diff --git a/docs/future-improvements.md b/docs/future-improvements.md index fdba1bd81f..100bd10dc1 100644 --- a/docs/future-improvements.md +++ b/docs/future-improvements.md @@ -229,13 +229,18 @@ the protocol layer. ## User-space programs -### Replace per-program `-Wl,-z,stack-size` workarounds with a real shadow-stack overflow guard -`wasm-ld` reserves a default 64 KiB shadow stack (the linear-memory region the -compiler uses for spilled locals, `alloca`, and address-taken locals). The -shadow stack grows **downward** from `__stack_high`, and `wasm-ld` places it +### Add a real shadow-stack overflow guard beyond the SDK's 8 MiB floor +Upstream `wasm-ld` reserves a default 64 KiB shadow stack (the linear-memory +region the compiler uses for spilled locals, `alloca`, and address-taken +locals). Kandelo's SDK raises executable links to an 8 MiB floor while +preserving larger explicit requests. That floor covers the mainstream +workloads that exposed the 64 KiB default, but it is a capacity policy rather +than an overflow guard. + +The shadow stack grows **downward** from `__stack_high`, and `wasm-ld` places it *immediately below* the `.data` / `.bss` segments in the same linear memory. There is no guard page, no stack-pointer bounds check, and no trap: a function -that consumes more than the remaining shadow-stack budget silently writes +that consumes more than the effective shadow-stack budget silently writes through `__stack_pointer` into whatever data segment happens to be just below it, corrupting unrelated globals. @@ -245,12 +250,11 @@ shadow-stack frame underflowed by ~108 KiB into PHP's `alloc_globals` data segment, silently corrupting `AG(mm_heap)`. The next `_efree` call dereferenced the now-bogus heap pointer and trapped — surfacing as "memory access out of bounds" inside the optimizer, with no indication that the actual cause was -stack overflow ~thousands of frames earlier. The PR's workaround is -`LDFLAGS=-Wl,-z,stack-size=4194304` (4 MiB) in `packages/registry/php/build-php.sh`. -That sidesteps the underflow for PHP's observed workload but doesn't *prevent* -the failure mode — a deeper recursion or a larger `alloca` will silently -corrupt data again, and every other large port we ship today (vim, nginx, -mariadb, etc.) has the same latent bug. +stack overflow ~thousands of frames earlier. The PHP recipe still requests +`LDFLAGS=-Wl,-z,stack-size=4194304` (4 MiB), which the SDK raises to its 8 MiB +floor. The larger reserve covers PHP's observed workload but doesn't *prevent* +the failure mode: a deeper recursion or a larger `alloca` can still silently +corrupt data, and every linked program has the same undetected-overflow risk. A real fix needs runtime detection so the failure surfaces as an obvious crash, not silent corruption. Possible approaches: @@ -280,10 +284,11 @@ crash, not silent corruption. Possible approaches: Once a real guard is in place, the per-program `-Wl,-z,stack-size=...` overrides should be audited: programs that genuinely need a larger shadow stack (PHP optimizer, deep parser stacks) keep the explicit override and -document why; everything else can drop the flag and rely on the default -+ guard. +document why; everything else can drop the package-local flag and rely on the +SDK floor plus the guard. -**Files:** `packages/registry/php/build-php.sh` (current 4 MiB workaround), +**Files:** `sdk/src/lib/flags.ts` and `sdk/kandelo/bin/wasm32posix-cc` (current +8 MiB floor), `packages/registry/php/build-php.sh` (current 4 MiB request), `libc/glue/channel_syscall.c` (likely site for a syscall-entry bounds check), `host/src/worker-main.ts` (instantiation-time wiring for stack bounds), plus any other `build-*.sh` that hits the same wall in the meantime. diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md index a18448a68d..d1348c9185 100644 --- a/docs/sdk-guide.md +++ b/docs/sdk-guide.md @@ -127,6 +127,11 @@ wasm32posix-ar rcs libfoo.a lib_a.o lib_b.o wasm32posix-cc main.c -L. -lfoo -o program.wasm ``` +The compiler wrapper preserves the caller's linker-input order. Objects, +explicit archives, `-l` libraries, and linker group controls reach Clang in +the same relative sequence supplied by the build system, as required by +normal static archive resolution. + ### With dynamic loading (dlopen) ```bash @@ -166,6 +171,7 @@ leave unresolved host imports in linked programs. -Wl,--import-memory # Memory provided by host -Wl,--shared-memory # Enable SharedArrayBuffer -Wl,--max-memory=1073741824 # 1GB max memory +-Wl,-z,stack-size=8388608 # 8 MiB main-thread shadow stack (see below) -Wl,--global-base=1114112 # Data segment start -Wl,--no-stack-first # LLVM 22+: preserve stack-after-data layout -Wl,--allow-undefined # Host imports are resolved at load time @@ -175,6 +181,95 @@ leave unresolved host imports in linked programs. -Wl,--export=__wasm_init_tls # TLS initialization ``` +#### Why an 8 MiB main-thread stack + +wasm-ld's default shadow stack is only ~64 KiB. WebAssembly has **no stack +guard page**, so a program that overflows the shadow stack does not fault at the +overflow — `__stack_pointer` simply keeps decrementing past `__data_end` and +silently overwrites whatever lives at the top of `.bss`. On this platform that +region holds the pthread/TLS globals (`__wasm_tp_storage`, the main thread's +`struct pthread`, and `__pthread_tsd_main`), so an overflow corrupts thread-local +storage and later surfaces as a **spurious "memory access out of bounds"** in an +unrelated function (e.g. `__pthread_getspecific` or `pthread_mutex_lock`) — far +from the actual overflow. Deep call chains in ordinary libraries hit this +easily; GTK's `gdk_pixbuf_new_from_file` → GObject type-registration → glib +chain is one confirmed case (see `docs/kandelo-lxde-desktop-demo.md`). + +POSIX does **not** mandate a default stack size — it is implementation-defined, +with `RLIMIT_STACK` governing the main thread. We reserve **8 MiB** because that +is the de-facto default soft `RLIMIT_STACK` on Linux/glibc (and macOS), so the +large body of C software written and tested on Linux already assumes it fits +within those bounds. Because a WebAssembly shadow stack cannot grow at runtime, +the reservation is fixed at link time. + +Scope and cost: +- **Main thread only.** This flag sizes the process's initial shadow stack. + Threads created via `pthread_create` get their own stacks from musl's + `__default_stacksize` (128 KiB by default), independent of this flag — an + 8 MiB main stack does **not** multiply per thread. +- **At least ~8 MiB of initial linear memory per process.** A larger stack raises + `__heap_base` 1:1, so each process instance reserves the extra space up front. + This is per-process; a `fork()`ed child is a separate address space with the + same configured main stack size. +- The engine's native Wasm call stack (operand stack / call frames) is separate + and host/engine-managed; it is not part of this linear-memory reservation. + +The SDK treats 8 MiB as a floor. It appends the larger of that floor and every +valid user stack request after the other linker arguments, where lld gives it +final precedence. The Node-hosted driver first asks pinned Clang for a `-###` +job trace without injecting executable glue. Compiler-only traces, including +`-fsyntax-only`, dependency generation, and analyzer jobs hidden in response +files, continue without link preparation. Confirmed executable links get a +second `-###` trace with the complete SDK link inputs, and the SDK scans the +exact `wasm-ld` argument vector Clang emits. This leaves option classification +and ordering in Clang: positional inputs, `-Wl,`, `-Xlinker`, and direct `-z` +transports cannot be misclassified by a duplicate SDK option table. A failed, +missing, or ambiguous linker trace aborts before the real link. These traces are +driver-only invocations and do not compile; they add two Clang processes per +executable link and one for an otherwise-unrecognized compiler-only mode. + +The Kandelo-native driver also uses a non-compiling Clang job trace to preserve +compiler-only modes before entering its manual link path. For a confirmed link, +it invokes the adjacent `wasm-ld` directly and scans the exact linker arguments +it has already classified. + +Relative lld response paths follow Clang's effective `-working-directory`, +including when that option comes from a top-level Clang response file, +configuration file, or environment override. The SDK reads the driver-owned cwd +slot in the pinned Clang trace and fails before linking if the trace omits or +contradicts that invariant. Debug, coverage, and file compilation-directory +options do not affect linker response resolution. + +LLVM 21 accepts the lld forms `-z stack-size=` and +`-zstack-size=`. Stack sizes follow LLVM 21's radix-0 integer syntax: +decimal, leading-zero or `0o` octal, `0x`/`0X` hexadecimal, and `0b`/`0B` +binary. + +Clang expands its response files when producing the trace. The SDK expands any +remaining lld response files, including nested `@file` references, using LLVM's +POSIX response-file tokenization rules. Expansion uses an iterative work stack, +not a nesting cutoff: both drivers inspect chains accepted by LLVM and reject +missing or recursive response files before the real link. To bound hostile or +accidental input, they also reject an expansion after 4,096 file expansions, +1,048,576 examined tokens, or 64 MiB of decoded response text. These are +explicit resource limits; exhausting one never falls back to the 8 MiB floor. +This keeps quoted or escaped paths distinct from linker options and prevents +object filenames containing `stack-size=` from changing the stack. Bare +`stack-size=` tokens, `-z=stack-size=`, and options after `--` are +not stack requests. Response-file contents are never rewritten. +The Node-hosted SDK accepts LLVM's UTF-8, UTF-16LE-BOM, and UTF-16BE-BOM +response encodings. The Kandelo-native Bash driver accepts UTF-8 response files +and rejects either UTF-16 BOM before compiling or linking: Bash variables cannot +retain UTF-16's embedded NUL bytes, and the packaged SDK does not declare a +transcoding tool. Generate UTF-8 response files when building inside Kandelo. +Invalid spellings stay visible to LLVM so it can reject them; a valid stack +larger than the SDK's fixed 1 GiB executable-memory maximum fails in the driver +instead of being silently replaced by the floor. Smaller legacy requests +therefore still receive 8 MiB, while programs such as SpiderMonkey that +explicitly need 16 MiB retain that larger reservation. Changing the platform +floor or scanner requires updating both `sdk/kandelo/bin/wasm32posix-cc` and +`sdk/src/lib/flags.ts`. + ### Files linked automatically When linking an executable (not compile-only), the SDK adds: diff --git a/sdk/kandelo/bin/wasm32posix-cc b/sdk/kandelo/bin/wasm32posix-cc index 6727457329..00299988de 100755 --- a/sdk/kandelo/bin/wasm32posix-cc +++ b/sdk/kandelo/bin/wasm32posix-cc @@ -31,6 +31,188 @@ find_tool() { return 1 } +is_gnu_response_whitespace() { + case "$1" in + ' '|$'\t'|$'\r'|$'\n') return 0 ;; + *) return 1 ;; + esac +} + +# Match LLVM's POSIX TokenizeGNUCommandLine response-file grammar without eval +# or shell expansion. The result is returned in the response_tokens array. +tokenize_gnu_response_file() { + local source="$1" + local token="" + local char quote + local index=0 + local length + response_tokens=() + + source="${source#$'\xef\xbb\xbf'}" + length="${#source}" + while (( index < length )); do + if [[ -z "$token" ]]; then + while (( index < length )); do + char="${source:index:1}" + is_gnu_response_whitespace "$char" || break + index=$((index + 1)) + done + (( index < length )) || break + fi + + char="${source:index:1}" + if [[ "$char" == "\\" && $((index + 1)) -lt length ]]; then + index=$((index + 1)) + token="${token}${source:index:1}" + index=$((index + 1)) + continue + fi + + if [[ "$char" == "'" || "$char" == '"' ]]; then + quote="$char" + index=$((index + 1)) + while (( index < length )); do + char="${source:index:1}" + [[ "$char" != "$quote" ]] || break + if [[ "$char" == "\\" && $((index + 1)) -lt length ]]; then + index=$((index + 1)) + char="${source:index:1}" + fi + token="${token}${char}" + index=$((index + 1)) + done + (( index < length )) || break + index=$((index + 1)) + continue + fi + + if is_gnu_response_whitespace "$char"; then + [[ -z "$token" ]] || response_tokens+=("$token") + token="" + index=$((index + 1)) + continue + fi + + token="${token}${char}" + index=$((index + 1)) + done + + [[ -z "$token" ]] || response_tokens+=("$token") +} + +response_file_is_active() { + local candidate="$1" + local active + for active in "${response_file_stack[@]}"; do + if [[ "$candidate" == "$active" || "$candidate" -ef "$active" ]]; then + return 0 + fi + done + return 1 +} + +# Expand standalone @file arguments with an explicit work stack. This avoids a +# nesting cutoff that could leave an lld-accepted stack request uninspected. +# Bounds are work/resource limits and fail before compilation or linking. +readonly max_response_file_expansions=4096 +readonly max_response_file_tokens=1048576 +readonly max_response_file_characters=67108864 + +expand_response_arguments() { + local arg path source kind last_index active_last index + local response_prefix + local nested_tokens + local response_file_expansions=0 + local response_file_tokens="$#" + local response_file_characters=0 + + response_tokens=() + response_file_stack=() + expanded_response_args=() + response_work_kinds=() + response_work_values=() + + if (( response_file_tokens > max_response_file_tokens )); then + die "response-file expansion exceeds the ${max_response_file_tokens}-token safety limit" + fi + + for ((index = $#; index >= 1; index--)); do + response_work_kinds+=("argument") + response_work_values+=("${!index}") + done + + while (( ${#response_work_kinds[@]} > 0 )); do + last_index=$((${#response_work_kinds[@]} - 1)) + kind="${response_work_kinds[$last_index]}" + arg="${response_work_values[$last_index]}" + unset "response_work_kinds[$last_index]" + unset "response_work_values[$last_index]" + + if [[ "$kind" == "leave" ]]; then + active_last=$((${#response_file_stack[@]} - 1)) + (( active_last >= 0 )) || die "internal response-file expansion stack underflow" + unset "response_file_stack[$active_last]" + continue + fi + + case "$arg" in + @?*) + path="${arg#@}" + [[ -r "$path" && ! -d "$path" ]] || \ + die "cannot inspect response file '$path' before linking" + response_file_is_active "$path" && \ + die "recursive response file '$path' cannot be inspected safely" + + response_file_expansions=$((response_file_expansions + 1)) + if (( response_file_expansions > max_response_file_expansions )); then + die "response-file expansion exceeds the ${max_response_file_expansions}-file safety limit" + fi + + response_prefix="" + LC_ALL=C IFS= read -r -n 2 response_prefix < "$path" || true + case "$response_prefix" in + $'\xff\xfe'|$'\xfe\xff') + # Bash variables cannot retain UTF-16's embedded NUL bytes, and + # the packaged SDK deliberately has no undeclared transcoder. + die "UTF-16 response file '$path' is unsupported by the Kandelo-native SDK driver;" \ + "rewrite it as UTF-8" + ;; + esac + source="$(< "$path")" + response_file_characters=$((response_file_characters + ${#source})) + if (( response_file_characters > max_response_file_characters )); then + die "response-file expansion exceeds the ${max_response_file_characters}-character safety limit" + fi + + tokenize_gnu_response_file "$source" + nested_tokens=("${response_tokens[@]}") + response_file_tokens=$((response_file_tokens + ${#nested_tokens[@]})) + if (( response_file_tokens > max_response_file_tokens )); then + die "response-file expansion exceeds the ${max_response_file_tokens}-token safety limit" + fi + + response_file_stack+=("$path") + response_work_kinds+=("leave") + response_work_values+=("$path") + for ((index = ${#nested_tokens[@]} - 1; index >= 0; index--)); do + response_work_kinds+=("argument") + response_work_values+=("${nested_tokens[$index]}") + done + ;; + *) + expanded_response_args+=("$arg") + ;; + esac + done +} + +original_args=("$@") +response_tokens=() +response_file_stack=() +expanded_response_args=() +expand_response_arguments "${original_args[@]}" +set -- "${expanded_response_args[@]}" + if [[ "$SDK_CXX" -eq 1 ]]; then CLANG="$(find_tool clang++ || true)" [[ -n "$CLANG" ]] || die "clang++ not found; set WASM_POSIX_LLVM_DIR" @@ -126,6 +308,47 @@ if [[ -d "$RESOURCE_DIR" ]]; then compile_flags+=(-resource-dir "$RESOURCE_DIR") fi +classify_clang_link_job() { + local trace line executable executable_name second + local compiler_jobs=0 + local linker_jobs=0 + local other_external_jobs=0 + + if ! trace="$("$CLANG" -### "${compile_flags[@]}" "${filtered[@]}" 2>&1 >/dev/null)"; then + printf '%s\n' "$trace" >&2 + die "clang -### failed while classifying the requested jobs" + fi + + while IFS= read -r line || [[ -n "$line" ]]; do + [[ "$line" =~ ^[[:space:]]+\" ]] || continue + tokenize_gnu_response_file "$line" + (( ${#response_tokens[@]} > 0 )) || continue + executable="${response_tokens[0]}" + executable_name="${executable##*/}" + executable_name="${executable_name%.exe}" + second="${response_tokens[1]:-}" + if [[ "$executable" == "$WASM_LD" || "$executable" -ef "$WASM_LD" ]]; then + linker_jobs=$((linker_jobs + 1)) + elif [[ "$second" == "-cc1" ]]; then + compiler_jobs=$((compiler_jobs + 1)) + # Wrapped LLVM installations may schedule Binaryen after wasm-ld. It is a + # post-link transform, not a second or replacement linker job. + elif [[ "$executable_name" =~ ^wasm-opt(-[0-9]+)?$ ]]; then + : + else + other_external_jobs=$((other_external_jobs + 1)) + fi + done <<< "$trace" + + if (( linker_jobs == 0 && other_external_jobs == 0 && compiler_jobs > 0 )); then + linking=0 + return + fi + if (( linker_jobs != 1 || other_external_jobs != 0 )); then + die "clang -### emitted ${linker_jobs} commands for the pinned linker ${WASM_LD}; expected exactly one" + fi +} + shared_link_flags=( -nostdlib -Wl,--experimental-pic @@ -135,6 +358,8 @@ shared_link_flags=( -Wl,--allow-undefined ) +max_executable_memory_size=1073741824 + exe_link_flags=( -nostdlib -Wl,--entry=_start @@ -142,8 +367,19 @@ exe_link_flags=( -Wl,--export=__heap_base -Wl,--import-memory -Wl,--shared-memory - -Wl,--max-memory=1073741824 + "-Wl,--max-memory=${max_executable_memory_size}" -Wl,--allow-undefined + # Reserve an 8 MiB main-thread shadow stack. wasm-ld's default is only ~64 KiB, + # and WebAssembly has no stack guard page, so a deep call chain silently + # overflows past __data_end into .bss and corrupts the pthread/TLS globals that + # live there (__wasm_tp_storage, __pthread_tsd_main) -> spurious "memory access + # out of bounds". POSIX leaves the default stack size implementation-defined, + # but 8 MiB is the de-facto Linux/glibc RLIMIT_STACK default that mainstream C + # software (GTK, etc.) is written and tested against, so matching it maximizes + # portability. Treat it as a floor: explicit larger requests remain effective. + # Sizes the main thread only; pthreads use musl __default_stacksize. Cost: at + # least ~8 MiB initial linear memory per process (raises __heap_base 1:1). Keep + # in sync with sdk/src/lib/flags.ts. See docs/sdk-guide.md. -Wl,--global-base=1114112 -Wl,--table-base=3 -Wl,--export-table @@ -157,6 +393,10 @@ exe_link_flags=( -Wl,--export=__abi_version ) +if [[ "$linking" -ne 0 ]]; then + classify_clang_link_job +fi + if [[ "$linking" -eq 0 ]]; then exec "$CLANG" "${compile_flags[@]}" "${filtered[@]}" fi @@ -166,11 +406,130 @@ compile_args=() source_inputs=() object_inputs=() user_link_args=() +main_thread_stack_size=8388608 + +parse_lld_stack_size() { + local raw="$1" + local radix digits significant_digits max_digits value + + # Match LLVM 21's radix-0 integer grammar. Invalid spellings stay in the + # original linker arguments so wasm-ld reports them truthfully. + if [[ "$raw" =~ ^0[xX]([0-9a-fA-F]+)$ ]]; then + radix=16 + digits="${BASH_REMATCH[1]}" + max_digits=8 + elif [[ "$raw" =~ ^0[bB]([01]+)$ ]]; then + radix=2 + digits="${BASH_REMATCH[1]}" + max_digits=31 + elif [[ "$raw" =~ ^0o([0-7]+)$ ]]; then + radix=8 + digits="${BASH_REMATCH[1]}" + max_digits=11 + elif [[ "$raw" =~ ^0[0-7]*$ ]]; then + radix=8 + digits="${raw#0}" + digits="${digits:-0}" + max_digits=11 + elif [[ "$raw" =~ ^[1-9][0-9]*$ ]]; then + radix=10 + digits="$raw" + max_digits=10 + else + return 1 + fi + + significant_digits="$digits" + while [[ "${#significant_digits}" -gt 1 && "${significant_digits:0:1}" == 0 ]]; do + significant_digits="${significant_digits#0}" + done + if [[ "${#significant_digits}" -gt "$max_digits" ]]; then + return 2 + fi + + case "$radix" in + 2) value=$((2#$significant_digits)) ;; + 8) value=$((8#$significant_digits)) ;; + 10) value=$((10#$significant_digits)) ;; + 16) value=$((16#$significant_digits)) ;; + esac + if (( value > max_executable_memory_size )); then + return 2 + fi + + printf '%s\n' "$value" +} + +consider_lld_stack_size() { + local raw="$1" + local requested_stack_size parse_status + + if requested_stack_size="$(parse_lld_stack_size "$raw")"; then + if (( requested_stack_size > main_thread_stack_size )); then + main_thread_stack_size="$requested_stack_size" + fi + return + else + parse_status=$? + fi + + if [[ "$parse_status" -eq 2 ]]; then + die "stack-size=$raw exceeds the SDK's ${max_executable_memory_size}-byte executable memory limit" + fi +} + +inspect_lld_stack_sizes() { + local index=0 + local arg requested_stack_size + local scan_args + + response_tokens=() + response_file_stack=() + expanded_response_args=() + expand_response_arguments "$@" + scan_args=("${expanded_response_args[@]}") + + while [[ "$index" -lt "${#scan_args[@]}" ]]; do + arg="${scan_args[$index]}" + [[ "$arg" != "--" ]] || break + case "$arg" in + -z) + index=$((index + 1)) + if [[ "$index" -lt "${#scan_args[@]}" ]]; then + arg="${scan_args[$index]}" + case "$arg" in + stack-size=*) + requested_stack_size="${arg#stack-size=}" + consider_lld_stack_size "$requested_stack_size" + ;; + esac + fi + ;; + -zstack-size=*) + requested_stack_size="${arg#-zstack-size=}" + consider_lld_stack_size "$requested_stack_size" + ;; + esac + index=$((index + 1)) + done +} idx=0 +clang_options_ended=0 while [[ "$idx" -lt "${#filtered[@]}" ]]; do arg="${filtered[$idx]}" + if [[ "$clang_options_ended" -eq 1 ]]; then + case "$arg" in + *.c|*.cc|*.cpp|*.cxx|*.C) source_inputs+=("$arg") ;; + *) object_inputs+=("$arg") ;; + esac + idx=$((idx + 1)) + continue + fi case "$arg" in + --) + clang_options_ended=1 + ;; -o) idx=$((idx + 1)) [[ "$idx" -lt "${#filtered[@]}" ]] || die "-o requires an output path" @@ -187,7 +546,19 @@ while [[ "$idx" -lt "${#filtered[@]}" ]]; do ;; -Wl,*) IFS=',' read -r -a wl_parts <<< "${arg#-Wl,}" - user_link_args+=("${wl_parts[@]}") + for part in "${wl_parts[@]}"; do + [[ -z "$part" ]] || user_link_args+=("$part") + done + ;; + -Xlinker) + idx=$((idx + 1)) + [[ "$idx" -lt "${#filtered[@]}" ]] || die "-Xlinker requires an argument" + user_link_args+=("${filtered[$idx]}") + ;; + -z) + idx=$((idx + 1)) + [[ "$idx" -lt "${#filtered[@]}" ]] || die "-z requires an argument" + user_link_args+=("-z" "${filtered[$idx]}") ;; -L) idx=$((idx + 1)) @@ -204,6 +575,9 @@ while [[ "$idx" -lt "${#filtered[@]}" ]]; do idx=$((idx + 1)) done +inspect_lld_stack_sizes "${user_link_args[@]}" +exe_link_flags+=("-Wl,-z,stack-size=${main_thread_stack_size}") + compiled_objects=() for src in "${source_inputs[@]}"; do base="$(basename "$src")" diff --git a/sdk/src/bin/c++.ts b/sdk/src/bin/c++.ts index 7870926d25..55e94b7c7c 100755 --- a/sdk/src/bin/c++.ts +++ b/sdk/src/bin/c++.ts @@ -9,8 +9,13 @@ async function main(): Promise { const arch = detectArch(); const toolchain = await resolveToolchain(arch); const userArgs = process.argv.slice(2); - await prepareExecutableLinker(userArgs, toolchain, arch); - const args = buildClangArgs(userArgs, toolchain, arch); + const executableLinker = await prepareExecutableLinker( + userArgs, + toolchain, + arch, + toolchain.cxx, + ); + const args = buildClangArgs(userArgs, toolchain, arch, executableLinker ?? undefined); const exitCode = await runPassthrough(toolchain.cxx, args); process.exit(exitCode); } diff --git a/sdk/src/bin/cc.ts b/sdk/src/bin/cc.ts index 247df83eb7..b1c05a0dc4 100755 --- a/sdk/src/bin/cc.ts +++ b/sdk/src/bin/cc.ts @@ -1,36 +1,214 @@ #!/usr/bin/env -S node --experimental-strip-types -import { readFileSync } from 'node:fs'; -import { join } from 'node:path'; +import { readFileSync, realpathSync } from 'node:fs'; +import { basename, isAbsolute, join, resolve } from 'node:path'; +import { TextDecoder } from 'node:util'; import { resolveLldMajor, resolveToolchain, type Toolchain } from '../lib/toolchain.ts'; import { compileFlags, + DEFAULT_MAIN_THREAD_STACK_SIZE, filterArgs, inferThreadSlotDeclaration, linkFlags, + mainThreadStackSize, + MAX_EXECUTABLE_MEMORY_SIZE, needsLinking, parseArgs, SHARED_LINK_FLAGS, THREAD_SLOT_USE_HOST_DEFAULT, threadSlotDeclarationDefine, + tokenizeGnuResponseFile, + type ResponseFileContents, } from '../lib/flags.ts'; -import { runPassthrough } from '../lib/exec.ts'; +import { run, runPassthrough } from '../lib/exec.ts'; import { isMain } from '../lib/is-main.ts'; import { type WasmArch, detectArch, targetTriple } from '../lib/arch.ts'; -export function buildClangArgs(userArgs: string[], toolchain: Toolchain, arch: WasmArch = 'wasm32'): string[] { +export function decodeLlvmResponseFile(bytes: Uint8Array): string { + const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); + if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) { + if ((buffer.length - 2) % 2 !== 0) throw new Error('odd-length UTF-16LE response file'); + return new TextDecoder('utf-16le', { fatal: true }).decode(buffer.subarray(2)); + } + if (buffer.length >= 2 && buffer[0] === 0xfe && buffer[1] === 0xff) { + if ((buffer.length - 2) % 2 !== 0) throw new Error('odd-length UTF-16BE response file'); + return new TextDecoder('utf-16be', { fatal: true }).decode(buffer.subarray(2)); + } + return buffer.toString('utf8'); +} + +function readLlvmResponseFile( + path: string, + workingDirectory = process.cwd(), +): ResponseFileContents | null { + try { + const resolvedPath = resolve(workingDirectory, path); + return { + contents: decodeLlvmResponseFile(readFileSync(resolvedPath)), + identity: realpathSync(resolvedPath), + }; + } catch { + return null; + } +} + +export type LinkerPreparation = + | { kind: 'no-link' } + | { kind: 'executable-link'; mainThreadStackSizeBytes: number }; + +function isPinnedLinker(actualPath: string | undefined, linkerPath: string): boolean { + if (actualPath === linkerPath) return true; + if (!actualPath) return false; + try { + return realpathSync(actualPath) === realpathSync(linkerPath); + } catch { + return false; + } +} + +function isExpectedNonLinkerJob(args: string[]): boolean { + if (args[1] === '-cc1') return true; + // Wrapped LLVM installations may schedule Binaryen after wasm-ld. It is a + // post-link transform, not a second or replacement linker job. + const executable = basename(args[0] ?? '').replace(/\.exe$/i, ''); + return /^wasm-opt(?:-[0-9]+)?$/.test(executable); +} + +function clangTraceCommands( + trace: string, + acceptsFirstLine: (args: string[], firstLine: string) => boolean, +): string[] { + const commands: string[] = []; + let offset = 0; + + while (offset < trace.length) { + const physicalLineEnd = trace.indexOf('\n', offset); + const firstLineEnd = physicalLineEnd === -1 ? trace.length : physicalLineEnd; + const firstLine = trace.slice(offset, firstLineEnd); + const firstLineArgs = tokenizeGnuResponseFile(firstLine); + if (!acceptsFirstLine(firstLineArgs, firstLine)) { + offset = physicalLineEnd === -1 ? trace.length : physicalLineEnd + 1; + continue; + } + + let quote: string | null = null; + let end = offset; + for (; end < trace.length; end++) { + const char = trace[end]; + if (char === '\\' && end + 1 < trace.length) { + end++; + continue; + } + if (quote !== null) { + if (char === quote) quote = null; + continue; + } + if (char === '"' || char === "'") { + quote = char; + continue; + } + if (char === '\n') break; + } + if (quote !== null) { + throw new Error('clang -### emitted an unterminated quoted command'); + } + + commands.push(trace.slice(offset, end)); + offset = end < trace.length ? end + 1 : end; + } + + return commands; +} + +export function linkerArgsFromClangTrace(trace: string, linkerPath: string): string[] | null { + const jobs = clangTraceCommands( + trace, + (args, firstLine) => + /^[\t ]+["']/.test(firstLine) && args[0] !== undefined && isAbsolute(args[0]), + ).map((line) => tokenizeGnuResponseFile(line)); + if (jobs.length === 0) { + throw new Error('clang -### emitted no recognizable jobs'); + } + + const matches = jobs.filter((args) => isPinnedLinker(args[0], linkerPath)); + const unexpectedJobs = jobs.filter((args) => + !isPinnedLinker(args[0], linkerPath) && !isExpectedNonLinkerJob(args)); + if (matches.length > 1 || unexpectedJobs.length > 0) { + throw new Error( + `clang -### emitted ${matches.length} commands for the pinned linker ${linkerPath}; expected exactly one`, + ); + } + if (matches.length === 0) { + if (!jobs.some((args) => args[1] === '-cc1')) { + throw new Error('clang -### emitted no compiler or pinned linker jobs'); + } + return null; + } + return matches[0].slice(1); +} + +export function workingDirectoryFromClangTrace( + trace: string, + initialWorkingDirectory = process.cwd(), +): string { + const tracedDirectories = new Set(); + + // The provisional argv always adds absolute glue C sources, so an executable + // link, including an object-only user link, emits cc1 jobs. Pinned LLVM 21 + // constructs this exact slot by pushing -resource-dir, its value, and then + // Args.AddLastArg(OPT_working_directory), before preprocessing and -Xclang + // arguments. This is the effective driver cwd even when a config file or + // CCC_OVERRIDE_OPTIONS supplied it; debug and coverage metadata are not. + for (const command of clangTraceCommands(trace, (args) => args.includes('-cc1'))) { + const args = tokenizeGnuResponseFile(command); + const resourceDirectoryIndex = args.indexOf('-resource-dir'); + if (resourceDirectoryIndex === -1 || args[resourceDirectoryIndex + 1] === undefined) { + throw new Error('clang -### cc1 command omitted its driver resource-directory slot'); + } + const followingIndex = resourceDirectoryIndex + 2; + if (args[followingIndex] === '-working-directory') { + const value = args[followingIndex + 1]; + if (value === undefined || value.length === 0) { + throw new Error('clang -### emitted a malformed driver working-directory slot'); + } + tracedDirectories.add(resolve(initialWorkingDirectory, value)); + } else if (args[followingIndex]?.startsWith('-working-directory')) { + throw new Error('clang -### emitted an ambiguous driver working-directory slot'); + } else { + tracedDirectories.add(resolve(initialWorkingDirectory)); + } + } + if (tracedDirectories.size !== 1) { + throw new Error('clang -### did not emit one consistent driver working directory'); + } + return tracedDirectories.values().next().value as string; +} + +function buildClangArgsInternal( + userArgs: string[], + toolchain: Toolchain, + arch: WasmArch = 'wasm32', + executableLinker?: LinkerPreparation, + reportWarnings = true, + classifyLink = false, +): string[] { const { filtered, warnings } = filterArgs(userArgs, arch); - for (const w of warnings) console.error(w); + if (reportWarnings) { + for (const w of warnings) console.error(w); + } const parsed = parseArgs(filtered); - const linking = needsLinking(parsed); + const linking = needsLinking(parsed) && executableLinker?.kind !== 'no-link'; const hasSourceFiles = parsed.sourceFiles.length > 0; const args: string[] = []; const target = `--target=${targetTriple(arch)}`; - // Inject compile flags when there are source files, compile-only modes, - // or when linking (since the glue .c file needs them). - if (hasSourceFiles || parsed.compileOnly || parsed.preprocessOnly || parsed.assemblyOnly || linking) { + // Inject compile flags for visible sources and compile-only modes, links + // that compile glue, and response-hidden compile jobs found by the trace. + if ( + hasSourceFiles || parsed.compileOnly || parsed.preprocessOnly || parsed.assemblyOnly || linking || + executableLinker?.kind === 'no-link' + ) { args.push(...compileFlags(arch)); } // Target is always needed (even for link-only, clang needs to know the target) @@ -43,11 +221,11 @@ export function buildClangArgs(userArgs: string[], toolchain: Toolchain, arch: W if (parsed.preprocessOnly) args.push('-E'); if (parsed.assemblyOnly) args.push('-S'); if (parsed.outputFile) args.push('-o', parsed.outputFile); - args.push(...parsed.otherArgs); - - args.push(...parsed.sourceFiles); - args.push(...parsed.objectFiles); - args.push(...parsed.archiveFiles); + // Static link semantics depend on the caller's exact ordering of objects, + // archives, -l flags, and linker group controls. Parsed classifications are + // for SDK decisions only; forwarding must never rebuild the command in + // type-based buckets. + args.push(...parsed.forwardedArgs); // -fPIC is consumed by parseArgs (so the linker can see `parsed.pic`), // but it must also reach clang at compile time so the resulting object @@ -61,6 +239,7 @@ export function buildClangArgs(userArgs: string[], toolchain: Toolchain, arch: W // linker path, clang can pick an unrelated ambient wasm-ld whose defaults // differ from the repository-pinned toolchain. args.push(`-fuse-ld=${join(toolchain.llvmDir, 'wasm-ld')}`); + if (classifyLink) return args; if (parsed.shared) { // Shared library build: no CRT, no libc, no syscall glue args.push(...SHARED_LINK_FLAGS); @@ -70,6 +249,22 @@ export function buildClangArgs(userArgs: string[], toolchain: Toolchain, arch: W 'wasm-ld version is unresolved; call prepareExecutableLinker() before building executable link arguments', ); } + if (!executableLinker || executableLinker.kind !== 'executable-link') { + throw new Error( + 'executable linker arguments are unprepared; call prepareExecutableLinker() and pass its result to buildClangArgs()', + ); + } + const preparedStackSize = executableLinker.mainThreadStackSizeBytes; + if ( + !Number.isSafeInteger(preparedStackSize) || + preparedStackSize < DEFAULT_MAIN_THREAD_STACK_SIZE || + preparedStackSize > MAX_EXECUTABLE_MEMORY_SIZE + ) { + throw new Error( + `prepared main-thread stack size must be an integer from ${DEFAULT_MAIN_THREAD_STACK_SIZE} ` + + `through ${MAX_EXECUTABLE_MEMORY_SIZE} bytes`, + ); + } // Executable build: link CRT, libc, and syscall glue const threadSlots = inferThreadSlotDeclaration(parsed, userArgs, { readFile: (path) => { @@ -98,7 +293,7 @@ export function buildClangArgs(userArgs: string[], toolchain: Toolchain, arch: W // it nor accepts --no-stack-first. Preserve Kandelo's established // stack-after-data layout explicitly only where the option exists. ...(toolchain.lldMajor >= 22 ? ['-Wl,--no-stack-first'] : []), - ...linkFlags(arch), + ...linkFlags(arch, preparedStackSize), ); } } @@ -106,24 +301,78 @@ export function buildClangArgs(userArgs: string[], toolchain: Toolchain, arch: W return args; } +export function buildClangArgs( + userArgs: string[], + toolchain: Toolchain, + arch: WasmArch = 'wasm32', + executableLinker?: LinkerPreparation, +): string[] { + return buildClangArgsInternal(userArgs, toolchain, arch, executableLinker, true); +} + export async function prepareExecutableLinker( userArgs: string[], toolchain: Toolchain, arch: WasmArch = 'wasm32', -): Promise { + compiler = toolchain.cc, +): Promise { const { filtered } = filterArgs(userArgs, arch); const parsed = parseArgs(filtered); - if (needsLinking(parsed) && !parsed.shared) { - toolchain.lldMajor = await resolveLldMajor(toolchain.llvmDir); + if (!needsLinking(parsed) || parsed.shared) return null; + + const classificationArgs = buildClangArgsInternal( + userArgs, + toolchain, + arch, + undefined, + false, + true, + ); + const classificationTrace = await run(compiler, ['-###', ...classificationArgs]); + if (classificationTrace.exitCode !== 0) { + throw new Error( + `clang -### failed while classifying the requested jobs:\n${classificationTrace.stderr.trim()}`, + ); + } + const linkerPath = join(toolchain.llvmDir, 'wasm-ld'); + const classifiedLinkerArgs = linkerArgsFromClangTrace(classificationTrace.stderr, linkerPath); + if (classifiedLinkerArgs === null) return { kind: 'no-link' }; + + toolchain.lldMajor = await resolveLldMajor(toolchain.llvmDir); + const provisional = buildClangArgsInternal(userArgs, toolchain, arch, { + kind: 'executable-link', + mainThreadStackSizeBytes: DEFAULT_MAIN_THREAD_STACK_SIZE, + }, false); + const trace = await run(compiler, ['-###', ...provisional]); + if (trace.exitCode !== 0) { + throw new Error(`clang -### failed while preparing the executable link:\n${trace.stderr.trim()}`); + } + + const linkerArgs = linkerArgsFromClangTrace(trace.stderr, linkerPath); + if (linkerArgs === null) { + throw new Error('clang -### omitted the pinned linker from a confirmed executable link'); } + let tracedWorkingDirectory: string | undefined; + return { + kind: 'executable-link', + mainThreadStackSizeBytes: mainThreadStackSize( + linkerArgs, + (path) => { + if (!isAbsolute(path)) { + tracedWorkingDirectory ??= workingDirectoryFromClangTrace(trace.stderr); + } + return readLlvmResponseFile(path, tracedWorkingDirectory); + }, + ), + }; } async function main(): Promise { const arch = detectArch(); const toolchain = await resolveToolchain(arch); const userArgs = process.argv.slice(2); - await prepareExecutableLinker(userArgs, toolchain, arch); - const args = buildClangArgs(userArgs, toolchain, arch); + const executableLinker = await prepareExecutableLinker(userArgs, toolchain, arch); + const args = buildClangArgs(userArgs, toolchain, arch, executableLinker ?? undefined); const exitCode = await runPassthrough(toolchain.cc, args); process.exit(exitCode); } diff --git a/sdk/src/lib/flags.ts b/sdk/src/lib/flags.ts index 9b2876d155..a2cdd3b8d5 100644 --- a/sdk/src/lib/flags.ts +++ b/sdk/src/lib/flags.ts @@ -20,7 +20,236 @@ export function compileFlags(arch: WasmArch): string[] { ]; } -export function linkFlags(arch: WasmArch): string[] { +export const DEFAULT_MAIN_THREAD_STACK_SIZE = 8 * 1024 * 1024; +export const MAX_EXECUTABLE_MEMORY_SIZE = 1024 * 1024 * 1024; + +type ParsedStackSize = + | { kind: 'valid'; value: number } + | { kind: 'invalid' } + | { kind: 'overflow' }; + +function parseLldStackSize(value: string): ParsedStackSize { + let digits: string; + let radix: 2 | 8 | 10 | 16; + + // Match LLVM 21's radix-0 integer grammar exactly. Invalid spellings are + // left for wasm-ld to reject so the compiler driver never rewrites input. + if (/^0[xX][0-9a-fA-F]+$/.test(value)) { + digits = value.slice(2); + radix = 16; + } else if (/^0[bB][01]+$/.test(value)) { + digits = value.slice(2); + radix = 2; + } else if (/^0o[0-7]+$/.test(value)) { + digits = value.slice(2); + radix = 8; + } else if (/^0[0-7]*$/.test(value)) { + digits = value.slice(1) || '0'; + radix = 8; + } else if (/^[1-9][0-9]*$/.test(value)) { + digits = value; + radix = 10; + } else { + return { kind: 'invalid' }; + } + + const significantDigits = digits.replace(/^0+/, '') || '0'; + const maxDigits = radix === 2 ? 31 : radix === 8 ? 11 : radix === 10 ? 10 : 8; + if (significantDigits.length > maxDigits) return { kind: 'overflow' }; + + const prefix = radix === 2 ? '0b' : radix === 8 ? '0o' : radix === 16 ? '0x' : ''; + const parsed = BigInt(`${prefix}${significantDigits}`); + if (parsed > BigInt(MAX_EXECUTABLE_MEMORY_SIZE)) return { kind: 'overflow' }; + + return { kind: 'valid', value: Number(parsed) }; +} + +export const MAX_RESPONSE_FILE_EXPANSIONS = 4096; +export const MAX_RESPONSE_FILE_TOKENS = 1024 * 1024; +export const MAX_RESPONSE_FILE_CHARACTERS = 64 * 1024 * 1024; + +export interface ResponseFileContents { + contents: string; + /** Canonical identity used only while this file is active in the expansion stack. */ + identity: string; +} + +export type ResponseFileReader = (path: string) => ResponseFileContents | null; + +function isGnuResponseWhitespace(char: string): boolean { + return char === ' ' || char === '\t' || char === '\r' || char === '\n'; +} + +/** Match LLVM's POSIX TokenizeGNUCommandLine response-file grammar. */ +export function tokenizeGnuResponseFile(source: string): string[] { + if (source.startsWith('\uFEFF')) source = source.slice(1); + + const tokens: string[] = []; + let token = ''; + + for (let i = 0; i < source.length; i++) { + if (token.length === 0) { + while (i < source.length && isGnuResponseWhitespace(source[i])) i++; + if (i === source.length) break; + } + + const char = source[i]; + if (char === '\\' && i + 1 < source.length) { + token += source[++i]; + continue; + } + + if (char === "'" || char === '"') { + const quote = char; + i++; + while (i < source.length && source[i] !== quote) { + if (source[i] === '\\' && i + 1 < source.length) i++; + token += source[i++]; + } + if (i === source.length) break; + continue; + } + + if (isGnuResponseWhitespace(char)) { + if (token.length > 0) tokens.push(token); + token = ''; + continue; + } + + token += char; + } + + if (token.length > 0) tokens.push(token); + return tokens; +} + +export function expandResponseFiles( + args: string[], + readResponseFile: ResponseFileReader, +): string[] { + const expanded: string[] = []; + const activeFiles = new Set(); + type WorkItem = + | { kind: 'argument'; value: string } + | { kind: 'leave'; identity: string }; + const work: WorkItem[] = []; + for (let index = args.length - 1; index >= 0; index--) { + work.push({ kind: 'argument', value: args[index] }); + } + + let fileExpansions = 0; + let examinedTokens = args.length; + let decodedCharacters = 0; + if (examinedTokens > MAX_RESPONSE_FILE_TOKENS) { + throw new Error( + `response-file expansion exceeds the ${MAX_RESPONSE_FILE_TOKENS}-token safety limit`, + ); + } + + while (work.length > 0) { + const item = work.pop() as WorkItem; + if (item.kind === 'leave') { + activeFiles.delete(item.identity); + continue; + } + + const arg = item.value; + if (!arg.startsWith('@') || arg.length === 1) { + expanded.push(arg); + continue; + } + const path = arg.slice(1); + const responseFile = readResponseFile(path); + if (responseFile === null) { + throw new Error(`cannot inspect response file ${JSON.stringify(path)} before linking`); + } + if (responseFile.identity.length === 0) { + throw new Error(`response file ${JSON.stringify(path)} has no canonical identity`); + } + if (activeFiles.has(responseFile.identity)) { + throw new Error(`recursive response file ${JSON.stringify(path)} cannot be inspected safely`); + } + + fileExpansions++; + if (fileExpansions > MAX_RESPONSE_FILE_EXPANSIONS) { + throw new Error( + `response-file expansion exceeds the ${MAX_RESPONSE_FILE_EXPANSIONS}-file safety limit`, + ); + } + decodedCharacters += responseFile.contents.length; + if (decodedCharacters > MAX_RESPONSE_FILE_CHARACTERS) { + throw new Error( + `response-file expansion exceeds the ${MAX_RESPONSE_FILE_CHARACTERS}-character safety limit`, + ); + } + + const nestedTokens = tokenizeGnuResponseFile(responseFile.contents); + examinedTokens += nestedTokens.length; + if (examinedTokens > MAX_RESPONSE_FILE_TOKENS) { + throw new Error( + `response-file expansion exceeds the ${MAX_RESPONSE_FILE_TOKENS}-token safety limit`, + ); + } + + activeFiles.add(responseFile.identity); + work.push({ kind: 'leave', identity: responseFile.identity }); + for (let index = nestedTokens.length - 1; index >= 0; index--) { + work.push({ kind: 'argument', value: nestedTokens[index] }); + } + } + + return expanded; +} + +/** + * Apply the SDK's stack-size floor while retaining explicit larger requests. + * Callers pass the exact argv emitted for wasm-ld by Clang's `-###` trace. That + * keeps Clang's option classification and ordering in Clang itself instead of + * duplicating its driver option table here. + */ +export function mainThreadStackSize( + linkerArgs: string[], + readResponseFile?: ResponseFileReader, +): number { + let result = DEFAULT_MAIN_THREAD_STACK_SIZE; + + const consider = (value: string): void => { + const requested = parseLldStackSize(value); + if (requested.kind === 'overflow') { + throw new Error( + `stack-size=${value} exceeds the SDK's ${MAX_EXECUTABLE_MEMORY_SIZE}-byte executable memory limit`, + ); + } + if (requested.kind === 'valid' && requested.value > result) result = requested.value; + }; + + const lldArgs = readResponseFile + ? expandResponseFiles(linkerArgs, readResponseFile) + : linkerArgs; + for (let i = 0; i < lldArgs.length; i++) { + const arg = lldArgs[i]; + if (arg === '--') break; + + if (arg === '-z') { + const value = lldArgs[++i]; + if (value?.startsWith('stack-size=')) { + consider(value.slice('stack-size='.length)); + } + continue; + } + + if (arg.startsWith('-zstack-size=')) { + consider(arg.slice('-zstack-size='.length)); + } + } + + return result; +} + +export function linkFlags( + arch: WasmArch, + mainThreadStackSizeBytes = DEFAULT_MAIN_THREAD_STACK_SIZE, +): string[] { return [ '-nostdlib', '-Wl,--entry=_start', @@ -28,8 +257,22 @@ export function linkFlags(arch: WasmArch): string[] { '-Wl,--export=__heap_base', '-Wl,--import-memory', '-Wl,--shared-memory', - '-Wl,--max-memory=1073741824', + `-Wl,--max-memory=${MAX_EXECUTABLE_MEMORY_SIZE}`, '-Wl,--allow-undefined', + // Reserve an 8 MiB main-thread shadow stack. wasm-ld's default is only + // ~64 KiB, and WebAssembly has no stack guard page, so a deep call chain + // silently overflows past __data_end into .bss and corrupts the pthread/TLS + // globals that live there (__wasm_tp_storage, __pthread_tsd_main), which + // then surfaces as a spurious "memory access out of bounds" far from the + // real fault. POSIX leaves the default stack size implementation-defined, + // but 8 MiB is the de-facto Linux/glibc RLIMIT_STACK default that mainstream + // C software (GTK, etc.) is written and tested against, so matching it + // maximizes portability. Treat it as a floor: callers retain explicit larger + // requests. This sizes only the main thread; pthreads get their own stacks + // from musl's __default_stacksize. Cost: at least ~8 MiB of initial linear + // memory per process (it raises __heap_base 1:1). Keep in sync with the bash + // wasm32posix-cc. See docs/sdk-guide.md. + `-Wl,-z,stack-size=${mainThreadStackSizeBytes}`, '-Wl,--global-base=1114112', '-Wl,--table-base=3', '-Wl,--export-table', @@ -131,6 +374,8 @@ export interface ParsedArgs { objectFiles: string[]; archiveFiles: string[]; otherArgs: string[]; + /** Arguments forwarded to clang, in the exact order supplied by the caller. */ + forwardedArgs: string[]; } const SOURCE_EXTS = new Set(['.c', '.cc', '.cpp', '.cxx', '.m', '.mm', '.i', '.ii']); @@ -162,6 +407,7 @@ export function parseArgs(args: string[]): ParsedArgs { objectFiles: [], archiveFiles: [], otherArgs: [], + forwardedArgs: [], }; for (let i = 0; i < args.length; i++) { @@ -199,8 +445,12 @@ export function parseArgs(args: string[]): ParsedArgs { } else if (FLAGS_WITH_VALUE.has(arg)) { // Flag that takes the next arg as its value — keep both as otherArgs result.otherArgs.push(arg); + result.forwardedArgs.push(arg); i++; - if (i < args.length) result.otherArgs.push(args[i]); + if (i < args.length) { + result.otherArgs.push(args[i]); + result.forwardedArgs.push(args[i]); + } } else if (!arg.startsWith('-')) { const ext = arg.substring(arg.lastIndexOf('.')); if (SOURCE_EXTS.has(ext)) { @@ -212,8 +462,10 @@ export function parseArgs(args: string[]): ParsedArgs { } else { result.otherArgs.push(arg); } + result.forwardedArgs.push(arg); } else { result.otherArgs.push(arg); + result.forwardedArgs.push(arg); } } diff --git a/sdk/test/cc.test.ts b/sdk/test/cc.test.ts index 1e3897b4bf..e735c8095d 100644 --- a/sdk/test/cc.test.ts +++ b/sdk/test/cc.test.ts @@ -1,7 +1,12 @@ import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { describe, it, expect } from 'vitest'; -import { buildClangArgs } from '../src/bin/cc.ts'; +import { + buildClangArgs, + decodeLlvmResponseFile, + linkerArgsFromClangTrace, + workingDirectoryFromClangTrace, +} from '../src/bin/cc.ts'; describe('buildClangArgs', () => { const toolchain = { @@ -15,9 +20,17 @@ describe('buildClangArgs', () => { sysroot: '/tmp/sysroot', glueDir: '/tmp/glue', }; + const build = ( + userArgs: string[], + selectedToolchain = toolchain, + mainThreadStackSizeBytes = 8 * 1024 * 1024, + ): string[] => buildClangArgs(userArgs, selectedToolchain, 'wasm32', { + kind: 'executable-link', + mainThreadStackSizeBytes, + }); it('compile-only: adds compile flags, no link flags', () => { - const args = buildClangArgs(['-c', 'foo.c', '-o', 'foo.o'], toolchain); + const args = build(['-c', 'foo.c', '-o', 'foo.o']); expect(args).toContain('--target=wasm32-unknown-unknown'); expect(args).toContain('--sysroot=/tmp/sysroot'); expect(args).toContain('-c'); @@ -27,7 +40,7 @@ describe('buildClangArgs', () => { }); it('compile+link: adds both compile and link flags plus glue', () => { - const args = buildClangArgs(['foo.c', '-o', 'foo.wasm'], toolchain); + const args = build(['foo.c', '-o', 'foo.wasm']); expect(args).toContain('--target=wasm32-unknown-unknown'); expect(args).toContain('-Wl,--entry=_start'); expect(args).toContain('-Wl,--import-memory'); @@ -37,8 +50,64 @@ describe('buildClangArgs', () => { expect(args.join(' ')).toContain('libc.a'); }); + it('uses the 8 MiB stack floor for default and smaller requests', () => { + const defaultArgs = build(['foo.c', '-o', 'foo.wasm']); + const smallerArgs = build([ + 'foo.c', '-Wl,-z,stack-size=1048576', '-o', 'foo.wasm', + ], toolchain); + + expect(defaultArgs.filter((arg) => arg.includes('stack-size=')).at(-1)) + .toBe('-Wl,-z,stack-size=8388608'); + expect(smallerArgs.filter((arg) => arg.includes('stack-size=')).at(-1)) + .toBe('-Wl,-z,stack-size=8388608'); + }); + + it('emits the prepared larger stack after preserving the original arguments', () => { + const userArgs = [ + 'foo.c', '-Wl,-z', '-iquote', '/tmp/include', + '-Wl,stack-size=16777216', '-o', 'foo.wasm', + ]; + const args = build(userArgs, toolchain, 16 * 1024 * 1024); + const forwarded = userArgs.slice(0, -2); + + expect(args.slice(args.indexOf(forwarded[0]), args.indexOf(forwarded.at(-1)!) + 1)) + .toEqual(forwarded); + expect(args.filter((arg) => arg.includes('stack-size=')).at(-1)) + .toBe('-Wl,-z,stack-size=16777216'); + }); + + it('rejects malformed UTF-16 response text like LLVM', () => { + expect(() => decodeLlvmResponseFile(Buffer.from([ + 0xff, 0xfe, 0x00, 0xd8, + ]))).toThrow(); + expect(() => decodeLlvmResponseFile(Buffer.from([ + 0xfe, 0xff, 0xd8, 0x00, + ]))).toThrow(); + expect(() => decodeLlvmResponseFile(Buffer.from([ + 0xff, 0xfe, 0x41, + ]))).toThrow(/odd-length UTF-16LE/); + }); + + it('decodes both LLVM UTF-16 response-file encodings', () => { + const contents = '-z\nstack-size=16777216\n'; + const littleEndian = Buffer.concat([ + Buffer.from([0xff, 0xfe]), + Buffer.from(contents, 'utf16le'), + ]); + const bigEndianContents = Buffer.from(contents, 'utf16le'); + bigEndianContents.swap16(); + const bigEndian = Buffer.concat([ + Buffer.from([0xfe, 0xff]), + bigEndianContents, + ]); + + for (const encoded of [littleEndian, bigEndian]) { + expect(decodeLlvmResponseFile(encoded)).toBe(contents); + } + }); + it('link-only: object files without -c get link flags plus compile flags for glue', () => { - const args = buildClangArgs(['foo.o', 'bar.o', '-o', 'out.wasm'], toolchain); + const args = build(['foo.o', 'bar.o', '-o', 'out.wasm']); expect(args).toContain('-Wl,--entry=_start'); expect(args.join(' ')).toContain('libc.a'); expect(args).toContain('--target=wasm32-unknown-unknown'); @@ -47,37 +116,63 @@ describe('buildClangArgs', () => { expect(args.join(' ')).toContain('channel_syscall.c'); }); + it('preserves user linker input order across argument categories', () => { + const userLinkArgs = [ + 'main.o', + '-Wl,--start-group', + '-lfoo', + 'libbar.a', + '-Wl,--end-group', + ]; + const args = build([...userLinkArgs, '-o', 'out.wasm']); + const forwarded = args.slice(args.indexOf('main.o'), args.indexOf('-Wl,--end-group') + 1); + expect(forwarded).toEqual(userLinkArgs); + }); + it('preprocess-only: no link flags', () => { - const args = buildClangArgs(['-E', 'foo.c'], toolchain); + const args = build(['-E', 'foo.c']); expect(args).not.toContain('-Wl,--entry=_start'); }); + it('honors an authoritative no-link trace for direct and response-file inputs', () => { + for (const userArgs of [ + ['-fsyntax-only', 'foo.c'], + ['@/tmp/syntax-only.rsp'], + ]) { + const args = buildClangArgs(userArgs, toolchain, 'wasm32', { kind: 'no-link' }); + expect(args).toContain('-fno-trapping-math'); + expect(args).not.toContain('-fuse-ld=/opt/llvm/bin/wasm-ld'); + expect(args).not.toContain('-Wl,--entry=_start'); + expect(args.join(' ')).not.toContain('channel_syscall.c'); + } + }); + it('filters ignored flags', () => { - const args = buildClangArgs(['-c', '-pthread', '-fPIC', 'foo.c'], toolchain); + const args = build(['-c', '-pthread', '-fPIC', 'foo.c']); expect(args).not.toContain('-pthread'); expect(args).toContain('-fPIC'); }); it('normalizes equivalent configure-supplied wasm target aliases', () => { - const args = buildClangArgs(['--target=wasm32-linux-musl', '-c', 'foo.c'], toolchain); + const args = build(['--target=wasm32-linux-musl', '-c', 'foo.c']); expect(args.filter((arg) => arg.startsWith('--target='))).toEqual(['--target=wasm32-unknown-unknown']); }); it('treats linker response lists as link commands', () => { - const args = buildClangArgs(['-fuse-ld=lld', '-o', 'out.wasm', '-Wl,@/tmp/objects.list'], toolchain); + const args = build(['-fuse-ld=lld', '-o', 'out.wasm', '-Wl,@/tmp/objects.list']); expect(args).toContain('-Wl,--entry=_start'); expect(args.join(' ')).toContain('channel_syscall.c'); expect(args.join(' ')).toContain('libc.a'); }); it('emits explicit process thread slot declarations into the glue compile', () => { - const args = buildClangArgs(['--kandelo-thread-slots=2', 'foo.c', '-o', 'foo.wasm'], toolchain); + const args = build(['--kandelo-thread-slots=2', 'foo.c', '-o', 'foo.wasm']); expect(args).toContain('-DWASM_POSIX_THREAD_SLOT_DECL=2'); expect(args).not.toContain('--kandelo-thread-slots=2'); }); it('pins lld to the same resolved LLVM tree as clang', () => { - const args = buildClangArgs(['foo.c', '-o', 'foo.wasm'], toolchain); + const args = build(['foo.c', '-o', 'foo.wasm']); expect(args).toContain('-fuse-ld=/opt/llvm/bin/wasm-ld'); }); @@ -91,6 +186,100 @@ describe('buildClangArgs', () => { ).toThrow(/wasm-ld version is unresolved/); }); + it('rejects executable links without the matching Clang trace preparation', () => { + expect(() => buildClangArgs(['foo.c', '-o', 'foo.wasm'], toolchain)) + .toThrow(/executable linker arguments are unprepared/); + expect(() => build(['foo.c', '-o', 'foo.wasm'], toolchain, 1024)) + .toThrow(/prepared main-thread stack size must be an integer/); + }); + + it('extracts the exact pinned wasm-ld argv from a Clang trace', () => { + const trace = [ + 'clang version 21.1.7', + ' "/opt/llvm/bin/clang-21" "-cc1" "-iquote" "/tmp/include"', + ' "/opt/llvm/bin/wasm-ld" "-m" "wasm32" "main.o" "-z" "stack-size=16777216"', + ].join('\n'); + + expect(linkerArgsFromClangTrace(trace, '/opt/llvm/bin/wasm-ld')).toEqual([ + '-m', 'wasm32', 'main.o', '-z', 'stack-size=16777216', + ]); + expect(() => linkerArgsFromClangTrace(trace, '/other/wasm-ld')) + .toThrow(/emitted 0 commands/); + + const noLinkTrace = + ' "/opt/llvm/bin/clang-21" "-cc1" "-triple" "wasm32-unknown-unknown"'; + expect(linkerArgsFromClangTrace(noLinkTrace, '/opt/llvm/bin/wasm-ld')).toBeNull(); + expect(() => linkerArgsFromClangTrace( + ' "/opt/llvm/bin/clang-21" "-cc1"\n "/opt/llvm/bin/wasm-ld" "one.o"\n' + + ' "/opt/llvm/bin/wasm-ld" "two.o"', + '/opt/llvm/bin/wasm-ld', + )).toThrow(/emitted 2 commands/); + expect(() => linkerArgsFromClangTrace( + ' "/opt/llvm/bin/clang-21" "-cc1"\n "/other/wasm-ld" "main.o"', + '/opt/llvm/bin/wasm-ld', + )).toThrow(/emitted 0 commands/); + expect(() => linkerArgsFromClangTrace( + ' "/opt/llvm/bin/clang-21" "-cc1"\n' + + ' "/opt/llvm/bin/wasm-ld" "main.o"\n "/other/wasm-ld" "other.o"', + '/opt/llvm/bin/wasm-ld', + )).toThrow(/emitted 1 commands/); + expect(linkerArgsFromClangTrace( + ' "/opt/llvm/bin/clang-21" "-cc1"\n' + + ' "/opt/llvm/bin/wasm-ld" "main.o"\n "/opt/bin/wasm-opt" "a.wasm"', + '/opt/llvm/bin/wasm-ld', + )).toEqual(['main.o']); + expect(() => linkerArgsFromClangTrace( + ' "/opt/bin/wasm-opt" "a.wasm"', + '/opt/llvm/bin/wasm-ld', + )).toThrow(/no compiler or pinned linker jobs/); + expect(() => linkerArgsFromClangTrace( + 'clang version 21.1.7', + '/opt/llvm/bin/wasm-ld', + )).toThrow(/no recognizable jobs/); + + const newlinePathTrace = + ' "/opt/llvm/bin/wasm-ld" "main\nobject.o" "-z" "stack-size=16777216"\n'; + expect(linkerArgsFromClangTrace(newlinePathTrace, '/opt/llvm/bin/wasm-ld')).toEqual([ + 'main\nobject.o', '-z', 'stack-size=16777216', + ]); + expect(linkerArgsFromClangTrace( + `"unrelated unterminated diagnostic\n${trace}`, + '/opt/llvm/bin/wasm-ld', + )).toEqual(['-m', 'wasm32', 'main.o', '-z', 'stack-size=16777216']); + expect(() => linkerArgsFromClangTrace( + ' "/opt/llvm/bin/wasm-ld" "unterminated\n', + '/opt/llvm/bin/wasm-ld', + )).toThrow(/unterminated quoted command/); + + const workingDirectoryTrace = [ + ' "/opt/llvm/bin/clang-21" "-cc1" "-ffile-compilation-dir=/spoofed" ' + + '"-resource-dir" "/opt/llvm/lib/clang/21" "-working-directory" "/tmp/build" ' + + '"-internal-isystem" "/opt/include" "-working-directory" "/xclang-only"', + ' "/opt/llvm/bin/wasm-ld" "main.o"', + ].join('\n'); + expect(workingDirectoryFromClangTrace( + workingDirectoryTrace, + '/tmp/project', + )).toBe('/tmp/build'); + + const initialDirectoryTrace = + ' "/opt/llvm/bin/clang-21" "-cc1" "-resource-dir" "/opt/llvm/lib/clang/21" ' + + '"-internal-isystem" "/opt/include" "-working-directory" "/xclang-only"'; + expect(workingDirectoryFromClangTrace( + initialDirectoryTrace, + '/tmp/project', + )).toBe('/tmp/project'); + expect(() => workingDirectoryFromClangTrace( + ' "/opt/llvm/bin/wasm-ld" "main.o"\n', + '/tmp/project', + )).toThrow(/did not emit one consistent driver working directory/); + expect(() => workingDirectoryFromClangTrace([ + workingDirectoryTrace, + ' "/opt/llvm/bin/clang-21" "-cc1" "-resource-dir" "/opt/llvm/lib/clang/21" ' + + '"-working-directory" "/other"', + ].join('\n'), '/tmp/project')).toThrow(/did not emit one consistent driver working directory/); + }); + it('pins the packaged SDK driver to clang\'s adjacent wasm-ld', () => { const script = readFileSync( join(import.meta.dirname, '../kandelo/bin/wasm32posix-cc'), @@ -102,7 +291,7 @@ describe('buildClangArgs', () => { }); it('preserves stack-after-data layout with LLD 22 and newer', () => { - const args = buildClangArgs( + const args = build( ['foo.c', '-o', 'foo.wasm'], { ...toolchain, lldMajor: 22 }, ); @@ -111,7 +300,7 @@ describe('buildClangArgs', () => { }); it('uses LLD 21 defaults without passing its unsupported negative option', () => { - const args = buildClangArgs( + const args = build( ['foo.c', '-o', 'foo.wasm'], { ...toolchain, lldMajor: 21 }, ); diff --git a/sdk/test/flags.test.ts b/sdk/test/flags.test.ts index ad3165cb4b..26c90c617c 100644 --- a/sdk/test/flags.test.ts +++ b/sdk/test/flags.test.ts @@ -1,15 +1,23 @@ import { describe, it, expect } from 'vitest'; import { COMPILE_FLAGS, + DEFAULT_MAIN_THREAD_STACK_SIZE, filterArgs, inferThreadSlotDeclaration, LINK_FLAGS, + MAX_EXECUTABLE_MEMORY_SIZE, + MAX_RESPONSE_FILE_EXPANSIONS, + mainThreadStackSize, needsLinking, parseArgs, THREAD_SLOT_NONE, THREAD_SLOT_USE_HOST_DEFAULT, } from '../src/lib/flags.ts'; +function responseFile(contents: string, identity = '/tmp/objects.list') { + return { contents, identity }; +} + describe('filterArgs', () => { it('passes through normal flags', () => { const result = filterArgs(['-O2', '-DFOO', '-Iinclude', 'main.c']); @@ -55,6 +63,25 @@ describe('filterArgs', () => { expect(result.filtered).toEqual(['-Wl,-z,stack-size=16777216', 'main.c']); }); + it('preserves valid and invalid stack-size spellings for wasm-ld', () => { + for (const value of [ + '0100000000', + '0o100000000', + '0x1000000', + '0X1000000', + '0b1000000000000000000000000', + '0B1000000000000000000000000', + '16777216z', + '08', + '0o8', + '0xg', + '0b2', + ]) { + const flag = `-Wl,-z,stack-size=${value}`; + expect(filterArgs([flag, 'main.c']).filtered).toEqual([flag, 'main.c']); + } + }); + it('removes equivalent wasm target aliases supplied by configure scripts', () => { const result = filterArgs([ '--target=wasm32-linux-musl', @@ -106,6 +133,25 @@ describe('parseArgs', () => { expect(parsed.archiveFiles).toEqual(['libbar.a']); }); + it('retains the original order of forwarded linker inputs and controls', () => { + const parsed = parseArgs([ + 'main.o', + '-Wl,--start-group', + '-lfoo', + 'libbar.a', + '-Wl,--end-group', + '-o', + 'out.wasm', + ]); + expect(parsed.forwardedArgs).toEqual([ + 'main.o', + '-Wl,--start-group', + '-lfoo', + 'libbar.a', + '-Wl,--end-group', + ]); + }); + it('handles -ofilename (no space) syntax', () => { const parsed = parseArgs(['-c', 'foo.c', '-ofoo.o']); expect(parsed.outputFile).toBe('foo.o'); @@ -160,6 +206,205 @@ describe('LINK_FLAGS', () => { expect(LINK_FLAGS).toContain('-Wl,--import-memory'); expect(LINK_FLAGS).toContain('-Wl,--shared-memory'); }); + + it('reserves an 8 MiB main-thread shadow stack (wasm-ld default ~64 KiB is too small)', () => { + expect(LINK_FLAGS).toContain('-Wl,-z,stack-size=8388608'); + }); +}); + +describe('mainThreadStackSize', () => { + it('uses 8 MiB when no explicit stack size is requested', () => { + expect(mainThreadStackSize(['main.o'])).toBe(DEFAULT_MAIN_THREAD_STACK_SIZE); + }); + + it('raises smaller requests to the SDK floor', () => { + expect(mainThreadStackSize(['-z', 'stack-size=1048576', 'main.o'])) + .toBe(DEFAULT_MAIN_THREAD_STACK_SIZE); + }); + + it('retains the largest request in the exact lld argv', () => { + expect(mainThreadStackSize([ + '-z', 'stack-size=1048576', + 'main.o', + '-z', 'stack-size=16777216', + ])).toBe(16 * 1024 * 1024); + }); + + it('recognizes both accepted lld -z spellings', () => { + for (const args of [ + ['-zstack-size=0x1000000', 'main.o'], + ['-z', 'stack-size=0o100000000', 'main.o'], + ]) { + expect(mainThreadStackSize(args)).toBe(16 * 1024 * 1024); + } + }); + + it('requires the stack-size operand to immediately follow -z', () => { + for (const args of [ + ['-z', 'main.o', 'stack-size=33554432'], + ['-z', '-lfoo', 'stack-size=33554432'], + ['-z', '-e', 'stack-size=33554432'], + ['-z', '-r', 'stack-size=33554432'], + ]) { + expect(mainThreadStackSize(args)).toBe(DEFAULT_MAIN_THREAD_STACK_SIZE); + } + }); + + it('retains larger requests in every integer radix accepted by LLVM 21', () => { + for (const value of [ + '16777216', + '0100000000', + '0o100000000', + '0x1000000', + '0X1000000', + '0b1000000000000000000000000', + '0B1000000000000000000000000', + ]) { + expect(mainThreadStackSize(['-z', `stack-size=${value}`, 'main.o'])) + .toBe(16 * 1024 * 1024); + } + }); + + it('treats leading-zero values as octal rather than padded decimal', () => { + expect(mainThreadStackSize(['-z', 'stack-size=020000000', 'main.o'])) + .toBe(DEFAULT_MAIN_THREAD_STACK_SIZE); + expect(mainThreadStackSize( + ['@/tmp/objects.list'], + () => responseFile('-z\nstack-size=020000000\n'), + )).toBe(DEFAULT_MAIN_THREAD_STACK_SIZE); + }); + + it('retains larger radix-prefixed requests in directly referenced response files', () => { + for (const value of [ + '16777216', + '0100000000', + '0o100000000', + '0x1000000', + '0X1000000', + '0b1000000000000000000000000', + '0B1000000000000000000000000', + ]) { + expect(mainThreadStackSize( + ['@/tmp/objects.list'], + () => responseFile(`first.o\n-z\nstack-size=${value}\n`), + )).toBe(16 * 1024 * 1024); + } + }); + + it('expands nested lld response files with LLVM GNU tokenization', () => { + const files: Record = { + '/tmp/outer.rsp': '"/tmp/not-stack-size=33554432.o" @/tmp/inner\\ file.rsp', + '/tmp/inner file.rsp': "-z 'stack-size=0x1000000'", + }; + const readResponseFile = (path: string) => + files[path] === undefined ? null : responseFile(files[path], path); + + expect(mainThreadStackSize(['@/tmp/outer.rsp', 'main.o'], readResponseFile)) + .toBe(16 * 1024 * 1024); + }); + + it('only treats exact lld -z operands as stack requests', () => { + for (const args of [ + ['stack-size=33554432', 'main.o'], + ['-z=stack-size=33554432', 'main.o'], + ['/tmp/not-stack-size=33554432.o'], + ['-z', '-zstack-size=33554432'], + ['--', '-zstack-size=33554432'], + ]) { + expect(mainThreadStackSize(args)).toBe(DEFAULT_MAIN_THREAD_STACK_SIZE); + } + + expect(mainThreadStackSize( + ['@/tmp/objects.rsp'], + () => responseFile('not-stack-size=33554432.o\nstack-size=33554432\n'), + )).toBe(DEFAULT_MAIN_THREAD_STACK_SIZE); + }); + + it('does not let invalid integer digits influence the floor', () => { + for (const value of ['16777216z', '08', '0o8', '0xg', '0b2']) { + expect(mainThreadStackSize(['-z', `stack-size=${value}`, 'main.o'])) + .toBe(DEFAULT_MAIN_THREAD_STACK_SIZE); + expect(mainThreadStackSize( + ['@/tmp/objects.list'], + () => responseFile(`-z\nstack-size=${value}\n`), + )).toBe(DEFAULT_MAIN_THREAD_STACK_SIZE); + } + }); + + it('rejects requests larger than the executable memory maximum', () => { + for (const value of [ + `${MAX_EXECUTABLE_MEMORY_SIZE + 1}`, + '010000000001', + '0o10000000001', + '0x40000001', + '0b1000000000000000000000000000001', + ]) { + expect(() => mainThreadStackSize(['-z', `stack-size=${value}`, 'main.o'])) + .toThrow(/exceeds the SDK's 1073741824-byte executable memory limit/); + expect(() => mainThreadStackSize( + ['@/tmp/objects.list'], + () => responseFile(`-z\nstack-size=${value}\n`), + )).toThrow(/exceeds the SDK's 1073741824-byte executable memory limit/); + } + }); + + it('inspects response chains deeper than the former recursive cutoff', () => { + const files: Record = {}; + for (let index = 0; index < 100; index++) { + files[`/tmp/deep-${index}.rsp`] = index === 99 + ? '-z stack-size=16777216' + : `@/tmp/deep-${index + 1}.rsp`; + } + const readResponseFile = (path: string) => + files[path] === undefined ? null : responseFile(files[path], path); + + expect(mainThreadStackSize(['@/tmp/deep-0.rsp'], readResponseFile)) + .toBe(16 * 1024 * 1024); + }); + + it('expands repeated files each time so cross-file option boundaries stay exact', () => { + const files: Record = { + '/tmp/z.rsp': '-z', + }; + const readResponseFile = (path: string) => + files[path] === undefined ? null : responseFile(files[path], path); + + expect(mainThreadStackSize([ + '@/tmp/z.rsp', 'stack-size=16777216', + '@/tmp/z.rsp', 'stack-size=33554432', + ], readResponseFile)).toBe(32 * 1024 * 1024); + }); + + it('rejects missing, recursive, and alias-recursive response files', () => { + expect(() => mainThreadStackSize(['@/tmp/missing.rsp'], () => null)) + .toThrow(/cannot inspect response file/); + + const recursiveFiles: Record = { + '/tmp/a.rsp': '@/tmp/b.rsp', + '/tmp/b.rsp': '@/tmp/a.rsp', + }; + expect(() => mainThreadStackSize( + ['@/tmp/a.rsp'], + (path) => responseFile(recursiveFiles[path], path), + )).toThrow(/recursive response file/); + + expect(() => mainThreadStackSize( + ['@/tmp/a.rsp'], + (path) => responseFile('@/tmp/alias.rsp', 'same-file'), + )).toThrow(/recursive response file/); + }); + + it('fails closed when response expansion exceeds the explicit file bound', () => { + expect(() => mainThreadStackSize( + ['@0'], + (path) => { + const index = Number(path); + return responseFile(`@${index + 1}`, path); + }, + )).toThrow( + `response-file expansion exceeds the ${MAX_RESPONSE_FILE_EXPANSIONS}-file safety limit`, + ); + }); }); describe('inferThreadSlotDeclaration', () => { diff --git a/sdk/test/integration.test.ts b/sdk/test/integration.test.ts index 416b446c3d..b9260ac70d 100644 --- a/sdk/test/integration.test.ts +++ b/sdk/test/integration.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, beforeAll } from 'vitest'; import { existsSync, unlinkSync, writeFileSync, mkdirSync, readFileSync } from 'node:fs'; -import { join, resolve, dirname } from 'node:path'; +import { basename, join, resolve, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { execFileSync } from 'node:child_process'; import { resolveToolchain } from '../src/lib/toolchain.ts'; @@ -66,8 +66,8 @@ describe('integration: compile C program', () => { writeFileSync(srcFile, 'int main(void) { return 0; }\n'); const userArgs = ['-###', srcFile, '-o', outFile]; - await prepareExecutableLinker(userArgs, toolchain); - const args = buildClangArgs(userArgs, toolchain); + const executableLinker = await prepareExecutableLinker(userArgs, toolchain); + const args = buildClangArgs(userArgs, toolchain, 'wasm32', executableLinker ?? undefined); const result = await run(toolchain.cc, args); expect(result.exitCode).toBe(0); @@ -76,6 +76,247 @@ describe('integration: compile C program', () => { try { unlinkSync(outFile); } catch {} }, 30_000); + it('preserves Clang non-linking modes directly and through response files', async () => { + const toolchain = await resolveToolchain(); + mkdirSync(TMP_DIR, { recursive: true }); + const source = join(TMP_DIR, 'non-link-mode.c'); + const modes = [ + ['syntax-only', '-fsyntax-only'], + ['dependencies', '-M'], + ['user-dependencies', '-MM'], + ['analyze', '--analyze'], + ] as const; + const responses = modes.map(([name]) => join(TMP_DIR, `non-link-${name}.rsp`)); + const outputs = modes.flatMap(([name]) => [ + join(TMP_DIR, `non-link-${name}-direct.out`), + join(TMP_DIR, `non-link-${name}-response.out`), + ]); + const sourceTreeAnalyzerOutput = join(SDK_ROOT, 'non-link-mode.plist'); + writeFileSync(source, 'int square(int value) { return value * value; }\n'); + expect(existsSync(sourceTreeAnalyzerOutput)).toBe(false); + + try { + for (let index = 0; index < modes.length; index++) { + const [, mode] = modes[index]; + const directOutput = outputs[index * 2]; + const responseOutput = outputs[index * 2 + 1]; + writeFileSync( + responses[index], + `${mode} "${source}" -o "${responseOutput}"\n`, + ); + for (const userArgs of [ + [mode, source, '-o', directOutput], + [`@${responses[index]}`], + ]) { + const preparation = await prepareExecutableLinker(userArgs, toolchain); + expect(preparation).toEqual({ kind: 'no-link' }); + const args = buildClangArgs(userArgs, toolchain, 'wasm32', preparation ?? undefined); + expect(args).not.toContain(`-fuse-ld=${join(toolchain.llvmDir, 'wasm-ld')}`); + expect(args.join(' ')).not.toContain('channel_syscall.c'); + execFileSync(process.execPath, [ + '--experimental-strip-types', + join(SDK_ROOT, 'src/bin/cc.ts'), + ...userArgs, + ], { + cwd: TMP_DIR, + env: process.env, + stdio: 'pipe', + }); + } + } + expect(existsSync(sourceTreeAnalyzerOutput)).toBe(false); + } finally { + try { unlinkSync(source); } catch {} + for (const path of [...responses, ...outputs]) { + try { unlinkSync(path); } catch {} + } + } + }, 30_000); + + it('retains larger stacks across non-intervening options and UTF-16 responses', async () => { + const toolchain = await resolveToolchain(); + mkdirSync(TMP_DIR, { recursive: true }); + const source = join(TMP_DIR, 'stack-floor-probe.c'); + const newlineSource = join(TMP_DIR, 'stack-floor-\nprobe.c'); + const newlineOutput = join(TMP_DIR, 'stack-floor-newline-path.wasm'); + const pchSource = join(TMP_DIR, 'stack-floor-prefix.c'); + const pch = join(TMP_DIR, 'stack-floor-prefix.pch'); + const object = join(TMP_DIR, 'stack-floor-probe.o'); + const driverResponse = join(TMP_DIR, 'stack-floor-driver.rsp'); + const objectOutput = join(TMP_DIR, 'stack-floor-object-only.wasm'); + const debugDirectoryOutput = join(TMP_DIR, 'stack-floor-debug-directory.wasm'); + const fileDirectoryOutput = join(TMP_DIR, 'stack-floor-file-directory.wasm'); + const configuredDirectoryOutput = join(TMP_DIR, 'stack-floor-configured-directory.wasm'); + const xclangOutput = join(TMP_DIR, 'stack-floor-xclang-working-directory.wasm'); + const compilationDirectoryConfig = join(TMP_DIR, 'stack-floor-compilation-dir.cfg'); + const workingDirectoryConfig = join(TMP_DIR, 'stack-floor-working-dir.cfg'); + const configuredWorkingDirectoryOutput = join(TMP_DIR, 'stack-floor-configured-cwd.wasm'); + const environmentWorkingDirectoryOutput = join(TMP_DIR, 'stack-floor-environment-cwd.wasm'); + const deepResponseOutput = join(TMP_DIR, 'stack-floor-deep-response.wasm'); + const deepResponses = Array.from({ length: 100 }, (_, index) => + join(TMP_DIR, `stack-floor-deep-${index}.rsp`)); + const floorOutput = join(TMP_DIR, 'stack-floor-default.wasm'); + const responseOutput = join(TMP_DIR, 'stack-floor-utf16-response.wasm'); + const response = join(TMP_DIR, 'stack-floor-utf16.rsp'); + const variants = [ + ['optimization', '-O2', '-g', '-fvisibility=hidden', '-L', TMP_DIR, '-static'], + ['iquote', '-iquote', TMP_DIR], + ['include-pch', '-include-pch', pch], + ['iframework', '-iframework', TMP_DIR], + ['working-directory', '-working-directory', TMP_DIR], + ]; + const variantOutputs = variants.map(([name]) => + join(TMP_DIR, `stack-floor-${name}.wasm`)); + const paths = [ + source, newlineSource, newlineOutput, pchSource, pch, object, + driverResponse, objectOutput, debugDirectoryOutput, fileDirectoryOutput, + configuredDirectoryOutput, xclangOutput, compilationDirectoryConfig, + workingDirectoryConfig, configuredWorkingDirectoryOutput, + environmentWorkingDirectoryOutput, deepResponseOutput, ...deepResponses, + floorOutput, responseOutput, response, + ...variantOutputs, + ]; + writeFileSync(source, 'int main(void) { return 0; }\n'); + if (process.platform !== 'win32') { + writeFileSync(newlineSource, 'int main(void) { return 0; }\n'); + } + writeFileSync(pchSource, '#define KANDELO_STACK_FLOOR_TEST 1\n'); + writeFileSync(response, Buffer.concat([ + Buffer.from([0xff, 0xfe]), + Buffer.from('-z\nstack-size=16777216\n', 'utf16le'), + ])); + writeFileSync( + driverResponse, + `-working-directory "${TMP_DIR}" -Wl,@${basename(response)}\n`, + ); + writeFileSync( + compilationDirectoryConfig, + '-ffile-compilation-dir=/unrelated\n', + ); + writeFileSync( + workingDirectoryConfig, + `-working-directory=${TMP_DIR}\n`, + ); + for (let index = 0; index < deepResponses.length; index++) { + writeFileSync( + deepResponses[index], + index === deepResponses.length - 1 + ? '-z stack-size=16777216\n' + : `@${basename(deepResponses[index + 1])}\n`, + ); + } + + const link = async (userArgs: string[]): Promise => { + const executableLinker = await prepareExecutableLinker(userArgs, toolchain); + const result = await run(toolchain.cc, buildClangArgs( + userArgs, + toolchain, + 'wasm32', + executableLinker ?? undefined, + )); + expect(result.exitCode, result.stderr).toBe(0); + }; + const stackPointer = (path: string): number => { + const dump = execFileSync('wasm-objdump', ['-x', path], { encoding: 'utf8' }); + const match = dump.match(/<__stack_pointer> - init i32=(\d+)/); + expect(match, dump).not.toBeNull(); + return Number(match?.[1]); + }; + + try { + const pchResult = await run(toolchain.cc, buildClangArgs([ + '-c', '-x', 'c-header', pchSource, '-o', pch, + ], toolchain)); + expect(pchResult.exitCode, pchResult.stderr).toBe(0); + const objectResult = await run(toolchain.cc, buildClangArgs([ + '-c', source, '-o', object, + ], toolchain)); + expect(objectResult.exitCode, objectResult.stderr).toBe(0); + + await link([source, '-o', floorOutput]); + for (let index = 0; index < variants.length; index++) { + const [, ...options] = variants[index]; + await link([ + source, '-Wl,-z', ...options, + '-Wl,stack-size=16777216', '-o', variantOutputs[index], + ]); + } + await link([ + source, '-working-directory', TMP_DIR, + `-Wl,@${basename(response)}`, '-o', responseOutput, + ]); + await link([object, `@${driverResponse}`, '-o', objectOutput]); + await link([ + source, '-working-directory', TMP_DIR, + '-fdebug-compilation-dir=/unrelated', + `-Wl,@${basename(response)}`, '-o', debugDirectoryOutput, + ]); + await link([ + source, '-working-directory', TMP_DIR, + '-ffile-compilation-dir=/unrelated', + `-Wl,@${basename(response)}`, '-o', fileDirectoryOutput, + ]); + await link([ + source, '-working-directory', TMP_DIR, + `--config=${compilationDirectoryConfig}`, + `-Wl,@${basename(response)}`, '-o', configuredDirectoryOutput, + ]); + await link([ + source, '-working-directory', TMP_DIR, + '-Xclang', '-working-directory', '-Xclang', SDK_ROOT, + `-Wl,@${basename(response)}`, '-o', xclangOutput, + ]); + await link([ + source, `--config=${workingDirectoryConfig}`, + `-Wl,@${basename(response)}`, '-o', configuredWorkingDirectoryOutput, + ]); + const previousOverride = process.env.CCC_OVERRIDE_OPTIONS; + try { + process.env.CCC_OVERRIDE_OPTIONS = `+-working-directory=${TMP_DIR}`; + await link([ + source, `-Wl,@${basename(response)}`, + '-o', environmentWorkingDirectoryOutput, + ]); + } finally { + if (previousOverride === undefined) delete process.env.CCC_OVERRIDE_OPTIONS; + else process.env.CCC_OVERRIDE_OPTIONS = previousOverride; + } + await link([ + source, '-working-directory', TMP_DIR, + `-Wl,@${basename(deepResponses[0])}`, '-o', deepResponseOutput, + ]); + if (process.platform !== 'win32') { + await link([ + newlineSource, '-Wl,-z', '-Wl,stack-size=16777216', + '-o', newlineOutput, + ]); + } + + const floorStackPointer = stackPointer(floorOutput); + const largeStackPointer = stackPointer(variantOutputs[0]); + expect(largeStackPointer - floorStackPointer).toBe(8 * 1024 * 1024); + for (const output of variantOutputs) { + expect(stackPointer(output)).toBe(largeStackPointer); + } + expect(stackPointer(responseOutput)).toBe(largeStackPointer); + expect(stackPointer(objectOutput)).toBe(largeStackPointer); + expect(stackPointer(debugDirectoryOutput)).toBe(largeStackPointer); + expect(stackPointer(fileDirectoryOutput)).toBe(largeStackPointer); + expect(stackPointer(configuredDirectoryOutput)).toBe(largeStackPointer); + expect(stackPointer(xclangOutput)).toBe(largeStackPointer); + expect(stackPointer(configuredWorkingDirectoryOutput)).toBe(largeStackPointer); + expect(stackPointer(environmentWorkingDirectoryOutput)).toBe(largeStackPointer); + expect(stackPointer(deepResponseOutput)).toBe(largeStackPointer); + if (process.platform !== 'win32') { + expect(stackPointer(newlineOutput)).toBe(largeStackPointer); + } + } finally { + for (const path of paths) { + try { unlinkSync(path); } catch {} + } + } + }, 30_000); + it('compiles a hello world program to .wasm', async () => { const toolchain = await resolveToolchain(); mkdirSync(TMP_DIR, { recursive: true }); @@ -92,8 +333,8 @@ describe('integration: compile C program', () => { `); const userArgs = [srcFile, '-o', outFile]; - await prepareExecutableLinker(userArgs, toolchain); - const args = buildClangArgs(userArgs, toolchain); + const executableLinker = await prepareExecutableLinker(userArgs, toolchain); + const args = buildClangArgs(userArgs, toolchain, 'wasm32', executableLinker ?? undefined); const result = await run(toolchain.cc, args); if (result.exitCode !== 0) { @@ -128,8 +369,8 @@ describe('integration: compile C program', () => { try { const userArgs = [srcFile, '-o', outFile]; - await prepareExecutableLinker(userArgs, toolchain); - const args = buildClangArgs(userArgs, toolchain); + const executableLinker = await prepareExecutableLinker(userArgs, toolchain); + const args = buildClangArgs(userArgs, toolchain, 'wasm32', executableLinker ?? undefined); const result = await run(toolchain.cc, args); if (result.exitCode !== 0) { console.error('clang stderr:', result.stderr); @@ -172,4 +413,88 @@ describe('integration: compile C program', () => { try { unlinkSync(srcFile); } catch {} try { unlinkSync(objFile); } catch {} }, 30_000); + + it('keeps direct helper objects ahead of dependent static libraries', async () => { + const toolchain = await resolveToolchain(); + mkdirSync(TMP_DIR, { recursive: true }); + + const mainSource = join(TMP_DIR, 'link-order-main.c'); + const mainObject = join(TMP_DIR, 'link-order-main.o'); + const directHelperSource = join(TMP_DIR, 'link-order-direct-helper.c'); + const directHelperObject = join(TMP_DIR, 'link-order-direct-helper.o'); + const apiSource = join(TMP_DIR, 'link-order-api.c'); + const apiObject = join(TMP_DIR, 'link-order-api.o'); + const archiveHelperSource = join(TMP_DIR, 'link-order-archive-helper.c'); + const archiveHelperObject = join(TMP_DIR, 'link-order-archive-helper.o'); + const providerArchive = join(TMP_DIR, 'liblink-order-provider.a'); + const output = join(TMP_DIR, 'link-order.wasm'); + const paths = [ + mainSource, + mainObject, + directHelperSource, + directHelperObject, + apiSource, + apiObject, + archiveHelperSource, + archiveHelperObject, + providerArchive, + output, + ]; + + writeFileSync(mainSource, ` + extern int link_order_api(void); + int main(void) { return link_order_api(); } + `); + writeFileSync(directHelperSource, ` + int link_order_helper(void) { return 42; } + `); + writeFileSync(apiSource, ` + extern int link_order_helper(void); + int link_order_api(void) { return link_order_helper() == 42 ? 0 : 1; } + `); + writeFileSync(archiveHelperSource, ` + int link_order_helper(void) { return 7; } + `); + + try { + for (const [source, object] of [ + [mainSource, mainObject], + [directHelperSource, directHelperObject], + [apiSource, apiObject], + [archiveHelperSource, archiveHelperObject], + ]) { + const compile = await run(toolchain.cc, buildClangArgs(['-c', source, '-o', object], toolchain)); + expect(compile.exitCode, compile.stderr).toBe(0); + } + const archive = await run(toolchain.ar, ['rcs', providerArchive, apiObject, archiveHelperObject]); + expect(archive.exitCode, archive.stderr).toBe(0); + + const linkArgs = [ + mainObject, + directHelperObject, + '-L', + TMP_DIR, + '-llink-order-provider', + '-o', + output, + ]; + const executableLinker = await prepareExecutableLinker(linkArgs, toolchain); + const link = await run(toolchain.cc, buildClangArgs( + linkArgs, + toolchain, + 'wasm32', + executableLinker ?? undefined, + )); + expect(link.exitCode, link.stderr).toBe(0); + + const module = new WebAssembly.Module(readFileSync(output)); + const imports = WebAssembly.Module.imports(module).map((entry) => entry.name); + expect(imports).not.toContain('link_order_api'); + expect(imports).not.toContain('link_order_helper'); + } finally { + for (const path of paths) { + try { unlinkSync(path); } catch {} + } + } + }, 30_000); }); diff --git a/sdk/test/native-cc.test.ts b/sdk/test/native-cc.test.ts new file mode 100644 index 0000000000..459acadb07 --- /dev/null +++ b/sdk/test/native-cc.test.ts @@ -0,0 +1,452 @@ +import { execFileSync } from 'node:child_process'; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, describe, expect, it } from 'vitest'; +import { buildClangArgs } from '../src/bin/cc.ts'; +import { resolveToolchain } from '../src/lib/toolchain.ts'; + +const sdkRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const nativeCc = join(sdkRoot, 'kandelo/bin/wasm32posix-cc'); +const tempDirs: string[] = []; + +function writeExecutable(path: string, content: string): void { + writeFileSync(path, content); + chmodSync(path, 0o755); +} + +afterEach(() => { + for (const path of tempDirs.splice(0)) rmSync(path, { recursive: true, force: true }); +}); + +describe('Kandelo-native cc driver', () => { + it('applies the floor and retains larger direct and response-file requests', () => { + const root = mkdtempSync(join(tmpdir(), 'kandelo-native-cc-')); + tempDirs.push(root); + + const llvm = join(root, 'llvm'); + const sysroot = join(root, 'sysroot'); + const glue = join(root, 'glue'); + const glueObjects = join(root, 'glue-objects'); + const capture = join(root, 'linker-args.txt'); + const source = join(root, 'main.c'); + mkdirSync(llvm); + mkdirSync(join(sysroot, 'lib'), { recursive: true }); + mkdirSync(glue); + mkdirSync(glueObjects); + + writeExecutable(join(llvm, 'clang'), `#!/usr/bin/env bash +set -e +if [[ \${1:-} == -### ]]; then + case "\${WASM_POSIX_TEST_TRACE_MODE:-link}" in + foreign) + printf ' "/other/wasm-ld" "-m" "wasm32"\n' >&2 + ;; + duplicate) + printf ' "%s/wasm-ld" "-m" "wasm32"\n' "$(dirname "$0")" >&2 + printf ' "%s/wasm-ld" "-m" "wasm32"\n' "$(dirname "$0")" >&2 + ;; + *) + printf ' "%s/wasm-ld" "-m" "wasm32"\n' "$(dirname "$0")" >&2 + ;; + esac + exit 0 +fi +while [[ $# -gt 0 ]]; do + if [[ $1 == -o ]]; then : > "$2"; exit 0; fi + shift +done +`); + writeExecutable(join(llvm, 'wasm-ld'), `#!/usr/bin/env bash +set -e +if [[ \${1:-} == --version ]]; then printf '%s\n' 'LLD 21.1.7'; exit 0; fi +printf '%s\n' "$@" > "$WASM_POSIX_TEST_CAPTURE" +while [[ $# -gt 0 ]]; do + if [[ $1 == -o ]]; then : > "$2"; exit 0; fi + shift +done +`); + writeFileSync(source, 'int main(void) { return 0; }\n'); + for (const path of [ + join(sysroot, 'lib/libc.a'), + join(sysroot, 'lib/crt1.o'), + join(glue, 'channel_syscall.c'), + join(glueObjects, 'channel_syscall.o'), + join(glueObjects, 'compiler_rt.o'), + join(glueObjects, 'cxxrt.o'), + ]) writeFileSync(path, ''); + + const env = { + ...process.env, + WASM_POSIX_GLUE_DIR: glue, + WASM_POSIX_GLUE_OBJ_DIR: glueObjects, + WASM_POSIX_LLVM_DIR: llvm, + WASM_POSIX_SYSROOT: sysroot, + WASM_POSIX_TEST_CAPTURE: capture, + }; + const responseValues = { + decimalLarge: '16777216', + leadingOctalLarge: '0100000000', + explicitOctalLarge: '0o100000000', + hexLarge: '0x1000000', + hexUpperLarge: '0X1000000', + binaryLarge: '0b1000000000000000000000000', + binaryUpperLarge: '0B1000000000000000000000000', + leadingOctalSmall: '020000000', + invalidDecimal: '16777216z', + invalidOctal: '08', + invalidExplicitOctal: '0o8', + invalidHex: '0xg', + invalidBinary: '0b2', + overflowHex: '0x40000001', + }; + const responses = Object.fromEntries( + Object.entries(responseValues).map(([name, value]) => { + const path = join(root, `${name}.rsp`); + writeFileSync(path, `-z\nstack-size=${value}\n`); + return [name, path]; + }), + ); + const nestedResponse = join(root, 'inner stack.rsp'); + const outerResponse = join(root, 'outer.rsp'); + const clangResponse = join(root, 'clang.rsp'); + writeFileSync(nestedResponse, "-z 'stack-size=0x1000000'\n"); + writeFileSync( + outerResponse, + `"${join(root, 'not-stack-size=33554432.o')}" @${nestedResponse.replace(' ', '\\ ')}\n`, + ); + writeFileSync( + clangResponse, + '-Xlinker -z -Xlinker stack-size=0x1000000\n', + ); + + const cases: Array<{ + linkArgs: string[]; + expected: string; + preserved?: string[]; + }> = [ + { linkArgs: [], expected: 'stack-size=8388608' }, + { + linkArgs: ['-Wl,-z,stack-size=1048576'], + expected: 'stack-size=8388608', + preserved: ['stack-size=1048576'], + }, + { + linkArgs: ['-Wl,-z,stack-size=020000000'], + expected: 'stack-size=8388608', + preserved: ['stack-size=020000000'], + }, + ...[ + '16777216', + '0100000000', + '0o100000000', + '0x1000000', + '0X1000000', + '0b1000000000000000000000000', + '0B1000000000000000000000000', + ].map((value) => ({ + linkArgs: [`-Wl,-z,stack-size=${value}`], + expected: 'stack-size=16777216', + preserved: [`stack-size=${value}`], + })), + { + linkArgs: ['-Wl,-zstack-size=0x1000000'], + expected: 'stack-size=16777216', + preserved: ['-zstack-size=0x1000000'], + }, + { + linkArgs: ['-Xlinker', '-z', '-Xlinker', 'stack-size=0o100000000'], + expected: 'stack-size=16777216', + preserved: ['-z', 'stack-size=0o100000000'], + }, + { + linkArgs: ['-z', 'stack-size=0b1000000000000000000000000'], + expected: 'stack-size=16777216', + preserved: ['-z', 'stack-size=0b1000000000000000000000000'], + }, + { + linkArgs: [ + '-Wl,-z', '-O2', '-g', '-fvisibility=hidden', + '-Wl,stack-size=16777216', + ], + expected: 'stack-size=16777216', + preserved: ['-z', 'stack-size=16777216'], + }, + { + linkArgs: [`@${clangResponse}`], + expected: 'stack-size=16777216', + preserved: ['-z', 'stack-size=0x1000000'], + }, + { + linkArgs: [`-Wl,@${outerResponse}`], + expected: 'stack-size=16777216', + preserved: [`@${outerResponse}`], + }, + ...[ + 'decimalLarge', + 'leadingOctalLarge', + 'explicitOctalLarge', + 'hexLarge', + 'hexUpperLarge', + 'binaryLarge', + 'binaryUpperLarge', + ].map((name) => ({ + linkArgs: [`-Wl,@${responses[name]}`], + expected: 'stack-size=16777216', + preserved: [`@${responses[name]}`], + })), + { + linkArgs: [`-Wl,@${responses.leadingOctalSmall}`], + expected: 'stack-size=8388608', + preserved: [`@${responses.leadingOctalSmall}`], + }, + ...[ + '16777216z', + '08', + '0o8', + '0xg', + '0b2', + ].map((value) => ({ + linkArgs: [`-Wl,-z,stack-size=${value}`], + expected: 'stack-size=8388608', + preserved: [`stack-size=${value}`], + })), + ...[ + 'invalidDecimal', + 'invalidOctal', + 'invalidExplicitOctal', + 'invalidHex', + 'invalidBinary', + ].map((name) => ({ + linkArgs: [`-Wl,@${responses[name]}`], + expected: 'stack-size=8388608', + preserved: [`@${responses[name]}`], + })), + { + linkArgs: ['-Wl,stack-size=33554432'], + expected: 'stack-size=8388608', + preserved: ['stack-size=33554432'], + }, + { + linkArgs: ['-Wl,-z=stack-size=33554432'], + expected: 'stack-size=8388608', + preserved: ['-z=stack-size=33554432'], + }, + { + linkArgs: ['-Xlinker', 'not-stack-size=33554432.o'], + expected: 'stack-size=8388608', + preserved: ['not-stack-size=33554432.o'], + }, + { + linkArgs: ['-Wl,-z,-zstack-size=33554432'], + expected: 'stack-size=8388608', + preserved: ['-z', '-zstack-size=33554432'], + }, + ]; + + for (const { linkArgs, expected, preserved } of cases) { + execFileSync('bash', [nativeCc, source, ...linkArgs, '-o', join(root, 'out.wasm')], { + cwd: root, + env, + }); + const emitted = readFileSync(capture, 'utf8').trim().split('\n'); + expect(emitted.filter((arg) => arg.startsWith('stack-size=')).at(-1)).toBe(expected); + for (const arg of preserved ?? []) expect(emitted).toContain(arg); + } + + for (const [name, value] of Object.entries(responseValues)) { + expect(readFileSync(responses[name], 'utf8')).toBe(`-z\nstack-size=${value}\n`); + } + expect(readFileSync(nestedResponse, 'utf8')).toBe("-z 'stack-size=0x1000000'\n"); + expect(readFileSync(outerResponse, 'utf8')).toContain('not-stack-size=33554432.o'); + expect(readFileSync(clangResponse, 'utf8')) + .toBe('-Xlinker -z -Xlinker stack-size=0x1000000\n'); + + for (const linkArgs of [ + ['-Wl,-z,stack-size=1073741825'], + ['-Wl,-zstack-size=1073741825'], + ['-Xlinker', '-z', '-Xlinker', 'stack-size=1073741825'], + [`-Wl,@${responses.overflowHex}`], + ]) { + expect(() => execFileSync( + 'bash', + [nativeCc, source, ...linkArgs, '-o', join(root, 'overflow.wasm')], + { cwd: root, env, stdio: 'pipe' }, + )).toThrow(/exceeds the SDK's 1073741824-byte executable memory limit/); + } + + const utf16Little = join(root, 'utf16-le.rsp'); + const utf16Big = join(root, 'utf16-be.rsp'); + const utf16Contents = '-z\nstack-size=16777216\n'; + writeFileSync(utf16Little, Buffer.concat([ + Buffer.from([0xff, 0xfe]), + Buffer.from(utf16Contents, 'utf16le'), + ])); + const bigEndianContents = Buffer.from(utf16Contents, 'utf16le'); + bigEndianContents.swap16(); + writeFileSync(utf16Big, Buffer.concat([ + Buffer.from([0xfe, 0xff]), + bigEndianContents, + ])); + + for (const linkArgs of [ + [`@${utf16Little}`], + [`-Wl,@${utf16Big}`], + ]) { + rmSync(capture, { force: true }); + expect(() => execFileSync( + 'bash', + [nativeCc, source, ...linkArgs, '-o', join(root, 'utf16.wasm')], + { cwd: root, env, stdio: 'pipe' }, + )).toThrow(/UTF-16 response file .* is unsupported .* rewrite it as UTF-8/); + expect(existsSync(capture)).toBe(false); + } + + const cycleA = join(root, 'cycle-a.rsp'); + const cycleB = join(root, 'cycle-b.rsp'); + const aliasSource = join(root, 'alias-source.rsp'); + const aliasLink = join(root, 'alias-link.rsp'); + writeFileSync(cycleA, `@${cycleB}\n`); + writeFileSync(cycleB, `@${cycleA}\n`); + writeFileSync(aliasSource, `@${aliasLink}\n`); + symlinkSync(aliasSource, aliasLink); + for (const { linkArg, message } of [ + { linkArg: `-Wl,@${cycleA}`, message: /recursive response file/ }, + { linkArg: `-Wl,@${aliasSource}`, message: /recursive response file/ }, + { linkArg: `-Wl,@${join(root, 'missing.rsp')}`, message: /cannot inspect response file/ }, + ]) { + rmSync(capture, { force: true }); + expect(() => execFileSync( + 'bash', + [nativeCc, source, linkArg, '-o', join(root, 'rejected-response.wasm')], + { cwd: root, env, stdio: 'pipe' }, + )).toThrow(message); + expect(existsSync(capture)).toBe(false); + } + + for (const traceMode of ['foreign', 'duplicate']) { + rmSync(capture, { force: true }); + expect(() => execFileSync( + 'bash', + [nativeCc, source, '-o', join(root, 'rejected-trace.wasm')], + { + cwd: root, + env: { ...env, WASM_POSIX_TEST_TRACE_MODE: traceMode }, + stdio: 'pipe', + }, + )).toThrow(/expected exactly one/); + expect(existsSync(capture)).toBe(false); + } + }, 30_000); + + it('retains an interleaved larger request in a real LLVM link', async () => { + const root = mkdtempSync(join(tmpdir(), 'kandelo-native-cc-real-')); + tempDirs.push(root); + const toolchain = await resolveToolchain(); + const glueObjects = join(root, 'glue-objects'); + const source = join(root, 'main.c'); + const floorOutput = join(root, 'floor.wasm'); + const largeOutput = join(root, 'large.wasm'); + const deepOutput = join(root, 'deep.wasm'); + const deepResponses = Array.from({ length: 100 }, (_, index) => + join(root, `deep-${index}.rsp`)); + mkdirSync(glueObjects); + writeFileSync(source, 'int main(void) { return 0; }\n'); + for (let index = 0; index < deepResponses.length; index++) { + writeFileSync( + deepResponses[index], + index === deepResponses.length - 1 + ? '-z stack-size=16777216\n' + : `@${deepResponses[index + 1]}\n`, + ); + } + + for (const name of ['channel_syscall', 'compiler_rt', 'cxxrt']) { + const output = join(glueObjects, `${name}.o`); + execFileSync(toolchain.cc, buildClangArgs([ + '-c', join(toolchain.glueDir, `${name}.c`), '-o', output, + ], toolchain), { stdio: 'pipe' }); + } + + const env = { + ...process.env, + WASM_POSIX_GLUE_DIR: toolchain.glueDir, + WASM_POSIX_GLUE_OBJ_DIR: glueObjects, + WASM_POSIX_LLVM_DIR: toolchain.llvmDir, + WASM_POSIX_SYSROOT: toolchain.sysroot, + }; + const invoke = (args: string[]): void => { + execFileSync('bash', [nativeCc, '--kandelo-thread-slots=-1', source, ...args], { + cwd: root, + env, + stdio: 'pipe', + }); + }; + const stackPointer = (path: string): number => { + const dump = execFileSync('wasm-objdump', ['-x', path], { encoding: 'utf8' }); + const match = dump.match(/<__stack_pointer> - init i32=(\d+)/); + expect(match, dump).not.toBeNull(); + return Number(match?.[1]); + }; + + invoke(['-o', floorOutput]); + invoke([ + '-Wl,-z', '-O2', '-g', '-fvisibility=hidden', + '-Wl,stack-size=16777216', '-o', largeOutput, + ]); + invoke([`-Wl,@${deepResponses[0]}`, '-o', deepOutput]); + + expect(stackPointer(largeOutput) - stackPointer(floorOutput)) + .toBe(8 * 1024 * 1024); + expect(stackPointer(deepOutput)).toBe(stackPointer(largeOutput)); + }, 30_000); + + it('preserves real Clang non-linking modes directly and through response files', async () => { + const root = mkdtempSync(join(tmpdir(), 'kandelo-native-cc-no-link-')); + tempDirs.push(root); + const toolchain = await resolveToolchain(); + const source = join(root, 'non-link-mode.c'); + const modes = [ + ['syntax-only', '-fsyntax-only'], + ['dependencies', '-M'], + ['user-dependencies', '-MM'], + ['analyze', '--analyze'], + ] as const; + writeFileSync(source, 'int square(int value) { return value * value; }\n'); + const env = { + ...process.env, + WASM_POSIX_GLUE_DIR: toolchain.glueDir, + WASM_POSIX_GLUE_OBJ_DIR: join(root, 'unused-glue-objects'), + WASM_POSIX_LLVM_DIR: toolchain.llvmDir, + WASM_POSIX_SYSROOT: toolchain.sysroot, + }; + + for (const [name, mode] of modes) { + const response = join(root, `${name}.rsp`); + const directOutput = join(root, `${name}-direct.out`); + const responseOutput = join(root, `${name}-response.out`); + writeFileSync(response, `${mode} "${source}" -o "${responseOutput}"\n`); + for (const args of [ + [mode, source, '-o', directOutput], + [`@${response}`], + ]) { + execFileSync('bash', [nativeCc, ...args], { + cwd: root, + env, + stdio: 'pipe', + }); + } + } + expect(existsSync(join(root, 'a.out'))).toBe(false); + }, 30_000); +});