From 7ddc7613816c86adab371ff99155d19329d3978d Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 10 Nov 2026 04:48:12 -0800 Subject: [PATCH 01/10] AOT: faster loops, async code and WebAssembly, smaller image tables, stack traces as in bun run, assembler fixes On 873ab3a4, the merge of oven-sh/WebKit#1025. 545 files, +23,771 -4,315: Source/JavaScriptCore/aot 476 files +21,514 -3,920 . Source/JavaScriptCore/assembler 7 files +553 -60 . Source/JavaScriptCore/b3 24 files +456 -72 . Source/JavaScriptCore/wasm 21 files +950 -197 . Source/JavaScriptCore/runtime 10 files +125 -4 . Source/JavaScriptCore/yarr 3 files +123 -62 . Source/JavaScriptCore/llint 3 files +46 -0 . Source/JavaScriptCore/interpreter 1 file +4 -0. Nothing in heap/, WTF or bmalloc. Executables - Stack traces and error messages are closer to `bun run`: positions of constructions, destructuring and `await`, names of methods under computed keys, paths relative to the folder of the build. An async function that awaits itself through `Promise.all` is in a trace once. - Modules: `require()` of an ES module whose graph holds a file that cannot be loaded runs the CommonJS files in front of it. Errors reach every load of a graph. A file that is imported as text and as JSON no longer mixes up its forms. After an imported CommonJS file throws, the next one that an import loads is the main module, as under `bun run`. `bun test --aot` - Mocks: a package module that failed at its first load, chains of mocks whose factories take their original, mocks of files that import each other in a cycle or that fail, `resetModules()` inside a running graph. WebAssembly in the image - Frames are shrink-wrapped, the bounds checking register is not saved, direct calls pass the first mutable i32 global in a register, functions of Go's compiler are entered behind the switch on their parameter, an access that a succeeded one covers needs no check, on x64 comparisons read memory. A trap takes no lock. The build of esbuild-wasm takes 77.1 G instructions, 80.2 before its three changes to B3 and Air. Plain Bun too - arm64 with DotProd: `i32x4.relaxed_dot_i8x16_i7x16_add_s(x, x, c)` in the optimizing JIT multiplied by the addend where the result took x's register. Fixed. - The data segments of a WebAssembly module lie in one buffer, each behind a short header: esbuild.wasm has 98,450 of them. Compiled ahead of time, its short job takes 6,800 page faults instead of 8,170. - Corrected assembler operations that nothing calls with such operands: equal registers, negative immediates and masks, pairs of floats at far offsets, blinded additions, shifts by 32 and more. Eleven native tests (assembler/testmasm.cpp). Speed (instructions, by our own benchmark rows) - Async code and generators: an awaited call of a compiled function continues it in place, 687 to 202 a call; a pipeline of generators is joined, 81 to 35 an element; iterators and generators in nested loops are closed and stepped in place. - Sorting: `sort((a, b) => a - b)` from 65 numbers on goes by digits, 1,286 to 52 an element for 20,000 integers; `sort(comparator)` of typed arrays is compiled. - Strings: substrings, parseInt(), keys of a `Map` that are joined, one character of a string, split(), destructuring, TextEncoder and TextDecoder (3,811 to 1,465 a line). - Loops: lists in fields, rows of rows, nested loops, counters of any kind, views under strides and masks, sums of unknown values, BigInts, remainders, method chains, throwing callees. - Objects: comparisons of one layout, `instanceof`, `Object.hasOwn()` and `Object.keys()` of records, getters and setters of host and class objects, static fields, spread copies, constructor functions below native classes. - Results of functions of node:path and node:fs have types. JSON.stringify() of arrays of primitives, json() bodies of a declared type, regular expressions that search for two units. - arm64: shifted, extended and constant operands, shared sums, proven integers. Size and build - Image tables are coded: positions, fault sites, side tables, function records, identifiers, shown strings, names for heap snapshots. Constants are compared and loaded in place, and element accesses under a constant index take fewer bytes. - Builds that took quadratic time: chains of classes and interfaces, unions with many tests, barrels with many stars under mocks, nested array patterns. - Dead stack words are no longer cleared at entries from native code. Checks - --aot-harden=proof-checks verifies each stage. `Wire::schemaHash` changes, so this engine and Bun's side go together. Not built as one tree. The known differences from main are in Bun's message. --- Source/JavaScriptCore/aot/AOT.cmake | 2 + Source/JavaScriptCore/aot/Sources.txt | 2 + .../builtins/AOTBuiltinArrayOperations.cpp | 153 ++++ .../aot/builtins/AOTBuiltinCallEmitter.cpp | 9 +- .../aot/builtins/AOTBuiltinCallEmitter.h | 2 + .../builtins/AOTBuiltinCallEmitterNumbers.cpp | 16 + .../aot/builtins/AOTBuiltinResultTypes.cpp | 4 +- .../aot/builtins/AOTBuiltinTable.cpp | 14 + .../aot/builtins/AOTBuiltinTable.h | 1 + .../builtins/AOTHashTableBuiltinEmitter.cpp | 2 +- .../builtins/AOTMathBuiltinEmitterNumbers.cpp | 3 +- .../aot/builtins/AOTMathBuiltinOperations.cpp | 14 +- .../aot/builtins/typedarray.builtins.ts | 54 +- .../aot/codegen/AOTAccessEmitter.cpp | 78 +- .../aot/codegen/AOTAccessEmitter.h | 3 + .../aot/codegen/AOTAllocationEmitter.cpp | 11 +- .../aot/codegen/AOTAllocationEmitter.h | 2 +- .../aot/codegen/AOTArrayBuiltinEmitter.cpp | 32 +- .../aot/codegen/AOTArrayBuiltinEmitter.h | 1 + .../aot/codegen/AOTArrayIntrinsicEmitter.cpp | 11 + .../aot/codegen/AOTArrayIntrinsicEmitter.h | 1 + .../codegen/AOTArrayStorageBuiltinEmitter.cpp | 21 +- .../codegen/AOTArrayStorageBuiltinEmitter.h | 2 +- .../aot/codegen/AOTCastEmitter.cpp | 270 ++++++- .../aot/codegen/AOTCastEmitter.h | 29 +- .../aot/codegen/AOTClosureTestEmitter.cpp | 22 +- .../aot/codegen/AOTClosureTestEmitter.h | 15 +- .../aot/codegen/AOTCodeGenerator.cpp | 244 +++++- .../aot/codegen/AOTCodeGenerator.h | 79 +- .../aot/codegen/AOTConstantEmitter.cpp | 6 + .../aot/codegen/AOTConstantEmitter.h | 1 + .../codegen/AOTConstantEmitterClosures.cpp | 15 +- .../aot/codegen/AOTControlEmitter.cpp | 2 +- .../aot/codegen/AOTConversionEmitter.cpp | 89 +- .../aot/codegen/AOTConversionEmitter.h | 19 +- .../aot/codegen/AOTElementAccessEmitter.cpp | 223 ++++- .../aot/codegen/AOTElementAccessEmitter.h | 11 +- .../codegen/AOTElementAccessEmitterStores.cpp | 13 +- .../AOTElementAccessEmitterTypedArrays.cpp | 9 +- .../aot/codegen/AOTElementEmitterLiterals.cpp | 6 +- .../aot/codegen/AOTGuessedAccessEmitter.cpp | 13 +- .../JavaScriptCore/aot/codegen/AOTHeaps.cpp | 6 +- .../aot/codegen/AOTOperationEmitter.h | 1 + .../aot/codegen/AOTOperatorEmitter.cpp | 64 +- .../aot/codegen/AOTOperatorEmitter.h | 15 +- .../codegen/AOTOperatorEmitterArithmetic.cpp | 148 +++- .../aot/codegen/AOTOperatorEmitterBitwise.cpp | 8 +- .../codegen/AOTOperatorEmitterComparisons.cpp | 60 +- .../codegen/AOTOperatorEmitterConversions.cpp | 33 +- .../codegen/AOTOperatorEmitterEquality.cpp | 95 ++- .../aot/codegen/AOTPureOperations.h | 1 + .../aot/codegen/AOTReadsThatAssume.txt | 13 +- .../aot/codegen/AOTRepresentationEmitter.cpp | 9 + .../aot/codegen/AOTRuntimeCallEmitter.cpp | 7 +- .../aot/codegen/AOTSavedFrameEmitter.cpp | 74 +- .../aot/codegen/AOTSavedFrameEmitter.h | 15 +- .../aot/codegen/AOTShapeTestEmitter.cpp | 6 +- .../aot/codegen/AOTSlotAccessEmitter.cpp | 8 +- .../aot/codegen/AOTSlotEmitter.cpp | 5 +- .../aot/codegen/AOTSlotEmitter.h | 3 + .../aot/codegen/AOTStraightAllocations.cpp | 2 +- .../codegen/AOTStraightArrayIntrinsics.cpp | 3 +- .../aot/codegen/AOTStraightCasts.cpp | 10 +- .../aot/codegen/AOTStraightEmitter.cpp | 3 +- .../codegen/AOTStringBuilderPartEmitter.cpp | 6 +- .../aot/codegen/AOTStringBuiltinEmitter.cpp | 13 +- .../aot/codegen/AOTStringEmitter.cpp | 238 +++++- .../aot/codegen/AOTStringEmitter.h | 34 +- .../codegen/AOTStringEmitterComparisons.cpp | 27 +- .../aot/codegen/AOTStringSwitch.cpp | 5 +- .../aot/codegen/AOTTypeTestEmitter.cpp | 9 +- .../aot/driver/AOTResultSections.cpp | 10 +- .../aot/facts/AOTEnforcedFields.cpp | 9 +- Source/JavaScriptCore/aot/facts/AOTFacts.cpp | 34 +- Source/JavaScriptCore/aot/facts/AOTFacts.h | 3 +- .../aot/facts/AOTFactsOfProgram.cpp | 54 +- .../aot/facts/AOTFactsRegExpTexts.cpp | 14 +- Source/JavaScriptCore/aot/facts/AOTLink.cpp | 61 +- .../aot/facts/AOTLinkAccesses.cpp | 358 +++++++- .../JavaScriptCore/aot/facts/AOTLinkCalls.cpp | 137 +++- .../aot/facts/AOTLinkEffects.cpp | 5 +- .../JavaScriptCore/aot/facts/AOTLinkFlow.cpp | 356 ++++++-- Source/JavaScriptCore/aot/facts/AOTLinkFlow.h | 185 ++++- .../aot/facts/AOTLinkKnownEffectsInlines.h | 52 +- .../aot/facts/AOTLinkLayouts.cpp | 215 ++++- .../aot/facts/AOTLinkPrototypes.cpp | 109 ++- .../JavaScriptCore/aot/facts/AOTLinkReach.cpp | 6 +- .../aot/facts/AOTLinkStrings.cpp | 5 +- .../JavaScriptCore/aot/facts/AOTLinkTables.h | 25 + Source/JavaScriptCore/aot/facts/AOTScan.cpp | 202 ++++- .../JavaScriptCore/aot/facts/AOTThinSummary.h | 65 +- .../aot/facts/AOTUnitedFamilies.h | 16 +- .../aot/image/AOTCallSiteTable.cpp | 763 +++++++++++++----- .../aot/image/AOTCallSiteTable.h | 43 +- .../aot/image/AOTCodeLayout.cpp | 8 +- .../aot/image/AOTCodedTexts.cpp | 162 ++++ .../JavaScriptCore/aot/image/AOTCodedTexts.h | 38 + .../aot/image/AOTFaultTable.cpp | 411 ++++++---- .../aot/image/AOTFieldTableEncoder.cpp | 2 +- .../aot/image/AOTFunctionProgramTables.cpp | 76 +- .../aot/image/AOTFunctionSources.cpp | 72 +- .../aot/image/AOTFunctionTable.cpp | 117 ++- .../aot/image/AOTFunctionTable.h | 1 - .../aot/image/AOTIdentifierNumbering.h | 1 + Source/JavaScriptCore/aot/image/AOTImage.cpp | 75 +- Source/JavaScriptCore/aot/image/AOTImage.h | 49 +- .../aot/image/AOTImageBuilder.cpp | 126 ++- .../aot/image/AOTImageBuilder.h | 5 +- .../JavaScriptCore/aot/image/AOTImageFormat.h | 245 +++--- .../JavaScriptCore/aot/image/AOTImageText.cpp | 84 +- .../JavaScriptCore/aot/image/AOTImageText.h | 77 +- .../aot/image/AOTLayoutTables.cpp | 47 +- .../aot/image/AOTModuleLists.cpp | 4 +- .../aot/image/AOTModuleTables.cpp | 3 +- .../JavaScriptCore/aot/image/AOTNumbering.cpp | 31 +- .../JavaScriptCore/aot/image/AOTNumbering.h | 31 + .../aot/image/AOTPositionEncoder.h | 37 + .../aot/image/AOTPrefixCode.cpp | 189 +++++ .../JavaScriptCore/aot/image/AOTPrefixCode.h | 310 +++++++ .../aot/image/AOTProgramTableCheck.cpp | 16 +- .../aot/image/AOTRegExpCompiler.h | 3 + .../aot/image/AOTRegExpSection.cpp | 21 +- .../aot/image/AOTRelocation.cpp | 22 + .../JavaScriptCore/aot/image/AOTRelocation.h | 20 + .../aot/image/AOTSectionWriter.cpp | 35 +- .../aot/image/AOTSectionWriter.h | 46 +- .../aot/image/AOTSideTableCodes.h | 309 +++++++ .../aot/image/AOTSideTables.cpp | 317 +++++++- .../aot/image/AOTStaticsTables.cpp | 3 +- .../aot/image/AOTStoredStrings.cpp | 29 +- .../aot/image/AOTStringUses.cpp | 43 + .../aot/image/AOTTableParts.cpp | 5 +- .../aot/image/AOTWasmInterface.h | 38 +- .../aot/image/AOTWasmModules.cpp | 185 ++++- .../JavaScriptCore/aot/image/AOTWasmModules.h | 3 + Source/JavaScriptCore/aot/ir/AOTBodyIndex.cpp | 70 +- Source/JavaScriptCore/aot/ir/AOTBodyIndex.h | 240 ++++-- .../aot/ir/AOTBodyIndexBuilder.h | 11 +- .../aot/ir/AOTBodyVerifierOperations.cpp | 6 +- .../aot/ir/AOTDeclaredFunctionsInScopes.cpp | 3 +- .../aot/ir/AOTEffectsOfOperations.cpp | 28 +- .../aot/ir/AOTEffectsOfOperations.h | 2 + Source/JavaScriptCore/aot/ir/AOTIRProgram.cpp | 107 ++- Source/JavaScriptCore/aot/ir/AOTIRProgram.h | 31 +- .../JavaScriptCore/aot/ir/AOTMemoryLocation.h | 4 +- Source/JavaScriptCore/aot/ir/AOTOpcode.h | 9 +- Source/JavaScriptCore/aot/ir/AOTResultRules.h | 1 + .../aot/ir/AOTResultRulesConstants.cpp | 50 ++ .../aot/ir/AOTResultRulesDeclared.cpp | 9 +- .../aot/ir/AOTResultRulesInternal.h | 2 +- .../aot/ir/AOTResultRulesNatural.cpp | 16 + .../aot/ir/AOTResultRulesRefined.cpp | 6 + Source/JavaScriptCore/aot/ir/AOTTypeArena.h | 50 +- Source/JavaScriptCore/aot/ir/AOTTypeTable.cpp | 22 +- Source/JavaScriptCore/aot/ir/AOTTypeTable.h | 28 +- .../aot/ir/generator/check-runtime.ts | 1 + .../aot/ir/generator/emit-cpp-operations.ts | 18 +- .../aot/ir/generator/emit-cpp-packed.ts | 2 +- .../aot/ir/schema/compile-call.def | 4 +- .../aot/ir/schema/flow-records.def | 2 +- .../aot/ir/schema/image-tables.def | 32 +- .../aot/ir/schema/intrinsic-operations.def | 5 +- .../aot/ir/schema/operations.def | 3 +- .../aot/ir/schema/prepared-operations.def | 4 +- .../ir/schema/runtime-builtin-operations.def | 5 + .../aot/ir/schema/runtime-operations.def | 1 + .../aot/ir/schema/selected-operations.def | 10 +- .../aot/ir/schema/statistics.def | 24 +- .../machine/AOTMachineAllocationChecker.cpp | 4 + .../aot/machine/AOTMachineAllocator.cpp | 3 +- .../aot/machine/AOTMachineArena.h | 34 + .../aot/machine/AOTMachineBuilder.cpp | 78 +- .../aot/machine/AOTMachineBuilder.h | 10 + .../aot/machine/AOTMachineEncoder.h | 53 +- .../AOTMachineEncoderComparisonsARM64.cpp | 91 +++ .../AOTMachineEncoderComparisonsX86_64.cpp | 69 +- .../aot/machine/AOTMachineEncoderControl.cpp | 44 +- .../AOTMachineEncoderIntegersARM64.cpp | 164 +++- .../machine/AOTMachineEncoderMemoryX86_64.cpp | 9 + .../aot/machine/AOTMachineEncoderMoves.cpp | 37 +- .../AOTMachineEncoderNumbersX86_64.cpp | 104 ++- .../aot/machine/AOTMachineFusion.cpp | 160 +++- .../aot/machine/AOTMachineLiveness.h | 51 +- .../aot/machine/AOTMachineTarget.h | 36 +- .../aot/machine/AOTMachineTargetARM64.cpp | 56 +- .../aot/machine/AOTMachineTesting.h | 1 + .../aot/object/AOTClassGuards.cpp | 29 + .../aot/object/AOTClassTable.cpp | 11 +- .../JavaScriptCore/aot/object/AOTClassTable.h | 2 + .../JavaScriptCore/aot/object/AOTClosure.cpp | 29 +- Source/JavaScriptCore/aot/object/AOTClosure.h | 1 + .../aot/object/AOTClosureTable.cpp | 12 +- .../aot/object/AOTFunctionData.h | 73 +- .../aot/object/AOTFunctionNames.h | 1 + .../aot/object/AOTFunctionTables.cpp | 36 +- .../aot/object/AOTObjectBuilder.cpp | 14 +- .../aot/object/AOTObjectModel.cpp | 2 +- .../aot/object/AOTObjectWords.h | 2 +- .../aot/object/AOTProgramTables.h | 47 +- .../aot/object/AOTPropertyWalk.cpp | 31 + .../aot/object/AOTPropertyWalk.h | 3 + .../aot/object/AOTThisAllocation.cpp | 2 + .../aot/plan/AOTCallingConventions.cpp | 36 +- .../aot/plan/AOTFieldsStorage.cpp | 2 +- Source/JavaScriptCore/aot/plan/AOTPlan.cpp | 2 +- Source/JavaScriptCore/aot/plan/AOTPlan.h | 3 +- .../aot/plan/AOTPromotedEnvironments.cpp | 22 +- .../JavaScriptCore/aot/plan/AOTShapeTest.cpp | 2 + .../JavaScriptCore/aot/plan/AOTSlotPlan.cpp | 136 +++- Source/JavaScriptCore/aot/plan/AOTSlotPlan.h | 4 + .../aot/plan/AOTUnionConsecutiveSets.cpp | 102 ++- .../aot/plan/AOTUnionConsecutiveSets.h | 1 + .../aot/plan/AOTUnionShapeNumbers.cpp | 4 +- .../JavaScriptCore/aot/plan/AOTUnionSlots.cpp | 199 ++++- .../aot/runtime/AOTArrayOperations.cpp | 87 +- .../aot/runtime/AOTAsyncResumption.h | 4 +- .../aot/runtime/AOTBuiltinCells.cpp | 8 + .../aot/runtime/AOTCallOperations.cpp | 7 +- .../aot/runtime/AOTCompiledFrameIterator.cpp | 4 +- .../aot/runtime/AOTCoroutineWords.h | 2 + .../aot/runtime/AOTErrorMessages.cpp | 2 + .../aot/runtime/AOTGrowingText.cpp | 86 +- .../aot/runtime/AOTGrowingText.h | 3 + .../aot/runtime/AOTInstance.cpp | 15 +- .../JavaScriptCore/aot/runtime/AOTInstance.h | 4 +- .../aot/runtime/AOTInterpreterSlowPaths.cpp | 17 +- .../aot/runtime/AOTInterpreterSlowPaths.h | 1 + .../aot/runtime/AOTIterationOperations.cpp | 181 ++++- .../aot/runtime/AOTJSONParseOperations.cpp | 4 +- Source/JavaScriptCore/aot/runtime/AOTJobs.cpp | 48 +- .../aot/runtime/AOTModuleEvaluation.cpp | 170 +++- .../aot/runtime/AOTModuleHost.h | 9 +- .../aot/runtime/AOTModuleLinkWalk.h | 27 +- .../aot/runtime/AOTModuleLinking.cpp | 194 +++-- .../aot/runtime/AOTModuleNamespace.h | 3 +- .../aot/runtime/AOTModuleRecord.h | 2 + .../aot/runtime/AOTModuleTable.cpp | 60 ++ .../aot/runtime/AOTModuleTable.h | 15 + .../JavaScriptCore/aot/runtime/AOTModules.cpp | 517 ++++++++++-- .../JavaScriptCore/aot/runtime/AOTModules.h | 118 ++- .../aot/runtime/AOTOperatorOperations.cpp | 21 +- .../aot/runtime/AOTProgramStatics.h | 1 + .../aot/runtime/AOTProgramStrings.cpp | 5 +- .../aot/runtime/AOTPropertyCachesAnswers.cpp | 73 +- .../runtime/AOTPropertyCachesOperations.cpp | 25 +- .../aot/runtime/AOTPropertyCachesStringKeys.h | 142 +++- .../aot/runtime/AOTPropertyCachesWords.h | 5 + .../aot/runtime/AOTRealmStartup.h | 1 + .../aot/runtime/AOTRegExpOperations.cpp | 44 +- .../aot/runtime/AOTRegExpRecords.h | 6 +- .../aot/runtime/AOTRuntimeCheckFailure.cpp | 2 +- .../aot/runtime/AOTRuntimeFunctions.h | 2 + .../aot/runtime/AOTRuntimeHooks.cpp | 23 + .../aot/runtime/AOTRuntimeHooks.h | 3 + .../aot/runtime/AOTRuntimeRecords.h | 4 +- .../aot/runtime/AOTRuntimeTable.h | 7 + .../aot/runtime/AOTRuntimeWords.h | 3 +- .../aot/runtime/AOTStackTrace.cpp | 103 ++- .../aot/runtime/AOTStackTrace.h | 1 + Source/JavaScriptCore/aot/runtime/AOTState.h | 16 + .../aot/runtime/AOTStringBuilderStorage.cpp | 115 ++- .../aot/runtime/AOTStringBuilderStorage.h | 11 + .../aot/runtime/AOTStringOperations.cpp | 453 +++++++++-- .../aot/runtime/AOTTestRunFactories.cpp | 74 +- .../aot/runtime/AOTTestRunOriginals.cpp | 19 +- .../aot/runtime/AOTTextsOfFewDigits.h | 47 +- .../aot/runtime/AOTUnwinding.cpp | 17 +- .../JavaScriptCore/aot/runtime/AOTVMState.h | 45 ++ .../aot/stubs/AOTStubArgumentThunks.cpp | 19 +- .../JavaScriptCore/aot/stubs/AOTStubCalls.cpp | 14 +- .../JavaScriptCore/aot/stubs/AOTStubCalls.h | 14 +- .../aot/stubs/AOTStubCoreFamilies.def | 4 + .../aot/stubs/AOTStubHelpersValues.cpp | 10 +- .../aot/stubs/AOTStubsArithmetic.cpp | 29 +- .../aot/stubs/AOTStubsBoxedCalls.cpp | 60 +- .../aot/stubs/AOTStubsBoxedCallsOfStrings.cpp | 7 +- .../aot/stubs/AOTStubsCallAdapters.cpp | 82 +- .../aot/stubs/AOTStubsCallAdapters.h | 3 +- .../aot/stubs/AOTStubsCallFamilies.def | 1 + .../aot/stubs/AOTStubsCallHost.cpp | 34 + .../aot/stubs/AOTStubsCallHost.h | 1 + .../aot/stubs/AOTStubsCalls.cpp | 21 +- .../aot/stubs/AOTStubsClosures.cpp | 45 +- .../aot/stubs/AOTStubsCollections.cpp | 25 +- .../aot/stubs/AOTStubsCollections.h | 7 +- .../aot/stubs/AOTStubsConstants.cpp | 38 + .../aot/stubs/AOTStubsConstants.h | 3 + .../aot/stubs/AOTStubsCoroutineResumption.cpp | 4 +- .../aot/stubs/AOTStubsCoroutineSteps.cpp | 8 +- .../aot/stubs/AOTStubsCoroutineSuspension.cpp | 118 ++- .../aot/stubs/AOTStubsElements.cpp | 66 ++ .../aot/stubs/AOTStubsElements.h | 5 +- .../aot/stubs/AOTStubsExceptions.cpp | 24 + .../aot/stubs/AOTStubsExceptions.h | 2 + .../aot/stubs/AOTStubsHashTables.h | 2 - .../aot/stubs/AOTStubsKeyedAccess.cpp | 9 +- .../aot/stubs/AOTStubsPropertyInstanceOf.cpp | 7 +- .../stubs/AOTStubsPropertyReadHandlers.cpp | 11 +- .../stubs/AOTStubsPropertyStoreHandlers.cpp | 6 +- .../aot/stubs/AOTStubsRegExpLiterals.cpp | 11 +- .../aot/stubs/AOTStubsStringConcatenation.cpp | 155 ++-- .../aot/stubs/AOTStubsStringConversions.cpp | 33 +- .../aot/stubs/AOTStubsStringFamilies.def | 5 + .../aot/stubs/AOTStubsStringHelpers.cpp | 17 +- .../aot/stubs/AOTStubsStringLiterals.cpp | 113 ++- .../aot/stubs/AOTStubsStringSubstrings.cpp | 31 +- .../aot/stubs/AOTStubsStrings.cpp | 61 +- .../aot/stubs/AOTStubsStrings.h | 35 +- .../JavaScriptCore/aot/support/AOTLargeList.h | 2 + Source/JavaScriptCore/aot/support/AOTSort.h | 42 + .../JavaScriptCore/aot/support/AOTThreads.h | 6 +- .../aot/tests/AOTFFIStubTests.cpp | 4 +- .../aot/tests/AOTFactsTestCommands.cpp | 23 +- .../aot/tests/AOTGrowthOfWorkTests.cpp | 49 ++ .../aot/tests/AOTIDSetTests.cpp | 20 + .../aot/tests/AOTImageBuildTests.cpp | 4 +- .../aot/tests/AOTImageCodeLayoutTests.cpp | 16 + .../aot/tests/AOTImageFaultTableTests.cpp | 9 +- .../aot/tests/AOTImageFunctionTableTests.cpp | 96 ++- .../aot/tests/AOTImageRegExpTableTests.cpp | 12 +- .../aot/tests/AOTImageSiteTableTests.cpp | 131 ++- .../aot/tests/AOTImageTableTests.cpp | 18 + .../aot/tests/AOTImageTestBodies.h | 5 +- .../aot/tests/AOTImageTextTests.cpp | 113 ++- .../aot/tests/AOTMachineAllocationTests.cpp | 2 +- .../aot/tests/AOTMachineOperationTests.cpp | 63 +- .../aot/tests/AOTMachineRandomProcedure.cpp | 90 ++- .../aot/tests/AOTMachineRandomTests.cpp | 29 +- .../aot/tests/AOTModuleProgramTests.cpp | 2 +- .../aot/tests/AOTProgramTestCommands.cpp | 48 +- .../aot/tests/AOTPropertyLookupTableTests.cpp | 90 +++ .../aot/tests/AOTPropertyLookupTests.cpp | 3 +- .../aot/tests/AOTRegExpCompilerTests.cpp | 16 +- .../aot/tests/AOTRuntimeOperationTests.cpp | 3 +- .../aot/tests/AOTSavedFrameTests.cpp | 258 ------ .../aot/tests/AOTStubBlobTests.cpp | 4 +- .../AOTStubBoxedCallsOfLanguageTests.cpp | 18 +- .../aot/tests/AOTStubPropertyStoreTests.cpp | 4 +- .../aot/tests/AOTStubRuntimeTests.cpp | 6 + .../aot/tests/AOTStubValueTests.cpp | 60 ++ .../tests/AOTUnionConsecutiveSetsTests.cpp | 49 ++ .../aot/tests/AOTWebAssemblyImageTests.cpp | 1 + .../aot/tests/host/stand-in.builtins.ts | 2 +- ...keeps-no-field-that-a-type-alone-spells.ir | 1 + ...t-nothing-evaluates-is-not-in-the-image.ir | 5 +- ...a-type-alone-spells-is-not-in-the-image.ir | 1 + ...-that-nothing-names-is-not-in-the-image.ir | 5 +- ...alk-alone-rules-out-is-not-in-the-image.ir | 5 +- ...nd-function-passes-a-test-for-functions.ir | 2 +- ...w-functions-that-met-reaches-them-alone.ir | 2 +- ...et-reach-unseen-code-without-the-others.ir | 2 +- .../tests/link/bit-operation-gives-int32.ir | 2 +- ...that-has-met-both-unites-their-families.ir | 78 +- ...that-have-never-met-share-no-parameters.ir | 76 +- ...-helper-only-names-are-not-united-there.ir | 102 ++- ...t-to-unseen-code-and-the-others-stay-in.ir | 76 +- ...are-reached-where-one-of-them-is-called.ir | 52 +- .../held-contents-of-escaped-receivers.ir | 126 +++ ...one-is-called-by-the-rows-of-its-target.ir | 2 +- .../regexp-texts/export-under-an-index.ir | 31 + .../eight-closures-are-a-set.ir | 71 ++ .../four-closures-are-a-set.ir | 47 -- ...-are-lost.ir => nine-closures-are-lost.ir} | 42 +- ...o-whoever-the-rest-list-is-passed-on-to.ir | 58 +- ...-elements-that-the-table-lists-has-code.ir | 8 +- ...ches-for-are-tested-in-a-hardened-build.ir | 2 +- ...-loop-is-called-in-the-copy-by-its-rows.ir | 2 +- .../spread-push-in-checked-code.ir | 53 ++ ...rray-storage-pointer-read-after-reserve.ir | 2 +- ...stale-array-storage-pointer-is-rejected.ir | 2 +- ...held-against-the-length-of-another-list.ir | 2 +- .../direct-calls/list-callback-used-twice.ir | 57 ++ ...-a-loop.ir => list-callback-with-calls.ir} | 10 +- .../unboxed-results-of-escaped-closures.ir | 33 + ...-are-made-on-the-way-that-hands-them-on.ir | 10 +- ...ise-settles-to-is-expected-of-the-await.ir | 24 +- ...e-all-written-out-is-copied-into-a-loop.ir | 8 +- .../leaf-copied-into-recursive-function.ir | 77 ++ ...text-keeps-no-builder-alive-at-an-await.ir | 46 +- ...ng-text-that-a-turn-reads-gets-a-string.ir | 132 ++- ...xt-in-a-builder-is-asked-of-the-builder.ir | 7 +- ...steps-by-a-whole-number-has-no-fraction.ir | 2 +- ...um-lies-inside-or-the-turn-begins-again.ir | 3 +- ...test-where-the-position-may-be-negative.ir | 4 +- .../loops/builtin-test-of-record-folds.ir | 40 + .../capacity-of-declared-array-literal.ir | 48 ++ .../loops/deleted-field-of-fresh-record.ir | 51 ++ .../walk/loops/final-read-of-local-list.ir | 101 +++ .../tests/walk/loops/keys-of-two-records.ir | 85 ++ .../walk/loops/loop-over-keys-of-record.ir | 96 +++ .../object-prototype-method-of-record.ir | 75 ++ .../walk/loops/own-property-test-of-record.ir | 96 +++ .../loops/property-read-again-behind-test.ir | 81 ++ .../pushes-onto-declared-array-literal.ir | 70 ++ .../literal-or-number-stored-to-slot.ir | 49 ++ ...in-heirs-with-named-members-are-guessed.ir | 109 +++ .../aot/walk/AOTArrayStorageForms.h | 4 +- Source/JavaScriptCore/aot/walk/AOTCode.cpp | 2 +- Source/JavaScriptCore/aot/walk/AOTCode.h | 3 +- .../aot/walk/AOTCodeVerifier.cpp | 24 +- .../aot/walk/AOTInlineBuiltinForms.cpp | 2 +- .../aot/walk/AOTRepresentationRules.cpp | 14 +- .../aot/walk/AOTSumOverRecords.cpp | 2 +- Source/JavaScriptCore/aot/walk/AOTSweep.cpp | 2 + Source/JavaScriptCore/aot/walk/AOTSweep.h | 3 +- .../aot/walk/AOTSweepStores.cpp | 304 +++++-- Source/JavaScriptCore/aot/walk/AOTWalk.cpp | 276 ++++++- Source/JavaScriptCore/aot/walk/AOTWalk.h | 171 +++- .../JavaScriptCore/aot/walk/AOTWalkBirths.h | 2 +- .../aot/walk/AOTWalkBirthsListed.cpp | 2 + .../aot/walk/AOTWalkBuiltins.cpp | 157 +++- .../JavaScriptCore/aot/walk/AOTWalkCalls.cpp | 34 +- .../aot/walk/AOTWalkChoices.cpp | 12 + .../aot/walk/AOTWalkCopiesOwnKeys.cpp | 22 +- .../aot/walk/AOTWalkCoroutineAwaitedCalls.cpp | 10 +- .../aot/walk/AOTWalkCoroutineCalls.h | 4 + .../aot/walk/AOTWalkCoroutinePromises.h | 1 + .../aot/walk/AOTWalkCoroutineThenables.cpp | 5 + .../aot/walk/AOTWalkCoroutines.cpp | 9 +- .../aot/walk/AOTWalkCountersOfLoops.cpp | 540 ++++++++++++- .../aot/walk/AOTWalkCountersOfLoops.h | 6 + .../aot/walk/AOTWalkEnvironmentCells.cpp | 294 ++++++- .../aot/walk/AOTWalkEscapeAnalysis.cpp | 2 +- .../aot/walk/AOTWalkFolding.cpp | 54 +- .../aot/walk/AOTWalkFrontsOfLoops.cpp | 6 + .../aot/walk/AOTWalkGeneratorStepsByHand.cpp | 94 ++- .../aot/walk/AOTWalkGeneratorsInLoops.cpp | 227 +++++- .../aot/walk/AOTWalkGeneratorsInLoops.h | 7 + .../aot/walk/AOTWalkGuesses.cpp | 624 ++++++++++++-- .../aot/walk/AOTWalkInlining.cpp | 212 ++++- .../aot/walk/AOTWalkIteration.cpp | 221 ++++- Source/JavaScriptCore/aot/walk/AOTWalkKnown.h | 6 + .../aot/walk/AOTWalkKnownAnswers.cpp | 48 +- .../aot/walk/AOTWalkKnownElements.cpp | 24 +- .../aot/walk/AOTWalkKnownRows.cpp | 16 +- .../aot/walk/AOTWalkLinksOfChains.h | 2 +- .../aot/walk/AOTWalkListReadsInLoops.cpp | 495 ++++++++++-- .../aot/walk/AOTWalkListReadsInLoops.h | 33 +- .../aot/walk/AOTWalkListedOwnKeys.cpp | 16 + .../aot/walk/AOTWalkListedOwnKeys.h | 1 + .../aot/walk/AOTWalkLoopInvariants.cpp | 192 ++++- .../JavaScriptCore/aot/walk/AOTWalkLoops.cpp | 145 +++- .../aot/walk/AOTWalkLoopsBuiltTwice.cpp | 336 +++++++- .../aot/walk/AOTWalkLoopsOverGenerators.cpp | 102 ++- .../aot/walk/AOTWalkNumberExpectations.cpp | 20 + .../aot/walk/AOTWalkNumberForms.h | 1 + .../aot/walk/AOTWalkNumberGuesses.h | 1 + .../aot/walk/AOTWalkNumberKeys.cpp | 2 +- .../aot/walk/AOTWalkNumberOperators.cpp | 89 ++ .../aot/walk/AOTWalkNumberQuotients.cpp | 74 +- .../aot/walk/AOTWalkNumbers.cpp | 34 +- .../aot/walk/AOTWalkReplacements.cpp | 6 +- .../aot/walk/AOTWalkStableBindings.cpp | 60 ++ Source/JavaScriptCore/aot/walk/AOTWalkState.h | 6 +- .../aot/walk/AOTWalkStringCharacters.cpp | 5 +- .../aot/walk/AOTWalkStringComparisons.cpp | 5 +- .../aot/walk/AOTWalkStringFlattening.cpp | 26 +- .../aot/walk/AOTWalkStringOperandsAsMade.cpp | 85 +- .../aot/walk/AOTWalkStringPieces.cpp | 80 +- .../aot/walk/AOTWalkStringSearches.cpp | 30 +- .../JavaScriptCore/aot/walk/AOTWalkStrings.h | 4 + .../aot/walk/AOTWalkTailCalls.cpp | 8 + .../aot/walk/AOTWalkTailCallsPastCleanups.cpp | 28 +- .../walk/AOTWalkTestsOfParameterGuards.cpp | 17 +- .../aot/walk/AOTWalkTextBuilderReads.cpp | 37 +- .../aot/walk/AOTWalkTextBuilderWebs.cpp | 87 +- .../aot/walk/AOTWalkTurnsWrittenOut.cpp | 248 +++++- .../aot/walk/AOTWalkTypedAccesses.cpp | 225 +++++- .../aot/walk/AOTWalkTypedElements.cpp | 484 +++++++++-- .../aot/walk/AOTWalkTypedTexts.cpp | 46 ++ .../aot/walk/AOTWalkUnionInstanceOf.cpp | 62 +- .../aot/walk/AOTWalkUnionMethodCalls.cpp | 56 +- .../aot/walk/AOTWalkUnionTableCalls.cpp | 113 ++- .../aot/walk/AOTWalkWrittenDescriptors.cpp | 2 + .../JavaScriptCore/assembler/ARM64Assembler.h | 10 - .../JavaScriptCore/assembler/LinkBuffer.cpp | 2 +- .../JavaScriptCore/assembler/MacroAssembler.h | 27 +- .../assembler/MacroAssemblerARM64.h | 25 +- .../assembler/MacroAssemblerX86_64.h | 112 ++- .../JavaScriptCore/assembler/X86Assembler.h | 32 + Source/JavaScriptCore/assembler/testmasm.cpp | 405 +++++++++- .../b3/B3EliminateCommonSubexpressions.cpp | 3 +- Source/JavaScriptCore/b3/B3FixSSA.cpp | 49 +- Source/JavaScriptCore/b3/B3LowerToAir.cpp | 49 +- Source/JavaScriptCore/b3/B3PatchpointValue.h | 4 + .../JavaScriptCore/b3/B3RangeReductions.cpp | 20 +- Source/JavaScriptCore/b3/B3RangeReductions.h | 1 + Source/JavaScriptCore/b3/B3ReduceStrength.cpp | 3 + Source/JavaScriptCore/b3/B3ThreadJumps.cpp | 18 + .../b3/air/AirAllocateRegistersByGreedy.cpp | 81 +- .../b3/air/AirCheckSpillSlots.h | 3 +- Source/JavaScriptCore/b3/air/AirCode.cpp | 1 + .../JavaScriptCore/b3/air/AirExtendedForms.h | 8 +- Source/JavaScriptCore/b3/air/AirInst.h | 2 + .../JavaScriptCore/b3/air/AirOpcode.opcodes | 5 + Source/JavaScriptCore/b3/air/AirPolicy.cpp | 3 + Source/JavaScriptCore/b3/air/AirPolicy.h | 8 + .../b3/air/AirReuseConditionFlags.cpp | 50 +- .../b3/air/AirReuseConditionFlags.h | 5 +- .../b3/air/AirShrinkWrapFrame.cpp | 80 +- Source/JavaScriptCore/b3/air/AirValidate.cpp | 2 +- .../JavaScriptCore/b3/air/testair_bun_flags.h | 10 +- .../JavaScriptCore/b3/testb3_bun_ranges.cpp | 46 ++ .../b3/testb3_bun_reductions.cpp | 51 +- .../b3/testb3_bun_thread_jumps.cpp | 26 + .../interpreter/Interpreter.cpp | 4 + .../JavaScriptCore/llint/LLIntSlowPaths.cpp | 7 + Source/JavaScriptCore/llint/LLIntSlowPaths.h | 1 + .../llint/LowLevelInterpreterAOT.asm | 38 + Source/JavaScriptCore/runtime/Error.cpp | 2 + .../runtime/InternalFunction.cpp | 7 + .../runtime/JSCustomGetterFunction.h | 3 + Source/JavaScriptCore/runtime/JSFunction.cpp | 3 +- Source/JavaScriptCore/runtime/JSMicrotask.cpp | 2 +- .../JavaScriptCore/runtime/JSModuleLoader.cpp | 7 +- Source/JavaScriptCore/runtime/JSONObject.cpp | 91 +++ .../JavaScriptCore/runtime/LiteralParser.cpp | 10 + .../runtime/ObjectConstructor.cpp | 2 +- Source/JavaScriptCore/runtime/StackFrame.cpp | 2 + Source/JavaScriptCore/wasm/WasmCallee.cpp | 12 + Source/JavaScriptCore/wasm/WasmCallee.h | 3 + .../JavaScriptCore/wasm/WasmCalleeGroup.cpp | 8 + Source/JavaScriptCore/wasm/WasmCalleeGroup.h | 1 + .../JavaScriptCore/wasm/WasmCodeInImage.cpp | 64 +- Source/JavaScriptCore/wasm/WasmCodeInImage.h | 13 +- .../wasm/WasmCompilationContext.h | 3 + .../wasm/WasmFaultSignalHandler.cpp | 13 +- Source/JavaScriptCore/wasm/WasmFormat.cpp | 50 +- Source/JavaScriptCore/wasm/WasmFormat.h | 86 +- .../wasm/WasmIPIntGenerator.cpp | 1 + Source/JavaScriptCore/wasm/WasmIPIntPlan.cpp | 7 +- Source/JavaScriptCore/wasm/WasmIPIntPlan.h | 1 + .../wasm/WasmIPIntSlowPaths.cpp | 2 + Source/JavaScriptCore/wasm/WasmModule.cpp | 17 +- .../wasm/WasmModuleInformation.h | 3 +- .../wasm/WasmOMGIRGenerator.cpp | 711 ++++++++++++++-- .../JavaScriptCore/wasm/WasmOMGIRGenerator.h | 2 + .../JavaScriptCore/wasm/WasmSectionParser.cpp | 94 +-- .../wasm/js/JSWebAssemblyInstance.cpp | 15 +- .../wasm/js/WebAssemblyModuleRecord.cpp | 41 +- Source/JavaScriptCore/yarr/YarrJIT.cpp | 74 +- .../yarr/YarrSharedRoutines.cpp | 106 ++- .../JavaScriptCore/yarr/YarrSharedRoutines.h | 5 +- 543 files changed, 23674 insertions(+), 4218 deletions(-) create mode 100644 Source/JavaScriptCore/aot/image/AOTCodedTexts.cpp create mode 100644 Source/JavaScriptCore/aot/image/AOTCodedTexts.h create mode 100644 Source/JavaScriptCore/aot/image/AOTPositionEncoder.h create mode 100644 Source/JavaScriptCore/aot/image/AOTPrefixCode.cpp create mode 100644 Source/JavaScriptCore/aot/image/AOTPrefixCode.h create mode 100644 Source/JavaScriptCore/aot/image/AOTSideTableCodes.h create mode 100644 Source/JavaScriptCore/aot/tests/link/held-contents-of-escaped-receivers.ir create mode 100644 Source/JavaScriptCore/aot/tests/link/regexp-texts/export-under-an-index.ir create mode 100644 Source/JavaScriptCore/aot/tests/link/sets-of-closures/eight-closures-are-a-set.ir delete mode 100644 Source/JavaScriptCore/aot/tests/link/sets-of-closures/four-closures-are-a-set.ir rename Source/JavaScriptCore/aot/tests/link/sets-of-closures/{five-closures-are-lost.ir => nine-closures-are-lost.ir} (54%) create mode 100644 Source/JavaScriptCore/aot/tests/walk/builtin-calls/spread-push-in-checked-code.ir create mode 100644 Source/JavaScriptCore/aot/tests/walk/direct-calls/list-callback-used-twice.ir rename Source/JavaScriptCore/aot/tests/walk/direct-calls/{method-of-a-list-handed-a-function-that-calls-others-is-called-in-a-loop.ir => list-callback-with-calls.ir} (86%) create mode 100644 Source/JavaScriptCore/aot/tests/walk/direct-calls/unboxed-results-of-escaped-closures.ir create mode 100644 Source/JavaScriptCore/aot/tests/walk/inlining/leaf-copied-into-recursive-function.ir create mode 100644 Source/JavaScriptCore/aot/tests/walk/loops/builtin-test-of-record-folds.ir create mode 100644 Source/JavaScriptCore/aot/tests/walk/loops/capacity-of-declared-array-literal.ir create mode 100644 Source/JavaScriptCore/aot/tests/walk/loops/deleted-field-of-fresh-record.ir create mode 100644 Source/JavaScriptCore/aot/tests/walk/loops/final-read-of-local-list.ir create mode 100644 Source/JavaScriptCore/aot/tests/walk/loops/keys-of-two-records.ir create mode 100644 Source/JavaScriptCore/aot/tests/walk/loops/loop-over-keys-of-record.ir create mode 100644 Source/JavaScriptCore/aot/tests/walk/loops/object-prototype-method-of-record.ir create mode 100644 Source/JavaScriptCore/aot/tests/walk/loops/own-property-test-of-record.ir create mode 100644 Source/JavaScriptCore/aot/tests/walk/loops/property-read-again-behind-test.ir create mode 100644 Source/JavaScriptCore/aot/tests/walk/loops/pushes-onto-declared-array-literal.ir create mode 100644 Source/JavaScriptCore/aot/tests/walk/typed-slots/literal-or-number-stored-to-slot.ir create mode 100644 Source/JavaScriptCore/aot/tests/walk/typed-slots/plain-heirs-with-named-members-are-guessed.ir diff --git a/Source/JavaScriptCore/aot/AOT.cmake b/Source/JavaScriptCore/aot/AOT.cmake index bf7f8c16e180f..31b7493fe5ed5 100644 --- a/Source/JavaScriptCore/aot/AOT.cmake +++ b/Source/JavaScriptCore/aot/AOT.cmake @@ -74,8 +74,10 @@ set(JavaScriptCore_AOT_EXPORTED_HEADERS aot/image/AOTImageForward.h aot/image/AOTImageLoading.h aot/image/AOTImageText.h + aot/image/AOTPrefixCode.h aot/image/AOTRegExpSummary.h aot/image/AOTRuntimeDescription.h + aot/image/AOTSideTableCodes.h aot/image/AOTStoredStrings.h aot/image/AOTVarint.h aot/image/AOTWasmInterface.h diff --git a/Source/JavaScriptCore/aot/Sources.txt b/Source/JavaScriptCore/aot/Sources.txt index e13643d8d1df9..e64857f35ca9e 100644 --- a/Source/JavaScriptCore/aot/Sources.txt +++ b/Source/JavaScriptCore/aot/Sources.txt @@ -201,6 +201,7 @@ aot/image/AOTCallSiteTable.cpp aot/image/AOTClassRecords.cpp aot/image/AOTCodeLayout.cpp aot/image/AOTCodeLinker.cpp +aot/image/AOTCodedTexts.cpp aot/image/AOTConstantPool.cpp aot/image/AOTConstantTables.cpp aot/image/AOTDataTable.cpp @@ -222,6 +223,7 @@ aot/image/AOTModuleTables.cpp aot/image/AOTNumberConstants.cpp aot/image/AOTNumberFunctions.cpp aot/image/AOTNumbering.cpp +aot/image/AOTPrefixCode.cpp aot/image/AOTProgramTablesDriver.cpp aot/image/AOTRegExpCompiler.cpp aot/image/AOTRegExpInProcessLoader.cpp diff --git a/Source/JavaScriptCore/aot/builtins/AOTBuiltinArrayOperations.cpp b/Source/JavaScriptCore/aot/builtins/AOTBuiltinArrayOperations.cpp index d9059a5a0716e..41691865bf10e 100644 --- a/Source/JavaScriptCore/aot/builtins/AOTBuiltinArrayOperations.cpp +++ b/Source/JavaScriptCore/aot/builtins/AOTBuiltinArrayOperations.cpp @@ -28,9 +28,12 @@ #include #include #include +#include +#include #include #include #include +#include #include #include #include @@ -429,12 +432,39 @@ JSC_DEFINE_JIT_OPERATION(operationAOTArraySearchOfAnyValue, EncodedJSValue, (Ins OPERATION_RETURN(scope, encodedJSValue()); } +static bool tryUnshiftItemFast(VM& vm, JSArray* array, JSValue item) +{ + IndexingType mode = array->indexingMode(); + bool itemFits = mode == ArrayWithContiguous || (mode == ArrayWithInt32 && item.isInt32()) || (mode == ArrayWithDouble && item.isNumber() && item.asNumber() == item.asNumber()); + if (!itemFits) + return false; + Butterfly* butterfly = array->butterfly(); + unsigned length = butterfly->publicLength(); + if (length > JSArray::shiftThreshold || length >= butterfly->vectorLength()) + return false; + butterfly->setPublicLength(length + 1); + WriteBarrier* elements = butterfly->contiguous().data(); + if (mode == ArrayWithContiguous && vm.heap.mutatorShouldBeFenced()) + gcSafeMemmove(elements + 1, elements, length * sizeof(JSValue)); + else + memmove(static_cast(elements + 1), static_cast(elements), length * sizeof(JSValue)); + if (mode == ArrayWithDouble) + butterfly->contiguousDouble().atUnsafe(0) = item.asNumber(); + else + elements[0].setWithoutWriteBarrier(item); + if (mode == ArrayWithContiguous) + vm.writeBarrier(array); + return true; +} + JSC_DEFINE_JIT_OPERATION(operationAOTArrayUnshift, EncodedJSValue, (Instance* instance, JSCell* cell, EncodedJSValue item)) { AOT_OPERATION_BEGIN(instance, Entry::operationAOTArrayUnshift); JSArray* array = cellArgument(cell, Entry::operationAOTArrayUnshift); if (mayRejectOrInterceptWrites(array) || array->structure()->holesMustForwardToPrototype(array)) [[unlikely]] OPERATION_RETURN(scope, encodedJSValue()); + if (tryUnshiftItemFast(vm, array, JSValue::decode(item))) [[likely]] + OPERATION_RETURN(scope, JSValue::encode(jsNumber(array->length()))); uint64_t length = array->length(); unshift(globalObject, array, 0, 0, 1, length); OPERATION_RETURN_IF_EXCEPTION(scope, encodedJSValue()); @@ -445,6 +475,127 @@ JSC_DEFINE_JIT_OPERATION(operationAOTArrayUnshift, EncodedJSValue, (Instance* in OPERATION_RETURN(scope, JSValue::encode(jsNumber(length + 1))); } +static constexpr unsigned bitsOfSortDigit = 11; +static constexpr unsigned numberOfSortDigits = 1 << bitsOfSortDigit; + +template static constexpr size_t minLengthSortedByDigits = sizeof(Key) > sizeof(uint32_t) ? 2048 : 512; + +template +static std::span sortKeysByDigits(std::span keys, std::span scratch) +{ + constexpr unsigned numberOfPasses = (sizeof(Key) * CHAR_BIT + bitsOfSortDigit - 1) / bitsOfSortDigit; + using CountsOfPass = std::array; + auto digitOf = [](Key key, unsigned pass) -> unsigned { + return static_cast(key >> (pass * bitsOfSortDigit)) & (numberOfSortDigits - 1); + }; + void* memory = nullptr; + if (!tryFastZeroedMalloc(numberOfPasses * sizeof(CountsOfPass)).getValue(memory)) + return { }; + auto counts = std::span { static_cast(memory), numberOfPasses }; + for (Key key : keys) { + for (unsigned pass = 0; pass < numberOfPasses; ++pass) + ++counts[pass][digitOf(key, pass)]; + } + for (unsigned pass = 0; pass < numberOfPasses; ++pass) { + CountsOfPass& places = counts[pass]; + if (places[digitOf(keys[0], pass)] == keys.size()) + continue; + uint32_t place = 0; + for (uint32_t& count : places) + place += std::exchange(count, place); + for (Key key : keys) + scratch[places[digitOf(key, pass)]++] = key; + std::swap(keys, scratch); + } + fastFree(memory); + return keys; +} + +template +static bool sortElementsByKey(std::span elements, NOESCAPE const auto& tryMakeKey, NOESCAPE const auto& elementOf) +{ + bool usesDigits = elements.size() >= minLengthSortedByDigits && elements.size() <= std::numeric_limits::max(); + void* memory = nullptr; + if (!tryFastMalloc((usesDigits ? 2 : 1) * elements.size() * sizeof(Key)).getValue(memory)) + return false; + auto keys = std::span { static_cast(memory), elements.size() }; + bool hasKeys = true; + for (size_t i = 0; i < elements.size() && hasKeys; ++i) + hasKeys = tryMakeKey(elements[i], keys[i]); + std::span sorted; + if (hasKeys && usesDigits) + sorted = sortKeysByDigits(keys, std::span { static_cast(memory) + elements.size(), elements.size() }); + else if (hasKeys) { + std::sort(keys.begin(), keys.end()); + sorted = keys; + } + for (size_t i = 0; i < sorted.size(); ++i) + elements[i] = elementOf(sorted[i]); + fastFree(memory); + return !sorted.empty(); +} + +static bool tryMakeSortKeyOfDouble(double number, uint64_t mask, uint64_t& key) +{ + constexpr uint64_t signBit = 1ULL << 63; + uint64_t bits = std::bit_cast(number); + if (number != number || bits == signBit) [[unlikely]] + return false; + key = (bits ^ (static_cast(static_cast(bits) >> 63) | signBit)) ^ mask; + return true; +} + +static double doubleOfSortKey(uint64_t key, uint64_t mask) +{ + constexpr uint64_t signBit = 1ULL << 63; + key ^= mask; + return std::bit_cast(key ^ (key & signBit ? signBit : ~0ULL)); +} + +JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationAOTSortNumbers, size_t, (Instance*, JSCell* cell, int32_t direction)) +{ + JSArray* array = cellArgument(cell, Entry::operationAOTSortNumbers); + Butterfly& butterfly = *array->butterfly(); + size_t length = butterfly.publicLength(); + uint64_t mask = direction < 0 ? ~0ULL : 0; + switch (array->indexingMode()) { + case ArrayWithDouble: + return sortElementsByKey(std::span { butterfly.contiguousDouble().data(), length }, [&](double number, uint64_t& key) { + return tryMakeSortKeyOfDouble(number, mask, key); + }, [&](uint64_t key) { + return doubleOfSortKey(key, mask); + }); + case ArrayWithInt32: + case ArrayWithContiguous: { + auto elements = std::span { std::bit_cast(butterfly.contiguous().data()), length }; + bool holdsOnlyInt32s = std::ranges::all_of(elements, [](EncodedJSValue element) { + return JSValue::decode(element).isInt32(); + }); + if (holdsOnlyInt32s) { + auto flipped = [&](uint32_t bits) { + return bits ^ static_cast(mask) ^ (1U << 31); + }; + return sortElementsByKey(elements, [&](EncodedJSValue element, uint32_t& key) { + key = flipped(static_cast(JSValue::decode(element).asInt32())); + return true; + }, [&](uint32_t key) { + return JSValue::encode(jsNumber(static_cast(flipped(key)))); + }); + } + if (array->indexingMode() == ArrayWithInt32) + return false; + return sortElementsByKey(elements, [&](EncodedJSValue element, uint64_t& key) { + JSValue number = JSValue::decode(element); + return number.isNumber() && tryMakeSortKeyOfDouble(number.asNumber(), mask, key); + }, [&](uint64_t key) { + return JSValue::encode(jsNumber(doubleOfSortKey(key, mask))); + }); + } + default: + return false; + } +} + JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationAOTNewArrayIterator, JSCell*, (Instance* instance, JSCell* array, uint32_t kind)) { AOT_OPERATION_BEGIN(instance, Entry::operationAOTNewArrayIterator); @@ -479,6 +630,8 @@ JSC_DEFINE_JIT_OPERATION(operationAOTArrayUnshiftGeneric, EncodedJSValue, (Insta JSArray* array = cellArgument(cell, Entry::operationAOTArrayUnshiftGeneric); if (mayRejectOrInterceptWrites(array) || array->structure()->holesMustForwardToPrototype(array)) [[unlikely]] OPERATION_RETURN(scope, JSValue::encode(callMethodInEngine(instance, Builtins::ArrayPrototypeUnshift, array, JSValue::decode(item)))); + if (tryUnshiftItemFast(vm, array, JSValue::decode(item))) [[likely]] + OPERATION_RETURN(scope, JSValue::encode(jsNumber(array->length()))); uint64_t length = array->length(); unshift(globalObject, array, 0, 0, 1, length); OPERATION_RETURN_IF_EXCEPTION(scope, encodedJSValue()); diff --git a/Source/JavaScriptCore/aot/builtins/AOTBuiltinCallEmitter.cpp b/Source/JavaScriptCore/aot/builtins/AOTBuiltinCallEmitter.cpp index c297eb5ba89e0..6af5e7ce5ea32 100644 --- a/Source/JavaScriptCore/aot/builtins/AOTBuiltinCallEmitter.cpp +++ b/Source/JavaScriptCore/aot/builtins/AOTBuiltinCallEmitter.cpp @@ -208,7 +208,9 @@ void BuiltinCallEmitter::generateCallBuiltin(const ValueContext& value) stopUnlessNativeResultIsAsWritten(value, result, overload.builtin, overload.mode == BuiltinCallMode::Construct); if (collectionsBefore) { Boxed none = boxedConstant(JSValue()); - callVoidOperation(value, RuntimeOperation::CheckFreshResult, { result, *collectionsBefore, word32(static_cast(overload.builtin.index())), receiver ? boxedOperand(value, receiver) : none, arguments.empty() ? none : boxedOperand(value, arguments[0]) }); + const Operation* receiverOperation = receiver ? &m_code.at(underlyingCellOf(m_code, receiver)) : nullptr; + bool hasOriginalReceiver = receiverOperation && (receiverOperation->opcode != Opcode::ToObject || receiverOperation->calledBuiltin() != overload.builtin); + callVoidOperation(value, RuntimeOperation::CheckFreshResult, { result, *collectionsBefore, word32(static_cast(overload.builtin.index())), hasOriginalReceiver ? boxedOperand(value, receiver) : none, arguments.empty() ? none : boxedOperand(value, arguments[0]) }); } defineAs(value, result, typeOf(value.id())); }; @@ -449,6 +451,11 @@ Boxed BuiltinCallEmitter::callBuiltinFunction(const OperationContext& context, B bool constructs = overload.mode == BuiltinCallMode::Construct; unsigned count = static_cast(arguments.size()); std::optional stub = stubThatCallsBuiltin(m_builtins, overload, count); + if (overload.mode == BuiltinCallMode::Get && receiver && !count && stub && stub->family() == StubFamily::CallHostFunctionWithoutBoxedCall && m_builtins.at(overload.builtin).origin == BuiltinOrigin::Host) { + const Operation* call = context.valueID() ? &operationAt(context.valueID()) : nullptr; + if (call && call->opcode == Opcode::CallBuiltin && call->overload() == id && call->builtinCallForm() == BuiltinCallForm::Whole) + stub = callHostGetter(); + } bool leavesFunctionUnfetched = stub && stub->family() == StubFamily::CallBoxedBuiltin && fetchesFunctionOnlyToLeave(static_cast(stub->form()), count) && keyOfCalleeOf(overload).hasCell(m_builtins); Boxed callee = leavesFunctionUnfetched ? Boxed(zeroExtended(numberOf(m_mentions.builtinCell(keyOfCalleeOf(overload)))).value()) : boxCell(calleeOf(context, overload)); Boxed thisValue = undefined(); diff --git a/Source/JavaScriptCore/aot/builtins/AOTBuiltinCallEmitter.h b/Source/JavaScriptCore/aot/builtins/AOTBuiltinCallEmitter.h index 99d2453ef9f7e..cd6c5c437bbd2 100644 --- a/Source/JavaScriptCore/aot/builtins/AOTBuiltinCallEmitter.h +++ b/Source/JavaScriptCore/aot/builtins/AOTBuiltinCallEmitter.h @@ -29,6 +29,8 @@ class BuiltinCallEmitter final : public BuiltinFamilyEmitter { void generateRemainderOfIntegers(const ValueContext&); void generateReciprocalOfDivisor(const ValueContext&); void generateRemainderByReciprocal(const ValueContext&); + void generateWholeDivisor(const ValueContext&); + void generateRemainderByWholeDivisor(const ValueContext&); void generateToInt32(const ValueContext&); void generateToUint32(const ValueContext&); void generateToIntegerOrInfinity(const ValueContext&); diff --git a/Source/JavaScriptCore/aot/builtins/AOTBuiltinCallEmitterNumbers.cpp b/Source/JavaScriptCore/aot/builtins/AOTBuiltinCallEmitterNumbers.cpp index 11eec559d0014..9edb95f6371de 100644 --- a/Source/JavaScriptCore/aot/builtins/AOTBuiltinCallEmitterNumbers.cpp +++ b/Source/JavaScriptCore/aot/builtins/AOTBuiltinCallEmitterNumbers.cpp @@ -10,6 +10,7 @@ #include "AOTConstantEmitter.h" #include "AOTConversionEmitter.h" +#include "AOTOperatorEmitter.h" #include "AOTIRProgram.h" #include "AOTThrowEmitter.h" #include "AOTTypeTable.h" @@ -299,6 +300,21 @@ void BuiltinCallEmitter::generateToInt32(const ValueContext& value) defineAs(value, int32BitsOf(value->valueOperand(), value->givesOnlyLowestBits()), m_program.at(TypeTable::int32)); } +void BuiltinCallEmitter::generateWholeDivisor(const ValueContext& value) +{ + Number divisor = numberOperand(value->divisorOperand()); + ConversionEmitter& conversions = emitter(); + Integer whole = conversions.truncateUnchecked(divisor); + Integer wholeOrZero = select(equal(conversions.bitsOf(toNumber(whole)), conversions.bitsOf(divisor)), whole, integer(0)); + defineAs(value, select(below(wholeOrZero, integer(maxExactInteger + 1)), wholeOrZero, integer(0)), typeOf(value.id())); +} + +void BuiltinCallEmitter::generateRemainderByWholeDivisor(const ValueContext& value) +{ + Number remainder = emitter().remainderByWholeDivisor(value, numberOperand(value->dividendOperand()), numberOperand(value->divisorOperand()), integerOperand(value->wholeDivisorOperand())); + defineAs(value, remainder, typeOf(value.id())); +} + void BuiltinCallEmitter::generateToUint32(const ValueContext& value) { defineAs(value, zeroExtended(int32BitsOf(value->valueOperand(), false)), m_program.at(TypeTable::uint32)); diff --git a/Source/JavaScriptCore/aot/builtins/AOTBuiltinResultTypes.cpp b/Source/JavaScriptCore/aot/builtins/AOTBuiltinResultTypes.cpp index 28e7bd62c3edd..670ed920b3278 100644 --- a/Source/JavaScriptCore/aot/builtins/AOTBuiltinResultTypes.cpp +++ b/Source/JavaScriptCore/aot/builtins/AOTBuiltinResultTypes.cpp @@ -95,6 +95,8 @@ static TypeID applyRangeRuleToBuiltinResult(const BuiltinTypeContext& context, c const BuiltinResult& result = overload.result; if (result.rangeRule == BuiltinRangeRule::None || !result.rangeOperand) return declared; + if (result.rangeRule == BuiltinRangeRule::IntegerPart) + declared = context.types.intersectionOf(TypeArena::fromPrimitives(Primitive::NumberWithoutFraction), declared); TypeID onlyOperand; std::span rangeOperands; @@ -129,8 +131,6 @@ static TypeID applyRangeRuleToBuiltinResult(const BuiltinTypeContext& context, c Range range = computeResultRange(result.rangeRule, ranges.span()); return types.intersectionOf(takesOnlyIntegers ? types.integer(range) : typeOfRuledResultOfFractions(types, overload, range), declared); } - if (result.rangeRule == BuiltinRangeRule::IntegerPart && rangeOperands.front() && types.at(rangeOperands.front()).isNumber()) - return types.intersectionOf(TypeArena::fromPrimitives(Primitive::NumberWithoutFraction), declared); return declared; } diff --git a/Source/JavaScriptCore/aot/builtins/AOTBuiltinTable.cpp b/Source/JavaScriptCore/aot/builtins/AOTBuiltinTable.cpp index 3fc4c8fc97647..ddfb7c20db8a0 100644 --- a/Source/JavaScriptCore/aot/builtins/AOTBuiltinTable.cpp +++ b/Source/JavaScriptCore/aot/builtins/AOTBuiltinTable.cpp @@ -413,6 +413,20 @@ bool BuiltinTable::ignoresReceiver(BuiltinID builtin) const }); } +bool BuiltinTable::ignoresArgument(BuiltinID builtin, unsigned argument) const +{ + BuiltinID canonical = canonicalID(builtin); + if (at(canonical).origin != BuiltinOrigin::Engine || argument > BuiltinOperand::maxArgumentIndex) + return false; + std::span rows = overloadsOf(canonical); + bool noRowSpeaksOfIt = !rows.empty() && std::ranges::all_of(rows, [&](const BuiltinOverload& row) { + return !row.isUndeclared() && !parameterFor(row, BuiltinOperand::argument(argument)); + }); + return noRowSpeaksOfIt && std::ranges::all_of(overloadSummariesOf(canonical), [](const BuiltinOverloadSummary& overload) { + return overload.callee == BuiltinCallee::HostFunction || overload.callee == BuiltinCallee::InternalFunction; + }); +} + bool BuiltinTable::mayDefineOnNewInstances(BuiltinClassID builtinClass, BuiltinName name) const { if (listsOwnPropertiesOfNewInstances(builtinClass)) diff --git a/Source/JavaScriptCore/aot/builtins/AOTBuiltinTable.h b/Source/JavaScriptCore/aot/builtins/AOTBuiltinTable.h index aa23a1502e5c6..b4a6dec678096 100644 --- a/Source/JavaScriptCore/aot/builtins/AOTBuiltinTable.h +++ b/Source/JavaScriptCore/aot/builtins/AOTBuiltinTable.h @@ -1000,6 +1000,7 @@ class BuiltinTable { bool listsOwnPropertiesOfNewInstances(BuiltinClassID) const; bool mayDefineOnNewInstances(BuiltinClassID, BuiltinName) const; JS_EXPORT_PRIVATE bool ignoresReceiver(BuiltinID) const; + bool ignoresArgument(BuiltinID, unsigned argument) const; bool definesCharactersOnNewInstances(BuiltinClassID) const; std::span classesWithInstanceMember(BuiltinName) const; diff --git a/Source/JavaScriptCore/aot/builtins/AOTHashTableBuiltinEmitter.cpp b/Source/JavaScriptCore/aot/builtins/AOTHashTableBuiltinEmitter.cpp index b2aea54aef85f..64034befcdf78 100644 --- a/Source/JavaScriptCore/aot/builtins/AOTHashTableBuiltinEmitter.cpp +++ b/Source/JavaScriptCore/aot/builtins/AOTHashTableBuiltinEmitter.cpp @@ -63,7 +63,7 @@ Boxed HashTableBuiltinEmitter::normalizedNumber(Number key) LBasicBlock fitsInt32 = newBlock(); LBasicBlock otherwise = newBlock(); Join join(m_out); - Word32 truncated = low32(conversions.toInteger(key)); + Word32 truncated = conversions.truncateToWord(key); m_out.branch(equal(toNumber(truncated), key).value(), normalEdge(fitsInt32), unlikelyEdge(otherwise)); m_out.appendTo(fitsInt32); diff --git a/Source/JavaScriptCore/aot/builtins/AOTMathBuiltinEmitterNumbers.cpp b/Source/JavaScriptCore/aot/builtins/AOTMathBuiltinEmitterNumbers.cpp index b8180e32117d3..02fa91feb0468 100644 --- a/Source/JavaScriptCore/aot/builtins/AOTMathBuiltinEmitterNumbers.cpp +++ b/Source/JavaScriptCore/aot/builtins/AOTMathBuiltinEmitterNumbers.cpp @@ -134,8 +134,7 @@ void MathBuiltinEmitter::generateParseInt(const InlineCall& call) if (ValueID argument = call.argumentOrNull(1)) radix = int32Operand(argument); CellPointer string = cellOperand(call.argument(0)); - missUnless(call, logicalNot(isRopeString(string))); - CharacterView view = emitter().viewOfFlatString(string); + CharacterView view = emitter().viewWithoutResolving(call.context(), string, rareEdge(missBlockOf(call))); Number result = emitter().selectByWidth(view.is8Bit, [&](bool is8Bit) { return callLeaf(is8Bit ? RuntimeOperation::ParseInt8 : RuntimeOperation::ParseInt16, { view.characters, view.length, radix.value_or(word32(0)) }); }, Machine::EdgeClass::Normal); diff --git a/Source/JavaScriptCore/aot/builtins/AOTMathBuiltinOperations.cpp b/Source/JavaScriptCore/aot/builtins/AOTMathBuiltinOperations.cpp index dc02238f35b02..208179a56cc8f 100644 --- a/Source/JavaScriptCore/aot/builtins/AOTMathBuiltinOperations.cpp +++ b/Source/JavaScriptCore/aot/builtins/AOTMathBuiltinOperations.cpp @@ -80,13 +80,18 @@ JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationAOTMathF16Round, double, (double x)) return Float16 { x }; } +static NEVER_INLINE double AOT_PRESERVE_MOST_LEAF_ATTRIBUTES parseInt8Slow(const Latin1Character* characters, uint32_t length, int32_t radix) +{ + return parseInt(unsafeMakeSpan(characters, length), radix); +} + extern "C" double AOT_PRESERVE_MOST_LEAF_ATTRIBUTES operationAOTParseInt8(const Latin1Character* characters, uint32_t length, int32_t radix) { if (!radix || radix == 10) { if (std::optional value = valueOfFewDigits(unsafeMakeSpan(characters, length))) return *value; } - return parseInt(unsafeMakeSpan(characters, length), radix); + [[clang::musttail]] return parseInt8Slow(characters, length, radix); } extern "C" double AOT_PRESERVE_MOST_LEAF_ATTRIBUTES operationAOTParseInt16(const char16_t* characters, uint32_t length, int32_t radix) @@ -94,11 +99,16 @@ extern "C" double AOT_PRESERVE_MOST_LEAF_ATTRIBUTES operationAOTParseInt16(const return parseInt(unsafeMakeSpan(characters, length), radix); } +static NEVER_INLINE double AOT_PRESERVE_MOST_LEAF_ATTRIBUTES stringToNumber8Slow(const Latin1Character* characters, uint32_t length) +{ + return jsToNumber(StringView { unsafeMakeSpan(characters, length) }); +} + extern "C" double AOT_PRESERVE_MOST_LEAF_ATTRIBUTES operationAOTStringToNumber8(const Latin1Character* characters, uint32_t length) { if (std::optional value = valueOfFewDigits(unsafeMakeSpan(characters, length))) return *value; - return jsToNumber(StringView { unsafeMakeSpan(characters, length) }); + [[clang::musttail]] return stringToNumber8Slow(characters, length); } extern "C" double AOT_PRESERVE_MOST_LEAF_ATTRIBUTES operationAOTStringToNumber16(const char16_t* characters, uint32_t length) diff --git a/Source/JavaScriptCore/aot/builtins/typedarray.builtins.ts b/Source/JavaScriptCore/aot/builtins/typedarray.builtins.ts index 5684cdc1c178c..4e97df74f70e8 100644 --- a/Source/JavaScriptCore/aot/builtins/typedarray.builtins.ts +++ b/Source/JavaScriptCore/aot/builtins/typedarray.builtins.ts @@ -167,6 +167,37 @@ const fold = (name: string) => [ }, ] satisfies b.FunctionField[]; +const sortingWithComparator = (returns: b.TableType, receiverUses: readonly b.UseSpec[]) => + ({ + ...operandCodeEffects, + args: { comparator: b.callable }, + returns, + uses: { + this: receiverUses, + comparator: [b.called({ afterwards: "TouchesOnlyWhatRowNames", times: "Repeatedly", this: b.undefined, args: [elementOfReceiver, elementOfReceiver], result: convertedByNativeCode })], + }, + implementation: b.hostCall, + }) satisfies b.FunctionField; + +const typedArrayOfNumbers = b.union( + b.instance("Int8Array"), + b.instance("Uint8Array"), + b.instance("Uint8ClampedArray"), + b.instance("Int16Array"), + b.instance("Uint16Array"), + b.instance("Int32Array"), + b.instance("Uint32Array"), + b.instance("Float16Array"), + b.instance("Float32Array"), + b.instance("Float64Array"), +); + +const sortingOfNumbers = { + ...sortingWithComparator(self, changesElements), + this: typedArrayOfNumbers, + implementation: b.compiled("prelude/array-sort", "sortTypedArray"), +} satisfies b.FunctionField; + const sorting = (nativeSymbol: string, returns: b.TableType, receiverUses: readonly b.UseSpec[]) => [ { @@ -179,18 +210,7 @@ const sorting = (nativeSymbol: string, returns: b.TableType, receiverUses: reado nativeSymbols: [nativeSymbol], implementation: b.hostCall, }, - { - args: { comparator: b.callable }, - returns, - capability: "js", - throws: true, - runsOnlyOperandCode: true, - uses: { - this: receiverUses, - comparator: [b.called({ afterwards: "TouchesOnlyWhatRowNames", times: "Repeatedly", this: b.undefined, args: [elementOfReceiver, elementOfReceiver], result: convertedByNativeCode })], - }, - implementation: b.hostCall, - }, + sortingWithComparator(returns, receiverUses), ] satisfies b.FunctionField[]; const copyOfRange = { @@ -371,6 +391,14 @@ export default b.declarations( uses: { this: changesElements, source: [readsElements] }, implementation: b.inline("TypedArraySet", { otherwise: b.hostCall }), }, + { + args: { source: b.notPassed }, + returns: b.never, + capability: "heap", + throws: true, + uses: { this: b.inspected }, + implementation: b.hostCall, + }, { ...operandCodeEffects, args: { source: b.any, offset: relativeIndex.optional }, @@ -395,7 +423,7 @@ export default b.declarations( writes: [elements], }, ], - sort: sorting("typedArrayViewProtoFuncSort", self, changesElements), + sort: [sortingOfNumbers, ...sorting("typedArrayViewProtoFuncSort", self, changesElements)], subarray: [ { ...viewOfRange, diff --git a/Source/JavaScriptCore/aot/codegen/AOTAccessEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTAccessEmitter.cpp index 8b98d8885beb2..a6999dfd707d5 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTAccessEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTAccessEmitter.cpp @@ -879,6 +879,8 @@ void AccessEmitter::returnWhatMethodOfBuiltinClassGives(const ValueContext& read ConversionEmitter& conversions = emitter(); auto boxed = [&](ValueID id) { ValueID made = inFrontOfBox(id); + if (std::optional word = generator().findBoxed(made)) + return word->value(); return conversions.box(read, get(made), typeOf(made)).value(); }; for (ValueID argument : call.argumentsOperands()) @@ -1327,6 +1329,16 @@ Condition AccessEmitter::hasByVal(const OperationContext& context, Boxed receive return Condition(m_out.castToInt32(callStub(context, inByValCached(PresenceQuestion::In), { receiver.value(), key.value() }).first)); } +LValue AccessEmitter::tryUnboxConstantIndex(Boxed key) +{ + if (!key.value()->hasInt64()) + return nullptr; + JSValue constant = JSValue::decode(key.value()->asInt64()); + if (!constant.isInt32() || constant.asInt32() < 0) + return nullptr; + return m_out.constInt64(constant.asInt32()); +} + std::optional AccessEmitter::elementKeyOf(const ValueContext& value) { ValueID property = value->propertyOperand(); @@ -1366,6 +1378,41 @@ static std::optional stubFormOfExpectedElements(ExpectedE RELEASE_ASSERT_NOT_REACHED(); } +bool AccessEmitter::readsElementsInLoop(const ValueContext& value, ValueID object) +{ + bool standsInRareBlock = m_code.at(value.block()).frequency == BlockFrequency::Rare; + return !standsInRareBlock && m_generator.isInLoop(value.block()) && typeOf(object).hasObjects(); +} + +Boxed AccessEmitter::presentElementOrElse(ValueID objectOperand, const ElementKey& key, LBasicBlock outOfLine) +{ + ElementAccessEmitter& elements = emitter(); + Boxed object = in(objectOperand); + Integer index; + if (key.isInteger) + index = Integer(key.value); + else { + Boxed boxedKey(key.value); + branchUnless(isInt32(boxedKey), outOfLine); + index = signExtended(unboxInt32(boxedKey)); + } + Type known = typeOf(objectOperand); + if (known.primitives) { + Condition isPresentCell = isCell(object); + if (known.primitives & Primitive::Absent) + isPresentCell = logicalAnd(isPresentCell, isNotEmpty(object)); + branchUnless(isPresentCell, outOfLine); + } + CellPointer cell = unboxCell(object); + branchUnless(elements.holdsBoxedElements(cell), outOfLine); + ArrayElements storage = elements.elementsOf(cell); + LengthTest test = elements.testLength(index, storage); + branchUnless(test.isInside, outOfLine); + Boxed element = elements.loadBoxedElement(storage, test.indexInside); + branchUnless(isNotEmpty(element), outOfLine); + return element; +} + void AccessEmitter::generateElemGet(const ValueContext& value) { Boxed object = in(value->objectOperand()); @@ -1425,6 +1472,17 @@ void AccessEmitter::generateElemGet(const ValueContext& value) define(value, join.finish()); return; } + bool keyIsIndex = key->isInteger || value->keyIsIntegral() || typeOf(value->propertyOperand()).isNumber(); + if (!extraPropertiesWord && !expected && keyIsIndex && readsElementsInLoop(value, value->objectOperand())) { + LBasicBlock outOfLine = newBlock(); + Join join(m_out); + join.add(presentElementOrElse(value->objectOperand(), *key, outOfLine)); + + m_out.appendTo(outOfLine); + join.add(Boxed(callStub(value, stub, { object.value(), key->value }, { .isOnRarePath = true }).first)); + define(value, join.finish()); + return; + } if (!extraPropertiesWord) { define(value, Boxed(callStub(value, stub, { object.value(), key->value }).first)); return; @@ -1476,7 +1534,11 @@ void AccessEmitter::generateElemSet(const ValueContext& value) ElementAccessEmitter& elements = emitter(); const PlacedSlot* extraPropertiesWord = elements.extraPropertiesWordOfReadByPosition(value); if (!extraPropertiesWord) { - callStub(value, stub, { object.value(), key->value, stored.value() }); + LValue constantIndex = key->isInteger || expected ? nullptr : tryUnboxConstantIndex(Boxed(key->value)); + if (constantIndex) + callStub(value, putByValAtConstantIndex(forMode(value->mode(), JSC::ECMAMode::strict(), JSC::ECMAMode::sloppy())), { object.value(), constantIndex, stored.value() }); + else + callStub(value, stub, { object.value(), key->value, stored.value() }); return; } LBasicBlock outOfLine = newBlock(); @@ -1506,7 +1568,19 @@ void AccessEmitter::generateElemSet(const ValueContext& value) void AccessEmitter::generateElemHas(const ValueContext& value) { Boxed object = in(value->objectOperand()); - defineTest(value, hasByVal(value, object, in(value->propertyOperand()))); + Boxed key = in(value->propertyOperand()); + if (!typeOf(value->propertyOperand()).isNumber() || !readsElementsInLoop(value, value->objectOperand())) { + defineTest(value, hasByVal(value, object, key)); + return; + } + LBasicBlock outOfLine = newBlock(); + Join join(m_out); + presentElementOrElse(value->objectOperand(), ElementKey { key.value(), false }, outOfLine); + join.add(word32(1)); + + m_out.appendTo(outOfLine); + join.add(Word32(hasByVal(value, object, key).value())); + defineTest(value, Condition(join.finish().value())); } void AccessEmitter::generateElemDelete(const ValueContext& value) diff --git a/Source/JavaScriptCore/aot/codegen/AOTAccessEmitter.h b/Source/JavaScriptCore/aot/codegen/AOTAccessEmitter.h index 347e6c9649b91..75b226cc945f4 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTAccessEmitter.h +++ b/Source/JavaScriptCore/aot/codegen/AOTAccessEmitter.h @@ -89,6 +89,7 @@ class AccessEmitter final : public OperationEmitter { Condition hasById(const OperationContext&, Boxed receiver, PropertyKeyID); Condition findsName(const OperationContext&, Boxed receiver, PropertyKeyID, PresenceQuestion); Boxed getByVal(const OperationContext&, Boxed receiver, Boxed key); + LValue tryUnboxConstantIndex(Boxed key); bool isSymbolOfRealm(PropertyKeyID key) const { return !m_plan.hasOneInstance() && !liesInThread(key) && m_program.at(key).kind == PropertyKeyKind::CreatedSymbol; } Boxed cellOfKey(const OperationContext&, PropertyKeyID); void putByVal(const OperationContext&, Boxed receiver, Boxed key, Boxed value, PutMode); @@ -149,6 +150,8 @@ class AccessEmitter final : public OperationEmitter { bool isInteger; }; std::optional elementKeyOf(const ValueContext&); + bool readsElementsInLoop(const ValueContext&, ValueID object); + Boxed presentElementOrElse(ValueID objectOperand, const ElementKey&, LBasicBlock outOfLine); Boxed keyOperand(const OperationContext&, PropertyKeyID key, ValueID computedKey); struct ExcludedKeys { Word32 run; diff --git a/Source/JavaScriptCore/aot/codegen/AOTAllocationEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTAllocationEmitter.cpp index 79e098d2f5e8a..04862c211b3dd 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTAllocationEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTAllocationEmitter.cpp @@ -153,7 +153,9 @@ void AllocationEmitter::generateNewObject(const ValueContext& value) } CellPointer cell = allocateWithFields(value, allocation, value->fields(), value->valuesOperands(), value->storage()); if (ValueID prototype = value->prototypeOperand()) - storePrototype(value, cell, prototype); + storePrototype(value, cell, in(prototype)); + else if (value->storage() == AllocationStorage::Heap && m_program.at(layout).kind == LayoutKind::Instance && !m_plan.slots().hasDirectInstances(layout)) + storePrototype(value, cell, unboxCell(loadField(loadStructure(cell), m_heaps.Structure_prototype))); else define(value, cell); } @@ -684,9 +686,8 @@ CellPointer AllocationEmitter::allocateInstance(const ValueContext& value, const return cell; } -void AllocationEmitter::storePrototype(const ValueContext& value, CellPointer cell, ValueID prototype) +void AllocationEmitter::storePrototype(const ValueContext& value, CellPointer cell, CellPointer requested) { - CellPointer requested = in(prototype); Boxed sitePrototype = loadField(loadStructure(cell), m_heaps.Structure_prototype); Condition hasSitePrototype = equal(boxCell(requested), sitePrototype); auto useCell = [&] { @@ -696,6 +697,10 @@ void AllocationEmitter::storePrototype(const ValueContext& value, CellPointer ce MentionedAllocation allocation = emitter().allocationOf(value, value->layout(), value->fields()); return callOperation(value, RuntimeOperation::SetConstructPrototype, { cell, requested, numberOf(allocation.mention), word32(value->fields().empty()) }, { .isOnRarePath = true }); }; + if (!m_plan.slots().hasDirectInstances(value->layout())) { + define(value, setByOperation()); + return; + } if (newObjectMayHaveOtherLayout(m_program, value.value()) || !generator().isInLoop(value.block())) { define(value, either(hasSitePrototype, useCell, setByOperation)); return; diff --git a/Source/JavaScriptCore/aot/codegen/AOTAllocationEmitter.h b/Source/JavaScriptCore/aot/codegen/AOTAllocationEmitter.h index 1f70f1b0288ca..1ac57fd7004db 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTAllocationEmitter.h +++ b/Source/JavaScriptCore/aot/codegen/AOTAllocationEmitter.h @@ -70,7 +70,7 @@ class AllocationEmitter final : public OperationEmitter { void storeInitialFields(CellPointer, LayoutID, const InitialFields&); void storeEmptySlots(CellPointer, LayoutID, std::span emptySlots, ZeroedBy, BlockID); void storeFieldsInExtension(const ValueContext&, CellPointer, LayoutID, const InitialFields&); - void storePrototype(const ValueContext&, CellPointer, ValueID prototype); + void storePrototype(const ValueContext&, CellPointer, CellPointer requested); bool looksAtPrototypeBeforeAllocating(const ValueContext&); CellPointer allocateInstance(const ValueContext&, const MentionedAllocation&, ValueID prototype); CellPointer allocateCopyFromOtherLayout(const ValueContext&, ValueID copy); diff --git a/Source/JavaScriptCore/aot/codegen/AOTArrayBuiltinEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTArrayBuiltinEmitter.cpp index 432f084a1f1f2..a3e1c533e1175 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTArrayBuiltinEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTArrayBuiltinEmitter.cpp @@ -15,8 +15,10 @@ #include "AOTRepresentation.h" #include "AOTResultRules.h" #include "AOTStubsArrays.h" +#include "AOTStubsElements.h" #include "AOTTypeTestEmitter.h" #include "ArrayConventions.h" +#include "JSArray.h" #include "JSType.h" #include @@ -57,7 +59,7 @@ TypeID ArrayBuiltinEmitter::elementTypeOf(const InlineCall& call, TypeLevel leve ArrayStorageFormList ArrayBuiltinEmitter::candidateFormsOf(const InlineCall& call) { TypeID contents = elementTypeOf(call, TypeLevel::Expected); - const ArrayStorageForm& expected = ArrayAllocationEmitter(generator()).formAtAllocation(typeOf(call.receiver())); + const ArrayStorageForm& expected = elements().expectedFormOf(call.receiver()); bool expectsContents = !isNeverType(contents); ArrayStorageFormList forms { &expected }; for (const ArrayStorageForm& form : arrayStorageForms) { @@ -160,6 +162,20 @@ Operand ArrayBuiltinEmitter::removeFirstElement(const InlineCall& call, const Ar return first; } +Operand ArrayBuiltinEmitter::shiftLongArray(const InlineCall& call, const ArrayStorageForm& form, const ArrayElements& storage, Integer length) +{ + Integer last = sub(length, integer(1)); + missUnless(call, below(last, integer(JSArray::shiftThreshold))); + Operand first = elements().loadRawElement(form, storage, integer(0)); + missUnless(call, logicalNot(elements().isHole(form, first))); + if (form.holdsCells) + missUnless(call, logicalNot(Condition(m_out.load8ZeroExt32(instanceField(m_heaps.collectorState, AOT_INSTANCE_OFFSET(mutatorShouldBeFenced)))))); + callStub(call.context(), moveElementsDown(), { storage.butterfly.value(), last.value() }); + elements().storeRawElement(form, storage, last, elements().rawHole(form)); + elements().setPublicLength(storage, last); + return first; +} + void ArrayBuiltinEmitter::removeElementAtEnd(const InlineCall& call, bool removesFirst) { if (!checkChosenForm(call, call.canMiss())) @@ -175,7 +191,17 @@ void ArrayBuiltinEmitter::removeElementAtEnd(const InlineCall& call, bool remove if (mayHaveNoElements) branchUnless(isNotZero(length), hasNoElements); if (removesFirst) { - missUnless(call, belowOrEqual(length, integer(maxLengthForElementwiseMove))); + Condition areFew = belowOrEqual(length, integer(maxLengthForElementwiseMove)); + if (!generator().paysForCodeInLine(call.context().block())) + missUnless(call, areFew); + else { + LBasicBlock few = newBlock(); + LBasicBlock many = newBlock(); + branch(areFew, { normalEdge(few), normalEdge(many) }); + m_out.appendTo(many); + join.add(resultFromRawElement(call, form, shiftLongArray(call, form, storage, length))); + m_out.appendTo(few); + } join.add(resultFromRawElement(call, form, removeFirstElement(call, form, storage, missBlockOf(call)))); return; } @@ -304,7 +330,7 @@ Integer ArrayBuiltinEmitter::indexOfNumber(const InlineCall& call, CellPointer a auto searchIntegers = [&] { LBasicBlock isInt32 = newBlock(); LBasicBlock isNoInt32 = newBlock(); - Word32 truncated = low32(emitter().toInteger(searched)); + Word32 truncated = emitter().truncateToWord(searched); branch(equal(toNumber(truncated), searched), { normalEdge(isInt32), normalEdge(isNoInt32) }); m_out.appendTo(isInt32); diff --git a/Source/JavaScriptCore/aot/codegen/AOTArrayBuiltinEmitter.h b/Source/JavaScriptCore/aot/codegen/AOTArrayBuiltinEmitter.h index 7855a191bef7a..f60cefdf94f0f 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTArrayBuiltinEmitter.h +++ b/Source/JavaScriptCore/aot/codegen/AOTArrayBuiltinEmitter.h @@ -73,6 +73,7 @@ class ArrayBuiltinEmitter final : public BuiltinFamilyEmitter { void removeElementAtEnd(const InlineCall&, bool removesFirst); Operand removeFirstElement(const InlineCall&, const ArrayStorageForm&, const ArrayElements&, LBasicBlock isAbsent); + Operand shiftLongArray(const InlineCall&, const ArrayStorageForm&, const ArrayElements&, Integer length); bool resultGetsNamedProperties(const InlineCall&); bool resultIsExpectedToHoldDoubles(const InlineCall&); diff --git a/Source/JavaScriptCore/aot/codegen/AOTArrayIntrinsicEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTArrayIntrinsicEmitter.cpp index 44ce7be149e75..3dac6131da91a 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTArrayIntrinsicEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTArrayIntrinsicEmitter.cpp @@ -272,6 +272,8 @@ void ArrayIntrinsicEmitter::putInBounds(const ValueContext& value, const ArraySt } if (!elements().boxIsMadeInFormOfType(stored) && !AOT::hasFlag(build().options().testing, Wire::TestingSwitch::SkipInt32Reboxing)) rawElement = emitter().inFormOfType(rawElement, typeOf(stored)); + if (elements().checksBounds()) + elements().stopUnlessInside(value, logicalOr(isInt32(rawElement.as()), elements().hasArrayStorage(elements().askedAnew(array), ArrayStorage::Contiguous))); elements().storeWordOfOperand(value, stored, rawElement.as(), storage, position); barrierAfterStore(value, array, typeOf(stored)); } @@ -310,6 +312,8 @@ void ArrayIntrinsicEmitter::putNumberInBounds(const ValueContext& value, const A void ArrayIntrinsicEmitter::generatePutContiguousIndex(const ValueContext& value) { + if (value->hasNumericStorage() && elements().checksBounds()) + elements().stopUnlessInside(value, elements().hasWritableNumericStorage(elements().askedAnew(in(value->arrayOperand())))); if (!value->goesByKindOfStorage()) { putInBounds(value, contiguous); return; @@ -780,6 +784,13 @@ void ArrayIntrinsicEmitter::generateTryFillNewArray(const ValueContext& value) define(value, join.finish()); } +void ArrayIntrinsicEmitter::generateSortNumbers(const ValueContext& value) +{ + CellPointer array = in(value->arrayOperand()); + Integer direction = integerOperand(value->directionOperand()); + define(value, callOperation(value, RuntimeOperation::SortNumbers, { array, low32(direction) })); +} + void ArrayIntrinsicEmitter::generateTruncateNewArray(const ValueContext& value) { static constexpr int64_t smallUnusedCapacity = 8; diff --git a/Source/JavaScriptCore/aot/codegen/AOTArrayIntrinsicEmitter.h b/Source/JavaScriptCore/aot/codegen/AOTArrayIntrinsicEmitter.h index 92fec9f7243c4..769412cce69cd 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTArrayIntrinsicEmitter.h +++ b/Source/JavaScriptCore/aot/codegen/AOTArrayIntrinsicEmitter.h @@ -40,6 +40,7 @@ class ArrayIntrinsicEmitter final : public BuiltinFamilyEmitter { void generateTryPutIndex(const ValueContext&); void generateTryCopyFilledElements(const ValueContext&); void generateTryPutElements(const ValueContext&); + void generateSortNumbers(const ValueContext&); void generateTryFillNewArray(const ValueContext&); void generateTruncateNewArray(const ValueContext&); diff --git a/Source/JavaScriptCore/aot/codegen/AOTArrayStorageBuiltinEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTArrayStorageBuiltinEmitter.cpp index fa0bfa78b6ec8..5188c905a4258 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTArrayStorageBuiltinEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTArrayStorageBuiltinEmitter.cpp @@ -47,7 +47,14 @@ void ArrayStorageBuiltinEmitter::generateArrayPush(const InlineCall& call) void ArrayStorageBuiltinEmitter::pushOne(const InlineCall& call, CellPointer array) { ValueID pushed = call.argument(0); - ArrayStorage expected = ArrayAllocationEmitter(generator()).formAtAllocation(typeOf(call.receiver())).kind; + Representation representation = call.canMiss() && !resultIsRead(call) ? Representation::Void : resultRepresentationOf(call); + BlockID block = call.context().block(); + if (m_code.at(block).frequency == BlockFrequency::Rare && !generator().isInLoop(block)) { + finishInlineCall(call, pushOutOfLine(call, array, pushed, representation)); + return; + } + + ArrayStorage expected = elements().expectedFormOf(call.receiver()).kind; ArrayStorage declared = arrayStorageDeclaredBy(m_program, types(), typeIDOf(call.receiver())).predicted(); ArrayStorageFormList forms = elements().candidateFormsFor(pushed, expected); if (!hasArrayStorageForm(declared) && representationOf(pushed) == Representation::Int32) { @@ -73,7 +80,7 @@ void ArrayStorageBuiltinEmitter::pushOne(const InlineCall& call, CellPointer arr bool mayPushCell = typeOf(pushed).hasCells(); LBasicBlock isStored = newBlock(); LBasicBlock slowPath = newBlock(); - OperandJoin join(m_out, resultRepresentationOf(call)); + OperandJoin join(m_out, representation); build().statistics().count(StatisticID::ArrayPushesInlined); ArrayElements storage = elements().elementsOf(array); @@ -100,17 +107,19 @@ void ArrayStorageBuiltinEmitter::pushOne(const InlineCall& call, CellPointer arr m_out.appendTo(isStored); Word32 newLength = add(length, word32(1)); elements().setPublicLength(storage, newLength); - join.add(lengthResult(call, zeroExtended(newLength))); + join.add(representation == Representation::Void ? Operand() : lengthResult(call, zeroExtended(newLength))); m_out.appendTo(slowPath); - join.add(pushOutOfLine(call, array, pushed)); + join.add(pushOutOfLine(call, array, pushed, representation)); finishInlineCall(call, join.finish()); } -Operand ArrayStorageBuiltinEmitter::pushOutOfLine(const InlineCall& call, CellPointer array, ValueID pushed) +Operand ArrayStorageBuiltinEmitter::pushOutOfLine(const InlineCall& call, CellPointer array, ValueID pushed, Representation representation) { Boxed newLength = elements().appendOutOfLine(call.context(), array, pushed); - return emitter().unbox(newLength, resultTypeOf(call), resultRepresentationOf(call)); + if (representation == Representation::Void) + return { }; + return emitter().unbox(newLength, resultTypeOf(call), representation); } void ArrayStorageBuiltinEmitter::generateArrayIsArray(const InlineCall& call) diff --git a/Source/JavaScriptCore/aot/codegen/AOTArrayStorageBuiltinEmitter.h b/Source/JavaScriptCore/aot/codegen/AOTArrayStorageBuiltinEmitter.h index 5f1aff90969c5..e847b3403edf6 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTArrayStorageBuiltinEmitter.h +++ b/Source/JavaScriptCore/aot/codegen/AOTArrayStorageBuiltinEmitter.h @@ -24,7 +24,7 @@ class ArrayStorageBuiltinEmitter final : public OperationEmitter { ElementAccessEmitter& elements() { return emitter(); } Operand lengthResult(const InlineCall&, Integer length); void pushOne(const InlineCall&, CellPointer array); - Operand pushOutOfLine(const InlineCall&, CellPointer array, ValueID pushed); + Operand pushOutOfLine(const InlineCall&, CellPointer array, ValueID pushed, Representation); }; static_assert(sizeof(ArrayStorageBuiltinEmitter) == sizeof(OperationEmitter)); diff --git a/Source/JavaScriptCore/aot/codegen/AOTCastEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTCastEmitter.cpp index a5368cc52dab3..fa5d7be3e3447 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTCastEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTCastEmitter.cpp @@ -15,6 +15,7 @@ #include "AOTEffectsNet.h" #include "AOTInliningAcrossFiles.h" #include "AOTOpcodeSets.h" +#include "AOTSlotEmitter.h" #include "AOTStubsClosures.h" #include "AOTStubsValues.h" #include "AOTThrowEmitter.h" @@ -25,6 +26,46 @@ namespace JSC { namespace AOT { namespace Typed { +void CastEmitter::generateNarrowed(const ValueContext& value) +{ + Operand operand = get(value->valueOperand()); + if (operand.is() && value->testOperand() && !value->isOnTakenEdge()) { + const Operation& test = operationAt(value->testOperand()); + bool isNotUndefined = test.opcode == Opcode::TypeTest && test.valueOperand() == value->valueOperand() && types().isSubset(TypeTable::undefined, test.testedType()); + if (LValue word = isNotUndefined ? emitter().findWordOfSlot(operand.value) : nullptr) + operand = Boxed(word); + } + defineNarrowed(value, operand); +} + +bool CastEmitter::tryTestWordOfSlot(const ValueContext& test, Boxed value, const TypeTestPlan& plan) +{ + constexpr int64_t bitsOutsideUndefined = ~static_cast(JSValue::ValueUndefined); + LValue word = emitter().findWordOfSlot(value.value()); + if (!word || !Machine::isEncodableBitImmediate(Width64, bitsOutsideUndefined)) + return false; + bool asksForNothing = plan.verdict == TypeTestVerdict::NeedsTest && (plan.passingPrimitives & Primitive::Undefined) && !(plan.passingPrimitives & ~Primitive::UndefinedOrNull) && !plan.integers && plan.stringLiterals.isEmpty() && plan.cellTests.isEmpty() && plan.objects != ObjectsVerdict::AllPass; + if (!asksForNothing) + return false; + bool nullFails = plan.failingPrimitives & Primitive::Null; + if (nullFails && !test.isBranchCondition()) + return false; + Condition holdsNothing(m_out.testIsZero64(word, m_out.constInt64(bitsOutsideUndefined))); + if (!nullFails) { + if (test.isBranchCondition()) + branch(holdsNothing, test.branchTargets()); + else + define(test, holdsNothing); + return true; + } + BranchTargets targets = test.branchTargets(); + LBasicBlock mayHoldNull = newBlock(); + m_out.branch(holdsNothing.value(), normalEdge(mayHoldNull), targets.notTaken); + m_out.appendToContinuation(mayHoldNull); + m_out.branch(m_out.equal(word, m_out.constInt64(JSValue::ValueNull)), targets.notTaken, targets.taken); + return true; +} + void CastEmitter::generateCheckInitialized(const ValueContext& value) { ValueID source = value->valueOperand(); @@ -186,13 +227,196 @@ ALWAYS_INLINE_INTO_OTHER_FILES bool CastEmitter::tryBranchOnNumberAndHandItOn(co return true; } +ValueID CastEmitter::sumBehindTestForNumber(const ValueContext& test) +{ + if (test->testedType() != TypeTable::number || generator().keepsValuesInHeapFrame()) + return { }; + auto isNumberTested = [&](ValueID id) { + const Operation& narrowed = m_code.at(id); + return narrowed.opcode == Opcode::Narrowed && narrowed.isOnTakenEdge() && narrowed.testOperand() == test.id() && narrowed.valueOperand() == test->valueOperand(); + }; + auto isTermOfBoxedSum = [&](ValueID id) { + return representationOf(id) == Representation::JSValue || representationOf(id) == Representation::Int32; + }; + ValueID number; + BlockID from = test.block(); + ValueID condition = test.id(); + for (bool isBehindOtherTest : { false, true }) { + const Terminator& end = m_code.at(from).terminator; + if (end.kind != TerminatorKind::Branch || end.conditionOperand() != condition) + return { }; + const Edge& passes = m_code.at(end.taken()); + const Edge& fails = m_code.at(end.notTaken()); + if (!passes.arguments().empty() || !fails.arguments().empty() || m_code.at(fails.to).frequency != BlockFrequency::Rare || generator().singlePredecessorOf(passes.to) != from || generator().leavesTurnLaidOut(passes.to)) + return { }; + ValueID next; + for (ValueID id : m_code.valuesOf(passes.to)) { + if (m_code.at(id).opcode != Opcode::Narrowed) { + next = id; + break; + } + if (isNumberTested(id)) { + if (number) + return { }; + number = id; + } + } + if (!next || !number) + return { }; + const Operation& operation = m_code.at(next); + if (operation.opcode == Opcode::Add || operation.opcode == Opcode::Sub) { + bool takesNumber = operation.leftOperand() == number || operation.rightOperand() == number; + bool isBoxedSum = operation.form() == OperatorForm::Numbers && operation.representation == Representation::JSValue && isTermOfBoxedSum(operation.leftOperand()) && isTermOfBoxedSum(operation.rightOperand()); + return takesNumber && isBoxedSum ? next : ValueID(); + } + bool isTestOfOtherOperand = operation.opcode == Opcode::TypeTest && operation.testedType() == TypeTable::number && m_code.useCountOf(number) == 1; + if (isBehindOtherTest || !isTestOfOtherOperand) + return { }; + from = passes.to; + condition = next; + } + return { }; +} + +bool CastEmitter::tryDeferTestToSum(const ValueContext& test, Operand operand) +{ + if (!operand.is() || !(typeOf(test->valueOperand()).primitives & Primitive::Integer) || AOT::hasFlag(build().options().testing, Wire::TestingSwitch::CheckEffectsAtRunTime)) + return false; + ValueID sum = sumBehindTestForNumber(test); + if (!sum) + return false; + BranchTargets targets = test.branchTargets(); + m_numberChecksOfSums.ensure(sum.index(), [] { return Vector(); }).iterator->value.append({ operand.as(), targets.notTaken }); + m_out.jump(targets.taken); + build().statistics().count(StatisticID::NumberTestsDeferredToSums); + return true; +} + +Vector CastEmitter::takeNumberChecksOf(ValueID sum) +{ + return m_numberChecksOfSums.isEmpty() ? Vector() : m_numberChecksOfSums.take(sum.index()); +} + std::optional CastEmitter::takeIntegerHandedOnTo(ValueID unboxing) { return m_integersHandedOn.takeOptional(unboxing.index()); } +FunctionID CastEmitter::closureThatTestAsksFor(ValueID id) +{ + const Operation& test = m_code.at(id); + if (test.opcode != Opcode::TypeTest || types().at(test.testedType()).primitives) + return { }; + ValueID source = test.valueOperand(); + Representation representation = representationOf(source); + if (representation != Representation::JSValue && (representation != Representation::Cell || test.asksInFull())) + return { }; + bool mayBeHole = test.asksInFull() ? !hasTestedForHole(operationAt(definitionOf(source))) || generator().holeIsLeftToTestBehind(definitionOf(source)) : typeOf(source).primitives & Primitive::Absent; + if (mayBeHole) + return { }; + const TypeTestPlan& plan = test.asksInFull() ? typeTestPlanFor(TypeTable::anything, representation, test.testedType()) : planFor(source, test.testedType()); + bool asksForOneClosure = plan.verdict == TypeTestVerdict::NeedsTest && !plan.passingPrimitives && plan.stringLiterals.isEmpty() && plan.objects == ObjectsVerdict::NeedTest && plan.cellTests.size() == 1 && plan.cellTests[0].kind == CellTestKind::IsClosureOf; + return asksForOneClosure && emitter().closureWordNames(plan.cellTests[0].function) ? plan.cellTests[0].function : FunctionID(); +} + +bool CastEmitter::tryBeginClosureChain(const ValueContext& test) +{ + FunctionID function = closureThatTestAsksFor(test.id()); + if (!function || generator().keepsValuesInHeapFrame() || generator().isLayingOutTurns() || generator().testOfBranch(test.block()) != test.id()) + return false; + auto bringsNothing = [&](EdgeID id) { + const Edge& edge = m_code.at(id); + return edge.arguments().empty() && m_code.at(edge.to).parameters().empty(); + }; + struct TestInChain { + BlockID block; + FunctionID function; + }; + Vector tests; + BlockID block = test.block(); + for (ValueID asking = test.id();;) { + const Terminator& end = m_code.at(block).terminator; + if (!bringsNothing(end.taken()) || !bringsNothing(end.notTaken())) + break; + tests.append({ block, function }); + BlockID next = m_code.at(end.notTaken()).to; + if (generator().singlePredecessorOf(next) != block || m_code.at(next).handler != m_code.at(block).handler) + break; + ValueID nextTest = generator().testOfBranch(next); + std::span values = m_code.valuesOf(next); + if (!nextTest || values.back() != nextTest) + break; + auto isNarrowedByFailure = [&](ValueID id) { + const Operation& narrowed = m_code.at(id); + return narrowed.opcode == Opcode::Narrowed && !narrowed.isOnTakenEdge() && narrowed.testOperand() == asking && narrowed.valueOperand() == m_code.at(asking).valueOperand() && narrowed.representation == representationOf(narrowed.valueOperand()); + }; + ValueID asked = m_code.at(nextTest).opcode == Opcode::TypeTest ? m_code.at(nextTest).valueOperand() : ValueID(); + bool asksSameValue = asked && (asked == m_code.at(asking).valueOperand() || (!m_code.isConstant(asked) && m_code.blockOf(asked) == next && isNarrowedByFailure(asked))); + if (!asksSameValue || !std::ranges::all_of(values.first(values.size() - 1), isNarrowedByFailure)) + break; + function = closureThatTestAsksFor(nextTest); + if (!function || std::ranges::any_of(tests, [&](const TestInChain& earlier) { return earlier.function == function; })) + break; + asking = nextTest; + block = next; + } + if (tests.size() < 2) + return false; + + const Edge& otherwise = m_code.at(m_code.at(tests.last().block).terminator.notTaken()); + uint32_t number = m_closureChains.size(); + Vector cases; + for (uint32_t place = 0; place < tests.size(); ++place) { + const Edge& taken = m_code.at(m_code.at(tests[place].block).terminator.taken()); + cases.append({ tests[place].function, generator().edgeWithoutArguments(taken) }); + generator().expectOtherJumps(taken.to); + if (place) + m_placesInClosureChains.set(tests[place].block.index(), PlaceInClosureChain { number, place }); + } + generator().expectOtherJumps(otherwise.to); + + Machine::DeclaredAccessesSuspension suspension(m_out); + BranchTargets targets = test.branchTargets(); + ValueID source = test->valueOperand(); + bool isBoxed = representationOf(source) == Representation::JSValue; + if (isBoxed && (test->asksInFull() || (typeOf(source).primitives & ~cellPrimitives))) { + LBasicBlock isCellBlock = newBlock(); + m_out.branch(isCell(in(source)).value(), normalEdge(isCellBlock), generator().edgeWithoutArguments(otherwise)); + m_out.appendToContinuation(isCellBlock); + } + CellPointer closure = isBoxed ? unboxCell(in(source)) : in(source); + Word32 word = emitter().upperHalfOfWordAtSlimClosureWord(closure); + m_out.branch(equal(word, emitter().upperHalfOfClosureWordOf(tests[0].function)).value(), targets.taken, targets.notTaken); + m_closureChains.append({ closure, word, WTF::move(cases) }); + build().statistics().count(StatisticID::ClosureTestsChained); + return true; +} + +bool CastEmitter::tryContinueClosureChain(const ValueContext& test) +{ + if (m_closureChains.isEmpty()) + return false; + auto place = m_placesInClosureChains.find(test.block().index()); + if (place == m_placesInClosureChains.end() || generator().isLayingOutTurns()) + return false; + const ClosureChain& chain = m_closureChains[place->value.chain]; + const ClosureTestEmitter::ClosureCase& closureCase = chain.cases[place->value.place]; + bool isLast = place->value.place + 1 == chain.cases.size(); + Machine::DeclaredAccessesSuspension suspension(m_out); + BranchTargets targets = test.branchTargets(); + LBasicBlock hasOtherWord = isLast ? newBlock() : nullptr; + m_out.branch(equal(chain.word, emitter().upperHalfOfClosureWordOf(closureCase.function)).value(), targets.taken, isLast ? normalEdge(hasOtherWord) : targets.notTaken); + if (!isLast) + return true; + m_out.appendToContinuation(hasOtherWord); + emitter().branchOnChangedClosure(chain.closure, chain.cases.span(), targets.notTaken); + return true; +} + void CastEmitter::generateTypeTest(const ValueContext& value) { + if (value.isBranchCondition() && (tryContinueClosureChain(value) || tryBeginClosureChain(value))) + return; ValueID source = value->valueOperand(); Operand operand = get(source); if (value->asksInFull()) @@ -207,15 +431,34 @@ void CastEmitter::generateTypeTest(const ValueContext& value) passesWhateverComes = row.part == PartOfEffects::TestsOfGuard && unit.role == CodeRole::ParameterGuard && unit.function.index() == row.row; } const TypeTestPlan& plan = passesWhateverComes ? typeTestPlanFor(whatMayCome, Representation::JSValue, whatMayCome) : value->asksInFull() ? typeTestPlanFor(whatMayCome, Representation::JSValue, value->testedType()) : planFor(source, value->testedType()); + if (!passesWhateverComes && !value->asksInFull() && operand.is() && tryTestWordOfSlot(value, operand.as(), plan)) + return; if (!value.isBranchCondition()) { define(value, hasType(operand, plan)); return; } - if (!passesWhateverComes && !value->asksInFull() && (tryBranchOnInt32AndHandItOn(value, operand) || tryBranchOnNumberAndHandItOn(value, operand))) + if (!passesWhateverComes && !value->asksInFull() && (tryBranchOnInt32AndHandItOn(value, operand) || tryBranchOnNumberAndHandItOn(value, operand) || tryDeferTestToSum(value, operand))) return; branchOnType(operand, plan, value.branchTargets()); } +void CastEmitter::generateIsUnlockedInstance(const ValueContext& value) +{ + bool mayBeLocked = instancesMayBeLocked(m_program, m_facts.linked(), value->testedType()); + Operand operand = get(value->valueOperand()); + if (value.isBranchCondition()) { + if (mayBeLocked) + branch(boolean(false), value.branchTargets()); + else + branchOnType(operand, planFor(value->valueOperand(), value->testedType()), value.branchTargets()); + return; + } + if (mayBeLocked) + define(value, boolean(false)); + else + define(value, hasType(operand, planFor(value->valueOperand(), value->testedType()))); +} + auto CastEmitter::questionForStubThatTells(TypeID type) -> std::optional { if (!types().at(type).isJavaScriptValue()) @@ -257,6 +500,29 @@ auto CastEmitter::questionForStubThatTells(TypeID type) -> std::optional(test->valueOperand()); + LBasicBlock stops = newBlock(); + LBasicBlock continuation = test.isBranchCondition() ? nullptr : newBlock(); + BlockEdge hasCheckedType = continuation ? normalEdge(continuation) : test.branchTargets().notTaken; + if (!continuation) { + BranchTargets targets = test.branchTargets(); + LBasicBlock isNotFalse = newBlock(); + LBasicBlock isNoBoolean = newBlock(); + m_out.branch(equal(found, boxedConstant(jsBoolean(false))).value(), targets.notTaken, normalEdge(isNotFalse)); + m_out.appendTo(isNotFalse); + m_out.branch(equal(found, boxedConstant(jsBoolean(true))).value(), targets.taken, rareEdge(isNoBoolean)); + m_out.appendTo(isNoBoolean); + } + emitter().emitPerFunctionBranch(found, typeTestPlanFor(check.typeWithoutProof(), Representation::JSValue, check.checkedType()), { hasCheckedType, rareEdge(stops) }); + m_out.appendTo(stops); + emitter().crashOnFalseProof(test, found, check.source(), FalseProofWord::summarize(types().at(check.checkedType()).primitives, types().at(check.checkedType()).hasObjects())); + if (continuation) + m_out.appendTo(continuation); +} + void CastEmitter::generateCheckProof(const ValueContext& value) { if (value->isLeftToTestBehind()) @@ -288,6 +554,8 @@ void CastEmitter::generateCheckProof(const ValueContext& value) bool looksAtCell = sourcesOfProofsCheckedUnder(build().options().testing); if (!asksForType && !looksAtCell) return; + if (!looksAtCell && generator().leaveCheckToTruthTest(value)) + return; bool asksStub = looksAtCell && !generator().isInitializingCell(); if (asksStub) { if (std::optional asked = asksForType ? questionForStubThatTells(value->checkedType()) : std::optional(QuestionForStub { checkProofOfKinds(), KindsOfProof::all, std::nullopt, std::nullopt })) { diff --git a/Source/JavaScriptCore/aot/codegen/AOTCastEmitter.h b/Source/JavaScriptCore/aot/codegen/AOTCastEmitter.h index b5e05281a453b..1bda30f198948 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTCastEmitter.h +++ b/Source/JavaScriptCore/aot/codegen/AOTCastEmitter.h @@ -7,6 +7,7 @@ #if ENABLE(AOT) +#include "AOTClosureTestEmitter.h" #include "AOTOperationEmitter.h" #include "AOTStubKey.h" @@ -19,13 +20,15 @@ class CastEmitter final : public OperationEmitter { { } - void generateNarrowed(const ValueContext& value) { defineNarrowed(value, get(value->valueOperand())); } + void generateNarrowed(const ValueContext&); void generateCheckInitialized(const ValueContext&); void generateRequireObjectCoercible(const ValueContext&); void requireObjectCoercible(const ValueContext&); void generateRequireObject(const ValueContext&); void generateTypeTest(const ValueContext&); + void generateIsUnlockedInstance(const ValueContext&); void generateCheckProof(const ValueContext&); + void checkBooleanOrBranch(const ValueContext& test, ValueID check); struct QuestionForStub { StubKey stub; uint32_t kinds; @@ -34,6 +37,11 @@ class CastEmitter final : public OperationEmitter { }; std::optional questionForStubThatTells(TypeID); std::optional takeIntegerHandedOnTo(ValueID unboxing); + struct DeferredNumberCheck { + Boxed value; + BlockEdge failing; + }; + Vector takeNumberChecksOf(ValueID sum); private: const TypeTestPlan& planFor(ValueID operand, TypeID to) { return typeTestPlanFor(typeIDOf(operand), representationOf(operand), to); } @@ -41,8 +49,27 @@ class CastEmitter final : public OperationEmitter { bool tryBranchOnInt32AndHandItOn(const ValueContext& test, Operand); ValueID unboxingToDoubleBehindTestForNumber(const ValueContext& test); bool tryBranchOnNumberAndHandItOn(const ValueContext& test, Operand); + ValueID sumBehindTestForNumber(const ValueContext& test); + bool tryDeferTestToSum(const ValueContext& test, Operand); + bool tryTestWordOfSlot(const ValueContext& test, Boxed, const TypeTestPlan&); + FunctionID closureThatTestAsksFor(ValueID test); + bool tryBeginClosureChain(const ValueContext& test); + bool tryContinueClosureChain(const ValueContext& test); + + struct ClosureChain { + CellPointer closure; + Word32 word; + Vector cases; + }; + struct PlaceInClosureChain { + uint32_t chain; + uint32_t place; + }; + Vector m_closureChains; + UncheckedKeyHashMap, WTF::UnsignedWithZeroKeyHashTraits> m_placesInClosureChains; UncheckedKeyHashMap, WTF::UnsignedWithZeroKeyHashTraits> m_integersHandedOn; + UncheckedKeyHashMap, DefaultHash, WTF::UnsignedWithZeroKeyHashTraits> m_numberChecksOfSums; }; } } } // namespace JSC::AOT::Typed diff --git a/Source/JavaScriptCore/aot/codegen/AOTClosureTestEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTClosureTestEmitter.cpp index a8f3be76c0583..110b188d2d27c 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTClosureTestEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTClosureTestEmitter.cpp @@ -105,6 +105,21 @@ void ClosureTestEmitter::branchOnClosureTest(CellPointer cell, FunctionID functi branchOnIsChangedClosureWithNumber(cell, isSlim, m_mentions.add(NumberSpace::ClosureFunction, function), targets); } +void ClosureTestEmitter::branchOnChangedClosure(CellPointer cell, std::span cases, BlockEdge otherwise) +{ + LBasicBlock isChanged = newBlock(); + m_out.branch(hasMutableCellType(cell, SlimClosureWithPropertiesType).value(), rareEdge(isChanged), otherwise); + + m_out.appendTo(isChanged); + Word32 number = numberOfChangedClosure(cell); + for (const ClosureCase& closureCase : cases) { + LBasicBlock next = newBlock(); + m_out.branch(equal(number, closureNumberOf(closureCase.function)).value(), closureCase.target, normalEdge(next)); + m_out.appendToContinuation(next); + } + m_out.jump(otherwise); +} + Condition ClosureTestEmitter::isClosureOfAny(CellPointer cell, std::span functions) { return asBoolean([&](BranchTargets targets) { @@ -307,7 +322,12 @@ CellPointer ClosureTestEmitter::closureIfCreated(FunctionID function) Word32 entry = closureTableEntry(function); Condition isNotCreated = isZero(entry); CellPointer none(m_out.constInt64(0)); - return select(isNotCreated, none, decode(NarrowReference(add(entry, 1).value()))); + return select(isNotCreated, none, closureInTableEntry(entry)); +} + +CellPointer ClosureTestEmitter::closureInTableEntry(Word32 entry) +{ + return decode(NarrowReference(add(entry, 1).value())); } Word32 ClosureTestEmitter::closureTableEntry(FunctionID function) diff --git a/Source/JavaScriptCore/aot/codegen/AOTClosureTestEmitter.h b/Source/JavaScriptCore/aot/codegen/AOTClosureTestEmitter.h index 540105e87b9a3..a030436d5b594 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTClosureTestEmitter.h +++ b/Source/JavaScriptCore/aot/codegen/AOTClosureTestEmitter.h @@ -21,11 +21,21 @@ class ClosureTestEmitter final : public OperationEmitter { Condition hasFunctionCellType(CellPointer); Condition isSlimClosureAsCreated(CellPointer); CellPointer closureIfCreated(FunctionID); + Word32 closureTableEntry(FunctionID); + CellPointer closureInTableEntry(Word32); std::optional mentionOfUpperHalfThatNames(FunctionID); + bool closureWordNames(FunctionID) const; + Word32 upperHalfOfWordAtSlimClosureWord(CellPointer); + Word32 upperHalfOfClosureWordOf(FunctionID); + struct ClosureCase { + FunctionID function; + BlockEdge target; + }; + void branchOnChangedClosure(CellPointer, std::span, BlockEdge otherwise); + private: FunctionKindData closureKindOf(FunctionID) const; - bool closureWordNames(FunctionID) const; void branchOnClosureTest(CellPointer, FunctionID, BranchTargets); struct ClosureNumberRun { FunctionID first; @@ -35,15 +45,12 @@ class ClosureTestEmitter final : public OperationEmitter { void branchOnClosureNumberInRuns(Word32, std::span ascendingRuns, int32_t addend, bool mayLieBelowFirstRun, BranchTargets); void branchOnIsChangedClosureWithNumber(CellPointer, bool isSlim, uint32_t mentionOfClosureNumber, BranchTargets); Word32 numberOfChangedClosure(CellPointer); - Word32 upperHalfOfWordAtSlimClosureWord(CellPointer); ptrdiff_t offsetOfClosureHeader(FunctionKindData) const; - Word32 closureTableEntry(FunctionID); TypedPointer closureCellOf(FunctionID); Condition hasClosureStructure(CellPointer, FunctionID); Condition hasClosureWordOf(CellPointer, bool isSlim, FunctionID); Word32 closureNumberOf(FunctionID, int32_t addend = 0); uint32_t mentionOfUpperHalfOfClosureWordOf(FunctionID); - Word32 upperHalfOfClosureWordOf(FunctionID); }; diff --git a/Source/JavaScriptCore/aot/codegen/AOTCodeGenerator.cpp b/Source/JavaScriptCore/aot/codegen/AOTCodeGenerator.cpp index b1644e8f830de..132b066b97d21 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTCodeGenerator.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTCodeGenerator.cpp @@ -111,6 +111,8 @@ GenerationResult CodeGenerator::run() m_looksBehindEachOperation |= m_keepsValuesInHeapFrame; } + findStringsThatMayStayRopes(); + definePinnedValues(m_out, m_pinned); emitter().generateEntries(); for (size_t position = 0; position < m_order.size(); ++position) { @@ -197,7 +199,7 @@ GenerationResult CodeGenerator::run() loop->listsAskedForKind.removeAllMatching([](const ListAskedForKind& asked) { return asked.otherKindsAreNotAsked; }); - askKindsReadWhereLoopIsComeBackTo(*loop); + versionTurnsByKindsRead(*loop); findSumsAnew(*loop); } if (!m_hasFailed && !loop->boundsAskedInFront.isEmpty()) { @@ -389,7 +391,7 @@ void CodeGenerator::orderBlocks() uint32_t numberOfBlocks = code().numberOfBlocks(); m_singlePredecessors = allocateSpanOfAllBitsSet(arena, numberOfBlocks); m_scopeAtEndOfBlock = allocateZeroedSpan(arena, numberOfBlocks); - std::span order = allocateSpan(arena, numberOfBlocks); + Machine::UncheckedSpan order = allocateSpan(arena, numberOfBlocks); uint32_t numberOrdered = 0; Vector steps; @@ -480,7 +482,7 @@ void CodeGenerator::orderBlocks() Step step { entries[entry].block, BlockID(), StepKind::OtherWayIn }; while (enter(step) || takeStep(step)) { } } - m_order = order.last(numberOrdered); + m_order = order.span().last(numberOrdered); for (const WayIntoBlock& way : m_waysIntoJoins) m_singlePredecessors[way.to.index()] = BlockID(); } @@ -499,8 +501,8 @@ void CodeGenerator::findDominators() m_dominatorsByPlace = allocateSpanOfAllBitsSet(arena, numberOfPlaces); m_dominatorsByPlace[root] = root; - std::span firstWaysOut = allocateSpanOfAllBitsSet(arena, numberOfPlaces); - std::span nextWaysOut = allocateSpan(arena, m_waysIntoJoins.size()); + Machine::UncheckedSpan firstWaysOut = allocateSpanOfAllBitsSet(arena, numberOfPlaces); + Machine::UncheckedSpan nextWaysOut = allocateSpan(arena, m_waysIntoJoins.size()); for (uint32_t index = 0; index < m_waysIntoJoins.size(); ++index) { const WayIntoBlock& way = m_waysIntoJoins[index]; if (!way.from) { @@ -512,8 +514,8 @@ void CodeGenerator::findDominators() first = index; } - std::span firstWaysIn = allocateSpanOfAllBitsSet(arena, numberOfPlaces); - std::span nextWaysIn = allocateSpan(arena, m_waysIntoJoins.size()); + Machine::UncheckedSpan firstWaysIn = allocateSpanOfAllBitsSet(arena, numberOfPlaces); + Machine::UncheckedSpan nextWaysIn = allocateSpan(arena, m_waysIntoJoins.size()); Vector targetsOfWaysThatComeRound; for (uint32_t place = 1; place < numberOfPlaces; ++place) { for (uint32_t index = firstWaysOut[place]; index != noWay; index = nextWaysOut[index]) { @@ -627,7 +629,7 @@ void CodeGenerator::numberDominatorTree() uint32_t size = ++m_subtreesByPlace[place].size; m_subtreesByPlace[m_dominatorsByPlace[place]].size += size; } - std::span nextNumbersBelow = allocateSpan(arena, numberOfPlaces); + Machine::UncheckedSpan nextNumbersBelow = allocateSpan(arena, numberOfPlaces); nextNumbersBelow[0] = 0; for (size_t place = 1; place < numberOfPlaces; ++place) { uint32_t& nextNumberBelowDominator = nextNumbersBelow[m_dominatorsByPlace[place]]; @@ -751,6 +753,79 @@ bool CodeGenerator::isStoredIntoByName(ValueID made) const return false; } +void CodeGenerator::findStringsThatMayStayRopes() +{ + if (m_keepsValuesInHeapFrame || !code().opcodesOfValues().contains(Opcode::FlattenString)) + return; + struct ReadOfString { + ValueID string; + BlockID block; + }; + Vector reads; + Vector candidates; + Vector readOtherwise; + Vector positionsOfHeads; + auto isCandidate = [&](ValueID value) { + return value && opcodeOf(value) == Opcode::FlattenString && code().at(value).mayStayRope(); + }; + auto noteOtherRead = [&](ValueID value) { + if (isCandidate(value) && !readOtherwise.contains(value)) + readOtherwise.append(value); + }; + for (size_t position = 0; position < m_order.size(); ++position) { + BlockID id = m_order[position]; + const Block& block = code().at(id); + if (block.isLoopHead) + positionsOfHeads.append(position); + for (ValueID user : code().valuesOf(id)) { + const Operation& operation = code().at(user); + if (isCandidate(user)) + candidates.append(user); + if (operation.opcode != Opcode::GetStringCodeUnit) + operation.forEachChild(noteOtherRead); + else if (isCandidate(operation.stringOperand())) + reads.append({ operation.stringOperand(), id }); + } + block.terminator.forEachChild(noteOtherRead); + for (uint32_t edge = 0; edge < block.terminator.numberOfEdges; ++edge) { + for (ValueID argument : code().at(block.terminator.edge(edge)).arguments()) + noteOtherRead(argument); + } + } + for (ValueID string : candidates) { + if (readOtherwise.contains(string)) + continue; + BlockID home = code().blockOf(string); + Vector positionsOfCarriers; + bool isReadInRowOfIterations = false; + for (const ReadOfString& read : reads) { + if (read.string != string) + continue; + std::optional nearest; + for (size_t position : positionsOfHeads) { + BlockID head = m_order[position]; + if (head != home && dominates(home, head) && dominates(head, read.block)) + nearest = position; + } + if (!nearest || positionsOfCarriers.contains(*nearest)) + continue; + isReadInRowOfIterations |= !!findCountedLoopAt(*nearest); + positionsOfCarriers.append(*nearest); + } + if (isReadInRowOfIterations) + continue; + m_stringsThatMayStayRopes.append(string); + build().statistics().count(StatisticID::StringsResolvedOnRead); + std::ranges::sort(positionsOfCarriers); + for (size_t position : positionsOfCarriers) { + BlockID head = m_order[position]; + bool isPassedArguments = std::ranges::any_of(code().at(head).parameters(), [&](ValueID parameter) { return !!m_values[parameter.index()]; }); + if (isPassedArguments) + m_stringsCarriedByLoops.append({ string, head, m_out.phiIn(blockFor(head), machineTypeOf(Pointer::representation)), m_out.phiIn(blockFor(head), machineTypeOf(Word32::representation)) }); + } + } +} + void CodeGenerator::createPhis() { constexpr unsigned mostBlocksThatHandParameterOn = 4; @@ -1106,6 +1181,8 @@ NEVER_INLINE ValueID CodeGenerator::readLeftToBranchOnItsTruthSlow(BlockID id, s bool asksTruthOfRead = asked.opcode == Opcode::ToBoolean && asked.form() == OperatorForm::Generic && asked.valueOperand() == read && code().useCountOf(read) == 1; return asksTruthOfRead && emitter().asksTruthOfRawElement(read) ? read : ValueID(); } + if (asked.opcode == Opcode::ToBoolean && (asked.form() == OperatorForm::Generic || asked.form() == OperatorForm::Numbers) && asked.valueOperand() == read && code().useCountOf(read) == 1 && emitter().asksTruthOfNumberInList(read)) + return read; if (code().at(id).isInLoop) return { }; if (code().useCountOf(read) == 1 && code().onlyUserOf(read) == test && emitter().integerThatLengthIsAskedFor(id, read, test)) @@ -1195,6 +1272,35 @@ bool CodeGenerator::isReadAgainBehindItsOwnTruthAlone(const ValueContext& read) return code().useCountOf(read.id()) == uses; } +bool CodeGenerator::isReadBehindOwnUndefinedTest(const ValueContext& read) const +{ + const Block& block = code().at(read.block()); + if (block.terminator.kind != TerminatorKind::Branch) + return false; + ValueID test = block.terminator.conditionOperand(); + if (!test || opcodeOf(test) != Opcode::TypeTest || code().at(test).valueOperand() != read.id() || !std::ranges::contains(read.operationsBehind(), test)) + return false; + TypeArena& types = inputs().types; + const Type& type = types.at(read->type); + TypeID tested = code().at(test).testedType(); + if (type.primitives & (Primitive::Null | Primitive::Absent) || !types.isSubset(TypeTable::undefined, tested) || !types.isSubset(tested, TypeTable::nullish)) + return false; + uint32_t uses = 1; + BlockID behind = read.block(); + for (EdgeID way = block.terminator.notTaken(); m_singlePredecessors[code().at(way).to.index()] == behind;) { + behind = code().at(way).to; + for (ValueID id : code().valuesOf(behind)) { + const Operation& narrowed = code().at(id); + uses += narrowed.opcode == Opcode::Narrowed && !narrowed.isOnTakenEdge() && narrowed.testOperand() == test && narrowed.valueOperand() == read.id() && narrowed.representation == read->representation; + } + const Terminator& end = code().at(behind).terminator; + if (end.kind != TerminatorKind::Jump) + break; + way = end.next(); + } + return code().useCountOf(read.id()) == uses; +} + void CodeGenerator::generateReturnedCall(const TerminatorContext& terminator, ValueID call) { ValueContext context(terminator.block(), call, code().at(call), handlerOf(terminator.block())); @@ -1230,6 +1336,24 @@ bool CodeGenerator::leavesHoleToTestBehind(const ValueContext& read) return true; } +bool CodeGenerator::leaveCheckToTruthTest(const ValueContext& check) +{ + const Type& checked = inputs().types.at(check->checkedType()); + if (!(checked.primitives & Primitive::Boolean) || !checked.hasOnlyPrimitives(Primitive::Boolean | Primitive::UndefinedOrNull | Primitive::Absent)) + return false; + BlockID home = check.block(); + const Block& block = code().at(home); + ValueID test = testReadByBranchAlone(home, block.terminator); + if (!test || code().at(test).opcode != Opcode::ToBoolean || code().at(test).form() != OperatorForm::Bits || code().at(test).valueOperand() != check->valueOperand()) + return false; + std::span values = code().valuesOf(block); + if (values.size() < 2 || values.back() != test || values[values.size() - 2] != check.id()) + return false; + m_truthTestWithCheck = test; + m_checkLeftToTruthTest = check.id(); + return true; +} + void CodeGenerator::generateBlock(BlockID id, const Block& block) { std::span values = code().valuesOf(block); @@ -1245,7 +1369,7 @@ void CodeGenerator::generateBlock(BlockID id, const Block& block) m_blocksBegun.quickSet(id.index()); if (std::exchange(m_nextBlockGoesOnHere, false)) RELEASE_ASSERT(blockFor(id) == m_out.m_block && currentBlockIsOpen()); - else if (BlockID predecessor = m_singlePredecessors[id.index()]) + else if (BlockID predecessor = m_singlePredecessors[id.index()]; predecessor && !m_blocksWithOtherJumps.get(id.index())) m_out.appendToSuccessor(blockFor(id), m_scopeAtEndOfBlock[predecessor.index()]); else m_out.appendTo(blockFor(id)); @@ -1349,7 +1473,7 @@ static constexpr int64_t largestStepOfCounter = 16; static constexpr unsigned mostOperationsInTurnLaidOut = 16; static constexpr OpcodeSet operationsOfTurnsLaidOut { Opcode::Narrowed, Opcode::Box, Opcode::Unbox, Opcode::IntegerToNumber, Opcode::NumberToInteger, Opcode::WidenInteger, Opcode::NarrowInteger, Opcode::ToInt32, - Opcode::Add, Opcode::Sub, Opcode::Mul, Opcode::Div, Opcode::Mod, Opcode::RemainderByReciprocal, Opcode::Increment, Opcode::Decrement, Opcode::Negate, + Opcode::Add, Opcode::Sub, Opcode::Mul, Opcode::Div, Opcode::Mod, Opcode::RemainderByReciprocal, Opcode::RemainderByWholeDivisor, Opcode::Increment, Opcode::Decrement, Opcode::Negate, Opcode::ToNumber, Opcode::BitAnd, Opcode::BitOr, Opcode::BitXor, Opcode::BitNot, Opcode::LShift, Opcode::RShift, Opcode::URShift, Opcode::Less, Opcode::LessEq, Opcode::Greater, Opcode::GreaterEq, Opcode::StrictEq, Opcode::Eq, Opcode::TypeTest, Opcode::GetStringCodeUnit, Opcode::GetTypedArrayElement, Opcode::GetArrayElement, Opcode::IsElementPresent, @@ -1624,7 +1748,7 @@ std::optional CodeGenerator::findCountedLoopAt(size_ loop.listsAskedForKind.append({ operation.arrayOperand(), ArrayStorage::Double, false, true, true }); continue; } - if (operation.opcode != Opcode::GetArrayElement || !operation.admitsInt32Storage() || code().isConstant(operation.arrayOperand())) + if (operation.opcode != Opcode::GetArrayElement || !operation.admitsInt32Storage() || operation.arrayStorage() != ArrayStorage::Double || code().isConstant(operation.arrayOperand())) continue; ValueID list = operation.arrayOperand(); if (ValueID holdsDoubles = operation.holdsDoublesOperand()) { @@ -1810,14 +1934,23 @@ std::optional CodeGenerator::findCountedLoopAt(size_ loop.mayComeBack = false; }); } - askKindsReadWhereLoopIsComeBackTo(loop); + versionTurnsByKindsRead(loop); findSumsOfIntegers(loop, back); return loop; } -void CodeGenerator::askKindsReadWhereLoopIsComeBackTo(CountedLoop& loop) +void CodeGenerator::versionTurnsByKindsRead(CountedLoop& loop) const { - if (!loop.mayComeBack || !loop.listsAskedForKind.isEmpty() || loop.listsWhoseKindIsRead.isEmpty()) + if (!loop.listsAskedForKind.isEmpty() || loop.listsWhoseKindIsRead.isEmpty()) + return; + auto testsIndexingTypeAlone = [&](ValueID value) { + const Operation& read = code().at(value); + return read.opcode != Opcode::GetArrayElement || !loop.listsWhoseKindIsRead.contains(read.arrayOperand()) || loop.accessesUnderCounterBelowBound.contains(value); + }; + bool readsAreBelowBound = loop.storesIntoLists && loop.viewsAskedForKind.isEmpty() && std::ranges::all_of(loop.turn, [&](BlockID block) { + return std::ranges::all_of(code().valuesOf(block), testsIndexingTypeAlone); + }); + if (!loop.mayComeBack && !readsAreBelowBound) return; loop.secondTurnsGoByKindsRead = loop.listsWhoseKindIsRead.size() > 1; for (ValueID list : std::exchange(loop.listsWhoseKindIsRead, { })) @@ -2476,6 +2609,8 @@ void CodeGenerator::layOutTurns(const CountedLoop& loop) m_out.appendTo(turns); m_headOfTurnsLaidOut = head; leavesTurnsLaidOut = true; + if (comesBack || (headOfRest != writtenHead && !noTurnComesToNextFront)) + m_blocksOfLoopLaidOut = findBlocksOfLoop(loop); for (const LoopCarriedValueStore& store : loop.loopCarriedValueStores) { m_valuesLeftOutOfTurnsLaidOut.append(store.box); m_valuesLeftOutOfTurnsLaidOut.append(store.store); @@ -2541,6 +2676,19 @@ void CodeGenerator::layOutTurns(const CountedLoop& loop) } return blockLeavingTurn; }; + LBasicBlock blockEndingLoop = nullptr; + m_blockEndingLoop = [&] { + if (!blockEndingLoop) { + blockEndingLoop = blockHandingOver(comesBack ? phisIntoWritten : phisOfRest, comesBack ? intoWritten : headOfRest, [&](size_t index) -> LValue { + if (const SumOfIntegers* sum = sumAt(index)) + return m_out.doubleAdd(sum->base, m_out.intToDouble(valuesAheadOfTurn[index])); + if (index == indexOfCounter && turnLaidOut) + return m_out.add(firstCounter, m_out.constantOfType(firstCounter->type(), turnLaidOut * loop.step)); + return forWrittenLoop(index, valuesAheadOfTurn[index]); + }); + } + return blockEndingLoop; + }; for (unsigned index = 0; index < turn.size(); ++index) { LBasicBlock& block = m_blocks[turn[index].index()]; if (!index) { @@ -2592,6 +2740,8 @@ void CodeGenerator::layOutTurns(const CountedLoop& loop) m_holesLeaveTurnsLaidOut = false; m_wordsOfElementsInTurnLaidOut.shrink(0); m_blockLeavingTurnLaidOut = nullptr; + m_blockEndingLoop = nullptr; + m_blocksOfLoopLaidOut.shrink(0); m_nextBlockGoesOnHere = false; emitter().forgetWidthsFound(); m_out.setOrigin(Machine::Origin()); @@ -2742,7 +2892,7 @@ BlockEdge CodeGenerator::edgeTo(const TerminatorContext& terminator, uint32_t ta const Edge& edge = code().at(terminator->edge(targetIndex)); const Block& blockAtEnd = code().at(edge.to); if (leavesTurnLaidOut(edge.to)) [[unlikely]] - return edgeLeavingTurnLaidOut(blockAtEnd); + return edgeLeavingTurnLaidOut(edge.to); BlockEdge target(blockFor(edge.to), edgeClassLeadingTo(blockAtEnd.frequency)); for (ValueID parameter : blockAtEnd.parameters()) { if (m_values[parameter.index()]) @@ -2763,9 +2913,49 @@ std::optional CodeGenerator::edgeThatBringsNothing(EdgeID id) return BlockEdge(block, edgeClassLeadingTo(blockAtEnd.frequency)); } -NEVER_INLINE BlockEdge CodeGenerator::edgeLeavingTurnLaidOut(const Block& blockAtEnd) +BlockEdge CodeGenerator::edgeWithoutArguments(const Edge& edge) +{ + const Block& blockAtEnd = code().at(edge.to); + RELEASE_ASSERT(edge.arguments().empty() && blockAtEnd.parameters().empty()); + return BlockEdge(blockFor(edge.to), edgeClassLeadingTo(blockAtEnd.frequency)); +} + +Vector CodeGenerator::findBlocksOfLoop(const CountedLoop& loop) const +{ + BlockID head = m_order[loop.positionOfHead]; + Vector found { head }; + Vector pending; + BitVector isFound(code().numberOfBlocks()); + isFound.quickSet(head.index()); + auto add = [&](BlockID block) { + if (code().isReached(block) && !isFound.quickSet(block.index())) + pending.append(block); + }; + for (BlockID block : loop.turn) + add(block); + code().forEachIncomingEdgeUnsealed(head, [&](const Edge& edge) { + if (!m_scopeAtEndOfBlock[edge.from.index()].block) + add(edge.from); + }); + while (!pending.isEmpty()) { + BlockID block = pending.takeLast(); + found.append(block); + code().forEachIncomingEdgeUnsealed(block, [&](const Edge& edge) { + add(edge.from); + }); + } + std::ranges::sort(found); + return found; +} + +LBasicBlock CodeGenerator::blockForLeavingEdge(BlockID target) { - return BlockEdge(m_blockLeavingTurnLaidOut(), edgeClassLeadingTo(blockAtEnd.frequency)); + return m_blocksOfLoopLaidOut.isEmpty() || std::ranges::binary_search(m_blocksOfLoopLaidOut, target) ? m_blockLeavingTurnLaidOut() : m_blockEndingLoop(); +} + +NEVER_INLINE BlockEdge CodeGenerator::edgeLeavingTurnLaidOut(BlockID target) +{ + return BlockEdge(blockForLeavingEdge(target), edgeClassLeadingTo(code().at(target).frequency)); } NEVER_INLINE BlockEdge CodeGenerator::edgeThatPassesArguments(const TerminatorContext& terminator, const Edge& edge, std::span declaredResults) @@ -2796,7 +2986,7 @@ void CodeGenerator::jumpTo(const TerminatorContext& terminator, uint32_t targetI return; } if (leavesTurnLaidOut(edge.to)) [[unlikely]] { - m_out.jump(BlockEdge(m_blockLeavingTurnLaidOut(), edgeClassLeadingTo(code().at(edge.to).frequency))); + m_out.jump(BlockEdge(blockForLeavingEdge(edge.to), edgeClassLeadingTo(code().at(edge.to).frequency))); return; } LBasicBlock& machineBlock = m_blocks[edge.to.index()]; @@ -2869,6 +3059,10 @@ void CodeGenerator::passArguments(const Edge& edge, std::span par if (m_values[parameters[index].index()]) passArgument(parameters[index], passed[index]); } + for (const StringCarriedByLoop& carried : m_stringsCarriedByLoops) { + if (carried.head == edge.to) + emitter().passOpenedRope(carried.string, edge.from, carried.characters, carried.implFlags); + } } NEVER_INLINE Operand CodeGenerator::getSlow(ValueID id, const Operation& operation) @@ -2887,6 +3081,10 @@ NEVER_INLINE Operand CodeGenerator::getSlow(ValueID id, const Operation& operati LValue lowered = m_values[id.index()]; if (!lowered) [[unlikely]] return zeroInPlaceOfValueOfFailedCode(representation); + if (m_valuesDefinedBoxed.get(id.index())) [[unlikely]] { + reportBoxedNumberUse(id); + return zeroInPlaceOfValueOfFailedCode(representation); + } if (representation == Representation::Double && (m_carriesParametersBoxed || m_followsNumbersOutOfBytes)) [[unlikely]] return getNumberInFormKept(id, lowered); if (representation == Representation::OptionalDouble && holdsAnyNaN(id)) [[unlikely]] @@ -2914,6 +3112,12 @@ void CodeGenerator::failAtNumberOrNothingReadForItsBits(ValueID id) fail(*m_operation, InternalErrorKind::UnsupportedConversion, makeString('v', id.index(), " ("_s, nameOf(opcodeOf(id)), ") is kept as its reader left it, for users that take it so, and is read by one that does not"_s)); } +void CodeGenerator::reportBoxedNumberUse(ValueID id) +{ + ASSERT_WITH_MESSAGE(!m_valuesDefinedBoxed.get(id.index()), "v%u is defined boxed for its only user, a Box", id.index()); + fail(*m_operation, InternalErrorKind::UnsupportedConversion, makeString('v', id.index(), " ("_s, nameOf(opcodeOf(id)), ") is defined boxed for its only user, a Box, and another operation reads it"_s)); +} + void CodeGenerator::verifyThatKeptValueIsNamedInThisStretch(ValueID id) { if (emitter().isReadAsNamedInAnotherStretch(id)) @@ -2931,7 +3135,7 @@ ALWAYS_INLINE_INTO_OTHER_FILES void CodeGenerator::verifyThatDefinitionDominates return; } if (!m_dominators) - m_dominators.emplace(code(), m_order); + m_dominators.emplace(code(), order()); if (code().isReached(home) && m_dominators->dominates(home, user)) return; fail(*m_operation, InternalErrorKind::UseNotDominated, makeString('v', id.index(), " ("_s, nameOf(opcodeOf(id)), ") is used in b"_s, user.index(), ", which b"_s, home.index(), ", where it is defined, does not dominate"_s)); @@ -2939,7 +3143,7 @@ ALWAYS_INLINE_INTO_OTHER_FILES void CodeGenerator::verifyThatDefinitionDominates Operand CodeGenerator::getWithAnyNaN(ValueID id) { - if (!holdsAnyNaN(id)) + if (!holdsAnyNaN(id) || m_valuesDefinedBoxed.get(id.index())) return get(id); return { Representation::Double, m_values[id.index()] }; } diff --git a/Source/JavaScriptCore/aot/codegen/AOTCodeGenerator.h b/Source/JavaScriptCore/aot/codegen/AOTCodeGenerator.h index 2ee4b2d7624f7..ee03a673d4414 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTCodeGenerator.h +++ b/Source/JavaScriptCore/aot/codegen/AOTCodeGenerator.h @@ -121,6 +121,14 @@ class CodeGenerator { bool isLayingOutTurns() const { return !!m_headOfTurnsLaidOut; } bool leavesTurnLaidOut(BlockID target) const { return m_headOfTurnsLaidOut && target != m_headOfTurnsLaidOut && !std::ranges::contains(m_turnLaidOut, target); } bool isInLoop(BlockID block) const { return code().at(block).isInLoop; } + bool mayStayRope(ValueID flatString) const { return m_stringsThatMayStayRopes.contains(flatString); } + void forEachLoopThatCarries(ValueID flatString, NOESCAPE const Invocable auto& functor) const + { + for (const StringCarriedByLoop& carried : m_stringsCarriedByLoops) { + if (carried.string == flatString) + functor(carried.head, carried.characters, carried.implFlags); + } + } bool isStoredIntoByName(ValueID) const; bool takesOpenForm(const OperationContext&, StateOfKnowledge = StateOfKnowledge::Always) const; bool takesFormOfLoops(const OperationContext&, StateOfKnowledge = StateOfKnowledge::Always) const; @@ -138,6 +146,8 @@ class CodeGenerator { std::optional kindOfListFoundInFrontOfTurnsLaidOut(ValueID list) const; bool liesInsideByQuestionInFrontOfTurnsLaidOut(ValueID read) const; bool leavesHoleToTestBehind(const ValueContext& read); + bool leaveCheckToTruthTest(const ValueContext& check); + ValueID checkLeftToTruthTest(ValueID test) const { return m_truthTestWithCheck == test ? m_checkLeftToTruthTest : ValueID(); } bool holeIsLeftToTestBehind(ValueID read) const { return m_elementsWhoseHolesAreLeftToTestsBehind.get(read.index()); } LValue& wordWhoseHoleIsLeftToTestAtWork() { return m_wordWhoseHoleIsLeftToTestAtWork; } bool holdsByQuestionInFrontOfTurnsLaidOut(ValueID relation) const { return m_relationsAskedInFrontOfTurnsLaidOut.contains(relation); } @@ -167,7 +177,7 @@ class CodeGenerator { LValue wordOf(ValueID id) const { return m_values[id.index()]; } void nameAgain(ValueID id, LValue lowered) { m_values[id.index()] = lowered; } bool keepsValuesInHeapFrame() const { return m_keepsValuesInHeapFrame; } - std::span order() const { return m_order; } + std::span order() const { return m_order.span(); } BlockID onlyPredecessorOf(BlockID block) const { return m_singlePredecessors[block.index()]; } bool blockDominates(BlockID above, BlockID below); BlockID nearestDominatorOf(BlockID); @@ -200,6 +210,33 @@ class CodeGenerator { RELEASE_ASSERT(!value.isBranchCondition()); defineValue(value.id(), value.representation(), result); } + bool isOnlyBoxed(ValueID id) const + { + ValueID user = code().useCountOf(id) == 1 ? code().onlyUserOf(id) : ValueID(); + return user && !m_keepsValuesInHeapFrame && code().at(user).opcode == Opcode::Box && code().at(user).representation == Representation::JSValue; + } + bool isOnlyTakenAsPosition(ValueID id) const + { + const Operation& unboxed = code().at(id); + if (unboxed.opcode != Opcode::Unbox || unboxed.representation != Representation::Double || m_keepsValuesInHeapFrame || code().useCountOf(id) != 1 || !code().onlyUserOf(id)) + return false; + const Operation& access = code().at(code().onlyUserOf(id)); + if ((access.opcode != Opcode::GetTypedArrayElement && access.opcode != Opcode::PutTypedArrayElement) || access.positionOperand() != id || access.block() != unboxed.block() || access.isInsideView() || access.level() == KnowledgeLevel::Expected) + return false; + return code().at(unboxed.valueOperand()).representation == Representation::JSValue && !isCarriedBoxed(unboxed.valueOperand()); + } + void defineBoxed(const ValueContext& value, Boxed result) + { + RELEASE_ASSERT(!value.isBranchCondition()); + defineValue(value.id(), Representation::JSValue, result); + m_valuesDefinedBoxed.set(value.id().index()); + } + void forgetBoxed(ValueID id) + { + if (m_valuesDefinedBoxed.get(id.index())) [[unlikely]] + m_valuesDefinedBoxed.clear(id.index()); + } + std::optional findBoxed(ValueID id) const { return m_valuesDefinedBoxed.get(id.index()) && m_values[id.index()] ? std::optional { Boxed(m_values[id.index()]) } : std::nullopt; } void defineTest(const ValueContext&, Condition); void defineResults(const ValueContext&, std::span); Operand resultAt(ValueID producer, uint32_t index) const; @@ -211,6 +248,9 @@ class CodeGenerator { LBasicBlock blockFor(BlockID) const; BlockEdge edgeTo(const TerminatorContext&, uint32_t targetIndex, std::span declaredResults = { }); std::optional edgeThatBringsNothing(EdgeID); + BlockEdge edgeWithoutArguments(const Edge&); + void expectOtherJumps(BlockID block) { m_blocksWithOtherJumps.set(block.index()); } + ValueID testOfBranch(BlockID id) const { return testReadByBranchAlone(id, code().at(id).terminator); } bool isTakenIntoSwitchInFront(BlockID block) const { return m_switchesTakenIntoSwitchInFront.get(block.index()); } void noteSwitchTakenIntoSwitchInFront(BlockID block) { m_switchesTakenIntoSwitchInFront.set(block.index()); } void jumpTo(const TerminatorContext&, uint32_t targetIndex, std::span declaredResults = { }); @@ -230,6 +270,7 @@ class CodeGenerator { }; std::optional waysOfTruthOf(const ValueContext& read) const; bool isReadAgainBehindItsOwnTruthAlone(const ValueContext& read) const; + bool isReadBehindOwnUndefinedTest(const ValueContext& read) const; void generateReturnedCall(const TerminatorContext&, ValueID call); template T& emitter(); @@ -296,6 +337,15 @@ class CodeGenerator { private: BlockID m_blockWhoseUsersOfLiteralsAreListed; Vector> m_usersOfLiteralsInBlock; + struct StringCarriedByLoop { + ValueID string; + BlockID head; + LValue characters; + LValue implFlags; + }; + void findStringsThatMayStayRopes(); + Vector m_stringsThatMayStayRopes; + Vector m_stringsCarriedByLoops; public: private: Vector, 2> m_scheduledLifetimeEnds; @@ -372,12 +422,15 @@ class CodeGenerator { return getSlow(id, operation); } BitVector m_elementsWhoseHolesAreLeftToTestsBehind; + ValueID m_truthTestWithCheck; + ValueID m_checkLeftToTruthTest; LValue m_wordWhoseHoleIsLeftToTestAtWork { nullptr }; UncheckedKeyHashMap, WTF::UnsignedWithZeroKeyHashTraits> m_elementsMadePresentAtHeadsOfBlocks; Operand getSlow(ValueID, const Operation&); Operand zeroInPlaceOfValueOfFailedCode(Representation); Operand getNumberInFormKept(ValueID, LValue lowered); COLD NEVER_INLINE void failAtNumberOrNothingReadForItsBits(ValueID); + COLD NEVER_INLINE void reportBoxedNumberUse(ValueID); Operand makeInWantedForm(ValueID, const Operation&, Representation wanted); Operand makeConstant(ValueID, const Operation&); void defineValue(ValueID id, Representation representation, Operand operand) @@ -393,7 +446,7 @@ class CodeGenerator { RELEASE_ASSERT(!lowered && operand.value && operand.value->type() == machineTypeOf(operand.representation)); lowered = operand.value; } - BlockEdge edgeLeavingTurnLaidOut(const Block& blockAtEnd); + BlockEdge edgeLeavingTurnLaidOut(BlockID blockAtEnd); BlockEdge edgeThatPassesArguments(const TerminatorContext&, const Edge&, std::span declaredResults); void passArguments(const Edge&, std::span parameters, std::span declaredResults); void passArgument(ValueID parameter, Operand); @@ -470,7 +523,7 @@ class CodeGenerator { }; bool findStoresOfWhatComesRound(CountedLoop&, const Edge& back) const; void storeLoopCarriedValue(const CountedLoop&, const LoopCarriedValueStore&, LValue); - static void askKindsReadWhereLoopIsComeBackTo(CountedLoop&); + void versionTurnsByKindsRead(CountedLoop&) const; struct LimitOfWrittenLoop { BlockID asking; Opcode relation { Opcode::Less }; @@ -541,13 +594,14 @@ class CodeGenerator { bool m_notesForms { false }; bool m_keepsValuesInHeapFrame { false }; BitVector m_switchesTakenIntoSwitchInFront; + BitVector m_blocksWithOtherJumps; bool m_looksBehindEachOperation { false }; BitVector m_storesBehindWhichOwnerIsRemembered; PinnedValues m_pinned; std::span m_values; std::span m_resultRanges; Vector m_results; - std::span m_blocks; + Machine::UncheckedSpan m_blocks; bool m_nextBlockGoesOnHere { false }; Vector m_blocksLeftOut; BlockID m_headOfTurnsLaidOut; @@ -558,6 +612,10 @@ class CodeGenerator { Vector m_argumentsForNextTurn; std::span m_turnLaidOut; WTF::Function m_blockLeavingTurnLaidOut; + WTF::Function m_blockEndingLoop; + Vector m_blocksOfLoopLaidOut; + Vector findBlocksOfLoop(const CountedLoop&) const; + LBasicBlock blockForLeavingEdge(BlockID target); Vector m_sumsOfIntegers; Vector m_valuesLeftOutOfTurnsLaidOut; std::span m_listsAskedForKindInFrontOfTurnsLaidOut; @@ -573,15 +631,15 @@ class CodeGenerator { bool m_asksElementsBehindTurns { false }; LValue m_boxedWordsOfElementsInTurns { nullptr }; Vector, 2> m_wordsOfElementsInTurnLaidOut; - std::span m_order; - std::span m_singlePredecessors; - std::span m_scopeAtEndOfBlock; + Machine::UncheckedSpan m_order; + Machine::UncheckedSpan m_singlePredecessors; + Machine::UncheckedSpan m_scopeAtEndOfBlock; BlockID m_onlyEntryBlock; bool m_firstBlockHasOtherWaysIn { false }; Vector m_waysIntoJoins; - std::span m_places; - std::span m_dominatorsByPlace; - std::span m_subtreesByPlace; + Machine::UncheckedSpan m_places; + Machine::UncheckedSpan m_dominatorsByPlace; + Machine::UncheckedSpan m_subtreesByPlace; std::array m_homesProven; BlockID m_userOfHomesProven; std::optional m_dominators; @@ -597,6 +655,7 @@ class CodeGenerator { bool m_hasFailed { false }; bool m_carriesParametersBoxed { false }; BitVector m_parametersCarriedBoxed; + BitVector m_valuesDefinedBoxed; bool m_followsNumbersOutOfBytes { false }; BitVector m_valuesHoldingAnyNaN; BitVector m_valuesReadForBitsOfNaN; diff --git a/Source/JavaScriptCore/aot/codegen/AOTConstantEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTConstantEmitter.cpp index 98ae5617ea5b3..041a09575f825 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTConstantEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTConstantEmitter.cpp @@ -307,6 +307,12 @@ CellPointer ConstantEmitter::cellMadeByStub(const OperationContext& context, Stu }); } +bool ConstantEmitter::loadsStringConstantByStub(const OperationContext& context, StringID id) +{ + std::span text = types().bytesOf(id); + return !text.empty() && !singleLatin1CharacterOf(text) && formOfConstantsWithCellsAt(context).form == Form::Closed; +} + CellPointer ConstantEmitter::stringConstant(const OperationContext& context, StringID id) { std::span text = types().bytesOf(id); diff --git a/Source/JavaScriptCore/aot/codegen/AOTConstantEmitter.h b/Source/JavaScriptCore/aot/codegen/AOTConstantEmitter.h index b088d2d0ee5bf..4b6d29740b6f1 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTConstantEmitter.h +++ b/Source/JavaScriptCore/aot/codegen/AOTConstantEmitter.h @@ -45,6 +45,7 @@ class ConstantEmitter final : public OperationEmitter { void generateEmptyArgumentList(const ValueContext& value) { defineCell(value, CellPointer(loadImmutablePointer(AOT_INSTANCE_OFFSET(realm.emptyArgumentList)).value())); } CellPointer stringConstant(const OperationContext&, StringID); + bool loadsStringConstantByStub(const OperationContext&, StringID); CellPointer keyConstant(const OperationContext&, PropertyKeyID); CellPointer bigIntConstant(const OperationContext&, StringID digits); CellPointer builtinCell(const OperationContext& context, BuiltinID id) { return builtinCell(context, BuiltinCellKey::object(id)); } diff --git a/Source/JavaScriptCore/aot/codegen/AOTConstantEmitterClosures.cpp b/Source/JavaScriptCore/aot/codegen/AOTConstantEmitterClosures.cpp index 249daf2669558..2cc0b7305db00 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTConstantEmitterClosures.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTConstantEmitterClosures.cpp @@ -133,9 +133,18 @@ CellPointer ConstantEmitter::singletonClosure(const OperationContext& context, F }; static constexpr OpcodeSet closures { Opcode::NewClosure }; m_out.noteFormOf(closures, Form::Open, StateOfKnowledge::Singleton); - CellPointer created = emitter().closureIfCreated(function); - return either(isNotEmpty(boxCell(created)), [&] { - return created; + ClosureTestEmitter& tests = emitter(); + if (m_plan.usesWideReferences()) { + CellPointer created = tests.closureIfCreated(function); + return either(isNotEmpty(boxCell(created)), [&] { + return created; + }, [&] { + return askStub(true); + }); + } + Word32 tableEntry = tests.closureTableEntry(function); + return either(Condition(m_out.notZero32(tableEntry.value())), [&] { + return tests.closureInTableEntry(tableEntry); }, [&] { return askStub(true); }); diff --git a/Source/JavaScriptCore/aot/codegen/AOTControlEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTControlEmitter.cpp index 051a66e62c983..6d78ba2ab7cb4 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTControlEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTControlEmitter.cpp @@ -32,7 +32,7 @@ void ControlEmitter::generateSwitch(const TerminatorContext& terminator) static_assert(undefinedDouble > PNaNAsBits && nullDouble > PNaNAsBits && absentDouble > PNaNAsBits); static_assert(undefinedDouble >> 52 == 0x7ff && nullDouble >> 52 == 0x7ff && absentDouble >> 52 == 0x7ff); Number number = representationOf(subject) == Representation::Double ? in(subject) : Number(in(subject).value()); - Integer truncated = emitter().toInteger(number); + Integer truncated = emitter().truncateUnchecked(number); LBasicBlock isIntegral = newBlock(); LBasicBlock isNoInteger = newBlock(); branch(equal(toNumber(truncated), number), { normalEdge(isIntegral), BlockEdge(isNoInteger, otherwise.edgeClass()) }); diff --git a/Source/JavaScriptCore/aot/codegen/AOTConversionEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTConversionEmitter.cpp index eebbc538b6345..137944b5536e5 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTConversionEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTConversionEmitter.cpp @@ -278,13 +278,13 @@ Operand ConversionEmitter::unbox(Boxed value, const Type& type, Representation t return either(isInt32(value), [&] { return unboxInt32(value); }, [&] { - return lowHalfOfIntegerToldInsideInt32(toInteger(unboxDoubleThatIsAsked(value))); + return lowHalfOfIntegerToldInsideInt32(truncateUnchecked(unboxDoubleThatIsAsked(value))); }); case Representation::Int64: return either(isInt32(value), [&] { return signExtended(unboxInt32(value)); }, [&] { - return toInteger(unboxDoubleThatIsAsked(value)); + return truncateUnchecked(unboxDoubleThatIsAsked(value)); }); case Representation::Double: if (!(type.primitives & Primitive::Integer)) @@ -384,7 +384,7 @@ Operand ConversionEmitter::inRepresentation(const OperationContext& context, Ope auto producedInteger = [&] { if (produced.is()) return signExtended(produced.as()); - return produced.is() ? produced.as() : toInteger(produced.as()); + return produced.is() ? produced.as() : truncateUnchecked(produced.as()); }; switch (wanted) { @@ -437,7 +437,18 @@ Boxed ConversionEmitter::boxNumber(Operand number) return boxNumberByStub(value); } - Word32 low = low32(toInteger(value)); +#if CPU(ARM64) + if (convertsWithJavaScriptSemantics()) { + Integer exact = exactInt32OrAllOnes(value); + return selectByBranch(greaterThanOrEqual(exact, integer(0)), [&] { + return boxInt32(low32(exact)); + }, [&] { + return boxDouble(value); + }); + } +#endif + + Word32 low = truncateToWord(value); Integer bitsOfLow = bitsOf(toNumber(low)); return selectByBranch(equal(bitsOfLow, bitsOf(value)), [&] { return boxInt32(low); @@ -450,7 +461,14 @@ Word32 ConversionEmitter::int32OrElse(Number value, LBasicBlock isNoInt32) { if (std::optional converted = convertedInt32Of(value)) return *converted; - Word32 low = low32(toInteger(value)); +#if CPU(ARM64) + if (convertsWithJavaScriptSemantics()) { + Integer exact = exactInt32OrAllOnes(value); + branchUnless(greaterThanOrEqual(exact, integer(0)), isNoInt32); + return low32(exact); + } +#endif + Word32 low = truncateToWord(value); branchUnless(equal(bitsOf(toNumber(low)), bitsOf(value)), isNoInt32); return low; } @@ -497,10 +515,11 @@ Number ConversionEmitter::toNumber(Integer value, Range range) Integer ConversionEmitter::toInteger(Number value, Range range, bool standsInLoop) { LValue number = value.value(); + static_assert(maxTrackedInteger < 1ll << 62, "a tracked integer is within int64"); if (number->opcode() != Machine::IToD || number->child(0)->type() != Machine::Int64 || range.isEmpty() || !range.isSubsetOf(Range::exact())) - return toInteger(value); + return truncateUnchecked(value); if (!standsInLoop && (range.maximum == maxExactInteger || range.minimum == -maxExactInteger)) - return toInteger(value); + return truncateUnchecked(value); Integer converted(number->child(0)); if (range.maximum == maxExactInteger) converted = select(greaterThan(converted, maxExactInteger), maxExactInteger, converted); @@ -509,12 +528,19 @@ Integer ConversionEmitter::toInteger(Number value, Range range, bool standsInLoo return converted; } -Integer ConversionEmitter::toInteger(Number value) +std::optional ConversionEmitter::findTruncation(Number value) { if (value.value()->hasDouble() && std::abs(value.value()->asDouble()) < static_cast(maxTrackedInteger)) return integer(static_cast(value.value()->asDouble())); if (std::optional converted = convertedInt32Of(value)) return signExtended(*converted); + return std::nullopt; +} + +Integer ConversionEmitter::toInteger(Number value) +{ + if (std::optional found = findTruncation(value)) + return *found; return Integer(conversionInRegisters(m_out, Machine::Int64, value, truncationToInt64(0), [](CCallHelpers& jit, const Machine::PatchParameters& params) { jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr()); #if CPU(ARM64) @@ -525,13 +551,32 @@ Integer ConversionEmitter::toInteger(Number value) } #if CPU(ARM64) -Integer ConversionEmitter::toNearestInteger(Number value) +bool ConversionEmitter::convertsWithJavaScriptSemantics() +{ + return MacroAssembler::supportsDoubleToInt32ConversionUsingJavaScriptSemantics(build().targetFeatures()); +} + +Integer ConversionEmitter::exactInt32OrAllOnes(Number value) +{ + return Integer(conversionInRegisters(m_out, Machine::Int64, value, pureFunction(PureOperation::ExactInt32, 0, true), [](CCallHelpers& jit, const Machine::PatchParameters& params) { + jit.convertDoubleToInt32UsingJavaScriptSemantics(params[1].fpr(), params[0].gpr()); + jit.m_assembler.csinv<64>(params[0].gpr(), params[0].gpr(), ARM64Registers::zr, ARM64Assembler::ConditionEQ); + })); +} +#endif + +Integer ConversionEmitter::truncateUnchecked(Number value) { +#if CPU(ARM64) + if (std::optional found = findTruncation(value)) + return *found; return Integer(conversionInRegisters(m_out, Machine::Int64, value, truncationToInt64(1), [](CCallHelpers& jit, const Machine::PatchParameters& params) { jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr()); })); -} +#else + return toInteger(value); #endif +} static constexpr Range integersNeverRounded { -(1ll << 51), 1ll << 51 }; static constexpr Range productsBehindNumbers { -(1ll << 62), 1ll << 62 }; @@ -748,7 +793,7 @@ ConversionEmitter::ExactInteger ConversionEmitter::computeIntegerBehind(LValue n return inWords ? low32(wide).value() : wide.value(); }; auto wholeNumberReadFromStorage = [&](LValue read) { - Word32 word = low32(toInteger(Number(read))); + Word32 word = truncateToWord(Number(read)); return ExactInteger { .value = ofWord(word), .holdsWhile = equal(toNumber(word), Number(read)) }; }; auto whileBoth = [&](Condition first, Condition second) { @@ -884,6 +929,19 @@ Word32 ConversionEmitter::toInt32OfBoxedNumber(Boxed value) }); } +Word32 ConversionEmitter::truncateNullableNumber(Boxed value) +{ + return either(isInt32(value), [&] { + return unboxInt32(value); + }, [&] { + return either(isNumber(value), [&] { + return toInt32(unboxDouble(value)); + }, [&] { + return word32(0); + }); + }); +} + Word32 ConversionEmitter::toInt32OfAnyNumber(Number value, BitsOfInt32 bits) { #if CPU(ARM64) @@ -893,9 +951,9 @@ Word32 ConversionEmitter::toInt32OfAnyNumber(Number value, BitsOfInt32 bits) })); } #endif - Integer truncated = toInteger(value); if (bits == BitsOfInt32::Lowest) - return low32(truncated); + return low32(toInteger(value)); + Integer truncated = truncateUnchecked(value); Condition fits; if (isX86_64()) fits = notEqual(truncated, std::numeric_limits::min()); @@ -1060,6 +1118,11 @@ Condition ConversionEmitter::isMarker(Operand operand) return greaterThanOrEqual(bitsOf(operand), static_cast(markerBits(Primitive::Undefined, operand.representation))); } +Condition ConversionEmitter::isNullOrAbsent(OptionalNumber operand) +{ + return greaterThanOrEqual(bitsOf(operand), static_cast(markerBits(Primitive::Undefined, Representation::OptionalDouble) + 1)); +} + Condition ConversionEmitter::isAbsent(Operand operand) { switch (operand.representation) { diff --git a/Source/JavaScriptCore/aot/codegen/AOTConversionEmitter.h b/Source/JavaScriptCore/aot/codegen/AOTConversionEmitter.h index da5f3fd6cb99f..3a29d836a7f85 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTConversionEmitter.h +++ b/Source/JavaScriptCore/aot/codegen/AOTConversionEmitter.h @@ -42,11 +42,16 @@ class ConversionEmitter final : public OperationEmitter { Integer toInteger(Number, Range, bool standsInLoop); Word32 int32OrElse(Number, LBasicBlock isNoInt32); Word32 int32OrElse(Integer, Range, LBasicBlock isNoInt32); -#if CPU(ARM64) - Integer toNearestInteger(Number); -#endif + Integer truncateUnchecked(Number); + Word32 truncateToWord(Number value) { return low32(truncateUnchecked(value)); } std::optional exactIntegerBehind(Number); void noteIntegerBehind(Number number, LValue integer) { m_integersNotedBehindNumbers.add(number.value(), integer); } + void noteIndexBehind(Number number, Integer index) { m_indexesBehindNumbers.add(number.value(), index.value()); } + std::optional indexBehind(Number number) const + { + LValue index = m_indexesBehindNumbers.get(number.value()); + return index ? std::optional(Integer(index)) : std::nullopt; + } void noteInt32ElementBehindNumberOrNaN(Number number, Boxed element) { if (number.value()->opcode() == Machine::Select) @@ -59,6 +64,7 @@ class ConversionEmitter final : public OperationEmitter { enum class BitsOfInt32 : uint8_t { All, Lowest }; static constexpr unsigned numberOfLowestBits = numberOfLowestBitsOfInt32; Word32 toInt32(Number, BitsOfInt32 = BitsOfInt32::All); + Word32 truncateNullableNumber(Boxed); Word32 toInt32OfBoxedNumber(Boxed); Number purified(Number); Number withoutPayload(Number); @@ -70,6 +76,7 @@ class ConversionEmitter final : public OperationEmitter { Operand fromBits(Integer, Representation); static uint64_t markerBits(uint16_t primitive, Representation); Condition isMarker(Operand); + Condition isNullOrAbsent(OptionalNumber); Condition isMarker(Operand operand, uint64_t marker) { if (std::optional narrow = operand.is() ? narrowWordBehind(operand.as()) : std::nullopt) @@ -101,6 +108,11 @@ class ConversionEmitter final : public OperationEmitter { bool comesRoundToIntegerBehind(LValue number); std::optional describeIntegerBehind(LValue number, bool mayTestWhatWasRead, const IntegersBehindNumbers& operands); ExactInteger computeIntegerBehind(LValue number, const IntegerBehindNumber&, bool inWords, const ExactIntegers& operands); + std::optional findTruncation(Number); +#if CPU(ARM64) + bool convertsWithJavaScriptSemantics(); + Integer exactInt32OrAllOnes(Number); +#endif Word32 toInt32OfAnyNumber(Number, BitsOfInt32); Operand convertMarkers(Operand source, const Type&, Representation to, std::optional unmarked = std::nullopt); Integer integerPartOf(Operand, const Type&); @@ -116,6 +128,7 @@ class ConversionEmitter final : public OperationEmitter { bool m_boxesNumbersByStub { false }; Place m_placeThatBoxesByStub { Place::Anywhere }; UncheckedKeyHashMap m_integersNotedBehindNumbers; + UncheckedKeyHashMap m_indexesBehindNumbers; UncheckedKeyHashMap m_int32ElementsBehindNumbersOrNaN; UncheckedKeyHashSet m_numbersOfInt32Elements; std::array m_integersBehindNumbers; diff --git a/Source/JavaScriptCore/aot/codegen/AOTElementAccessEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTElementAccessEmitter.cpp index 2df31ec4e0cf2..45347eae37414 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTElementAccessEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTElementAccessEmitter.cpp @@ -206,6 +206,22 @@ static_assert([] { return true; }()); +static constexpr uint32_t bitsOutsideNumericShapes = ~static_cast(DoubleShape - Int32Shape); + +Condition ElementAccessEmitter::hasWritableNumericStorage(CellPointer array) +{ + return hasNoBits(sub(indexingMode(array, IndexingShapeAndWritabilityMask), word32(IsArray | Int32Shape)), bitsOutsideNumericShapes); +} + +static_assert([] { + for (uint32_t byte = 0; byte <= 0xff; ++byte) { + bool passesTest = !((byte - (IsArray | Int32Shape)) & bitsOutsideNumericShapes); + if (passesTest != (byte == (IsArray | Int32Shape) || byte == (IsArray | DoubleShape))) + return false; + } + return true; +}()); + Condition ElementAccessEmitter::holdsBoxedElements(CellPointer array) { return hasNoBits(sub(loadByte(array, m_heaps.JSArray_indexingTypeAndMisc), word32(Int32Shape)), bitsOfShapeOutsideBoxedShapes); @@ -299,7 +315,7 @@ Operand ElementAccessEmitter::convertRawElement(const OperationContext& context, if (form.element == Representation::Double) { stopUnlessDoubleIsOf(context, rawElement, result); if ((holdsInteger(representation) || representation == Representation::Int32) && types().at(types().intersectionOf(result, TypeTable::number)).isInteger()) { - Integer integer = emitter().toInteger(rawElement.as()); + Integer integer = emitter().truncateUnchecked(rawElement.as()); if (representation == Representation::Int32) return low32(integer); return Operand(representation, integer.value()); @@ -329,7 +345,7 @@ void ElementAccessEmitter::stopUnlessDoubleIsOf(const OperationContext& context, if (Type claimed = types().at(numbers); claimed.isInteger() && claimed.integers != Range::int32()) { ConversionEmitter& conversions = emitter(); Number number(element.value); - Integer truncated = conversions.toInteger(number); + Integer truncated = conversions.truncateUnchecked(number); emitter().checkOrCrash(context, equal(conversions.bitsOf(toNumber(truncated)), conversions.bitsOf(number)), ProofSource::Contents); emitter().checkOrCrash(context, emitter().isInRange(truncated, claimed.integers), ProofSource::Contents); return; @@ -407,15 +423,42 @@ std::optional ElementAccessEmitter::indexBesideWholeNumber(ValueID posi if (!(std::abs(addend) <= INT32_MAX) || addend != std::trunc(addend)) return std::nullopt; - Integer truncated = emitter().toInteger(Number(whole)); + Integer truncated = emitter().truncateUnchecked(Number(whole)); if (isNotAnIndex && !isX86_64()) branchUnless(equal(toNumber(truncated), Number(whole)), isNotAnIndex); int64_t step = static_cast(addend); return add(truncated, integer(sum.opcode == Opcode::Add ? step : -step)); } +std::optional ElementAccessEmitter::wordOfPosition(ValueID position) +{ + if (std::optional word = generator().findBoxed(position)) + return word; + const Operation& unboxed = operationAt(definitionOf(position)); + if (unboxed.opcode != Opcode::Unbox || unboxed.representation != Representation::Double || unboxed.block() != generator().blockBeingGenerated() || generator().keepsValuesInHeapFrame()) + return std::nullopt; + if (representationOf(unboxed.valueOperand()) != Representation::JSValue || !(typeOf(position).primitives & Primitive::Integer) || generator().isCarriedBoxed(unboxed.valueOperand())) + return std::nullopt; + return in(unboxed.valueOperand()); +} + +Integer ElementAccessEmitter::indexOfWordOrElse(Boxed word, bool keyIsIntegral, LBasicBlock isNotAnIndex, NaNIndexMode nanMode) +{ + return either(isInt32(word), [&] { + return signExtended(unboxInt32(word)); + }, [&] { + Number number = unboxDouble(word); + Integer index = emitter().truncateUnchecked(number); + if (isNotAnIndex && (!keyIsIntegral || !isX86_64())) + branchUnless(isX86_64() && nanMode == NaNIndexMode::ProduceOutOfBoundsIndex ? equalOrUnordered(toNumber(index), number) : equal(toNumber(index), number), isNotAnIndex); + return index; + }); +} + Integer ElementAccessEmitter::indexOrElse(ValueID position, bool keyIsIntegral, LBasicBlock isNotAnIndex, NaNIndexMode nanMode) { + if (std::optional word = wordOfPosition(position)) + return indexOfWordOrElse(*word, keyIsIntegral, isNotAnIndex, nanMode); Operand operand = get(position); switch (operand.representation) { case Representation::Int32: @@ -427,9 +470,14 @@ Integer ElementAccessEmitter::indexOrElse(ValueID position, bool keyIsIntegral, Number number(operand.value); if (std::optional exact = emitter().exactIntegerBehind(number)) return *exact; + if (std::optional index = emitter().indexBehind(number)) { + if (isNotAnIndex && nanMode == NaNIndexMode::TakeSlowPath) + branchUnless(greaterThanOrEqual(*index, integer(0)), isNotAnIndex); + return *index; + } if (std::optional beside = keyIsIntegral ? indexBesideWholeNumber(position, number, isNotAnIndex) : std::nullopt) return *beside; - Integer index = emitter().toInteger(number); + Integer index = emitter().truncateUnchecked(number); if (isNotAnIndex && (!keyIsIntegral || !isX86_64())) { bool truncatesNaNOutOfBounds = isX86_64() && nanMode == NaNIndexMode::ProduceOutOfBoundsIndex; branchUnless(truncatesNaNOutOfBounds ? equalOrUnordered(toNumber(index), number) : equal(toNumber(index), number), isNotAnIndex); @@ -452,8 +500,9 @@ Integer ElementAccessEmitter::indexOrElse(ValueID position, bool keyIsIntegral, Integer ElementAccessEmitter::indexOfAccessOrElse(const ValueContext& access, LBasicBlock isNotAnIndex, NaNIndexMode nanMode) { ValueID position = access->positionOperand(); - if (Operand key = get(position); access->keyIsIntegral() && key.is() && checksBounds()) { - Number number = key.as(); + if (access->keyIsIntegral() && checksBounds() && representationOf(position) == Representation::Double) { + std::optional word = generator().findBoxed(position); + Number number = word ? emitter().unbox(*word, typeOf(position), Representation::Double).as() : in(position); emitter().requireOrCrash(access, equalOrUnordered(trunc(number), number), std::nullopt, ProofSource::OtherOperation); } return indexOrElse(position, access->keyIsIntegral(), isNotAnIndex, nanMode); @@ -461,6 +510,8 @@ Integer ElementAccessEmitter::indexOfAccessOrElse(const ValueContext& access, LB Boxed ElementAccessEmitter::boxedPosition(ValueID position) { + if (std::optional word = wordOfPosition(position)) + return *word; Operand operand = get(position); switch (operand.representation) { case Representation::Int32: @@ -598,7 +649,7 @@ void ElementAccessEmitter::holdKindOfList(const ValueContext& access, StorageIsW return; CellPointer list = askedAnew(in(access->arrayOperand())); bool admitsInt32Storage = Operation::opcodesWithAdmitsInt32Storage.contains(access.opcode()) && access->admitsInt32Storage(); - Condition holds = admitsInt32Storage ? hasDoubleOrInt32Storage(list) : hasArrayStorage(list, form->kind, isWritten == StorageIsWritten::Yes); + Condition holds = admitsInt32Storage ? form->kind == ArrayStorage::Contiguous ? holdsBoxedElements(list) : hasDoubleOrInt32Storage(list) : hasArrayStorage(list, form->kind, isWritten == StorageIsWritten::Yes); emitter().requireOrCrash(access, holds, std::nullopt, ProofSource::RememberedFact); } @@ -626,12 +677,42 @@ ArrayElements ElementAccessEmitter::elementsOf(const ValueContext& access, CellP return { array, *storage }; } +bool ElementAccessEmitter::addsConstantToIndex(ValueID position, LValue sum) +{ + static constexpr int32_t greatestConstant = std::numeric_limits::max(); + if (sum->opcode() != Machine::Add || !sum->child(1)->hasInt32()) + return false; + LValue index = sum->child(0); + int32_t constant = sum->child(1)->asInt32(); + if (index->hasInt32() || !Machine::clearsUpperHalf(index) || constant < -greatestConstant || constant > greatestConstant) + return false; + for (ValueID named = position; index->opcode() != Machine::Phi; named = operationAt(named).valueOperand()) { + if (generator().code().useCountOf(named) != 1) + return false; + if (operationAt(named).opcode != Opcode::Narrowed) + break; + } + if (constant < 0) + return true; + const Operation& definition = operationAt(definitionOf(position)); + for (uint32_t operandIndex = 0; operandIndex < definition.numberOfOperands(); ++operandIndex) { + ValueID operand = definition.operand(operandIndex); + if (operand && representationOf(operand) == Representation::Int32 && generator().isGenerated(operand) && get(operand).value == index && typeOf(operand).integers.minimum >= 0) + return true; + } + return false; +} + Integer ElementAccessEmitter::indexInsideLength(ValueID position) { Operand operand = get(position); - if (operand.representation == Representation::Int32) - return Integer(m_out.extendWordThatIsNotNegative(operand.as().value())); - return indexOrElse(position, true, nullptr); + if (operand.representation != Representation::Int32) + return indexOrElse(position, true, nullptr); + LValue word = operand.as().value(); + if (!addsConstantToIndex(position, word)) + return Integer(m_out.extendWordThatIsNotNegative(word)); + build().statistics().count(StatisticID::ConstantsFoldedIntoElementAddresses); + return Integer(m_out.add(m_out.extendWordThatIsNotNegative(word->child(0)), m_out.constInt64(static_cast(word->child(1)->asInt32())))); } static_assert(MAX_STORAGE_VECTOR_LENGTH <= static_cast(std::numeric_limits::max())); @@ -697,7 +778,7 @@ Operand ElementAccessEmitter::whatReadGivesWhereNothingIsFound(const ValueContex return convertElementThatHasBeenAsked(read, arrayStorageFormOf(ArrayStorage::Double), number(PNaN), representation, TypeTable::number); } -Operand ElementAccessEmitter::readElementInsideLength(const ValueContext& value, const ArrayStorageForm& form, const ArrayElements& elements, Integer index, LBasicBlock isHole) +Operand ElementAccessEmitter::readElementInsideLength(const ValueContext& value, const ArrayStorageForm& form, const ArrayElements& elements, Integer index, LBasicBlock isHole, std::optional representation) { RELEASE_ASSERT(!readsHoleAsNaN(value) || form.element == Representation::Double); Operand rawElement = loadRawElement(form, elements, index); @@ -707,7 +788,7 @@ Operand ElementAccessEmitter::readElementInsideLength(const ValueContext& value, m_out.appendTo(holdsValue); } else if (!readsHoleAsNaN(value)) checkElementPresenceProof(value, logicalNot(this->isHole(form, rawElement))); - return convertRawElement(value, form, rawElement, value.representation(), typeIDOf(value.id())); + return convertRawElement(value, form, rawElement, representation.value_or(value.representation()), typeIDOf(value.id())); } const PlacedSlot* ElementAccessEmitter::extraPropertiesWordOfReadByPosition(const ValueContext& read) @@ -782,14 +863,11 @@ void ElementAccessEmitter::generateGetArrayElement(const ValueContext& value) define(value, emitter().zeroOf(value.representation())); return; } + generator().forgetBoxed(value.id()); if (liesInsideLength(generator(), value)) holdPositionInsideList(value); if (value.isBranchCondition()) { - static_assert(!static_cast(JSValue::ValueEmpty)); - const ArrayStorageForm& form = arrayStorageFormOf(*kindOfListFoundInFrontOfTurns(value, value->arrayOperand())); - Operand raw = loadRawElement(form, elementsOf(value, in(value->arrayOperand())), indexInsideLength(value->positionOperand())); - branch(form.kind == ArrayStorage::Double ? isNeitherZeroNorNaN(raw.as()) : isNotZero(low32(raw.as())), value.branchTargets()); - build().statistics().count(StatisticID::UnrolledLoopRawTruthinessTests); + branchOnTruthOfNumberInList(value); return; } if (GuessedAccessEmitter::testsForItself(value)) { @@ -806,7 +884,7 @@ void ElementAccessEmitter::generateGetArrayElement(const ValueContext& value) form = findArrayStorageForm(*found); if (!form) readWithAnyStorage(value); - else if (value->admitsInt32Storage()) + else if (value->admitsInt32Storage() && form->kind == ArrayStorage::Double) readFromDoubleOrInt32Storage(value); else if (liesInsideLength(generator(), value)) readInsideLength(value, *form); @@ -842,6 +920,58 @@ static bool isAlwaysArrayIndex(const Type& position) return position.isInteger() && position.integers.isSubsetOf({ 0, MAX_ARRAY_INDEX }); } +bool ElementAccessEmitter::asksTruthOfNumberInList(ValueID read) +{ + const Operation& operation = generator().code().at(read); + if (operation.opcode != Opcode::GetArrayElement || !isNumericStorage(operation.arrayStorage()) || operation.level() != KnowledgeLevel::Proven || operation.asksCellWhereNothingIsFound() || operation.readsHoleAsNaN()) + return false; + return (operation.isInBounds() || isAlwaysArrayIndex(typeOf(operation.positionOperand()))) && typeOf(read).hasOnlyPrimitives(Primitive::Number | Primitive::Undefined | Primitive::Absent); +} + +void ElementAccessEmitter::branchOnTruthOfNumberInList(const ValueContext& value) +{ + static_assert(!static_cast(JSValue::ValueEmpty)); + BranchTargets targets = value.branchTargets(); + CellPointer array = in(value->arrayOperand()); + std::optional found = kindOfListFoundInFrontOfTurns(value, value->arrayOperand()); + if (found) + build().statistics().count(StatisticID::UnrolledLoopRawTruthinessTests); + else + holdKindOfList(value); + ArrayElements elements = elementsOf(value, array); + bool isInsideLength = liesInsideLength(generator(), value); + std::optional indexMadeInFront; + if (!isInsideLength) { + LengthTest test = testLength(value, indexOfAccessOrElse(value, nullptr, NaNIndexMode::ProduceOutOfBoundsIndex), elements); + LBasicBlock isInBounds = newBlock(); + branch(test.isInside, { normalEdge(isInBounds), targets.notTaken }); + m_out.appendTo(isInBounds); + indexMadeInFront = test.indexInside.value()->opcode() == Machine::ZExt32 ? Integer(m_out.extendWordThatIsNotNegative(test.indexInside.value()->child(0))) : test.indexInside; + } + auto branchOnTruth = [&](const ArrayStorageForm& form) { + Operand raw = loadRawElement(form, elements, indexMadeInFront ? *indexMadeInFront : indexInsideLength(value->positionOperand())); + branch(form.kind == ArrayStorage::Double ? isNeitherZeroNorNaN(raw.as()) : isNotZero(low32(raw.as())), targets); + }; + if (found || !value->admitsInt32Storage()) { + branchOnTruth(arrayStorageFormOf(found.value_or(value->arrayStorage()))); + return; + } + if (value->holdsDoublesOperand() && generator().isGenerated(value->holdsDoublesOperand())) { + Condition holdsDoubles = in(value->holdsDoublesOperand()); + if (checksBounds()) + emitter().requireOrCrash(value, Condition(m_out.equal(holdsDoubles.value(), hasArrayStorage(askedAnew(array), ArrayStorage::Double).value())), std::nullopt, ProofSource::RememberedFact); + LBasicBlock isOfDoubles = newBlock(); + LBasicBlock isOfInt32s = newBlock(); + m_out.branch(holdsDoubles.value(), normalEdge(isOfDoubles), normalEdge(isOfInt32s)); + m_out.appendTo(isOfDoubles); + branchOnTruth(arrayStorageFormOf(ArrayStorage::Double)); + m_out.appendTo(isOfInt32s); + branchOnTruth(arrayStorageFormOf(ArrayStorage::Int32)); + return; + } + forDoubleAndInt32Storage(*this, m_out, array, branchOnTruth); +} + bool ElementAccessEmitter::hasOneArmInLine(const ValueContext& read) { return holdsUndefined(read.representation()); @@ -1033,10 +1163,11 @@ Operand ElementAccessEmitter::numberOfInt32Element(const ValueContext& value, Bo return convertElementThatHasBeenAsked(value, arrayStorageFormOf(ArrayStorage::Double), number, value.representation(), typeIDOf(value.id())); } -Operand ElementAccessEmitter::readNumberInsideLength(const ValueContext& value, const ArrayStorageForm& form, const ArrayElements& elements, Integer index, LBasicBlock isHole) +Operand ElementAccessEmitter::readNumberInsideLength(const ValueContext& value, const ArrayStorageForm& form, const ArrayElements& elements, Integer index, LBasicBlock isHole, std::optional representation) { + RELEASE_ASSERT(!representation || !readsHoleAsNaN(value)); if (!readsHoleAsNaN(value) || form.element == Representation::Double) - return readElementInsideLength(value, form, elements, index, isHole); + return readElementInsideLength(value, form, elements, index, isHole, representation); RELEASE_ASSERT(form.element == Representation::Int32); Boxed rawElement = loadRawElement(form, elements, index).as(); @@ -1074,12 +1205,21 @@ void ElementAccessEmitter::readFromDoubleOrInt32Storage(const ValueContext& valu { CellPointer array = in(value->arrayOperand()); ValueID position = value->positionOperand(); - Representation representation = value.representation(); RELEASE_ASSERT(value->arrayStorage() == ArrayStorage::Double); + bool isOnlyBoxed = !readsHoleAsNaN(value) && generator().isOnlyBoxed(value.id()); + Representation representation = isOnlyBoxed ? Representation::JSValue : value.representation(); + std::optional representationOfElement = isOnlyBoxed ? std::optional { representation } : std::nullopt; + auto defineElement = [&](Operand element) { + if (isOnlyBoxed) { + build().statistics().count(StatisticID::NumberElementsReadBoxed); + generator().defineBoxed(value, element.as()); + } else + define(value, element); + }; bool isInsideLength = liesInsideLength(generator(), value); if (std::optional found = isInsideLength && findsElement(value) ? kindOfListFoundInFrontOfTurns(value, value->arrayOperand()) : std::nullopt) { - define(value, readNumberInsideLength(value, arrayStorageFormOf(*found), elementsOf(value, array), indexInsideLength(position), nullptr)); + defineElement(readNumberInsideLength(value, arrayStorageFormOf(*found), elementsOf(value, array), indexInsideLength(position), nullptr, representationOfElement)); return; } @@ -1088,15 +1228,21 @@ void ElementAccessEmitter::readFromDoubleOrInt32Storage(const ValueContext& valu OperandJoin join(m_out, representation); ArrayElements elements = elementsOf(value, array); - Integer index = isInsideLength ? indexInsideLength(position) : indexOfAccessOrElse(value, isNotAnIndex, NaNIndexMode::ProduceOutOfBoundsIndex); - Integer indexInside = index; + bool isExtendedInArm = isInsideLength && representationOf(position) == Representation::Int32; + std::optional indexMadeInFront; + if (!isExtendedInArm) + indexMadeInFront = isInsideLength ? indexInsideLength(position) : indexOfAccessOrElse(value, isNotAnIndex, NaNIndexMode::ProduceOutOfBoundsIndex); + std::optional indexInside = indexMadeInFront; + auto indexInArm = [&] { + return indexInside ? *indexInside : indexInsideLength(position); + }; if (!isInsideLength) { - LengthTest test = testLength(value, index, elements); + LengthTest test = testLength(value, *indexMadeInFront, elements); branchUnless(test.isInside, isAbsent); indexInside = test.indexInside; } if (std::optional found = kindOfListFoundInFrontOfTurns(value, value->arrayOperand())) - join.add(readNumberInsideLength(value, arrayStorageFormOf(*found), elements, indexInside, isAbsent)); + join.add(readNumberInsideLength(value, arrayStorageFormOf(*found), elements, indexInArm(), isAbsent, representationOfElement)); else if (LValue holdsDoubles = value->holdsDoublesOperand() && generator().isGenerated(value->holdsDoublesOperand()) ? in(value->holdsDoublesOperand()).value() : generator().holdsDoublesAsReadInFrontOfTurnsLaidOut(value->arrayOperand())) { if (checksBounds()) { emitter().requireOrCrash(value, Condition(m_out.equal(holdsDoubles, hasArrayStorage(askedAnew(array), ArrayStorage::Double).value())), std::nullopt, ProofSource::RememberedFact); @@ -1106,26 +1252,26 @@ void ElementAccessEmitter::readFromDoubleOrInt32Storage(const ValueContext& valu LBasicBlock isOfInt32s = newBlock(); m_out.branch(holdsDoubles, normalEdge(isOfDoubles), normalEdge(isOfInt32s)); m_out.appendTo(isOfDoubles); - join.add(readNumberInsideLength(value, arrayStorageFormOf(ArrayStorage::Double), elements, indexInside, isAbsent)); + join.add(readNumberInsideLength(value, arrayStorageFormOf(ArrayStorage::Double), elements, indexInArm(), isAbsent, representationOfElement)); m_out.appendTo(isOfInt32s); - join.add(readNumberInsideLength(value, arrayStorageFormOf(ArrayStorage::Int32), elements, indexInside, isAbsent)); + join.add(readNumberInsideLength(value, arrayStorageFormOf(ArrayStorage::Int32), elements, indexInArm(), isAbsent, representationOfElement)); } else { forDoubleAndInt32Storage(*this, m_out, array, [&](const ArrayStorageForm& form) { - join.add(readNumberInsideLength(value, form, elements, indexInside, isAbsent)); + join.add(readNumberInsideLength(value, form, elements, indexInArm(), isAbsent, representationOfElement)); }); } if (isAbsent) { m_out.appendTo(isAbsent); if (isNotAnIndex) - branchUnless(belowOrEqual(index, integer(MAX_ARRAY_INDEX)), isNotAnIndex); + branchUnless(belowOrEqual(*indexMadeInFront, integer(MAX_ARRAY_INDEX)), isNotAnIndex); join.add(whatReadGivesWhereNothingIsFound(value, representation)); } if (isNotAnIndex) { m_out.appendTo(isNotAnIndex); join.add(emitter().unbox(readOutOfLine(value, array), typeOf(value.id()), representation)); } - define(value, join.finish()); + defineElement(join.finish()); } void ElementAccessEmitter::branchOnPresentElementInDoubleOrInt32Storage(const ValueContext& test, BranchTargets targets) @@ -1138,6 +1284,19 @@ void ElementAccessEmitter::branchOnPresentElementInDoubleOrInt32Storage(const Va branchOnElementPresence(test, &arrayStorageFormOf(*found), elements, index, targets); return; } + if (test->holdsDoublesOperand() && generator().isGenerated(test->holdsDoublesOperand())) { + Condition holdsDoubles = in(test->holdsDoublesOperand()); + if (checksBounds()) + emitter().requireOrCrash(test, Condition(m_out.equal(holdsDoubles.value(), hasArrayStorage(askedAnew(array), ArrayStorage::Double).value())), std::nullopt, ProofSource::RememberedFact); + LBasicBlock isOfDoubles = newBlock(); + LBasicBlock isOfInt32s = newBlock(); + m_out.branch(holdsDoubles.value(), normalEdge(isOfDoubles), normalEdge(isOfInt32s)); + m_out.appendTo(isOfDoubles); + branchOnElementPresence(test, &arrayStorageFormOf(ArrayStorage::Double), elements, index, targets); + m_out.appendTo(isOfInt32s); + branchOnElementPresence(test, &arrayStorageFormOf(ArrayStorage::Int32), elements, index, targets); + return; + } forDoubleAndInt32Storage(*this, m_out, array, [&](const ArrayStorageForm& form) { branchOnElementPresence(test, &form, elements, index, targets); }); @@ -1321,6 +1480,8 @@ void ElementAccessEmitter::generateGetStringCodeUnit(const ValueContext& value) StringEmitter& strings = emitter(); CharacterView view = strings.viewOf(value, value->stringOperand()); Word32 position = strings.word32Of(value->positionOperand()); + if (checksBounds() && view.possibleRope) + view.length = strings.lengthOf(view.owner); if (checksBounds()) stopUnlessInside(value, below(position, view.length)); ValueID wholeNumber = value->positionOperand(); diff --git a/Source/JavaScriptCore/aot/codegen/AOTElementAccessEmitter.h b/Source/JavaScriptCore/aot/codegen/AOTElementAccessEmitter.h index 666cfe7f2dd15..65648ecd50001 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTElementAccessEmitter.h +++ b/Source/JavaScriptCore/aot/codegen/AOTElementAccessEmitter.h @@ -60,6 +60,8 @@ class ElementAccessEmitter final : public OperationEmitter { void generateIsElementInStorage(const ValueContext&); void generateGetElementInStorageOrAbsent(const ValueContext&); bool asksTruthOfRawElement(ValueID read); + bool asksTruthOfNumberInList(ValueID read); + void branchOnTruthOfNumberInList(const ValueContext&); void generateGetStringLength(const ValueContext&); void generateGetStringCodeUnit(const ValueContext&); @@ -87,6 +89,7 @@ class ElementAccessEmitter final : public OperationEmitter { Condition holdsBoxedElements(CellPointer array); Condition holdsWritableBoxedElements(CellPointer array); Condition takesNumbersByKindOfStorage(CellPointer array, bool admitsInt32Storage); + Condition hasWritableNumericStorage(CellPointer array); Condition hasWritableShape(Word32 modeWithWritability, const ArrayStorageForm& form) { return equal(modeWithWritability, word32(form.indexingType)); } CellPointer askedAnew(CellPointer cell) { return CellPointer(m_out.opaque(cell.value())); } @@ -121,6 +124,7 @@ class ElementAccessEmitter final : public OperationEmitter { ArrayStorageFormList candidateFormsFor(ValueID stored, ArrayStorage expected); ArrayStorageFormList formExpectedToHold(ValueID list, ValueID stored); bool goesByKindOfListOfNumbers(ValueID list, ValueID stored); + const ArrayStorageForm& expectedFormOf(ValueID array); ArrayStorageFormList formsOfListOfNumbersFor(ValueID stored, bool hasArmForEveryKind); ArrayStorageFormList formsThatMayHold(ValueID list, ValueID stored); void storeInProvenList(const ValueContext&, const ArrayStorageFormList&); @@ -132,8 +136,11 @@ class ElementAccessEmitter final : public OperationEmitter { enum class NaNIndexMode : bool { TakeSlowPath, ProduceOutOfBoundsIndex }; Integer indexOrElse(ValueID position, bool keyIsIntegral, LBasicBlock isNotAnIndex, NaNIndexMode = NaNIndexMode::TakeSlowPath); + std::optional wordOfPosition(ValueID position); + Integer indexOfWordOrElse(Boxed, bool keyIsIntegral, LBasicBlock isNotAnIndex, NaNIndexMode); Integer indexOfAccessOrElse(const ValueContext& access, LBasicBlock isNotAnIndex, NaNIndexMode = NaNIndexMode::TakeSlowPath); Integer indexInsideLength(ValueID position); + bool addsConstantToIndex(ValueID position, LValue sum); Integer indexFoundBelowLength(Integer index); Condition isBelowLengthOfView(Integer index, Integer length, TypedArrayType); std::optional indexBesideWholeNumber(ValueID position, Number, LBasicBlock isNotAnIndex); @@ -145,7 +152,7 @@ class ElementAccessEmitter final : public OperationEmitter { const PlacedSlot* extraPropertiesWordOfReadByPosition(const ValueContext& read); Boxed readElementOfExtraProperties(CellPointer object, const PlacedSlot& extraPropertiesWord, Integer index, LBasicBlock notFound); void overwriteElementOfExtraProperties(const ValueContext& store, CellPointer object, const PlacedSlot& extraPropertiesWord, Integer index, Boxed stored, LBasicBlock notFound); - Operand readElementInsideLength(const ValueContext&, const ArrayStorageForm&, const ArrayElements&, Integer index, LBasicBlock isHole); + Operand readElementInsideLength(const ValueContext&, const ArrayStorageForm&, const ArrayElements&, Integer index, LBasicBlock isHole, std::optional = std::nullopt); void branchOnPresentElement(const ValueContext& test, const ArrayStorageForm&, BranchTargets); void branchOnPresentElementInAnyStorage(const ValueContext& test, BranchTargets); @@ -172,7 +179,7 @@ class ElementAccessEmitter final : public OperationEmitter { void readWithOneArmInLine(const ValueContext&, const ArrayStorageForm* knownForm); void readFromDoubleOrInt32Storage(const ValueContext&); Operand numberOfInt32Element(const ValueContext&, Boxed element); - Operand readNumberInsideLength(const ValueContext&, const ArrayStorageForm&, const ArrayElements&, Integer index, LBasicBlock isHole); + Operand readNumberInsideLength(const ValueContext&, const ArrayStorageForm&, const ArrayElements&, Integer index, LBasicBlock isHole, std::optional = std::nullopt); Boxed readOutOfLine(const ValueContext&, CellPointer array); bool findsElement(const ValueContext& read); Boxed readBoxedElement(const ValueContext&, const ArrayElements&, Integer index, LBasicBlock isAbsent); diff --git a/Source/JavaScriptCore/aot/codegen/AOTElementAccessEmitterStores.cpp b/Source/JavaScriptCore/aot/codegen/AOTElementAccessEmitterStores.cpp index 7703cf7a6fc58..bbd46f3a61227 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTElementAccessEmitterStores.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTElementAccessEmitterStores.cpp @@ -116,6 +116,14 @@ bool ElementAccessEmitter::goesByKindOfListOfNumbers(ValueID list, ValueID store return type.primitives & Primitive::Number && type.hasOnlyPrimitives(Primitive::Number | Primitive::UndefinedOrNull) && isDeclaredToHoldNumbersAlone(m_program, types(), typeIDOf(list)); } +const ArrayStorageForm& ElementAccessEmitter::expectedFormOf(ValueID array) +{ + ArrayStorage declared = arrayStorageDeclaredBy(m_program, types(), typeIDOf(array)).predicted(); + if (hasArrayStorageForm(declared)) + return arrayStorageFormOf(declared); + return arrayStorageFormOf(isDeclaredToHoldNumbersAlone(m_program, types(), typeIDOf(array)) ? ArrayStorage::Int32 : ArrayStorage::Contiguous); +} + ArrayStorageFormList ElementAccessEmitter::formsThatMayHold(ValueID list, ValueID stored) { return goesByKindOfListOfNumbers(list, stored) ? formsOfListOfNumbersFor(stored, true) : candidateFormsFor(stored, ArrayStorage::Other); @@ -123,7 +131,8 @@ ArrayStorageFormList ElementAccessEmitter::formsThatMayHold(ValueID list, ValueI ArrayStorageFormList ElementAccessEmitter::formExpectedToHold(ValueID list, ValueID stored) { - if (goesByKindOfListOfNumbers(list, stored)) + bool isTestedForNumber = representationOf(stored) == Representation::JSValue && typeOf(stored).primitives & Primitive::Number && isDeclaredToHoldNumbersAlone(m_program, types(), typeIDOf(list)); + if (isTestedForNumber || goesByKindOfListOfNumbers(list, stored)) return formsOfListOfNumbersFor(stored, false); ArrayStorageFormList forms = candidateFormsFor(stored, storageExpectedToHold(representationOf(stored))); if (forms.size() > 1) @@ -517,7 +526,7 @@ Condition ElementAccessEmitter::tryStoreElement(const ValueContext& store, CellP Join join(m_out); Word32 mode = indexingMode(array, IndexingShapeAndWritabilityMask); ArrayElements elements = elementsOf(array); - forEachFormByShape(mode, &ElementAccessEmitter::hasWritableShape, candidateFormsFor(stored, storageExpectedToHold(representationOf(stored))), otherwise, [&](const ArrayStorageForm& form) { + forEachFormByShape(mode, &ElementAccessEmitter::hasWritableShape, candidateFormsFor(stored, typeOf(stored).isInt32() ? ArrayStorage::Int32 : storageExpectedToHold(representationOf(stored))), otherwise, [&](const ArrayStorageForm& form) { Operand rawElement = *rawElementToStore(form, stored, otherwise); if (leavesStorageAsItIs) { LengthTest test = testLength(index, elements); diff --git a/Source/JavaScriptCore/aot/codegen/AOTElementAccessEmitterTypedArrays.cpp b/Source/JavaScriptCore/aot/codegen/AOTElementAccessEmitterTypedArrays.cpp index 15bc0db571bc8..54465107c4440 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTElementAccessEmitterTypedArrays.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTElementAccessEmitterTypedArrays.cpp @@ -100,7 +100,7 @@ void ElementAccessEmitter::askViewWherePositionIsNotBelowNamedLength(LBasicBlock bool ElementAccessEmitter::becomesIndexWithoutTest(const ValueContext& access) { - return get(access->positionOperand()).representation != Representation::Double || (access->keyIsIntegral() && isX86_64()); + return representationOf(access->positionOperand()) != Representation::Double || (access->keyIsIntegral() && isX86_64()); } void ElementAccessEmitter::holdPositionInsideView(const ValueContext& access, Integer index, const TypedArrayElements& elements) @@ -302,11 +302,11 @@ void ElementAccessEmitter::generateGetTypedArrayElement(const ValueContext& valu LBasicBlock isNotBelowNamedLength = named ? newBlock() : nullptr; TypedArrayElements elements = named ? *named : typedArrayElementsOf(value->viewOperand(), otherwise); branchUnless(isBelowLengthOfView(index, elements.length, type), named ? isNotBelowNamedLength : outside); - if (nothingIsNaN) + if (nothingIsNaN && isFloat(type)) join.add(loadBytesOfTypedArrayElement(elements, indexFoundBelowLength(index), type)); else { Operand element = loadElement(elements, indexFoundBelowLength(index)); - join.add(TypedArraySupport(generator()).inRepresentation(value, representation, element, elementType)); + join.add(TypedArraySupport(generator()).inRepresentation(value, nothingIsNaN ? Representation::Double : representation, element, elementType)); } if (named) @@ -345,7 +345,8 @@ void ElementAccessEmitter::generatePutTypedArrayElement(const ValueContext& valu auto storeElement = [&](const TypedArrayElements& elements, Integer index) { Operand element = elementToStore(); if (!isOfSeveralKinds) { - storeTypedArrayElement(elements, index, type, element); + bool isByte = isClamped(type) && typeOf(storedValue).isInteger() && typeOf(storedValue).integers.isSubsetOf(Range { 0, UINT8_MAX }) && !element.is(); + storeTypedArrayElement(elements, index, isByte ? TypeUint8 : type, element); return; } if (std::optional found = kindOfViewFoundInFrontOfTurns(value, value->viewOperand())) { diff --git a/Source/JavaScriptCore/aot/codegen/AOTElementEmitterLiterals.cpp b/Source/JavaScriptCore/aot/codegen/AOTElementEmitterLiterals.cpp index ada510d04fa39..2c1e4ee7c2ddc 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTElementEmitterLiterals.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTElementEmitterLiterals.cpp @@ -53,6 +53,10 @@ void GuardEmitter::generateHasArrayStorage(const ValueContext& value) } if (value->admitsInt32Storage()) { CellPointer list = in(value->arrayOperand()); + if (value->isWritable()) { + defineTest(value, kind == ArrayStorage::Contiguous ? emitter().holdsWritableBoxedElements(list) : emitter().hasWritableNumericStorage(list)); + return; + } defineTest(value, kind == ArrayStorage::Contiguous ? emitter().holdsBoxedElements(list) : emitter().hasDoubleOrInt32Storage(list)); return; } @@ -69,7 +73,7 @@ void GuardEmitter::generateIsElementPresent(const ValueContext& value) const ArrayStorageForm* form = findArrayStorageForm(value->arrayStorage()); emitter().holdKindOfList(value); defineTestByBranching(value, [&](BranchTargets targets) { - if (form && value->admitsInt32Storage()) + if (form && form->kind == ArrayStorage::Double && value->admitsInt32Storage()) emitter().branchOnPresentElementInDoubleOrInt32Storage(value, targets); else if (form) emitter().branchOnPresentElement(value, *form, targets); diff --git a/Source/JavaScriptCore/aot/codegen/AOTGuessedAccessEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTGuessedAccessEmitter.cpp index b30a57e44f633..cf9c966f809d8 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTGuessedAccessEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTGuessedAccessEmitter.cpp @@ -519,14 +519,18 @@ void GuessedAccessEmitter::readNumberOfProvenList(const ValueContext& value) OperandJoin join(m_out, representation); bool isWhole = types().at(types().withoutPrimitives(typeIDOf(value.id()), Primitive::Undefined)).isNumberWithoutFraction(); ArrayStorageFormList forms { &arrayStorageFormOf(isWhole ? ArrayStorage::Int32 : ArrayStorage::Double), &arrayStorageFormOf(isWhole ? ArrayStorage::Double : ArrayStorage::Int32), &arrayStorageFormOf(ArrayStorage::Contiguous) }; - elements.forEachFormByShape(elements.indexingMode(list, arrayStorageTestMask), &ElementAccessEmitter::hasShape, forms, outOfLine, [&](const ArrayStorageForm& form) { + auto readInForm = [&](const ArrayStorageForm& form) { LengthTest test = elements.testLength(value, index, storage); if (!value->isInBounds()) continueGuessedAccessIf(m_out, test.isInside, rareEdge(outOfLine)); Operand element = elements.loadRawElement(form, storage, test.indexInside); continueGuessedAccessIf(m_out, logicalNot(elements.isHole(form, element)), rareEdge(outOfLine)); join.add(elements.convertRawElement(value, form, element, representation, typeIDOf(value.id()))); - }, ElementAccessEmitter::FirstForm::IsLikeTheOthers); + }; + if (m_code.at(value.block()).frequency == BlockFrequency::Rare) + m_out.jump(outOfLine); + else + elements.forEachFormByShape(elements.indexingMode(list, arrayStorageTestMask), &ElementAccessEmitter::hasShape, forms, outOfLine, readInForm, ElementAccessEmitter::FirstForm::IsLikeTheOthers); m_out.appendTo(outOfLine); elements.stopUnlessListIsAsMadeWhereNothingIsFound(value, list); @@ -561,7 +565,10 @@ void GuessedAccessEmitter::generateElementRead(const ValueContext& value) return; } if (value.opcode() == Opcode::GetArrayElement) { - define(value, Boxed(callStub(value, getByValKeepingRegisters(), { in(value->arrayOperand()).value(), in(value->positionOperand()).value() }).first)); + Boxed position = in(value->positionOperand()); + LValue constantIndex = emitter().tryUnboxConstantIndex(position); + StubKey stub = constantIndex ? getByValAtConstantIndex() : getByValKeepingRegisters(); + define(value, Boxed(callStub(value, stub, { in(value->arrayOperand()).value(), constantIndex ? constantIndex : position.value() }).first)); return; } diff --git a/Source/JavaScriptCore/aot/codegen/AOTHeaps.cpp b/Source/JavaScriptCore/aot/codegen/AOTHeaps.cpp index 50687e9051ba6..707078a45104b 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTHeaps.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTHeaps.cpp @@ -59,7 +59,7 @@ static unsigned bucketIndexOf(ElementBase base) { LocationMasks masks; masks.add(MemoryLocation::elements(base)); - uint64_t mask = masks.maskOf(IRHeap::Elements) & ~Typed::markOfElementsOfArraysNotStoredAsRawDoubles; + uint64_t mask = masks.maskOf(IRHeap::Elements) & ~Typed::markOfElementsThatMayHoldCells; RELEASE_ASSERT(std::popcount(mask) == 1); return std::countr_zero(mask); } @@ -168,7 +168,7 @@ const Machine::AbstractHeap& AOTHeaps::at(const Program& program, const MemoryLo uint64_t mask = Typed::maskOf(program, location); if (heap == IRHeap::Elements) - mask &= ~Typed::markOfElementsOfArraysNotStoredAsRawDoubles; + mask &= ~Typed::markOfElementsThatMayHoldCells; if (std::popcount(mask) != 1) return entire(heap); @@ -188,7 +188,7 @@ void AOTHeaps::appendHeaps(const LocationMasks& masks, HeapList& list) const return; } if (heap == IRHeap::Elements) - mask &= ~Typed::markOfElementsOfArraysNotStoredAsRawDoubles; + mask &= ~Typed::markOfElementsThatMayHoldCells; for (uint64_t remainingBits = mask; remainingBits; remainingBits &= remainingBits - 1) list.append(&bucketOf(heap, std::countr_zero(remainingBits))); for (const Machine::AbstractHeap* fieldHeap : m_fieldHeapsBelow[static_cast(heap)]) diff --git a/Source/JavaScriptCore/aot/codegen/AOTOperationEmitter.h b/Source/JavaScriptCore/aot/codegen/AOTOperationEmitter.h index 9bbd1a8d28c41..dd8d280ea09d9 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTOperationEmitter.h +++ b/Source/JavaScriptCore/aot/codegen/AOTOperationEmitter.h @@ -48,6 +48,7 @@ struct CallOptions { bool followsThrow { false }; bool convertsNoCells { false }; bool callerSuspendsAtReturn { false }; + bool rejectsOnThrow { false }; bool returnsWord32 { false }; }; diff --git a/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitter.cpp index a7d07374e713d..60b0d826a469f 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitter.cpp @@ -27,6 +27,7 @@ namespace { struct ArithmeticOperatorTraits { BinaryOperator stub; bool hasStubWithDouble; + RuntimeOperation onBigInts; }; struct UnaryOperatorTraits { @@ -40,15 +41,15 @@ static ArithmeticOperatorTraits traitsOfArithmeticOperator(ArithmeticOperator op { switch (op) { case ArithmeticOperator::Add: - return { BinaryOperator::Add, true }; + return { BinaryOperator::Add, true, RuntimeOperation::ValueAdd }; case ArithmeticOperator::Sub: - return { BinaryOperator::Sub, true }; + return { BinaryOperator::Sub, true, RuntimeOperation::ValueSub }; case ArithmeticOperator::Mul: - return { BinaryOperator::Mul, true }; + return { BinaryOperator::Mul, true, RuntimeOperation::ValueMul }; case ArithmeticOperator::Div: - return { BinaryOperator::Div, true }; + return { BinaryOperator::Div, true, RuntimeOperation::ValueDiv }; case ArithmeticOperator::Mod: - return { BinaryOperator::Mod, false }; + return { BinaryOperator::Mod, false, RuntimeOperation::ValueMod }; case ArithmeticOperator::Pow: break; } @@ -103,6 +104,8 @@ Operand OperatorEmitter::arithmeticResult(const ValueContext& value, ArithmeticO m_out.noteForm(op == ArithmeticOperator::Pow ? Form::Runtime : Form::Closed, StateOfKnowledge::Generic); Boxed x = in(left); Boxed y = in(right); + if (sumsInt32sInPlace(value, op)) + return sumOfAnyValues(value, op, x, y); return arithmeticOnBoxedOperands(value, op, x, y); } case OperatorForm::Integers: @@ -120,12 +123,12 @@ Operand OperatorEmitter::arithmeticResult(const ValueContext& value, ArithmeticO }; Operand x = termOfBoxedSum(left); Operand y = termOfBoxedSum(right); - return sumOfBoxedNumbers(op, x, typeOf(left), y, typeOf(right)); + return sumOfBoxedNumbers(op, x, typeOf(left), y, typeOf(right), emitter().takeNumberChecksOf(value.id()).span()); } if (handsOnBitsOfNaN(value.value()) && (generator().holdsAnyNaN(left) || generator().holdsAnyNaN(right))) { Number x = asNumericOperandWithAnyNaN(left); Number y = asNumericOperandWithAnyNaN(right); - return generator().settleNaNOf(value, arithmeticOnNumbers(op, x, y)); + return generator().settleNaNOf(value, arithmeticOnNumbers(op, x, y, &value)); } if (representationOf(left) == Representation::OptionalDouble && representationOf(right) == Representation::OptionalDouble && markersMayMeetInArithmetic(op, typeOf(left), typeOf(right))) { build().statistics().count(StatisticID::ArithmeticOnMarkers); @@ -133,7 +136,7 @@ Operand OperatorEmitter::arithmeticResult(const ValueContext& value, ArithmeticO } Number x = asNumericOperand(left, &value); Number y = asNumericOperand(right, &value); - return arithmeticOnNumbers(op, x, y); + return arithmeticOnNumbers(op, x, y, &value); } case OperatorForm::NumberAndValue: if (op != ArithmeticOperator::Pow && traitsOfArithmeticOperator(op).hasStubWithDouble) @@ -143,6 +146,8 @@ Operand OperatorEmitter::arithmeticResult(const ValueContext& value, ArithmeticO m_out.noteForm(op == ArithmeticOperator::Pow ? Form::Runtime : Form::Closed, StateOfKnowledge::BigInts); Boxed x = boxCell(in(left)); Boxed y = boxCell(in(right)); + if (op != ArithmeticOperator::Pow && generator().isInLoop(value.block())) + return unboxCell(callOperation(value, traitsOfArithmeticOperator(op).onBigInts, { x, y })); return unboxCell(arithmeticOnBoxedOperands(value, op, x, y)); } case OperatorForm::Strings: @@ -185,6 +190,36 @@ Boxed OperatorEmitter::arithmeticOnBoxedOperands(const ValueContext& value, Arit return Boxed(callBinaryStub(value, traitsOfArithmeticOperator(op).stub, left, right)); } +bool OperatorEmitter::sumsInt32sInPlace(const ValueContext& value, ArithmeticOperator op) +{ + if ((op != ArithmeticOperator::Add && op != ArithmeticOperator::Sub) || !generator().isInLoop(value.block()) || standsInRareBlock(value)) + return false; + return typeOf(value->leftOperand()).primitives & Primitive::Integer && typeOf(value->rightOperand()).primitives & Primitive::Integer; +} + +Boxed OperatorEmitter::sumOfAnyValues(const ValueContext& value, ArithmeticOperator op, Boxed left, Boxed right) +{ + bool isLeftInt32 = typeOf(value->leftOperand()).isInt32(); + bool isRightInt32 = typeOf(value->rightOperand()).isInt32(); + Boxed tested = isLeftInt32 ? right : isRightInt32 ? left : Boxed(m_out.bitAnd(left.value(), right.value())); + LBasicBlock int32Case = newBlock(); + LBasicBlock otherCase = newBlock(); + Join join(m_out); + Condition areInt32 = isInt32(tested); + ScopeFork fork = m_out.forkScope(); + m_out.branch(areInt32.value(), normalEdge(int32Case), unlikelyEdge(otherCase)); + + m_out.appendToArm(int32Case, fork); + Integer x = signExtended(low32(left)); + Integer y = signExtended(low32(right)); + Integer exact = op == ArithmeticOperator::Add ? add(x, y) : sub(x, y); + join.add(emitter().boxInteger(exact, { 2 * static_cast(INT32_MIN), 2 * static_cast(INT32_MAX) + 1 })); + + m_out.appendToArm(otherCase, fork); + join.add(arithmeticOnBoxedOperands(value, op, left, right)); + return join.finishBehind(fork); +} + Number OperatorEmitter::arithmeticOnNumberAndValue(const ValueContext& value, ArithmeticOperator op) { bool numberIsOnRight = !typeOf(value->leftOperand()).isNumber(); @@ -202,7 +237,7 @@ Number OperatorEmitter::arithmeticOnNumberAndValue(const ValueContext& value, Ar if (standsInRareBlock(value)) return byStub(); auto withNumber = [&](Number otherNumber) { - return numberIsOnRight ? arithmeticOnNumbers(op, otherNumber, numberOperand) : arithmeticOnNumbers(op, numberOperand, otherNumber); + return numberIsOnRight ? arithmeticOnNumbers(op, otherNumber, numberOperand, &value) : arithmeticOnNumbers(op, numberOperand, otherNumber, &value); }; LBasicBlock int32Case = newBlock(); LBasicBlock noInt32Case = newBlock(); @@ -403,12 +438,23 @@ Operand OperatorEmitter::convertedToNumber(const ValueContext& value, const Func return failWithZero(value); } +bool OperatorEmitter::isCutToInt32Alone(ValueID number) +{ + ValueID user = generator().code().useCountOf(number) == 1 ? generator().code().onlyUserOf(number) : ValueID(); + return user && (opcodeOf(user) == Opcode::ToInt32 || opcodeOf(user) == Opcode::ToUint32); +} + void OperatorEmitter::generateToNumber(const ValueContext& value) { if (ValueID operand = value->valueOperand(); handsOnBitsOfNaN(value.value()) && generator().holdsAnyNaN(operand)) { define(value, generator().settleNaNOf(value, generator().getWithAnyNaN(operand).as())); return; } + if (ValueID operand = value->valueOperand(); representationOf(operand) == Representation::OptionalDouble && value->form() == OperatorForm::Numbers && isCutToInt32Alone(value.id())) { + m_out.noteForm(Form::Open, StateOfKnowledge::Numbers); + define(value, asNumber(get(operand), typeOf(operand), PayloadOfNaN::Stays)); + return; + } define(value, convertedToNumber(value, [&](Boxed operand) -> Operand { m_out.noteForm(Form::Closed, StateOfKnowledge::Generic); return Number(callStub(value, toNumberAsDouble(), { operand.value() }).first); diff --git a/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitter.h b/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitter.h index 928b1baeb547b..8a0bc15c52755 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitter.h +++ b/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitter.h @@ -7,10 +7,12 @@ #if ENABLE(AOT) +#include "AOTCastEmitter.h" #include "AOTOperationEmitter.h" #include "AOTStubsArithmetic.h" #include "AOTStubsPropertyKeys.h" #include +#include namespace JSC { namespace AOT { namespace Typed { @@ -26,6 +28,7 @@ class OperatorEmitter final : public OperationEmitter { { } + Number remainderByWholeDivisor(const ValueContext&, Number dividend, Number divisor, Integer wholeDivisor); void generateAdd(const ValueContext& value) { defineHeldToItsType(value, arithmeticResult(value, ArithmeticOperator::Add)); } void generateSub(const ValueContext& value) { defineHeldToItsType(value, arithmeticResult(value, ArithmeticOperator::Sub)); } void generateMul(const ValueContext& value) { defineHeldToItsType(value, arithmeticResult(value, ArithmeticOperator::Mul)); } @@ -52,6 +55,7 @@ class OperatorEmitter final : public OperationEmitter { void generateNumberOrNaN(const ValueContext&); void generateToNumeric(const ValueContext&); void generateToNumber(const ValueContext&); + bool isCutToInt32Alone(ValueID number); void generateInstanceOf(const ValueContext&); bool leaveOperandToInstanceOf(const ValueContext& read, InstanceOfOperand, NumberedImmediate); void generateHasInPrototypeChain(const ValueContext&); @@ -83,7 +87,8 @@ class OperatorEmitter final : public OperationEmitter { Number asNumericOperand(ValueID, const OperationContext* user = nullptr); Number numericOperandAsItComes(ValueID); Number asNumericOperandWithAnyNaN(ValueID); - Number asNumber(Operand, const Type&); + enum class PayloadOfNaN : bool { Goes, Stays }; + Number asNumber(Operand, const Type&, PayloadOfNaN = PayloadOfNaN::Goes); Number asNumberLikeOperand(ValueID); template Operand withIntegerOperand(const ValueContext&, const Functor&); template T passingOrConverted(Boxed, const TypeTestPlan&, Form formOfConversion, const Functor& convert); @@ -92,10 +97,12 @@ class OperatorEmitter final : public OperationEmitter { bool standsInRareBlock(const ValueContext& value) const { return m_code.at(value.block()).frequency == BlockFrequency::Rare; } Operand arithmeticOnIntegers(const ValueContext&, ArithmeticOperator); Operand arithmeticOnCheckedIntegers(const ValueContext&, ArithmeticOperator); - Number arithmeticOnNumbers(ArithmeticOperator, Number, Number); - Boxed sumOfBoxedNumbers(ArithmeticOperator, Operand left, const Type& leftType, Operand right, const Type& rightType); + Number arithmeticOnNumbers(ArithmeticOperator, Number, Number, const ValueContext* = nullptr); + Boxed sumOfBoxedNumbers(ArithmeticOperator, Operand left, const Type& leftType, Operand right, const Type& rightType, std::span = { }); Number arithmeticOnNumberAndValue(const ValueContext&, ArithmeticOperator); Boxed arithmeticOnBoxedOperands(const ValueContext&, ArithmeticOperator, Boxed left, Boxed right); + bool sumsInt32sInPlace(const ValueContext&, ArithmeticOperator); + Boxed sumOfAnyValues(const ValueContext&, ArithmeticOperator, Boxed left, Boxed right); Operand addStrings(const ValueContext&); CellPointer concatenationOfParts(const ValueContext&); Operand unaryOnIntegers(const ValueContext&, UnaryOperator); @@ -106,11 +113,13 @@ class OperatorEmitter final : public OperationEmitter { template Condition compare(std::optional, T, T); void generateComparisonOfNumbers(const ValueContext&, std::optional); Condition relationOnStringCells(const ValueContext&, Relation, CellPointer left, CellPointer right); + Condition relationOnBigInts(const ValueContext&, Relation, CellPointer left, CellPointer right); Condition relationOnNullableStrings(const ValueContext&, Relation); enum class ValueInRelation : uint8_t { MayBeInt32, IsNoInt32 }; Condition relationOnValues(const ValueContext&, Relation, ValueInRelation = ValueInRelation::MayBeInt32); bool generateRelationOfIntegerAndValue(const ValueContext&, Relation); bool generateRelationOfBoxedNumberAndValue(const ValueContext&, Relation); + bool generateRelationOfDoubleAndValue(const ValueContext&, Relation); bool generateRelationOfValuesThatAreLikelyInt32s(const ValueContext&, Relation); struct OperandLeftToInstanceOf { diff --git a/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterArithmetic.cpp b/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterArithmetic.cpp index ea71db62a4053..579acde154cb8 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterArithmetic.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterArithmetic.cpp @@ -228,7 +228,7 @@ class IntegerArithmeticEmitter final : public InlineMathEmitter { return result; } - Number remainder(Number dividend, Number divisor) + Number remainder(const ValueContext* value, Number dividend, Number divisor) { static constexpr OpcodeSet remainders { Opcode::Mod }; std::optional constantDivisor = constantNumberOf(divisor.value()); @@ -256,28 +256,51 @@ class IntegerArithmeticEmitter final : public InlineMathEmitter { m_out.noteFormOf(remainders, Form::Open, constantWholeDivisor ? StateOfKnowledge::NumbersAndWholeDivisor : StateOfKnowledge::Numbers); LBasicBlock genericPath = newBlock(); Join join(m_out); + Vector indexes; + bool isKey = value && mayBeKey(*value); + auto addWhole = [&](Integer remainder) { + if (isKey) + indexes.append(m_out.anchor(remainder.value())); + join.add(toNumber(remainder)); + }; + std::optional provenDividend = value ? provenNonNegativeInteger(*value, (*value)->leftOperand(), dividend) : std::nullopt; if (constantWholeDivisor) - join.add(toNumber(remainderOfNonNegative(nonNegativeIntegerOrElse(dividend, genericPath), integer(*constantWholeDivisor)))); + addWhole(remainderOfNonNegative(provenDividend ? *provenDividend : nonNegativeIntegerOrElse(dividend, genericPath), integer(*constantWholeDivisor))); else { #if CPU(ARM64) - if (MacroAssembler::supportsDoubleToInt32ConversionUsingJavaScriptSemantics(build().targetFeatures())) { + if (!provenDividend && MacroAssembler::supportsDoubleToInt32ConversionUsingJavaScriptSemantics(build().targetFeatures())) { LBasicBlock widerPath = newBlock(); Word32 narrowDividend = nonNegativeInt32OrElse(dividend, widerPath); Word32 narrowDivisor = nonNegativeInt32OrElse(divisor, widerPath); branchUnless(isNotZero(narrowDivisor), genericPath); - join.add(toNumber(remainderOfNonNegative(narrowDividend, narrowDivisor))); + addWhole(zeroExtended(remainderOfNonNegative(narrowDividend, narrowDivisor))); m_out.appendTo(widerPath); } #endif - Integer wholeDividend = nonNegativeIntegerOrElse(dividend, genericPath); - Integer wholeDivisor = nonNegativeIntegerOrElse(divisor, genericPath); - branchUnless(isNotZero(wholeDivisor), genericPath); - join.add(toNumber(remainderOfNonNegative(wholeDividend, wholeDivisor))); + Integer wholeDivisor = positiveIntegerOrElse(divisor, genericPath); + Integer wholeDividend = provenDividend ? *provenDividend : wholeDividendOrElse(dividend, join, indexes, genericPath, isKey); + addWhole(remainderOfNonNegative(wholeDividend, wholeDivisor)); } - m_out.appendTo(genericPath); - join.add(callLeaf(RuntimeOperation::Fmod, { dividend, divisor })); - return join.finish(); + return joinedWithLibrary(join, indexes, genericPath, dividend, divisor, isKey); + } + + Number remainderByWholeDivisor(const ValueContext& value, Number dividend, Number divisor, Integer wholeDivisor) + { + if (checksProofs()) + emitter().requireOrCrash(value, logicalOr(equal(wholeDivisor, integer(0)), equal(toNumber(wholeDivisor), divisor)), std::nullopt, ProofSource::OtherOperation); + LBasicBlock genericPath = newBlock(); + Join join(m_out); + Vector indexes; + std::optional provenDividend = provenNonNegativeInteger(value, value->dividendOperand(), dividend); + branchUnless(isNotZero(wholeDivisor), genericPath); + bool isKey = mayBeKey(value); + Integer wholeDividend = provenDividend ? *provenDividend : wholeDividendOrElse(dividend, join, indexes, genericPath, isKey); + Integer remainder = remainderOfNonNegative(wholeDividend, wholeDivisor); + if (isKey) + indexes.append(m_out.anchor(remainder.value())); + join.add(toNumber(remainder)); + return joinedWithLibrary(join, indexes, genericPath, dividend, divisor, isKey); } private: @@ -291,6 +314,51 @@ class IntegerArithmeticEmitter final : public InlineMathEmitter { bool checksProofs() { return emittersCheckProofs(); } + bool mayBeKey(const ValueContext& value) + { + static constexpr OpcodeSet usersOfNumbersAlone = allNumericOperators | allRelations | allEqualities | allBitwiseOperators | OpcodeSet { Opcode::ToBoolean, Opcode::Box, Opcode::NumberToInteger }; + const Code& code = generator().code(); + if (code.useCountOf(value.id()) != 1) + return code.useCountOf(value.id()); + ValueID user = code.onlyUserOf(value.id()); + return user && !usersOfNumbersAlone.contains(code.at(user).opcode); + } + + Integer wholeDividendOrElse(Number dividend, Join& join, Vector& indexes, LBasicBlock genericPath, bool isKey) + { + if (!isX86_64() || (dividend.value()->opcode() == Machine::IToD && dividend.value()->child(0)->type() == Machine::Int32)) + return nonNegativeIntegerOrElse(dividend, genericPath); + LBasicBlock isNotPositive = newBlock(); + LBasicBlock isZero = newBlock(); + LBasicBlock continuation = newBlock(); + Integer whole = positiveIntegerOrElse(dividend, isNotPositive); + m_out.jump(continuation); + m_out.appendTo(isNotPositive); + branch(equal(dividend, 0), { rareEdge(isZero), normalEdge(genericPath) }); + m_out.appendTo(isZero); + if (isKey) + indexes.append(m_out.anchor(integer(0).value())); + join.add(dividend); + m_out.appendTo(continuation); + return whole; + } + + Number joinedWithLibrary(Join& join, Vector& indexes, LBasicBlock genericPath, Number dividend, Number divisor, bool isKey) + { + m_out.appendTo(genericPath); + Number ofAnyNumbers = callLeaf(RuntimeOperation::Fmod, { dividend, divisor }); + if (!isKey) { + join.add(ofAnyNumbers); + return join.finish(); + } + Integer whole = emitter().truncateUnchecked(ofAnyNumbers); + indexes.append(m_out.anchor(select(equal(toNumber(whole), ofAnyNumbers), whole, integer(-1)).value())); + join.add(ofAnyNumbers); + Number result = join.finish(); + emitter().noteIndexBehind(result, Integer(m_out.phi(machineTypeOf(Integer::representation), indexes))); + return result; + } + template T heldToWhatNumbersGive(const ValueContext& value, ArithmeticOperator op, T left, T right, T result) { @@ -505,22 +573,54 @@ class IntegerArithmeticEmitter final : public InlineMathEmitter { } #endif ConversionEmitter& conversions = emitter(); - Word32 narrow = low32(conversions.toInteger(number)); + Word32 narrow = conversions.truncateToWord(number); Integer bitsOfNarrow = conversions.bitsOf(toNumber(narrow)); branchUnless(equal(bitsOfNarrow, conversions.bitsOf(number)), otherwise); branchUnless(greaterThanOrEqual(narrow, word32(0)), otherwise); return narrow; } + std::optional provenNonNegativeInteger(const ValueContext& value, ValueID operand, Number number) + { + const Type& type = typeOf(operand); + if (!type.isInteger() || type.integers.minimum < 0) + return std::nullopt; + Integer whole; + if (number.value()->opcode() != Machine::IToD) + whole = emitter().truncateUnchecked(number); + else if (LValue converted = number.value()->child(0); converted->type() == Machine::Int32) + whole = zeroExtended(Word32(converted)); + else + whole = Integer(converted); + if (checksProofs()) { + Condition isInRange = logicalAnd(greaterThanOrEqual(whole, integer(0)), lessThanOrEqual(whole, integer(type.integers.maximum))); + emitter().requireOrCrash(value, logicalAnd(equal(toNumber(whole), number), isInRange), std::nullopt, ProofSource::OtherOperation); + } + return whole; + } + + Integer positiveIntegerOrElse(Number number, LBasicBlock otherwise) + { + if (!isX86_64() || (number.value()->opcode() == Machine::IToD && number.value()->child(0)->type() == Machine::Int32)) { + Integer whole = nonNegativeIntegerOrElse(number, otherwise); + branchUnless(isNotZero(whole), otherwise); + return whole; + } + Integer whole = emitter().truncateUnchecked(number); + branchUnless(greaterThan(whole, integer(0)), otherwise); + branchUnless(equalOrUnordered(toNumber(whole), number), otherwise); + return whole; + } + Integer nonNegativeIntegerOrElse(Number number, LBasicBlock otherwise) { if (number.value()->opcode() == Machine::IToD && number.value()->child(0)->type() == Machine::Int32) return zeroExtended(nonNegativeInt32OrElse(number, otherwise)); ConversionEmitter& conversions = emitter(); - Integer whole = conversions.toInteger(number); + Integer whole = conversions.truncateUnchecked(number); Integer bitsOfWhole = conversions.bitsOf(toNumber(whole)); branchUnless(equal(bitsOfWhole, conversions.bitsOf(number)), otherwise); - branchUnless(greaterThanOrEqual(whole, integer(0)), otherwise); + branchUnless(isARM64() ? below(whole, integer(1ll << 62)) : greaterThanOrEqual(whole, integer(0)), otherwise); return whole; } }; @@ -673,7 +773,7 @@ class PowerEmitter final : public InlineMathEmitter { } // anonymous namespace -Number OperatorEmitter::asNumber(Operand operand, const Type& type) +Number OperatorEmitter::asNumber(Operand operand, const Type& type, PayloadOfNaN payload) { if (operand.is()) return toNumber(operand.as()); @@ -686,7 +786,7 @@ Number OperatorEmitter::asNumber(Operand operand, const Type& type) ConversionEmitter& conversions = emitter(); Number number(unboxed.value); - if (type.primitives & ~(Primitive::Number | Primitive::Null)) + if (payload == PayloadOfNaN::Goes && type.primitives & ~(Primitive::Number | Primitive::Null)) number = conversions.withoutPayload(number); if (!(type.primitives & Primitive::Null)) return number; @@ -743,7 +843,7 @@ Number OperatorEmitter::asNumericOperandWithAnyNaN(ValueID id) return generator().holdsAnyNaN(id) ? generator().getWithAnyNaN(id).as() : asNumericOperand(id); } -Boxed OperatorEmitter::sumOfBoxedNumbers(ArithmeticOperator op, Operand left, const Type& leftType, Operand right, const Type& rightType) +Boxed OperatorEmitter::sumOfBoxedNumbers(ArithmeticOperator op, Operand left, const Type& leftType, Operand right, const Type& rightType, std::span deferredChecks) { RELEASE_ASSERT(op == ArithmeticOperator::Add || op == ArithmeticOperator::Sub); RELEASE_ASSERT(left.is() || right.is()); @@ -769,6 +869,11 @@ Boxed OperatorEmitter::sumOfBoxedNumbers(ArithmeticOperator op, Operand left, co join.add(conversions.boxInteger(exact, { 2 * static_cast(INT32_MIN), 2 * static_cast(INT32_MAX) + 1 })); m_out.appendToArm(numberCase, fork); + for (const CastEmitter::DeferredNumberCheck& check : deferredChecks) { + LBasicBlock holdsNumber = newBlock(); + m_out.branch(isNumber(check.value).value(), normalEdge(holdsNumber), check.failing); + m_out.appendTo(holdsNumber); + } bool oneAloneWasAsked = left.is() != right.is(); auto numberIn = [&](Operand operand, const Type& type) { if (!oneAloneWasAsked || !operand.is()) @@ -861,7 +966,12 @@ Operand OperatorEmitter::arithmeticOnCheckedIntegers(const ValueContext& value, return is32Bit ? integers.remainderOfIntegers(value) : integers.remainderOfIntegers(value); } -Number OperatorEmitter::arithmeticOnNumbers(ArithmeticOperator op, Number left, Number right) +Number OperatorEmitter::remainderByWholeDivisor(const ValueContext& value, Number dividend, Number divisor, Integer wholeDivisor) +{ + return IntegerArithmeticEmitter(generator()).remainderByWholeDivisor(value, dividend, divisor, wholeDivisor); +} + +Number OperatorEmitter::arithmeticOnNumbers(ArithmeticOperator op, Number left, Number right, const ValueContext* value) { switch (op) { case ArithmeticOperator::Add: @@ -873,7 +983,7 @@ Number OperatorEmitter::arithmeticOnNumbers(ArithmeticOperator op, Number left, case ArithmeticOperator::Div: return div(left, right); case ArithmeticOperator::Mod: - return IntegerArithmeticEmitter(generator()).remainder(left, right); + return IntegerArithmeticEmitter(generator()).remainder(value, left, right); case ArithmeticOperator::Pow: return pow(left, right); } diff --git a/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterBitwise.cpp b/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterBitwise.cpp index f255e74cf0834..1879d852105b2 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterBitwise.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterBitwise.cpp @@ -73,11 +73,15 @@ Operand OperatorEmitter::bitwiseResult(const ValueContext& value, BitwiseOperato return low32(Integer(in(id).value())); case Representation::Double: case Representation::OptionalDouble: + if (representationOf(id) == Representation::OptionalDouble && !generator().holdsAnyNaN(id)) + return emitter().toInt32(asNumber(in(id), typeOf(id), PayloadOfNaN::Stays), bits); return emitter().toInt32(asNumericOperandWithAnyNaN(id), bits); case Representation::JSValue: - if (!typeOf(id).isNumber()) + if (typeOf(id).isNumber()) + return emitter().toInt32OfBoxedNumber(in(id)); + if (!typeOf(id).hasOnlyPrimitives(Primitive::Number | Primitive::UndefinedOrNull)) break; - return emitter().toInt32OfBoxedNumber(in(id)); + return emitter().truncateNullableNumber(in(id)); case Representation::Unchosen: case Representation::Void: case Representation::Boolean: diff --git a/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterComparisons.cpp b/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterComparisons.cpp index d3b5baea34dc8..096e24502722f 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterComparisons.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterComparisons.cpp @@ -88,6 +88,29 @@ static Condition compareNonNegative(Output& out, Relation relation, LValue left, RELEASE_ASSERT_NOT_REACHED(); } +Condition OperatorEmitter::relationOnBigInts(const ValueContext& value, Relation relation, CellPointer left, CellPointer right) +{ + auto callRelation = [&] { + return callOperation(value, traitsOfRelation(relation).operation, { boxCell(left), boxCell(right) }); + }; + if (!generator().isInLoop(value.block())) + return callRelation(); + Word32 leftLength = loadField(left, m_heaps.JSBigInt_length); + Word32 rightLength = loadField(right, m_heaps.JSBigInt_length); + Word32 signs = bitAnd(bitOr(loadByte(left, m_heaps.JSBigInt_sign), loadByte(right, m_heaps.JSBigInt_sign)), TypeInfoPerCellBit); + return either(belowOrEqual(bitOr(bitOr(leftLength, rightLength), signs), 1), [&] { + return selectByBranch(equal(leftLength, rightLength), [&] { + return selectByBranch(isZero(leftLength), [&] { + return boolean(relation == Relation::LessOrEqual || relation == Relation::GreaterOrEqual); + }, [&] { + return compareNonNegative(m_out, relation, loadField(left, m_heaps.JSBigInt_data).value(), loadField(right, m_heaps.JSBigInt_data).value()); + }); + }, [&] { + return compareNonNegative(m_out, relation, leftLength.value(), rightLength.value()); + }); + }, callRelation); +} + template Condition OperatorEmitter::compare(std::optional relation, T left, T right) { @@ -136,15 +159,15 @@ void OperatorEmitter::generateComparisonOfNumbers(const ValueContext& value, std Integer wide = signExtended(low32(boxed)); branch(compare(relation, integerIsOnRight ? wide : integer, integerIsOnRight ? integer : wide), targets); m_out.appendTo(numberCase); - Number unboxed = asNumber(boxed, integerIsOnRight ? leftType : rightType); + Number unboxed = asNumber(boxed, integerIsOnRight ? leftType : rightType, PayloadOfNaN::Stays); Number converted = toNumber(integer); branch(compare(relation, integerIsOnRight ? unboxed : converted, integerIsOnRight ? converted : unboxed), targets); }); return; } if (!mayBeInt32(left, leftType) || !mayBeInt32(right, rightType)) { - Number leftNumber = asNumber(left, leftType); - defineTest(value, compare(relation, leftNumber, asNumber(right, rightType))); + Number leftNumber = asNumber(left, leftType, PayloadOfNaN::Stays); + defineTest(value, compare(relation, leftNumber, asNumber(right, rightType, PayloadOfNaN::Stays))); return; } @@ -159,8 +182,8 @@ void OperatorEmitter::generateComparisonOfNumbers(const ValueContext& value, std Word32 leftInt32 = int32Of(left); branch(compare(relation, leftInt32, int32Of(right)), targets); m_out.appendTo(numberCase); - Number leftNumber = asNumber(left, leftType); - branch(compare(relation, leftNumber, asNumber(right, rightType)), targets); + Number leftNumber = asNumber(left, leftType, PayloadOfNaN::Stays); + branch(compare(relation, leftNumber, asNumber(right, rightType, PayloadOfNaN::Stays)), targets); }); } @@ -171,7 +194,7 @@ void OperatorEmitter::generateRelation(const ValueContext& value, Relation relat Condition result; switch (value->form()) { case OperatorForm::NumberAndValue: - if (generateRelationOfIntegerAndValue(value, relation) || generateRelationOfBoxedNumberAndValue(value, relation)) + if (generateRelationOfIntegerAndValue(value, relation) || generateRelationOfBoxedNumberAndValue(value, relation) || generateRelationOfDoubleAndValue(value, relation)) return; [[fallthrough]]; case OperatorForm::Generic: @@ -200,7 +223,7 @@ void OperatorEmitter::generateRelation(const ValueContext& value, Relation relat return; case OperatorForm::BigInts: m_out.noteForm(Form::Runtime, StateOfKnowledge::BigInts); - result = callOperation(value, traitsOfRelation(relation).operation, { boxCell(in(left)), boxCell(in(right)) }); + result = relationOnBigInts(value, relation, in(left), in(right)); break; case OperatorForm::Strings: { m_out.noteForm(Form::Closed, StateOfKnowledge::Strings); @@ -349,6 +372,29 @@ bool OperatorEmitter::generateRelationOfBoxedNumberAndValue(const ValueContext& return true; } +bool OperatorEmitter::generateRelationOfDoubleAndValue(const ValueContext& value, Relation relation) +{ + bool numberIsOnRight = !typeOf(value->leftOperand()).isNumber(); + ValueID numberID = numberIsOnRight ? value->rightOperand() : value->leftOperand(); + ValueID otherID = numberIsOnRight ? value->leftOperand() : value->rightOperand(); + if (representationOf(numberID) != Representation::Double || !(typeOf(otherID).primitives & Primitive::Number) || generator().code().at(value.block()).frequency == BlockFrequency::Rare) + return false; + m_out.noteForm(Form::Open, StateOfKnowledge::NumberAndValue); + Number number = in(numberID); + Boxed other = in(otherID); + defineTestByBranching(value, [&](BranchTargets targets) { + LBasicBlock int32Case = newBlock(); + LBasicBlock otherCase = newBlock(); + m_out.branch(isInt32(other).value(), normalEdge(int32Case), unlikelyEdge(otherCase)); + m_out.appendTo(int32Case); + Number otherNumber = toNumber(low32(other)); + branch(numberIsOnRight ? compare(relation, otherNumber, number) : compare(relation, number, otherNumber), targets); + m_out.appendTo(otherCase); + branch(relationOnValues(value, relation, ValueInRelation::IsNoInt32), targets); + }); + return true; +} + bool OperatorEmitter::generateRelationOfValuesThatAreLikelyInt32s(const ValueContext& value, Relation relation) { if (generator().code().at(value.block()).frequency == BlockFrequency::Rare) diff --git a/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterConversions.cpp b/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterConversions.cpp index 21cd5482285c3..6ede6a0fab027 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterConversions.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterConversions.cpp @@ -8,6 +8,7 @@ #if ENABLE(AOT) +#include "AOTCastEmitter.h" #include "AOTConstantEmitter.h" #include "AOTConversionEmitter.h" #include "AOTIRProgram.h" @@ -120,6 +121,13 @@ class ToStringEmitter final : public OperationEmitter { return convertByType(context, get(id), typeIDOf(id)); } + StringEmitter::StringPart partOfConcatenation(const OperationContext& context, ValueID id) + { + if (StringID constant = emitter().findNumberedStringConstant(context, id)) + return constant; + return convertAddend(context, id); + } + CellPointer convertByType(const OperationContext& context, Operand operand, TypeID typeID) { Type type = types().at(typeID); @@ -152,7 +160,7 @@ class ToStringEmitter final : public OperationEmitter { CellPointer convertDouble(const OperationContext& context, Number value) { ConversionEmitter& conversions = emitter(); - Word32 whole = low32(conversions.toInteger(value)); + Word32 whole = conversions.truncateToWord(value); Integer bitsOfWhole = conversions.bitsOf(toNumber(whole)); return selectByBranch(equal(bitsOfWhole, conversions.bitsOf(value)), [&] { return emitter().integerToString(context, whole); @@ -243,6 +251,11 @@ void OperatorEmitter::generateToBoolean(const ValueContext& value) case OperatorForm::Bits: case OperatorForm::Booleans: m_out.noteForm(Form::Open, stateOf(value->form())); + if (ValueID check = generator().checkLeftToTruthTest(value.id())) { + emitter().checkBooleanOrBranch(value, check); + if (value.isBranchCondition()) + return; + } defineTest(value, truthiness.isTruthy(value, get(tested), typeOf(tested))); return; case OperatorForm::Unselected: @@ -287,11 +300,11 @@ CellPointer OperatorEmitter::concatenationOfParts(const ValueContext& value) ToStringEmitter toStrings(generator()); if (!value->isLikelyKey()) { Vector firstOccurrences = firstOccurrencesOfStrcatParts(parts); - Vector fibers; + Vector fibers; for (uint32_t index = 0; index < parts.size(); ++index) { uint32_t first = firstOccurrences[index]; bool isConvertedAlready = first < index && !typeOf(parts[index]).hasObjects(); - fibers.append(isConvertedAlready ? fibers[first] : toStrings.convertAddend(value, parts[index])); + fibers.append(isConvertedAlready ? fibers[first] : toStrings.partOfConcatenation(value, parts[index])); } return emitter().concatenate(value, fibers.span()); } @@ -314,11 +327,17 @@ Operand OperatorEmitter::addStrings(const ValueContext& value) ValueID right = value->rightOperand(); bool resultIsBoxed = value.representation() == Representation::JSValue; switch (value->form()) { - case OperatorForm::Strings: - return strings.concatenate(value, { in(left), in(right) }); + case OperatorForm::Strings: { + auto partOf = [&](ValueID operand) -> StringEmitter::StringPart { + if (StringID constant = strings.findNumberedStringConstant(value, operand)) + return constant; + return in(operand); + }; + return strings.concatenate(value, { partOf(left), partOf(right) }); + } case OperatorForm::StringAndPrimitive: { - CellPointer first = toStrings.convertAddend(value, left); - CellPointer second = toStrings.convertAddend(value, right); + StringEmitter::StringPart first = toStrings.partOfConcatenation(value, left); + StringEmitter::StringPart second = toStrings.partOfConcatenation(value, right); return strings.concatenate(value, { first, second }); } case OperatorForm::NullableStrings: { diff --git a/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterEquality.cpp b/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterEquality.cpp index 7c974c2d0c536..e3b1937311cc6 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterEquality.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterEquality.cpp @@ -12,6 +12,7 @@ #include "AOTRepresentationRules.h" #include "AOTRuntimeCallEmitter.h" #include "AOTStringEmitter.h" +#include "AOTStubsStrings.h" #include "AOTStubsValues.h" #include "AOTTypeArena.h" #include "AOTTypeTable.h" @@ -37,6 +38,29 @@ class EqualityEmitter final : public OperationEmitter { return Condition(m_out.castToInt32(callStub(context, AOT::equal(isStrict), { left.value(), right.value() }, options).first)); } + Condition equalityOnBigInts(const ValueContext& value, bool isStrict, CellPointer left, CellPointer right) + { + auto callEqual = [&] { + return callEqualStub(value, isStrict, boxCell(left), boxCell(right)); + }; + if (!generator().isInLoop(value.block())) + return callEqual(); + Word32 length = loadField(left, m_heaps.JSBigInt_length); + return selectByBranch(equal(length, loadField(right, m_heaps.JSBigInt_length)), [&] { + return either(belowOrEqual(length, 1), [&] { + return selectByBranch(isZero(length), [&] { + return boolean(true); + }, [&] { + Condition haveSameDigit = equal(loadField(left, m_heaps.JSBigInt_data), loadField(right, m_heaps.JSBigInt_data)); + Condition haveSameSign = equal(loadByte(left, m_heaps.JSBigInt_sign), loadByte(right, m_heaps.JSBigInt_sign)); + return Condition(m_out.bitAnd(haveSameDigit.value(), haveSameSign.value())); + }); + }, callEqual); + }, [&] { + return boolean(false); + }); + } + bool comparesOnlyNumbers(const ValueContext& value) { Vector provenTypes; @@ -82,10 +106,47 @@ class EqualityEmitter final : public OperationEmitter { return callEqualStub(value, false, x, y); } + Condition isMultipleOfConstant(ValueID remainder, ValueID comparedWith) + { + const Operation& divided = operationAt(definitionOf(remainder)); + const Operation& zero = operationAt(definitionOf(comparedWith)); + if (divided.opcode != Opcode::RemainderOfIntegers || zero.opcode != Opcode::Integer || zero.integer() || m_code.useCountOf(remainder) != 1) + return { }; + ValueID dividend = divided.dividendOperand(); + Representation representation = representationOf(dividend); + Range dividends = typeOf(dividend).integers; + Range divisors = typeOf(divided.divisorOperand()).integers; + bool isSigned = dividends.minimum < 0; + if ((representation != Representation::Int32 && representation != Representation::Int64) || !divisors.isSingleValue() || !dividends.isSubsetOf(isSigned ? Range::int32() : Range::uint32())) + return { }; + int64_t divisor = std::abs(divisors.minimum); + if (divisor < 3 || divisor > std::numeric_limits::max() || hasOneBitSet(static_cast(divisor))) + return { }; + + unsigned shift = std::countr_zero(static_cast(divisor)); + uint32_t oddPart = static_cast(divisor) >> shift; + uint32_t inverse = oddPart; + while (inverse * oddPart != 1) + inverse *= 2 - inverse * oddPart; + uint32_t offset = isSigned ? (std::numeric_limits::max() / oddPart) >> shift << shift : 0; + uint32_t greatestQuotient = isSigned ? 2 * offset >> shift : std::numeric_limits::max() / static_cast(divisor); + Word32 quotient = mul(representation == Representation::Int32 ? in(dividend) : low32(in(dividend)), word32(static_cast(inverse))); + if (offset) + quotient = add(quotient, word32(static_cast(offset))); + if (shift) + quotient = Word32(m_out.bitOr(m_out.lShr(quotient.value(), shift), m_out.shl(quotient.value(), 32 - shift))); + build().statistics().count(StatisticID::MultiplesTestedWithoutRemainder); + return belowOrEqual(quotient, word32(static_cast(greatestQuotient))); + } + Condition equalityOnIntegers(const ValueContext& value) { ValueID left = value->leftOperand(); ValueID right = value->rightOperand(); + if (Condition isMultiple = isMultipleOfConstant(left, right)) + return isMultiple; + if (Condition isMultiple = isMultipleOfConstant(right, left)) + return isMultiple; switch (formOfOperands(value)) { case Representation::Int32: { Word32 x = in(left); @@ -331,8 +392,11 @@ class EqualityEmitter final : public OperationEmitter { return equalsWrittenText(value, ComparedOperand::Value, in(boxed).value(), definition.string(), everyCellIsString); } m_out.noteForm(Form::Closed, StateOfKnowledge::StringAndValue); - Boxed string = boxCell(in(typed)); - return callEqualStub(value, true, string, in(boxed)); + CellPointer string = in(typed); + Boxed other = in(boxed); + if (mayBeEmpty(boxed)) + return callEqualStub(value, true, boxCell(string), other); + return Condition(m_out.castToInt32(callStub(value, stringEqualsValue(), { string.value(), other.value() }).first)); } Condition equalityOnValues(const ValueContext& value, bool isStrict) @@ -361,8 +425,7 @@ class EqualityEmitter final : public OperationEmitter { }); return; } - bool asksBothTagsInLine = value->expectsInt32s() && generator().code().at(value.block()).frequency != BlockFrequency::Rare; - if (asksBothTagsInLine && !bitsDecide(kindsComparedBy(value)) && !generator().takesOpenForm(value, StateOfKnowledge::Generic)) { + if (asksForInt32sFirst(value) && !bitsDecide(kindsComparedBy(value)) && !generator().takesOpenForm(value, StateOfKnowledge::Generic)) { m_out.noteForm(Form::Closed, StateOfKnowledge::Generic); defineTestByBranching(value, [&](BranchTargets targets) { LBasicBlock areInt32s = newBlock(); @@ -506,6 +569,11 @@ class EqualityEmitter final : public OperationEmitter { return { typesAreExhaustive, mayBothBe(Primitive::WholeNumber), mayBothBe(Primitive::Number), mayBothBeStrings, mayBothBeStrings || mayBothBe(Primitive::BigInt) }; } + bool asksForInt32sFirst(const ValueContext& value) + { + return value->opcode == Opcode::StrictEq && value->expectsInt32s() && generator().code().at(value.block()).frequency != BlockFrequency::Rare; + } + static bool bitsDecide(const ComparedKinds& kinds) { return !kinds.mayBothBeNumbers && !kinds.mayBeEqualByContents; } void decideWithoutContents(const ValueContext& value, Boxed x, Boxed y, BranchTargets targets, NOESCAPE const Invocable auto& answerByStub, NOESCAPE const Invocable auto& answerForStrings) @@ -524,6 +592,16 @@ class EqualityEmitter final : public OperationEmitter { return; } + bool areInt32sDecided = mayBothBeNumbers && asksForInt32sFirst(value); + if (areInt32sDecided) { + LBasicBlock areInt32s = newBlock(); + LBasicBlock areNotBothInt32s = newBlock(); + m_out.branch(isInt32(Boxed(m_out.bitAnd(x.value(), y.value()))).value(), normalEdge(areInt32s), rareEdge(areNotBothInt32s)); + m_out.appendTo(areInt32s); + branch(equal(x, y), targets); + m_out.appendTo(areNotBothInt32s); + } + bool comparesStringsHere = generator().takesFormOfLoops(value, StateOfKnowledge::Generic) && mayBothBeStrings; LBasicBlock hasOtherBits = newBlock(); @@ -545,8 +623,11 @@ class EqualityEmitter final : public OperationEmitter { m_out.appendTo(hasOtherBits); LBasicBlock areCells = mayBeEqualByContents ? newBlock() : nullptr; - LBasicBlock areNotBothCells = mayBothBeNumbers ? newBlock() : nullptr; - m_out.branch(areBothCells(x, y).value(), areCells ? normalEdge(areCells) : targets.notTaken, areNotBothCells ? normalEdge(areNotBothCells) : targets.notTaken); + LBasicBlock areNotBothCells = mayBothBeNumbers && !areInt32sDecided ? newBlock() : nullptr; + auto edgeBesideCells = [&] { + return areNotBothCells ? normalEdge(areNotBothCells) : areInt32sDecided ? edgeToStub() : targets.notTaken; + }; + m_out.branch(areBothCells(x, y).value(), areCells ? normalEdge(areCells) : targets.notTaken, edgeBesideCells()); if (areNotBothCells) { m_out.appendTo(areNotBothCells); m_out.branch(isInt32(Boxed(m_out.bitAnd(x.value(), y.value()))).value(), targets.notTaken, edgeToStub()); @@ -681,7 +762,7 @@ void OperatorEmitter::generateEquality(const ValueContext& value, bool isStrict) case OperatorForm::BigInts: { m_out.noteForm(Form::Closed, StateOfKnowledge::BigInts); CellPointer x = in(left); - result = equality.callEqualStub(value, isStrict, boxCell(x), boxCell(in(right))); + result = equality.equalityOnBigInts(value, isStrict, x, in(right)); break; } case OperatorForm::Strings: diff --git a/Source/JavaScriptCore/aot/codegen/AOTPureOperations.h b/Source/JavaScriptCore/aot/codegen/AOTPureOperations.h index 69947aee28528..6ffe653d2e251 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTPureOperations.h +++ b/Source/JavaScriptCore/aot/codegen/AOTPureOperations.h @@ -18,6 +18,7 @@ enum class PureOperation : uint16_t { BuiltinCell, ToInt32, TruncateToInt64, + ExactInt32, NonNegativeInt32, RuntimeOperation, BoxNumber, diff --git a/Source/JavaScriptCore/aot/codegen/AOTReadsThatAssume.txt b/Source/JavaScriptCore/aot/codegen/AOTReadsThatAssume.txt index 03bd64276bc90..84768c16d56c5 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTReadsThatAssume.txt +++ b/Source/JavaScriptCore/aot/codegen/AOTReadsThatAssume.txt @@ -4,22 +4,21 @@ JSString_value | codegen/AOTHeaps.cpp | 1 | makes the heap another name of JSRop JSString_value | codegen/AOTHeaps.h | 1 | the declaration of the heap JSString_value | codegen/AOTPrimitiveEmitter.h | 1 | isRopeString(): it is the test JSString_value | codegen/AOTStringBuiltinEmitter.cpp | 1 | is8Bit() tests the rope bit next to the load: the flat side alone reads through the word, the other side reads the width off the word itself -JSString_value | codegen/AOTStringEmitter.cpp | 7 | implOfFlatString() takes a FlatStringCell; viewOfFlatString() is listed below; three test the rope bit next to the load; one reads behind the stub that has resolved the rope in place, or off the base of a substring, which is flat; one reads what FlattenString has just made +JSString_value | codegen/AOTStringEmitter.cpp | 9 | implOfFlatString() takes a FlatStringCell; viewOfFlatString() is listed below; five test the rope bit next to the load; one reads behind the stub that has resolved the rope in place, or off the base of a substring, which is flat; one reads what FlattenString has just made JSString_value | codegen/AOTStringEmitterComparisons.cpp | 1 | tests the rope bit next to the load JSString_value | codegen/AOTStringSwitch.cpp | 1 | tests the rope bit next to the load -viewOfFlatString | builtins/AOTMathBuiltinEmitterNumbers.cpp | 1 | behind isRopeString() of the same cell -viewOfFlatString | codegen/AOTStringBuiltinEmitter.cpp | 5 | each behind isRopeString() of the same cell +viewOfFlatString | codegen/AOTStringBuiltinEmitter.cpp | 2 | each behind isRopeString() of the same cell viewOfFlatString | codegen/AOTStringEmitter.cpp | 1 | the definition viewOfFlatString | codegen/AOTStringEmitter.h | 1 | the declaration -viewOfFlatString | codegen/AOTStringEmitterComparisons.cpp | 1 | behind isRopeString() of the same cell +offsetOfValue untested | runtime/AOTStringOperations.cpp | 1 | A count of uses added to the first word of a substring, which holds flags alone, and nothing is read through it. offsetOfValue untested | stubs/AOTStubsArrayReads.cpp | 2 | The base of a substring, which is flat: both makers of substrings (AOTStubsStringSubstrings.cpp) and jsSubstringOfResolved() store a flat base alone. The word of an element whose cell says that it holds an atom is only compared with the address of the atom that is sought: nothing is read through it. offsetOfValue untested | stubs/AOTStubsBoxedCalls.cpp | 1 | A lambda: its first call is tested on the next line, its second loads the word of the same string again, and a flat string stays flat. offsetOfValue untested | stubs/AOTStubsCollections.cpp | 2 | generateGrowCompactTable() is reached from emitInsert() alone, behind its lookup: a key that is a string has passed emitFindStringKey() (AOTStubsHashTables.cpp), and the form that compares bits is chosen for a type without strings. A key of the table, which is flat and has its hash. offsetOfValue untested | stubs/AOTStubsHashTables.cpp | 8 | emitCompareCharactersInImage(): the key has passed emitFindStringKey(), where a rope leaves. The key has passed emitFindStringKey(), where a rope leaves and a string without a hash gets one. A candidate is a key of the table, which is flat and has its hash: emitFindStringKey() and runtime/AOTCompactTable.cpp let no other in. At a table in the image two words are only compared, the key's with that of the cell of a constant: nothing is read through either, which is right for a rope too. The same candidate where characters are sought. offsetOfValue untested | stubs/AOTStubsKeyedAccess.cpp | 2 | A count of visits added to the first word of a substring, which holds flags alone, and nothing is read through it. The base of a substring. offsetOfValue untested | stubs/AOTStubsStringHelpers.cpp | 2 | The base of a substring, twice. -offsetOfValue untested | stubs/AOTStubsStringLiterals.cpp | 1 | The first piece of a rope that is no substring, and packPiece() tests each piece. The base of a substring. -offsetOfValue untested | stubs/AOTStubsStringSubstrings.cpp | 2 | makeSubstringOfFlatString() trusts its caller: generateFinishRegExpExec(), and all four jumps to that stub bring a flat subject: finishMatchAndReturn() behind loadFlatSubject() or flattenSubject(), the three of generateRegExpMatchLiteral() behind the test at its head. -offsetOfValue untested | stubs/AOTStubsStrings.cpp | 6 | The base of a substring, twice. branchOnOrderOfFlatStrings() loads the words of both strings again: its two callers, generateCompareStrings() and emitStringOrder() (AOTStubsArithmetic.cpp), have tested both right before, and a flat string stays flat. emitStringLocaleComparePrelude() loads the words of both strings again: it has tested both at its head and calls nothing in between. +offsetOfValue untested | stubs/AOTStubsStringLiterals.cpp | 2 | The first piece of a rope that is no substring, and packPiece() tests each piece. The base of a substring. generateNarrowCharactersOfString() reads through the word behind the substring bit alone, which no flat string has. +offsetOfValue untested | stubs/AOTStubsStringSubstrings.cpp | 3 | The base of a substring, of which one character is taken. makeSubstringOfFlatString() trusts its caller: generateFinishRegExpExec(), and all four jumps to that stub bring a flat subject: finishMatchAndReturn() behind loadFlatSubject() or flattenSubject(), the three of generateRegExpMatchLiteral() behind the test at its head. +offsetOfValue untested | stubs/AOTStubsStrings.cpp | 7 | The base of a substring, three times. branchOnOrderOfFlatStrings() loads the words of both strings again: its two callers, generateCompareStrings() and emitStringOrder() (AOTStubsArithmetic.cpp), have tested both right before, and a flat string stays flat. emitStringLocaleComparePrelude() loads the words of both strings again: it has tested both at its head and calls nothing in between. offsetOfValue untested | tests/AOTStubBoxedCallsOfLanguageTests.cpp | 2 | The test lays down the word of a string that it makes up, flat and as a rope. Nothing is read through it. offsetOfValue untested | tests/AOTStubPropertyReadTests.cpp | 4 | Two tests lay down the word of a string that they make up, flat and as a rope. Nothing is read through it. diff --git a/Source/JavaScriptCore/aot/codegen/AOTRepresentationEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTRepresentationEmitter.cpp index 2ab480c7d813a..8fe3f8cb30e98 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTRepresentationEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTRepresentationEmitter.cpp @@ -120,6 +120,11 @@ void RepresentationEmitter::generate(const ValueContext& value) define(value, get(converted)); return; } + if (std::optional boxed = generator().findBoxed(converted)) [[unlikely]] { + RELEASE_ASSERT(value.opcode() == Opcode::Box && value.representation() == Representation::JSValue); + define(value, *boxed); + return; + } if (ValueID sum = generator().sumBoxedBy(value.value())) [[unlikely]] { define(value, emitter().boxedSum(sum)); return; @@ -128,6 +133,10 @@ void RepresentationEmitter::generate(const ValueContext& value) define(value, emitter().boxInteger(in(converted), Range { 0, IndexingHeader::maximumLength - 1 })); return; } + if (value.opcode() == Opcode::Unbox && generator().isOnlyTakenAsPosition(value.id())) { + generator().defineBoxed(value, in(converted)); + return; + } if (value.opcode() == Opcode::Unbox) { if (std::optional handedOn = emitter().takeIntegerHandedOnTo(value.id())) { define(value, *handedOn); diff --git a/Source/JavaScriptCore/aot/codegen/AOTRuntimeCallEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTRuntimeCallEmitter.cpp index ca92a8db4289c..70384ccdb240d 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTRuntimeCallEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTRuntimeCallEmitter.cpp @@ -16,6 +16,7 @@ #include "AOTHeaps.h" #include "AOTInliningAcrossFiles.h" #include "AOTPureOperations.h" +#include "AOTRelocation.h" #include "AOTSavedFrameEmitter.h" #include "AOTStaticsEmitter.h" #include "AOTStubCatalog.h" @@ -260,7 +261,7 @@ void CallGenerator::generate(CCallHelpers& jit, const Machine::PatchParameters& } emitCallInstruction(jit, called, moves); if (neverReturns) - jit.breakpoint(); + emitUnreachable(jit); else if (branchesOnFlags) emitBranchOnFlags(jit, params); } @@ -1666,6 +1667,10 @@ RecordedSite RuntimeCallEmitter::recordedSiteOf(const OperationContext& context, bool isTailCall = origin.isTailCall || options.callerHasLeft; uint8_t kind = callSiteKindsByColumns[(options.constructs ? 1 : 0) | (isTailCall ? 2 : 0) | (isInsideCallee ? 4 : 0) | (calledBuiltin ? 8 : 0)]; RELEASE_ASSERT(kind != noCallSiteKind); + if (options.rejectsOnThrow) { + RELEASE_ASSERT(static_cast(kind) == CallSiteKind::Plain); + return { .origin = origin, .kind = CallSiteKind::AwaitedCall }; + } return { .origin = origin, .kind = static_cast(kind), .calledBuiltin = calledBuiltin, .constructedCallee = options.constructedCallee }; } diff --git a/Source/JavaScriptCore/aot/codegen/AOTSavedFrameEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTSavedFrameEmitter.cpp index 478e242354578..f589ab7f019d1 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTSavedFrameEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTSavedFrameEmitter.cpp @@ -261,7 +261,7 @@ void SavedFrameEmitter::generateAwaitCallBySavingFrame(const ValueContext& value for (ValueID argument : value->argumentsOperands().first(value->numberOfVariadicOperands() - numberOfHeldOperandsOf(value.value()))) arguments.append(get(argument)); arguments.append(undefined()); - Operand returned = emitter().callDirect(value, callee, arguments.span(), { .callerSuspendsAtReturn = true }); + Operand returned = emitter().callDirect(value, callee, arguments.span(), { .callerSuspendsAtReturn = true, .rejectsOnThrow = conventions().of(callee.function).coroutineNeverSuspends }); RELEASE_ASSERT(isSameWord(returned.representation, Representation::JSValue)); define(value, Boxed(suspend(value, awaitCallBySavingFrame(), { returned.value }, value->promiseOperand(), RunsCodeOfAwaited::BehindFunction).first)); } @@ -325,49 +325,57 @@ void SavedFrameEmitter::findHeapWordsAliveAtSuspensions() return; for (const LiveAtSuspension& record : m_code.suspensions()) { for (ValueID live : m_code.liveAt(record.suspension)) { - if (m_code.at(live).opcode == Opcode::NewHeapWords && !m_heapWordsAliveAtSuspensions.contains(live)) - m_heapWordsAliveAtSuspensions.append(live); + if (m_code.at(live).opcode == Opcode::NewHeapWords) + m_heapWordsAliveAtSuspensions.set(live.index()); } } } -bool SavedFrameEmitter::areNamedOnlyByReadsAndStoresOfWords(ValueID made) const +void SavedFrameEmitter::findUsesOfHeapWords() { - uint32_t usersNotSeen = m_code.useCountOf(made); - if (usersNotSeen == std::numeric_limits::max()) - return false; + if (std::exchange(m_hasFoundUsesOfHeapWords, true)) + return; static_assert(!OperandIndex::GetHeapFrameWord::frame && !OperandIndex::PutHeapFrameWord::frame); - uint32_t numberOfBlocks = m_code.numberOfBlocks(); - uint32_t first = m_code.blockOf(made).index(); - for (uint32_t step = 0; usersNotSeen && step < numberOfBlocks; ++step) { - BlockID block((first + step) % numberOfBlocks); + auto isMadeHere = [&](ValueID operand) { + return operand && m_code.at(operand).opcode == Opcode::NewHeapWords; + }; + auto noteOtherUse = [&](ValueID operand) { + if (isMadeHere(operand)) + m_usesOfHeapWords.add(operand.index(), UsesOfHeapWords()).iterator->value.isUsedOtherwise = true; + }; + for (uint32_t index = 0; index < m_code.numberOfBlocks(); ++index) { + BlockID block(index); if (!m_code.isReached(block)) continue; - bool areNamedOtherwise = false; for (ValueID id : m_code.valuesOf(block)) { const Operation& operation = m_code.at(id); bool takesWordsFirst = operation.opcode == Opcode::GetHeapFrameWord || operation.opcode == Opcode::PutHeapFrameWord; operation.forEachOperand([&](ValueID operand, uint32_t operandIndex) { - if (operand != made) - return; - if (takesWordsFirst && !operandIndex) - --usersNotSeen; - else - areNamedOtherwise = true; + if (!takesWordsFirst || operandIndex) + noteOtherUse(operand); + else if (isMadeHere(operand)) + ++m_usesOfHeapWords.add(operand.index(), UsesOfHeapWords()).iterator->value.numberOfAccesses; }); } for (ValueID operand : m_code.at(block).terminator.operands()) - areNamedOtherwise |= operand == made; - if (m_code.at(block).subject == made) - --usersNotSeen; + noteOtherUse(operand); + if (ValueID subject = m_code.at(block).subject; isMadeHere(subject)) + ++m_usesOfHeapWords.add(subject.index(), UsesOfHeapWords()).iterator->value.numberOfAccesses; m_code.forEachEdgeOf(block, [&](EdgeID, const Edge& edge) { for (ValueID argument : edge.arguments()) - areNamedOtherwise |= argument == made; + noteOtherUse(argument); }); - if (areNamedOtherwise) - return false; } - return !usersNotSeen; +} + +bool SavedFrameEmitter::areNamedOnlyByReadsAndStoresOfWords(ValueID made) +{ + uint32_t numberOfUses = m_code.useCountOf(made); + if (numberOfUses == std::numeric_limits::max()) + return false; + findUsesOfHeapWords(); + UsesOfHeapWords uses = m_usesOfHeapWords.get(made.index()); + return !uses.isUsedOtherwise && uses.numberOfAccesses == numberOfUses; } Machine::StackSlot* SavedFrameEmitter::slotOfHeapWordsInFrame(ValueID frame) @@ -375,18 +383,16 @@ Machine::StackSlot* SavedFrameEmitter::slotOfHeapWordsInFrame(ValueID frame) const Operation& made = m_code.at(frame); if (made.opcode != Opcode::NewHeapWords || !m_code.has(Mechanism::CellsInFrames, m_facts)) return nullptr; - for (const HeapWordsInFrame& words : m_heapWordsInFrame) { - if (words.made == frame) - return words.slot; - } + auto added = m_slotsOfHeapWords.add(frame.index(), nullptr); + if (!added.isNewEntry) + return added.iterator->value; findHeapWordsAliveAtSuspensions(); - Machine::StackSlot* slot = nullptr; - if (!m_heapWordsAliveAtSuspensions.contains(frame) && areNamedOnlyByReadsAndStoresOfWords(frame)) { - slot = m_proc.addStackSlot(made.numberOfWords() * sizeof(uint64_t)); + if (!m_heapWordsAliveAtSuspensions.get(frame.index()) && areNamedOnlyByReadsAndStoresOfWords(frame)) { + Machine::StackSlot* slot = m_proc.addStackSlot(made.numberOfWords() * sizeof(uint64_t)); slot->setIsDeadAtSuspensions(true); + added.iterator->value = slot; } - m_heapWordsInFrame.append(HeapWordsInFrame { frame, slot }); - return slot; + return added.iterator->value; } TypedPointer SavedFrameEmitter::wordInFrame(Machine::StackSlot* slot, uint32_t index) diff --git a/Source/JavaScriptCore/aot/codegen/AOTSavedFrameEmitter.h b/Source/JavaScriptCore/aot/codegen/AOTSavedFrameEmitter.h index 226dae898d28c..cc2fafae4f9cc 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTSavedFrameEmitter.h +++ b/Source/JavaScriptCore/aot/codegen/AOTSavedFrameEmitter.h @@ -69,12 +69,13 @@ class SavedFrameEmitter final : public OperationEmitter { void defineValueAndWord(const ValueContext&, StubCallResult); void branchUnlessUnmodifiedGenerator(Operand, const Type&, LBasicBlock otherwise); - struct HeapWordsInFrame { - ValueID made; - Machine::StackSlot* slot { nullptr }; + struct UsesOfHeapWords { + uint32_t numberOfAccesses { 0 }; + bool isUsedOtherwise { false }; }; void findHeapWordsAliveAtSuspensions(); - bool areNamedOnlyByReadsAndStoresOfWords(ValueID made) const; + void findUsesOfHeapWords(); + bool areNamedOnlyByReadsAndStoresOfWords(ValueID made); Machine::StackSlot* slotOfHeapWordsInFrame(ValueID frame); TypedPointer wordInFrame(Machine::StackSlot*, uint32_t index); @@ -165,8 +166,10 @@ class SavedFrameEmitter final : public OperationEmitter { std::span m_headsOfLoopsInsideStretchesBehind; std::span m_orderOfBlocks; - Vector m_heapWordsInFrame; - Vector m_heapWordsAliveAtSuspensions; + UncheckedKeyHashMap, WTF::UnsignedWithZeroKeyHashTraits> m_slotsOfHeapWords; + UncheckedKeyHashMap, WTF::UnsignedWithZeroKeyHashTraits> m_usesOfHeapWords; + BitVector m_heapWordsAliveAtSuspensions; + bool m_hasFoundUsesOfHeapWords { false }; bool m_hasLookedForHeapWordsAliveAtSuspensions { false }; }; diff --git a/Source/JavaScriptCore/aot/codegen/AOTShapeTestEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTShapeTestEmitter.cpp index 774705ca09761..b042b03c9d975 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTShapeTestEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTShapeTestEmitter.cpp @@ -85,15 +85,13 @@ Condition ShapeTestEmitter::bearsNoMark(CellPointer cell) void ShapeTestEmitter::setMark(CellPointer cell) { CodeGenerator::InitializationScope scope(generator(), CodeGenerator::FillsCellOfHeap::No); - TypedPointer byteWithMark = m_out.address(m_heaps.AOTObject_shapeWord, cell.value(), offsetOfByteWithMark); - m_out.store32As8(bitAnd(Word32(m_out.load8ZeroExt32(byteWithMark)), word32(static_cast(~byteWithMarkIsUnmarked))).value(), byteWithMark); + storeField(cell, m_heaps.AOTObject_shapeWord, bitAnd(loadField(cell, m_heaps.AOTObject_shapeWord), word32(static_cast(~shapeWordIsUnmarked)))); } void ShapeTestEmitter::clearMark(CellPointer cell) { CodeGenerator::InitializationScope scope(generator(), CodeGenerator::FillsCellOfHeap::No); - TypedPointer byteWithMark = m_out.address(m_heaps.AOTObject_shapeWord, cell.value(), offsetOfByteWithMark); - m_out.store32As8(bitOr(Word32(m_out.load8ZeroExt32(byteWithMark)), word32(byteWithMarkIsUnmarked)).value(), byteWithMark); + storeField(cell, m_heaps.AOTObject_shapeWord, bitOr(loadField(cell, m_heaps.AOTObject_shapeWord), word32(static_cast(shapeWordIsUnmarked)))); } Word32 ShapeTestEmitter::orderWordOf(CellPointer cell) diff --git a/Source/JavaScriptCore/aot/codegen/AOTSlotAccessEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTSlotAccessEmitter.cpp index d52548648267e..810b72f44ae22 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTSlotAccessEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTSlotAccessEmitter.cpp @@ -359,6 +359,12 @@ Operand SlotAccessEmitter::loadSlotOrUndefined(const ValueContext& value, CellPo build().statistics().count(StatisticID::EmptySlotConversionsSkipped); return loadSlot(cell, access); } + bool holdsMarkedNumber = representation == SlotRepresentation::OptionalDouble || representation == SlotRepresentation::OptionalInt64; + bool answersMayBeAskedAgain = AOT::hasFlag(build().options().testing, Wire::TestingSwitch::CheckWalkAtRunTime) || AOT::hasFlag(build().options().testing, Wire::TestingSwitch::CheckBoundsInEmittersOnly); + if (holdsMarkedNumber && value.representation() == Typed::representationOf(representation) && !access.slot.isOutOfLine() && !checksWhatSlotsHold && !answersMayBeAskedAgain && generator().isReadBehindOwnUndefinedTest(value)) { + build().statistics().count(StatisticID::EmptySlotConversionsSkipped); + return loadSlot(cell, access); + } if (traits.hasUndefined) return inResultRepresentation(value, slots.loadSlotOrUndefined(cell, access.slot, locationOfSlotAccess(access), extensionWordOfSlotAccess(access))); @@ -427,7 +433,7 @@ Operand SlotAccessEmitter::readSlot(const ValueContext& value, CellPointer cell, return readIfMemberHasField(value, cell, access, [&] { CellPointer base = baseOfSlot(cell, access); if (sourcesOfProofsCheckedUnder(build().options().testing) & bitOfSourceOfProofs(ProofSource::Field)) - emitter().requireOrCrash(value, emitter().holdsValue(base, access.slot, locationOfSlotAccess(access), extensionWordOfSlotAccess(access)), std::nullopt, ProofSource::Field); + emitter().requireOrCrash(value, emitter().isFilled(base, access.slot, locationOfSlotAccess(access), extensionWordOfSlotAccess(access)), std::nullopt, ProofSource::Field); return inResultRepresentation(value, loadFilledSlot(value, base, access)); }); } diff --git a/Source/JavaScriptCore/aot/codegen/AOTSlotEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTSlotEmitter.cpp index 6736844269b88..360dac153c7b1 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTSlotEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTSlotEmitter.cpp @@ -538,7 +538,10 @@ Operand SlotEmitter::loadSlotOrUndefined(CellPointer cell, const PlacedSlot& slo Operand loaded = load(pointer, representation); if (representsAbsentAsZero(representation)) { Condition holdsAbsent = isAbsent(loaded, representation); - return Operand(loaded.representation, m_out.select(holdsAbsent.value(), constantWithBits(loaded.value->type(), traits.undefinedBits), loaded.value)); + LValue loadedOrUndefined = m_out.select(holdsAbsent.value(), constantWithBits(loaded.value->type(), traits.undefinedBits), loaded.value); + if (loaded.is() && loadedOrUndefined != loaded.value) + m_wordsOfSlots.set(loadedOrUndefined, loaded.value); + return Operand(loaded.representation, loadedOrUndefined); } LBasicBlock isValue = newBlock(); diff --git a/Source/JavaScriptCore/aot/codegen/AOTSlotEmitter.h b/Source/JavaScriptCore/aot/codegen/AOTSlotEmitter.h index bb8cebb35c5f1..3a67fcd996c9a 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTSlotEmitter.h +++ b/Source/JavaScriptCore/aot/codegen/AOTSlotEmitter.h @@ -34,6 +34,7 @@ class SlotEmitter final : public OperationEmitter { Operand loadSlot(CellPointer, const PlacedSlot&, const MemoryLocation&, const PlacedSlot* extensionWord = nullptr); Operand loadSlot(CellPointer, const PlacedSlot&, const MemoryLocation&, const PlacedSlot* extensionWord, const Type& loadedType); Operand loadSlotOrUndefined(CellPointer, const PlacedSlot&, const MemoryLocation&, const PlacedSlot* extensionWord = nullptr); + LValue findWordOfSlot(LValue loadedOrUndefined) const { return m_wordsOfSlots.get(loadedOrUndefined); } Operand loadSlotThatHoldsValue(CellPointer, const PlacedSlot&, const MemoryLocation&, const PlacedSlot* extensionWord = nullptr); Operand loadSlotOrLeaveIfEmpty(CellPointer, const PlacedSlot&, const MemoryLocation&, const PlacedSlot* extensionWord, BlockEdge isEmpty); static constexpr bool storeLooksAtType(SlotRepresentation slot) { return slot == SlotRepresentation::OptionalInt32; } @@ -86,6 +87,8 @@ class SlotEmitter final : public OperationEmitter { LValue loadBits(TypedPointer, SlotRepresentation); void storeBits(TypedPointer, SlotRepresentation, LValue); LValue constantWithBits(LType, uint64_t bits); + + UncheckedKeyHashMap m_wordsOfSlots; }; } } } // namespace JSC::AOT::Typed diff --git a/Source/JavaScriptCore/aot/codegen/AOTStraightAllocations.cpp b/Source/JavaScriptCore/aot/codegen/AOTStraightAllocations.cpp index aaf59da1bbde0..2be138aef4703 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTStraightAllocations.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTStraightAllocations.cpp @@ -202,7 +202,7 @@ static void generateNewObject(StraightEmitter& emitter, const BodyValue& value) } } } - bool canBeMade = hasSlots ? placed && !placed->isBornUnfinished : program.at(layout).kind == LayoutKind::Record; + bool canBeMade = hasSlots ? placed && !placed->isBornUnfinished && (prototype || program.at(layout).kind != LayoutKind::Instance || plan.slots().hasDirectInstances(layout)) : program.at(layout).kind == LayoutKind::Record; if (!canBeMade || (isEnvironment && numberListed != fields.size()) || (prototype && (!hasSlots || isEnvironment || !fields.isEmpty()))) { emitter.fail(value); return; diff --git a/Source/JavaScriptCore/aot/codegen/AOTStraightArrayIntrinsics.cpp b/Source/JavaScriptCore/aot/codegen/AOTStraightArrayIntrinsics.cpp index f571f0dacd248..e52e25142e161 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTStraightArrayIntrinsics.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTStraightArrayIntrinsics.cpp @@ -23,7 +23,7 @@ OpcodeSet StraightEmitter::opcodesOfArrayIntrinsics() Opcode::IsPristineArray, Opcode::ArrayStorageKind, Opcode::HasContiguousIndex, Opcode::GetContiguousIndex, Opcode::GetDoubleIndex, Opcode::NewContiguousArray, Opcode::NewArrayLike, - Opcode::TryPutByValDirect, Opcode::TryPutIndex, Opcode::TryCopyFilledElements, Opcode::TryPutElements, Opcode::TryFillNewArray, + Opcode::TryPutByValDirect, Opcode::TryPutIndex, Opcode::TryCopyFilledElements, Opcode::TryPutElements, Opcode::SortNumbers, Opcode::TryFillNewArray, Opcode::TruncateNewArray, }; } @@ -197,6 +197,7 @@ void StraightEmitter::generateArrayIntrinsic(const BodyValue& value) case Opcode::TryPutByValDirect: case Opcode::TryPutIndex: case Opcode::TryPutElements: + case Opcode::SortNumbers: case Opcode::TryFillNewArray: jit().move(CCallHelpers::TrustedImm32(JSValue::ValueFalse), takeResultRegister()); define(value); diff --git a/Source/JavaScriptCore/aot/codegen/AOTStraightCasts.cpp b/Source/JavaScriptCore/aot/codegen/AOTStraightCasts.cpp index c5a8603ce60bb..c31ed33234229 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTStraightCasts.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTStraightCasts.cpp @@ -14,7 +14,7 @@ namespace JSC { namespace AOT { namespace Typed { OpcodeSet StraightEmitter::opcodesOfCasts() { - return inputIdentityOpcodes | OpcodeSet { Opcode::CheckInitialized, Opcode::RequireObjectCoercible, Opcode::RequireObject, Opcode::TypeTest }; + return inputIdentityOpcodes | OpcodeSet { Opcode::CheckInitialized, Opcode::RequireObjectCoercible, Opcode::RequireObject, Opcode::TypeTest, Opcode::IsUnlockedInstance }; } static ThrowKind throwKindOfStraightForm(NullishAccess access) @@ -102,6 +102,14 @@ void StraightEmitter::generateCast(const BodyValue& value) case Opcode::TypeTest: generateTypeTest(*this, value); return; + case Opcode::IsUnlockedInstance: + if (!instancesMayBeLocked(program(), facts().linked(), value.testedType())) { + generateTypeTest(*this, value); + return; + } + jit().move(CCallHelpers::TrustedImm32(JSValue::ValueFalse), takeResultRegister()); + define(value); + return; default: break; } diff --git a/Source/JavaScriptCore/aot/codegen/AOTStraightEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTStraightEmitter.cpp index 59ebbed2658fe..d5fcdd83838b7 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTStraightEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTStraightEmitter.cpp @@ -12,6 +12,7 @@ #include "AOTBuiltinTableBindings.h" #include "AOTCellFormat.h" #include "AOTModuleEmitter.h" +#include "AOTRelocation.h" #include "AOTStubCatalog.h" #include "AOTStubsArithmetic.h" #include "AOTRepresentationRules.h" @@ -902,7 +903,7 @@ void StraightEmitter::emitCallInstruction(StubKey key, const StubSignature& sign m_records.traps.callThrowSites.append(CallThrowSite { returnSite, WTF::move(words), wordsThatAreMentions }); } if (options.neverReturns) - m_jit->breakpoint(); + emitUnreachable(*m_jit); } void StraightEmitter::callStub(const BodyValue& value, StubKey key, std::initializer_list arguments, const StraightCallOptions& options) diff --git a/Source/JavaScriptCore/aot/codegen/AOTStringBuilderPartEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTStringBuilderPartEmitter.cpp index b9fe7686cfc21..a15e7d54e1217 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTStringBuilderPartEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTStringBuilderPartEmitter.cpp @@ -184,7 +184,7 @@ void StringBuilderPartEmitter::generateStringBuilderAppendNumber(const ValueCont return; } Number appended = in(value->valueOperand()); - Integer truncated = emitter().toInteger(appended); + Integer truncated = emitter().truncateUnchecked(appended); Condition isWhole = equal(toNumber(truncated), appended); Condition hasFewDigits = logicalAnd(isWhole, below(truncated, StringBuilderEmitter::firstIntegerWrittenByCall)); emitArmsOfStringBuilderPart(m_out, hasFewDigits, Machine::EdgeClass::Normal, [&] { @@ -387,7 +387,7 @@ void StringBuilderPartEmitter::generateStackStringBuilderAppendNumber(const Valu return; } Number appended = in(value->valueOperand()); - Integer truncated = emitter().toInteger(appended); + Integer truncated = emitter().truncateUnchecked(appended); Condition isWhole = equal(toNumber(truncated), appended); Condition hasFewDigits = logicalAnd(isWhole, below(truncated, StringBuilderEmitter::firstIntegerWrittenByCall)); builders().defineCursorBehindPart(value, place.record, emitArmsOfStackStringBuilderPart(m_out, hasFewDigits, Machine::EdgeClass::Normal, [&] { @@ -448,7 +448,7 @@ void StringBuilderPartEmitter::generateStackStringBuilderAppendCodeUnit(const Va m_out.store32As8(codeUnit.value(), builders().bytesAt(place.cursor)); return builders().advance(value, place.record, offsetBy(place.cursor, 1)); }, [&] { - return appendString(value, place, callOperation(value, RuntimeOperation::CharToString, { codeUnit })); + return callOperation(value, RuntimeOperation::StackStringBuilderAppendCodeUnit, { place.record, place.cursor, codeUnit }, { .isOnRarePath = true }); })); } diff --git a/Source/JavaScriptCore/aot/codegen/AOTStringBuiltinEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTStringBuiltinEmitter.cpp index c834d4440b9da..b31d908625765 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTStringBuiltinEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTStringBuiltinEmitter.cpp @@ -282,10 +282,8 @@ void StringBuiltinEmitter::generateTrim(const InlineCall& call, TrimmedEnds ends LBasicBlock byCall = newBlock(); Join join(m_out); - Condition stringIsFlat = logicalNot(isRopeString(string)); ScopeFork fork = m_out.forkScope(); - continueIf(stringIsFlat, byCall); - CharacterView view = strings().viewOfFlatString(string); + CharacterView view = strings().viewWithoutResolving(call.context(), string, unlikelyEdge(byCall)); continueIf(view.is8Bit, byCall); continueIf(isNotZero(view.length), byCall); CharacterView narrowView { view.characters, view.length, boolean(true), view.owner }; @@ -312,7 +310,7 @@ void StringBuiltinEmitter::generateStringLocaleCompare(const InlineCall& call) void StringBuiltinEmitter::generateStringConcat(const InlineCall& call) { - Vector parts; + Vector parts; parts.append(stringOperand(call.receiver())); for (ValueID argument : call.arguments()) parts.append(stringOperand(argument)); @@ -358,9 +356,8 @@ Condition StringBuiltinEmitter::hasCharactersAt(const InlineCall& call, CellPoin return loadBytes(m_out, width, TypedPointer(characters, pointer.value())); }; - continueIf(logicalNot(isRopeString(string)), generic); continueIf(logicalNot(isRopeString(other)), generic); - CharacterView subject = strings().viewOfFlatString(string); + CharacterView subject = strings().viewWithoutResolving(call.context(), string, unlikelyEdge(generic)); CharacterView sought = strings().viewOfFlatString(other); continueIf(logicalAnd(subject.is8Bit, sought.is8Bit), generic); continueIf(belowOrEqual(sought.length, word32(maxLengthComparedInLine)), generic); @@ -431,11 +428,9 @@ void StringBuiltinEmitter::generateSearch(const InlineCall& call, SearchDirectio LBasicBlock generic = newBlock(); Join join(m_out); - Condition stringIsFlat = logicalNot(isRopeString(string)); ScopeFork fork = m_out.forkScope(); - continueIf(stringIsFlat, generic); continueIf(logicalNot(isRopeString(other)), generic); - CharacterView subject = strings().viewOfFlatString(string); + CharacterView subject = strings().viewWithoutResolving(call.context(), string, unlikelyEdge(generic)); CharacterView sought = strings().viewOfFlatString(other); continueIf(equal(subject.is8Bit, sought.is8Bit), generic); join.add(strings().find(subject, sought, position, direction)); diff --git a/Source/JavaScriptCore/aot/codegen/AOTStringEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTStringEmitter.cpp index d2327ec96803b..47e9bc9cfc881 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTStringEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTStringEmitter.cpp @@ -8,6 +8,7 @@ #if ENABLE(AOT) +#include "AOTConstantEmitter.h" #include "AOTConversionEmitter.h" #include "AOTElementAccessEmitter.h" #include "AOTPropertyCachesStringKeys.h" @@ -174,14 +175,63 @@ CharacterView StringEmitter::viewOfString(const OperationContext& context, CellP return viewOfSource({ joinedCharacters, joinedLength, joinedFlags }, string); } +CharacterView StringEmitter::viewWithoutResolving(const OperationContext& context, CellPointer string, BlockEdge otherwise) +{ + Pointer fiber = loadField(string, m_heaps.JSString_value); + Condition isFlat = hasNoBits(fiber, JSString::isRopeInPointer); + if (!context.block() || !generator().isInLoop(context.block())) { + continueIf(isFlat, otherwise); + return viewOfSource(sourceInImpl(fiber), string); + } + + LBasicBlock flat = newBlock(); + LBasicBlock rope = newBlock(); + LBasicBlock substring = newBlock(); + LBasicBlock continuation = newBlock(); + Vector characters; + Vector lengths; + Vector implFlags; + auto join = [&](const CharacterSource& source) { + characters.append(m_out.anchor(source.characters.value())); + lengths.append(m_out.anchor(source.length.value())); + implFlags.append(m_out.anchor(source.implFlags.value())); + m_out.jump(continuation); + }; + ScopeFork fork = m_out.forkScope(); + m_out.branch(isFlat.value(), normalEdge(flat), unlikelyEdge(rope)); + + m_out.appendToArm(flat, fork); + join(sourceInImpl(fiber)); + + m_out.appendToArm(rope, fork); + m_out.branch(m_out.testNonZeroPtr(fiber.value(), JSRopeString::isSubstringInPointer), normalEdge(substring), otherwise); + + m_out.appendToContinuation(substring); + StubCallResult found = callStub(context, substringCharacters(), { string.value() }); + join({ Pointer(found.first), loadField(string, m_heaps.JSRopeString_length), flagsInImpl(Pointer(found.second)) }); + + m_out.appendToJoin(continuation, fork); + Pointer joinedCharacters(m_out.phi(machineTypeOf(Pointer::representation), characters)); + Word32 joinedLength(m_out.phi(machineTypeOf(Word32::representation), lengths)); + Word32 joinedFlags(m_out.phi(machineTypeOf(Word32::representation), implFlags)); + return viewOfSource({ joinedCharacters, joinedLength, joinedFlags }, string); +} + CharacterView StringEmitter::viewOf(const OperationContext& context, ValueID string) { std::optional flat = flatStringByType(string); if (!flat) return viewOfString(context, in(string)); CellPointer cell = flat->cell(); - if (auto opened = m_openedStrings.find(cell.value()); opened != m_openedStrings.end()) - return viewOfSource(opened->value.source, cell); + if (auto opened = m_openedStrings.find(cell.value()); opened != m_openedStrings.end()) { + if (!opened->value.mayBeRope) + return viewOfSource(opened->value.source, cell); + const OpenedRope& rope = openedRopeAt(string, context.block()); + CharacterView view = viewOfSource({ rope.characters, Word32(), rope.implFlags }, cell); + view.possibleRope = string; + view.implFlags = rope.implFlags; + return view; + } return viewOfSource(sourceInImpl(implOfFlatString(*flat)), cell); } @@ -228,8 +278,70 @@ Word32 StringEmitter::loadCodeUnit(Pointer characters, Word32 index, bool is8Bit return Word32(m_out.load16ZeroExt32(m_out.baseIndex(m_heaps.characters16, characters.value(), offset))); } +const StringEmitter::OpenedRope& StringEmitter::openedRopeAt(ValueID string, BlockID block) const +{ + for (size_t index = m_openedRopes.size(); index--;) { + const OpenedRope& opened = m_openedRopes[index]; + if (opened.string == string && m_generator.blockDominates(opened.block, block)) + return opened; + } + RELEASE_ASSERT_NOT_REACHED(); +} + +void StringEmitter::passOpenedRope(ValueID string, BlockID from, LValue characters, LValue implFlags) +{ + const OpenedRope& opened = openedRopeAt(string, from); + m_out.upsilon(opened.characters.value(), characters); + m_out.upsilon(opened.implFlags.value(), implFlags); +} + +Word32 StringEmitter::codeUnitOfPossibleRope(const CharacterView& view, Word32 index) +{ + LBasicBlock narrow = newBlock(); + LBasicBlock other = newBlock(); + LBasicBlock wide = newBlock(); + LBasicBlock unresolved = newBlock(); + LBasicBlock resolved = newBlock(); + LBasicBlock rope = newBlock(); + LBasicBlock continuation = newBlock(); + Vector units; + Vector characters; + Vector implFlags; + auto join = [&](Word32 unit, Pointer charactersOfUnit, Word32 flagsOfUnit) { + ensureStillAliveHere(view.owner); + units.append(m_out.anchor(unit.value())); + characters.append(m_out.anchor(charactersOfUnit.value())); + implFlags.append(m_out.anchor(flagsOfUnit.value())); + m_out.jump(continuation); + }; + m_out.branch(view.is8Bit.value(), normalEdge(narrow), unlikelyEdge(other)); + m_out.appendTo(narrow); + join(loadCodeUnit(view.characters, index, true), view.characters, view.implFlags); + + m_out.appendTo(other); + m_out.branch(m_out.notZero64(view.characters.value()), normalEdge(wide), unlikelyEdge(unresolved)); + m_out.appendTo(wide); + join(loadCodeUnit(view.characters, index, false), view.characters, view.implFlags); + + m_out.appendTo(unresolved); + LValue operands[] = { view.owner.value(), index.value() }; + Word32 unit = low32(Integer(emitter().callStubWithoutSite(stringCodeUnitAt(), operands).first)); + Pointer fiber = loadField(view.owner, m_heaps.JSString_value); + m_out.branch(hasNoBits(fiber, JSString::isRopeInPointer).value(), normalEdge(resolved), rareEdge(rope)); + m_out.appendTo(resolved); + join(unit, loadField(fiber, m_heaps.StringImpl_data), flagsInImpl(fiber)); + m_out.appendTo(rope); + join(unit, view.characters, view.implFlags); + + m_out.appendTo(continuation); + m_openedRopes.append({ view.possibleRope, generator().blockBeingGenerated(), Pointer(m_out.phi(machineTypeOf(Pointer::representation), characters)), Word32(m_out.phi(machineTypeOf(Word32::representation), implFlags)) }); + return Word32(m_out.phi(machineTypeOf(Word32::representation), units)); +} + Word32 StringEmitter::codeUnitAt(const CharacterView& view, Word32 index, std::optional indexInWholeNumber) { + if (view.possibleRope) + return codeUnitOfPossibleRope(view, index); return selectByWidth(view.is8Bit, [&](bool is8Bit) { Word32 codeUnit = loadCodeUnit(view.characters, index, is8Bit, indexInWholeNumber); ensureStillAliveHere(view.owner); @@ -327,7 +439,7 @@ Integer StringEmitter::truncateToInteger(Number position) position = select(lessThan(position, negativeLimit), negativeLimit, position); return emitter().toInteger(position); #else - return emitter().toNearestInteger(position); + return emitter().truncateUnchecked(position); #endif } @@ -513,8 +625,6 @@ void StringEmitter::generateNewStringSplitArray(const ValueContext& value) void StringEmitter::generateFlattenString(const ValueContext& value) { CellPointer string = in(value->stringOperand()); - if (ValueID isRead = value->isReadOperand()) - string = select(in(isRead), string, emptyString()); LBasicBlock isFlat = newBlock(); LBasicBlock isRope = newBlock(); LBasicBlock continuation = newBlock(); @@ -526,18 +636,54 @@ void StringEmitter::generateFlattenString(const ValueContext& value) m_out.appendToArm(isFlat, fork); ValueFromBlock cellWhenFlat = m_out.anchor(string.value()); - ValueFromBlock implWhenFlat = m_out.anchor(fiber.value()); + if (generator().mayStayRope(value.id())) { + ValueFromBlock charactersWhenFlat = m_out.anchor(loadField(fiber, m_heaps.StringImpl_data).value()); + ValueFromBlock flagsWhenFlat = m_out.anchor(flagsInImpl(fiber).value()); + m_out.jump(continuation); + + m_out.appendToArm(isRope, fork); + ValueFromBlock cellWhenRope = m_out.anchor(string.value()); + ValueFromBlock charactersWhenRope = m_out.anchor(m_out.constIntPtr(0)); + ValueFromBlock flagsWhenRope = m_out.anchor(word32(0).value()); + m_out.jump(continuation); + + m_out.appendToJoin(continuation, fork); + CellPointer opened(m_out.phi(machineTypeOf(CellPointer::representation), cellWhenFlat, cellWhenRope)); + Pointer characters(m_out.phi(machineTypeOf(Pointer::representation), charactersWhenFlat, charactersWhenRope)); + Word32 flags(m_out.phi(machineTypeOf(Word32::representation), flagsWhenFlat, flagsWhenRope)); + m_openedStrings.add(opened.value(), OpenedString { fiber, { characters, Word32(), flags }, true }); + m_openedRopes.append({ value.id(), value.block(), characters, flags }); + generator().forEachLoopThatCarries(value.id(), [&](BlockID head, LValue carriedCharacters, LValue carriedFlags) { + m_openedRopes.append({ value.id(), head, Pointer(carriedCharacters), Word32(carriedFlags) }); + }); + define(value, opened); + return; + } + Vector cells { cellWhenFlat }; + Vector impls { m_out.anchor(fiber.value()) }; m_out.jump(continuation); + auto join = [&](CellPointer cell) { + cells.append(m_out.anchor(cell.value())); + impls.append(m_out.anchor(loadField(cell, m_heaps.JSString_value).value())); + m_out.jump(continuation); + }; m_out.appendToArm(isRope, fork); - CellPointer flattened(callStub(value, flattenString(), { string.value() }).first); - ValueFromBlock cellWhenRope = m_out.anchor(flattened.value()); - ValueFromBlock implWhenRope = m_out.anchor(loadField(flattened, m_heaps.JSString_value).value()); - m_out.jump(continuation); + LBasicBlock isNotRead = value->isReadOperands().empty() ? nullptr : newBlock(); + for (ValueID isRead : value->isReadOperands()) { + LBasicBlock mayBeRead = newBlock(); + m_out.branch(in(isRead).value(), normalEdge(mayBeRead), normalEdge(isNotRead)); + m_out.appendTo(mayBeRead); + } + join(CellPointer(callStub(value, flattenString(), { string.value() }).first)); + if (isNotRead) { + m_out.appendTo(isNotRead); + join(emptyString()); + } m_out.appendToJoin(continuation, fork); - CellPointer flat(m_out.phi(machineTypeOf(CellPointer::representation), cellWhenFlat, cellWhenRope)); - Pointer impl(m_out.phi(machineTypeOf(Pointer::representation), implWhenFlat, implWhenRope)); + CellPointer flat(m_out.phi(machineTypeOf(CellPointer::representation), cells)); + Pointer impl(m_out.phi(machineTypeOf(Pointer::representation), impls)); m_openedStrings.add(flat.value(), OpenedString { impl, sourceInImpl(impl) }); define(value, flat); } @@ -554,19 +700,67 @@ void StringEmitter::generateStringRange(const ValueContext& value) defineResults(value, { range.start, range.end }); } -CellPointer StringEmitter::concatenate(const OperationContext& context, std::span strings) +StringID StringEmitter::findNumberedStringConstant(const OperationContext& context, ValueID id) { - if (strings.empty()) - return emptyString(); + if (opcodeOf(id) != Opcode::String) + return { }; + StringID constant = operationAt(id).string(); + return emitter().loadsStringConstantByStub(context, constant) ? constant : StringID(); +} - CellPointer result = strings[0]; - for (size_t i = 1; i < strings.size();) { - Vector operands { result.value(), strings[i++].value() }; - if (i < strings.size()) - operands.append(strings[i++].value()); - result = CellPointer(callStub(context, AOT::concatenate(static_cast(operands.size())), operands.span()).first); +CellPointer StringEmitter::cellOf(const OperationContext& context, const StringPart& part) +{ + return part.cell ? *part.cell : emitter().stringConstant(context, part.constant); +} + +static std::optional chooseConstantsOfConcatenation(std::span parts) +{ + auto isConstant = [&](size_t index) { + return !parts[index].cell; + }; + if (parts.size() == 2) { + if (isConstant(0)) + return ConstantsOfConcatenation::FirstOfTwo; + return isConstant(1) ? std::optional(ConstantsOfConcatenation::SecondOfTwo) : std::nullopt; + } + if (isConstant(0)) + return isConstant(2) ? ConstantsOfConcatenation::FirstAndThirdOfThree : ConstantsOfConcatenation::FirstOfThree; + if (isConstant(2)) + return ConstantsOfConcatenation::ThirdOfThree; + return isConstant(1) ? std::optional(ConstantsOfConcatenation::SecondOfThree) : std::nullopt; +} + +CellPointer StringEmitter::concatenateTwoOrThree(const OperationContext& context, std::span parts) +{ + std::optional constants = chooseConstantsOfConcatenation(parts); + Vector operands; + for (unsigned index = 0; index < parts.size(); ++index) { + if (constants && isConstantOperand(*constants, index)) + operands.append(zeroExtended(numberOf(m_mentions.add(NumberSpace::StringCell, parts[index].constant))).value()); + else + operands.append(cellOf(context, parts[index]).value()); + } + StubKey stub = constants ? concatenateWithConstants(*constants) : AOT::concatenate(static_cast(parts.size())); + return CellPointer(callStub(context, stub, operands.span()).first); +} + +CellPointer StringEmitter::concatenate(const OperationContext& context, std::span parts) +{ + if (parts.empty()) + return emptyString(); + if (parts.size() == 1) + return cellOf(context, parts[0]); + + bool beginsWithStringTwice = parts[0].cell && parts[1].cell && parts[0].cell->value() == parts[1].cell->value(); + size_t numberOfPartsJoined = std::min(parts.size(), beginsWithStringTwice ? 2 : 3); + CellPointer joined = concatenateTwoOrThree(context, parts.first(numberOfPartsJoined)); + while (numberOfPartsJoined < parts.size()) { + Vector next { joined, parts[numberOfPartsJoined++] }; + if (numberOfPartsJoined < parts.size()) + next.append(parts[numberOfPartsJoined++]); + joined = concatenateTwoOrThree(context, next.span()); } - return result; + return joined; } CellPointer StringEmitter::concatenateKey(const OperationContext& context, std::span primitives, KeyUse keyUse) diff --git a/Source/JavaScriptCore/aot/codegen/AOTStringEmitter.h b/Source/JavaScriptCore/aot/codegen/AOTStringEmitter.h index 45145054aef41..27c1c7c6c2abf 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTStringEmitter.h +++ b/Source/JavaScriptCore/aot/codegen/AOTStringEmitter.h @@ -22,6 +22,8 @@ struct CharacterView { Word32 length; Condition is8Bit; CellPointer owner; + ValueID possibleRope { }; + Word32 implFlags { }; }; struct CharacterSource { @@ -112,8 +114,11 @@ class StringEmitter final : public OperationEmitter { std::optional flatStringByType(ValueID string); Pointer implOfFlatString(FlatStringCell); CharacterView viewOfString(const OperationContext&, CellPointer string); + CharacterView viewWithoutResolving(const OperationContext&, CellPointer string, BlockEdge otherwise); Word32 codeUnitAt(const CharacterView&, Word32 index, std::optional indexInWholeNumber = std::nullopt); + Word32 codeUnitOfPossibleRope(const CharacterView&, Word32 index); + void passOpenedRope(ValueID string, BlockID from, LValue characters, LValue implFlags); Word32 codePointAt(const CharacterView&, Word32 index); CellPointer stringFromCodeUnit(const OperationContext&, Word32 codeUnit); CellPointer substring(const OperationContext&, CellPointer string, CodeUnitRange); @@ -138,8 +143,22 @@ class StringEmitter final : public OperationEmitter { Word32 find(const CharacterView& subject, const CharacterView& sought, Word32 position, SearchDirection); Word32 findCodeUnit(const CharacterView& subject, Word32 codeUnit, Word32 position, SearchDirection); - CellPointer concatenate(const OperationContext&, std::span); - CellPointer concatenate(const OperationContext& context, std::initializer_list strings) { return concatenate(context, std::span(strings.begin(), strings.size())); } + struct StringPart { + StringPart(CellPointer cell) + : cell(cell) + { + } + StringPart(StringID constant) + : constant(constant) + { + } + + std::optional cell { }; + StringID constant { }; + }; + StringID findNumberedStringConstant(const OperationContext&, ValueID); + CellPointer concatenate(const OperationContext&, std::span); + CellPointer concatenate(const OperationContext& context, std::initializer_list parts) { return concatenate(context, std::span(parts.begin(), parts.size())); } enum class KeyUse : uint8_t { AnyKey, NameOfProperty }; CellPointer concatenateKey(const OperationContext&, std::span primitives, KeyUse); CellPointer integerToString(const OperationContext& context, Word32 value) { return integerToString(context, signExtended(value)); } @@ -149,6 +168,8 @@ class StringEmitter final : public OperationEmitter { private: static constexpr unsigned maxLengthComparedInPlace = maxLengthOfTextComparedInPlace; + CellPointer cellOf(const OperationContext&, const StringPart&); + CellPointer concatenateTwoOrThree(const OperationContext&, std::span); Word32 flagsInImpl(Pointer impl); CharacterSource sourceInImpl(Pointer impl); CharacterSource sourceOfRope(const OperationContext&, CellPointer rope, Pointer fiber); @@ -172,7 +193,16 @@ class StringEmitter final : public OperationEmitter { struct OpenedString { Pointer impl; CharacterSource source; + bool mayBeRope { false }; + }; + struct OpenedRope { + ValueID string; + BlockID block; + Pointer characters; + Word32 implFlags; }; + const OpenedRope& openedRopeAt(ValueID string, BlockID) const; + Vector m_openedRopes; UncheckedKeyHashMap m_openedStrings; Vector m_flagsFoundNarrow; Vector m_flagsFoundWide; diff --git a/Source/JavaScriptCore/aot/codegen/AOTStringEmitterComparisons.cpp b/Source/JavaScriptCore/aot/codegen/AOTStringEmitterComparisons.cpp index e8e22ca97a6fa..fc115126fa1d4 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTStringEmitterComparisons.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTStringEmitterComparisons.cpp @@ -396,8 +396,12 @@ void StringEmitter::generateFindLiteral(const ValueContext& value, SearchDirecti Word32 position = word32Of(value->positionOperand()); if (isForward && !position.value()->isInt32(0)) holdRangeInside(value, position, position); + bool readsRange = !!value->startOperand(); + auto viewOfSubject = [&] { + return readsRange ? viewOfRange(value) : viewOf(value, string); + }; if (!isForward && position.value()->isInt32(0) && canCompareInline(literal)) { - CharacterView view = viewOf(value, string); + CharacterView view = viewOfSubject(); Word32 length = word32(text.length()); Condition isAtStart = asBoolean([&](BranchTargets targets) { continueIf(aboveOrEqual(view.length, length), targets.notTaken); @@ -413,11 +417,12 @@ void StringEmitter::generateFindLiteral(const ValueContext& value, SearchDirecti return low32(callOperation(value, operation, { in(string), numberOfText, position })); }; if (!text.is8Bit()) { + RELEASE_ASSERT(!readsRange); define(value, findInCell()); return; } - CharacterView view = viewOf(value, string); + CharacterView view = viewOfSubject(); if (!isForward && text.length() == 1) { Word32 codeUnit = word32(text[0]); define(value, findCodeUnit(view, codeUnit, select(lessThan(position, 0), 0, position), direction)); @@ -425,9 +430,10 @@ void StringEmitter::generateFindLiteral(const ValueContext& value, SearchDirecti } Word32 found = selectByWidth(view.is8Bit, [&](bool is8Bit) { - if (!is8Bit) + if (!is8Bit && !readsRange) return findInCell(); - RuntimeOperation operation = isForward ? RuntimeOperation::StringFindLiteral8 : RuntimeOperation::StringReverseFindLiteral8; + RuntimeOperation operation = isForward ? (is8Bit ? RuntimeOperation::StringFindLiteral8 : RuntimeOperation::StringFindLiteral16) + : (is8Bit ? RuntimeOperation::StringReverseFindLiteral8 : RuntimeOperation::StringReverseFindLiteral16); return low32(callWithoutSite(operation, { view.characters, view.length, numberOfText, position })); }); ensureStillAliveHere(view.owner); @@ -441,23 +447,22 @@ void StringEmitter::generateStringIndexOfCharacter(const ValueContext& value) Word32 position = word32Of(value->positionOperand()); if (!position.value()->isInt32(0)) holdRangeInside(value, position, position); + if (value->startOperand()) { + define(value, findCodeUnit(viewOfRange(value), codeUnit, position, SearchDirection::Forward)); + return; + } if (typeOf(string).isOpaque(OpaqueKind::FlatString)) { define(value, findCodeUnit(viewOf(value, string), codeUnit, position, SearchDirection::Forward)); return; } CellPointer cell = in(string); - LBasicBlock flat = newBlock(); LBasicBlock rope = newBlock(); Join join(m_out); - Condition isRope = isRopeString(cell); ScopeFork fork = m_out.forkScope(); - m_out.branch(isRope.value(), unlikelyEdge(rope), normalEdge(flat)); - - m_out.appendToArm(flat, fork); - join.add(findCodeUnit(viewOfFlatString(cell), codeUnit, position, SearchDirection::Forward)); + join.add(findCodeUnit(viewWithoutResolving(value, cell, unlikelyEdge(rope)), codeUnit, position, SearchDirection::Forward)); - m_out.appendToArm(rope, fork); + m_out.appendTo(rope); join.add(low32(callOperation(value, RuntimeOperation::StringIndexOfCharacter, { cell, codeUnit, position }))); define(value, join.finishBehind(fork)); } diff --git a/Source/JavaScriptCore/aot/codegen/AOTStringSwitch.cpp b/Source/JavaScriptCore/aot/codegen/AOTStringSwitch.cpp index 54785d138e1ed..4d88253c00ea3 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTStringSwitch.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTStringSwitch.cpp @@ -253,9 +253,8 @@ class StringSwitchEmitter final : public OperationEmitter { } m_strings.continueIf(belowOrEqual(sub(m_strings.lengthOf(string), word32(static_cast(shortest))), word32(static_cast(longest - shortest))), m_otherwise); } - Pointer answer(callStub(m_terminator, narrowCharactersOfString(), { string.value() }).first); - Word32 lengthWrittenDown(loadBytes(m_out, sizeof(uint32_t), m_out.address(m_heaps.characters8.atAnyIndex(), answer.value(), OBJECT_OFFSETOF(NarrowCharactersOfString, length)))); - goOnWith(Pointer(m_out.add(answer.value(), m_out.constIntPtr(OBJECT_OFFSETOF(NarrowCharactersOfString, characters)))), lengthWrittenDown); + StubCallResult narrow = callStub(m_terminator, narrowCharactersOfString(), { string.value() }); + goOnWith(Pointer(narrow.first), low32(Integer(narrow.second))); m_out.appendToJoin(hasCharacters, fork); switchOverLengthGroups(string, { Pointer(characters), Word32(length), boolean(true), string }, cases); diff --git a/Source/JavaScriptCore/aot/codegen/AOTTypeTestEmitter.cpp b/Source/JavaScriptCore/aot/codegen/AOTTypeTestEmitter.cpp index 67692ed461318..241391380d170 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTTypeTestEmitter.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTTypeTestEmitter.cpp @@ -524,6 +524,8 @@ Condition TypeTestEmitter::passesByMarker(Operand operand, const TypeTestPlan& p Condition disagrees; if (!agreeing) disagrees = isMarker(operand); + else if (operand.representation == Representation::OptionalDouble && agreeing == Primitive::Undefined && disagreeing == (testedMarkerPrimitives & ~Primitive::Undefined)) + disagrees = emitter().isNullOrAbsent(operand.as()); else { for (uint16_t primitive : testedMarkerKinds) { if (disagreeing & primitive) @@ -556,7 +558,7 @@ Condition TypeTestEmitter::doublePasses(Number number, const TypeTestPlan& plan) Condition otherNumberPasses = boolean(true); if (failing & Primitive::Double) otherNumberPasses = passing & Primitive::FractionalNumber ? isFractional(m_out, number) : logicalNot(isFractional(m_out, number)); - Integer truncated = emitter().toInteger(number); + Integer truncated = emitter().truncateUnchecked(number); Condition isInteger = isExactIntegerIn(number, truncated, Range::full()); return select(isInteger, integerPasses(truncated, plan), otherNumberPasses); } @@ -591,6 +593,7 @@ Condition TypeTestEmitter::isInRange(Integer value, Range range) Condition TypeTestEmitter::isExactIntegerIn(Number number, Integer truncated, Range range) { + ASSERT(range.isSubsetOf(Range::full())); ConversionEmitter& conversions = emitter(); if (range == Range::int32()) { Integer bitsOfLowHalf = conversions.bitsOf(toNumber(low32(truncated))); @@ -617,10 +620,10 @@ Condition TypeTestEmitter::isIntegerIn(Number number, Range range) Condition isExactInt32(patchpoint); if (Range::int32().isSubsetOf(range)) return isExactInt32; - return logicalAnd(isExactInt32, isInRange(conversions.toInteger(number), range)); + return logicalAnd(isExactInt32, isInRange(conversions.truncateUnchecked(number), range)); } #endif - return isExactIntegerIn(number, conversions.toInteger(number), range); + return isExactIntegerIn(number, conversions.truncateUnchecked(number), range); } } } } // namespace JSC::AOT::Typed diff --git a/Source/JavaScriptCore/aot/driver/AOTResultSections.cpp b/Source/JavaScriptCore/aot/driver/AOTResultSections.cpp index 3038edc9d2922..516a7bc34d720 100644 --- a/Source/JavaScriptCore/aot/driver/AOTResultSections.cpp +++ b/Source/JavaScriptCore/aot/driver/AOTResultSections.cpp @@ -180,12 +180,10 @@ static void writeWorkerSections(const Typed::Program& program, const Typed::Fact startsWithoutName.append(Wire::WorkerStartRecord { .witness = start }); setRecordsOfResultSection(output, Wire::ResultSection::WorkerStartsWithoutName, startsWithoutName); - const Vector& workerURLs = facts.tables().program.workerURLs; - Vector urls; - urls.reserveInitialCapacity(workerURLs.size()); - for (const Wire::Witness& witness : workerURLs) - urls.append(Wire::WorkerURLRecord { .witness = witness }); - setRecordsOfResultSection(output, Wire::ResultSection::WorkerURLs, urls); + Vector startsFromURLs; + for (const Wire::Witness& start : facts.tables().program.workerStartsFromURLs) + startsFromURLs.append(Wire::WorkerURLStartRecord { .witness = start }); + setRecordsOfResultSection(output, Wire::ResultSection::WorkerStartsFromURLs, startsFromURLs); } static void writeLackedFunctionSection(const Typed::Facts& facts, CompileOutput& output) diff --git a/Source/JavaScriptCore/aot/facts/AOTEnforcedFields.cpp b/Source/JavaScriptCore/aot/facts/AOTEnforcedFields.cpp index 10d308df63323..1a5cd9e450aed 100644 --- a/Source/JavaScriptCore/aot/facts/AOTEnforcedFields.cpp +++ b/Source/JavaScriptCore/aot/facts/AOTEnforcedFields.cpp @@ -119,10 +119,11 @@ static TypeID typeWrittenInFamilyInEnforcedFields(const Program& program, TypeAr LayoutID declaring = declaredLayoutOf(program, layout); std::span views = program.viewsAbove(declaring); if (views.size() >= Program::fewestViewsIndexedByKey) { - for (const Program::FieldWrittenAbove& listed : program.fieldsWrittenAbove(declaring, key)) { - if (!add(listed.view, listed.field)) - return { }; - } + bool isAnswered = program.visitFieldsWrittenAbove(declaring, key, [&](Program::FieldOfView listed) { + return add(listed.view, listed.field); + }); + if (!isAnswered) + return { }; } else { for (LayoutID view : views) { if (std::optional field = program.at(view).findField(key); field && !add(view, *field)) diff --git a/Source/JavaScriptCore/aot/facts/AOTFacts.cpp b/Source/JavaScriptCore/aot/facts/AOTFacts.cpp index 92c763f980d14..f99fc57c3e9b4 100644 --- a/Source/JavaScriptCore/aot/facts/AOTFacts.cpp +++ b/Source/JavaScriptCore/aot/facts/AOTFacts.cpp @@ -30,6 +30,14 @@ namespace JSC { namespace AOT { namespace Typed { +bool instancesMayBeLocked(const Program& program, const LinkTables& linked, TypeID testedType) +{ + std::span layouts = program.types.listOf(program.at(testedType).layouts); + return layouts.empty() || std::ranges::any_of(layouts, [&](LayoutID id) { + return linked.layout(id).instancesMayBeLocked; + }); +} + ASCIILiteral nameOf(FactKind kind) { switch (kind) { @@ -685,6 +693,10 @@ static void fillMembersOfClassesForFacts(const Program& program, const ThinSumma } rows.firstRowOfOwner.append(static_cast(rows.rows.size())); + for (const ClassMember& member : rows.rows) { + if (member.key) + tables.keysOfMembersOfClass.set(member.key.index()); + } tables.placesOfMembersOfClassByKey.resize(rows.rows.size()); for (uint32_t layout = 0; layout < numberOfLayouts; ++layout) { std::span members = rows.of(layout); @@ -698,14 +710,24 @@ static void fillMembersOfClassesForFacts(const Program& program, const ThinSumma std::span Facts::placesOfMembersUnder(LayoutID classLayout, ClassSide side, PropertyKeyID key) const { + if (key && !m_tables.keysOfMembersOfClass.get(key.index())) + return { }; std::span members = membersOf(classLayout); if (members.empty()) return { }; std::span places = m_tables.placesOfMembersOfClassByKey.span().subspan(m_tables.membersOfClass.firstRowOfOwner[classLayout.index()], members.size()); - auto found = std::ranges::equal_range(places, std::pair { side, key }, { }, [&](uint32_t place) { - return std::pair { members[place].side, members[place].key }; - }); - return { found.begin(), found.end() }; + auto wordOf = [](ClassSide sideOfMember, PropertyKeyID keyOfMember) { + return static_cast(sideOfMember) << 32 | keyOfMember.bits(); + }; + auto wordOfPlace = [&](uint32_t place) { + return wordOf(members[place].side, members[place].key); + }; + const uint64_t word = wordOf(side, key); + auto first = std::ranges::lower_bound(places, word, { }, wordOfPlace); + auto end = first; + while (end != places.end() && wordOfPlace(*end) == word) + ++end; + return { first, end }; } BlockID blockWhereListsOfArgumentsAreFirstNamed(const Program& program, const BodyIndexTable& bodies, FunctionID id) @@ -892,7 +914,9 @@ FunctionID Facts::functionThatCallEntersByBinding(const BodyIndex& index, const bool Facts::testMentionsWhatIsOutOfReach(TypeReader types, TypeID tested, NOESCAPE const ScopedLambda& isInReach) const { - Type type = types.at(tested); + if (holdsOnlyPrimitives(tested)) + return false; + const Type& type = types.recordOf(tested); if (!type.hasObjects()) return false; bool mentionsIt = types.at(type.layouts).anyOfWithin(m_tables.firstLayoutOfNamespace, m_tables.endOfLayoutsOfNamespaces, [&](LayoutID layout) { diff --git a/Source/JavaScriptCore/aot/facts/AOTFacts.h b/Source/JavaScriptCore/aot/facts/AOTFacts.h index 1911a5d62a34a..ab3bf4905dc52 100644 --- a/Source/JavaScriptCore/aot/facts/AOTFacts.h +++ b/Source/JavaScriptCore/aot/facts/AOTFacts.h @@ -77,7 +77,7 @@ struct ProgramFactTables { Vector workerEntries; Vector workerStartsWithoutName; - Vector workerURLs; + Vector workerStartsFromURLs; Vector missingLibraryFunctions; Vector missingLibraryModules; Vector regExpTextsInSight; @@ -125,6 +125,7 @@ struct FactTables { BitVector functionsTakenToRunOnce; FactRows membersOfClass; Vector placesOfMembersOfClassByKey; + BitVector keysOfMembersOfClass; BitVector layoutsThatParsersMayMake; LayoutSetID layoutsThatStayWithCompiledCode; diff --git a/Source/JavaScriptCore/aot/facts/AOTFactsOfProgram.cpp b/Source/JavaScriptCore/aot/facts/AOTFactsOfProgram.cpp index 22c5a2f2b6ddc..90134250de27b 100644 --- a/Source/JavaScriptCore/aot/facts/AOTFactsOfProgram.cpp +++ b/Source/JavaScriptCore/aot/facts/AOTFactsOfProgram.cpp @@ -405,7 +405,7 @@ void RowsOfProgramFacts::listSpecifiersOfLibrary(const Program& program) } struct FindingOfProgramFacts { - enum class Kind : uint8_t { UseOfBuiltin, ReadOfAnyMember, WriteOnBuiltin, PrototypeChangeOfBuiltin, NameReadOffUnknownValue, ReadThatNoRowAnswers, PartOfClassIsUsed, KeysOfBuiltinListed, OptionThatRunsCode, InstanceThatMayAnswerEveryName, UnseenCode, WorkerEntry, WorkerStartWithoutName, WorkerURL, URLThatLeavesBody, NamedModule, LoaderHandedToText, CallOfEvent, BuiltinSpelledInText, LightFormIsAskedForMore, CallUnderComputedKey, StandsAboveClass, NamespaceIsGivenByImportCall }; + enum class Kind : uint8_t { UseOfBuiltin, ReadOfAnyMember, WriteOnBuiltin, PrototypeChangeOfBuiltin, NameReadOffUnknownValue, ReadThatNoRowAnswers, PartOfClassIsUsed, KeysOfBuiltinListed, OptionThatRunsCode, InstanceThatMayAnswerEveryName, UnseenCode, WorkerEntry, WorkerStartWithoutName, WorkerStartFromURL, NamedModule, LoaderHandedToText, CallOfEvent, BuiltinSpelledInText, LightFormIsAskedForMore, CallUnderComputedKey, StandsAboveClass, NamespaceIsGivenByImportCall }; Kind kind { Kind::UseOfBuiltin }; uint8_t detail { 0 }; @@ -571,7 +571,7 @@ class BodyLookOfProgramFacts { BuiltinID builtinSurelyAt(ValueID value) const { return value && !m_mayBeSomethingElse.get(value.index()) ? m_builtinOfValue[value.index()] : BuiltinID(); } BuiltinID builtinHeldBy(VariableID variable) const { return variable.index() < m_builtinsHeldByVariables.size() ? m_builtinsHeldByVariables[variable.index()] : BuiltinID(); } ValueID originOfURL(ValueID value) const { return value && !m_originOfURLOfValue.isEmpty() ? m_originOfURLOfValue[value.index()] : ValueID(); } - ValueID originOfURLOrOfItsText(ValueID) const; + bool isURLOrItsText(ValueID) const; BuiltinID holderOfMethodServed(ValueID) const; BuiltinID classOfPartAt(ValueID value) const { return value && !m_classOfPartOfValue.isEmpty() ? m_classOfPartOfValue[value.index()] : BuiltinID(); } BuiltinID builtinThatLinkLedToAt(ValueID value) const { return value && !m_builtinThatLinkLedToOfValue.isEmpty() ? m_builtinThatLinkLedToOfValue[value.index()] : BuiltinID(); } @@ -788,10 +788,6 @@ void BodyLookOfProgramFacts::noteFate(ValueID value, Fate fate, const auto& plac note(Finding::Kind::PartOfClassIsUsed, 0, objectOfClass.index(), witnessAt(place)); if (BuiltinID above = builtinThatLinkLedToAt(value); above && fate != Fate::IsInspected) noteUse(above, BuiltinUse::Escaped, place); - if (fate != Fate::IsLetGo && fate != Fate::IsFollowedByLink && fate != Fate::IsAskedForAnyMember) - return; - if (ValueID origin = originOfURL(value)) - note(Finding::Kind::URLThatLeavesBody, 0, Wire::none, witnessAt(m_index.value(origin))); } void BodyLookOfProgramFacts::handOn(ValueID from, ValueID to) @@ -1333,18 +1329,18 @@ BodyLookOfProgramFacts::Fate BodyLookOfProgramFacts::fateOf(const BodyValue& use return fateAt(BuiltinOperand::argument(argument)); } -ValueID BodyLookOfProgramFacts::originOfURLOrOfItsText(ValueID value) const +bool BodyLookOfProgramFacts::isURLOrItsText(ValueID value) const { - if (ValueID origin = originOfURL(value)) - return origin; + if (originOfURL(value)) + return true; BodyValue read = stripCasts(value); if (!m_rows.href || read.opcode() != Opcode::PropGet || builtinMemberNameOf(m_program, read.as().key()).name != m_rows.href) - return { }; + return false; ValueID origin = originOfURL(read.operand(0)); if (!origin || !callsThatSpellOutArgumentsInProgramFacts.contains(m_index.opcodeOf(origin))) - return { }; + return false; std::span written = m_index.value(origin).variadicOperands(); - return !written.empty() && stripCasts(written[0]).opcode() == Opcode::String ? origin : ValueID(); + return !written.empty() && stripCasts(written[0]).opcode() == Opcode::String; } void BodyLookOfProgramFacts::noteConstructionWhoseOperandsNobodySees(BuiltinID row, const Witness& witness) @@ -1365,7 +1361,7 @@ void BodyLookOfProgramFacts::noteStartOfWorker(const BodyValue& call, bool spell return note(Finding::Kind::WorkerStartWithoutName, 0, Wire::none, witnessAt(call)); bool mayHaveOptions = !m_rows.fileArguments; - bool namesFile = false; + bool isFromURL = false; bool mayGetUnknownFile = false; Vector paths; std::span arguments = call.variadicOperands(); @@ -1379,16 +1375,17 @@ void BodyLookOfProgramFacts::noteStartOfWorker(const BodyValue& call, bool spell } if (opcode == Opcode::String) paths.append(passed.as().string()); - else if (ValueID origin = originOfURLOrOfItsText(arguments[argument])) { - namesFile = true; - note(Finding::Kind::WorkerURL, 0, Wire::none, witnessAt(m_index.value(origin))); - } else + else if (isURLOrItsText(arguments[argument])) + isFromURL = true; + else mayGetUnknownFile |= !isNullish; } for (StringID path : paths) note(Finding::Kind::WorkerEntry, mayHaveOptions, path.index(), witnessAt(call)); - if (mayGetUnknownFile || (paths.isEmpty() && !namesFile)) + if (mayGetUnknownFile || (paths.isEmpty() && !isFromURL)) note(Finding::Kind::WorkerStartWithoutName, 0, Wire::none, witnessAt(call)); + else if (isFromURL) + note(Finding::Kind::WorkerStartFromURL, 0, Wire::none, witnessAt(call)); } bool BodyLookOfProgramFacts::listKeysOfLiteral(ValueID value, Vector& keys) const @@ -3096,8 +3093,7 @@ void WritesOnBuiltinsInProgramFacts::take(const FindingOfProgramFacts& finding) case Kind::BuiltinSpelledInText: case Kind::WorkerEntry: case Kind::WorkerStartWithoutName: - case Kind::WorkerURL: - case Kind::URLThatLeavesBody: + case Kind::WorkerStartFromURL: case Kind::NamedModule: case Kind::LoaderHandedToText: case Kind::LightFormIsAskedForMore: @@ -3557,7 +3553,6 @@ class BuilderOfProgramFacts { std::optional m_codeThatNobodyNamesMayLeave; BitVector m_partOfClassIsUsed; Vector m_callsOfEventsNamedInBodies; - Vector m_urlsThatLeaveBodies; Vector m_whereProgramReadsNames; Vector m_callsUnderComputedKeys; Witness m_whereNameOfRunIsCalled { noWitness }; @@ -3899,11 +3894,8 @@ void BuilderOfProgramFacts::take(std::span findings) case Finding::Kind::LoaderHandedToText: m_loaderIsHandedToText = true; break; - case Finding::Kind::WorkerURL: - m_program.workerURLs.append(finding.witness); - break; - case Finding::Kind::URLThatLeavesBody: - m_urlsThatLeaveBodies.append(finding.witness); + case Finding::Kind::WorkerStartFromURL: + m_program.workerStartsFromURLs.append(finding.witness); break; case Finding::Kind::NamedModule: if (finding.detail) { @@ -4191,7 +4183,7 @@ void BuilderOfProgramFacts::noteWhatRowsMake(BuiltinID canonical) bool mayPrintWhatNobodyKnows = mayBeHandedWhatNobodyKnows && std::ranges::any_of(overload.parameters, [&](const BuiltinParameter& parameter) { return m_builtins.isPrintedByFormatter(canonical, parameter); }); - bool mayDescribeWhatNobodyKnows = mayBeHandedWhatNobodyKnows && (m_instancesLoseTheirConstructor || unseenCodeCanRun()) && std::ranges::any_of(overload.parameters, [](const BuiltinParameter& parameter) { + bool mayDescribeWhatNobodyKnows = mayBeHandedWhatNobodyKnows && !m_builtins.isUnder(canonical, Builtins::NativeFunctions) && (m_instancesLoseTheirConstructor || unseenCodeCanRun()) && std::ranges::any_of(overload.parameters, [](const BuiltinParameter& parameter) { return parameter.roles.contains(BuiltinOperandRole::DescribedWhenRefused); }); if (mayPrintWhatNobodyKnows || mayDescribeWhatNobodyKnows) @@ -6013,14 +6005,12 @@ void BuilderOfProgramFacts::listWorkers() } entries.shrink(kept); - Vector& urls = m_program.workerURLs; - if (isNamedByOperation) - urls.appendVector(m_urlsThatLeaveBodies); + Vector& startsFromURLs = m_program.workerStartsFromURLs; auto placeOf = [](const Witness& witness) { return std::tuple(witness.function, witness.value); }; - AOT::sort(urls, { }, placeOf); - urls.shrink(std::ranges::unique(urls, { }, placeOf).begin() - urls.begin()); + AOT::sort(startsFromURLs, { }, placeOf); + startsFromURLs.shrink(std::ranges::unique(startsFromURLs, { }, placeOf).begin() - startsFromURLs.begin()); } bool BuilderOfProgramFacts::needsOneInstanceAtMost() const diff --git a/Source/JavaScriptCore/aot/facts/AOTFactsRegExpTexts.cpp b/Source/JavaScriptCore/aot/facts/AOTFactsRegExpTexts.cpp index cc2578c26fd15..478c7fe87ea15 100644 --- a/Source/JavaScriptCore/aot/facts/AOTFactsRegExpTexts.cpp +++ b/Source/JavaScriptCore/aot/facts/AOTFactsRegExpTexts.cpp @@ -1013,13 +1013,19 @@ JSValue RehearsalOfRegExpTexts::namespaceOf(ModuleID id) JSObject* made = constructEmptyObject(m_realm.get()); uint64_t numberOfExports = module.exports.size() + module.syntheticExports.size(); payForPlacesMadeOutOfCharacters(numberOfExports, numberOfExports); + auto define = [&](StringID name, VariableID exported) { + JSValue value = variable(exported); + auto scope = DECLARE_TOP_EXCEPTION_SCOPE(vm); + made->putDirectMayBeIndex(m_realm.get(), Identifier::fromString(vm, charactersOf(name)), value); + return !hasThrown(scope); + }; for (const ModuleExport& exported : module.exports) { - if (exported.kind == ModuleExportKind::Variable && exported.variable) - made->putDirect(vm, Identifier::fromString(vm, charactersOf(exported.name)), variable(exported.variable)); + if (exported.kind == ModuleExportKind::Variable && exported.variable && !define(exported.name, exported.variable)) + return unknown(); } for (const SyntheticExport& synthetic : module.syntheticExports) { - if (synthetic.variable) - made->putDirect(vm, Identifier::fromString(vm, charactersOf(synthetic.name)), variable(synthetic.variable)); + if (synthetic.variable && !define(synthetic.name, synthetic.variable)) + return unknown(); } if (m_statesOfModules[id.index()] == StateOfModule::HasRun) m_namespaces[id.index()] = made; diff --git a/Source/JavaScriptCore/aot/facts/AOTLink.cpp b/Source/JavaScriptCore/aot/facts/AOTLink.cpp index 65f1eefb1c113..9779de6f6fef5 100644 --- a/Source/JavaScriptCore/aot/facts/AOTLink.cpp +++ b/Source/JavaScriptCore/aot/facts/AOTLink.cpp @@ -23,6 +23,7 @@ #include "AOTPhaseTimes.h" #include "AOTStronglyConnectedComponents.h" #include +#include #include #include #include @@ -647,6 +648,7 @@ class WriterOfLink { void writeCallsThatFindBuiltins(); void noteCallersThatFindBuiltins(); void writeResultsOfReadsByKey(); + void writeNativeCallResults(); void writeContentsAtOperations(); void writeReceiversThatStayAsMade(); void writeReadsOffViewsOfWrittenLengths(); @@ -801,6 +803,7 @@ void WriterOfLink::writeWhatMakesTypes(TypeArena& types) writeCoroutinesOfLink(m_program, m_flow, types, m_tables); writeCatches(); writeResultsOfReadsByKey(); + writeNativeCallResults(); writeContentsAtOperations(); } @@ -2002,6 +2005,54 @@ void WriterOfLink::writeResultsOfReadsByKey() } } +void WriterOfLink::writeNativeCallResults() +{ + if (writesWhatIsExpected()) + return; + const BuiltinTable& builtins = m_program.builtins(); + m_tables.nativeCallResults = collectInParallel(loops(), m_numberOfFunctions, [&](unsigned index, Vector& calls) { + if (!m_flow.reached.get(index)) + return; + FunctionID caller(index); + const ThinSummary& summary = m_flow.summaries.at(caller); + uint32_t firstCall = m_flow.firstCallOfFunction[index]; + BitVector nativeCalls; + Vector sites; + for (uint32_t number = 0; number < summary.calls.size(); ++number) { + const LinkFlow::CalleesOfCall& callees = m_flow.calleesOfCallsAsSettled()[firstCall + number]; + if (callees.builtins().isEmpty()) + continue; + const SummaryCall& call = summary.calls[number]; + bool hasOtherCallees = !callees.listed().isEmpty() || !callees.shared().isEmpty() || std::ranges::any_of(callees.builtins().span(), [&](BuiltinID callee) { + return builtins.at(callee).scope != BuiltinScope::Module; + }); + if (!hasOtherCallees && call.site && !call.isMadeByProtocol && !call.isConstruction && !call.passesList) + nativeCalls.set(number); + } + if (nativeCalls.isEmpty()) + return; + for (const SummaryCall& call : summary.calls) + sites.append(call.site); + std::ranges::sort(sites); + size_t first = calls.size(); + for (uint32_t node = 1; node < summary.nodes.size(); ++node) { + const SummaryNode& result = summary.nodes[node]; + if (result.kind != SummaryNode::Kind::ResultOfCall || result.second || !nativeCalls.get(result.first)) + continue; + ValueID site = summary.calls[result.first].site; + const LinkType& given = m_flow.typeAt(m_flow.nodeOf(caller, node)); + bool holdsObjects = given.objects != LinkType::Objects::None || given.functions != LinkType::Functions::None || given.classes != LinkType::Classes::None || given.builtin || given.hasUnknownObjects || given.holdsLostBuiltins; + if (given.isNever() || holdsObjects || std::ranges::equal_range(sites, site).size() != 1) + continue; + if (TypeID proven = typeIDOf(given); proven && holdsOnlyPrimitives(proven)) + calls.append(LinkedNativeCallResult { caller, site, proven }); + } + std::ranges::sort(calls.mutableSpan().subspan(first), { }, &LinkedNativeCallResult::site); + }); + for (const LinkedNativeCallResult& call : m_tables.nativeCallResults) + m_tables.nativeFunctionCallers.set(call.function.index()); +} + void WriterOfLink::writeContentsAtOperations() { struct Operation { @@ -2230,6 +2281,7 @@ ALWAYS_INLINE void WriterOfLink::writeFunction(FunctionID id) return; const Function& function = m_program.at(id); + linked.isIterationCallback = m_flow.iterationCallbacks.get(index); linked.everyCallerIsKnown = everyCallerIsKnownOf(index); linked.holdsInt32Promise = m_flow.summaries.at(id).has(SummaryTrait::HoldsInt32Promise); bool sharesNoCall = linked.everyCallerIsKnown && !m_functionsSharingCalls.get(index) && !m_functionsThatTextsOfLibraryName.get(index); @@ -2240,6 +2292,11 @@ ALWAYS_INLINE void WriterOfLink::writeFunction(FunctionID id) linked.result = typeIDOf(m_flow.typeAt(m_flow.result(id, 0))); if (std::optional established = function.results[0].established(); established && !linked.result) linked.result = unlessAnything(*established); + if (m_flow.summaries.at(id).has(SummaryTrait::ReturnsUInt32) && linked.result && holdsOnlyPrimitives(linked.result) && primitivesInID(linked.result) == Primitive::Integer) { + constexpr Range uint32 { 0, std::numeric_limits::max() }; + if (std::optional narrower = integersInID(linked.result).isSubsetOf(uint32) ? std::nullopt : tryMakeTypeOfPrimitives(Primitive::Integer, uint32)) + linked.result = *narrower; + } } writeParameters(id, linked); } @@ -2278,6 +2335,8 @@ void WriterOfLink::findFunctionsThatRunAtMostOnce() uint8_t& number = numberOfCallsThatEnter[call.callee.index()]; number = std::min(number + 1, several); } + for (uint32_t index = 0; index < m_numberOfFunctions; ++index) + m_tables.functions[index].hasSingleCallSite = numberOfCallsThatEnter[index] == 1 && m_tables.functions[index].everyCallHasOneCallee; Vector functionsToFollow; for (size_t body : bodiesThatLoaderEntersAtMostOnce(m_program)) { @@ -3553,7 +3612,7 @@ static void stopWhereLinkFindsLessFromMore(Program& program, BuildContext& build const LinkedLayout& ofMore = givenMore.layouts[index]; if (!ofProgram.isAllocated) continue; - if (!ofMore.isAllocated || (ofProgram.reachesUnseenCode && !ofMore.reachesUnseenCode) || !ofMore.possibleChanges.containsAll(ofProgram.possibleChanges)) + if (!ofMore.isAllocated || (ofProgram.hasDirectInstances && !ofMore.hasDirectInstances) || (ofProgram.reachesUnseenCode && !ofMore.reachesUnseenCode) || !ofMore.possibleChanges.containsAll(ofProgram.possibleChanges)) note("layout "_s, index, ": what befalls its objects"_s); if (ofMore.onlyFunctionThatMakesObjects && ofMore.onlyFunctionThatMakesObjects != ofProgram.onlyFunctionThatMakesObjects) note("layout "_s, index, ": who makes its objects"_s); diff --git a/Source/JavaScriptCore/aot/facts/AOTLinkAccesses.cpp b/Source/JavaScriptCore/aot/facts/AOTLinkAccesses.cpp index 48c7660cf2a49..d912ea12ea777 100644 --- a/Source/JavaScriptCore/aot/facts/AOTLinkAccesses.cpp +++ b/Source/JavaScriptCore/aot/facts/AOTLinkAccesses.cpp @@ -26,6 +26,7 @@ #include "AOTResultRules.h" #include #include +#include #include #include @@ -176,11 +177,13 @@ class AccessInLinkAccesses { bool isRead() const { return m_access.kind == Kind::Read; } void noteSiteOfCopyThatSpellsStore(); + void layAccessorAsWrittenOnPrototypeOf(LayoutID); bool readsThroughSuperWhatIsOnlyCalled() const; bool isNamed() const { return !!m_access.key; } bool isTakenByLoader() const { return isRead() && m_function && isLibraryCode(m_program, m_program.at(m_function)) && std::ranges::contains(m_flow.summaries.at(m_function).namesThatLoaderTakes, m_access.value, &ThinSummary::NameThatLoaderTakes::read); } bool readKey(); bool readsOrStoresUnderTextsThatBodyWrites(); + bool isFinalListRead(); void give(const LinkType&); void giveSymbol(PropertyKeyID); @@ -190,6 +193,11 @@ class AccessInLinkAccesses { void giveWhatIsHeldAt(FlowNodeID); void giveWhatIsEstablishedOf(const DeclaredType&); void storeInto(FlowNodeID); + LayoutID classTestedByInstanceOf(); + bool isExcludedByInstanceOf(LayoutID); + LayoutID findClassRefusedByTest(); + LayoutID classRefusedByTest(); + bool isRefusedByTest(LayoutID); bool storesTextNumberOrBooleanThatIsWritten() const; bool isStoreByBodyOfFileInItsOwnText() const; bool storesWhatSetterOfPrototypeLinkPassesOver() const; @@ -198,7 +206,7 @@ class AccessInLinkAccesses { enum class UnknownIs : uint8_t { ByEscapeAlone, InSight }; enum class TellsName : uint8_t { No, Yes }; void cannotTell(UnknownIs = UnknownIs::InSight, TellsName = TellsName::Yes); - void cannotTellWhatRuns(); + void cannotTellWhatRuns(UnknownIs = UnknownIs::InSight); void noteChange(LayoutID layout, LayoutChange change) { @@ -213,6 +221,8 @@ class AccessInLinkAccesses { } template void forEachLayoutUpTheBases(LayoutID, const Functor&) const; + std::optional m_classTestedByInstanceOf; + std::optional m_classRefusedByTest; FlowNodeID extraPropertyReadNode(LayoutID id) { return m_flow.extraPropertyReadNode(id); } FlowNodeID foundBesideFieldsUnderKey() { return m_flow.foundBesideFieldsUnder(m_access.key); } void storeBesideFieldsOf(LayoutID); @@ -330,6 +340,7 @@ class AccessInLinkAccesses { bool m_keyCannotNameBuiltin { false }; bool m_keyIsNumberBooleanOrNullish { false }; bool m_keyIsSymbolAsFarAsIsSeen { false }; + bool m_seenKeysAreNumbers { false }; bool m_keyMayNamePrototypeLink { false }; bool m_hasThrown { false }; bool m_wasAskedToThrow { false }; @@ -442,6 +453,169 @@ void AccessInLinkAccesses::storeInto(FlowNodeID to) m_flow.addEdge(m_value, to); } +static ValueID valueBehindCastsAndProofs(const BodyIndex& index, ValueID value) +{ + while (true) { + ValueID stripped = index.stripCasts(value); + if (ValueID subject = index.valueNarrowedInto(stripped)) + stripped = subject; + if (stripped == value) + return value; + value = stripped; + } +} + +static LayoutID classNamedByBinding(const Program& program, const BodyIndex& index, ValueID value) +{ + while (!index.isParameter(value) && index.opcodeOf(value) == Opcode::CheckInitialized) + value = index.value(value).operand(0); + if (index.isParameter(value) || index.opcodeOf(value) != Opcode::GetModuleVariable) + return { }; + const Variable& variable = program.at(index.value(value).as().variable()); + if (!variable.flags.contains(VariableFlags::IsImmutable) || !variable.declared.isEstablished() || !variable.declared.written()) + return { }; + Type declared = program.at(variable.declared.written()); + declared.primitives &= ~Primitive::Absent; + FunctionID constructor = singleFunctionOf(TypeReader(program), declared); + return constructor && isClassConstructorOf(program.at(constructor).kind) ? program.at(constructor).constructedClass : LayoutID(); +} + +LinkFlow::ReadsBehindInstanceOf LinkFlow::findReadsBehindInstanceOf(const Program& program, const BodyIndex& index, const ThinSummary& summary) +{ + struct Place { + uint32_t subject { 0 }; + uint32_t dominatorNumber { 0 }; + uint32_t lastBelow { 0 }; + ValueID read; + ValueID test; + LayoutID tested; + }; + ReadsBehindInstanceOf found; + Vector places; + HashSet, WTF::UnsignedWithZeroKeyHashTraits> subjects; + for (uint32_t number = 0; number < index.numberOfBlocks(); ++number) { + BlockID block(number); + std::optional edge = index.isReachable(block) ? index.onlyIncomingEdge(block) : std::nullopt; + if (!edge || edge->targetIndex) + continue; + BodyTerminator end = index.terminator(edge->from); + if (end.kind() != TerminatorKind::Branch) + continue; + ValueID condition = end.as().conditionOperand(); + while (!index.isParameter(condition) && index.opcodeOf(condition) == Opcode::ToBoolean) + condition = index.value(condition).operand(0); + if (index.isParameter(condition) || index.opcodeOf(condition) != Opcode::InstanceOf) + continue; + auto test = index.value(condition).as(); + LayoutID tested = classNamedByBinding(program, index, test.constructorOperand()); + if (!tested) + continue; + uint32_t subject = valueBehindCastsAndProofs(index, test.valueOperand()).index(); + subjects.add(subject); + places.append(Place { subject, index.dominatorNumber(block), index.lastDominatorNumberBelow(block), { }, condition, tested }); + } + if (places.isEmpty()) + return found; + for (const SummaryAccess& access : summary.accesses) { + if (access.kind != SummaryAccess::Kind::Read || !access.key || access.site.index() >= index.numberOfValues() || index.isParameter(access.site) || index.opcodeOf(access.site) != Opcode::PropGet) + continue; + uint32_t subject = valueBehindCastsAndProofs(index, index.value(access.site).as().objectOperand()).index(); + if (subjects.contains(subject)) + places.append(Place { subject, index.dominatorNumber(index.blockOf(access.site)), 0, access.site, { }, { } }); + } + std::ranges::sort(places, [](const Place& first, const Place& second) { + return std::tuple(first.subject, first.dominatorNumber, !!first.read, first.read.bits()) < std::tuple(second.subject, second.dominatorNumber, !!second.read, second.read.bits()); + }); + Vector testsAround; + for (const Place& place : places) { + while (!testsAround.isEmpty() && (testsAround.last()->subject != place.subject || testsAround.last()->lastBelow < place.dominatorNumber)) + testsAround.removeLast(); + if (!place.read) + testsAround.append(&place); + else if (!testsAround.isEmpty()) + found.bySite.append(ReadBehindInstanceOf { place.read, testsAround.last()->test, testsAround.last()->tested }); + } + std::ranges::sort(found.bySite, { }, &ReadBehindInstanceOf::site); + for (uint32_t number = 0; number < summary.accesses.size(); ++number) { + const SummaryAccess& access = summary.accesses[number]; + auto read = std::ranges::lower_bound(found.bySite, access.site, { }, &ReadBehindInstanceOf::site); + if (access.kind == SummaryAccess::Kind::Read && access.value != noSummaryNode && read != found.bySite.end() && read->site == access.site) + found.accessesByTest.append({ read->test, number }); + } + std::ranges::sort(found.accessesByTest); + return found; +} + +LayoutID AccessInLinkAccesses::classTestedByInstanceOf() +{ + if (!m_classTestedByInstanceOf) { + m_classTestedByInstanceOf = LayoutID(); + if (const LinkFlow::ReadsBehindInstanceOf* reads = m_shared || !isRead() || !isNamed() ? nullptr : m_flow.readsBehindInstanceOf(m_function)) { + auto read = std::ranges::lower_bound(reads->bySite, m_access.site, { }, &LinkFlow::ReadBehindInstanceOf::site); + if (read != reads->bySite.end() && read->site == m_access.site) + m_classTestedByInstanceOf = read->tested; + } + } + return *m_classTestedByInstanceOf; +} + +bool AccessInLinkAccesses::isExcludedByInstanceOf(LayoutID id) +{ + if (m_program.at(id).kind != LayoutKind::Instance || m_program.at(id).has(LayoutFlags::IsOfCreatedObject)) + return false; + LayoutID tested = classTestedByInstanceOf(); + if (!tested) + return false; + size_t stepsLeft = m_program.layouts.size(); + for (LayoutID current = declaredLayoutOf(m_program, id); current && stepsLeft; --stepsLeft) { + const Layout& layout = m_program.at(current); + if (current == tested || !layout.constructor || !isClassConstructorOf(m_program.at(layout.constructor).kind) || (layout.has(LayoutFlags::HasHeritage) && !layout.base)) + return false; + current = layout.base; + } + return true; +} + +LayoutID AccessInLinkAccesses::findClassRefusedByTest() +{ + const BodyIndex* index = m_shared || isRead() || !isNamed() || m_program.at(m_function).kind != FunctionKind::DerivedConstructor ? nullptr : bodiesLookedAt(m_flow.looks).find(m_function); + if (!index || !index->opcodes().contains(Opcode::IsUnlockedInstance) || m_access.site.index() >= index->numberOfValues() || index->isParameter(m_access.site)) + return { }; + BodyValue store = index->value(m_access.site); + ValueID object = store.opcode() == Opcode::PropDefine ? store.as().objectOperand() : store.opcode() == Opcode::PropSet ? store.as().objectOperand() : ValueID(); + std::optional edge = object ? index->onlyIncomingEdge(index->blockOf(m_access.site)) : std::nullopt; + if (!edge || !edge->targetIndex) + return { }; + BodyTerminator end = index->terminator(edge->from); + if (end.kind() != TerminatorKind::Branch) + return { }; + ValueID condition = end.as().conditionOperand(); + if (index->isParameter(condition) || index->opcodeOf(condition) != Opcode::IsUnlockedInstance || index->stripCasts(index->value(condition).operand(0)) != index->stripCasts(object)) + return { }; + const Type& tested = m_program.at(index->value(condition).as().testedType()); + std::span layouts = m_program.types.listOf(tested.layouts); + if (layouts.size() != 1 || m_flow.instancesMayBeLocked(layouts[0])) + return { }; + FlowNodeID told = m_flow.whereInstancesMayBeLocked(layouts[0]); + FlowObserver observer { FlowObserver::Kind::MemberOfAccess, m_function, m_record }; + if (laysWatchAt(told, observer)) + m_flow.observe(told, observer); + return m_flow.instancesMayBeLocked(layouts[0]) ? LayoutID() : layouts[0]; +} + +LayoutID AccessInLinkAccesses::classRefusedByTest() +{ + if (!m_classRefusedByTest) + m_classRefusedByTest = findClassRefusedByTest(); + return *m_classRefusedByTest; +} + +bool AccessInLinkAccesses::isRefusedByTest(LayoutID id) +{ + LayoutID refused = m_program.at(id).kind == LayoutKind::Instance && !m_program.at(id).has(LayoutFlags::IsOfCreatedObject) ? classRefusedByTest() : LayoutID(); + return refused && m_flow.standsAtOrBelow(declaredLayoutOf(m_program, id), refused); +} + bool AccessInLinkAccesses::isStoreByBodyOfFileInItsOwnText() const { const Function& whoStores = m_program.at(m_function); @@ -717,10 +891,10 @@ void AccessInLinkAccesses::cannotTell(UnknownIs unknownIs, TellsName tellsName) m_flow.addEdge(m_receiver, m_flow.unseen()); } -void AccessInLinkAccesses::cannotTellWhatRuns() +void AccessInLinkAccesses::cannotTellWhatRuns(UnknownIs unknownIs) { m_flow.addEdge(m_subject, m_flow.unseen()); - cannotTell(); + cannotTell(unknownIs); } template @@ -806,6 +980,8 @@ bool AccessInLinkAccesses::readKey() m_keyCannotNameBuiltin = !holdsObjects && !(key.primitives & LinkType::closed(Primitive::AnyString | Primitive::AnySymbol)); m_keyIsNumberBooleanOrNullish = !holdsObjects && !(key.primitives & ~LinkType::closed(Primitive::Number | Primitive::Boolean | Primitive::UndefinedOrNull)); m_keyIsSymbolAsFarAsIsSeen = hasNode && LinkFlow::isSymbolAsFarAsIsSeen(key); + LinkType seenKeys = key.directPart(); + m_seenKeysAreNumbers = !holdsObjectsForLinkAccesses(seenKeys) && !(seenKeys.primitives & ~LinkType::closed(Primitive::Number)); bool keyIsTextOrNumber = !holdsObjects && !(key.primitives & LinkType::closed(Primitive::AnySymbol)); m_keyMayNamePrototypeLink = (holdsObjects || key.primitives & Primitive::AnyString) && keyThatBeginsSoMayNamePrototypeLink(m_program, m_access) && !(keyIsTextOrNumber && storesWhatSetterOfPrototypeLinkPassesOver()); return true; @@ -928,6 +1104,7 @@ bool AccessInLinkAccesses::readsOrStoresUnderTextsThatBodyWrites() underName.m_keyIsOnlyNumbers = false; underName.m_keyCannotNameBuiltin = false; underName.m_keyIsSymbolAsFarAsIsSeen = false; + underName.m_seenKeysAreNumbers = false; underName.m_keyMayNamePrototypeLink = false; underName.m_before = { }; underName.resolve(); @@ -935,6 +1112,117 @@ bool AccessInLinkAccesses::readsOrStoresUnderTextsThatBodyWrites() return true; } +static void collectListAliases(const BodyIndex& index, BitVector& aliases) +{ + bool grows = true; + auto unite = [&](ValueID one, ValueID other) { + if (!one || !other || aliases.get(one.index()) == aliases.get(other.index())) + return; + aliases.set(one.index()); + aliases.set(other.index()); + grows = true; + }; + while (std::exchange(grows, false)) { + for (uint32_t number = 0; number < index.numberOfBlocks(); ++number) { + BodyBlock block = index.block(BlockID(number)); + for (uint32_t place = 0; place < block.numberOfParameters(); ++place) + unite(block.parameter(place), index.valueNarrowedInto(block.parameter(place))); + for (BodyValue value : block.values()) { + if (opcodesThatStateTypes.contains(value.opcode())) + unite(value.id(), value.operand(0)); + } + for (BodyTarget target : index.terminator(block.id()).targets()) { + std::span arguments = target.arguments(); + for (uint32_t place = 0; place < arguments.size(); ++place) + unite(arguments[place], index.block(target.block()).parameter(place)); + } + } + } +} + +static bool isLocalListMethod(const Program& program, PropertyKeyID key) +{ + const BuiltinName methods[] = { BuiltinNames::push, BuiltinNames::pop, BuiltinNames::shift, BuiltinNames::unshift, BuiltinNames::indexOf, BuiltinNames::includes, BuiltinNames::at }; + return std::ranges::any_of(methods, [&](BuiltinName method) { + return key == program.keyOf(method); + }); +} + +static bool usesListLocally(const Program& program, const BodyIndex& index, const BitVector& aliases, const BodyValue& user, uint32_t operandIndex) +{ + switch (user.opcode()) { + case Opcode::PropGet: + return operandIndex == OperandIndex::PropGet::object; + case Opcode::ElemGet: + return operandIndex == OperandIndex::ElemGet::object; + case Opcode::ElemSet: + return operandIndex == OperandIndex::ElemSet::object; + case Opcode::Call: { + if (operandIndex != OperandIndex::Call::thisValue) + return false; + ValueID callee = index.stripCasts(user.as().calleeOperand()); + if (index.isParameter(callee) || index.opcodeOf(callee) != Opcode::PropGet) + return false; + auto method = index.value(callee).as(); + return aliases.get(method.objectOperand().index()) && isLocalListMethod(program, method.key()); + } + default: + return opcodesThatStateTypes.contains(user.opcode()); + } +} + +bool AccessInLinkAccesses::isFinalListRead() +{ + const BodyIndex* index = isRead() && !isNamed() && !m_shared && m_receiver == m_subject && !m_program.at(m_function).isCoroutine() ? bodiesLookedAt(m_flow.looks).find(m_function) : nullptr; + if (!index || m_access.site.index() >= index->numberOfValues() || index->isParameter(m_access.site) || index->opcodeOf(m_access.site) != Opcode::ElemGet) + return false; + BodyBlock last = index->block(index->blockOf(m_access.site)); + if (last.hasHandler() || index->terminator(last.id()).numberOfTargets()) + return false; + ValueID list = index->value(m_access.site).as().objectOperand(); + ValueID root = index->stripCasts(list); + auto isRoot = [&](ValueID value) { + return index->opcodeOf(value) == Opcode::NewArray || (index->opcodeOf(value) == Opcode::Data && !!m_program.at(index->value(value).as().data()).root.tryAs()); + }; + if (index->isParameter(root) ? index->blockOf(root) == entryBlock : !isRoot(root)) + return false; + + BitVector aliases(index->numberOfValues()); + aliases.set(list.index()); + collectListAliases(*index, aliases); + for (uint32_t number = 0; number < index->numberOfBlocks(); ++number) { + BodyBlock block = index->block(BlockID(number)); + for (uint32_t place = 0; place < block.numberOfParameters(); ++place) { + if (aliases.get(block.parameter(place).index()) && (block.id() == entryBlock || block.isHandlerBlock())) + return false; + } + for (BodyValue value : block.values()) { + if (aliases.get(value.id().index()) && !opcodesThatStateTypes.contains(value.opcode()) && !isRoot(value.id())) + return false; + bool isBehindRead = block.id() == last.id() && value.id().index() > m_access.site.index(); + bool staysLocal = true; + value.forEachOperand([&](ValueID operand, uint32_t operandIndex) { + staysLocal &= !operand || !aliases.get(operand.index()) || (!isBehindRead && usesListLocally(m_program, *index, aliases, value, operandIndex)); + }); + if (!staysLocal) + return false; + } + bool staysInBlocks = true; + index->terminator(block.id()).forEachOperand([&](ValueID operand, uint32_t) { + staysInBlocks &= !operand || !aliases.get(operand.index()); + }); + if (!staysInBlocks) + return false; + if (block.hasHandler()) { + for (ValueID argument : block.handler().arguments()) { + if (aliases.get(argument.index())) + return false; + } + } + } + return true; +} + std::span AccessInLinkAccesses::definitionsByDescriptorAtSite() const { const ThinSummary& summary = m_flow.summaries.at(m_function); @@ -1055,7 +1343,7 @@ void AccessInLinkAccesses::resolve() } else m_flow.noteWhatIsStoredOn(subject, stored); } - if (!isNamed() && !m_keyIsOnlyNumbers && m_access.kind != Kind::Define && !membersOfObjectPrototypeAreWhatTableLists()) + if (!isNamed() && !m_keyIsOnlyNumbers && m_access.kind != Kind::Define && !membersOfObjectPrototypeAreWhatTableLists() && !isFinalListRead()) m_flow.addEdge(m_subject, m_flow.unseen()); if ((subject.hasUnknownObjects && !m_before.hasUnknownObjects) || (subject.hasDirectUnknownObjects() && !m_before.hasDirectUnknownObjects())) cannotTell(subject.hasDirectUnknownObjects() ? UnknownIs::InSight : UnknownIs::ByEscapeAlone); @@ -1166,6 +1454,8 @@ void AccessInLinkAccesses::resolveUnderPrivateName() void AccessInLinkAccesses::resolveOnLayout(LayoutID id, const LinkType& pooled) { + if (isRefusedByTest(id)) + return; const Layout& layout = m_program.at(id); switch (layout.kind) { case LayoutKind::Record: @@ -1251,7 +1541,7 @@ void AccessInLinkAccesses::resolveOnLayout(LayoutID id, const LinkType& pooled) return; bool goesOnAboveClasses = members.lookupGoesOnAboveClasses() && !isPrivateName; FlowNodeID foundOnPrototypes; - if (!isPrivateName && m_flow.isBelowConstructorFunction(id)) { + if (!isPrivateName && m_flow.isBelowFollowedPrototype(id)) { if (isRead()) foundOnPrototypes = m_flow.foundOnPrototypesAbove(id, m_access.key, LinkFlow::AccessorsAbove::AreEnteredByCaller); else if (!isDefinition) @@ -1282,6 +1572,8 @@ void AccessInLinkAccesses::resolveOnLayout(LayoutID id, const LinkType& pooled) giveConstructorsOfRowAbove(id); if (m_nameClass == NameClass::Constructor && layout.kind == LayoutKind::Instance && layout.constructor && !mayFindClassAbove) give(closureForLinkAccesses(layout.constructor)); + else if (BuiltinID member; !field && isExcludedByInstanceOf(id) && lookUpFrom(Builtins::ObjectPrototype, member) == LookupInLinkAccesses::Absent) + return; else if (!giveWhatIsFoundFrom(Builtins::ObjectPrototype)) m_flow.escape(id); return; @@ -1502,6 +1794,15 @@ void AccessInLinkAccesses::resolveOnFamily(FamilyID family) if (mayComeInFrontOfDefinition) givePrimitives(Primitive::Undefined); } + if (LayoutID tested = classTestedByInstanceOf()) { + if (auto read = m_flow.findFamilyReadBehindInstanceOf(family, m_access.key, tested)) { + giveWhatIsHeldAt(read->fields); + giveWhatIsHeldAt(read->besideFields); + return; + } + } + if (LayoutID refused = classRefusedByTest(); refused && m_flow.everyMemberStandsAtOrBelow(family, refused)) + return; if (isRead()) giveWhatIsHeldAt(m_flow.fieldReadThrough(family, m_access.key)); else @@ -1593,14 +1894,15 @@ void AccessInLinkAccesses::resolveOnPrototype(LayoutID id) LinkFlow::FoundMembers members = m_flow.findMembers(id, ClassSide::Prototype, m_access.key); layOnReceiverUnlessAccessorAnswers(members); bool isPrivateName = m_program.at(m_access.key).kind == PropertyKeyKind::PrivateName; - if (!isDefinition && !isPrivateName && m_flow.isBelowConstructorFunction(id)) { + if (!isDefinition && !isPrivateName && m_flow.isBelowFollowedPrototype(id)) { if (!isRead()) m_flow.lookForSetterOnPrototypesAbove(id, m_access.key, LinkFlow::AccessorsAbove::AreEnteredByCaller); else if (FlowNodeID foundOnPrototypes = m_flow.foundOnPrototypesAbove(id, m_access.key, LinkFlow::AccessorsAbove::AreEnteredByCaller)) giveFrom(foundOnPrototypes); enterAccessorsOnPrototypesAbove(id); } - if (!isRead() && !isPrivateName && m_receiver == m_subject && m_flow.isOfConstructorFunction(id)) { + bool laysFollowedName = m_flow.isOfConstructorFunction(id) || (!isRead() && m_flow.followsNamesOnPrototypeOf(id) && m_nameClass != NameClass::Constructor && members.lookupGoesOnAboveClasses() && !m_flow.instancesMayHoldName(id, m_access.key)); + if (!isRead() && !isPrivateName && m_receiver == m_subject && laysFollowedName) { bool findsAccessor = std::ranges::any_of(members, [](const LinkFlow::FoundMember& found) { return found.kind != MethodKind::Method; }); @@ -1610,7 +1912,7 @@ void AccessInLinkAccesses::resolveOnPrototype(LayoutID id) return; } if (isDefinition && m_access.laysAccessorAsWritten && m_flow.isDefinedByDescriptorSomewhere(m_access.key)) - m_flow.layAccessorOnPrototypeOf(id, m_access.key, m_access.writtenGetter, m_access.writtenSetter); + layAccessorAsWrittenOnPrototypeOf(id); else if (isDefinition && m_value == m_flow.unseen()) m_flow.defineOnPrototypeOf(id, m_access.key); else @@ -2080,12 +2382,14 @@ void AccessInLinkAccesses::enterAccessorAt(FlowNodeID member) tellAgainOnceClimbsWhoeverTells(member); const LinkType held = m_flow.typeAt(member); - if (held.primitives) + if (held.hasUnknownObjects ? held.directPrimitives() : held.primitives) givePrimitives(Primitive::Undefined); if (held.builtin) m_flow.note(LinkFlow::Note::BuiltinUsedViaFlow, m_table.canonicalID(held.builtin).index()); - if (held.hasUnknownObjects || held.builtin || held.objects != LinkType::Objects::None || held.classes != LinkType::Classes::None) + if (held.hasDirectUnknownObjects() || held.builtin || held.objects != LinkType::Objects::None || held.classes != LinkType::Classes::None) cannotTellWhatRuns(); + else if (held.hasUnknownObjects) + cannotTellWhatRuns(UnknownIs::ByEscapeAlone); switch (held.functions) { case LinkType::Functions::None: @@ -2282,7 +2586,12 @@ void AccessInLinkAccesses::resolveOnClasses(const LinkType& instances) return; } bool namesNoMemberAbove = !isElement && keyNamesNoMemberAbove(instances); - if (!isElement && !namesNoMemberAbove) + if (!isElement && !namesNoMemberAbove && isFinalListRead()) { + m_flow.addEdge(m_flow.contentsOf(instances, SlotOfContents::underAnyName()), m_flow.unseen()); + forEachListOfElements(instances, [&](FlowNodeID elements) { + m_flow.addEdge(elements, m_flow.unseen()); + }); + } else if (!isElement && !namesNoMemberAbove) m_flow.addEdge(m_subject, m_flow.unseen()); if (isRead() && !isElement) giveFrom(m_flow.contentsOf(instances, SlotOfContents::underAnyName())); @@ -2307,7 +2616,7 @@ void AccessInLinkAccesses::resolveOnClasses(const LinkType& instances) givePrimitives(Primitive::Undefined); storeIntoUnseen(); if (!isElement && !namesNoMemberAbove) - cannotTell(UnknownIs::InSight, tellsOfInstances() ? TellsName::No : TellsName::Yes); + cannotTell(m_seenKeysAreNumbers ? UnknownIs::ByEscapeAlone : UnknownIs::InSight, tellsOfInstances() ? TellsName::No : TellsName::Yes); return; } @@ -2338,11 +2647,13 @@ void AccessInLinkAccesses::resolveOnClasses(const LinkType& instances) m_flow.arrive(m_flow.contentsOf(instances, SlotOfContents::isChanged()), m_flow.fromTypeID(TypeTable::undefined)); if (!isRead() && (isNamed() ? m_nameClass != NameClass::Index : !m_access.keyIsSurelyIndex)) m_flow.arrive(m_flow.contentsOf(instances, SlotOfContents::noLongerAsMade()), m_flow.fromTypeID(TypeTable::undefined)); - BuiltinID prototype = isOneClass && (isElement || namesNoMemberAbove) && hasElements ? prototypeOfInstancesOf(builtinClass) : BuiltinID(); - if (prototype && indexesAreAbsentFrom(prototype)) + bool namesOnlyOwnElements = isElement || namesNoMemberAbove; + BuiltinID prototype = isOneClass && (namesOnlyOwnElements || m_seenKeysAreNumbers) && hasElements ? prototypeOfInstancesOf(builtinClass) : BuiltinID(); + bool indexesAreAbsentAbove = prototype && indexesAreAbsentFrom(prototype); + if (indexesAreAbsentAbove) givePrimitives(Primitive::Undefined); - else - cannotTell(UnknownIs::InSight, !isElement && tellsOfInstances() ? TellsName::No : TellsName::Yes); + if (!indexesAreAbsentAbove || !namesOnlyOwnElements) + cannotTell(indexesAreAbsentAbove ? UnknownIs::ByEscapeAlone : UnknownIs::InSight, !isElement && tellsOfInstances() ? TellsName::No : TellsName::Yes); } void AccessInLinkAccesses::resolveUnderNameOnClass(const LinkType& instances) @@ -2418,13 +2729,20 @@ void AccessInLinkAccesses::resolveUnderNameOnClass(const LinkType& instances) void AccessInLinkAccesses::handOn(const LinkType& type) { - if (m_access.whatHoldsNothing != noSummaryNode) - m_flow.arriveFromSource(m_flow.nodeOf(m_function, m_access.whatHoldsNothing), type); + if (uint32_t node = m_access.whatHoldsNothing(); node != noSummaryNode) + m_flow.arriveFromSource(m_flow.nodeOf(m_function, node), type); +} + +void AccessInLinkAccesses::layAccessorAsWrittenOnPrototypeOf(LayoutID id) +{ + RELEASE_ASSERT(!m_shared); + const SummaryWrittenAccessor& written = m_flow.summaries.at(m_function).writtenAccessorOf(m_record); + m_flow.layAccessorOnPrototypeOf(id, m_access.key, written.getter, written.setter); } void AccessInLinkAccesses::resolveContents(const LinkType& subject) { - bool settles = m_access.whatHoldsNothing != noSummaryNode; + bool settles = m_access.whatHoldsNothing() != noSummaryNode; if (uint16_t primitives = (subject.primitives & ~m_before.primitives) | (subject.directPrimitives() & ~m_before.directPrimitives())) { LinkType handed = primitivesForLinkAccesses(primitives); handed.escapeOnlyPrimitives = handed.primitives & ~LinkType::closed(subject.directPrimitives()); @@ -2521,7 +2839,7 @@ bool AccessInLinkAccesses::settlesPromiseThatCallHasJustMade() void AccessInLinkAccesses::resolveContentsOfInstances(const LinkType& instances) { - bool settles = m_access.whatHoldsNothing != noSummaryNode; + bool settles = m_access.whatHoldsNothing() != noSummaryNode; bool isOneClass = instances.classes != LinkType::Classes::Mixed; if (!isOneClass && m_access.slot.indexOfInternalField()) { diff --git a/Source/JavaScriptCore/aot/facts/AOTLinkCalls.cpp b/Source/JavaScriptCore/aot/facts/AOTLinkCalls.cpp index bce4a0bf92b4f..34ef9a0f95aa1 100644 --- a/Source/JavaScriptCore/aot/facts/AOTLinkCalls.cpp +++ b/Source/JavaScriptCore/aot/facts/AOTLinkCalls.cpp @@ -331,6 +331,11 @@ static LinkType primitivesWrittenForLinkCalls(const BuiltinType& written) return primitivesForLinkCalls(LinkType::primitivesOf(primitivesOfBuiltinType(written), integerRangeWrittenBy(written))); } +static LinkType wholeNumbersForLinkCalls() +{ + return primitivesForLinkCalls(Primitive::NumberWithoutFraction); +} + static LinkType closuresForLinkCalls(FunctionID function) { LinkType type; @@ -844,6 +849,7 @@ class CallInLinkCalls { static QuestionsInLinkCalls questionsOfComparisonOfPrivateFields(); void inspectWhatExpectedBuiltinCompares(); void enter(FunctionID callee); + void noteCustomInstanceOf(); private: using Source = SourceInLinkCalls; @@ -997,7 +1003,8 @@ class CallInLinkCalls { LinkType objectsThatNoNodeHolds(LayoutID); Values resultOf(FunctionID callee, uint32_t index, const Passed&, const Values& constructed, uint8_t distance); void noteWhatIsMadeThrough(FunctionID callee, const Passed&); - bool namesOnlyClassesAtOrBelow(LayoutID, const Values& newTargets); + enum class NewTargets : uint8_t { OwnClasses, OwnOrEscapedClasses, OwnOrUnseenClasses, OtherClasses }; + NewTargets classifyNewTargets(LayoutID above, const Values& newTargets); bool addObjectsMadeForNewTarget(Values&, FunctionID callee, const Passed&, uint8_t distance); Values passToLostFunctions(const Passed&, const LinkType& callee, const Source*); Values passToUnseen(const Passed&); @@ -1094,7 +1101,7 @@ class CallInLinkCalls { void addContentsOfSlot(RowCall&, const Values& holders, BuiltinSlotID, Values&); void addIteratedValues(const Values& iterables, Values&); Values valuesIteratedOf(const Values& iterables); - void addResultsOfCallbacks(RowCall&, BuiltinOperand, bool isResultAsItIsAnswered, Values&); + void addResultsOfCallbacks(RowCall&, BuiltinOperand, bool isResultAsItIsAnswered, bool containsCallbackResults, Values&); void addPrototypes(const Values& objects, Values&); Values contentsOf(const Values& holders, NamesRead = NamesRead::All); void settle(Values&, bool rowHasJustMadeThem = false); @@ -1182,6 +1189,7 @@ class CallInLinkCalls { bool mayBePrintedWholeWhereItIsRefused(const LinkType&, bool isLiteralMadeForCall); bool isLiteralMadeForCall(const RowCall&, BuiltinOperand) const; void followWhatHandlersAnswerForFormatter(RowCall&, const BuiltinHandlerCall&); + bool findsNoElements(const RowCall&, const BuiltinPropertyAction&); bool gatherClosuresWrittenAt(const RowCall&, ValueID written, ValueID writer, std::span path, Vector&); bool makesInstanceWithNothingCalledUnder(ValueID callOrConstruction, BuiltinName step, PropertyKeyID); enum class SubjectOfInspection : uint8_t { Layout, Function, Site }; @@ -1674,6 +1682,7 @@ void CallInLinkCalls::callUnseen(UnknownIs unknownIs, ReceiverOfUnseenCode recei m_state.callsThatEnterMembersOfTable[m_number] = 1; if (levelBefore >= level) return; + noteCustomInstanceOf(); if (levelBefore) { for (const ResultInLinkCalls& result : results()) m_flow.arrive(result.node, LinkType::anything()); @@ -2039,7 +2048,7 @@ ALWAYS_INLINE void CallInLinkCalls::callName() for (uint16_t group : primitivesWithClassesForLinkCalls) { if (isSettledButForOwnCallees() || isDoneButForWhatIsInSight()) break; - uint16_t present = receiver.primitives & group; + uint16_t present = (receiver.hasUnknownObjects ? receiver.directPrimitives() : receiver.primitives) & group; if (!present) continue; MemberOfTable found = findOnInstancesOf(primitiveClassOf(LinkType::inNamesOfPrimitive(present))); @@ -2058,7 +2067,7 @@ ALWAYS_INLINE void CallInLinkCalls::callName() if (receiver.hasUnknownObjects) { UnknownIs how = howUnknownIs(receiver); bool holdsWhatCanBeRead = false; - if (how == UnknownIs::InSight && !m_shared && !isSettledButForOwnCallees() && listsAloneHaveMemberCalledThatMakesList(holdsWhatCanBeRead)) { + if (how == UnknownIs::InSight && !m_shared && !isSettledButForOwnCallees() && !m_flow.rowsSpeakOfKeyFor(m_caller, m_call.key) && listsAloneHaveMemberCalledThatMakesList(holdsWhatCanBeRead)) { SetForScope makesListAtOneSite(m_makesWhatRowsMakeAtOneSiteOfProgram, !holdsWhatCanBeRead); callMemberOfTable(findOnObjectsOfClassItself(BuiltinClasses::Array), valuesOf(unseenObjectsForLinkCalls()), WhenAbsent::CallThrows, ReceiverOfUnseenCode::UnseenObjectItself); how = UnknownIs::ByEscapeAlone; @@ -2838,8 +2847,20 @@ void CallInLinkCalls::enter(FunctionID callee) enter(callee, passedByCall()); } +void CallInLinkCalls::noteCustomInstanceOf() +{ + if (m_shared || !m_call.isMadeByProtocol || m_call.key != m_program.keyOf(m_builtins.nameOf(WellKnownSymbol::HasInstance))) + return; + const LinkFlow::ReadsBehindInstanceOf* reads = m_flow.readsBehindInstanceOf(m_caller); + if (!reads) + return; + for (auto [test, access] : std::ranges::equal_range(reads->accessesByTest, m_call.site, { }, &std::pair::first)) + m_flow.arrive(nodeAt(m_summary.accesses[access].value), primitivesForLinkCalls(Primitive::Undefined)); +} + void CallInLinkCalls::enter(FunctionID callee, const Passed& passed) { + noteCustomInstanceOf(); if (lists(callee) && callees().builtins().isEmpty() && !callees().mayEnterUnseen && !passed.newTarget) { std::optional thisParameter = m_program.at(callee).thisParameterIndex(); if (!passed.isConstruction && passed.receiver && thisParameter) @@ -3150,30 +3171,37 @@ void CallInLinkCalls::noteWhatIsMadeThrough(FunctionID callee, const Passed& pas m_flow.addEdge(m_flow.objectsMadeThroughConstructorOf(m_program.at(m_caller).constructedClass), m_flow.objectsMadeThroughConstructorOf(function.constructedClass)); } -bool CallInLinkCalls::namesOnlyClassesAtOrBelow(LayoutID above, const Values& newTargets) +auto CallInLinkCalls::classifyNewTargets(LayoutID above, const Values& newTargets) -> NewTargets { - bool namesOnlyThose = true; + NewTargets named = NewTargets::OwnClasses; + auto add = [&](NewTargets added) { + named = std::max(named, added); + }; forEachType(newTargets, [&](const LinkType& newTarget, const Source*) { switch (newTarget.functions) { case LinkType::Functions::None: break; case LinkType::Functions::Closure: { const Function& constructor = m_program.at(newTarget.function); - namesOnlyThose &= isClassConstructorOf(constructor.kind) && constructor.constructedClass && m_flow.standsAtOrBelow(constructor.constructedClass, above); + if (!isClassConstructorOf(constructor.kind) || !constructor.constructedClass || !m_flow.standsAtOrBelow(constructor.constructedClass, above)) + add(NewTargets::OtherClasses); break; } case LinkType::Functions::Constructors: - namesOnlyThose &= m_flow.everyMemberStandsAtOrBelow(newTarget.constructorsOf, above); + if (!m_flow.everyMemberStandsAtOrBelow(newTarget.constructorsOf, above)) + add(NewTargets::OtherClasses); break; case LinkType::Functions::Set: case LinkType::Functions::Lost: - namesOnlyThose = false; + add(NewTargets::OtherClasses); break; } - if (newTarget.hasUnknownObjects || newTarget.builtin || newTarget.objects != LinkType::Objects::None || newTarget.classes != LinkType::Classes::None) - namesOnlyThose = false; + if (newTarget.builtin || newTarget.objects != LinkType::Objects::None || newTarget.classes != LinkType::Classes::None) + add(NewTargets::OtherClasses); + else if (newTarget.hasUnknownObjects) + add(newTarget.hasDirectUnknownObjects() ? NewTargets::OwnOrUnseenClasses : NewTargets::OwnOrEscapedClasses); }); - return namesOnlyThose; + return named; } bool CallInLinkCalls::addObjectsMadeForNewTarget(Values& values, FunctionID callee, const Passed& passed, uint8_t distance) @@ -3189,12 +3217,23 @@ bool CallInLinkCalls::addObjectsMadeForNewTarget(Values& values, FunctionID call return false; LayoutID constructed = m_program.at(m_caller).constructedClass; FunctionID constructorAtTop = m_program.at(m_flow.topOfRow(constructed)).constructor; - bool namesOnlyClassesOfRow = namesOnlyClassesAtOrBelow(constructed, *passed.newTarget); - if (!namesOnlyClassesOfRow && !isClassConstructorOf(m_program.at(constructorAtTop).kind)) + NewTargets named = classifyNewTargets(constructed, *passed.newTarget); + if (named != NewTargets::OwnClasses && !isClassConstructorOf(m_program.at(constructorAtTop).kind)) return false; add(values, m_flow.objectsMadeThroughConstructorOf(constructed), notObservedForLinkCalls, TypeTable::object); - if (!namesOnlyClassesOfRow) + switch (named) { + case NewTargets::OwnClasses: + break; + case NewTargets::OwnOrEscapedClasses: + add(values, unseenObjectsForLinkCalls().asEscapeOnly()); + break; + case NewTargets::OwnOrUnseenClasses: + add(values, unseenObjectsForLinkCalls()); + break; + case NewTargets::OtherClasses: add(values, m_flow.result(constructorAtTop, 0), distance); + break; + } return true; } @@ -3318,12 +3357,16 @@ ValuesInLinkCalls CallInLinkCalls::callBuiltin(BuiltinID builtin, const Passed& m_flow.testRuns().noteHolderOfSpy(holder, { }, m_caller, m_call.site); }); } - auto holdsNothing = [&](const Values& values) { + auto holdsNothing = [&](const Values& values, bool isReceiver = false) { bool holdsSomething = false; forEachType(values, [&](const LinkType& type, const Source* source) { holdsSomething = true; auto letNamespaceGo = [&](LayoutID layout) { - if (m_program.at(layout).kind == LayoutKind::ModuleNamespace) + const Layout& record = m_program.at(layout); + if (record.kind != LayoutKind::ModuleNamespace) + return; + bool isIgnored = isReceiver && record.namespaceOf && m_program.at(record.namespaceOf).kind == ModuleKind::RuntimeSource && m_builtins.ignoresReceiver(call.builtin); + if (!isIgnored) escape(objectsForLinkCalls(layout), source); }; if (type.objects == LinkType::Objects::Layout) @@ -3333,8 +3376,12 @@ ValuesInLinkCalls CallInLinkCalls::callBuiltin(BuiltinID builtin, const Passed& }); return !holdsSomething; }; - if ((passed.receiver && holdsNothing(*passed.receiver)) || std::ranges::any_of(passed.arguments, holdsNothing)) + if (passed.receiver && holdsNothing(*passed.receiver, true)) return { }; + for (const Values& argument : passed.arguments) { + if (holdsNothing(argument)) + return { }; + } if (!mayGoByRows || !rowsSpeakOf(call)) return callOutOfSight(call); @@ -3749,6 +3796,11 @@ void CallInLinkCalls::describeWhatRowRefuses(const RowCall& call, const BuiltinO if (!parameter.roles.contains(BuiltinOperandRole::DescribedWhenRefused) || !fitOf(call, parameter).mayBeOutside) return; throwAnything(); + if (m_builtins.isUnder(call.builtin, Builtins::NativeFunctions)) { + callees().mayEnterUnseen = true; + escape(valuesAt(call, parameter.operand)); + return; + } questions.describesLiteralMadeForCall = isLiteralMadeForCall(call, parameter.operand); ask(valuesAt(call, parameter.operand), questions); }; @@ -4043,10 +4095,12 @@ ALWAYS_INLINE void CallInLinkCalls::addFit(const LinkType& type, const Source* s bool isNamed = std::ranges::any_of(accepted.parts, [&](const BuiltinTypePart& part) { return part.kind == BuiltinTypePartKind::Builtin && m_builtins.canonicalID(part.builtin) == m_builtins.canonicalID(type.builtin); }); + bool isConstructor = m_builtins.at(m_builtins.canonicalID(type.builtin)).kind == BuiltinKind::Constructor; bool takesOnlyWhatProgramMakes = !takesAnyFunction && !accepted.flags.contains(BuiltinTypeFlag::HasPlainData) && !accepted.parts.empty() && std::ranges::all_of(accepted.parts, [&](const BuiltinTypePart& part) { - return part.kind == BuiltinTypePartKind::Instance && m_builtins.at(part.builtinClass).hasClosureInstances; + return part.kind == BuiltinTypePartKind::Instance && (m_builtins.at(part.builtinClass).hasClosureInstances || (isConstructor && part.builtinClass == BuiltinClasses::BoundFunction)); }); - note(takesSomeObject && !takesOnlyWhatProgramMakes, takesAnyObject || isNamed, takesSomeObject && !takesOnlyWhatProgramMakes); + bool isAcceptedBuiltin = takesAnyFunction && isCallableBuiltin(m_builtins, m_builtins.canonicalID(type.builtin)); + note(takesSomeObject && !takesOnlyWhatProgramMakes, takesAnyObject || isNamed || isAcceptedBuiltin, takesSomeObject && !takesOnlyWhatProgramMakes); } switch (type.classes) { @@ -4102,7 +4156,8 @@ void CallInLinkCalls::applyRow(RowCall& call) applyParameter(call, parameter); } for (uint32_t argument = 0; argument < passed.arguments.size(); ++argument) { - if (argument > BuiltinOperand::maxArgumentIndex || !m_builtins.parameterFor(row, BuiltinOperand::argument(argument))) + bool isTaken = argument <= BuiltinOperand::maxArgumentIndex && m_builtins.parameterFor(row, BuiltinOperand::argument(argument)); + if (!isTaken && !m_builtins.ignoresArgument(call.builtin, argument)) escape(passed.arguments[argument]); } if (passed.list && !m_builtins.parameterFor(row, BuiltinOperand::argument(BuiltinOperand::maxArgumentIndex))) @@ -4207,7 +4262,7 @@ void CallInLinkCalls::applyParameter(RowCall& call, const BuiltinParameter& para changed = saidByTable.questions; if (std::exchange(changed->listKeys, false)) tellKeysBecomeTextsAmong(values); - if (changed->describeObjects && !fitOf(call, parameter).mayBeOutside) + if (changed->describeObjects && (m_builtins.isUnder(call.builtin, Builtins::NativeFunctions) || !fitOf(call, parameter).mayBeOutside)) changed->describeObjects = false; changed->describesLiteralMadeForCall = changed->describeObjects && isLiteralMadeForCall(call, parameter.operand); changed->areAskedByFormatter = isPrinted; @@ -4228,6 +4283,12 @@ void CallInLinkCalls::applyParameter(RowCall& call, const BuiltinParameter& para void CallInLinkCalls::applyCallback(RowCall& call, const BuiltinCallback& callback) { + if (callback.times == BuiltinCallbackTimes::Repeatedly || callback.times == BuiltinCallbackTimes::PerElement) { + forEachType(valuesAt(call, callback.callee), [&](const LinkType& type, const Source*) { + if (type.functions == LinkType::Functions::Closure) + m_flow.iterationCallbacks.set(type.function.index()); + }); + } PassedMadeInLinkCalls passed; passed.isConstruction = callback.constructs; if (callback.constructs) { @@ -4967,8 +5028,23 @@ void CallInLinkCalls::callHandlers(RowCall& call, const BuiltinHandlerCall& hand inspect(printed, questionsOfFormatter); } +bool CallInLinkCalls::findsNoElements(const RowCall& call, const BuiltinPropertyAction& action) +{ + if (action.kind != BuiltinPropertyActionKind::ReadsByKey || action.key.kind != BuiltinKeyKind::Indexes || m_program.admitsReplacedBuiltins) + return false; + bool mayFind = false; + forEachType(valuesAt(call, action.object), [&](const LinkType& type, const Source*) { + LinkType beside = type; + beside.primitives = 0; + mayFind |= !beside.isNever() || type.holdsLostBuiltins || (type.primitives & LinkType::closed(Primitive::AnyString)); + }); + return !mayFind; +} + void CallInLinkCalls::applyAction(RowCall& call, const BuiltinPropertyAction& action) { + if (findsNoElements(call, action)) + return; bool reads = false; bool knowsThatLinkIsLeftAlone = false; switch (action.kind) { @@ -5400,6 +5476,8 @@ ALWAYS_INLINE void CallInLinkCalls::addResult(RowCall& call) add(call.result, primitivesForLinkCalls(Primitive::Null)); else if (result.testedOperand) add(call.result, primitivesForLinkCalls(Primitive::Boolean)); + else if (result.rangeRule == BuiltinRangeRule::IntegerPart) + add(call.result, wholeNumbersForLinkCalls()); else addValuesOfType(call, result.type, true, call.result); } @@ -5679,7 +5757,7 @@ void CallInLinkCalls::addValuesOfPart(RowCall& call, const BuiltinTypePart& part } case BuiltinTypePartKind::CallbackResult: { Values answers; - addResultsOfCallbacks(call, *part.operand, isResult && !part.isSettled, answers); + addResultsOfCallbacks(call, *part.operand, isResult && !part.isSettled, !isResult && !part.isSettled, answers); if (part.isSettled) settle(answers); add(values, answers); @@ -5994,11 +6072,16 @@ ValuesInLinkCalls CallInLinkCalls::valuesIteratedOf(const Values& iterables) return iterated; } -void CallInLinkCalls::addResultsOfCallbacks(RowCall& call, BuiltinOperand operand, bool isResultAsItIsAnswered, Values& values) +void CallInLinkCalls::addResultsOfCallbacks(RowCall& call, BuiltinOperand operand, bool isResultAsItIsAnswered, bool containsCallbackResults, Values& values) { - bool usesHaveLaidAnswersOfBuiltins = isResultAsItIsAnswered && std::ranges::all_of(call.row->flow.callbacks, [&](const BuiltinCallback& callback) { - return callback.callee != operand || callback.resultUse == BuiltinCallbackResultUse::Returned; - }); + auto usesLayAnswers = [&](const BuiltinCallback& callback) { + if (callback.callee != operand) + return true; + if (isResultAsItIsAnswered) + return callback.resultUse == BuiltinCallbackResultUse::Returned; + return callback.resultUse == BuiltinCallbackResultUse::Stored && callback.resultTo.kind == BuiltinPlaceKind::ResultSlot && m_builtins.at(callback.resultTo.slot).owner == BuiltinClasses::Array; + }; + bool usesHaveLaidAnswersOfBuiltins = (isResultAsItIsAnswered || containsCallbackResults) && std::ranges::all_of(call.row->flow.callbacks, usesLayAnswers); bool constructs = std::ranges::any_of(call.row->flow.callbacks, [&](const BuiltinCallback& callback) { return callback.callee == operand && callback.constructs; }); diff --git a/Source/JavaScriptCore/aot/facts/AOTLinkEffects.cpp b/Source/JavaScriptCore/aot/facts/AOTLinkEffects.cpp index e8bbba6f47669..5f9d6e1b24bf3 100644 --- a/Source/JavaScriptCore/aot/facts/AOTLinkEffects.cpp +++ b/Source/JavaScriptCore/aot/facts/AOTLinkEffects.cpp @@ -251,6 +251,7 @@ class CalleeSummarizerForLinkEffects { struct GivenArgumentsInKnownEffects { uint32_t parametersTakenAsExpected { 0 }; uint32_t parametersCalled { 0 }; + Vector parametersThatBringObjectsAlone; }; static EffectsOfComponentInKnownEffects knownEffectsOfUnknownCode() { return { Effects::unknownCode(), LocationMasks::top(), objectsOfAnyLayout }; } @@ -270,7 +271,9 @@ class CalleeSummarizerForLinkEffects { static void takeKnownEffects(const Effects& all, const LocationMasks& writesOnReturn, uint64_t layoutsWritten, WayInKnownEffects, EffectsOfComponentInKnownEffects& into); void takeKnownEffectsOfFunction(FunctionID callee, WayInKnownEffects, EffectsOfComponentInKnownEffects& into) const; bool mergeKnownEffectsOfCodeOutside(const CodeEntered&, uint32_t component, WayInKnownEffects, EffectsOfComponentInKnownEffects&) const; - void addEffectsOfOperatorsLeftApart(FunctionID, const ThinSummary&, EffectsOfComponentInKnownEffects&) const; + void addEffectsOfOperatorsLeftApart(FunctionID, const ThinSummary&, EffectsOfComponentInKnownEffects&, GivenArgumentsInKnownEffects&) const; + std::optional primitivesExpectedOfParameter(FunctionID, uint32_t entryParameter) const; + uint32_t parametersExpectedPrimitiveThatBringObjectsAlone(FunctionID, const ThinSummary&, uint32_t node, GivenArgumentsInKnownEffects&) const; bool readMayMakeClosureOfField(const LinkType& subject, PropertyKeyID) const; void addEffectsOfAccessItself(FunctionID, const ThinSummary&, const SummaryAccess&, const LinkType& subject, PropertyKeyID, WayInKnownEffects, EffectsOfComponentInKnownEffects&); static std::optional entryParameterAt(const ThinSummary&, uint32_t node); diff --git a/Source/JavaScriptCore/aot/facts/AOTLinkFlow.cpp b/Source/JavaScriptCore/aot/facts/AOTLinkFlow.cpp index cd0f1702fdc0d..b11d80c9dfa97 100644 --- a/Source/JavaScriptCore/aot/facts/AOTLinkFlow.cpp +++ b/Source/JavaScriptCore/aot/facts/AOTLinkFlow.cpp @@ -8,6 +8,7 @@ #if ENABLE(AOT) +#include "AOTBodyIndex.h" #include "AOTBuildContext.h" #include "AOTBuiltinMembers.h" #include "AOTBuiltinTable.h" @@ -264,9 +265,10 @@ class EdgesOfBusyNodesInLinkFlow { bool insert(const Slot& slot) { size_t mask = m_slots.size() - 1; + Slot* slots = m_slots.mutableSpan().data(); uint64_t words = (static_cast(slot.from) << 32 | slot.to) ^ static_cast(slot.passes) << 15; for (size_t place = (words * 0x9e3779b97f4a7c15ull) >> m_shift;; place = (place + 1) & mask) { - Slot& held = m_slots[place]; + Slot& held = slots[place]; if (held.from == UINT32_MAX) { held = slot; return true; @@ -292,12 +294,37 @@ class EdgesOfBusyNodesInLinkFlow { unsigned m_shift { 64 }; }; +class ListedFlowObserver { +public: + ListedFlowObserver(const FlowObserver& observer, uint32_t next) + : m_kindAndFunction(static_cast(observer.kind) << bitsOfFunction | (observer.function.bits() + 1)) + , m_record(observer.record) + , m_next(next) + { + RELEASE_ASSERT(observer.function.bits() + 1 < 1u << bitsOfFunction); + } + + FlowObserver::Kind kind() const { return static_cast(m_kindAndFunction >> bitsOfFunction); } + FlowObserver observer() const { return { kind(), FunctionID::fromBits((m_kindAndFunction & ((1u << bitsOfFunction) - 1)) - 1), m_record }; } + uint32_t next() const { return m_next; } + +private: + static constexpr unsigned bitsOfFunction = 27; + static_assert(static_cast(FlowObserver::Kind::FieldIsOnlyAsked) < 1u << (32 - bitsOfFunction)); + + uint32_t m_kindAndFunction; + uint32_t m_record; + uint32_t m_next; +}; +static_assert(sizeof(ListedFlowObserver) == 12); + struct alignas(128) LinkFlow::Lane { WTF_DEPRECATED_MAKE_STRUCT_FAST_ALLOCATED(Lane); unsigned index { 0 }; SegmentedVector edges; - SegmentedVector, 1 << 16> observers; + SegmentedVector observers; + LinkReferenceTable::RecentNumbers recentNumbers; HashSet, WTF::UnsignedWithZeroKeyHashTraits> nodesThatAccessorsFoundAsked; EdgesOfBusyNodesInLinkFlow edgesOfBusyNodes; Vector numbersOfNodesToTell; @@ -349,6 +376,7 @@ struct alignas(128) LinkFlow::Lane { Vector notes; HashSet, WTF::UnsignedWithZeroKeyHashTraits> notesListed; std::array notesListedLately { }; + BitVector layoutsNotedWithBases; HashMap, DefaultHash, WTF::UnsignedWithZeroKeyHashTraits> fieldsNotNotedEmpty; LinkType typeObservedBefore; uint64_t numberOfObserversTold { 0 }; @@ -361,6 +389,7 @@ struct alignas(128) LinkFlow::Lane { Vector layoutsMetInFamilies; Vector typesAtUnseen; HashMap, WTF::UnsignedWithZeroKeyHashTraits> testsOfEdges; + HashMap, DefaultHash, WTF::UnsignedWithZeroKeyHashTraits> readsBehindInstanceOf; uint64_t numberOfNodesTakenOffWorklist { 0 }; uint64_t scatteredNumber { 88172645463325252ull }; }; @@ -493,6 +522,11 @@ LinkFlow::LinkFlow(Program& program, const ThinSummaries& summaries, const Famil fieldsWrittenAfterConstruction.ensureSize(m_firstFieldOfLayout.last()); fieldsThatMayBeEmpty.ensureSize(m_firstFieldOfLayout.last()); fieldsNotOfBirth.ensureSize(m_firstFieldOfLayout.last()); + m_classesWithInstancesThatMayBeLocked.ensureSize(program.layouts.size()); + for (uint32_t index = 0; index < program.layouts.size(); ++index) { + if (const Layout& layout = program.layouts[index]; layout.kind != LayoutKind::Instance || !isFoundByShapeTest(layout)) + noteInstancesMayBeLocked(LayoutID(index)); + } gettersThatMayBeAbsent.ensureSize(m_firstFieldOfLayout.last()); settersThatMayBeAbsent.ensureSize(m_firstFieldOfLayout.last()); @@ -514,6 +548,7 @@ void LinkFlow::makeTablesThatNeedNoNodes() size_t numberOfFunctions = program.functions.size(); size_t numberOfLayouts = program.layouts.size(); reached.ensureSize(numberOfFunctions); + iterationCallbacks.ensureSize(numberOfFunctions); lostFunctions.ensureSize(numberOfFunctions); escapedFunctions.ensureSize(numberOfFunctions); libraryFunctionsEntered.ensureSize(numberOfFunctions); @@ -766,7 +801,7 @@ void LinkFlow::numberNodes(Vector&& numbersOfMemberNodesOfSides) m_nodes.grow(build.parallelLoops(), next); m_listsOfNodes.reserve(roomForNodes); m_listsOfNodes.grow(build.parallelLoops(), next); - m_nodes[unseen().index()].type = LinkType::anything(); + setTypeOfNode(unseen().index(), LinkType::anything(), LinkType { }); for (uint32_t node = m_firstMemberNodeOfSide[0]; node < m_nodesOfFunctions[0].firstParameter; ++node) m_nodes[node].typeOfPosition = TypeTable::anything; auto bind = [&](FlowNodeID node, const DeclaredType& declared) { @@ -888,6 +923,91 @@ void LinkFlow::numberNodes(Vector&& numbersOfMemberNodesOfSides) }); } +LinkReferenceTable::LinkReferenceTable() + : m_room(MallocSpan>::zeroedMalloc(numberOfBlocks * sizeof(std::atomic))) + , m_blocks(m_room.mutableSpan().data()) +{ + auto* records = new References[recordsInBlock]; + records[0] = LinkType::noReferences(); + m_room[0].store(records, std::memory_order_relaxed); +} + +LinkReferenceTable::~LinkReferenceTable() +{ + size_t end = (m_nextNumber.load(std::memory_order_relaxed) + recordsInBlock - 1) / recordsInBlock; + for (auto& block : m_room.mutableSpan().first(end)) + delete[] block.load(std::memory_order_relaxed); +} + +uint32_t LinkReferenceTable::takeNumber(Shard& shard) +{ + if (shard.next == shard.end) { + shard.next = m_nextNumber.fetch_add(recordsTakenAtOnce, std::memory_order_relaxed); + shard.end = shard.next + recordsTakenAtOnce; + RELEASE_ASSERT(shard.next < shard.end); + auto& block = m_room[shard.next / recordsInBlock]; + References* expected = block.load(std::memory_order_acquire); + if (!expected) { + auto* records = new References[recordsInBlock]; + if (!block.compare_exchange_strong(expected, records, std::memory_order_acq_rel)) + delete[] records; + } + } + return shard.next++; +} + +uint32_t LinkReferenceTable::numberOf(const References& references, RecentNumbers& recentNumbers) +{ + uint64_t hash = hashOf(references); + Recent& recent = recentNumbers[hash >> 32 & (recentNumbers.size() - 1)]; + if (recent.references != references) + recent = { references, findOrAdd(references, hash) }; + return recent.number; +} + +uint32_t LinkReferenceTable::findOrAdd(const References& references, uint64_t hash) +{ + Shard& shard = m_shards[hash % numberOfShards]; + while (shard.isLocked.load(std::memory_order_relaxed) || shard.isLocked.exchange(true, std::memory_order_acquire)) + simde_mm_pause(); + if (4 * static_cast(shard.size) >= 3 * shard.slots.size()) + growSlots(shard); + size_t mask = shard.slots.size() - 1; + size_t slot = hash / numberOfShards & mask; + uint32_t number; + while ((number = shard.slots[slot]) && at(number) != references) + slot = (slot + 1) & mask; + if (!number) { + number = takeNumber(shard); + m_room[number / recordsInBlock].load(std::memory_order_relaxed)[number % recordsInBlock] = references; + shard.slots[slot] = number; + ++shard.size; + } + shard.isLocked.store(false, std::memory_order_release); + return number; +} + +uint32_t LinkFlow::numberOfReferences(const LinkType::References& references) +{ + return m_references.numberOf(references, s_laneAtWork ? s_laneAtWork->recentNumbers : m_recentNumbers); +} + +void LinkReferenceTable::growSlots(Shard& shard) +{ + Vector larger; + larger.fill(0, std::max(32, 2 * shard.slots.size())); + size_t mask = larger.size() - 1; + for (uint32_t number : shard.slots) { + if (!number) + continue; + size_t slot = hashOf(at(number)) / numberOfShards & mask; + while (larger[slot]) + slot = (slot + 1) & mask; + larger[slot] = number; + } + shard.slots = WTF::move(larger); +} + FlowNodeID LinkFlow::makeNode() { if (m_severalThreadsTell) @@ -1119,6 +1239,18 @@ void LinkFlow::loseFunctionsOfSet(const LinkType& type) const bool LinkFlow::mayStandInOneSet(std::span members) const { + static constexpr size_t mostMembersThatAreNoSmallLeaves = 4; + static constexpr uint32_t weightOfSmallLeaf = 16; + auto isSmallLeaf = [&](FunctionID member) { + if (!program.at(member).hasBody()) + return false; + const ThinSummary& summary = summaries.at(member); + return summary.weightOfOperations <= weightOfSmallLeaf && !summary.has(SummaryTrait::HasLoops) && std::ranges::all_of(summary.calls, [](const SummaryCall& call) { + return call.callee == SummaryCall::Callee::Builtin || call.isMadeByProtocol; + }); + }; + if (members.size() > mostMembersThatAreNoSmallLeaves && !std::ranges::all_of(members, isSmallLeaf)) + return false; for (size_t first = 0; first < members.size(); ++first) { LayoutID classOfFirst = classOfConstructor(members[first]); for (size_t second = first + 1; classOfFirst && second < members.size(); ++second) { @@ -2283,7 +2415,7 @@ void LinkFlow::arrive(FlowNodeID node, const LinkType& arriving, PropertyKeyID s } if (type.isNever()) return; - const LinkType& held = typeAt(node); + LinkType held = m_flowIsAtRest || node.index() >= m_nodes.size() ? typeAt(node) : headOfNode(node.index()); bool namesNothing = type.objects == LinkType::Objects::None && type.functions == LinkType::Functions::None && type.classes == LinkType::Classes::None && !type.builtin && (!(type.primitives & LinkType::oneSymbol) || (held.directPrimitives() & Primitive::Symbol && !symbol) || !(type.directPrimitives() & LinkType::oneSymbol)); if (namesNothing && node != unseen() && bringsNoPrimitiveInLinkFlow(type.primitives, held.primitives) && type.raisesNoFlagOver(held) && (!type.holdsLostBuiltins || held.holdsLostBuiltins || isNodeOfLibrary(node.index())) && bringsNothingToShadowsInLinkFlow(type, held)) @@ -2301,16 +2433,18 @@ ALWAYS_INLINE_INTO_OTHER_FILES void LinkFlow::receive(FlowNodeID node, const Lin } size_t index = node.index(); + Node& target = m_nodes[index]; + LinkType held = headHeldBy(target); bool namesNothing = type.objects == LinkType::Objects::None && type.functions == LinkType::Functions::None && type.classes == LinkType::Classes::None && !type.builtin - && (!type.holdsLostBuiltins || m_nodes[index].type.holdsLostBuiltins || isNodeOfLibrary(index)) && (!(type.primitives & LinkType::oneSymbol) || (m_nodes[index].type.directPrimitives() & Primitive::Symbol && !symbol) || !(type.directPrimitives() & LinkType::oneSymbol)); - if (namesNothing && bringsNoPrimitiveInLinkFlow(type.primitives, m_nodes[index].type.primitives) && type.raisesNoFlagOver(m_nodes[index].type) && bringsNothingToShadowsInLinkFlow(type, m_nodes[index].type)) + && (!type.holdsLostBuiltins || held.holdsLostBuiltins || isNodeOfLibrary(index)) && (!(type.primitives & LinkType::oneSymbol) || (held.directPrimitives() & Primitive::Symbol && !symbol) || !(type.directPrimitives() & LinkType::oneSymbol)); + if (namesNothing && bringsNoPrimitiveInLinkFlow(type.primitives, held.primitives) && type.raisesNoFlagOver(held) && bringsNothingToShadowsInLinkFlow(type, held)) return; if (type.marks & m_marksThatAreUsed) [[unlikely]] { receiveWithoutMarksThatAreUsed(node, type, symbol); return; } - TypeID bound = m_nodes[index].typeOfPosition; + TypeID bound = target.typeOfPosition; LinkType arriving = type; if (bound == TypeTable::anything) arriving.primitives &= LinkType::closed(Primitive::All); @@ -2324,7 +2458,7 @@ ALWAYS_INLINE_INTO_OTHER_FILES void LinkFlow::receive(FlowNodeID node, const Lin arriving.escapeOnlyPrimitives |= added; arriving.counts = LinkType::Counts::Unknown; } - LinkType before = m_nodes[index].type; + LinkType before = typeHeldBy(target); bool heldNothing = !before.primitives && before.objects == LinkType::Objects::None && before.functions == LinkType::Functions::None && before.classes == LinkType::Classes::None && !before.hasUnknownObjects && !before.holdsLostBuiltins && !before.builtin; if (heldNothing && !(arriving.hasUnknownObjects && arriving.builtin) && !(arriving.primitives & LinkType::oneSymbol)) { arriving.escapeOnlyPrimitives = withoutLiteralsInLinkFlow(arriving.primitives) & ~withoutLiteralsInLinkFlow(arriving.directPrimitives()); @@ -2335,7 +2469,7 @@ ALWAYS_INLINE_INTO_OTHER_FILES void LinkFlow::receive(FlowNodeID node, const Lin return; if (m_breachesOfClosure) [[unlikely]] noteBreachOfClosure({ .node = node, .before = before, .after = arriving }); - m_nodes[index].type = arriving; + setTypeHeldBy(target, arriving, before); if (!m_isOnWorklist.quickSet(index)) { Lane& lane = laneOf(node); lane.worklist.append(node); @@ -2404,7 +2538,7 @@ ALWAYS_INLINE_INTO_OTHER_FILES void LinkFlow::receive(FlowNodeID node, const Lin if (!after.holdsSameWithShadowsAs(before)) { if (m_breachesOfClosure) [[unlikely]] noteBreachOfClosure({ .node = node, .before = before, .after = after }); - m_nodes[index].type = after; + setTypeOfNode(index, after, before); if (!m_isOnWorklist.quickSet(index)) { Lane& lane = laneOf(node); lane.worklist.append(node); @@ -2427,7 +2561,8 @@ NEVER_INLINE void LinkFlow::arriveAtOnce(FlowNodeID node, const LinkType& type, ALWAYS_INLINE_INTO_OTHER_FILES bool LinkFlow::layEdge(Lane& lane, FlowNodeID from, FlowNodeID to, TypeID passes) { - uint32_t head = m_listsOfNodes[from.index()].firstEdge; + ListsOfNode& lists = m_listsOfNodes[from.index()]; + uint32_t head = lists.firstEdge; uint32_t numberLookedThrough = 0; uint32_t next = head; bool isKnownToBeNew = false; @@ -2437,9 +2572,11 @@ ALWAYS_INLINE_INTO_OTHER_FILES bool LinkFlow::layEdge(Lane& lane, FlowNodeID fro isKnownToBeNew = true; } bool isAskedOfSet = !isKnownToBeNew && m_hasEdgesInSet.quickGet(from.index()); - for (; !isKnownToBeNew && !isAskedOfSet && next != UINT32_MAX && numberLookedThrough < numberOfEdgesLookedThroughInLinkFlow; next = lane.edges[next].next, ++numberLookedThrough) { - if (lane.edges[next].to == to && lane.edges[next].passes == passes) + for (; !isKnownToBeNew && !isAskedOfSet && next != UINT32_MAX && numberLookedThrough < numberOfEdgesLookedThroughInLinkFlow; ++numberLookedThrough) { + const Edge& edge = lane.edges[next]; + if (edge.to == to && edge.passes == passes) return false; + next = edge.next; } if (isAskedOfSet || numberLookedThrough == numberOfEdgesLookedThroughInLinkFlow) { uint32_t numberOfFrom = static_cast(from.index()); @@ -2452,7 +2589,7 @@ ALWAYS_INLINE_INTO_OTHER_FILES bool LinkFlow::layEdge(Lane& lane, FlowNodeID fro } if (head == UINT32_MAX && m_accessorsWaitAt.quickGet(from.index())) [[unlikely]] wakeAccessorsThatWaitAt(lane, from); - m_listsOfNodes[from.index()].firstEdge = static_cast(lane.edges.size()); + lists.firstEdge = static_cast(lane.edges.size()); lane.edges.append(Edge { to, passes, head }); return true; } @@ -2476,7 +2613,7 @@ NEVER_INLINE void LinkFlow::addEdgeAtOnce(FlowNodeID from, FlowNodeID to, TypeID { if (from == unseen()) { UnknownByEscapeAlone ofEscapeAlone(UnknownByEscapeAlone::marksAtWork() == UnknownByEscapeAlone::Marks::OfEscape); - const LinkType& held = m_nodes[from.index()].type; + LinkType held = typeOfNode(from.index()); if (passes) arrive(to, passedBy(held, passes)); else @@ -2484,31 +2621,31 @@ NEVER_INLINE void LinkFlow::addEdgeAtOnce(FlowNodeID from, FlowNodeID to, TypeID return; } UnknownByEscapeAlone keepsItsOwnMarks(false); - if (layEdge(laneOf(from), from, to, passes) && !m_nodes[from.index()].type.isNever()) - arrive(to, passedByTestOfEdge(m_nodes[from.index()].type, passes), symbolSeenAt(from)); + if (layEdge(laneOf(from), from, to, passes) && !isNeverAt(from.index())) + arrive(to, passedByTestOfEdge(typeOfNode(from.index()), passes), symbolSeenAt(from)); } NEVER_INLINE void LinkFlow::wakeAccessorsThatWaitAt(Lane& lane, FlowNodeID node) { m_accessorsWaitAt.quickClear(node.index()); - for (uint32_t listed = m_listsOfNodes[node.index()].firstObserver; listed != UINT32_MAX; listed = lane.observers[listed].second) { - if (lane.observers[listed].first.kind == FlowObserver::Kind::FieldOfAccessorLaid) - lane.newObservers.append(std::pair { lane.observers[listed].first, node }); + for (uint32_t listed = m_listsOfNodes[node.index()].firstObserver; listed != UINT32_MAX; listed = lane.observers[listed].next()) { + if (lane.observers[listed].kind() == FlowObserver::Kind::FieldOfAccessorLaid) + lane.newObservers.append(std::pair { lane.observers[listed].observer(), node }); } } bool LinkFlow::isAskedByMoreThanAccessorsThatWait(Lane& lane, FlowNodeID node) { const ListsOfNode& lists = m_listsOfNodes[node.index()]; - if (lists.firstEdge != UINT32_MAX || !m_nodes[node.index()].type.isNever()) + if (lists.firstEdge != UINT32_MAX || !isNeverAt(node.index())) return true; if (m_accessorsWaitAt.quickGet(node.index())) return false; if (lane.nodesThatAccessorsFoundAsked.contains(node.index())) return true; bool isAsked = false; - for (uint32_t listed = lists.firstObserver; listed != UINT32_MAX && !isAsked; listed = lane.observers[listed].second) - isAsked = lane.observers[listed].first.kind != FlowObserver::Kind::FieldOfAccessorLaid; + for (uint32_t listed = lists.firstObserver; listed != UINT32_MAX && !isAsked; listed = lane.observers[listed].next()) + isAsked = lane.observers[listed].kind() != FlowObserver::Kind::FieldOfAccessorLaid; if (isAsked) lane.nodesThatAccessorsFoundAsked.add(node.index()); return isAsked; @@ -2543,7 +2680,7 @@ ALWAYS_INLINE_INTO_OTHER_FILES void LinkFlow::putObserver(Lane& lane, FlowNodeID wakeAccessorsThatWaitAt(lane, node); ListsOfNode& lists = m_listsOfNodes[node.index()]; uint32_t index = static_cast(lane.observers.size()); - lane.observers.append(std::pair { observer, lists.firstObserver }); + lane.observers.append(ListedFlowObserver(observer, lists.firstObserver)); lists.firstObserver = index; } @@ -2551,8 +2688,8 @@ NEVER_INLINE bool LinkFlow::isObservedAlready(FlowNodeID node, const FlowObserve { Lane& lane = laneOf(node); bool isThere = false; - for (uint32_t listed = m_listsOfNodes[node.index()].firstObserver; listed != UINT32_MAX && !isThere; listed = lane.observers[listed].second) - isThere = lane.observers[listed].first == observer; + for (uint32_t listed = m_listsOfNodes[node.index()].firstObserver; listed != UINT32_MAX && !isThere; listed = lane.observers[listed].next()) + isThere = lane.observers[listed].observer() == observer; for (auto& holder : m_lanes) { for (const ObserverPut& put : holder->observersPut[laneIndexOf(node)]) isThere |= put.node == node && put.observer == observer; @@ -2925,6 +3062,11 @@ void LinkFlow::note(Note note, uint32_t number, uint32_t more) lane.notes.append({ note, number, more }); } +BitVector& LinkFlow::layoutsNotedWithBases() +{ + return m_severalThreadsTell ? s_laneAtWork->layoutsNotedWithBases : m_layoutsNotedWithBases; +} + Vector& LinkFlow::fieldsThatLaneAtWorkHasNotNotedEmpty(LayoutID layout) { Lane& lane = m_severalThreadsTell ? *s_laneAtWork : *m_firstLane; @@ -3048,12 +3190,15 @@ void LinkFlow::applyNote(Note note, uint32_t number, uint32_t more) possibleChanges[number].add(OptionSet::fromRaw(static_cast(more))); if (!(more & changeIsByStoreInSight)) changesByMoreThanStoresInSight[number].add(OptionSet::fromRaw(static_cast(more))); + if (more & static_cast(LayoutChange::IsFrozen)) + noteInstancesMayBeLocked(LayoutID(number)); return; case Note::LayoutGainsWhatUnseenCodeGaveSources: layoutsThatGainWhatUnseenCodeGaveSources.set(number); return; case Note::LayoutDefinedOnUnderWrittenKeys: layoutsDefinedOnUnderWrittenKeys.set(number); + noteInstancesMayBeLocked(LayoutID(number)); return; case Note::LayoutDefinedOnUnderKeyThatIsNotWritten: layoutsDefinedOnUnderKeysThatAreNotWritten.set(number); @@ -3406,7 +3551,7 @@ ALWAYS_INLINE_INTO_OTHER_FILES void LinkFlow::send(Lane& sender, FlowNodeID to, noteReachesUnseen(type); return; } - const LinkType& held = m_nodes[to.index()].type; + LinkType held = headOfNode(to.index()); bool namesNothing = type.objects == LinkType::Objects::None && type.functions == LinkType::Functions::None && type.classes == LinkType::Classes::None && !type.builtin && (!(type.primitives & LinkType::oneSymbol) || (held.directPrimitives() & Primitive::Symbol && !symbol) || !(type.directPrimitives() & LinkType::oneSymbol)); if (namesNothing && bringsNoPrimitiveInLinkFlow(type.primitives, held.primitives) && type.raisesNoFlagOver(held) && (!type.holdsLostBuiltins || held.holdsLostBuiltins || isNodeOfLibrary(to.index())) && bringsNothingToShadowsInLinkFlow(type, held)) @@ -3427,20 +3572,23 @@ void LinkFlow::sendFromLane(unsigned index) auto [node, before] = takeNode(lane); ++lane.numberOfNodesTakenOffWorklist; m_isOnWorklist.quickClear(node.index()); - const LinkType& type = m_nodes[node.index()].type; + LinkType type = typeOfNode(node.index()); PropertyKeyID symbol = LinkType::holdsOneSymbol(type.directPrimitives()) ? symbolNotedAt(node) : PropertyKeyID(); const ListsOfNode& lists = m_listsOfNodes[node.index()]; for (uint32_t next = lists.firstEdge; next != UINT32_MAX;) { - size_t numberOfEdges = 0; - for (; next != UINT32_MAX && numberOfEdges < lengthOfStretch; next = edges[numberOfEdges++].next) { - edges[numberOfEdges] = lane.edges[next]; - __builtin_prefetch(&m_nodes[edges[numberOfEdges].to.index()].type); + Edge* endOfEdges = edges.data(); + Edge* const endOfStretch = endOfEdges + lengthOfStretch; + while (next != UINT32_MAX && endOfEdges != endOfStretch) { + Edge& edge = *endOfEdges++; + edge = lane.edges[next]; + m_nodes.prefetch(edge.to.index()); + next = edge.next; } - for (size_t position = 0; position < numberOfEdges; ++position) { - if (edges[position].passes) - send(lane, edges[position].to, passedByTestOfEdge(type, edges[position].passes), symbol); + for (const Edge* edge = edges.data(); edge != endOfEdges; ++edge) { + if (edge->passes) + send(lane, edge->to, passedByTestOfEdge(type, edge->passes), symbol); else - send(lane, edges[position].to, type, symbol); + send(lane, edge->to, type, symbol); } } if (lists.firstObserver != UINT32_MAX && !m_isToBeTold.quickSet(node.index())) @@ -3741,12 +3889,12 @@ void LinkFlow::receiveInLane(unsigned index) if (!(sender->lanesOfFilledBoxesInThisPass >> index & 1)) continue; Vector& box = sender->boxes[index]; - static constexpr size_t ahead = 12; - for (size_t place = 0; place < box.size(); ++place) { - if (place + ahead < box.size()) - __builtin_prefetch(&m_nodes[box[place + ahead].node.index()]); - const ArrivalHeldBack& arrival = box[place]; - receive(arrival.node, arrival.type, arrival.symbol); + static constexpr ptrdiff_t ahead = 12; + const ArrivalHeldBack* const endOfBox = box.end(); + for (const ArrivalHeldBack* arrival = box.begin(); arrival != endOfBox; ++arrival) { + if (endOfBox - arrival > ahead) + m_nodes.prefetch(arrival[ahead].node.index()); + receive(arrival->node, arrival->type, arrival->symbol); } box.shrink(0); } @@ -3781,7 +3929,7 @@ void LinkFlow::receiveInLane(unsigned index) return std::ranges::lexicographical_compare(layoutsOfSet(one), layoutsOfSet(other)); }); } - LinkType before = m_nodes[node.index()].type; + LinkType before = typeOfNode(node.index()); LinkType after = before; if (!objects.isEmpty()) foldObjects(after, objects.span()); @@ -3806,7 +3954,7 @@ void LinkFlow::receiveInLane(unsigned index) continue; if (m_breachesOfClosure) [[unlikely]] noteBreachOfClosure({ .node = node, .before = before, .after = after }); - m_nodes[node.index()].type = after; + setTypeOfNode(node.index(), after, before); if (!m_isOnWorklist.quickSet(node.index())) { lane.worklist.append(node); lane.typesBeforeClimb.append(before); @@ -3931,7 +4079,7 @@ void LinkFlow::applyDeferredUpdates(unsigned index) Vector& puts = holder->observersPut[index]; for (size_t place = 0; place < puts.size(); ++place) { if (place + ahead < puts.size()) - __builtin_prefetch(&m_listsOfNodes[puts[place + ahead].node.index()]); + m_listsOfNodes.prefetch(puts[place + ahead].node.index()); putObserver(lane, puts[place].node, puts[place].observer); } puts.shrink(0); @@ -3943,14 +4091,15 @@ void LinkFlow::applyDeferredUpdates(unsigned index) Vector& edgesLaid = holder->edgesLaid[index]; for (size_t place = 0; place < edgesLaid.size(); ++place) { if (place + ahead < edgesLaid.size()) { - __builtin_prefetch(&m_listsOfNodes[edgesLaid[place + ahead].from.index()]); - __builtin_prefetch(&m_nodes[edgesLaid[place + ahead].from.index()]); - __builtin_prefetch(&m_nodes[edgesLaid[place + ahead].to.index()]); + const EdgeLaid& next = edgesLaid[place + ahead]; + m_listsOfNodes.prefetch(next.from.index()); + m_nodes.prefetch(next.from.index()); + m_nodes.prefetch(next.to.index()); } const EdgeLaid& laid = edgesLaid[place]; if (!layEdge(lane, laid.from, laid.to, laid.passes)) continue; - const LinkType& type = m_nodes[laid.from.index()].type; + LinkType type = typeOfNode(laid.from.index()); if (type.isNever()) continue; PropertyKeyID symbol = LinkType::holdsOneSymbol(type.directPrimitives()) ? symbolNotedAt(laid.from) : PropertyKeyID(); @@ -4036,8 +4185,9 @@ void LinkFlow::sortTellsOfLane(unsigned index) switch (m_order) { case OrderOfWorklist::NewestFirst: numbers.shrink(0); - for (uint32_t place = 0; place < lane.nodesToTell.size(); ++place) - numbers.append(static_cast(lane.nodesToTell[place].first.index()) << 32 | place); + numbers.reserveCapacity(lane.nodesToTell.size()); + for (const auto& [node, before] : lane.nodesToTell) + numbers.append(static_cast(node.index()) << 32 | static_cast(numbers.size())); sortNumbersInLinkFlow(numbers, lane.roomToSort, m_nodes.size()); break; case OrderOfWorklist::OldestFirst: @@ -4062,8 +4212,12 @@ void LinkFlow::sortTellsOfLane(unsigned index) uint32_t place = m_order == OrderOfWorklist::NewestFirst ? static_cast(numbers[turn]) : turn; FlowNodeID node = lane.nodesToTell[place].first; m_isToBeTold.quickClear(node.index()); - for (uint32_t listed = m_listsOfNodes[node.index()].firstObserver; listed != UINT32_MAX; listed = lane.observers[listed].second) - hand(lane.observers[listed].first, node, place); + for (uint32_t listed = m_listsOfNodes[node.index()].firstObserver; listed != UINT32_MAX;) { + FlowObserver observer = lane.observers[listed].observer(); + uint32_t next = lane.observers[listed].next(); + hand(observer, node, place); + listed = next; + } } for (auto& [observer, node] : lane.newObservers) hand(observer, node, UINT32_MAX); @@ -4077,8 +4231,9 @@ void LinkFlow::tellInLane(unsigned index) for (size_t place = 0; place < tells.size(); ++place) { if (place + 4 < tells.size()) { FunctionID function = tells[place + 4].observer.function; - __builtin_prefetch(&summaries.at(function).calls); - __builtin_prefetch(&summaries.at(function).accesses); + const ThinSummary& summary = summaries.at(function); + __builtin_prefetch(&summary.calls); + __builtin_prefetch(&summary.accesses); __builtin_prefetch(&m_nodesOfFunctions[function.index()]); __builtin_prefetch(&firstCallOfFunction[function.index()]); } @@ -4282,12 +4437,12 @@ void LinkFlow::flow() auto tellObserversOf = [&](FlowNodeID node, const LinkType& before) { Lane& lane = laneOf(node); - LinkType type = m_nodes[node.index()].type; + LinkType type = typeOfNode(node.index()); for (uint32_t index = m_listsOfNodes[node.index()].firstObserver; index != UINT32_MAX;) { size_t numberOfObservers = 0; for (; index != UINT32_MAX && numberOfObservers < lengthOfStretch; ++numberOfObservers) { - observers[numberOfObservers] = lane.observers[index].first; - index = lane.observers[index].second; + observers[numberOfObservers] = lane.observers[index].observer(); + index = lane.observers[index].next(); FunctionID function = observers[numberOfObservers].function; __builtin_prefetch(&summaries.at(function).calls); __builtin_prefetch(&summaries.at(function).accesses); @@ -4359,7 +4514,7 @@ void LinkFlow::flow() tellOnLanes(); while (!m_newObservers.isEmpty()) { auto [observer, node] = takeNewObserver(); - LinkType type = m_nodes[node.index()].type; + LinkType type = typeOfNode(node.index()); tell(observer, type, LinkType { }); } flushFirstLaneNotes(); @@ -4450,7 +4605,7 @@ Vector LinkFlow::findBreachesOfClosure() Vector edges; Vector observers; for (size_t index = 0; index < numberOfNodes; ++index) { - LinkType type = m_nodes[index].type; + LinkType type = typeOfNode(index); if (type.isNever() || index == unseen().index()) continue; m_nodeGoneOver = FlowNodeID(index); @@ -4462,8 +4617,8 @@ Vector LinkFlow::findBreachesOfClosure() for (const Edge& edge : edges) arrive(edge.to, passedByTestOfEdge(type, edge.passes), symbolSeenAt(FlowNodeID(index))); observers.shrink(0); - for (uint32_t listed = m_listsOfNodes[index].firstObserver; listed != UINT32_MAX; listed = lane.observers[listed].second) - observers.append(lane.observers[listed].first); + for (uint32_t listed = m_listsOfNodes[index].firstObserver; listed != UINT32_MAX; listed = lane.observers[listed].next()) + observers.append(lane.observers[listed].observer()); for (const FlowObserver& observer : observers) { m_observerToldOnceMore = observer; tell(observer, type, LinkType { }); @@ -4682,6 +4837,7 @@ bool LinkFlow::tieInWhatGrew() hasTied |= tieInLostLayouts(); hasTied |= tieInLostFunctions(); hasTied |= openLayoutsThatChanged(); + hasTied |= leaveFieldsEmptyThatCopiesPassOver(); hasTied |= tieStoresUnderUnknownNames(); hasTied |= letMembersOfLostEscape(); hasTied |= tieAppendedFieldsOfLostLayouts(); @@ -4937,6 +5093,17 @@ TypeID LinkFlow::toTypeID(TypeArena& types, const LinkType& type) const return types.unionOf(written, types.builtinClass(type.builtinClass, contents)); } +const LinkFlow::ReadsBehindInstanceOf* LinkFlow::readsBehindInstanceOf(FunctionID reader) +{ + const BodyIndex* index = bodiesLookedAt(looks).find(reader); + if (!index || !index->opcodes().contains(Opcode::InstanceOf)) + return nullptr; + auto& known = s_laneAtWork ? s_laneAtWork->readsBehindInstanceOf : m_readsBehindInstanceOf; + return known.ensure(reader.index(), [&] { + return makeUniqueWithoutFastMallocCheck(findReadsBehindInstanceOf(program, *index, summaries.at(reader))); + }).iterator->value.get(); +} + LinkType LinkFlow::passedByTestOfEdgeSlowCase(const LinkType& type, TypeID test) { if (type.isNever() || holdsOnlyPrimitives(test)) @@ -6073,7 +6240,7 @@ bool LinkFlow::hasNothingAbove(LayoutID id) m_layoutsWithNothingAbove.ensureSize(program.layouts.size()); for (uint32_t index = 0; index < program.layouts.size(); ++index) { const Layout& layout = program.layouts[index]; - if (keepsValuesInFieldsInLinkFlow(layout.kind) && !layout.base && !m_layoutsBelowConstructorFunctions.get(index) && membersOf(LayoutID(index), ClassSide::Prototype).empty()) + if (keepsValuesInFieldsInLinkFlow(layout.kind) && !layout.base && !m_layoutsBelowFollowedPrototypes.get(index) && membersOf(LayoutID(index), ClassSide::Prototype).empty()) m_layoutsWithNothingAbove.quickSet(index); } } @@ -6089,7 +6256,7 @@ LinkFlow::SortedMembersOfFamily& LinkFlow::sortedMembersOf(FamilyID family) Vector withNothingAbove; for (LayoutID id : program.types.listOf(families.layoutsOf(family))) { if (!hasNothingAbove(id)) { - if (program.at(id).kind == LayoutKind::Instance && !program.at(id).has(LayoutFlags::IsOfCreatedObject) && !m_layoutsBelowConstructorFunctions.get(id.index()) && !namesMayBeIncomplete(id, ClassSide::Prototype)) { + if (program.at(id).kind == LayoutKind::Instance && !program.at(id).has(LayoutFlags::IsOfCreatedObject) && !m_layoutsBelowFollowedPrototypes.get(id.index()) && !namesMayBeIncomplete(id, ClassSide::Prototype)) { sorted->instancesOfClasses.append(id); sorted->someInstanceOfClassMayHaveAppendedFields |= mayHaveAppendedFieldsInLinkFlow(program.at(id)); if (const Layout& layout = program.at(id); layout.has(LayoutFlags::IsNeverAllocated) || layout.has(LayoutFlags::IsWrittenUnderDictionaryType) || layout.writtenBuiltinBase || layout.kind == LayoutKind::BuiltinResult) @@ -6265,7 +6432,7 @@ bool LinkFlow::tieReadThroughClassesThatDeclare(const SortedMembersOfFamily& sor tieRead(aggregate, topmost, key); LayoutID last; for (const DeclarerOfMember& declarer : declarers) { - if (declarer.classLayout != std::exchange(last, declarer.classLayout) && isMemberOf(family, declarer.classLayout) && !m_layoutsBelowConstructorFunctions.get(declarer.classLayout.index())) + if (declarer.classLayout != std::exchange(last, declarer.classLayout) && isMemberOf(family, declarer.classLayout) && !m_layoutsBelowFollowedPrototypes.get(declarer.classLayout.index())) tieRead(aggregate, declarer.classLayout, key); } if (auto born = m_fieldsBornBelowClassesByKey.find(key.bits()); born != m_fieldsBornBelowClassesByKey.end()) { @@ -6322,6 +6489,61 @@ void LinkFlow::tieReadThroughSortedMembers(FlowNodeID aggregate, FamilyID family sorted.readsWithoutLostMember.append(ReadThroughFamily { key, aggregate }); } +std::optional LinkFlow::findFamilyReadBehindInstanceOf(FamilyID family, PropertyKeyID key, LayoutID tested) +{ + LockerOfWhatIsShared lockerOfWhatIsShared(*this); + if (!isAbsentFromObjectPrototype(key) || nameClassOf(key) == NameClass::Proto || program.at(key).kind == PropertyKeyKind::PrivateName || mayBeNamedByNumber(key)) + return std::nullopt; + SortedMembersOfFamily& sorted = sortedMembersOf(family); + if (!sorted.everyMemberIsOfWrittenClass) { + auto isOfWrittenClass = [&](LayoutID id) { + const Layout& layout = program.at(id); + return layout.constructor && isClassConstructorOf(program.at(layout.constructor).kind) && (!layout.has(LayoutFlags::HasHeritage) || layout.base); + }; + bool isWritten = sorted.tiedOneByOne.isEmpty() && sorted.withNothingAbove.isEmpty() && std::ranges::all_of(sorted.instancesOfClasses, isOfWrittenClass); + for (LayoutID topmost : sorted.topmostInstancesOfClasses) { + for (LayoutID above = program.at(topmost).base; isWritten && above; above = program.at(above).base) + isWritten = isOfWrittenClass(above); + } + sorted.everyMemberIsOfWrittenClass = isWritten; + if (isWritten) { + for (LayoutID member : sorted.instancesOfClasses) + sorted.placesOfMembers.append(m_stretchesAtAndBelowLayouts[member.index()].first); + std::ranges::sort(sorted.placesOfMembers); + } + } + if (!*sorted.everyMemberIsOfWrittenClass) + return std::nullopt; + auto added = sorted.membersThatListNames.add(key.bits(), SortedMembersOfFamily::MembersThatListName()); + SortedMembersOfFamily::MembersThatListName& listed = added.iterator->value; + if (added.isNewEntry) { + auto list = [&](LayoutID id) { + listed.places.append(m_stretchesAtAndBelowLayouts[id.index()].first); + if (!listed.fields) + listed.fields = makeNode(); + UnknownByEscapeAlone ofEscapeAlone(UnknownByEscapeAlone::marksAtWork() == UnknownByEscapeAlone::Marks::OfEscape); + tieRead(listed.fields, id, key); + }; + for (LayoutID listing : program.layoutsWithName(key)) { + if (isMemberOf(family, listing)) + list(listing); + } + for (LayoutID leftOut : sorted.instancesOfClassesThatIndexOfNamesLeavesOut) { + if (findField(program, leftOut, key)) + list(leftOut); + } + std::ranges::sort(listed.places); + } + auto numberBelowTested = [&](const Vector& places) { + const StretchInTreeOfBases& below = m_stretchesAtAndBelowLayouts[tested.index()]; + return std::ranges::lower_bound(places, below.end) - std::ranges::lower_bound(places, below.first); + }; + auto numberListed = numberBelowTested(listed.places); + if (!numberListed || numberListed != numberBelowTested(sorted.placesOfMembers)) + return std::nullopt; + return FamilyReadBehindInstanceOf { listed.fields, sorted.foundBesideTheirFields }; +} + FlowNodeID LinkFlow::fieldReadThrough(FamilyID family, PropertyKeyID key) { return nodeAskedOnceByLane(MadeOnDemand::FieldReadThrough, static_cast(family.index()) << 32 | key.bits(), [&]() -> FlowNodeID { @@ -7170,8 +7392,8 @@ Vector LinkFlow::observersOf(FlowNodeID node) const if (node.index() >= m_listsOfNodes.size()) return observers; const Lane& lane = *m_lanes[laneIndexOf(node)]; - for (uint32_t listed = m_listsOfNodes[node.index()].firstObserver; listed != UINT32_MAX; listed = lane.observers[listed].second) - observers.append(lane.observers[listed].first); + for (uint32_t listed = m_listsOfNodes[node.index()].firstObserver; listed != UINT32_MAX; listed = lane.observers[listed].next()) + observers.append(lane.observers[listed].observer()); return observers; } diff --git a/Source/JavaScriptCore/aot/facts/AOTLinkFlow.h b/Source/JavaScriptCore/aot/facts/AOTLinkFlow.h index d34ddca38be2b..11cbc7f9360a2 100644 --- a/Source/JavaScriptCore/aot/facts/AOTLinkFlow.h +++ b/Source/JavaScriptCore/aot/facts/AOTLinkFlow.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -221,6 +222,16 @@ struct LinkType { using Words = std::array; ALWAYS_INLINE Words words() const { return std::bit_cast(*this); } + using References = std::array; + ALWAYS_INLINE uint64_t head() const { return words()[0]; } + ALWAYS_INLINE References references() const + { + Words held = words(); + return { held[1], held[2], held[3] }; + } + static ALWAYS_INLINE References noReferences() { return LinkType { }.references(); } + static ALWAYS_INLINE LinkType of(uint64_t head, const References& references) { return std::bit_cast(Words { head, references[0], references[1], references[2] }); } + static ALWAYS_INLINE LinkType ofHead(uint64_t head) { return of(head, noReferences()); } static ALWAYS_INLINE Words wordsOfShadows() { LinkType marked; @@ -269,14 +280,14 @@ struct LinkType { }; static_assert(std::has_unique_object_representations_v); -template -class SetsOfFourInLinkFlow { - WTF_MAKE_NONCOPYABLE(SetsOfFourInLinkFlow); +template +class SmallSetsInLinkFlow { + WTF_MAKE_NONCOPYABLE(SmallSetsInLinkFlow); public: - static constexpr size_t mostMembers = 4; + static constexpr size_t mostMembers = mostMembersOfSet; - SetsOfFourInLinkFlow() = default; - ~SetsOfFourInLinkFlow() + SmallSetsInLinkFlow() = default; + ~SmallSetsInLinkFlow() { for (auto& part : m_parts) delete[] part.load(std::memory_order_relaxed); @@ -295,11 +306,18 @@ class SetsOfFourInLinkFlow { RELEASE_ASSERT(sortedMembers.size() >= 2 && sortedMembers.size() <= mostMembers); Members members { }; std::ranges::copy(sortedMembers, members.begin()); - std::pair words { static_cast(members[0].bits()) << 32 | members[1].bits(), static_cast(members[2].bits()) << 32 | members[3].bits() }; + auto words = [&](size_t first) -> Words { + return { static_cast(members[first].bits()) << 32 | members[first + 1].bits(), static_cast(members[first + 2].bits()) << 32 | members[first + 3].bits() }; + }; + Key key; + if constexpr (mostMembers == 4) + key = words(0); + else + key = { words(0), words(4) }; while (m_isLocked.load(std::memory_order_relaxed) || m_isLocked.exchange(true, std::memory_order_acquire)) simde_mm_pause(); uint32_t next = m_size.load(std::memory_order_relaxed); - auto added = m_numbers.add(words, next); + auto added = m_numbers.add(key, next); if (added.isNewEntry) { auto [part, position] = placeOf(next); Members* records = m_parts[part].load(std::memory_order_relaxed); @@ -318,6 +336,9 @@ class SetsOfFourInLinkFlow { private: using Members = std::array; + using Words = std::pair; + static_assert(mostMembers == 4 || mostMembers == 8); + using Key = std::conditional_t>; static constexpr unsigned log2OfSetsInFirstPart = 10; static constexpr size_t numberOfParts = 32 - log2OfSetsInFirstPart; @@ -334,12 +355,65 @@ class SetsOfFourInLinkFlow { } std::array, numberOfParts> m_parts { }; - UncheckedKeyHashMap, uint32_t> m_numbers; + UncheckedKeyHashMap m_numbers; std::atomic m_size { 0 }; std::atomic m_isLocked { false }; }; -using SetsOfLayoutsInLinkFlow = SetsOfFourInLinkFlow; -using SetsOfFunctionsInLinkFlow = SetsOfFourInLinkFlow; +WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN + +class LinkReferenceTable { + WTF_MAKE_NONCOPYABLE(LinkReferenceTable); +public: + using References = LinkType::References; + + JS_EXPORT_PRIVATE LinkReferenceTable(); + JS_EXPORT_PRIVATE ~LinkReferenceTable(); + + ALWAYS_INLINE const References& at(uint32_t number) const { return m_blocks[number >> log2OfRecordsInBlock].load(std::memory_order_relaxed)[number & (recordsInBlock - 1)]; } + + struct Recent { + References references { LinkType::noReferences() }; + uint32_t number { 0 }; + }; + using RecentNumbers = std::array; + JS_EXPORT_PRIVATE uint32_t numberOf(const References&, RecentNumbers&); + +private: + static constexpr unsigned log2OfRecordsInBlock = 18; + static constexpr uint32_t recordsInBlock = 1u << log2OfRecordsInBlock; + static constexpr size_t numberOfBlocks = 1u << (32 - log2OfRecordsInBlock); + static constexpr unsigned numberOfShards = 128; + static constexpr uint32_t recordsTakenAtOnce = 64; + + struct Shard { + Vector slots; + uint32_t size { 0 }; + uint32_t next { 0 }; + uint32_t end { 0 }; + std::atomic isLocked { false }; + }; + + static ALWAYS_INLINE uint64_t hashOf(const References& references) + { + uint64_t hash = references[0] * 0x9e3779b97f4a7c15ull; + hash = (std::rotl(hash, 29) ^ references[1]) * 0xbf58476d1ce4e5b9ull; + hash = (std::rotl(hash, 31) ^ references[2]) * 0x94d049bb133111ebull; + return hash ^ hash >> 32; + } + uint32_t findOrAdd(const References&, uint64_t hash); + uint32_t takeNumber(Shard&); + void growSlots(Shard&); + + MallocSpan> m_room; + std::atomic* m_blocks; + std::array m_shards; + std::atomic m_nextNumber { recordsTakenAtOnce }; +}; + +WTF_ALLOW_UNSAFE_BUFFER_USAGE_END + +using SetsOfLayoutsInLinkFlow = SmallSetsInLinkFlow; +using SetsOfFunctionsInLinkFlow = SmallSetsInLinkFlow; class SetsOfClassesInLinkFlow { WTF_MAKE_NONCOPYABLE(SetsOfClassesInLinkFlow); @@ -638,8 +712,11 @@ class LinkFlow { LayoutID classOfConstructor(FunctionID) const; NameClass nameClassOf(PropertyKeyID key) const { return m_nameClassesOfKeys[key.index()]; } + BitVector findClassesWithFollowedPrototypes(); + bool instancesMayHoldName(LayoutID, PropertyKeyID) const; LayoutID layoutOfConstructorFunction(FunctionID) const; bool isOfConstructorFunction(LayoutID layout) const { return m_layoutsOfConstructorFunctions.get(layout.index()); } + bool followsNamesOnPrototypeOf(LayoutID layout) const { return m_layoutsWithFollowedPrototypes.get(layout.index()); } FlowNodeID storedOnPrototypeOf(LayoutID, PropertyKeyID, bool isLaidByBodyOfFile); void noteInstancesLaidAsPrototype(const LinkType&); enum class AccessorsAbove : bool { AreEnteredByNobody, AreEnteredByCaller }; @@ -654,6 +731,7 @@ class LinkFlow { void letAccessorsOnPrototypesAboveEscape(LayoutID, PropertyKeyID); Vector halvesOfAccessorsOnPrototypesAbove(LayoutID, PropertyKeyID, bool ofSetters); bool isBelowConstructorFunction(LayoutID layout) const { return m_layoutsBelowConstructorFunctions.get(layout.index()); } + bool isBelowFollowedPrototype(LayoutID layout) const { return m_layoutsBelowFollowedPrototypes.get(layout.index()); } bool mayNotBeLinkedYetBelow(LayoutID from, LayoutID classAbove) const; FlowNodeID objectsMadeForClassBelowConstructorFunction(LayoutID); FlowNodeID objectsMadeThroughConstructorOf(LayoutID); @@ -781,25 +859,27 @@ class LinkFlow { FlowNodeID madeForLostFunctionsWithoutClass(); void noteCallOfLostFunctions(const LinkType& callee); - LinkType typeAt(FlowNodeID id) const + ALWAYS_INLINE LinkType typeAt(FlowNodeID id) const { if (id.index() >= m_nodes.size()) return { }; - const LinkType& type = m_nodes[id.index()].type; - if (!m_flowIsAtRest) - return type; + LinkType type = typeOfNode(id.index()); + return m_flowIsAtRest ? typeSeenAtRest(type) : type; + } + LinkType typeSeenAtRest(const LinkType& type) const + { if (ViewAtWork::current()) return type.directPart(); if (type.hasUnknownObjects && m_typesAreSettled && (type.functions != LinkType::Functions::None || type.classes != LinkType::Classes::None)) return settled(type); return type; } - void prefetchTypeAt(FlowNodeID id) const { __builtin_prefetch(&m_nodes[id.index()].type); } - bool holdsInstancesOfOneSiteAt(FlowNodeID id) const { return id.index() < m_nodes.size() && m_nodes[id.index()].type.classes == LinkType::Classes::Site; } - bool hasUnknownObjectsAt(FlowNodeID id) const { return id.index() < m_nodes.size() && m_nodes[id.index()].type.hasUnknownObjects; } + void prefetchTypeAt(FlowNodeID id) const { __builtin_prefetch(&m_nodes[id.index()]); } + bool holdsInstancesOfOneSiteAt(FlowNodeID id) const { return id.index() < m_nodes.size() && headOfNode(id.index()).classes == LinkType::Classes::Site; } + bool hasUnknownObjectsAt(FlowNodeID id) const { return id.index() < m_nodes.size() && headOfNode(id.index()).hasUnknownObjects; } FunctionID closureAloneAt(FlowNodeID id) const { - if (id.index() >= m_nodes.size() || m_nodes[id.index()].type.functions != LinkType::Functions::Closure) + if (id.index() >= m_nodes.size() || headOfNode(id.index()).functions != LinkType::Functions::Closure) return { }; return typeAt(id).function; } @@ -1018,7 +1098,7 @@ class LinkFlow { private: const LinkFlow* m_flow; }; - enum class MadeOnDemand : uint8_t { Contents, ContentsOfFamily, TakenForConstant, FieldReadThroughLost, FieldStoredThroughLost, FieldReadThrough, FieldStoredThrough, FoundOnPrototypesAbove, FoundBesideFieldsUnder, ExtraPropertyReadNode, ExtraPropertyStoreNode, FoundWhereFieldIsEmpty, AppendedField, AppendedStoreNode, FoundOnClosuresOf, FoundOnClassObjectsFrom, LaidOnClosuresOf, WrittenAtMember, WhereLossIsTold, ParameterOfLostFunctions, ReceiverOfLostFunctions, ResultOfLostFunctions, LaidOnLostClosures, NameAddedByTests, FunctionOnObjectPrototype, AnyFieldDefinedOn, AnyFieldReadThrough, AnyFieldStoredThrough, ResultOfOperator }; + enum class MadeOnDemand : uint8_t { WhereInstancesMayBeLocked, Contents, ContentsOfFamily, TakenForConstant, FieldReadThroughLost, FieldStoredThroughLost, FieldReadThrough, FieldStoredThrough, FoundOnPrototypesAbove, FoundBesideFieldsUnder, ExtraPropertyReadNode, ExtraPropertyStoreNode, FoundWhereFieldIsEmpty, AppendedField, AppendedStoreNode, FoundOnClosuresOf, FoundOnClassObjectsFrom, LaidOnClosuresOf, WrittenAtMember, WhereLossIsTold, ParameterOfLostFunctions, ReceiverOfLostFunctions, ResultOfLostFunctions, LaidOnLostClosures, NameAddedByTests, FunctionOnObjectPrototype, AnyFieldDefinedOn, AnyFieldReadThrough, AnyFieldStoredThrough, ResultOfOperator }; static constexpr size_t numberOfKindsMadeOnDemand = static_cast(MadeOnDemand::ResultOfOperator) + 1; uint32_t askedOnceByLane(MadeOnDemand kind, uint64_t words, const Invocable auto& make) const { @@ -1194,6 +1274,7 @@ class LinkFlow { const TestRunsInLink& testRuns() const { return *m_testRuns; } BitVector reached; + BitVector iterationCallbacks; BitVector allocatedLayouts; BitVector lostLayouts; UnitedFamilies unitedLayouts; @@ -1536,7 +1617,26 @@ class LinkFlow { Vector& fieldsThatLaneAtWorkHasNotNotedEmpty(LayoutID) LIFETIME_BOUND; void noteConstructionsOf(FunctionID); FlowNodeID foundWhereFieldIsEmpty(FieldReference); + bool instancesMayBeLocked(LayoutID classLayout) const { return m_classesWithInstancesThatMayBeLocked.get(classLayout.index()); } + void noteInstancesMayBeLocked(LayoutID); + FlowNodeID whereInstancesMayBeLocked(LayoutID classLayout); bool mayReadInFrontOfDefinition(FunctionID reader, const SummaryAccess&, LayoutID) const; + struct ReadBehindInstanceOf { + ValueID site; + ValueID test; + LayoutID tested; + }; + struct ReadsBehindInstanceOf { + Vector bySite; + Vector> accessesByTest; + }; + static ReadsBehindInstanceOf findReadsBehindInstanceOf(const Program&, const BodyIndex&, const ThinSummary&); + const ReadsBehindInstanceOf* readsBehindInstanceOf(FunctionID); + struct FamilyReadBehindInstanceOf { + FlowNodeID fields; + FlowNodeID besideFields; + }; + std::optional findFamilyReadBehindInstanceOf(FamilyID, PropertyKeyID, LayoutID tested); bool superCallMayAnswerWithOlderObject(LayoutID id) const { return m_layoutsWhoseSuperCallMayAnswerWithOlderObject.get(id.index()); } bool constructionMayAnswerWithOlderObject(LayoutID id) const { return m_layoutsWhoseConstructionMayAnswerWithOlderObject.get(id.index()); } bool everyMemberStandsAtOrBelow(FamilyID family, LayoutID above) const { return m_stretchesAtAndBelowLayouts[above.index()].holds(m_stretchesOfFamilies[family.index()]); } @@ -1563,15 +1663,36 @@ class LinkFlow { }; struct Node { - LinkType type; + uint64_t head { LinkType { }.head() }; + uint32_t references { 0 }; TypeID typeOfPosition; }; - static_assert(sizeof(Node) == 36); + static_assert(sizeof(Node) == 16); struct ListsOfNode { uint32_t firstEdge { UINT32_MAX }; uint32_t firstObserver { UINT32_MAX }; }; LargeList m_nodes; + LinkReferenceTable m_references; + LinkReferenceTable::RecentNumbers m_recentNumbers; + uint32_t numberOfReferences(const LinkType::References&); + static ALWAYS_INLINE LinkType headHeldBy(const Node& node) { return LinkType::ofHead(node.head); } + ALWAYS_INLINE LinkType typeHeldBy(const Node& node) const { return LinkType::of(node.head, m_references.at(node.references)); } + ALWAYS_INLINE void setTypeHeldBy(Node& node, const LinkType& type, const LinkType& before) + { + node.head = type.head(); + LinkType::References references = type.references(); + if (references != before.references()) + node.references = references == LinkType::noReferences() ? 0 : numberOfReferences(references); + } + ALWAYS_INLINE LinkType headOfNode(size_t node) const { return headHeldBy(m_nodes[node]); } + ALWAYS_INLINE LinkType typeOfNode(size_t node) const { return typeHeldBy(m_nodes[node]); } + ALWAYS_INLINE void setTypeOfNode(size_t node, const LinkType& type, const LinkType& before) { setTypeHeldBy(m_nodes[node], type, before); } + ALWAYS_INLINE bool isNeverAt(size_t node) const + { + const Node& held = m_nodes[node]; + return !held.references && headHeldBy(held).isNever(); + } using SetOfNumbers = HashSet, WTF::UnsignedWithZeroKeyHashTraits>; bool hasPreparation(PropertyKeyID symbol) const { return m_everySymbolOfLibraryMayHavePreparation || m_symbolsWithPreparation.contains(symbol.bits()); } struct Preparation { @@ -1797,6 +1918,8 @@ class LinkFlow { void noteFieldMayBeEmpty(FieldReference); Vector m_nodesFoundWhereFieldsAreEmpty; + BitVector m_classesWithInstancesThatMayBeLocked; + HashMap, WTF::UnsignedWithZeroKeyHashTraits> m_nodesToldWhereInstancesMayBeLocked; void indexConstructorFunctions(); uint32_t recordOfNameOnPrototype(LayoutID, PropertyKeyID); @@ -1806,9 +1929,17 @@ class LinkFlow { bool letPrototypesFoundThroughLostEscape(); class CopiesOfMembers; bool layMembersThatCopiesBring(); + bool leaveFieldsEmptyThatCopiesPassOver(); + std::optional> m_functionsThatMakeCopies; + BitVector m_layoutsOfPlainFieldsAlone; + bool copyMayPassOverFieldOf(LayoutID); + BitVector m_layoutsThatParsersMake; HashSet, WTF::UnsignedWithZeroKeyHashTraits> m_edgesLaidByCopiesOfMembers; BitVector m_layoutsOfConstructorFunctions; BitVector m_layoutsBelowConstructorFunctions; + BitVector m_layoutsWithFollowedPrototypes; + BitVector m_layoutsBelowFollowedPrototypes; + BitVector m_namesStoredOnObjects; Vector m_nearestConstructorFunctionsFrom; BitVector m_prototypesPassedByLookups; BitVector m_prototypesWithDefinitions; @@ -2023,6 +2154,8 @@ class LinkFlow { Vector m_namesOnClosures; UncheckedKeyHashMap, IntHash, WTF::UnsignedWithZeroKeyHashTraits> m_nodesThatLeaveWithClassObjects; BitVector m_classObjectsThatAreOpen; + BitVector m_layoutsNotedWithBases; + BitVector& layoutsNotedWithBases(); Vector m_newestNameOnClosuresOf; void ensureExtraPropertyNodes(LayoutID); void open(LayoutID); @@ -2185,6 +2318,7 @@ class LinkFlow { uint16_t primitives { 0 }; }; HashMap, WTF::UnsignedWithZeroKeyHashTraits> m_testsOfEdges; + HashMap, DefaultHash, WTF::UnsignedWithZeroKeyHashTraits> m_readsBehindInstanceOf; HashMap, WTF::UnsignedWithZeroKeyHashTraits> m_nodesOnDemand; Vector m_extraPropertyNodes; @@ -2228,6 +2362,13 @@ class LinkFlow { HashMap, DefaultHash, WTF::UnsignedWithZeroKeyHashTraits> membersWithNothingAboveByKey; uint32_t numberWithNothingAbove { 0 }; FlowNodeID foundBesideTheirFields; + struct MembersThatListName { + Vector places; + FlowNodeID fields; + }; + HashMap, WTF::UnsignedWithZeroKeyHashTraits> membersThatListNames; + std::optional everyMemberIsOfWrittenClass; + Vector placesOfMembers; uint32_t numberLost { 0 }; Vector readsWithoutLostMember; }; diff --git a/Source/JavaScriptCore/aot/facts/AOTLinkKnownEffectsInlines.h b/Source/JavaScriptCore/aot/facts/AOTLinkKnownEffectsInlines.h index a4c9f424b2c0b..fb9ab2a737b15 100644 --- a/Source/JavaScriptCore/aot/facts/AOTLinkKnownEffectsInlines.h +++ b/Source/JavaScriptCore/aot/facts/AOTLinkKnownEffectsInlines.h @@ -287,7 +287,38 @@ bool CalleeSummarizerForLinkEffects::mergeKnownEffectsOfCodeOutside(const CodeEn return everythingStandsOutside; } -void CalleeSummarizerForLinkEffects::addEffectsOfOperatorsLeftApart(FunctionID id, const ThinSummary& summary, EffectsOfComponentInKnownEffects& effects) const +std::optional CalleeSummarizerForLinkEffects::primitivesExpectedOfParameter(FunctionID id, uint32_t entryParameter) const +{ + std::span parameters = m_tables.parametersOf(id); + TypeID expected = entryParameter < parameters.size() ? parameters[entryParameter].writtenOrSeenToArrive() : TypeID(); + if (!expected || !holdsOnlyPrimitives(expected)) + return std::nullopt; + Type type = m_program.types.at(expected); + return LinkType::closed(LinkType::primitivesOf(type.primitives, type.integers)); +} + +uint32_t CalleeSummarizerForLinkEffects::parametersExpectedPrimitiveThatBringObjectsAlone(FunctionID id, const ThinSummary& summary, uint32_t node, GivenArgumentsInKnownEffects& given) const +{ + if (node == noSummaryNode || summary.nodes[node].kind != SummaryNode::Kind::Join) + return 0; + static constexpr uint32_t othersBringObjects = ~0u; + if (given.parametersThatBringObjectsAlone.isEmpty()) { + given.parametersThatBringObjectsAlone.fill(0, summary.nodes.size()); + for (const SummaryEdge& edge : summary.edges) { + if (edge.to >= summary.nodes.size() || summary.nodes[edge.to].kind != SummaryNode::Kind::Join) + continue; + uint32_t& parameters = given.parametersThatBringObjectsAlone[edge.to]; + if (parameters == othersBringObjects) + continue; + std::optional parameter = edge.passes == TypeTable::object ? entryParameterAt(summary, edge.from) : std::nullopt; + parameters = parameter && primitivesExpectedOfParameter(id, *parameter) ? parameters | 1u << *parameter : othersBringObjects; + } + } + uint32_t parameters = given.parametersThatBringObjectsAlone[node]; + return parameters == othersBringObjects ? 0 : parameters; +} + +void CalleeSummarizerForLinkEffects::addEffectsOfOperatorsLeftApart(FunctionID id, const ThinSummary& summary, EffectsOfComponentInKnownEffects& effects, GivenArgumentsInKnownEffects& given) const { std::optional boundOfOperators = summary.ownEffectsOfOperators(); if (!boundOfOperators) @@ -298,7 +329,16 @@ void CalleeSummarizerForLinkEffects::addEffectsOfOperatorsLeftApart(FunctionID i bool mayTakeOtherValues = false; for (uint32_t node : summary.operandsOfOperatorsLeftApart) { LinkType type = typeAt(id, summary, node); - mayTakeOtherValues |= type.isNever() || mayBeObjectForLinkEffects(type) || (type.primitives & ~plainPrimitives); + if (type.isNever() || mayBeObjectForLinkEffects(type) || (type.primitives & ~plainPrimitives)) { + std::optional parameter = type.isNever() ? std::nullopt : entryParameterAt(summary, node); + std::optional expected = parameter ? primitivesExpectedOfParameter(id, *parameter) : std::nullopt; + if (expected && !(*expected & ~plainPrimitives)) { + given.parametersTakenAsExpected |= 1u << *parameter; + mayTakeStrings |= !!(*expected & ~whatBecomesNumber); + continue; + } + mayTakeOtherValues = true; + } mayTakeStrings |= !!(type.primitives & ~whatBecomesNumber); } @@ -459,6 +499,10 @@ bool CalleeSummarizerForLinkEffects::judgeCallGivenArguments(FunctionID id, cons return true; } case SummaryCall::Callee::Name: { + if (uint32_t bringers = call.isMadeByProtocol ? parametersExpectedPrimitiveThatBringObjectsAlone(id, summary, call.receiver, given) : 0) { + given.parametersTakenAsExpected |= bringers; + return true; + } std::optional parameter = entryParameterAt(summary, call.receiver); std::optional expected = parameter ? typeExpectedOfParameter(id, *parameter) : std::nullopt; if (!expected || !call.key || findOwnMember(m_program, m_builtins.canonicalID(Builtins::ObjectPrototype), call.key)) @@ -495,6 +539,7 @@ auto CalleeSummarizerForLinkEffects::knownEffectsOfBody(uint32_t function) -> Ef bool isKnown = !keepsDefaults(id, summary, { SummaryTrait::Suspends, SummaryTrait::HasDirectEval }) && !summary.ownEffectsMayCallUnknownCode() && !handsObjectToCodeNobodyNames(id, summary) && !listingOfKeysMayRunCode(id, summary); EffectsOfComponentInKnownEffects effects; + GivenArgumentsInKnownEffects given; if (isKnown) { Effects ownEffects = summary.ownEffects(); effects.all = ownEffects; @@ -517,7 +562,7 @@ auto CalleeSummarizerForLinkEffects::knownEffectsOfBody(uint32_t function) -> Ef effects.writesOnReturn.add(m_program, MemoryLocation::moduleVariable(VariableID(node.first))); } } - addEffectsOfOperatorsLeftApart(id, summary, effects); + addEffectsOfOperatorsLeftApart(id, summary, effects, given); for (const SummaryChange& change : summary.changes) { if (changeMayRunCode(id, change)) { isKnown = false; @@ -527,7 +572,6 @@ auto CalleeSummarizerForLinkEffects::knownEffectsOfBody(uint32_t function) -> Ef } } - GivenArgumentsInKnownEffects given; bool entersOwnComponent = false; for (uint32_t index = 0; isKnown && index < summary.calls.size(); ++index) { const SummaryCall& call = summary.calls[index]; diff --git a/Source/JavaScriptCore/aot/facts/AOTLinkLayouts.cpp b/Source/JavaScriptCore/aot/facts/AOTLinkLayouts.cpp index 3ed5d79fb6de0..f4f8b19d3e0f8 100644 --- a/Source/JavaScriptCore/aot/facts/AOTLinkLayouts.cpp +++ b/Source/JavaScriptCore/aot/facts/AOTLinkLayouts.cpp @@ -1762,6 +1762,29 @@ void LinkFlow::noteFieldMayBeEmpty(FieldReference reference) note(Note::FieldMayBeEmpty, cell); } +void LinkFlow::noteInstancesMayBeLocked(LayoutID id) +{ + for (LayoutID current = declaredLayoutOf(program, id); current; current = program.at(current).base) { + if (m_classesWithInstancesThatMayBeLocked.set(current.index())) + return; + if (auto told = m_nodesToldWhereInstancesMayBeLocked.find(current.index()); told != m_nodesToldWhereInstancesMayBeLocked.end()) + arrive(FlowNodeID(told->value), fromTypeID(TypeTable::undefined)); + } +} + +FlowNodeID LinkFlow::whereInstancesMayBeLocked(LayoutID classLayout) +{ + return nodeAskedOnceByLane(MadeOnDemand::WhereInstancesMayBeLocked, classLayout.index(), [&]() -> FlowNodeID { + if (auto told = m_nodesToldWhereInstancesMayBeLocked.find(classLayout.index()); told != m_nodesToldWhereInstancesMayBeLocked.end()) + return FlowNodeID(told->value); + FlowNodeID node = makeNode(); + m_nodesToldWhereInstancesMayBeLocked.add(classLayout.index(), node.index()); + if (instancesMayBeLocked(classLayout)) + arrive(node, fromTypeID(TypeTable::undefined)); + return node; + }); +} + FlowNodeID LinkFlow::foundWhereFieldIsEmpty(FieldReference reference) { return nodeAskedOnceByLane(MadeOnDemand::FoundWhereFieldIsEmpty, static_cast(reference.layout.index()) << 32 | reference.field, [&]() -> FlowNodeID { @@ -1937,10 +1960,106 @@ void LinkFlow::forEachFieldLeftEmptyByRowOfConstructors(LayoutID made, uint32_t } } +bool LinkFlow::copyMayPassOverFieldOf(LayoutID layout) +{ + uint32_t index = layout.index(); + if (m_layoutsThatParsersMake.isEmpty()) { + m_layoutsThatParsersMake.ensureSize(program.layouts.size()); + for (const ExpectedResult& expected : program.expectedResults) { + for (LayoutID made : expected.layouts) { + if (expected.flags.contains(ExpectedResultFlags::IsMadeByParser)) + m_layoutsThatParsersMake.quickSet(made.index()); + } + } + } + return possibleChanges[index].containsAny({ LayoutChange::LosesProperty, LayoutChange::GetsAccessor }) || layoutsDefinedOnUnderWrittenKeys.get(index) || layoutsDefinedOnUnderKeysThatAreNotWritten.get(index) + || layoutsGivenAccessorsByDescriptorsInSight.get(index) || layoutsWithKnownAccessors.get(index) || lostLayouts.get(index) || escapedLayouts.get(index) || m_layoutsThatParsersMake.quickGet(index); +} + +bool LinkFlow::leaveFieldsEmptyThatCopiesPassOver() +{ + if (!m_functionsThatMakeCopies) { + m_functionsThatMakeCopies = Vector(summaries.functionsThatMakeCopies()); + m_layoutsOfPlainFieldsAlone.ensureSize(program.layouts.size()); + for (FunctionID id : *m_functionsThatMakeCopies) { + summaries.at(id).forEachLayoutLeftEmptyAndCopy([&](LayoutID layout, std::span, uint32_t copy) { + if (copy == ThinSummary::noCopy || program.at(layout).kind != LayoutKind::Record || m_layoutsOfPlainFieldsAlone.get(layout.index())) + return; + bool hasPlainFieldsAlone = std::ranges::all_of(program.at(layout).fields, [](const Field& field) { + return field.key && field.form.isEmpty(); + }); + if (hasPlainFieldsAlone) + m_layoutsOfPlainFieldsAlone.set(layout.index()); + }); + } + } + bool someIsLeft = false; + m_functionsThatMakeCopies->removeAllMatching([&](FunctionID id) { + if (!reached.get(id.index())) + return false; + bool someIsFilled = false; + const ThinSummary& summary = summaries.at(id); + summary.forEachLayoutLeftEmptyAndCopy([&](LayoutID layout, std::span fieldsHeld, uint32_t copy) { + if (copy == ThinSummary::noCopy) + return; + const LinkType& source = typeAt(nodeOf(id, summary.changes[copy].other)); + if (source.isNever()) { + someIsFilled = true; + return; + } + LinkType objects; + objects.objects = source.objects; + objects.layout = source.layout; + objects.family = source.family; + bool holdsRecordsAlone = source == objects && (source.objects == LinkType::Objects::Layout || source.objects == LinkType::Objects::Set || source.objects == LinkType::Objects::Family); + bool holdsOtherLayouts = false; + if (holdsRecordsAlone) { + forEachLayoutOf(source, [&](LayoutID copied) { + holdsOtherLayouts |= copied != layout; + holdsRecordsAlone = holdsRecordsAlone && program.at(copied).kind == LayoutKind::Record && !copyMayPassOverFieldOf(copied); + }); + } + if (holdsRecordsAlone && !holdsOtherLayouts && m_layoutsOfPlainFieldsAlone.get(layout.index())) { + someIsFilled = true; + return; + } + auto copyFills = [&](uint32_t field) { + PropertyKeyID key = program.at(layout).fields[field].key; + bool isFilled = holdsRecordsAlone && !!key; + if (isFilled) { + forEachLayoutOf(source, [&](LayoutID copied) { + if (!isFilled) + return; + const Layout& from = program.at(copied); + bool standsInSamePlace = field < from.fields.size() && from.fields[field].key == key; + std::optional found = standsInSamePlace ? std::optional(FieldReference { copied, field }) : findField(program, copied, key); + isFilled = found && fieldAt(program, *found).form.isEmpty() && !fieldsThatMayBeEmpty.get(m_firstFieldOfLayout[found->layout.index()] + found->field); + }); + } + return isFilled; + }; + for (uint32_t field = 0; field < program.at(layout).fields.size(); ++field) { + if (std::ranges::binary_search(fieldsHeld, field) || fieldsThatMayBeEmpty.get(m_firstFieldOfLayout[layout.index()] + field)) + continue; + if (copyFills(field)) { + someIsFilled = true; + continue; + } + noteFieldMayBeEmpty({ layout, field }); + someIsLeft = true; + } + }); + return !someIsFilled; + }); + return someIsLeft; +} + void LinkFlow::noteConstructionsOf(FunctionID id) { noteBirthsOf(id); - summaries.at(id).forEachLayoutLeftEmpty([&](LayoutID layout, std::span fieldsHeld) { + summaries.at(id).forEachLayoutLeftEmptyAndCopy([&](LayoutID layout, std::span fieldsHeld, uint32_t copy) { + if (copy != ThinSummary::noCopy) + return; fieldsThatLaneAtWorkHasNotNotedEmpty(layout).removeAllMatching([&](uint32_t field) { if (std::ranges::binary_search(fieldsHeld, field)) return false; @@ -2059,6 +2178,7 @@ class WriterOfLinkLayouts { bool everyCallerIsKnown(FunctionID function) const { return m_functionsWhoseCallersAreAllKnown.get(function.index()); } BitVector m_functionsWhoseCallersAreAllKnown; bool mayBeAllocated(LayoutID) const; + bool isConstructedOnlyBySuper(LayoutID) const; bool mayBeKnownByFamily(LayoutID id) const { return m_flow.layoutsMetInFamilies.get(id.index()) || m_flow.lostLayouts.get(id.index()) || hasEscaped(id.index()); } bool canHoldAppendedFields(LayoutID) const; bool canNameAppendedField(LayoutID, PropertyKeyID) const; @@ -2137,11 +2257,19 @@ class WriterOfLinkLayouts { FunctionID method; bool isWrittenOnce { false }; bool isWrittenAgain { false }; + bool isCopied { false }; }; HashMap, WTF::UnsignedWithZeroKeyHashTraits> m_namesWrittenOnPrototypes; HashSet, WTF::UnsignedWithZeroKeyHashTraits> m_keysWrittenOnPrototypes; HashMap, WTF::UnsignedWithZeroKeyHashTraits> m_accessorsWrittenInMaps; Vector> m_namesLaidOutOfMaps; + struct CopiedPrototypeMember { + uint64_t copy; + uint64_t original; + }; + Vector m_copiedPrototypeMembers; + void noteCopiedPrototypeMember(FunctionID, const ThinSummary&, const SummaryAccess& store, uint64_t copy); + void findFunctionsOfCopiedPrototypeMembers(); BitVector m_fieldsWrittenLate; BitVector m_fieldsGivenValues; @@ -3245,9 +3373,12 @@ void WriterOfLinkLayouts::noteHalfOfAccessorLaidOnPrototype(FunctionID id, const void WriterOfLinkLayouts::noteNameWrittenOnPrototype(FunctionID id, const ThinSummary& summary, const SummaryAccess& access) { - if (access.key && access.mapOfWrittenAccessor) { - auto added = m_accessorsWrittenInMaps.add(static_cast(access.mapOfWrittenAccessor.index()) << 32 | access.key.bits(), NameWrittenOnPrototype { access.writtenGetter, access.writtenSetter, { }, true }); - if (!added.isNewEntry && (added.iterator->value.getter != access.writtenGetter || added.iterator->value.setter != access.writtenSetter)) + SummaryWrittenAccessor accessor; + if (access.laysAccessorAsWritten) + accessor = summary.writtenAccessorOf(static_cast(&access - summary.accesses.begin())); + if (access.key && accessor.map) { + auto added = m_accessorsWrittenInMaps.add(static_cast(accessor.map.index()) << 32 | access.key.bits(), NameWrittenOnPrototype { accessor.getter, accessor.setter, { }, true }); + if (!added.isNewEntry && (added.iterator->value.getter != accessor.getter || added.iterator->value.setter != accessor.setter)) added.iterator->value.isWrittenOnce = false; } if (access.key && access.kind == SummaryAccess::Kind::Store && access.value != noSummaryNode && summary.nodes[access.value].kind == SummaryNode::Kind::Closure && summary.nodes[access.subject].kind == SummaryNode::Kind::ResultOfAccess) { @@ -3261,7 +3392,7 @@ void WriterOfLinkLayouts::noteNameWrittenOnPrototype(FunctionID id, const ThinSu return; NameWrittenOnPrototype written; if (access.kind == SummaryAccess::Kind::Define && access.laysAccessorAsWritten) - written = { access.writtenGetter, access.writtenSetter, { }, true }; + written = { accessor.getter, accessor.setter, { }, true }; else if (!access.isByDescriptor && access.value != noSummaryNode && summary.nodes[access.value].kind == SummaryNode::Kind::Closure) written = { { }, { }, FunctionID(summary.nodes[access.value].first), true }; m_keysWrittenOnPrototypes.add(key.bits()); @@ -3271,6 +3402,7 @@ void WriterOfLinkLayouts::noteNameWrittenOnPrototype(FunctionID id, const ThinSu added.iterator->value.isWrittenAgain = true; return; } + noteCopiedPrototypeMember(id, summary, access, static_cast(layout.index()) << 32 | key.bits()); if (access.kind != SummaryAccess::Kind::Define || access.descriptorFromElsewhere == noSummaryNode || summary.nodes[access.descriptorFromElsewhere].kind != SummaryNode::Kind::ResultOfAccess) return; const SummaryAccess& taken = summary.accesses[summary.nodes[access.descriptorFromElsewhere].first]; @@ -3281,6 +3413,37 @@ void WriterOfLinkLayouts::noteNameWrittenOnPrototype(FunctionID id, const ThinSu m_namesLaidOutOfMaps.append({ static_cast(layout.index()) << 32 | key.bits(), static_cast(map.layout.index()) << 32 | taken.key.bits() }); } +void WriterOfLinkLayouts::noteCopiedPrototypeMember(FunctionID id, const ThinSummary& summary, const SummaryAccess& store, uint64_t copy) +{ + if (store.kind != SummaryAccess::Kind::Store || store.isByDescriptor || store.value == noSummaryNode || summary.nodes[store.value].kind != SummaryNode::Kind::ResultOfAccess) + return; + const SummaryAccess& read = summary.accesses[summary.nodes[store.value].first]; + if (read.kind != SummaryAccess::Kind::Read || !read.key) + return; + LayoutID original = findLayoutWhosePrototypeObjectIsAt(summary, read.subject); + if (!original) + original = onlyPrototypeOfConstructorFunctionInLinkLayouts(m_flow, typeAt(id, read.subject)); + if (original) + m_copiedPrototypeMembers.append({ copy, static_cast(original.index()) << 32 | read.key.bits() }); +} + +void WriterOfLinkLayouts::findFunctionsOfCopiedPrototypeMembers() +{ + for (auto [copy, original] : m_copiedPrototypeMembers) { + auto found = m_namesWrittenOnPrototypes.find(original); + auto copied = m_namesWrittenOnPrototypes.find(copy); + if (found == m_namesWrittenOnPrototypes.end() || !found->value.isWrittenOnce || !found->value.method || copied->value.isWrittenAgain) + continue; + const Function& method = m_program.at(found->value.method); + const Function& constructor = m_program.at(m_program.at(LayoutID(static_cast(copy >> 32))).constructor); + if (!isLibraryCode(m_program, method) || !isLibraryCode(m_program, constructor) || (method.module != constructor.module && !method.captures.isEmpty())) + continue; + copied->value.method = found->value.method; + copied->value.isWrittenOnce = true; + copied->value.isCopied = true; + } +} + bool WriterOfLinkLayouts::isWrittenOnPrototypeAbove(LayoutID layout, PropertyKeyID key) const { if (!m_keysWrittenOnPrototypes.contains(key.bits())) @@ -3558,6 +3721,10 @@ void WriterOfLinkLayouts::noteNamesWrittenOfLayouts() } } + HashMap, DefaultHash, WTF::UnsignedWithZeroKeyHashTraits> namesUnderKey; + for (uint32_t place = 0; place < m_namesNotListed.size(); ++place) + namesUnderKey.add(m_namesNotListed[place].key.bits(), Vector()).iterator->value.append(place); + Vector mayHideBehindUnknownObjects; for (uint32_t index = 0; index < numberOfLayouts; ++index) { const Layout& written = m_program.layouts[index]; @@ -3572,7 +3739,25 @@ void WriterOfLinkLayouts::noteNamesWrittenOfLayouts() for (const Field& field : written.fields) { if (!field.key || field.form.contains(FieldForm::IsAccessor) || field.flags.contains(FieldFlags::IsMethod)) continue; + bool isRequired = field.flags.contains(FieldFlags::IsRequired); + auto noteDescription = [&](NameNotListed& name) { + name.isRequiredBySomeWrittenType |= isRequired; + if (m_program.at(name.layout).supertypes.contains(LayoutID(index))) + (isRequired ? name.isRequiredBySomeTypeOfThis : name.isOptionalInSomeTypeOfThis) = true; + }; bool isStoredOnUnknownObjects = m_keysStoredOnUnknownObjects.contains(field.key.bits()); + if (!isStoredOnUnknownObjects) { + auto names = namesUnderKey.find(field.key.bits()); + if (names == namesUnderKey.end()) + continue; + if (names->value.size() < described.size()) { + for (uint32_t place : names->value) { + if (std::ranges::binary_search(described, m_namesNotListed[place].layout)) + noteDescription(m_namesNotListed[place]); + } + continue; + } + } for (size_t place = 0; place < described.size(); ++place) { LayoutID id = described[place]; auto noted = m_pairsNotListed.find(static_cast(id.index()) << 32 | field.key.bits()); @@ -3581,11 +3766,7 @@ void WriterOfLinkLayouts::noteNamesWrittenOfLayouts() continue; noted = m_pairsNotListed.find(static_cast(id.index()) << 32 | field.key.bits()); } - NameNotListed& name = m_namesNotListed[noted->value]; - bool isRequired = field.flags.contains(FieldFlags::IsRequired); - name.isRequiredBySomeWrittenType |= isRequired; - if (m_program.at(id).supertypes.contains(LayoutID(index))) - (isRequired ? name.isRequiredBySomeTypeOfThis : name.isOptionalInSomeTypeOfThis) = true; + noteDescription(m_namesNotListed[noted->value]); } } } @@ -4130,6 +4311,14 @@ bool WriterOfLinkLayouts::mayBeAllocated(LayoutID id) const RELEASE_ASSERT_NOT_REACHED(); } +bool WriterOfLinkLayouts::isConstructedOnlyBySuper(LayoutID id) const +{ + const Layout& layout = m_program.at(id); + if (layout.kind != LayoutKind::Instance || !layout.constructor || layout.isDerived() || m_flow.allocatedLayouts.get(id.index()) || m_isMadeAtNoNode.quickGet(id.index())) + return false; + return m_program.at(layout.constructor).kind == FunctionKind::BaseConstructor && everyCallerIsKnown(layout.constructor); +} + void WriterOfLinkLayouts::writeLayouts() { bool everyPrototypeMayChange = objectPrototypeMayGainName(); @@ -4147,6 +4336,7 @@ void WriterOfLinkLayouts::writeLayouts() linked.hasConstructorFunctionInRow = haveConstructorFunctionInRow.quickGet(index); linked.family = m_flow.familiesBehindFlow->nearestCommonFamily(id, id); linked.isAllocated = mayBeAllocated(id); + linked.instancesMayBeLocked = m_flow.instancesMayBeLocked(id); linked.possibleChanges = LinkedLayout { }.possibleChanges; linked.listsEveryMemberAbove = !m_flow.namesMayBeIncomplete(id, ClassSide::Prototype); linked.listsEveryStaticMember = !m_flow.namesMayBeIncomplete(id, ClassSide::Static); @@ -4160,6 +4350,7 @@ void WriterOfLinkLayouts::writeLayouts() if (m_flow.reading() == WhatUnseenCodeDoes::Anything) linked.reachesUnseenCode = false; + linked.hasDirectInstances = linked.isAllocated && (linked.reachesUnseenCode || !isConstructedOnlyBySuper(id)); linked.possibleChanges = m_changes[index]; linked.possibleChanges.add(m_flow.possibleChangesOf(index)); if (writtenBaseOf(m_program.at(id)) && m_program.at(id).inheritedFields != InheritedFields::AllOfRow) @@ -4660,6 +4851,8 @@ void WriterOfLinkLayouts::writeMembers() name->value = accessor->value; } + findFunctionsOfCopiedPrototypeMembers(); + for (const LinkFlow::NameOnPrototype& name : m_flow.namesOnPrototypes()) { if (!name.mayHoldAccessor() && (!name.isStoredInto || !name.key)) continue; @@ -4681,6 +4874,8 @@ void WriterOfLinkLayouts::writeMembers() linked.expectedGetter = written->value.getter; linked.expectedSetter = written->value.setter; } + if (written->value.isCopied && !linked.expected && !name.mayHoldAccessor()) + linked.expected = written->value.method; m_namesWrittenOnPrototypes.remove(written); } m_tables.namesOnPrototypes.append(linked); diff --git a/Source/JavaScriptCore/aot/facts/AOTLinkPrototypes.cpp b/Source/JavaScriptCore/aot/facts/AOTLinkPrototypes.cpp index 7747a58e2861d..3029c3d4975b6 100644 --- a/Source/JavaScriptCore/aot/facts/AOTLinkPrototypes.cpp +++ b/Source/JavaScriptCore/aot/facts/AOTLinkPrototypes.cpp @@ -73,7 +73,7 @@ static uint64_t wordOfNameInLinkPrototypes(LayoutID layout, PropertyKeyID key) void LinkFlow::indexConstructorFunctions() { size_t numberOfLayouts = program.layouts.size(); - for (BitVector* byLayout : { &m_layoutsOfConstructorFunctions, &m_layoutsBelowConstructorFunctions, &m_prototypesPassedByLookups, &m_prototypesWithDefinitions, &m_prototypesAboveLostLayouts, &m_prototypesPassedByNumbers }) + for (BitVector* byLayout : { &m_layoutsOfConstructorFunctions, &m_layoutsBelowConstructorFunctions, &m_layoutsWithFollowedPrototypes, &m_layoutsBelowFollowedPrototypes, &m_prototypesPassedByLookups, &m_prototypesWithDefinitions, &m_prototypesAboveLostLayouts, &m_prototypesPassedByNumbers }) byLayout->ensureSize(numberOfLayouts); m_nearestConstructorFunctionsFrom.fill(LayoutID(), numberOfLayouts); auto noteConstructorFunctionAbove = [&](LayoutID below, LayoutID above) { @@ -97,6 +97,81 @@ void LinkFlow::indexConstructorFunctions() noteConstructorFunctionAbove(below, LayoutID(index)); } } + + BitVector classesWithFollowedPrototypes = findClassesWithFollowedPrototypes(); + for (size_t index = 0; index < numberOfLayouts; ++index) { + if (!m_layoutsOfConstructorFunctions.quickGet(index) && !classesWithFollowedPrototypes.quickGet(index)) + continue; + m_layoutsWithFollowedPrototypes.quickSet(index); + m_layoutsBelowFollowedPrototypes.quickSet(index); + for (LayoutID below : program.layoutsExpectedBelow(LayoutID(index))) + m_layoutsBelowFollowedPrototypes.quickSet(below.index()); + } +} + +static LayoutID classNamedAtInLinkPrototypes(const Program& program, FunctionID asker, const SummaryNode& node) +{ + FunctionID constructor; + auto takeFrom = [&](TypeID written) { + if (!written || holdsOnlyPrimitives(written)) + return; + Type type = program.at(written); + type.primitives &= ~Primitive::Absent; + constructor = !type.primitives && type.isJavaScriptValue() ? singleFunctionOf(TypeReader(program), type) : FunctionID(); + }; + switch (node.kind) { + case SummaryNode::Kind::Closure: + constructor = FunctionID(node.first); + break; + case SummaryNode::Kind::Variable: + takeFrom(program.provenTypeOf(program.variables[node.first].declared)); + break; + case SummaryNode::Kind::Capture: + takeFrom(program.provenTypeOf(program.at(node.second ? FunctionID(node.second - 1) : asker).captures[node.first])); + break; + default: + break; + } + LayoutID constructed = constructor && isClassConstructorOf(program.at(constructor).kind) ? program.at(constructor).constructedClass : LayoutID(); + bool isClass = constructed && program.at(constructed).kind == LayoutKind::Instance && program.at(constructed).constructor == constructor && !program.at(constructed).has(LayoutFlags::IsConstructorFunction); + return isClass ? constructed : LayoutID(); +} + +bool LinkFlow::instancesMayHoldName(LayoutID layout, PropertyKeyID key) const +{ + return m_namesStoredOnObjects.get(key.index()) || program.at(layout).findField(key) || std::ranges::any_of(program.layoutsExpectedBelow(layout), [&](LayoutID below) { + return !!program.at(below).findField(key); + }); +} + +BitVector LinkFlow::findClassesWithFollowedPrototypes() +{ + PropertyKeyID prototype = program.keyOf(BuiltinNames::prototype); + Vector, 8> namesOnPrototypes; + for (uint32_t index = 0; index < program.functions.size(); ++index) { + if (isLibraryCode(program, program.functions[index])) + continue; + FunctionID function(index); + const ThinSummary& summary = summaries.at(function); + for (const SummaryAccess& access : summary.accesses) { + bool isWrite = access.kind == SummaryAccess::Kind::Store || access.kind == SummaryAccess::Kind::Define; + if (!isWrite || !access.key || access.subject == noSummaryNode) + continue; + const SummaryNode& object = summary.nodes[access.subject]; + const SummaryAccess* read = object.kind == SummaryNode::Kind::ResultOfAccess ? &summary.accesses[object.first] : nullptr; + if (read && read->kind == SummaryAccess::Kind::Read && read->key == prototype && read->subject != noSummaryNode) { + if (LayoutID written = classNamedAtInLinkPrototypes(program, function, summary.nodes[read->subject])) + namesOnPrototypes.append({ written, access.key }); + } else if (!classNamedAtInLinkPrototypes(program, function, object)) + m_namesStoredOnObjects.set(access.key.index()); + } + } + BitVector classes(program.layouts.size()); + for (auto [layout, key] : namesOnPrototypes) { + if (!instancesMayHoldName(layout, key)) + classes.quickSet(layout.index()); + } + return classes; } LayoutID LinkFlow::layoutOfConstructorFunction(FunctionID function) const @@ -135,11 +210,11 @@ bool LinkFlow::mayBeDefinedUnderUnknownKeyOnPrototypeOf(LayoutID layout, Propert bool LinkFlow::mayLieOnPrototypeAbove(LayoutID layout, PropertyKeyID key, bool countsKeysThatNobodyKnows) const { LockerOfWhatIsShared lockerOfWhatIsShared(*this); - if (!isBelowConstructorFunction(layout)) + if (!isBelowFollowedPrototype(layout)) return false; size_t stepsLeft = program.layouts.size(); for (LayoutID current = layout; current && stepsLeft; current = program.at(current).base, --stepsLeft) { - if (!isOfConstructorFunction(current)) + if (!followsNamesOnPrototypeOf(current)) continue; const NameOnPrototype* underKey = findNameOnPrototype(current, key); if (underKey && (underKey->isStoredInto || underKey->mayHoldAccessor())) @@ -165,7 +240,7 @@ static void openHalvesOfAccessorInLinkPrototypes(LinkFlow& flow, LinkFlow::NameO uint32_t LinkFlow::recordOfNameOnPrototype(LayoutID layout, PropertyKeyID key) { LockerOfWhatIsShared lockerOfWhatIsShared(*this); - RELEASE_ASSERT(isOfConstructorFunction(layout)); + RELEASE_ASSERT(followsNamesOnPrototypeOf(layout)); uint32_t record = static_cast(m_namesOnPrototypes.size()); auto added = m_recordsOfNamesOnPrototypes.add(wordOfNameInLinkPrototypes(layout, key), record); if (!added.isNewEntry) @@ -229,7 +304,7 @@ void LinkFlow::forEachPrototypePassedAbove(LayoutID layout, PropertyKeyID key, c { size_t stepsLeft = program.layouts.size(); for (LayoutID current = layout; current && stepsLeft; current = program.at(current).base, --stepsLeft) { - if (isOfConstructorFunction(current)) { + if (followsNamesOnPrototypeOf(current)) { uint32_t record = recordOfNameOnPrototype(current, key); m_namesOnPrototypes[record].isPassedByLookups = true; m_prototypesPassedByLookups.set(current.index()); @@ -272,12 +347,12 @@ bool LinkFlow::letAccessorOnPrototypeEscape(uint32_t record) void LinkFlow::letAccessorsOnPrototypesAboveEscape(LayoutID layout, PropertyKeyID key) { - if (!isDefinedByDescriptorSomewhere(key) || !m_layoutsBelowConstructorFunctions.get(layout.index())) + if (!isDefinedByDescriptorSomewhere(key) || !m_layoutsBelowFollowedPrototypes.get(layout.index())) return; LockerOfWhatIsShared lockerOfWhatIsShared(*this); size_t stepsLeft = program.layouts.size(); for (LayoutID current = layout; current && stepsLeft; current = program.at(current).base, --stepsLeft) { - if (!isOfConstructorFunction(current)) + if (!followsNamesOnPrototypeOf(current)) continue; uint32_t record = recordOfNameOnPrototype(current, key); m_namesOnPrototypes[record].isPassedByLookupsThatEnterNoAccessor = true; @@ -289,12 +364,12 @@ void LinkFlow::letAccessorsOnPrototypesAboveEscape(LayoutID layout, PropertyKeyI Vector LinkFlow::halvesOfAccessorsOnPrototypesAbove(LayoutID layout, PropertyKeyID key, bool ofSetters) { Vector halves; - if (!isDefinedByDescriptorSomewhere(key) || !m_layoutsBelowConstructorFunctions.get(layout.index())) + if (!isDefinedByDescriptorSomewhere(key) || !m_layoutsBelowFollowedPrototypes.get(layout.index())) return halves; LockerOfWhatIsShared lockerOfWhatIsShared(*this); size_t stepsLeft = program.layouts.size(); for (LayoutID current = layout; current && stepsLeft; current = program.at(current).base, --stepsLeft) { - if (isOfConstructorFunction(current)) { + if (followsNamesOnPrototypeOf(current)) { uint32_t record = recordOfNameOnPrototype(current, key); halves.append(ofSetters ? m_namesOnPrototypes[record].setter : m_namesOnPrototypes[record].getter); } @@ -365,7 +440,7 @@ void LinkFlow::layAccessorOnPrototypeOf(LayoutID layout, PropertyKeyID key, Func FlowNodeID LinkFlow::foundOnPrototypesAbove(LayoutID layout, PropertyKeyID key, AccessorsAbove accessors) { - if (!m_layoutsBelowConstructorFunctions.get(layout.index())) + if (!m_layoutsBelowFollowedPrototypes.get(layout.index())) return { }; if (accessors == AccessorsAbove::AreEnteredByNobody) letAccessorsOnPrototypesAboveEscape(layout, key); @@ -394,7 +469,7 @@ FlowNodeID LinkFlow::foundOnPrototypesAbove(LayoutID layout, PropertyKeyID key, FlowNodeID LinkFlow::foundOnPrototypesAboveUnderNumbers(LayoutID layout) { LockerOfWhatIsShared lockerOfWhatIsShared(*this); - if (!m_layoutsBelowConstructorFunctions.get(layout.index())) + if (!m_layoutsBelowFollowedPrototypes.get(layout.index())) return { }; passPrototypesAboveUnderNumbers(layout); return foundBesideFieldsUnderAnyNumber(); @@ -402,12 +477,12 @@ FlowNodeID LinkFlow::foundOnPrototypesAboveUnderNumbers(LayoutID layout) void LinkFlow::passPrototypesAboveUnderNumbers(LayoutID layout) { - if (!m_layoutsBelowConstructorFunctions.get(layout.index())) + if (!m_layoutsBelowFollowedPrototypes.get(layout.index())) return; LockerOfWhatIsShared lockerOfWhatIsShared(*this); size_t stepsLeft = program.layouts.size(); for (LayoutID current = layout; current && stepsLeft; current = program.at(current).base, --stepsLeft) { - if (!isOfConstructorFunction(current) || m_prototypesPassedByNumbers.set(current.index()) || !m_prototypesWithDefinitions.get(current.index())) + if (!followsNamesOnPrototypeOf(current) || m_prototypesPassedByNumbers.set(current.index()) || !m_prototypesWithDefinitions.get(current.index())) continue; bool mayFindAccessor = false; for (uint32_t record = firstNameOnPrototypeOf(current); record != UINT32_MAX; record = m_namesOnPrototypes[record].nextOfLayout) { @@ -425,7 +500,7 @@ void LinkFlow::passPrototypesAboveUnderNumbers(LayoutID layout) void LinkFlow::lookForSetterOnPrototypesAbove(LayoutID layout, PropertyKeyID key, AccessorsAbove accessors) { - if (!m_layoutsBelowConstructorFunctions.get(layout.index())) + if (!m_layoutsBelowFollowedPrototypes.get(layout.index())) return; if (accessors == AccessorsAbove::AreEnteredByNobody) letAccessorsOnPrototypesAboveEscape(layout, key); @@ -499,11 +574,11 @@ void LinkFlow::defineOnPrototypeOf(LayoutID layout, PropertyKeyID key, UnknownKe void LinkFlow::tiePrototypesAboveLostLayout(LayoutID lost) { - if (!m_layoutsBelowConstructorFunctions.get(lost.index())) + if (!m_layoutsBelowFollowedPrototypes.get(lost.index())) return; size_t stepsLeft = program.layouts.size(); for (LayoutID current = lost; current && stepsLeft; current = program.at(current).base, --stepsLeft) { - if (!isOfConstructorFunction(current) || m_prototypesAboveLostLayouts.set(current.index())) + if (!followsNamesOnPrototypeOf(current) || m_prototypesAboveLostLayouts.set(current.index())) continue; for (uint32_t record = firstNameOnPrototypeOf(current); record != UINT32_MAX; record = m_namesOnPrototypes[record].nextOfLayout) { NameOnPrototype name = m_namesOnPrototypes[record]; @@ -532,7 +607,7 @@ bool LinkFlow::letPrototypesFoundThroughLostEscape() void LinkFlow::openPrototypeOf(LayoutID layout) { - if (!isOfConstructorFunction(layout)) + if (!followsNamesOnPrototypeOf(layout)) return; for (uint32_t record = firstNameOnPrototypeOf(layout); record != UINT32_MAX; record = m_namesOnPrototypes[record].nextOfLayout) { FlowNodeID stored = m_namesOnPrototypes[record].stored; diff --git a/Source/JavaScriptCore/aot/facts/AOTLinkReach.cpp b/Source/JavaScriptCore/aot/facts/AOTLinkReach.cpp index 7eafde81e03f9..d9ef77499231f 100644 --- a/Source/JavaScriptCore/aot/facts/AOTLinkReach.cpp +++ b/Source/JavaScriptCore/aot/facts/AOTLinkReach.cpp @@ -651,6 +651,7 @@ void StateOfLinkReach::noteNamedByReachedCode(FunctionID named) void StateOfLinkReach::loseLayout(LayoutID id) { if (!m_flow.lostLayouts.set(id.index())) { + m_flow.noteInstancesMayBeLocked(id); m_flow.noteLayoutToTie(id); m_steps.append(QueuedStep { Step::TieLostLayout, id.index() }); } @@ -680,6 +681,7 @@ void StateOfLinkReach::escapeLayout(LayoutID id) } if (m_flow.escapedLayouts.set(id.index())) return; + m_flow.noteInstancesMayBeLocked(id); m_steps.append(QueuedStep { Step::OpenLayout, id.index() }); if (m_program.at(id).kind == LayoutKind::ModuleNamespace) { m_flow.arrive(m_flow.whereInstancesLoseTheirConstructor(), LinkType::anything()); @@ -2470,8 +2472,8 @@ void LinkFlow::arriveFromSource(FlowNodeID node, TypeID id) void LinkFlow::noteAllocatedWithRowAbove(LayoutID id) { - size_t stepsLeft = program.layouts.size(); - for (LayoutID current = id; current && stepsLeft; current = program.at(current).base, --stepsLeft) + BitVector& noted = layoutsNotedWithBases(); + for (LayoutID current = id; current && !noted.set(current.index()); current = program.at(current).base) noteAllocated(current); } diff --git a/Source/JavaScriptCore/aot/facts/AOTLinkStrings.cpp b/Source/JavaScriptCore/aot/facts/AOTLinkStrings.cpp index 5b62e453cca31..3336c79352a93 100644 --- a/Source/JavaScriptCore/aot/facts/AOTLinkStrings.cpp +++ b/Source/JavaScriptCore/aot/facts/AOTLinkStrings.cpp @@ -16,9 +16,10 @@ namespace JSC { namespace AOT { namespace Typed { bool keyThatBeginsSoMayNamePrototypeLink(const Program& program, const SummaryAccess& access) { - if (!access.beginningOfKey) + StringID beginningOfKey = access.beginningOfKey(); + if (!beginningOfKey) return true; - std::span beginning = program.string(access.beginningOfKey); + std::span beginning = program.string(beginningOfKey); std::span nameOfLink = program.string(program.stringOf(BuiltinNames::__proto__)); return beginning.size() <= nameOfLink.size() && std::ranges::equal(beginning, nameOfLink.first(beginning.size())); } diff --git a/Source/JavaScriptCore/aot/facts/AOTLinkTables.h b/Source/JavaScriptCore/aot/facts/AOTLinkTables.h index 7e45e32464d95..66eabbef58bb1 100644 --- a/Source/JavaScriptCore/aot/facts/AOTLinkTables.h +++ b/Source/JavaScriptCore/aot/facts/AOTLinkTables.h @@ -42,8 +42,10 @@ struct LinkedFunction { bool everyCallerIsKnown { false }; bool everyCallHasOneCallee { false }; bool everyTakerOfResultKnowsCallee { false }; + bool hasSingleCallSite { false }; bool ringsOfTailCallsAreListed { false }; bool runsAtMostOnce { false }; + bool isIterationCallback { false }; bool isNamedInItsCycle { true }; bool liesUnderNameOrIsLost { false }; bool isInQueueFromStart { false }; @@ -114,7 +116,9 @@ inline constexpr uint32_t maxAppendedFieldsOutOfLine = maxExtensionGranules; struct LinkedLayout { FamilyID family; bool isAllocated { true }; + bool hasDirectInstances { true }; bool reachesUnseenCode { true }; + bool instancesMayBeLocked { true }; OptionSet possibleChanges { LayoutChange::IsFrozen, LayoutChange::GainsProperty, LayoutChange::LosesProperty, LayoutChange::ChangesPrototype, LayoutChange::GetsAccessor, LayoutChange::GetsObjectAbove }; bool isRedefinedByLiteralsAlone { false }; bool isDefinedOnByDescriptorsThatLayValuesAlone { false }; @@ -252,6 +256,12 @@ struct LinkedResultOfRead { TypeID proven; }; +struct LinkedNativeCallResult { + FunctionID function; + ValueID site; + TypeID proven; +}; + struct LinkedContentsAtOperation { FunctionID function; ValueID site; @@ -423,6 +433,10 @@ struct LinkedFunctionUnderName { friend constexpr auto operator<=>(const LinkedFunctionUnderName&, const LinkedFunctionUnderName&) = default; }; +class Program; +struct LinkTables; +bool instancesMayBeLocked(const Program&, const LinkTables&, TypeID testedType); + struct LinkTables { std::unique_ptr familiesBehindFlow; BitVector exportsThatStayIn; @@ -444,6 +458,8 @@ struct LinkTables { Vector coroutines; const LinkedCoroutine* findCoroutine(FunctionID) const; Vector resultsOfReadsByKey; + Vector nativeCallResults; + BitVector nativeFunctionCallers; Vector contentsAtOperations; Vector receiversThatStayAsMade; Vector receiversThatNobodyChanges; @@ -730,6 +746,15 @@ struct LinkTables { }); return found ? found->key : PropertyKeyID(); } + TypeID nativeCallResult(FunctionID function, ValueID site) const + { + if (!nativeFunctionCallers.get(function.index())) + return { }; + const LinkedNativeCallResult* found = findListedUnder(nativeCallResults.span(), placeOfSite(function, site), [](const LinkedNativeCallResult& call) { + return placeOfSite(call.function, call.site); + }); + return found ? found->proven : TypeID(); + } TypeID resultOfReadByKey(FunctionID function, ValueID site) const { const LinkedResultOfRead* found = findListedUnder(resultsOfReadsByKey.span(), placeOfSite(function, site), [](const LinkedResultOfRead& read) { diff --git a/Source/JavaScriptCore/aot/facts/AOTScan.cpp b/Source/JavaScriptCore/aot/facts/AOTScan.cpp index 2f3a55764d9c7..6fd6379583cdd 100644 --- a/Source/JavaScriptCore/aot/facts/AOTScan.cpp +++ b/Source/JavaScriptCore/aot/facts/AOTScan.cpp @@ -14,6 +14,7 @@ #include "AOTBuiltinTable.h" #include "AOTDiagnostics.h" #include "AOTEffectsDeclared.h" +#include "AOTEffectsOfOperations.h" #include "AOTFoldingOfConstants.h" #include "AOTIRProgram.h" #include "AOTInliningAcrossFiles.h" @@ -328,6 +329,7 @@ struct ObjectBeingMadeInScan { Vector parametersHandedTo; Vector definedInTurn; Vector keysThatHoldItself; + uint32_t copyAtBirth { ThinSummary::noCopy }; }; struct ObjectHandedToBlockInScan { @@ -965,6 +967,7 @@ class ScanOfBody { const Vector& keysOfVariables() const { return m_keysOfVariables; } const Vector& membersUnderVariables() const { return m_membersUnderVariables; } const Vector& functionsThatCopyMembers() const { return m_functionsThatCopyMembers; } + const Vector& functionsThatMakeCopies() const { return m_functionsThatMakeCopies; } const Vector& makersOfClosures() const { return m_makersOfClosures; } private: @@ -1120,6 +1123,7 @@ class ScanOfBody { void addOperand(const BodyValue&, uint32_t operandIndex); void addVariadicOperand(const BodyValue&, uint32_t place); void addConstant(const BodyValue&); + uint32_t nodeOfNativeBinding(BuiltinID); void addNatural(const BodyValue&, const NaturalType&); bool prototypeMaySelectOtherObject(const BodyValue&, LayoutID) const; void addAllocation(const BodyValue&, LayoutID); @@ -1181,6 +1185,7 @@ class ScanOfBody { BodyValue definitionBehindCasts(ValueID value) const { return definitionOf(m_roots[value.index()]); } SummaryCall* findCallThatMade(ValueID); void noteResultReadAsKey(SummaryCall&); + void noteIfParameterIsReadAsKey(ValueID key, uint16_t& parameters); ALWAYS_INLINE void noteLeaving(ValueID value, WayOfLeavingInScan way = WayOfLeavingInScan::IsSeenFromOutside) { uint32_t root = m_roots[value.index()]; @@ -1254,6 +1259,7 @@ class ScanOfBody { void noteEvaluationOf(ModuleID, ValueID site, uint32_t block); bool answersWithResultOfBody(const BodyValue&) const; void noteRecordOf(ModuleID); + void noteRecordOf(VariableID); bool listsFirstStoresOfBody() const; void lookForFirstStoresAndUses(const BodyValue&); void noteUseByBodyOfModule(ValueID operand) @@ -1286,6 +1292,8 @@ class ScanOfBody { void followAllocation(const BodyValue&); void leaveEveryFieldEmpty(LayoutID); void leaveFieldsEmptyButFor(LayoutID, std::span fieldsHeld); + void noteCopyAtBirth(const BodyValue& copy, ValueID target); + bool leaveFieldsToCopy(const ObjectBeingMadeInScan&, LayoutID); void listFieldsHeldBy(const ObjectBeingMadeInScan&, const Layout&, Vector& fieldsHeld); bool isListedByAllocation(uint32_t root, PropertyKeyID) const; std::span fillsOf(uint32_t root, SetsOfFillsInScan::Set) LIFETIME_BOUND; @@ -1342,6 +1350,16 @@ class ScanOfBody { } return oneNodeForOrigins(list); } + uint32_t subjectNodeFor(ListInScan list) + { + uint32_t atom = list.count == 1 ? m_atoms[list.first] : 0; + if (atom & parameterBit) { + ListInScan resolved = m_resolved[atom & ~parameterBit]; + if (resolved.count == 1 && TypeTable::isAnythingButNullish(m_origins[resolved.first].passes)) + return m_origins[resolved.first].node; + } + return oneNodeFor(list); + } uint32_t oneNodeForOrigins(ListInScan); bool mayHaveFewSources(ListInScan) const; bool gatherSourcesOf(ListInScan, Vector& sources, Vector& parametersGoneThrough); @@ -1354,6 +1372,7 @@ class ScanOfBody { void nameGettersOfDescriptors(); void layDescriptorsInSight(); static constexpr uint32_t changeOfNoMap = UINT32_MAX; + void forgetWrittenAccessorOf(uint32_t access); void nameAccessorWrittenInDescriptor(uint32_t access, uint32_t rootOfDescriptor, uint32_t positionOfDefinition, uint32_t changeOfEntryInMap = changeOfNoMap, uint32_t placeWhereMapTakesEntry = 0); FunctionID functionWrittenAt(const SummaryNode&) const; std::span storesInto(uint32_t root); @@ -1486,6 +1505,7 @@ class ScanOfBody { Vector m_keysOfVariables; Vector m_membersUnderVariables; Vector m_functionsThatCopyMembers; + Vector m_functionsThatMakeCopies; Vector m_makersOfClosures; HashSet, WTF::UnsignedWithZeroKeyHashTraits> m_placesOfAccessorsLaid; bool hasAccessorLaidUnder(uint32_t root, PropertyKeyID key) const { return !m_accessorsLaid.isEmpty() && m_placesOfAccessorsLaid.contains(static_cast(root) << 32 | key.bits()); } @@ -1502,6 +1522,7 @@ class ScanOfBody { }; Vector m_heritages; Vector> m_returnsOfCoroutine; + bool m_someReturnIsNoShift { false }; Vector m_edgesLeadingBack; bool m_listsDefinitions { false }; struct ValueMadeForVariableInScan { @@ -1643,7 +1664,7 @@ struct ReferencesOfScan { scan.m_summary.sitesThatHandOverCaptures.append(site); } } else if constexpr (std::is_same_v) - scan.noteRecordOf(scan.m_program.at(id).module); + scan.noteRecordOf(id); else if constexpr (std::is_same_v) { if (((role == ModuleReferenceKind::Target) || ...)) { scan.noteFunctionNamed(scan.m_program.at(id).body); @@ -1928,6 +1949,7 @@ ThinSummary::ThinSummary(SummaryBeingMade& made) visit(placesOfWrittenLists, made.placesOfWrittenLists); visit(definitionsByDescriptorInOrderOfSites, made.definitionsByDescriptorInOrderOfSites); visit(callsAtSitesOfCopiesThatSpellStores, made.callsAtSitesOfCopiesThatSpellStores); + visit(writtenAccessors, made.writtenAccessors); }; SummaryStorage& storage = nodes.m_storage; @@ -1997,11 +2019,14 @@ ThinSummary::ThinSummary(SummaryBeingMade& made) made.layoutsOfOwnWrites = 0; made.ownEffects = { }; made.ownEffectsOfOperators = std::nullopt; + made.makesCopies = false; } void ScanOfBody::run(FunctionID id, ThinSummary& result) { summarize(id); + if (m_summary.makesCopies) + m_functionsThatMakeCopies.append(id); if (!m_summary.copiesOfMembers.isEmpty()) m_functionsThatCopyMembers.append(id); result = ThinSummary(m_summary); @@ -2089,6 +2114,7 @@ void ScanOfBody::reset() m_readsNotYetNeeded.shrink(0); m_heritages.shrink(0); m_returnsOfCoroutine.shrink(0); + m_someReturnIsNoShift = false; m_edgesLeadingBack.shrink(0); m_listsDefinitions = listsDefinitions(); m_valuesMadeForVariables.shrink(0); @@ -2467,12 +2493,22 @@ void ScanOfBody::addConstant(const BodyValue& value) } return; } + if (opcode == Opcode::NativeBinding) { + addNode(nodeOfNativeBinding(value.as().builtin())); + return; + } if (opcode == Opcode::String) noteTextThatMayNameEvent(value.as().string()); TypeID type = typeOfPrimitiveConstantInScan(value); addNode(type ? constant(type) : 0); } +uint32_t ScanOfBody::nodeOfNativeBinding(BuiltinID binding) +{ + BuiltinID function = findNativeFunction(m_program.builtins(), binding); + return function ? nodeFor(SummaryNode::Kind::Builtin, function.index()) : 0; +} + void ScanOfBody::noteTextThatMayNameEvent(StringID text) { BuiltinName name = m_program.strings.builtinName(text); @@ -2874,7 +2910,7 @@ ListInScan ScanOfBody::settle(ValueID site, ListInScan list, uint32_t block) uint32_t access = addAccess(SummaryAccess::Kind::Read, site, list, { }); m_summary.accesses[access].slot = ContentsSlot::Value; uint32_t whatIsNoPromise = newJoin(); - m_summary.accesses[access].whatHoldsNothing = whatIsNoPromise; + m_summary.accesses[access].whatHoldsNothingOrBeginningOfKey = whatIsNoPromise; uint32_t valueOfPromise = resultOfRead(access); ListInScan objects = objectsAmong(listOfNode(whatIsNoPromise)); @@ -3175,16 +3211,35 @@ BodyValue ScanOfBody::definitionOf(uint32_t value) const PrimitiveKnownToFolding ScanOfBody::primitiveHeldBy(ValueID value) { - BodyValue definition = definitionOf(value.index()); - if (!definition || !operationsThatConstantsMayDecide.contains(definition.opcode())) + ValueID unvisited; + auto findPrimitive = [&](ValueID operand) -> PrimitiveKnownToFolding { + BodyValue definition = definitionOf(operand.index()); + if (!definition || !operationsThatConstantsMayDecide.contains(definition.opcode())) + return { }; + if (m_primitivesAsked[operand.index()].stamp == m_stampOfBody) + return m_primitivesAsked[operand.index()].known; + unvisited = operand; return { }; - if (m_primitivesAsked[value.index()].stamp != m_stampOfBody) { - PrimitiveKnownToFolding known = primitiveMadeOfConstants(m_program, definition, scopedLambda([&](ValueID operand) { - return primitiveHeldBy(operand); + }; + PrimitiveKnownToFolding known = findPrimitive(value); + if (!unvisited) + return known; + + Vector pending { unvisited }; + while (!pending.isEmpty()) { + ValueID current = pending.last(); + unvisited = { }; + known = primitiveMadeOfConstants(m_program, definitionOf(current.index()), scopedLambda([&](ValueID operand) { + return findPrimitive(operand); })); - m_primitivesAsked[value.index()] = { m_stampOfBody, known }; + if (unvisited) { + pending.append(unvisited); + continue; + } + m_primitivesAsked[current.index()] = { m_stampOfBody, known }; + pending.removeLast(); } - return m_primitivesAsked[value.index()].known; + return known; } void ScanOfBody::noteLeavingOfObjectBeingMade(uint32_t root, WayOfLeavingInScan way) @@ -3375,6 +3430,9 @@ ALWAYS_INLINE bool ScanOfBody::noteEffectsOf(const BodyValue& value, Opcode opco const Effects* declaredEffects = opcode != Opcode::IsNotFrozen ? m_declaredEffectsOfEveryValue[static_cast(opcode)] : nullptr; if (!declaredEffects) declaredEffects = &declaredEffectsOfOperation(m_program, opcode, value, effectsStorage); + static constexpr Effects effectsOfNamespaceRead = Effects::ofReadByNameThatRunsNoCode(); + if (opcode == Opcode::ModuleNamespace && namespaceReadRunsNoCode(m_program, value)) + declaredEffects = &effectsOfNamespaceRead; if (opcode == Opcode::Builtin && standsForFirstReadOffHolder(m_program.builtins(), value.as().builtin(), value.as().isNotReadFromHolder())) { if (declaredEffects != &effectsStorage) effectsStorage = *declaredEffects; @@ -3752,11 +3810,13 @@ ALWAYS_INLINE_INTO_OTHER_FILES void ScanOfBody::readAccess(const BodyValue& valu } if (SummaryCall* madeKey = !key && computedKey ? findCallThatMade(computedKey) : nullptr) noteResultReadAsKey(*madeKey); + if (!key && computedKey) + noteIfParameterIsReadAsKey(computedKey, m_summary.entryParametersReadAsNames); ListInScan subject = object ? listOf(object) : listOfNode(0); auto addAccessUnderKey = [&] { uint32_t added = addAccessUnder(kind, value.id(), subject, key, computedKey); m_summary.accesses[added].keyIsNameThatLoopOverSubjectGave = kind == SummaryAccess::Kind::Read && !key && computedKey && object && isNameThatLoopOverGave(computedKey, object); - m_summary.accesses[added].beginningOfKey = beginningOfKey; + m_summary.accesses[added].whatHoldsNothingOrBeginningOfKey = beginningOfKey.bits(); return added; }; if (object) { @@ -4125,6 +4185,7 @@ void ScanOfBody::readOperand(const BodyValue& value, ValueID operand, uint32_t o ValueID source = info.assignedFromOperand ? value.operand(*info.assignedFromOperand) : ValueID(); addChange(SummaryChange::Kind::CopiesInto, value.id(), list, { }, source ? listOf(source) : listOfNode(0)); m_summary.changes.last().definesWhatItCopies = !declaredBoundInTable(value.opcode()).mayCallUnknownCode; + noteCopyAtBirth(value, value.operand(operandIndex)); m_summary.changes.last().subjectIsMadeInBody = isMadeInBody(value.operand(operandIndex)); break; } @@ -4728,6 +4789,37 @@ void ScanOfBody::leaveFieldsEmptyButFor(LayoutID id, std::span f m_summary.fieldsHeldOfLayoutsLeftEmpty.append(fieldsHeld); } +void ScanOfBody::noteCopyAtBirth(const BodyValue& copy, ValueID target) +{ + if (copy.opcode() != Opcode::CopyDataProperties) + return; + auto operation = copy.as(); + uint32_t root = m_roots[target.index()]; + if (!operation.excludedKeys().isEmpty() || !operation.excludedOperands().empty() || !(m_uses[root] & isFollowedWhileMade) || m_blocks[root] != m_blockBeingRead || m_firstLeft[root] < copy.id().index()) + return; + BodyValue made = definitionOf(root); + if (!made || made.opcode() != Opcode::NewObject || m_program.at(made.as().layout()).kind != LayoutKind::Record) + return; + ObjectBeingMadeInScan& object = objectBeingMade(root); + if (object.kind == ObjectBeingMadeKindInScan::OfOneLayout && !object.positionOfFinish && object.copyAtBirth == ThinSummary::noCopy) + object.copyAtBirth = static_cast(m_summary.changes.size() - 1); +} + +bool ScanOfBody::leaveFieldsToCopy(const ObjectBeingMadeInScan& object, LayoutID id) +{ + if (object.copyAtBirth == ThinSummary::noCopy || !object.positionOfFinish || object.isGivenUp) + return false; + listFieldsHeldBy(object, m_program.at(id), m_fieldsThatObjectHolds); + if (m_fieldsThatObjectHolds.size() == m_program.at(id).fields.size()) + return true; + m_summary.makesCopies = true; + m_summary.fieldsHeldOfLayoutsLeftEmpty.append(id.index()); + m_summary.fieldsHeldOfLayoutsLeftEmpty.append(static_cast(m_fieldsThatObjectHolds.size()) | ThinSummary::isFilledByCopy); + m_summary.fieldsHeldOfLayoutsLeftEmpty.append(object.copyAtBirth); + m_summary.fieldsHeldOfLayoutsLeftEmpty.append(m_fieldsThatObjectHolds.span()); + return true; +} + bool ScanOfBody::isListedByAllocation(uint32_t root, PropertyKeyID key) const { BodyValue made = definitionOf(root); @@ -4993,7 +5085,7 @@ void ScanOfBody::listConstructions() LayoutID id = layoutOfAllocation(definitionOf(object.root)); const Layout& layout = m_program.at(id); bool goesByLiteralAlone = layout.kind == LayoutKind::Record && !object.endsWhereItLeaves; - if (object.hasLeft || goesByLiteralAlone) + if ((object.hasLeft || goesByLiteralAlone) && !leaveFieldsToCopy(object, id)) m_placesOfObjectsOfOneLayout.append({ id, static_cast(&object - m_objectsBeingMade.begin()) }); continue; } @@ -5291,10 +5383,14 @@ ALWAYS_INLINE void ScanOfBody::readCallByValue(const BodyValue& value, SummaryCa if (arguments.size()) firstArgument = *arguments.begin(); } - if (SummaryCall* madeKey = firstArgument ? findCallThatMade(firstArgument) : nullptr) { + SummaryCall* madeKey = firstArgument ? findCallThatMade(firstArgument) : nullptr; + if (madeKey || (firstArgument && m_roots[firstArgument.index()] < m_function->numberOfEntryParameters())) { const BuiltinTable& table = m_program.builtins(); - if (findOwnMember(m_program, table.at(BuiltinClasses::Map).prototype, read.key()) || findOwnMember(m_program, table.at(BuiltinClasses::Set).prototype, read.key())) + bool isKeyOfCollection = findOwnMember(m_program, table.at(BuiltinClasses::Map).prototype, read.key()) || findOwnMember(m_program, table.at(BuiltinClasses::Set).prototype, read.key()); + if (isKeyOfCollection && madeKey) noteResultReadAsKey(*madeKey); + else if (isKeyOfCollection) + noteIfParameterIsReadAsKey(firstArgument, m_summary.entryParametersReadAsKeys); } if (m_uses[definition.id().index()] & mayComeInFrontOfDefinition) readByName(definition.id().index()); @@ -5505,6 +5601,22 @@ void ScanOfBody::noteRecordOf(ModuleID module) named.append(module); } +static bool isFilledAfterBodyReturns(const Program& program, VariableID id) +{ + const Variable& variable = program.at(id); + if (!variable.module || variable.initialization != VariableInitialization::Uninitialized) + return false; + const Module& module = program.at(variable.module); + return loaderFillsVariablesOffValueOf(module) && std::ranges::contains(module.syntheticExports, id, &SyntheticExport::variable); +} + +void ScanOfBody::noteRecordOf(VariableID variable) +{ + ModuleID module = m_program.at(variable).module; + if (module != m_function->module && !isFilledAfterBodyReturns(m_program, variable)) + noteRecordOf(module); +} + bool ScanOfBody::listsDefinitions() const { if (!m_function->isModuleBody() || !m_function->module || m_function->isCoroutine()) @@ -5768,6 +5880,15 @@ ALWAYS_INLINE const uint32_t* ScanOfBody::readTerminator(const BodyTerminator& t escape(listOf(operand)); } }); + if (kind == TerminatorKind::Return || kind == TerminatorKind::ReturnCall) { + ValueID returned = kind == TerminatorKind::Return ? terminator.operand(0) : ValueID(); + BodyValue made = returned ? definitionOf(m_roots[returned.index()]) : BodyValue(); + m_someReturnIsNoShift |= !made || made.opcode() != Opcode::URShift; + if (m_someReturnIsNoShift) + m_summary.traits.remove(SummaryTrait::ReturnsUInt32); + else + m_summary.traits.add(SummaryTrait::ReturnsUInt32); + } if (kind == TerminatorKind::Return && !terminator.operand(0) && m_function->isCoroutine()) m_returnsOfCoroutine.append(std::pair { listOfNode(constant(TypeTable::undefined)), block }); if (kind == TerminatorKind::Return && !terminator.operand(0) && !m_function->isCoroutine()) { @@ -5810,8 +5931,10 @@ ALWAYS_INLINE const uint32_t* ScanOfBody::readTerminator(const BodyTerminator& t if (proof != Proof::None) subject = test ? test.operand(operandIndex) : terminator.operand(operandIndex); TypeID passes; - if (subject && !targetIndex && test && test.opcode() == Opcode::TypeTest) - passes = test.as().testedType(); + if (subject && test && test.opcode() == Opcode::TypeTest) { + TypeID tested = test.as().testedType(); + passes = targetIndex ? TypeTable::anythingBut(tested) : tested; + } if (SummaryCall* madeAnswer = passes == TypeTable::object ? findCallThatMade(subject) : nullptr) m_answerTestedForObject = { target.block().index(), position, madeAnswer->site, { } }; if (subject) @@ -6095,6 +6218,13 @@ SummaryCall* ScanOfBody::findCallThatMade(ValueID value) return record != m_summary.calls.end() && record->site == made.id() ? &*record : nullptr; } +void ScanOfBody::noteIfParameterIsReadAsKey(ValueID key, uint16_t& parameters) +{ + uint32_t entryParameter = m_roots[key.index()]; + if (entryParameter < std::min(m_function->numberOfEntryParameters(), 16)) + parameters |= 1u << entryParameter; +} + void ScanOfBody::noteResultReadAsKey(SummaryCall& call) { call.resultIsReadAsKey = true; @@ -6526,6 +6656,11 @@ OriginInScan ScanOfBody::behindTestThatIsWritten(OriginInScan origin, TypeID pas { if (!origin.node) return { constant(passes), { } }; + if (TypeTable::passesEveryObject(passes)) { + SummaryNode::Kind kind = m_summary.nodes[origin.node].kind; + if (kind == SummaryNode::Kind::Allocation || kind == SummaryNode::Kind::ClassInstance || kind == SummaryNode::Kind::Closure || kind == SummaryNode::Kind::Builtin) + return origin; + } return narrowerOfTwoTests(origin, passes); } @@ -6645,7 +6780,7 @@ void ScanOfBody::applySinks() m_summary.arguments[sink.target] = oneNodeFor(sink.value); break; case SinkKindInScan::SubjectOfCall: - m_summary.calls[sink.target].subject = oneNodeFor(sink.value); + m_summary.calls[sink.target].subject = subjectNodeFor(sink.value); break; case SinkKindInScan::ReceiverOfCall: m_summary.calls[sink.target].receiver = oneNodeFor(sink.value); @@ -6654,7 +6789,7 @@ void ScanOfBody::applySinks() m_summary.calls[sink.target].newTarget = oneNodeFor(sink.value); break; case SinkKindInScan::SubjectOfAccess: - m_summary.accesses[sink.target].subject = oneNodeFor(sink.value); + m_summary.accesses[sink.target].subject = subjectNodeFor(sink.value); break; case SinkKindInScan::ReceiverOfAccess: m_summary.accesses[sink.target].receiver = oneNodeFor(sink.value); @@ -6666,10 +6801,10 @@ void ScanOfBody::applySinks() m_summary.accesses[sink.target].value = oneNodeFor(sink.value); break; case SinkKindInScan::SubjectOfChange: - m_summary.changes[sink.target].subject = oneNodeFor(sink.value); + m_summary.changes[sink.target].subject = subjectNodeFor(sink.value); break; case SinkKindInScan::OtherOfChange: - m_summary.changes[sink.target].other = oneNodeFor(sink.value); + m_summary.changes[sink.target].other = m_summary.changes[sink.target].kind == SummaryChange::Kind::CopiesInto ? subjectNodeFor(sink.value) : oneNodeFor(sink.value); break; case SinkKindInScan::ValueOfMember: { uint32_t value = oneNodeFor(sink.value); @@ -6688,10 +6823,10 @@ void ScanOfBody::applySinks() m_summary.changes[sink.target].valueInSight = oneNodeFor(sink.value); break; case SinkKindInScan::TargetOfCopyOfMembers: - m_summary.copiesOfMembers[sink.target].target = oneNodeFor(sink.value); + m_summary.copiesOfMembers[sink.target].target = subjectNodeFor(sink.value); break; case SinkKindInScan::SourceOfCopyOfMembers: - m_summary.copiesOfMembers[sink.target].source = oneNodeFor(sink.value); + m_summary.copiesOfMembers[sink.target].source = subjectNodeFor(sink.value); break; case SinkKindInScan::SubjectOfInspection: m_summary.inspections[sink.target].subject = oneNodeFor(sink.value); @@ -6973,6 +7108,13 @@ FunctionID ScanOfBody::functionWrittenAt(const SummaryNode& held) const return variable.initialization == VariableInitialization::Function && variable.flags.contains(VariableFlags::IsImmutable) ? variable.function : FunctionID(); } +void ScanOfBody::forgetWrittenAccessorOf(uint32_t access) +{ + m_summary.writtenAccessors.removeAllMatching([&](const SummaryWrittenAccessor& written) { + return written.access == access; + }); +} + void ScanOfBody::nameAccessorWrittenInDescriptor(uint32_t access, uint32_t root, uint32_t position, uint32_t changeOfEntryInMap, uint32_t placeWhereMapTakesEntry) { bool hasLeftIntoMapAlone = changeOfEntryInMap != changeOfNoMap && m_firstLeft[root] == placeWhereMapTakesEntry && m_lastLeft[root] == placeWhereMapTakesEntry; @@ -7084,8 +7226,8 @@ void ScanOfBody::nameAccessorWrittenInDescriptor(uint32_t access, uint32_t root, return; } m_summary.accesses[access].laysAccessorAsWritten = true; - m_summary.accesses[access].writtenGetter = functions[0]; - m_summary.accesses[access].writtenSetter = functions[1]; + forgetWrittenAccessorOf(access); + m_summary.writtenAccessors.append({ access, functions[0], functions[1], { } }); if (!hasLeftIntoMapAlone) return; uint32_t nodes[2] { halves[0] ? oneNodeFor(*halves[0]) : noSummaryNode, halves[1] ? oneNodeFor(*halves[1]) : noSummaryNode }; @@ -7189,9 +7331,7 @@ void ScanOfBody::nameKeysOfDefinitionsByMaps() access.keyNode = keyNode; access.value = valueWhereNoneIsInSight; access.laysAccessorAsWritten = false; - access.writtenGetter = { }; - access.writtenSetter = { }; - access.mapOfWrittenAccessor = { }; + forgetWrittenAccessorOf(placeOfAccess); access.descriptorFromElsewhere = noSummaryNode; isFirst = false; eachEntryLaysAccessorAsWritten &= key && descriptor; @@ -7212,7 +7352,7 @@ void ScanOfBody::nameKeysOfDefinitionsByMaps() return; if (m_summary.accesses[placeOfAccess].laysAccessorAsWritten) { if (made.opcode() == Opcode::NewObject) - m_summary.accesses[placeOfAccess].mapOfWrittenAccessor = made.as().layout(); + m_summary.writtenAccessors.last().map = made.as().layout(); return; } uint32_t from = oneNodeFor(listOf(descriptor)); @@ -7395,6 +7535,7 @@ void ScanOfBody::finish() sortWithoutRepeatsInScan(m_summary.functionsNamed); listWhatWalkAsksOfNodesAndCalls(); listStoresOnResult(); + AOT::stableSort(m_summary.writtenAccessors, { }, &SummaryWrittenAccessor::access); AOT::stableSort(m_summary.definitionsByDescriptorInOrderOfSites, { }, [&](uint32_t change) { return m_summary.changes[change].site.index(); }); @@ -7745,6 +7886,13 @@ ThinSummaries scanBodies(const Program& program, Diagnostics& diagnostics, const summaries.setSetsOfLayoutsWrittenInBodies(WTF::move(setsOfLayoutsWritten)); summaries.setNumberOfValuesScanned(numberOfValuesScanned); + Vector functionsThatMakeCopies; + for (const std::unique_ptr& scan : scans) { + if (scan) + functionsThatMakeCopies.appendVector(scan->functionsThatMakeCopies()); + } + std::ranges::sort(functionsThatMakeCopies); + summaries.setFunctionsThatMakeCopies(WTF::move(functionsThatMakeCopies)); nameKeysOfMembersByVariablesInScan(program, scans, summaries); collectSymbolValuedVariables(program, scans, summaries); listFunctionsThatCopyMembersInScan(numberOfFunctions, scans, summaries); diff --git a/Source/JavaScriptCore/aot/facts/AOTThinSummary.h b/Source/JavaScriptCore/aot/facts/AOTThinSummary.h index 17cfbb247212c..9b1dfc067295f 100644 --- a/Source/JavaScriptCore/aot/facts/AOTThinSummary.h +++ b/Source/JavaScriptCore/aot/facts/AOTThinSummary.h @@ -163,11 +163,22 @@ class WrittenUnit { }; struct SummaryEdge { - uint32_t from { 0 }; + SummaryEdge() = default; + SummaryEdge(uint32_t from, uint32_t to, TypeID passes, bool runsCodeOfValue) + : from(from) + , runsCodeOfValue(runsCodeOfValue) + , to(to) + , passes(passes) + { + RELEASE_ASSERT(this->from == from); + } + + uint32_t from : 31 { 0 }; + uint32_t runsCodeOfValue : 1 { false }; uint32_t to { 0 }; TypeID passes; - bool runsCodeOfValue { false }; }; +static_assert(sizeof(SummaryEdge) == 12); struct SummaryCall { enum class Callee : uint8_t { @@ -238,14 +249,10 @@ struct SummaryAccess { PropertyKeyID key; uint32_t keyNode { noSummaryNode }; - StringID beginningOfKey; uint32_t value { noSummaryNode }; SlotOfContents slot; - uint32_t whatHoldsNothing { noSummaryNode }; + uint32_t whatHoldsNothingOrBeginningOfKey { noSummaryNode }; uint32_t thrownTo { noSummaryNode }; - FunctionID writtenGetter; - FunctionID writtenSetter; - LayoutID mapOfWrittenAccessor; union { uint32_t descriptorFromElsewhere { noSummaryNode }; uint32_t writtenIndexOfRead; @@ -253,9 +260,20 @@ struct SummaryAccess { bool isToContents() const { return !!slot; } + uint32_t whatHoldsNothing() const { return isToContents() ? whatHoldsNothingOrBeginningOfKey : noSummaryNode; } + StringID beginningOfKey() const { return isToContents() ? StringID() : StringID::fromBits(whatHoldsNothingOrBeginningOfKey); } bool laysValueInSightUnderName() const { return !isToContents() && (kind == Kind::Store || (kind == Kind::Define && value && value != noSummaryNode)); } }; +static_assert(sizeof(SummaryAccess) == 48 && StringID().bits() == noSummaryNode); + +struct SummaryWrittenAccessor { + uint32_t access { 0 }; + FunctionID getter; + FunctionID setter; + LayoutID map; +}; + struct SummaryOperator { static constexpr uint32_t mostOperands = 2; @@ -369,6 +387,7 @@ enum class SummaryTrait : uint16_t { StoresUnderComputedKey = 1 << 7, OnlyPassesOnGatheredArguments = 1 << 8, HoldsInt32Promise = 1 << 9, + ReturnsUInt32 = 1 << 10, }; struct OtherNameOfObjectBeingMade { @@ -434,6 +453,8 @@ struct SummaryWithoutLists { uint32_t weightOfOperations { 0 }; uint32_t weightThatSwitchesSave { 0 }; uint32_t numberOfOwnCalls { 0 }; + uint16_t entryParametersReadAsNames { 0 }; + uint16_t entryParametersReadAsKeys { 0 }; uint32_t numberOfArgumentsOfOwnCalls { 0 }; OptionSet traits; bool operatorsLeftApartMayJoinTexts { false }; @@ -667,8 +688,16 @@ struct ThinSummary : SummaryWithoutLists { SummaryList placesOfWrittenLists; SummaryList definitionsByDescriptorInOrderOfSites; SummaryList callsAtSitesOfCopiesThatSpellStores; + SummaryList writtenAccessors; }; + const SummaryWrittenAccessor& writtenAccessorOf(uint32_t access) const LIFETIME_BOUND + { + auto found = std::ranges::lower_bound(writtenAccessors, access, { }, &SummaryWrittenAccessor::access); + RELEASE_ASSERT(found != writtenAccessors.end() && found->access == access); + return *found; + } + uint64_t layoutsOfOwnWrites() const { return wordsOfEffects.isEmpty() ? 0 : wordsOfEffects[0]; } bool ownEffectsMayCallUnknownCode() const; JS_EXPORT_PRIVATE Effects ownEffects() const; @@ -691,14 +720,25 @@ struct ThinSummary : SummaryWithoutLists { } } - void forEachLayoutLeftEmpty(NOESCAPE const Invocable fieldsHeld)> auto& functor) const + static constexpr uint32_t isFilledByCopy = 1u << 31; + static constexpr uint32_t noCopy = UINT32_MAX; + void forEachLayoutLeftEmptyAndCopy(NOESCAPE const Invocable fieldsHeld, uint32_t copy)> auto& functor) const { std::span rest = fieldsHeldOfLayoutsLeftEmpty.span(); while (!rest.empty()) { - functor(LayoutID(rest[0]), rest.subspan(2, rest[1])); - rest = rest.subspan(2 + rest[1]); + bool isFilled = rest[1] & isFilledByCopy; + size_t first = isFilled ? 3 : 2; + uint32_t numberHeld = rest[1] & ~isFilledByCopy; + functor(LayoutID(rest[0]), rest.subspan(first, numberHeld), isFilled ? rest[2] : noCopy); + rest = rest.subspan(first + numberHeld); } } + void forEachLayoutLeftEmpty(NOESCAPE const Invocable fieldsHeld)> auto& functor) const + { + forEachLayoutLeftEmptyAndCopy([&](LayoutID layout, std::span fieldsHeld, uint32_t) { + functor(layout, fieldsHeld); + }); + } bool leavesEmpty(FieldReference field) const { @@ -713,6 +753,7 @@ struct ThinSummary : SummaryWithoutLists { static_assert(sizeof(ThinSummary) <= 128); struct SummaryBeingMade : SummaryWithoutLists { + bool makesCopies { false }; Vector nodes; Vector edges; Vector calls; @@ -771,6 +812,7 @@ struct SummaryBeingMade : SummaryWithoutLists { Vector placesOfWrittenLists; Vector definitionsByDescriptorInOrderOfSites; Vector callsAtSitesOfCopiesThatSpellStores; + Vector writtenAccessors; uint64_t layoutsOfOwnWrites { 0 }; Effects ownEffects; std::optional ownEffectsOfOperators; @@ -804,6 +846,8 @@ class ThinSummaries { void setNumberOfValuesScanned(uint64_t number) { m_numberOfValuesScanned = number; } std::span functionsThatCopyMembers() const LIFETIME_BOUND { return m_functionsThatCopyMembers.span(); } void setFunctionsThatCopyMembers(Vector&& functions) { m_functionsThatCopyMembers = WTF::move(functions); } + std::span functionsThatMakeCopies() const LIFETIME_BOUND { return m_functionsThatMakeCopies.span(); } + void setFunctionsThatMakeCopies(Vector&& functions) { m_functionsThatMakeCopies = WTF::move(functions); } struct MakerOfClosures { FunctionID closure; FunctionID maker; @@ -824,6 +868,7 @@ class ThinSummaries { SetsOfLayoutsWritten m_setsOfLayoutsWrittenInBodies; uint64_t m_numberOfValuesScanned { 0 }; Vector m_functionsThatCopyMembers; + Vector m_functionsThatMakeCopies; Vector m_makersAboveCopiesOfMembers; Vector m_symbolsOfVariables; }; diff --git a/Source/JavaScriptCore/aot/facts/AOTUnitedFamilies.h b/Source/JavaScriptCore/aot/facts/AOTUnitedFamilies.h index 379390b58fe15..e7aabb66da023 100644 --- a/Source/JavaScriptCore/aot/facts/AOTUnitedFamilies.h +++ b/Source/JavaScriptCore/aot/facts/AOTUnitedFamilies.h @@ -11,6 +11,8 @@ #include #include +WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN + namespace JSC { namespace AOT { namespace Typed { class UnitedFamilies { @@ -38,10 +40,11 @@ class UnitedFamilies { { if (member >= m_parent.size()) return member; - for (uint32_t parent = parentOf(member); parent != member; parent = parentOf(member)) { - uint32_t above = parentOf(parent); + uint32_t* parents = m_parent.mutableSpan().data(); + for (uint32_t parent = parentOf(parents, member); parent != member; parent = parentOf(parents, member)) { + uint32_t above = parentOf(parents, parent); if (above != parent) - WTF::atomicStore(&m_parent[member], above, std::memory_order_relaxed); + WTF::atomicStore(parents + member, above, std::memory_order_relaxed); member = above; } return member; @@ -51,7 +54,8 @@ class UnitedFamilies { { if (member >= m_parent.size()) return member; - for (uint32_t parent = parentOf(member); parent != member; parent = parentOf(member)) + const uint32_t* parents = m_parent.span().data(); + for (uint32_t parent = parentOf(parents, member); parent != member; parent = parentOf(parents, member)) member = parent; return member; } @@ -88,7 +92,7 @@ class UnitedFamilies { } private: - uint32_t parentOf(uint32_t member) const { return WTF::atomicLoad(&m_parent[member], std::memory_order_relaxed); } + static uint32_t parentOf(const uint32_t* parents, uint32_t member) { return WTF::atomicLoad(const_cast(parents) + member, std::memory_order_relaxed); } mutable Vector m_parent; Vector m_next; @@ -97,4 +101,6 @@ class UnitedFamilies { } } } // namespace JSC::AOT::Typed +WTF_ALLOW_UNSAFE_BUFFER_USAGE_END + #endif // ENABLE(AOT) diff --git a/Source/JavaScriptCore/aot/image/AOTCallSiteTable.cpp b/Source/JavaScriptCore/aot/image/AOTCallSiteTable.cpp index 8c2c97a50be4d..575267ffe6d73 100644 --- a/Source/JavaScriptCore/aot/image/AOTCallSiteTable.cpp +++ b/Source/JavaScriptCore/aot/image/AOTCallSiteTable.cpp @@ -15,6 +15,7 @@ #include "AOTImageFormat.h" #include "AOTLinkedFunctions.h" #include "AOTNumbering.h" +#include "AOTPositionEncoder.h" #include "AOTSort.h" #include "AOTStubKey.h" #include "AOTStubsPolls.h" @@ -35,22 +36,6 @@ namespace JSC { namespace AOT { namespace { -struct SiteCheckpoint { - static constexpr unsigned sitesBetween = 64; - static uint64_t numberFor(uint64_t numberOfSites) { return numberOfSites ? (numberOfSites - 1) / sitesBetween : 0; } - static SiteCheckpoint at(const uint8_t* first, uint64_t index) - { - SiteCheckpoint result; - memcpy(&result, first + index * sizeof(SiteCheckpoint), sizeof(SiteCheckpoint)); - return result; - } - - uint32_t offset; - uint32_t position; - uint32_t frame; -}; -static_assert(sizeof(SiteCheckpoint) == 12 && sizeof(SiteCheckpoint::position) == 4 && sizeof(SiteCheckpoint::frame) == 4); - struct SiteWithCode { uint32_t returnOffset; uint32_t site; @@ -66,32 +51,80 @@ struct FirstReadDetails { uint32_t value; }; -ALWAYS_INLINE void updateFrame(const CallSiteRow& row, unsigned& frame) +ALWAYS_INLINE void updateFrame(const CallSiteDetails& site, unsigned& frame) { - if (!row.hasDetails) - return; - if (row.details.kind != CallSiteKind::FirstRead) [[likely]] { - frame = row.details.frame; + if (site.kind != CallSiteKind::FirstRead) [[likely]] { + frame = site.frame; return; } - FirstReadDetails details = FirstReadDetails::decode(row.details.frame); + FirstReadDetails details = FirstReadDetails::decode(site.frame); if (!details.hasNoSite) frame = details.value; } -struct StreamHead : CallSiteHead { - const uint8_t* sites() const { return checkpoints + SiteCheckpoint::numberFor(numberOfSites) * sizeof(SiteCheckpoint); } +struct PrologueCheckOffsets { + uint32_t callOffset; + uint32_t resumeOffset; +}; +struct StreamHead { + SideTableReader readerAt(const uint8_t* begin) const { return reader.at(begin); } + const uint8_t* firstBlock() const { return checkpoints + SiteCheckpoint::numberFor(numberOfSites) * sizeof(SiteCheckpoint); } + + SideTableReader reader; + uint64_t numberOfSites { 0 }; + uint64_t numberOfSitesWithoutCode { 0 }; + uint64_t numberOfInlineFrames { 0 }; + Vector prologueChecks; + const uint8_t* handlers { nullptr }; + const uint8_t* inlineFrameParents { nullptr }; + const uint8_t* inlineFunctions { nullptr }; const uint8_t* checkpoints { nullptr }; }; struct CallSiteScratch { Vector returnSites; Vector laterRunOfReturnSites; - Vector head; - Vector frames; - Vector checkpoints; - Vector sites; + Vector head; + Vector parents; + Vector functions; + Vector sites; + Vector blocks; +}; + +struct InlineFrameSymbols { + ALWAYS_INLINE friend void appendVarint(InlineFrameSymbols& row, uint64_t value) + { + if (std::exchange(row.writesFunction, !row.writesFunction)) + row.functions.appendVarint(SideTableByte::InlineFunction, SideTableByte::InlineFunctionContinuation, value); + else + row.parents.appendVarint(SideTableByte::InlineFrameParent, value); + } + + SideTableSymbolWriter parents; + SideTableSymbolWriter functions; + bool writesFunction { false }; +}; + +struct SiteDetailsSymbols { + ALWAYS_INLINE friend void appendVarint(SiteDetailsSymbols& details, uint64_t value) + { + if (std::exchange(details.writesBuiltin, false)) { + details.symbols.appendVarint(SideTableByte::CalledBuiltin, SideTableByte::CalledBuiltinContinuation, value); + return; + } + if (details.followsInlined) + details.symbols.appendVarint(SideTableByte::SiteDetailsAfterInlined, SideTableByte::SiteDetailsAfterInlinedContinuation, value); + else + details.symbols.appendVarint(SideTableByte::SiteDetails, SideTableByte::SiteDetailsContinuation, value); + SiteDetailsWord word { value }; + details.followsInlined = word.isInlined(); + details.writesBuiltin = namesBuiltinOf(word.kind()); + } + + SideTableSymbolWriter& symbols; + bool followsInlined { false }; + bool writesBuiltin { false }; }; } // anonymous namespace @@ -183,6 +216,8 @@ static size_t inlineFrameCount(const SiteDescription& sites) static void encodeSites(CallSiteScratch& scratch, const CompiledCode& code) { + constexpr size_t maxSymbolsOfDistance = 5; + constexpr size_t maxSymbolsOfSite = 1 + CallSiteDetails::maxSize + maxSymbolsOfDistance; const SiteDescription& sites = code.sites; std::span returnSites = scratch.returnSites.span(); size_t numberOfInlineFrames = inlineFrameCount(sites); @@ -192,48 +227,68 @@ static void encodeSites(CallSiteScratch& scratch, const CompiledCode& code) unsigned previousFrame = 0; bool previousIsFirstRead = false; uint32_t number = sites.numberOfSitesWithoutCode; - scratch.checkpoints.shrink(0); + scratch.blocks.shrink(0); scratch.sites.shrink(0); - scratch.sites.grow(returnSites.size() * CallSiteRow::maxSize); - uint8_t* firstSiteByte = scratch.sites.mutableSpan().data(); - WriterOfReservedBytes siteBytes(firstSiteByte); - unsigned writtenCount = 0; - for (auto [offset, site, isFirstRead, firstReadColumnDelta] : returnSites) { - ++number; - RELEASE_ASSERT(site == number); - RELEASE_ASSERT(previousIsFirstRead ? offset == previousOffset : offset > previousOffset); - if (writtenCount && !(writtenCount % SiteCheckpoint::sitesBetween)) { - SiteCheckpoint checkpoint { static_cast(previousOffset / codeOffsetUnit), static_cast(siteBytes.end() - firstSiteByte), previousFrame }; - scratch.checkpoints.append(asByteSpan(checkpoint)); - } - ++writtenCount; - unsigned frame = positions[number].inlineFrame; - RELEASE_ASSERT(frame <= numberOfInlineFrames); - while (!flaggedSites.empty() && flaggedSites.front().site < number) - flaggedSites = flaggedSites.subspan(1); - CallSiteRow row { .distance = offset - previousOffset }; - bool isFlagged = !flaggedSites.empty() && flaggedSites.front().site == number; - if (isFirstRead || firstReadColumnDelta) { - RELEASE_ASSERT(isFirstRead || (!isFlagged && frame == previousFrame)); - row.details.kind = CallSiteKind::FirstRead; - } else if (isFlagged) { - row.details.kind = flaggedSites.front().kind; - row.details.calledBuiltin = flaggedSites.front().calledBuiltin; + scratch.sites.grow(returnSites.size() * maxSymbolsOfSite + SiteCheckpoint::numberFor(returnSites.size()) + 1); + PrefixSymbol* firstSymbol = scratch.sites.mutableSpan().data(); + SideTableSymbolWriter symbols(firstSymbol); + std::array distances; + while (!returnSites.empty()) { + std::span block = returnSites.first(std::min(returnSites.size(), SiteCheckpoint::sitesBetween)); + returnSites = returnSites.subspan(block.size()); + CallSiteStreamHead::Block head { 0, static_cast(previousOffset / codeOffsetUnit), previousFrame }; + PrefixSymbol* firstSymbolOfBlock = symbols.end(); + symbols.append(SideTableByte::SitesWithDetails, 0); + SideTableSymbolWriter distanceSymbols(distances.data()); + SiteDetailsSymbols detailsSymbols { symbols }; + unsigned sitesWithDetails = 0; + unsigned sitesSkipped = 0; + bool followsDetails = false; + for (auto [offset, site, isFirstRead, firstReadColumnDelta] : block) { + ++number; + RELEASE_ASSERT(site == number); + RELEASE_ASSERT(previousIsFirstRead ? offset == previousOffset : offset > previousOffset); + unsigned frame = positions[number].inlineFrame; + RELEASE_ASSERT(frame <= numberOfInlineFrames); + while (!flaggedSites.empty() && flaggedSites.front().site < number) + flaggedSites = flaggedSites.subspan(1); + uint32_t distance = offset - previousOffset; + RELEASE_ASSERT(!(distance % codeOffsetUnit)); + if (followsDetails) + distanceSymbols.appendVarint(SideTableByte::DistanceAfterDetails, SideTableByte::DistanceAfterDetailsContinuation, distance / codeOffsetUnit); + else + distanceSymbols.appendVarint(SideTableByte::Distance, SideTableByte::DistanceContinuation, distance / codeOffsetUnit); + CallSiteDetails details; + bool isFlagged = !flaggedSites.empty() && flaggedSites.front().site == number; + if (isFirstRead || firstReadColumnDelta) { + RELEASE_ASSERT(isFirstRead || (!isFlagged && frame == previousFrame)); + details.kind = CallSiteKind::FirstRead; + } else if (isFlagged) { + details.kind = flaggedSites.front().kind; + details.calledBuiltin = flaggedSites.front().calledBuiltin; + } + followsDetails = frame != previousFrame || details.kind != CallSiteKind::Plain; + if (followsDetails) { + details.frame = details.kind == CallSiteKind::FirstRead ? FirstReadDetails { !isFirstRead, isFirstRead ? frame : firstReadColumnDelta }.encode() : frame; + symbols.append(SideTableByte::SitesSkipped, static_cast(sitesSkipped)); + details.append(detailsSymbols); + ++sitesWithDetails; + sitesSkipped = 0; + } else + ++sitesSkipped; + previousOffset = offset; + previousFrame = frame; + previousIsFirstRead = isFirstRead; } - row.hasDetails = frame != previousFrame || row.details.kind != CallSiteKind::Plain; - if (row.details.kind == CallSiteKind::FirstRead) - row.details.frame = FirstReadDetails { !isFirstRead, isFirstRead ? frame : firstReadColumnDelta }.encode(); - else if (row.hasDetails) - row.details.frame = frame; - row.append(siteBytes); - previousOffset = offset; - previousFrame = frame; - previousIsFirstRead = isFirstRead; + *firstSymbolOfBlock |= static_cast(sitesWithDetails); + symbols.append(std::span(distances.data(), distanceSymbols.end())); + head.numberOfSymbols = static_cast(symbols.end() - firstSymbolOfBlock); + scratch.blocks.append(head); } - scratch.sites.shrink(siteBytes.end() - firstSiteByte); + scratch.sites.shrink(symbols.end() - firstSymbol); } -static void encodeStream(Vector& stream, CallSiteScratch& scratch, const CompiledCode& code, const FunctionNumbers& numbers, const Typed::CallingConventions& conventions) +static void encodeStream(Vector& stream, CallSiteScratch& scratch, const CompiledCode& code, const FunctionNumbers& numbers, const Typed::CallingConventions& conventions) { const SiteDescription& sites = code.sites; std::span prologueChecks = code.frame.prologueChecks.span(); @@ -252,20 +307,17 @@ static void encodeStream(Vector& stream, CallSiteScratch& scratch, cons if (returnSites.isEmpty() && prologueChecks.empty()) return; - CallSiteHead head; - head.numberOfSites = returnSites.size(); - head.hasPrologueChecks = !prologueChecks.empty(); - head.prologueChecks.reserveInitialCapacity(prologueChecks.size()); - for (const PrologueCheck& check : prologueChecks) - head.prologueChecks.append({ .callOffset = check.callOffset, .resumeOffset = check.resumeOffset }); - head.numberOfSitesWithoutCode = sites.numberOfSitesWithoutCode; + CallSiteStreamHead head { }; + scratch.parents.shrink(0); + scratch.functions.shrink(0); if (sites.hasEncodedInlineFrames()) { std::span inlineFrames = sites.inlineFrames.span(); RELEASE_ASSERT(inlineFrames.size() - 1 <= sites.numberOfSitesWithoutCode); - scratch.frames.shrink(0); - scratch.frames.grow((inlineFrames.size() - 1) * InlineFrameRow::maxSize); - uint8_t* firstFrameByte = scratch.frames.mutableSpan().data(); - WriterOfReservedBytes frames(firstFrameByte); + scratch.parents.grow((inlineFrames.size() - 1) * InlineFrameRow::maxSize); + scratch.functions.grow((inlineFrames.size() - 1) * InlineFrameRow::maxSize); + PrefixSymbol* firstParent = scratch.parents.mutableSpan().data(); + PrefixSymbol* firstFunction = scratch.functions.mutableSpan().data(); + InlineFrameSymbols rows { SideTableSymbolWriter(firstParent), SideTableSymbolWriter(firstFunction) }; for (unsigned frame = 1; frame < inlineFrames.size(); ++frame) { RELEASE_ASSERT(inlineFrames[frame].parent < frame); RELEASE_ASSERT(inlineFrames[frame].callSite == sites.numberOfSitesWithoutCode - (inlineFrames.size() - 1) + frame); @@ -273,24 +325,52 @@ static void encodeStream(Vector& stream, CallSiteScratch& scratch, cons Typed::CodeUnit shown = code.body.callees[inlineFrames[frame].callee]; FunctionID function = calledBuiltin ? functionInFramesOf(calledBuiltin, numbers.numberOfFunctions()) : shown.role == Typed::CodeRole::Body ? shown.function : numbers.idOf(conventions.indexOf(shown)); RELEASE_ASSERT(function); - InlineFrameRow { .isTailCall = inlineFrames[frame].isTailCall, .isConstruct = inlineFrames[frame].isConstruct, .parent = inlineFrames[frame].parent, .function = function }.append(frames); + InlineFrameRow { .isTailCall = inlineFrames[frame].isTailCall, .isConstruct = inlineFrames[frame].isConstruct, .parent = inlineFrames[frame].parent, .function = function }.append(rows); } + scratch.parents.shrink(rows.parents.end() - firstParent); + scratch.functions.shrink(rows.functions.end() - firstFunction); head.hasInlineFrames = true; - head.numberOfInlineFrames = inlineFrameCount(sites); - head.inlineFrames = std::span(firstFrameByte, frames.end()); } + + constexpr size_t maxSymbolsOfWord = 10; scratch.head.shrink(0); - head.append(scratch.head); + scratch.head.grow((4 + 2 * prologueChecks.size()) * maxSymbolsOfWord); + PrefixSymbol* firstHeadSymbol = scratch.head.mutableSpan().data(); + SideTableSymbolWriter headSymbols(firstHeadSymbol); + headSymbols.appendVarint(SideTableByte::CallSiteHead, SideTableByte::CallSiteHeadContinuation, static_cast(returnSites.size()) << 2 | static_cast(!prologueChecks.empty()) << 1 | head.hasInlineFrames); + if (!prologueChecks.empty()) { + headSymbols.appendVarint(SideTableByte::PrologueCheckCount, prologueChecks.size()); + for (const PrologueCheck& check : prologueChecks) { + RELEASE_ASSERT(!(check.callOffset % codeOffsetUnit) && !(check.resumeOffset % codeOffsetUnit)); + headSymbols.appendVarint(SideTableByte::PrologueCall, SideTableByte::PrologueCallContinuation, check.callOffset / codeOffsetUnit); + headSymbols.appendVarint(SideTableByte::PrologueResume, check.resumeOffset / codeOffsetUnit); + } + } + headSymbols.appendVarint(SideTableByte::SitesWithoutCode, sites.numberOfSitesWithoutCode); + if (head.hasInlineFrames) + headSymbols.appendVarint(SideTableByte::InlineFrameCount, inlineFrameCount(sites)); + scratch.head.shrink(headSymbols.end() - firstHeadSymbol); + encodeSites(scratch, code); - stream.reserveInitialCapacity(scratch.head.size() + scratch.checkpoints.size() + scratch.sites.size()); + head.numberOfHeadSymbols = static_cast(scratch.head.size()); + head.numberOfParentSymbols = static_cast(scratch.parents.size()); + head.numberOfFunctionSymbols = static_cast(scratch.functions.size()); + head.numberOfBlocks = static_cast(scratch.blocks.size()); + auto symbolsOfBytes = [](std::span bytes) { + return spanReinterpretCast(bytes); + }; + stream.reserveInitialCapacity((sizeof(head) + scratch.blocks.sizeInBytes()) / sizeof(PrefixSymbol) + scratch.head.size() + scratch.parents.size() + scratch.functions.size() + scratch.sites.size()); + stream.append(symbolsOfBytes(asByteSpan(head))); + stream.append(symbolsOfBytes(asBytes(scratch.blocks.span()))); stream.append(scratch.head.span()); - stream.append(scratch.checkpoints.span()); + stream.append(scratch.parents.span()); + stream.append(scratch.functions.span()); stream.append(scratch.sites.span()); } -Vector> encodeCallSites(const LinkedFunctions& functions, const FunctionNumbers& numbers, const Typed::CallingConventions& conventions, BuildContext& context) +Vector> encodeCallSites(const LinkedFunctions& functions, const FunctionNumbers& numbers, const Typed::CallingConventions& conventions, BuildContext& context) { - Vector> streams(functions.size()); + Vector> streams(functions.size()); ChunksOfIndices chunks = divideIntoChunks(static_cast(functions.size())); forEachIndexInParallel(context.parallelLoops(), chunks.size(), [&](unsigned chunk) { CallSiteScratch scratch; @@ -304,27 +384,79 @@ Vector> encodeCallSites(const LinkedFunctions& functions, const return streams; } -static StreamHead readHead(const uint8_t* cursor) +SideTableParts SideTableParts::read(ImageFunction function, std::span section, uint32_t offset) { - StreamHead head { CallSiteHead::read(cursor) }; - head.checkpoints = cursor; - return head; + SideTableParts parts; + SideTableReader reader(section, section.data() + function.sideTables); + reader.next(SideTableByte::Flags); + parts.frame = SideTableFrame::read(reader, offset); + if (function.has(ImageFunctionFlag::HasFrameMaps)) { + parts.firstFrameMapWord = static_cast(reader.nextVarint(SideTableByte::FrameMapStart, SideTableByte::FrameMapStartContinuation)); + parts.numberOfFrameMapWords = static_cast(reader.nextVarint(SideTableByte::FrameMapSize)); + } + parts.hasHandlers = function.has(ImageFunctionFlag::HasHandlers); + if (function.has(ImageFunctionFlag::HasConstructedCallees)) { + for (uint64_t size = reader.nextVarint(SideTableByte::Continuation); size; --size) + parts.constructedCallees.append(reader.next(SideTableByte::ConstructedCallee)); + } + if (function.has(ImageFunctionFlag::HasCallSites)) + parts.callSites = reader; + return parts; } -static const uint8_t* findCallSites(const Image& image, uint32_t function) +std::optional SideTableParts::constructedCalleeAt(uint32_t site) const { - const ImageFunction& record = image.function(function); - if (!record.has(ImageFunctionFlag::HasCallSites)) - return nullptr; - return SideTableParts::read(record, image.section(ImageSection::SideTables).data()).callSites; + const uint8_t* cursor = constructedCallees.begin(); + const uint8_t* end = constructedCallees.end(); + uint64_t listed = 0; + while (cursor < end && listed < site) { + listed += readVarint(cursor); + uint32_t function = static_cast(readVarint(cursor)); + if (listed == site) + return function; + } + return std::nullopt; +} + +static StreamHead readHead(const SideTableParts& parts) +{ + SideTableReader reader = *parts.callSites; + StreamHead head { .reader = reader }; + uint64_t bits = reader.nextVarint(SideTableByte::CallSiteHead, SideTableByte::CallSiteHeadContinuation); + bool hasInlineFrames = bits & 1; + bool hasPrologueChecks = bits & 2; + head.numberOfSites = bits >> 2; + if (hasPrologueChecks) { + for (uint64_t count = reader.nextVarint(SideTableByte::PrologueCheckCount); count; --count) { + uint64_t call = reader.nextVarint(SideTableByte::PrologueCall, SideTableByte::PrologueCallContinuation); + uint64_t resume = reader.nextVarint(SideTableByte::PrologueResume); + head.prologueChecks.append({ static_cast(call * codeOffsetUnit), static_cast(resume * codeOffsetUnit) }); + } + } + head.numberOfSitesWithoutCode = reader.nextVarint(SideTableByte::SitesWithoutCode); + if (hasInlineFrames) + head.numberOfInlineFrames = reader.nextVarint(SideTableByte::InlineFrameCount); + const uint8_t* next = reader.alignToByte(); + auto passPart = [&] { + size_t size = static_cast(readVarint(next)); + return std::exchange(next, next + size); + }; + if (parts.hasHandlers) + head.handlers = passPart(); + if (hasInlineFrames) { + head.inlineFrameParents = passPart(); + head.inlineFunctions = passPart(); + } + head.checkpoints = next; + return head; } static std::optional readHead(const Image& image, uint32_t function) { - const uint8_t* callSites = findCallSites(image, function); - if (!callSites) + ImageFunction record = image.function(function); + if (!record.has(ImageFunctionFlag::HasCallSites)) return std::nullopt; - return readHead(callSites); + return readHead(SideTableParts::read(record, image.section(ImageSection::SideTables))); } void InlineFrameReader::read(const Image& image, uint32_t function, unsigned last) @@ -380,7 +512,7 @@ void InlineFrameReader::decode(const Image& image, uint32_t function, unsigned l m_image = ℑ m_function = function; m_frames.shrink(0); - m_rest = { head->inlineFrames.data(), head->numberOfInlineFrames }; + m_rest = { head->readerAt(head->inlineFrameParents), head->readerAt(head->inlineFunctions), head->numberOfInlineFrames }; callSite = static_cast(head->numberOfSitesWithoutCode - head->numberOfInlineFrames); } unsigned numberOfDecodedFrames = static_cast(m_frames.size()); @@ -388,7 +520,8 @@ void InlineFrameReader::decode(const Image& image, uint32_t function, unsigned l if (last > m_frames.capacity()) m_frames.reserveCapacity(std::max(last, std::min(2 * m_frames.capacity(), numberOfDecodedFrames + m_rest.count))); for (unsigned frame = numberOfDecodedFrames + 1; frame <= last; ++frame) { - InlineFrameRow row = InlineFrameRow::read(m_rest.rows); + InlineFrameCursor cursor { *m_rest.parents, *m_rest.functions }; + InlineFrameRow row = InlineFrameRow::read(cursor); RELEASE_ASSERT(row.parent < frame); m_frames.unsafeAppendWithoutCapacityCheck(InlineFrameRecord { row, CallSiteIndex(++callSite) }); } @@ -396,23 +529,28 @@ void InlineFrameReader::decode(const Image& image, uint32_t function, unsigned l } SiteFrames::SiteFrames(const Image& image, uint32_t function) - : SiteFrames(findCallSites(image, function)) { + std::optional head = readHead(image, function); + if (!head || !head->numberOfSites) + return; + m_remainingInlineFrames = head->numberOfInlineFrames; + if (m_remainingInlineFrames) + m_inlineFrameParents = head->readerAt(head->inlineFrameParents); + RELEASE_ASSERT(m_remainingInlineFrames <= head->numberOfSitesWithoutCode); + m_remainingSuspensionSites = head->numberOfSitesWithoutCode - m_remainingInlineFrames; + m_remainingSitesWithCode = head->numberOfSites; + m_checkpoints = head->checkpoints; + m_firstBlock = head->firstBlock(); + m_sites = head->readerAt(m_firstBlock); + startBlock(m_firstBlock); } -SiteFrames::SiteFrames(const uint8_t* callSites) +void SiteFrames::startBlock(const uint8_t* begin) { - if (!callSites) - return; - StreamHead head = readHead(callSites); - if (!head.numberOfSites) - return; - m_remainingInlineFrames = head.numberOfInlineFrames; - m_inlineFrames = head.inlineFrames.data(); - RELEASE_ASSERT(m_remainingInlineFrames <= head.numberOfSitesWithoutCode); - m_remainingSuspensionSites = head.numberOfSitesWithoutCode - m_remainingInlineFrames; - m_remainingSitesWithCode = head.numberOfSites; - m_sites = head.sites(); + m_sites = m_sites->at(begin); + m_details.emplace(*m_sites); + m_siteInBlock = 0; + ++m_nextBlock; } unsigned SiteFrames::next() @@ -423,12 +561,15 @@ unsigned SiteFrames::next() } if (m_remainingInlineFrames) { --m_remainingInlineFrames; - return InlineFrameRow::read(m_inlineFrames).parent; + return static_cast(m_inlineFrameParents->nextVarint(SideTableByte::InlineFrameParent) >> 2); } if (!m_remainingSitesWithCode) return 0; --m_remainingSitesWithCode; - updateFrame(CallSiteRow::read(m_sites), m_frame); + if (m_siteInBlock == SiteCheckpoint::sitesBetween) + startBlock(m_firstBlock + SiteCheckpoint::at(m_checkpoints, m_nextBlock - 1).position); + if (m_details->isAt(m_siteInBlock++)) + updateFrame(m_details->next(), m_frame); return m_frame; } @@ -441,8 +582,9 @@ void SiteFrames::skip(uint64_t numberOfSites) next(); uint64_t numberOfCheckpoints = SiteCheckpoint::numberFor(m_remainingSitesWithCode); if (uint64_t precedingCount = std::min(numberOfSites / SiteCheckpoint::sitesBetween, numberOfCheckpoints)) { - SiteCheckpoint checkpoint = SiteCheckpoint::at(m_sites - numberOfCheckpoints * sizeof(SiteCheckpoint), precedingCount - 1); - m_sites += checkpoint.position; + SiteCheckpoint checkpoint = SiteCheckpoint::at(m_checkpoints, precedingCount - 1); + m_nextBlock = precedingCount; + startBlock(m_firstBlock + checkpoint.position); m_frame = checkpoint.frame; m_remainingSitesWithCode -= precedingCount * SiteCheckpoint::sitesBetween; numberOfSites -= precedingCount * SiteCheckpoint::sitesBetween; @@ -457,7 +599,7 @@ static std::optional findCallSite(const StreamHead* head, uint32 if (callOffset + sizeOfNearCall == returnOffset) return std::nullopt; } - const uint8_t* cursor = head->sites(); + const uint8_t* block = head->firstBlock(); uint32_t offset = 0; uint64_t number = head->numberOfSitesWithoutCode; unsigned frame = 0; @@ -476,7 +618,7 @@ static std::optional findCallSite(const StreamHead* head, uint32 SiteCheckpoint checkpoint = SiteCheckpoint::at(head->checkpoints, precedingCount - 1); offset = checkpoint.offset * codeOffsetUnit; frame = checkpoint.frame; - cursor += checkpoint.position; + block += checkpoint.position; number += precedingCount * SiteCheckpoint::sitesBetween; count -= precedingCount * SiteCheckpoint::sitesBetween; } @@ -484,24 +626,50 @@ static std::optional findCallSite(const StreamHead* head, uint32 uint64_t numberInFront = 0; unsigned frameInFront = 0; std::optional firstReadSite; + struct DetailedSite { + unsigned site; + CallSiteDetails details; + }; + Vector detailedSites; + SideTableReader reader = head->readerAt(block); + const SideTableReader::DistanceCodes codes = reader.distanceCodes(); + const DetailedSite* nextDetailed = nullptr; + unsigned siteInBlock = SiteCheckpoint::sitesBetween; + bool followsDetails = false; while (count--) { - CallSiteRow row = CallSiteRow::read(cursor); - offset += row.distance; - updateFrame(row, frame); + if (siteInBlock == SiteCheckpoint::sitesBetween) { + reader.alignToByte(); + detailedSites.shrink(0); + for (SiteDetailsReader details(reader); !details.isAtEnd();) { + unsigned site = details.site(); + detailedSites.append({ site, details.next() }); + } + nextDetailed = detailedSites.begin(); + siteInBlock = 0; + followsDetails = false; + } + offset += static_cast(reader.nextDistance(codes, followsDetails) * codeOffsetUnit); + followsDetails = nextDetailed != detailedSites.end() && nextDetailed->site == siteInBlock; + ++siteInBlock; + CallSiteDetails details; + if (followsDetails) { + details = nextDetailed++->details; + updateFrame(details, frame); + } ++number; if (offset > returnOffset) break; if (offset < returnOffset) { - hasSiteInFront = row.details.kind == CallSiteKind::Plain; + hasSiteInFront = details.kind == CallSiteKind::Plain; numberInFront = number; frameInFront = frame; continue; } CallSiteIndex callSite(static_cast(number)); - if (row.details.kind != CallSiteKind::FirstRead) - return CallSiteRecord { callSite, frame, row.details.kind, row.details.calledBuiltin, firstReadSite }; - if (FirstReadDetails details = FirstReadDetails::decode(row.details.frame); details.hasNoSite) - return CallSiteRecord { callSite, frame, CallSiteKind::Plain, { }, firstReadSiteInFrontOf(callSite, details.value) }; + if (details.kind != CallSiteKind::FirstRead) + return CallSiteRecord { callSite, frame, details.kind, details.calledBuiltin, firstReadSite }; + if (FirstReadDetails firstRead = FirstReadDetails::decode(details.frame); firstRead.hasNoSite) + return CallSiteRecord { callSite, frame, CallSiteKind::Plain, { }, firstReadSiteInFrontOf(callSite, firstRead.value) }; firstReadSite = callSite; } if (!hasSiteInFront) @@ -519,26 +687,64 @@ std::optional Image::callSiteAt(uint32_t function, uint32_t retu std::optional Image::constructedCalleeAt(uint32_t function, CallSiteIndex site) const { - const ImageFunction& record = this->function(function); + ImageFunction record = this->function(function); if (!record.has(ImageFunctionFlag::HasConstructedCallees)) [[likely]] return std::nullopt; - std::optional callee = SideTableParts::read(record, section(ImageSection::SideTables).data()).constructedCalleeAt(site.bits()); + std::optional callee = SideTableParts::read(record, section(ImageSection::SideTables)).constructedCalleeAt(site.bits()); return callee ? std::optional(FunctionID(*callee)) : std::nullopt; } +static std::optional findHandlerAmong(const Invocable()> auto& nextRow, CallSiteIndex site) +{ + uint64_t endOfLast = 0; + int64_t target = 0; + while (std::optional row = nextRow()) { + uint64_t start = endOfLast + row->sitesSkipped; + endOfLast = start + row->furtherSites + 1; + target += row->targetChange; + if (site.bits() < start) + return std::nullopt; + if (site.bits() < endOfLast) + return HandlerRecord { static_cast(target), row->type }; + } + return std::nullopt; +} + +static std::optional findHandler(const StreamHead& head, CallSiteIndex site) +{ + if (!head.handlers) + return std::nullopt; + SideTableReader reader = head.readerAt(head.handlers); + uint64_t rowsLeft = reader.nextVarint(SideTableByte::HandlerRows); + return findHandlerAmong([&]() -> std::optional { + if (!rowsLeft) + return std::nullopt; + --rowsLeft; + HandlerCursor cursor { reader }; + return HandlerRow::read(cursor); + }, site); +} + +std::optional Image::handlerAt(uint32_t function, CallSiteIndex site) const +{ + std::optional head = readHead(*this, function); + return head ? findHandler(*head, site) : std::nullopt; +} + ReturnSiteInfo Image::returnSiteInfoAt(uint32_t function, uint32_t returnOffset) const { - const ImageFunction& record = this->function(function); - SideTableParts parts = SideTableParts::read(record, section(ImageSection::SideTables).data()); + ImageFunction record = this->function(function); + SideTableParts parts = SideTableParts::read(record, section(ImageSection::SideTables), returnOffset ? returnOffset - 1 : 0); ReturnSiteInfo result; + result.hasConstructedCallees = record.has(ImageFunctionFlag::HasConstructedCallees); if (parts.callSites) { - StreamHead head = readHead(parts.callSites); + StreamHead head = readHead(parts); result.site = findCallSite(&head, returnOffset); + if (result.site) + result.handler = findHandler(head, result.site->callSite); } - if (result.site && !parts.handlers.empty()) - result.handler = findHandler(parts.handlers, result.site->callSite); - if (hasFrameAtReturnOffset(function, parts, returnOffset)) - result.frame = section(ImageSection::Frames)[parts.frame]; + if (hasFrameAtReturnOffset(function, parts.frame, returnOffset)) + result.frame = section(ImageSection::Frames)[parts.frame.number]; if (record.has(ImageFunctionFlag::SpillsArgumentRegisters)) result.frame.calleeSavesStart |= ImageFrame::liesBelowSpilledArgumentsBit; return result; @@ -596,30 +802,37 @@ std::optional findHandler(std::span handlers, Call { const uint8_t* cursor = handlers.data(); const uint8_t* end = cursor + handlers.size(); - uint64_t endOfLast = 0; - int64_t target = 0; - while (cursor < end) { - HandlerRow row = HandlerRow::read(cursor); - uint64_t start = endOfLast + row.sitesSkipped; - endOfLast = start + row.furtherSites + 1; - target += row.targetChange; - if (site.bits() < start) + return findHandlerAmong([&]() -> std::optional { + if (cursor >= end) return std::nullopt; - if (site.bits() < endOfLast) - return HandlerRecord { static_cast(target), row.type }; - } - return std::nullopt; + return HandlerRow::read(cursor); + }, site); } static constexpr size_t maxZerosInFrontOfStart = 3; -static_assert(sitesBetweenPositionCheckpoints * (maxZerosInFrontOfStart + ConstructionStartRow::maxSize + PositionRow::maxSize) <= std::numeric_limits::max(), "a checkpoint stores the size of a run of positions in 16 bits"); +static constexpr size_t maxSizeOfPosition = maxZerosInFrontOfStart + ConstructionStartRow::maxSize + PositionRow::maxSize; +static_assert(sitesBetweenPositionCheckpoints * maxSizeOfPosition * PrefixCode::maxLength / 8 < std::numeric_limits::max(), "a checkpoint stores the size of a run of positions in 16 bits"); namespace { +enum class PreviousRow : uint8_t { None, OnOtherLine, OnSameLine }; + +enum RowCode : unsigned { + ColumnDeltaContinuation = 9, + LineDeltaContinuation, + SourceStart, + SourceContinuation, + ColumnStart, + OtherByte, + NumberOfRowCodes +}; +static_assert(NumberOfRowCodes == PositionIndex::numberOfCodes); + struct PositionState { int64_t line { 0 }; int64_t column { 0 }; uint32_t source { 0 }; + PreviousRow previousRow { PreviousRow::None }; LineColumn constructionStart { 0, 0 }; bool startIsForCallers { false }; bool startIsWhereWritten { false }; @@ -652,15 +865,94 @@ static uint32_t sourceNumber(const SitePosition& position) return position.source ? position.source.index() + 1 : 0; } -static void appendConstructionStart(WriterOfReservedBytes& stream, const SitePosition& position, const SiteConstructionStart& constructionStart) +static unsigned firstByteCode(bool followsInlineFrames, bool isSameFrame, PreviousRow previousRow) +{ + unsigned kindOfFrame = !followsInlineFrames ? 0 : isSameFrame ? 1 : 2; + return kindOfFrame * 3 + std::to_underlying(previousRow); +} + +template +static void forEachRemainingByte(uint8_t firstByte, unsigned codeOfFirst, const NextByte& nextByte) +{ + uint8_t byte = firstByte; + auto finishVarint = [&](unsigned code) { + while (byte & 0x80) + byte = nextByte(code); + }; + if (!byte) { + for (size_t zeros = 0; !byte && zeros < maxZerosInFrontOfStart; ++zeros) + byte = nextByte(OtherByte); + bool startsOnSameLine = byte & 1; + finishVarint(OtherByte); + if (!startsOnSameLine) { + byte = nextByte(OtherByte); + finishVarint(OtherByte); + } + byte = nextByte(codeOfFirst); + } + bool isOnSameLine = byte & 1; + bool isInOtherSource = !isOnSameLine && byte & 2; + finishVarint(isOnSameLine ? ColumnDeltaContinuation : LineDeltaContinuation); + if (isOnSameLine) + return; + if (isInOtherSource) { + byte = nextByte(SourceStart); + finishVarint(SourceContinuation); + } + byte = nextByte(ColumnStart); + finishVarint(OtherByte); +} + +namespace { + +class RowSymbols { +public: + explicit RowSymbols(PrefixSymbol* firstReservedSymbol) + : m_symbols(firstReservedSymbol) + { + } + + PrefixSymbol* end() const { return m_symbols.end(); } + void appendStartMarker(unsigned codeOfFirst, size_t number) + { + for (size_t zeros = 0; zeros < number; ++zeros) + m_symbols.append(zeros ? static_cast(OtherByte) : codeOfFirst, 0); + } + ALWAYS_INLINE void appendColumnChange(unsigned codeOfFirst, int64_t change) + { + m_symbols.appendVarint(codeOfFirst, ColumnDeltaContinuation, zigZagEncode(change) << 1 | 1); + } + ALWAYS_INLINE void appendLineChange(unsigned codeOfFirst, int64_t change, std::optional otherSource, uint32_t column) + { + m_symbols.appendVarint(codeOfFirst, LineDeltaContinuation, zigZagEncode(change) << 2 | static_cast(otherSource.has_value()) << 1); + if (otherSource) + m_symbols.appendVarint(SourceStart, SourceContinuation, *otherSource); + m_symbols.appendVarint(ColumnStart, OtherByte, column); + } + friend void appendVarint(RowSymbols& rows, uint64_t value) { rows.m_symbols.appendVarint(OtherByte, OtherByte, value); } + +private: + PrefixSymbolWriter m_symbols; +}; + +struct PositionInFront { + uint32_t line; + uint32_t column; + uint32_t source; + uint32_t numberOfRun; + PreviousRow previousRow; +}; + +} // anonymous namespace + +static void appendConstructionStart(RowSymbols& rows, unsigned codeOfFirst, const SitePosition& position, const SiteConstructionStart& constructionStart) { bool startsOnSameLine = constructionStart.line == position.line; bool startsInFront = startsOnSameLine ? constructionStart.column < position.column : constructionStart.line < position.line; if (constructionStart.isWhereWritten) { bool startsBehind = startsOnSameLine ? constructionStart.column > position.column : constructionStart.line > position.line; if (startsBehind) { - for (size_t zeros = 0; zeros < maxZerosInFrontOfStart; ++zeros) - stream.append(0); + rows.appendStartMarker(codeOfFirst, maxZerosInFrontOfStart); ConstructionStartRow start { .isOnSameLine = startsOnSameLine }; if (startsOnSameLine) start.columnsBack = constructionStart.column - position.column; @@ -668,12 +960,10 @@ static void appendConstructionStart(WriterOfReservedBytes& stream, const SitePos start.linesBack = constructionStart.line - position.line; start.column = constructionStart.column; } - start.append(stream); + start.append(rows); } } else if (constructionStart.line && (startsInFront || constructionStart.isForCallers)) { - stream.append(0); - if (constructionStart.isForCallers) - stream.append(0); + rows.appendStartMarker(codeOfFirst, constructionStart.isForCallers ? 2 : 1); startsOnSameLine |= !startsInFront; ConstructionStartRow start { .isOnSameLine = startsOnSameLine }; if (startsOnSameLine) @@ -682,26 +972,23 @@ static void appendConstructionStart(WriterOfReservedBytes& stream, const SitePos start.linesBack = position.line - constructionStart.line; start.column = constructionStart.column; } - start.append(stream); + start.append(rows); } } -static void appendPosition(WriterOfReservedBytes& stream, PositionState& last, const SitePosition& position) +static ALWAYS_INLINE void appendPosition(RowSymbols& rows, unsigned codeOfFirst, PositionInFront& last, const SitePosition& position) { uint32_t source = sourceNumber(position); - PositionRow row { .isOnSameLine = position.line == last.line && source == last.source }; - if (row.isOnSameLine) - row.columnChange = static_cast(position.column) - last.column; - else { - row.isInOtherSource = source != last.source; - row.lineChange = static_cast(position.line) - last.line; - row.source = row.isInOtherSource ? source : 0; - row.column = position.column; + if (position.line == last.line && source == last.source) { + rows.appendColumnChange(codeOfFirst, static_cast(position.column) - last.column); + last.previousRow = PreviousRow::OnSameLine; + } else { + rows.appendLineChange(codeOfFirst, static_cast(position.line) - last.line, source == last.source ? std::nullopt : std::optional(source), position.column); + last.line = position.line; + last.source = source; + last.previousRow = PreviousRow::OnOtherLine; } - row.append(stream); - last.line = position.line; last.column = position.column; - last.source = source; } OwnPosition OwnPosition::of(const SiteDescription& sites) @@ -709,13 +996,7 @@ OwnPosition OwnPosition::of(const SiteDescription& sites) return { sites.positions[0].line, sourceNumber(sites.positions[0]) }; } -void encodePositions(Vector& stream, const SiteDescription& sites, const uint8_t*, OwnPosition ofFunctionInFront) -{ - Vector roomForPositions; - encodePositions(stream, roomForPositions, sites, ofFunctionInFront); -} - -void encodePositions(Vector& stream, Vector& roomForPositions, const SiteDescription& sites, OwnPosition ofFunctionInFront) +void PositionEncoder::encode(const SiteDescription& sites, OwnPosition ofFunctionInFront) { RELEASE_ASSERT(!sites.positions.isEmpty()); const SitePosition* positionsOfSites = sites.positions.begin(); @@ -735,40 +1016,43 @@ void encodePositions(Vector& stream, Vector& roomForPositions, return site > numberOfSuspensionSites ? inlineFrames[site - numberOfSuspensionSites].parent : 0; }; - PositionState start { positionsOfSites[0].line, positionsOfSites[0].column, sourceNumber(positionsOfSites[0]) }; + PositionInFront start { positionsOfSites[0].line, positionsOfSites[0].column, sourceNumber(positionsOfSites[0]), 0, PreviousRow::None }; Vector checkpoints; - constexpr size_t maxSizeOfPosition = maxZerosInFrontOfStart + ConstructionStartRow::maxSize + PositionRow::maxSize; - roomForPositions.shrink(0); - roomForPositions.grow((count - 1) * maxSizeOfPosition); - uint8_t* firstPositionByte = roomForPositions.mutableSpan().data(); - WriterOfReservedBytes positions(firstPositionByte); - const uint8_t* lastRunEnd = firstPositionByte; - Vector states; - states.fill(start, followsInlineFrames ? numberOfInlineFrames + 1 : 1); - Vector checkpointsOfStates; - checkpointsOfStates.fill(0, states.size()); - uint32_t numberOfCheckpoints = 0; + m_roomForSymbols.shrink(0); + m_roomForSymbols.grow((count - 1) * maxSizeOfPosition); + PrefixSymbol* firstSymbol = m_roomForSymbols.mutableSpan().data(); + RowSymbols rows(firstSymbol); + const PrefixSymbol* lastRunEnd = firstSymbol; + Vector positionsInFront; + positionsInFront.fill(start, followsInlineFrames ? numberOfInlineFrames + 1 : 1); + uint32_t numberOfRun = 0; + unsigned frameInFront = 0; for (size_t site = 1; site < count; ++site) { if (!(site % sitesBetweenPositionCheckpoints)) { - size_t bytesSinceLast = positions.end() - lastRunEnd; - ASSERT(bytesSinceLast <= sitesBetweenPositionCheckpoints * maxSizeOfPosition); - checkpoints.append(static_cast(bytesSinceLast)); - checkpoints.append(static_cast(bytesSinceLast >> 8)); - lastRunEnd = positions.end(); - ++numberOfCheckpoints; + size_t symbolsSinceLast = rows.end() - lastRunEnd; + ASSERT(symbolsSinceLast <= sitesBetweenPositionCheckpoints * maxSizeOfPosition); + checkpoints.append(static_cast(symbolsSinceLast)); + checkpoints.append(static_cast(symbolsSinceLast >> 8)); + lastRunEnd = rows.end(); + start.numberOfRun = ++numberOfRun; + frameInFront = 0; } unsigned frame = frameOf(site); - if (std::exchange(checkpointsOfStates[frame], numberOfCheckpoints) != numberOfCheckpoints) - states[frame] = start; + PositionInFront& inFront = positionsInFront[frame]; + if (inFront.numberOfRun != numberOfRun) + inFront = start; + unsigned codeOfFirst = firstByteCode(followsInlineFrames, frame == std::exchange(frameInFront, frame), inFront.previousRow); if (!constructionStarts.empty() && constructionStarts.front().site == site) { - appendConstructionStart(positions, positionsOfSites[site], constructionStarts.front()); + appendConstructionStart(rows, codeOfFirst, positionsOfSites[site], constructionStarts.front()); constructionStarts = constructionStarts.subspan(1); } - appendPosition(positions, states[frame], positionsOfSites[site]); + appendPosition(rows, codeOfFirst, inFront, positionsOfSites[site]); } RELEASE_ASSERT(constructionStarts.empty()); - size_t sizeOfPositions = positions.end() - firstPositionByte; + std::span symbols(firstSymbol, rows.end()); + PositionCodes::count(m_counts, symbols); + m_symbols.append(symbols); bool hasCheckpoints = !checkpoints.isEmpty(); bool isInOtherSource = start.source != ofFunctionInFront.source; std::array roomForHead; @@ -776,16 +1060,55 @@ void encodePositions(Vector& stream, Vector& roomForPositions, PositionsHead { .followsInlineFrames = followsInlineFrames, .hasCheckpoints = hasCheckpoints, - .size = checkpoints.size() + sizeOfPositions, + .size = checkpoints.size() + symbols.size(), .isInOtherSource = isInOtherSource, - .lineChange = start.line - (isInOtherSource ? 0 : static_cast(ofFunctionInFront.line)), + .lineChange = static_cast(start.line) - (isInOtherSource ? 0 : ofFunctionInFront.line), .sourceChange = static_cast(start.source) - ofFunctionInFront.source, .column = positionsOfSites[0].column, .count = hasCheckpoints ? count : 0, }.append(head); - stream.append(std::span(roomForHead.data(), head.end())); - stream.appendVector(checkpoints); - stream.append(std::span(firstPositionByte, sizeOfPositions)); + m_heads.append(std::span(roomForHead.data(), head.end())); + m_heads.appendVector(checkpoints); +} + +void PositionEncoder::appendNextCoded(Vector& stream, const PositionCodes& codes) +{ + const uint8_t* begin = m_heads.span().data(); + const uint8_t* cursor = begin + m_nextHead; + PositionsHead head = PositionsHead::read(cursor); + size_t numberOfCheckpoints = head.hasCheckpoints ? (head.count - 1) / sitesBetweenPositionCheckpoints : 0; + const uint8_t* checkpoints = cursor; + cursor += numberOfCheckpoints * sizeof(uint16_t); + m_nextHead = cursor - begin; + std::span symbols = m_symbols.subspan(m_nextSymbol, head.size - numberOfCheckpoints * sizeof(uint16_t)); + m_nextSymbol += symbols.size(); + + m_roomForBytes.shrink(0); + m_roomForBytes.grow(symbols.size() * PrefixCode::maxLength / 8 + numberOfCheckpoints + 1); + uint8_t* firstCodedByte = m_roomForBytes.mutableSpan().data(); + uint8_t* lastRunEnd = firstCodedByte; + Vector codedCheckpoints; + for (size_t run = 0; run <= numberOfCheckpoints; ++run) { + bool isLast = run == numberOfCheckpoints; + size_t sizeOfRun = isLast ? symbols.size() : checkpoints[2 * run] | checkpoints[2 * run + 1] << 8; + uint8_t* endOfCodedRun = codes.write(lastRunEnd, symbols.first(sizeOfRun)); + symbols = symbols.subspan(sizeOfRun); + if (!isLast) { + size_t size = endOfCodedRun - lastRunEnd; + codedCheckpoints.append(static_cast(size)); + codedCheckpoints.append(static_cast(size >> 8)); + } + lastRunEnd = endOfCodedRun; + } + + std::span codedRuns(firstCodedByte, lastRunEnd); + head.size = codedCheckpoints.size() + codedRuns.size(); + std::array roomForHead; + WriterOfReservedBytes headBytes(roomForHead.data()); + head.append(headBytes); + stream.append(std::span(roomForHead.data(), headBytes.end())); + stream.appendVector(codedCheckpoints); + stream.append(codedRuns); } static PositionsEntry readEntry(const uint8_t*& cursor, OwnPosition& ofFunctionInFront) @@ -829,6 +1152,7 @@ static void readPosition(const uint8_t*& cursor, PositionState& state) state.source = row.source; state.column = row.column; } + state.previousRow = row.isOnSameLine ? PreviousRow::OnSameLine : PreviousRow::OnOtherLine; state.constructionStart = { 0, 0 }; if (!start) return; @@ -840,12 +1164,29 @@ static void readPosition(const uint8_t*& cursor, PositionState& state) state.constructionStart = { static_cast(state.line - start->linesBack), start->column }; } +static bool readCodedPosition(PrefixCodeReader& reader, const uint8_t* section, unsigned codeOfFirst, PositionState& state) +{ + unsigned firstByte = reader.read(PositionCodes::code(section, codeOfFirst)); + if (firstByte == PrefixCode::endOfStream) + return false; + std::array bytes; + size_t size = 0; + bytes[size++] = static_cast(firstByte); + forEachRemainingByte(bytes[0], codeOfFirst, [&](unsigned code) { + RELEASE_ASSERT(size < bytes.size()); + return bytes[size++] = static_cast(reader.read(PositionCodes::code(section, code))); + }); + const uint8_t* cursor = bytes.data(); + readPosition(cursor, state); + return true; +} + static SourcePosition sourcePositionFor(const PositionState& state) { return { { static_cast(state.line), static_cast(state.column) }, state.source ? ModuleID(state.source - 1) : ModuleID(), state.constructionStart, state.startIsForCallers, state.startIsWhereWritten }; } -std::optional decodePosition(const Image& image, uint32_t function, const uint8_t* entryInSection, OwnPosition ofFunctionInFront, CallSiteIndex site) +std::optional decodePosition(const Image& image, uint32_t function, const uint8_t* section, const uint8_t* entryInSection, OwnPosition ofFunctionInFront, CallSiteIndex site) { PositionsEntry entry = readEntry(entryInSection, ofFunctionInFront); const PositionState& start = entry.start; @@ -876,15 +1217,17 @@ std::optional decodePosition(const Image& image, uint32_t functi frames->skip(first - 1); } Vector states { start }; + PrefixCodeReader reader(cursor, entry.end); unsigned frame = 0; for (uint64_t i = first; i <= number; ++i) { - if (cursor == entry.end) - return std::nullopt; + unsigned frameInFront = frame; if (frames) frame = frames->next(); while (frame >= states.size()) states.append(start); - readPosition(cursor, states[frame]); + unsigned codeOfFirst = firstByteCode(entry.followsInlineFrames, frame == frameInFront, states[frame].previousRow); + if (!readCodedPosition(reader, section, codeOfFirst, states[frame])) + return std::nullopt; } return sourcePositionFor(states[frame]); } @@ -905,14 +1248,12 @@ static std::optional findPosition(const Image& image, uint32_t f const ImageFunction& record = image.function(function); if (!record.has(ImageFunctionFlag::HasPositions)) return std::nullopt; - const uint8_t* table = image.section(ImageSection::Positions).data(); - uint32_t placeOfFirstEntry; - memcpy(&placeOfFirstEntry, table + function / PositionIndex::functionsBetween * sizeof(uint32_t), sizeof(uint32_t)); - const uint8_t* cursor = table + PositionIndex::numberOfEntriesFor(image.section(ImageSection::Functions).size()) * sizeof(uint32_t) + placeOfFirstEntry; + const uint8_t* section = image.section(ImageSection::Positions).data(); + const uint8_t* cursor = PositionIndex::firstEntry(section, function / PositionIndex::functionsBetween, image.header().counts.functions); OwnPosition ofFunctionInFront; for (unsigned entriesInFront = function % PositionIndex::functionsBetween; entriesInFront; --entriesInFront) readEntry(cursor, ofFunctionInFront); - return decodePosition(image, function, cursor, ofFunctionInFront, site); + return decodePosition(image, function, section, cursor, ofFunctionInFront, site); } std::optional Image::positionOf(uint32_t function, CallSiteIndex site) const diff --git a/Source/JavaScriptCore/aot/image/AOTCallSiteTable.h b/Source/JavaScriptCore/aot/image/AOTCallSiteTable.h index 7e0baef048ced..b4e9792109627 100644 --- a/Source/JavaScriptCore/aot/image/AOTCallSiteTable.h +++ b/Source/JavaScriptCore/aot/image/AOTCallSiteTable.h @@ -11,13 +11,16 @@ #include "AOTCodeSites.h" #include "AOTImageForward.h" #include "AOTLimits.h" +#include "AOTPrefixCode.h" #include "AOTSchemaPackedRecords.h" +#include "AOTSideTableCodes.h" #include "AOTVector.h" #include "CallFrame.h" #include "HandlerInfo.h" #include "LineColumn.h" #include #include +#include namespace JSC { namespace AOT { @@ -38,6 +41,7 @@ struct CallSiteRecord { bool isConstruct() const { return isConstructOf(kind); } bool isTailCall() const { return isTailCallOf(kind); } bool isInsideCallee() const { return isInsideCalleeOf(kind); } + bool isAwaited() const { return isAwaitedOf(kind); } CallSiteKind kind { CallSiteKind::Plain }; BuiltinID calledBuiltin { }; @@ -55,7 +59,8 @@ struct InlineFrameRecord : InlineFrameRow { }; struct UndecodedInlineFrames { - const uint8_t* rows { nullptr }; + std::optional parents; + std::optional functions; uint64_t count { 0 }; }; @@ -104,15 +109,22 @@ class InlineFrameReader { }; class SiteFrames { + WTF_MAKE_NONCOPYABLE(SiteFrames); public: SiteFrames(const Image&, uint32_t function); - explicit SiteFrames(const uint8_t* callSites); unsigned next(); void skip(uint64_t numberOfSites); private: - const uint8_t* m_inlineFrames { nullptr }; - const uint8_t* m_sites { nullptr }; + void startBlock(const uint8_t* begin); + + std::optional m_inlineFrameParents; + std::optional m_sites; + std::optional m_details; + const uint8_t* m_checkpoints { nullptr }; + const uint8_t* m_firstBlock { nullptr }; + uint64_t m_nextBlock { 0 }; + unsigned m_siteInBlock { 0 }; uint64_t m_remainingSuspensionSites { 0 }; uint64_t m_remainingInlineFrames { 0 }; uint64_t m_remainingSitesWithCode { 0 }; @@ -120,7 +132,22 @@ class SiteFrames { }; inline FunctionID functionInFramesOf(BuiltinID builtin, size_t numberOfFunctions) { return FunctionID(numberOfFunctions + builtin.index()); } -Vector> encodeCallSites(const LinkedFunctions&, const FunctionNumbers&, const Typed::CallingConventions&, BuildContext&); + +struct CallSiteStreamHead { + struct Block { + uint32_t numberOfSymbols; + uint32_t offset; + uint32_t frame; + }; + + uint32_t numberOfHeadSymbols; + uint32_t numberOfParentSymbols; + uint32_t numberOfFunctionSymbols; + uint32_t numberOfBlocks; + uint32_t hasInlineFrames; +}; + +Vector> encodeCallSites(const LinkedFunctions&, const FunctionNumbers&, const Typed::CallingConventions&, BuildContext&); std::optional endOfLastReturnSite(const CompiledCode&); void* resumeAddressAfterStackCheck(const void* returnAddress); @@ -151,7 +178,7 @@ struct SourcePosition { bool startIsWhereWritten { false }; }; -static constexpr unsigned sitesBetweenPositionCheckpoints = 512; +static constexpr unsigned sitesBetweenPositionCheckpoints = 128; struct OwnPosition { static OwnPosition of(const SiteDescription&); @@ -160,9 +187,7 @@ struct OwnPosition { uint32_t source { 0 }; }; -void encodePositions(Vector& stream, const SiteDescription&, const uint8_t* callSites, OwnPosition ofFunctionInFront); -void encodePositions(Vector& stream, Vector& roomForPositions, const SiteDescription&, OwnPosition ofFunctionInFront); -std::optional decodePosition(const Image&, uint32_t function, const uint8_t* entry, OwnPosition ofFunctionInFront, CallSiteIndex); +std::optional decodePosition(const Image&, uint32_t function, const uint8_t* section, const uint8_t* entry, OwnPosition ofFunctionInFront, CallSiteIndex); void forgetRememberedPositions(); } } // namespace JSC::AOT diff --git a/Source/JavaScriptCore/aot/image/AOTCodeLayout.cpp b/Source/JavaScriptCore/aot/image/AOTCodeLayout.cpp index 4d866a30fad08..a921db5cc2bf4 100644 --- a/Source/JavaScriptCore/aot/image/AOTCodeLayout.cpp +++ b/Source/JavaScriptCore/aot/image/AOTCodeLayout.cpp @@ -245,10 +245,14 @@ static void layOutWhatFollowsFunctions(const CodeLayoutInputs& inputs, CodeLayou const RegExpSection& regExpSection = inputs.regExpSection; size_t webAssemblyCodeOffset = WTF::roundUpToMultipleOf(inputs.webAssemblyCodeAlignment, functionsEnd); size_t ffiCodeOffset = WTF::roundUpToMultipleOf(inputs.ffiCodeAlignment, webAssemblyCodeOffset + inputs.webAssemblyCodeSize); - size_t executableSize = WTF::roundUpToMultipleOf(ffiCodeOffset + inputs.ffiCodeSize + regExpSection.dataOffset); + size_t endOfFFICode = ffiCodeOffset + inputs.ffiCodeSize; + size_t executableSize = WTF::roundUpToMultipleOf(endOfFFICode + regExpSection.dataOffset); size_t regExpCodeOffset = executableSize - regExpSection.dataOffset; size_t constantsOffset = WTF::roundUpToMultipleOf(regExpCodeOffset + regExpSection.bytes.size()); - size_t ffiDirectoryOffset = constantsOffset + inputs.constantsSize; + size_t sizeOfAlignedConstants = WTF::roundUpToMultipleOf(inputs.constantsSize); + if (sizeOfAlignedConstants && !inputs.ffiCodeSize && regExpSection.bytes.isEmpty() && sizeOfAlignedConstants <= executableSize - endOfFFICode) + constantsOffset = executableSize - sizeOfAlignedConstants; + size_t ffiDirectoryOffset = std::max(executableSize, constantsOffset + inputs.constantsSize); if (inputs.ffiCodeSize) ffiDirectoryOffset = WTF::roundUpToMultipleOf(ffiDirectoryOffset); layout.codeSize = ffiDirectoryOffset + inputs.ffiDirectorySize; diff --git a/Source/JavaScriptCore/aot/image/AOTCodedTexts.cpp b/Source/JavaScriptCore/aot/image/AOTCodedTexts.cpp new file mode 100644 index 0000000000000..2ba9a46052c9c --- /dev/null +++ b/Source/JavaScriptCore/aot/image/AOTCodedTexts.cpp @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2026 Oven, Inc. All rights reserved. + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "config.h" +#include "AOTCodedTexts.h" + +#if ENABLE(AOT) + +#include "AOTPrefixCode.h" +#include "AOTThreads.h" +#include +#include +#include +#include +#include +#include +#include + +WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN + +namespace JSC { namespace AOT { + +namespace { + +constexpr std::array punctuation { 0, '.', '(', ')', '_', '$', ' ', '"', '\'', '[', ']' }; +constexpr unsigned uppercaseCode = 26; +constexpr unsigned digitCode = 27; +constexpr unsigned firstPunctuationCode = 28; +constexpr unsigned otherByteCode = firstPunctuationCode + punctuation.size(); +constexpr unsigned prefixLengthCode = otherByteCode + 1; +constexpr unsigned numberOfTextCodes = prefixLengthCode + 1; + +using TextCodes = PrefixCodes; + +constexpr auto codeByPreviousByte = [] { + std::array codes { }; + codes.fill(otherByteCode); + for (unsigned letter = 'a'; letter <= 'z'; ++letter) + codes[letter] = letter - 'a'; + for (unsigned letter = 'A'; letter <= 'Z'; ++letter) + codes[letter] = uppercaseCode; + for (unsigned digit = '0'; digit <= '9'; ++digit) + codes[digit] = digitCode; + for (size_t index = 0; index < punctuation.size(); ++index) + codes[punctuation[index]] = static_cast(firstPunctuationCode + index); + return codes; +}(); + +constexpr size_t sizeOfIndexFor(uint32_t count) { return (count + CodedTexts::textsInGroup - 1) / CodedTexts::textsInGroup * sizeof(uint32_t); } +constexpr size_t maxSharedPrefixLength = std::numeric_limits::max(); + +} // anonymous namespace + +bool CodedTexts::canHold(std::span text) +{ + return !text.empty() && std::ranges::all_of(text, [](uint8_t byte) { return byte && byte < 0x80; }); +} + +CodedTexts::CodedTexts(std::span bytes, uint32_t count) + : m_bytes(bytes) + , m_count(count) +{ +} + +Vector CodedTexts::at(uint32_t number) const +{ + RELEASE_ASSERT(number < m_count); + const uint8_t* codes = m_bytes.data(); + const uint8_t* index = TextCodes::endOfCodes(codes); + uint32_t place; + memcpy(&place, index + number / textsInGroup * sizeof(place), sizeof(place)); + PrefixCodeReader reader(index + sizeOfIndexFor(m_count) + place, codes + m_bytes.size()); + Vector text; + for (unsigned entry = 0;; ++entry) { + text.shrink(entry ? std::min(reader.read(TextCodes::code(codes, prefixLengthCode)), text.size()) : 0); + uint8_t previousByte = text.isEmpty() ? 0 : text.last(); + for (;;) { + unsigned byte = reader.read(TextCodes::code(codes, codeByPreviousByte[previousByte])); + if (!byte || byte == PrefixCode::endOfStream) + break; + previousByte = static_cast(byte); + text.append(previousByte); + } + if (entry == number % textsInGroup) + return text; + } +} + +Vector CodedTexts::encode(std::span> ascendingTexts, const ParallelLoops& loops) +{ + if (ascendingTexts.empty()) + return { }; + auto numberOfGroups = static_cast((ascendingTexts.size() + textsInGroup - 1) / textsInGroup); + Vector> symbolsOfGroups(numberOfGroups); + TextCodes::Counts counts { }; + Lock lockOfCounts; + forEachRangeOfIndicesInParallel(loops, numberOfGroups, [&](size_t firstGroup, size_t endOfGroups) { + TextCodes::Counts countsOfRange { }; + for (size_t group = firstGroup; group < endOfGroups; ++group) { + size_t first = group * textsInGroup; + std::span texts = ascendingTexts.subspan(first, std::min(textsInGroup, ascendingTexts.size() - first)); + size_t maxSize = 0; + for (std::span text : texts) + maxSize += text.size() + 2; + Vector& symbolsOfGroup = symbolsOfGroups[group]; + symbolsOfGroup.grow(maxSize); + PrefixSymbol* firstSymbol = symbolsOfGroup.mutableSpan().data(); + PrefixSymbolWriter symbols(firstSymbol); + std::span previousText; + for (std::span text : texts) { + RELEASE_ASSERT(canHold(text)); + size_t front = 0; + if (!previousText.empty()) { + size_t longest = std::min({ text.size(), previousText.size(), maxSharedPrefixLength }); + while (front < longest && text[front] == previousText[front]) + ++front; + symbols.append(prefixLengthCode, static_cast(front)); + } + uint8_t previousByte = front ? text[front - 1] : 0; + for (uint8_t byte : text.subspan(front)) + symbols.append(codeByPreviousByte[std::exchange(previousByte, byte)], byte); + symbols.append(codeByPreviousByte[previousByte], 0); + previousText = text; + } + symbolsOfGroup.shrink(symbols.end() - firstSymbol); + TextCodes::count(countsOfRange, symbolsOfGroup.span()); + } + Locker locker { lockOfCounts }; + TextCodes::add(counts, countsOfRange); + }); + + TextCodes codes(counts); + Vector> groups(numberOfGroups); + forEachIndexInParallel(loops, numberOfGroups, [&](unsigned group) { + std::span symbols = symbolsOfGroups[group].span(); + groups[group].grow(symbols.size() * PrefixCode::maxLength / 8 + 1); + uint8_t* firstByte = groups[group].mutableSpan().data(); + groups[group].shrink(codes.write(firstByte, symbols) - firstByte); + }); + + Vector bytes(codes.bytes()); + size_t index = bytes.size(); + bytes.grow(index + sizeOfIndexFor(static_cast(ascendingTexts.size()))); + size_t firstGroup = bytes.size(); + for (unsigned group = 0; group < numberOfGroups; ++group) { + RELEASE_ASSERT(bytes.size() - firstGroup <= std::numeric_limits::max()); + uint32_t place = static_cast(bytes.size() - firstGroup); + memcpy(bytes.mutableSpan().data() + index + group * sizeof(place), &place, sizeof(place)); + bytes.appendVector(groups[group]); + } + while (bytes.size() % sizeof(uint32_t)) + bytes.append(0); + return bytes; +} + +} } // namespace JSC::AOT + +WTF_ALLOW_UNSAFE_BUFFER_USAGE_END + +#endif // ENABLE(AOT) diff --git a/Source/JavaScriptCore/aot/image/AOTCodedTexts.h b/Source/JavaScriptCore/aot/image/AOTCodedTexts.h new file mode 100644 index 0000000000000..8ed451f9d0d53 --- /dev/null +++ b/Source/JavaScriptCore/aot/image/AOTCodedTexts.h @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2026 Oven, Inc. All rights reserved. + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#if ENABLE(AOT) + +#include "AOTVector.h" +#include +#include + +namespace JSC { namespace AOT { + +class ParallelLoops; + +class CodedTexts { +public: + static constexpr unsigned textsInGroup = 8; + + CodedTexts() = default; + JS_EXPORT_PRIVATE CodedTexts(std::span bytes, uint32_t count); + + uint32_t count() const { return m_count; } + JS_EXPORT_PRIVATE Vector at(uint32_t number) const; + + static bool canHold(std::span text); + JS_EXPORT_PRIVATE static Vector encode(std::span> ascendingTexts, const ParallelLoops&); + +private: + std::span m_bytes; + uint32_t m_count { 0 }; +}; + +} } // namespace JSC::AOT + +#endif // ENABLE(AOT) diff --git a/Source/JavaScriptCore/aot/image/AOTFaultTable.cpp b/Source/JavaScriptCore/aot/image/AOTFaultTable.cpp index a0ac874515542..283770dec6666 100644 --- a/Source/JavaScriptCore/aot/image/AOTFaultTable.cpp +++ b/Source/JavaScriptCore/aot/image/AOTFaultTable.cpp @@ -17,7 +17,9 @@ #include "AOTInstanceLayout.h" #include "AOTLinkedFunctions.h" #include "AOTOperationData.h" +#include "AOTPrefixCode.h" #include "AOTSectionWriter.h" +#include "AOTSort.h" #include "AOTState.h" #include "AOTStubDescriptor.h" #include "AOTStubKey.h" @@ -65,63 +67,37 @@ constexpr uint32_t faultSitesBetweenCheckpoints = 64; struct FaultTableHeader { uint32_t numberOfSites; - uint32_t numberOfNullChecks; uint32_t numberOfFirstWords; - uint32_t numberOfSecondWords; uint32_t defaultSecondWord; - uint32_t numberOfThirdWords; - uint32_t bytesOfSecondWord; - uint32_t reserved; + uint32_t sizeOfCodes; }; struct FaultCheckpoint { - bool hasSecondWord(uint32_t siteBehindCheckpoint) const { return sitesWithSecondWord[siteBehindCheckpoint / 32] >> (siteBehindCheckpoint % 32) & 1; } - uint32_t numberOfSecondWordsInFrontOf(uint32_t siteBehindCheckpoint) const - { - uint64_t sites = sitesWithSecondWord[0] | static_cast(sitesWithSecondWord[1]) << 32; - return static_cast(std::popcount(sites & ((uint64_t(1) << siteBehindCheckpoint) - 1))); - } - uint32_t previousSiteOffset; uint32_t stepsPosition; - uint32_t firstNullCheck; - uint32_t firstSecondWord; - uint32_t sitesWithSecondWord[2]; }; -static_assert(faultSitesBetweenCheckpoints == 64); -struct FaultSiteWord { - uint32_t nullCheck; - uint32_t word; -}; - -static_assert(sizeof(FaultTableHeader) == 32 && sizeof(FaultCheckpoint) == 24 && sizeof(FaultSiteWord) == 8); +static_assert(sizeof(FaultTableHeader) == 16 && sizeof(FaultCheckpoint) == 8); struct FaultTableLayout { - FaultTableLayout(uint64_t numberOfSites, uint64_t numberOfNullChecks, uint64_t numberOfFirstWords, uint64_t numberOfSecondWords, uint64_t bytesOfSecondWord, uint64_t numberOfThirdWords) + FaultTableLayout(uint64_t numberOfSites, uint64_t numberOfFirstWords, uint64_t sizeOfCodes) : numberOfCheckpoints((numberOfSites + faultSitesBetweenCheckpoints - 1) / faultSitesBetweenCheckpoints) - , hasNarrowWordNumbers(numberOfFirstWords <= std::numeric_limits::max() + 1u) , firstWordsOffset(sizeof(FaultTableHeader) + numberOfCheckpoints * sizeof(FaultCheckpoint)) - , thirdWordsOffset(firstWordsOffset + numberOfFirstWords * sizeof(uint32_t)) - , wordNumbersOffset(thirdWordsOffset + numberOfThirdWords * sizeof(FaultSiteWord)) - , secondWordsOffset(wordNumbersOffset + numberOfNullChecks * (hasNarrowWordNumbers ? sizeof(uint16_t) : sizeof(uint32_t))) - , stepsOffset(secondWordsOffset + numberOfSecondWords * bytesOfSecondWord) + , codesOffset(firstWordsOffset + numberOfFirstWords * sizeof(uint32_t)) + , stepsOffset(codesOffset + sizeOfCodes) { } explicit FaultTableLayout(const FaultTableHeader& header) - : FaultTableLayout(header.numberOfSites, header.numberOfNullChecks, header.numberOfFirstWords, header.numberOfSecondWords, header.bytesOfSecondWord, header.numberOfThirdWords) + : FaultTableLayout(header.numberOfSites, header.numberOfFirstWords, header.sizeOfCodes) { } uint64_t sizeWithSteps(uint64_t sizeOfSteps) const { return roundUpToMultipleOf(stepsOffset + sizeOfSteps); } uint64_t numberOfCheckpoints; - bool hasNarrowWordNumbers; uint64_t firstWordsOffset; - uint64_t thirdWordsOffset; - uint64_t wordNumbersOffset; - uint64_t secondWordsOffset; + uint64_t codesOffset; uint64_t stepsOffset; }; @@ -137,6 +113,92 @@ struct CollectedSite { std::array words; }; +enum SiteByte : unsigned { StepStart, StepContinuation, NumberStart, NumberContinuation, LaterWordsCode, ShortWordLow, ShortWordHigh, LaterWordsVarint, NumberOfSiteBytes }; +using FaultSiteCodes = PrefixCodes; + +class LaterWordsOfBlock { +public: + static constexpr size_t maxSize = 1 + 2 * sizeOfVarint(UINT32_MAX); + + static constexpr bool isShort(uint8_t code) { return code >= firstShort; } + static constexpr unsigned numberOfVarints(uint8_t code) { return code == codeOfTwoWords ? 2 : code == codeOfWide; } + + explicit LaterWordsOfBlock(uint32_t defaultWord) + : m_defaultWord(defaultWord) + { + } + + void append(PrefixSymbolWriter& symbols, std::span words) + { + uint32_t word = words[0]; + if (words.size() > 1) { + symbols.append(LaterWordsCode, codeOfTwoWords); + symbols.appendVarint(LaterWordsVarint, LaterWordsVarint, word); + symbols.appendVarint(LaterWordsVarint, LaterWordsVarint, words[1]); + return; + } + if (word == m_defaultWord) { + symbols.append(LaterWordsCode, codeOfDefault); + return; + } + for (uint8_t index = 0; index < m_size; ++index) { + if (m_words[index] == word) { + symbols.append(LaterWordsCode, static_cast(firstRepeated + index)); + return; + } + } + if (word < endOfShortWords) { + symbols.append(LaterWordsCode, static_cast(firstShort + (word >> 16))); + symbols.append(ShortWordLow, static_cast(word)); + symbols.append(ShortWordHigh, static_cast(word >> 8)); + } else { + symbols.append(LaterWordsCode, codeOfWide); + symbols.appendVarint(LaterWordsVarint, LaterWordsVarint, word); + } + note(word); + } + + std::array read(const uint8_t*& cursor) + { + uint8_t code = *cursor++; + if (code == codeOfDefault) + return { m_defaultWord, 0 }; + if (code == codeOfTwoWords) { + uint32_t first = static_cast(readVarint(cursor)); + return { first, static_cast(readVarint(cursor)) }; + } + if (code < codeOfWide) + return { m_words[code - firstRepeated], 0 }; + uint32_t word; + if (code == codeOfWide) + word = static_cast(readVarint(cursor)); + else { + word = static_cast(code - firstShort) << 16 | static_cast(cursor[1]) << 8 | cursor[0]; + cursor += 2; + } + note(word); + return { word, 0 }; + } + +private: + static constexpr uint8_t codeOfDefault = 0; + static constexpr uint8_t firstRepeated = 1; + static constexpr uint8_t codeOfWide = 62; + static constexpr uint8_t codeOfTwoWords = 63; + static constexpr uint8_t firstShort = 64; + static constexpr uint32_t endOfShortWords = (256u - firstShort) << 16; + + void note(uint32_t word) + { + if (m_size < m_words.size()) + m_words[m_size++] = word; + } + + uint32_t m_defaultWord; + uint8_t m_size { 0 }; + std::array m_words; +}; + void writeFaultSites(std::span sites, const CompiledCode& code, uint32_t bodyStart) { RELEASE_ASSERT(code.frame.hasFrame() || !code.frame.calleeSaveRegisters.registerCount()); @@ -191,6 +253,42 @@ uint32_t findMostFrequentWord(const ParallelLoops& loops, Vector&& wor return mostFrequentWord; } +constexpr size_t maxSizeOfSite = FaultStep::maxSize + sizeOfVarint(uint64_t(UINT32_MAX) << 1) + LaterWordsOfBlock::maxSize; + +struct StepSymbols { + friend void appendVarint(StepSymbols& step, uint64_t value) { step.symbols.appendVarint(StepStart, StepContinuation, value); } + + PrefixSymbolWriter& symbols; +}; + +template +void readSiteBytes(const NextByte& nextByte) +{ + uint8_t byte = nextByte(StepStart); + bool isLoopPoll = byte & 2; + while (byte & 0x80) + byte = nextByte(StepContinuation); + if (isLoopPoll) + return; + byte = nextByte(NumberStart); + bool hasLaterWords = byte & 1; + while (byte & 0x80) + byte = nextByte(NumberContinuation); + if (!hasLaterWords) + return; + uint8_t code = nextByte(LaterWordsCode); + if (LaterWordsOfBlock::isShort(code)) { + nextByte(ShortWordLow); + nextByte(ShortWordHigh); + return; + } + for (unsigned left = LaterWordsOfBlock::numberOfVarints(code); left; --left) { + do { + byte = nextByte(LaterWordsVarint); + } while (byte & 0x80); + } +} + Vector encodeFaultTable(std::span sites, const ParallelLoops& loops) { auto numberOfBlocks = static_cast((sites.size() + faultSitesBetweenCheckpoints - 1) / faultSitesBetweenCheckpoints); @@ -199,59 +297,19 @@ Vector encodeFaultTable(std::span sites, const Par return sites.subspan(first, std::min(faultSitesBetweenCheckpoints, sites.size() - first)); }; - Vector> steps(numberOfBlocks); - Vector checkpoints(numberOfBlocks); - forEachIndexInParallel(loops, numberOfBlocks, [&](unsigned block) { - uint32_t previousOffset = block ? sites[block * faultSitesBetweenCheckpoints - 1].offset : 0; - uint32_t offsetBeforeBlock = previousOffset; - uint32_t nullChecksInBlock = 0; - steps[block].grow(sitesOf(block).size() * FaultStep::maxSize); - uint8_t* firstStepByte = steps[block].mutableSpan().data(); - WriterOfReservedBytes stepBytes(firstStepByte); + Vector everyFirstWord = collectInParallel(loops, numberOfBlocks, [&](unsigned block, Vector& words) { for (const CollectedSite& site : sitesOf(block)) { - RELEASE_ASSERT(site.offset > previousOffset); - site.stepFrom(previousOffset).append(stepBytes); - previousOffset = site.offset; - nullChecksInBlock += site.isNullCheck(); - } - steps[block].shrink(stepBytes.end() - firstStepByte); - checkpoints[block] = { offsetBeforeBlock, 0, nullChecksInBlock, 0, { 0, 0 } }; - }); - size_t sizeOfSteps = 0; - uint32_t numberOfNullChecks = 0; - for (unsigned block = 0; block < numberOfBlocks; ++block) { - checkpoints[block].stepsPosition = static_cast(sizeOfSteps); - sizeOfSteps += steps[block].size(); - uint32_t nullChecksInBlock = std::exchange(checkpoints[block].firstNullCheck, numberOfNullChecks); - numberOfNullChecks += nullChecksInBlock; - } - - auto collectSparseWords = [&](unsigned index, std::optional omittedWord) { - return collectInParallel(loops, numberOfBlocks, [&](unsigned block, Vector& words) { - uint32_t nullCheck = checkpoints[block].firstNullCheck; - for (const CollectedSite& site : sitesOf(block)) { - if (index < site.numberOfWords && site.words[index] != omittedWord) - words.append(FaultSiteWord { nullCheck, site.words[index] }); - nullCheck += site.isNullCheck(); + if (site.isNullCheck()) { + RELEASE_ASSERT(site.numberOfWords); + words.append(site.words[0]); } - }); - }; - Vector firstWordsWithNullChecks = collectInParallel(loops, numberOfBlocks, [&](unsigned block, Vector& words) { - uint32_t nullCheck = checkpoints[block].firstNullCheck; - for (const CollectedSite& site : sitesOf(block)) { - if (!site.isNullCheck()) - continue; - RELEASE_ASSERT(site.numberOfWords); - words.append(static_cast(site.words[0]) << 32 | nullCheck); - ++nullCheck; } }); - sortByWordOnPoolKeepingEqualElementsInOrder(loops, firstWordsWithNullChecks.mutableSpan(), [](uint64_t wordWithNullCheck) { - return wordWithNullCheck >> 32; + sortByWordOnPoolKeepingEqualElementsInOrder(loops, everyFirstWord.mutableSpan(), [](uint32_t word) { + return word; }); Vector firstWords; - for (uint64_t wordWithNullCheck : firstWordsWithNullChecks) { - uint32_t word = static_cast(wordWithNullCheck >> 32); + for (uint32_t word : everyFirstWord) { if (firstWords.isEmpty() || firstWords.last() != word) firstWords.append(word); } @@ -261,81 +319,93 @@ Vector encodeFaultTable(std::span sites, const Par firstWords[i] = firstWords[i + 1]; } removeRepeatedElements(firstWords); + + Vector usesOfFirstWord(FillWith { }, firstWords.size(), 0u); + size_t indexOfFirstWord = 0; + for (uint32_t word : everyFirstWord) { + while (firstWords[indexOfFirstWord] < word) + ++indexOfFirstWord; + ++usesOfFirstWord[indexOfFirstWord]; + } + Vector mostUsedFirst(firstWords.size()); + for (size_t index = 0; index < firstWords.size(); ++index) + mostUsedFirst[index] = static_cast(index); + AOT::sort(mostUsedFirst, [&](uint32_t a, uint32_t b) { + return usesOfFirstWord[a] != usesOfFirstWord[b] ? usesOfFirstWord[a] > usesOfFirstWord[b] : a < b; + }); + Vector numberOfFirstWord(firstWords.size()); + for (size_t number = 0; number < mostUsedFirst.size(); ++number) + numberOfFirstWord[mostUsedFirst[number]] = static_cast(number); + uint32_t defaultSecondWord = findMostFrequentWord(loops, collectInParallel(loops, sites.size(), [&](unsigned site, Vector& words) { if (sites[site].numberOfWords > 1) words.append(sites[site].words[1]); })); - auto hasSecondWordOfItsOwn = [&](const CollectedSite& site) { - return site.numberOfWords > 1 && site.words[1] != defaultSecondWord; - }; - Vector secondWords = collectInParallel(loops, numberOfBlocks, [&](unsigned block, Vector& words) { - std::span sitesOfBlock = sitesOf(block); - for (size_t site = 0; site < sitesOfBlock.size(); ++site) { - if (!hasSecondWordOfItsOwn(sitesOfBlock[site])) + + Vector> symbolsOfBlocks(numberOfBlocks); + Vector checkpoints(numberOfBlocks); + forEachIndexInParallel(loops, numberOfBlocks, [&](unsigned block) { + uint32_t previousOffset = block ? sites[block * faultSitesBetweenCheckpoints - 1].offset : 0; + checkpoints[block] = { previousOffset, 0 }; + symbolsOfBlocks[block].grow(sitesOf(block).size() * maxSizeOfSite); + PrefixSymbol* firstSymbol = symbolsOfBlocks[block].mutableSpan().data(); + PrefixSymbolWriter symbols(firstSymbol); + StepSymbols stepSymbols { symbols }; + LaterWordsOfBlock laterWords(defaultSecondWord); + for (const CollectedSite& site : sitesOf(block)) { + RELEASE_ASSERT(site.offset > previousOffset); + site.stepFrom(previousOffset).append(stepSymbols); + previousOffset = site.offset; + if (!site.isNullCheck()) continue; - words.append(sitesOfBlock[site].words[1]); - checkpoints[block].sitesWithSecondWord[site / 32] |= 1u << (site % 32); + size_t index = std::ranges::lower_bound(firstWords, site.words[0]) - firstWords.begin(); + bool hasLaterWords = site.numberOfWords > 1; + symbols.appendVarint(NumberStart, NumberContinuation, static_cast(numberOfFirstWord[index]) << 1 | hasLaterWords); + if (hasLaterWords) + laterWords.append(symbols, std::span(site.words).subspan(1, site.numberOfWords - 1)); } + symbolsOfBlocks[block].shrink(symbols.end() - firstSymbol); + }); + + FaultSiteCodes::Counts counts { }; + for (const Vector& symbols : symbolsOfBlocks) + FaultSiteCodes::count(counts, symbols.span()); + FaultSiteCodes codes(counts); + Vector> steps(numberOfBlocks); + forEachIndexInParallel(loops, numberOfBlocks, [&](unsigned block) { + std::span symbols = symbolsOfBlocks[block].span(); + steps[block].grow(symbols.size() * PrefixCode::maxLength / 8 + 1); + uint8_t* firstByte = steps[block].mutableSpan().data(); + steps[block].shrink(codes.write(firstByte, symbols) - firstByte); }); - uint32_t numberOfSecondWordsSoFar = 0; - uint32_t largestSecondWord = 0; - for (FaultCheckpoint& checkpoint : checkpoints) { - checkpoint.firstSecondWord = numberOfSecondWordsSoFar; - numberOfSecondWordsSoFar += static_cast(std::popcount(checkpoint.sitesWithSecondWord[0]) + std::popcount(checkpoint.sitesWithSecondWord[1])); + + size_t sizeOfSteps = 0; + for (unsigned block = 0; block < numberOfBlocks; ++block) { + checkpoints[block].stepsPosition = static_cast(sizeOfSteps); + sizeOfSteps += steps[block].size(); } - RELEASE_ASSERT(numberOfSecondWordsSoFar == secondWords.size()); - for (uint32_t word : secondWords) - largestSecondWord = std::max(largestSecondWord, word); - uint32_t bytesOfSecondWord = (std::bit_width(largestSecondWord | 1u) + 7) / 8; - Vector thirdWords = collectSparseWords(2, std::nullopt); FaultTableHeader header { .numberOfSites = static_cast(sites.size()), - .numberOfNullChecks = numberOfNullChecks, .numberOfFirstWords = static_cast(firstWords.size()), - .numberOfSecondWords = static_cast(secondWords.size()), .defaultSecondWord = defaultSecondWord, - .numberOfThirdWords = static_cast(thirdWords.size()), - .bytesOfSecondWord = bytesOfSecondWord, - .reserved = 0, + .sizeOfCodes = static_cast(codes.bytes().size()), }; FaultTableLayout layout(header); Vector bytes; bytes.grow(layout.sizeWithSteps(sizeOfSteps)); std::span section = bytes.mutableSpan(); - forEachRangeOfIndicesInParallel(loops, section.size(), [&](size_t first, size_t end) { - zeroSpan(section.subspan(first, end - first)); - }); + zeroSpan(section.subspan(layout.stepsOffset + sizeOfSteps)); memcpySpan(section, asByteSpan(header)); memcpySpan(section.subspan(sizeof(FaultTableHeader)), asBytes(checkpoints.span())); - memcpySpan(section.subspan(layout.firstWordsOffset), asBytes(firstWords.span())); - memcpySpan(section.subspan(layout.thirdWordsOffset), asBytes(thirdWords.span())); - forEachIndexInParallel(loops, numberOfBlocks, [&](unsigned block) { - uint32_t end = block + 1 < numberOfBlocks ? checkpoints[block + 1].firstSecondWord : static_cast(secondWords.size()); - for (uint32_t number = checkpoints[block].firstSecondWord; number < end; ++number) { - uint8_t* bytesOfWord = section.data() + layout.secondWordsOffset + static_cast(number) * bytesOfSecondWord; - for (uint32_t byte = 0; byte < bytesOfSecondWord; ++byte) - bytesOfWord[byte] = static_cast(secondWords[number] >> (8 * byte)); - } - }); + auto* firstWordsByNumber = reinterpret_cast(section.data() + layout.firstWordsOffset); + for (size_t number = 0; number < mostUsedFirst.size(); ++number) + firstWordsByNumber[number] = firstWords[mostUsedFirst[number]]; + memcpySpan(section.subspan(layout.codesOffset), codes.bytes().span()); forEachIndexInParallel(loops, numberOfBlocks, [&](unsigned block) { memcpySpan(section.subspan(layout.stepsOffset + checkpoints[block].stepsPosition), steps[block].span()); }); - forEachRangeOfIndicesInParallel(loops, firstWordsWithNullChecks.size(), [&](size_t first, size_t end) { - uint8_t* wordNumbers = section.data() + layout.wordNumbersOffset; - size_t number = std::ranges::lower_bound(firstWords, static_cast(firstWordsWithNullChecks[first] >> 32)) - firstWords.begin(); - for (size_t index = first; index < end; ++index) { - uint32_t word = static_cast(firstWordsWithNullChecks[index] >> 32); - uint32_t nullCheck = static_cast(firstWordsWithNullChecks[index]); - while (number < firstWords.size() && firstWords[number] < word) - ++number; - if (layout.hasNarrowWordNumbers) - reinterpret_cast(wordNumbers)[nullCheck] = static_cast(number); - else - reinterpret_cast(wordNumbers)[nullCheck] = static_cast(number); - } - }); return bytes; } @@ -347,51 +417,31 @@ struct Fault { }; struct FaultTableView { - explicit FaultTableView(const uint8_t* bytes) - : header(*reinterpret_cast(bytes)) + explicit FaultTableView(std::span bytes) + : header(*reinterpret_cast(bytes.data())) , layout(header) - , checkpoints(reinterpret_cast(bytes + sizeof(FaultTableHeader)), layout.numberOfCheckpoints) - , firstWords(reinterpret_cast(bytes + layout.firstWordsOffset), header.numberOfFirstWords) - , secondWords(bytes + layout.secondWordsOffset) - , thirdWords(reinterpret_cast(bytes + layout.thirdWordsOffset), header.numberOfThirdWords) - , wordNumbers(bytes + layout.wordNumbersOffset) - , steps(bytes + layout.stepsOffset) + , checkpoints(reinterpret_cast(bytes.data() + sizeof(FaultTableHeader)), layout.numberOfCheckpoints) + , firstWords(reinterpret_cast(bytes.data() + layout.firstWordsOffset), header.numberOfFirstWords) + , codes(bytes.data() + layout.codesOffset) + , steps(bytes.subspan(layout.stepsOffset)) { } - static std::optional findWord(std::span words, uint32_t nullCheck) + std::array readThrowSite(const uint8_t*& cursor, LaterWordsOfBlock& laterWords) const { - auto entry = std::ranges::lower_bound(words, nullCheck, { }, &FaultSiteWord::nullCheck); - if (entry == words.end() || entry->nullCheck != nullCheck) - return std::nullopt; - return entry->word; - } - - uint32_t secondWordOf(const FaultCheckpoint& checkpoint, uint32_t siteBehindCheckpoint) const - { - if (!checkpoint.hasSecondWord(siteBehindCheckpoint)) - return header.defaultSecondWord; - const uint8_t* bytesOfWord = secondWords + static_cast(checkpoint.firstSecondWord + checkpoint.numberOfSecondWordsInFrontOf(siteBehindCheckpoint)) * header.bytesOfSecondWord; - uint32_t word = 0; - for (uint32_t byte = 0; byte < header.bytesOfSecondWord; ++byte) - word |= static_cast(bytesOfWord[byte]) << (8 * byte); - return word; - } - - std::array throwSiteOf(const FaultCheckpoint& checkpoint, uint32_t siteBehindCheckpoint, uint32_t nullCheck) const - { - uint32_t number = layout.hasNarrowWordNumbers ? reinterpret_cast(wordNumbers)[nullCheck] : reinterpret_cast(wordNumbers)[nullCheck]; - return { firstWords[number], secondWordOf(checkpoint, siteBehindCheckpoint), findWord(thirdWords, nullCheck).value_or(0) }; + uint64_t number = readVarint(cursor); + std::array later { header.defaultSecondWord, 0 }; + if (number & 1) + later = laterWords.read(cursor); + return { firstWords[number >> 1], later[0], later[1] }; } const FaultTableHeader& header; FaultTableLayout layout; std::span checkpoints; std::span firstWords; - const uint8_t* secondWords; - std::span thirdWords; - const uint8_t* wordNumbers; - const uint8_t* steps; + const uint8_t* codes; + std::span steps; }; } // anonymous namespace @@ -426,7 +476,7 @@ Vector encodeFaultSites(const LinkedFunctions& functions, std::span(maxSectionSize, numberOfSites * sizeof(CollectedSite)), context)) return { }; @@ -452,7 +502,7 @@ std::optional Image::faultSiteAt(const void* pc) const if (offset >= header().code.functionsEnd) return std::nullopt; - FaultTableView table(bytes.data()); + FaultTableView table(bytes); auto nextCheckpoint = std::ranges::lower_bound(table.checkpoints, static_cast(offset), { }, &FaultCheckpoint::previousSiteOffset); if (nextCheckpoint == table.checkpoints.begin()) return std::nullopt; @@ -460,22 +510,31 @@ std::optional Image::faultSiteAt(const void* pc) const uint32_t firstSite = static_cast(checkpoint - table.checkpoints.begin()) * faultSitesBetweenCheckpoints; uint32_t numberOfSteps = std::min(faultSitesBetweenCheckpoints, table.header.numberOfSites - firstSite); - const uint8_t* cursor = table.steps + checkpoint->stepsPosition; + PrefixCodeReader reader(table.steps.data() + checkpoint->stepsPosition, table.steps.data() + table.steps.size()); + std::array siteBytes; uint64_t currentOffset = checkpoint->previousSiteOffset; - uint32_t nullCheck = checkpoint->firstNullCheck; + LaterWordsOfBlock laterWords(table.header.defaultSecondWord); for (uint32_t i = 0; i < numberOfSteps; ++i) { + size_t size = 0; + readSiteBytes([&](unsigned code) { + RELEASE_ASSERT(size < siteBytes.size()); + return siteBytes[size++] = static_cast(reader.read(FaultSiteCodes::code(table.codes, code))); + }); + const uint8_t* cursor = siteBytes.data(); FaultStep step = FaultStep::read(cursor); currentOffset += step.distance; - if (currentOffset < offset) { - nullCheck += !step.isLoopPoll; - continue; - } if (currentOffset > offset) return std::nullopt; + bool isFound = currentOffset == offset; bool hasFrame = !step.isFrameless; - if (step.isLoopPoll) - return FaultSite { FaultSiteKind::LoopPoll, hasFrame, { } }; - return FaultSite { FaultSiteKind::NullCheck, hasFrame, table.throwSiteOf(*checkpoint, i, nullCheck) }; + if (step.isLoopPoll) { + if (isFound) + return FaultSite { FaultSiteKind::LoopPoll, hasFrame, { } }; + continue; + } + std::array throwSite = table.readThrowSite(cursor, laterWords); + if (isFound) + return FaultSite { FaultSiteKind::NullCheck, hasFrame, throwSite }; } return std::nullopt; } diff --git a/Source/JavaScriptCore/aot/image/AOTFieldTableEncoder.cpp b/Source/JavaScriptCore/aot/image/AOTFieldTableEncoder.cpp index 9299246ca5171..ce548140974b6 100644 --- a/Source/JavaScriptCore/aot/image/AOTFieldTableEncoder.cpp +++ b/Source/JavaScriptCore/aot/image/AOTFieldTableEncoder.cpp @@ -239,7 +239,7 @@ static TypeRecordForFieldTable describeTypeForFieldTable(const Typed::Type& type static LayoutID layoutOfRecordForFieldTable(const TableEncoder& encoder, uint32_t shape) { const Typed::ShapeTable& shapes = encoder.plan.shapes(); - if (!shapes.isAssigned(shape) || !encoder.plan.isInUse(shapes.shape(shape).layout)) + if (!shapes.isAssigned(shape) || !encoder.plan.isAllocatable(shapes.shape(shape).layout)) return { }; return shapes.shape(shape).layout; } diff --git a/Source/JavaScriptCore/aot/image/AOTFunctionProgramTables.cpp b/Source/JavaScriptCore/aot/image/AOTFunctionProgramTables.cpp index e46dd41d0d15c..2b507f7177e4b 100644 --- a/Source/JavaScriptCore/aot/image/AOTFunctionProgramTables.cpp +++ b/Source/JavaScriptCore/aot/image/AOTFunctionProgramTables.cpp @@ -12,6 +12,7 @@ #include "AOTIRProgram.h" #include "AOTPhaseTimes.h" #include "AOTPlan.h" +#include "AOTPrefixCode.h" #include "AOTRepresentation.h" #include "AOTSort.h" #include "AOTThreads.h" @@ -117,15 +118,74 @@ class FunctionTablesByRange { Vector m_records; Vector m_unsharedRecords; Vector m_groupsOfNames; - Vector m_sources; + Vector m_sources; Vector m_kinds; Vector m_lengths; Vector m_captureCounts; - Vector m_cellNumbers; + Vector m_cells; }; } // anonymous namespace +Vector SparseNumberTable::create(std::span cells) +{ + Vector words(FillWith { }, NumberGroup::numberOfGroupWords(cells.size()), 0); + for (size_t first = 0; first < cells.size(); first += NumberGroup::size) { + std::span cellsOfGroup = cells.subspan(first, std::min(NumberGroup::size, cells.size() - first)); + uint32_t bits = 0; + Vector list; + bool follows = true; + for (size_t entry = 0; entry < cellsOfGroup.size(); ++entry) { + uint32_t cell = cellsOfGroup[entry]; + if (!cell) + continue; + RELEASE_ASSERT(!(cell & NumberGroup::isListedFlag)); + bits |= 1u << entry; + follows &= list.isEmpty() || cell == list.last() + 1; + list.append(cell); + } + size_t group = first / NumberGroup::size * NumberGroup::numberOfWords; + words[group] = bits; + if (!bits) + continue; + if (follows) { + words[group + 1] = list[0]; + continue; + } + RELEASE_ASSERT(words.size() < NumberGroup::isListedFlag); + words[group + 1] = static_cast(words.size()) | NumberGroup::isListedFlag; + words.appendVector(list); + } + return words; +} + +Vector AscendingNumberTable::create(std::span functions) +{ + Vector words(FillWith { }, NumberGroup::numberOfGroupWords(functions.size()), 0); + size_t endOfBits = words.size() * 32; + for (size_t first = 0; first < functions.size(); first += NumberGroup::size) { + std::span functionsOfGroup = functions.subspan(first, std::min(NumberGroup::size, functions.size() - first)); + size_t group = first / NumberGroup::size * NumberGroup::numberOfWords; + words[group] = functionsOfGroup.front(); + RELEASE_ASSERT(endOfBits < NumberGroup::isListedFlag); + if (!std::ranges::is_sorted(functionsOfGroup) || functionsOfGroup.back() - functionsOfGroup.front() > maxSpan) { + words[group + 1] = static_cast(words.size()) | NumberGroup::isListedFlag; + words.append(functionsOfGroup); + endOfBits = words.size() * 32; + continue; + } + words[group + 1] = static_cast(endOfBits); + for (size_t entry = 1; entry < functionsOfGroup.size(); ++entry) { + endOfBits += functionsOfGroup[entry] - functionsOfGroup[entry - 1]; + while (words.size() * 32 <= endOfBits) + words.append(0); + words[endOfBits / 32] |= 1u << endOfBits % 32; + ++endOfBits; + } + } + return words; +} + static bool isShownByName(const TableInputs& inputs, FunctionID id) { return inputs.bodyFacts.functionsWithFrames.get(id.index()); @@ -275,7 +335,7 @@ FunctionTablesByRange::FunctionTablesByRange(const TableEncoder& encoder) m_kinds.grow(numberOfClosureFunctions); m_lengths.grow(numberOfClosureFunctions); m_captureCounts.grow(numberOfClosureFunctions); - m_cellNumbers.grow(numberOfClosureFunctions); + m_cells.grow(numberOfClosureFunctions); } static void uniteEqualFunctionRecords(Vector& records) @@ -362,7 +422,7 @@ void FunctionTablesByRange::collectClosureFunction(uint32_t index, CodeUnit unit m_captureCounts[number] = counts.encoded(); if ((m_lengths[number] == countsAreListed || m_captureCounts[number] == countsAreListed) && !AOT::hasFlag(m_encoder.build.options().injectedFaults, Wire::FaultInjection::DropLargeCountEntries)) functionsWithLargeCounts.append({ Number24(index), counts, { static_cast(length), static_cast(length >> 8) } }); - m_cellNumbers[number] = Number24(unit.role == Typed::CodeRole::GenericCallEntry ? m_encoder.numbers.closureCellNumber(unit.function) : 0); + m_cells[number] = unit.role == Typed::CodeRole::GenericCallEntry ? m_encoder.numbers.closureCellNumber(unit.function) : 0; } void FunctionTablesByRange::collectNamesOfCaptures(uint32_t index, CodeUnit unit, Vector& named) const @@ -430,7 +490,7 @@ void FunctionTablesByRange::collectFunction(uint32_t index, RowsOfFunctionRange& collectName(index, builtin, rows.names); return; } - m_sources[index - m_firstDerived] = Number24(unit.function.index()); + m_sources[index - m_firstDerived] = unit.function.index(); if (index < m_firstClosureFunction || index >= m_endOfClosureFunctions) return; collectClosureFunction(index, unit, record.kind, rows.functionsWithLargeCounts); @@ -498,14 +558,16 @@ void FunctionTablesByRange::forEachTable(const auto& functor) const functor(ProgramTableIndex::functionLengths, m_lengths); functor(ProgramTableIndex::closureCaptureCounts, m_captureCounts); functor(ProgramTableIndex::functionsWithLargeCounts, m_functionsWithLargeCounts.rows); - functor(ProgramTableIndex::closureCellNumbers, m_cellNumbers); + functor(ProgramTableIndex::closureCells, m_cells); functor(ProgramTableIndex::namedCaptures, m_bytesOfNamedCaptures); functor(ProgramTableIndex::constructEntrypoints, m_constructEntrypoints.rows); } void FunctionTablesByRange::appendTo(TablePart& part) { - m_bytesOfNamedCaptures = encodeNamedCaptures(m_namedCaptures.rows.span()); + m_bytesOfNamedCaptures = encodeVarintBytes(encodeNamedCaptures(m_namedCaptures.rows.span()).span()); + m_sources = AscendingNumberTable::create(m_sources.span()); + m_cells = SparseNumberTable::create(m_cells.span()); uint64_t room = 0; forEachTable([&](ProgramTableIndex, const auto& rows) { room += rows.sizeInBytes() + sizeof(uint64_t); diff --git a/Source/JavaScriptCore/aot/image/AOTFunctionSources.cpp b/Source/JavaScriptCore/aot/image/AOTFunctionSources.cpp index 21f95e1067c5d..2eec4176a9ce9 100644 --- a/Source/JavaScriptCore/aot/image/AOTFunctionSources.cpp +++ b/Source/JavaScriptCore/aot/image/AOTFunctionSources.cpp @@ -50,9 +50,16 @@ struct ModuleRetainedRanges { uint32_t end; }; -struct HashedSourceRun { - uint64_t hashAndSize; - uint32_t run; +class HashedSourceRun { +public: + static uint64_t create(std::span bytes, uint32_t run) + { + constexpr unsigned bitsOfSize = 8; + uint64_t hash = StringHasher::computeHashAndMaskTop8Bits(byteCast(bytes)); + return (hash << bitsOfSize | (bytes.size() & ((1u << bitsOfSize) - 1))) << 32 | run; + } + static uint32_t group(uint64_t hashedRun) { return static_cast(hashedRun >> 32); } + static uint32_t run(uint64_t hashedRun) { return static_cast(hashedRun); } }; struct SourceRunLayout { @@ -70,24 +77,29 @@ static SourceRunLayout computeSourceRunLayout(const ParallelLoops& loops, std::s SourceRunLayout runLayout; runLayout.offsets.grow(runs.size()); runLayout.firstEqualRun.grow(runs.size()); - Vector hashed; + Vector hashed; hashed.grow(runs.size()); forEachIndexInParallel(loops, static_cast(runs.size()), [&](unsigned run) { - std::span bytes = runs[run]; - hashed[run] = { static_cast(StringHasher::computeHashAndMaskTop8Bits(byteCast(bytes))) << 32 | bytes.size(), run }; + hashed[run] = HashedSourceRun::create(runs[run], run); runLayout.firstEqualRun[run] = run; }); - parallelSort(loops, hashed.mutableSpan(), [](const HashedSourceRun& a, const HashedSourceRun& b) { - return a.hashAndSize != b.hashAndSize ? a.hashAndSize < b.hashAndSize : a.run < b.run; - }); + parallelSort(loops, hashed.mutableSpan(), std::ranges::less { }); forEachRangeOfIndicesInParallel(loops, hashed.size(), [&](size_t at, size_t end) { - while (at && at < end && hashed[at].hashAndSize == hashed[at - 1].hashAndSize) + while (at && at < end && HashedSourceRun::group(hashed[at]) == HashedSourceRun::group(hashed[at - 1])) ++at; + Vector differentRuns; while (at < end) { - HashedSourceRun first = hashed[at]; - for (++at; at < hashed.size() && hashed[at].hashAndSize == first.hashAndSize; ++at) { - if (equalSpans(runs[first.run], runs[hashed[at].run])) - runLayout.firstEqualRun[hashed[at].run] = first.run; + uint32_t group = HashedSourceRun::group(hashed[at]); + differentRuns.shrink(0); + for (; at < hashed.size() && HashedSourceRun::group(hashed[at]) == group; ++at) { + uint32_t run = HashedSourceRun::run(hashed[at]); + size_t equal = differentRuns.findIf([&](uint32_t earlier) { + return equalSpans(runs[earlier], runs[run]); + }); + if (equal != notFound) + runLayout.firstEqualRun[run] = differentRuns[equal]; + else + differentRuns.append(run); } } }); @@ -324,6 +336,32 @@ static std::optional findQuotedSourceLocations(const Tabl return result; } +static size_t findFirstLocation(std::span locations, uint64_t location, size_t nearby) +{ + size_t first = 0; + size_t end = locations.size(); + if (nearby < end && locations[nearby] < location) { + first = nearby + 1; + for (size_t step = 1; end - first > step; step *= 2) { + if (locations[first + step] >= location) { + end = first + step; + break; + } + first += step + 1; + } + } else if (nearby < end) { + end = nearby; + for (size_t step = 1; end > step; step *= 2) { + if (locations[end - step] < location) { + first = end - step + 1; + break; + } + end -= step; + } + } + return std::ranges::lower_bound(locations.subspan(first, end - first), location) - locations.begin(); +} + static bool markReportableQuotedSources(const TableEncoder& encoder, QuotedSourceLocations& sources) { const LinkedFunctions& functions = encoder.inputs.functions; @@ -332,13 +370,15 @@ static bool markReportableQuotedSources(const TableEncoder& encoder, QuotedSourc forEachIndexInParallel(encoder.build.parallelLoops(), static_cast(functions.size()), [&](unsigned position) { size_t lastModule = numberOfModules; uint64_t lastLocation = 0; + size_t lastPlace = 0; auto report = [&](size_t module, uint64_t location) { if (module >= numberOfModules || (module == lastModule && location == lastLocation)) return; + std::span locations = sources.locationsOf(module); + lastPlace = findFirstLocation(locations, location, module == lastModule ? lastPlace : locations.size()); lastModule = module; lastLocation = location; - std::span locations = sources.locationsOf(module); - for (size_t at = std::ranges::lower_bound(locations, location) - locations.begin(); at < locations.size() && locations[at] == location; ++at) { + for (size_t at = lastPlace; at < locations.size() && locations[at] == location; ++at) { AtomicRef isReportable(sources.isReportable[sources.firstOfModule[module] + at]); if (!isReportable.load(std::memory_order_relaxed)) isReportable.store(1, std::memory_order_relaxed); diff --git a/Source/JavaScriptCore/aot/image/AOTFunctionTable.cpp b/Source/JavaScriptCore/aot/image/AOTFunctionTable.cpp index 5bee4950113b2..8a937e3a60b19 100644 --- a/Source/JavaScriptCore/aot/image/AOTFunctionTable.cpp +++ b/Source/JavaScriptCore/aot/image/AOTFunctionTable.cpp @@ -131,31 +131,94 @@ std::optional collectFrames(const LinkedFunctions& functions, BuildC return result; } -static Vector encodeFunctions(const FunctionTableInputs& inputs, const ParallelLoops& loops) +Vector ImageFunctions::encode(std::span sideTables, std::span numbers) { - Vector records(inputs.functions.size()); - forEachRangeOfIndicesInParallel(loops, inputs.functions.size(), [&](size_t first, size_t end) { - for (size_t position = first; position < end; ++position) { - const CompiledCode& code = inputs.functions[position]; - const SiteDescription& sites = code.sites; - OptionSet flags = inputs.sideTableFlags[position]; - flags.set(ImageFunctionFlag::HasInlineFrames, sites.hasEncodedInlineFrames()); - flags.set(ImageFunctionFlag::HasPositions, inputs.hasPositions && !sites.positions.isEmpty()); - flags.set(ImageFunctionFlag::SpillsArgumentRegisters, code.frame.spillsArgumentRegisters); - records[position] = ImageFunction::create(inputs.sideTableOffsets[position], code.function.index(), flags); + RELEASE_ASSERT(sideTables.size() == numbers.size()); + Vector groups; + Vector listedNumbers; + Vector steps; + Vector listedPlaces; + UncheckedKeyHashMap, WTF::UnsignedWithZeroKeyHashTraits> entryOfPlace; + uint32_t firstPlaceBehind = 0; + for (size_t index = 0; index < numbers.size(); ++index) { + size_t entry = index % functionsInGroup; + if (!entry) { + groups.append(0); + groups.append(static_cast(listedNumbers.size())); + groups.append(firstPlaceBehind); } + if (entry && numbers[index] == numbers[index - 1] + 1) + groups[groups.size() - wordsOfGroup] |= 1u << entry; + else + listedNumbers.append(Number24(numbers[index])); + + uint32_t place = sideTables[index]; + firstPlaceBehind = std::max(firstPlaceBehind, place + 1); + if (uint32_t placeOfGroup = groups.last(); place >= placeOfGroup && place - placeOfGroup < isListedPlace) + steps.append(static_cast(place - placeOfGroup)); + else { + uint32_t listed = entryOfPlace.ensure(place, [&] { + listedPlaces.append(place); + return static_cast(listedPlaces.size() - 1); + }).iterator->value; + steps.append(static_cast(isListedPlace | listed)); + } + } + + bool needsWholePlaces = listedPlaces.size() > isListedPlace; + uint32_t firstWord = needsWholePlaces ? hasWholePlaces : static_cast(listedPlaces.size()); + size_t count = numbers.size(); + Vector words(FillWith { }, firstListWord(firstWord, count) + numberOfListedPlaces(firstWord) + wordsOfNumbers(listedNumbers.size()), 0); + std::span span = words.mutableSpan(); + span[0] = firstWord; + span[1] = static_cast(listedNumbers.size()); + span[2] = static_cast(count); + if (needsWholePlaces) + memcpySpan(span.subspan(wordsOfHead), sideTables); + else { + memcpySpan(asWritableBytes(span.subspan(wordsOfHead)), asBytes(steps.span())); + memcpySpan(span.subspan(firstListWord(firstWord, count)), listedPlaces.span()); + } + memcpySpan(span.subspan(firstGroupWord(firstWord, count)), groups.span()); + memcpySpan(asWritableBytes(span.subspan(firstListWord(firstWord, count) + numberOfListedPlaces(firstWord))), asBytes(listedNumbers.span())); + return bytesOf(words); +} + +static Vector encodeFunctions(const FunctionTableInputs& inputs) +{ + Vector numbers(inputs.functions.size(), [&](size_t position) { + return inputs.functions[position].function.index(); }); - return records; + return ImageFunctions::encode(inputs.sideTableOffsets, numbers.span()); } -static Vector encodeFunctionStarts(const Vector& starts) +Vector FunctionStarts::encode(std::span starts) { constexpr uint32_t endOfStarts = std::numeric_limits::max(); - Vector bytes; - bytes.reserveInitialCapacity((starts.size() + 1) * sizeof(uint32_t)); - bytes.append(asBytes(starts.span())); - bytes.append(asByteSpan(endOfStarts)); - return bytes; + size_t numberOfEntries = starts.size() + 1; + auto startAt = [&](size_t index) { + return index < starts.size() ? starts[index] : endOfStarts; + }; + size_t firstGroup = numberOfWordsOfSteps(starts.size()); + size_t numberOfGroups = numberOfGroupsFor(starts.size()); + Vector words; + words.fill(0, firstGroup + numberOfGroups); + for (size_t group = 0; group < numberOfGroups; ++group) { + size_t first = group * functionsInGroup; + size_t last = std::min(first + functionsInGroup, numberOfEntries) - 1; + uint32_t start = startAt(first); + if (start < wideGroup && startAt(last) - start <= std::numeric_limits::max()) { + words[firstGroup + group] = start; + auto* steps = reinterpret_cast(words.mutableSpan().data()); + for (size_t index = first; index <= last; ++index) + steps[index] = static_cast(startAt(index) - start); + continue; + } + words[firstGroup + group] = wideGroup | safeCast(words.size() - firstGroup); + for (size_t index = first; index < first + functionsInGroup; ++index) + words.append(startAt(index)); + } + return Vector(asBytes(words.span())); } static Vector encodeCodeGranules(const CodeLayout& layout) @@ -271,16 +334,16 @@ EncodedFunctionTables encodeFunctionTables(const FunctionTableInputs& inputs, Bu const CodeLayout& layout = inputs.layout; const ParallelLoops& loops = context.parallelLoops(); RELEASE_ASSERT(layout.bodyStarts.size() == functions.size()); - RELEASE_ASSERT(inputs.sideTableFlags.size() == functions.size() && inputs.sideTableOffsets.size() == functions.size()); + RELEASE_ASSERT(inputs.sideTableOffsets.size() == functions.size()); if (!checkFunctionLimits(inputs, context) || !checkClosureFunctionRange(inputs, context)) return { }; EncodedFunctionTables tables; - tables.functions = bytesOf(encodeFunctions(inputs, loops)); + tables.functions = encodeFunctions(inputs); tables.closureWords = bytesOf(encodeClosureWords(inputs, context)); runTasksInParallel(loops, { - [&] { tables.functionStarts = encodeFunctionStarts(layout.bodyStarts); }, + [&] { tables.functionStarts = FunctionStarts::encode(layout.bodyStarts.span()); }, [&] { tables.codeGranules = bytesOf(encodeCodeGranules(layout)); }, [&] { tables.methodTable = bytesOf(encodeMethodTable(inputs, context)); }, }); @@ -293,7 +356,7 @@ std::optional Image::imageFunctionFromClosureWord(FunctionID id) const if (!word) return std::nullopt; CodeLocation entry = locate(std::bit_cast(ClosureWord::addressOf(word, std::bit_cast(code().data())))); - if (entry.kind != CodeKind::Function || function(entry.function).function() != id.index()) + if (entry.kind != CodeKind::Function || rootFunctionOf(entry.function) != id) return std::nullopt; return entry.function; } @@ -308,18 +371,18 @@ std::optional Image::imageFunctionOf(FunctionID id) const return imageFunctionFromClosureWord(id); std::call_once(m_didSortFunctionsByID, [&] { for (uint32_t function = 0; function < header().counts.functions; ++function) { - if (!hasClosureWord(this->function(function).function())) + if (!hasClosureWord(rootFunctionOf(function).index())) m_functionsByID.append(function); } AOT::sort(m_functionsByID, { }, [&](uint32_t function) { - return this->function(function).function(); + return rootFunctionOf(function).index(); }); }); std::span functionsByID = m_functionsByID.span(); auto found = std::ranges::lower_bound(functionsByID, id.index(), { }, [&](uint32_t function) -> uint32_t { - return this->function(function).function(); + return rootFunctionOf(function).index(); }); - if (found == functionsByID.end() || function(*found).function() != id.index()) + if (found == functionsByID.end() || rootFunctionOf(*found) != id) return std::nullopt; return *found; } @@ -329,7 +392,7 @@ std::span Image::frameMapWords(uint32_t function) const const ImageFunction& record = this->function(function); if (!record.has(ImageFunctionFlag::HasFrameMaps)) return { }; - SideTableParts parts = SideTableParts::read(record, section(ImageSection::SideTables).data()); + SideTableParts parts = SideTableParts::read(record, section(ImageSection::SideTables)); return section(ImageSection::FrameMaps).subspan(parts.firstFrameMapWord, parts.numberOfFrameMapWords); } diff --git a/Source/JavaScriptCore/aot/image/AOTFunctionTable.h b/Source/JavaScriptCore/aot/image/AOTFunctionTable.h index c781ea934ce89..4a44edf92c19c 100644 --- a/Source/JavaScriptCore/aot/image/AOTFunctionTable.h +++ b/Source/JavaScriptCore/aot/image/AOTFunctionTable.h @@ -41,7 +41,6 @@ struct FunctionTableInputs { const CodeLayout& layout; const SharedCode& sharedCode; const CodeIndex& index; - std::span> sideTableFlags; std::span sideTableOffsets; const ProgramTables& programTables; bool hasPositions; diff --git a/Source/JavaScriptCore/aot/image/AOTIdentifierNumbering.h b/Source/JavaScriptCore/aot/image/AOTIdentifierNumbering.h index 213052117bc92..8b96d44051700 100644 --- a/Source/JavaScriptCore/aot/image/AOTIdentifierNumbering.h +++ b/Source/JavaScriptCore/aot/image/AOTIdentifierNumbering.h @@ -32,6 +32,7 @@ struct NameCandidate { struct NamesAndStringsToNumber { Vector names; BitVector numberedConstants; + Vector codedStrings; Vector firstFunctionsNamingTexts; }; diff --git a/Source/JavaScriptCore/aot/image/AOTImage.cpp b/Source/JavaScriptCore/aot/image/AOTImage.cpp index 9cd837d6a58e1..b11581da837c3 100644 --- a/Source/JavaScriptCore/aot/image/AOTImage.cpp +++ b/Source/JavaScriptCore/aot/image/AOTImage.cpp @@ -165,8 +165,10 @@ static bool hasWellFormedTables(const ImageHeader& header, std::span(bytes.data() + rangeOf(ImageSection::Functions).offset), static_cast(numberOfElements(ImageSection::Functions)) }; + if (rangeOf(ImageSection::SideTables).size < (numberOfFunctions ? SideTableCodes::sizeOfPlaces : 0) + || !ImageFunctions::hasSize(functions, numberOfFunctions) + || numberOfElements(ImageSection::FunctionStarts) < FunctionStarts::numberOfWordsWithoutLists(numberOfFunctions)) return false; if (code.functionsEnd > header.codeSize || code.codeGranuleShift >= 32) return false; @@ -182,12 +184,15 @@ static bool hasWellFormedTables(const ImageHeader& header, std::span(copy) + code.sharedCodeSize; } + uint64_t endOfConstants = endOfCodePart(code.constantsOffset, code.constantsSize); + bool hasValidConstants = header.executableSize <= code.constantsOffset + || (!code.ffiCodeSize && code.ffiCodeOffset <= code.constantsOffset && endOfConstants <= header.executableSize && code.regExpCodeOffset == header.executableSize); bool hasValidCodeParts = code.functionsEnd <= code.webAssemblyCodeOffset && endOfCodePart(code.webAssemblyCodeOffset, code.webAssemblyCodeSize) <= code.ffiCodeOffset && endOfCodePart(code.ffiCodeOffset, code.ffiCodeSize) <= code.regExpCodeOffset && code.regExpCodeOffset <= header.executableSize - && header.executableSize <= code.constantsOffset - && endOfCodePart(code.constantsOffset, code.constantsSize) <= header.codeSize + && hasValidConstants + && endOfConstants <= header.codeSize && (!code.ffiCodeSize || !(code.ffiCodeOffset % imagePageSize)); if (!hasValidCodeParts || rangeOf(ImageSection::ProgramTables).size < sizeof(ProgramTables)) return false; @@ -211,7 +216,8 @@ static bool hasWellFormedTables(const ImageHeader& header, std::span= header.numbers.numberOfStringWords && numberOfElements(ImageSection::StringHeaders) >= header.numbers.endOfStringHeaders; } @@ -284,8 +290,8 @@ std::optional> Image::flags(std::span bytes) Image::Image(std::span bytes, const uint8_t* code) : m_bytes(bytes) , m_code(code) - , m_functions(section(ImageSection::Functions).data()) - , m_functionStarts(section(ImageSection::FunctionStarts).data()) + , m_functions(section(ImageSection::Functions).data(), header().counts.functions) + , m_functionStarts(section(ImageSection::FunctionStarts).data(), header().counts.functions) , m_codeGranules(section(ImageSection::CodeGranules).data()) , m_text(*this) { @@ -320,6 +326,8 @@ void Image::publish(std::unique_ptr&& unpublishedImage) std::span Image::ffiDirectory() const { const ImageCodeRanges& ranges = header().code; + if (!ranges.ffiCodeSize) + return { }; uint64_t offset = WTF::roundUpToMultipleOf(endOfCodePart(ranges.constantsOffset, ranges.constantsSize)); if (offset > header().codeSize) return { }; @@ -362,7 +370,7 @@ CodeLocation Image::locate(const void* pc) const return { CodeKind::Stub, isReturnSite ? site->kind : StubFrameKind::Plain, 0, inSharedCode }; } } - const uint32_t* starts = m_functionStarts; + const FunctionStarts& starts = m_functionStarts; uint32_t index = m_codeGranules[offset >> header.code.codeGranuleShift]; if (offset < starts[index]) return { }; @@ -404,17 +412,60 @@ bool Image::usesCallConvention(uint32_t offsetInSharedCode) const }); } +SideTableReader Image::sideTable(uint32_t function) const +{ + RELEASE_ASSERT(function < header().counts.functions); + std::span sideTables = section(ImageSection::SideTables); + return SideTableReader(sideTables, sideTables.data() + m_functions.sideTableOf(function)); +} + +ImageFunction Image::function(uint32_t function) const +{ + return { m_functions.sideTableOf(function), sideTable(function).next(SideTableByte::Flags) }; +} + +const ImageFrame& Image::frameOf(uint32_t function) const +{ + static constexpr ImageFrame noFrame { 0, 0, 0 }; + SideTableReader reader = sideTable(function); + reader.next(SideTableByte::Flags); + SideTableFrame frame = SideTableFrame::read(reader, 0); + return frame.hasFrame ? section(ImageSection::Frames)[frame.number] : noFrame; +} + bool Image::hasFrameAt(uint32_t function, uint32_t offset) const { - return SideTableParts::read(this->function(function), section(ImageSection::SideTables).data()).hasFrameAt(offset); + SideTableReader reader = sideTable(function); + reader.next(SideTableByte::Flags); + return SideTableFrame::read(reader, offset).coversOffset; +} + +static uint32_t offsetBeforeReturn(uint32_t returnOffset) +{ + return returnOffset ? returnOffset - 1 : 0; } bool Image::hasFrameAtReturnOffset(uint32_t function, uint32_t returnOffset) const { - return hasFrameAtReturnOffset(function, SideTableParts::read(this->function(function), section(ImageSection::SideTables).data()), returnOffset); + SideTableReader reader = sideTable(function); + reader.next(SideTableByte::Flags); + return hasFrameAtReturnOffset(function, SideTableFrame::read(reader, offsetBeforeReturn(returnOffset)), returnOffset); +} + +ImageFrame Image::frameAtReturnOffset(uint32_t function, uint32_t returnOffset) const +{ + ImageFrame frame { }; + SideTableReader reader = sideTable(function); + auto flags = OptionSet::fromRaw(reader.next(SideTableByte::Flags)); + SideTableFrame frameInTable = SideTableFrame::read(reader, offsetBeforeReturn(returnOffset)); + if (hasFrameAtReturnOffset(function, frameInTable, returnOffset) && frameInTable.hasFrame) + frame = section(ImageSection::Frames)[frameInTable.number]; + if (flags.contains(ImageFunctionFlag::SpillsArgumentRegisters)) + frame.calleeSavesStart |= ImageFrame::liesBelowSpilledArgumentsBit; + return frame; } -bool Image::hasFrameAtReturnOffset(uint32_t function, const SideTableParts& parts, uint32_t returnOffset) const +bool Image::hasFrameAtReturnOffset(uint32_t function, const SideTableFrame& frame, uint32_t returnOffset) const { if (!returnOffset) return false; @@ -426,7 +477,7 @@ bool Image::hasFrameAtReturnOffset(uint32_t function, const SideTableParts& part return fault->hasFrame; } } - return parts.hasFrameAt(returnOffset - 1); + return frame.coversOffset; } static std::span characterBytesOf(const StringImpl& string) { return string.is8Bit() ? asBytes(string.span8()) : asBytes(string.span16()); } diff --git a/Source/JavaScriptCore/aot/image/AOTImage.h b/Source/JavaScriptCore/aot/image/AOTImage.h index 20e3cd9a466f6..ec5616635aafe 100644 --- a/Source/JavaScriptCore/aot/image/AOTImage.h +++ b/Source/JavaScriptCore/aot/image/AOTImage.h @@ -23,6 +23,7 @@ #include "AOTVector.h" #include "AOTWire.h" #include "JSExportMacros.h" +#include #include #include #include @@ -77,6 +78,7 @@ struct ReturnSiteInfo { std::optional site; std::optional handler; ImageFrame frame { }; + bool hasConstructedCallees { false }; }; class Image { @@ -139,7 +141,7 @@ class Image { std::span bytes() const { return m_bytes; } template std::span section(ImageSection) const; std::span code() const { return { m_code, static_cast(header().codeSize) }; } - std::span regExpCode() const { return code().subspan(header().code.regExpCodeOffset, header().code.constantsOffset - header().code.regExpCodeOffset); } + std::span regExpCode() const { return code().subspan(header().code.regExpCodeOffset, std::max(header().code.constantsOffset, header().code.regExpCodeOffset) - header().code.regExpCodeOffset); } std::span webAssemblyCode() const { return code().subspan(header().code.webAssemblyCodeOffset, header().code.webAssemblyCodeSize); } std::span ffiCode() const { return code().subspan(header().code.ffiCodeOffset, header().code.ffiCodeSize); } std::span ffiDirectory() const; @@ -161,15 +163,17 @@ class Image { JS_EXPORT_PRIVATE CodeLocation locate(const void* pc) const; - const ImageFunction& function(uint32_t function) const; + ImageFunction function(uint32_t function) const; + const FunctionStarts& functionStarts() const LIFETIME_BOUND { return m_functionStarts; } const uint8_t* codeOf(uint32_t function) const; size_t sizeOfCodeOf(uint32_t function) const; const ImageFrame& frameOf(uint32_t function) const; + SideTableReader sideTable(uint32_t function) const; FunctionID rootFunctionOf(uint32_t function) const; uint64_t closureWord(FunctionID) const; JS_EXPORT_PRIVATE const void* stubAddress(StubKey) const; - enum class FrequentStub : uint8_t { EnterClosure, EnterSlimClosure, EnterModuleBody, EnterClosureForConstruct, EnterHandlerOfJob, EnterSavedFrame, RejectInFrame, RejectInFrameBelowSpilledArguments }; + enum class FrequentStub : uint8_t { EnterClosure, EnterSlimClosure, EnterModuleBody, EnterClosureForConstruct, EnterHandlerOfJob, EnterSavedFrame, RejectInFrame, RejectInFrameBelowSpilledArguments, RejectAwaitedCall }; template void* frequentStub(FrequentStub, const Functor&) const; JS_EXPORT_PRIVATE std::span stubAddressesOf(StubFamily) const; std::span stubAddressesSortedWith(StubFamily) const; @@ -189,11 +193,12 @@ class Image { std::span frameMapWords(uint32_t function) const; bool hasFrameAt(uint32_t function, uint32_t offset) const; bool hasFrameAtReturnOffset(uint32_t function, uint32_t returnOffset) const; - bool hasFrameAtReturnOffset(uint32_t function, const SideTableParts&, uint32_t returnOffset) const; + bool hasFrameAtReturnOffset(uint32_t function, const SideTableFrame&, uint32_t returnOffset) const; ImageFrame frameAtReturnOffset(uint32_t function, uint32_t returnOffset) const; ReturnSiteInfo returnSiteInfoAt(uint32_t function, uint32_t returnOffset) const; std::optional callSiteAt(uint32_t function, uint32_t returnOffset) const; + JS_EXPORT_PRIVATE std::optional handlerAt(uint32_t function, CallSiteIndex) const; std::optional constructedCalleeAt(uint32_t function, CallSiteIndex) const; std::optional prologueCheckAt(const void* returnAddress) const; @@ -212,7 +217,7 @@ class Image { JS_EXPORT_PRIVATE static std::atomic s_current; static std::atomic s_lastPublished; - mutable std::array, 8> m_frequentStubs { }; + mutable std::array, 9> m_frequentStubs { }; mutable std::atomic m_decompressSource { nullptr }; mutable DecompressedSourceBlocks m_decompressedFunctionSources; mutable DecompressedSourceBlocks m_decompressedQuotedSources; @@ -222,8 +227,8 @@ class Image { std::optional m_positionOfCodeInFile; std::span m_bytes; const uint8_t* m_code; - const ImageFunction* m_functions; - const uint32_t* m_functionStarts; + ImageFunctions m_functions; + FunctionStarts m_functionStarts; const uint32_t* m_codeGranules; ImageText m_text; mutable std::once_flag m_didSortFunctionsByID; @@ -237,12 +242,6 @@ std::span Image::section(ImageSection section) const return { reinterpret_cast(m_bytes.data() + range.offset), range.size / sizeof(T) }; } -inline const ImageFunction& Image::function(uint32_t function) const -{ - RELEASE_ASSERT(function < header().counts.functions); - return m_functions[function]; -} - inline const uint8_t* Image::codeOf(uint32_t function) const { RELEASE_ASSERT(function < header().counts.functions); @@ -252,32 +251,14 @@ inline const uint8_t* Image::codeOf(uint32_t function) const inline size_t Image::sizeOfCodeOf(uint32_t function) const { RELEASE_ASSERT(function < header().counts.functions); - const uint32_t* starts = m_functionStarts; + const FunctionStarts& starts = m_functionStarts; return (function + 1 < header().counts.functions ? starts[function + 1] : header().code.functionsEnd) - starts[function]; } -inline const ImageFrame& Image::frameOf(uint32_t function) const -{ - static constexpr ImageFrame noFrame { 0, 0, 0 }; - SideTableParts parts = SideTableParts::read(this->function(function), section(ImageSection::SideTables).data()); - if (parts.frameRanges.empty()) - return noFrame; - return section(ImageSection::Frames)[parts.frame]; -} - -inline ImageFrame Image::frameAtReturnOffset(uint32_t function, uint32_t returnOffset) const -{ - ImageFrame frame { }; - if (hasFrameAtReturnOffset(function, returnOffset)) - frame = frameOf(function); - if (this->function(function).has(ImageFunctionFlag::SpillsArgumentRegisters)) - frame.calleeSavesStart |= ImageFrame::liesBelowSpilledArgumentsBit; - return frame; -} - inline FunctionID Image::rootFunctionOf(uint32_t function) const { - return FunctionID(this->function(function).function()); + RELEASE_ASSERT(function < header().counts.functions); + return FunctionID(m_functions.numberOf(function)); } inline uint64_t Image::closureWord(FunctionID function) const diff --git a/Source/JavaScriptCore/aot/image/AOTImageBuilder.cpp b/Source/JavaScriptCore/aot/image/AOTImageBuilder.cpp index bb2aaae208a23..75f60388bb682 100644 --- a/Source/JavaScriptCore/aot/image/AOTImageBuilder.cpp +++ b/Source/JavaScriptCore/aot/image/AOTImageBuilder.cpp @@ -11,6 +11,7 @@ #include "AOTBuildContext.h" #include "AOTBuiltinIDs.h" #include "AOTCallSiteTable.h" +#include "AOTCodedTexts.h" #include "AOTCodeTemplates.h" #include "AOTFacts.h" #include "AOTFaultTable.h" @@ -286,10 +287,15 @@ struct StringHeaderNumbers { bool hasRecords() const { return form == ImageForm::PositionIndependent; } uint64_t sizeOfHeaders() const { return static_cast(count) * (hasRecords() ? sizeof(ImageStringRecord) : sizeof(ImageStringHeader)); } uint64_t sizeOfList() const { return static_cast(numberOfListedWords) * sizeof(uint32_t); } + uint32_t numberOfStringWords() const { return ofStrings->numberOfWords() - endOfIdentifiers; } ImageForm form { ImageForm::Absolute }; + const StringNumbers* ofStrings { nullptr }; Vector byString; Vector listedWordIndexes; + Vector codedTexts; + Vector otherIdentifiers; + uint32_t endOfIdentifiers { 0 }; uint32_t count { 0 }; uint32_t numberOfListedWords { 0 }; uint64_t largestNumberNeeded { 0 }; @@ -390,33 +396,23 @@ static size_t countFunctionsWithCode(const Typed::Program& program, const Linked return count; } -static Vector measureSideTablesOfFunctions(const SideTables& sideTables, std::span> callSites, bool hasPositions, const ParallelLoops& loops) +static Vector measureSideTablesOfFunctions(const SideTables& sideTables, bool hasPositions, const ParallelLoops& loops) { - size_t numberOfFunctions = callSites.size(); - RELEASE_ASSERT(sideTables.offsets.size() == numberOfFunctions && sideTables.flags.size() == numberOfFunctions); - Vector sizes(FillWith { }, numberOfFunctions, 0); - const uint8_t* streams = sideTables.bytes.span().data(); + size_t numberOfFunctions = sideTables.sizes.size(); + Vector sizes = sideTables.sizes; + if (!hasPositions) + return sizes; const uint8_t* positions = sideTables.positions.span().data(); size_t numberOfGroups = PositionIndex::numberOfEntriesFor(numberOfFunctions); forEachIndexInParallel(loops, static_cast(numberOfGroups), [&](unsigned group) { - const uint8_t* cursor = nullptr; - if (hasPositions) { - uint32_t placeOfFirstEntry; - memcpy(&placeOfFirstEntry, positions + group * sizeof(uint32_t), sizeof(uint32_t)); - cursor = positions + numberOfGroups * sizeof(uint32_t) + placeOfFirstEntry; - } + const uint8_t* cursor = PositionIndex::firstEntry(positions, group, numberOfFunctions); size_t first = static_cast(group) * PositionIndex::functionsBetween; size_t end = std::min(numberOfFunctions, first + PositionIndex::functionsBetween); for (size_t position = first; position < end; ++position) { - SideTableParts parts = SideTableParts::read(ImageFunction::create(sideTables.offsets[position], 0, sideTables.flags[position]), streams); - uint64_t size = parts.frameRanges.size() + parts.handlers.size() + callSites[position].size(); - if (cursor) { - const uint8_t* entry = cursor; - PositionsHead head = PositionsHead::read(cursor); - cursor += head.size; - size += cursor - entry; - } - sizes[position] = clampTo(size); + const uint8_t* entry = cursor; + PositionsHead head = PositionsHead::read(cursor); + cursor += head.size; + sizes[position] += static_cast(cursor - entry); } }); return sizes; @@ -447,16 +443,26 @@ static TextRecords textRecordsIn(const Vector& strings, const Vector stringsInOrderOfText, ImageForm form) +static StringHeaderNumbers numberStringHeaders(const TextRecords& text, const ImageNumbers& imageNumbers, ImageForm form) { StringHeaderNumbers numbers; numbers.form = form; + numbers.ofStrings = &imageNumbers.stringNumbers(); numbers.byString.fill(0, text.strings.size()); if (numbers.hasRecords()) numbers.listedWordIndexes.fill(0, text.strings.size()); uint64_t count = text.identifiers.size(); + numbers.endOfIdentifiers = static_cast(text.identifiers.size()); + RELEASE_ASSERT(numbers.endOfIdentifiers <= numbers.ofStrings->numberOfWords()); + for (uint32_t number = 0; number < text.identifiers.size(); ++number) { + StringID textOfIdentifier = text.identifiers[number].text(); + bool isOther = text.identifiers[number].kind() != PropertyKeyKind::Text; + if (textOfIdentifier && isOther) + numbers.otherIdentifiers.append({ number, numbers.ofStrings->requiredNumberOf(textOfIdentifier) + 1 }); + RELEASE_ASSERT(isOther || !number || numbers.ofStrings->requiredNumberOf(textOfIdentifier) == number); + } uint64_t numberOfListedWords = 0; - for (size_t index : listStringsInOrderOfText(text.strings.size(), stringsInOrderOfText)) { + for (size_t index : listStringsInOrderOfText(text.strings.size(), imageNumbers.stringsInOrderOfText())) { const ImageString& string = text.strings[index]; bool needsRecord = numbers.hasRecords() && string.hasHeader() && string.range(text.longStrings).length; if (needsRecord && !ImageListedString::fitsOneWord(string.range(text.longStrings))) { @@ -474,10 +480,24 @@ static StringHeaderNumbers numberStringHeaders(const TextRecords& text, std::spa } numbers.count = static_cast(count); numbers.numberOfListedWords = static_cast(numberOfListedWords); - numbers.largestNumberNeeded = std::max(count, numberOfListedWords); + numbers.largestNumberNeeded = std::max({ count, numberOfListedWords, imageNumbers.stringNumbers().coded().size() }); return numbers; } +static Vector encodeCodedTexts(const Typed::StringTable& strings, const StringNumbers& numbers, BuildContext& context) +{ + Vector> texts(numbers.coded().size(), [&](size_t index) { + return strings[numbers.coded()[index]]; + }); + Vector bytes = CodedTexts::encode(texts.span(), context.parallelLoops()); + if (checksImageWhileBuilding(context)) { + CodedTexts coded(bytes.span(), static_cast(texts.size())); + for (uint32_t number = 0; number < texts.size(); ++number) + RELEASE_ASSERT(equalSpans(std::span(coded.at(number).span()), texts[number])); + } + return bytes; +} + [[nodiscard]] static bool checkStringHeaderNumbers(const StringHeaderNumbers& numbers, BuildContext& context) { static_assert(defaultLimitValue(LimitID::MaxListedStrings, hostArchitecture()) == ImageString::hashMask + 1); @@ -497,11 +517,11 @@ static std::optional computeImageExtent(const ImageSections& sectio if (static_cast(index) == ImageSection::StringHeaders) size = stringHeaders.sizeOfHeaders(); else if (static_cast(index) == ImageSection::Strings) - size = size / sizeof(ImageString) * sizeof(ImageStringWord); + size = static_cast(stringHeaders.numberOfStringWords()) * sizeof(ImageStringWord) + stringHeaders.codedTexts.size(); else if (static_cast(index) == ImageSection::LongStrings) size = stringHeaders.sizeOfList(); else if (static_cast(index) == ImageSection::Identifiers) - size = size / sizeof(ImageIdentifier) * sizeof(ImageIdentifierWord); + size = ImageOtherIdentifier::offsetFor(size / sizeof(ImageIdentifier)) + stringHeaders.otherIdentifiers.sizeInBytes(); bool isClonedByPages = static_cast(index) == ImageSection::MethodTable && sections.programTables().instanceSizes.sizeOfMethodTable; tablesEnd = isClonedByPages ? WTF::roundUpToMultipleOf(tablesEnd) : WTF::roundUpToMultipleOf(tablesEnd); extent.sections[index] = { static_cast(tablesEnd), static_cast(size) }; @@ -583,6 +603,8 @@ static void writeHeader(ImageHeader& header, const ImageWriterInputs& parts, con .endOfClosureNumbers = functionNumbers.endOfClosureFunctions().index() - functionNumbers.firstClosureFunction().index(), .endOfIdentifiers = endOfIdentifiers, .endOfStringHeaders = stringHeaders.count, + .numberOfStringWords = stringHeaders.numberOfStringWords(), + .numberOfCodedTexts = static_cast(stringHeaders.ofStrings->coded().size()), }; if (Typed::reachesModuleObjects(parts.program)) flags.add(ImageFlag::ShowsWaysBetweenModuleObjects); @@ -637,6 +659,15 @@ static std::span sectionIn(std::span image, const ImageExtent& exten return spanReinterpretCast(image.subspan(extent[section].offset, extent[section].size)); } +static ImageStringWord wordOfString(const TextRecords& text, const StringHeaderNumbers& numbers, size_t index) +{ + const ImageString& string = text.strings[index]; + ImageStringWord word = string.inImage(numbers.byString[index]); + if (string.hasText() && string.range(text.longStrings).length < 2) + word.bits |= ImageString::isShorterThanTwoFlag; + return word; +} + static void writeStrings(std::span image, const TextRecords& text, const StringHeaderNumbers& numbers, const ImageExtent& extent, const ParallelLoops& loops) { std::span records; @@ -645,20 +676,26 @@ static void writeStrings(std::span image, const TextRecords& text, cons records = sectionIn(image, extent, ImageSection::StringHeaders); else headers = sectionIn(image, extent, ImageSection::StringHeaders); - std::span stringsInImage = sectionIn(image, extent, ImageSection::Strings); + std::span stringsInImage = sectionIn(image, extent, ImageSection::Strings).first(numbers.numberOfStringWords()); + memcpySpan(image.subspan(extent[ImageSection::Strings].offset + stringsInImage.size_bytes()), numbers.codedTexts.span()); std::span listedStrings = sectionIn(image, extent, ImageSection::LongStrings); - std::span identifiersInImage = sectionIn(image, extent, ImageSection::Identifiers); + std::span identifiersInImage = sectionIn(image, extent, ImageSection::Identifiers); for (size_t number = 0; number < text.identifiers.size(); ++number) - identifiersInImage[number] = text.identifiers[number].inImage(); + identifiersInImage[number] = text.identifiers[number].word(0).byteInImage(); + memcpySpan(identifiersInImage.subspan(ImageOtherIdentifier::offsetFor(text.identifiers.size())), asBytes(numbers.otherIdentifiers.span())); forEachRangeOfIndicesInParallel(loops, text.strings.size(), [&](size_t first, size_t end) { for (size_t index = first; index < end; ++index) { const ImageString& string = text.strings[index]; - stringsInImage[index] = string.inImage(numbers.byString[index]); + if (std::optional number = numbers.ofStrings->numberOf(StringID(static_cast(index))); number && *number < numbers.ofStrings->numberOfWords()) { + ImageStringWord word = wordOfString(text, numbers, index); + if (*number >= numbers.endOfIdentifiers) + stringsInImage[*number - numbers.endOfIdentifiers] = word; + else + RELEASE_ASSERT((word.bits & ~(ImageString::is16BitFlag | ImageString::isShorterThanTwoFlag)) == ImageStringWord::ofIdentifier(*number).bits); + } if (!string.hasText()) continue; ImageLongString range = string.range(text.longStrings); - if (range.length < 2) - stringsInImage[index].bits |= ImageString::isShorterThanTwoFlag; if (!string.hasHeader()) { ImageListedString::write(listedStrings.subspan(numbers.byString[index]), range); if (string.hasListedHash()) @@ -689,11 +726,10 @@ static void writeStrings(std::span image, const TextRecords& text, cons }); } -static void writeWordsOfStringConstants(std::span image, const ImageExtent& extent) +static void writeWordsOfStringConstants(std::span image, const TextRecords& text, const StringHeaderNumbers& numbers, const ImageExtent& extent) { - std::span stringsInImage = sectionIn(image, extent, ImageSection::Strings); for (uint32_t& constant : sectionIn(image, extent, ImageSection::StringConstants)) - constant = stringsInImage[constant].bits; + constant = wordOfString(text, numbers, constant).bits; } static void writeRangeOfShapeSets(std::span image, const CodeLinkerInputs& linker, const ImageExtent& extent) @@ -742,7 +778,7 @@ static Vector writeImage(const ImageWriterInputs& parts, PhaseTimes& ti times.didFinish("copy sections"_s); writeStrings(image.mutableSpan(), text, stringHeaders, *extent, loops); - writeWordsOfStringConstants(image.mutableSpan(), *extent); + writeWordsOfStringConstants(image.mutableSpan(), text, stringHeaders, *extent); writeRangeOfShapeSets(image.mutableSpan(), parts.linker, *extent); times.didFinish("write strings"_s); @@ -816,7 +852,7 @@ BuiltImage buildImage(ImageInputs&& inputs, BuildContext& context) EncodedText text; StringHeaderNumbers stringHeaders; Vector namesOfSites; - Vector> callSites; + Vector> callSites; SideTables sideTables; const bool hasPositions = !context.options().obfuscates(); @@ -825,14 +861,13 @@ BuiltImage buildImage(ImageInputs&& inputs, BuildContext& context) compiledRegExps = compileRegExps(program, numbers.regExpRequests(), context); steps.didFinish("compile regular expressions"_s); }; - auto layOutAndDescribeRegExps = [&] { + auto layOutRegExps = [&] { StepTimes steps(context, nameOfLaneSteps); std::optional section = layOutRegExpSection(compiledRegExps.results.span(), context.assemblerPolicy(), templates, &context); if (!section) return; regExpSection = WTF::move(*section); - regExps = encodeRegExpRecords({ numbers.regExpRequests(), compiledRegExps, regExpSection }, context); - steps.didFinish("lay out and describe regular expressions"_s); + steps.didFinish("lay out regular expressions"_s); }; std::optional mentions; @@ -864,7 +899,7 @@ BuiltImage buildImage(ImageInputs&& inputs, BuildContext& context) steps.didFinish("encode side tables"_s); if (diagnostics.hasErrors()) return; - result.sideTableBytes = measureSideTablesOfFunctions(sideTables, callSites.span(), hasPositions, context.parallelLoops()); + result.sideTableBytes = measureSideTablesOfFunctions(sideTables, hasPositions, context.parallelLoops()); steps.didFinish("measure side tables"_s); }, [&] { @@ -964,19 +999,25 @@ BuiltImage buildImage(ImageInputs&& inputs, BuildContext& context) steps.didFinish("encode text"_s); if (diagnostics.hasErrors()) return; - stringHeaders = numberStringHeaders(textRecordsIn(text.strings, text.longStrings, text.identifiers), numbers.stringsInOrderOfText(), context.recordedOptions().form()); + stringHeaders = numberStringHeaders(textRecordsIn(text.strings, text.longStrings, text.identifiers), numbers, context.recordedOptions().form()); + stringHeaders.codedTexts = encodeCodedTexts(program.strings, numbers.stringNumbers(), context); steps.didFinish("number string headers"_s); }, [&] { compileRegExpsOfImage(); if (!diagnostics.hasErrors()) - layOutAndDescribeRegExps(); + layOutRegExps(); }, }); times.didFinish("lanes in front of the layout of code"_s); if (diagnostics.hasErrors()) return result; + regExps = encodeRegExpRecords({ numbers.regExpRequests(), compiledRegExps, regExpSection, numbers.stringNumbers() }, context); + times.didFinish("describe regular expressions"_s); + if (diagnostics.hasErrors()) + return result; + CodeLayoutInputs layoutInputs { .functions = functions, .callees = callees, @@ -1020,7 +1061,6 @@ BuiltImage buildImage(ImageInputs&& inputs, BuildContext& context) .layout = result.layout, .sharedCode = result.sharedCode, .index = *index, - .sideTableFlags = sideTables.flags.span(), .sideTableOffsets = sideTables.offsets.span(), .programTables = sections.programTables(), .hasPositions = hasPositions, diff --git a/Source/JavaScriptCore/aot/image/AOTImageBuilder.h b/Source/JavaScriptCore/aot/image/AOTImageBuilder.h index c8c6ad8143a3b..fcb9bee9dd98d 100644 --- a/Source/JavaScriptCore/aot/image/AOTImageBuilder.h +++ b/Source/JavaScriptCore/aot/image/AOTImageBuilder.h @@ -12,6 +12,7 @@ #include "AOTConstantPool.h" #include "AOTFFIStubs.h" #include "AOTImageFormat.h" +#include "AOTPrefixCode.h" #include "AOTLinkedFunctions.h" #include "AOTNumbering.h" #include "AOTRegExpCompiler.h" @@ -32,7 +33,7 @@ class StubCatalog; struct SideTables { Vector bytes; Vector offsets; - Vector> flags; + Vector sizes; Vector positions; Vector frameMapWords; Vector frames; @@ -86,7 +87,7 @@ void reportLinkFailure(LinkFailure, FunctionID, BuildContext&); OptionSet imageFlagsAdmittedBy(Wire::UnseenCodeCause); OptionSet flagsForUnseenCode(const Typed::Facts&, const CompileOptions&); ResolvedCallees resolveCallees(const LinkedFunctions&, const Typed::Plan&, BuildContext&); -SideTables encodeSideTables(const LinkedFunctions&, std::span> callSites, BuildContext&); +SideTables encodeSideTables(const LinkedFunctions&, std::span> callSites, BuildContext&); struct CalledSharedCode { Vector stubKeys; Vector stubKeysOfArgumentMoves; diff --git a/Source/JavaScriptCore/aot/image/AOTImageFormat.h b/Source/JavaScriptCore/aot/image/AOTImageFormat.h index 16e4a9680e898..484ad2d77c346 100644 --- a/Source/JavaScriptCore/aot/image/AOTImageFormat.h +++ b/Source/JavaScriptCore/aot/image/AOTImageFormat.h @@ -10,12 +10,15 @@ #include "AOTClosureWord.h" #include "AOTCompileOptions.h" #include "AOTLimits.h" +#include "AOTPrefixCode.h" +#include "AOTRecordTypes.h" #include "AOTRuntimeTable.h" #include "AOTStubFrames.h" #include "AOTVarint.h" #include "AOTVector.h" #include "CPU.h" #include "CPUFeatureSet.h" +#include "JSExportMacros.h" #include "StackAlignment.h" #include #include @@ -48,8 +51,41 @@ static constexpr size_t sizeOfNearCall = isX86_64() ? 5 : sizeof(uint32_t); static constexpr size_t sizeOfJump = isX86_64() ? 5 : sizeof(uint32_t); static constexpr size_t codeOffsetUnit = isX86_64() ? 1 : sizeof(uint32_t); +class FunctionStarts { +public: + static constexpr unsigned groupShift = 5; + static constexpr uint32_t functionsInGroup = 1u << groupShift; + static constexpr uint32_t wideGroup = 1u << 31; + + static constexpr size_t numberOfWordsOfSteps(size_t numberOfFunctions) { return (numberOfFunctions + 2) / 2; } + static constexpr size_t numberOfGroupsFor(size_t numberOfFunctions) { return (numberOfFunctions + functionsInGroup) / functionsInGroup; } + static constexpr size_t numberOfWordsWithoutLists(size_t numberOfFunctions) { return numberOfWordsOfSteps(numberOfFunctions) + numberOfGroupsFor(numberOfFunctions); } + JS_EXPORT_PRIVATE static Vector encode(std::span starts); + + FunctionStarts(const uint32_t* words, size_t numberOfFunctions) + : m_steps(reinterpret_cast(words)) + , m_groups(words + numberOfWordsOfSteps(numberOfFunctions)) + { + } + + uint32_t operator[](uint32_t function) const + { + uint32_t group = m_groups[function >> groupShift]; + if (group & wideGroup) [[unlikely]] + return m_groups[(group & ~wideGroup) + function % functionsInGroup]; + return group + m_steps[function]; + } + + const uint16_t* steps() const { return m_steps; } + const uint32_t* groups() const { return m_groups; } + +private: + const uint16_t* m_steps; + const uint32_t* m_groups; +}; + #define FOR_EACH_AOT_IMAGE_SECTION(macro) \ - macro(Functions, ImageFunction) \ + macro(Functions, uint32_t) \ macro(FunctionStarts, uint32_t) \ macro(CodeGranules, uint32_t) \ macro(InnerEntries, ImageInnerEntry) \ @@ -63,7 +99,7 @@ static constexpr size_t codeOffsetUnit = isX86_64() ? 1 : sizeof(uint32_t); macro(Strings, ImageStringWord) \ macro(LongStrings, uint32_t) \ macro(Text, uint8_t) \ - macro(Identifiers, ImageIdentifierWord) \ + macro(Identifiers, uint8_t) \ macro(StringConstants, uint32_t) \ macro(NamesOfSites, uint16_t) \ macro(StringHeaders, ImageStringHeaderOrRecord) \ @@ -134,6 +170,8 @@ struct ImageNumberRanges { uint32_t endOfClosureNumbers; uint32_t endOfIdentifiers; uint32_t endOfStringHeaders; + uint32_t numberOfStringWords; + uint32_t numberOfCodedTexts; }; struct ImageHeader { @@ -183,25 +221,64 @@ enum class ImageFunctionFlag : uint8_t { }; struct ImageFunction { - static constexpr unsigned numberOfFunctionBits = 24; - static constexpr unsigned flagsShift = numberOfFunctionBits; - static constexpr uint32_t functionMask = (1u << numberOfFunctionBits) - 1; + OptionSet flags() const { return OptionSet::fromRaw(flagBits); } + bool has(ImageFunctionFlag flag) const { return flags().contains(flag); } + + uint32_t sideTables; + uint8_t flagBits; +}; - static ImageFunction create(uint32_t sideTables, uint32_t function, OptionSet flags) +class ImageFunctions { +public: + static constexpr unsigned functionsInGroup = 32; + static constexpr size_t wordsOfGroup = 3; + static constexpr uint16_t isListedPlace = 1u << 15; + static constexpr uint32_t hasWholePlaces = UINT32_MAX; + + ImageFunctions(const uint32_t* words, uint32_t count) + : m_places(words + wordsOfHead) + , m_groups(words + firstGroupWord(words[0], count)) + , m_listedPlaces(words + firstListWord(words[0], count)) + , m_listedNumbers(reinterpret_cast(m_listedPlaces + numberOfListedPlaces(words[0]))) + , m_hasWholePlaces(words[0] == hasWholePlaces) { - RELEASE_ASSERT(function <= functionMask); - return { sideTables, function | static_cast(flags.toRaw()) << flagsShift }; } - uint32_t function() const { return functionAndFlags & functionMask; } - OptionSet flags() const { return OptionSet::fromRaw(static_cast(functionAndFlags >> flagsShift)); } - bool has(ImageFunctionFlag flag) const { return flags().contains(flag); } + uint32_t sideTableOf(uint32_t index) const + { + if (m_hasWholePlaces) [[unlikely]] + return m_places[index]; + uint16_t step = reinterpret_cast(m_places)[index]; + return step & isListedPlace ? m_listedPlaces[step & ~isListedPlace] : group(index)[2] + step; + } + uint32_t numberOf(uint32_t index) const + { + uint32_t entry = index % functionsInGroup; + uint32_t listed = ~group(index)[0] & UINT32_MAX >> (functionsInGroup - 1 - entry); + uint32_t lastListed = functionsInGroup - 1 - std::countl_zero(listed); + return m_listedNumbers[group(index)[1] + std::popcount(listed) - 1] + (entry - lastListed); + } - uint32_t sideTables; - uint32_t functionAndFlags; + static bool hasSize(std::span words, size_t count) { return words.size() >= wordsOfHead && words[2] == count && words.size() == firstListWord(words[0], count) + numberOfListedPlaces(words[0]) + wordsOfNumbers(words[1]); } + JS_EXPORT_PRIVATE static Vector encode(std::span sideTables, std::span numbers); + +private: + static constexpr size_t wordsOfHead = 3; + + static constexpr size_t numberOfListedPlaces(uint32_t firstWord) { return firstWord == hasWholePlaces ? 0 : firstWord; } + static constexpr size_t firstGroupWord(uint32_t firstWord, size_t count) { return wordsOfHead + (firstWord == hasWholePlaces ? count : (count + 1) / 2); } + static constexpr size_t firstListWord(uint32_t firstWord, size_t count) { return firstGroupWord(firstWord, count) + (count + functionsInGroup - 1) / functionsInGroup * wordsOfGroup; } + static constexpr size_t wordsOfNumbers(size_t count) { return (count * sizeof(Number24) + sizeof(uint32_t) - 1) / sizeof(uint32_t); } + + const uint32_t* group(uint32_t index) const { return m_groups + index / functionsInGroup * wordsOfGroup; } + + const uint32_t* m_places; + const uint32_t* m_groups; + const uint32_t* m_listedPlaces; + const Number24* m_listedNumbers; + bool m_hasWholePlaces; }; -static_assert(sizeof(ImageFunction) == 8 && std::has_unique_object_representations_v); -static_assert(defaultLimitValue(LimitID::ImageFunctions, hostArchitecture()) <= 1ull << ImageFunction::numberOfFunctionBits); +static_assert(defaultLimitValue(LimitID::ImageFunctions, hostArchitecture()) <= static_cast(Number24::max) + 1); static_assert(defaultLimitValue(LimitID::ImageClosureFunctions, hostArchitecture()) == ClosureWord::firstReservedFunction); static_assert(defaultLimitValue(LimitID::CodeBytes, hostArchitecture()) <= ClosureWord::maxSizeOfCode); @@ -253,141 +330,19 @@ static_assert(sizeof(StubAddressInImage) == 16); static constexpr uint32_t offsetOfAbsentStub = 0xffffffffu; -struct SideTableInputs { - OptionSet flags() const; - - std::span frameRanges; - uint32_t frame { 0 }; - std::optional> frameMapRange; - std::span handlers; - std::span> constructedCallees; - std::span callSites; -}; - -struct SideTableParts { - static SideTableParts read(const ImageFunction&, const uint8_t* sideTables); - static void append(Vector& stream, const SideTableInputs&); - static void appendFrameChange(Vector& bytes, uint32_t previousChange, uint32_t change); - - bool hasFrameAt(uint32_t offset) const; - std::optional constructedCalleeAt(uint32_t site) const; - - std::span frameRanges; - uint32_t frame { 0 }; - uint32_t firstFrameMapWord { 0 }; - uint32_t numberOfFrameMapWords { 0 }; - std::span handlers; - std::span constructedCallees; - const uint8_t* callSites { nullptr }; -}; - struct PositionIndex { static constexpr unsigned functionsBetween = 32; + static constexpr unsigned numberOfCodes = 15; static constexpr size_t numberOfEntriesFor(size_t numberOfFunctions) { return (numberOfFunctions + functionsBetween - 1) / functionsBetween; } -}; -inline SideTableParts SideTableParts::read(const ImageFunction& function, const uint8_t* sideTables) -{ - SideTableParts parts; - const uint8_t* cursor = sideTables + function.sideTables; - size_t frameRangesSize = static_cast(readVarint(cursor)); - parts.frameRanges = unsafeMakeSpan(cursor, frameRangesSize); - cursor += frameRangesSize; - if (frameRangesSize) - parts.frame = static_cast(readVarint(cursor)); - if (function.has(ImageFunctionFlag::HasFrameMaps)) { - parts.firstFrameMapWord = static_cast(readVarint(cursor)); - parts.numberOfFrameMapWords = static_cast(readVarint(cursor)); - } - if (function.has(ImageFunctionFlag::HasHandlers)) { - size_t size = static_cast(readVarint(cursor)); - parts.handlers = unsafeMakeSpan(cursor, size); - cursor += size; - } - if (function.has(ImageFunctionFlag::HasConstructedCallees)) { - size_t size = static_cast(readVarint(cursor)); - parts.constructedCallees = unsafeMakeSpan(cursor, size); - cursor += size; - } - if (function.has(ImageFunctionFlag::HasCallSites)) - parts.callSites = cursor; - return parts; -} - -inline OptionSet SideTableInputs::flags() const -{ - OptionSet flags; - flags.set(ImageFunctionFlag::HasFrameMaps, frameMapRange && frameMapRange->second); - flags.set(ImageFunctionFlag::HasHandlers, !handlers.empty()); - flags.set(ImageFunctionFlag::HasConstructedCallees, !constructedCallees.empty()); - flags.set(ImageFunctionFlag::HasCallSites, !callSites.empty()); - return flags; -} - -inline void SideTableParts::append(Vector& stream, const SideTableInputs& inputs) -{ - OptionSet flags = inputs.flags(); - appendVarint(stream, inputs.frameRanges.size()); - stream.append(inputs.frameRanges); - if (!inputs.frameRanges.empty()) - appendVarint(stream, inputs.frame); - if (flags.contains(ImageFunctionFlag::HasFrameMaps)) { - appendVarint(stream, inputs.frameMapRange->first); - appendVarint(stream, inputs.frameMapRange->second); - } - if (flags.contains(ImageFunctionFlag::HasHandlers)) { - appendVarint(stream, inputs.handlers.size()); - stream.append(inputs.handlers); - } - if (flags.contains(ImageFunctionFlag::HasConstructedCallees)) { - Vector rows; - uint32_t previous = 0; - for (auto [site, function] : inputs.constructedCallees) { - RELEASE_ASSERT(site > previous); - appendVarint(rows, site - std::exchange(previous, site)); - appendVarint(rows, function); - } - appendVarint(stream, rows.size()); - stream.appendVector(rows); - } - stream.append(inputs.callSites); -} - -inline std::optional SideTableParts::constructedCalleeAt(uint32_t site) const -{ - const uint8_t* cursor = constructedCallees.data(); - const uint8_t* end = cursor + constructedCallees.size(); - uint64_t listed = 0; - while (cursor < end && listed < site) { - listed += readVarint(cursor); - uint32_t function = static_cast(readVarint(cursor)); - if (listed == site) - return function; - } - return std::nullopt; -} - -inline void SideTableParts::appendFrameChange(Vector& bytes, uint32_t previousChange, uint32_t change) -{ - RELEASE_ASSERT(previousChange <= change && !(previousChange % codeOffsetUnit) && !(change % codeOffsetUnit)); - appendVarint(bytes, (change - previousChange) / codeOffsetUnit); -} - -inline bool SideTableParts::hasFrameAt(uint32_t offset) const -{ - const uint8_t* cursor = frameRanges.data(); - const uint8_t* end = cursor + frameRanges.size(); - uint64_t position = offset / codeOffsetUnit; - uint64_t change = 0; - bool hasFrame = false; - while (cursor < end) { - change += readVarint(cursor); - if (position < change) - return hasFrame; - hasFrame = !hasFrame; + static const uint8_t* firstEntry(const uint8_t* section, size_t group, size_t numberOfFunctions) + { + const uint8_t* index = PrefixCodes::endOfCodes(section); + uint32_t place; + memcpy(&place, index + group * sizeof(uint32_t), sizeof(uint32_t)); + return index + numberOfEntriesFor(numberOfFunctions) * sizeof(uint32_t) + place; } - return hasFrame; -} +}; } } // namespace JSC::AOT diff --git a/Source/JavaScriptCore/aot/image/AOTImageText.cpp b/Source/JavaScriptCore/aot/image/AOTImageText.cpp index 500d246e65a77..8fcc7ebac79e9 100644 --- a/Source/JavaScriptCore/aot/image/AOTImageText.cpp +++ b/Source/JavaScriptCore/aot/image/AOTImageText.cpp @@ -8,11 +8,15 @@ #if ENABLE(AOT) +#include "AOTCodedTexts.h" #include "AOTImage.h" #include "AOTState.h" #include "AOTVector.h" +#include #include #include +#include +#include #include #include #include @@ -38,17 +42,38 @@ static_assert(ImageString::hashMask == StringHasher::maskHash && ImageIdentifier static_assert(!(imageSectionAlignment % alignof(ImageStringRecord)) && !(imageSectionAlignment % alignof(ImageStringHeader))); +struct ImageText::DecodedTexts { + static constexpr size_t numberOfRecentTexts = 512; + + struct RecentText { + uint32_t numberPlusOne; + uint8_t length; + std::array characters; + }; + + Lock lock; + Vector recentTexts WTF_GUARDED_BY_LOCK(lock); + HashMap, WTF::UnsignedWithZeroKeyHashTraits> viewedTexts WTF_GUARDED_BY_LOCK(lock); +}; + ImageText::ImageText(const Image& image) - : m_strings(image.section(ImageSection::Strings)) + : m_endOfIdentifiers(image.header().numbers.endOfIdentifiers) + , m_strings(image.section(ImageSection::Strings).first(image.header().numbers.numberOfStringWords)) + , m_codedTexts(image.section(ImageSection::Strings).subspan(m_strings.size_bytes())) + , m_numberOfCodedTexts(image.header().numbers.numberOfCodedTexts) + , m_decodedTexts(makeUniqueWithoutFastMallocCheck()) , m_listedStrings(image.section(ImageSection::LongStrings)) , m_text(image.section(ImageSection::Text)) - , m_identifiers(image.section(ImageSection::Identifiers)) + , m_identifiers(image.section(ImageSection::Identifiers).first(image.header().numbers.endOfIdentifiers)) + , m_otherIdentifiers(spanReinterpretCast(image.section(ImageSection::Identifiers).subspan(ImageOtherIdentifier::offsetFor(m_identifiers.size())))) , m_stringConstants(image.section(ImageSection::StringConstants)) #if !ENABLE(AOT_POSITION_INDEPENDENT_IMAGE) , m_stringHeaders(image.section(ImageSection::StringHeaders).first(image.header().numbers.endOfStringHeaders)) , m_addressOfText(image.header().address + image.header().sections[static_cast(ImageSection::Text)].offset) { } + +ImageText::~ImageText() = default; #else , m_stringRecords(image.section(ImageSection::StringHeaders)) { @@ -136,8 +161,44 @@ StringImpl* ImageText::headerOf(ImageStringWord word) const #endif } +uint32_t ImageText::textOfListedIdentifier(uint32_t number) const +{ + auto other = std::ranges::lower_bound(m_otherIdentifiers, number, { }, &ImageOtherIdentifier::number); + return other != m_otherIdentifiers.end() && other->number == number ? other->textPlusOne : 0; +} + +String ImageText::createStringOfCodedText(ImageStringWord word) const +{ + DecodedTexts& decoded = *m_decodedTexts; + Locker locker { decoded.lock }; + if (decoded.recentTexts.isEmpty()) + decoded.recentTexts.fill(DecodedTexts::RecentText { }, DecodedTexts::numberOfRecentTexts); + DecodedTexts::RecentText& recent = decoded.recentTexts[word.number() % DecodedTexts::numberOfRecentTexts]; + if (recent.numberPlusOne != word.number() + 1) { + Vector text = CodedTexts(m_codedTexts, m_numberOfCodedTexts).at(word.number()); + if (text.size() > recent.characters.size()) + return String(byteCast(text.span())); + recent.numberPlusOne = word.number() + 1; + recent.length = static_cast(text.size()); + std::ranges::copy(text, recent.characters.begin()); + } + return String(byteCast(std::span(recent.characters).first(recent.length))); +} + +NEVER_INLINE StringView ImageText::viewOfCodedText(ImageStringWord word) const +{ + DecodedTexts& decoded = *m_decodedTexts; + Locker locker { decoded.lock }; + return decoded.viewedTexts.ensure(word.number(), [&] { + Vector text = CodedTexts(m_codedTexts, m_numberOfCodedTexts).at(word.number()); + return String(byteCast(text.span())); + }).iterator->value; +} + String ImageText::createStringWithoutHeader(ImageStringWord word) const { + if (word.isCoded()) + return createStringOfCodedText(word); StringView characters = view(word); if (characters.isEmpty()) return emptyString(); @@ -150,16 +211,25 @@ StringView ImageText::view(ImageStringWord word) const { RELEASE_ASSERT_WITH_MESSAGE(word.hasText(), "AOT: the image has no text for this string because nothing referenced it at build time"); ImageLongString range { 0, 0 }; - if (!word.hasHeader()) + bool is16Bit = word.is16Bit(); + if (!word.hasHeader()) { + if (word.isCoded()) [[unlikely]] + return viewOfCodedText(word); range = ImageListedString::read(m_listedStrings.subspan(word.number())); + } #if ENABLE(AOT_POSITION_INDEPENDENT_IMAGE) - else - range = rangeFor(m_stringRecords[word.number()]); + else { + const ImageStringRecord& record = m_stringRecords[word.number()]; + range = rangeFor(record); + is16Bit = record.is16Bit(); + } #else - else if (const ImageStringHeader& header = headerAt(word.number()); header.length) + else if (const ImageStringHeader& header = headerAt(word.number()); header.length) { range = { static_cast(header.characters - m_addressOfText), header.length }; + is16Bit = !header.is8Bit(); + } #endif - if (word.is16Bit()) + if (is16Bit) return StringView(spanReinterpretCast(m_text.subspan(range.offset, range.length * sizeof(char16_t)))); return StringView(byteCast(m_text.subspan(range.offset, range.length))); } diff --git a/Source/JavaScriptCore/aot/image/AOTImageText.h b/Source/JavaScriptCore/aot/image/AOTImageText.h index 10edb8ed97ab5..9507bec2f38a5 100644 --- a/Source/JavaScriptCore/aot/image/AOTImageText.h +++ b/Source/JavaScriptCore/aot/image/AOTImageText.h @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN @@ -42,6 +44,7 @@ struct ImageString { static constexpr uint32_t hasTextFlag = 1u << 28; static constexpr uint32_t hasListedHashFlag = 1u << 29; static constexpr uint32_t isShorterThanTwoFlag = 1u << 30; + static constexpr uint32_t isCodedFlag = 1u << 31; bool hasText() const { return !!location; } bool is16Bit() const { return hashAndFlags & is16BitFlag; } @@ -59,12 +62,15 @@ struct ImageString { static_assert(sizeof(ImageString) == 8 && isImageRecord); struct ImageStringWord { + static constexpr ImageStringWord ofIdentifier(uint32_t number) { return { ImageString::hasTextFlag | ImageString::isAtomFlag | ImageString::hasIdentifierNumberFlag | number }; } + bool hasText() const { return !!bits; } bool is16Bit() const { return bits & ImageString::is16BitFlag; } bool isAtom() const { return bits & ImageString::isAtomFlag; } bool hasIdentifierNumber() const { return bits & ImageString::hasIdentifierNumberFlag; } bool hasHeader() const { return hasText() && !(bits & ImageString::hasNoHeaderFlag); } bool hasListedHash() const { return bits & ImageString::hasListedHashFlag; } + bool isCoded() const { return bits & ImageString::isCodedFlag; } uint32_t number() const { return bits & ImageString::hashMask; } uint32_t bits; @@ -136,6 +142,7 @@ struct ImageStringHeader { } uint32_t hash() const { return hashAndFlags >> StringImpl::s_flagCount; } + bool is8Bit() const { return hashAndFlags & StringImpl::flagIs8Bit(); } uint32_t refCount; uint32_t length; @@ -184,7 +191,7 @@ struct ImageIdentifier { uint32_t hashOrSymbol() const { return bits & hashMask; } PropertyKeyKind kind() const { return static_cast(bits >> kindShift & ((1u << kindBits) - 1)); } NameClass nameClass() const { return static_cast(bits >> nameClassShift); } - ImageIdentifierWord inImage() const; + ImageIdentifierWord word(uint32_t textNumberPlusOne) const; uint32_t textPlusOne; uint32_t bits; @@ -194,6 +201,9 @@ static_assert(sizeof(ImageIdentifier) == 8 && isImageRecord); struct ImageIdentifierWord { static constexpr uint32_t textMask = ImageIdentifier::hashMask; + uint8_t byteInImage() const { return bits >> ImageIdentifier::kindShift; } + static ImageIdentifierWord ofByteInImage(uint8_t byte) { return { static_cast(byte) << ImageIdentifier::kindShift }; } + StringID text() const { return bits & textMask ? StringID((bits & textMask) - 1) : StringID(); } PropertyKeyKind kind() const { return static_cast(bits >> ImageIdentifier::kindShift & ((1u << ImageIdentifier::kindBits) - 1)); } NameClass nameClass() const { return kind() == PropertyKeyKind::WellKnownSymbol ? NameClass::Ordinary : static_cast(bits >> ImageIdentifier::nameClassShift); } @@ -205,45 +215,59 @@ struct ImageIdentifierWord { uint32_t bits; }; -static_assert(sizeof(ImageIdentifierWord) == 4 && isImageRecord); +static_assert(sizeof(ImageIdentifierWord) == 4 && ImageIdentifier::kindShift == 24); + +struct ImageOtherIdentifier { + static constexpr size_t offsetFor(size_t numberOfIdentifiers) { return (numberOfIdentifiers + sizeof(uint32_t) - 1) / sizeof(uint32_t) * sizeof(uint32_t); } + static constexpr bool isSizeOfSection(size_t size, size_t numberOfIdentifiers) { return size >= offsetFor(numberOfIdentifiers) && !((size - offsetFor(numberOfIdentifiers)) % (2 * sizeof(uint32_t))); } + + uint32_t number; + uint32_t textPlusOne; +}; +static_assert(sizeof(ImageOtherIdentifier) == 8 && isImageRecord); static_assert(numberOfEnumerators <= 1u << ImageIdentifier::nameClassBits); -inline ImageIdentifierWord ImageIdentifier::inImage() const +inline ImageIdentifierWord ImageIdentifier::word(uint32_t textNumberPlusOne) const { - RELEASE_ASSERT(textPlusOne <= ImageIdentifierWord::textMask); + RELEASE_ASSERT(textNumberPlusOne <= ImageIdentifierWord::textMask); uint32_t kindBitsInPlace = static_cast(kind()) << kindShift; if (kind() == PropertyKeyKind::WellKnownSymbol) { RELEASE_ASSERT(nameClass() == NameClass::Ordinary); - return { textPlusOne | kindBitsInPlace | hashOrSymbol() << nameClassShift }; + return { textNumberPlusOne | kindBitsInPlace | hashOrSymbol() << nameClassShift }; } - return { textPlusOne | kindBitsInPlace | static_cast(nameClass()) << nameClassShift }; + return { textNumberPlusOne | kindBitsInPlace | static_cast(nameClass()) << nameClassShift }; } class ImageText { -#if ENABLE(AOT_POSITION_INDEPENDENT_IMAGE) WTF_MAKE_NONCOPYABLE(ImageText); -#endif public: JS_EXPORT_PRIVATE explicit ImageText(const Image&); -#if ENABLE(AOT_POSITION_INDEPENDENT_IMAGE) JS_EXPORT_PRIVATE ~ImageText(); -#endif - uint32_t numberOfStrings() const { return static_cast(m_strings.size()); } + uint32_t numberOfStrings() const { return endOfStringsWithWords() + m_numberOfCodedTexts; } + uint32_t endOfStringsWithWords() const { return m_endOfIdentifiers + static_cast(m_strings.size()); } uint32_t numberOfIdentifiers() const { return static_cast(m_identifiers.size()); } uint32_t numberOfStringConstants() const { return static_cast(m_stringConstants.size()); } - bool hasText(StringID string) const { return m_strings[string.index()].hasText(); } - bool isAtom(StringID string) const { return m_strings[string.index()].isAtom(); } - ImageStringWord wordOf(StringID string) const { return m_strings[string.index()]; } + bool hasText(StringID string) const { return wordOf(string).hasText(); } + bool isAtom(StringID string) const { return wordOf(string).isAtom(); } + ImageStringWord wordOf(StringID string) const + { + uint32_t number = string.index(); + if (number < m_endOfIdentifiers) [[likely]] + return { number ? ImageStringWord::ofIdentifier(number).bits : 0 }; + if (number < endOfStringsWithWords()) + return m_strings[number - m_endOfIdentifiers]; + return { ImageString::hasTextFlag | ImageString::hasNoHeaderFlag | ImageString::isCodedFlag | (number - endOfStringsWithWords()) }; + } StringView view(StringID string) const { return view(wordOf(string)); } ALWAYS_INLINE StringView viewInline(StringID string) const { ImageStringWord word = wordOf(string); #if !ENABLE(AOT_POSITION_INDEPENDENT_IMAGE) - if ((word.bits & (ImageString::hasTextFlag | ImageString::hasNoHeaderFlag | ImageString::is16BitFlag)) == ImageString::hasTextFlag) [[likely]] { + if ((word.bits & (ImageString::hasTextFlag | ImageString::hasNoHeaderFlag)) == ImageString::hasTextFlag) [[likely]] { const ImageStringHeader& header = headerAt(word.number()); - if (header.length) [[likely]] + if (header.length && header.is8Bit()) [[likely]] return StringView(unsafeMakeSpan(std::bit_cast(header.characters), header.length)); } #endif @@ -260,7 +284,12 @@ class ImageText { JS_EXPORT_PRIVATE String createAtom(ImageStringWord) const; StringImpl* tryGetHeader(ImageStringWord) const; - const ImageIdentifierWord& identifier(uint32_t number) const { return m_identifiers[number]; } + ImageIdentifierWord identifier(uint32_t number) const + { + ImageIdentifierWord word = ImageIdentifierWord::ofByteInImage(m_identifiers[number]); + word.bits |= word.kind() == PropertyKeyKind::Text ? (number ? number + 1 : 0) : textOfListedIdentifier(number); + return word; + } #if ENABLE(AOT_POSITION_INDEPENDENT_IMAGE) std::span stringRecords() const { return m_stringRecords; } StringView recordView(uint32_t number) const { return view(m_stringRecords[number]); } @@ -276,7 +305,7 @@ class ImageText { UniquedStringImpl* identifierAtoms() const { return reinterpret_cast(const_cast(m_stringHeaders.data())); } #endif ImageStringWord stringConstant(uint32_t number) const { return { m_stringConstants[number] }; } - bool hasListedHash(StringID string) const { return m_strings[string.index()].hasListedHash(); } + bool hasListedHash(StringID string) const { return wordOf(string).hasListedHash(); } private: static bool hasHeader(ImageStringWord word) { return !word.hasText() || word.hasHeader(); } @@ -290,11 +319,21 @@ class ImageText { static constexpr size_t lazyHeadersAlignment = 2 * MB; #endif String createStringWithoutHeader(ImageStringWord) const; + String createStringOfCodedText(ImageStringWord) const; + JS_EXPORT_PRIVATE uint32_t textOfListedIdentifier(uint32_t number) const; + StringView viewOfCodedText(ImageStringWord) const; + + struct DecodedTexts; + const uint32_t m_endOfIdentifiers; std::span m_strings; + std::span m_codedTexts; + uint32_t m_numberOfCodedTexts { 0 }; + const std::unique_ptr m_decodedTexts; std::span m_listedStrings; std::span m_text; - std::span m_identifiers; + std::span m_identifiers; + std::span m_otherIdentifiers; std::span m_stringConstants; #if ENABLE(AOT_POSITION_INDEPENDENT_IMAGE) std::span m_stringRecords; diff --git a/Source/JavaScriptCore/aot/image/AOTLayoutTables.cpp b/Source/JavaScriptCore/aot/image/AOTLayoutTables.cpp index 305643812e2bd..79c5d1920acb3 100644 --- a/Source/JavaScriptCore/aot/image/AOTLayoutTables.cpp +++ b/Source/JavaScriptCore/aot/image/AOTLayoutTables.cpp @@ -127,7 +127,7 @@ static void forEachLayoutRecord(const TableInputs& inputs, NOESCAPE const Invoca { const Typed::ShapeTable& shapes = inputs.plan.shapes(); for (uint32_t shape = 1; shape <= shapes.numberOfShapes(); ++shape) { - if (shapes.isAssigned(shape) && inputs.plan.isInUse(shapes.shape(shape).layout)) + if (shapes.isAssigned(shape) && inputs.plan.isAllocatable(shapes.shape(shape).layout)) functor(shapes.shape(shape).layout, shape); } for (size_t index = 0; index < inputs.program.layouts.size(); ++index) { @@ -529,7 +529,7 @@ static bool classOfLayoutRecordIsEvaluatedOnce(const TableInputs& inputs, Layout static uint32_t findShapeThatParsersMake(const TableEncoder& encoder, LayoutID id, std::span fields) { const Typed::ShapeTable& shapes = encoder.plan.shapes(); - if (encoder.program.at(id).kind != LayoutKind::Record || !encoder.plan.isInUse(id) || !encoder.plan.layout(id).holdsCompiledObjects() || !encoder.facts.parsersMayMake(id)) + if (encoder.program.at(id).kind != LayoutKind::Record || !encoder.plan.isAllocatable(id) || !encoder.plan.layout(id).holdsCompiledObjects() || !encoder.facts.parsersMayMake(id)) return 0; uint32_t shape = shapes.generalShapeOf(id); if (!shapes.isAssigned(shape) || !shapes.shape(shape).impossibleMutations.isEmpty()) @@ -733,7 +733,7 @@ static void encodeLayoutRecords(TableEncoder& encoder, std::span(encoder.numbers.allocations().size()) * (sizeof(AllocationData) + sizeof(ConstructedAllocationData)); for (size_t index = 0; index < records.size(); ++index) { if (records[index].shapeThatParsersMake) - mostBytesOfTables += encoder.firstParts->fieldsOfLayoutRecords[index].fields.size() * sizeof(ParsedRecordFieldData); + mostBytesOfTables += encoder.firstParts->fieldsOfLayoutRecords[index].fields.size() * (sizeof(Number24) + sizeof(uint32_t)); } forEachLayoutRecord(encoder.inputs, [&](LayoutID id, uint32_t) { const LayoutRecordParts& parts = records[id.index()]; @@ -828,7 +828,13 @@ static AllocationRows collectAllocationRows(const TableEncoder& encoder) return rows; } -static Vector collectParsedRecordFields(const TableEncoder& encoder, std::span records) +struct ParsedRecordTables { + Vector names; + Vector starts; + Vector shapes; +}; + +static ParsedRecordTables collectParsedRecords(const TableEncoder& encoder, std::span records) { Vector keys; for (size_t index = 0; index < records.size(); ++index) { @@ -842,9 +848,24 @@ static Vector collectParsedRecordFields(const TableEncode radixSortKeepingEqualKeysInOrder<64>(keys, [](uint64_t key) { return key; }); - return Vector(keys.size(), [&](size_t index) { - return ParsedRecordFieldData { Number24(static_cast(keys[index] >> 40)), Number24(static_cast(keys[index]) & Number24::max) }; - }); + if (keys.isEmpty()) + return { }; + Vector placeOfName(FillWith { }, encoder.numbers.identifiers().count(), 0); + Vector starts; + ParsedRecordTables tables; + tables.shapes.reserveInitialCapacity(keys.size()); + for (uint64_t key : keys) { + uint32_t& place = placeOfName[key >> 40]; + if (!place) { + starts.append(tables.shapes.size()); + place = starts.size(); + } + tables.shapes.append(Number24(static_cast(key) & Number24::max)); + } + starts.append(tables.shapes.size()); + tables.names = SparseNumberTable::create(placeOfName.span()); + tables.starts = AscendingNumberTable::create(starts.span()); + return tables; } static Vector collectConstructorsOfLayouts(const TableEncoder& encoder) @@ -853,7 +874,7 @@ static Vector collectConstructorsOfLayouts(const TableEncoder& for (size_t index = 0; index < encoder.program.layouts.size(); ++index) { LayoutID id(index); FunctionID constructor = encoder.program.at(id).constructor; - if (!constructor || !(encoder.plan.isInUse(id) || hasRecordOfClass(encoder.inputs, id))) + if (!constructor || !(encoder.plan.isAllocatable(id) || hasRecordOfClass(encoder.inputs, id))) continue; if (std::optional entrypoint = encoder.findEntrypoint(constructor)) constructors.append({ entrypoint->index(), static_cast(index) }); @@ -896,7 +917,7 @@ void encodeLayoutTables(TableEncoder& encoder) times.didFinish("parts of records"_s); if (encoder.build.diagnostics().hasErrors()) return; - Vector parsedRecordFields; + ParsedRecordTables parsedRecords; AllocationRows allocationRows; Vector constructors; runLanes(encoder.build.parallelLoops(), { @@ -906,8 +927,8 @@ void encodeLayoutTables(TableEncoder& encoder) }, [&] { StepTimes timesOfLane(encoder.build, nameOfSteps); - parsedRecordFields = collectParsedRecordFields(encoder, records.span()); - timesOfLane.didFinish("fields of parsed records"_s); + parsedRecords = collectParsedRecords(encoder, records.span()); + timesOfLane.didFinish("parsed records"_s); }, [&] { StepTimes timesOfLane(encoder.build, nameOfSteps); @@ -918,7 +939,9 @@ void encodeLayoutTables(TableEncoder& encoder) }, }); destroyInBackground(encoder.build.backgroundJobs(), WTF::move(records)); - encoder.part.setTable(ProgramTableIndex::parsedRecordFields, parsedRecordFields); + encoder.part.setTable(ProgramTableIndex::parsedRecordNames, parsedRecords.names); + encoder.part.setTable(ProgramTableIndex::parsedRecordStarts, parsedRecords.starts); + encoder.part.setTable(ProgramTableIndex::parsedRecordShapes, parsedRecords.shapes); encoder.part.setTable(ProgramTableIndex::allocations, allocationRows.allocations); encoder.part.setTable(ProgramTableIndex::constructedAllocations, allocationRows.constructedAllocations); encoder.part.setTable(ProgramTableIndex::constructors, constructors); diff --git a/Source/JavaScriptCore/aot/image/AOTModuleLists.cpp b/Source/JavaScriptCore/aot/image/AOTModuleLists.cpp index ced7a767f0c81..e934151626736 100644 --- a/Source/JavaScriptCore/aot/image/AOTModuleLists.cpp +++ b/Source/JavaScriptCore/aot/image/AOTModuleLists.cpp @@ -200,8 +200,10 @@ struct NumbersOfImageForModuleLists { struct IDsNotedForModuleLists { void addTo(NamesOfTables& names) const { - for (uint32_t id : strings) + for (uint32_t id : strings) { names.strings.set(id); + names.modulePathStrings.set(id); + } for (uint32_t id : textsNamedAsKeys) names.textsNamedAsKeys.set(id); for (uint32_t id : variables) diff --git a/Source/JavaScriptCore/aot/image/AOTModuleTables.cpp b/Source/JavaScriptCore/aot/image/AOTModuleTables.cpp index 15e55f43704d6..2246f3c797ad1 100644 --- a/Source/JavaScriptCore/aot/image/AOTModuleTables.cpp +++ b/Source/JavaScriptCore/aot/image/AOTModuleTables.cpp @@ -173,6 +173,7 @@ void nameWhatModuleTablesName(const TableInputs& inputs, const GatheredMentions& auto noteString = [&](StringID id) -> uint32_t { names.strings.set(id.index()); + names.modulePathStrings.set(id.index()); return 0; }; nameWhatModuleListsName(inputs, answers, loops, names); @@ -434,7 +435,7 @@ static void listStateOfLibraryModulesInClasses(const TableEncoder& encoder, size { const Typed::Program& program = encoder.program; for (size_t index = first; index < end; ++index) { - FunctionID constructor = encoder.plan.isInUse(LayoutID(static_cast(index))) ? program.layouts[index].constructor : FunctionID(); + FunctionID constructor = encoder.plan.isAllocatable(LayoutID(static_cast(index))) ? program.layouts[index].constructor : FunctionID(); ModuleID module = constructor ? program.at(constructor).module : ModuleID(); if (isLibraryModuleWithBodyForModuleTables(program, module)) found.append(StateOfLibraryModuleData::create(module.index(), StateOfLibraryModuleData::Kind::Classes, static_cast(index), 1)); diff --git a/Source/JavaScriptCore/aot/image/AOTNumbering.cpp b/Source/JavaScriptCore/aot/image/AOTNumbering.cpp index ace6b9447d1ba..76b97e93e9640 100644 --- a/Source/JavaScriptCore/aot/image/AOTNumbering.cpp +++ b/Source/JavaScriptCore/aot/image/AOTNumbering.cpp @@ -258,9 +258,7 @@ ALWAYS_INLINE_INTO_OTHER_FILES std::optional ImageNumbers::numberOfInL return m_identifiers->numberOfText(StringID(key[0])); case NumberSpace::String: case NumberSpace::ShownString: - if (!m_namedStrings.get(key[0])) - return std::nullopt; - return key[0]; + return m_stringNumbers.numberOf(StringID(key[0])); case NumberSpace::StringCell: return m_stringConstants->numberOf(StringID(key[0])); case NumberSpace::AtomOfString: @@ -409,8 +407,7 @@ ALWAYS_INLINE_INTO_OTHER_FILES std::optional ImageNumbers::valueOf(con break; case NumberSpace::String: case NumberSpace::ShownString: - if (m_namedStrings.get(id)) - number = id; + number = m_stringNumbers.numberOf(StringID(id)); break; case NumberSpace::StringCell: number = m_stringConstants->numberOf(StringID(id)); @@ -449,6 +446,7 @@ NamesOfTables::NamesOfTables(const Typed::Program& program) , textsNamedAsKeys(program.strings.size()) , strings(program.strings.size()) , stringsWithOtherUsesThanShowing(program.strings.size()) + , modulePathStrings(program.strings.size()) , stringConstants(program.strings.size()) , bigInts(program.strings.size()) , variables(program.variables.size()) @@ -462,6 +460,7 @@ void NamesOfTables::merge(const NamesOfTables& other) textsNamedAsKeys.merge(other.textsNamedAsKeys); strings.merge(other.strings); stringsWithOtherUsesThanShowing.merge(other.stringsWithOtherUsesThanShowing); + modulePathStrings.merge(other.modulePathStrings); stringConstants.merge(other.stringConstants); bigInts.merge(other.bigInts); allocations.appendVector(other.allocations); @@ -470,6 +469,28 @@ void NamesOfTables::merge(const NamesOfTables& other) functionsWithClosureCell.merge(other.functionsWithClosureCell); } +StringNumbers::StringNumbers(const BitVector& named, Vector&& sortedCodedStrings, size_t numberOfStrings, const IdentifierNumbering* identifiers) + : m_numbers(FillWith { }, numberOfStrings, noNumber) + , m_numberOfWords(identifiers ? identifiers->count() : 0) + , m_coded(WTF::move(sortedCodedStrings)) +{ + for (uint32_t number = 1; number < m_numberOfWords; ++number) { + StringID text = identifiers->at(number).text; + if (text && identifiers->numberOfText(text) == number) + m_numbers[text.index()] = number; + } + BitVector isCoded(numberOfStrings); + for (StringID id : m_coded) + isCoded.quickSet(id.index()); + for (size_t index : named) { + if (m_numbers[index] == noNumber && !isCoded.quickGet(index)) + m_numbers[index] = m_numberOfWords++; + } + uint32_t number = m_numberOfWords; + for (StringID id : m_coded) + m_numbers[id.index()] = number++; +} + static uint64_t countThingsNumbered(const ImageNumbers& numbers) { const PlacedStatics& statics = numbers.statics(); diff --git a/Source/JavaScriptCore/aot/image/AOTNumbering.h b/Source/JavaScriptCore/aot/image/AOTNumbering.h index 78f9996c421d5..23cc2897b6311 100644 --- a/Source/JavaScriptCore/aot/image/AOTNumbering.h +++ b/Source/JavaScriptCore/aot/image/AOTNumbering.h @@ -462,6 +462,34 @@ struct NumberedKeys { Vector hashSlots; }; +class StringNumbers { +public: + StringNumbers() = default; + JS_EXPORT_PRIVATE StringNumbers(const BitVector& named, Vector&& sortedCodedStrings, size_t numberOfStrings, const IdentifierNumbering* = nullptr); + + uint32_t numberOfWords() const { return m_numberOfWords; } + std::span coded() const { return m_coded.span(); } + std::optional numberOf(StringID id) const + { + if (id.index() >= m_numbers.size() || m_numbers[id.index()] == noNumber) + return std::nullopt; + return m_numbers[id.index()]; + } + uint32_t requiredNumberOf(StringID id) const + { + std::optional number = numberOf(id); + RELEASE_ASSERT(number); + return *number; + } + +private: + static constexpr uint32_t noNumber = UINT32_MAX; + + Vector m_numbers; + uint32_t m_numberOfWords { 0 }; + Vector m_coded; +}; + class ImageNumbers { WTF_MAKE_NONCOPYABLE(ImageNumbers); public: @@ -479,6 +507,7 @@ class ImageNumbers { InstanceSizes instanceSizes() const { return m_instanceSizes; } const BitVector& namedStrings() const { return m_namedStrings; } + const StringNumbers& stringNumbers() const { return m_stringNumbers; } const BitVector& textsOfNames() const { return m_textsOfNames; } const BitVector& otherStringsWithText() const { return m_otherStringsWithText; } std::span stringsInOrderOfText() const { return m_stringsInOrderOfText.span(); } @@ -551,6 +580,7 @@ class ImageNumbers { PlacedStatics m_statics; InstanceSizes m_instanceSizes { }; BitVector m_namedStrings; + StringNumbers m_stringNumbers; BitVector m_textsOfNames; BitVector m_otherStringsWithText; Vector m_stringsInOrderOfText; @@ -682,6 +712,7 @@ struct NamesOfTables { BitVector textsNamedAsKeys; BitVector strings; BitVector stringsWithOtherUsesThanShowing; + BitVector modulePathStrings; BitVector stringConstants; BitVector bigInts; Vector allocations; diff --git a/Source/JavaScriptCore/aot/image/AOTPositionEncoder.h b/Source/JavaScriptCore/aot/image/AOTPositionEncoder.h new file mode 100644 index 0000000000000..9fe2f0c5db71b --- /dev/null +++ b/Source/JavaScriptCore/aot/image/AOTPositionEncoder.h @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2026 Oven, Inc. All rights reserved. + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#if ENABLE(AOT) + +#include "AOTCallSiteTable.h" +#include "AOTImageFormat.h" +#include "AOTPrefixCode.h" +#include "AOTVector.h" + +namespace JSC { namespace AOT { + +using PositionCodes = PrefixCodes; + +class PositionEncoder { +public: + void encode(const SiteDescription&, OwnPosition ofFunctionInFront); + const PositionCodes::Counts& counts() const { return m_counts; } + void appendNextCoded(Vector& stream, const PositionCodes&); + +private: + Vector m_heads; + Vector m_symbols; + Vector m_roomForSymbols; + Vector m_roomForBytes; + PositionCodes::Counts m_counts { }; + size_t m_nextHead { 0 }; + size_t m_nextSymbol { 0 }; +}; + +} } // namespace JSC::AOT + +#endif // ENABLE(AOT) diff --git a/Source/JavaScriptCore/aot/image/AOTPrefixCode.cpp b/Source/JavaScriptCore/aot/image/AOTPrefixCode.cpp new file mode 100644 index 0000000000000..ce98b34fa11ab --- /dev/null +++ b/Source/JavaScriptCore/aot/image/AOTPrefixCode.cpp @@ -0,0 +1,189 @@ +/* + * Copyright (C) 2026 Oven, Inc. All rights reserved. + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "config.h" +#include "AOTPrefixCode.h" + +#if ENABLE(AOT) + +#include +#include +#include +#include + +WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN + +namespace JSC { namespace AOT { + +PrefixCode::Lengths PrefixCode::findLengths(const Counts& counts) +{ + struct Leaf { + uint32_t count; + uint16_t symbol; + }; + + Vector leaves; + leaves.append({ 0, endOfStream }); + for (unsigned byte = 0; byte < endOfStream; ++byte) { + if (counts[byte]) + leaves.append({ counts[byte], static_cast(byte) }); + } + Lengths lengths { }; + size_t numberOfLeaves = leaves.size(); + if (numberOfLeaves == 1) + return lengths; + + for (;;) { + std::sort(leaves.begin() + 1, leaves.end(), [](const Leaf& a, const Leaf& b) { + return a.count != b.count ? a.count < b.count : a.symbol < b.symbol; + }); + size_t numberOfNodes = 2 * numberOfLeaves - 1; + std::array weights; + std::array parents; + for (size_t leaf = 0; leaf < numberOfLeaves; ++leaf) + weights[leaf] = leaves[leaf].count; + size_t nextLeaf = 0; + size_t nextInner = numberOfLeaves; + for (size_t node = numberOfLeaves; node < numberOfNodes; ++node) { + auto takeLightest = [&]() -> size_t { + bool takesLeaf = nextLeaf < numberOfLeaves && (nextInner == node || weights[nextLeaf] <= weights[nextInner]); + size_t lightest = takesLeaf ? nextLeaf++ : nextInner++; + parents[lightest] = static_cast(node); + return lightest; + }; + size_t first = takeLightest(); + size_t second = takeLightest(); + weights[node] = weights[first] + weights[second]; + } + std::array depths; + depths[numberOfNodes - 1] = 0; + for (size_t node = numberOfNodes - 1; node--;) + depths[node] = depths[parents[node]] + 1; + if (depths[0] <= maxLength) { + for (size_t leaf = 0; leaf < numberOfLeaves; ++leaf) + lengths[leaves[leaf].symbol] = static_cast(depths[leaf]); + return lengths; + } + for (Leaf& leaf : leaves.mutableSpan().subspan(1)) + leaf.count -= leaf.count / 2; + } +} + +PrefixCode::PrefixCode(const Counts& counts) +{ + Lengths lengths = findLengths(counts); + unsigned word = 0; + for (unsigned length = 1; length <= maxLength; ++length) { + word <<= 1; + unsigned numberOfWords = 0; + for (unsigned byte = 0; byte < endOfStream; ++byte) { + if (lengths[byte] != length) + continue; + m_words[byte] = { static_cast(word++), static_cast(length) }; + m_bytes.append(static_cast(byte)); + ++numberOfWords; + } + RELEASE_ASSERT(numberOfWords <= std::numeric_limits::max()); + m_wordsOfLength[length - 1] = static_cast(numberOfWords); + } +} + +unsigned PrefixCode::bitsOfShortWords(unsigned most) const +{ + unsigned longest = 1; + for (const Word& word : m_words) + longest = std::max(longest, word.length); + return std::min(longest, most); +} + +void PrefixCode::append(Vector& bytes, unsigned bitsOfShortWords) const +{ + Vector shortWords(FillWith { }, size_t { 1 } << bitsOfShortWords, 0); + for (unsigned byte = 0; byte < endOfStream; ++byte) { + auto [bits, length] = m_words[byte]; + if (!length || length > bitsOfShortWords) + continue; + unsigned bitsBehind = bitsOfShortWords - length; + std::fill_n(shortWords.mutableSpan().data() + (bits << bitsBehind), 1u << bitsBehind, static_cast(length << 8 | byte)); + } + bytes.append(asBytes(shortWords.span())); + bytes.append(std::span(m_wordsOfLength)); + bytes.appendVector(m_bytes); +} + +namespace { + +constexpr uint32_t bytesAreNotCoded = 1u << 31; +constexpr unsigned numberOfVarintCodes = 16; +using VarintCodes = PrefixCodes; + +constexpr unsigned codeOfVarintByte(uint8_t byteInFront, uint8_t secondByteInFront) +{ + return (byteInFront >> 7) << 3 | (secondByteInFront >> 7) << 2 | (byteInFront & 3); +} + +} // anonymous namespace + +Vector encodeVarintBytes(std::span bytes) +{ + if (bytes.empty()) + return { }; + Vector symbols; + symbols.grow(bytes.size()); + PrefixSymbolWriter writer(symbols.mutableSpan().data()); + uint8_t byteInFront = 0; + uint8_t secondByteInFront = 0; + for (uint8_t byte : bytes) { + writer.append(codeOfVarintByte(byteInFront, secondByteInFront), byte); + secondByteInFront = std::exchange(byteInFront, byte); + } + VarintCodes::Counts counts { }; + VarintCodes::count(counts, symbols.span()); + VarintCodes codes(counts); + + RELEASE_ASSERT(bytes.size() < bytesAreNotCoded); + uint32_t size = static_cast(bytes.size()); + Vector result; + result.append(asByteSpan(size)); + result.appendVector(codes.bytes()); + size_t sizeInFront = result.size(); + result.grow(sizeInFront + symbols.size() * PrefixCode::maxLength / 8 + 1); + result.shrink(codes.write(result.mutableSpan().data() + sizeInFront, symbols.span()) - result.mutableSpan().data()); + if (result.size() < bytes.size() + sizeof(size)) + return result; + size |= bytesAreNotCoded; + result.shrink(0); + result.append(asByteSpan(size)); + result.append(bytes); + return result; +} + +Vector decodeVarintBytes(std::span coded) +{ + if (coded.empty()) + return { }; + uint32_t size; + memcpy(&size, coded.data(), sizeof(size)); + coded = coded.subspan(sizeof(size)); + if (size & bytesAreNotCoded) + return Vector(coded.first(size & ~bytesAreNotCoded)); + PrefixCodeReader reader(VarintCodes::endOfCodes(coded.data()), coded.data() + coded.size()); + Vector bytes; + bytes.reserveInitialCapacity(size); + uint8_t byteInFront = 0; + uint8_t secondByteInFront = 0; + for (uint32_t index = 0; index < size; ++index) { + unsigned byte = reader.read(VarintCodes::code(coded.data(), codeOfVarintByte(byteInFront, secondByteInFront))); + secondByteInFront = std::exchange(byteInFront, static_cast(byte)); + bytes.append(byteInFront); + } + return bytes; +} + +} } // namespace JSC::AOT + +WTF_ALLOW_UNSAFE_BUFFER_USAGE_END + +#endif // ENABLE(AOT) diff --git a/Source/JavaScriptCore/aot/image/AOTPrefixCode.h b/Source/JavaScriptCore/aot/image/AOTPrefixCode.h new file mode 100644 index 0000000000000..8ca4dac82af7c --- /dev/null +++ b/Source/JavaScriptCore/aot/image/AOTPrefixCode.h @@ -0,0 +1,310 @@ +/* + * Copyright (C) 2026 Oven, Inc. All rights reserved. + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#if ENABLE(AOT) + +#include "AOTVector.h" +#include "JSExportMacros.h" +#include +#include +#include +#include +#include +#include +#include +#include + +WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN + +namespace JSC { namespace AOT { + +class PrefixCode { +public: + static constexpr unsigned maxLength = 15; + static constexpr unsigned endOfStream = 256; + static constexpr unsigned bitsOfShortWord = 7; + static constexpr size_t sizeOfShortWords(unsigned bits) { return sizeof(uint16_t) << bits; } + static constexpr size_t maxSize(unsigned bits) { return sizeOfShortWords(bits) + maxLength + endOfStream; } + using Counts = std::array; + + struct Word { + uint16_t bits { 0 }; + uint8_t length { 0 }; + }; + + explicit PrefixCode(const Counts&); + + const Word& word(uint8_t byte) const { return m_words[byte]; } + unsigned bitsOfShortWords(unsigned most) const; + void append(Vector&, unsigned bitsOfShortWords) const; + +private: + using Lengths = std::array; + + static Lengths findLengths(const Counts&); + + std::array m_words { }; + std::array m_wordsOfLength { }; + Vector m_bytes; +}; + +class PrefixCodeWriter { +public: + explicit PrefixCodeWriter(uint8_t* firstReservedByte) + : m_end(firstReservedByte) + { + } + + ALWAYS_INLINE void append(PrefixCode::Word word) + { + ASSERT(word.length); + m_bits = m_bits << word.length | word.bits; + m_numberOfBits += word.length; + if (m_numberOfBits < 32) + return; + m_numberOfBits -= 32; + uint32_t bytes = flipBytes(static_cast(m_bits >> m_numberOfBits)); + memcpy(m_end, &bytes, sizeof(bytes)); + m_end += sizeof(bytes); + } + + ALWAYS_INLINE uint8_t* finish() + { + if (unsigned numberOfOnes = -m_numberOfBits % 8) + append({ static_cast((1u << numberOfOnes) - 1), static_cast(numberOfOnes) }); + while (m_numberOfBits) { + m_numberOfBits -= 8; + *m_end++ = static_cast(m_bits >> m_numberOfBits); + } + return m_end; + } + +private: + uint8_t* m_end; + uint64_t m_bits { 0 }; + unsigned m_numberOfBits { 0 }; +}; + +using PrefixSymbol = uint16_t; + +class PrefixSymbolWriter { +public: + explicit PrefixSymbolWriter(PrefixSymbol* firstReservedSymbol) + : m_end(firstReservedSymbol) + { + } + + PrefixSymbol* end() const { return m_end; } + void append(std::span symbols) + { + if (!symbols.empty()) + memcpy(m_end, symbols.data(), symbols.size_bytes()); + m_end += symbols.size(); + } + void append(unsigned code, uint8_t byte) { *m_end++ = static_cast(code << 8 | byte); } + void appendVarint(unsigned codeOfFirst, unsigned codeOfOthers, uint64_t value) + { + unsigned code = codeOfFirst; + for (; value >= 0x80; value >>= 7) { + append(code, static_cast(value) | 0x80); + code = codeOfOthers; + } + append(code, static_cast(value)); + } + +private: + PrefixSymbol* m_end; +}; + +struct ShortWordsOfOneSize { + static constexpr unsigned bits(unsigned) { return PrefixCode::bitsOfShortWord; } +}; + +struct PrefixCodeView { + static constexpr unsigned bitsOfShift = 6; + + PrefixCodeView(const uint8_t* codes, uint32_t place) + : shortWords(codes + (place >> bitsOfShift)) + , shift(place & ((1u << bitsOfShift) - 1)) + { + } + + const uint8_t* endOfShortWords() const { return shortWords + (sizeof(uint16_t) << (64 - shift)); } + + const uint8_t* shortWords; + unsigned shift; +}; + +template +class PrefixCodes { + static constexpr size_t maxSizeOfCodes() + { + size_t size = 0; + for (unsigned code = 0; code < numberOfCodes; ++code) + size += PrefixCode::maxSize(ShortWords::bits(code)); + return size; + } + +public: + using Counts = std::array; + static constexpr size_t sizeOfPlaces = (numberOfCodes + 1) * sizeof(uint32_t); + static constexpr size_t maxSize = sizeOfPlaces + maxSizeOfCodes(); + static_assert(maxSize < (1u << (32 - PrefixCodeView::bitsOfShift))); + + explicit PrefixCodes(const Counts& counts) + { + m_bytes.grow(sizeOfPlaces); + for (unsigned number = 0; number < numberOfCodes; ++number) { + unsigned first = number * PrefixCode::endOfStream; + PrefixCode::Counts countsOfCode; + std::copy_n(counts.begin() + first, countsOfCode.size(), countsOfCode.begin()); + PrefixCode code(countsOfCode); + for (unsigned byte = 0; byte < PrefixCode::endOfStream; ++byte) + m_words[first + byte] = code.word(static_cast(byte)); + unsigned bitsOfShortWords = code.bitsOfShortWords(ShortWords::bits(number)); + writePlace(number, bitsOfShortWords); + code.append(m_bytes, bitsOfShortWords); + } + writePlace(numberOfCodes, 1); + } + + static void count(Counts& counts, std::span symbols) + { + uint32_t* countOf = counts.data(); + for (PrefixSymbol symbol : symbols) + ++countOf[symbol]; + } + static void add(Counts& sums, const Counts& counts) + { + for (size_t symbol = 0; symbol < counts.size(); ++symbol) + sums[symbol] += counts[symbol]; + } + + uint8_t* write(uint8_t* firstReservedByte, std::span symbols, std::span laterSymbols = { }) const + { + PrefixCodeWriter writer(firstReservedByte); + const PrefixCode::Word* words = m_words.data(); + for (PrefixSymbol symbol : symbols) + writer.append(words[symbol]); + for (PrefixSymbol symbol : laterSymbols) + writer.append(words[symbol]); + return writer.finish(); + } + + const Vector& bytes() const { return m_bytes; } + + static PrefixCodeView code(const uint8_t* bytes, unsigned number) + { + uint32_t place; + memcpy(&place, bytes + number * sizeof(place), sizeof(place)); + return { bytes, place }; + } + static const uint8_t* endOfCodes(const uint8_t* bytes) { return code(bytes, numberOfCodes).shortWords; } + +private: + void writePlace(unsigned number, unsigned bitsOfShortWords) + { + uint32_t place = static_cast(m_bytes.size()) << PrefixCodeView::bitsOfShift | (64 - bitsOfShortWords); + memcpy(m_bytes.mutableSpan().data() + number * sizeof(place), &place, sizeof(place)); + } + + std::array m_words; + Vector m_bytes; +}; + +Vector encodeVarintBytes(std::span); +JS_EXPORT_PRIVATE Vector decodeVarintBytes(std::span); + +class PrefixCodeReader { +public: + PrefixCodeReader(const uint8_t* begin, const uint8_t* end) + : m_cursor(begin) + , m_end(end) + { + } + + ALWAYS_INLINE unsigned read(PrefixCodeView code) + { + if (m_numberOfBits < PrefixCode::maxLength) + fill(); + uint16_t shortWord; + memcpy(&shortWord, code.shortWords + sizeof(shortWord) * (m_bits >> code.shift), sizeof(shortWord)); + unsigned length = shortWord >> 8; + unsigned byte = shortWord & 0xff; + if (!length) [[unlikely]] { + LongWord word = findLongWord(code.endOfShortWords(), static_cast(m_bits >> (64 - PrefixCode::maxLength))); + length = word.length; + byte = word.byte; + } + m_bits <<= length; + m_numberOfBits -= length; + return byte; + } + + const uint8_t* alignToByte() + { + unsigned numberOfBitsDropped = m_numberOfBits % 8; + m_bits <<= numberOfBitsDropped; + m_numberOfBits -= numberOfBitsDropped; + return m_cursor - m_numberOfBits / 8; + } + +private: + struct LongWord { + unsigned length; + unsigned byte; + }; + + ALWAYS_INLINE void fill() + { + uint32_t next; + if (m_end - m_cursor >= static_cast(sizeof(next))) [[likely]] { + memcpy(&next, m_cursor, sizeof(next)); + next = flipBytes(next); + m_cursor += sizeof(next); + } else { + next = lastBytes(m_cursor, m_end); + m_cursor = m_end; + } + m_bits |= static_cast(next) << (32 - m_numberOfBits); + m_numberOfBits += 32; + } + + NEVER_INLINE static uint32_t lastBytes(const uint8_t* cursor, const uint8_t* end) + { + uint32_t next = UINT32_MAX; + for (unsigned shift = 24; cursor < end; shift -= 8) + next ^= static_cast(static_cast(~*cursor++)) << shift; + return next; + } + + NEVER_INLINE static LongWord findLongWord(const uint8_t* numbersOfWords, unsigned nextBits) + { + const uint8_t* bytes = numbersOfWords + PrefixCode::maxLength; + unsigned firstWord = 0; + for (unsigned length = 1; length <= PrefixCode::maxLength; ++length) { + unsigned word = nextBits >> (PrefixCode::maxLength - length); + unsigned number = numbersOfWords[length - 1]; + if (word - firstWord < number) + return { length, bytes[word - firstWord] }; + bytes += number; + firstWord = (firstWord + number) << 1; + } + return { 0, PrefixCode::endOfStream }; + } + + const uint8_t* m_cursor; + const uint8_t* m_end; + uint64_t m_bits { 0 }; + unsigned m_numberOfBits { 0 }; +}; + +} } // namespace JSC::AOT + +WTF_ALLOW_UNSAFE_BUFFER_USAGE_END + +#endif // ENABLE(AOT) diff --git a/Source/JavaScriptCore/aot/image/AOTProgramTableCheck.cpp b/Source/JavaScriptCore/aot/image/AOTProgramTableCheck.cpp index 80a30227e1d0f..c70ebb116a86c 100644 --- a/Source/JavaScriptCore/aot/image/AOTProgramTableCheck.cpp +++ b/Source/JavaScriptCore/aot/image/AOTProgramTableCheck.cpp @@ -17,6 +17,7 @@ #include "AOTObjectVisiting.h" #include "AOTObjectWords.h" #include "AOTPlan.h" +#include "AOTPrefixCode.h" #include "AOTPropertyCachesFront.h" #include "AOTRuntimeType.h" #include "AOTStringUses.h" @@ -467,7 +468,6 @@ static void checkOrderOfProgramTables(const JoinedTables& check) checkSorted(tables.addedPrototypeNames(), std::identity { }, "addedPrototypeNames"_s); checkSorted(tables.namesOfStatements(), &NameOfStatementData::module, "namesOfStatements"_s); checkSorted(tables.requestedOriginals(), &RequestedOriginalData::function, "requestedOriginals"_s); - checkSorted(tables.parsedRecordFields(), &ParsedRecordFieldData::nameNumber, "parsedRecordFields"_s); checkSorted(tables.quotedSourceGroups(), [](const QuotedSourceGroupData& group) { return std::tuple { group.module, group.line, group.column }; }, "quotedSourceGroups"_s); checkSorted(tables.sourceStamps(), &SourceStampData::module, "sourceStamps"_s); checkSorted(tables.starsAsWritten(), &StarAsWrittenData::module, "starsAsWritten"_s); @@ -569,14 +569,14 @@ static void checkFunctionTables(const JoinedTables& check) const Typed::CallingConventions& conventions = check.plan.conventions(); const FunctionNumbers& functions = check.numbers.functions(); size_t numberOfClosureFunctions = tables.closureFunctions.end - tables.closureFunctions.first; - std::span cellNumbers = tables.closureCellNumbers(); + SparseNumberTable cells(tables.closureCells()); bool hasRowForEachFunction = tables.numberOfFunctions() == functions.numberOfFunctions() && tables.closureFunctionKinds().size() == numberOfClosureFunctions && tables.functionLengths().size() == numberOfClosureFunctions - && tables.closureCaptureCounts().size() == numberOfClosureFunctions && cellNumbers.size() == numberOfClosureFunctions; + && tables.closureCaptureCounts().size() == numberOfClosureFunctions && tables.closureCells().size() >= NumberGroup::numberOfGroupWords(numberOfClosureFunctions); if (!check.expect(hasRowForEachFunction, "TableOutOfBounds"_s, { }, "functionRecordNumbers"_s)) return; for (uint32_t number = 0; number < numberOfClosureFunctions; ++number) { CodeUnit unit = conventions.unitAt(functions.unitOf(FunctionID(tables.closureFunctionWithNumber(number)))); - uint32_t cell = cellNumbers[number]; + uint32_t cell = cells.at(number); bool isCellOfFunction = unit.role != Typed::CodeRole::GenericCallEntry || cell == check.numbers.closureCellNumber(unit.function); if (!check.expect(isCellOfFunction && (!cell || cell < tables.numberOfClosureCells), "ClosureCellDiffersFromNumbers"_s, unit.function)) return; @@ -614,7 +614,7 @@ static void checkNamesForHeapSnapshots(const JoinedTables& check) Vector variables; uint32_t lastModule = 0; uint32_t endOfClosureNumbers = 0; - bool endsWhereModuleEnds = forEachNamedVariable(tables.namedVariables(), [&](uint32_t module, size_t) { + bool endsWhereModuleEnds = forEachNamedVariable(decodeVarintBytes(tables.namedVariables()).span(), [&](uint32_t module, size_t) { lastModule = module; }, [&](const VariableData& variable) { variables.append(variable); @@ -634,7 +634,7 @@ static void checkNamesForHeapSnapshots(const JoinedTables& check) }), "TableOutOfBounds"_s, { }, "namedVariables"_s); Vector captures; - bool endsWhereClosureNumberEnds = forEachNamedCapture(tables.namedCaptures(), [&](const NamedCaptureData& capture) { + bool endsWhereClosureNumberEnds = forEachNamedCapture(decodeVarintBytes(tables.namedCaptures()).span(), [&](const NamedCaptureData& capture) { captures.append(capture); }); check.expect(endsWhereClosureNumberEnds && ascendsStrictlyInTable(captures.span(), [](const NamedCaptureData& capture) { return capture.closureNumber << 8 | capture.slot; }), "TableNotSorted"_s, { }, "namedCaptures"_s); @@ -662,7 +662,7 @@ static void checkListedFunctions(const JoinedTables& check) } for (size_t index = 0; index < check.program.layouts.size(); ++index) { FunctionID constructor = check.program.layouts[index].constructor; - if (!constructor || !check.plan.isInUse(LayoutID(index))) + if (!constructor || !check.plan.isAllocatable(LayoutID(index))) continue; std::optional entry = check.encoder.findEntrypoint(constructor); check.expect(!entry || std::ranges::binary_search(tables.constructors(), entry->index(), { }, &ConstructorData::function), "FunctionNotListed"_s, LayoutID(index), "constructors"_s); @@ -839,7 +839,7 @@ static void checkLayoutRecords(const JoinedTables& check) for (uint32_t shape = 0; shape < offsets.size(); ++shape) { uint32_t offset = offsets[shape]; LayoutID id = shapes.isAssigned(shape) ? shapes.shape(shape).layout : LayoutID(); - if (id && !check.plan.isInUse(id)) + if (id && !check.plan.isAllocatable(id)) id = { }; if (!check.expect(!offset == !id, "ShapeWithoutLayout"_s, id) || !id) continue; diff --git a/Source/JavaScriptCore/aot/image/AOTRegExpCompiler.h b/Source/JavaScriptCore/aot/image/AOTRegExpCompiler.h index 8c098d58a6828..7b77c6ed34a26 100644 --- a/Source/JavaScriptCore/aot/image/AOTRegExpCompiler.h +++ b/Source/JavaScriptCore/aot/image/AOTRegExpCompiler.h @@ -172,10 +172,13 @@ struct RegExpSection { JS_EXPORT_PRIVATE std::optional layOutRegExpSection(std::span, const AssemblerPolicy&, const CodeTemplates&, BuildContext*); +class StringNumbers; + struct RegExpRecordInputs { const RegExpRequests& requests; const CompiledRegExps& compiled; const RegExpSection& section; + const StringNumbers& strings; }; inline constexpr uint32_t regExpOutputSizeWithoutSubpatterns = 2; diff --git a/Source/JavaScriptCore/aot/image/AOTRegExpSection.cpp b/Source/JavaScriptCore/aot/image/AOTRegExpSection.cpp index 153861e55adef..666553c340909 100644 --- a/Source/JavaScriptCore/aot/image/AOTRegExpSection.cpp +++ b/Source/JavaScriptCore/aot/image/AOTRegExpSection.cpp @@ -9,6 +9,7 @@ #if ENABLE(AOT) && ENABLE(YARR_JIT) #include "AOTBuildContext.h" +#include "AOTNumbering.h" #include "AOTSectionLinker.h" #include "AOTSectionWriter.h" #include "AOTSort.h" @@ -424,13 +425,13 @@ std::optional layOutRegExpSection(std::span return RegExpSectionBuilder(regExps, policy, templates, context).build(); } -static uint32_t encodeRegExpNamedGroups(SectionWriter& details, const RegExpRequest& request, const RegExpSummary& summary) +static uint32_t encodeRegExpNamedGroups(SectionWriter& details, const RegExpRequest& request, const RegExpSummary& summary, const StringNumbers& strings) { Vector words; for (StringID name : request.captureGroupNames) - words.append(name ? name.index() + 1 : 0); + words.append(name ? strings.requiredNumberOf(name) + 1 : 0); for (const RegExpNamedGroupStrings& group : request.namedGroups) { - words.append(group.name.index()); + words.append(strings.requiredNumberOf(group.name)); words.append(static_cast(group.parenIndices.size())); words.appendVector(group.parenIndices); } @@ -446,7 +447,7 @@ static uint32_t encodeRegExpNamedGroups(SectionWriter& details, const RegExpRequ return offset; } -static void encodeRegExpMetadata(ImageRegExp& record, const RegExpRequest& request, const RegExpSummary& summary, SectionWriter& details) +static void encodeRegExpMetadata(ImageRegExp& record, const RegExpRequest& request, const RegExpSummary& summary, const StringNumbers& strings, SectionWriter& details) { record.numSubpatterns = summary.numSubpatterns; record.minimumSize = summary.minimumSize; @@ -455,7 +456,7 @@ static void encodeRegExpMetadata(ImageRegExp& record, const RegExpRequest& reque record.hasNamedGroups = summary.hasNamedGroups(); RELEASE_ASSERT(!record.hasNamedGroups || !record.hasCharacterSet()); if (record.hasNamedGroups) - record.details = encodeRegExpNamedGroups(details, request, summary); + record.details = encodeRegExpNamedGroups(details, request, summary, strings); if (record.hasCharacterSet()) { RELEASE_ASSERT(summary.characterSet); const Yarr::CharacterSetWithRanges& set = *summary.characterSet; @@ -467,13 +468,13 @@ static void encodeRegExpMetadata(ImageRegExp& record, const RegExpRequest& reque } } -static ImageRegExp createRegExpRecord(const RegExpRequest& request, const CompiledRegExp& compiled, const RegExpSummary& summary, const std::array& entries, SectionWriter& details) +static ImageRegExp createRegExpRecord(const RegExpRequest& request, const CompiledRegExp& compiled, const RegExpSummary& summary, const std::array& entries, const StringNumbers& strings, SectionWriter& details) { RegExpRecordForm form = compiled.form; ASSERT(request.flags == ImageRegExp::significantFlags(request.flags)); RELEASE_ASSERT((form == RegExpRecordForm::NoMetadata) == (summary.metadata == RegExpMetadata::Unknown)); ImageRegExp record { }; - record.pattern = request.pattern.index(); + record.pattern = strings.requiredNumberOf(request.pattern); record.rawFlags = request.flags.toRaw(); record.rawForm = static_cast(form); record.isLiteral = request.origin == RegExpRequestOrigin::Literal; @@ -482,7 +483,7 @@ static ImageRegExp createRegExpRecord(const RegExpRequest& request, const Compil case RegExpMetadata::Unknown: break; case RegExpMetadata::Known: - encodeRegExpMetadata(record, request, summary, details); + encodeRegExpMetadata(record, request, summary, strings, details); break; } @@ -496,7 +497,7 @@ static ImageRegExp createRegExpRecord(const RegExpRequest& request, const Compil break; case RegExpRecordForm::Atom: RELEASE_ASSERT(request.atom); - record.atom = request.atom.index(); + record.atom = strings.requiredNumberOf(request.atom); break; case RegExpRecordForm::NoCode: case RegExpRecordForm::NoMetadata: @@ -538,7 +539,7 @@ RegExpSections encodeRegExpRecords(const RegExpRecordInputs& inputs, BuildContex const RegExpSummary* summary = context.regExpSummaries().summaryIfComputed(request.pattern, request.flags); RELEASE_ASSERT(summary); - records.append(createRegExpRecord(request, compiled, *summary, inputs.section.entries[index], details)); + records.append(createRegExpRecord(request, compiled, *summary, inputs.section.entries[index], inputs.strings, details)); hashes.append(ImageRegExpHash { ImageRegExp::hashOf(summary->patternHash, request.flags), recordNumber }); if (compiled.form == RegExpRecordForm::Code || compiled.form == RegExpRecordForm::NoCode) maxOutputSize = std::max(maxOutputSize, (summary->numSubpatterns + 1) * 2 + summary->numDuplicateNamedCaptureGroups); diff --git a/Source/JavaScriptCore/aot/image/AOTRelocation.cpp b/Source/JavaScriptCore/aot/image/AOTRelocation.cpp index e33835705bf80..09c00fb7d8861 100644 --- a/Source/JavaScriptCore/aot/image/AOTRelocation.cpp +++ b/Source/JavaScriptCore/aot/image/AOTRelocation.cpp @@ -77,6 +77,16 @@ void emitPadding(CCallHelpers& jit) #endif } +void emitUnreachable(CCallHelpers& jit) +{ +#if CPU(X86_64) + constexpr uint8_t undefinedInstruction = 0xd6; + jit.m_assembler.buffer().putByte(static_cast(undefinedInstruction)); +#else + jit.oops(); +#endif +} + RelocationRecorder::RelocationRecorder(CCallHelpers& jit, const AssemblerPolicy& policy) : m_jit(jit) { @@ -96,6 +106,18 @@ void RelocationRecorder::load64(CodeSymbol symbol, GPRReg dest) append(m_jit.loadPCRelative64(dest), RelocationKind::Load64, symbol); } +#if CPU(X86_64) +void RelocationRecorder::compare64(GPRReg left, CodeSymbol symbol) +{ + append(m_jit.comparePCRelative64(left), RelocationKind::Load64, symbol); +} + +void RelocationRecorder::loadDouble(CodeSymbol symbol, FPRReg dest) +{ + append(m_jit.loadPCRelativeDouble(dest), RelocationKind::Load64, symbol); +} +#endif + void RelocationRecorder::emitNumber32(CodeSymbol symbol) { CCallHelpers::Label site = m_jit.label(); diff --git a/Source/JavaScriptCore/aot/image/AOTRelocation.h b/Source/JavaScriptCore/aot/image/AOTRelocation.h index b7f557f7f5813..368f6d0e90933 100644 --- a/Source/JavaScriptCore/aot/image/AOTRelocation.h +++ b/Source/JavaScriptCore/aot/image/AOTRelocation.h @@ -123,6 +123,7 @@ class JS_EXPORT_PRIVATE ImageLinkBuffer { }; JS_EXPORT_PRIVATE void emitPadding(CCallHelpers&); +JS_EXPORT_PRIVATE void emitUnreachable(CCallHelpers&); enum class InstanceWordWidth : uint8_t { Byte, Word32, Word64 }; @@ -205,6 +206,10 @@ class JS_EXPORT_PRIVATE RelocationRecorder { void moveAddress(CodeSymbol, GPRReg dest, PCRelativeReach); void load64(CodeSymbol, GPRReg dest); +#if CPU(X86_64) + void loadDouble(CodeSymbol, FPRReg dest); + void compare64(GPRReg left, CodeSymbol); +#endif ALWAYS_INLINE uint32_t call(CodeSymbol symbol) { CCallHelpers::Label site = m_jit.label(); @@ -253,6 +258,20 @@ class JS_EXPORT_PRIVATE RelocationRecorder { } #endif +#if CPU(X86_64) + template + auto withNumberAsImmediate(CodeSymbol symbol, const Functor& functor) + { + auto& buffer = m_jit.m_assembler.buffer(); + auto result = functor(placeholderOfNumber); + std::span immediate = WTF::unsafeMakeSpan(static_cast(buffer.data()), buffer.codeSize()).last(sizeof(int32_t)); + RELEASE_ASSERT(WTF::unalignedLoad(immediate.data()) == placeholderOfNumber); + WTF::unalignedStore(immediate.data(), 0); + append(CCallHelpers::Jump(buffer.label().labelAtOffset(-static_cast(sizeof(int32_t)))).label(), RelocationKind::Number32, symbol); + return result; + } +#endif + uint32_t numberOfRelocations() const { return m_pending.size(); } void reserveRoomFor(size_t number) { m_pending.reserveCapacity(number); } @@ -262,6 +281,7 @@ class JS_EXPORT_PRIVATE RelocationRecorder { private: #if CPU(X86_64) static constexpr int32_t placeholderOfInstanceWordDistance = 0x12345678; + static constexpr int32_t placeholderOfNumber = 0x23456789; #endif struct PendingRelocation { diff --git a/Source/JavaScriptCore/aot/image/AOTSectionWriter.cpp b/Source/JavaScriptCore/aot/image/AOTSectionWriter.cpp index d0f50697907c3..c3e5759c043ec 100644 --- a/Source/JavaScriptCore/aot/image/AOTSectionWriter.cpp +++ b/Source/JavaScriptCore/aot/image/AOTSectionWriter.cpp @@ -102,11 +102,11 @@ struct HashedStream { } // anonymous namespace -static Vector sortStreamsByHash(const ParallelLoops& loops, std::span> streams) +static Vector sortStreamsByHash(const ParallelLoops& loops, std::span> streams) { Vector entries = collectInParallel(loops, streams.size(), [&](unsigned index, Vector& hashed) { if (!streams[index].isEmpty()) - hashed.append({ hashOfStream(streams[index].span()), index }); + hashed.append({ hashOfStream(asBytes(streams[index].span())), index }); }); radixSortKeepingEqualKeysInOrder<32>(entries, [](const HashedStream& entry) { return entry.hash; @@ -114,13 +114,8 @@ static Vector sortStreamsByHash(const ParallelLoops& loops, std::s return entries; } -Vector SectionWriter::appendAll(std::span> streams) +Vector findFirstEqualStreams(std::span> streams, const ParallelLoops& loops) { - Vector offsets(FillWith { }, streams.size(), 0); - if (m_didOverflow) - return offsets; - - const ParallelLoops& loops = m_context.parallelLoops(); Vector entries = sortStreamsByHash(loops, streams); Vector firstEqualStream(streams.size(), [](size_t index) { return static_cast(index); @@ -145,29 +140,7 @@ Vector SectionWriter::appendAll(std::span> strea } } }); - - size_t newSize = m_bytes.size(); - for (size_t index = 0; index < streams.size(); ++index) { - if (streams[index].isEmpty() || firstEqualStream[index] != index) - continue; - offsets[index] = static_cast(newSize); - newSize += streams[index].size(); - } - if (!canGrowTo(newSize)) { - offsets.fill(0); - return offsets; - } - growSectionTo(m_bytes, newSize); - std::span section = m_bytes.mutableSpan(); - forEachRangeOfIndicesInParallel(loops, streams.size(), [&](size_t first, size_t end) { - for (size_t index = first; index < end; ++index) { - if (firstEqualStream[index] != index) - offsets[index] = offsets[firstEqualStream[index]]; - else if (!streams[index].isEmpty()) - memcpySpan(section.subspan(offsets[index]), streams[index].span()); - } - }); - return offsets; + return firstEqualStream; } } } // namespace JSC::AOT diff --git a/Source/JavaScriptCore/aot/image/AOTSectionWriter.h b/Source/JavaScriptCore/aot/image/AOTSectionWriter.h index 958031332057f..8058af6be858c 100644 --- a/Source/JavaScriptCore/aot/image/AOTSectionWriter.h +++ b/Source/JavaScriptCore/aot/image/AOTSectionWriter.h @@ -79,11 +79,12 @@ template void sortByWordOnPoolKeepingEqualElementsInOrder(const ParallelLoops& loops, std::span list, const Invocable auto& wordOf) { static_assert(std::is_trivially_copyable_v && alignof(T) <= alignof(std::max_align_t)); - static constexpr size_t fewestSortedByBytes = 1 << 14; - static constexpr size_t valuesOfByte = 256; + static constexpr size_t fewestSortedByDigits = 1 << 14; + static constexpr unsigned bitsInDigit = 8; + static constexpr size_t valuesOfDigit = 1 << bitsInDigit; size_t count = list.size(); - if (count < fewestSortedByBytes) { + if (count < fewestSortedByDigits) { AOT::stableSortInLine(list, { }, wordOf); return; } @@ -104,30 +105,38 @@ void sortByWordOnPoolKeepingEqualElementsInOrder(const ParallelLoops& loops, std Vector scratch; scratch.grow(count * sizeof(T)); Vector places; - places.grow(valuesOfByte * numberOfRanges); + places.grow(valuesOfDigit * numberOfRanges); std::span from = list; std::span to(reinterpret_cast(scratch.mutableSpan().data()), count); - for (unsigned shift = 0; shift < 64; shift += 8) { - if (!static_cast(bitsThatDiffer >> shift)) - continue; + while (bitsThatDiffer) { + unsigned shift = std::countr_zero(bitsThatDiffer); + bitsThatDiffer &= ~(static_cast(valuesOfDigit - 1) << shift); + auto digitOf = [&](const T& element) { + return static_cast(wordOf(element) >> shift) & (valuesOfDigit - 1); + }; forEachRangeOfIndicesInParallel(loops, count, [&](size_t first, size_t end) { - std::array numbersInRange { }; + std::array numbersInRange { }; for (const T& element : from.subspan(first, end - first)) - ++numbersInRange[static_cast(wordOf(element) >> shift)]; - size_t range = first / indicesInRangeOfLoop; - for (size_t value = 0; value < valuesOfByte; ++value) - places[value * numberOfRanges + range] = numbersInRange[value]; + ++numbersInRange[digitOf(element)]; + uint32_t* placeOfRange = places.mutableSpan().data() + first / indicesInRangeOfLoop; + for (uint32_t number : numbersInRange) { + *placeOfRange = number; + placeOfRange += numberOfRanges; + } }); uint32_t next = 0; for (uint32_t& place : places) next += std::exchange(place, next); forEachRangeOfIndicesInParallel(loops, count, [&](size_t first, size_t end) { - std::array placesOfRange; - size_t range = first / indicesInRangeOfLoop; - for (size_t value = 0; value < valuesOfByte; ++value) - placesOfRange[value] = places[value * numberOfRanges + range]; + std::array placesOfRange; + const uint32_t* placeOfRange = places.span().data() + first / indicesInRangeOfLoop; + for (uint32_t& place : placesOfRange) { + place = *placeOfRange; + placeOfRange += numberOfRanges; + } + T* elements = to.data(); for (const T& element : from.subspan(first, end - first)) - to[placesOfRange[static_cast(wordOf(element) >> shift)]++] = element; + elements[placesOfRange[digitOf(element)]++] = element; }); std::swap(from, to); } @@ -176,13 +185,14 @@ Vector bytesOf(const Vector& values) [[nodiscard]] bool checkListSize(uint64_t count, size_t sizeOfElement, BuildContext&); void growSectionTo(Vector& bytes, size_t newSize); +Vector findFirstEqualStreams(std::span> streams, const ParallelLoops&); + class SectionWriter { WTF_MAKE_NONCOPYABLE(SectionWriter); public: explicit SectionWriter(BuildContext&); uint32_t append(std::span, size_t alignment); - Vector appendAll(std::span> streams); size_t size() const { return m_bytes.size(); } bool didOverflow() const { return m_didOverflow; } Vector takeBytes(); diff --git a/Source/JavaScriptCore/aot/image/AOTSideTableCodes.h b/Source/JavaScriptCore/aot/image/AOTSideTableCodes.h new file mode 100644 index 0000000000000..22afd8bffb630 --- /dev/null +++ b/Source/JavaScriptCore/aot/image/AOTSideTableCodes.h @@ -0,0 +1,309 @@ +/* + * Copyright (C) 2026 Oven, Inc. All rights reserved. + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#if ENABLE(AOT) + +#include "AOTImageFormat.h" +#include "AOTPrefixCode.h" +#include "AOTSchemaPackedRecords.h" +#include "AOTVector.h" +#include +#include +#include +#include +#include +#include +#include + +WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN + +namespace JSC { namespace AOT { + +enum class SideTableByte : uint8_t { + Flags, + FrameChangesSize, + FirstFrameChange, + LaterFrameChange, + Frame, + FrameMapStart, + FrameMapStartContinuation, + FrameMapSize, + HandlerRows, + HandlerStart, + HandlerSites, + HandlerTarget, + HandlerTargetContinuation, + ConstructedCallee, + CallSiteHead, + CallSiteHeadContinuation, + PrologueCheckCount, + PrologueCall, + PrologueCallContinuation, + PrologueResume, + SitesWithoutCode, + InlineFrameCount, + InlineFrameParent, + InlineFunction, + InlineFunctionContinuation, + SitesWithDetails, + SitesSkipped, + SiteDetails, + SiteDetailsContinuation, + SiteDetailsAfterInlined, + SiteDetailsAfterInlinedContinuation, + CalledBuiltin, + CalledBuiltinContinuation, + Distance, + DistanceContinuation, + DistanceAfterDetails, + DistanceAfterDetailsContinuation, + Continuation, +}; +static constexpr unsigned sideTableByteCount = std::to_underlying(SideTableByte::Continuation) + 1; + +struct SideTableShortWords { + static constexpr unsigned bits(unsigned code) + { + switch (static_cast(code)) { + case SideTableByte::Distance: + case SideTableByte::DistanceAfterDetails: + return 10; + case SideTableByte::LaterFrameChange: + case SideTableByte::CallSiteHead: + case SideTableByte::PrologueCall: + case SideTableByte::InlineFunction: + case SideTableByte::InlineFunctionContinuation: + return 9; + default: + return 7; + } + } +}; +using SideTableCodes = PrefixCodes; + +static constexpr size_t sideTablePadding = 8; + +struct SiteCheckpoint { + static constexpr unsigned sitesBetween = 64; + static constexpr uint64_t numberFor(uint64_t numberOfSites) { return numberOfSites ? (numberOfSites - 1) / sitesBetween : 0; } + static SiteCheckpoint at(const uint8_t* first, uint64_t index) + { + SiteCheckpoint result; + memcpy(&result, first + index * sizeof(SiteCheckpoint), sizeof(SiteCheckpoint)); + return result; + } + + uint32_t offset; + uint32_t position; + uint32_t frame; +}; +static_assert(sizeof(SiteCheckpoint) == 12); + +struct SiteDetailsWord { + static constexpr unsigned maskOfKind = std::bit_ceil(std::to_underlying(CallSiteKind::AwaitedCall) + 1u) - 1; + + CallSiteKind kind() const { return static_cast(bits & maskOfKind); } + bool isInlined() const { return bits > maskOfKind; } + + uint64_t bits; +}; + +class SideTableSymbolWriter { +public: + explicit SideTableSymbolWriter(PrefixSymbol* firstReservedSymbol) + : m_symbols(firstReservedSymbol) + { + } + + PrefixSymbol* end() const { return m_symbols.end(); } + ALWAYS_INLINE void append(SideTableByte kind, uint8_t byte) { m_symbols.append(std::to_underlying(kind), byte); } + ALWAYS_INLINE void appendVarint(SideTableByte first, SideTableByte continuation, uint64_t value) { m_symbols.appendVarint(std::to_underlying(first), std::to_underlying(continuation), value); } + ALWAYS_INLINE void appendVarint(SideTableByte first, uint64_t value) { appendVarint(first, SideTableByte::Continuation, value); } + void append(std::span symbols) { m_symbols.append(symbols); } + +private: + PrefixSymbolWriter m_symbols; +}; + +class SideTableReader { +public: + SideTableReader(std::span section, const uint8_t* begin) + : m_codes(section.data()) + , m_end(section.data() + section.size()) + , m_reader(begin, m_end) + { + } + + ALWAYS_INLINE uint8_t next(SideTableByte kind) + { + return static_cast(m_reader.read(SideTableCodes::code(m_codes, std::to_underlying(kind)))); + } + ALWAYS_INLINE uint64_t nextVarint(SideTableByte first, SideTableByte continuation = SideTableByte::Continuation) + { + uint8_t byte = next(first); + uint64_t value = byte & 0x7f; + for (unsigned shift = 7; byte & 0x80 && shift < 64; shift += 7) { + byte = next(continuation); + value |= static_cast(byte & 0x7f) << shift; + } + return value; + } + struct DistanceCodes { + PrefixCodeView distance; + PrefixCodeView distanceAfterDetails; + }; + DistanceCodes distanceCodes() const + { + return { SideTableCodes::code(m_codes, std::to_underlying(SideTableByte::Distance)), SideTableCodes::code(m_codes, std::to_underlying(SideTableByte::DistanceAfterDetails)) }; + } + ALWAYS_INLINE uint64_t nextDistance(const DistanceCodes& codes, bool followsDetails) + { + uint8_t byte = static_cast(m_reader.read(followsDetails ? codes.distanceAfterDetails : codes.distance)); + if (!(byte & 0x80)) [[likely]] + return byte; + SideTableByte continuation = followsDetails ? SideTableByte::DistanceAfterDetailsContinuation : SideTableByte::DistanceContinuation; + return (byte & 0x7f) | nextVarint(continuation, continuation) << 7; + } + const uint8_t* alignToByte() { return m_reader.alignToByte(); } + SideTableReader at(const uint8_t* begin) const { return SideTableReader(std::span(m_codes, m_end), begin); } + +private: + const uint8_t* m_codes; + const uint8_t* m_end; + PrefixCodeReader m_reader; +}; + +struct HandlerCursor { + ALWAYS_INLINE friend uint64_t readVarint(HandlerCursor& cursor) + { + switch (cursor.word++) { + case 0: + return cursor.reader.nextVarint(SideTableByte::HandlerStart); + case 1: + return cursor.reader.nextVarint(SideTableByte::HandlerSites); + default: + return cursor.reader.nextVarint(SideTableByte::HandlerTarget, SideTableByte::HandlerTargetContinuation); + } + } + + SideTableReader& reader; + unsigned word { 0 }; +}; + +class SiteDetailsReader { +public: + explicit SiteDetailsReader(SideTableReader& reader) + : m_reader(reader) + , m_numberLeft(static_cast(reader.nextVarint(SideTableByte::SitesWithDetails))) + { + skipSites(); + } + + bool isAtEnd() const { return !m_numberLeft; } + bool isAt(unsigned site) const { return m_numberLeft && m_site == site; } + unsigned site() const { return m_site; } + CallSiteDetails next() + { + CallSiteDetails details = CallSiteDetails::read(*this); + --m_numberLeft; + ++m_site; + skipSites(); + return details; + } + + ALWAYS_INLINE friend uint64_t readVarint(SiteDetailsReader& details) + { + SideTableReader& reader = details.m_reader; + if (std::exchange(details.m_readsBuiltin, false)) + return reader.nextVarint(SideTableByte::CalledBuiltin, SideTableByte::CalledBuiltinContinuation); + SiteDetailsWord word { details.m_followsInlined ? reader.nextVarint(SideTableByte::SiteDetailsAfterInlined, SideTableByte::SiteDetailsAfterInlinedContinuation) + : reader.nextVarint(SideTableByte::SiteDetails, SideTableByte::SiteDetailsContinuation) }; + details.m_followsInlined = word.isInlined(); + details.m_readsBuiltin = namesBuiltinOf(word.kind()); + return word.bits; + } + +private: + void skipSites() + { + if (m_numberLeft) + m_site += static_cast(m_reader.nextVarint(SideTableByte::SitesSkipped)); + } + + SideTableReader& m_reader; + unsigned m_numberLeft; + unsigned m_site { 0 }; + bool m_followsInlined { false }; + bool m_readsBuiltin { false }; +}; + +struct InlineFrameCursor { + ALWAYS_INLINE friend uint64_t readVarint(InlineFrameCursor& cursor) + { + if (std::exchange(cursor.readsFunction, true)) + return cursor.functions.nextVarint(SideTableByte::InlineFunction, SideTableByte::InlineFunctionContinuation); + return cursor.parents.nextVarint(SideTableByte::InlineFrameParent); + } + + SideTableReader& parents; + SideTableReader& functions; + bool readsFunction { false }; +}; + +struct SideTableFrame { + static SideTableFrame read(SideTableReader& reader, uint32_t offset) + { + SideTableFrame result; + uint64_t size = reader.nextVarint(SideTableByte::FrameChangesSize); + if (!size) + return result; + uint64_t position = offset / codeOffsetUnit; + uint64_t change = 0; + uint64_t distance = 0; + unsigned shift = 0; + SideTableByte kind = SideTableByte::FirstFrameChange; + while (size--) { + uint8_t byte = reader.next(kind); + kind = SideTableByte::LaterFrameChange; + distance |= static_cast(byte & 0x7f) << (shift & 63); + if (byte & 0x80) { + shift += 7; + continue; + } + change += distance; + result.coversOffset ^= position >= change; + distance = 0; + shift = 0; + } + result.hasFrame = true; + result.number = static_cast(reader.nextVarint(SideTableByte::Frame)); + return result; + } + + bool hasFrame { false }; + bool coversOffset { false }; + uint32_t number { 0 }; +}; + +struct SideTableParts { + JS_EXPORT_PRIVATE static SideTableParts read(ImageFunction, std::span section, uint32_t offset = 0); + + std::optional constructedCalleeAt(uint32_t site) const; + + SideTableFrame frame; + uint32_t firstFrameMapWord { 0 }; + uint32_t numberOfFrameMapWords { 0 }; + Vector constructedCallees; + std::optional callSites; + bool hasHandlers { false }; +}; + +} } // namespace JSC::AOT + +WTF_ALLOW_UNSAFE_BUFFER_USAGE_END + +#endif // ENABLE(AOT) diff --git a/Source/JavaScriptCore/aot/image/AOTSideTables.cpp b/Source/JavaScriptCore/aot/image/AOTSideTables.cpp index 4afc6a6066c56..7522c5aef378c 100644 --- a/Source/JavaScriptCore/aot/image/AOTSideTables.cpp +++ b/Source/JavaScriptCore/aot/image/AOTSideTables.cpp @@ -12,10 +12,16 @@ #include "AOTCallSiteTable.h" #include "AOTFunctionTable.h" #include "AOTPhaseTimes.h" +#include "AOTPositionEncoder.h" +#include "AOTPrefixCode.h" #include "AOTSectionWriter.h" +#include "AOTSideTableCodes.h" #include "AOTSort.h" #include "AOTThreads.h" +#include "AOTVarint.h" #include +#include +#include #include #include #include @@ -35,8 +41,7 @@ struct SideTableScratch { Vector frameRanges; Vector handlers; Vector, 8> constructedCallees; - Vector stream; - Vector roomForPositions; + Vector symbols; }; struct HashedFrameMaps { @@ -112,6 +117,12 @@ static void forEachOffsetAskedForFrameInOrder(Vector& offsets, con } } +static void appendFrameChange(Vector& bytes, uint32_t previousChange, uint32_t change) +{ + RELEASE_ASSERT(previousChange <= change && !(previousChange % codeOffsetUnit) && !(change % codeOffsetUnit)); + appendVarint(bytes, (change - previousChange) / codeOffsetUnit); +} + static void encodeFrameRanges(SideTableScratch& scratch, const CompiledCode& code) { Vector& bytes = scratch.frameRanges; @@ -135,7 +146,7 @@ static void encodeFrameRanges(SideTableScratch& scratch, const CompiledCode& cod uint32_t endOfLastUnitAsked = 0; forEachOffsetAskedForFrameInOrder(scratch.offsetsAskedForFrame, code, [&](uint32_t offset) ALWAYS_INLINE_LAMBDA { if (hasFrameAt(offset) != hasFrame) { - SideTableParts::appendFrameChange(bytes, previousChange, endOfLastUnitAsked); + appendFrameChange(bytes, previousChange, endOfLastUnitAsked); previousChange = endOfLastUnitAsked; hasFrame = !hasFrame; } @@ -177,9 +188,27 @@ static FrameMapPlaces findPlacesOfFrameMaps(const LinkedFunctions& functions, co return places; } -static Vector joinPositionsOfChunks(const ChunksOfIndices& chunks, std::span> positionsOfChunks, std::span placesInChunks, BuildContext& context) +static Vector joinPositionsOfChunks(const ChunksOfIndices& chunks, std::span encoders, BuildContext& context) { - RELEASE_ASSERT(positionsOfChunks.size() == chunks.size()); + RELEASE_ASSERT(encoders.size() == chunks.size()); + PositionCodes::Counts counts { }; + for (const PositionEncoder& encoder : encoders) + PositionCodes::add(counts, encoder.counts()); + PositionCodes codes(counts); + const Vector& bytesOfCodes = codes.bytes(); + + Vector> positionsOfChunks(chunks.size()); + Vector placesInChunks; + placesInChunks.fill(0, PositionIndex::numberOfEntriesFor(chunks.firstIndexOf.last())); + forEachIndexInParallel(context.parallelLoops(), chunks.size(), [&](unsigned chunk) { + Vector& positions = positionsOfChunks[chunk]; + for (unsigned position = chunks.firstIndexOf[chunk]; position < chunks.firstIndexOf[chunk + 1]; ++position) { + if (!(position % PositionIndex::functionsBetween)) + placesInChunks[position / PositionIndex::functionsBetween] = static_cast(positions.size()); + encoders[chunk].appendNextCoded(positions, codes); + } + }); + size_t sizeOfIndex = placesInChunks.size() * sizeof(uint32_t); uint64_t sizeOfEntries = 0; Vector placeOfChunk(positionsOfChunks.size()); @@ -187,12 +216,13 @@ static Vector joinPositionsOfChunks(const ChunksOfIndices& chunks, std: placeOfChunk[chunk] = static_cast(sizeOfEntries); sizeOfEntries += positionsOfChunks[chunk].size(); } - if (!checkSectionSize(sizeOfIndex + sizeOfEntries, context)) + if (!checkSectionSize(bytesOfCodes.size() + sizeOfIndex + sizeOfEntries, context)) return { }; - Vector result(sizeOfIndex + sizeOfEntries); - std::span index = result.mutableSpan().first(sizeOfIndex); - std::span entries = result.mutableSpan().subspan(sizeOfIndex); + Vector result(bytesOfCodes.size() + sizeOfIndex + sizeOfEntries); + memcpySpan(result.mutableSpan(), bytesOfCodes.span()); + std::span index = result.mutableSpan().subspan(bytesOfCodes.size(), sizeOfIndex); + std::span entries = result.mutableSpan().subspan(bytesOfCodes.size() + sizeOfIndex); forEachIndexInParallel(context.parallelLoops(), chunks.size(), [&](unsigned chunk) { memcpySpan(entries.subspan(placeOfChunk[chunk]), positionsOfChunks[chunk].span()); size_t endOfGroups = PositionIndex::numberOfEntriesFor(chunks.firstIndexOf[chunk + 1]); @@ -204,7 +234,235 @@ static Vector joinPositionsOfChunks(const ChunksOfIndices& chunks, std: return result; } -SideTables encodeSideTables(const LinkedFunctions& functions, std::span> callSites, BuildContext& context) +namespace { + +struct SideTableInputs { + OptionSet flags() const; + + OptionSet flagsOfFunction; + std::span frameRanges; + uint32_t frame { 0 }; + std::optional> frameMapRange; + std::span handlers; + std::span> constructedCallees; + std::span callSites; +}; + +OptionSet SideTableInputs::flags() const +{ + OptionSet flags = flagsOfFunction; + flags.set(ImageFunctionFlag::HasFrameMaps, frameMapRange && frameMapRange->second); + flags.set(ImageFunctionFlag::HasHandlers, !handlers.empty()); + flags.set(ImageFunctionFlag::HasConstructedCallees, !constructedCallees.empty()); + flags.set(ImageFunctionFlag::HasCallSites, !callSites.empty()); + return flags; +} + +struct SideTableStream { + static SideTableStream read(std::span words) + { + SideTableStream stream; + auto take = [&](size_t size) { + return std::exchange(words, words.subspan(size)).first(size); + }; + auto takeBytes = [&](std::span bytes) { + memcpySpan(bytes, asBytes(take(bytes.size() / sizeof(PrefixSymbol)))); + }; + std::array numbersOfSymbols; + takeBytes(asMutableByteSpan(numbersOfSymbols)); + stream.own = take(numbersOfSymbols[0]); + stream.handlers = take(numbersOfSymbols[1]); + if (words.empty()) + return stream; + takeBytes(asMutableByteSpan(stream.head)); + stream.blocks = take(stream.head.numberOfBlocks * sizeof(CallSiteStreamHead::Block) / sizeof(PrefixSymbol)); + stream.callSites = words; + return stream; + } + + CallSiteStreamHead::Block block(size_t index) const + { + constexpr size_t size = sizeof(CallSiteStreamHead::Block); + CallSiteStreamHead::Block result; + memcpySpan(asMutableByteSpan(result), asBytes(blocks).subspan(index * size, size)); + return result; + } + + std::span own; + std::span handlers; + CallSiteStreamHead head { }; + std::span blocks; + std::span callSites; +}; + +} // anonymous namespace + +static void appendStream(Vector& stream, Vector& room, const SideTableInputs& inputs) +{ + constexpr size_t maxSymbolsOfWord = 10; + OptionSet flags = inputs.flags(); + room.shrink(0); + room.grow(8 * maxSymbolsOfWord + inputs.frameRanges.size() + inputs.handlers.size() + inputs.constructedCallees.size() * 2 * maxSymbolsOfWord); + PrefixSymbol* firstSymbol = room.mutableSpan().data(); + SideTableSymbolWriter symbols(firstSymbol); + symbols.append(SideTableByte::Flags, static_cast(flags.toRaw())); + symbols.appendVarint(SideTableByte::FrameChangesSize, inputs.frameRanges.size()); + if (!inputs.frameRanges.empty()) { + SideTableByte kind = SideTableByte::FirstFrameChange; + for (uint8_t byte : inputs.frameRanges) + symbols.append(std::exchange(kind, SideTableByte::LaterFrameChange), byte); + symbols.appendVarint(SideTableByte::Frame, inputs.frame); + } + if (flags.contains(ImageFunctionFlag::HasFrameMaps)) { + symbols.appendVarint(SideTableByte::FrameMapStart, SideTableByte::FrameMapStartContinuation, inputs.frameMapRange->first); + symbols.appendVarint(SideTableByte::FrameMapSize, inputs.frameMapRange->second); + } + if (flags.contains(ImageFunctionFlag::HasConstructedCallees)) { + Vector rows; + uint32_t previous = 0; + for (auto [site, function] : inputs.constructedCallees) { + RELEASE_ASSERT(site > previous); + appendVarint(rows, site - std::exchange(previous, site)); + appendVarint(rows, function); + } + symbols.appendVarint(SideTableByte::Continuation, rows.size()); + for (uint8_t byte : rows) + symbols.append(SideTableByte::ConstructedCallee, byte); + } + PrefixSymbol* endOfOwn = symbols.end(); + if (flags.contains(ImageFunctionFlag::HasHandlers)) { + Vector words; + for (const uint8_t* cursor = inputs.handlers.data(); cursor < inputs.handlers.data() + inputs.handlers.size();) + words.append(readVarint(cursor)); + constexpr size_t wordsOfRow = 3; + RELEASE_ASSERT(!(words.size() % wordsOfRow)); + symbols.appendVarint(SideTableByte::HandlerRows, words.size() / wordsOfRow); + for (size_t word = 0; word < words.size(); word += wordsOfRow) { + symbols.appendVarint(SideTableByte::HandlerStart, words[word]); + symbols.appendVarint(SideTableByte::HandlerSites, words[word + 1]); + symbols.appendVarint(SideTableByte::HandlerTarget, SideTableByte::HandlerTargetContinuation, words[word + 2]); + } + } + std::array numbersOfSymbols { static_cast(endOfOwn - firstSymbol), static_cast(symbols.end() - endOfOwn) }; + stream.reserveInitialCapacity(sizeof(numbersOfSymbols) / sizeof(PrefixSymbol) + (symbols.end() - firstSymbol) + inputs.callSites.size()); + stream.append(spanReinterpretCast(asByteSpan(numbersOfSymbols))); + stream.append(std::span(firstSymbol, symbols.end())); + stream.append(inputs.callSites); +} + +static uint8_t* writeStream(uint8_t* to, const SideTableStream& stream, const SideTableCodes& codes) +{ + std::span left = stream.callSites; + auto take = [&](size_t size) { + return std::exchange(left, left.subspan(size)).first(size); + }; + auto writePart = [&](std::span symbols) { + constexpr size_t maxSizeOfVarint = 10; + uint8_t* part = to + maxSizeOfVarint; + uint8_t* endOfPart = codes.write(part, symbols); + WriterOfReservedBytes sizeOfPart(to); + appendVarint(sizeOfPart, endOfPart - part); + memmove(sizeOfPart.end(), part, endOfPart - part); + to = sizeOfPart.end() + (endOfPart - part); + }; + to = codes.write(to, stream.own, take(stream.head.numberOfHeadSymbols)); + if (!stream.handlers.empty()) + writePart(stream.handlers); + if (stream.callSites.empty()) + return to; + if (stream.head.hasInlineFrames) { + writePart(take(stream.head.numberOfParentSymbols)); + writePart(take(stream.head.numberOfFunctionSymbols)); + } + uint32_t numberOfBlocks = stream.head.numberOfBlocks; + uint8_t* checkpoints = to; + uint8_t* firstBlock = to += numberOfBlocks ? (numberOfBlocks - 1) * sizeof(SiteCheckpoint) : 0; + for (uint32_t number = 0; number < numberOfBlocks; ++number) { + CallSiteStreamHead::Block block = stream.block(number); + if (number) { + SiteCheckpoint checkpoint { block.offset, static_cast(to - firstBlock), block.frame }; + memcpy(checkpoints + (number - 1) * sizeof(SiteCheckpoint), &checkpoint, sizeof(SiteCheckpoint)); + } + to = codes.write(to, take(block.numberOfSymbols)); + } + RELEASE_ASSERT(left.empty()); + return to; +} + +static void codeStreams(SideTables& result, std::span> streams, BuildContext& context) +{ + const ParallelLoops& loops = context.parallelLoops(); + Vector firstEqualStream = findFirstEqualStreams(streams, loops); + Vector distinctStreams; + for (uint32_t position = 0; position < streams.size(); ++position) { + if (firstEqualStream[position] == position) + distinctStreams.append(position); + } + + ChunksOfIndices chunks = divideIntoChunks(static_cast(distinctStreams.size())); + auto forEachStreamOf = [&](unsigned chunk, const auto& functor) { + for (unsigned number = chunks.firstIndexOf[chunk]; number < chunks.firstIndexOf[chunk + 1]; ++number) + functor(distinctStreams[number], SideTableStream::read(streams[distinctStreams[number]].span())); + }; + Vector countsOfChunks(chunks.size()); + forEachIndexInParallel(loops, chunks.size(), [&](unsigned chunk) { + countsOfChunks[chunk].fill(0); + forEachStreamOf(chunk, [&](uint32_t, const SideTableStream& stream) { + SideTableCodes::count(countsOfChunks[chunk], stream.own); + SideTableCodes::count(countsOfChunks[chunk], stream.handlers); + SideTableCodes::count(countsOfChunks[chunk], stream.callSites); + }); + }); + SideTableCodes::Counts counts { }; + for (const SideTableCodes::Counts& countsOfChunk : countsOfChunks) + SideTableCodes::add(counts, countsOfChunk); + SideTableCodes codes(counts); + + Vector> bytesOfChunks(chunks.size()); + result.offsets.grow(streams.size()); + forEachIndexInParallel(loops, chunks.size(), [&](unsigned chunk) { + size_t maxSize = 0; + forEachStreamOf(chunk, [&](uint32_t position, const SideTableStream& stream) { + constexpr size_t maxSizeOfPartEnd = 16; + maxSize += streams[position].size() * sizeof(PrefixSymbol) + (4 + stream.head.numberOfBlocks) * maxSizeOfPartEnd; + }); + Vector& bytes = bytesOfChunks[chunk]; + bytes.grow(maxSize); + uint8_t* begin = bytes.mutableSpan().data(); + uint8_t* end = begin; + forEachStreamOf(chunk, [&](uint32_t position, const SideTableStream& stream) { + result.offsets[position] = static_cast(end - begin); + end = writeStream(end, stream, codes); + }); + RELEASE_ASSERT(end <= begin + maxSize); + bytes.shrink(end - begin); + }); + + uint64_t size = codes.bytes().size(); + Vector firstByteOfChunk(chunks.size()); + for (unsigned chunk = 0; chunk < chunks.size(); ++chunk) { + firstByteOfChunk[chunk] = static_cast(size); + size += bytesOfChunks[chunk].size(); + } + size += sideTablePadding; + if (!checkSectionSize(size, context)) { + result.offsets.fill(0); + return; + } + result.bytes.grow(static_cast(size)); + std::span section = result.bytes.mutableSpan(); + memcpySpan(section, codes.bytes().span()); + memsetSpan(section.last(sideTablePadding), std::numeric_limits::max()); + forEachIndexInParallel(loops, chunks.size(), [&](unsigned chunk) { + memcpySpan(section.subspan(firstByteOfChunk[chunk]), bytesOfChunks[chunk].span()); + for (unsigned number = chunks.firstIndexOf[chunk]; number < chunks.firstIndexOf[chunk + 1]; ++number) + result.offsets[distinctStreams[number]] += firstByteOfChunk[chunk]; + }); + for (uint32_t position = 0; position < streams.size(); ++position) + result.offsets[position] = result.offsets[firstEqualStream[position]]; +} + +SideTables encodeSideTables(const LinkedFunctions& functions, std::span> callSites, BuildContext& context) { RELEASE_ASSERT(callSites.size() == functions.size()); const ParallelLoops& loops = context.parallelLoops(); @@ -226,17 +484,14 @@ SideTables encodeSideTables(const LinkedFunctions& functions, std::spanframes); result.frameMapWords.grow(frameMaps.numberOfWords); - result.flags.grow(numberOfFunctions); + result.sizes.grow(numberOfFunctions); std::span frameMapWords = result.frameMapWords.mutableSpan(); - Vector> streams(numberOfFunctions); + Vector> streams(numberOfFunctions); ChunksOfIndices chunks = divideIntoChunks(numberOfFunctions); - Vector> positionsOfChunks(chunks.size()); - Vector placesInChunks; - placesInChunks.fill(0, PositionIndex::numberOfEntriesFor(numberOfFunctions)); + Vector encoders(hasPositions ? chunks.size() : 0); forEachIndexInParallel(loops, chunks.size(), [&](unsigned chunk) { SideTableScratch scratch; - Vector& positions = positionsOfChunks[chunk]; for (unsigned position = chunks.firstIndexOf[chunk]; position < chunks.firstIndexOf[chunk + 1]; ++position) { const CompiledCode& code = functions[position]; const SiteDescription& sites = code.sites; @@ -256,7 +511,12 @@ SideTables encodeSideTables(const LinkedFunctions& functions, std::span flagsOfFunction; + flagsOfFunction.set(ImageFunctionFlag::HasInlineFrames, sites.hasEncodedInlineFrames()); + flagsOfFunction.set(ImageFunctionFlag::HasPositions, hasPositions && !sites.positions.isEmpty()); + flagsOfFunction.set(ImageFunctionFlag::SpillsArgumentRegisters, code.frame.spillsArgumentRegisters); SideTableInputs inputs { + .flagsOfFunction = flagsOfFunction, .frameRanges = scratch.frameRanges.span(), .frame = frameOf[position], .frameMapRange = frameMapRange, @@ -264,32 +524,27 @@ SideTables encodeSideTables(const LinkedFunctions& functions, std::span(scratch.frameRanges.size() + scratch.handlers.size() + callSites[position].size()); if (!hasPositions) continue; bool beginsGroup = !(position % PositionIndex::functionsBetween); - if (beginsGroup) - placesInChunks[position / PositionIndex::functionsBetween] = static_cast(positions.size()); - encodePositions(positions, scratch.roomForPositions, sites, beginsGroup ? OwnPosition { } : OwnPosition::of(functions[position - 1].sites)); + encoders[chunk].encode(sites, beginsGroup ? OwnPosition { } : OwnPosition::of(functions[position - 1].sites)); } }); times.didFinish("encode"_s); - SectionWriter writer(context); - result.offsets = writer.appendAll(streams.span()); - result.bytes = writer.takeBytes(); - times.didFinish("hold equal streams once"_s); - result.positions = joinPositionsOfChunks(chunks, positionsOfChunks.span(), placesInChunks.span(), context); - times.didFinish("join positions"_s); + codeStreams(result, streams.span(), context); + times.didFinish("code streams, hold equal ones once"_s); + if (hasPositions) + result.positions = joinPositionsOfChunks(chunks, encoders.mutableSpan(), context); + times.didFinish("code and join positions"_s); forEachIndexInParallel(loops, chunks.size(), [&](unsigned chunk) { for (unsigned position = chunks.firstIndexOf[chunk]; position < chunks.firstIndexOf[chunk + 1]; ++position) streams[position].clear(); - positionsOfChunks[chunk].clear(); + if (hasPositions) + encoders[chunk] = { }; }); times.didFinish("free the streams"_s); return result; diff --git a/Source/JavaScriptCore/aot/image/AOTStaticsTables.cpp b/Source/JavaScriptCore/aot/image/AOTStaticsTables.cpp index 96c211626f292..402d3f1dec5c0 100644 --- a/Source/JavaScriptCore/aot/image/AOTStaticsTables.cpp +++ b/Source/JavaScriptCore/aot/image/AOTStaticsTables.cpp @@ -11,6 +11,7 @@ #include "AOTGlobalBindings.h" #include "AOTIRProgram.h" #include "AOTPlan.h" +#include "AOTPrefixCode.h" #include "AOTSort.h" #include "AOTThreads.h" #include @@ -315,7 +316,7 @@ static void encodeNamesOfVariables(TableEncoder& encoder) writer.endModule(closureNumbers[module].span()); } RELEASE_ASSERT(first == named.size()); - encoder.part.setTable(ProgramTableIndex::namedVariables, writer.bytes()); + encoder.part.setTable(ProgramTableIndex::namedVariables, encodeVarintBytes(writer.bytes().span())); } static Vector collectBitsOfNamesOfClassGuards(const TableEncoder& encoder) diff --git a/Source/JavaScriptCore/aot/image/AOTStoredStrings.cpp b/Source/JavaScriptCore/aot/image/AOTStoredStrings.cpp index 76e40fda6ce00..75584847225bb 100644 --- a/Source/JavaScriptCore/aot/image/AOTStoredStrings.cpp +++ b/Source/JavaScriptCore/aot/image/AOTStoredStrings.cpp @@ -8,6 +8,7 @@ #if ENABLE(AOT) +#include "AOTCodedTexts.h" #include "AOTImageFormat.h" #include "AOTImageText.h" #include "AOTProgramTables.h" @@ -16,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -174,20 +176,31 @@ VerdictOnStoredStrings ReaderOfImage::read() auto bytesOfWords = section(ImageSection::Strings); auto bytesOfListedStrings = section(ImageSection::LongStrings); auto bytesOfHeaders = section(ImageSection::StringHeaders); + auto bytesOfConstants = section(ImageSection::StringConstants); auto text = section(ImageSection::Text); - if (!bytesOfWords || !bytesOfListedStrings || !bytesOfHeaders || !text) + if (!bytesOfWords || !bytesOfListedStrings || !bytesOfHeaders || !bytesOfConstants || !text) return refuse("a section lies outside the image"); RecordsInFile words(*bytesOfWords); + RecordsInFile constants(*bytesOfConstants); RecordsInFile listedStrings(*bytesOfListedStrings); RecordsInFile headers(*bytesOfHeaders); RecordsInFile records(*bytesOfHeaders); bool hasRecords = m_header.options.form() == ImageForm::PositionIndependent; uint64_t addressOfText = m_header.address + m_header.sections[static_cast(ImageSection::Text)].offset; + size_t numberOfWords = m_header.numbers.numberOfStringWords; + if (numberOfWords > words.size()) + return refuse("a section lies outside the image"); + CodedTexts codedTexts(bytesOfWords->subspan(numberOfWords * sizeof(ImageStringWord)), m_header.numbers.numberOfCodedTexts); + Vector covered; - for (size_t index = 0; index < words.size(); ++index) { - ImageStringWord word = words[index]; - if (!word.hasText()) + HashSet bitsOfWords; + size_t endOfIdentifiers = m_header.numbers.endOfIdentifiers; + size_t endOfWords = endOfIdentifiers + numberOfWords; + for (size_t index = 0; index < endOfWords + constants.size(); ++index) { + ImageStringWord word = index < endOfIdentifiers ? ImageStringWord::ofIdentifier(static_cast(index)) : index < endOfWords ? words[index - endOfIdentifiers] : constants[index - endOfWords]; + bool is16Bit = word.is16Bit(); + if (!word.hasText() || !bitsOfWords.add(word.bits).isNewEntry) continue; std::optional range = ImageLongString { 0, 0 }; if (!word.hasHeader()) @@ -196,28 +209,32 @@ VerdictOnStoredStrings ReaderOfImage::read() range = std::nullopt; else if (hasRecords) { ImageStringRecord record = records[word.number()]; + is16Bit = record.is16Bit(); if (uint32_t length = record.rangeOrIndex & ImageString::lengthMask) range = ImageLongString { record.rangeOrIndex >> ImageString::lengthBits, length }; else if (record.hasText()) range = listedString(listedStrings, record.rangeOrIndex >> ImageString::lengthBits); } else if (ImageStringHeader header = headers[word.number()]; header.length) { + is16Bit = !header.is8Bit(); uint64_t offset = header.characters - addressOfText; range = offset <= text->size() ? std::optional(ImageLongString { static_cast(offset), header.length }) : std::nullopt; } - uint64_t sizeOfUnit = word.is16Bit() ? sizeof(char16_t) : sizeof(Latin1Character); + uint64_t sizeOfUnit = is16Bit ? sizeof(char16_t) : sizeof(Latin1Character); if (!range || range->offset + range->length * sizeOfUnit > text->size()) return refuse("string ", index, " lies outside the section of text"); if (!range->length) continue; std::span bytes = text->subspan(range->offset, range->length * sizeOfUnit); covered.append({ range->offset, range->offset + bytes.size() }); - if (word.is16Bit()) { + if (is16Bit) { Vector units(range->length); memcpy(units.mutableSpan().data(), bytes.data(), bytes.size()); appendString(byteCast(StringView(units.span()).utf8().span())); } else appendLatin1String(bytes); } + for (uint32_t number = 0; number < codedTexts.count(); ++number) + appendLatin1String(codedTexts.at(number).span()); VerdictOnStoredStrings verdictOnText = reportSection("Text"_s, m_header.sections[static_cast(ImageSection::Text)].offset, *text, covered); return std::max(verdictOnText, readTextsOfProgramTables()); } diff --git a/Source/JavaScriptCore/aot/image/AOTStringUses.cpp b/Source/JavaScriptCore/aot/image/AOTStringUses.cpp index 4c73e4f44f033..a401654952c43 100644 --- a/Source/JavaScriptCore/aot/image/AOTStringUses.cpp +++ b/Source/JavaScriptCore/aot/image/AOTStringUses.cpp @@ -9,6 +9,7 @@ #if ENABLE(AOT) #include "AOTBuildContext.h" +#include "AOTCodedTexts.h" #include "AOTFieldTable.h" #include "AOTIRProgram.h" #include "AOTIdentifierNumbering.h" @@ -265,6 +266,40 @@ static Vector sortStringsByFirstFunction(const ParallelLoops& loops, c }); } +static Vector sortTextsForCoding(const Typed::StringTable& strings, const BitVector& candidates, const ParallelLoops& loops) +{ + struct TextByFront { + uint64_t front; + StringID id; + }; + Vector texts; + for (size_t index : candidates) { + StringID id(index); + std::span text = strings[id]; + if (!CodedTexts::canHold(text)) + continue; + uint64_t front = 0; + for (size_t place = 0; place < sizeof(front); ++place) + front = front << 8 | (place < text.size() ? text[place] : 0); + texts.append({ front, id }); + } + sortByWordOnPoolKeepingEqualElementsInOrder(loops, texts.mutableSpan(), [](const TextByFront& text) { + return text.front; + }); + for (size_t begin = 0; begin < texts.size();) { + size_t end = begin + 1; + while (end < texts.size() && texts[end].front == texts[begin].front) + ++end; + AOT::sort(texts.mutableSpan().subspan(begin, end - begin), [&](const TextByFront& a, const TextByFront& b) { + return std::ranges::lexicographical_compare(strings[a.id], strings[b.id]); + }); + begin = end; + } + return texts.map([](const TextByFront& text) { + return text.id; + }); +} + void ImageNumbersBuilder::collectNamesAndStringsWithText() { const Typed::Program& program = inputs.program; @@ -307,6 +342,13 @@ void ImageNumbersBuilder::collectNamesAndStringsWithText() numbers.m_textsOfNames.quickSet(name.identifier.text.index()); } numbers.m_otherStringsWithText.exclude(numbers.m_textsOfNames); + + BitVector candidates = numbers.m_stringsOnlyShown; + candidates.exclude(toNumber->numberedConstants); + candidates.exclude(tables.modulePathStrings); + toNumber->codedStrings = sortTextsForCoding(program.strings, candidates, context.parallelLoops()); + for (StringID id : toNumber->codedStrings) + numbers.m_otherStringsWithText.quickClear(id.index()); }, [&] { toNumber->firstFunctionsNamingTexts = findFirstFunctionsNamingTexts(program, mentions, names); @@ -385,6 +427,7 @@ void ImageNumbersBuilder::numberIdentifiersAndStrings() context.statistics().count(StatisticID::NamesWithoutColumn, mentionedWithoutColumn); context.statistics().count(StatisticID::StringConstantsNumbered, constants.count()); + numbers.m_stringNumbers = StringNumbers(numbers.m_namedStrings, WTF::move(toNumber->codedStrings), numberOfStrings, &identifiers); numbers.m_identifiers = makeUniqueWithoutFastMallocCheck(WTF::move(identifiers)); numbers.m_stringConstants = makeUniqueWithoutFastMallocCheck(WTF::move(constants)); destroyInBackground(context.backgroundJobs(), WTF::move(toNumber)); diff --git a/Source/JavaScriptCore/aot/image/AOTTableParts.cpp b/Source/JavaScriptCore/aot/image/AOTTableParts.cpp index 05a00699abfa7..7f36de6a9564d 100644 --- a/Source/JavaScriptCore/aot/image/AOTTableParts.cpp +++ b/Source/JavaScriptCore/aot/image/AOTTableParts.cpp @@ -431,9 +431,10 @@ TableEncoder::TableEncoder(const TableInputs& inputs, const ImageNumbers& number uint32_t TableEncoder::stringNumber(StringID id) const { StringID string = id ? id : program.strings.emptyString(); - if (!numbers.namedStrings().get(string.index())) + std::optional number = numbers.stringNumbers().numberOf(string); + if (!number) fail("StringWithoutRecord"_s, { }, makeString('\'', program.strings.toWTFString(string), '\'')); - return string.index(); + return number.value_or(0); } uint32_t TableEncoder::stringConstantNumber(StringID id) const diff --git a/Source/JavaScriptCore/aot/image/AOTWasmInterface.h b/Source/JavaScriptCore/aot/image/AOTWasmInterface.h index e1e87ad8c9ecf..ed1943a49d23d 100644 --- a/Source/JavaScriptCore/aot/image/AOTWasmInterface.h +++ b/Source/JavaScriptCore/aot/image/AOTWasmInterface.h @@ -127,9 +127,16 @@ enum class ImageStub : uint32_t { ThrowException, ThrowStackOverflow, CatchRunway, + FaultWithoutFrame, +}; +static constexpr uint32_t numberOfImageStubs = 4; +static_assert(static_cast(ImageStub::FaultWithoutFrame) + 1 == numberOfImageStubs); + +enum class ImageEntry : uint16_t { + Primary, + Direct, + DirectWithZero, }; -static constexpr uint32_t numberOfImageStubs = 3; -static_assert(static_cast(ImageStub::CatchRunway) + 1 == numberOfImageStubs); constexpr uint32_t imageSymbolOf(ImageStub stub) { return static_cast(stub); } inline uint32_t imageSymbolOf(FunctionCodeIndex function) { return numberOfImageStubs + static_cast(function.rawIndex()); } @@ -138,10 +145,18 @@ struct DirectoryInImage { uint32_t numberOfModules; uint32_t numberOfDeclinedModules; uint32_t catchRunway; - uint32_t unused; + uint32_t faultWithoutFrame; }; static_assert(sizeof(DirectoryInImage) == 16); +struct FramelessRangeInImage { + friend constexpr auto operator<=>(const FramelessRangeInImage&, const FramelessRangeInImage&) = default; + + uint32_t begin; + uint32_t end; +}; +static_assert(sizeof(FramelessRangeInImage) == 8); + struct FunctionInImage { uint32_t entry; uint32_t size; @@ -176,9 +191,11 @@ struct ModuleInImage { uint32_t numberOfJSEntrypoints; uint32_t jsEntrypoints; uint32_t usesVectors; + uint32_t framelessRanges; + uint32_t numberOfFramelessRanges; uint32_t unused; }; -static_assert(sizeof(ModuleInImage) == 72); +static_assert(sizeof(ModuleInImage) == 80); struct DeclinedModuleInImage { std::array digest; @@ -209,12 +226,13 @@ struct ImagePointerTable { static constexpr uint32_t restoreFrameCallee = numberOfOperations + 3; static constexpr uint32_t restoreFrameReturnAddress = numberOfOperations + 4; static constexpr uint32_t structureIDBase = numberOfOperations + 5; + static constexpr uint32_t throwFromFaultHandler = numberOfOperations + 6; #if GIGACAGE_ENABLED - static constexpr uint32_t primitiveGigacageDisableRequest = numberOfOperations + 6; - static constexpr uint32_t primitiveGigacageBase = numberOfOperations + 7; - static constexpr uint32_t numberOfSharedPointers = numberOfOperations + 8; + static constexpr uint32_t primitiveGigacageDisableRequest = numberOfOperations + 7; + static constexpr uint32_t primitiveGigacageBase = numberOfOperations + 8; + static constexpr uint32_t numberOfSharedPointers = numberOfOperations + 9; #else - static constexpr uint32_t numberOfSharedPointers = numberOfOperations + 6; + static constexpr uint32_t numberOfSharedPointers = numberOfOperations + 7; #endif static constexpr uint32_t slotOf(ImageCodeOperation operation) { return static_cast(operation); } @@ -242,7 +260,9 @@ struct ModuleForImage { Vector javaScriptImportStubs; Vector jsEntrypoints; WTF::Vector savedRegisters; + WTF::Vector framelessRanges; uint32_t catchRunway { 0 }; + uint32_t faultWithoutFrame { 0 }; }; struct StructLayoutInImage { @@ -264,6 +284,8 @@ struct CodeForImage { uint32_t pointerSpace { 0 }; unsigned threadedJumps { 0 }; unsigned firstStateTests { 0 }; + static constexpr GPRReg passedGlobalGPR = GPRInfo::wasmScratchGPR0; + std::optional passedGlobal { }; }; } } // namespace JSC::Wasm diff --git a/Source/JavaScriptCore/aot/image/AOTWasmModules.cpp b/Source/JavaScriptCore/aot/image/AOTWasmModules.cpp index 895227bafbcf2..f76cff99540af 100644 --- a/Source/JavaScriptCore/aot/image/AOTWasmModules.cpp +++ b/Source/JavaScriptCore/aot/image/AOTWasmModules.cpp @@ -323,10 +323,27 @@ class WasmRelocationLowering final : public B3::RelocationLowering { bool m_isOutOfReach { false }; }; +static std::optional findGlobalToPass(const Wasm::ModuleInformation& info) +{ + bool mayUseBaselineTier = info.m_usesTailCalls.loadRelaxed() || info.m_usesLegacyExceptions.loadRelaxed() || info.m_usesModernExceptions.loadRelaxed(); + if (!info.memoryCount() || mayUseBaselineTier) + return std::nullopt; + for (uint32_t index = 0; index < info.globals.size(); ++index) { + const Wasm::GlobalInformation& global = info.globals[index]; + if (global.mutability != Wasm::Mutability::Mutable || !global.type.isI32()) + continue; + if (global.bindingMode == Wasm::GlobalInformation::BindingMode::EmbeddedInInstance) + return index; + return std::nullopt; + } + return std::nullopt; +} + struct WasmModuleCompiler { Ref module; Ref group; const Wasm::ModuleInformation& info { module->moduleInformation() }; + std::optional passedGlobal { findGlobalToPass(info) }; Vector jsEntrypointTypes { }; Wasm::ImagePointerTable pointers { }; UncheckedKeyHashMap typeIndices { }; @@ -369,6 +386,69 @@ struct WasmModuleCompiler { return layout; } + static uint32_t findZeroBranchTarget(std::span code) + { +#if CPU(X86_64) + static_assert(GPRInfo::argumentGPR0 == X86Registers::edi); + auto displacementAt = [&](size_t offset, size_t size) { + int32_t displacement = static_cast(code[offset]); + if (size == sizeof(int32_t)) + memcpySpan(asMutableByteSpan(displacement), code.subspan(offset, size)); + return displacement; + }; + constexpr uint8_t shortJumpIfZero = 0x74; + constexpr uint8_t shortJumpIfNotZero = 0x75; + constexpr std::array testEDI { 0x85, 0xff }; + if (code.size() < 16 || !spanHasPrefix(code, std::span { testEDI })) + return 0; + size_t offset = testEDI.size(); + size_t size = 1; + uint8_t opcode = code[offset++]; + if (opcode == 0x0f) { + opcode = code[offset++] - 0x10; + size = sizeof(int32_t); + } + uint32_t next = static_cast(offset + size); + if (opcode == shortJumpIfNotZero) + return next; + if (opcode == shortJumpIfZero) + return next + displacementAt(offset, size); + return 0; +#elif CPU(ARM64) + uint32_t instruction; + if (code.size() < sizeof(instruction)) + return 0; + memcpySpan(asMutableByteSpan(instruction), code.first(sizeof(instruction))); + constexpr uint32_t compareAndBranchOfW0 = 0xfe00001f; + constexpr uint32_t branchIfZero = 0x34000000; + static_assert(GPRInfo::argumentGPR0 == ARM64Registers::x0); + if ((instruction & compareAndBranchOfW0 & ~0x01000000u) != branchIfZero) + return 0; + bool branchesIfNotZero = instruction & 0x01000000; + int32_t displacement = static_cast(instruction << 8) >> 13 << 2; + return branchesIfNotZero ? sizeof(instruction) : displacement; +#else + UNUSED_PARAM(code); + return 0; +#endif + } + + static Vector findFramelessRanges(const B3::Air::Code& code, ImageLinkBuffer& buffer) + { + Vector result; + uint32_t begin = 0; + auto append = [&](uint32_t end) { + if (begin < end) + result.append({ begin, end }); + }; + for (const B3::Air::FrameRange& range : code.frameRanges()) { + append(buffer.linkBuffer().offsetOf(range.begin)); + begin = buffer.linkBuffer().offsetOf(range.end); + } + append(static_cast(buffer.bytes().size())); + return result; + } + WasmFunctionFailure compileWithOptimizingTier(Wasm::FunctionCodeIndex functionIndex, Wasm::ImageCodeFlavor flavor, BuildContext& buildContext, WasmFunctionCode& result) const { Wasm::FunctionSpaceIndex functionIndexSpace = info.toSpaceIndex(functionIndex); @@ -385,19 +465,45 @@ struct WasmModuleCompiler { policy.upstream.allowsSIMD = true; bool usesTraps = flavor == Wasm::ImageCodeFlavor::Trapping && info.memoryCount(); policy.upstream.usesWasmFaultSignalHandler = usesTraps; + if (usesTraps) + policy.air.registers.unsavedCalleeSaves.add(GPRInfo::wasmBoundsCheckingSizeRegister, IgnoreVectors); policy.optimization.narrowsIntegers = true; + policy.optimization.reducesWithRanges = true; policy.selection.extendsInstructionSelection = true; policy.air.layout.mergesIdenticalBlocks = true; + policy.air.frame.emitsFrame = true; + policy.air.frame.shrinkWraps = true; + policy.air.frame.needsLikelyFramelessPath = true; + policy.air.frame.hoistsFrameAboveSwitches = true; + policy.air.frame.maxBytesAllocatedBeforeStackCheck = 4 * KB; + policy.air.frame.registersClobberedByFrameSetup = RegisterSet::macroClobberedGPRs(); + policy.air.registers.chargesForCalleeSaves = true; + policy.air.splitting.splitsAroundClobbersEarly = true; + policy.optimization.estimatesFrequencies = true; + policy.optimization.threadsJumps = true; + policy.optimization.foldsImpliedComparisons = true; + policy.optimization.splitsBranchConditions = true; + policy.optimization.sinksValues = true; + policy.optimization.placesConstantsByFrequency = true; + policy.optimization.mergesAdjacentConstantStores = true; + policy.air.registers.usesSecondPreferredRegister = true; + policy.air.peepholes.hoistsCommonMoves = true; policy.addressing.isPositionIndependent = true; policy.addressing.dataSectionSpace = static_cast(WasmRelocationSpace::Constants); policy.addressing.jumpTableSpace = static_cast(WasmRelocationSpace::JumpTables); policy.addressing.relocationLowering = &lowering; policy.air.peepholes.emitsShufflesInPairOrder = true; + policy.air.peepholes.reusesConditionFlags = true; policy.targetFeatures = assemblerPolicy.features; Wasm::CodeForImage codeForImage { pointers, typeIndices, structLayouts, recorder, flavor }; codeForImage.dependsOnFlavor = true; codeForImage.policy = &policy; codeForImage.pointerSpace = static_cast(WasmRelocationSpace::Pointers); + if (usesTraps) { + codeForImage.passedGlobal = passedGlobal; + policy.air.frame.sinksFrameBelowEntry = passedGlobal && Wasm::isResumable(info, functionIndex, *passedGlobal); + policy.air.registers.zeroExtended.add(Wasm::CodeForImage::passedGlobalGPR, IgnoreVectors); + } context.codeForImage = &codeForImage; MemoryMode mode = usesTraps ? MemoryMode::Signaling : MemoryMode::BoundsChecking; auto compiled = Wasm::parseAndCompileOMG(context, profiledCallee.get(), callee.get(), info.functions[functionIndex], signature, unlinkedCalls, module.get(), group.get(), info, mode, Wasm::CompilationMode::OMGMode, functionIndex, UINT32_MAX); @@ -426,9 +532,17 @@ struct WasmModuleCompiler { result.usesVectors = false; result.code = RelocatableCode { Vector(buffer.bytes()), WTF::move(*relocations) }; result.savedRegisters = *savedRegisters; + result.framelessRanges = findFramelessRanges(context.procedure->code(), buffer); result.dependsOnFlavor = true; result.threadedJumps = codeForImage.threadedJumps; result.firstStateTests = codeForImage.firstStateTests; + if (codeForImage.passedGlobal) { + result.directEntry = buffer.linkBuffer().offsetOf(context.directEntry); + if (auto& withZero = context.directEntryWithZero) + result.directEntryWithZero = buffer.linkBuffer().offsetOf(*withZero); + else + result.directEntryWithZero = result.directEntry + findZeroBranchTarget(buffer.bytes().subspan(result.directEntry)); + } for (size_t directCallee : (*compiled)->outgoingJITDirectCallees) result.directCallees.set(directCallee); return { }; @@ -550,6 +664,13 @@ struct WasmModuleCompiler { stubOf(Wasm::ImageStub::CatchRunway) = generate("stub"_s, 2, [](CCallHelpers& jit, RelocationRecorder&) { Wasm::emitCatchInWasm(jit); }); + stubOf(Wasm::ImageStub::FaultWithoutFrame) = generate("stub"_s, 3, [](CCallHelpers& jit, RelocationRecorder&) { + jit.emitFunctionPrologue(); + jit.storePtr(GPRInfo::wasmContextInstancePointer, CCallHelpers::addressFor(CallFrameSlot::codeBlock)); + jit.loadPtr(CCallHelpers::Address(GPRInfo::wasmContextInstancePointer, JSWebAssemblyInstance::offsetOfImageCodePointers()), GPRInfo::argumentGPR1); + jit.loadPtr(CCallHelpers::Address(GPRInfo::argumentGPR1, Wasm::ImagePointerTable::throwFromFaultHandler * sizeof(void*)), GPRInfo::argumentGPR1); + jit.farJump(GPRInfo::argumentGPR1, CFunctionPtrTag); + }); for (unsigned index = 0; index < info.importFunctionCount(); ++index) { code.importStubs.append(generateWithoutRelocations("stub of import"_s, index, [&](CCallHelpers& jit) { @@ -665,12 +786,29 @@ static void compileFunctions(std::span> } } std::ranges::stable_sort(tasks, std::ranges::greater { }, &WasmFunctionTask::bodySize); - forEachIndexInParallel(context.parallelLoops(), safeCast(tasks.size()), [&](unsigned index) { - const WasmFunctionTask& task = tasks[index]; - bool isChecking = task.flavor == Wasm::ImageCodeFlavor::Checking; - WasmFunctionCode& result = (isChecking ? task.module->code.functionsThatCheck : task.module->code.functions)[task.function]; - (isChecking ? task.module->checkingFailures : task.module->failures)[task.function] = task.module->compiler.compileFunction(Wasm::FunctionCodeIndex(task.function), task.flavor, context, result); - }); + auto compile = [&] { + forEachIndexInParallel(context.parallelLoops(), safeCast(tasks.size()), [&](unsigned index) { + const WasmFunctionTask& task = tasks[index]; + bool isChecking = task.flavor == Wasm::ImageCodeFlavor::Checking; + WasmFunctionCode& result = (isChecking ? task.module->code.functionsThatCheck : task.module->code.functions)[task.function]; + result = { }; + (isChecking ? task.module->checkingFailures : task.module->failures)[task.function] = task.module->compiler.compileFunction(Wasm::FunctionCodeIndex(task.function), task.flavor, context, result); + }); + }; + compile(); + Vector modulesWithoutDirectEntries; + for (auto& module : modules) { + if (!module || !module->compiler.passedGlobal || std::ranges::all_of(module->code.functions, &WasmFunctionCode::isOptimized)) + continue; + module->compiler.passedGlobal = std::nullopt; + modulesWithoutDirectEntries.append(module.get()); + } + if (!modulesWithoutDirectEntries.isEmpty()) { + tasks.removeAllMatching([&](const WasmFunctionTask& task) { + return task.flavor == Wasm::ImageCodeFlavor::Checking || !modulesWithoutDirectEntries.contains(task.module); + }); + compile(); + } for (auto& module : modules) { if (!module) continue; @@ -698,7 +836,9 @@ static WasmModuleOrDecline finishWasmModule(PendingWasmModule& module, const Cod unsigned baselineCheckingCopies = 0; unsigned threadedJumps = 0; unsigned firstStateTests = 0; + unsigned directEntries = 0; auto countCopy = [&](const WasmFunctionCode& copy) { + directEntries += !!copy.directEntry; ++(copy.isOptimized ? optimizedCopies : baselineCopies); threadedJumps += copy.threadedJumps; firstStateTests += copy.firstStateTests; @@ -717,6 +857,7 @@ static WasmModuleOrDecline finishWasmModule(PendingWasmModule& module, const Cod context.statistics().count(StatisticID::WasmBaselineCheckingCopies, baselineCheckingCopies); context.statistics().count(StatisticID::WasmJumpsThreaded, threadedJumps); context.statistics().count(StatisticID::WasmFirstStateTests, firstStateTests); + context.statistics().count(StatisticID::WasmDirectEntries, directEntries); return linkWasmModule(WTF::move(module.code), templates, context); } @@ -758,10 +899,11 @@ static BitVector findFunctionsWithTwoCopies(const WasmModuleCode& module) namespace { struct ModuleUnits { - uint32_t append(const RelocatableCode& code, std::optional flavor = std::nullopt) + uint32_t append(const RelocatableCode& code, std::optional flavor = std::nullopt, uint32_t directEntry = 0, uint32_t directEntryWithZero = 0) { units.append(LinkUnit { code.bytes.span(), code.relocations.span(), codeUnitAlignment }); flavors.append(flavor); + directEntries.append({ directEntry, directEntryWithZero }); return size() - 1; } @@ -769,6 +911,7 @@ struct ModuleUnits { { units.append(LinkUnit { bytes.span(), { }, codeUnitAlignment }); flavors.append(std::nullopt); + directEntries.append({ 0, 0 }); return size() - 1; } @@ -776,6 +919,7 @@ struct ModuleUnits { Vector units; Vector> flavors; + Vector> directEntries; uint32_t firstStub { 0 }; uint32_t firstImportStub { 0 }; uint32_t firstJavaScriptImportStub { 0 }; @@ -814,7 +958,7 @@ static ModuleUnits collectUnits(const WasmModuleCode& module, const BitVector& f auto& trapping = units.functions[static_cast(Wasm::ImageCodeFlavor::Trapping)]; auto& checking = units.functions[static_cast(Wasm::ImageCodeFlavor::Checking)]; for (uint32_t function = 0; function < module.functions.size(); ++function) - trapping.append(units.append(module.functions[function].code, Wasm::ImageCodeFlavor::Trapping)); + trapping.append(units.append(module.functions[function].code, Wasm::ImageCodeFlavor::Trapping, module.functions[function].directEntry, module.functions[function].directEntryWithZero)); for (uint32_t function = 0; function < module.functions.size(); ++function) { if (functionsWithTwoCopies.get(function)) checking.append(units.append(codeOf(module, function, Wasm::ImageCodeFlavor::Checking).code, Wasm::ImageCodeFlavor::Checking)); @@ -835,7 +979,12 @@ static std::optional resolve(const ModuleUnits& units, uint32_t unit std::optional flavor = units.flavors[unit]; if (!flavor || function >= units.functions[0].size()) return std::nullopt; - return UnitTarget { units.functions[static_cast(*flavor)][function] }; + uint32_t target = units.functions[static_cast(*flavor)][function]; + if (static_cast(relocation.entry) == Wasm::ImageEntry::Primary) + return UnitTarget { target }; + uint32_t offset = units.directEntries[target][static_cast(relocation.entry) == Wasm::ImageEntry::DirectWithZero]; + RELEASE_ASSERT(offset); + return UnitTarget { target, offset }; } static Wasm::ImageDeclineReason reportSectionLinkFailure(const SectionLinkFailure& failure, BuildContext& context) @@ -870,6 +1019,7 @@ WasmModuleOrDecline linkWasmModule(WasmModuleCode&& module, const CodeTemplates& result.digest = module.digest; result.sourceSize = module.sourceSize; result.catchRunway = offsets[units.firstStub + static_cast(Wasm::ImageStub::CatchRunway)]; + result.faultWithoutFrame = offsets[units.firstStub + static_cast(Wasm::ImageStub::FaultWithoutFrame)]; result.importStubs.append(offsets.span().subspan(units.firstImportStub, module.importStubs.size())); result.javaScriptImportStubs.append(offsets.span().subspan(units.firstJavaScriptImportStub, module.javaScriptImportStubs.size())); @@ -886,6 +1036,8 @@ WasmModuleOrDecline linkWasmModule(WasmModuleCode&& module, const CodeTemplates& handler.code += entry; result.handlers.append(handler); } + for (Wasm::FramelessRangeInImage range : function.framelessRanges) + result.framelessRanges.append({ entry + range.begin, entry + range.end }); return Wasm::FunctionInImage { entry, static_cast(function.code.bytes.size()), firstHandler, static_cast(function.handlers.size()), function.savedRegistersIndex, function.usesVectors, 0 }; }; auto& trapping = result.functions[static_cast(Wasm::ImageCodeFlavor::Trapping)]; @@ -901,6 +1053,9 @@ WasmModuleOrDecline linkWasmModule(WasmModuleCode&& module, const CodeTemplates& checking.append(functionsWithTwoCopies.get(index) ? describe(index, Wasm::ImageCodeFlavor::Checking) : trapping[index]); } + std::ranges::sort(result.framelessRanges); + auto duplicates = std::ranges::unique(result.framelessRanges); + result.framelessRanges.shrink(duplicates.begin() - result.framelessRanges.begin()); result.code = WTF::move(section->bytes); result.savedRegisters = WTF::move(module.savedRegisters); return result; @@ -929,6 +1084,10 @@ static Wasm::ModuleInImage appendModule(Vector& code, Vector& if (static_cast(handler.type) != Wasm::HandlerType::Delegate) handler.code += base; } + for (auto& range : module.framelessRanges) { + range.begin += base; + range.end += base; + } for (uint32_t& stub : module.importStubs) stub += base; for (uint32_t& stub : module.javaScriptImportStubs) @@ -952,6 +1111,8 @@ static Wasm::ModuleInImage appendModule(Vector& code, Vector& record.numberOfJSEntrypoints = static_cast(module.jsEntrypoints.size()); record.jsEntrypoints = appendToDirectory(directory, asBytes(module.jsEntrypoints.span())); record.savedRegisters = appendToDirectory(directory, asBytes(module.savedRegisters.span())); + record.numberOfFramelessRanges = static_cast(module.framelessRanges.size()); + record.framelessRanges = appendToDirectory(directory, asBytes(module.framelessRanges.span())); return record; } @@ -963,7 +1124,7 @@ bool WasmSectionBuilder::append(Wasm::ModuleForImage&& module) uint64_t base = WTF::roundUpToMultipleOf(m_codeSize); uint64_t codeSize = base + module.code.size(); uint64_t directorySize = m_directorySize + sizeof(Wasm::ModuleInImage) + arraySize(module.functions[0]) + arraySize(module.functions[1]) + arraySize(module.handlers) - + arraySize(module.importStubs) + arraySize(module.javaScriptImportStubs) + arraySize(module.jsEntrypoints) + arraySize(module.savedRegisters); + + arraySize(module.importStubs) + arraySize(module.javaScriptImportStubs) + arraySize(module.jsEntrypoints) + arraySize(module.savedRegisters) + arraySize(module.framelessRanges); if (codeSize > maxSectionOffset || directorySize > maxSectionOffset) return false; m_modules.append(PlacedModule { WTF::move(module), static_cast(base) }); @@ -1011,8 +1172,10 @@ WebAssemblyImageInputs WasmSectionBuilder::finish(BuildContext& context) section.directory.reserveInitialCapacity(m_directorySize); Wasm::DirectoryInImage header { section.numberOfModules, static_cast(m_declinedModules.size()), 0, 0 }; - if (!m_modules.isEmpty()) + if (!m_modules.isEmpty()) { header.catchRunway = m_modules[0].base + m_modules[0].module.catchRunway; + header.faultWithoutFrame = m_modules[0].base + m_modules[0].module.faultWithoutFrame; + } appendToDirectory(section.directory, asByteSpan(header)); size_t recordsOffset = section.directory.size(); section.directory.insertFill(recordsOffset, 0, m_modules.size() * sizeof(Wasm::ModuleInImage)); diff --git a/Source/JavaScriptCore/aot/image/AOTWasmModules.h b/Source/JavaScriptCore/aot/image/AOTWasmModules.h index 19183f5c49ec3..3fd624eb529df 100644 --- a/Source/JavaScriptCore/aot/image/AOTWasmModules.h +++ b/Source/JavaScriptCore/aot/image/AOTWasmModules.h @@ -40,12 +40,15 @@ struct WasmFunctionCode { Vector handlers; BitVector directCallees; Wasm::SavedRegistersInImage savedRegisters { 0, 0, 0 }; + Vector framelessRanges; uint16_t savedRegistersIndex { 0 }; bool usesVectors { false }; bool dependsOnFlavor { false }; bool isOptimized { false }; unsigned threadedJumps { 0 }; unsigned firstStateTests { 0 }; + uint32_t directEntry { 0 }; + uint32_t directEntryWithZero { 0 }; }; struct WasmJSEntrypointCode { diff --git a/Source/JavaScriptCore/aot/ir/AOTBodyIndex.cpp b/Source/JavaScriptCore/aot/ir/AOTBodyIndex.cpp index a600586f54faa..2020d2114b09d 100644 --- a/Source/JavaScriptCore/aot/ir/AOTBodyIndex.cpp +++ b/Source/JavaScriptCore/aot/ir/AOTBodyIndex.cpp @@ -123,7 +123,7 @@ size_t BodyIndexTable::sizeInBytes() const result += capacityInBytes(worker->edges) + capacityInBytes(worker->retreatingEdges); result += capacityInBytes(worker->firstPredecessor) + capacityInBytes(worker->predecessors); result += capacityInBytes(worker->sizesOfSubtrees) + capacityInBytes(worker->nextNumbersBelow); - result += capacityInBytes(worker->worklist) + capacityInBytes(worker->loops) + capacityInBytes(worker->mentionsOfIterations); + result += capacityInBytes(worker->worklist) + capacityInBytes(worker->loops) + capacityInBytes(worker->mentionsOfIterations) + capacityInBytes(worker->blocks); } return result; } @@ -201,12 +201,41 @@ const BodyIndex& BodyIndexBuilder::finish() adoptMentionsOfIterations(); computeDominators(); numberDominatorTree(); - m_index->m_blocksInDominatorOrder = m_blocksInDominatorOrder.first(m_blocks[entryBlock.index()].lastDominatorNumberBelow + 1); + m_index->m_numberOfReachableBlocks = m_blocks[entryBlock.index()].lastDominatorNumberBelow + 1; findLoops(); adoptLoops(); + adoptBlocks(); return *m_index; } +void BodyIndexBuilder::adoptBlocks() +{ + if (!m_hasNarrowNumbers) { + std::span memory = m_worker.allocate(bytesOfArrayInBodyIndex(m_blocks.size())); + std::span blocks = takeArrayOfBodyIndex(memory, m_blocks.size()); + std::ranges::copy(m_blocks, blocks.begin()); + m_index->m_wideBlocks = blocks.data(); + return; + } + using NarrowBlockPlace = BodyIndex::NarrowBlockPlace; + std::span memory = m_worker.allocate(bytesOfArrayInBodyIndex(m_blocks.size())); + std::span blocks = takeArrayOfBodyIndex(memory, m_blocks.size()); + for (size_t block = 0; block < m_blocks.size(); ++block) + new (NotNull, &blocks[block]) NarrowBlockPlace(m_blocks[block]); + m_index->m_narrowBlocks = blocks.data(); + m_index->m_hasNarrowBlocks = true; +} + +BodyIndex::RareParts& BodyIndexBuilder::ensureRareParts() +{ + if (!m_rareParts) { + std::span memory = m_worker.allocate(bytesOfArrayInBodyIndex(1)); + m_rareParts = new (NotNull, takeArrayOfBodyIndex(memory, 1).data()) BodyIndex::RareParts; + m_index->m_rareParts = m_rareParts; + } + return *m_rareParts; +} + void BodyIndexBuilder::begin() { const Body& body = m_function.body; @@ -214,26 +243,36 @@ void BodyIndexBuilder::begin() size_t numberOfValues = body.numberOfValues(); RELEASE_ASSERT(body.words().size() < BodyIndex::isParameterBit); + m_hasNarrowNumbers = BodyIndex::fitsNarrowNumbers(body); std::span memory = m_worker.allocate(bytesOfArrayInBodyIndex(1) - + bytesOfArrayInBodyIndex(numberOfBlocks) + bytesOfArrayInBodyIndex(numberOfBlocks) - + 2 * bytesOfArrayInBodyIndex(numberOfValues) + + bytesOfArrayInBodyIndex(numberOfValues) + + (m_hasNarrowNumbers ? bytesOfArrayInBodyIndex(numberOfValues) : bytesOfArrayInBodyIndex(numberOfValues)) + bytesOfArrayInBodyIndex(numberOfValues)); m_index = new (NotNull, takeArrayOfBodyIndex(memory, 1).data()) BodyIndex; - m_blocks = takeArrayOfBodyIndex(memory, numberOfBlocks); m_blocksInDominatorOrder = takeArrayOfBodyIndex(memory, numberOfBlocks); m_placesOfValues = takeArrayOfBodyIndex(memory, numberOfValues); - m_onlyUsersOfValues = takeArrayOfBodyIndex(memory, numberOfValues); + if (m_hasNarrowNumbers) + m_narrowOnlyUsersOfValues = takeArrayOfBodyIndex(memory, numberOfValues); + else + m_onlyUsersOfValues = takeArrayOfBodyIndex(memory, numberOfValues); m_usesOfValues = takeArrayOfBodyIndex(memory, numberOfValues); - std::uninitialized_default_construct(m_blocks.begin(), m_blocks.end()); + m_worker.blocks.shrink(0); + m_worker.blocks.grow(numberOfBlocks); + m_blocks = m_worker.blocks.mutableSpan(); m_index->m_body = body; - m_index->m_blocks = m_blocks; - m_index->m_placesOfValues = m_placesOfValues; - m_index->m_onlyUsersOfValues = m_onlyUsersOfValues; - m_index->m_usesOfValues = m_usesOfValues; + m_index->m_wideBlocks = m_blocks.data(); + m_index->m_blocksInDominatorOrder = m_blocksInDominatorOrder.data(); + m_index->m_placesOfValues = m_placesOfValues.data(); + if (m_hasNarrowNumbers) + m_index->m_narrowOnlyUsersOfValues = m_narrowOnlyUsersOfValues.data(); + else + m_index->m_onlyUsersOfValues = m_onlyUsersOfValues.data(); + m_index->m_hasNarrowUsers = m_hasNarrowNumbers; + m_index->m_usesOfValues = m_usesOfValues.data(); m_worker.edges.shrink(0); m_worker.retreatingEdges.shrink(0); @@ -378,7 +417,7 @@ void BodyIndexBuilder::adoptMentionsOfIterations() std::span memory = m_worker.allocate(bytesOfArrayInBodyIndex(found.size())); std::span mentions = takeArrayOfBodyIndex(memory, found.size()); std::ranges::copy(found, mentions.begin()); - m_index->m_mentionsOfIterations = mentions; + ensureRareParts().mentionsOfIterations = mentions; } void BodyIndexBuilder::adoptLoops() @@ -465,8 +504,9 @@ void BodyIndexBuilder::adoptLoops() }); } std::ranges::copy(found, loops.begin()); - m_index->m_loops = loops; - m_index->m_membersOfLoops = members; + BodyIndex::RareParts& rareParts = ensureRareParts(); + rareParts.loops = loops; + rareParts.membersOfLoops = members; } ValueID BodyIndex::stripCastsSlow(ValueID id) const @@ -512,7 +552,7 @@ std::optional BodyIndex::edgeResultOfAnyTerminator(const BodyTer result.isOnTakenEdge = !targetIndex; } else result.caseIndex = targetIndex; - result.parameter = ValueID(m_blocks[target.block().index()].firstParameter + static_cast(target.arguments().size()) + proofIndex); + result.parameter = ValueID(firstParameterOf(target.block()) + static_cast(target.arguments().size()) + proofIndex); return result; } diff --git a/Source/JavaScriptCore/aot/ir/AOTBodyIndex.h b/Source/JavaScriptCore/aot/ir/AOTBodyIndex.h index 5bf99e1a443f2..842ad3cc5ba70 100644 --- a/Source/JavaScriptCore/aot/ir/AOTBodyIndex.h +++ b/Source/JavaScriptCore/aot/ir/AOTBodyIndex.h @@ -16,9 +16,12 @@ #include #include #include +#include #include #include +WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN + namespace JSC { namespace AOT { namespace Typed { class Program; @@ -182,10 +185,10 @@ class BodyIndex { bool holdsAnyOf(const OpcodeSet& wanted) const { return m_opcodes.intersects(wanted); } BlockID blockOf(ValueID id) const { - if (m_blocks.size() < 2 || id.index() < m_blocks[1].firstParameter) + uint32_t numberOfBlocks = this->numberOfBlocks(); + if (numberOfBlocks < 2) return BlockID(0u); - auto next = std::ranges::upper_bound(m_blocks, id.index(), { }, &BlockPlace::firstParameter); - return BlockID(static_cast(next - m_blocks.begin()) - 1); + return m_hasNarrowBlocks ? findBlockOf(m_narrowBlocks, numberOfBlocks, id) : findBlockOf(m_wideBlocks, numberOfBlocks, id); } uint32_t useCount(ValueID id) const { return m_usesOfValues[id.index()] & maxCountedUses; } @@ -195,11 +198,7 @@ class BodyIndex { { if (!hasOneUse(id)) return std::nullopt; - auto kind = static_cast(m_usesOfValues[id.index()] >> bitsOfUseCount); - uint32_t user = m_onlyUsersOfValues[id.index()]; - if (kind == BodyUseKind::Operand) - return BodyUser { kind, ValueID(user), { } }; - return BodyUser { kind, { }, BlockID(user) }; + return userOf(id); } ValueID onlyUsingValue(ValueID id) const { @@ -210,18 +209,17 @@ class BodyIndex { { if (isUnused(id)) return std::nullopt; - auto kind = static_cast(m_usesOfValues[id.index()] >> bitsOfUseCount); - uint32_t user = m_onlyUsersOfValues[id.index()]; - if (kind == BodyUseKind::Operand) - return BodyUser { kind, ValueID(user), { } }; - return BodyUser { kind, { }, BlockID(user) }; + return userOf(id); } std::span mentionsOfIteration(ValueID open) const { - auto found = std::ranges::equal_range(m_mentionsOfIterations, open.index(), { }, [](const BodyMentionOfIteration& mention) { + if (!m_rareParts) + return { }; + std::span mentions = m_rareParts->mentionsOfIterations; + auto found = std::ranges::equal_range(mentions, open.index(), { }, [](const BodyMentionOfIteration& mention) { return mention.open.index(); }); - return m_mentionsOfIterations.subspan(found.begin() - m_mentionsOfIterations.begin(), found.size()); + return mentions.subspan(found.begin() - mentions.begin(), found.size()); } ValueID stripCasts(ValueID id) const { @@ -232,47 +230,81 @@ class BodyIndex { BodyBlock block(BlockID id) const { - const BlockPlace& place = m_blocks[id.index()]; - return m_body.blockAt(place.wordOffset, place.wordOffsetOfTerminator, id, ValueID(place.firstParameter)); + return withBlockPlace(id, [&](const auto& place) ALWAYS_INLINE_LAMBDA { + return m_body.blockAt(place.wordOffset, place.wordOffsetOfTerminator, id, ValueID(static_cast(place.firstParameter))); + }); + } + ALWAYS_INLINE BodyTerminator terminator(BlockID id) const + { + return withBlockPlace(id, [&](const auto& place) ALWAYS_INLINE_LAMBDA { + return m_body.terminatorAt(place.wordOffsetOfTerminator, id); + }); } - ALWAYS_INLINE BodyTerminator terminator(BlockID id) const { return m_body.terminatorAt(m_blocks[id.index()].wordOffsetOfTerminator, id); } - bool isReachable(BlockID id) const { return m_blocks[id.index()].flags & IsReachable; } - uint32_t numberOfIncomingEdges(BlockID id) const { return m_blocks[id.index()].numberOfIncomingEdges; } - bool isEnteredByHandlerEdge(BlockID id) const { return m_blocks[id.index()].flags & IsEnteredByHandlerEdge; } + bool isReachable(BlockID id) const { return flagsOf(id) & IsReachable; } + uint32_t numberOfIncomingEdges(BlockID id) const + { + return withBlockPlace(id, [](const auto& place) ALWAYS_INLINE_LAMBDA -> uint32_t { + return place.numberOfIncomingEdges; + }); + } + bool isEnteredByHandlerEdge(BlockID id) const { return flagsOf(id) & IsEnteredByHandlerEdge; } std::optional onlyIncomingEdge(BlockID id) const { - const BlockPlace& place = m_blocks[id.index()]; - if (place.numberOfIncomingEdges != 1 || place.flags & IsEnteredByHandlerEdge || !place.immediateDominator) - return std::nullopt; - RELEASE_ASSERT(place.wordOffsetOfFirstIncomingTarget); - return BodyIncomingEdge { place.immediateDominator, place.indexOfFirstIncomingTarget, BodyTarget(&m_body.words()[place.wordOffsetOfFirstIncomingTarget]) }; + return withBlockPlace(id, [&](const auto& place) ALWAYS_INLINE_LAMBDA -> std::optional { + BlockID from = place.dominator(); + if (place.numberOfIncomingEdges != 1 || place.flags & IsEnteredByHandlerEdge || !from) + return std::nullopt; + RELEASE_ASSERT(place.wordOffsetOfFirstIncomingTarget); + return BodyIncomingEdge { from, place.indexOfFirstIncomingTarget, BodyTarget(&m_body.words()[place.wordOffsetOfFirstIncomingTarget]) }; + }); } - BlockID immediateDominator(BlockID id) const { return m_blocks[id.index()].immediateDominator; } - std::span blocksInDominatorOrder() const { return m_blocksInDominatorOrder; } - uint32_t dominatorNumber(BlockID id) const { return m_blocks[id.index()].dominatorNumber; } - uint32_t lastDominatorNumberBelow(BlockID id) const { return m_blocks[id.index()].lastDominatorNumberBelow; } + BlockID immediateDominator(BlockID id) const + { + return withBlockPlace(id, [](const auto& place) ALWAYS_INLINE_LAMBDA { + return place.dominator(); + }); + } + std::span blocksInDominatorOrder() const { return unsafeMakeSpan(m_blocksInDominatorOrder, m_numberOfReachableBlocks); } + uint32_t dominatorNumber(BlockID id) const + { + return withBlockPlace(id, [](const auto& place) ALWAYS_INLINE_LAMBDA -> uint32_t { + return place.dominatorNumber; + }); + } + uint32_t lastDominatorNumberBelow(BlockID id) const + { + return withBlockPlace(id, [](const auto& place) ALWAYS_INLINE_LAMBDA -> uint32_t { + return place.lastDominatorNumberBelow; + }); + } bool dominates(BlockID from, BlockID to) const { - if (!isReachable(from) || !isReachable(to)) - return false; - uint32_t number = dominatorNumber(to); - return dominatorNumber(from) <= number && number <= lastDominatorNumberBelow(from); + return m_hasNarrowBlocks ? dominatesAmong(m_narrowBlocks, from, to) : dominatesAmong(m_wideBlocks, from, to); } bool strictlyDominates(BlockID from, BlockID to) const { return from != to && dominates(from, to); } - uint32_t numberOfLoops() const { return static_cast(m_loops.size()); } - bool hasLoops() const { return !m_loops.empty(); } - const BodyLoop& loop(LoopID id) const { return m_loops[id.index()]; } - std::span membersOf(LoopID id) const { return m_membersOfLoops.subspan(m_loops[id.index()].firstMember, m_loops[id.index()].numberOfMembers); } - LoopID loopOf(BlockID id) const { return m_blocks[id.index()].loop; } - bool isLoopHead(BlockID id) const { return m_blocks[id.index()].flags & IsLoopHead; } + uint32_t numberOfLoops() const { return m_rareParts ? static_cast(m_rareParts->loops.size()) : 0; } + bool hasLoops() const { return numberOfLoops(); } + const BodyLoop& loop(LoopID id) const { return m_rareParts->loops[id.index()]; } + std::span membersOf(LoopID id) const { return m_rareParts->membersOfLoops.subspan(loop(id).firstMember, loop(id).numberOfMembers); } + LoopID loopOf(BlockID id) const + { + return withBlockPlace(id, [](const auto& place) ALWAYS_INLINE_LAMBDA { + return place.innermostLoop(); + }); + } + bool isLoopHead(BlockID id) const { return flagsOf(id) & IsLoopHead; } bool isInLoop(BlockID block, LoopID loop) const { LoopID holder = loopOf(block); - return holder && m_loops[holder.index()].placeAmongLoops - m_loops[loop.index()].placeAmongLoops < m_loops[loop.index()].numberOfLoopsWithin; + return holder && this->loop(holder).placeAmongLoops - this->loop(loop).placeAmongLoops < this->loop(loop).numberOfLoopsWithin; + } + uint32_t loopDepthOf(BlockID id) const + { + LoopID holder = loopOf(id); + return holder ? loop(holder).depth : 0; } - uint32_t loopDepthOf(BlockID id) const { return loopOf(id) ? m_loops[loopOf(id).index()].depth : 0; } bool hasIrreducibleFlow() const { return m_hasIrreducibleFlow; } JS_EXPORT_PRIVATE std::optional edgeResult(BlockID from, uint32_t targetIndex, uint32_t proofIndex) const; @@ -304,7 +336,7 @@ class BodyIndex { result.isOnTakenEdge = !targetIndex; } else result.caseIndex = targetIndex; - result.parameter = ValueID(m_blocks[target.block().index()].firstParameter + static_cast(target.arguments().size()) + proofIndex); + result.parameter = ValueID(firstParameterOf(target.block()) + static_cast(target.arguments().size()) + proofIndex); return result; } JS_EXPORT_PRIVATE std::optional edgeResultOfAnyTerminator(const BodyTerminator&, uint32_t targetIndex, BodyTarget, uint32_t proofIndex) const; @@ -355,6 +387,9 @@ class BodyIndex { }; struct BlockPlace { + BlockID dominator() const { return immediateDominator; } + LoopID innermostLoop() const { return loop; } + uint32_t wordOffset { 0 }; uint32_t wordOffsetOfTerminator { 0 }; uint32_t firstParameter { 0 }; @@ -369,21 +404,120 @@ class BodyIndex { }; static_assert(sizeof(BlockPlace) == 40); + struct NarrowBlockPlace { + explicit NarrowBlockPlace(const BlockPlace& place) + : wordOffset(static_cast(place.wordOffset)) + , wordOffsetOfTerminator(static_cast(place.wordOffsetOfTerminator)) + , firstParameter(static_cast(place.firstParameter)) + , dominatorPlusOne(static_cast(place.immediateDominator.bits() + 1)) + , loopPlusOne(static_cast(place.loop.bits() + 1)) + , dominatorNumber(static_cast(place.dominatorNumber)) + , lastDominatorNumberBelow(static_cast(place.lastDominatorNumberBelow)) + , indexOfFirstIncomingTarget(static_cast(place.indexOfFirstIncomingTarget)) + , wordOffsetOfFirstIncomingTarget(static_cast(place.wordOffsetOfFirstIncomingTarget)) + , numberOfIncomingEdges(place.numberOfIncomingEdges) + , flags(place.flags) + { + } + + BlockID dominator() const { return BlockID::fromBits(static_cast(dominatorPlusOne) - 1); } + LoopID innermostLoop() const { return LoopID::fromBits(static_cast(loopPlusOne) - 1); } + + uint16_t wordOffset; + uint16_t wordOffsetOfTerminator; + uint16_t firstParameter; + uint16_t dominatorPlusOne; + uint16_t loopPlusOne; + uint16_t dominatorNumber; + uint16_t lastDominatorNumberBelow; + uint16_t indexOfFirstIncomingTarget; + uint16_t wordOffsetOfFirstIncomingTarget; + uint16_t numberOfIncomingEdges; + uint16_t flags; + }; + static_assert(sizeof(NarrowBlockPlace) == 22); + static_assert(!(BlockID().bits() + 1) && !(LoopID().bits() + 1)); + + struct RareParts { + std::span loops; + std::span membersOfLoops; + std::span mentionsOfIterations; + }; + BodyIndex() = default; + static bool fitsNarrowNumbers(const Body& body) { return body.words().size() <= UINT16_MAX; } + + template + ALWAYS_INLINE std::invoke_result_t withBlockPlace(BlockID id, const Functor& functor) const + { + ASSERT(id.index() < numberOfBlocks()); + if (m_hasNarrowBlocks) [[likely]] + return functor(m_narrowBlocks[id.index()]); + return functor(m_wideBlocks[id.index()]); + } + + uint16_t flagsOf(BlockID id) const + { + return withBlockPlace(id, [](const auto& place) ALWAYS_INLINE_LAMBDA { + return place.flags; + }); + } + uint32_t firstParameterOf(BlockID id) const + { + return withBlockPlace(id, [](const auto& place) ALWAYS_INLINE_LAMBDA -> uint32_t { + return place.firstParameter; + }); + } + + template + static ALWAYS_INLINE bool dominatesAmong(const Place* blocks, BlockID from, BlockID to) + { + const Place& above = blocks[from.index()]; + const Place& below = blocks[to.index()]; + return (above.flags & below.flags & IsReachable) && above.dominatorNumber <= below.dominatorNumber && below.dominatorNumber <= above.lastDominatorNumberBelow; + } + + template + static BlockID findBlockOf(const Place* blocks, uint32_t numberOfBlocks, ValueID id) + { + if (id.index() < blocks[1].firstParameter) + return BlockID(0u); + const Place* next = std::ranges::upper_bound(blocks, blocks + numberOfBlocks, id.index(), { }, [](const Place& place) -> uint32_t { + return place.firstParameter; + }); + return BlockID(static_cast(next - blocks) - 1); + } + + BodyUser userOf(ValueID id) const + { + auto kind = static_cast(m_usesOfValues[id.index()] >> bitsOfUseCount); + uint32_t user = m_hasNarrowUsers ? m_narrowOnlyUsersOfValues[id.index()] : m_onlyUsersOfValues[id.index()]; + if (kind == BodyUseKind::Operand) + return BodyUser { kind, ValueID(user), { } }; + return BodyUser { kind, { }, BlockID(user) }; + } + JS_EXPORT_PRIVATE ValueID stripCastsSlow(ValueID cast) const; Body m_body; - std::span m_blocks; - std::span m_blocksInDominatorOrder; - std::span m_loops; - std::span m_membersOfLoops; - std::span m_placesOfValues; - std::span m_onlyUsersOfValues; - std::span m_usesOfValues; - std::span m_mentionsOfIterations; - OpcodeSet m_opcodes; + union { + const BlockPlace* m_wideBlocks { nullptr }; + const NarrowBlockPlace* m_narrowBlocks; + }; + const uint32_t* m_placesOfValues { nullptr }; + union { + const uint32_t* m_onlyUsersOfValues { nullptr }; + const uint16_t* m_narrowOnlyUsersOfValues; + }; + const uint8_t* m_usesOfValues { nullptr }; + const BlockID* m_blocksInDominatorOrder { nullptr }; + const RareParts* m_rareParts { nullptr }; + uint32_t m_numberOfReachableBlocks { 0 }; + bool m_hasNarrowBlocks { false }; + bool m_hasNarrowUsers { false }; bool m_hasIrreducibleFlow { false }; + OpcodeSet m_opcodes; mutable std::atomic m_liveness { nullptr }; mutable std::atomic m_keptByWalk { nullptr }; }; @@ -490,4 +624,6 @@ class BodyLiveness { } } } // namespace JSC::AOT::Typed +WTF_ALLOW_UNSAFE_BUFFER_USAGE_END + #endif // ENABLE(AOT) diff --git a/Source/JavaScriptCore/aot/ir/AOTBodyIndexBuilder.h b/Source/JavaScriptCore/aot/ir/AOTBodyIndexBuilder.h index cc97c983e10da..9cd8ba68e754f 100644 --- a/Source/JavaScriptCore/aot/ir/AOTBodyIndexBuilder.h +++ b/Source/JavaScriptCore/aot/ir/AOTBodyIndexBuilder.h @@ -39,6 +39,7 @@ struct BodyIndexTable::Worker { Vector worklist; Vector loops; Vector mentionsOfIterations; + Vector blocks; }; inline constexpr OpcodeSet opcodesThatMayNameIterationsInBodyIndex { @@ -70,6 +71,8 @@ class BodyIndexBuilder { void indexWords(); void adoptLoops(); void adoptMentionsOfIterations(); + void adoptBlocks(); + BodyIndex::RareParts& ensureRareParts(); void computeDominators(); BlockID nearestCommonDominator(BlockID, BlockID) const; @@ -86,8 +89,11 @@ class BodyIndexBuilder { std::span m_blocksInDominatorOrder; std::span m_placesOfValues; std::span m_onlyUsersOfValues; + std::span m_narrowOnlyUsersOfValues; std::span m_usesOfValues; OpcodeSet m_opcodes; + BodyIndex::RareParts* m_rareParts { nullptr }; + bool m_hasNarrowNumbers { false }; }; ALWAYS_INLINE void BodyIndexBuilder::fileBlock(BlockID block, uint32_t wordOffset, uint32_t firstParameter, uint32_t numberOfParameters, uint32_t numberOfValues) @@ -122,7 +128,10 @@ ALWAYS_INLINE void BodyIndexBuilder::fileUse(ValueID used, BodyUseKind kind, uin uint32_t count = uses & BodyIndex::maxCountedUses; if (!count) { uses = static_cast(static_cast(kind) << BodyIndex::bitsOfUseCount | 1); - m_onlyUsersOfValues[used.index()] = user; + if (m_hasNarrowNumbers) [[likely]] + m_narrowOnlyUsersOfValues[used.index()] = static_cast(user); + else + m_onlyUsersOfValues[used.index()] = user; return; } if (count < BodyIndex::maxCountedUses) diff --git a/Source/JavaScriptCore/aot/ir/AOTBodyVerifierOperations.cpp b/Source/JavaScriptCore/aot/ir/AOTBodyVerifierOperations.cpp index dfbe4e371ddb0..5bc28fa3ef6a7 100644 --- a/Source/JavaScriptCore/aot/ir/AOTBodyVerifierOperations.cpp +++ b/Source/JavaScriptCore/aot/ir/AOTBodyVerifierOperations.cpp @@ -290,7 +290,7 @@ void BodyVerifier::verifyRuleOfOpcode(const BodyBlock& block, uint32_t indexInBl return; } - case Opcode::Upcast: case Opcode::NarrowByField: case Opcode::Check: case Opcode::CheckInitialized: case Opcode::RequireObjectCoercible: case Opcode::RequireObject: case Opcode::TypeTest: + case Opcode::Upcast: case Opcode::NarrowByField: case Opcode::Check: case Opcode::CheckInitialized: case Opcode::RequireObjectCoercible: case Opcode::RequireObject: case Opcode::TypeTest: case Opcode::IsUnlockedInstance: case Opcode::Undefined: case Opcode::Null: case Opcode::Absent: case Opcode::Boolean: case Opcode::Integer: case Opcode::Double: case Opcode::String: case Opcode::BigInt: case Opcode::Data: case Opcode::GetModuleVariable: case Opcode::PutModuleVariable: case Opcode::ScopeHasBinding: case Opcode::NewDictionary: case Opcode::AppendElement: case Opcode::AppendSpread: case Opcode::AppendHole: case Opcode::NewRegExp: case Opcode::NewTemplateObject: @@ -319,7 +319,7 @@ void BodyVerifier::verifyRuleOfOpcode(const BodyBlock& block, uint32_t indexInBl case Opcode::PutContiguousIndex: case Opcode::PutDoubleIndex: case Opcode::GetOwnContiguousIndex: case Opcode::GetOwnDoubleIndex: case Opcode::GetOwnInt32Index: case Opcode::PutOwnContiguousIndex: case Opcode::PutOwnDoubleIndex: case Opcode::PutOwnInt32Index: case Opcode::NewContiguousArray: case Opcode::NewArrayLike: - case Opcode::TryPutByValDirect: case Opcode::TryPutIndex: case Opcode::TryCopyFilledElements: case Opcode::TryPutElements: case Opcode::TryFillNewArray: + case Opcode::TryPutByValDirect: case Opcode::TryPutIndex: case Opcode::TryCopyFilledElements: case Opcode::TryPutElements: case Opcode::SortNumbers: case Opcode::TryFillNewArray: case Opcode::TruncateNewArray: case Opcode::CollectionTable: case Opcode::CollectionIteratorClosedMark: case Opcode::GetCollectionIteratorTable: case Opcode::PutCollectionIteratorTable: case Opcode::IsCollectionIteratorClosedMark: case Opcode::MapNext: case Opcode::SetNext: @@ -342,7 +342,7 @@ void BodyVerifier::verifyRuleOfOpcode(const BodyBlock& block, uint32_t indexInBl case Opcode::GetEnumeratorIndexedLength: case Opcode::IsElementInStorage: case Opcode::GetElementInStorageOrAbsent: case Opcode::GetStringLength: case Opcode::GetStringCodeUnit: case Opcode::GetArgument: case Opcode::GetRestLength: case Opcode::CreateRest: case Opcode::MarkEnvironmentEscaped: case Opcode::ReleaseEnvironment: - case Opcode::CallBuiltin: case Opcode::DivideIntegers: case Opcode::RemainderOfIntegers: case Opcode::ReciprocalOfDivisor: case Opcode::RemainderByReciprocal: case Opcode::ToInt32: case Opcode::ToUint32: case Opcode::ToIntegerOrInfinity: case Opcode::NewSetFromData: case Opcode::NewMapFromData: + case Opcode::CallBuiltin: case Opcode::DivideIntegers: case Opcode::RemainderOfIntegers: case Opcode::ReciprocalOfDivisor: case Opcode::RemainderByReciprocal: case Opcode::WholeDivisor: case Opcode::RemainderByWholeDivisor: case Opcode::ToInt32: case Opcode::ToUint32: case Opcode::ToIntegerOrInfinity: case Opcode::NewSetFromData: case Opcode::NewMapFromData: case Opcode::NewFormatterWithPattern: case Opcode::SetUpFormatterWithOptions: case Opcode::FormatDateWithPattern: case Opcode::FormatNumberWithPattern: case Opcode::NewArgumentList: case Opcode::NewArrayOfGiven: case Opcode::SharedRegExpLiteral: case Opcode::GetBuiltinValue: case Opcode::GetWatchedBuiltinValue: case Opcode::GetWatchedBuiltinValueOrAbsent: case Opcode::InitializeBuiltin: case Opcode::DestructureIterable: case Opcode::AppendIterable: case Opcode::EnumeratorGetByVal: case Opcode::RequireAttachedTypedArray: case Opcode::RequireLengthFitsList: case Opcode::CreateAsyncFromSyncIterator: diff --git a/Source/JavaScriptCore/aot/ir/AOTDeclaredFunctionsInScopes.cpp b/Source/JavaScriptCore/aot/ir/AOTDeclaredFunctionsInScopes.cpp index 60c76deb04b29..6ef9cc45e32b1 100644 --- a/Source/JavaScriptCore/aot/ir/AOTDeclaredFunctionsInScopes.cpp +++ b/Source/JavaScriptCore/aot/ir/AOTDeclaredFunctionsInScopes.cpp @@ -25,7 +25,8 @@ FunctionID functionDeclaredInScopeSlot(const Program& program, LayoutID layout, Type type = program.at(written); if (!type.isJavaScriptValue() || type.primitives) return { }; - return singleFunctionOf(program, type); + FunctionID function = singleFunctionOf(program, type); + return function && isClassConstructorOf(program.at(function).kind) ? FunctionID() : function; } LayoutID scopeThatCaptureTakes(const Program& program, FunctionID function, uint32_t capture) diff --git a/Source/JavaScriptCore/aot/ir/AOTEffectsOfOperations.cpp b/Source/JavaScriptCore/aot/ir/AOTEffectsOfOperations.cpp index 30d7523ed9fbc..eed9f861b72cd 100644 --- a/Source/JavaScriptCore/aot/ir/AOTEffectsOfOperations.cpp +++ b/Source/JavaScriptCore/aot/ir/AOTEffectsOfOperations.cpp @@ -156,7 +156,7 @@ constexpr auto effectsOfEveryOperationWithRoomForAll = [] { continue; Effects effects = withDeclaredKindOfThrow(opcode, builder.effects()); if (opcode == Opcode::PutDoubleIndex || opcode == Opcode::PutOwnDoubleIndex) { - uint64_t written = effects.writes.maskOf(AbstractHeap::Elements) & ~markOfElementsOfArraysNotStoredAsRawDoubles; + uint64_t written = effects.writes.maskOf(AbstractHeap::Elements) & ~markOfElementsThatMayHoldCells; effects.writes.remove(AbstractHeapSet { AbstractHeap::Elements }); effects.writes.add(AbstractHeap::Elements, written); } @@ -193,7 +193,7 @@ static_assert(effectsOfEveryOperation.opcodesNamingOnlyEntireHeaps.contains(Opco static_assert([] { bool rawStoresLackMarkAlone = true; for (Opcode opcode : { Opcode::PutDoubleIndex, Opcode::PutOwnDoubleIndex }) - rawStoresLackMarkAlone &= effectsOfEveryOperationWith(opcode).writes.maskOf(AbstractHeap::Elements) == (locationMaskOfElementBase(ElementBase::Array) & ~markOfElementsOfArraysNotStoredAsRawDoubles); + rawStoresLackMarkAlone &= effectsOfEveryOperationWith(opcode).writes.maskOf(AbstractHeap::Elements) == (locationMaskOfElementBase(ElementBase::Array) & ~markOfElementsThatMayHoldCells); for (Opcode opcode : { Opcode::PutContiguousIndex, Opcode::PutOwnContiguousIndex, Opcode::PutOwnInt32Index }) rawStoresLackMarkAlone &= effectsOfEveryOperationWith(opcode).writes.maskOf(AbstractHeap::Elements) == locationMaskOfElementBase(ElementBase::Array); return rawStoresLackMarkAlone && std::popcount(locationMaskOfElementBase(ElementBase::Array)) == 2; @@ -261,6 +261,8 @@ static constexpr OpcodeSet opcodesWithEffectsFittedToOperands = [] { opcodes.add(Opcode::NewArrayLike); if (effectRuleOf(Opcode::PutArrayElement) == EffectRule::Declared) opcodes.add(Opcode::PutArrayElement); + opcodes.add(Opcode::PutContiguousIndex); + opcodes.add(Opcode::ModuleNamespace); return opcodes; }(); @@ -756,9 +758,25 @@ static bool definesElementOfListForEffects(const TypingContext& context, Operand return onlyClass && onlyClass->builtinClass == BuiltinClasses::Array; } +template +bool namespaceReadRunsNoCode(const Program& program, const Operation& read) +{ + constexpr OptionSet excludedFlags { ModuleFlags::MayBeReevaluated, ModuleFlags::MayBeMocked, ModuleFlags::MayBeEvicted, ModuleFlags::IsLoadedByLoader }; + if (read.ofModuleItself() || read.asksAnew() || read.phase() != ImportPhase::Evaluation) + return false; + const Module& module = program.at(read.module()); + if (module.kind != ModuleKind::SourceText || module.exportRule != SyntheticExportRule::None || module.flags.containsAny(excludedFlags)) + return false; + return std::ranges::none_of(module.requests, [](const ModuleRequest& request) { + return request.isDynamicStarExport; + }); +} + template static Effects declaredEffectsWithOperands(const TypingContext& context, Opcode opcode, const Holder& operation, OperandTypes operandTypes, TypeID result) { + if (opcode == Opcode::ModuleNamespace && namespaceReadRunsNoCode(context.program, operation)) + return Effects::ofReadByNameThatRunsNoCode(); if (opcode == Opcode::ElemDefine && definesElementOfListForEffects(context, operandTypes)) { Effects effects = declaredBoundOf(Opcode::PutArrayElement); effects.writes.add(perObjectHeaps); @@ -773,6 +791,11 @@ static Effects declaredEffectsWithOperands(const TypingContext& context, Opcode } if (opcode == Opcode::NewArrayLike && effects.mayThrow && lengthIsThatOfListForEffects(context, operandTypes)) effects.mayThrow = ThrowEffect::onlyForLackOfMemory(); + if (opcode == Opcode::PutContiguousIndex && operation.hasNumericStorage()) { + uint64_t written = effects.writes.maskOf(AbstractHeap::Elements) & ~markOfElementsThatMayHoldCells; + effects.writes.remove(AbstractHeapSet { AbstractHeap::Elements }); + effects.writes.add(AbstractHeap::Elements, written); + } if (opcode == Opcode::PutArrayElement && effects.mayThrow && operation.fillsPlaceOfListMadeHere()) effects.mayThrow = ThrowEffect::onlyForLackOfMemory(); if (opcode == Opcode::Strcat && effects.mayThrow && joinsTextsThatFitForEffects(context, operation, operandTypes)) @@ -833,6 +856,7 @@ Effects effectsOfTerminator(const TypingContext& context, const Terminator& term return effects; } +template bool namespaceReadRunsNoCode(const Program&, const BodyValue&); template Effects effectsOf(const TypingContext&, const Draft&, OperandTypes, TypeID, const Effects*); template Effects effectsOf(const TypingContext&, const Operation&, OperandTypes, TypeID, const Effects*); template Effects effectsOf(const TypingContext&, const BodyValue&, OperandTypes, TypeID, const Effects*); diff --git a/Source/JavaScriptCore/aot/ir/AOTEffectsOfOperations.h b/Source/JavaScriptCore/aot/ir/AOTEffectsOfOperations.h index 8ca3c244f975e..495aef21e0fbe 100644 --- a/Source/JavaScriptCore/aot/ir/AOTEffectsOfOperations.h +++ b/Source/JavaScriptCore/aot/ir/AOTEffectsOfOperations.h @@ -19,6 +19,8 @@ class BodyIndexTable; template Effects effectsOf(const TypingContext&, const Operation&, OperandTypes, TypeID result, const Effects* foundByCaller = nullptr); template Effects effectsOfTerminator(const TypingContext&, const Terminator&, OperandTypes, const Effects* foundByCaller = nullptr); +template bool namespaceReadRunsNoCode(const Program&, const Operation&); + Effects effectsOfOperator(const TypingContext&, Opcode, std::span operandTypes, TypeID result); Effects effectsOfProtocolsOn(const TypingContext&, OptionSet, TypeID operand); bool operandsMayRunCode(Opcode); diff --git a/Source/JavaScriptCore/aot/ir/AOTIRProgram.cpp b/Source/JavaScriptCore/aot/ir/AOTIRProgram.cpp index 154034d38efac..ac331dacf7af8 100644 --- a/Source/JavaScriptCore/aot/ir/AOTIRProgram.cpp +++ b/Source/JavaScriptCore/aot/ir/AOTIRProgram.cpp @@ -413,19 +413,12 @@ std::span Program::viewsAbove(LayoutID id) const return m_derivedTables.viewsAbove.subspan(first[id.index()], first[id.index() + 1] - first[id.index()]); } -std::span Program::fieldsWrittenAbove(LayoutID id, PropertyKeyID key) const +std::span Program::fieldsOfViews(PropertyKeyID key) const { - const Vector& rows = m_derivedTables.rowOfFieldsWrittenAbove; - if (!id || id.index() >= rows.size() || rows[id.index()] == UINT32_MAX) + const Vector& first = m_derivedTables.firstFieldsByKey; + if (!key || key.index() + 1 >= first.size()) return { }; - const Vector& first = m_derivedTables.firstFieldWrittenAbove; - uint32_t row = rows[id.index()]; - std::span ofLayout = m_derivedTables.fieldsWrittenAbove.subspan(first[row], first[row + 1] - first[row]); - auto found = std::ranges::lower_bound(ofLayout, key, { }, &FieldWrittenAbove::key); - auto behindFound = found; - while (behindFound != ofLayout.end() && behindFound->key == key) - ++behindFound; - return ofLayout.subspan(found - ofLayout.begin(), behindFound - found); + return m_derivedTables.fieldsOfViewsByKey.subspan(first[key.index()], first[key.index() + 1] - first[key.index()]); } void Program::indexViews(const ParallelLoops& loops) @@ -494,59 +487,53 @@ void Program::indexViews(const ParallelLoops& loops) return entry.view; }); - Vector layoutsWithRows; + BitVector isIndexedByKey(numberOfLayouts); for (uint32_t index = 0; index < numberOfLayouts; ++index) { - if (viewsAbove(LayoutID(index)).size() >= fewestViewsIndexedByKey) - layoutsWithRows.append(index); - } - AOT::stableSort(layoutsWithRows, [&](uint32_t a, uint32_t b) { - return std::ranges::lexicographical_compare(viewsAbove(LayoutID(a)), viewsAbove(LayoutID(b))); - }); - Vector& rowOf = m_derivedTables.rowOfFieldsWrittenAbove; - rowOf.fill(UINT32_MAX, numberOfLayouts); - Vector firstLayoutOfRow; - for (uint32_t index : layoutsWithRows) { - if (firstLayoutOfRow.isEmpty() || !std::ranges::equal(viewsAbove(LayoutID(firstLayoutOfRow.last())), viewsAbove(LayoutID(index)))) - firstLayoutOfRow.append(index); - rowOf[index] = static_cast(firstLayoutOfRow.size() - 1); + std::span views = viewsAbove(LayoutID(index)); + if (views.size() < fewestViewsIndexedByKey) + continue; + for (LayoutID view : views) + isIndexedByKey.quickSet(view.index()); } - auto numberOfRows = static_cast(firstLayoutOfRow.size()); - Vector& firstField = m_derivedTables.firstFieldWrittenAbove; - firstField.fill(0, numberOfRows + 1); - ChunksOfIndices division = divideIntoChunks(numberOfRows); - Vector> chunks(division.size()); - forEachIndexInParallel(loops, division.size(), [&](unsigned chunk) { - Vector& found = chunks[chunk]; - size_t mostFields = 0; - for (unsigned row = division.firstIndexOf[chunk]; row < division.firstIndexOf[chunk + 1]; ++row) { - for (LayoutID view : viewsAbove(LayoutID(firstLayoutOfRow[row]))) - mostFields += layouts[view.index()].fields.size(); - } - found.reserveInitialCapacity(mostFields); - for (unsigned row = division.firstIndexOf[chunk]; row < division.firstIndexOf[chunk + 1]; ++row) { - std::span views = viewsAbove(LayoutID(firstLayoutOfRow[row])); - size_t firstOfLayout = found.size(); - for (LayoutID view : views) { - const Vector& fields = layouts[view.index()].fields; - for (uint32_t field = 0; field < fields.size(); ++field) { - if (fields[field].key && !fields[field].flags.contains(FieldFlags::IsInherited)) - found.append({ fields[field].key, view, field }); - } - } - std::span ofLayout = found.mutableSpan().subspan(firstOfLayout); - AOT::stableSort(ofLayout, { }, [](const FieldWrittenAbove& entry) { - return entry.key; - }); - auto repeated = std::ranges::unique(ofLayout, [](const FieldWrittenAbove& a, const FieldWrittenAbove& b) { - return a.key == b.key && a.view == b.view; - }); - found.shrink(firstOfLayout + (repeated.begin() - ofLayout.begin())); - firstField[row + 1] = static_cast(found.size() - firstOfLayout); + struct FieldUnderKey { + PropertyKeyID key; + FieldOfView field; + }; + Vector fieldsUnderKeys = collectInParallel(loops, numberOfLayouts, [&](unsigned index, Vector& found) { + if (!isIndexedByKey.quickGet(index)) + return; + const Vector& fields = layouts[index].fields; + for (uint32_t field = 0; field < fields.size(); ++field) { + if (fields[field].key && !fields[field].flags.contains(FieldFlags::IsInherited)) + found.append({ fields[field].key, { LayoutID(index), field } }); } }); - for (uint32_t row = 0; row < numberOfRows; ++row) - firstField[row + 1] += firstField[row]; - m_derivedTables.fieldsWrittenAbove = joinChunks(loops, WTF::move(chunks)); + uint32_t numberOfKeys = 0; + for (const FieldUnderKey& entry : fieldsUnderKeys) + numberOfKeys = std::max(numberOfKeys, entry.key.index() + 1); + Vector& firstField = m_derivedTables.firstFieldsByKey; + firstField.fill(0, numberOfKeys + 1); + for (const FieldUnderKey& entry : fieldsUnderKeys) + ++firstField[entry.key.index() + 1]; + for (uint32_t key = 0; key < numberOfKeys; ++key) + firstField[key + 1] += firstField[key]; + Vector fieldsByKey; + fieldsByKey.grow(fieldsUnderKeys.size()); + Vector nextField(firstField); + for (const FieldUnderKey& entry : fieldsUnderKeys) + fieldsByKey[nextField[entry.key.index()]++] = entry.field; + + uint32_t numberKept = 0; + for (uint32_t key = 0; key < numberOfKeys; ++key) { + uint32_t firstOfKey = std::exchange(firstField[key], numberKept); + for (uint32_t place = firstOfKey; place < firstField[key + 1]; ++place) { + if (place == firstOfKey || fieldsByKey[place].view != fieldsByKey[numberKept - 1].view) + fieldsByKey[numberKept++] = fieldsByKey[place]; + } + } + firstField[numberOfKeys] = numberKept; + fieldsByKey.shrink(numberKept); + m_derivedTables.fieldsOfViewsByKey = WTF::move(fieldsByKey); } void Program::internBuiltinNameKeys() diff --git a/Source/JavaScriptCore/aot/ir/AOTIRProgram.h b/Source/JavaScriptCore/aot/ir/AOTIRProgram.h index 2a523050ee72d..e25f8635cc4e2 100644 --- a/Source/JavaScriptCore/aot/ir/AOTIRProgram.h +++ b/Source/JavaScriptCore/aot/ir/AOTIRProgram.h @@ -154,13 +154,33 @@ class Program final { std::span layoutsExpectedBelow(LayoutID) const LIFETIME_BOUND; std::span layoutsWithName(PropertyKeyID) const LIFETIME_BOUND; std::span viewsAbove(LayoutID) const LIFETIME_BOUND; - struct FieldWrittenAbove { - PropertyKeyID key; + struct FieldOfView { LayoutID view; uint32_t field { 0 }; }; static constexpr size_t fewestViewsIndexedByKey = 8; - std::span fieldsWrittenAbove(LayoutID, PropertyKeyID) const LIFETIME_BOUND; + std::span fieldsOfViews(PropertyKeyID) const LIFETIME_BOUND; + template + bool visitFieldsWrittenAbove(LayoutID id, PropertyKeyID key, const Functor& functor) const + { + std::span fields = fieldsOfViews(key); + std::span views = viewsAbove(id); + auto nextField = fields.begin(); + auto nextView = views.begin(); + while (nextField != fields.end() && nextView != views.end()) { + if (nextField->view < *nextView) + nextField = std::ranges::lower_bound(nextField, fields.end(), *nextView, { }, &FieldOfView::view); + else if (*nextView < nextField->view) + nextView = std::ranges::lower_bound(nextView, views.end(), nextField->view); + else { + if (!functor(*nextField)) + return false; + ++nextField; + ++nextView; + } + } + return true; + } bool prototypeMaySelectOtherClass(LayoutID) const; void indexClasses(); void indexNames(); @@ -205,9 +225,8 @@ class Program final { Vector layoutsWithName; Vector firstViewAbove; Vector viewsAbove; - Vector rowOfFieldsWrittenAbove; - Vector firstFieldWrittenAbove; - Vector fieldsWrittenAbove; + Vector firstFieldsByKey; + Vector fieldsOfViewsByKey; Vector builtinNameKeys; PropertyKeyID keyOfWordOnWhatSeesAccesses; Vector registeredSymbolNames; diff --git a/Source/JavaScriptCore/aot/ir/AOTMemoryLocation.h b/Source/JavaScriptCore/aot/ir/AOTMemoryLocation.h index 216565e89b49f..ad61197c82dc2 100644 --- a/Source/JavaScriptCore/aot/ir/AOTMemoryLocation.h +++ b/Source/JavaScriptCore/aot/ir/AOTMemoryLocation.h @@ -313,7 +313,7 @@ constexpr uint64_t locationMaskOfBit(std::optional bit, uint32_t offset return uint64_t { 1 } << ((*bit + offset) & 63); } -constexpr uint64_t markOfElementsOfArraysNotStoredAsRawDoubles = uint64_t { 1 } << 63; +constexpr uint64_t markOfElementsThatMayHoldCells = uint64_t { 1 } << 63; constexpr uint64_t locationMaskOfElementBase(ElementBase base) { @@ -321,7 +321,7 @@ constexpr uint64_t locationMaskOfElementBase(ElementBase base) case ElementBase::Unknown: return LocationMasks::entireHeap; case ElementBase::Array: - return (uint64_t { 1 } << static_cast(base)) | markOfElementsOfArraysNotStoredAsRawDoubles; + return (uint64_t { 1 } << static_cast(base)) | markOfElementsThatMayHoldCells; case ElementBase::Arguments: case ElementBase::Object: return uint64_t { 1 } << static_cast(base); diff --git a/Source/JavaScriptCore/aot/ir/AOTOpcode.h b/Source/JavaScriptCore/aot/ir/AOTOpcode.h index bf76ed2669f16..87f84ea2d05bb 100644 --- a/Source/JavaScriptCore/aot/ir/AOTOpcode.h +++ b/Source/JavaScriptCore/aot/ir/AOTOpcode.h @@ -73,6 +73,7 @@ class EnumeratorBitSet { public: using Word = typename WordOfEnumeratorBitSet::Type; static constexpr unsigned bitsPerWord = sizeof(Word) * 8; + static constexpr unsigned numberOfWords = std::bit_ceil((count + bitsPerWord - 1) / bitsPerWord); constexpr EnumeratorBitSet() = default; constexpr EnumeratorBitSet(std::initializer_list enumerators) @@ -81,8 +82,8 @@ class EnumeratorBitSet { add(enumerator); } - constexpr void add(E enumerator) { m_words[static_cast(enumerator) / bitsPerWord] |= static_cast(1) << (static_cast(enumerator) % bitsPerWord); } - constexpr bool contains(E enumerator) const { return m_words[static_cast(enumerator) / bitsPerWord] >> (static_cast(enumerator) % bitsPerWord) & 1; } + constexpr void add(E enumerator) { m_words[wordOf(enumerator)] |= static_cast(1) << (static_cast(enumerator) % bitsPerWord); } + constexpr bool contains(E enumerator) const { return m_words[wordOf(enumerator)] >> (static_cast(enumerator) % bitsPerWord) & 1; } constexpr bool isEmpty() const { Word all = 0; @@ -128,7 +129,9 @@ class EnumeratorBitSet { friend constexpr bool operator==(const EnumeratorBitSet&, const EnumeratorBitSet&) = default; private: - std::array m_words { }; + static constexpr unsigned wordOf(E enumerator) { return static_cast(enumerator) / bitsPerWord % numberOfWords; } + + std::array m_words { }; }; using OpcodeSet = EnumeratorBitSet>; diff --git a/Source/JavaScriptCore/aot/ir/AOTResultRules.h b/Source/JavaScriptCore/aot/ir/AOTResultRules.h index d07404c57ff5a..a559c842f882f 100644 --- a/Source/JavaScriptCore/aot/ir/AOTResultRules.h +++ b/Source/JavaScriptCore/aot/ir/AOTResultRules.h @@ -217,6 +217,7 @@ Range quotientsRoundedEitherWay(Range dividends, Range divisors); template TypeID typeOfConstant(const TypingContext&, const Operation&, ObjectID created = { }); TypeID typeOfKeyConstant(const TypingContext&, PropertyKeyID); TypeID typeOfBuiltinObject(const TypingContext&, BuiltinID); +BuiltinID findNativeFunction(const BuiltinTable&, BuiltinID binding); TypeID typeOfData(const TypingContext&, const DataNode&, ObjectID created = { }); TypeID typeOfAllocatedLayout(const TypingContext&, LayoutID, ObjectID created = { }); TypeID typeOfDeclaredLayout(const TypingContext&, LayoutID); diff --git a/Source/JavaScriptCore/aot/ir/AOTResultRulesConstants.cpp b/Source/JavaScriptCore/aot/ir/AOTResultRulesConstants.cpp index 095456dfaaa5d..6a0b4680e8faa 100644 --- a/Source/JavaScriptCore/aot/ir/AOTResultRulesConstants.cpp +++ b/Source/JavaScriptCore/aot/ir/AOTResultRulesConstants.cpp @@ -16,6 +16,7 @@ #include "AOTInliningAcrossFiles.h" #include "AOTObjects.h" #include "AOTVector.h" +#include #include #include @@ -79,6 +80,53 @@ TypeID typeOfBuiltinObject(const TypingContext& context, BuiltinID builtin) return typeOfBuiltinConstant({ context.program, types }, builtin); } +static bool hasCompleteSignature(const BuiltinTable& builtins, BuiltinID function) +{ + if (builtins.at(function).marks || builtins.letsInUnseenCode(function) || !builtins.calledFunctionsOf(function).empty() || !builtins.moduleLoadsOf(function).empty()) + return false; + return std::ranges::all_of(builtins.overloadsOf(function), [](const BuiltinOverload& overload) { + if (overload.mode == BuiltinCallMode::Get) + return true; + bool runsCode = overload.effects.capability == BuiltinCapability::JS && !overload.flags.contains(BuiltinOverloadFlag::RunsOnlyOperandCode); + return !overload.isUndeclared() && !runsCode && overload.flow.callbacks.empty() && overload.flow.handlerCalls.empty() && overload.flow.propertyActions.empty(); + }); +} + +static bool hasDescribedResult(const BuiltinTable& builtins, BuiltinID function) +{ + return std::ranges::any_of(builtins.overloadsOf(function), [&](const BuiltinOverload& overload) { + if (overload.mode != BuiltinCallMode::Call || !overload.result.type) + return false; + const BuiltinType& answer = builtins.at(overload.result.type); + return !answer.isAnything() && (!(answer.primitives - BuiltinPrimitive::Undefined).isEmpty() || !answer.stringLiterals.empty() || !answer.parts.empty()); + }); +} + +BuiltinID findNativeFunction(const BuiltinTable& builtins, BuiltinID binding) +{ + BuiltinID answered; + for (const BuiltinOverload& overload : builtins.overloadsOf(binding)) { + if (overload.mode != BuiltinCallMode::Get) + continue; + if (answered || overload.isUndeclared() || overload.resultIsDeclaredOnly() || !overload.result.type) + return { }; + const BuiltinType& written = builtins.at(overload.result.type); + if (!written.flags.isEmpty() || !written.primitives.isEmpty() || !written.stringLiterals.empty() || written.parts.size() != 1 || written.parts[0].kind != BuiltinTypePartKind::Builtin) + return { }; + answered = builtins.canonicalID(written.parts[0].builtin); + if (builtins.at(answered).kind != BuiltinKind::Function) + return { }; + } + bool isDescribed = answered && hasDescribedResult(builtins, answered) && hasCompleteSignature(builtins, binding) && hasCompleteSignature(builtins, answered); + return isDescribed ? answered : BuiltinID(); +} + +static TypeID typeOfNativeBinding(const TypingContext& context, BuiltinID binding) +{ + BuiltinID function = findNativeFunction(context.program.builtins(), binding); + return function ? context.types.forBuiltin(function) : context.types.anything(UnknownSeed::UndescribedNativeResult); +} + static TypeID typeOfBuiltinObjectOfModule(const TypingContext& context, BuiltinID builtin, ModuleID module) { const BuiltinTable& builtins = context.program.builtins(); @@ -257,6 +305,8 @@ ALWAYS_INLINE_INTO_OTHER_FILES TypeID typeOfConstant(const TypingContext& contex return typeOfBuiltinObject(context, operation.builtin()); case Opcode::ModuleBuiltin: return typeOfBuiltinObjectOfModule(context, operation.builtin(), operation.module()); + case Opcode::NativeBinding: + return typeOfNativeBinding(context, operation.builtin()); case Opcode::KeyConstant: return typeOfKeyConstant(context, operation.key()); case Opcode::Data: { diff --git a/Source/JavaScriptCore/aot/ir/AOTResultRulesDeclared.cpp b/Source/JavaScriptCore/aot/ir/AOTResultRulesDeclared.cpp index 936cda445e108..66d91a9fe7eeb 100644 --- a/Source/JavaScriptCore/aot/ir/AOTResultRulesDeclared.cpp +++ b/Source/JavaScriptCore/aot/ir/AOTResultRulesDeclared.cpp @@ -114,11 +114,10 @@ TypeID tryTypeWrittenAbove(const TypingContext& context, LayoutID layout, Proper LayoutID declaring = declaredLayoutOf(program, layout); std::span views = program.viewsAbove(declaring); if (views.size() >= Program::fewestViewsIndexedByKey) { - for (const Program::FieldWrittenAbove& listed : program.fieldsWrittenAbove(declaring, key)) { - if (!add(listed.view, listed.field)) - return { }; - } - return isWritten ? written.finish() : TypeID(); + bool isAnswered = program.visitFieldsWrittenAbove(declaring, key, [&](Program::FieldOfView listed) { + return add(listed.view, listed.field); + }); + return isAnswered && isWritten ? written.finish() : TypeID(); } for (LayoutID view : views) { if (std::optional field = program.at(view).findField(key); field && !add(view, *field)) diff --git a/Source/JavaScriptCore/aot/ir/AOTResultRulesInternal.h b/Source/JavaScriptCore/aot/ir/AOTResultRulesInternal.h index 551bf9f0a3981..c8784e889bf06 100644 --- a/Source/JavaScriptCore/aot/ir/AOTResultRulesInternal.h +++ b/Source/JavaScriptCore/aot/ir/AOTResultRulesInternal.h @@ -21,7 +21,7 @@ TypeID wholeRemainderType(TypeArena&, Range dividends, Range divisors); inline constexpr OpcodeSet opcodesRefinedBeyondFixedNatural { Opcode::URShift, Opcode::ToNumber, Opcode::StrictEq, Opcode::ToBoolean, Opcode::LogicalNot, Opcode::ToString, Opcode::ToPropertyKey, - Opcode::ToObject, Opcode::ToThis, Opcode::TypeTest, Opcode::GetTypedArrayElement, Opcode::GetTypedArrayLength, Opcode::GetStringLength, Opcode::DivideIntegers, Opcode::RemainderOfIntegers, Opcode::RemainderByReciprocal, Opcode::ToInt32, + Opcode::ToObject, Opcode::ToThis, Opcode::TypeTest, Opcode::GetTypedArrayElement, Opcode::GetTypedArrayLength, Opcode::GetStringLength, Opcode::DivideIntegers, Opcode::RemainderOfIntegers, Opcode::RemainderByReciprocal, Opcode::WholeDivisor, Opcode::RemainderByWholeDivisor, Opcode::ToInt32, Opcode::ToUint32, Opcode::ToIntegerOrInfinity, Opcode::ToLength, Opcode::IntegerBound, Opcode::GetSuperBase }; inline const OpcodeSet& opcodesWithNaturalTypeRefinedByOperation() { return opcodesRefinedBeyondFixedNatural; } diff --git a/Source/JavaScriptCore/aot/ir/AOTResultRulesNatural.cpp b/Source/JavaScriptCore/aot/ir/AOTResultRulesNatural.cpp index 86965cb854b3a..a60a528e0a218 100644 --- a/Source/JavaScriptCore/aot/ir/AOTResultRulesNatural.cpp +++ b/Source/JavaScriptCore/aot/ir/AOTResultRulesNatural.cpp @@ -536,8 +536,24 @@ static TypeID numberResultWithinBounds(TypeArena& types, Opcode opcode, NaturalO } } +static bool isIntegerOrNullish(const NaturalOperand& operand) +{ + return operand.isMadeOfPrimitives && operand.primitives & Primitive::UndefinedOrNull && operand.hasOnlyPrimitives(Primitive::Integer | Primitive::UndefinedOrNull); +} + static TypeID numberResultOf(TypeArena& types, Opcode opcode, NaturalOperands operands) { + if (allBitwiseOperators.contains(opcode) && std::ranges::any_of(operands, isIntegerOrNullish)) { + Vector converted; + for (const NaturalOperand& operand : operands) { + converted.append(operand); + if (!isIntegerOrNullish(operand)) + continue; + converted.last().primitives = Primitive::Integer; + converted.last().integers = operand.integers.unionWith(Range::one(0)); + } + return numberResultOf(types, opcode, converted.span()); + } if ((opcode == Opcode::ToNumber || opcode == Opcode::ToNumeric) && operands[0].isBoolean()) return naturalTypeOfIntegers(types, { operands[0].primitives & Primitive::False ? 0 : 1, operands[0].primitives & Primitive::True ? 1 : 0 }); if (shiftOperators.contains(opcode)) diff --git a/Source/JavaScriptCore/aot/ir/AOTResultRulesRefined.cpp b/Source/JavaScriptCore/aot/ir/AOTResultRulesRefined.cpp index 70a3531249bff..5dbb850a9b7f3 100644 --- a/Source/JavaScriptCore/aot/ir/AOTResultRulesRefined.cpp +++ b/Source/JavaScriptCore/aot/ir/AOTResultRulesRefined.cpp @@ -243,6 +243,12 @@ static TypeID refinedTypeOfOperation(const TypingContext& context, const Operati return refinedTypeOfWholeRemainder(types, operands[OperandIndex::RemainderOfIntegers::dividend], operands[OperandIndex::RemainderOfIntegers::divisor], upperBound); case Opcode::RemainderByReciprocal: return refinedTypeOfWholeRemainder(types, operands[OperandIndex::RemainderByReciprocal::dividend], operands[OperandIndex::RemainderByReciprocal::divisor], upperBound); + case Opcode::WholeDivisor: + return types.integer(Range { 0, maxExactInteger }); + case Opcode::RemainderByWholeDivisor: { + TypeID number = numberResultOf(types, Opcode::Mod, operands); + return number ? number : upperBound; + } case Opcode::IntegerBound: return refinedTypeOfIntegerBound(types, operands[OperandIndex::IntegerBound::number], upperBound); case Opcode::GetStringLength: diff --git a/Source/JavaScriptCore/aot/ir/AOTTypeArena.h b/Source/JavaScriptCore/aot/ir/AOTTypeArena.h index 16340b1c08538..82791b73f7a03 100644 --- a/Source/JavaScriptCore/aot/ir/AOTTypeArena.h +++ b/Source/JavaScriptCore/aot/ir/AOTTypeArena.h @@ -21,32 +21,29 @@ namespace JSC { namespace AOT { namespace Typed { enum class DeclaredParts : uint8_t { FromEither, FromSecond }; enum class AnswerKeptInArena : uint8_t { - FirstOfWalk = 0, - CellsOfFieldArm = 57, - ObjectSetOfList = 61, - NumberOfLayoutList = 62, - LayoutSetOfList = 63, - FirstOfAppliedReads = 64, - FirstOfAppliedWrites = 72, - TypeInCells = 78, - TypeFound = 79, - ReadOrAssignmentRunsNoCodeOnLayouts = 80, - LayoutsAreFoundByShapeTest = 81, - ReadRunsNoCodeOnLayoutsOfObjects = 82, - NumberOfObjectList = 84, - FieldsOfSlotRead = 85, - TypeInFilledSlot = 101, - TestOfGuess = 102, - ShapesOfLayouts = 105, - PartWhoseClassesHaveGuards = 106, - PartOfWrittenSetThatFactoriesFill = 107, - PartThatHasNameAsked = 108, - ConstantMemberOfBuiltin = 109, - RecordsThatFileWrites = 110, - UnitMayMentionFunctions = 111, - LiteralsWithoutNames = 112, - ReadFindsFieldWithoutWrittenType = 113, - LayoutsThatStayWithCompiledCode = 114, + FirstOfWalk, + LastOfWalk = FirstOfWalk + 54, + ObjectSetOfList, + NumberOfLayoutList, + LayoutSetOfList, + NumberOfObjectList, + TypeInFilledSlot, + TestOfGuess, + LayoutsOfGuessTest, + ShapesOfLayouts, + PartWhoseClassesHaveGuards, + PartWithNamedMembersAbove, + PartOfWrittenSetThatFactoriesFill, + PartThatHasNameAsked, + ConstantMemberOfBuiltin, + RecordsThatFileWrites, + WrittenRecordCoverage, + UnitMayMentionFunctions, + LiteralsWithoutNames, + ReadFindsFieldWithoutWrittenType, + LayoutsThatStayWithCompiledCode, + FieldOfLayoutsIsStable, + NumberOfAnswers, }; enum class MakerOfType : uint8_t { @@ -68,6 +65,7 @@ enum class MakerOfType : uint8_t { inline constexpr uint32_t bitsOfOperationInTypeMemo = 8; inline constexpr uint8_t firstQuestionFromOutsideInTypeMemo = 128; +static_assert(static_cast(AnswerKeptInArena::NumberOfAnswers) <= firstQuestionFromOutsideInTypeMemo); constexpr bool wordOfTypeMemoLiesInThread(uint32_t bits) { diff --git a/Source/JavaScriptCore/aot/ir/AOTTypeTable.cpp b/Source/JavaScriptCore/aot/ir/AOTTypeTable.cpp index fa8fd36201632..31b559fddff76 100644 --- a/Source/JavaScriptCore/aot/ir/AOTTypeTable.cpp +++ b/Source/JavaScriptCore/aot/ir/AOTTypeTable.cpp @@ -681,6 +681,12 @@ TypeTable::TypeTable(const BuiltinTable& builtins, const Program* program) }; internRangeAtStart(int16, { INT16_MIN, INT16_MAX }); internRangeAtStart(uint16, { 0, UINT16_MAX }); + internAtStart(anythingButUndefined, Primitive::All & ~Primitive::Undefined, true); + internAtStart(anythingButNull, Primitive::All & ~Primitive::Null, true); + internAtStart(anythingButNullish, Primitive::All & ~Primitive::UndefinedOrNull, true); + internAtStart(anythingButString, Primitive::All & ~Primitive::String, true); + internAtStart(anythingButNumber, Primitive::All & ~Primitive::Number, true); + internAtStart(anythingButBoolean, Primitive::All & ~Primitive::Boolean, true); RELEASE_ASSERT(m_types.size() == numberOfTypesInternedAtStart); } @@ -727,6 +733,14 @@ void TypeTable::endStableNumbersOfPropertyKeys() endStableNumbersOfPropertyKeys(ParallelLoops(1)); } +static bool keyIsLookedUpAsName(const Program& program, const PropertyKey& key, PropertyKeyID id) +{ + if (key.kind == PropertyKeyKind::PrivateName) + return false; + NameClass nameClass = program.nameClassOf(id); + return !mayBeElementKey(nameClass) && nameClass != NameClass::Proto; +} + void TypeTable::endStableNumbersOfPropertyKeys(const ParallelLoops& loops) { bool isFirstCall = m_numberOfKeysWithStableNumbers == UINT32_MAX; @@ -735,9 +749,13 @@ void TypeTable::endStableNumbersOfPropertyKeys(const ParallelLoops& loops) return; m_locationBitsOfKeys.grow(m_numberOfKeysWithStableNumbers); + m_keysLookedUpAsNames.grow(m_numberOfKeysWithStableNumbers); forEachRangeOfIndicesInParallel(loops, m_numberOfKeysWithStableNumbers, [&](size_t first, size_t end) { - for (size_t position = first; position < end; ++position) - m_locationBitsOfKeys[position] = locationBitOfKeyInTypeTable(*m_program, m_propertyKeys.at(static_cast(position))); + for (size_t position = first; position < end; ++position) { + const PropertyKey& key = m_propertyKeys.at(static_cast(position)); + m_locationBitsOfKeys[position] = locationBitOfKeyInTypeTable(*m_program, key); + m_keysLookedUpAsNames[position] = keyIsLookedUpAsName(*m_program, key, PropertyKeyID(static_cast(position))); + } }); } diff --git a/Source/JavaScriptCore/aot/ir/AOTTypeTable.h b/Source/JavaScriptCore/aot/ir/AOTTypeTable.h index b799c83d12cf3..03aa0e6f9291a 100644 --- a/Source/JavaScriptCore/aot/ir/AOTTypeTable.h +++ b/Source/JavaScriptCore/aot/ir/AOTTypeTable.h @@ -160,7 +160,31 @@ class TypeTable { static constexpr TypeID cellOrMarker = idAt(IDHome::Program, 5); static constexpr TypeID int16 = idAt(IDHome::Program, 6); static constexpr TypeID uint16 = idAt(IDHome::Program, 7); - static constexpr uint32_t numberOfTypesInternedAtStart = 8; + static constexpr TypeID anythingButUndefined = idAt(IDHome::Program, 8); + static constexpr TypeID anythingButNull = idAt(IDHome::Program, 9); + static constexpr TypeID anythingButNullish = idAt(IDHome::Program, 10); + static constexpr TypeID anythingButString = idAt(IDHome::Program, 11); + static constexpr TypeID anythingButNumber = idAt(IDHome::Program, 12); + static constexpr TypeID anythingButBoolean = idAt(IDHome::Program, 13); + static constexpr uint32_t numberOfTypesInternedAtStart = 14; + static constexpr bool passesEveryObject(TypeID id) { return id == anything || (id.bits() >= anythingButUndefined.bits() && id.bits() <= anythingButBoolean.bits()); } + static constexpr bool isAnythingButNullish(TypeID id) { return id == anythingButUndefined || id == anythingButNull || id == anythingButNullish; } + static constexpr TypeID anythingBut(TypeID primitives) + { + if (primitives == undefined) + return anythingButUndefined; + if (primitives == null) + return anythingButNull; + if (primitives == nullish) + return anythingButNullish; + if (primitives == string) + return anythingButString; + if (primitives == number) + return anythingButNumber; + if (primitives == boolean) + return anythingButBoolean; + return { }; + } static constexpr TypeID valuesOf(TypeID held) { return held == anythingOrAbsent ? anything : held; } const BuiltinTable& builtins() const { return m_builtins; } @@ -203,6 +227,7 @@ class TypeTable { void endStableNumbersOfPropertyKeys(const ParallelLoops&); void addKeysOfTexts(uint32_t numberOfStrings, const ParallelLoops&); bool hasStableNumber(PropertyKeyID key) const { return key.index() < m_numberOfKeysWithStableNumbers; } + bool isLookedUpAsName(PropertyKeyID key) const { return key.index() < m_keysLookedUpAsNames.size() && m_keysLookedUpAsNames[key.index()]; } std::optional locationBitOf(PropertyKeyID key) const { if (key.index() >= m_locationBitsOfKeys.size()) @@ -299,6 +324,7 @@ class TypeTable { std::array m_addressesOfPositionZeroInKeySegments { }; uint32_t m_numberOfKeysWithStableNumbers { UINT32_MAX }; Vector m_locationBitsOfKeys; + Vector m_keysLookedUpAsNames; Vector m_typeOfWireIndex; Vector m_typesOfLiteralStrings; }; diff --git a/Source/JavaScriptCore/aot/ir/generator/check-runtime.ts b/Source/JavaScriptCore/aot/ir/generator/check-runtime.ts index ebd5c6e9600e8..a53b1ecf64dc1 100644 --- a/Source/JavaScriptCore/aot/ir/generator/check-runtime.ts +++ b/Source/JavaScriptCore/aot/ir/generator/check-runtime.ts @@ -144,6 +144,7 @@ const justifiedDifferences: ReadonlyMap = new Map([ ["AppendValuesOfMap names an index of Elements", "as AppendHole"], ["SetArrayLength names an index of Elements", "as AppendHole"], ["TruncateNewArray names an index of Elements", "as AppendHole"], + ["SortNumbers names an index of Elements", "as AppendHole"], ["SetMappedArgument names an index of Elements", "a row of the run time has no index; the receiver is an arguments object"], ["NewArray leaves out throws", "for want of memory alone: the cell is null and the next use faults"], ["CreateRest leaves out throws", "as NewArray"], diff --git a/Source/JavaScriptCore/aot/ir/generator/emit-cpp-operations.ts b/Source/JavaScriptCore/aot/ir/generator/emit-cpp-operations.ts index f236a87976e0d..f5447f3a30bdb 100644 --- a/Source/JavaScriptCore/aot/ir/generator/emit-cpp-operations.ts +++ b/Source/JavaScriptCore/aot/ir/generator/emit-cpp-operations.ts @@ -240,22 +240,20 @@ struct PositionTable { // Which case of a switch each member takes, in one table that every copy of the switch reads. template struct CaseTable { - static constexpr std::array> tabulate() + static constexpr size_t numberOfPlaces = std::bit_ceil(numberOfEnumerators); + + static constexpr std::array tabulate() { - std::array> cases { }; + std::array cases { }; + cases.fill(std::numeric_limits::max()); for (E member : allMembersOf) cases[static_cast(member)] = find(member); return cases; } - static constexpr std::array> cases = tabulate(); + static constexpr std::array cases = tabulate(); - static uint8_t at(E member) - { - size_t index = static_cast(member); - RELEASE_ASSERT(index < cases.size()); - return cases[index]; - } + static uint8_t at(E member) { return cases[static_cast(member) % numberOfPlaces]; } };`.split("\n"); const positionIncludes = `#include "AOTSchemaFlagMasks.h" @@ -434,6 +432,8 @@ function operationsHeader(schema: Schema): string[] { '"AOTSchemaOperandIndexes.h"', "", "", + "", + "", "", "", "", diff --git a/Source/JavaScriptCore/aot/ir/generator/emit-cpp-packed.ts b/Source/JavaScriptCore/aot/ir/generator/emit-cpp-packed.ts index 239ce7fd5fcc0..3d23c879315df 100644 --- a/Source/JavaScriptCore/aot/ir/generator/emit-cpp-packed.ts +++ b/Source/JavaScriptCore/aot/ir/generator/emit-cpp-packed.ts @@ -248,7 +248,7 @@ function structLines(record: PackedRecord): string[] { "}", ], [ - `static ${name} read(const uint8_t*& cursor)`, + `static ${name} read(auto& cursor)`, "{", ...indent([`${name} result;`, ...guardedLines(words, "result.", readLinesOf), "return result;"]), "}", diff --git a/Source/JavaScriptCore/aot/ir/schema/compile-call.def b/Source/JavaScriptCore/aot/ir/schema/compile-call.def index 1c42b44707707..b8ab1aecd6e70 100644 --- a/Source/JavaScriptCore/aot/ir/schema/compile-call.def +++ b/Source/JavaScriptCore/aot/ir/schema/compile-call.def @@ -23,7 +23,7 @@ enum ResultSection { MissingLibraryFunctions WorkerStartsWithoutName BytesByForm BytesByModule FunctionNames LayoutNames - WorkerURLs + WorkerStartsFromURLs MissingLibraryModules } @@ -46,7 +46,7 @@ record FeatureUseRecord abi { feature: Feature builtin: Builtin? witness: Witnes record LocaleUseRecord abi { kind: LocaleUseKind tag: TextRange builtin: Builtin? witness: Witness } record WorkerEntryRecord abi { path: TextRange witness: Witness } record WorkerStartRecord abi { witness: Witness } -record WorkerURLRecord abi { witness: Witness } +record WorkerURLStartRecord abi { witness: Witness } record AccessRecord abi { location: SourceLocation kind: AccessKind level: KnowledgeLevel cause: UnprovenCause layout: Layout? copies: UInt32 function: Function? cost: AccessCost loopDepth: UInt32 diff --git a/Source/JavaScriptCore/aot/ir/schema/flow-records.def b/Source/JavaScriptCore/aot/ir/schema/flow-records.def index 6db189bacc27a..984665a4798fa 100644 --- a/Source/JavaScriptCore/aot/ir/schema/flow-records.def +++ b/Source/JavaScriptCore/aot/ir/schema/flow-records.def @@ -5,7 +5,7 @@ enum BuiltinLocationState { | engine Unknown Constant } flags ObjectMutation { | engine DeleteField RedefineProperty AddProperty ChangeIntegrity ChangePrototype BecomePrototype - StoreOfOtherType + StoreOfOtherType ConstructDirectly } flags FieldStability { | engine IsNeverDeleted IsNeverRedefined } diff --git a/Source/JavaScriptCore/aot/ir/schema/image-tables.def b/Source/JavaScriptCore/aot/ir/schema/image-tables.def index 7a27f58edf296..827a2b4f00881 100644 --- a/Source/JavaScriptCore/aot/ir/schema/image-tables.def +++ b/Source/JavaScriptCore/aot/ir/schema/image-tables.def @@ -1,28 +1,20 @@ -enum CallSiteKind (isFirstRead: Bool, isConstruct: Bool, isTailCall: Bool, isInsideCallee: Bool, namesBuiltin: Bool) { +enum CallSiteKind (isFirstRead: Bool, isConstruct: Bool, isTailCall: Bool, isInsideCallee: Bool, namesBuiltin: Bool, isAwaited: Bool) { | engine - Plain(false, false, false, false, false) - Construct(false, true, false, false, false) - TailCall(false, false, true, false, false) - TailConstruct(false, true, true, false, false) - FirstRead(true, false, false, false, false) - InsideBuiltin(false, false, false, false, true) - InsideCallee(false, false, true, true, false) - InsideBuiltinCallee(false, false, true, true, true) - ConstructInsideBuiltin(false, true, false, false, true) - ConstructInsideBuiltinCallee(false, true, true, true, true) + Plain(false, false, false, false, false, false) + Construct(false, true, false, false, false, false) + TailCall(false, false, true, false, false, false) + TailConstruct(false, true, true, false, false, false) + FirstRead(true, false, false, false, false, false) + InsideBuiltin(false, false, false, false, true, false) + InsideCallee(false, false, true, true, false, false) + InsideBuiltinCallee(false, false, true, true, true, false) + ConstructInsideBuiltin(false, true, false, false, true, false) + ConstructInsideBuiltinCallee(false, true, true, true, true, false) + AwaitedCall(false, false, false, false, false, true) } -packed PrologueCheckRow { callOffset: CodeOffset ; resumeOffset: CodeOffset } -packed CallSiteHead { - hasInlineFrames: Bool hasPrologueChecks: Bool numberOfSites: UInt64 - ; prologueChecks: List if hasPrologueChecks - ; numberOfSitesWithoutCode: UInt64 - ; numberOfInlineFrames: UInt64 if hasInlineFrames - ; inlineFrames: Bytes if hasInlineFrames -} packed InlineFrameRow { isTailCall: Bool isConstruct: Bool parent: UInt32 ; function: Function } packed CallSiteDetails { kind: CallSiteKind frame: UInt32 ; calledBuiltin: Builtin if namesBuiltin(kind) } -packed CallSiteRow { hasDetails: Bool distance: CodeOffset ; details: CallSiteDetails if hasDetails } packed HandlerRow { sitesSkipped: UInt32 ; furtherSites: UInt32 ; type: HandlerType[2] targetChange: Signed } diff --git a/Source/JavaScriptCore/aot/ir/schema/intrinsic-operations.def b/Source/JavaScriptCore/aot/ir/schema/intrinsic-operations.def index 7f8523aab3fb4..692bce01ef94f 100644 --- a/Source/JavaScriptCore/aot/ir/schema/intrinsic-operations.def +++ b/Source/JavaScriptCore/aot/ir/schema/intrinsic-operations.def @@ -24,7 +24,7 @@ operation StepSetIterator (iterator: Object/inspects+readsContents+writesContent family RuntimeIntrinsic AsRead..Placed ; isIntrinsic = true operation RequireBuiltinModule (runtimeModule: RuntimeModule) -> unknown ; unknownCode ; runtime = RequireBuiltinModule ; flow = Copies ; flowRule = Module ; form = Runtime -operation NativeBinding (builtin: Builtin) -> unknown ; Allocates UsesRuntime throws ; runtime = NativeBinding ; flow = Unsound ; flowRule = Module ; form = Runtime +operation NativeBinding (builtin: Builtin) -> constant ; Allocates UsesRuntime throws ; runtime = NativeBinding ; flow = TableProves ; flowRule = Module ; form = Runtime operation LinkTimeConstant (constant: LinkTimeConstant) -> unknown ; CreatesConstant ; runtime = LinkTimeConstant ; flow = Unsound ; form = Open operation HasJSType (value: Any/inspects, first: CheckedJSType, last: CheckedJSType) -> natural(boolean) ; none ; branches = May ; flow = Checks ; hasOwnFoldingRule = true ; form = Open operation FindsDeclaredMember (object: Any/inspects, key: PropertyKey, declaredBy?: Any/inspects, engine conditions: List, engine memberDeclared?: Function(Closure), with AccessStamp) -> natural(boolean) ; reads(PrototypeLinks, MemberGuards[conditions]) ; proves = Shape ; access = Has ; receiver = object ; branches = May ; flow = TableProves ; form = Open @@ -49,7 +49,7 @@ operation ArrayStorageKind (array: Array/inspects) -> natural(ArrayStorage) ; re operation HasContiguousIndex (array: Array/inspects, position: Integer/inspects, engine storage?: ArrayStoragePointer/inspects) [| engine isInBounds] -> natural(boolean) ; reads(ArrayStorage, Elements[Array]) ; branches = May ; flow = TableProves ; form = Open operation GetContiguousIndex (array: Array/inspects+readsContents, position: Integer/inspects, engine storage?: ArrayStoragePointer/inspects) -> unknown => declared(array) ; reads(ArrayStorage, Elements[Array]) ; flow = Copies ; flowRule = Heap ; form = Open operation GetDoubleIndex (array: Array/inspects+readsContents, position: Integer/inspects, engine storage?: ArrayStoragePointer/inspects) -> natural(number) ; reads(ArrayStorage, Elements[Array]) ; flow = TableProves ; flowRule = Heap ; form = Open -operation PutContiguousIndex (array: Array/inspects+writesContents, position: Integer/inspects, value: Any/retains(array), engine storage?: ArrayStoragePointer/inspects) [| engine goesByKindOfStorage isInBounds] -> none ; reads(ArrayStorage) writes(Elements[Array]) ; runsOwnerBarrier = true ; remembersOwner = true ; flow = Copies ; flowRule = Heap ; form = Open +operation PutContiguousIndex (array: Array/inspects+writesContents, position: Integer/inspects, value: Any/retains(array), engine storage?: ArrayStoragePointer/inspects) [| engine goesByKindOfStorage isInBounds hasNumericStorage] -> none ; reads(ArrayStorage) writes(Elements[Array]) ; runsOwnerBarrier = true ; remembersOwner = true ; flow = Copies ; flowRule = Heap ; form = Open operation PutDoubleIndex (array: Array/inspects+writesContents, position: Integer/inspects, value: Number/inspects, engine storage?: ArrayStoragePointer/inspects) [| engine isInBounds] -> none ; reads(ArrayStorage) writes(Elements[Array]) ; flow = Copies ; flowRule = Heap ; form = Open operation GetOwnContiguousIndex (array: Array/inspects+readsContents, position: Integer/inspects, engine storage?: ArrayStoragePointer/inspects) -> unknown => declared(array) ; reads(ArrayStorage, Elements[Array]) ; relies = InBounds ; flow = Copies ; flowRule = Heap ; form = Open operation GetOwnDoubleIndex (array: Array/inspects+readsContents, position: Integer/inspects, engine storage?: ArrayStoragePointer/inspects) -> natural(number) ; reads(ArrayStorage, Elements[Array]) ; relies = InBounds ; flow = TableProves ; flowRule = Heap ; form = Open @@ -63,6 +63,7 @@ operation TryPutByValDirect (array: Array/inspects+writesContents, position: Int operation TryPutIndex (array: Array/inspects+writesContents, position: Integer/inspects, value: Any/retains(array)) -> natural(boolean) ; reads(ArrayStorage, RuntimeState) writes(ArrayStorage, Elements[Array]) ; runsOwnerBarrier = true ; flow = TableProves ; flowRule = Heap ; form = Open operation TryCopyFilledElements (array: Array/inspects+writesContents, source: Array/inspects+readsContents+retainsContents, count: Integer/inspects) -> natural(boolean) ; reads(ArrayStorage, Elements[Array]) writes(ArrayStorage, Elements[Array]) ; flow = TableProves ; flowRule = Heap ; form = Open operation TryPutElements (array: Array/inspects+writesContents, source: Array/inspects+readsContents+retainsContents, count: Integer/inspects) -> natural(boolean) ; reads(ArrayStorage, Elements[Array]) writes(Elements[Array]) ; flow = TableProves ; flowRule = Heap ; form = Open +operation SortNumbers (array: Array/inspects+readsContents+writesContents, direction: Integer/inspects) -> natural(boolean) ; reads(ArrayStorage, Elements[Array]) writes(Elements[Array]) ; runtime = SortNumbers ; flow = TableProves ; flowRule = Heap ; form = Closed operation TryFillNewArray (array: Array/inspects+writesContents, value: Any/retains(array)) -> natural(boolean) ; reads(ArrayStorage, RuntimeState) writes(ArrayStorage, Elements[Array]) ; flow = TableProves ; flowRule = Heap ; form = Open operation TruncateNewArray (array: Array/inspects+returns+writesContents, length: Integer/inspects) -> allocation(Array) => natural(same(array)) ; GrowsArray writes(ShapeWords) ; runtime = TruncateNewArray ; flow = Copies ; flowRule = Heap ; form = Open diff --git a/Source/JavaScriptCore/aot/ir/schema/operations.def b/Source/JavaScriptCore/aot/ir/schema/operations.def index 2bbd4dab7162e..1e0d34ea08ee8 100644 --- a/Source/JavaScriptCore/aot/ir/schema/operations.def +++ b/Source/JavaScriptCore/aot/ir/schema/operations.def @@ -49,7 +49,7 @@ operation PutEnvironmentField (environment: Layout/inspects, layout: Layout, fie operation ScopeHasBinding (scope: Object/inspects, key: PropertyKey) -> natural(boolean) ; unknownCode ; runtime = ScopeHasBinding ; flow = TableProves ; flowRule = Heap ; form = Runtime family Allocation AsRead..Placed -operation NewObject (layout: Layout, fields: List, prototype?: Object/retains(result), values...: Declared/retains(result), with CellStamp) -> allocation(layout) ; Allocates writes(MemberGuards) ; runtime = AllocateSlow ; interpreted = Allocation ; allocatesInline = true ; allocatesAfterLastSafepoint = true ; replaces = CreateIteratorResult ; flow = Allocates ; flowRule = Heap ; closedUnder = Always, ObjectInFrame ; openUnder = CellInFrameFilledOnce ; formDependsOnLoop = true ; openIn = Loop +operation NewObject (layout: Layout, fields: List, prototype?: Object/retains(result), values...: Declared/retains(result), with CellStamp) [| engine endsNoGuards] -> allocation(layout) ; Allocates writes(MemberGuards) ; runtime = AllocateSlow ; interpreted = Allocation ; allocatesInline = true ; allocatesAfterLastSafepoint = true ; replaces = CreateIteratorResult ; flow = Allocates ; flowRule = Heap ; closedUnder = Always, ObjectInFrame ; openUnder = CellInFrameFilledOnce ; formDependsOnLoop = true ; openIn = Loop operation NewDictionary () -> natural(object) ; Allocates ; runtime = NewDictionary ; allocatesInline = true ; flow = Allocates ; flowRule = NewDictionary ; form = Closed operation FinishObject (object: Object/returns) -> natural(same(object)) ; none ; interpreted = Cast ; flow = Copies ; passThrough = Mark ; form = Open operation NewArray (contents: Type, holes: List, elements...: Declared/retains(result), engine storage: AllocationStorage, engine literalStorage: ArrayLiteralStorage, engine arrayStorage: ArrayStorage) -> allocation(Array) ; Allocates ; runtime = NewArray ; interpreted = Allocation ; allocatesInline = true ; flow = Allocates ; flowRule = Heap ; formDependsOnLoop = true ; form = Closed @@ -136,6 +136,7 @@ operation CheckInitialized (value: Any/returns, name: String) -> cast ; throws ; operation RequireObjectCoercible (value: Any/returns, nullishAccess: NullishAccess, name?: String) -> cast ; throws ; interpreted = Cast ; flow = Checks ; runsBeforeCallee = true ; hasKnownValueRule = true ; passThrough = PassedCheck ; form = Open operation RequireObject (value: Any/returns, runtimeError: RuntimeErrorKind) -> cast ; throws ; interpreted = Cast ; flow = Checks ; hasKnownValueRule = true ; passThrough = PassedCheck ; openUnder = PassesTest, NeedsTest ; runtimeUnder = FailsTest operation TypeTest (value: Any/inspects, testedType: Type) [| engine asksInFull] -> natural(boolean) ; none ; proves = TestedType ; branches = May ; interpreted = Test ; flow = Checks ; hasKnownValueRule = true ; hasOwnFoldingRule = true ; form = Open +operation IsUnlockedInstance (value: Any/inspects, testedType: Type) -> natural(boolean) ; none ; branches = May ; interpreted = Test ; flow = Checks ; hasOwnFoldingRule = true ; form = Open family Call AsRead..Placed operation Call (callee: Any/calls, thisValue?: Any/retains, arguments...: Any/retains, calleeKey?: PropertyKey, calleeName?: String, expectedResult?: ExpectedResult, calledBuiltin?: Builtin, callNumber?: UInt32, engine methodOfReceiverAsMade?: Builtin, with CallStamp) [isNeverInlined isDoneInPlace | engine methodIsNamedByKeyAlone] -> unknown ; unknownCode by Call ; accessClass = IndirectCall ; flow = Copies ; runsBeforeCallee = true ; flowRule = Call ; entersByValue = Call ; form = Closed diff --git a/Source/JavaScriptCore/aot/ir/schema/prepared-operations.def b/Source/JavaScriptCore/aot/ir/schema/prepared-operations.def index af430f082b823..c88e09c7b3d81 100644 --- a/Source/JavaScriptCore/aot/ir/schema/prepared-operations.def +++ b/Source/JavaScriptCore/aot/ir/schema/prepared-operations.def @@ -21,7 +21,7 @@ engine operation HasNumberOfBoundArguments (function: Object/inspects, count: UI engine operation GetWordOfBoundFunction (function: Object/inspects, index: UInt32) -> unknown ; reads(RuntimeState) ; form = Open operation IsHolderWithoutAddedNames (builtin: Builtin) -> natural(boolean) ; reads(NamedProperties) ; AsRead..Placed ; flow = TableProves ; form = Open operation HasArrayStorage (array: Array/inspects, arrayStorage: ArrayStorage, engine data?: Data) [isWritable | engine admitsInt32Storage] -> natural(boolean) ; reads(ArrayStorage) ; proves = ArrayStorage ; form = Open -operation IsElementPresent (array: Array/inspects, position: Number/inspects, storage?: ArrayStoragePointer/inspects, length...: Integer/inspects, arrayStorage: ArrayStorage) [keyIsIntegral | engine admitsInt32Storage isInBounds] -> natural(boolean) ; reads(ArrayStorage, Elements[Array]) ; proves = InBounds ; relies = ArrayStorage ; form = Open +operation IsElementPresent (array: Array/inspects, position: Number/inspects, storage?: ArrayStoragePointer/inspects, holdsDoubles?: Boolean/inspects, length...: Integer/inspects, arrayStorage: ArrayStorage) [keyIsIntegral | engine admitsInt32Storage isInBounds] -> natural(boolean) ; reads(ArrayStorage, Elements[Array]) ; proves = InBounds ; relies = ArrayStorage ; form = Open operation IsNotThenable (value: Any/inspects) -> natural(boolean) ; reads(NamedProperties, ObjectFields, PrototypeLinks, RuntimeState) ; proves = NotThenable ; replaces = ReturnAsyncResult, ResolvePromise ; formDependsOnLoop = true ; openIn = Loop ; form = Closed operation AreMembersUnchanged (conditions: List) -> natural(boolean) ; reads(MemberGuards[conditions]) ; proves = MemberUnchanged ; form = Open operation HasKeyOrder (object: Layout/inspects, layout: Layout, fields: List) [| engine hasNoFlags] -> natural(boolean) ; reads(OrderWords) ; proves = KeyOrder ; relies = Shape ; form = Open @@ -101,6 +101,8 @@ operation DivideIntegers (dividend: Integer/inspects, divisor: Integer/inspects, operation RemainderOfIntegers (dividend: Integer/inspects, divisor: Integer/inspects) -> natural(integer) ; none ; form = Open operation ReciprocalOfDivisor (divisor: Integer/inspects) -> natural(Reciprocal) ; none ; form = Open operation RemainderByReciprocal (dividend: Integer/inspects, divisor: Integer/inspects, reciprocal: Reciprocal/inspects) -> natural(integer) ; none ; form = Open +operation WholeDivisor (divisor: Number/inspects) -> natural(integer) ; none ; form = Open +operation RemainderByWholeDivisor (dividend: Number/inspects, divisor: Number/inspects, wholeDivisor: Integer/inspects) -> natural(number) ; none ; form = Open operation ToInt32 (value: Number/inspects) [| engine givesOnlyLowestBits] -> natural(int32) ; none ; runtime = ToInt32 ; hasKnownValueRule = true ; form = Open operation ToUint32 (value: Number/inspects) -> natural(uint32) ; none ; runtime = ToInt32 ; hasKnownValueRule = true ; form = Open operation ToIntegerOrInfinity (value: Number/inspects) -> natural(number) ; none ; form = Open diff --git a/Source/JavaScriptCore/aot/ir/schema/runtime-builtin-operations.def b/Source/JavaScriptCore/aot/ir/schema/runtime-builtin-operations.def index 6267dece82e0c..2b39868b9cc8b 100644 --- a/Source/JavaScriptCore/aot/ir/schema/runtime-builtin-operations.def +++ b/Source/JavaScriptCore/aot/ir/schema/runtime-builtin-operations.def @@ -28,6 +28,7 @@ runtime MoveBytes : void (uint8_t* target, const uint8_t* source, uint64_t count runtime NewArrayWithIndexingType : JSCell* (uint32_t count, const EncodedJSValue* values, uint32_t indexingType) ; reads(all) writes(AllocatorState, RuntimeState) throws collects runtime NewArrayFromDoubles : JSCell* (uint32_t countAndNumberMask, double, double, double, double) ; reads(all) writes(AllocatorState, RuntimeState) throws collects runtime NewArrayIterator : JSCell* (JSCell* array, uint32_t kind) ; RuntimeAllocates +runtime SortNumbers : size_t (JSCell* array, int32_t direction) ; reads(all) writes(Elements) runtime TruncateNewArray : JSCell* (JSCell* array, uint32_t length) ; RuntimeGrowsArray writes(ShapeWords) runtime NewArrayWithHoles : JSCell* (JSCell* source, uint64_t length, uint32_t indexingType, uint32_t flags) ; reads(all) writes(AllocatorState, RuntimeState) throws collects runtime NewArrayWithHolesAtSite : JSCell* (void* profile, JSCell* source, uint64_t length, uint32_t seedIndexingType) ; reads(all) writes(AllocatorState, RuntimeState) throws collects @@ -92,6 +93,7 @@ runtime StringUTF8Length8 : size_t (const Latin1Character* characters, uint32_t runtime StringUTF8Length16 : size_t (const char16_t* characters, uint32_t length) ; reads(StringStorage) ; leaf preserveMost runtime StringImplsAreEqual : size_t (void* first, void* second) ; reads(StringStorage) ; leaf preserveMost runtime CompareStringImpls : int64_t (void* first, void* second) ; reads(StringStorage) ; leaf preserveMost +runtime CompareStringsInPlace : int64_t (JSCell* first, JSCell* second) ; reads(StringStorage) ; leaf preserveMost runtime StringImplEqualsImageString : size_t (void* impl, uint32_t string) ; reads(StringStorage) ; leaf preserveMost runtime CharactersEqualImageString : size_t (const Latin1Character* characters, uint32_t length, uint32_t string) ; reads(StringStorage) ; leaf preserveMost runtime HashStringImpl : size_t (void* impl) ; reads(StringStorage) writes(StringStorage) ; leaf preserveMost @@ -99,6 +101,8 @@ runtime FindKeptStringOfTwoCharacters : size_t (void* vm, uint32_t units) ; read runtime StringFindLiteral8 : size_t (const Latin1Character* characters, uint32_t length, uint32_t text, uint32_t start) ; reads(all) runtime StringReverseFindLiteral8 : size_t (const Latin1Character* characters, uint32_t length, uint32_t text, int32_t position) ; reads(all) +runtime StringFindLiteral16 : size_t (const char16_t* characters, uint32_t length, uint32_t text, uint32_t start) ; reads(all) +runtime StringReverseFindLiteral16 : size_t (const char16_t* characters, uint32_t length, uint32_t text, int32_t position) ; reads(all) runtime StringEqualsLiteral : size_t (JSCell* string, uint32_t text) ; reads(all) runtime StringEqualsLiteralInRegister : size_t (JSCell* string, uint64_t text) ; reads(all) runtime StringIndexOfLiteral, StringLastIndexOfLiteral : size_t (JSCell* string, uint32_t text, int32_t position) ; RuntimeCreatesString @@ -112,6 +116,7 @@ runtime StringToLowerCase, StringToUpperCase : JSCell* (JSCell* string) ; Runtim runtime StringRepeat : JSCell* (JSCell* string, uint32_t count) ; RuntimeCreatesString runtime StringPad : JSCell* (JSCell* string, uint32_t length, JSCell* filler, uint32_t isAtStart) ; RuntimeCreatesString runtime NarrowCharactersOfString : size_t (JSCell* string) ; reads(all) +runtime StringCodeUnitAt : size_t (JSCell* string, uint32_t index) ; reads(all) writes(StringStorage, AllocatorState) effects RuntimeMatchesRegExp = reads(all) writes(RegExpState, StringStorage, AllocatorState, RuntimeState) throws collects runtime RegExpExec : EncodedJSValue (JSCell* regExp, JSCell* string) ; unknownCode diff --git a/Source/JavaScriptCore/aot/ir/schema/runtime-operations.def b/Source/JavaScriptCore/aot/ir/schema/runtime-operations.def index e3b325e0b297c..43b9384d67322 100644 --- a/Source/JavaScriptCore/aot/ir/schema/runtime-operations.def +++ b/Source/JavaScriptCore/aot/ir/schema/runtime-operations.def @@ -111,6 +111,7 @@ runtime StringBuilderToString, StringBuilderFinish : JSCell* (JSCell* builder, u runtime StringBuilderDiscard : void (JSCell* builder) ; reads(all) writes(StringBuilderContents, RuntimeState) runtime StackStringBuilderGrow : uint8_t* (uint8_t* builder, uint8_t* cursor) ; reads(all) writes(StringStorage, StringBuilderContents, AllocatorState, RuntimeState) throws collects runtime StackStringBuilderAppend : uint8_t* (uint8_t* builder, uint8_t* cursor, EncodedJSValue primitive) ; reads(all) writes(StringStorage, StringBuilderContents, AllocatorState, RuntimeState) throws collects +runtime StackStringBuilderAppendCodeUnit : uint8_t* (uint8_t* builder, uint8_t* cursor, uint32_t codeUnit) ; reads(all) writes(StringStorage, StringBuilderContents, AllocatorState, RuntimeState) throws collects runtime StackStringBuilderAppendStringRange : uint8_t* (uint8_t* builder, uint8_t* cursor, JSCell* string, int32_t start, int32_t end) ; reads(all) writes(StringStorage, StringBuilderContents, AllocatorState, RuntimeState) throws collects runtime StackStringBuilderToString : JSCell* (uint8_t* builder, uint8_t* cursor) ; reads(all) writes(StringStorage, StringBuilderContents, AllocatorState, RuntimeState) throws collects runtime StackStringBuilderFinish : JSCell* (uint8_t* builder, uint8_t* cursor, uint32_t isLikelyKey) ; reads(all) writes(StringStorage, StringBuilderContents, AllocatorState, RuntimeState) throws collects diff --git a/Source/JavaScriptCore/aot/ir/schema/selected-operations.def b/Source/JavaScriptCore/aot/ir/schema/selected-operations.def index e1376f96d13fe..d373c43eea512 100644 --- a/Source/JavaScriptCore/aot/ir/schema/selected-operations.def +++ b/Source/JavaScriptCore/aot/ir/schema/selected-operations.def @@ -15,7 +15,7 @@ operation FulfillFreshPromise (promise: Object/inspects+writesContents, value: A family String Selected..Placed operation Substring (string: String/inspects, start: Integer/inspects, end: Integer/inspects) -> natural(string) ; CreatesString ; Prepared..Placed ; interpreted = Allocation ; replaces = CallBuiltin, ElemGet ; runtime = Substring ; hasOwnFoldingRule = true ; form = Closed -operation FlattenString (string: String/inspects+returns, isRead?: Boolean/inspects) -> natural(FlatString) ; ResolvesRopes ; Prepared..Placed ; replaces = CallBuiltin, ElemGet, StrictEq, Substring, StringEqualsLiteral, StringOrderToCharacter, StringStartsWithLiteral, StringEndsWithLiteral, StringIndexOfLiteral, StringIndexOfCharacter, StringLastIndexOfLiteral, GetStringCodeUnit, GetStringLength ; runtime = Flatten ; form = Closed +operation FlattenString (string: String/inspects+returns, isRead...: Boolean/inspects) [| engine mayStayRope] -> natural(FlatString) ; ResolvesRopes ; Prepared..Placed ; replaces = CallBuiltin, ElemGet, StrictEq, Substring, StringEqualsLiteral, StringOrderToCharacter, StringStartsWithLiteral, StringEndsWithLiteral, StringIndexOfLiteral, StringIndexOfCharacter, StringLastIndexOfLiteral, GetStringCodeUnit, GetStringLength ; runtime = Flatten ; form = Closed operation StringRange (length: Integer/inspects, first: Number/inspects, second?: Number/inspects, method: StringRangeMethod) -> results(natural(int[0, 2147483647]), natural(int[0, 2147483647])) ; none ; Prepared..Placed ; form = Open operation StringOrder (left: String/inspects, right: String/inspects) -> natural(int[-1, 1]) ; ResolvesRopes ; replaces = Less, LessEq, Greater, GreaterEq, CallBuiltin ; runtime = CompareStrings ; form = Closed operation StringOrderToCharacter (string: String/inspects, codeUnit: UInt32) -> natural(int[-1, 1]) ; ResolvesRopes ; replaces = Less, LessEq, Greater, GreaterEq ; form = Open @@ -23,9 +23,9 @@ operation StringEqualsLiteral (string: String/inspects, length?: Integer/inspect operation ValueEqualsLiteral (value: Any/inspects, length?: Integer/inspects, literal: String) [| engine hasLeftLoop] -> natural(boolean) ; reads(StringStorage) ; proves = Equality ; branches = May ; replaces = StrictEq ; runtime = StringEqualsLiteral ; formDependsOnLoop = true ; openUnder = EmptyLiteral, LengthOfSubject ; closedUnder = Always, LiteralComparedByStub ; openIn = Loop operation StringStartsWithLiteral (string: String/inspects, literal: String) -> natural(boolean) ; ResolvesRopes ; branches = May ; replaces = CallBuiltin ; form = Open operation StringEndsWithLiteral (string: String/inspects, literal: String) -> natural(boolean) ; ResolvesRopes ; branches = May ; replaces = CallBuiltin ; form = Open -operation StringIndexOfLiteral (string: String/inspects, position: Integer/inspects, literal: String) -> natural(int[-1, 2147483647]) ; ResolvesRopes ; replaces = CallBuiltin ; runtime = StringIndexOfLiteral ; form = Runtime -operation StringIndexOfCharacter (string: String/inspects, position: Integer/inspects, codeUnit: UInt32) -> natural(int[-1, 2147483647]) ; ResolvesRopes ; replaces = CallBuiltin ; runtime = StringIndexOfCharacter ; form = Runtime -operation StringLastIndexOfLiteral (string: String/inspects, position: Integer/inspects, literal: String) -> natural(int[-1, 2147483647]) ; ResolvesRopes ; Prepared..Placed ; replaces = CallBuiltin ; runtime = StringLastIndexOfLiteral ; form = Runtime +operation StringIndexOfLiteral (string: String/inspects, position: Integer/inspects, start?: Integer/inspects, end?: Integer/inspects, literal: String) -> natural(int[-1, 2147483647]) ; ResolvesRopes ; replaces = CallBuiltin ; runtime = StringIndexOfLiteral ; form = Runtime +operation StringIndexOfCharacter (string: String/inspects, position: Integer/inspects, start?: Integer/inspects, end?: Integer/inspects, codeUnit: UInt32) -> natural(int[-1, 2147483647]) ; ResolvesRopes ; replaces = CallBuiltin ; runtime = StringIndexOfCharacter ; form = Runtime +operation StringLastIndexOfLiteral (string: String/inspects, position: Integer/inspects, start?: Integer/inspects, end?: Integer/inspects, literal: String) -> natural(int[-1, 2147483647]) ; ResolvesRopes ; Prepared..Placed ; replaces = CallBuiltin ; runtime = StringLastIndexOfLiteral ; form = Runtime operation StringRangeEqualsLiteral (string: String/inspects, start: Integer/inspects, end: Integer/inspects, literal: String) -> natural(boolean) ; ResolvesRopes ; branches = May ; replaces = CallBuiltin, Substring, StrictEq, Eq ; form = Closed operation StringRangeToNumber (string: String/inspects, start: Integer/inspects, end: Integer/inspects) -> natural(number) ; ResolvesRopes ; replaces = CallBuiltin, Substring, ToNumber, ToNumeric ; form = Runtime operation StringRangeParseInt (string: String/inspects, start: Integer/inspects, end: Integer/inspects, radix?: Integer/inspects) -> natural(number) ; ResolvesRopes ; replaces = CallBuiltin, Substring ; form = Runtime @@ -50,7 +50,7 @@ engine operation StackStringBuilderAppendNumber (builder: StringBuilder/inspects engine operation StackStringBuilderAppendString (builder: StringBuilder/inspects, cursor: BuilderCursor/inspects, value: String/inspects) -> natural(BuilderCursor) ; AppendsToBuilder ; runtime = StackStringBuilderAppend ; base = builder ; form = Closed engine operation StackStringBuilderAppendStringRange (builder: StringBuilder/inspects, cursor: BuilderCursor/inspects, string: String/inspects, start: Integer/inspects, end: Integer/inspects) -> natural(BuilderCursor) ; AppendsToBuilder ; runtime = StackStringBuilderAppendStringRange ; base = builder ; form = Closed engine operation StackStringBuilderAppendBoolean (builder: StringBuilder/inspects, cursor: BuilderCursor/inspects, value: Boolean/inspects) -> natural(BuilderCursor) ; AppendsToBuilder ; runtime = StackStringBuilderAppend ; base = builder ; form = Open -engine operation StackStringBuilderAppendCodeUnit (builder: StringBuilder/inspects, cursor: BuilderCursor/inspects, value: Integer/inspects) -> natural(BuilderCursor) ; AppendsToBuilder ; runtime = StackStringBuilderAppend ; base = builder ; form = Open +engine operation StackStringBuilderAppendCodeUnit (builder: StringBuilder/inspects, cursor: BuilderCursor/inspects, value: Integer/inspects) -> natural(BuilderCursor) ; AppendsToBuilder ; runtime = StackStringBuilderAppendCodeUnit ; base = builder ; form = Open engine operation StackStringBuilderAppendStringOrNullish (builder: StringBuilder/inspects, cursor: BuilderCursor/inspects, value: Any/inspects) -> natural(BuilderCursor) ; AppendsToBuilder ; runtime = StackStringBuilderAppend ; base = builder ; form = Closed engine operation StackStringBuilderAppendValue (builder: StringBuilder/inspects, cursor: BuilderCursor/inspects, value: Any/inspects) -> natural(BuilderCursor) ; AppendsToBuilder ; runtime = StackStringBuilderAppend ; base = builder ; form = Runtime diff --git a/Source/JavaScriptCore/aot/ir/schema/statistics.def b/Source/JavaScriptCore/aot/ir/schema/statistics.def index 6c2bd056c928c..60eca1aacff2d 100644 --- a/Source/JavaScriptCore/aot/ir/schema/statistics.def +++ b/Source/JavaScriptCore/aot/ir/schema/statistics.def @@ -50,14 +50,14 @@ enum StatisticID (stage: StatisticStage) { ObjectSpreadsRemovedAsEmpty URLAllocationsElided IteratorsSteppedWithoutCall IteratorMethodsCalledDirectly AsyncArrayIterationsIndexed GeneratorMethodsInlinedIntoIterations SpeculatedNextMethodsGuarded DeclaredNextMethodsGuarded OwnKeyWalksExpanded FieldInitializationChecksEmitted ObjectInitializationChecksEmitted IncompleteAllocationsTrapped CrossModuleVariableChecksEmitted - LibraryFunctionsCalledDirectly LibraryResultsCheckedAgainstSignatures SortComparatorCallsElided ArraySpeciesLookupsElided ByteLiteralsEncoded + LibraryFunctionsCalledDirectly ListCallbacksCopiedWithCalls MethodChainsKeptInArm LibraryResultsCheckedAgainstSignatures SortComparatorCallsElided ArraySpeciesLookupsElided ByteLiteralsEncoded BuiltinCallsOnNarrowRow BuiltinCallsOnWideRow BuiltinCallsOnOnlyRow BuiltinCallsOnLibraryFunction BuiltinCallsLeftGeneric CollectionLookupsHoisted ResultsCarriedAroundLoops MapEntryIndicesCached MapEntryIndicesSurvivingInserts MapResultsGuardedAsNumbers Int32GuardsHoistedForComparisons InstanceOfMissesWalkedInline InstanceOfMissesDecidedByHierarchy InstanceOfTestsJumpThreaded InstanceOfTestsDecidedByHierarchy InstanceOfGuardsHoisted InstanceOfGuardsNotHoisted VariableReadsHoistedWithinFrame VariableReadsNotHoisted PreheadersProvenUnreachable AccessesProven AccessesGuarded AccessesUnknown ValuesInDouble LiteralArgumentsScalarized ArrayOrStringLoopsVersioned CounterIndexedLoopsVersioned LoopsReversionedWithoutConditionalSpeculations SlotsProvenFilledByTruthiness LoopsReversionedWithoutHoistedGuards LoopsVersionedInsideOuterVersions - LoopsVersionedForAnyArray TypedArraySubclassLoopsVersioned CollectionCallRunsGuardedOnce DuplicateCollectionStoresRemoved LoopsReversionedForOtherIndexingType + LoopsVersionedForAnyArray TypedArraySubclassLoopsVersioned CollectionCallRunsGuardedOnce DuplicateCollectionStoresRemoved LoopsReversionedForOtherIndexingType CopiesForInt32sStruck EnteringElementsGuardedAsNumbers SumsSpecializedAsNumbers SiteCachedReadsHoisted SiteCachedReadsBesideCalls ArgumentSpeculationsPassedToInlinees PartialInlinesAfterDominatingTests InlineeGuardsHoistedPastStores InlineeSpeculationsHoistedPastStores CapturedVariableReadsHoisted InvariantKeyReadsHoisted KeyedRowLookupsHoisted KeyedRowLookupsRevalidated StaticFieldGuardsHoisted StaticFieldGuardsReused InvariantBasesGuardedAsArrays @@ -81,12 +81,12 @@ enum StatisticID (stage: StatisticStage) { UnrolledLoopRawTruthinessTests UnrolledLoopsForInt32Truthiness UnrolledLoopsWithSlotStores LocalArrayMethodEffectsNarrowed ReceiversSpeculatedForNamedCallbacks ValuesWalked ValuesWalkedAndKept CallsInlined OperationsBuilt OperationsSwept - DirectCalls IndirectCalls SpeculationsEmitted WalkLoopsVersioned LoopCarriedObjectsShapeChecked ArrayOrUint8ArrayLoopsVersioned VariableReceiverGuardsHoisted LoopVersionsFailingIntoFallback ArraysGuardedBesideArrays IterationRestartsEmitted StoresUndoneAtIterationRestarts SiteCachedReadsEmitted SiteCachedStoresEmittedInline SiteCachedReadGroupsEmitted SiteCachedReadsGrouped CallsInlinedAfterFailedGuards TypedArrayRowStoreRegionsGuarded ArraySpeculationsWithTypedArrayFallback Int32SpeculationsWithNumberFallback ArrayParameterChecksSpanningBranches TypedArrayRowsGuardedInIterations AbsentElementWideningsAvoided EmptySlotCasesEmitted LoopVisits LoopVisitsUndone LoopVisitsAborted LoopEffectsPredictedFromSource LoopEffectPredictionsRefuted LoopVersioningAttemptsFailed FreshObjectSlotLossesIgnored FreshObjectFactLossesIgnored + DirectCalls IndirectCalls SpeculationsEmitted WalkLoopsVersioned LoopCarriedObjectsShapeChecked ArrayOrUint8ArrayLoopsVersioned VariableReceiverGuardsHoisted LoopVersionsFailingIntoFallback ArraysGuardedBesideArrays IterationRestartsEmitted StoresUndoneAtIterationRestarts SiteCachedReadsEmitted SiteCachedStoresEmittedInline SiteCachedReadGroupsEmitted SiteCachedReadsGrouped CallsInlinedAfterFailedGuards InlineeNumberGuards TypedArrayRowStoreRegionsGuarded ArraySpeculationsWithTypedArrayFallback Int32SpeculationsWithNumberFallback ArrayParameterChecksSpanningBranches TypedArrayRowsGuardedInIterations AbsentElementWideningsAvoided EmptySlotCasesEmitted LoopVisits LoopVisitsUndone LoopVisitsAborted LoopEffectsPredictedFromSource LoopEffectPredictionsRefuted LoopVersioningAttemptsFailed FreshObjectSlotLossesIgnored FreshObjectFactLossesIgnored ObjectAssignCallsExpanded JoinsDuplicatedPerPredecessor NarrowedValuesMergedAtJoins RepeatedConditionsJumpThreaded CalleesSpeculatedFromFieldWrites FreshObjectsMergedAtJoins AllocationsSunk AllocationsTypedByRunningConstructor PlainInstanceChecksLeftGeneric MemberLookupChecksLeftGeneric EntryParameterProofsChecked CallResultProofsChecked FieldProofsChecked VariableProofsChecked CaptureProofsChecked ScopeSlotProofsChecked ContentsProofsChecked OtherCallResultProofsChecked OtherOperationProofsChecked TestProofsChecked RememberedFactProofsChecked HoistedCallProofsChecked LifetimeProofsChecked UntestableProofsSkipped ProofsImpliedByRepresentation UnsatisfiableProofChecks ProofChecksFolded ProofCheckOperationsRunningCode FreshCellsSkippedAtReturns - AllocateInFramesChanges ClosuresInFrames KnownInductionsUsed InvariantNumberGuardsHoisted SumsGuardedAsNumbers LoopsRevisitedWithoutSumGuards CachedLengthsRevalidatedInPreheaders BuiltinMembersCheckedForReplacement LoopBoundsClampedToTypedArrays Int32OrUint32GuardsHoisted ArrayBoundsChecksHoisted TypedArrayBoundsChecksHoisted + AllocateInFramesChanges ClosuresInFrames KnownInductionsUsed InvariantNumberGuardsHoisted CarriedIndexGuardsHoisted StringsResolvedOnRead SumsGuardedAsNumbers LoopsRevisitedWithoutSumGuards CachedLengthsRevalidatedInPreheaders BuiltinMembersCheckedForReplacement LoopBoundsClampedToTypedArrays Int32OrUint32GuardsHoisted ArrayBoundsChecksHoisted TypedArrayBoundsChecksHoisted ArraysSpeculatedFromIndexedReads ArraysSpeculatedFromLocationUses ArraysSpeculatedFromElementUses StringsSpeculatedFromLocationUses TypedArrayLengthGuardsAllowingSubclasses @@ -104,7 +104,7 @@ enum StatisticID (stage: StatisticStage) { SlicesFoldedToArrayLiterals ArrayLiteralsAllocatedForLoops IteratorStepHoleChecksRemoved - ArraysGuardedInIterations ElementsGuardedInIterations RowsGuardedForSiblingIndexingType + ListsGuardedForReads WholeNumberListsAskedTwice BoxedStoresByKind StepsRestatedBehindTests StoringLoopsLeftAsLoops AddendsWhoseSignIsLeftToTurns ReadsGuardedForNumbers DifferencesShownFromZeroUp IntegersOfViewsReadAsDoubles ConstantsFoldedIntoElementAddresses PositionsShownByTestedTerms LengthsAskedForUnrolledTurns ArraysGuardedInIterations ElementsGuardedInIterations RowsGuardedForSiblingIndexingType ArraysGuardedAtLoopHeaders ArrayGuardsHoistedPastStores ArraysSpeculatedFromLoopBounds @@ -119,7 +119,7 @@ enum StatisticID (stage: StatisticStage) { RetainedObjectFactLossesIgnored RowStoreRegionsGuarded PartialRegionFallbacksEmitted RowStoreRegionsRestartingIterations MethodCallsDispatchedByShape PropertyReadsDispatchedByShape SpeculatedReceiversDispatchedByShape - ConstructsDispatchedByCallee CallsDispatchedByCallee SlotClosureLoopsVersioned JoinsDuplicatedPerCallee ConstructorReadsResolvedToClasses + ConstructsDispatchedByCallee CallsDispatchedByCallee ClosureTestsChained SlotClosureLoopsVersioned JoinsDuplicatedPerCallee ConstructorReadsResolvedToClasses MethodCallsThroughTable IteratorStepsThreadedUnderHandlers JoinsDuplicatedPerKeyString AccessesDispatchedByKeyString ConstantArrayElementsFolded ConstantKeyLoopsFullyUnrolled @@ -141,12 +141,12 @@ enum StatisticID (stage: StatisticStage) { StringOperandsGuardedInIterations NumberOperandsGuardedInIterations BodiesReadForBirths ConstructorFieldsFoldedIntoAllocations KeyOrderChecksBeforeStores RemaindersByHoistedReciprocal - DivisorVariableReadsHoisted AbsenceTestedVariableReadsHoisted CalleeVariableReadsHoisted ImportedFunctionReadsHoisted LoopStoredVariableReadsHoisted RawVariableReadsHoisted + DivisorVariableReadsHoisted AbsenceTestedVariableReadsHoisted CalleeVariableReadsHoisted ImportedFunctionReadsHoisted LoopStoredVariableReadsHoisted SlotsCarriedAroundLoops CarriedSlotsMergedAtJoins SumsWithUndefinedTruncated RawVariableReadsHoisted BuiltinSubclassesCheckedByStructure - ZeroOrOnePhisFolded NarrowedBindingsFoldedAtJoins NarrowedArgumentsFoldedAtJoins + ZeroOrOnePhisFolded NarrowedBindingsFoldedAtJoins NarrowedArgumentsFoldedAtJoins UndoableStoresMergedAtJoins ParameterGuardsSkipped ParameterGuardsEntered [Emit] - NativeFunctionsCalledDirectly NativeFunctionTestsEmitted DoublesRecomputedAsIntegers UnboxingsFusedIntoTypeChecks CollectionStepsSkippingDeletedPrefix MapEntryAllocationsElided LoopsRotated + NativeFunctionsCalledDirectly NativeFunctionTestsEmitted DoublesRecomputedAsIntegers UnboxingsFusedIntoTypeChecks NumberTestsDeferredToSums CollectionStepsSkippingDeletedPrefix MapEntryAllocationsElided LoopsRotated InnerPointersAcrossSafepoints NullChecksFoldedIntoAccesses Int64ComparisonsWithBoxedNumbers ClosureChecksByNumberRanges OperationsEmitted MachineOperationsAppended MachineOperationsScanned MachineOperationsAllocated MachineOperationsEncoded ValuesSpilled LaterUsesScannedForLiveness BlocksVisitedByLivenessSearches @@ -159,7 +159,7 @@ enum StatisticID (stage: StatisticStage) { VectorsOfViewsRead AccessesInsideViews UnrolledLoopsEmitted CodeUnitIndexExtensionsRemoved UnrolledLoopStoresSunk UnrolledLoopIntegerSums IntegralStoresWithoutDoubleConversion - Int32StoresSharingBoxedPath BoxedReadsWithDoublePath UnrolledLoopsForIndexingType UnrolledLoopsForTypedArrays UnrolledIterationsForTypedArrays UnrolledLoopRemaindersShared UnrolledLoopTripChecksReused UnrolledLoopsForMaskedIndexes UnrolledLoopComparisonsHoisted UnrolledLoopTypeProofsChecked UnrolledLoopIntegerElementSums RecordedLengthLoopsFullyUnrolled ClampedCapacitiesReserved StringsSpeculatedFromElementComparisons ArrayLiteralAllocationsSunk ArrayLiteralsAllocatedWithCapacity ArrayLiteralsAllocatedFromSpread UntypedKeyStoresAppendingInline CapturedArrayReadsHoisted LaterStoreBoundsChecksHoisted Int32KeysGuardedInIterations ModuleConstantReadsHoisted AbsentArrayElementWideningsAvoided StoredValuesNarrowedToInt32 UnrolledLoopsWithReentry UnrolledLoopsWithSharedReentry + Int32StoresSharingBoxedPath BoxedReadsWithDoublePath NumberElementsReadBoxed UnrolledLoopsForIndexingType UnrolledLoopsForTypedArrays UnrolledIterationsForTypedArrays UnrolledLoopRemaindersShared UnrolledLoopTripChecksReused UnrolledLoopsForMaskedIndexes UnrolledLoopComparisonsHoisted UnrolledLoopTypeProofsChecked UnrolledLoopIntegerElementSums RecordedLengthLoopsFullyUnrolled ClampedCapacitiesReserved StringsSpeculatedFromElementComparisons ArrayLiteralAllocationsSunk ArrayLiteralsAllocatedWithCapacity ArrayLiteralsAllocatedFromSpread UntypedKeyStoresAppendingInline CapturedArrayReadsHoisted LaterStoreBoundsChecksHoisted Int32KeysGuardedInIterations ModuleConstantReadsHoisted AbsentArrayElementWideningsAvoided StoredValuesNarrowedToInt32 UnrolledLoopsWithReentry UnrolledLoopsWithSharedReentry UnrolledLoopLengthChecksHoisted LengthsStoredIntoProvenLists ArrayPushesInlined @@ -169,7 +169,7 @@ enum StatisticID (stage: StatisticStage) { SuperBasesFromClassRecords SuperCallsDevirtualized SuperMethodLookupsGuarded IteratorAllocationsElidedInInlinees AlternateShapesSpeculatedAtJoins - RemaindersByConditionalSubtraction RemaindersByProvenConstantDivisor + RemaindersByConditionalSubtraction RemaindersByProvenConstantDivisor MultiplesTestedWithoutRemainder ReusedLoadsVerified FreshResultsChecked [Image] @@ -179,7 +179,7 @@ enum StatisticID (stage: StatisticStage) { ClosedPrototypeObjects ClosedClassObjects FunctionsWithClosedClosures ClosedSidesInheritingAddedNames ImageFunctionsEmitted CodeBytesEmitted StubBytes ConstantBytesEmitted TableBytesEmitted StringBytesEmitted Veneers StubCopiesEmitted RegExpsCompiled RegExpsInterpreted - WasmFunctionsCompiled WasmOptimizedCopies WasmBaselineCopies WasmOptimizedCheckingCopies WasmBaselineCheckingCopies WasmJumpsThreaded WasmFirstStateTests WasmFunctionsWithTwoCopies WasmCodeBytesEmitted WasmSecondCopyBytesEmitted + WasmFunctionsCompiled WasmOptimizedCopies WasmBaselineCopies WasmOptimizedCheckingCopies WasmBaselineCheckingCopies WasmJumpsThreaded WasmFirstStateTests WasmDirectEntries WasmFunctionsWithTwoCopies WasmCodeBytesEmitted WasmSecondCopyBytesEmitted FunctionBytesEmitted FarInstanceWordBytesEmitted VeneerBytesEmitted SharedCodeBytesEmitted ArgumentThunkBytes FinalJumpBytesRemoved FFICodeBytesEmitted FFIDirectoryBytesEmitted RegExpCodeBytesEmitted RegExp16BitCodeBytes RegExpTableBytesEmitted diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineAllocationChecker.cpp b/Source/JavaScriptCore/aot/machine/AOTMachineAllocationChecker.cpp index 15b848b525a4d..6fdfc9fa15490 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineAllocationChecker.cpp +++ b/Source/JavaScriptCore/aot/machine/AOTMachineAllocationChecker.cpp @@ -723,6 +723,10 @@ bool AllocationChecker::checkLeaf(const Operation* reader, const Operation* user return fail(FailureKind::WrongValueInRegister, reader); [[fallthrough]]; case Location::Immediate: + if (location.isImmediate() && leaf->hasNoPosition() && comparesConstantInMemory(*user, childIndex, *leaf)) { + ++numberOfMemoryOperands; + return acceptsMemoryOperand(*user, childIndex) || fail(FailureKind::WrongValueInRegister, reader); + } if (!leaf->hasNoPosition() || !standsInInstruction(m_liveness, user, childIndex, leaf)) return fail(FailureKind::WrongValueInRegister, reader); return true; diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineAllocator.cpp b/Source/JavaScriptCore/aot/machine/AOTMachineAllocator.cpp index 342942d9cb4b7..61be69cd1d8ea 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineAllocator.cpp +++ b/Source/JavaScriptCore/aot/machine/AOTMachineAllocator.cpp @@ -901,7 +901,8 @@ static NEVER_INLINE Location placeAllocatedLeafInRegisterOrMemory(Allocator& all allocator.noteFrameUse(); if ((allocator.m_valueFacts[leaf->index()].lastUse <= allocator.m_position || !allocator.hasFreeRegister(bank, candidates)) && takeMemoryOperandForAllocatedLeaf(allocator, user, childIndex, taken)) return allocator.memoryLocationOf(leaf); - } + } else if (comparesConstantInMemory(*user, childIndex, *leaf) && takeMemoryOperandForAllocatedLeaf(allocator, user, childIndex, taken)) + return Location::immediate(); reg = allocator.bringIntoRegister(leaf, candidates, hintIfAmong(allocator.hintOf(leaf), candidates)); if (!reg) { allocator.m_proc.fail(FailureKind::OutOfRegisters, user); diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineArena.h b/Source/JavaScriptCore/aot/machine/AOTMachineArena.h index f06e379bc3057..46fcfdc7e57b1 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineArena.h +++ b/Source/JavaScriptCore/aot/machine/AOTMachineArena.h @@ -175,6 +175,40 @@ class Arena { bool m_forgetsRoomsLetGoAtReset { false }; }; +template +class UncheckedSpan { +public: + UncheckedSpan() = default; + template + UncheckedSpan(std::span span) + : m_data(span.data()) + , m_size(span.size()) + { + } + template + UncheckedSpan(UncheckedSpan other) + : m_data(other.data()) + , m_size(other.size()) + { + } + + T& operator[](size_t index) const + { + ASSERT(index < m_size); + return m_data[index]; + } + T* data() const { return m_data; } + T* begin() const { return m_data; } + T* end() const { return m_data + m_size; } + std::span span() const { return { m_data, m_size }; } + size_t size() const { return m_size; } + bool empty() const { return !m_size; } + +private: + T* m_data { nullptr }; + size_t m_size { 0 }; +}; + template class ArenaVector { static_assert(std::is_trivially_destructible_v); diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineBuilder.cpp b/Source/JavaScriptCore/aot/machine/AOTMachineBuilder.cpp index 7701c05dce872..dff1a48552215 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineBuilder.cpp +++ b/Source/JavaScriptCore/aot/machine/AOTMachineBuilder.cpp @@ -338,6 +338,50 @@ static ALWAYS_INLINE MachineBuilderAddress machineBuilderAddressOf(Builder& buil return { base, static_cast(distance) }; } +static bool machineBuilderAccessTakesSharedSum(const MachineBuilderAddress& address) +{ + if (!isARM64() || !address.offset || address.base->opcode() != Add || address.base->hasNoPosition()) + return false; + const Operation* pointer = address.base->child(0); + return pointer->isArgumentReg(cellCageBaseGPR) || pointer->has(OperationFlag::IsInnerPointer); +} + +#if CPU(ARM64) +struct MachineBuilderIndexOfSum { + explicit MachineBuilderIndexOfSum(const Operation* sum) + : word(sum->child(1)) + { + if (word->opcode() == Shl && word->child(1)->hasInt32()) { + shift = static_cast(word->child(1)->asInt32()) & 63; + word = word->child(0); + } + if (word->opcode() == ZExt32 || word->opcode() == SExt32) { + extension = word->opcode(); + word = word->child(0); + } + } + friend bool operator==(const MachineBuilderIndexOfSum&, const MachineBuilderIndexOfSum&) = default; + + const Operation* word; + unsigned shift { 0 }; + Opcode extension { Nop }; +}; + +Operation* Builder::sharedSumFor(Operation* sum) +{ + ASSERT(sum->opcode() == Add && sum->type() == Int64); + MachineBuilderIndexOfSum index(sum); + Operation*& last = m_lastSharedSums[(sum->child(0)->index() * 7 + index.word->index() * 3 + index.shift) % m_lastSharedSums.size()]; + if (last && last->child(0) == sum->child(0) && MachineBuilderIndexOfSum(last) == index) { + NumberingKey key = machineBuilderKeyOf(Add, Int64, last->child(0), last->child(1)); + if (Operation* visible = findNumbered(key, Numbering::hashOf(key))) + return visible; + } + last = sum; + return sum; +} +#endif + static constexpr unsigned machineBuilderNumberOfSetsOfHeapTraits = 32; static_assert(static_cast(HeapTrait::Immutable) < machineBuilderNumberOfSetsOfHeapTraits && static_cast(HeapTrait::HoldsInnerPointer) < machineBuilderNumberOfSetsOfHeapTraits); static_assert(static_cast(HeapTrait::HoldsPermanentCell) < machineBuilderNumberOfSetsOfHeapTraits && static_cast(HeapTrait::IsFilledOnce) < machineBuilderNumberOfSetsOfHeapTraits); @@ -686,7 +730,8 @@ Operation* Builder::binary(Opcode opcode, Operation* left, Operation* right) ASSERT(opcode != RotR || right->hasInt32()); sealPendingPatch(); - if (isCommutative(opcode) && machineBuilderShouldSwapOperands(left, right)) + bool comparesLinkedNumber = (opcode == Equal || opcode == NotEqual) && (left->opcode() == NumberOf) != (right->opcode() == NumberOf) && !left->isConstant() && !right->isConstant(); + if (comparesLinkedNumber ? left->opcode() == NumberOf : isCommutative(opcode) && machineBuilderShouldSwapOperands(left, right)) std::swap(left, right); Type type = typeOfResult(opcode, left); @@ -967,6 +1012,9 @@ Operation* Builder::load(Opcode opcode, Type type, Operation* base, ptrdiff_t of } MachineBuilderAddress address = machineBuilderAddressOf(*this, base, offset, !traps); + bool takesSharedSum = !traps && machineBuilderAccessTakesSharedSum(address); + if (takesSharedSum) + address.base = sharedSumFor(address.base); NumberingKey key = machineBuilderKeyOf(opcode, type, address.base, nullptr, nullptr, machineBuilderPayloadOfAccess(address.offset, heap.index())); uint32_t hash = 0; if (!traps) { @@ -977,8 +1025,10 @@ Operation* Builder::load(Opcode opcode, Type type, Operation* base, ptrdiff_t of return existing; } } - if (Operation* ownBase = duplicateIfFusable(address.base); ownBase != address.base) - return load(opcode, type, ownBase, address.offset, heap, trapping); + if (!takesSharedSum) { + if (Operation* ownBase = duplicateIfFusable(address.base); ownBase != address.base) + return load(opcode, type, ownBase, address.offset, heap, trapping); + } Operation* result = appendNewMemoryAccess(opcode, type, address.base); result->m_payload.memory.offset = address.offset; @@ -994,6 +1044,26 @@ Operation* Builder::load(Opcode opcode, Type type, Operation* base, ptrdiff_t of return result; } +Operation* Builder::makeSumOfStore(Operation* sum) +{ + if (!isX86_64() || sum->opcode() != Add || sum->hasNoPosition() || sum->isNarrowReference()) + return duplicateIfFusable(sum); + Operation* base = sum->child(0); + Operation* index = sum->child(1); + Operation* amount = index->opcode() == Shl && index->child(1)->hasInt32() ? index->child(1) : nullptr; + Operation* scaled = amount ? index->child(0) : index; + if (base->hasNoPosition() || scaled->hasNoPosition() || scaled->isNarrowReference() || !isEncodableIndexedAddress(Width64, amount ? static_cast(amount->asInt32()) & 63 : 0, IndexExtension::None, 0)) + return duplicateIfFusable(sum); + if (scaled->opcode() == ZExt32 && !scaled->child(0)->hasNoPosition() && clearsUpperHalf(scaled->child(0))) { + Operation* extension = appendNewComputation(ZExt32, Int64, scaled->child(0)); + extension->m_flags |= scaled->m_flags & static_cast(OperationFlag::ExtendsWordThatIsNotNegative); + scaled = extension; + } + Operation* own = appendNewComputation(Add, Int64, base, amount ? appendNewComputation(Shl, Int64, scaled, amount) : scaled); + own->m_flags |= sum->m_flags & static_cast(OperationFlag::IsInnerPointer); + return own; +} + Operation* Builder::store(Opcode opcode, Operation* value, Operation* base, ptrdiff_t offset, const AbstractHeap& heap) { ASSERT(isStore(opcode) && heap.isInitialized()); @@ -1009,7 +1079,7 @@ Operation* Builder::store(Opcode opcode, Operation* value, Operation* base, ptrd MachineBuilderAddress address = machineBuilderAddressOf(*this, base, offset, true); Operation* ownValue = duplicateIfFusable(value); - Operation* ownBase = duplicateIfFusable(address.base); + Operation* ownBase = machineBuilderAccessTakesSharedSum(address) ? sharedSumFor(address.base) : value->type().isFloat() ? makeSumOfStore(address.base) : duplicateIfFusable(address.base); Operation* result = appendNewMemoryAccess(opcode, Void, ownValue, ownBase); result->m_payload.memory.offset = address.offset; result->m_payload.memory.heap = heap.index(); diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineBuilder.h b/Source/JavaScriptCore/aot/machine/AOTMachineBuilder.h index 021f76da8b51e..871e727ebe4e6 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineBuilder.h +++ b/Source/JavaScriptCore/aot/machine/AOTMachineBuilder.h @@ -9,6 +9,7 @@ #include "AOTMachine.h" #include "AOTMachineNumbering.h" +#include #include #include @@ -223,6 +224,12 @@ class Builder { return index < m_epochOfLastInheritedFind.size() && m_epochOfLastInheritedFind[index] == m_scopeEpoch; } Operation* appendLoadLike(const Operation* load, Operation* base); + Operation* makeSumOfStore(Operation* sum); +#if CPU(ARM64) + Operation* sharedSumFor(Operation* sum); +#else + Operation* sharedSumFor(Operation* sum) { return sum; } +#endif void verifyReusedLoad(Operation* load, Operation* baseAsked, int32_t offsetAsked); void noteNumberedLoadForStores(Operation* load); void stopIfLoadOfStoredWordStands(const Operation* store); @@ -282,6 +289,9 @@ class Builder { std::array m_smallConstants64 { }; ArenaVector m_epochOfLastInheritedFind; uint32_t m_scopeEpoch { 0 }; +#if CPU(ARM64) + std::array m_lastSharedSums { }; +#endif bool m_hasInheritedFindsInEpoch { false }; bool m_isDuplicating { false }; bool m_isVerifyingReusedLoad { false }; diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineEncoder.h b/Source/JavaScriptCore/aot/machine/AOTMachineEncoder.h index 127155d516c2d..1a95d7696e927 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineEncoder.h +++ b/Source/JavaScriptCore/aot/machine/AOTMachineEncoder.h @@ -11,43 +11,13 @@ #include "AOTMachineFrame.h" #include "AOTMachineSlots.h" #include "AllowMacroScratchRegisterUsage.h" +#include "MarkedBlockCage.h" #include namespace JSC { namespace AOT { namespace Machine { WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN -template -class UncheckedSpan { -public: - UncheckedSpan() = default; - template - UncheckedSpan(std::span span) - : m_data(span.data()) - , m_size(span.size()) - { - } - template - UncheckedSpan(UncheckedSpan other) - : m_data(other.data()) - , m_size(other.size()) - { - } - - T& operator[](size_t index) const - { - ASSERT(index < m_size); - return m_data[index]; - } - T* data() const { return m_data; } - size_t size() const { return m_size; } - bool empty() const { return !m_size; } - -private: - T* m_data { nullptr }; - size_t m_size { 0 }; -}; - inline Operation* childWithoutCheck(const Operation* operation, unsigned index) { ASSERT(index < operation->numChildren()); @@ -128,6 +98,8 @@ struct Test { const Operation* loadOnLeft { nullptr }; AddressForm addressOnLeft { }; bool rightIsNumberOfBit { false }; + IndexExtension extensionOfRight { IndexExtension::None }; + const Operation* linkedNumberOnRight { nullptr }; }; struct FrameAtBlock { @@ -144,6 +116,22 @@ enum class MemoryOfEncodedMove : bool { IsRead, IsWritten }; inline constexpr unsigned minBytesOfTeardownSharedByReturns = isARM64() ? 12 : 8; +inline bool tryMoveSmallConstant(CCallHelpers& jit, int32_t constant, GPRReg destination) +{ +#if CPU(X86_64) + static_assert(!static_cast(MarkedBlockCage::maxSize)); + if (!isRepresentableAs(constant)) + return false; + jit.m_assembler.leal_mr(constant, cellCageBaseGPR, destination); + return true; +#else + UNUSED_PARAM(jit); + UNUSED_PARAM(constant); + UNUSED_PARAM(destination); + return false; +#endif +} + class Encoder { WTF_MAKE_NONCOPYABLE(Encoder); public: @@ -403,7 +391,10 @@ class Encoder { MacroAssembler::Jump jumpOnOverflow(bool whenItOverflowed); void encodeNumber(const Step&); + void materializeBitsInFPR(uint64_t bits, Width, FPRReg destination); bool blockHasWrittenNumberRegisterInFrontOf(const Step&, FPRReg); + bool writesNumberRegister(const Step&, FPRReg) const; + bool numberRegisterMayBeWrittenShortlyInFrontOf(const Step&, FPRReg) const; void encodeLoad(const Step&); void encodeStore(const Step&); diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineEncoderComparisonsARM64.cpp b/Source/JavaScriptCore/aot/machine/AOTMachineEncoderComparisonsARM64.cpp index c657dfab9bb9f..f07cc5631a9af 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineEncoderComparisonsARM64.cpp +++ b/Source/JavaScriptCore/aot/machine/AOTMachineEncoderComparisonsARM64.cpp @@ -110,6 +110,78 @@ static ComparedNumbers comparedNumbersOf(const Test& test) return { left, InvalidFPRReg, areFloats }; } +template +static void compareWithUInt12(CCallHelpers& jit, GPRReg left, int64_t constant) +{ + uint64_t magnitude = constant < 0 ? uint64_t { 0 } - static_cast(constant) : static_cast(constant); + int shift = magnitude & ~uint64_t { 0xfff } ? 12 : 0; + RELEASE_ASSERT(!(magnitude & ~(uint64_t { 0xfff } << shift))); + UInt12 immediate(static_cast(magnitude >> shift)); + if (constant < 0) + jit.m_assembler.cmn(left, immediate, shift); + else + jit.m_assembler.cmp(left, immediate, shift); +} + +static void compareWithExtendedWord(CCallHelpers& jit, const Test& test) +{ + ASSERT(test.kind == Test::CompareIntegers && test.width == Width64 && test.extensionOfRight != IndexExtension::None); + ARM64Assembler::ExtendType type = test.extensionOfRight == IndexExtension::ZeroExtend32 ? ARM64Assembler::UXTW : ARM64Assembler::SXTW; + jit.m_assembler.cmp<64>(gprOfComparisonOperand(test.left), gprOfComparisonOperand(test.right), type, 0); +} + +static std::optional constantComparisonOfTest(const Test& test) +{ + if (test.kind != Test::CompareIntegers || !test.right.isImmediate()) + return std::nullopt; + std::optional found = findConstantComparison(test.relation, test.width, test.rightConstant); + RELEASE_ASSERT(found); + return found; +} + +static ARM64Assembler::Condition setFlagsForConstantComparison(CCallHelpers& jit, const Test& test, const ConstantComparison& comparison) +{ + bool is32 = test.width == Width32; + GPRReg left = gprOfComparisonOperand(test.left); + if (comparison.kind != ConstantComparison::HighBits) { + if (is32) + compareWithUInt12<32>(jit, left, comparison.constant); + else + compareWithUInt12<64>(jit, left, comparison.constant); + if (comparison.kind == ConstantComparison::Overflow) + return comparison.holdsOnOverflow ? ARM64Assembler::ConditionVS : ARM64Assembler::ConditionVC; + return static_cast(*conditionsOfComparison(comparison.relation).ofIntegers); + } + + bool isSigned = comparison.relation == LessThan || comparison.relation == GreaterEqual; + ARM64Assembler::ShiftType shift = isSigned ? ARM64Assembler::ASR : ARM64Assembler::LSR; + if (is32) + jit.m_assembler.cmp<32>(ARM64Registers::zr, left, shift, comparison.shift); + else + jit.m_assembler.cmp<64>(ARM64Registers::zr, left, shift, comparison.shift); + switch (comparison.relation) { + case LessThan: + return ARM64Assembler::ConditionGE; + case GreaterEqual: + return ARM64Assembler::ConditionLT; + case Below: + return ARM64Assembler::ConditionEQ; + case AboveEqual: + return ARM64Assembler::ConditionNE; + default: + break; + } + RELEASE_ASSERT_NOT_REACHED(); +} + +static MacroAssembler::Jump jumpOnFlags(CCallHelpers& jit, ARM64Assembler::Condition condition) +{ + jit.m_assembler.b_cond(condition); + AssemblerLabel label = jit.m_assembler.labelIgnoringWatchpoints(); + jit.m_assembler.nop(); + return MacroAssembler::Jump(label, ARM64Assembler::JumpCondition, condition); +} + static MacroAssembler::Jump branchOnComparedIntegers(CCallHelpers& jit, MacroAssembler::RelationalCondition condition, const Test& test) { bool is32 = test.width == Width32; @@ -152,7 +224,15 @@ static MacroAssembler::Jump branchOnComparedNumbers(CCallHelpers& jit, MacroAsse static MacroAssembler::RelationalCondition setFlagsForTestOfIntegers(CCallHelpers& jit, const Test& test) { + std::optional comparison = constantComparisonOfTest(test); + if (comparison && (comparison->kind != ConstantComparison::Immediate || comparison->relation != test.relation)) + return static_cast(setFlagsForConstantComparison(jit, test, *comparison)); + MacroAssembler::RelationalCondition condition = relationalConditionOfTest(test); + if (test.extensionOfRight != IndexExtension::None) { + compareWithExtendedWord(jit, test); + return condition; + } bool asksForEquality = condition == MacroAssembler::Equal || condition == MacroAssembler::NotEqual; bool testsBits = test.kind == Test::TestBits; bool is32 = test.width == Width32; @@ -277,7 +357,18 @@ MacroAssembler::JumpList Encoder::branchOn(const Test& test, bool whenItHolds) MacroAssembler::JumpList jumps; switch (test.kind) { case Test::CompareIntegers: { + std::optional comparison = constantComparisonOfTest(test); + if (comparison && (comparison->kind != ConstantComparison::Immediate || comparison->relation != test.relation)) { + ARM64Assembler::Condition condition = setFlagsForConstantComparison(m_jit, test, *comparison); + jumps.append(jumpOnFlags(m_jit, whenItHolds ? condition : ARM64Assembler::invert(condition))); + return jumps; + } MacroAssembler::RelationalCondition condition = relationalConditionOfTest(test); + if (test.extensionOfRight != IndexExtension::None) { + compareWithExtendedWord(m_jit, test); + jumps.append(jumpOnFlags(m_jit, static_cast(whenItHolds ? condition : MacroAssembler::invert(condition)))); + return jumps; + } jumps.append(branchOnComparedIntegers(m_jit, whenItHolds ? condition : MacroAssembler::invert(condition), test)); return jumps; } diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineEncoderComparisonsX86_64.cpp b/Source/JavaScriptCore/aot/machine/AOTMachineEncoderComparisonsX86_64.cpp index dd7dd7526287d..3dc52bfec6072 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineEncoderComparisonsX86_64.cpp +++ b/Source/JavaScriptCore/aot/machine/AOTMachineEncoderComparisonsX86_64.cpp @@ -9,7 +9,10 @@ #if ENABLE(AOT) && CPU(X86_64) #include "AllowMacroScratchRegisterUsage.h" +#include +#include #include +#include #include namespace JSC { namespace AOT { namespace Machine { @@ -150,6 +153,7 @@ struct X86Comparison { X86ComparedOperand left; X86ComparedOperand right; bool rightIsNumberOfBit { false }; + const Operation* linkedNumberOnRight { nullptr }; }; static X86ComparedOperand x86ComparedOperandAt(const Encoder& encoder, Location location, int64_t constant) @@ -199,7 +203,7 @@ static ALWAYS_INLINE X86Comparison x86ComparisonOfTest(Encoder& encoder, const T { RELEASE_ASSERT(!test.left.isImmediate()); X86ComparedOperand left = test.loadOnLeft ? x86ComparedOperandReadByFusedLoad(encoder, test.loadOnLeft, test.addressOnLeft) : x86ComparedOperandAt(encoder, test.left, 0); - return X86Comparison { test.kind, test.relation, test.width, left, x86ComparedOperandAt(encoder, test.right, test.rightConstant), test.rightIsNumberOfBit }; + return X86Comparison { test.kind, test.relation, test.width, left, x86ComparedOperandAt(encoder, test.right, test.rightConstant), test.rightIsNumberOfBit, test.linkedNumberOnRight }; } static X86ComparedOperand takeX86ComparedOperand(Encoder& encoder, const Operation* comparison, unsigned childIndex, LeafCursor& cursor) @@ -222,12 +226,16 @@ static X86ComparedOperand takeX86ComparedOperand(Encoder& encoder, const Operati static X86Comparison takeX86Comparison(Encoder& encoder, const Operation* operation, LeafCursor& cursor) { X86ComparedOperand left = takeX86ComparedOperand(encoder, operation, 0, cursor); - X86ComparedOperand right = takeX86ComparedOperand(encoder, operation, 1, cursor); + const Operation* rightChild = operation->child(1); + bool comparesLinkedNumber = rightChild->opcode() == NumberOf && cursor.peek().isImmediate(); + if (comparesLinkedNumber) + cursor.take(); + X86ComparedOperand right = comparesLinkedNumber ? X86ComparedOperand::forConstant(0) : takeX86ComparedOperand(encoder, operation, 1, cursor); RELEASE_ASSERT(!left.mentions(scratchGPR) || !right.mentions(scratchGPR)); RELEASE_ASSERT(!left.mentions(scratchFPR) || !right.mentions(scratchFPR)); Type type = operation->child(0)->type(); - return X86Comparison { type.isFloat() ? Test::CompareNumbers : Test::CompareIntegers, operation->opcode(), widthOf(type), left, right }; + return X86Comparison { type.isFloat() ? Test::CompareNumbers : Test::CompareIntegers, operation->opcode(), widthOf(type), left, right, false, comparesLinkedNumber ? rightChild : nullptr }; } static void bringX86ComparedIntegerIntoScratch(CCallHelpers& jit, Width width, X86ComparedOperand& operand, const X86ComparedOperand& other) @@ -350,8 +358,16 @@ static ALWAYS_INLINE std::optional tryEmitF return condition; } -static MacroAssembler::RelationalCondition emitFlagsOfX86Comparison(CCallHelpers& jit, X86Comparison comparison) +static MacroAssembler::RelationalCondition emitFlagsOfX86Comparison(Encoder& encoder, X86Comparison comparison) { + if (const Operation* number = std::exchange(comparison.linkedNumberOnRight, nullptr)) { + RELEASE_ASSERT(comparison.kind == Test::CompareIntegers && comparison.width == Width32 && comparison.right.isConstant()); + return encoder.m_relocations.withNumberAsImmediate(number->symbol(), [&](int32_t placeholder) { + comparison.right = X86ComparedOperand::forConstant(placeholder); + return emitFlagsOfX86Comparison(encoder, comparison); + }); + } + CCallHelpers& jit = encoder.m_jit; bool testsBits = comparison.kind == Test::TestBits; RELEASE_ASSERT(testsBits || comparison.kind == Test::CompareIntegers); RELEASE_ASSERT(!testsBits || comparison.relation == Equal || comparison.relation == NotEqual); @@ -379,8 +395,15 @@ static MacroAssembler::RelationalCondition emitFlagsOfX86Comparison(CCallHelpers if (left.isConstant()) bringX86ComparedIntegerIntoScratch(jit, width, left, right); - if (right.isConstant() && width == Width64 && !isRepresentableAs(right.constant)) + if (right.isConstant() && width == Width64 && !isRepresentableAs(right.constant)) { + if (!testsBits && left.isInRegister()) { + auto bytes = std::bit_cast>(right.constant); + uint32_t index = encoder.m_proc.addConstantData(std::span(bytes), sizeof(uint64_t), ConstantKind::Number); + encoder.m_relocations.compare64(left.gpr(), CodeSymbol { SymbolSpace::Constant, index }); + return *condition; + } bringX86ComparedIntegerIntoScratch(jit, width, right, left); + } if (left.isInMemory() && right.isConstant()) { if (std::optional conditionInPlace = tryEmitFlagsOfX86WordInMemory(jit, comparison.kind, left, right.constant, width, *condition)) return *conditionInPlace; @@ -510,14 +533,15 @@ static MacroAssembler::JumpList branchOnX86ComparedNumbers(CCallHelpers& jit, X8 return jumps; } -static MacroAssembler::JumpList branchOnX86Comparison(CCallHelpers& jit, const X86Comparison& comparison, bool whenItHolds) +static MacroAssembler::JumpList branchOnX86Comparison(Encoder& encoder, const X86Comparison& comparison, bool whenItHolds) { + CCallHelpers& jit = encoder.m_jit; switch (comparison.kind) { case Test::CompareNumbers: return branchOnX86ComparedNumbers(jit, comparison, whenItHolds); case Test::CompareIntegers: case Test::TestBits: { - MacroAssembler::RelationalCondition condition = emitFlagsOfX86Comparison(jit, comparison); + MacroAssembler::RelationalCondition condition = emitFlagsOfX86Comparison(encoder, comparison); return MacroAssembler::JumpList(jit.branchOnFlags(whenItHolds ? condition : MacroAssembler::invert(condition))); } } @@ -585,9 +609,10 @@ static void moveArmOfX86Select(Encoder& encoder, const X86SelectArm& arm, Reg de case Location::Immediate: { RELEASE_ASSERT(canMoveArmOfX86SelectWithoutClobberingFlags(arm)); int64_t constant = arm.value->asInt(); - if (type == Int32 || isRepresentableAs(constant)) - jit.moveWithoutClobberingFlags(MacroAssembler::TrustedImm32(static_cast(constant)), destination.gpr()); - else + if (type == Int32 || isRepresentableAs(constant)) { + if (!tryMoveSmallConstant(jit, static_cast(constant), destination.gpr())) + jit.moveWithoutClobberingFlags(MacroAssembler::TrustedImm32(static_cast(constant)), destination.gpr()); + } else jit.moveWithoutClobberingFlags(MacroAssembler::TrustedImm64(constant), destination.gpr()); return; } @@ -636,7 +661,7 @@ static void encodeX86SelectByConditionalMove(Encoder& encoder, X86Comparison com return; } - MacroAssembler::RelationalCondition condition = emitFlagsOfX86Comparison(jit, comparison); + MacroAssembler::RelationalCondition condition = emitFlagsOfX86Comparison(encoder, comparison); if (!movesWhenTestHolds) condition = MacroAssembler::invert(condition); GPRReg source = placeArms(); @@ -649,13 +674,13 @@ static void encodeX86SelectByBranching(Encoder& encoder, const X86Comparison& co Location inDestination = Location::reg(destination); if (otherwise.location == inDestination) { - MacroAssembler::JumpList done = branchOnX86Comparison(jit, comparison, false); + MacroAssembler::JumpList done = branchOnX86Comparison(encoder, comparison, false); moveArmOfX86SelectAfterTest(encoder, whenItHolds, destination); done.link(&jit); return; } if (whenItHolds.location == inDestination) { - MacroAssembler::JumpList done = branchOnX86Comparison(jit, comparison, true); + MacroAssembler::JumpList done = branchOnX86Comparison(encoder, comparison, true); moveArmOfX86SelectAfterTest(encoder, otherwise, destination); done.link(&jit); return; @@ -664,13 +689,13 @@ static void encodeX86SelectByBranching(Encoder& encoder, const X86Comparison& co bool testReadsDestination = comparison.left.mentions(destination) || comparison.right.mentions(destination); if (!testReadsDestination && !otherwise.location.isImmediate()) { moveArmOfX86Select(encoder, otherwise, destination); - MacroAssembler::JumpList done = branchOnX86Comparison(jit, comparison, false); + MacroAssembler::JumpList done = branchOnX86Comparison(encoder, comparison, false); moveArmOfX86SelectAfterTest(encoder, whenItHolds, destination); done.link(&jit); return; } - MacroAssembler::JumpList holds = branchOnX86Comparison(jit, comparison, true); + MacroAssembler::JumpList holds = branchOnX86Comparison(encoder, comparison, true); moveArmOfX86SelectAfterTest(encoder, otherwise, destination); MacroAssembler::Jump done = jit.jump(); holds.link(&jit); @@ -681,7 +706,7 @@ static void encodeX86SelectByBranching(Encoder& encoder, const X86Comparison& co MacroAssembler::JumpList Encoder::branchOn(const Test& test, bool whenItHolds) { AllowMacroScratchRegisterUsage allowScratch(m_jit); - return branchOnX86Comparison(m_jit, x86ComparisonOfTest(*this, test), whenItHolds); + return branchOnX86Comparison(*this, x86ComparisonOfTest(*this, test), whenItHolds); } static ALWAYS_INLINE std::optional tryEmitFlagsOfX86Register(CCallHelpers& jit, bool testsBits, Opcode relation, Width width, GPRReg left, Location right, int64_t constant) @@ -743,7 +768,7 @@ static ALWAYS_INLINE std::optional tryEmitF static std::optional tryEmitFlagsOfX86TestOfRegister(CCallHelpers& jit, const Test& test) { - if (test.loadOnLeft || test.rightIsNumberOfBit || !test.left.isRegister()) + if (test.loadOnLeft || test.rightIsNumberOfBit || test.linkedNumberOnRight || !test.left.isRegister()) return std::nullopt; RELEASE_ASSERT(test.kind == Test::TestBits || test.kind == Test::CompareIntegers); return tryEmitFlagsOfX86Register(jit, test.kind == Test::TestBits, test.relation, test.width, test.left.gpr(), test.right, test.rightConstant); @@ -797,7 +822,7 @@ static ALWAYS_INLINE MacroAssembler::RelationalCondition emitFlagsOfX86WordReadB return *condition; AllowMacroScratchRegisterUsage allowScratch(encoder.m_jit); X86ComparedOperand word = x86ComparedOperandReadByFusedLoad(encoder, load, address); - return emitFlagsOfX86Comparison(encoder.m_jit, X86Comparison { testsBits ? Test::TestBits : Test::CompareIntegers, relation, widthOf(load->type()), word, x86ComparedOperandAt(encoder, right, constant), false }); + return emitFlagsOfX86Comparison(encoder, X86Comparison { testsBits ? Test::TestBits : Test::CompareIntegers, relation, widthOf(load->type()), word, x86ComparedOperandAt(encoder, right, constant), false }); } std::optional Encoder::tryEmitFlagsOfPredicate(const Step& step, const Operation* predicate) @@ -839,6 +864,8 @@ std::optional Encoder::tryEmitFlagsOfPredic const Operation* left = inlineChildWithoutCheck(fused, 0); const Operation* right = inlineChildWithoutCheck(fused, 1); + if (left->opcode() == NumberOf || right->opcode() == NumberOf) + return std::nullopt; if (isFused(right)) { if (isFused(left) || !isLoad(right->opcode())) return std::nullopt; @@ -888,9 +915,9 @@ MacroAssembler::Jump Encoder::jumpOnIntegers(const Test& test, bool whenItHolds) ASSERT(test.kind != Test::CompareNumbers); AllowMacroScratchRegisterUsage allowScratch(m_jit); std::optional conditionOfRegister = tryEmitFlagsOfX86TestOfRegister(m_jit, test); - if (!conditionOfRegister && test.loadOnLeft && test.addressOnLeft.kind == AddressForm::InstanceSymbol && !test.rightIsNumberOfBit) + if (!conditionOfRegister && test.loadOnLeft && test.addressOnLeft.kind == AddressForm::InstanceSymbol && !test.rightIsNumberOfBit && !test.linkedNumberOnRight) conditionOfRegister = tryEmitFlagsOfX86InstanceWord(*this, test.kind == Test::TestBits, test.relation, test.width, test.loadOnLeft, test.addressOnLeft.symbol, test.right, test.rightConstant); - MacroAssembler::RelationalCondition condition = conditionOfRegister ? *conditionOfRegister : emitFlagsOfX86Comparison(m_jit, x86ComparisonOfTest(*this, test)); + MacroAssembler::RelationalCondition condition = conditionOfRegister ? *conditionOfRegister : emitFlagsOfX86Comparison(*this, x86ComparisonOfTest(*this, test)); return m_jit.branchOnFlags(whenItHolds ? condition : MacroAssembler::invert(condition)); } @@ -918,7 +945,7 @@ void Encoder::encodeComparison(const Step& step) bool clearsDestinationFirst = !comparison.left.mentions(destination) && !comparison.right.mentions(destination); if (clearsDestinationFirst) m_jit.move(MacroAssembler::TrustedImm32(0), destination); - MacroAssembler::RelationalCondition condition = emitFlagsOfX86Comparison(m_jit, comparison); + MacroAssembler::RelationalCondition condition = emitFlagsOfX86Comparison(*this, comparison); m_jit.m_assembler.setCC_r(static_cast(condition), destination); if (!clearsDestinationFirst) m_jit.zeroExtend8To32(destination, destination); diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineEncoderControl.cpp b/Source/JavaScriptCore/aot/machine/AOTMachineEncoderControl.cpp index 9bcf01b8360e5..edf13421e40e1 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineEncoderControl.cpp +++ b/Source/JavaScriptCore/aot/machine/AOTMachineEncoderControl.cpp @@ -21,8 +21,11 @@ static ALWAYS_INLINE Test testOfEncodedOperands(Encoder& encoder, Test::Kind kin { const Operation* left = childWithoutCheck(fused, 0); const Operation* right = childWithoutCheck(fused, 1); - bool leftIsFused = encoder.m_liveness.isFused(left) && left->opcode() != Trunc; - bool rightIsFused = encoder.m_liveness.isFused(right) && right->opcode() != Trunc; + auto isFusedOperand = [&](const Operation* operand) ALWAYS_INLINE_LAMBDA { + return encoder.m_liveness.isFused(operand) && operand->opcode() != Trunc && !(isARM64() && namesWord(operand)); + }; + bool leftIsFused = isFusedOperand(left); + bool rightIsFused = isFusedOperand(right); if (leftIsFused && left->opcode() == Patch) { RELEASE_ASSERT(isX86_64() && kind == Test::TestBits && right->isInt(1)); @@ -35,6 +38,20 @@ static ALWAYS_INLINE Test testOfEncodedOperands(Encoder& encoder, Test::Kind kin return test; } + if (isARM64() && (leftIsFused || rightIsFused)) { + const Operation* extension = rightIsFused ? right : left; + RELEASE_ASSERT(kind == Test::CompareIntegers && leftIsFused != rightIsFused && extension->type() == Int64 && (extension->opcode() == SExt32 || extension->opcode() == ZExt32)); + Test test { kind, rightIsFused ? relation : flippedComparison(relation), Width64, Location(), Location(), 0 }; + if (rightIsFused) + test.left = cursor.take(); + test.right = cursor.take(); + if (leftIsFused) + test.left = cursor.take(); + RELEASE_ASSERT(test.left.isRegister() && test.right.isRegister()); + test.extensionOfRight = extension->opcode() == SExt32 ? IndexExtension::SignExtend32 : IndexExtension::ZeroExtend32; + return test; + } + if (leftIsFused || rightIsFused) { RELEASE_ASSERT(isX86_64() && kind != Test::CompareNumbers && leftIsFused != rightIsFused); Test test { kind, relation, widthOf(left->type()), Location(), Location(), 0 }; @@ -53,8 +70,12 @@ static ALWAYS_INLINE Test testOfEncodedOperands(Encoder& encoder, Test::Kind kin } RELEASE_ASSERT(test.right); if (test.right.isImmediate()) { - RELEASE_ASSERT(other->hasInt()); - test.rightConstant = other->asInt(); + if (other->opcode() == NumberOf) + test.linkedNumberOnRight = other; + else { + RELEASE_ASSERT(other->hasInt()); + test.rightConstant = other->asInt(); + } } return test; } @@ -63,12 +84,16 @@ static ALWAYS_INLINE Test testOfEncodedOperands(Encoder& encoder, Test::Kind kin Location rightLocation = cursor.take(); RELEASE_ASSERT(leftLocation && rightLocation && !leftLocation.isImmediate()); - int64_t rightConstant = 0; + Test test { kind, relation, widthOf(left->type()), leftLocation, rightLocation, 0 }; if (rightLocation.isImmediate()) { - RELEASE_ASSERT(right->isConstant()); - rightConstant = right->hasInt() ? right->asInt() : right->constantBits(); + if (right->opcode() == NumberOf) + test.linkedNumberOnRight = right; + else { + RELEASE_ASSERT(right->isConstant()); + test.rightConstant = right->hasInt() ? right->asInt() : right->constantBits(); + } } - return Test { kind, relation, widthOf(left->type()), leftLocation, rightLocation, rightConstant }; + return test; } static Test nonZeroTestOfEncodedValue(Encoder& encoder, const Operation* value, Location location) @@ -160,8 +185,7 @@ void Encoder::encodeRareTerminator(Block* block, const Step& step) } RELEASE_ASSERT(opcode == Oops); - AllowMacroScratchRegisterUsage allowScratch(m_jit); - m_jit.oops(); + emitUnreachable(m_jit); } static ALWAYS_INLINE void addPendingJumpsOnTest(Encoder& encoder, const Test& test, bool whenItHolds, const Block* target) diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineEncoderIntegersARM64.cpp b/Source/JavaScriptCore/aot/machine/AOTMachineEncoderIntegersARM64.cpp index 9f8e871a602d5..12f7ed7265994 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineEncoderIntegersARM64.cpp +++ b/Source/JavaScriptCore/aot/machine/AOTMachineEncoderIntegersARM64.cpp @@ -96,6 +96,154 @@ static void encodeARM64ScaledSum(Encoder& encoder, const Step& step) RELEASE_ASSERT_NOT_REACHED(); } +static const Operation* fusedOperandOf(const Encoder& encoder, const Operation* operation) +{ + for (const Operation* child : operation->children()) { + if (encoder.isFused(child) && child->opcode() != Trunc && !namesWord(child)) + return child; + } + return nullptr; +} + +static GPRReg takeIntegerRegister(LeafCursor& leaves) +{ + Location location = leaves.take(); + RELEASE_ASSERT(location.isRegister() && location.reg().isGPR()); + return location.gpr(); +} + +template +static void emitWithShiftedOperandARM64(ARM64Assembler& assembler, Opcode opcode, GPRReg destination, GPRReg left, GPRReg right, ARM64Assembler::ShiftType shift, unsigned amount) +{ + switch (opcode) { + case Add: + assembler.add(destination, left, right, shift, amount); + return; + case Sub: + assembler.sub(destination, left, right, shift, amount); + return; + case BitAnd: + assembler.and_(destination, left, right, shift, amount); + return; + case BitOr: + assembler.orr(destination, left, right, shift, amount); + return; + case BitXor: + assembler.eor(destination, left, right, shift, amount); + return; + default: + break; + } + RELEASE_ASSERT_NOT_REACHED(); +} + +static void encodeWithShiftedOperandARM64(Encoder& encoder, const Step& step, const Operation* shift) +{ + const Operation* operation = step.operation; + ASSERT(acceptsShiftedOperand(operation->opcode())); + bool shiftIsFirst = operation->child(0) == shift; + RELEASE_ASSERT(isShiftByConstant(shift) && (!shiftIsFirst || isCommutative(operation->opcode()))); + + LeafCursor leaves(step); + GPRReg other = shiftIsFirst ? InvalidGPRReg : takeIntegerRegister(leaves); + GPRReg shifted = takeIntegerRegister(leaves); + leaves.take(); + if (shiftIsFirst) + other = takeIntegerRegister(leaves); + + ARM64Assembler::ShiftType type = shift->opcode() == Shl ? ARM64Assembler::LSL : shift->opcode() == ZShr ? ARM64Assembler::LSR : ARM64Assembler::ASR; + if (operation->type() == Int32) + emitWithShiftedOperandARM64<32>(encoder.m_jit.m_assembler, operation->opcode(), step.destination.gpr(), other, shifted, type, amountOfShift(shift)); + else + emitWithShiftedOperandARM64<64>(encoder.m_jit.m_assembler, operation->opcode(), step.destination.gpr(), other, shifted, type, amountOfShift(shift)); +} + +static void encodeBitFieldARM64(Encoder& encoder, const Step& step, BitField field) +{ + LeafCursor leaves(step); + GPRReg source = takeIntegerRegister(leaves); + GPRReg destination = step.destination.gpr(); + ARM64Assembler& assembler = encoder.m_jit.m_assembler; + if (step.operation->type() == Int32) { + if (field.extendsSign) + assembler.sbfx<32>(destination, source, field.lowestBit, field.numberOfBits); + else + assembler.ubfx<32>(destination, source, field.lowestBit, field.numberOfBits); + return; + } + if (field.extendsSign) + assembler.sbfx<64>(destination, source, field.lowestBit, field.numberOfBits); + else + assembler.ubfx<64>(destination, source, field.lowestBit, field.numberOfBits); +} + +static void encodeWithExtendedOperandARM64(Encoder& encoder, const Step& step, const Operation* extension) +{ + const Operation* sum = step.operation; + RELEASE_ASSERT(sum->type() == Int64 && (sum->opcode() == Add || (sum->opcode() == Sub && sum->child(1) == extension))); + bool extensionIsFirst = sum->child(0) == extension; + + LeafCursor leaves(step); + GPRReg other = extensionIsFirst ? InvalidGPRReg : takeIntegerRegister(leaves); + GPRReg word = takeIntegerRegister(leaves); + if (extensionIsFirst) + other = takeIntegerRegister(leaves); + + GPRReg destination = step.destination.gpr(); + ARM64Assembler& assembler = encoder.m_jit.m_assembler; + bool subtracts = sum->opcode() == Sub; + ARM64Assembler::ExtendType type = extension->opcode() == ZExt32 ? ARM64Assembler::UXTW : ARM64Assembler::SXTW; + if (subtracts) + assembler.sub<64>(destination, other, word, type, 0); + else + assembler.add<64>(destination, other, word, type, 0); +} + +static void encodeSumWithProductARM64(Encoder& encoder, const Step& step, const Operation* product) +{ + const Operation* sum = step.operation; + RELEASE_ASSERT(sum->opcode() == Add || (sum->opcode() == Sub && sum->child(1) == product)); + bool productIsFirst = sum->child(0) == product; + + LeafCursor leaves(step); + GPRReg addend = productIsFirst ? InvalidGPRReg : takeIntegerRegister(leaves); + GPRReg left = takeIntegerRegister(leaves); + GPRReg right = takeIntegerRegister(leaves); + if (productIsFirst) + addend = takeIntegerRegister(leaves); + + GPRReg destination = step.destination.gpr(); + ARM64Assembler& assembler = encoder.m_jit.m_assembler; + bool subtracts = sum->opcode() == Sub; + if (sum->type() == Int32) { + if (subtracts) + assembler.msub<32>(destination, left, right, addend); + else + assembler.madd<32>(destination, left, right, addend); + return; + } + if (subtracts) + assembler.msub<64>(destination, left, right, addend); + else + assembler.madd<64>(destination, left, right, addend); +} + +static bool tryEncodeFusedOperandARM64(Encoder& encoder, const Step& step) +{ + const Operation* fused = fusedOperandOf(encoder, step.operation); + if (!fused) + return false; + if (fused->opcode() == Mul) + encodeSumWithProductARM64(encoder, step, fused); + else if (fused->opcode() == SExt32 || fused->opcode() == ZExt32) + encodeWithExtendedOperandARM64(encoder, step, fused); + else if (std::optional field = bitFieldOf(step.operation)) + encodeBitFieldARM64(encoder, step, *field); + else + encodeWithShiftedOperandARM64(encoder, step, fused); + return true; +} + bool Encoder::tryEncodeSumAndConstant(const Step&, const Step&) { return false; @@ -148,7 +296,7 @@ void Encoder::encodeInteger(const Step& step) ASSERT(operation->type().isInt()); #if ASSERT_ENABLED for (const Operation* child : operation->children()) - ASSERT(!m_liveness.isFused(child) || child->opcode() == Trunc || addsScaledIndex(m_liveness, operation)); + ASSERT(!m_liveness.isFused(child) || child->opcode() == Trunc || child->opcode() == Mul || child->opcode() == SExt32 || child->opcode() == ZExt32 || isShiftByConstant(child)); #endif RELEASE_ASSERT(step.destination.isRegister()); @@ -178,6 +326,8 @@ void Encoder::encodeInteger(const Step& step) encodeARM64ScaledSum(*this, step); return; } + if (tryEncodeFusedOperandARM64(*this, step)) + return; IntegerSources sources = integerSourcesOrConstant(*this, step); if (sources.constant && is32) m_jit.add32(integerConstant32Of(sources.constant), sources.left, destination); @@ -195,6 +345,8 @@ void Encoder::encodeInteger(const Step& step) encodeSumThatSetsFlags(step); return; } + if (tryEncodeFusedOperandARM64(*this, step)) + return; IntegerSources sources = integerSourcesOrConstant(*this, step); if (sources.constant && is32) m_jit.sub32(sources.left, integerConstant32Of(sources.constant), destination); @@ -262,6 +414,8 @@ void Encoder::encodeInteger(const Step& step) } case BitAnd: { + if (tryEncodeFusedOperandARM64(*this, step)) + return; IntegerSources sources = integerSourcesOrConstant(*this, step); if (sources.constant && is32) m_jit.and32(integerConstant32Of(sources.constant), sources.left, destination); @@ -275,6 +429,8 @@ void Encoder::encodeInteger(const Step& step) } case BitOr: { + if (tryEncodeFusedOperandARM64(*this, step)) + return; IntegerSources sources = integerSourcesOrConstant(*this, step); if (sources.constant && is32) m_jit.or32(integerConstant32Of(sources.constant), sources.left, destination); @@ -288,6 +444,8 @@ void Encoder::encodeInteger(const Step& step) } case BitXor: { + if (tryEncodeFusedOperandARM64(*this, step)) + return; IntegerSources sources = integerSourcesOrConstant(*this, step); if (sources.constant && is32) m_jit.xor32(integerConstant32Of(sources.constant), sources.left, destination); @@ -314,6 +472,8 @@ void Encoder::encodeInteger(const Step& step) } case SShr: { + if (tryEncodeFusedOperandARM64(*this, step)) + return; IntegerSources sources = integerSourcesOrConstant(*this, step); if (sources.constant && is32) m_jit.rshift32(sources.left, integerConstant32Of(sources.constant), destination); @@ -327,6 +487,8 @@ void Encoder::encodeInteger(const Step& step) } case ZShr: { + if (tryEncodeFusedOperandARM64(*this, step)) + return; IntegerSources sources = integerSourcesOrConstant(*this, step); if (sources.constant && is32) m_jit.urshift32(sources.left, integerConstant32Of(sources.constant), destination); diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineEncoderMemoryX86_64.cpp b/Source/JavaScriptCore/aot/machine/AOTMachineEncoderMemoryX86_64.cpp index e9dda99fa0a84..aa6663dc7789b 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineEncoderMemoryX86_64.cpp +++ b/Source/JavaScriptCore/aot/machine/AOTMachineEncoderMemoryX86_64.cpp @@ -42,6 +42,11 @@ static ALWAYS_INLINE void withOperandOfLoadOrStore(const AddressForm& form, cons RELEASE_ASSERT_NOT_REACHED(); } +static bool onlyFaultsOnEmptyPointer(const Encoder& encoder, const Operation* load, const AddressForm& form) +{ + return load->opcode() == Load8Z && load->traps() && encoder.factsOf(load).lastUse == noPosition && form.kind == AddressForm::BaseOffset && form.offset >= 0 && form.offset < static_cast(sizeof(uint64_t)); +} + static ALWAYS_INLINE void emitX86Load(Encoder& encoder, const Operation* load, const AddressForm& form, Reg destination) { CCallHelpers& jit = encoder.m_jit; @@ -50,6 +55,10 @@ static ALWAYS_INLINE void emitX86Load(Encoder& encoder, const Operation* load, c bool isNumber = type.isFloat(); Width width = accessWidthOf(load->opcode(), type); ASSERT(destination.isFPR() == isNumber); + if (onlyFaultsOnEmptyPointer(encoder, load, form)) { + jit.load32(MacroAssembler::Address(form.base), destination.gpr()); + return; + } withOperandOfLoadOrStore(form, [&](auto address) ALWAYS_INLINE_LAMBDA { switch (width) { case Width8: diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineEncoderMoves.cpp b/Source/JavaScriptCore/aot/machine/AOTMachineEncoderMoves.cpp index e9f22ebc6bcf2..7674ff7459408 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineEncoderMoves.cpp +++ b/Source/JavaScriptCore/aot/machine/AOTMachineEncoderMoves.cpp @@ -86,6 +86,9 @@ static bool tryMaterializeBitsBesidePinnedRegister(CCallHelpers& jit, uint64_t b static void materializeBitsInGPR(CCallHelpers& jit, uint64_t bits, Width width, GPRReg destination) { + bool hasOnlyLowHalf = !encodedMoveIsWide(width) || isRepresentableAs(bits); + if (bits && hasOnlyLowHalf && tryMoveSmallConstant(jit, static_cast(static_cast(bits)), destination)) + return; if (!encodedMoveIsWide(width)) { jit.move(CCallHelpers::TrustedImm32(static_cast(static_cast(bits))), destination); return; @@ -105,6 +108,18 @@ static void materializeBitsInGPR(CCallHelpers& jit, uint64_t bits, Width width, jit.move(CCallHelpers::TrustedImm64(static_cast(bits)), destination); } +#if CPU(X86_64) +static bool isLoadedIntoGPR(uint64_t bits) +{ + if (isRepresentableAs(bits) || isRepresentableAs(static_cast(bits))) + return false; + constexpr unsigned shiftOfTag = 48; + constexpr uint64_t lowBytesOfLongDouble = (1ull << 40) - 1; + bool isBoxedInteger = bits >> shiftOfTag == static_cast(JSValue::NumberTag) >> shiftOfTag; + return (isBoxedInteger || !(bits >> shiftOfTag) || !(bits & lowBytesOfLongDouble)) && !findPinnedConstant(bits); +} +#endif + static bool bitsAreCheapestFromImmediates(uint64_t bits) { #if CPU(ARM64) @@ -117,9 +132,9 @@ static bool bitsAreCheapestFromImmediates(uint64_t bits) #endif } -static void materializeBitsInFPR(Encoder& encoder, uint64_t bits, Width width, FPRReg destination) +void Encoder::materializeBitsInFPR(uint64_t bits, Width width, FPRReg destination) { - CCallHelpers& jit = encoder.m_jit; + CCallHelpers& jit = m_jit; if (!encodedMoveIsWide(width)) { jit.move32ToFloat(CCallHelpers::TrustedImm32(static_cast(static_cast(bits))), destination); return; @@ -142,10 +157,14 @@ static void materializeBitsInFPR(Encoder& encoder, uint64_t bits, Width width, F } std::array bytes = std::bit_cast>(bits); - uint32_t index = encoder.m_proc.addConstantData(std::span(bytes), sizeof(uint64_t), ConstantKind::Number); + uint32_t index = m_proc.addConstantData(std::span(bytes), sizeof(uint64_t), ConstantKind::Number); +#if CPU(X86_64) + m_relocations.loadDouble(CodeSymbol { SymbolSpace::Constant, index }, destination); +#else GPRReg address = jit.scratchRegister(); - encoder.m_relocations.moveAddress(CodeSymbol { SymbolSpace::Constant, index }, address, PCRelativeReach::Far); + m_relocations.moveAddress(CodeSymbol { SymbolSpace::Constant, index }, address, PCRelativeReach::Far); jit.loadDouble(MacroAssembler::Address(address), destination); +#endif } static ALWAYS_INLINE void materializeInGPR(Encoder& encoder, const Operation* operation, GPRReg destination) @@ -232,10 +251,18 @@ void Encoder::encodeMaterialize(const Operation* operation, Reg destination) uint64_t bits = static_cast(operation->constantBits()); if (width == Width32) bits = static_cast(bits); +#if CPU(X86_64) + if (destination.isGPR() && encodedMoveIsWide(width) && isLoadedIntoGPR(bits)) { + auto bytes = std::bit_cast>(bits); + uint32_t index = m_proc.addConstantData(std::span(bytes), sizeof(uint64_t), ConstantKind::Number); + m_relocations.load64(CodeSymbol { SymbolSpace::Constant, index }, destination.gpr()); + return; + } +#endif if (destination.isGPR()) materializeBitsInGPR(m_jit, bits, width, destination.gpr()); else - materializeBitsInFPR(*this, bits, width, destination.fpr()); + materializeBitsInFPR(bits, width, destination.fpr()); return; } diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineEncoderNumbersX86_64.cpp b/Source/JavaScriptCore/aot/machine/AOTMachineEncoderNumbersX86_64.cpp index c7bb64cf6b49c..d246513adea56 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineEncoderNumbersX86_64.cpp +++ b/Source/JavaScriptCore/aot/machine/AOTMachineEncoderNumbersX86_64.cpp @@ -9,6 +9,7 @@ #if ENABLE(AOT) && CPU(X86_64) #include "AllowMacroScratchRegisterUsage.h" +#include #include #include @@ -152,6 +153,70 @@ bool Encoder::blockHasWrittenNumberRegisterInFrontOf(const Step& step, FPRReg re return m_numberRegistersWrittenInBlock & maskOf(Reg(reg)); } +bool Encoder::writesNumberRegister(const Step& step, FPRReg reg) const +{ + if (step.destination.isRegister() && step.destination.reg() == Reg(reg)) + return true; + if (step.kind == StepKind::Swap && step.sources[0].isRegister() && step.sources[0].reg() == Reg(reg)) + return true; + if (step.patchAssignment == UINT32_MAX) + return false; + const PatchAssignment& assignment = m_allocation.patchAssignment(step.patchAssignment); + return (assignment.fpScratch & maskOf(Reg(reg))) || std::ranges::any_of(assignment.locations, [&](const Location& location) { + return location.isRegister() && location.reg() == Reg(reg); + }); +} + +bool Encoder::numberRegisterMayBeWrittenShortlyInFrontOf(const Step& step, FPRReg reg) const +{ + static constexpr unsigned stepsOfLastWriters = 16; + static constexpr unsigned mostBlocks = 8; + struct EndOfBlock { + const Block* block; + unsigned stepsLeft; + }; + Vector ends; + auto isWrittenBetween = [&](const Step* begin, const Step* end, unsigned& stepsLeft) { + WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN + for (; end > begin && stepsLeft; --stepsLeft) { + if (writesNumberRegister(*--end, reg)) + return true; + } + WTF_ALLOW_UNSAFE_BUFFER_USAGE_END + return false; + }; + auto appendBlocksInFrontOf = [&](const Block* block, unsigned stepsLeft) { + for (const Block* predecessor : placeOf(block).predecessors) { + if (ends.size() == mostBlocks) + return false; + ends.append(EndOfBlock { predecessor, stepsLeft }); + } + return true; + }; + + WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN + const BlockSteps& steps = stepsOf(m_block); + if (&step < m_steps + steps.begin || &step >= m_steps + steps.end) + return true; + unsigned stepsLeft = stepsOfLastWriters; + if (isWrittenBetween(m_steps + steps.begin, &step, stepsLeft) || (stepsLeft && !appendBlocksInFrontOf(m_block, stepsLeft))) + return true; + for (unsigned index = 0; index < ends.size(); ++index) { + auto [block, stepsLeftThere] = ends[index]; + const BlockSteps& stepsThere = stepsOf(block); + if (stepsThere.terminator != UINT32_MAX && writesNumberRegister(m_steps[stepsThere.terminator], reg)) + return true; + if (isWrittenBetween(m_steps + stepsThere.beginOfEdgeMoves, m_steps + stepsThere.endOfEdgeMoves, stepsLeftThere)) + return true; + if (isWrittenBetween(m_steps + stepsThere.begin, m_steps + stepsThere.end, stepsLeftThere)) + return true; + if (stepsLeftThere && !appendBlocksInFrontOf(block, stepsLeftThere)) + return true; + } + WTF_ALLOW_UNSAFE_BUFFER_USAGE_END + return false; +} + template static void encodeX86NumberUnary(Encoder& encoder, const Step* stepThatWaitsLongForDestination, const X86NumberSource& source, Width widthOfSource, FPRReg destination, const Functor& emit) { @@ -267,8 +332,8 @@ static void encodeX86NumberNegOrAbs(Encoder& encoder, Opcode opcode, Width width auto moveMaskTo = [&](FPRReg mask) { if (isDouble) { - int64_t signBit = std::bit_cast(-0.0); - jit.move64ToDouble(CCallHelpers::TrustedImm64(negates ? signBit : ~signBit), mask); + uint64_t signBit = std::bit_cast(-0.0); + encoder.materializeBitsInFPR(negates ? signBit : ~signBit, Width64, mask); } else { int32_t signBit = std::bit_cast(-0.0f); jit.move32ToFloat(CCallHelpers::TrustedImm32(negates ? signBit : ~signBit), mask); @@ -451,7 +516,29 @@ static void encodeX86NumberMinOrMax(Encoder& encoder, Opcode opcode, Width width doneWithUnordered.link(&jit); } -static void encodeX86NumberFromInteger(Encoder& encoder, Width widthOfInteger, const X86NumberSource& source, FPRReg destination) +static bool mayBeFirstOfX86MinOrMax(Opcode opcode, const Operation* number) +{ + if (number->opcode() == IToD) + return opcode == FMin; + if (!number->hasDouble() || std::isnan(number->asDouble())) + return false; + return number->asDouble() || std::signbit(number->asDouble()) == (opcode == FMax); +} + +static void encodeX86MinOrMaxOfDoubleKnownToComeFirst(Encoder& encoder, Opcode opcode, const X86NumberSource& first, const X86NumberSource& second, FPRReg destination) +{ + CCallHelpers& jit = encoder.m_jit; + FPRReg registerOfResult = first.isRegister() || !second.isRegister() ? destination : scratchFPR; + FPRReg firstFPR = x86NumberInRegister(encoder, first, Width64, registerOfResult); + FPRReg secondFPR = x86NumberInRegister(encoder, second, Width64, scratchFPR); + if (opcode == FMin) + jit.minOrSecondDouble(firstFPR, secondFPR, registerOfResult); + else + jit.maxOrSecondDouble(firstFPR, secondFPR, registerOfResult); + jit.moveDouble(registerOfResult, destination); +} + +static void encodeX86NumberFromInteger(Encoder& encoder, const Step& step, Width widthOfInteger, const X86NumberSource& source, FPRReg destination) { CCallHelpers& jit = encoder.m_jit; auto convert = [&](auto operand) { @@ -461,7 +548,8 @@ static void encodeX86NumberFromInteger(Encoder& encoder, Width widthOfInteger, c jit.convertInt32ToDouble(operand, destination); }; - jit.moveZeroToDouble(destination); + if (encoder.numberRegisterMayBeWrittenShortlyInFrontOf(step, destination)) + jit.moveZeroToDouble(destination); if (source.isRegister()) { convert(source.reg.gpr()); return; @@ -530,6 +618,12 @@ void Encoder::encodeNumber(const Step& step) case FMax: { X86NumberSource left = takeSource(0); X86NumberSource right = takeSource(1); + for (unsigned known = 0; width == Width64 && known < 2; ++known) { + if (!mayBeFirstOfX86MinOrMax(opcode, operation->child(known))) + continue; + encodeX86MinOrMaxOfDoubleKnownToComeFirst(*this, opcode, known ? right : left, known ? left : right, step.destination.fpr()); + return; + } encodeX86NumberMinOrMax(*this, opcode, width, left, right, step.destination.fpr()); return; } @@ -569,7 +663,7 @@ void Encoder::encodeNumber(const Step& step) } case IToD: ASSERT(operation->type() == Double); - encodeX86NumberFromInteger(*this, widthOf(operation->child(0)->type()), takeSource(0), step.destination.fpr()); + encodeX86NumberFromInteger(*this, step, widthOf(operation->child(0)->type()), takeSource(0), step.destination.fpr()); return; case BitwiseCast: ASSERT(width == widthOf(operation->child(0)->type())); diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineFusion.cpp b/Source/JavaScriptCore/aot/machine/AOTMachineFusion.cpp index 9536ffa8a6dea..6441141467e19 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineFusion.cpp +++ b/Source/JavaScriptCore/aot/machine/AOTMachineFusion.cpp @@ -131,6 +131,14 @@ static IndexExtension fusionExtensionOf(const Operation* extensionOrNull) return extensionOrNull->opcode() == ZExt32 ? IndexExtension::ZeroExtend32 : IndexExtension::SignExtend32; } +static ALWAYS_INLINE bool fusionCanAbsorbExtension(const ValueFacts* values, const Block* block, const Operation* extension) +{ + ASSERT(extension->opcode() == ZExt32 || extension->opcode() == SExt32); + if (!isARM64() || extension->owner() == block || extension->isNarrowReference() || fusionChildrenOf(extension)[0]->isNarrowReference()) + return fusionCanAbsorb(values, block, extension); + return extension->useCount() == 1 && values[extension->index()].lastUse == noPosition; +} + static unsigned fuseAddressBelow(ValueFacts* values, unsigned numberOfLeaves, const Operation* access, Operation* sum) { ASSERT(sum == access->lastChild()); @@ -157,7 +165,7 @@ static unsigned fuseAddressBelow(ValueFacts* values, unsigned numberOfLeaves, co } Operation* extension = nullptr; const Operation* extended = nullptr; - if ((shifted->opcode() == ZExt32 || shifted->opcode() == SExt32) && fusionCanAbsorb(values, block, shifted)) { + if ((shifted->opcode() == ZExt32 || shifted->opcode() == SExt32) && fusionCanAbsorbExtension(values, block, shifted)) { extension = shifted; extended = fusionChildrenOf(shifted)[0]; } @@ -355,6 +363,70 @@ static NEVER_INLINE bool fuseLoadTestedByBranch(ValueFacts* values, const Operat return fuseLoadThatTestCanRead(values, load, 1); } +static ALWAYS_INLINE bool fusionCanAbsorbInteger(const ValueFacts* values, const Block* block, const Operation* candidate) +{ + return fusionCanAbsorb(values, block, candidate) && !candidate->isNarrowReference() && !candidate->has(OperationFlag::IsInnerPointer); +} + +static void fuseOperandAndTruncation(ValueFacts* values, const Operation* operand) +{ + fusionMark(values, operand, ValueFlag::IsFused); + const Operation* word = fusionChildrenOf(operand)[0]; + if (word->opcode() == Trunc && !fusionChildrenOf(word)[0]->hasNoPosition() && fusionCanAbsorbInteger(values, operand->owner(), word)) + fusionMark(values, word, ValueFlag::IsFused); +} + +static bool fusionOperandIsInRegister(const Operation* user, unsigned index, const Operation* operand) +{ + return !operand->hasNoPosition() || (operand->isConstant() && !acceptsImmediate(*user, index, *operand)); +} + +static ALWAYS_INLINE bool fusionReadsWordsByName(Opcode opcode) +{ + return isARM64() && (opcode == Add || opcode == Sub || opcode == Mul || opcode == BitAnd || opcode == BitOr || opcode == BitXor || opcode == Shl || opcode == SShr || opcode == ZShr || isComparison(opcode)); +} + +static bool fuseNamesOfWordsBelow(ValueFacts* values, const Operation* user) +{ + if (!fusionReadsWordsByName(user->opcode())) + return false; + bool hasFused = false; + Operation* const* operands = fusionChildrenOf(user); + for (unsigned index : { 0u, 1u }) { + const Operation* name = operands[index]; + if (!namesWord(name) || name->isNarrowReference() || user->has(OperationFlag::IsInnerPointer)) + continue; + const Operation* word = fusionChildrenOf(name)[0]; + if (word->hasNoPosition() || word->isNarrowReference() || values[name->index()].has(ValueFlag::IsFused) || !fusionCanAbsorbExtension(values, user->owner(), name)) + continue; + fusionMark(values, name, ValueFlag::IsFused); + hasFused = true; + } + return hasFused; +} + +static bool fuseExtendedOperandBelow(ValueFacts* values, const Operation* user) +{ + bool hasFused = fuseNamesOfWordsBelow(values, user); + Opcode opcode = user->opcode(); + if (!acceptsExtendedOperand(opcode)) + return hasFused; + Operation* const* operands = fusionChildrenOf(user); + if (operands[0]->type() != Int64) + return hasFused; + for (unsigned index : { 1u, 0u }) { + const Operation* extension = operands[index]; + if ((extension->opcode() != SExt32 && extension->opcode() != ZExt32) || namesWord(extension) || (!index && (opcode == Sub || user->has(OperationFlag::IsInnerPointer)))) + continue; + const Operation* word = fusionChildrenOf(extension)[0]; + if (word->hasNoPosition() || word->isNarrowReference() || !fusionOperandIsInRegister(user, 1 - index, operands[1 - index]) || !fusionCanAbsorbInteger(values, user->owner(), extension)) + continue; + fuseOperandAndTruncation(values, extension); + return true; + } + return hasFused; +} + static ALWAYS_INLINE bool fuseShiftBelowBitTest(ValueFacts* values, const Block* block, Operation* const* operands, unsigned numberOfLeaves) { const Operation* shifted = operands[0]; @@ -479,6 +551,8 @@ static ALWAYS_INLINE bool fusePredicateBelow(const BackwardScan& scan, ValueFact } if (opcodeOfUser != Select && ((fusionTraitsOf(operands[0]) | fusionTraitsOf(operands[1])) & FusionTrait::IsTruncation)) fuseTruncationsBelowTest(values, block, operands); + if (isARM64() && fusionIsComparison(opcodeOfTested)) + fuseExtendedOperandBelow(values, tested); if (!mayFuseLoad || (!fusionIsLoad(operands[0]->opcode()) && !fusionIsLoad(operands[1]->opcode()))) continue; if (std::optional numberOfLeavesAdded = fuseLoadBelowTest(values, user, tested, numberOfLeaves)) { @@ -566,6 +640,8 @@ static ALWAYS_INLINE bool fuseOneTestInto(ValueFacts* values, Operation* user, c ASSERT(fusionLeavesInStepOf(scan, user) == 2); if (traits & FusionTrait::IsTruncation) return fuseTruncationsBelowTest(values, user->owner(), operands); + if constexpr (isARM64() && !predicateIsBitAnd) + fuseExtendedOperandBelow(values, predicate); return true; } @@ -677,10 +753,77 @@ static bool fuseLoadBelowComputation(ValueFacts* values, const Operation* user, return true; } +static bool fuseShiftBelowBitField(ValueFacts* values, const Operation* user) +{ + if (!bitFieldOf(user)) + return false; + const Operation* shift = fusionChildrenOf(user)[0]; + if (fusionChildrenOf(shift)[0]->hasNoPosition() || !fusionCanAbsorbInteger(values, user->owner(), shift)) + return false; + fuseOperandAndTruncation(values, shift); + return true; +} + +static bool fuseShiftedOperandBelow(ValueFacts* values, const Operation* user) +{ + Opcode opcode = user->opcode(); + if (!acceptsShiftedOperand(opcode) || !user->type().isInt()) + return false; + Operation* const* operands = fusionChildrenOf(user); + for (unsigned index : { 1u, 0u }) { + const Operation* shift = operands[index]; + if ((!index && !isCommutative(opcode)) || !isShiftByConstant(shift) || !amountOfShift(shift)) + continue; + if (opcode == Add && shift->opcode() == Shl && user->type() == Int64) + continue; + if (!fusionOperandIsInRegister(user, 1 - index, operands[1 - index]) || fusionChildrenOf(shift)[0]->hasNoPosition() || !fusionCanAbsorbInteger(values, user->owner(), shift)) + continue; + fuseOperandAndTruncation(values, shift); + return true; + } + return false; +} + +static constexpr unsigned maxOperationsBetweenProductAndSum = 8; + +static bool fusedProductCanMoveTo(const Operation* product, const Operation* sum) +{ + const Block* block = sum->owner(); + unsigned begin = product->indexInBlock() + 1; + unsigned end = sum->indexInBlock(); + if (end - begin > maxOperationsBetweenProductAndSum) + return false; + for (unsigned index = begin; index < end; ++index) { + if (block->at(index)->opcode() == Patch) + return false; + } + return true; +} + +static bool fuseProductBelowSum(ValueFacts* values, const Operation* user) +{ + Opcode opcode = user->opcode(); + if ((opcode != Add && opcode != Sub) || !user->type().isInt()) + return false; + Operation* const* operands = fusionChildrenOf(user); + for (unsigned index : { 1u, 0u }) { + const Operation* product = operands[index]; + if ((!index && opcode == Sub) || product->opcode() != Mul || operands[1 - index]->hasNoPosition()) + continue; + if (!fusionCanAbsorbInteger(values, user->owner(), product) || !fusedProductCanMoveTo(product, user)) + continue; + fusionMark(values, product, ValueFlag::IsFused); + return true; + } + return false; +} + enum FusibleChild : uint8_t { FusibleTruncation = 1 << 0, FusibleShift = 1 << 1, FusibleLoad = 1 << 2, + FusibleProduct = 1 << 3, + FusibleExtension = 1 << 4, }; static constexpr std::array fusibleChildrenByOpcode = [] { @@ -688,6 +831,13 @@ static constexpr std::array fusibleChildrenByOpcode = kinds[Trunc] = FusibleTruncation; kinds[Shl] = FusibleShift; kinds[Load] = isX86_64() ? FusibleLoad : 0; + if (isARM64()) { + kinds[ZShr] = FusibleShift; + kinds[SShr] = FusibleShift; + kinds[Mul] = FusibleProduct; + kinds[SExt32] = FusibleExtension; + kinds[ZExt32] = FusibleExtension; + } return kinds; }(); @@ -705,6 +855,14 @@ static NEVER_INLINE bool fuseIntoComputation(ValueFacts* values, const Operation bool hasFused = (kinds & FusibleTruncation) && fuseTruncationsBelow(values, user, children, numberOfChildren); if ((kinds & FusibleShift) && fuseScaledIndexBelow(values, user)) return true; + if (isARM64()) { + if (((kinds & FusibleShift) && (fuseShiftBelowBitField(values, user) || fuseShiftedOperandBelow(values, user))) || ((kinds & FusibleProduct) && fuseProductBelowSum(values, user))) { + fuseNamesOfWordsBelow(values, user); + return true; + } + if ((kinds & FusibleExtension) && !fusionIsComparison(user->opcode()) && fuseExtendedOperandBelow(values, user)) + return true; + } if (!isX86_64() || !(kinds & FusibleLoad)) return hasFused; for (unsigned index = numberOfChildren; index--;) { diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineLiveness.h b/Source/JavaScriptCore/aot/machine/AOTMachineLiveness.h index bbdb27318d2a3..e61e36798b268 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineLiveness.h +++ b/Source/JavaScriptCore/aot/machine/AOTMachineLiveness.h @@ -10,6 +10,7 @@ #include "AOTMachineOrder.h" #include "AOTMachineTarget.h" #include +#include #include namespace JSC { namespace AOT { namespace Machine { @@ -195,6 +196,54 @@ struct OverflowCheckedSum { JS_EXPORT_PRIVATE OverflowCheckedSum overflowCheckedSumOf(const Liveness&, const Operation* sumOrBranch); +inline bool namesWord(const Operation* operation) { return operation->opcode() == ZExt32 && clearsUpperHalf(operation->child(0)); } + +inline bool isShiftByConstant(const Operation* operation) +{ + Opcode opcode = operation->opcode(); + return (opcode == Shl || opcode == ZShr || opcode == SShr) && operation->child(1)->hasInt32(); +} + +inline unsigned amountOfShift(const Operation* shift) +{ + ASSERT(isShiftByConstant(shift)); + return static_cast(shift->child(1)->asInt32()) & (shift->type() == Int32 ? 31 : 63); +} + +struct BitField { + unsigned lowestBit; + unsigned numberOfBits; + bool extendsSign; +}; + +inline std::optional bitFieldOf(const Operation* operation) +{ + Opcode opcode = operation->opcode(); + if (opcode != BitAnd && opcode != ZShr && opcode != SShr) + return std::nullopt; + const Operation* shift = operation->child(0); + const Operation* constant = operation->child(1); + if (!constant->hasInt() || !isShiftByConstant(shift)) + return std::nullopt; + + unsigned width = operation->type() == Int32 ? 32 : 64; + unsigned amount = amountOfShift(shift); + if (opcode == BitAnd) { + uint64_t mask = static_cast(constant->asInt()); + if (width == 32) + mask = static_cast(mask); + unsigned numberOfBits = static_cast(std::popcount(mask)); + if (shift->opcode() == Shl || !mask || (mask & (mask + 1)) || amount + numberOfBits > width) + return std::nullopt; + return BitField { amount, numberOfBits, false }; + } + + unsigned rightShiftAmount = amountOfShift(operation); + if (shift->opcode() != Shl || rightShiftAmount < amount) + return std::nullopt; + return BitField { rightShiftAmount - amount, width - rightShiftAmount, opcode == SShr }; +} + struct WayOutOfBranch { bool holds(Position position) const { return begin <= position && position < end; } @@ -309,7 +358,7 @@ ALWAYS_INLINE void forEachLeaf(const Liveness& liveness, const Operation* operat inline bool addsScaledIndex(const Liveness& liveness, const Operation* operation) { - if (operation->opcode() != Add || liveness.isFused(operation)) + if (operation->opcode() != Add || operation->type() != Int64 || liveness.isFused(operation)) return false; auto isFusedShift = [&](const Operation* child) { return child->opcode() == Shl && liveness.isFused(child); diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineTarget.h b/Source/JavaScriptCore/aot/machine/AOTMachineTarget.h index 54b95bcbf2a1c..f74a697b7ac42 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineTarget.h +++ b/Source/JavaScriptCore/aot/machine/AOTMachineTarget.h @@ -82,6 +82,11 @@ inline bool x86TargetAddressAcceptsBase(const Operation& access, const Operation return distance >= -x86TargetMaxDistanceFromFrameAddress && distance <= x86TargetMaxDistanceFromFrameAddress; } +ALWAYS_INLINE bool comparesConstantInMemory(const Operation& user, unsigned childIndex, const Operation& constant) +{ + return childIndex == 1 && (user.opcode() == Equal || user.opcode() == NotEqual) && constant.hasInt() && !isX86TargetImmediate(constant); +} + ALWAYS_INLINE bool acceptsImmediate(const Operation& user, unsigned childIndex, const Operation& constant) { if (!constant.hasNoPosition()) @@ -94,8 +99,6 @@ ALWAYS_INLINE bool acceptsImmediate(const Operation& user, unsigned childIndex, case BitAnd: case BitOr: case BitXor: - case Equal: - case NotEqual: case LessThan: case GreaterThan: case LessEqual: @@ -107,6 +110,10 @@ ALWAYS_INLINE bool acceptsImmediate(const Operation& user, unsigned childIndex, case EqualOrUnordered: return childIndex == 1 && isX86TargetImmediate(constant); + case Equal: + case NotEqual: + return childIndex == 1 && (constant.opcode() == NumberOf || isX86TargetImmediate(constant)); + case Shl: case SShr: case ZShr: @@ -185,11 +192,29 @@ ALWAYS_INLINE bool acceptsImmediate(const Operation& user, unsigned childIndex, RELEASE_ASSERT_NOT_REACHED(); } #else +constexpr bool comparesConstantInMemory(const Operation&, unsigned, const Operation&) { return false; } JS_EXPORT_PRIVATE bool acceptsImmediate(const Operation& user, unsigned childIndex, const Operation& constant); #endif JS_EXPORT_PRIVATE bool isEncodableBitImmediate(Width, int64_t bits); +#if CPU(ARM64) +struct ConstantComparison { + enum Kind : uint8_t { + Immediate, + Overflow, + HighBits, + }; + + Kind kind; + Opcode relation; + int64_t constant; + unsigned shift { 0 }; + bool holdsOnOverflow { false }; +}; +std::optional findConstantComparison(Opcode relation, Width, int64_t constant); +#endif + #if CPU(X86_64) ALWAYS_INLINE bool acceptsMemoryOperand(const Operation& user, unsigned childIndex) { @@ -304,6 +329,13 @@ JS_EXPORT_PRIVATE bool isEncodableIndexedAddress(Width accessWidth, unsigned shi JS_EXPORT_PRIVATE bool isEncodableScaledSum(unsigned shift, IndexExtension); JS_EXPORT_PRIVATE bool scaledSumAcceptsBase(const Operation& base); +constexpr bool acceptsShiftedOperand(Opcode opcode) +{ + return isARM64() && (opcode == Add || opcode == Sub || opcode == BitAnd || opcode == BitOr || opcode == BitXor); +} + +constexpr bool acceptsExtendedOperand(Opcode opcode) { return isARM64() && (opcode == Add || opcode == Sub || isComparison(opcode)); } + constexpr bool resultPrefersRegisterOfFirstChild() { return isX86_64(); } constexpr bool resultOverwritesFirstChild(Opcode opcode) { diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineTargetARM64.cpp b/Source/JavaScriptCore/aot/machine/AOTMachineTargetARM64.cpp index 923a707a8533d..12ecd09bcd7b6 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineTargetARM64.cpp +++ b/Source/JavaScriptCore/aot/machine/AOTMachineTargetARM64.cpp @@ -110,6 +110,60 @@ static constexpr bool isARM64TargetArithmeticImmediate(int64_t value) static_assert(isARM64TargetArithmeticImmediate(0xfff) && isARM64TargetArithmeticImmediate(-0xfff) && isARM64TargetArithmeticImmediate(0xfff000) && isARM64TargetArithmeticImmediate(-0xfff000)); static_assert(!isARM64TargetArithmeticImmediate(0x1001) && !isARM64TargetArithmeticImmediate(0x1000000) && !isARM64TargetArithmeticImmediate(std::numeric_limits::min())); +static constexpr bool isARM64TargetArithmeticMagnitude(uint64_t magnitude) +{ + return !(magnitude & ~uint64_t { 0xfff }) || !(magnitude & ~uint64_t { 0xfff000 }); +} + +std::optional findConstantComparison(Opcode relation, Width width, int64_t constant) +{ + ASSERT(width == Width32 || width == Width64); + if (isARM64TargetArithmeticImmediate(constant)) + return ConstantComparison { ConstantComparison::Immediate, relation, constant }; + + bool is32 = width == Width32; + if (relation == Above || relation == BelowEqual || relation == AboveEqual || relation == Below) { + uint64_t greatest = is32 ? UINT32_MAX : UINT64_MAX; + uint64_t bound = is32 ? static_cast(constant) : static_cast(constant); + if (relation == Above || relation == BelowEqual) { + if (bound == greatest) + return std::nullopt; + ++bound; + relation = relation == Above ? AboveEqual : Below; + int64_t canonical = is32 ? static_cast(bound) : static_cast(bound); + if (bound && isARM64TargetArithmeticImmediate(canonical)) + return ConstantComparison { ConstantComparison::Immediate, relation, canonical }; + } + if (!std::has_single_bit(bound)) + return std::nullopt; + return ConstantComparison { ConstantComparison::HighBits, relation, 0, static_cast(std::countr_zero(bound)) }; + } + + if (relation != LessThan && relation != LessEqual && relation != GreaterThan && relation != GreaterEqual) + return std::nullopt; + int64_t least = is32 ? std::numeric_limits::min() : std::numeric_limits::min(); + int64_t greatest = is32 ? std::numeric_limits::max() : std::numeric_limits::max(); + ASSERT(constant >= least && constant <= greatest); + if (relation == LessEqual || relation == GreaterThan) { + if (constant == greatest) + return std::nullopt; + ++constant; + relation = relation == LessEqual ? LessThan : GreaterEqual; + if (isARM64TargetArithmeticImmediate(constant)) + return ConstantComparison { ConstantComparison::Immediate, relation, constant }; + } + + uint64_t distanceFromLeast = static_cast(constant) - static_cast(least); + if (distanceFromLeast && isARM64TargetArithmeticMagnitude(distanceFromLeast)) + return ConstantComparison { ConstantComparison::Overflow, relation, static_cast(distanceFromLeast), 0, relation == LessThan }; + uint64_t distanceToGreatest = static_cast(greatest) - static_cast(constant) + 1; + if (isARM64TargetArithmeticMagnitude(distanceToGreatest)) + return ConstantComparison { ConstantComparison::Overflow, relation, -static_cast(distanceToGreatest), 0, relation == GreaterEqual }; + if (constant > 0 && std::has_single_bit(static_cast(constant))) + return ConstantComparison { ConstantComparison::HighBits, relation, 0, static_cast(std::countr_zero(static_cast(constant))) }; + return std::nullopt; +} + static bool isARM64TargetZeroNumber(const Operation& constant) { if (constant.hasDouble()) @@ -209,7 +263,7 @@ bool acceptsImmediate(const Operation& user, unsigned childIndex, const Operatio if (childIndex != 1) return false; if (constant.hasInt()) - return isARM64TargetArithmeticImmediate(constant.asInt()); + return !!findConstantComparison(user.opcode(), widthOf(constant.type()), constant.asInt()); return isARM64TargetZeroNumber(constant); case BitAnd: diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineTesting.h b/Source/JavaScriptCore/aot/machine/AOTMachineTesting.h index b058384e7e1fe..3533912a72800 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineTesting.h +++ b/Source/JavaScriptCore/aot/machine/AOTMachineTesting.h @@ -63,6 +63,7 @@ struct RandomProcedureOptions { bool withLandingPads { true }; bool withNumbers { true }; bool withScaledSums { false }; + bool withCombinedOperands { false }; bool withArmsThatReturn { false }; }; struct RandomProgram; diff --git a/Source/JavaScriptCore/aot/object/AOTClassGuards.cpp b/Source/JavaScriptCore/aot/object/AOTClassGuards.cpp index eb4d2fc9ea3fe..f223163032da0 100644 --- a/Source/JavaScriptCore/aot/object/AOTClassGuards.cpp +++ b/Source/JavaScriptCore/aot/object/AOTClassGuards.cpp @@ -23,6 +23,7 @@ #include "AOTObject.h" #include "AOTObjectInternal.h" #include "AOTObjectModel.h" +#include "AOTObjectModelCaches.h" #include "AOTProgramIdentifiers.h" #include "AOTPropertyCachesAnswers.h" #include "AOTPropertyWalk.h" @@ -1146,6 +1147,34 @@ bool ClassTable::nobodyHearsOfValueUnder(const LayoutData& layout, ClassSide sid return true; } +bool ClassTable::addsStaticUnobserved(VM& vm, const Closure& classObject, const LayoutData& layout, Structure* before, UniquedStringImpl* uid) const +{ + if (!hasNoGuardLeft(layout, ClassSide::Static) || uid->isSymbol() || isImplicitMemberOfClassHolder(vm, ClassSide::Static, uid) || uid == vm.propertyNames->displayName.impl() || parseIndex(PropertyName(uid)) || !closureAdmitsName(vm, &classObject, uid)) + return false; + std::optional key = identifiersOf(m_state.instance).numberOf(uid); + for (const LayoutData* current = &layout; current; current = assumedBaseOf(*current)) { + if (hasComputedMemberNames(*current) || (key && declaresMember(*current, ClassSide::Static, *key))) + return false; + } + JSValue prototype = classObject.getDirect(vm, vm.propertyNames->prototype); + if (!prototype || !prototype.isObject() || !before->storedPrototype().isObject()) + return false; + JSObject* parent = asObject(before->storedPrototype()); + JSValue parentOfPrototype = asObject(prototype)->getPrototypeDirect(); + JSCell* parentOfPrototypeCell = parentOfPrototype.isCell() ? parentOfPrototype.asCell() : nullptr; + const ClassStructureCache& cache = m_state.objects->caches->classStructures; + ClassStructures kept = cache.find(layout.index, parent, parentOfPrototypeCell); + if (!kept && parent == m_state.instance.globalObject->functionPrototype()) + kept = cache.find(layout.index, nullptr, parentOfPrototypeCell); + if (!kept || sharesStructureOfRootClasses(kept.classObjectStructure)) + return false; + for (Structure* structure = before; structure; structure = structure->previousID()) { + if (structure == kept.classObjectStructure) + return true; + } + return false; +} + void ClassTable::holdMemberToProof(VM& vm, const LayoutData& layout, ClassSide side, JSObject* holder, UniquedStringImpl* uid) { if (!layout.classData().isChangedInSightAlone(side) || vm.aotState().nativeTestChangesObjectsUnseen) diff --git a/Source/JavaScriptCore/aot/object/AOTClassTable.cpp b/Source/JavaScriptCore/aot/object/AOTClassTable.cpp index cb85b3273d58e..abbd4a57d9bfa 100644 --- a/Source/JavaScriptCore/aot/object/AOTClassTable.cpp +++ b/Source/JavaScriptCore/aot/object/AOTClassTable.cpp @@ -203,6 +203,11 @@ Structure* ClassTable::classObjectStructureInheritingFrom(VM& vm, JSObject* pare return structure; } +bool ClassTable::sharesStructureOfRootClasses(Structure* structure) const +{ + return structure && structure == m_classObjectStructures.get(m_state.instance.globalObject->functionPrototype()); +} + Closure* ClassTable::createClassObject(VM& vm, const LayoutData& classLayout) { ClosureTable& closures = m_state.objects->closures; @@ -965,7 +970,7 @@ void ClassDefinition::run(JSValue heritage, JSValue prototypeParent) m_prototype->structure()->setMayBePrototype(true); if (scope.exception()) [[unlikely]] return; - if (m_mayShareStructures && !isShared() && m_hasOnlyPlainMembers) + if (m_mayShareStructures && !isShared() && m_hasOnlyPlainMembers && !m_classes.sharesStructureOfRootClasses(m_classObject->structure())) m_state.objects->caches->classStructures.add(m_classLayout.index, m_classObjectParent, prototypeParentCell(), { m_classObject->structure(), m_prototype->structure() }); m_earlyStores.restoreTo(m_globalObject, m_classObject, m_requestedPart == ClassPart::Everything ? StoreTime::BeforeDefinition : StoreTime::AfterDefinition); m_earlyPrototypeStores.restoreTo(m_globalObject, m_prototype, StoreTime::AfterDefinition); @@ -1321,7 +1326,7 @@ bool ClassTable::tryDefineAgainWithKeptStructures(JSGlobalObject* globalObject, { VM& vm = globalObject->vm(); const ClassData& classData = classLayout.classData(); - if (classData.numberOfMethods || classData.numberOfStaticMethods || classData.numberOfStaticFields || classLayout.assumedBase() != LayoutData::noBase) + if (classData.numberOfMethods || classData.numberOfStaticMethods || classLayout.assumedBase() != LayoutData::noBase) return false; if (classObject->butterfly() || classObject->mayBePrototype()) return false; @@ -1352,6 +1357,8 @@ bool ClassTable::tryDefineAgainWithKeptStructures(JSGlobalObject* globalObject, classObject->setStructure(vm, kept.classObjectStructure); classObject->putDirectOffset(vm, offsetOfPrototype, prototype); prototype->putDirectOffset(vm, offsetOfConstructor, classObject); + if (classData.numberOfStaticFields) + markFunctionPropertiesDeclaredAsFields(vm, classObject, classLayout); if (structureOfHolderLeftOut) { nameClassObjectOnPrototypeMadeLeftOut(vm, classLayout, classObject, prototype); return true; diff --git a/Source/JavaScriptCore/aot/object/AOTClassTable.h b/Source/JavaScriptCore/aot/object/AOTClassTable.h index 3478c90c9634a..bffd67b74c8ed 100644 --- a/Source/JavaScriptCore/aot/object/AOTClassTable.h +++ b/Source/JavaScriptCore/aot/object/AOTClassTable.h @@ -176,6 +176,8 @@ class ClassTable { bool countsGuardsOfNamesBehindArming(const LayoutData& classLayout) const { return m_classesThatCountGuardsOfNamesBehindArming.get(m_tables.rankOfClass(classLayout.index)); } static bool leavesDefinitionToRecord(const LayoutData& classLayout) { return classLayout.createsMembersLazily && classLayout.createsClassObjectOnMutation && classLayout.classData().guardsHoldWhileClear; } void disarmMemberGuards(const LayoutData&); + bool addsStaticUnobserved(VM&, const Closure& classObject, const LayoutData&, Structure* before, UniquedStringImpl*) const; + bool sharesStructureOfRootClasses(Structure*) const; void didChangeMember(const LayoutData&, ClassSide, UniquedStringImpl*); void holdDefinitionToNameInRecord(const LayoutData&, ClassSide, uint32_t index, UniquedStringImpl*, const JSObject* function) const; void didDefineMember(VM&, const LayoutData&, ClassSide, uint32_t index, JSObject* holder, UniquedStringImpl*, JSObject* function); diff --git a/Source/JavaScriptCore/aot/object/AOTClosure.cpp b/Source/JavaScriptCore/aot/object/AOTClosure.cpp index 01b21baa86304..a695511485f87 100644 --- a/Source/JavaScriptCore/aot/object/AOTClosure.cpp +++ b/Source/JavaScriptCore/aot/object/AOTClosure.cpp @@ -364,8 +364,11 @@ static Structure* structureSharedWithClassesInheritingFrom(VM& vm, const Closure if (!classLayout.staticMethods().empty() || !classLayout.staticFields().empty()) return nullptr; State& state = classObject.state(); - if (parent != state.instance.globalObject->functionPrototype() && classLayout.evaluation() != EvaluationState::ProvenOnce) - return nullptr; + if (classLayout.evaluation() != EvaluationState::ProvenOnce) { + bool isFirstEvaluation = state.objects->closures.closureIfCreated(state.objects->classes.functionOfOwnClosure(classObject)) == &classObject; + if (parent != state.instance.globalObject->functionPrototype() || !isFirstEvaluation) + return nullptr; + } return state.objects->classes.classObjectStructureInheritingFrom(vm, parent); } @@ -563,25 +566,30 @@ bool holdsWhatIsFoundAbove(VM& vm, const JSObject* object, UniquedStringImpl* ui return own->getter() == found->getter() && own->setter() == found->setter(); } -void holdClosureToLackingNameSlowCase(VM& vm, const JSCell* closure, UniquedStringImpl* uid) +bool closureAdmitsName(VM& vm, const JSCell* closure, UniquedStringImpl* uid) { std::optional number = numberOfClosureHeldToListedNames(vm, closure); if (!number) - return; + return true; const CommonIdentifiers& names = *vm.propertyNames; if (uid == names.name.impl() || uid == names.length.impl() || uid == names.prototype.impl() || uid == names.arguments.impl() || uid == names.caller.impl()) - return; + return true; if (uid->isSymbol() && static_cast(uid)->isPrivate()) - return; + return true; std::optional name = identifiersOf(*Instance::owning(closure)).numberOf(uid); - if (name && std::ranges::binary_search(programTables().closurePropertyNames(), ClosurePropertyNameData { closureFunctionWithNumber(*number).index(), *name })) + return name && std::ranges::binary_search(programTables().closurePropertyNames(), ClosurePropertyNameData { closureFunctionWithNumber(*number).index(), *name }); +} + +void holdClosureToLackingNameSlowCase(VM& vm, const JSCell* closure, UniquedStringImpl* uid) +{ + if (closureAdmitsName(vm, closure, uid)) return; const JSObject* object = static_cast(closure); unsigned attributes = 0; JSValue value = object->getDirect(vm, PropertyName(uid), attributes); if (!value || holdsWhatIsFoundAbove(vm, object, uid, value, attributes)) return; - crashOnFalseProofOfClosure(*number, "AddProperty"_s); + crashOnFalseProofOfClosure(*numberOfClosureHeldToListedNames(vm, closure), "AddProperty"_s); } void holdClosureToItsPrototype(VM& vm, const JSCell* closure) @@ -753,10 +761,11 @@ bool Closure::put(JSCell* cell, JSGlobalObject* globalObject, PropertyName name, if (slot.thisValue() == JSValue(closure)) holdClosureToLackingNameSlowCase(vm, closure, name.uid()); if (isStoreToClassObject) { + bool callsSetter = slot.type() == PutPropertySlot::SetterProperty; bool replacesUndeclaredProperty = slot.type() == PutPropertySlot::ExistingProperty && !classes.hasStaticMethodNamed(*classLayout, name.uid()); - if (!replacesUndeclaredProperty) + if (!replacesUndeclaredProperty && !callsSetter) slot.disableCaching(); - if (slot.type() != PutPropertySlot::SetterProperty) { + if (!callsSetter) { classes.holdMemberToProof(vm, *classLayout, ClassSide::Static, closure, name.uid()); classes.didChangeMember(*classLayout, ClassSide::Static, name.uid()); } diff --git a/Source/JavaScriptCore/aot/object/AOTClosure.h b/Source/JavaScriptCore/aot/object/AOTClosure.h index f952040352de8..66b15520877e6 100644 --- a/Source/JavaScriptCore/aot/object/AOTClosure.h +++ b/Source/JavaScriptCore/aot/object/AOTClosure.h @@ -95,6 +95,7 @@ ALWAYS_INLINE bool isCompiledClosure(const JSCell* cell) { return cell->type() = JSValue findOnChainAbove(VM&, const JSObject*, UniquedStringImpl*, unsigned& attributes); bool holdsWhatIsFoundAbove(VM&, const JSObject*, UniquedStringImpl*, JSValue, unsigned attributes); void holdClosureToLackingNameSlowCase(VM&, const JSCell* closure, UniquedStringImpl*); +bool closureAdmitsName(VM&, const JSCell* closure, UniquedStringImpl*); ALWAYS_INLINE void holdClosureToLackingName(VM& vm, const JSCell* cell, UniquedStringImpl* uid) { if (isCompiledCalleeType(cell->type())) [[unlikely]] diff --git a/Source/JavaScriptCore/aot/object/AOTClosureTable.cpp b/Source/JavaScriptCore/aot/object/AOTClosureTable.cpp index 737a333ba6f7a..011d9dfad9db5 100644 --- a/Source/JavaScriptCore/aot/object/AOTClosureTable.cpp +++ b/Source/JavaScriptCore/aot/object/AOTClosureTable.cpp @@ -47,12 +47,6 @@ static std::span allocateClosureCells(State& state) return { static_cast(instance.closureCells), numberOfCells * sizeof(uint32_t) }; } -static uint32_t cellNumberOf(const ProgramTables& tables, FunctionID function) -{ - ClosureFunctionRangeData range = tables.closureFunctions; - return function.index() >= range.first && function.index() < range.end ? tables.closureCellNumbers()[function.index() - range.first] : 0; -} - static RememberedSlot* narrowClosureCells(const Instance& instance) { ASSERT(!instance.options().useWideReferences()); @@ -230,7 +224,7 @@ JSObject* ClosureTable::calleeOfModuleBodies(VM& vm, FunctionID body) JSCell* ClosureTable::closureIfCreated(FunctionID function) const { - uint32_t number = cellNumberOf(m_tables, function); + uint32_t number = m_tables.cellOfClosureFunction(function.index()); if (m_state.instance.options().useWideReferences()) { JSCell* cell = static_cast(m_state.instance.closureCells)[number]; return cell ? cell : m_state.objects->classes.classObjectHeldWhileReached(function); @@ -241,7 +235,7 @@ JSCell* ClosureTable::closureIfCreated(FunctionID function) const bool ClosureTable::hasCellFor(FunctionID function) const { - return cellNumberOf(m_tables, function); + return m_tables.cellOfClosureFunction(function.index()); } bool ClosureTable::hasSingletonClosure(FunctionID function) const @@ -255,7 +249,7 @@ void ClosureTable::didCreateClosureFor(FunctionID function, JSCell* cell) m_state.objects->classes.didRegisterClosure(function); if (!m_tables.kindOfFunction(function.index()).isInRuntimeModule) m_state.instance.closureTable.cards = m_writtenRegions.cards(); - uint32_t number = cellNumberOf(m_tables, function); + uint32_t number = m_tables.cellOfClosureFunction(function.index()); RELEASE_ASSERT(number); if (m_state.instance.options().useWideReferences()) { static_cast(m_state.instance.closureCells)[number] = cell; diff --git a/Source/JavaScriptCore/aot/object/AOTFunctionData.h b/Source/JavaScriptCore/aot/object/AOTFunctionData.h index ecb59d79194d7..66d6d4ea8ab58 100644 --- a/Source/JavaScriptCore/aot/object/AOTFunctionData.h +++ b/Source/JavaScriptCore/aot/object/AOTFunctionData.h @@ -10,6 +10,8 @@ #include "AOTRecordTypes.h" #include "AOTVarint.h" #include "CodeType.h" +#include "JSExportMacros.h" +#include #include #include #include @@ -133,6 +135,73 @@ struct ClosureFunctionRangeData { }; static_assert(sizeof(ClosureFunctionRangeData) == 12 && isImageRecord); +struct NumberGroup { + static constexpr unsigned size = 32; + static constexpr unsigned numberOfWords = 2; + static constexpr uint32_t isListedFlag = 1u << 31; + static constexpr size_t numberOfGroupWords(size_t numberOfEntries) { return (numberOfEntries + size - 1) / size * numberOfWords; } +}; + +class SparseNumberTable { +public: + explicit SparseNumberTable(std::span words) + : m_words(words) + { + } + + uint32_t at(uint32_t entry) const + { + std::span group = m_words.subspan(entry / NumberGroup::size * NumberGroup::numberOfWords, NumberGroup::numberOfWords); + uint32_t bit = 1u << entry % NumberGroup::size; + if (!(group[0] & bit)) + return 0; + uint32_t rank = std::popcount(group[0] & (bit - 1)); + return group[1] & NumberGroup::isListedFlag ? m_words[(group[1] & ~NumberGroup::isListedFlag) + rank] : group[1] + rank; + } + + JS_EXPORT_PRIVATE static Vector create(std::span numbers); + +private: + std::span m_words; +}; + +class AscendingNumberTable { +public: + static constexpr uint32_t maxSpan = 512; + + explicit AscendingNumberTable(std::span words) + : m_words(words) + { + } + + uint32_t at(uint32_t index) const + { + std::span group = m_words.subspan(index / NumberGroup::size * NumberGroup::numberOfWords, NumberGroup::numberOfWords); + uint32_t entry = index % NumberGroup::size; + uint32_t place = group[1]; + if (place & NumberGroup::isListedFlag) + return m_words[(place & ~NumberGroup::isListedFlag) + entry]; + if (!entry) + return group[0]; + uint32_t word = place / 32; + uint32_t bits = m_words[word] & ~0u << place % 32; + uint32_t onesLeft = entry; + while (static_cast(std::popcount(bits)) < onesLeft) { + onesLeft -= std::popcount(bits); + bits = m_words[++word]; + } + while (--onesLeft) + bits &= bits - 1; + uint32_t placeOfOne = word * 32 + std::countr_zero(bits); + return group[0] + (placeOfOne - place) - (entry - 1); + } + + JS_EXPORT_PRIVATE static Vector create(std::span numbers); + +private: + std::span m_words; +}; + struct FunctionSources { uint32_t numberOfEntries; uint32_t entriesOffset; @@ -346,13 +415,13 @@ bool forEachNamedCapture(std::span bytes, const auto& visit) v(functionRecordNumbers, uint8_t) \ v(functionRecords, FunctionRecordData) \ v(unsharedFunctionRecords, UnsharedFunctionRecordData) \ - v(sourceFunctions, Number24) \ + v(sourceFunctions, uint32_t) \ v(implementedBuiltins, ImplementedBuiltinData) \ v(closureFunctionKinds, FunctionKindData) \ v(functionLengths, uint8_t) \ v(closureCaptureCounts, uint8_t) \ v(functionsWithLargeCounts, FunctionWithLargeCountsData) \ - v(closureCellNumbers, Number24) \ + v(closureCells, uint32_t) \ v(namedCaptures, uint8_t) \ v(functionSources, FunctionSources) \ v(quotedSourceGroups, QuotedSourceGroupData) \ diff --git a/Source/JavaScriptCore/aot/object/AOTFunctionNames.h b/Source/JavaScriptCore/aot/object/AOTFunctionNames.h index f91a6400c6b96..03079a7c6c14c 100644 --- a/Source/JavaScriptCore/aot/object/AOTFunctionNames.h +++ b/Source/JavaScriptCore/aot/object/AOTFunctionNames.h @@ -31,6 +31,7 @@ JSString* functionSourceText(JSGlobalObject*, FunctionID); std::optional> functionSourceBytes(FunctionID); std::optional> quotedSourceBytes(uint32_t start, uint32_t length); +String computedNameInFrames(VM&, const FunctionDescription&); String writtenFunctionName(FunctionID); String initialNameProperty(FunctionID); JSString* initialNamePropertyString(VM&, State&, FunctionID); diff --git a/Source/JavaScriptCore/aot/object/AOTFunctionTables.cpp b/Source/JavaScriptCore/aot/object/AOTFunctionTables.cpp index 90267ffa728cc..4eb9ff25b3593 100644 --- a/Source/JavaScriptCore/aot/object/AOTFunctionTables.cpp +++ b/Source/JavaScriptCore/aot/object/AOTFunctionTables.cpp @@ -25,6 +25,7 @@ #include "AOTProgramStrings.h" #include "AOTSlimClosure.h" #include "AOTState.h" +#include "AOTVMState.h" #include "FunctionRareData.h" #include "JSCInlines.h" #include "NativeExecutable.h" @@ -230,6 +231,35 @@ String functionName(const FunctionDescription& description, NameForm form) RELEASE_ASSERT_NOT_REACHED(); } +static JSString* shareComputedName(VM& vm, FunctionID function, const String& name) +{ + VMState::ComputedFunctionName ownName { .hash = name.hash(), .length = name.length() }; + VMState::ComputedFunctionName& shared = vm.aotState().computedFunctionNames.add(sourceFunctionOf(function).index(), WTF::move(ownName)).iterator->value; + if (shared.hash != name.hash() || shared.length != name.length()) + shared.isOneOfSeveral = true; + if (JSString* string = shared.name.get(); string && !shared.isOneOfSeveral) { + if (string->tryGetValue().data == name) + return string; + shared.isOneOfSeveral = true; + } + JSString* string = jsString(vm, name); + shared.name = shared.isOneOfSeveral ? Weak() : Weak(string); + return string; +} + +String computedNameInFrames(VM& vm, const FunctionDescription& description) +{ + if (runsInBodyWithoutName(description)) + return emptyString(); + FunctionID source = sourceFunctionOf(description.function()); + if (functionKindData(source).isGenerator) + return emptyString(); + const auto& names = vm.aotState().computedFunctionNames; + auto found = names.find(source.index()); + JSString* name = found == names.end() ? nullptr : found->value.name.get(); + return name ? String(name->tryGetValueWithoutGC().data) : emptyString(); +} + static String computedFunctionName(JSGlobalObject* globalObject, const Identifier& key, MethodKind kind) { VM& vm = globalObject->vm(); @@ -260,10 +290,12 @@ void setComputedFunctionName(VM& vm, JSObject* closure, const Identifier& key, M auto scope = DECLARE_THROW_SCOPE(vm); String name = computedFunctionName(closure->realm(), key, kind); RETURN_IF_EXCEPTION(scope, void()); - if (std::optional function = functionOf(closure); function && !closure->getDirect(vm, vm.propertyNames->name) && initialNameProperty(sourceFunctionOf(*function)) == name) + std::optional function = functionOf(closure); + if (function && !closure->getDirect(vm, vm.propertyNames->name) && initialNameProperty(sourceFunctionOf(*function)) == name) return; + JSString* nameString = function ? shareComputedName(vm, *function, name) : jsString(vm, name); CompiledCallee::reifyProperties(vm, closure); - closure->putDirect(vm, vm.propertyNames->name, jsString(vm, name), PropertyAttribute::ReadOnly | PropertyAttribute::DontEnum); + closure->putDirect(vm, vm.propertyNames->name, nameString, PropertyAttribute::ReadOnly | PropertyAttribute::DontEnum); if (isCompiledClosure(closure)) { FunctionRareData* rareData = uncheckedDowncast(closure)->ensureRareData(vm); rareData->setHasReifiedName(); diff --git a/Source/JavaScriptCore/aot/object/AOTObjectBuilder.cpp b/Source/JavaScriptCore/aot/object/AOTObjectBuilder.cpp index fa7bc691fa23f..f7cc2a1368ca0 100644 --- a/Source/JavaScriptCore/aot/object/AOTObjectBuilder.cpp +++ b/Source/JavaScriptCore/aot/object/AOTObjectBuilder.cpp @@ -190,24 +190,24 @@ std::unique_ptr ObjectBuilder::tryCreatePlanOfSmallestRecord( { VM& vm = *state.instance.vm; countEvent(vm, RuntimeCounterID::RecordPlanChoices); - std::span recordsWithRarestName; + std::span shapesWithRarestName; for (UniquedStringImpl* name : names) { std::optional number = identifiersOf(state.instance).numberOf(name); if (!number) return nullptr; - std::span recordsWithName { std::ranges::equal_range(state.tables.parsedRecordFields(), *number, { }, &ParsedRecordFieldData::nameNumber) }; - if (recordsWithName.empty()) + std::span shapesWithName = state.tables.parsedRecordShapesWithName(*number); + if (shapesWithName.empty()) return nullptr; - if (recordsWithRarestName.empty() || recordsWithName.size() < recordsWithRarestName.size()) - recordsWithRarestName = recordsWithName; + if (shapesWithRarestName.empty() || shapesWithName.size() < shapesWithRarestName.size()) + shapesWithRarestName = shapesWithName; } RuntimeLayoutTable& layouts = state.objects->layouts; size_t recordsAsked = 0; std::unique_ptr chosen; - for (const ParsedRecordFieldData& row : recordsWithRarestName) { + for (Number24 shape : shapesWithRarestName) { ++recordsAsked; - const LayoutData& layout = layouts.layoutForShape(row.shape); + const LayoutData& layout = layouts.layoutForShape(shape); if (layout.numberOfFields < names.size() || !canBuildWithoutCompilerKnowledge(layout)) continue; std::unique_ptr plan = tryCreatePlan(state, layout, names); diff --git a/Source/JavaScriptCore/aot/object/AOTObjectModel.cpp b/Source/JavaScriptCore/aot/object/AOTObjectModel.cpp index f2ec97dbf51c9..7636ddfa91581 100644 --- a/Source/JavaScriptCore/aot/object/AOTObjectModel.cpp +++ b/Source/JavaScriptCore/aot/object/AOTObjectModel.cpp @@ -1117,7 +1117,7 @@ AllocationSiteRecord& StructureTable::fillAllocationSite(VM& vm, uint32_t number .header = kind == AllocatedCellKind::Environment ? Environment::headerFor(structure, allocation.shape) : cellHeaderFor(structure), .orderAndShapeWords = orderAndShapeWordsFor(allocation), .allocator = allocator ? allocator : &m_allocatorWithoutRoom, - .prototype = kind == AllocatedCellKind::Object ? static_cast(JSValue::encode(structure->storedPrototype())) : 0, + .prototype = kind == AllocatedCellKind::Object && !m_tables.layoutForShape(allocation.shape).impossibleMutations().contains(ObjectMutation::ConstructDirectly) ? static_cast(JSValue::encode(structure->storedPrototype())) : 0, }; m_writtenRecords.record(&record); if (kind == AllocatedCellKind::Object && noteWhetherLivesWhilePrototypeLives(m_tables.layoutForShape(allocation.shape))) diff --git a/Source/JavaScriptCore/aot/object/AOTObjectWords.h b/Source/JavaScriptCore/aot/object/AOTObjectWords.h index 2eb41ac08f302..41f3de39ef6bf 100644 --- a/Source/JavaScriptCore/aot/object/AOTObjectWords.h +++ b/Source/JavaScriptCore/aot/object/AOTObjectWords.h @@ -119,7 +119,7 @@ struct PrototypeTransition { }; static constexpr uint32_t numberOfPrototypeTransitions = 1024; -static constexpr unsigned prototypeTransitionHashShift = 6; +static constexpr unsigned prototypeTransitionHashShift = 4; static constexpr uint32_t indexOfPrototypeTransition(uint32_t before, EncodedJSValue prototype) { return ((before ^ static_cast(prototype)) >> prototypeTransitionHashShift) % numberOfPrototypeTransitions; } static constexpr uint32_t maxIndexOfFirstElementStoredContiguously = 7; diff --git a/Source/JavaScriptCore/aot/object/AOTProgramTables.h b/Source/JavaScriptCore/aot/object/AOTProgramTables.h index 6fdc65899c10f..14e9b42ff5ddf 100644 --- a/Source/JavaScriptCore/aot/object/AOTProgramTables.h +++ b/Source/JavaScriptCore/aot/object/AOTProgramTables.h @@ -15,6 +15,7 @@ #include "AOTPagedBitSet.h" #include "AOTRuntimeType.h" #include +#include #include #include #include @@ -53,14 +54,6 @@ struct TableRange { }; static_assert(sizeof(TableRange) == 8 && isImageRecord); -struct ParsedRecordFieldData { - Number24 name; - Number24 shape; - - uint32_t nameNumber() const { return name; } -}; -static_assert(sizeof(ParsedRecordFieldData) == 6 && isImageRecord); - struct MethodTableDependentData { uint32_t guardBit; uint32_t distanceToEntry; @@ -109,7 +102,9 @@ constexpr uint32_t methodTableEntryOf(uint32_t originOfSelector, uint32_t shape) v(classesBelowWithGuardsOfNames, ClassBelowWithGuardOfNameData) \ v(closurePropertyNames, ClosurePropertyNameData) \ v(addedPrototypeNames, AddedPrototypeNameData) \ - v(parsedRecordFields, ParsedRecordFieldData) + v(parsedRecordNames, uint32_t) \ + v(parsedRecordStarts, uint32_t) \ + v(parsedRecordShapes, Number24) #define FOR_EACH_AOT_PROGRAM_TABLE(v) \ FOR_EACH_AOT_OBJECT_PROGRAM_TABLE(v) \ @@ -306,6 +301,22 @@ struct ProgramTables { uint8_t number = functionRecordNumbers()[function]; if (number != unsharedFunctionRecord) [[likely]] return functionRecords()[number]; + return unsharedRecordOfFunction(function); + } + bool areUnsharedRecordsConsistent() const + { + std::span numbers = functionRecordNumbers(); + return std::ranges::all_of(unsharedFunctionRecords(), [&](const UnsharedFunctionRecordData& entry) { + return entry.function < numbers.size() && numbers[entry.function] == unsharedFunctionRecord; + }); + } + NEVER_INLINE const FunctionRecordData& unsharedRecordOfFunction(uint32_t function) const + { + static constinit std::atomic checkedTables { nullptr }; + if (checkedTables.load(std::memory_order_relaxed) != this) [[unlikely]] { + RELEASE_ASSERT(areUnsharedRecordsConsistent()); + checkedTables.store(this, std::memory_order_relaxed); + } return findRequiredEntry(unsharedFunctionRecords(), function, &UnsharedFunctionRecordData::function).record; } const FunctionTraceData& traceOfFunction(uint32_t function) const { return recordOfFunction(function).trace; } @@ -319,9 +330,21 @@ struct ProgramTables { } uint32_t sourceFunctionOf(uint32_t function) const { - std::span sources = sourceFunctions(); - size_t firstListed = numberOfFunctions() - sources.size(); - return function < firstListed ? function : sources[function - firstListed]; + std::span sources = sourceFunctions(); + return sources.empty() || function < closureFunctions.first ? function : AscendingNumberTable(sources).at(function - closureFunctions.first); + } + std::span parsedRecordShapesWithName(uint32_t name) const + { + uint32_t place = parsedRecordNames().empty() ? 0 : SparseNumberTable(parsedRecordNames()).at(name); + if (!place) + return { }; + AscendingNumberTable starts(parsedRecordStarts()); + uint32_t first = starts.at(place - 1); + return parsedRecordShapes().subspan(first, starts.at(place) - first); + } + uint32_t cellOfClosureFunction(uint32_t function) const + { + return function >= closureFunctions.first && function < closureFunctions.end ? SparseNumberTable(closureCells()).at(function - closureFunctions.first) : 0; } static constexpr uint32_t numberOfFunctionNameGroupWords(size_t numberOfFunctions) { return static_cast((numberOfFunctions + 31) / 32 * 2); } std::optional nameOfFunction(uint32_t function) const diff --git a/Source/JavaScriptCore/aot/object/AOTPropertyWalk.cpp b/Source/JavaScriptCore/aot/object/AOTPropertyWalk.cpp index 8d6f4c5fde863..5c7ee0651e291 100644 --- a/Source/JavaScriptCore/aot/object/AOTPropertyWalk.cpp +++ b/Source/JavaScriptCore/aot/object/AOTPropertyWalk.cpp @@ -368,6 +368,37 @@ Result forEachOwnPropertyAsCreatedAfterPreparing(JSGlobalObject* globalObject, J return forEachOwnPropertyAsCreated(vm, object, visitor, elements, Preparations::AreDone); } +bool structureGuaranteesAbsence(Structure* structure, UniquedStringImpl* name) +{ + return Object::structureGuaranteesAbsence(structure, name); +} + +bool appendDifferingValues(JSObject* first, JSObject* second, IdenticalValues identicalValues, MarkedArgumentBuffer& pairs) +{ + if (!isCompiledObject(first) || !isCompiledObject(second) || !haveSameFieldsInKeyOrder(first, second) || hasExtraProperties(first)) + return false; + const Object& left = *uncheckedDowncast(first); + const Object& right = *uncheckedDowncast(second); + if (left.layout().hasFieldsWithSymbolKeys) + return false; + auto fields = left.fieldsInKeyOrder(); + if (!fields) + return false; + for (auto& field : *fields) { + JSValue leftValue = left.readListedField(field); + JSValue rightValue = right.readListedField(field); + if (leftValue == rightValue && (!leftValue || identicalValues == IdenticalValues::Skip)) + continue; + if (!leftValue || !rightValue) { + pairs.clear(); + return false; + } + pairs.appendWithCrashOnOverflow(leftValue); + pairs.appendWithCrashOnOverflow(rightValue); + } + return true; +} + bool hasGetterToCall(JSObject* object) { const Object& typed = *uncheckedDowncast(object); diff --git a/Source/JavaScriptCore/aot/object/AOTPropertyWalk.h b/Source/JavaScriptCore/aot/object/AOTPropertyWalk.h index 824ffb4b10181..ca83dca0f0c24 100644 --- a/Source/JavaScriptCore/aot/object/AOTPropertyWalk.h +++ b/Source/JavaScriptCore/aot/object/AOTPropertyWalk.h @@ -51,6 +51,7 @@ enum class Presence : uint8_t { Present, Absent, Unknown }; enum class Integrity : uint8_t { NonExtensible, Sealed, Frozen }; enum class Elements : bool { NeedGenericPath, Skip }; enum class Preparations : bool { NeedGenericPath, AreDone }; +enum class IdenticalValues : bool { Skip, Append }; ALWAYS_INLINE bool isCompiledObject(const JSCell* cell) { return cell->type() == CompiledObjectType; } JS_EXPORT_PRIVATE bool wouldHaveArrayStorage(JSObject* compiledObject); @@ -76,6 +77,8 @@ Presence hasOwnProperty(JSObject*, UniquedStringImpl*); const CachedPropertyList* tryGetOwnPropertyList(VM&, JSObject* compiledObject, JSObject*& extraProperties); Presence hasPropertyOnChain(Structure*, UniquedStringImpl*); bool ensureExtraPropertiesStructure(VM&, JSObject*); +JS_EXPORT_PRIVATE bool structureGuaranteesAbsence(Structure*, UniquedStringImpl*); +JS_EXPORT_PRIVATE bool appendDifferingValues(JSObject*, JSObject*, IdenticalValues, MarkedArgumentBuffer&); JS_EXPORT_PRIVATE JSValue getDirect(VM&, const JSObject*, PropertyName, unsigned* attributesOrNull); JS_EXPORT_PRIVATE bool ordinaryGetOwnPropertySlot(JSGlobalObject*, JSObject*, PropertyName, PropertySlot&); JS_EXPORT_PRIVATE bool isOrdinaryPrototypeChain(JSObject* first, JSObject* last); diff --git a/Source/JavaScriptCore/aot/object/AOTThisAllocation.cpp b/Source/JavaScriptCore/aot/object/AOTThisAllocation.cpp index 1e60f05f3efc5..b0a2fea323e49 100644 --- a/Source/JavaScriptCore/aot/object/AOTThisAllocation.cpp +++ b/Source/JavaScriptCore/aot/object/AOTThisAllocation.cpp @@ -123,6 +123,7 @@ void ClassTable::keepWordsOfObjectWithoutFields(const LayoutData& layout, const Object* ClassTable::createWithoutFields(VM& vm, Structure* structure, const LayoutData& layout) { Object* object = Object::create(vm, structure, layout, layout.size, orderWordOf(0, 0)); + validateMutation(vm, *object, ObjectMutation::ConstructDirectly); CompiledStructure* compiled = uncheckedDowncast(structure); if (!compiled->wordsOfObjectWithoutFields()) { if (!wordsOfObjectWithoutFields(layout)) @@ -377,6 +378,7 @@ JSObject* ClassTable::instanceWithPrototype(VM& vm, Object* allocated, JSObject* const LayoutData& allocatedClass = classOfInstances(allocatedLayout); const LayoutData* requested = isDefinedByClass(allocatedClass) ? classWithPrototype(prototype) : nullptr; if (!requested || requested->index == allocatedClass.index) { + validateMutation(vm, *allocated, ObjectMutation::ConstructDirectly); if (!requested && isDefinedByClass(allocatedClass)) validateMutation(vm, *allocated, ObjectMutation::ChangePrototype); allocated->setPrototypeAtCreation(vm, prototype); diff --git a/Source/JavaScriptCore/aot/plan/AOTCallingConventions.cpp b/Source/JavaScriptCore/aot/plan/AOTCallingConventions.cpp index 00d72455bbd8a..fe787d67dcd8b 100644 --- a/Source/JavaScriptCore/aot/plan/AOTCallingConventions.cpp +++ b/Source/JavaScriptCore/aot/plan/AOTCallingConventions.cpp @@ -767,7 +767,27 @@ static void boxResultsOfMethodsWhoseNamesakesReturnOtherwise(const Program& prog }); } -static void boxResultsThatNobodyTakesRaw(const Program& program, const LinkTables& linked, BitVector& functionsThatReturnRawValues, CallingConventionParts& parts) +static BitVector collectDirectlyCalledClosures(const Program& program, const LinkTables& linked, const LinkTables& held) +{ + BitVector closures(program.functions.size()); + auto addCallees = [&](std::span calls) { + for (const LinkedCall& call : calls) { + FunctionKind kind = program.at(call.callee).kind; + if (!call.isInTailPosition && (kind == FunctionKind::Plain || kind == FunctionKind::Arrow)) + closures.quickSet(call.callee.index()); + } + }; + addCallees(linked.calls); + addCallees(held.calls); + return closures; +} + +static bool isUnboxedDouble(Representation form) +{ + return form == Representation::Double || form == Representation::OptionalDouble; +} + +static void boxResultsThatNobodyTakesRaw(const Program& program, const LinkTables& linked, const BitVector& directlyCalledClosures, BitVector& functionsThatReturnRawValues, CallingConventionParts& parts) { auto returnsRawValue = [&](FunctionID id) { return functionsThatReturnRawValues.get(id.index()); @@ -785,7 +805,7 @@ static void boxResultsThatNobodyTakesRaw(const Program& program, const LinkTable Vector functionsWithoutTakers; for (size_t index : functionsThatReturnRawValues) { FunctionID id(static_cast(index)); - if (!numberOfTakers[index] && !hasTakersThatLinkDoesNotList(id)) + if (!numberOfTakers[index] && !hasTakersThatLinkDoesNotList(id) && !directlyCalledClosures.get(index)) functionsWithoutTakers.append(id); } while (!functionsWithoutTakers.isEmpty()) { @@ -793,7 +813,7 @@ static void boxResultsThatNobodyTakesRaw(const Program& program, const LinkTable functionsThatReturnRawValues.clear(boxed.index()); parts.formsOfResults[parts.conventions[boxed.index()].firstResult] = Representation::JSValue; for (const LinkedCall& call : std::ranges::equal_range(linked.calls, boxed, { }, &LinkedCall::caller)) { - if (call.isInTailPosition && returnsRawValue(call.callee) && !--numberOfTakers[call.callee.index()]) + if (call.isInTailPosition && returnsRawValue(call.callee) && !--numberOfTakers[call.callee.index()] && !directlyCalledClosures.get(call.callee.index())) functionsWithoutTakers.append(call.callee); } } @@ -827,6 +847,7 @@ CallingConventions computeCallingConventions(const Program& program, const Facts parts.formsOfEntryParameters.fill(Representation::JSValue, numberOfEntryParameters); BitVector functionsThatReturnRawValues(numberOfFunctions); + BitVector directlyCalledClosures = returnsRawValues ? collectDirectlyCalledClosures(program, linked, facts.held()) : BitVector(); forEachRangeOfIndicesInParallel(loops, numberOfFunctions, [&](size_t firstFunction, size_t endOfFunctions) { for (size_t index = firstFunction; index < endOfFunctions; ++index) { FunctionID id(static_cast(index)); @@ -840,8 +861,11 @@ CallingConventions computeCallingConventions(const Program& program, const Facts formsOfResults[result] = boxedFormOfDeclaredPosition(program, function.results[result]); if (returnsRawValues && convention.has(CodeRole::Body) && mayReturnInFormOfItsOwn(function) && formsOfResults.back() == Representation::JSValue) { const LinkedFunction& proven = linked.function(id); - if (proven.everyTakerOfResultKnowsCallee && proven.ringsOfTailCallsAreListed) - formsOfResults.back() = formOfResultProvenToBe(program, proven.result); + if (proven.ringsOfTailCallsAreListed && (proven.everyTakerOfResultKnowsCallee || directlyCalledClosures.get(index))) { + Representation form = formOfResultProvenToBe(program, proven.result); + if (proven.everyTakerOfResultKnowsCallee || isUnboxedDouble(form)) + formsOfResults.back() = form; + } if (formsOfResults.back() != Representation::JSValue) functionsThatReturnRawValues.quickSet(index); } @@ -852,7 +876,7 @@ CallingConventions computeCallingConventions(const Program& program, const Facts } }); boxResultsOfMethodsWhoseNamesakesReturnOtherwise(program, linked, functionsThatReturnRawValues, parts); - boxResultsThatNobodyTakesRaw(program, linked, functionsThatReturnRawValues, parts); + boxResultsThatNobodyTakesRaw(program, linked, directlyCalledClosures, functionsThatReturnRawValues, parts); return CallingConventions(WTF::move(parts)); } diff --git a/Source/JavaScriptCore/aot/plan/AOTFieldsStorage.cpp b/Source/JavaScriptCore/aot/plan/AOTFieldsStorage.cpp index f2aa9bcc009cc..5c398a423e4b4 100644 --- a/Source/JavaScriptCore/aot/plan/AOTFieldsStorage.cpp +++ b/Source/JavaScriptCore/aot/plan/AOTFieldsStorage.cpp @@ -916,7 +916,7 @@ void checkStorageOfFields(const Program& program, const Facts& facts, const Plan build.diagnostics().add(InternalError { .kind = InternalErrorKind::PlacementCheckFailed, .layout = layout, .detail = String(words) }); }; auto hasCellsInUse = [&](LayoutID id) { - return plan.isInUse(id) && plan.layout(id).holdsCompiledObjects() && !plan.layout(id).slots.isEmpty(); + return plan.isAllocatable(id) && plan.layout(id).holdsCompiledObjects() && !plan.layout(id).slots.isEmpty(); }; const LinkTables& linked = facts.linked(); for (uint32_t index = 0; index < program.layouts.size(); ++index) { diff --git a/Source/JavaScriptCore/aot/plan/AOTPlan.cpp b/Source/JavaScriptCore/aot/plan/AOTPlan.cpp index 961b08e17d079..3435a19a11582 100644 --- a/Source/JavaScriptCore/aot/plan/AOTPlan.cpp +++ b/Source/JavaScriptCore/aot/plan/AOTPlan.cpp @@ -1403,7 +1403,7 @@ void CheckerOfPlan::checkLayout(LayoutID id) checkArea(id, planned, SlotStorage::Inline, isEnvironment ? offsetOfFirstEnvironmentVariable : offsetOfFirstField, planned.size, RuleOfPlan::SlotOutsideCell); checkArea(id, planned, SlotStorage::OutOfLine, offsetOfFirstExtensionSlot, std::min(planned.extensionSize, maxExtensionSize), RuleOfPlan::SlotOutsideExtension); - if (!m_plan.isInUse(id) || !(planned.holdsCompiledObjects() || isEnvironment)) + if (!m_plan.isAllocatable(id) || !(planned.holdsCompiledObjects() || isEnvironment)) return; expect(m_plan.shapes().isAssigned(m_plan.shapes().generalShapeOf(id)), RuleOfPlan::ShapeMissing, id); for (FieldIndex field = 0; field < planned.numberOfFields(); ++field) diff --git a/Source/JavaScriptCore/aot/plan/AOTPlan.h b/Source/JavaScriptCore/aot/plan/AOTPlan.h index e895fc53943e8..2e54e186bd554 100644 --- a/Source/JavaScriptCore/aot/plan/AOTPlan.h +++ b/Source/JavaScriptCore/aot/plan/AOTPlan.h @@ -128,7 +128,8 @@ class Plan { const SlotPlan& slots() const { return m_parts.slots; } const PlannedLayout& layout(LayoutID id) const { return slots().layout(id); } const PlacedSlot& slot(LayoutID id, FieldIndex field) const { return slots().slot(id, field); } - bool isInUse(LayoutID id) const { return !slots().isNeverAllocated(id); } + bool isInUse(LayoutID id) const { return slots().hasDirectInstances(id); } + bool isAllocatable(LayoutID id) const { return !slots().isNeverAllocated(id); } bool usesWideReferences() const { return m_parts.usesWideReferences; } const ShapeTable& shapes() const { return m_parts.shapes; } diff --git a/Source/JavaScriptCore/aot/plan/AOTPromotedEnvironments.cpp b/Source/JavaScriptCore/aot/plan/AOTPromotedEnvironments.cpp index e268340b96358..0707a2127eae0 100644 --- a/Source/JavaScriptCore/aot/plan/AOTPromotedEnvironments.cpp +++ b/Source/JavaScriptCore/aot/plan/AOTPromotedEnvironments.cpp @@ -117,6 +117,8 @@ class PlannerOfPromotedEnvironments { uint32_t rootOf(uint32_t node); void unite(uint32_t, uint32_t); void addSource(uint32_t node, uint32_t source); + static bool agrees(TypesOfCapture ofSource, TypesOfCapture ofReader) { return ofSource.established == ofReader.established && (!ofReader.proven || ofSource.proven == ofReader.proven); } + static void noteReader(Source&, FunctionID reader, uint32_t capture, TypesOfCapture); bool mayLieInFrameOfCaller(FunctionID, const BodyIndex&, ValueID) const; void followCapturedValues(FunctionID, const ThinSummary&, const BodyIndex&, bool serves); bool goThroughWhatStandsInFunctionsThatRunOnce(); @@ -131,6 +133,7 @@ class PlannerOfPromotedEnvironments { bool isExportsObjectThatStays(LayoutID) const; bool isEnteredUnderNoHandler(FunctionID, unsigned stepsLeft = 8) const; bool storesForGood(FunctionID maker, const BodyIndex&, ValueID closure, FunctionID holder, BlockID lastBlockOnEveryWayToReturn) const; + bool isClassToken(TypeID type) const { return type && m_program.at(type).isOpaque(OpaqueKind::ClassToken); } bool listFunctionsWhoseClosuresAreHeld(Candidate, TypesOfCapture, bool& holdsClosures); void strike(Candidate); void noteThatFunctionHoldsWord(uint32_t function); @@ -318,6 +321,17 @@ void PlannerOfPromotedEnvironments::unite(uint32_t first, uint32_t second) addSource(first, m_sourcesOfClasses[second]); } +void PlannerOfPromotedEnvironments::noteReader(Source& source, FunctionID reader, uint32_t capture, TypesOfCapture types) +{ + if (!source.types.proven && types.proven && source.types.established == types.established) { + source.firstReader = reader; + source.captureOfFirstReader = capture; + source.types = types; + return; + } + source.readersAgree &= agrees(source.types, types); +} + bool PlannerOfPromotedEnvironments::mayLieInFrameOfCaller(FunctionID id, const BodyIndex& index, ValueID value) const { while (!index.isParameter(value) && inputPassThroughOpcodes.contains(index.opcodeOf(value)) && index.value(value).operand(0)) @@ -428,7 +442,7 @@ void PlannerOfPromotedEnvironments::followCapturedValues(FunctionID id, const Th m_sources.append(Source { id, handed, callee, capture, types, true, mayLieInFrameOfCaller(id, index, handed), { } }); source = firstSource + static_cast(m_sources.size()) - 1; } else - m_sources[source - firstSource].readersAgree &= m_sources[source - firstSource].types == types; + noteReader(m_sources[source - firstSource], callee, capture, types); addSource(node, source); } } @@ -470,7 +484,7 @@ std::optional PlannerOfPromotedEnviron } auto node = m_nodesOfCaptures.find(m_facts.linked().firstCaptureOfFunction[id.index()] + capture); uint32_t found = node == m_nodesOfCaptures.end() ? noSource : m_sourcesOfClasses[rootOf(node->value)]; - if (found < firstSource || m_sources[found - firstSource].types != typesOfCapture(id, capture)) + if (found < firstSource || !agrees(m_sources[found - firstSource].types, typesOfCapture(id, capture))) return std::nullopt; return found - firstSource; } @@ -637,6 +651,8 @@ FunctionID PlannerOfPromotedEnvironments::makerThatMakesOnEveryWay(FunctionID ho bool PlannerOfPromotedEnvironments::listFunctionsWhoseClosuresAreHeld(Candidate candidate, TypesOfCapture types, bool& holdsClosures) { + if (isClassToken(types.established)) + return m_isCapturedByClosureThatNeverDies.quickGet(candidate); for (TypeID id : { types.proven, types.established }) { if (!id) continue; @@ -716,7 +732,7 @@ void PlannerOfPromotedEnvironments::keepWhatCouldBeLetGo() return type && (m_program.at(type).primitives & Primitive::Absent); }; bool holdsClosures = false; - bool serves = source.readersAgree && m_program.at(source.types.established).isJavaScriptValue() && !mayBeAbsent(source.types.established) && !mayBeAbsent(source.types.proven) + bool serves = source.readersAgree && (m_program.at(source.types.established).isJavaScriptValue() || isClassToken(source.types.established)) && !mayBeAbsent(source.types.established) && !mayBeAbsent(source.types.proven) && listFunctionsWhoseClosuresAreHeld(candidate, source.types, holdsClosures); if (!serves || (holdsClosures && source.mayLieInFrameOfCaller)) struckFromStart.append(candidate); diff --git a/Source/JavaScriptCore/aot/plan/AOTShapeTest.cpp b/Source/JavaScriptCore/aot/plan/AOTShapeTest.cpp index 137d4c5502eb3..2f4fd0d81740c 100644 --- a/Source/JavaScriptCore/aot/plan/AOTShapeTest.cpp +++ b/Source/JavaScriptCore/aot/plan/AOTShapeTest.cpp @@ -767,6 +767,8 @@ static OptionSet mutationsRuledOutForShapeTest(const Program& pr if (!layout.possibleChanges.contains(change)) ruledOut.add(mutation); } + if (!layout.hasDirectInstances) + ruledOut.add(ObjectMutation::ConstructDirectly); uint32_t numberDeclared = static_cast(program.at(id).fields.size()); for (uint32_t field = 0; field < numberDeclared; ++field) { if (!fieldAt(program, declarationOf(program, { id, field })).form.isEmpty()) diff --git a/Source/JavaScriptCore/aot/plan/AOTSlotPlan.cpp b/Source/JavaScriptCore/aot/plan/AOTSlotPlan.cpp index 09de51ff97860..5a6b470f57f27 100644 --- a/Source/JavaScriptCore/aot/plan/AOTSlotPlan.cpp +++ b/Source/JavaScriptCore/aot/plan/AOTSlotPlan.cpp @@ -201,7 +201,7 @@ bool SlotPlan::listMembersAskedAbout(const Program& program, PropertyKeyID key, listed.areListed = true; for (LayoutID member : members) { const Layout& layout = program.at(member); - listed.numberInUse += !isNeverAllocated(member); + listed.numberInUse += hasDirectInstances(member); if (writtenBaseOf(layout) && layout.inheritedFields != InheritedFields::AllOfRow) listed.askedAboutEveryName.append(member); } @@ -228,7 +228,7 @@ FieldAccess SlotPlan::fieldAccessOfOneLayout(const Program& program, PropertyKey { FieldAccess access; access.key = key; - if (isNeverAllocated(receiver) || !holdsCompiledObjects(receiver)) + if (!hasDirectInstances(receiver) || !holdsCompiledObjects(receiver)) return access; std::optional field = findField(program, receiver, key); if (!field) @@ -263,7 +263,7 @@ FieldAccess SlotPlan::fieldAccessOfLayouts(const Program& program, PropertyKeyID bool coversEveryMember = true; uint32_t numberInUse = 0; for (LayoutID member : receivers) { - if (isNeverAllocated(member)) + if (!hasDirectInstances(member)) continue; ++numberInUse; const PlacedSlot* placed = nullptr; @@ -330,7 +330,7 @@ std::optional SlotPlan::recordOfStoredField(const Program& program, Pr constexpr OptionSet flagsOfPrivateNames { FieldFlags::IsPrivate, FieldFlags::IsBrand }; std::optional index; for (LayoutID member : receivers) { - if (isNeverAllocated(member)) + if (!hasDirectInstances(member)) continue; if (!holdsCompiledObjects(member)) return std::nullopt; @@ -375,7 +375,7 @@ StoredFieldForm SlotPlan::formOfStoredField(const Program& program, PropertyKeyI { std::optional form; for (LayoutID member : receivers) { - if (isNeverAllocated(member)) + if (!hasDirectInstances(member)) continue; std::optional field = findField(program, member, key); if (!field) @@ -690,10 +690,108 @@ static void chooseListedFieldsOutOfLineForSlotPlan(const Program& program, const } } -static void placeFieldsSharedByFamiliesForSlotPlan(const Program& program, const Families& families, std::span storage, PlacesOfFieldsInSlotPlan& result) +static BitVector findFamiliesThatMeetForSlotPlan(const Program& program, const Facts& facts, const Families& families) +{ + const LinkTables& linked = facts.linked(); + TypeReader reader(program); + size_t numberOfTypes = program.types.numberOfTypes(); + BitVector result(families.size()); + BitVector familiesNamedWhole(families.size()); + BitVector typesAsked(numberOfTypes); + UncheckedKeyHashSet, WTF::UnsignedWithZeroKeyHashTraits> setsAsked; + Vector typesToAsk; + Vector lookOfFamily(FillWith { }, families.size(), 0); + Vector metFrom(FillWith { }, families.size(), 0); + uint32_t look = 0; + auto meet = [&](LayoutSetID layouts) { + if (!layouts || holdsOneMember(layouts) || !setsAsked.add(layouts.bits()).isNewEntry) + return; + if (FamilyID family = families.familyOf(layouts)) { + familiesNamedWhole.quickSet(family.index()); + return; + } + ++look; + reader.at(layouts).forEach([&](LayoutID member) { + uint32_t inside = families.size() + member.index(); + for (FamilyID family = families.nearestFamilyOf(member); family; family = families.nearestFamilyAbove(family)) { + uint32_t number = family.index(); + if (std::exchange(lookOfFamily[number], look) == look) { + if (metFrom[number] != inside) + result.quickSet(number); + break; + } + metFrom[number] = inside; + inside = number; + } + }); + }; + auto ask = [&](TypeID type) { + if (type && !holdsOnlyPrimitives(type) && liesInProgram(type) && positionOf(type) < numberOfTypes && !typesAsked.quickSet(positionOf(type))) + typesToAsk.append(type); + while (!typesToAsk.isEmpty()) { + Type read = reader.at(typesToAsk.takeLast()); + meet(read.layouts); + for (const ClassPart& part : classPartsOf(reader, read)) { + for (TypeID contents : part.writtenContents) { + if (contents && !holdsOnlyPrimitives(contents) && liesInProgram(contents) && positionOf(contents) < numberOfTypes && !typesAsked.quickSet(positionOf(contents))) + typesToAsk.append(contents); + } + } + } + }; + auto askOfCell = [&](const LinkedCell& cell, const DeclaredType& declared) { + ask(cell.proven ? cell.proven : declared.written()); + }; + for (uint32_t index = 0; index < program.functions.size() && index < linked.functions.size(); ++index) { + if (!linked.functions[index].isReached) + continue; + ask(linked.functions[index].result); + for (const LinkedParameter& parameter : linked.parametersOf(FunctionID(index))) + ask(parameter.proven ? parameter.proven : parameter.writtenOrSeenToArrive()); + } + for (size_t index = 0; index < linked.variables.size(); ++index) + askOfCell(linked.variables[index], program.variables[index].declared); + for (uint32_t index = 0; index < program.layouts.size() && index < linked.layouts.size(); ++index) { + if (!linked.layouts[index].isAllocated) + continue; + const auto& fields = program.layouts[index].fields; + for (uint32_t field = 0; field < fields.size(); ++field) { + if (const LinkedCell* cell = linked.findCellOfField({ LayoutID(index), field })) + askOfCell(*cell, fields[field].declared); + } + } + for (const LinkedCell& cell : linked.appendedFields) + ask(cell.proven); + for (uint32_t number = families.size(); number--;) { + FamilyID above = families.nearestFamilyAbove(FamilyID(number)); + if (above && familiesNamedWhole.quickGet(above.index())) + familiesNamedWhole.quickSet(number); + } + result.merge(familiesNamedWhole); + return result; +} + +static bool formsOfRowsDifferForSlotPlan(const PlacesOfFieldsInSlotPlan& places, std::span rows) +{ + bool someHoldAnyValue = false; + bool someHoldNumbers = false; + for (uint32_t row : rows) { + someHoldAnyValue |= places.formsProven[row] == FormOfPlaceInSlotPlan::AnyValue; + someHoldNumbers |= places.formsProven[row] == FormOfPlaceInSlotPlan::Integer || places.formsProven[row] == FormOfPlaceInSlotPlan::Number; + } + return someHoldAnyValue && someHoldNumbers; +} + +static void placeFieldsSharedByFamiliesForSlotPlan(const Program& program, const Facts& facts, const Families& families, std::span storage, PlacesOfFieldsInSlotPlan& result) { Vector members; Vector rows; + std::optional familiesThatMeet; + auto meets = [&](uint32_t family) { + if (!familiesThatMeet) + familiesThatMeet = findFamiliesThatMeetForSlotPlan(program, facts, families); + return familiesThatMeet->quickGet(family); + }; const BitVector& familiesThatMayAgree = result.familiesThatMayAgreeOnSlots; for (uint32_t number = families.size(); number--;) { findMembersThatAgreeOnSlotsForSlotPlan(program, families, familiesThatMayAgree, FamilyID(number), storage, members); @@ -714,11 +812,15 @@ static void placeFieldsSharedByFamiliesForSlotPlan(const Program& program, const return; rows.append(result.rowOf({ member, *listed })); } + bool numbersStandApart = !result.formsProven.isEmpty() && formsOfRowsDifferForSlotPlan(result, rows.span()) && !meets(number); + uint32_t place = static_cast(result.formsOfPlaces.size()); for (uint32_t row : rows) { result.offsets[row] = end; - result.places[row] = static_cast(result.formsOfPlaces.size()); + result.places[row] = place + (numbersStandApart && result.formsProven[row] != FormOfPlaceInSlotPlan::AnyValue); } result.formsOfPlaces.append(FormOfPlaceInSlotPlan::Unclaimed); + if (numbersStandApart) + result.formsOfPlaces.append(FormOfPlaceInSlotPlan::Unclaimed); end += bytesPerSlotInSlotPlan; }); for (LayoutID member : members) @@ -920,8 +1022,10 @@ static void numberRowsForSlotPlan(const Program& program, PlacesOfFieldsInSlotPl result.firstRowOfLayout.append(static_cast(numberOfRows)); } -static void placeFieldsForSlotPlan(const Program& program, const LinkTables& link, const LinkTables& held, const BitVector& layoutsWithSlotsThatStoresMayEmpty, const Families* families, const LayoutsBelowTheirBases& layoutsBelowTheirBases, std::span storage, const StorageOfLayouts& storageOfLayouts, bool hasTypedSlots, PlacesOfFieldsInSlotPlan& result) +static void placeFieldsForSlotPlan(const Program& program, const Facts& facts, const BitVector& layoutsWithSlotsThatStoresMayEmpty, const Families* families, const LayoutsBelowTheirBases& layoutsBelowTheirBases, std::span storage, const StorageOfLayouts& storageOfLayouts, bool hasTypedSlots, PlacesOfFieldsInSlotPlan& result) { + const LinkTables& link = facts.linked(); + const LinkTables& held = facts.held(); const BitVector& layoutsThatRepeatTheirBases = storageOfLayouts.layoutsThatRepeatTheirBases; uint32_t numberOfLayouts = static_cast(program.layouts.size()); uint32_t numberOfRows = result.firstRowOfLayout.last(); @@ -937,7 +1041,7 @@ static void placeFieldsForSlotPlan(const Program& program, const LinkTables& lin chooseListedFieldsOutOfLineForSlotPlan(program, held, *families, storage, result); placeAppendedFieldsForSlotPlan(program, link, families, storage, result); if (families) { - placeFieldsSharedByFamiliesForSlotPlan(program, *families, storage, result); + placeFieldsSharedByFamiliesForSlotPlan(program, facts, *families, storage, result); auto hasCells = [&](LayoutID id) { return storage[id.index()].storage == LayoutStorage::CompiledObject && storage[id.index()].hasCells(); }; @@ -1501,14 +1605,19 @@ void holdDoorsToProofsInSlotPlan(Program& program, const Facts& facts, const Par for (uint32_t index = range * layoutsInRange; index < endOfRange; ++index) { LayoutKind kind = program.layouts[index].kind; bool hasDoors = kind == LayoutKind::Record || kind == LayoutKind::Instance || kind == LayoutKind::Container; + bool mayBeStable = (kind == LayoutKind::Record || kind == LayoutKind::Instance) && !program.layouts[index].has(LayoutFlags::IsConstructorFunction); PlannedLayout& planned = parts.layouts[index]; for (FieldIndex field = 0; field < planned.numberOfFields(); ++field) { PlacedSlot& slot = planned.slots[field]; FieldReference reference { LayoutID(index), field }; + if (const LinkedCell* cell = mayBeStable ? facts.linked().findCellOfField(reference) : nullptr) + slot.isStable = cell->everyWriterIsKnown && !cell->isWrittenAfterConstruction && cell->isFilledAtEveryRead; TypeID proven = facts.linked().provenOfField(reference); + slot.isStable &= !!proven; if (!proven) continue; slot.everyStoreFits = !slot.type || types.isSubset(proven, slot.type); + slot.isStable &= slot.everyStoreFits; if (!slot.everyStoreFits || !hasDoors || !slot.indexInLayoutRecord || !slot.hasStorage() || slot.holdsAccessor || slot.deferredFunction) continue; fieldsWithDoorsInRanges[range].append(reference); @@ -1590,6 +1699,11 @@ StorageOfLayouts chooseStorageOfLayouts(const Program& program, const Facts& fac const LinkTables& linked = facts.linked(); StorageOfLayouts result; result.layoutsNeverAllocated = findLayoutsNeverAllocatedForSlotPlan(program, linked, loops); + result.layoutsWithoutDirectInstances.ensureSize(program.layouts.size()); + for (size_t index = 0; index < std::min(program.layouts.size(), linked.layouts.size()); ++index) { + if (!linked.layouts[index].hasDirectInstances) + result.layoutsWithoutDirectInstances.quickSet(index); + } for (const TaggedMaker& maker : linked.taggedMakers) { for (const TaggedLayout& member : program.taggedFamilies[maker.family].members) { if (member.layout.index() < linked.layouts.size() && linked.layout(member.layout).isAllocated) @@ -1628,6 +1742,8 @@ SlotPlanParts computeSlotPlanParts(Program& program, const Facts& facts, BuildCo unsigned numberOfLayouts = static_cast(program.layouts.size()); SlotPlanParts parts; parts.layoutsNeverAllocated = storageOfLayouts.layoutsNeverAllocated; + parts.layoutsWithoutDirectInstances = storageOfLayouts.layoutsWithoutDirectInstances; + parts.layoutsWithoutDirectInstances.merge(parts.layoutsNeverAllocated); std::span storage = storageOfLayouts.layouts.span(); bool hasTypedSlots = hasTypedSlotsInSlotPlan(facts); const Families* families = withoutMechanismsMissingDependencies(facts.mechanisms()).contains(Mechanism::TypedOperations) ? &facts.families() : nullptr; @@ -1650,7 +1766,7 @@ SlotPlanParts computeSlotPlanParts(Program& program, const Facts& facts, BuildCo EnvironmentsHandedBack environmentsHandedBack; runLanes(build.parallelLoops(), { [&] { - placeFieldsForSlotPlan(program, facts.linked(), facts.held(), layoutsWithSlotsThatStoresMayEmpty, families, layoutsBelowTheirBases, storage, storageOfLayouts, hasTypedSlots, places); + placeFieldsForSlotPlan(program, facts, layoutsWithSlotsThatStoresMayEmpty, families, layoutsBelowTheirBases, storage, storageOfLayouts, hasTypedSlots, places); if (hasTypedSlots) chooseFormsOfPlacesForSlotPlan(program, facts, facts.linked(), families, storage, storageOfLayouts.layoutsThatRepeatTheirBases, places); forEachIndexInParallel(build.parallelLoops(), numberOfLayouts, [&](unsigned index) { diff --git a/Source/JavaScriptCore/aot/plan/AOTSlotPlan.h b/Source/JavaScriptCore/aot/plan/AOTSlotPlan.h index 5a7337914385c..c4c061d933b85 100644 --- a/Source/JavaScriptCore/aot/plan/AOTSlotPlan.h +++ b/Source/JavaScriptCore/aot/plan/AOTSlotPlan.h @@ -57,6 +57,7 @@ struct StorageOfLayouts { bool holdsCompiledObjectsInUse(LayoutID id) const { return of(id).holdsCompiledObjects() && !isNeverAllocated(id); } BitVector layoutsNeverAllocated; + BitVector layoutsWithoutDirectInstances; BitVector layoutsThatRepeatTheirBases; Vector nearestLayoutsAboveThatDoNotRepeat; Vector layouts; @@ -104,6 +105,7 @@ struct PlacedSlot { bool deferredClosureCapturesObject { false }; bool deferredClosureCapturesEnvironment { false }; bool mayTakeWordLeftInCell { false }; + bool isStable { false }; uint32_t offset { 0 }; std::optional indexInLayoutRecord; FunctionID deferredFunction; @@ -345,6 +347,7 @@ struct SlotPlanParts { BitVector functionsWhosePrototypePropertyStaysAsMade; Vector classCreation; BitVector layoutsNeverAllocated; + BitVector layoutsWithoutDirectInstances; BitVector keysWithColumn; BitVector keysOfSlotsThatHoldMarks; Vector firstModuleThatWritesFieldUnder; @@ -408,6 +411,7 @@ class SlotPlan { const PlannedLayout& layout(LayoutID id) const { return id.index() < m_parts.layouts.size() ? m_parts.layouts[id.index()] : m_unplannedLayout; } bool holdsCompiledObjects(LayoutID id) const { return layout(id).holdsCompiledObjects(); } bool isNeverAllocated(LayoutID id) const { return m_parts.layoutsNeverAllocated.get(id.index()); } + bool hasDirectInstances(LayoutID id) const { return !m_parts.layoutsWithoutDirectInstances.get(id.index()); } bool someLayoutInUseHoldsOtherThanCompiledObjects(uint32_t wordOfLayouts, uint64_t layoutsInWord) const { return wordOfLayouts >= m_layoutsInUseThatHoldOtherThanCompiledObjects.size() || (m_layoutsInUseThatHoldOtherThanCompiledObjects[wordOfLayouts] & layoutsInWord); } const PlacedSlot& slot(LayoutID id, FieldIndex field) const diff --git a/Source/JavaScriptCore/aot/plan/AOTUnionConsecutiveSets.cpp b/Source/JavaScriptCore/aot/plan/AOTUnionConsecutiveSets.cpp index ae428c14dfbef..b31d43746c8fb 100644 --- a/Source/JavaScriptCore/aot/plan/AOTUnionConsecutiveSets.cpp +++ b/Source/JavaScriptCore/aot/plan/AOTUnionConsecutiveSets.cpp @@ -25,6 +25,7 @@ struct ConsecutiveSets::Tree { explicit Tree(uint32_t numberOfLeaves); bool tryKeepTogether(std::span leaves); + bool tryKeepNestedSetTogether(std::span leaves); Vector order(std::span keyOfLeaf) const; private: @@ -87,18 +88,29 @@ struct ConsecutiveSets::Tree { void replaceInParent(uint32_t node, uint32_t replacement); void beginVisiting(uint32_t node); - uint32_t markLeaves(std::span leaves); + uint32_t markNodes(std::span nodes); void countPertinentChildren(); - uint32_t labelPertinentNodes(uint32_t numberOfLeavesInSet); + uint32_t labelPertinentNodes(uint32_t numberOfNodesInSet); bool tryLabel(uint32_t node, bool isPertinentRoot); uint32_t takeFullChildrenOfP(uint32_t node); PartialRow flatten(uint32_t node, uint32_t q); void reducePertinentRootP(uint32_t root); + uint32_t findNestedSetOf(uint32_t leaf); void reducePertinentRootQ(uint32_t root); + struct NestedSet { + uint32_t parent { none }; + uint32_t node { none }; + uint32_t numberOfLeaves { 1 }; + uint32_t lastEnclosingSet { 0 }; + }; + Vector m_nodes; Vector m_queue; + Vector m_nestedSets; + Vector m_nestedSetsInside; + uint32_t m_numberOfNestedSets { 0 }; uint32_t m_numberOfLeaves { 0 }; uint32_t m_root { none }; uint32_t m_epoch { 0 }; @@ -222,19 +234,19 @@ void ConsecutiveSets::Tree::beginVisiting(uint32_t index) node.firstFullChild = none; } -uint32_t ConsecutiveSets::Tree::markLeaves(std::span leaves) +uint32_t ConsecutiveSets::Tree::markNodes(std::span nodes) { ++m_epoch; RELEASE_ASSERT(m_epoch); m_queue.shrink(0); - for (uint32_t leaf : leaves) { - RELEASE_ASSERT(leaf < m_numberOfLeaves); - if (at(leaf).epoch == m_epoch) + for (uint32_t node : nodes) { + RELEASE_ASSERT(node < m_nodes.size()); + if (at(node).epoch == m_epoch) continue; - beginVisiting(leaf); - at(leaf).label = Label::Full; - at(leaf).numberOfPertinentLeaves = 1; - m_queue.append(leaf); + beginVisiting(node); + at(node).label = Label::Full; + at(node).numberOfPertinentLeaves = 1; + m_queue.append(node); } return static_cast(m_queue.size()); } @@ -256,13 +268,13 @@ void ConsecutiveSets::Tree::countPertinentChildren() } } -uint32_t ConsecutiveSets::Tree::labelPertinentNodes(uint32_t numberOfLeavesInSet) +uint32_t ConsecutiveSets::Tree::labelPertinentNodes(uint32_t numberOfNodesInSet) { - m_queue.shrink(numberOfLeavesInSet); + m_queue.shrink(numberOfNodesInSet); for (size_t head = 0; head < m_queue.size(); ++head) { uint32_t node = m_queue[head]; - bool isPertinentRoot = at(node).numberOfPertinentLeaves == numberOfLeavesInSet; - if (at(node).kind != Kind::Leaf && !tryLabel(node, isPertinentRoot)) + bool isPertinentRoot = at(node).numberOfPertinentLeaves == numberOfNodesInSet; + if (head >= numberOfNodesInSet && !tryLabel(node, isPertinentRoot)) return none; if (isPertinentRoot) return node; @@ -452,7 +464,7 @@ bool ConsecutiveSets::Tree::tryKeepTogether(std::span leaves) { if (leaves.size() < 2) return true; - uint32_t numberOfLeavesInSet = markLeaves(leaves); + uint32_t numberOfLeavesInSet = markNodes(leaves); if (numberOfLeavesInSet < 2) return true; countPertinentChildren(); @@ -468,6 +480,61 @@ bool ConsecutiveSets::Tree::tryKeepTogether(std::span leaves) return true; } +uint32_t ConsecutiveSets::Tree::findNestedSetOf(uint32_t leaf) +{ + uint32_t set = leaf; + while (m_nestedSets[set].parent != none) + set = m_nestedSets[set].parent; + while (leaf != set) + leaf = std::exchange(m_nestedSets[leaf].parent, set); + return set; +} + +bool ConsecutiveSets::Tree::tryKeepNestedSetTogether(std::span leaves) +{ + if (leaves.size() < 2) + return true; + if (m_nestedSets.isEmpty()) { + m_nestedSets.grow(m_numberOfLeaves); + for (uint32_t leaf = 0; leaf < m_numberOfLeaves; ++leaf) + m_nestedSets[leaf] = NestedSet { .node = leaf }; + } + ++m_numberOfNestedSets; + m_nestedSetsInside.shrink(0); + size_t numberOfLeavesInside = 0; + for (uint32_t leaf : leaves) { + uint32_t set = findNestedSetOf(leaf); + if (std::exchange(m_nestedSets[set].lastEnclosingSet, m_numberOfNestedSets) == m_numberOfNestedSets) + continue; + m_nestedSetsInside.append(set); + numberOfLeavesInside += m_nestedSets[set].numberOfLeaves; + } + if (numberOfLeavesInside != leaves.size()) + return false; + if (m_nestedSetsInside.size() == 1) + return true; + + Vector nodes; + for (uint32_t set : m_nestedSetsInside) + nodes.append(m_nestedSets[set].node); + uint32_t numberOfNodes = markNodes(nodes.span()); + RELEASE_ASSERT(numberOfNodes == nodes.size()); + countPertinentChildren(); + uint32_t root = labelPertinentNodes(numberOfNodes); + RELEASE_ASSERT(root != none && at(root).kind == Kind::P && !at(root).numberOfPartialChildren); + uint32_t node = root; + if (at(root).label != Label::Full) { + reducePertinentRootP(root); + node = at(root).endChildren[1]; + } + uint32_t united = m_nestedSetsInside[0]; + for (uint32_t set : m_nestedSetsInside.span().subspan(1)) + m_nestedSets[set].parent = united; + m_nestedSets[united].node = node; + m_nestedSets[united].numberOfLeaves = static_cast(numberOfLeavesInside); + return true; +} + Vector ConsecutiveSets::Tree::order(std::span keyOfLeaf) const { RELEASE_ASSERT(keyOfLeaf.size() == m_numberOfLeaves); @@ -531,6 +598,11 @@ bool ConsecutiveSets::tryKeepTogether(std::span leaves) return m_tree->tryKeepTogether(leaves); } +bool ConsecutiveSets::tryKeepNestedSetTogether(std::span leaves) +{ + return m_tree->tryKeepNestedSetTogether(leaves); +} + Vector ConsecutiveSets::order(std::span keyOfLeaf) const { return m_tree->order(keyOfLeaf); diff --git a/Source/JavaScriptCore/aot/plan/AOTUnionConsecutiveSets.h b/Source/JavaScriptCore/aot/plan/AOTUnionConsecutiveSets.h index 61c16e58a2135..5f5d7b34ae95b 100644 --- a/Source/JavaScriptCore/aot/plan/AOTUnionConsecutiveSets.h +++ b/Source/JavaScriptCore/aot/plan/AOTUnionConsecutiveSets.h @@ -22,6 +22,7 @@ class ConsecutiveSets { JS_EXPORT_PRIVATE ~ConsecutiveSets(); JS_EXPORT_PRIVATE bool tryKeepTogether(std::span leaves); + JS_EXPORT_PRIVATE bool tryKeepNestedSetTogether(std::span leaves); JS_EXPORT_PRIVATE Vector order(std::span keyOfLeaf) const; private: diff --git a/Source/JavaScriptCore/aot/plan/AOTUnionShapeNumbers.cpp b/Source/JavaScriptCore/aot/plan/AOTUnionShapeNumbers.cpp index 383b915b7ff21..17c589591802b 100644 --- a/Source/JavaScriptCore/aot/plan/AOTUnionShapeNumbers.cpp +++ b/Source/JavaScriptCore/aot/plan/AOTUnionShapeNumbers.cpp @@ -303,8 +303,10 @@ class OrderOfShapesInUnionShapeNumbers { ConsecutiveSets consecutiveSets(numberOfLeaves); Vector firstRefusedSetOfLeaf(FillWith { }, numberOfLeaves, inNoRefusedSet); uint32_t numberOfRefusedSets = 0; + bool allSetsAreNested = true; for (MentionedSet& set : m_sets) { - set.isKeptTogether = consecutiveSets.tryKeepTogether(leavesOf(set)); + allSetsAreNested = allSetsAreNested && set.isFamily && consecutiveSets.tryKeepNestedSetTogether(leavesOf(set)); + set.isKeptTogether = allSetsAreNested || consecutiveSets.tryKeepTogether(leavesOf(set)); RELEASE_ASSERT(set.isKeptTogether || !set.isFamily); if (set.isKeptTogether) continue; diff --git a/Source/JavaScriptCore/aot/plan/AOTUnionSlots.cpp b/Source/JavaScriptCore/aot/plan/AOTUnionSlots.cpp index 898b60b9d33ff..d6a6813516655 100644 --- a/Source/JavaScriptCore/aot/plan/AOTUnionSlots.cpp +++ b/Source/JavaScriptCore/aot/plan/AOTUnionSlots.cpp @@ -51,6 +51,44 @@ bool remainingRowsFitInUnionSlots(const WidthCountsInUnionSlots& room, const Wid return true; } +class CountsInUnionSlots { +public: + explicit CountsInUnionSlots(size_t numberOfWords) + : m_numberOfWords(numberOfWords) + { + } + + void add(std::span positions, uint32_t amount) + { + for (unsigned lowest = 0; amount >> lowest; ++lowest) { + if (!(amount >> lowest & 1)) + continue; + for (size_t word = 0; word < positions.size(); ++word) { + uint64_t carry = positions[word]; + for (unsigned plane = lowest; carry; ++plane) { + while (m_planes.size() <= plane * m_numberOfWords) + m_planes.insertFill(m_planes.size(), 0, m_numberOfWords); + uint64_t& bits = m_planes[plane * m_numberOfWords + word]; + carry = std::exchange(bits, bits ^ carry) & carry; + } + } + } + } + + template + void forEachCount(const Functor& functor) const + { + for (size_t index = 0; index < m_planes.size(); ++index) { + for (uint64_t bits = m_planes[index]; bits; bits &= bits - 1) + functor(index % m_numberOfWords * 64 + std::countr_zero(bits), 1u << (index / m_numberOfWords)); + } + } + +private: + size_t m_numberOfWords; + Vector m_planes; +}; + class CellInUnionSlots { public: bool isInitialized() const { return m_isInitialized; } @@ -69,6 +107,9 @@ class CellInUnionSlots { m_occupied[offset / 64] |= maskOf(offset, widestSlotInUnionSlots); for (unsigned logSize = 0; logSize < numberOfWidthsInUnionSlots; ++logSize) m_room[logSize] = (limit - firstByte) >> logSize; + m_freeWidestSlots.fill(0, limit / widestSlotInUnionSlots / 64 + 1); + for (uint32_t offset = roundUpToWidestSlotInUnionSlots(firstByte); offset + widestSlotInUnionSlots <= limit; offset += widestSlotInUnionSlots) + m_freeWidestSlots[offset / widestSlotInUnionSlots / 64] |= 1ull << (offset / widestSlotInUnionSlots % 64); } uint32_t limit() const { return m_limit; } @@ -88,6 +129,25 @@ class CellInUnionSlots { return remainingRowsFitInUnionSlots(roomAfterTaking(offset, logSize), unplaced); } + void keepPlaceableOffsets(std::span words, unsigned logSize) const + { + std::array fits = fitsInLargerBlocks(logSize); + for (size_t word = 0; word < words.size(); ++word) { + uint64_t placeable = 0; + if (logSize == numberOfWidthsInUnionSlots - 1) { + if (fits[0] && word < m_freeWidestSlots.size()) + placeable = m_freeWidestSlots[word]; + } else { + for (unsigned part = 0; part < 1u << logSize; ++part) { + size_t occupied = (word << logSize) + part; + if (occupied < m_occupied.size()) + placeable |= joinFirstBytes(placeableFirstBytes(m_occupied[occupied], logSize, fits), logSize) << (part * (64 >> logSize)); + } + } + words[word] &= placeable; + } + } + void place(uint32_t offset, unsigned logSize) { RELEASE_ASSERT(canPlace(offset, logSize)); @@ -138,14 +198,64 @@ class CellInUnionSlots { return room; } + std::array fitsInLargerBlocks(unsigned logSize) const + { + std::array fits { }; + if (!m_unplaced[logSize]) + return fits; + WidthCountsInUnionSlots unplaced = m_unplaced; + --unplaced[logSize]; + WidthCountsInUnionSlots room = m_room; + for (unsigned other = 0; other <= logSize; ++other) + room[other] -= 1u << (logSize - other); + for (unsigned larger = 0; logSize + larger < numberOfWidthsInUnionSlots; ++larger) { + if (larger) + --room[logSize + larger]; + fits[larger] = remainingRowsFitInUnionSlots(room, unplaced); + } + return fits; + } + + static uint64_t placeableFirstBytes(uint64_t occupied, unsigned logSize, const std::array& fits) + { + constexpr std::array firstBytesOfBlocks { UINT64_MAX, 0x5555555555555555, 0x1111111111111111, 0x0101010101010101 }; + std::array freeBlocks; + freeBlocks[0] = ~occupied; + for (unsigned log = 1; log < numberOfWidthsInUnionSlots; ++log) + freeBlocks[log] = freeBlocks[log - 1] & (freeBlocks[log - 1] >> (1u << (log - 1))) & firstBytesOfBlocks[log]; + uint64_t placeable = 0; + uint64_t inLargerBlocks = 0; + for (unsigned larger = numberOfWidthsInUnionSlots - logSize; larger--;) { + uint64_t blocks = freeBlocks[logSize + larger]; + for (unsigned log = logSize + larger; log > logSize; --log) + blocks |= blocks << (1u << (log - 1)); + if (fits[larger]) + placeable |= blocks & ~inLargerBlocks; + inLargerBlocks |= blocks; + } + return placeable; + } + + static uint64_t joinFirstBytes(uint64_t bits, unsigned logSize) + { + for (; logSize; --logSize) { + for (unsigned shift = 1; shift < 32; shift *= 2) + bits = (bits | bits >> shift) & masksOfEveryOtherRun[std::countr_zero(shift)]; + } + return bits; + } + static constexpr std::array masksOfEveryOtherRun { 0x3333333333333333, 0x0f0f0f0f0f0f0f0f, 0x00ff00ff00ff00ff, 0x0000ffff0000ffff, 0x00000000ffffffff }; + void take(uint32_t offset, unsigned logSize) { m_room = roomAfterTaking(offset, logSize); m_occupied[offset / 64] |= maskOf(offset, 1u << logSize); + m_freeWidestSlots[offset / widestSlotInUnionSlots / 64] &= ~(1ull << (offset / widestSlotInUnionSlots % 64)); m_end = std::max(m_end, offset + (1u << logSize)); } Vector m_occupied; + Vector m_freeWidestSlots; bool m_isInitialized { false }; uint32_t m_firstByte { 0 }; uint32_t m_limit { 0 }; @@ -789,51 +899,74 @@ class PlacerOfUnionSlots { if (unplaced.size() < 2) return; unsigned logSize = m_units[unplaced[0]].logWidth; - auto offsetAt = [&](size_t candidate) { - return offsetOfFirstField + (static_cast(candidate) << logSize); + struct OffsetsOfUnit { + UnitIndex unit { 0 }; + size_t firstWord { 0 }; + size_t numberOfWords { 0 }; }; - - Vector candidatesWithinLimits; - size_t numberOfCandidates = 0; + Vector unitsLeft; + Vector fittingOffsets; + size_t mostWords = 0; for (UnitIndex unit : unplaced) { uint32_t leastLimit = UINT32_MAX; for (uint32_t row : rowsOf(unit)) leastLimit = std::min(leastLimit, cellOfRow(row).limit()); - candidatesWithinLimits.append(static_cast(std::max(offsetOfFirstField, leastLimit) - offsetOfFirstField) >> logSize); - numberOfCandidates = std::max(numberOfCandidates, candidatesWithinLimits.last()); - } - if (!numberOfCandidates) - return; - - BitVector fits(unplaced.size() * numberOfCandidates); - Vector rowsThatFit(FillWith { }, numberOfCandidates, 0u); - for (size_t index = 0; index < unplaced.size(); ++index) { - for (size_t candidate = 0; candidate < candidatesWithinLimits[index]; ++candidate) { - if (!fitsEveryRow(unplaced[index], offsetAt(candidate))) - continue; - fits.quickSet(index * numberOfCandidates + candidate); - rowsThatFit[candidate] += m_units[unplaced[index]].numberOfRows; - } + size_t numberOfOffsets = std::max(offsetOfFirstField, leastLimit) >> logSize; + size_t numberOfWords = (numberOfOffsets + 63) / 64; + size_t firstWord = fittingOffsets.size(); + fittingOffsets.insertFill(firstWord, UINT64_MAX, numberOfWords); + std::span words = fittingOffsets.mutableSpan().subspan(firstWord); + words[0] &= UINT64_MAX << (offsetOfFirstField >> logSize); + if (numberOfOffsets % 64) + words.back() &= (1ull << (numberOfOffsets % 64)) - 1; + for (uint32_t row : rowsOf(unit)) + cellOfRow(row).keepPlaceableOffsets(words, logSize); + unitsLeft.append(OffsetsOfUnit { unit, firstWord, numberOfWords }); + mostWords = std::max(mostWords, numberOfWords); } + auto countRows = [&](std::span units) { + CountsInUnionSlots counts(mostWords); + for (const OffsetsOfUnit& offsets : units) + counts.add(fittingOffsets.subspan(offsets.firstWord, offsets.numberOfWords), m_units[offsets.unit].numberOfRows); + return counts; + }; + Vector rowsThatFit(FillWith { }, mostWords * 64, 0u); + countRows(unitsLeft.span()).forEachCount([&](size_t position, uint32_t count) { + rowsThatFit[position] += count; + }); + Vector unitsPlaced; while (true) { size_t best = std::ranges::max_element(rowsThatFit) - rowsThatFit.begin(); if (rowsThatFit[best] < 2) break; - uint32_t placeAtOffset = placeInUseAt(unitsOfGroup, offsetAt(best)); - for (size_t index = 0; index < unplaced.size(); ++index) { - if (!fits.quickGet(index * numberOfCandidates + best)) - continue; - uint32_t numberOfRows = m_units[unplaced[index]].numberOfRows; - if (!tryPlace(unplaced[index], offsetAt(best), placeAtOffset)) { - fits.quickClear(index * numberOfCandidates + best); - rowsThatFit[best] -= numberOfRows; - continue; - } - for (size_t candidate = 0; candidate < candidatesWithinLimits[index]; ++candidate) { - if (fits.quickClear(index * numberOfCandidates + candidate)) - rowsThatFit[candidate] -= numberOfRows; + uint32_t offset = static_cast(best) << logSize; + uint32_t placeAtOffset = placeInUseAt(unitsOfGroup, offset); + unitsPlaced.shrink(0); + unitsLeft.removeAllMatching([&](const OffsetsOfUnit& offsets) { + if (best / 64 >= offsets.numberOfWords) + return false; + uint64_t& word = fittingOffsets[offsets.firstWord + best / 64]; + uint64_t bit = 1ull << (best % 64); + if (!(word & bit)) + return false; + if (!tryPlace(offsets.unit, offset, placeAtOffset)) { + word &= ~bit; + rowsThatFit[best] -= m_units[offsets.unit].numberOfRows; + return false; } + unitsPlaced.append(offsets); + return true; + }); + if (unitsPlaced.size() < unitsLeft.size()) { + countRows(unitsPlaced.span()).forEachCount([&](size_t position, uint32_t count) { + rowsThatFit[position] -= count; + }); + } else { + rowsThatFit.fill(0); + countRows(unitsLeft.span()).forEachCount([&](size_t position, uint32_t count) { + rowsThatFit[position] += count; + }); } } unplaced.removeAllMatching([&](UnitIndex unit) { diff --git a/Source/JavaScriptCore/aot/runtime/AOTArrayOperations.cpp b/Source/JavaScriptCore/aot/runtime/AOTArrayOperations.cpp index f5641d0fe6dc7..7d0f135350c9f 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTArrayOperations.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTArrayOperations.cpp @@ -93,25 +93,17 @@ static bool tryAppendArray(JSGlobalObject* globalObject, VM& vm, JSArray* target return target->appendMemcpy(globalObject, vm, target->length(), source); } -static NEVER_INLINE bool tryAppendArrayWithHoles(JSGlobalObject* globalObject, VM& vm, JSArray* target, JSArray* source) +static ALWAYS_INLINE bool isPlainTarget(JSArray* target) +{ + IndexingType type = target->indexingType(); + return !isCopyOnWrite(target->indexingMode()) && (hasInt32(type) || hasDouble(type) || hasContiguous(type)) && !target->structure()->outOfLineCapacity() && !target->structure()->hasElementsStartOffset(); +} + +template +static ALWAYS_INLINE bool tryFillEmptyTarget(VM& vm, JSArray* target, unsigned length, NOESCAPE const ValueAt& valueAt) { - IndexingType type = source->indexingType(); - unsigned length = source->length(); - if (!hasInt32(type) && !hasDouble(type) && !hasContiguous(type)) - return false; - if (source->structure()->holesMustForwardToPrototype(source) || !copiesIntoDenseStorage(vm.arrayStoragePolicy(), static_cast(target->length()) + length)) - return false; - auto valueAt = [&](unsigned i) { - if (hasDouble(type)) { - double number = source->butterfly()->contiguousDouble().data()[i]; - return std::isnan(number) ? jsUndefined() : JSValue(JSValue::EncodeAsDouble, number); - } - JSValue value = source->butterfly()->contiguous().data()[i].get(); - return value ? value : jsUndefined(); - }; IndexingType targetType = target->indexingType(); - bool targetIsPlain = !isCopyOnWrite(target->indexingMode()) && (hasInt32(targetType) || hasDouble(targetType) || hasContiguous(targetType)) && !target->structure()->outOfLineCapacity() && !target->structure()->hasElementsStartOffset(); - if (!target->length() && length && targetIsPlain) { + if (!target->length() && length && isPlainTarget(target)) { if (hasContiguous(targetType) && target->butterfly()->vectorLength() >= length) { Butterfly* butterfly = target->butterfly(); butterfly->setPublicLength(length); @@ -142,6 +134,27 @@ static NEVER_INLINE bool tryAppendArrayWithHoles(JSGlobalObject* globalObject, V return true; } } + return false; +} + +static NEVER_INLINE bool tryAppendArrayWithHoles(JSGlobalObject* globalObject, VM& vm, JSArray* target, JSArray* source) +{ + IndexingType type = source->indexingType(); + unsigned length = source->length(); + if (!hasInt32(type) && !hasDouble(type) && !hasContiguous(type)) + return false; + if (source->structure()->holesMustForwardToPrototype(source) || !copiesIntoDenseStorage(vm.arrayStoragePolicy(), static_cast(target->length()) + length)) + return false; + auto valueAt = [&](unsigned i) { + if (hasDouble(type)) { + double number = source->butterfly()->contiguousDouble().data()[i]; + return std::isnan(number) ? jsUndefined() : JSValue(JSValue::EncodeAsDouble, number); + } + JSValue value = source->butterfly()->contiguous().data()[i].get(); + return value ? value : jsUndefined(); + }; + if (tryFillEmptyTarget(vm, target, length, valueAt)) + return true; Vector values(length, [&](size_t i) { return JSValue::encode(valueAt(i)); }); @@ -150,6 +163,46 @@ static NEVER_INLINE bool tryAppendArrayWithHoles(JSGlobalObject* globalObject, V return target->appendMemcpy(globalObject, vm, target->length(), ArrayWithContiguous, values.span()); } +static bool tryFillWithCodePoints(Instance& instance, JSArray* target, std::span units) +{ + VM& vm = *instance.vm; + if (target->length() || !isPlainTarget(target) || !target->tryMakeWritableContiguous(vm).data() || !target->ensureLength(vm, static_cast(units.size()))) + return false; + unsigned count = 0; + for (size_t i = 0; i < units.size(); ++i) { + bool isPair = U16_IS_LEAD(units[i]) && i + 1 < units.size() && U16_IS_TRAIL(units[i + 1]); + JSString* character = isPair ? jsNontrivialString(vm, String(units.subspan(i, 2))) : stringOfCharacter(instance, units[i]); + i += isPair; + target->butterfly()->contiguous().at(target, count++).set(vm, target, character); + } + target->butterfly()->setPublicLength(count); + return true; +} + +bool tryAppendCharacters(Instance& instance, JSArray* target, JSString* string) +{ + JSGlobalObject* globalObject = instance.globalObject; + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + auto view = string->view(globalObject); + RETURN_IF_EXCEPTION(scope, false); + if (view->isEmpty()) + return true; + if (!copiesIntoDenseStorage(vm.arrayStoragePolicy(), static_cast(target->length()) + view->length())) + return false; + if (!view->is8Bit()) { + bool isFilled = tryFillWithCodePoints(instance, target, view->span16()); + ensureStillAliveHere(string); + return isFilled; + } + std::span characters = view->span8(); + bool isFilled = tryFillEmptyTarget(vm, target, static_cast(characters.size()), [&](unsigned i) { + return JSValue(vm.smallStrings.singleCharacterString(characters[i])); + }); + ensureStillAliveHere(string); + return isFilled; +} + static void appendElements(JSGlobalObject* globalObject, VM& vm, JSArray* target, JSArray* source) { auto scope = DECLARE_THROW_SCOPE(vm); diff --git a/Source/JavaScriptCore/aot/runtime/AOTAsyncResumption.h b/Source/JavaScriptCore/aot/runtime/AOTAsyncResumption.h index 0a76790780fe2..af3e7eb0ab013 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTAsyncResumption.h +++ b/Source/JavaScriptCore/aot/runtime/AOTAsyncResumption.h @@ -45,8 +45,9 @@ struct AdapterEntry { JSCell* outerAsyncGenerator { nullptr }; JSCell* outerPromiseOfHandler { nullptr }; uintptr_t outerAsyncContextIsPutBack { false }; + const uint64_t* outerFrameThatMayContinue { nullptr }; }; -static_assert(sizeof(AdapterEntry) == 4 * sizeof(void*)); +static_assert(sizeof(AdapterEntry) == 5 * sizeof(void*)); struct AsyncGeneratorBodyEntry { Instance* instance { nullptr }; AdapterEntry adapterEntry; @@ -70,6 +71,7 @@ JSValue enterSavedFrame(VM&, Instance&, SavedFrame*, Typed::ResumeMode, JSValue void resumeSavedAsyncFunction(JSGlobalObject*, VM&, Instance&, SavedFrame*, Typed::ResumeMode, JSValue resolution); void settlePromiseOfHandler(VM&, JSCell* promise, JSValue returned, JSValue error); void takeAwayRightToContinue(Instance&); +const uint64_t* frameWithAwaitingCallers(const Instance&); JS_EXPORT_PRIVATE bool isSuspendedAsyncFunction(JSCell*); JS_EXPORT_PRIVATE bool isSuspendedModuleBody(VM&, JSValue coroutine); diff --git a/Source/JavaScriptCore/aot/runtime/AOTBuiltinCells.cpp b/Source/JavaScriptCore/aot/runtime/AOTBuiltinCells.cpp index e61a47a88ac09..5e945a35761e2 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTBuiltinCells.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTBuiltinCells.cpp @@ -604,6 +604,7 @@ JSValue BuiltinCells::fillWatchedValue(JSGlobalObject* realm, BuiltinCellKey key } return value; } + m_state.immortalCells->add(holder); if (m_watchesOfValues.size() <= place) m_watchesOfValues.grow(m_values.size()); m_watchesOfValues[place] = WTF::move(watch); @@ -961,6 +962,13 @@ bool locksBuiltins(JSGlobalObject* realm) return (realm->vm().hasCompiledCode() || locksBuiltinsWithoutImage) && realm->globalObjectMethodTable()->moduleHost; } +void acceptForeignBinaryData(JSGlobalObject* realm) +{ + Instance* instance = realm->aotInstance(); + if (instance && instance->state) + instance->state->acceptsForeignBinaryData = true; +} + void lockBuiltinsWithoutImage() { locksBuiltinsWithoutImage = true; diff --git a/Source/JavaScriptCore/aot/runtime/AOTCallOperations.cpp b/Source/JavaScriptCore/aot/runtime/AOTCallOperations.cpp index bbb4c3ef4ced8..ba1bc5ac05d8b 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTCallOperations.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTCallOperations.cpp @@ -422,12 +422,7 @@ JSC_DEFINE_JIT_OPERATION(operationAOTConstructWithArray, EncodedJSValue, (Instan JSC_DEFINE_JIT_OPERATION(operationAOTSuperConstruct, EncodedJSValue, (Instance* instance, EncodedJSValue parent, EncodedJSValue newTarget, uint32_t count, const EncodedJSValue* argumentValues)) { AOT_OPERATION_BEGIN(instance, Entry::operationAOTSuperConstruct); - MarkedArgumentBuffer arguments; - fill(arguments, count, argumentValues); - if (arguments.hasOverflowed()) [[unlikely]] { - throwOutOfMemoryError(globalObject, scope); - OPERATION_RETURN(scope, encodedJSValue()); - } + ArgList arguments(const_cast(argumentValues), count); OPERATION_RETURN(scope, JSValue::encode(constructValue(globalObject, vm, JSValue::decode(parent), JSValue::decode(newTarget), arguments))); } diff --git a/Source/JavaScriptCore/aot/runtime/AOTCompiledFrameIterator.cpp b/Source/JavaScriptCore/aot/runtime/AOTCompiledFrameIterator.cpp index 0fb27792c0ca2..a2ea3f43dcfc8 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTCompiledFrameIterator.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTCompiledFrameIterator.cpp @@ -8,6 +8,7 @@ #if ENABLE(AOT) +#include "AOTAsyncResumption.h" #include "AOTBuiltinIDs.h" #include "AOTCallSiteCache.h" #include "AOTCompiledCallee.h" @@ -752,6 +753,7 @@ void CompiledFrameIterator::advanceToCallerMachineFrame() m_cursor.engineCalleeStubFrame = nullptr; m_cursor.plainFunctionConstructFrame = nullptr; m_cursor.constructFrameMakesThis = false; + m_cursor.isAmongFramesThatWait |= m_instance && m_cursor.callFrame && m_cursor.callFrame == std::bit_cast(frameWithAwaitingCallers(*m_instance)); m_calleeWasEntered = true; enterMachineFrame(findCallerFunctionFrame(m_cursor, m_exit, m_runEnd)); } @@ -1081,7 +1083,7 @@ static CallSiteCache::Entry createFunctionEntry(const Image& image, InlineFrameR FunctionDescription innermostDescription(innermost.function); entry.innermostVisibility = innermostDescription.visibility(); entry.innermostIsAsync = innermostDescription.isAsync(); - entry.namesConstructedCallee = image.constructedCalleeAt(location.function, entry.site.callSite).has_value(); + entry.namesConstructedCallee = returnSite.hasConstructedCallees && image.constructedCalleeAt(location.function, entry.site.callSite).has_value(); entry.namesTailCallingBuiltin = entry.site.calledBuiltin && leavesByTailCall(entry.site.calledBuiltin); entry.outermostVisibleFunction = findOutermostVisibleFunction(image, reader, entry, true); entry.shownFrameCount = countShownFrames(image, reader, entry.code, entry.site, ShownFrameFilter(true), entry.site.isTailCall()); diff --git a/Source/JavaScriptCore/aot/runtime/AOTCoroutineWords.h b/Source/JavaScriptCore/aot/runtime/AOTCoroutineWords.h index dba9ec3f197c8..60a2d1ba9ccb3 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTCoroutineWords.h +++ b/Source/JavaScriptCore/aot/runtime/AOTCoroutineWords.h @@ -19,6 +19,8 @@ namespace AOT { class AsyncGeneratorStepInProgress; +inline constexpr int32_t mustSuspendAtNextAwaitBit = 1; + struct CoroutineWords { uint64_t savedFrameHeader; uint64_t plainGeneratorHeader; diff --git a/Source/JavaScriptCore/aot/runtime/AOTErrorMessages.cpp b/Source/JavaScriptCore/aot/runtime/AOTErrorMessages.cpp index a08f27721c35a..2f1ff7ea05bd2 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTErrorMessages.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTErrorMessages.cpp @@ -486,6 +486,8 @@ String changeThatCompiledCodeRulesOutMessage(ObjectMutation mutation) return "make this object a prototype"_s; case ObjectMutation::StoreOfOtherType: return "store a value of this type in this property"_s; + case ObjectMutation::ConstructDirectly: + return "construct an object of this class"_s; } RELEASE_ASSERT_NOT_REACHED(); }; diff --git a/Source/JavaScriptCore/aot/runtime/AOTGrowingText.cpp b/Source/JavaScriptCore/aot/runtime/AOTGrowingText.cpp index 6c9ece1e2c045..5367544e3e1d5 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTGrowingText.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTGrowingText.cpp @@ -10,10 +10,12 @@ #include "AOTInstance.h" #include "AOTRuntimeTable.h" +#include "AOTState.h" #include "AOTVMState.h" #include "JSCInlines.h" #include "LocalAllocatorInlines.h" #include +#include #include #include #include @@ -240,16 +242,86 @@ JSString* tryAppendToGrowingText(Instance& instance, JSString* text, std::spantryGetValueImpl(); - if (is8Bit) { - text->resolveToBuffer(std::span { charactersOf(characters) + start, length }); - writePieces(characters, start + length, pieces); - } else { - text->resolveToBuffer(std::span { charactersOf(characters) + start, length }); - writePieces(characters, start + length, pieces); - } + auto writeText = [&](CharacterType* room) { + writeWhatWasAddedTo(bottom, text, room + start + length); + writeString(room + start, bottom); + writePieces(characters, start + length, pieces); + }; + if (is8Bit) + writeText(charactersOf(characters)); + else + writeText(charactersOf(characters)); return createSubstringOfBuffer(instance, vm, buffer, start, static_cast(lengthOfAll), is8Bit); } +void noteResolvedText(Instance& instance, const JSString* text) +{ + instance.state->growingFlatText.lastResolved = text->tryGetValueImpl(); +} + +template +static bool tryResolveInGrowingBuffer(Instance& instance, JSRopeString* rope, StringImpl& text, std::span pieces) +{ + constexpr bool is8Bit = std::is_same_v; + GrowingFlatText& growing = instance.state->growingFlatText; + uint32_t length = rope->length(); + uint32_t lengthOfText = text.length(); + const CharacterType* characters = charactersOf(text); + if (growing.buffer && growing.buffer->tryGetValueImpl()->is8Bit() == is8Bit) { + StringImpl& room = *growing.buffer->tryGetValueImpl(); + EndOfText end = endOfTextIn(room); + const CharacterType* start = charactersOf(room); + if (characters > start && characters + lengthOfText == start + end && length - lengthOfText <= room.length() - end) { + writePieces(room, end, pieces); + rope->resolveBySharing(StringImpl::createSubstringSharingImpl(room, static_cast(characters - start), length).get()); + return true; + } + } + uint32_t header = lengthOfBufferHeader(is8Bit); + uint64_t capacity = header + 2 * static_cast(length); + constexpr uint32_t minResolvesInTurn = 3; + growing.resolvesInTurn = growing.lastResolved == &text ? growing.resolvesInTurn + 1 : 0; + if (growing.resolvesInTurn < minResolvesInTurn || capacity > JSString::MaxLength) + return false; + JSString* buffer = tryCreateTextBuffer(*instance.vm, is8Bit, static_cast(capacity)); + if (!buffer) + return false; + StringImpl& room = *buffer->tryGetValueImpl(); + copyCharacters(charactersOf(room) + header, text); + writePieces(room, header + lengthOfText, pieces); + growing.buffer = buffer; + rope->resolveBySharing(StringImpl::createSubstringSharingImpl(room, header, length).get()); + return true; +} + +bool tryResolveInGrowingBuffer(Instance& instance, JSRopeString* rope) +{ + constexpr uint32_t minLengthPerAddition = 8; + constexpr unsigned maxRopesAddedAtOnce = 4; + bool is8Bit = rope->is8Bit(); + std::array added; + size_t firstAdded = added.size(); + StringImpl* text = nullptr; + for (const JSString* current = rope; !text; current = firstPartOf(current)) { + if (current->isSubstring() || firstAdded < 2) + return false; + for (JSString* part : { thirdPartOf(current), secondPartOf(current) }) { + if (!part) + continue; + if (is8Bit && !part->is8Bit()) + return false; + added[--firstAdded] = part; + } + text = firstPartOf(current)->tryGetValueImpl(); + } + uint32_t length = rope->length(); + uint32_t lengthOfText = text->length(); + if (text->is8Bit() != is8Bit || lengthOfText < minLengthOfGrowingText || (length - lengthOfText) > lengthOfText / minLengthPerAddition) + return false; + std::span pieces = std::span { added }.subspan(firstAdded); + return is8Bit ? tryResolveInGrowingBuffer(instance, rope, *text, pieces) : tryResolveInGrowingBuffer(instance, rope, *text, pieces); +} + bool canGrowInPlace(Instance& instance, JSString* text) { return text->isRope() && bufferThatEndsWith(instance, text); diff --git a/Source/JavaScriptCore/aot/runtime/AOTGrowingText.h b/Source/JavaScriptCore/aot/runtime/AOTGrowingText.h index 938af3706f74a..312540a1295d1 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTGrowingText.h +++ b/Source/JavaScriptCore/aot/runtime/AOTGrowingText.h @@ -33,6 +33,9 @@ ALWAYS_INLINE JSString* joinTexts(JSGlobalObject* globalObject, Instance& instan JS_EXPORT_PRIVATE bool canGrowInPlace(Instance&, JSString* text); +bool tryResolveInGrowingBuffer(Instance&, JSRopeString*); +void noteResolvedText(Instance&, const JSString*); + } } // namespace JSC::AOT #endif // ENABLE(AOT) diff --git a/Source/JavaScriptCore/aot/runtime/AOTInstance.cpp b/Source/JavaScriptCore/aot/runtime/AOTInstance.cpp index 268d256c8c511..9ca5e91fe677f 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTInstance.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTInstance.cpp @@ -41,6 +41,7 @@ #include "AOTObjectModel.h" #include "AOTOperationData.h" #include "AOTOperationScope.h" +#include "AOTPrefixCode.h" #include "AOTProgramIdentifiers.h" #include "AOTProgramStatics.h" #include "AOTProgramStrings.h" @@ -76,6 +77,7 @@ #include "JSBoundFunction.h" #include "JSCInlines.h" #include "JSCJSValueInlines.h" +#include "JSCustomGetterFunction.h" #include "JSFunction.h" #include "JSModuleLoader.h" #include "JSPromise.h" @@ -239,7 +241,8 @@ static void fillImageWords(Instance& instance, VM& vm, const Image& image, Progr words.closureFunctionKinds = image.programTables().closureFunctionKinds().data(); words.regExps = image.section(ImageSection::RegExps).data(); words.regExpCode = image.regExpCode().data(); - words.functionStarts = image.section(ImageSection::FunctionStarts).data(); + words.functionSteps = image.functionStarts().steps(); + words.functionStartGroups = image.functionStarts().groups(); #if ENABLE(AOT_POSITION_INDEPENDENT_IMAGE) words.stringRecords = image.text().stringRecords().data(); words.lazyStringHeaderOffsets = image.text().lazyHeaderOffsets(); @@ -249,7 +252,8 @@ static void fillImageWords(Instance& instance, VM& vm, const Image& image, Progr words.sizeOfProgramIdentifiers = static_cast(vm.aotState().identifiers->count()) * sizeof(UniquedStringImpl); #endif static_assert(sizeof(ImageStringWord) == sizeof(uint32_t)); - words.stringWords = image.section(ImageSection::Strings).data(); + words.endOfIdentifiers = image.header().numbers.endOfIdentifiers; + words.stringWords = std::bit_cast(std::bit_cast(image.section(ImageSection::Strings).data()) - words.endOfIdentifiers * sizeof(uint32_t)); words.stringConstants = image.section(ImageSection::StringConstants).data(); static_assert(!(InstanceLayout::largestPageSize % InstanceLayout::sizeOfIndexOfSiteAnswers) && !(InstanceLayout::distanceToPageEdge % InstanceLayout::sizeOfIndexOfSiteAnswers)); static_assert(sizeof(uint16_t) == InstanceLayout::sizeOfIndexOfSiteAnswers); @@ -511,6 +515,8 @@ Instance* Instance::create(JSGlobalObject* realm) instance->effectsNet.exercised = makeEffectsNetExercised().data(); State::create(*instance); + instance->state->caches->addClearedEachCollection("growing flat text"_s, asMutableByteSpan(instance->state->growingFlatText)); + instance->state->caches->addClearedEachCollection("wide character strings"_s, asMutableByteSpan(instance->state->wideCharacterStrings)); instance->state->caches->addClearedEachCollection("chains above lists without names"_s, asMutableByteSpan(instance->arrays.chainsAboveListsWithoutNames)); instance->state->caches->addClearedEachCollection("structures of lists without names"_s, asMutableByteSpan(instance->arrays.structuresOfListsWithoutNames)); instance->state->caches->addClearedEachCollection("copies without frame"_s, asWritableBytes(instance->copiesWithoutFrame())); @@ -849,6 +855,7 @@ static void fillThunkAndPointers(void** entries, const Image& image) set(Entry::NonConstructorHostFunction, std::bit_cast(&callHostFunctionAsConstructor)); set(Entry::ErrorConstructorClassInfo, const_cast(ErrorConstructor::info())); set(Entry::BooleanConstructorClassInfo, const_cast(BooleanConstructor::info())); + set(Entry::CustomGetterClassInfo, const_cast(JSCustomGetterFunction::info())); set(Entry::NumberConstructorClassInfo, const_cast(NumberConstructor::info())); set(Entry::StringConstructorClassInfo, const_cast(StringConstructor::info())); set(Entry::GeneratorClassInfo, const_cast(Generator::info())); @@ -1040,6 +1047,7 @@ void VMState::willShutDownHeap() if (propertyCaches) propertyCaches->front().clear(); formatPatternState = nullptr; + computedFunctionNames.clear(); } void VMState::countWithoutImage(VM& vm) @@ -1066,6 +1074,7 @@ void VMState::releaseImage(VM& vm) propertyCaches = nullptr; functionTables = nullptr; cyclesOfImports = nullptr; + importersOfModules = nullptr; reexportedNames = nullptr; callSiteCache = nullptr; runtimeTable = nullptr; @@ -1644,7 +1653,7 @@ void ProgramStatics::willAnalyzeHeap(VM& vm) m_scopesOfClosures.variablesShown.clear(); if (m_firstNamedVariablesOfModules.isEmpty()) { m_placesOfModulesOfClosureNumbers.fill(UINT32_MAX, m_tables.closureFunctions.end - m_tables.closureFunctions.first); - bool endsWhereModuleEnds = forEachNamedVariable(m_tables.namedVariables(), [&](uint32_t module, size_t) { + bool endsWhereModuleEnds = forEachNamedVariable(decodeVarintBytes(m_tables.namedVariables()).span(), [&](uint32_t module, size_t) { m_modulesInHeapSnapshots.append(module); m_firstNamedVariablesOfModules.append(static_cast(m_namedVariables.size())); }, [&](const VariableData& variable) { diff --git a/Source/JavaScriptCore/aot/runtime/AOTInstance.h b/Source/JavaScriptCore/aot/runtime/AOTInstance.h index 724ff780161f7..9e8dfc1a4f3e1 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTInstance.h +++ b/Source/JavaScriptCore/aot/runtime/AOTInstance.h @@ -157,7 +157,9 @@ struct ImageWords { const FunctionKindData* closureFunctionKinds; const ImageRegExp* regExps; const uint8_t* regExpCode; - const uint32_t* functionStarts; + const uint16_t* functionSteps; + const uint32_t* functionStartGroups; + uint64_t endOfIdentifiers; }; struct LazyStringWords { diff --git a/Source/JavaScriptCore/aot/runtime/AOTInterpreterSlowPaths.cpp b/Source/JavaScriptCore/aot/runtime/AOTInterpreterSlowPaths.cpp index 55a5e04a10b81..589db7992c463 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTInterpreterSlowPaths.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTInterpreterSlowPaths.cpp @@ -8,6 +8,7 @@ #if ENABLE(AOT) +#include "AOTArrayAllocationProfile.h" #include "AOTChainDependents.h" #include "AOTClosure.h" #include "AOTCompiledStructure.h" @@ -140,7 +141,7 @@ static JSValue readBySlot(Instance& instance, const Slot& slot, JSCell* base, co case Slot::Kind::FieldGetter: case Slot::Kind::FieldCreation: case Slot::Kind::FieldOrInherited: - case Slot::Kind::FieldOrInheritedSetter: + case Slot::Kind::FieldOrInheritedAccessor: case Slot::Kind::FieldOrInheritedField: case Slot::Kind::Variable: case Slot::Kind::ExtraGetter: @@ -485,6 +486,20 @@ void didLinkNewObject(ObjectAllocationProfile& profile) allocatorWord(profile) = ObjectLiteralPlan::notPlannedYet; } +JSArray* createEmptyArray(JSGlobalObject* globalObject, JSC::ArrayAllocationProfile& profile) +{ + constexpr unsigned vectorLength = BASE_CONTIGUOUS_VECTOR_LEN_EMPTY; + Instance* instance = interpreterInstance(globalObject); + IndexingType indexingType = profile.selectIndexingType(); + Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(hasUndecided(indexingType) ? ArrayWithInt32 : indexingType); + bool hasStorageInCell = hasInt32(structure->indexingType()) || hasContiguous(structure->indexingType()); + JSArray* array = instance && hasStorageInCell && !structure->outOfLineCapacity() ? tryCreateArrayWithStorage(globalObject->vm(), structure, 0, vectorLength) : nullptr; + if (!array) + return constructEmptyArray(globalObject, &profile); + instance->findMissingShortArrayAllocator(vectorLength); + return profile.updateLastAllocation(array); +} + void planObjectLiteral(JSGlobalObject* globalObject, CodeBlock* codeBlock, const JSInstruction* instruction, ObjectAllocationProfile& profile) { if (allocatorWord(profile) != ObjectLiteralPlan::notPlannedYet) diff --git a/Source/JavaScriptCore/aot/runtime/AOTInterpreterSlowPaths.h b/Source/JavaScriptCore/aot/runtime/AOTInterpreterSlowPaths.h index e485d8c2ccaa9..5682972896852 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTInterpreterSlowPaths.h +++ b/Source/JavaScriptCore/aot/runtime/AOTInterpreterSlowPaths.h @@ -168,6 +168,7 @@ class ObjectLiteralPlans { void didLinkNewObject(ObjectAllocationProfile&); void planObjectLiteral(JSGlobalObject*, CodeBlock*, const JSInstruction*, ObjectAllocationProfile&); +JSArray* createEmptyArray(JSGlobalObject*, JSC::ArrayAllocationProfile&); struct SymbolTableOffsets { static constexpr ptrdiff_t offsetOfMaxScopeOffset = OBJECT_OFFSETOF(SymbolTable, m_lock) - sizeof(ScopeOffset); diff --git a/Source/JavaScriptCore/aot/runtime/AOTIterationOperations.cpp b/Source/JavaScriptCore/aot/runtime/AOTIterationOperations.cpp index dec45699d8965..ca6802eda5081 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTIterationOperations.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTIterationOperations.cpp @@ -38,6 +38,7 @@ #include "JSPropertyNameEnumeratorInlines.h" #include "JSSetIterator.h" #include "JSStringIteratorInlines.h" +#include #include namespace JSC { namespace AOT { @@ -148,6 +149,7 @@ class IterationInRunTime { bool open(JSValue iterable, WordingOfIteration); bool tryOpenUnseen(JSValue iterable); bool step(JSValue&); + bool tryStepToEndUnseen(MarkedArgumentBuffer& values); void close(); private: @@ -302,6 +304,31 @@ bool IterationInRunTime::step(JSValue& value) return true; } +bool IterationInRunTime::tryStepToEndUnseen(MarkedArgumentBuffer& values) +{ + if (m_stepper != Stepper::ArrayIterator) + return false; + auto* iterator = uncheckedDowncast(m_iterator.asCell()); + auto* array = uncheckedDowncast(iterator->iteratedObject()); + IterationKind kind = iterator->kind(); + int64_t first = iterator->index(); + uint32_t end = array->length(); + if (kind == IterationKind::Entries || first == JSArrayIterator::doneIndex || first >= end) + return false; + if (kind == IterationKind::Values) { + for (uint32_t index = first; index < end; ++index) { + if (!array->canGetIndexQuickly(index)) + return false; + } + } + iterator->setIndex(JSArrayIterator::doneIndex); + values.fill(m_vm, end - first, [&](JSValue* place) { + for (uint32_t index = first; index < end; ++index) + *place++ = kind == IterationKind::Keys ? jsNumber(index) : array->getIndexQuickly(index); + }); + return true; +} + void IterationInRunTime::close() { auto scope = DECLARE_THROW_SCOPE(m_vm); @@ -393,6 +420,28 @@ static bool tryTakeOutOfTable(JSGlobalObject* globalObject, Collection* collecti return true; } +static void takeCharacters(Instance& instance, JSString* string, std::span elements) +{ + JSGlobalObject* globalObject = instance.globalObject; + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + auto view = string->view(globalObject); + RETURN_IF_EXCEPTION(scope, void()); + StringView characters = view; + unsigned at = 0; + for (EncodedJSValue& element : elements) { + if (at >= characters.length()) + break; + char16_t unit = characters[at++]; + if (U16_IS_LEAD(unit) && at < characters.length() && U16_IS_TRAIL(characters[at])) { + std::array pair { unit, characters[at++] }; + element = JSValue::encode(jsNontrivialString(vm, String(std::span { pair }))); + } else + element = JSValue::encode(stringOfCharacter(instance, unit)); + } + ensureStillAliveHere(string); +} + static void destructureIterable(Instance& instance, JSGlobalObject* globalObject, JSValue iterable, std::span elements) { VM& vm = globalObject->vm(); @@ -422,6 +471,10 @@ static void destructureIterable(Instance& instance, JSGlobalObject* globalObject isTaken = tryTakeOutOfTable(globalObject, uncheckedDowncast(cell), elements); else if (cell->type() == JSMapType) isTaken = tryTakeOutOfTable(globalObject, uncheckedDowncast(cell), elements); + else if (cell->isString() && globalObject->isStringPrototypeIteratorProtocolFastAndNonObservable()) { + takeCharacters(instance, asString(cell), elements); + isTaken = true; + } RETURN_IF_EXCEPTION(scope, void()); if (isTaken) return; @@ -566,7 +619,7 @@ JSC_DEFINE_JIT_OPERATION(operationAOTEnumeratorGetByVal, EncodedJSValue, (Instan OPERATION_RETURN(scope, JSValue::encode(objectArgument(object, entry)->get(globalObject, identifier))); } -JSC_DEFINE_JIT_OPERATION(operationAOTDestructureIterable, void, (Instance* instance, EncodedJSValue iterable, uint32_t count, EncodedJSValue* results)) +static NEVER_INLINE OperationReturnType JIT_OPERATION_ATTRIBUTES destructureAnyIterable(Instance* instance, EncodedJSValue iterable, uint32_t count, EncodedJSValue* results) { AOT_OPERATION_BEGIN(instance, Entry::operationAOTDestructureIterable); if (JSValue::decode(iterable).isUndefinedOrNull()) [[unlikely]] { @@ -577,6 +630,26 @@ JSC_DEFINE_JIT_OPERATION(operationAOTDestructureIterable, void, (Instance* insta OPERATION_RETURN(scope); } +static ALWAYS_INLINE bool tryTakeLatin1Characters(Instance& instance, JSValue iterable, std::span elements) +{ + if (!iterable.isString() || !instance.globalObject->isStringPrototypeIteratorProtocolFastAndNonObservable()) + return false; + const StringImpl* impl = asString(iterable)->tryGetValueImpl(); + if (!impl || !impl->is8Bit()) + return false; + std::span characters = impl->span8(); + for (size_t index = 0; index < elements.size(); ++index) + elements[index] = JSValue::encode(index < characters.size() ? JSValue(jsSingleCharacterString(*instance.vm, characters[index])) : jsUndefined()); + return true; +} + +JSC_DEFINE_JIT_OPERATION(operationAOTDestructureIterable, void, (Instance* instance, EncodedJSValue iterable, uint32_t count, EncodedJSValue* results)) +{ + if (tryTakeLatin1Characters(*instance, JSValue::decode(iterable), unsafeMakeSpan(results, count))) + return { nullptr }; + AOT_MUST_TAIL_CALL return destructureAnyIterable(instance, iterable, count, results); +} + JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationAOTCreateAsyncFromSyncIterator, JSCell*, (Instance* instance, JSCell* iterator, EncodedJSValue next)) { constexpr Entry entry = Entry::operationAOTCreateAsyncFromSyncIterator; @@ -673,6 +746,20 @@ static JSFunction* makeFunctionOfLibrary(JSGlobalObject* globalObject, BuiltinID } +static std::optional findIndexingTypeOfValues(JSArray* target, std::span values) +{ + bool holdsInt32Alone = true; + bool holdsNumbersAlone = true; + for (EncodedJSValue encoded : values) { + if (!encoded) + return std::nullopt; + JSValue value = JSValue::decode(encoded); + holdsInt32Alone &= value.isInt32(); + holdsNumbersAlone &= value.isNumber() && value.asNumber() == value.asNumber(); + } + return holdsNumbersAlone && hasDouble(target->indexingType()) ? ArrayWithDouble : holdsInt32Alone ? ArrayWithInt32 : ArrayWithContiguous; +} + static bool tryAppendElementsOfArguments(JSGlobalObject* globalObject, JSArray* target, ClonedArguments* arguments) { if (!arguments->isIteratorProtocolFastAndNonObservable() || !hasContiguous(arguments->indexingType())) @@ -682,17 +769,63 @@ static bool tryAppendElementsOfArguments(JSGlobalObject* globalObject, JSArray* if (length > butterfly->publicLength() || static_cast(target->length()) + length > MAX_STORAGE_VECTOR_LENGTH) return false; std::span elements(std::bit_cast(butterfly->contiguous().data()), length); - bool holdsInt32Alone = true; - bool holdsNumbersAlone = true; - for (EncodedJSValue element : elements) { - if (!element) - return false; - JSValue value = JSValue::decode(element); - holdsInt32Alone &= value.isInt32(); - holdsNumbersAlone &= value.isNumber() && value.asNumber() == value.asNumber(); + std::optional kind = findIndexingTypeOfValues(target, elements); + return kind && (!length || target->appendMemcpy(globalObject, globalObject->vm(), target->length(), *kind, elements)); +} + +template +static bool tryListElementsOfView(JSArrayBufferView* view, Vector& values) +{ + size_t length = view->length(); + if (!values.tryReserveInitialCapacity(length)) + return false; + values.grow(length); + const auto* __restrict elements = uncheckedDowncast(view)->typedVector(); + EncodedJSValue* __restrict numbers = values.mutableSpan().data(); + for (size_t index = 0; index < length; ++index) + numbers[index] = JSValue::encode(View::Adaptor::toJSValue(nullptr, elements[index])); + return true; +} + +static bool tryListNumbersOfView(JSArrayBufferView* view, Vector& values) +{ + switch (view->type()) { + case Int8ArrayType: + return tryListElementsOfView(view, values); + case Uint8ArrayType: + return tryListElementsOfView(view, values); + case Uint8ClampedArrayType: + return tryListElementsOfView(view, values); + case Int16ArrayType: + return tryListElementsOfView(view, values); + case Uint16ArrayType: + return tryListElementsOfView(view, values); + case Int32ArrayType: + return tryListElementsOfView(view, values); + case Uint32ArrayType: + return tryListElementsOfView(view, values); + case Float16ArrayType: + return tryListElementsOfView(view, values); + case Float32ArrayType: + return tryListElementsOfView(view, values); + case Float64ArrayType: + return tryListElementsOfView(view, values); + default: + return false; } - IndexingType kind = holdsNumbersAlone && hasDouble(target->indexingType()) ? ArrayWithDouble : holdsInt32Alone ? ArrayWithInt32 : ArrayWithContiguous; - return !length || target->appendMemcpy(globalObject, globalObject->vm(), target->length(), kind, elements); +} + +static bool tryAppendNumbersOfView(JSGlobalObject* globalObject, JSArray* target, JSArrayBufferView* view) +{ + if (view->isOutOfBounds() || !view->isIteratorProtocolFastAndNonObservable() || static_cast(target->length()) + view->length() > MAX_STORAGE_VECTOR_LENGTH) + return false; + Vector values; + if (!tryListNumbersOfView(view, values)) + return false; + bool holdsInt32Alone = view->type() >= Int8ArrayType && view->type() <= Int32ArrayType; + std::optional kind = holdsInt32Alone ? hasDouble(target->indexingType()) ? ArrayWithDouble : ArrayWithInt32 : findIndexingTypeOfValues(target, values.span()); + RELEASE_ASSERT(kind); + return values.isEmpty() || target->appendMemcpy(globalObject, globalObject->vm(), target->length(), *kind, values.span()); } static void appendIterable(Instance& instance, JSGlobalObject* globalObject, JSArray* target, JSValue iterable) @@ -709,6 +842,18 @@ static void appendIterable(Instance& instance, JSGlobalObject* globalObject, JSA if (isReadByIndexUnseen(iterable)) RELEASE_AND_RETURN(scope, appendElementsOfList(globalObject, target, asArray(iterable))); if (cell->isString() || isTypedArrayType(cell->type())) { + if (cell->isString() && globalObject->isStringPrototypeIteratorProtocolFastAndNonObservable()) { + bool isAppended = tryAppendCharacters(instance, target, asString(cell)); + RETURN_IF_EXCEPTION(scope, void()); + if (isAppended) + return; + } + if (isTypedArrayType(cell->type())) { + bool isAppended = tryAppendNumbersOfView(globalObject, target, uncheckedDowncast(cell)); + RETURN_IF_EXCEPTION(scope, void()); + if (isAppended) + return; + } throwIfTooLongForList(globalObject, cell); RETURN_IF_EXCEPTION(scope, void()); JSCellButterfly* listed = CommonSlowPaths::trySpreadFast(globalObject, cell); @@ -763,13 +908,12 @@ static void appendIterable(Instance& instance, JSGlobalObject* globalObject, JSA if (!isOpen) return; MarkedArgumentBuffer values; - while (true) { + for (bool hasMore = !iteration.tryStepToEndUnseen(values); hasMore;) { JSValue value; - bool hasValue = iteration.step(value); + hasMore = iteration.step(value); RETURN_IF_EXCEPTION(scope, void()); - if (!hasValue) - break; - values.append(value); + if (hasMore) + values.append(value); } if (values.hasOverflowed() || static_cast(target->length()) + values.size() > MAX_STORAGE_VECTOR_LENGTH) [[unlikely]] { throwOutOfMemoryError(globalObject, scope); @@ -778,7 +922,10 @@ static void appendIterable(Instance& instance, JSGlobalObject* globalObject, JSA if (values.isEmpty()) return; ArgList listed(values); - bool isAppended = target->appendMemcpy(globalObject, vm, target->length(), ArrayWithContiguous, std::span(listed.data(), listed.size())); + std::span elements(listed.data(), listed.size()); + std::optional kind = findIndexingTypeOfValues(target, elements); + RELEASE_ASSERT(kind); + bool isAppended = target->appendMemcpy(globalObject, vm, target->length(), *kind, elements); RETURN_IF_EXCEPTION(scope, void()); if (isAppended) return; diff --git a/Source/JavaScriptCore/aot/runtime/AOTJSONParseOperations.cpp b/Source/JavaScriptCore/aot/runtime/AOTJSONParseOperations.cpp index a9977d520e1bc..14643cde4aad2 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTJSONParseOperations.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTJSONParseOperations.cpp @@ -81,7 +81,7 @@ JSONRuntime::JSONRuntime(Instance& instance, CacheRegistry& registry, ZeroedTabl , m_learnedNames(arena) { instance.json.runtime = this; - m_hasParsedRecords = instance.state && !instance.state->tables.parsedRecordFields().empty(); + m_hasParsedRecords = instance.state && !instance.state->tables.parsedRecordShapes().empty(); registry.add(CacheKind::ClearedEachCollection, "JSON source copy"_s, *this); } @@ -122,7 +122,7 @@ auto JSONRuntime::findOrCreateRecordPlan(VM& vm, Structure* structure) -> Record if (structureOfNamesInRecord && structureOfNamesInRecord != structure && m_structuresThatNoPlanFits && m_structuresThatNoPlanFits->get(structureOfNamesInRecord)) return { }; std::optional lastNameNumber = lastNameThatProgramSpells ? identifiersOf(m_instance).numberOf(lastNameThatProgramSpells) : std::nullopt; - if (!lastNameNumber || std::ranges::equal_range(m_instance.state->tables.parsedRecordFields(), *lastNameNumber, { }, &ParsedRecordFieldData::nameNumber).empty()) + if (!lastNameNumber || m_instance.state->tables.parsedRecordShapesWithName(*lastNameNumber).empty()) return { }; unsigned inlineCapacity = structure->inlineCapacity(); diff --git a/Source/JavaScriptCore/aot/runtime/AOTJobs.cpp b/Source/JavaScriptCore/aot/runtime/AOTJobs.cpp index cfc684aaa853b..4a3db4cd8f879 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTJobs.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTJobs.cpp @@ -70,14 +70,43 @@ static ALWAYS_INLINE SavedFrame* savedFrameIn(JSValue value) } static void markStepsWhoseConsumerHasAwaited(const Instance&, const uint64_t* frame); +static JSAsyncGenerator* asyncGeneratorEnteredByStub(const Instance&, const uint64_t* frame); + +static ALWAYS_INLINE bool mustSuspendAtNextAwait(const uint64_t* frame) { return std::bit_cast(frame) & mustSuspendAtNextAwaitBit; } +static ALWAYS_INLINE const uint64_t* withoutBit(const uint64_t* frame) { return std::bit_cast(std::bit_cast(frame) & ~static_cast(mustSuspendAtNextAwaitBit)); } + +static const uint64_t* frameThatMayContinue(const Instance& instance) +{ + const uint64_t* frame = instance.coroutines.frameThatMayContinue; + return mustSuspendAtNextAwait(frame) ? nullptr : frame; +} + +static const uint64_t* withoutRightToContinue(const Instance& instance, const uint64_t* frame) +{ + const uint64_t* bare = withoutBit(frame); + if (!bare || asyncGeneratorEnteredByStub(instance, bare)) + return nullptr; + return std::bit_cast(std::bit_cast(bare) | mustSuspendAtNextAwaitBit); +} + +const uint64_t* frameWithAwaitingCallers(const Instance& instance) +{ + const uint64_t* frame = withoutBit(instance.coroutines.frameThatMayContinue); + if (!frame) [[likely]] + return nullptr; + const void* returnAddress = std::bit_cast(frame[1]); + if (returnAddress == instance.stubs.returnAddress(StubFrameKind::SavedFrameAdapter) || returnAddress == instance.stubs.returnAddress(StubFrameKind::AsyncGeneratorBody)) + return nullptr; + return frame; +} static AdapterEntry willEnterAdapter(Instance& instance, JSCell* frame, JSCell* asyncGenerator, bool asyncContextIsPutBack) { instance.vm->didEnterVM = true; if (!instance.vm->isAsyncContextTrackingEnabled()) - markStepsWhoseConsumerHasAwaited(instance, instance.coroutines.frameThatMayContinue); + markStepsWhoseConsumerHasAwaited(instance, frameThatMayContinue(instance)); CoroutineWords& coroutines = instance.coroutines; - return { std::exchange(coroutines.enteredFrame, frame), std::exchange(coroutines.enteredAsyncGenerator, asyncGenerator), std::exchange(coroutines.promiseOfEnteredHandler, nullptr), !!std::exchange(coroutines.asyncContextIsPutBack, asyncContextIsPutBack) }; + return { std::exchange(coroutines.enteredFrame, frame), std::exchange(coroutines.enteredAsyncGenerator, asyncGenerator), std::exchange(coroutines.promiseOfEnteredHandler, nullptr), !!std::exchange(coroutines.asyncContextIsPutBack, asyncContextIsPutBack), withoutRightToContinue(instance, coroutines.frameThatMayContinue) }; } struct LastEnteredByAdapter { @@ -90,6 +119,7 @@ static LastEnteredByAdapter didLeaveAdapter(Instance& instance, const AdapterEnt { CoroutineWords& coroutines = instance.coroutines; coroutines.asyncContextIsPutBack = entry.outerAsyncContextIsPutBack; + coroutines.frameThatMayContinue = entry.outerFrameThatMayContinue; return { static_cast(std::exchange(coroutines.enteredFrame, entry.outerFrame)), std::exchange(coroutines.enteredAsyncGenerator, entry.outerAsyncGenerator), std::exchange(coroutines.promiseOfEnteredHandler, entry.outerPromiseOfHandler) }; } @@ -275,7 +305,6 @@ JSAsyncGenerator* didReturnFromAsyncGeneratorBody(const AsyncGeneratorBodyEntry& auto [lastFrame, lastAsyncGenerator, promiseOfHandler] = didLeaveAdapter(instance, entry.adapterEntry); if (lastFrame && lastFrame->kind() == CoroutineKind::AsyncGenerator) return uncheckedDowncast(lastAsyncGenerator); - instance.coroutines.frameThatMayContinue = nullptr; if (isTerminated) return nullptr; if (!lastFrame) { @@ -449,7 +478,10 @@ static void markStepsWhoseConsumerHasAwaited(const Instance& instance, const uin void takeAwayRightToContinue(Instance& instance) { - markStepsWhoseConsumerHasAwaited(instance, std::exchange(instance.coroutines.frameThatMayContinue, nullptr)); + const uint64_t* frame = frameThatMayContinue(instance); + markStepsWhoseConsumerHasAwaited(instance, frame); + if (frame) + instance.coroutines.frameThatMayContinue = withoutRightToContinue(instance, frame); } static void rejectAsyncGeneratorStep(Instance& instance, const uint64_t* slotsOfStub, JSAsyncGenerator* generator, JSValue error) @@ -537,9 +569,12 @@ JSValue enterSavedFrame(VM& vm, Instance& instance, SavedFrame* saved, ResumeMod static_assert(static_cast(JSGenerator::Argument::Value) == 3 && static_cast(JSGenerator::Argument::ResumeMode) == 4); ASSERT(&instance == Instance::owning(saved)); if (!instance.vm->isAsyncContextTrackingEnabled()) - markStepsWhoseConsumerHasAwaited(instance, instance.coroutines.frameThatMayContinue); + markStepsWhoseConsumerHasAwaited(instance, frameThatMayContinue(instance)); + const uint64_t* outerFrame = withoutRightToContinue(instance, instance.coroutines.frameThatMayContinue); CallData callData = CallData::forCompiledCode(instance.state->coroutines->savedFrameEntrypoint(), instance.globalObject); - return enterCompiledCode(vm, callData, saved, jsUndefined(), saved, jsUndefined(), jsNumber(0), sent, jsNumber(static_cast(engineResumeModeOf(mode)))); + JSValue returned = enterCompiledCode(vm, callData, saved, jsUndefined(), saved, jsUndefined(), jsNumber(0), sent, jsNumber(static_cast(engineResumeModeOf(mode)))); + instance.coroutines.frameThatMayContinue = outerFrame; + return returned; } void settlePromiseOfHandler(VM& vm, JSCell* cell, JSValue returned, JSValue error) @@ -553,7 +588,6 @@ void settlePromiseOfHandler(VM& vm, JSCell* cell, JSValue returned, JSValue erro static void finishAdapterEntry(VM& vm, Instance& instance, const AdapterEntry& entry, TopExceptionScope& scope, JSValue returned) { - instance.coroutines.frameThatMayContinue = nullptr; auto [lastFrame, lastAsyncGenerator, promiseOfHandler] = didLeaveAdapter(instance, entry); JSValue error; Exception* exception = scope.exception(); diff --git a/Source/JavaScriptCore/aot/runtime/AOTModuleEvaluation.cpp b/Source/JavaScriptCore/aot/runtime/AOTModuleEvaluation.cpp index 7c8d73f400b5d..5fd4627ca9ca7 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTModuleEvaluation.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTModuleEvaluation.cpp @@ -235,6 +235,7 @@ void Modules::innerModuleEvaluation(ModuleID root, EvaluationContext& context) m_modulesAskedForWhileTheyRan.append(root); auto settleImports = makeScopeExit([&] { settleImportsOfModulesThatHaveRun(root); + resetModulesThatHaveRun(); }); EvaluationFrames frames; EvaluationInProgress evaluation { frames, m_innermostEvaluation }; @@ -804,6 +805,10 @@ void Modules::didThrowInBody(ModuleID module, JSValue error) { if (m_originalsBeingEvaluated.get(module.index())) [[unlikely]] return; + auto forgetMainModule = makeScopeExit([&] { + if (m_mainCommonJSModule == module) + forgetFailedMainModule(); + }); if (data(module).kind == ModuleKind::CommonJs && m_instance.vm->hasPendingTerminationException()) [[unlikely]] { setStatus(module, ModuleStatus::Evaluated); return; @@ -814,6 +819,8 @@ void Modules::didThrowInBody(ModuleID module, JSValue error) else { if (hasSynchronousLoadFetched(module)) ensureLoaded(module).throwAfterFetchLoad = m_outermostSynchronousLoad; + if (LoadedModule* loaded = loadedIfExists(module); loaded && !loaded->wasFetchedDuringEvaluation) + loaded->wasLoadedDuringEvaluation = false; discardFailedEvaluation(module); } return; @@ -869,6 +876,27 @@ void Modules::lookAtCycleOf(ModuleID module, ModulesThatWaitForCycles& waiting) return stored == evaluated ? markOfEvaluationThatBegan(member) : stored; }); + Vector importers; + for (uint32_t place = 0; place < members.size(); ++place) { + if (evaluationOf[place]) + importers.append(place); + } + while (!importers.isEmpty()) { + uint32_t importer = importers.takeLast(); + for (const ModuleRequestData& request : m_table.requests(ModuleID(members[importer]))) { + if (request.isRequire || request.isDeferred) + continue; + ModuleID imported = request.targetModule(); + const CyclesOfImports::Member* target = cycles.find(imported); + if (!target || target->cycle != asked->cycle || evaluationOf[target->place]) + continue; + if (m_storedStatuses[imported.index()] >= evaluated || data(imported).bodyFunction() || data(imported).evaluatesItself() || !isOnCycleOfImports(imported)) + continue; + evaluationOf[target->place] = evaluationOf[importer]; + importers.append(target->place); + } + } + struct ImportTargets { std::span requests; uint32_t importer; @@ -892,7 +920,7 @@ void Modules::lookAtCycleOf(ModuleID module, ModulesThatWaitForCycles& waiting) }); BitVector unfinishedComponents(components.numberOfComponents); for (unsigned place = 0; place < members.size(); ++place) { - if (evaluationOf[place] && m_storedStatuses[members[place]] != evaluated) + if (evaluationOf[place] && m_storedStatuses[members[place]] > evaluated) unfinishedComponents.quickSet(components.componentOf[place]); } for (unsigned place = 0; place < members.size(); ++place) { @@ -2418,18 +2446,60 @@ uint32_t Modules::countStepsOfWholeGraph(const PlanOfLoad& plan, std::span std::numeric_limits::max() - 3 - m_table.size()) { - m_placesSeenByPlans.forgetAll(m_table.size()); - m_firstPlaceOfNextPlan = 1; + return m_placesSeenByPlans.takeMark(m_table.size()); +} + +bool Modules::tryFindCompiledModulesBehindNames(ModuleID root, UncheckedKeyHashMap& modulesBehindNames) +{ + VM& vm = *m_instance.vm; + auto scope = DECLARE_TOP_EXCEPTION_SCOPE(vm); + if (!m_evictedFiles.isEmpty() || !m_modulesWithPendingMock.isEmpty() || (host().hasResolvePlugins && host().hasResolvePlugins(m_instance.globalObject))) + return false; + BitVector visited(m_table.size()); + visited.quickSet(root.index()); + Vector toFollow { root }; + while (!toFollow.isEmpty()) { + ModuleID importer = toFollow.takeLast(); + for (const ModuleRequestData& request : m_table.requests(importer)) { + ModuleID target = request.targetModule(); + if (request.isRequire || request.isHoisted || !data(target).reachesUncompiledCode()) + continue; + if (data(target).isUncompiledCode()) { + if (modulesBehindNames.contains(target)) + continue; + if (fetchParametersOf(target)) + return false; + String path = resolvedKey(target).string(); + if (scope.exception()) [[unlikely]] { + TRY_CLEAR_EXCEPTION(scope, false); + return false; + } + std::optional compiled = m_table.findByPath(path); + if (!compiled || isReplacedByHost(*compiled, LoadKind::Import) || delegatesToHost(*compiled)) + return false; + ModuleID behind = moduleWithExportsOf(fileOf(*compiled)); + if (isMocked(behind) || isMocked(fileOf(behind))) + return false; + modulesBehindNames.add(target, behind); + target = behind; + if (!data(target).reachesUncompiledCode()) + continue; + } + if (status(target) == ModuleStatus::Unlinked && !isMocked(target) && !m_modulesWithPendingMock.get(target.index()) && !visited.quickSet(target.index())) + toFollow.append(target); + } } - return m_firstPlaceOfNextPlan++; + return true; } bool Modules::planLevelsOfLoad(ModuleID root) { using Found = PlanOfLoad::Found; using Level = PlanOfLoad::Level; - if (data(root).reachesUncompiledCode() || !m_queuedFetches.isEmpty()) + if (!m_queuedFetches.isEmpty()) + return false; + UncheckedKeyHashMap modulesBehindNames; + if (data(root).reachesUncompiledCode() && !tryFindCompiledModulesBehindNames(root, modulesBehindNames)) return false; auto isInHandsOfLoader = [&](ModuleID module) { @@ -2445,14 +2515,14 @@ bool Modules::planLevelsOfLoad(ModuleID root) PlacesSeenByPlans& placesSeen = m_placesSeenByPlans; Ref plan = adoptRef(*new PlanOfLoad); auto didEndSearch = makeScopeExit([&] { - m_firstPlaceOfNextPlan = firstPlaceFound + plan->found.size(); + m_placesSeenByPlans.setNextMark(firstPlaceFound + plan->found.size()); }); auto& found = plan->found; auto& levels = plan->levels; Vector firstRequestOfPlace; Vector requestedPlaces; placesSeen[root.index()] = firstPlaceFound; - found.append(Found { root, 0, 0, false, false, false, false }); + found.append(Found { root, 0, 0, false, false, false, false, false, false }); levels.append(Level { 0, 0, 0, false }); while (true) { Level next { static_cast(found.size()), 0, 0, false }; @@ -2466,11 +2536,22 @@ bool Modules::planLevelsOfLoad(ModuleID root) if (request.isHoisted || request.isRequire) continue; ModuleID target = request.targetModule(); + bool isBehindName = data(target).isUncompiledCode(); + if (isBehindName) [[unlikely]] { + auto behind = modulesBehindNames.find(target); + if (behind == modulesBehindNames.end()) + return false; + target = behind->value; + } uint32_t& placeSeen = placesSeen[target.index()]; if (placeSeen >= isNotFound) { if (placeSeen == isNotFound) continue; requestedPlaces.append(placeSeen - firstPlaceFound); + if (isBehindName) { + found[placeSeen - firstPlaceFound].leadsToFile = true; + found[placeSeen - firstPlaceFound].isBehindName = true; + } if (placeSeen < firstPlaceOfNext) { plan->asksAcrossLevels = true; continue; @@ -2507,7 +2588,8 @@ bool Modules::planLevelsOfLoad(ModuleID root) } placeSeen = firstPlaceFound + found.size(); requestedPlaces.append(found.size()); - found.append(Found { target, importer, laterRequests, isFile, isFile && isCommonJSModuleOrFacade(target), false, false }); + bool isCommonJSFile = isFile && isCommonJSModuleOrFacade(target); + found.append(Found { target, importer, laterRequests, isFile, isBehindName || isCommonJSFile, isBehindName, isBehindName && (!isFile || isCommonJSFile), false, false }); next.hasFiles |= isFile; if (laterRequests > next.laterRequests) { next.laterRequests = laterRequests; @@ -3981,6 +4063,8 @@ ModuleID Modules::formThatRequireFinds(ModuleID target) auto first = m_formsAnsweredFirst.find(ModuleMapKey { key.impl(), ScriptFetchParameters::Type::HostDefined }); if (first != m_formsAnsweredFirst.end() && forms.contains(first->value.module) && status(first->value.module) == ModuleStatus::Evaluated && !wasEvictedByRemovedMock(first->value.module)) return first->value.module; + if (forms.contains(target) && !m_table.fullPath(target).endsWith("?json"_s) && status(target) == ModuleStatus::Evaluated && !wasEvictedByRemovedMock(target)) + return target; for (ModuleID form : forms) { if (m_table.fullPath(form).endsWith("?json"_s) && status(form) == ModuleStatus::Evaluated && !wasEvictedByRemovedMock(form)) return form; @@ -4035,6 +4119,23 @@ bool Modules::isImportResolvedByLoader(ModuleID file) const }); } +bool Modules::hasPlannedLoadComeTo(ModuleID importer) const +{ + for (const std::unique_ptr& level : m_levelsOfLoads.values()) { + if (!level->plan) + continue; + PlanOfLoad& plan = *level->plan; + uint32_t numberOfLevels = std::min(plan.levelsRequested, plan.levels.size()); + for (; plan.levelsAmongModulesComeTo < numberOfLevels; ++plan.levelsAmongModulesComeTo) { + for (const PlanOfLoad::Found& found : plan.level(plan.levelsAmongModulesComeTo)) + plan.modulesComeTo.set(found.module.index()); + } + if (plan.modulesComeTo.get(importer.index())) + return true; + } + return false; +} + bool Modules::hasBeenImported(ModuleID target) const { if (namespaceIfCreated(target, ModulePhase::Evaluation)) @@ -4048,7 +4149,16 @@ bool Modules::hasBeenImported(ModuleID target) const ModuleID importer(import.importer); if (activeMock(importer) && !wasOriginalEvaluated(importer)) return false; - return wasImporterLoaded(importer) && momentLinked(importer) >= momentMockInItsPlaceEnded; + return (wasImporterLoaded(importer) || hasPlannedLoadComeTo(importer)) && momentLinked(importer) >= momentMockInItsPlaceEnded; + }); +} + +bool Modules::isUnevaluatedTextForm(ModuleID form) const +{ + if (!m_table.fullPath(form).endsWith("?text"_s) || status(form) == ModuleStatus::Evaluated || namespaceIfCreated(form, ModulePhase::Evaluation)) + return false; + return std::ranges::none_of(m_table.importsOfHeldFile(form), [&](const ImportOfHeldFileData& import) { + return status(ModuleID(import.importer)) > ModuleStatus::Linked; }); } @@ -4283,7 +4393,7 @@ JSValue Modules::answerRequire(ModuleID target, ModuleID referrer, RequiredBy re return original; } bool isRequiredByProgram = requiredBy == RequiredBy::Program || (referrer && !data(referrer).isOfLibrary()); - if (m_mocks && (isMocked(target) || m_modulesWithPendingMock.get(target.index())) && hasFailedWithoutMock(target)) [[unlikely]] + if (m_mocks && m_modulesWithPendingMock.get(target.index()) && hasFailedWithoutMock(target)) [[unlikely]] forgetLoadThatFailed(target); if (!m_rejectedRequires.isEmpty() && (!targetData.isOfLibrary() || isRequiredByProgram) && targetData.kind == ModuleKind::SourceText && !isMocked(target) && !vm.m_synchronousModuleQueue && !m_blockedRequiresWithError.get(target.index()) && !m_isResumingFactoryChainImport && isStillBlockedOnFactories(target)) [[unlikely]] { willRejectRequireAgain(target, target); @@ -4411,7 +4521,8 @@ JSValue Modules::answerRequire(ModuleID target, ModuleID referrer, RequiredBy re if (loaded.isKeptByRequire && loaded.requiredValue && !isAnsweredAsByLoader && status(target) == ModuleStatus::Failed) [[unlikely]] return loaded.requiredValue.get(); bool wasBeingRequired = false; - if (status(target) != ModuleStatus::Evaluated) { + bool isFailedUnderMock = status(target) == ModuleStatus::Failed && isMocked(target) && !m_mocksWithoutFactory.get(target.index()); + if (status(target) != ModuleStatus::Evaluated && !isFailedUnderMock) { wasBeingRequired = loaded.isBeingRequired = status(target) != ModuleStatus::Evaluating; bool wasKeptByRequire = std::exchange(loaded.isKeptByRequire, true); evaluateSynchronously(target, SynchronousEvaluationReason::Require); @@ -4573,6 +4684,33 @@ void Modules::decideMainCommonJSModule(std::optional file) m_firstFilesOfLoads.clear(); } +bool Modules::isInRequireMap(ModuleID file) +{ + const ModuleData& fileData = data(file); + if (hasNotBegun(file) || status(file) == ModuleStatus::Failed || !(isFileOfProgram(file) || fileData.kind == ModuleKind::Data)) + return false; + if (fileData.kind == ModuleKind::CommonJs) + return true; + LoadedModule* loaded = loadedIfExists(file); + return firstRequirerOf(file) || (loaded && loaded->firstUncompiledRequirer); +} + +void Modules::forgetFailedMainModule() +{ + ModuleID failed = *std::exchange(m_mainCommonJSModule, std::nullopt); + m_failedMainModules.set(failed.index()); + m_isMainCommonJSModuleDecided = false; + for (uint32_t index = 0; index < m_table.size() && !m_isMainCommonJSModuleDecided; ++index) { + ModuleID file(index); + m_isMainCommonJSModuleDecided = file != failed && isInRequireMap(file); + } +} + +bool Modules::isMainCommonJSModule(ModuleID file) +{ + return mainCommonJSModule() == file || (status(file) == ModuleStatus::Failed && m_failedMainModules.get(file.index())); +} + void Modules::lookForMainCommonJSModule() { if (m_isMainCommonJSModuleDecided) @@ -4582,7 +4720,7 @@ void Modules::lookForMainCommonJSModule() return decideMainCommonJSModule(file); } for (ModuleID file : m_firstFilesOfLoads) { - if (!wasRequestedByLoader(file)) + if (!wasRequestedByLoader(file) || status(file) == ModuleStatus::Failed) continue; bool someFileHasModuleObject = std::ranges::any_of(m_pendingFirstRequireTargets, [&](ModuleID required) { return status(required) != ModuleStatus::Failed; @@ -4618,7 +4756,7 @@ std::optional> Modules::newModuleObjectTraits(Modul OptionSet traits; if (data(module).hasValue()) traits.add(ModuleObjectTrait::HasExportsVariable); - if (mainCommonJSModule() == module) + if (isMainCommonJSModule(module)) traits.add(ModuleObjectTrait::IsMain); return traits; } @@ -5252,10 +5390,10 @@ void Modules::willLoadCopyFromDisk(ModuleID module, LoadKind way) JSObject* Modules::parentModuleObject(ModuleID module) { LoadedModule* loaded = loadedIfExists(module); - if (loaded && loaded->firstUncompiledRequirer && mainCommonJSModule() != module) + if (loaded && loaded->firstUncompiledRequirer && !isMainCommonJSModule(module)) return loaded->firstUncompiledRequirer; std::optional firstRequirer = firstRequirerOf(module); - if (!firstRequirer || mainCommonJSModule() == module) + if (!firstRequirer || isMainCommonJSModule(module)) return nullptr; return hidesModuleObject(*firstRequirer) ? moduleObjectOrPlaceholder(*firstRequirer) : moduleObject(*firstRequirer); } diff --git a/Source/JavaScriptCore/aot/runtime/AOTModuleHost.h b/Source/JavaScriptCore/aot/runtime/AOTModuleHost.h index cdce24c4b5af4..6ba036af0d5e7 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTModuleHost.h +++ b/Source/JavaScriptCore/aot/runtime/AOTModuleHost.h @@ -135,6 +135,8 @@ JS_EXPORT_PRIVATE bool isExternalSpecifier(StringView specifier, StringView impo JS_EXPORT_PRIVATE String pathBesideFile(StringView file, StringView written); JS_EXPORT_PRIVATE String modulePath(ModuleID); String moduleFilePath(ModuleID); +#define AOT_BUILD_FOLDER_PREFIX "./" +String moduleDisplayPath(ModuleID); String sourceProviderURL(ModuleID); JS_EXPORT_PRIVATE bool entryPointIsCommonJS(JSGlobalObject*); JS_EXPORT_PRIVATE bool isESModule(ModuleID); @@ -178,6 +180,7 @@ JS_EXPORT_PRIVATE void setModuleExports(JSGlobalObject*, ModuleID, JSValue); JS_EXPORT_PRIVATE JSObject* mainModuleObject(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE void willSetMainModulePath(JSGlobalObject*); JS_EXPORT_PRIVATE JSObject* parentModuleObject(JSGlobalObject*, ModuleID); +JS_EXPORT_PRIVATE JSObject* childModuleObject(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE bool canExposeModuleObject(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE bool forEachNewChild(JSGlobalObject*, ModuleID, const ScopedLambda&); JS_EXPORT_PRIVATE void clearNewChildren(JSGlobalObject*, ModuleID); @@ -202,6 +205,7 @@ JS_EXPORT_PRIVATE bool isRetainedByInFlightLoads(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE bool isInLoadGraph(JSGlobalObject*, ModuleID root, ModuleID); enum class LoadGraphMode : uint8_t { RespectMocks, OriginalRoot, IgnoreMocks }; JS_EXPORT_PRIVATE BitVector modulesInLoadGraph(JSGlobalObject*, ModuleID root, LoadGraphMode = LoadGraphMode::RespectMocks); +JS_EXPORT_PRIVATE BitVector unfinishedModulesInLoadGraph(JSGlobalObject*, ModuleID root, LoadGraphMode, const BitVector& unsettledMocks); JS_EXPORT_PRIVATE bool isCurrentImportTargetAfterWait(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE JSValue currentImportWaitState(JSGlobalObject*); using MissingExportListener = void (*)(const String& name, const String& exporter, const String& importer); @@ -270,6 +274,7 @@ JS_EXPORT_PRIVATE bool hasPendingMock(JSGlobalObject*, ModuleID file); JS_EXPORT_PRIVATE JSObject* requireCacheEntry(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE void forEachRequireCacheKey(JSGlobalObject*, const WTF::Function&); JS_EXPORT_PRIVATE void forEachEvaluatingAsyncModule(JSGlobalObject*, const WTF::Function&); +JS_EXPORT_PRIVATE void resetModulesAfterEvaluation(JSGlobalObject*, const WTF::Function& isKept); JS_EXPORT_PRIVATE void forEachModuleWithProgramEntry(JSGlobalObject*, const WTF::Function&); JS_EXPORT_PRIVATE void forEachTLASuspendedModule(JSGlobalObject*, const WTF::Function&); JS_EXPORT_PRIVATE void forEachLibraryModuleInRequireCache(JSGlobalObject*, const WTF::Function&); @@ -292,6 +297,7 @@ JS_EXPORT_PRIVATE bool wasDisplacedDuringEvaluation(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE bool wasRegisteredByFailedLoad(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE bool wasLinkedByFailedGraph(JSGlobalObject*, ModuleID, bool byEarlierTestFile = false); JS_EXPORT_PRIVATE bool wasLinkedBeforeLinkFailure(JSGlobalObject*, ModuleID); +JS_EXPORT_PRIVATE bool isLinkedForImporterYetToRun(JSGlobalObject*, ModuleID, std::optional asking); JS_EXPORT_PRIVATE bool isFailedImportRoot(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE bool wasDisplacedAfterFailure(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE bool wasLoadedBeforeTestFile(JSGlobalObject*, ModuleID); @@ -334,6 +340,7 @@ JS_EXPORT_PRIVATE bool hasDelayedImport(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE bool isRequestedByDetachedLoad(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE void retryMockInDetachedLoads(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE bool needsRetryInDetachedLoad(JSGlobalObject*, ModuleID); +JS_EXPORT_PRIVATE uint64_t resumingLoadFactoryCall(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE void removeUnusedLingeringMocks(JSGlobalObject*); JS_EXPORT_PRIVATE bool importsStatically(JSGlobalObject*, ModuleID importer, ModuleID imported); JS_EXPORT_PRIVATE void setNeedsMockReinstall(JSGlobalObject*, ModuleID, bool waits = true); @@ -403,7 +410,7 @@ JS_EXPORT_PRIVATE bool wasRequiredUnderActiveMock(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE bool wasRequiredUnderPreloadMock(JSGlobalObject*, ModuleID file); JS_EXPORT_PRIVATE JSObject* activeMockForNamespace(JSCell* moduleNamespace); JS_EXPORT_PRIVATE bool hasMockSlot(JSGlobalObject*, ModuleID); -JS_EXPORT_PRIVATE void willMockModule(JSGlobalObject*, ModuleID, JSValue requireReturnValue, bool hasCommonJSExports = false); +JS_EXPORT_PRIVATE void willMockModule(JSGlobalObject*, ModuleID, JSValue requireReturnValue, bool hasCommonJSExports = false, bool hasFactory = true); JS_EXPORT_PRIVATE void didMockModule(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE void didStopInstallingMock(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE void mockExport(JSGlobalObject*, ModuleID, UniquedStringImpl* name, JSValue); diff --git a/Source/JavaScriptCore/aot/runtime/AOTModuleLinkWalk.h b/Source/JavaScriptCore/aot/runtime/AOTModuleLinkWalk.h index e2c4c27989489..0b55403dc8585 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTModuleLinkWalk.h +++ b/Source/JavaScriptCore/aot/runtime/AOTModuleLinkWalk.h @@ -18,13 +18,26 @@ namespace JSC { namespace AOT { +struct WalksOfOneLink final : RefCounted { + WTF_DEPRECATED_MAKE_STRUCT_FAST_ALLOCATED(WalksOfOneLink); + + explicit WalksOfOneLink(const VisitedModuleRecords& visited) + : visited(&visited) + { + } + + const VisitedModuleRecords* visited; + Vector modulesWalked; + Vector roots; +}; + class ModuleLinkWalk { WTF_MAKE_NONCOPYABLE(ModuleLinkWalk); WTF_DEPRECATED_MAKE_FAST_ALLOCATED(ModuleLinkWalk); public: enum class Leaf : uint8_t { CapturesExports, IsUncompiled }; - ModuleLinkWalk(const Modules&, ModuleID root); + ModuleLinkWalk(const Modules&, ModuleID root, RefPtr&& earlier = nullptr); ModuleID root() const { return m_root; } bool bringUpToDate(const Modules&); @@ -34,6 +47,16 @@ class ModuleLinkWalk { template bool replay(ThrowScope&, const ShouldEnterFunctor&, const VisitLeafFunctor&, const DidLeaveFunctor&) const; + void forEachModuleOfProgram(NOESCAPE const Invocable auto& functor) const + { + for (const Step& step : m_steps) { + if (static_cast(step.kind) == StepKind::Enter || static_cast(step.kind) == StepKind::CapturesExports) + functor(ModuleID(step.module)); + } + for (uint32_t module : m_modulesThatOnlyBecomeLinked) + functor(ModuleID(module)); + } + private: using StepKind = StepOfLinkWalk; @@ -48,6 +71,8 @@ class ModuleLinkWalk { void append(ModuleID, StepKind, bool resolvesNames = false); ModuleID m_root; + RefPtr m_earlier; + size_t m_numberOfEarlierRoots { 0 }; unsigned m_linkEpoch; std::span m_plan; Vector m_steps; diff --git a/Source/JavaScriptCore/aot/runtime/AOTModuleLinking.cpp b/Source/JavaScriptCore/aot/runtime/AOTModuleLinking.cpp index ce03b402dc6c6..26c001724d9a5 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTModuleLinking.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTModuleLinking.cpp @@ -351,15 +351,23 @@ ALWAYS_INLINE_INTO_OTHER_FILES void ModuleLinkWalk::append(ModuleID module, Step m_steps.unsafeAppendWithoutCapacityCheck(Step { module.index(), static_cast(kind), resolvesNames, 0 }); } -ModuleLinkWalk::ModuleLinkWalk(const Modules& modules, ModuleID root) +ModuleLinkWalk::ModuleLinkWalk(const Modules& modules, ModuleID root, RefPtr&& earlier) : m_root(root) + , m_earlier(WTF::move(earlier)) + , m_numberOfEarlierRoots(m_earlier ? m_earlier->roots.size() : 0) , m_linkEpoch(modules.linkEpoch()) { + Vector passedOver; + if (m_earlier) { + passedOver = modules.modulesSettledForLinks(); + for (size_t index = 0; index < passedOver.size(); ++index) + passedOver[index] |= m_earlier->modulesWalked[index]; + } struct World { const ModuleData& data(uint32_t module) const { return table.data(ModuleID(module)); } bool isUnlinked(uint32_t module) const { return modules.status(ModuleID(module)) == ModuleStatus::Unlinked; } bool isSettledForLinks(uint32_t module) const { return modules.isSettledForLinks(ModuleID(module)); } - std::span modulesSettledForLinks() const { return modules.modulesSettledForLinks(); } + std::span modulesSettledForLinks() const { return passedOver.empty() ? modules.modulesSettledForLinks() : passedOver; } bool loadsModulesIndividually() const { return modules.loadsModulesIndividually(); } bool goesByAllRequests(uint32_t module) const { return modules.linkGoesByAllRequestsOf(ModuleID(module)); } void didGetStateOfItsOwn(uint32_t module) const { modules.didGetStateOfItsOwn(ModuleID(module)); } @@ -370,6 +378,7 @@ ModuleLinkWalk::ModuleLinkWalk(const Modules& modules, ModuleID root) bool resolvesNames(uint32_t module) const { return !table.data(ModuleID(module)).capturesExports() && (!table.requiredNames(ModuleID(module)).empty() || !table.syntheticExports(ModuleID(module)).empty()); } const Modules& modules; const ModuleTable& table; + std::span passedOver; }; struct Steps { uint32_t size() const { return static_cast(walk.m_steps.size()); } @@ -395,7 +404,7 @@ ModuleLinkWalk::ModuleLinkWalk(const Modules& modules, ModuleID root) return; } Steps steps { *this }; - walkModulesToLink(World { modules, table }, root.index(), steps); + walkModulesToLink(World { modules, table, passedOver.span() }, root.index(), steps); m_linkEpoch = modules.linkEpoch(); } @@ -403,6 +412,10 @@ bool ModuleLinkWalk::bringUpToDate(const Modules& modules) { if (m_linkEpoch != modules.linkEpoch()) return false; + for (size_t index = 0; index < m_numberOfEarlierRoots; ++index) { + if (modules.status(m_earlier->roots[index]) == ModuleStatus::Unlinked) + return false; + } if (hasPlan()) return modules.status(m_root) == ModuleStatus::Unlinked; std::span steps = m_steps.mutableSpan(); @@ -471,37 +484,43 @@ static void reachModulesByRequests(const ModuleTable& table, const BitVector& mo } } -static Box createMockedModules(const ModuleTable& table, BitVector&& modules, const MockedModules* earlier) +static Box createMockedModules(const ModuleTable& table, const ImportersOfModules& importers, BitVector&& modules, const MockedModules* earlier, std::optional added) { - Box mocks = Box::create(); + Box mocks = Box::create(importers); mocks->modules = WTF::move(modules); - BitVector below(table.size()); Vector work; - for (size_t module : mocks->modules) - appendModulesRequestedBy(table, ModuleID(module), work); - reachModulesByRequests(table, BitVector(table.size()), WTF::move(work), below); - below.merge(mocks->modules); - mocks->positionBelow.fill(MockedModules::notBelow, table.size()); - for (size_t module : below) - mocks->positionBelow[module] = mocks->numberBelow++; - - if (earlier) - mocks->namedByListsAlone = earlier->namedByListsAlone; - else { - BitVector isAskedFor(table.size()); - for (const ModuleRequestData& request : table.allRequests()) { - if (!request.isHoisted || table.data(request.targetModule()).kind == ModuleKind::SourceText) - isAskedFor.quickSet(request.targetModule().index()); - } - mocks->namedByListsAlone = BitVector(table.size()); - for (uint32_t module = 0; module < table.size(); ++module) { - if (!isAskedFor.quickGet(module)) - mocks->namedByListsAlone.quickSet(module); - } + if (earlier && added) { + mocks->positionBelow = earlier->positionBelow; + mocks->numberBelow = earlier->numberBelow; + work.append(*added); + } else { + mocks->positionBelow.fill(MockedModules::notBelow, table.size()); + for (size_t module : mocks->modules) + work.append(ModuleID(module)); + } + while (!work.isEmpty()) { + ModuleID module = work.takeLast(); + uint32_t& position = mocks->positionBelow[module.index()]; + if (position != MockedModules::notBelow) + continue; + position = mocks->numberBelow++; + appendModulesRequestedBy(table, module, work); + } + if (earlier && added && earlier->comeToMock) { + mocks->comeToMock = earlier->comeToMock; + importers.addModulesAbove({ *added }, *mocks->comeToMock); } return mocks; } +const ImportersOfModules& Modules::importersOfModules() const +{ + std::unique_ptr& importers = m_instance.vm->aotState().importersOfModules; + if (!importers) + importers = makeUnique(m_table); + return *importers; +} + void Modules::willMock(ModuleID mocked) { forgetLoadsRefusedByCheckOfNames(mocked); @@ -522,7 +541,7 @@ void Modules::willMock(ModuleID mocked) ++m_timesMocked.add(mocked, 0u).iterator->value; BitVector modules = m_mocks ? m_mocks->modules : BitVector(m_table.size()); modules.quickSet(mocked.index()); - m_mocks = createMockedModules(m_table, WTF::move(modules), m_mocks.get()); + m_mocks = createMockedModules(m_table, importersOfModules(), WTF::move(modules), m_mocks.get(), mocked); } void Modules::didRemoveMock(ModuleID mocked) @@ -547,52 +566,18 @@ void Modules::didRemoveMock(ModuleID mocked) } for (size_t module : others) momentOfFirstMockInTestFile(ModuleID(module)); - m_mocks = createMockedModules(m_table, WTF::move(others), m_mocks.get()); -} - -static BitVector findModulesThatComeToMock(const ModuleTable& table, const BitVector& mocked) -{ - auto forEachRequestFollowed = [&](auto&& visit) { - for (uint32_t importer = 0; importer < table.size(); ++importer) { - if (mocked.quickGet(importer)) - continue; - for (const ModuleRequestData& request : table.requests(ModuleID(importer))) { - if (!request.isHoisted) - visit(importer, request.targetModule().index()); - } - } - }; - Vector boundsOfImporters; - boundsOfImporters.fill(0, table.size() + 1); - forEachRequestFollowed([&](uint32_t, uint32_t imported) { - ++boundsOfImporters[imported]; - }); - for (uint32_t module = 0; module < table.size(); ++module) - boundsOfImporters[module + 1] += boundsOfImporters[module]; - Vector importers; - importers.fill(0, boundsOfImporters[table.size()]); - forEachRequestFollowed([&](uint32_t importer, uint32_t imported) { - importers[--boundsOfImporters[imported]] = importer; - }); - - BitVector comeToMock = mocked; - Vector work; - for (size_t module : mocked) - work.append(module); - while (!work.isEmpty()) { - uint32_t imported = work.takeLast(); - for (uint32_t position = boundsOfImporters[imported]; position < boundsOfImporters[imported + 1]; ++position) { - if (!comeToMock.quickSet(importers[position])) - work.append(importers[position]); - } - } - return comeToMock; + m_mocks = createMockedModules(m_table, importersOfModules(), WTF::move(others), nullptr, std::nullopt); } bool Modules::MocksInGraph::rootComesToMock() const { - if (!m_mocks->comeToMock) - m_mocks->comeToMock = findModulesThatComeToMock(m_table, m_mocks->modules); + if (!m_mocks->comeToMock) { + Vector mocked; + for (size_t module : m_mocks->modules) + mocked.append(ModuleID(module)); + m_mocks->comeToMock = BitVector(m_table.size()); + m_mocks->importers->addModulesAbove(WTF::move(mocked), *m_mocks->comeToMock); + } return m_mocks->comeToMock->quickGet(m_root.index()); } @@ -693,30 +678,53 @@ void Modules::link(const ModuleLinkWalk& walk) void Modules::captureSyntheticExports(ModuleID root, VisitedModuleRecords& visited) { auto scope = DECLARE_THROW_SCOPE(*m_instance.vm); - std::unique_ptr walk = takeLinkWalk(root); + if (data(root).linkFollowsRequests() && status(root) == ModuleStatus::Unlinked) { + if (ModuleRecord* record = recordIfExists(root); record && visited.contains(record)) + return; + } + std::unique_ptr walk = takeKeptLinkWalk(root); + if (!walk) { + countEvent(*m_instance.vm, RuntimeCounterID::ModuleLinkWalksCreated); + RefPtr earlier = m_walksOfLink && m_walksOfLink->visited == &visited && !m_walksOfLink->roots.isEmpty() ? m_walksOfLink : nullptr; + bool leavesOutEarlierWalks = !!earlier; + walk = makeUnique(*this, root, WTF::move(earlier)); + if (leavesOutEarlierWalks && !resolvesNamesOutsideLibrary(*walk, scope)) + walk = makeUnique(*this, root); + } captureSyntheticExports(*walk, visited); RETURN_IF_EXCEPTION(scope, void()); keepLinkWalk(WTF::move(walk)); } -void Modules::captureSyntheticExports(const ModuleLinkWalk& walk, VisitedModuleRecords& visited) +bool Modules::resolvesNamesOutsideLibrary(const ModuleLinkWalk& walk, ThrowScope& scope) const { - JSGlobalObject* realm = m_instance.globalObject; - auto scope = DECLARE_THROW_SCOPE(*m_instance.vm); - bool resolvesNames = false; walk.replay(scope, [&](ModuleID module) { return status(module) == ModuleStatus::Unlinked; }, [](ModuleLinkWalk::Leaf, ModuleID) { }, [&](ModuleID module, bool moduleResolvesNames) { - resolvesNames |= moduleResolvesNames && !findsItsNamesInLibraryAlone(module); + resolvesNames = resolvesNames || (moduleResolvesNames && !findsItsNamesInLibraryAlone(module)); }); + return resolvesNames; +} +void Modules::captureSyntheticExports(const ModuleLinkWalk& walk, VisitedModuleRecords& visited) +{ + JSGlobalObject* realm = m_instance.globalObject; + auto scope = DECLARE_THROW_SCOPE(*m_instance.vm); + + bool resolvesNames = resolvesNamesOutsideLibrary(walk, scope); MocksInGraph mocks = mocksInGraph(walk.root()); bool needsPath = data(walk.root()).reachesUncompiledCode(); + RefPtr walks = m_walksOfLink; + if (walks && walks->visited != &visited) + walks = nullptr; + if (!walks && needsPath) + walks = adoptRef(*new WalksOfOneLink(visited)); + SetForScope walksOfLink(m_walksOfLink, walks); Vector path; size_t visitedPathLength = 0; - walk.replay(scope, [&](ModuleID module) { + bool didEnterRoot = walk.replay(scope, [&](ModuleID module) { if (status(module) != ModuleStatus::Unlinked) return false; ModuleRecord* record = recordIfExists(module); @@ -762,6 +770,16 @@ void Modules::captureSyntheticExports(const ModuleLinkWalk& walk, VisitedModuleR path.removeLast(); visitedPathLength = std::min(visitedPathLength, path.size()); }); + RETURN_IF_EXCEPTION(scope, void()); + if (!walks || !didEnterRoot) + return; + if (walks->modulesWalked.isEmpty()) + walks->modulesWalked.fill(0, m_modulesSettledForLinks.size()); + walk.forEachModuleOfProgram([&](ModuleID module) { + if (!mocks.hides(module)) + walks->modulesWalked[module.index() / 64] |= 1ull << (module.index() % 64); + }); + walks->roots.append(walk.root()); } void Modules::finishLink(ModuleID root, LinkContext& context) @@ -810,8 +828,15 @@ void Modules::finishLink(const ModuleLinkWalk& walk, LinkContext& context) return; KeepsAnswers keepsAnswers(*this); - checkRequiredNames(modulesThatResolveNames.span()); - RETURN_IF_EXCEPTION(scope, void()); + for (const ModuleID& module : modulesThatResolveNames) { + checkRequiredNames(std::span(&module, 1)); + if (scope.exception()) [[unlikely]] { + m_linkFailures.set(walk.root(), module); + return; + } + } + if (!m_linkFailures.isEmpty()) [[unlikely]] + m_linkFailures.remove(walk.root()); for (ModuleID module : modulesThatResolveNames) { if (!isUnlinked(module)) @@ -1396,6 +1421,17 @@ JSValue Modules::exportsBeforeMock(ModuleID file, bool evenIfEvaluationFailed, O } if (JSValue original = isCommonJSFile && activeMock(file) ? ensureLoaded(file).exportsOfOriginalBesideMock.get() : JSValue()) return original; + if (!isCommonJSFile && isMocked(file) && m_originalsEvaluatedUnderMock.get(file.index()) && !m_modulesRetainedByInFlightLoads.get(file.index()) && status(file) == ModuleStatus::Evaluated && !delegatesWithoutPlugins(file)) { + if (JSValue error = ensureLoaded(file).errorOfModuleThatRanBesideMock.get()) [[unlikely]] { + throwException(realm, scope, error); + return { }; + } + if (owedBelowOriginal == OwedBelowOriginal::IsPaid) { + evaluateDueDependencies(file); + RETURN_IF_EXCEPTION(scope, { }); + } + RELEASE_AND_RETURN(scope, originalNamespace(file)); + } if (activeMock(file) || delegatesWithoutPlugins(file)) return { }; LoadedModule* loaded = loadedIfExists(file); diff --git a/Source/JavaScriptCore/aot/runtime/AOTModuleNamespace.h b/Source/JavaScriptCore/aot/runtime/AOTModuleNamespace.h index b2b8ce4793376..5ba9b43cf9dbe 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTModuleNamespace.h +++ b/Source/JavaScriptCore/aot/runtime/AOTModuleNamespace.h @@ -47,7 +47,7 @@ class ModuleNamespace final : public JSNonFinalObject { bool isAnsweredByMock() const; JSObject* activeMock() const; void setKeepsToModule(bool keepsToModule) { m_keepsToModule = keepsToModule; } - bool keepsToModule() const { return m_keepsToModule; } + bool keepsToModule() const; void setMockOfView(VM& vm, JSObject* mock) { m_mockOfView.set(vm, this, mock); } void clearMockOfView() { m_mockOfView.clear(); } AbstractModuleRecord* recordForResolution(); @@ -60,6 +60,7 @@ class ModuleNamespace final : public JSNonFinalObject { void refuseToShowExport(PropertyName); void replaceCapturedValue(JSGlobalObject*, const Identifier& name, JSValue); + DynamicExports dynamicExports() const { return { m_dynamicExportNames.get(), m_capturedValues.get(), m_liveSources.get() }; } void replaceDynamicExports(VM&, const DynamicExports&); JSValue capturedValueWithoutAllocating(UniquedStringImpl*) const; diff --git a/Source/JavaScriptCore/aot/runtime/AOTModuleRecord.h b/Source/JavaScriptCore/aot/runtime/AOTModuleRecord.h index ba6c06c1a2f5d..fcd9b61359fdc 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTModuleRecord.h +++ b/Source/JavaScriptCore/aot/runtime/AOTModuleRecord.h @@ -115,6 +115,7 @@ class ModuleRecord final : public CyclicModuleRecord { bool isRequestedByWalkedModule() const { return m_isRequestedByWalkedModule; } void setIsRequestedByWalkedModule() { m_isRequestedByWalkedModule = true; } void setHasUnresolvableRequest() { m_hasUnresolvableRequest = true; } + bool hasUnresolvableRequest() const { return m_hasUnresolvableRequest; } uint8_t willLoadImportedModule(const Identifier& key); static BitVector willRefetchFromHost(JSGlobalObject*); static void didRefetchFromHost(JSGlobalObject*, const BitVector& savedHostRequests); @@ -167,6 +168,7 @@ class ModuleRecord final : public CyclicModuleRecord { ModuleRecord(Instance&, Structure*, JSModuleLoader*, const Identifier& moduleKey, ModuleID); Modules& modules() const; void appendDynamicStarTargets(StarExportTargets&); + void appendDynamicStarTargets(const Identifier& exportName, StarExportTargets&); bool appendStarsThatMayProvide(const Identifier& exportName, unsigned index, StarExportTargets&); AbstractModuleRecord* recordBehindStar(ModuleID); diff --git a/Source/JavaScriptCore/aot/runtime/AOTModuleTable.cpp b/Source/JavaScriptCore/aot/runtime/AOTModuleTable.cpp index a48b401cbae85..3de06f3359c33 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTModuleTable.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTModuleTable.cpp @@ -56,6 +56,48 @@ const CyclesOfImports::Member* CyclesOfImports::find(ModuleID module) const return found != membersByModule.end() && found->module == module.index() ? &*found : nullptr; } +ImportersOfModules::ImportersOfModules(const ModuleTable& table) + : namedByListsAlone(table.size()) +{ + auto forEachRequest = [&](auto&& functor) { + for (uint32_t importer = 0; importer < table.size(); ++importer) { + for (const ModuleRequestData& request : table.requests(ModuleID(importer))) { + if (!request.isHoisted) + functor(importer, request.target()); + } + } + }; + firstImporterOf.fill(0, table.size() + 1); + forEachRequest([&](uint32_t, uint32_t imported) { + ++firstImporterOf[imported]; + }); + for (uint32_t module = 0; module < table.size(); ++module) + firstImporterOf[module + 1] += firstImporterOf[module]; + importers.fill(0, firstImporterOf[table.size()]); + forEachRequest([&](uint32_t importer, uint32_t imported) { + importers[--firstImporterOf[imported]] = importer; + }); + + BitVector isAskedFor(table.size()); + for (const ModuleRequestData& request : table.allRequests()) { + if (!request.isHoisted || table.data(request.targetModule()).kind == ModuleKind::SourceText) + isAskedFor.quickSet(request.target()); + } + for (uint32_t module = 0; module < table.size(); ++module) + namedByListsAlone.quickSet(module, !isAskedFor.quickGet(module)); +} + +void ImportersOfModules::addModulesAbove(Vector&& modules, BitVector& found) const +{ + while (!modules.isEmpty()) { + uint32_t module = modules.takeLast().index(); + if (found.quickSet(module)) + continue; + for (uint32_t position = firstImporterOf[module]; position < firstImporterOf[module + 1]; ++position) + modules.append(ModuleID(importers[position])); + } +} + ReexportedNames::ReexportedNames(const ModuleTable& table) : m_answers(table.programTables().requiredNames().size()) , m_seen(table.size()) @@ -601,6 +643,24 @@ String ModuleTable::fullPath(ModuleID module) const return path.toString(); } +bool ModuleTable::hasQueryInDirectory() const +{ + return std::ranges::any_of(m_tables->moduleDirectories(), [&](const ModuleDirectoryData& directory) { + return ProgramStrings::viewInImage(*m_tables, StringID(directory.name)).contains('?'); + }); +} + +bool ModuleTable::pathWithoutQueryEndsWith(ModuleID module, StringView end, bool hasQueryInDirectory) const +{ + StringView name = ProgramStrings::viewInImage(*m_tables, StringID(data(module).publicPathOrName)); + size_t query = name.reverseFind('?'); + StringView nameWithoutQuery = name.left(query); + if (nameWithoutQuery.length() >= end.length() && (query != notFound || !hasQueryInDirectory)) + return nameWithoutQuery.endsWith(end); + String path = fullPath(module); + return StringView(path).left(path.reverseFind('?')).endsWith(end); +} + String ModuleTable::displayPathText(ModuleID module) const { uint8_t charactersInFront = data(module).charactersInFrontOfDisplayPath; diff --git a/Source/JavaScriptCore/aot/runtime/AOTModuleTable.h b/Source/JavaScriptCore/aot/runtime/AOTModuleTable.h index b1d6f1e55681b..51ae49fefb461 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTModuleTable.h +++ b/Source/JavaScriptCore/aot/runtime/AOTModuleTable.h @@ -137,6 +137,7 @@ class ModuleTable { std::optional writerOfExport(ModuleID, unsigned place) const; const ModuleFailureData& failure(ModuleFailureID) const; std::optional loadFailure(ModuleID) const; + bool hasLoadFailures() const { return !m_tables->moduleFailures().empty() && m_tables->moduleFailures().back().isLoadOfModule(); } std::optional findByPath(StringView publicPath) const { return findByPath(publicPath, publicPath.hash()); } std::optional findByPath(StringView publicPath, unsigned hashOfPath) const; @@ -156,6 +157,8 @@ class ModuleTable { bool isRuntimeFunction(FunctionID) const; String fullPath(ModuleID) const; + bool hasQueryInDirectory() const; + bool pathWithoutQueryEndsWith(ModuleID, StringView end, bool hasQueryInDirectory) const; String displayPathText(ModuleID) const; String pathTextForLineColumns(ModuleID) const; bool hasFilePath(ModuleID module) const { return data(module).hasExportTable() || data(module).kind == ModuleKind::CommonJs; } @@ -195,6 +198,18 @@ struct CyclesOfImports { Vector firstMemberOf; }; +struct ImportersOfModules { + WTF_DEPRECATED_MAKE_STRUCT_FAST_ALLOCATED(ImportersOfModules); + + explicit ImportersOfModules(const ModuleTable&); + + void addModulesAbove(Vector&&, BitVector& found) const; + + Vector importers; + Vector firstImporterOf; + BitVector namedByListsAlone; +}; + struct ReexportedNames { WTF_DEPRECATED_MAKE_STRUCT_FAST_ALLOCATED(ReexportedNames); diff --git a/Source/JavaScriptCore/aot/runtime/AOTModules.cpp b/Source/JavaScriptCore/aot/runtime/AOTModules.cpp index d7d833fccd817..5acd6bf49bda1 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTModules.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTModules.cpp @@ -147,6 +147,7 @@ Modules::Modules(State& state) , m_loadsModulesByName(state.instance.vm->aotState().image->loadsModulesByName()) , m_refusesOriginalsBesideMocks(state.instance.vm->aotState().image->refusesOriginalsBesideMocks()) , m_showsEntriesOfRequireCache(state.instance.vm->aotState().image->showsEntriesOfRequireCache()) + , m_placesSeenByPlans(state.instance.vm->aotState().placesSeenByPlans) , m_mockOperations(host().mockOperations ? host().mockOperations() : nullptr) { m_instance.modules.status = state.tableArena.allocate(m_table.size()); @@ -493,7 +494,7 @@ void Modules::resetFailedEvaluations() { Vector failed; for (ModuleID module : m_records.keys()) { - if (hasResettablePackageFailure(module)) + if (hasResettablePackageFailure(module) && !(isMocked(module) && hasFailedWithoutMock(module))) failed.append(module); } for (ModuleID module : failed) @@ -790,6 +791,7 @@ bool Modules::isRequiredModulePatchPending(ModuleID module) bool Modules::delegatesToHost(ModuleID module) { + resetModulesThatHaveRun(); if (m_evictedFiles.isEmpty() && !m_table.hasSourceStamps()) return false; if (data(module).isUncompiledCode()) @@ -1018,6 +1020,8 @@ void Modules::setFailed(ModuleID module, JSValue error) record->retain(); record->setEvaluationError(*m_instance.vm, error); record->setStatus(CyclicModuleRecord::Status::Evaluated); + if (m_mainCommonJSModule == fileOf(module)) + forgetFailedMainModule(); } void Modules::throwEvaluationError(ModuleID module) @@ -1746,6 +1750,40 @@ auto Modules::starExportProviders(ModuleID module) const -> const Vectorvalue; } +auto Modules::starTargetsByName(ModuleID module) const -> const StarTargetsByName& +{ + std::unique_ptr& kept = m_starTargetsByName.add(module, nullptr).iterator->value; + if (kept) + return *kept; + auto stars = makeUnique(); + std::span asWritten = m_table.starsAsWritten(module); + for (const StarAsWrittenData& star : asWritten) + stars->targets.append(StarTarget { ModuleID(star.target), !!star.isFollowed }); + if (asWritten.empty()) { + for (ModuleRequestData request : m_table.dynamicStars(module)) + stars->targets.append(StarTarget { request.targetModule() }); + } + for (unsigned place = 0; place < stars->targets.size(); ++place) { + StarTarget star = stars->targets[place]; + bool hasTable = data(star.module).hasExportTable(); + bool listsItsNames = hasTable && data(star.module).isESModuleOfProgram() && !data(star.module).capturesExports() && !data(star.module).hasDynamicStarExport() && !data(star.module).mayBeMocked() && m_table.ambiguousExports(star.module).empty() + && std::ranges::none_of(m_table.requiredNames(star.module), [&](const RequiredNameData& requiredName) { + return !!m_table.findExportStatement(requiredName); + }); + if (star.isFollowed && !listsItsNames) + stars->targetsWithUnlistedNames.append(place); + if (!hasTable) + continue; + for (const ModuleExportData& entry : m_table.exports(star.module)) { + Vector& places = stars->targetsListingName.add(identifier(entry.name).impl(), Vector()).iterator->value; + if (places.isEmpty() || places.last() != place) + places.append(place); + } + } + kept = WTF::move(stars); + return *kept; +} + std::span Modules::requiredNamesOfStatementsThatExport(ModuleID module, const String& exportName) const { auto& byExportName = m_requiredNamesOfStatementsByExportName.ensure(module, [&] { @@ -1761,9 +1799,91 @@ std::span Modules::requiredNamesOfStatementsThatExport(ModuleID return found == byExportName.end() ? std::span() : found->value.span(); } +auto Modules::modulesOnReexportWay(ModuleID reexportingModule, unsigned indexOfReexport) const -> const Vector& +{ + return m_reexportWays.ensure(static_cast(reexportingModule.index()) << 32 | indexOfReexport, [&] { + const ModuleExportData& reexport = m_table.exports(reexportingModule)[indexOfReexport]; + Vector asked; + auto indexOfExport = [&](ModuleID module, uint32_t name) -> std::optional { + return data(module).hasExportTable() ? m_table.findExport(module, identifier(name).impl()) : std::nullopt; + }; + if (reexport.kind() == ModuleExportKind::Indirect && !reexport.isFromStarExport) { + for (uint32_t place : requiredNamesOfStatementsThatExport(reexportingModule, identifier(reexport.name).string())) + asked.append(m_table.requiredNames(reexportingModule)[place].exporterModule()); + return asked; + } + if (reexport.kind() == ModuleExportKind::Indirect) { + std::optional first = indexOfExport(reexportingModule, reexport.name); + if (!first) + return asked; + BitVector isSeen(m_table.size()); + isSeen.quickSet(reexportingModule.index()); + Vector toFollow; + auto follow = [&](ModuleID holder, unsigned index) { + const Vector& providers = starExportProviders(holder)[index]; + for (size_t next = providers.size(); next--;) + toFollow.append(providers[next]); + }; + follow(reexportingModule, *first); + while (!toFollow.isEmpty()) { + PlaceOfExport next = toFollow.takeLast(); + asked.append(next.module); + if (isSeen.quickSet(next.module.index())) + continue; + if (!m_table.exports(next.module)[next.index].isFromStarExport) + break; + follow(next.module, next.index); + } + return asked; + } + UncheckedKeyHashSet, WTF::UnsignedWithZeroKeyHashTraits> seen; + struct RequestedName { + ModuleID module; + uint32_t name; + }; + Vector toAsk; + toAsk.append(RequestedName { reexportingModule, reexport.name }); + while (!toAsk.isEmpty()) { + RequestedName next = toAsk.takeLast(); + if (!seen.add(static_cast(next.module.index()) << 32 | next.name).isNewEntry) + continue; + asked.append(next.module); + std::span requiredNames = m_table.requiredNames(next.module); + for (uint32_t place : requiredNamesOfStatementsThatExport(next.module, identifier(next.name).string())) + toAsk.append(RequestedName { requiredNames[place].exporterModule(), requiredNames[place].name }); + } + if (reexport.kind() != ModuleExportKind::Variable) + return asked; + if (!m_modulesWithSyntheticExport) { + m_modulesWithSyntheticExport.emplace(); + for (uint32_t index = 0; index < m_table.size(); ++index) { + for (const SyntheticExportData& entry : m_table.syntheticExports(ModuleID(index))) { + if (entry.hasVariable) + m_modulesWithSyntheticExport->add(std::bit_cast(entry.variable), Vector()).iterator->value.append(ModuleID(index)); + } + } + } + if (auto found = m_modulesWithSyntheticExport->find(std::bit_cast(reexport.variable)); found != m_modulesWithSyntheticExport->end()) + asked.appendVector(found->value); + return asked; + }).iterator->value; +} + +bool Modules::mayMockAnswerReexport(ModuleID reexportingModule, const ModuleExportData& reexportEntry) const +{ + if (!m_removedMocksInUse.isEmpty()) + return true; + std::span entries = m_table.exports(reexportingModule); + if (&reexportEntry < entries.data() || &reexportEntry >= entries.data() + entries.size()) + return true; + return std::ranges::any_of(modulesOnReexportWay(reexportingModule, static_cast(&reexportEntry - entries.data())), [&](ModuleID module) { + return m_modulesWithActiveMock.get(module.index()) || m_modulesWithActiveMock.get(fileOf(module).index()); + }); +} + auto Modules::findMockForReexportSlow(ModuleID reexportingModule, const ModuleExportData& reexportEntry, MockMode reexportingModuleMockMode) const -> std::optional { - if (!someMockIsSeen()) + if (!someMockIsSeen() || !mayMockAnswerReexport(reexportingModule, reexportEntry)) return std::nullopt; BitVector modulesWithMock = m_modulesWithActiveMock; for (ModuleID mocked : m_removedMocksInUse.keys()) @@ -2641,7 +2761,14 @@ bool Modules::setMockWithIntactOriginal(ModuleID module, JSObject* mock, JSValue if (auto* filed = dynamicDowncast(object)) filed->leaveStructureThatCachesKnow(*m_instance.vm); } - if (!made || !data(each).hasDynamicStarExport() || !reaching.quickGet(index)) + if (!data(each).hasDynamicStarExport() || !reaching.quickGet(index)) + continue; + if (JSObject* unfilled = other ? other->namespaceToRestore.get() : nullptr) { + other->namespaceToRestore.set(*this, nullptr); + if (!made && !setAside && !activeMock(each)) + other->namespaceBeforeMockInPlace.set(*this, unfilled); + } + if (!made) continue; LoadedModule& behindStars = ensureLoaded(each); if (!behindStars.namespaceBeforeMockInPlace && !activeMock(each)) @@ -2690,6 +2817,32 @@ bool Modules::setMockWithIntactOriginal(ModuleID module, JSObject* mock, JSValue return true; } +bool Modules::reachesStarCycle(ModuleID root) const +{ + BitVector isFinished(m_table.size()); + BitVector isOnPath(m_table.size()); + Vector, 8> path { { root, 0 } }; + isOnPath.quickSet(root.index()); + while (!path.isEmpty()) { + auto [module, place] = path.last(); + DynamicStars stars = m_table.dynamicStars(module); + if (place == stars.numberOfPlaces()) { + isOnPath.quickClear(module.index()); + isFinished.quickSet(module.index()); + path.removeLast(); + continue; + } + ++path.last().second; + std::optional request = stars.at(place); + if (!request || isFinished.quickGet(request->target())) + continue; + if (isOnPath.quickSet(request->target())) + return true; + path.append({ request->targetModule(), 0 }); + } + return false; +} + void Modules::removeMockWithIntactOriginal(ModuleID module) { auto scope = DECLARE_THROW_SCOPE(*m_instance.vm); @@ -2788,7 +2941,8 @@ void Modules::removeMockWithIntactOriginal(ModuleID module) LoadedModule* behindStars = loadedIfExists(each); if (JSObject* before = behindStars ? behindStars->namespaceBeforeMockInPlace.get() : nullptr) { behindStars->namespaceBeforeMockInPlace.set(*this, nullptr); - setNamespace(each, ModulePhase::Evaluation, before); + if (!reachesStarCycle(each)) + behindStars->namespaceToRestore.set(*this, before); } } if (JSObject* own = exporting.namespaceBeforeMockInPlace.get()) { @@ -2952,6 +3106,7 @@ JSValue Modules::capturedValue(ModuleID module, const Identifier& name) { JSGlobalObject* realm = m_instance.globalObject; auto scope = DECLARE_THROW_SCOPE(*m_instance.vm); + SetForScope isBypassingMocks(m_isBypassingMocks, false); captureExports(module); RETURN_IF_EXCEPTION(scope, { }); LoadedModule& loaded = ensureLoaded(module); @@ -3023,11 +3178,14 @@ JSObject* Modules::namespaceIfCreated(ModuleID module, ModulePhase phase) const void Modules::discardNamespace(ModuleID module) { + LoadedModule* loaded = loadedIfExists(module); + if (loaded && loaded->namespaceToRestore) + loaded->namespaceToRestore.set(*this, nullptr); JSObject* made = namespaceIfCreated(module, ModulePhase::Evaluation); if (!made) return; setNamespace(module, ModulePhase::Evaluation, nullptr); - LoadedModule* loaded = loadedIfExists(module); + loaded = loadedIfExists(module); if (!loaded || loaded->requiredValue.get() != JSValue(made)) return; loaded->requiredValue.set(*this, JSValue()); @@ -3038,7 +3196,7 @@ void Modules::discardNamespace(ModuleID module) void Modules::setNamespace(ModuleID module, ModulePhase phase, JSObject* namespaceObject) { if (phase == ModulePhase::Evaluation && data(module).holdsNamespaceInValue() && !m_modulesWithEarlierNamespaceInValue.get(module.index())) - setValue(module, namespaceObject); + ProgramStatics::writeNamespace(m_instance, data(module).value, namespaceObject); else ensureLoaded(module).namespaceObjectFor(phase).set(*this, namespaceObject); } @@ -3084,7 +3242,15 @@ JSObject* Modules::createNamespace(ModuleID module, ModulePhase phase, bool shou bool isRequired = phase == ModulePhase::Evaluation && m_modulesMarkedAsRequired.get(module.index()); if (phase == ModulePhase::Evaluation && m_modulesWithExtensibleNamespace.get(module.index())) shouldPreventExtensions = false; - ModuleNamespace* namespaceObject = ModuleNamespace::create(vm, namespaceStructure(shouldPreventExtensions), module, phase, { }); + LoadedModule* loadedBeforeMock = phase == ModulePhase::Evaluation ? loadedIfExists(module) : nullptr; + auto* restored = loadedBeforeMock ? dynamicDowncast(loadedBeforeMock->namespaceToRestore.get()) : nullptr; + ModuleNamespace::DynamicExports exportsBefore; + if (restored) [[unlikely]] { + loadedBeforeMock->namespaceToRestore.set(*this, nullptr); + exportsBefore = restored->dynamicExports(); + restored->replaceDynamicExports(vm, { }); + } + ModuleNamespace* namespaceObject = restored ? restored : ModuleNamespace::create(vm, namespaceStructure(shouldPreventExtensions), module, phase, { }); size_t place = m_namespacesBeingMade.size(); size_t outermostReadBefore = std::exchange(m_outermostNamespaceReadWhileBeingMade, notFound); auto forgetsNamespacesThatLackNames = makeScopeExit([&] { @@ -3107,7 +3273,13 @@ JSObject* Modules::createNamespace(ModuleID module, ModulePhase phase, bool shou size_t outermostRead = m_outermostNamespaceReadWhileBeingMade; bool lacksNames = phase == ModulePhase::Evaluation && outermostRead < place; m_outermostNamespaceReadWhileBeingMade = std::min(outermostReadBefore, outermostRead); - RETURN_IF_EXCEPTION(scope, nullptr); + if (scope.exception()) [[unlikely]] { + if (restored) { + restored->replaceDynamicExports(vm, exportsBefore); + ensureLoaded(module).namespaceToRestore.set(*this, restored); + } + return nullptr; + } namespaceObject->replaceDynamicExports(vm, exportObjects); if (isRequired && !namespaceObject->hasExport(vm.propertyNames->__esModule.impl())) { if (m_modulesWithDeferredMark.get(module.index())) @@ -5268,6 +5440,8 @@ void Modules::appendRequestsForLoader(ModuleID module, ModuleRecord* record, Mod std::unique_ptr level = m_levelsOfLoads.take(module); if (!level) return false; + if (level->plan) + level->plan->levelsRequested = std::max(level->plan->levelsRequested, level->depth + 1); if (level->next && level->plan && (!m_levelsOfLoads.contains(*level->next) || !isStillToBeLoaded(*level->next))) layWayOfLoadAgain(*level); if (level->next && level->plan && !m_instance.vm->synchronousModuleQueue()) @@ -5276,20 +5450,28 @@ void Modules::appendRequestsForLoader(ModuleID module, ModuleRecord* record, Mod for (const PlanOfLoad::Found& found : level->plan->level(level->depth + 1)) noteRequestByLoader(found.module); } - auto append = [&](ModuleID target) { + PlanOfLoad& plan = *level->plan; + auto append = [&](ModuleID target, bool isFetchedByHost) { Identifier key = keyOfRequest(target); record->appendRequestedModule(key, nullptr, ModulePhase::Evaluation); - record->didRequestCompiledModule(key, target); + if (!isFetchedByHost) + record->didRequestCompiledModule(key, target); }; bool isNextLeft = level->next.has_value(); auto appendNext = [&] { isNextLeft = false; - if (auto asked = m_levelsOfLoads.find(*level->next); asked != m_levelsOfLoads.end()) + bool isFetchedByHost = false; + if (auto asked = m_levelsOfLoads.find(*level->next); asked != m_levelsOfLoads.end()) { asked->value->isAskedFor = true; - append(*level->next); + isFetchedByHost = asked->value->plan == &plan && plan.found[asked->value->place].isFetchedByHost; + } + append(*level->next, isFetchedByHost); }; - PlanOfLoad& plan = *level->plan; - Vector listed; + struct Listed { + ModuleID module; + bool isFetchedByHost; + }; + Vector listed; if (level->depth + 1 < plan.levels.size()) { std::span below = plan.level(level->depth + 1); for (PlanOfLoad::Found& imported : below.subspan(std::ranges::lower_bound(below, level->place, { }, &PlanOfLoad::Found::importer) - below.begin())) { @@ -5297,7 +5479,12 @@ void Modules::appendRequestsForLoader(ModuleID module, ModuleRecord* record, Mod break; if (level->next != imported.module && (!imported.leadsToFile || !isLeftToLoad(plan, imported))) continue; - listed.append(imported.module); + listed.append({ imported.module, static_cast(imported.isFetchedByHost) }); + if (imported.isBehindName && isCommonJSModuleOrFacade(imported.module)) { + noteRequestByLoader(imported.module); + if (!m_isMainCommonJSModuleDecided) + m_firstFilesOfLoads.append(fileOf(imported.module)); + } if (imported.isFile || level->next == imported.module) continue; ModuleRecord* known = recordIfExists(imported.module); @@ -5312,19 +5499,23 @@ void Modules::appendRequestsForLoader(ModuleID module, ModuleRecord* record, Mod }).iterator->value->isAskedFor = true; } } - for (ModuleID imported : listed) { + for (auto [imported, isFetchedByHost] : listed) { + if (isFetchedByHost && isCommonJSModuleOrFacade(imported)) { + noteLoadOfFile(imported, loader()->registryEntry(keyOfRequest(imported)) ? LoadOfFile::HasComeToFile : LoadOfFile::IsRequested); + RETURN_IF_EXCEPTION(scope, true); + } if (level->next == imported) { appendNext(); continue; } - if (isCommonJSModuleOrFacade(imported)) { + if (isCommonJSModuleOrFacade(imported) && !isFetchedByHost) { m_filesLoadedWithoutLoader.set(imported.index()); if (status(imported) == ModuleStatus::Evaluated) continue; noteLoadOfFile(imported); RETURN_IF_EXCEPTION(scope, true); } - append(imported); + append(imported, isFetchedByHost); } if (isNextLeft) appendNext(); @@ -5333,9 +5524,25 @@ void Modules::appendRequestsForLoader(ModuleID module, ModuleRecord* record, Mod if (!m_levelsOfLoads.isEmpty() && appendNextOnWay()) return; if (!record->isRequestedByWalkedModule()) { + if (m_table.hasLoadFailures()) [[unlikely]] { + if (std::optional failure = findUnmockedFileFailure(module)) + RELEASE_AND_RETURN(scope, appendUnresolvableRequest(*failure, record)); + } bool mustWaitForFile = importsFileBeingEvaluated(module); bool endsAtStatement = statementThatEndsLoad(module).has_value(); bool isWalked = mustWaitForFile || endsAtStatement || canTellHowGraphIsLoaded(module); + bool isPlanned = !isWalked && data(module).reachesUncompiledCode() && !isInTransferredSynchronousLoad() && planLevelsOfLoad(module); + RETURN_IF_EXCEPTION(scope, void()); + if (isPlanned) { + if (!m_instance.vm->synchronousModuleQueue() && !m_loadsInFlight.hasRoot(module)) + didStartLoad(load.promise(), module, SettlingOfLoad::IsLookedFor); + m_loadsInFlight.forEachOfRoot(module, [](LoadInFlight& load) { + load.isWalked = true; + load.hasMarksBelowRoot = true; + }); + appendNextOnWay(); + return; + } if (!isWalked) { evaluateImportedCommonJSModules(module, EagerLoadJobPolicy::DeferJobs, ErrorOfImportedFile::IsLeftToLoader, &load); RETURN_IF_EXCEPTION(scope, void()); @@ -5353,7 +5560,9 @@ void Modules::appendRequestsForLoader(ModuleID module, ModuleRecord* record, Mod m_loadsInFlight.forEachOfRoot(module, [](LoadInFlight& load) { load.isWalked = true; }); - if (!data(module).isLoadedByLoader() && m_rootAwaitingLoader != module && !mustWaitForFile && !endsAtStatement && !isInTransferredSynchronousLoad() && planLevelsOfLoad(module)) { + isPlanned = !data(module).isLoadedByLoader() && m_rootAwaitingLoader != module && !mustWaitForFile && !endsAtStatement && !isInTransferredSynchronousLoad() && planLevelsOfLoad(module); + RETURN_IF_EXCEPTION(scope, void()); + if (isPlanned) { appendNextOnWay(); return; } @@ -5412,6 +5621,33 @@ void Modules::appendRequestsForLoader(ModuleID module, ModuleRecord* record, Mod RELEASE_AND_RETURN(scope, appendUnresolvableRequest(*statement, record)); } +std::optional Modules::findUnmockedFileFailure(ModuleID root) const +{ + std::optional failureOfRoot = loadFailure(root); + BitVector seen(m_table.size()); + Vector work { root }; + while (!work.isEmpty()) { + ModuleID module = work.takeLast(); + if (seen.quickSet(module.index()) || !data(module).isESModuleOfProgram()) + continue; + if (module != root) { + bool hasBegun = status(module) >= ModuleStatus::Evaluating && status(module) <= ModuleStatus::Evaluated; + if (hasBegun || isMocked(module) || activeMock(module)) + continue; + std::optional failure = loadFailure(module); + if (failure && failure != failureOfRoot && data(module).mayBeMocked() && m_table.failure(*failure).importerModule() == module && !m_table.failure(*failure).isAbsentExport() && !statementThatEndsLoad(module)) + return failure; + } + size_t firstBelow = work.size(); + for (const ModuleRequestData& request : m_table.imports(module)) { + if (!request.isHoisted && !request.isDeferred) + work.append(request.targetModule()); + } + std::ranges::reverse(work.mutableSpan().subspan(firstBelow)); + } + return std::nullopt; +} + std::optional Modules::statementThatEndsLoad(ModuleID module) const { std::optional endOfLoad = m_table.loadFailure(module); @@ -5688,6 +5924,13 @@ JSC_DEFINE_JIT_OPERATION(operationAOTRequireIfImported, EncodedJSValue, (Instanc Modules& modules = modulesOf(*instance); if (!modules.hasBeenImported(ModuleID(module)) || isLoadedWithoutType(modules, ModuleID(module))) OPERATION_RETURN(scope, JSValue::encode(jsUndefined())); + if (modules.isUnevaluatedTextForm(ModuleID(module))) [[unlikely]] { + String path = modules.table().fullPath(ModuleID(module)); + if (std::optional file = modules.table().findByPath(StringView(path).left(path.reverseFind('?'))); file && modules.data(*file).kind == ModuleKind::Data && modules.data(*file).requireResult == RequireResult::NamespaceIfImported) { + modules.moduleNamespace(*file, Modules::ModulePhase::Evaluation); + OPERATION_RETURN(scope, JSValue::encode(jsUndefined())); + } + } OPERATION_RETURN(scope, JSValue::encode(modules.require(ModuleID(module), ModuleID(referrer)))); } @@ -5964,6 +6207,7 @@ bool isExternalSpecifier(StringView specifier, StringView importerDirectory) struct PathOfModule { StringImpl* path; StringImpl* pathOfFile; + StringImpl* displayPath; bool isEvaluatedText; }; @@ -6007,7 +6251,12 @@ static PathOfModule pathOfModule(ModuleID module) }; StringImpl* kept = keep(path); String pathOfFile = evaluator || table->data(module).registryID() ? path : withoutQuery(*table, module, path); - return PathOfModule { kept, pathOfFile.length() == path.length() ? kept : keep(pathOfFile), !!evaluator }; + StringImpl* keptPathOfFile = pathOfFile.length() == path.length() ? kept : keep(pathOfFile); + String displayPath = evaluator ? String() : table->displayPathText(module); + bool isRelativeToFolderOfBuild = !displayPath.isEmpty() && displayPath != pathOfFile; + if (isRelativeToFolderOfBuild && !displayPath.startsWith("../"_s)) + displayPath = makeString(AOT_BUILD_FOLDER_PREFIX ""_s, displayPath); + return PathOfModule { kept, keptPathOfFile, isRelativeToFolderOfBuild ? keep(displayPath) : keptPathOfFile, !!evaluator }; }).iterator->value; } @@ -6021,6 +6270,11 @@ String moduleFilePath(ModuleID module) return pathOfModule(module).pathOfFile; } +String moduleDisplayPath(ModuleID module) +{ + return pathOfModule(module).displayPath; +} + String sourceProviderURL(ModuleID module) { PathOfModule found = pathOfModule(module); @@ -6384,7 +6638,8 @@ void Modules::followImportsOfModulesThatHaveRun() for (size_t index : m_recordsNotSeenToHaveRun) { ++numberLookedAt; lookAt(ModuleID(index)); - if (now[index] != inRecord || m_modulesSeenToHaveRun.quickGet(index) || !m_records.contains(ModuleID(index))) + ModuleRecord* record = recordIfExists(ModuleID(index)); + if (now[index] != inRecord || m_modulesSeenToHaveRun.quickGet(index) || !record || record->status() < CyclicModuleRecord::Status::Evaluating) settled.append(index); } for (size_t index : settled) @@ -6453,21 +6708,25 @@ bool hasPendingMock(JSGlobalObject* realm, ModuleID file) return modules && modules->hasPendingMock(file); } -bool hasRequireCacheEntry(JSGlobalObject* realm, ModuleID module) +static bool hasRequireCacheEntry(Modules* modules, ModuleID module, bool& importsAreFollowed) { - Modules* modules = modulesOf(realm); - if (!modules) - return false; const ModuleData& data = modules->data(module); if (data.isOfLibrary()) return modules->isLibraryModuleInRequireCache(module); if (data.isFacade() || !modules->table().hasFilePath(module) || modules->isEvicted(module) || modules->hasNoEntryBehindMock(module)) return false; - if (!modules->hasRun(module) && !modules->isListedBeforeItHasRun(module)) + if (!modules->hasRun(module, importsAreFollowed) && !modules->isListedBeforeItHasRun(module)) return false; return !isNamedByNumber(modules->table().fullPath(module), module); } +bool hasRequireCacheEntry(JSGlobalObject* realm, ModuleID module) +{ + Modules* modules = modulesOf(realm); + bool importsAreFollowed = false; + return modules && hasRequireCacheEntry(modules, module, importsAreFollowed); +} + JSObject* requireCacheEntry(JSGlobalObject* realm, ModuleID module) { ASSERT(hasRequireCacheEntry(realm, module)); @@ -6495,6 +6754,81 @@ void forEachEvaluatingAsyncModule(JSGlobalObject* realm, const WTF::Function& topmost) const +{ + BitVector below(m_table.size()); + BitVector imported(m_table.size()); + Vector evaluating; + for (unsigned index = 0; index < m_table.size(); ++index) { + if (status(ModuleID(index)) == ModuleStatus::Evaluating) + evaluating.append(ModuleID(index)); + } + Vector work = evaluating; + while (!work.isEmpty()) { + ModuleID module = work.takeLast(); + if (below.quickSet(module.index()) || data(module).isOfLibrary()) + continue; + for (const ModuleRequestData& request : m_table.imports(module)) { + if (request.isDeferred) + continue; + ModuleID target = moduleLoadedUnder(request.targetModule()); + imported.quickSet(target.index()); + work.append(target); + } + } + topmost.shrink(0); + for (ModuleID module : evaluating) { + if (!imported.quickGet(module.index())) + topmost.append(module); + } + if (topmost.isEmpty()) + topmost.append(evaluating.span()); + return below; +} + +void Modules::resetModulesAfterEvaluation(const WTF::Function& isKept) +{ + BitVector listed(m_table.size()); + for (ModuleID file : m_modulesToResetOnceRun) + listed.quickSet(file.index()); + for (size_t index : modulesBelowEvaluatingOnes(m_modulesThatResetWaitsFor)) { + ModuleID module(index); + ModuleStatus current = status(module); + if (current != ModuleStatus::Evaluating && current != ModuleStatus::Linked) + continue; + ModuleID file = fileOf(module); + if (data(file).isOfLibrary() || !m_table.hasFilePath(file) || isEvicted(file) || !mayBeEvicted(file) || listed.quickSet(file.index())) + continue; + if (!isKept(m_table.fullPath(file))) + m_modulesToResetOnceRun.append(file); + } +} + +void Modules::resetModulesOnceRun() +{ + bool hasOneEnded = m_modulesThatResetWaitsFor.isEmpty() || std::ranges::any_of(m_modulesThatResetWaitsFor, [&](ModuleID module) { + return status(module) != ModuleStatus::Evaluating; + }); + if (!hasOneEnded) + return; + BitVector below = modulesBelowEvaluatingOnes(m_modulesThatResetWaitsFor); + Vector haveRun; + m_modulesToResetOnceRun.removeAllMatching([&](ModuleID file) { + if (below.get(file.index()) || below.get(moduleWithExportsOf(file).index())) + return false; + haveRun.append(file); + return true; + }); + for (ModuleID file : haveRun) + didResetModule(m_instance.globalObject, file); +} + +void resetModulesAfterEvaluation(JSGlobalObject* realm, const WTF::Function& isKept) +{ + if (Modules* modules = modulesOf(realm)) + modules->resetModulesAfterEvaluation(isKept); +} + void forEachModuleWithProgramEntry(JSGlobalObject* realm, const WTF::Function& functor) { Modules* modules = modulesOf(realm); @@ -6521,10 +6855,11 @@ void forEachRequireCacheKey(JSGlobalObject* realm, const WTF::Functiontable(); BitVector isSeen(table.size()); Vector inOrder; + bool importsAreFollowed = false; auto askFor = [&](ModuleID module) { if (isSeen.quickSet(module.index())) return false; - if (!table.data(module).isOfLibrary() && hasRequireCacheEntry(realm, module)) + if (!table.data(module).isOfLibrary() && hasRequireCacheEntry(modules, module, importsAreFollowed)) inOrder.append(module); return true; }; @@ -6896,6 +7231,26 @@ BitVector modulesInLoadGraph(JSGlobalObject* realm, ModuleID root, LoadGraphMode return modules ? modules->modulesInLoadGraph(root, graphMode) : BitVector(); } +BitVector unfinishedModulesInLoadGraph(JSGlobalObject* realm, ModuleID root, LoadGraphMode graphMode, const BitVector& unsettledMocks) +{ + Modules* modules = modulesOf(realm); + if (!modules) + return { }; + BitVector result = modules->modulesInLoadGraph(root, graphMode); + result.set(root.index()); + bool importsAreFollowed = false; + Vector finished; + for (size_t index : result) { + ModuleID module(index); + ModuleID file = modules->table().moduleBehindFacade(module).value_or(module); + if (!unsettledMocks.get(file.index()) && !modules->isBeingEvaluated(module) && hasRequireCacheEntry(modules, module, importsAreFollowed)) + finished.append(index); + } + for (size_t index : finished) + result.quickClear(index); + return result; +} + bool isRetainedByInFlightLoads(JSGlobalObject* realm, ModuleID module) { Modules* modules = modulesOf(realm); @@ -7265,6 +7620,18 @@ void didCreateModuleObject(JSGlobalObject* realm, ModuleID module, JSObject* mod modules->retainModuleObjectFromRequire(module, moduleObject); } +JSObject* Modules::childModuleObject(ModuleID module) +{ + bool isShown = (isFileOfProgram(module) || data(module).kind == ModuleKind::Data) && !hidesModuleObject(module); + return isShown ? moduleObjectMadeByRequire(module) : nullptr; +} + +JSObject* childModuleObject(JSGlobalObject* realm, ModuleID module) +{ + Modules* modules = modulesThatShowWaysBetweenModuleObjects(realm); + return modules ? modules->childModuleObject(module) : nullptr; +} + void willRequireFromUncompiledCode(JSGlobalObject* realm, ModuleID module, JSObject* requirerModuleObject) { if (Modules* modules = modulesOf(realm)) @@ -7310,12 +7677,10 @@ void willSetExtensionLoader(JSGlobalObject* realm, StringView extension) if (!modules) return; const ModuleTable& table = modules->table(); + bool hasQueryInDirectory = table.hasQueryInDirectory(); for (unsigned index = 0; index < table.size(); ++index) { ModuleID module(index); - if (!table.hasFilePath(module) || modules->data(module).isOfLibrary()) - continue; - String path = table.fullPath(module); - if (!StringView(path).left(path.reverseFind('?')).endsWith(extension)) + if (!table.hasFilePath(module) || modules->data(module).isOfLibrary() || !table.pathWithoutQueryEndsWith(module, extension, hasQueryInDirectory)) continue; if (!modules->mayBeEvicted(module)) { throwLoaderChangeError(realm, LoaderChange::ReplaceLoaderFunction, "a loader in Module._extensions"_s); @@ -7521,6 +7886,12 @@ bool wasLinkedByFailedGraph(JSGlobalObject* realm, ModuleID module, bool byEarli return modules && modules->wasLinkedByFailedGraph(module, byEarlierTestFile); } +bool isLinkedForImporterYetToRun(JSGlobalObject* realm, ModuleID module, std::optional asking) +{ + Modules* modules = modulesOf(realm); + return modules && modules->isLinkedForImporterYetToRun(module, asking); +} + bool wasLinkedBeforeLinkFailure(JSGlobalObject* realm, ModuleID module) { Modules* modules = modulesOf(realm); @@ -7958,7 +8329,7 @@ void didChangePlugins(JSGlobalObject* realm) modules->forgetWhatNamesResolvedTo(); } -void willMockModule(JSGlobalObject* realm, ModuleID module, JSValue requireReturnValue, bool hasCommonJSExports) +void willMockModule(JSGlobalObject* realm, ModuleID module, JSValue requireReturnValue, bool hasCommonJSExports, bool hasFactory) { Modules* found = modulesOf(realm); if (!found) @@ -7968,6 +8339,7 @@ void willMockModule(JSGlobalObject* realm, ModuleID module, JSValue requireRetur modules.leaveImportersAbove(module); JSObject* exports = requireReturnValue.isObject() ? asObject(requireReturnValue) : constructEmptyObject(realm); modules.setMockHasCommonJSExports(module, hasCommonJSExports); + modules.setMockHasFactory(module, hasFactory); if (modules.data(module).isOfLibrary() || !modules.table().hasFilePath(module)) { auto scope = DECLARE_THROW_SCOPE(realm->vm()); modules.releaseRemovedMock(module); @@ -8560,6 +8932,24 @@ void ModuleRecord::appendDynamicStarTargets(StarExportTargets& targets) } } +void ModuleRecord::appendDynamicStarTargets(const Identifier& exportName, StarExportTargets& targets) +{ + auto scope = DECLARE_THROW_SCOPE(vm()); + Modules& modules = this->modules(); + if (!modules.data(m_module).hasDynamicStarExport() || (modules.standsForWholeModule(m_module) && !keepsToModule())) + return; + const Modules::StarTargetsByName& stars = modules.starTargetsByName(m_module); + stars.visitCandidates(exportName.impl(), [&](unsigned place) { + if (!stars.targets[place].isFollowed) + return true; + AbstractModuleRecord* target = recordBehindStar(stars.targets[place].module); + if (scope.exception()) [[unlikely]] + return false; + targets.append(target); + return true; + }); +} + AbstractModuleRecord* ModuleRecord::recordBehindStar(ModuleID asked) { Modules& modules = this->modules(); @@ -8592,35 +8982,25 @@ bool ModuleRecord::appendStarsThatMayProvide(const Identifier& exportName, unsig std::optional targetIndex = modules.data(asked).hasExportTable() ? modules.table().findExport(asked, exportName.impl()) : std::nullopt; return targetIndex && exportSameBinding(modules.table(), entry, modules.table().exports(asked)[*targetIndex]); }; - std::span asWritten = modules.table().starsAsWritten(m_module); - auto someStar = [&](auto&& holds) { - for (const StarAsWrittenData& star : asWritten) { - if (holds(ModuleID(star.target))) - return true; - } - if (!asWritten.empty()) - return false; - for (const ModuleRequestData& request : modules.table().requests(m_module)) { - if (request.isDynamicStarExport && holds(request.targetModule())) - return true; - } - return false; - }; - if (!someStar(isFirstProvider) && !someStar(providesBinding)) + const Modules::StarTargetsByName& stars = modules.starTargetsByName(m_module); + bool someStarProvidesBinding = false; + stars.visitCandidates(exportName.impl(), [&](unsigned place) { + someStarProvidesBinding = providesBinding(stars.targets[place].module); + return !someStarProvidesBinding; + }); + if (!someStarProvidesBinding) return false; - if (asWritten.empty()) { - appendDynamicStarTargets(targets); - RETURN_IF_EXCEPTION(scope, false); - return true; - } - for (const StarAsWrittenData& star : asWritten) { - ModuleID asked(star.target); - if (!star.isFollowed && !providesBinding(asked)) - continue; - AbstractModuleRecord* target = recordBehindStar(asked); - RETURN_IF_EXCEPTION(scope, false); + stars.visitCandidates(exportName.impl(), [&](unsigned place) { + Modules::StarTarget star = stars.targets[place]; + if (!star.isFollowed && !providesBinding(star.module)) + return true; + AbstractModuleRecord* target = recordBehindStar(star.module); + if (scope.exception()) [[unlikely]] + return false; targets.append(target); - } + return true; + }); + RETURN_IF_EXCEPTION(scope, false); return true; } @@ -8845,7 +9225,7 @@ auto ModuleRecord::resolveOwnExport(JSGlobalObject* globalObject, const Identifi return absent; } if (!isDefault && (!entry || entry->isFromStarExport || modules.leavesNameToStars(m_module, *index))) { - appendDynamicStarTargets(starTargets); + appendDynamicStarTargets(exportName, starTargets); RETURN_IF_EXCEPTION(scope, Resolution::error()); } bool isSeenAsEndedMock = entry && reexporter && modules.endedMockSeenBy(m_module, reexporter->m_module); @@ -9274,8 +9654,15 @@ AbstractModuleRecord* ModuleNamespace::recordForResolution() return modules().recordForResolution(m_module); } +bool ModuleNamespace::keepsToModule() const +{ + return m_keepsToModule || modules().isBypassingMocks(); +} + JSObject* ModuleNamespace::activeMock() const { + if (modules().isBypassingMocks()) [[unlikely]] + return nullptr; if (JSObject* ofView = m_mockOfView.get()) [[unlikely]] return ofView; return keepsToModule() ? nullptr : modules().activeMock(m_module); @@ -9377,14 +9764,14 @@ JSValue Modules::readOffNamespaceThatListsName(ModuleID reader, JSObject* namesp JSGlobalObject* realm = m_instance.globalObject; auto scope = DECLARE_THROW_SCOPE(*m_instance.vm); std::pair read { namespaceOfReader, name.impl() }; - if (m_namesBeingReadOffListers.contains(read) && !m_mocks && m_removedMocksInUse.isEmpty()) [[unlikely]] { - AbstractModuleRecord* record = recordForResolution(reader); + if (m_namesBeingReadOffListers.contains(read) && (m_isBypassingMocks || (!m_mocks && m_removedMocksInUse.isEmpty()))) [[unlikely]] { + AbstractModuleRecord* record = m_isBypassingMocks && activeMock(reader) ? ensureDisplacedRecord(reader) : recordForResolution(reader); RETURN_IF_EXCEPTION(scope, { }); AbstractModuleRecord::Resolution resolution = record->resolveExport(realm, name); RETURN_IF_EXCEPTION(scope, { }); auto* provider = resolution.type == AbstractModuleRecord::Resolution::Type::Resolved ? dynamicDowncast(resolution.moduleRecord) : nullptr; if (provider && provider->module() != reader && resolution.localName != m_instance.vm->propertyNames->starNamespacePrivateName) { - JSObject* namespaceOfProvider = moduleNamespace(provider->module(), ModulePhase::Evaluation); + JSObject* namespaceOfProvider = m_isBypassingMocks ? originalNamespace(provider->module()) : moduleNamespace(provider->module(), ModulePhase::Evaluation); RETURN_IF_EXCEPTION(scope, { }); RELEASE_AND_RETURN(scope, namespaceOfProvider->get(realm, resolution.localName)); } @@ -9393,6 +9780,10 @@ JSValue Modules::readOffNamespaceThatListsName(ModuleID reader, JSObject* namesp auto endRead = makeScopeExit([&] { m_namesBeingReadOffListers.removeLast(); }); + auto* readingNamespace = dynamicDowncast(namespaceOfReader); + auto* listingNamespace = dynamicDowncast(lister); + bool shouldBypassMocks = listingNamespace && (m_isBypassingMocks || (readingNamespace && readingNamespace->keepsToModule()) || (listingNamespace->activeMock() && importsOriginalDespiteMock(listingNamespace->module(), reader))); + SetForScope isBypassingMocks(m_isBypassingMocks, shouldBypassMocks); RELEASE_AND_RETURN(scope, lister->get(realm, name)); } diff --git a/Source/JavaScriptCore/aot/runtime/AOTModules.h b/Source/JavaScriptCore/aot/runtime/AOTModules.h index 7299a754e714b..1b36ce9346a15 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTModules.h +++ b/Source/JavaScriptCore/aot/runtime/AOTModules.h @@ -42,6 +42,7 @@ class ThrowScope; namespace AOT { class ModuleLinkWalk; +struct WalksOfOneLink; struct EvaluationInProgress; struct Instance; struct State; @@ -98,6 +99,7 @@ struct LinkContext { macro(JSObject*, activeMock) \ macro(JSObject*, recordBeforeMockInPlace) \ macro(JSObject*, namespaceBeforeMockInPlace) \ + macro(JSObject*, namespaceToRestore) \ macro(JSObject*, capturedExportsBeforeMockInPlace) \ macro(JSValue, requiredValueBeforeMockInPlace) \ macro(JSValue, requiredValueBeforeRefusal) \ @@ -134,10 +136,15 @@ struct EvaluationContext { struct MockedModules { static constexpr uint32_t notBelow = std::numeric_limits::max(); + explicit MockedModules(const ImportersOfModules& importers) + : importers(&importers) + { + } + + const ImportersOfModules* importers; BitVector modules; Vector positionBelow; uint32_t numberBelow { 0 }; - BitVector namedByListsAlone; std::optional comeToMock; UncheckedKeyHashMap> reachedFromRoots; }; @@ -171,6 +178,7 @@ class Modules { enum class MockMode : bool { Use, Bypass }; JSValue readExport(ModuleID, unsigned index, MockMode = MockMode::Use); JSValue readOffNamespaceThatListsName(ModuleID reader, JSObject* namespaceOfReader, JSObject* lister, const Identifier&); + bool isBypassingMocks() const { return m_isBypassingMocks; } JSValue readExportIfInitialized(ModuleID, unsigned index) const; bool isExportAmbiguous(ModuleID, unsigned index) const; bool tryReplaceExport(ModuleID, UniquedStringImpl* name, JSValue); @@ -394,6 +402,7 @@ class Modules { return mock != m_mocksWithIntactOriginal.end() && mock->value.isDisplaced; } bool setMockWithIntactOriginal(ModuleID, JSObject* mock, JSValue requireReturnValue, bool keepsEarlierMockForImporters = false); + bool reachesStarCycle(ModuleID) const; void removeMockWithIntactOriginal(ModuleID); bool seesMockOf(ModuleID mocked, ModuleID importer) const; bool keepsMockThatStandsWhileAnotherWaits(ModuleID mocked, ModuleID importer) const; @@ -437,6 +446,7 @@ class Modules { bool isMockLingering(ModuleID module) const { return m_modulesWithLingeringMock.get(module.index()); } bool isRequestedByDetachedLoad(ModuleID mocked) const { return m_resumingDetachedLoad && staticImportClosure(*m_resumingDetachedLoad, MockTraversal::Stop).get(mocked.index()); } void retryMockInDetachedLoads(ModuleID mocked); + uint64_t resumingLoadFactoryCall(ModuleID mocked) const; bool needsRetryInDetachedLoad(ModuleID mocked) const { return m_detachedLoads.containsIf([&](const auto& left) { return left.wait->status() == JSPromise::Status::Pending && left.modulesToRetry.get(mocked.index()); }); } void didResumeAfterFactories() { m_hasImportsBlockedOnFactories = false; } bool tryMarkMockLingering(ModuleID module, bool wasInstalledOnPreloadMock) @@ -565,16 +575,50 @@ class Modules { }; std::optional findMockForReexport(ModuleID reexportingModule, const ModuleExportData& reexportEntry, MockMode reexportingModuleMockMode = MockMode::Use) const; std::optional findMockForReexportSlow(ModuleID reexportingModule, const ModuleExportData& reexportEntry, MockMode reexportingModuleMockMode) const; + const Vector& modulesOnReexportWay(ModuleID reexportingModule, unsigned index) const; + bool mayMockAnswerReexport(ModuleID reexportingModule, const ModuleExportData& reexportEntry) const; struct PlaceOfExport { ModuleID module; unsigned index; }; const Vector>& starExportProviders(ModuleID) const; + struct StarTarget { + ModuleID module; + bool isFollowed { true }; + }; + struct StarTargetsByName { + WTF_DEPRECATED_MAKE_STRUCT_FAST_ALLOCATED(StarTargetsByName); + + Vector targets; + Vector targetsWithUnlistedNames; + UncheckedKeyHashMap, Vector> targetsListingName; + + template + bool visitCandidates(UniquedStringImpl* name, const Functor& functor) const + { + auto found = targetsListingName.find(name); + std::span listing = found == targetsListingName.end() ? std::span() : found->value.span(); + std::span others = targetsWithUnlistedNames.span(); + while (!listing.empty() || !others.empty()) { + bool takesListing = others.empty() || (!listing.empty() && listing.front() <= others.front()); + unsigned place = takesListing ? listing.front() : others.front(); + if (!listing.empty() && listing.front() == place) + listing = listing.subspan(1); + if (!others.empty() && others.front() == place) + others = others.subspan(1); + if (!functor(place)) + return false; + } + return true; + } + }; + const StarTargetsByName& starTargetsByName(ModuleID) const; enum class MockFilter : uint8_t { All, AddingNames }; bool isMockedThroughStarExports(ModuleID, MockFilter = MockFilter::All); bool isMockedThroughStarExportsSlow(ModuleID, MockFilter); void leaveOutWhatMockLacks(ModuleID, JSObject* exportsOfMock); void setMockHasCommonJSExports(ModuleID, bool); + void setMockHasFactory(ModuleID module, bool hasFactory) { m_mocksWithoutFactory.set(module.index(), !hasFactory); } void takeWhatMockAdds(ModuleID, bool comesAheadOfLoad); bool isAddedByMock(ModuleID, const Identifier&); void appendNamesAddedByMock(ModuleID, WTF::Vector&); @@ -629,7 +673,7 @@ class Modules { { if (!m_mocks || module == m_root) [[likely]] return false; - if (m_mocks->namedByListsAlone.quickGet(module.index())) + if (m_mocks->importers->namedByListsAlone.quickGet(module.index())) return true; uint32_t position = m_mocks->positionBelow[module.index()]; if (position == MockedModules::notBelow) @@ -697,8 +741,11 @@ class Modules { bool wasLinkedByFailedGraph(ModuleID, bool byEarlierTestFile); bool isImportedByFailedRoot(ModuleID, bool ofEarlierTestFile = false) const; bool isFailedImportRoot(ModuleID module) const { return m_importCallRoots.get(module.index()) && status(module) == ModuleStatus::Failed; } + std::optional moduleThatFailedToLink(ModuleID root) const; + bool hasLinkFailures() const { return m_errorsOfExportsThatBuildFoundAbsent || !m_linkFailures.isEmpty(); } bool wasLinkedAheadOfFailure(ModuleID root, ModuleID) const; bool wasLinkedBeforeLinkFailure(ModuleID) const; + bool isLinkedForImporterYetToRun(ModuleID, std::optional asking) const; std::optional failureByAbsentExport(ModuleID) const; bool tryEvaluateESModuleBeforeMock(ModuleID); bool startDetachedOriginalLoad(ModuleID); @@ -807,6 +854,7 @@ class Modules { void throwFailure(ModuleFailureID, bool isForRequire); void throwNewFailure(ModuleFailureID, bool isForRequire); std::optional loadFailure(ModuleID module) const { return m_table.loadFailure(module); } + std::optional findUnmockedFileFailure(ModuleID root) const; void throwExportNotFound(const RequiredNameData&); bool dropExportThatFindsNothing(ModuleID importer, const RequiredNameData&); void throwExportAmbiguous(const RequiredNameData&); @@ -822,9 +870,11 @@ class Modules { void setIsRunningPreloads(bool runsPreloads) { m_isRunningPreloads = runsPreloads; } std::optional firstModuleRequestedByHost() const { return m_firstModuleRequestedByHost; } bool hasBeenImported(ModuleID) const; + bool isUnevaluatedTextForm(ModuleID) const; bool isBeingFetched(ModuleID) const; bool isFetchedSynchronously(ModuleID) const; bool isImportResolvedByLoader(ModuleID) const; + bool hasPlannedLoadComeTo(ModuleID importer) const; enum class ImportResolver : bool { Build, Loader }; void runDataFileAgainForImport(ModuleID, ImportResolver); void reevaluateDataFile(ModuleID); @@ -853,6 +903,12 @@ class Modules { bool hasUncompiledRequester() const { return m_moduleRequester && m_moduleRequester->impl() != compiledCodeRequester().impl() && !isInHostRequest(); } void forEachTLASuspendedModule(const WTF::Function&); void evict(ModuleID, bool shouldRestoreIfRetained = false); + void resetModulesAfterEvaluation(const WTF::Function& isKept); + void resetModulesThatHaveRun() + { + if (!m_modulesToResetOnceRun.isEmpty()) [[unlikely]] + resetModulesOnceRun(); + } void forEachFailedModule(const WTF::Function&) const; void snapshotEvaluatedModules(); void retainNewlyEvaluatedModules(); @@ -1017,6 +1073,7 @@ class Modules { void didChangeStatusInRecord(ModuleID module) { m_recordsNotSeenToHaveRun.set(module.index()); } void lookAtCycleOf(ModuleID, ModulesThatWaitForCycles&) const; const CyclesOfImports& cyclesOfImports() const; + const ImportersOfModules& importersOfModules() const; JSValue callBodyOfModuleEnteredByLoader(ModuleID module) { return callBodyUnderOwnMark(module, false); } void evaluateModuleInRecord(ModuleID); bool beginModuleInRecord(ModuleID); @@ -1087,6 +1144,7 @@ class Modules { bool canTellHowGraphIsLoaded(ModuleID root); void appendRequestsForLoader(ModuleID, ModuleRecord*, ModuleGraphLoadingState&); std::optional statementThatEndsLoad(ModuleID) const; + bool isLeftByFailedLoad(ModuleID) const; void appendUnresolvableRequest(ModuleFailureID statement, ModuleRecord*); void keepFailureOfRequest(ModuleID, ModuleRecord*, const Identifier& name); ModuleRecord* appendRequestsOfUncompiledCode(ModuleID, ModuleRecord*); @@ -1189,6 +1247,7 @@ class Modules { void didBindExportsVariable(ModuleID); JSObject* mainModuleObject(ModuleID); JSObject* parentModuleObject(ModuleID); + JSObject* childModuleObject(ModuleID); bool canExposeModuleObject(ModuleID module) const { return data(module).kind != ModuleKind::CommonJs || data(module).hasValue(); } bool hidesModuleObject(ModuleID module) const { return !canExposeModuleObject(module) || data(module).hasUnescapedExports(); } bool forEachNewChild(ModuleID, const ScopedLambda&); @@ -1443,6 +1502,7 @@ class Modules { enum class NativeModuleLoading : bool { Eager, Deferred }; bool loadRuntimeModules(const ModuleLinkWalk&, NativeModuleLoading = NativeModuleLoading::Eager); void link(const ModuleLinkWalk&); + bool resolvesNamesOutsideLibrary(const ModuleLinkWalk&, ThrowScope&) const; void captureSyntheticExports(const ModuleLinkWalk&, VisitedModuleRecords&); void finishLink(const ModuleLinkWalk&, LinkContext&); void didLinkEntryPoint(); @@ -1480,11 +1540,16 @@ class Modules { void evaluateWithOwnStack(ModuleID root); Vector m_modulesAskedForWhileTheyRan; void settleImportsOfModulesThatHaveRun(ModuleID rootOfEvaluation); + BitVector modulesBelowEvaluatingOnes(Vector& topmost) const; + Vector m_modulesToResetOnceRun; + Vector m_modulesThatResetWaitsFor; + void resetModulesOnceRun(); size_t queuedJobCount(EagerLoadJobPolicy) const; void holdJobsBehind(size_t numberOfJobs, ModuleID file, EagerLoadJobPolicy = EagerLoadJobPolicy::DeferJobs); bool hasQueuedTicks() const { return host().hasQueuedTicks && host().hasQueuedTicks(m_instance.globalObject); } struct LevelOfLoad; struct PlanOfLoad; + bool tryFindCompiledModulesBehindNames(ModuleID root, UncheckedKeyHashMap&); bool planLevelsOfLoad(ModuleID root); bool isLeftAsLoaded(ModuleID) const; bool isModuleWithoutRecord(const ModuleData&, StoredModuleStatus) const; @@ -1515,6 +1580,9 @@ class Modules { std::optional mainCommonJSModule(); void decideMainCommonJSModule(std::optional); void lookForMainCommonJSModule(); + bool isInRequireMap(ModuleID file); + void forgetFailedMainModule(); + bool isMainCommonJSModule(ModuleID file); void noteFirstRequestFromHost(ModuleID); void noteLoadOfGraph(ModuleID root); JSObject* moduleObjectMadeByRequire(ModuleID); @@ -1580,6 +1648,7 @@ class Modules { std::unique_ptr takeKeptLinkWalk(ModuleID root); void keepLinkWalk(std::unique_ptr&&); Vector, 2> m_linkWalksOfLoads; + RefPtr m_walksOfLink; bool checkIsSettledForLinks(ModuleID) const; mutable unsigned m_linkEpoch { 0 }; mutable BitVector m_leadersWhoseLinkGoesByAllRequests; @@ -1615,11 +1684,13 @@ class Modules { JSObject* m_prototypeOfModuleObjects { nullptr }; JSObject* m_errorsOfExportsThatBuildFoundAbsent { nullptr }; Vector m_failuresByExportsThatBuildFoundAbsent; + UncheckedKeyHashMap m_linkFailures; Structure* m_extensibleNamespaceStructure { nullptr }; Identifier m_moduleExportsName; enum class RecordedRequire : uint8_t { Begun, BegunForEvictedModule, Returned, BegunAgain, BegunFirstESModuleRequire, ReturnedAndNotListed }; UncheckedKeyHashMap m_recordedRequires; std::optional m_mainCommonJSModule; + BitVector m_failedMainModules; bool m_isMainCommonJSModuleDecided { false }; bool m_didSearchBelowEntryPoint { false }; Vector m_pendingFirstRequireTargets; @@ -2032,9 +2103,11 @@ class Modules { struct Found { ModuleID module; uint32_t importer; - uint32_t laterRequests : 28; + uint32_t laterRequests : 26; uint32_t isFile : 1; uint32_t leadsToFile : 1; + uint32_t isBehindName : 1; + uint32_t isFetchedByHost : 1; uint32_t wasLoadedSince : 1; uint32_t isKnownToBeLeft : 1; }; @@ -2049,6 +2122,9 @@ class Modules { Vector found; Vector levels; Vector knownToBeLeft; + BitVector modulesComeTo; + uint32_t levelsAmongModulesComeTo { 0 }; + uint32_t levelsRequested { 0 }; uint32_t steps { 0 }; uint32_t stepsOfWholeGraph { 0 }; bool asksAcrossLevels { false }; @@ -2069,34 +2145,7 @@ class Modules { }; UncheckedKeyHashMap> m_levelsOfLoads; bool hasGraphRun(const LevelOfLoad& level) const { return status(level.plan->found[0].module) >= ModuleStatus::EvaluatingAsync; } - class PlacesSeenByPlans { - public: - size_t size() const { return m_numberOfModules; } - void forgetAll(size_t numberOfModules) - { - m_numberOfModules = numberOfModules; - m_groups.clear(); - m_groups.grow((numberOfModules + modulesInGroup - 1) / modulesInGroup); - } - uint32_t& operator[](size_t module) - { - std::unique_ptr& group = m_groups[module / modulesInGroup]; - if (!group) - group = makeUnique(); - return group->places[module % modulesInGroup]; - } - - private: - static constexpr size_t modulesInGroup = 64; - struct Group { - WTF_DEPRECATED_MAKE_STRUCT_FAST_ALLOCATED(Group); - std::array places { }; - }; - Vector> m_groups; - size_t m_numberOfModules { 0 }; - }; - PlacesSeenByPlans m_placesSeenByPlans; - uint32_t m_firstPlaceOfNextPlan { 1 }; + PlacesSeenByPlans& m_placesSeenByPlans; Vector m_rootsLoadedWithoutLoader; BitVector m_filesLoadedWithoutLoader; BitVector m_filesLoadedByRequire; @@ -2165,7 +2214,7 @@ class Modules { std::optional m_originalBeingEvaluated; bool needsOriginalBeforeFactory(ModuleID mocked) const { return m_modulesWithPendingMock.get(mocked.index()) && m_mocksRetriedAfterFailure.get(mocked.index()); } bool sweepAheadOfFactories(ModuleID root); - bool isOriginalDelegatedToHost(ModuleID mocked, const BitVector& held); + bool isOriginalDelegatedToHost(ModuleID mocked, const BitVector& held, const BitVector& isOnPath); enum class OriginalInSweep : uint8_t { Evaluated, DelegatedToHost, Failed }; OriginalInSweep runOriginalInSweep(ModuleID mocked); JSObject* actualKeyNamespace(ModuleID mocked); @@ -2223,6 +2272,7 @@ class Modules { }; Vector m_namespacesThatLackNames; Vector, 4> m_namesBeingReadOffListers; + bool m_isBypassingMocks { false }; struct CollectionOfNamesBehindStars { Vector, 1> namesOfEndedMocks; }; @@ -2344,6 +2394,9 @@ class Modules { Vector> m_takersOfNames; BitVector m_importersRetainedAboveMock; mutable UncheckedKeyHashMap>> m_starExportProviders; + mutable UncheckedKeyHashMap> m_starTargetsByName; + mutable UncheckedKeyHashMap, DefaultHash, WTF::UnsignedWithZeroKeyHashTraits> m_reexportWays; + mutable std::optional, IntHash, WTF::UnsignedWithZeroKeyHashTraits>> m_modulesWithSyntheticExport; std::optional m_moduleWithDeferredJobs; std::optional m_rootWhoseFilesRun; struct DeferredRequest { @@ -2391,6 +2444,7 @@ class Modules { Vector m_importsMadeByOwnFactory; BitVector m_mocksRetriedAfterFailure; BitVector m_modulesNeedingMockReinstall; + BitVector m_mocksWithoutFactory; BitVector m_modulesWithMockLoadError; ModuleID m_currentRequireTarget; bool m_shouldContinuePastPendingFactories { false }; diff --git a/Source/JavaScriptCore/aot/runtime/AOTOperatorOperations.cpp b/Source/JavaScriptCore/aot/runtime/AOTOperatorOperations.cpp index ba87a3529b46a..628629e27d790 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTOperatorOperations.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTOperatorOperations.cpp @@ -70,6 +70,17 @@ static ALWAYS_INLINE double numberOfValueThatIsNoCell(JSValue value) OPERATION_RETURN(scope, JSValue::encode(function(globalObject, left, right))); \ } +#define AOT_DEFINE_BITWISE_OPERATION(name, function, bigIntFunction) \ + JSC_DEFINE_JIT_OPERATION(name, EncodedJSValue, (Instance* instance, EncodedJSValue encodedLeft, EncodedJSValue encodedRight)) \ + { \ + AOT_OPERATION_BEGIN(instance, Entry::name); \ + JSValue left = JSValue::decode(encodedLeft); \ + JSValue right = JSValue::decode(encodedRight); \ + if (left.isHeapBigInt() && right.isHeapBigInt()) \ + OPERATION_RETURN(scope, JSValue::encode(JSBigInt::bigIntFunction(globalObject, left.asHeapBigInt(), right.asHeapBigInt()))); \ + OPERATION_RETURN(scope, JSValue::encode(function(globalObject, left, right))); \ + } + #define AOT_DEFINE_COMPARISON_OPERATION(name, ofHeapBigInts, relation, result) \ JSC_DEFINE_JIT_OPERATION(name, size_t, (Instance* instance, EncodedJSValue encodedLeft, EncodedJSValue encodedRight)) \ { \ @@ -114,11 +125,11 @@ AOT_DEFINE_NUMERIC_BINARY_OPERATION(operationAOTValueMul, jsMul, multiply, x * y AOT_DEFINE_NUMERIC_BINARY_OPERATION(operationAOTValueDiv, jsDiv, divide, x / y) AOT_DEFINE_NUMERIC_BINARY_OPERATION(operationAOTValueMod, jsRemainder, remainder, Math::fmodDouble(x, y)) AOT_DEFINE_BINARY_OPERATION(operationAOTValuePow, EncodedJSValue, JSValue::encode(jsPow(globalObject, left, right))) -AOT_DEFINE_BINARY_OPERATION(operationAOTValueBitAnd, EncodedJSValue, JSValue::encode(jsBitwiseAnd(globalObject, left, right))) -AOT_DEFINE_BINARY_OPERATION(operationAOTValueBitOr, EncodedJSValue, JSValue::encode(jsBitwiseOr(globalObject, left, right))) -AOT_DEFINE_BINARY_OPERATION(operationAOTValueBitXor, EncodedJSValue, JSValue::encode(jsBitwiseXor(globalObject, left, right))) -AOT_DEFINE_BINARY_OPERATION(operationAOTValueLShift, EncodedJSValue, JSValue::encode(jsLShift(globalObject, left, right))) -AOT_DEFINE_BINARY_OPERATION(operationAOTValueRShift, EncodedJSValue, JSValue::encode(jsRShift(globalObject, left, right))) +AOT_DEFINE_BITWISE_OPERATION(operationAOTValueBitAnd, jsBitwiseAnd, bitwiseAnd) +AOT_DEFINE_BITWISE_OPERATION(operationAOTValueBitOr, jsBitwiseOr, bitwiseOr) +AOT_DEFINE_BITWISE_OPERATION(operationAOTValueBitXor, jsBitwiseXor, bitwiseXor) +AOT_DEFINE_BITWISE_OPERATION(operationAOTValueLShift, jsLShift, leftShift) +AOT_DEFINE_BITWISE_OPERATION(operationAOTValueRShift, jsRShift, signedRightShift) AOT_DEFINE_COMPARISON_OPERATION(operationAOTCompareLess, == JSBigInt::ComparisonResult::LessThan, <, jsLess(globalObject, left, right)) AOT_DEFINE_COMPARISON_OPERATION(operationAOTCompareLessEq, != JSBigInt::ComparisonResult::GreaterThan, <=, jsLessEq(globalObject, left, right)) AOT_DEFINE_COMPARISON_OPERATION(operationAOTCompareGreater, == JSBigInt::ComparisonResult::GreaterThan, >, jsLess(globalObject, right, left)) diff --git a/Source/JavaScriptCore/aot/runtime/AOTProgramStatics.h b/Source/JavaScriptCore/aot/runtime/AOTProgramStatics.h index ed52855044b4a..fd644b32974df 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTProgramStatics.h +++ b/Source/JavaScriptCore/aot/runtime/AOTProgramStatics.h @@ -49,6 +49,7 @@ class ProgramStatics { static JSValue readVariableWithoutAllocating(const Instance&, SlotData); static bool isVariableEmpty(const Instance&, SlotData); static void writeVariable(Instance&, SlotData, JSValue); + static void writeNamespace(Instance& instance, SlotData slot, JSValue value) { write(instance, slot, value); } static void emptyVariableWhoseValueHasMoved(Instance&, SlotData); template void visitRoots(Visitor&); diff --git a/Source/JavaScriptCore/aot/runtime/AOTProgramStrings.cpp b/Source/JavaScriptCore/aot/runtime/AOTProgramStrings.cpp index d44ea48044c8b..30869581e483e 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTProgramStrings.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTProgramStrings.cpp @@ -10,6 +10,7 @@ #include "AOTHeapCacheRegistry.h" #include "AOTInstance.h" +#include "AOTPrefixCode.h" #include "AOTProgramIdentifiers.h" #include "AOTState.h" #include "AOTVector.h" @@ -78,6 +79,8 @@ JSString* ProgramStrings::string(VM& vm, ImageStringWord word, Made made) const { DeferGCForAWhile deferGC(vm); const ImageText& text = m_image.text(); + if (word.isCoded()) + return jsString(vm, text.createString(word)); if (StringImpl* header = text.tryGetHeader(word)) return jsOwnedString(vm, String { *header }); StringView characters = text.view(word); @@ -222,7 +225,7 @@ void ProgramStrings::willAnalyzeHeap(VM& vm) ensureName(field.name, field.slot); } if (!m_hasNamedVariablesForHeapSnapshots) { - bool endsWhereClosureNumberEnds = forEachNamedCapture(m_tables.namedCaptures(), [&](const NamedCaptureData& capture) { + bool endsWhereClosureNumberEnds = forEachNamedCapture(decodeVarintBytes(m_tables.namedCaptures()).span(), [&](const NamedCaptureData& capture) { identifiers.nameForHeapSnapshots(capture.closureNumber, capture.slot, AtomString(text(m_tables, StringID(capture.name))), capture.isVariableOfModule); }); RELEASE_ASSERT(endsWhereClosureNumberEnds); diff --git a/Source/JavaScriptCore/aot/runtime/AOTPropertyCachesAnswers.cpp b/Source/JavaScriptCore/aot/runtime/AOTPropertyCachesAnswers.cpp index 942eee97a39c3..50df4d521a5cc 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTPropertyCachesAnswers.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTPropertyCachesAnswers.cpp @@ -345,6 +345,8 @@ static void nameInheritedProperty(PropertyAnswer& answer, const ChainWalk& walk) answer.holder = walk.holder; } +static bool leavesFieldOut(State&, const LayoutData&, uint32_t orderWord, const FieldData&); + static PropertyAnswer classifyFieldRead(const ReadQuery& query, Object& object, const FieldData& field) { bool isAccessor = field.isAccessor(); @@ -365,10 +367,15 @@ static PropertyAnswer classifyFieldRead(const ReadQuery& query, Object& object, return uncacheableOnce(SlotAccess::Read); if (mayExistWithoutStorage) { bool findsPlainValue = walk.result == ChainWalkResult::Found && !(walk.attributes & PropertyAttribute::AccessorOrCustomAccessorOrValue) && isValidOffset(walk.offset); - if (isAccessor || walk.firstObjectWithEmptySlot || (walk.result != ChainWalkResult::Absent && !findsPlainValue)) + bool findsAccessor = walk.result == ChainWalkResult::Found && walk.attributes & PropertyAttribute::Accessor && isValidOffset(walk.offset) && leavesFieldOut(object.state(), object.layout(), object.orderWord(), field); + if (isAccessor || walk.firstObjectWithEmptySlot || (walk.result != ChainWalkResult::Absent && !findsPlainValue && !findsAccessor)) return answer; nameInheritedProperty(answer, walk); - answer.emptySlotMayBeUndefined = true; + if (findsAccessor) { + uint32_t orderWord = object.orderWord() & orderAndCountMask; + answer.orderWords = std::pair { orderWord, orderWord }; + } else + answer.emptySlotMayBeUndefined = true; } else if (walk.firstObjectWithEmptySlot) { if (walk.result != ChainWalkResult::Unknown && !isAccessor) { answer.kind = PropertyAnswerKind::FieldOrInheritedField; @@ -496,6 +503,16 @@ static PropertyAnswer classifyReadAboveReceiver(const ReadQuery& query) return answer; } +static bool readsPrivateMemberOfString(const ReadQuery& query) +{ + return query.base.isString() && PropertyName(query.uid).isPrivateName(); +} + +static bool readsPrototypeAccessor(const ReadQuery& query) +{ + return query.uid == query.globalObject->vm().propertyNames->underscoreProto.impl() && query.slot.isAccessor() && query.slot.slotBase() == query.globalObject->objectPrototype(); +} + PropertyAnswer classifyRead(const ReadQuery& query) { constexpr SlotAccess access = SlotAccess::Read; @@ -504,7 +521,7 @@ PropertyAnswer classifyRead(const ReadQuery& query) UniquedStringImpl* uid = query.uid; const PropertySlot& slot = query.slot; bool mayAllocate = query.mode == ChainWalkMode::MayAllocate; - if (!structure || isNeverCachedName(vm, uid)) + if (!structure || (isNeverCachedName(vm, uid) && !readsPrototypeAccessor(query) && !readsPrivateMemberOfString(query))) return PropertyAnswer::uncacheable(access); if (!query.base.isObject()) { if (query.base.isCell() && vm.aotState().propertyCaches->isSharedByInstances()) @@ -710,7 +727,10 @@ static bool isFunctionWithOrdinaryPut(VM& vm, JSCell* base, Structure* structure { if (isCompiledClosure(base)) { auto* closure = uncheckedDowncast(base); - if (closure->classData() || closure->kind().isConstructor) + if (const LayoutData* classLayout = closure->classData()) { + if (!closure->state().objects->classes.addsStaticUnobserved(vm, *closure, *classLayout, structure, uid)) + return false; + } else if (closure->kind().isConstructor) return false; } else if (base->type() != SlimClosureWithPropertiesType && (base->type() != JSFunctionType || base->classInfo() != JSBoundFunction::info())) return false; @@ -722,6 +742,15 @@ static bool isFunctionWithOrdinaryPut(VM& vm, JSCell* base, Structure* structure return prototype.isNull() || (prototype.isObject() && storesAreOrdinaryAlongPrototypeChain(asObject(prototype)->structure())); } +static bool definesUnobservedStatic(VM& vm, JSCell* base, Structure* structure, UniquedStringImpl* uid) +{ + if (!isCompiledClosure(base) || !structure->isStructureExtensible() || structure->mayHaveImmutableProperties() || Closure::isNameThatBuiltinFunctionsRefuse(vm, uid)) + return false; + auto* closure = uncheckedDowncast(base); + const LayoutData* classLayout = closure->classData(); + return classLayout && closure->state().objects->classes.addsStaticUnobserved(vm, *closure, *classLayout, structure, uid); +} + bool hasOrdinaryDefineForName(VM& vm, JSCell* base, Structure* structure, UniquedStringImpl* uid) { if (!base->isObject() || parseIndex(*uid) || !structure->isStructureExtensible() || structure->isCompiledStructure()) @@ -736,6 +765,18 @@ bool hasOrdinaryDefineForName(VM& vm, JSCell* base, Structure* structure, Unique return structure->classInfoForCells()->methodTable.defineOwnProperty == JSObject::defineOwnProperty && !typeInfo.overridesGetOwnPropertySlot() && !typeInfo.hasStaticPropertyTable(); } +static bool hasOrdinaryPutForName(VM& vm, JSCell* base, Structure* structure, UniquedStringImpl* uid) +{ + JSType type = structure->typeInfo().type(); + if (isTypedView(type)) + return !parseIndex(*uid) && !isCanonicalNumericIndexString(uid); + if (type == DirectArgumentsType || type == ScopedArgumentsType || type == ClonedArgumentsType) + return !parseIndex(*uid) && uid != vm.propertyNames->length.impl() && uid != vm.propertyNames->callee.impl() && uid != vm.propertyNames->iteratorSymbol.impl(); + if (isJSArray(base) || structure->classInfoForCells() == ErrorInstance::info()) + return hasOrdinaryDefineForName(vm, base, structure, uid); + return !structure->typeInfo().overridesPut(); +} + static std::optional classifyReplacement(const StoreQuery& query) { const PutPropertySlot& slot = query.slot; @@ -779,10 +820,9 @@ static std::optional classifyAddition(const StoreQuery& query) PropertyAnswer answer = PropertyAnswer::own(PropertyAnswerKind::Addition, access, offset); answer.newStructure = after; bool isStore = access == SlotAccess::Store; - bool isFunction = isStore && isFunctionWithOrdinaryPut(vm, query.base, before, query.uid); + bool isFunction = isStore ? isFunctionWithOrdinaryPut(vm, query.base, before, query.uid) : definesUnobservedStatic(vm, query.base, before, query.uid); if (isStore) { - bool hasOrdinarySet = isJSArray(query.base) ? hasOrdinaryDefineForName(vm, query.base, before, query.uid) : !before->typeInfo().overridesPut(); - if (!isFunction && !hasOrdinarySet) + if (!isFunction && !hasOrdinaryPutForName(vm, query.base, before, query.uid)) return std::nullopt; ChainWalk walk = walkChainAbove(query); if (storeThroughChain(walk) != StoreThroughChain::CreatesOwnProperty) @@ -874,9 +914,13 @@ static PropertyAnswer classifyFieldStore(const StoreQuery& query, Object& object bool hasMoved = object.structure() != before && !query.slot.isCacheableSetter(); if (hasMoved && !hasMovedBetweenSlotAndExtraProperties(query, object, *before, field)) return uncacheableOnce(access); - if (field.flags & fieldAttributeFlags || field.slot.representation() == SlotRepresentation::DeferredClosure) + bool hasAttributes = field.flags & fieldAttributeFlags; + bool isWritableInSlot = access == SlotAccess::Store && !(field.flags & (fieldIsReadOnly | fieldIsAccessor)) && object.isInSlot(field); + if ((hasAttributes && !isWritableInSlot) || field.slot.representation() == SlotRepresentation::DeferredClosure) return PropertyAnswer::uncacheable(access); PropertyAnswer answer { .kind = PropertyAnswerKind::Field, .access = access, .isOwn = true, .field = &field }; + if (hasAttributes) + return answer; answer.undefinedNeedsNoStorage = field.flags & fieldMayExistWithoutStorage && object.state().objects->layouts.slotAdmits(field, jsUndefined()); answer.fieldTakesAnyValue = access == SlotAccess::Store && !field.isOutOfLine() && field.slot.representation() == SlotRepresentation::JSValue && object.state().objects->layouts.typeOf(field).takesEveryValue(); if (!field.canLeaveSlot() || before->mayBePrototype() || hasMoved) @@ -1052,7 +1096,7 @@ PropertyAnswer classifyStore(const StoreQuery& query) } return answer; } - if (access == SlotAccess::Define && !hasOrdinaryDefineForName(vm, query.base, structureBefore, query.uid)) + if (access == SlotAccess::Define && !hasOrdinaryDefineForName(vm, query.base, structureBefore, query.uid) && !definesUnobservedStatic(vm, query.base, structureBefore, query.uid)) return PropertyAnswer::uncacheable(access); if (access == SlotAccess::Store && isLastIndexOfRegExp(vm, query.base, query.uid) && query.base->structure() == structureBefore) return PropertyAnswer::own(PropertyAnswerKind::LastIndexOfRegExp, access); @@ -1133,14 +1177,13 @@ static Slot slotForFieldOrInherited(const PropertyAnswer& answer, Structure* rec if (std::optional extensionWordOffset = locationOfHiddenWord(compiledStructureOf(receiverStructure).extensionSlot())) slot = Slot::tryCreateFieldOrInheritedField(receiverStructure->id(), flagsOfKind, answer.field->slot, answer.holder, *extensionWordOffset, cageBase); } else { - if (isRead && answer.kind == PropertyAnswerKind::FieldOrInheritedAccessor) + bool keepsOrderWordThatLeavesFieldOut = !!answer.orderWords; + bool readsField = isRead && !keepsOrderWordThatLeavesFieldOut; + if (readsField && answer.kind == PropertyAnswerKind::FieldOrInheritedAccessor) flagsOfKind |= Slot::inheritsGetter; - if (answer.emptySlotMayBeUndefined) - flagsOfKind |= Slot::emptySlotMayBeUndefined; - bool keepsOrderWordThatLeavesFieldOut = !isRead && answer.orderWords; - if (keepsOrderWordThatLeavesFieldOut) + if (answer.emptySlotMayBeUndefined || keepsOrderWordThatLeavesFieldOut) flagsOfKind |= Slot::emptySlotMayBeUndefined; - slot = Slot::tryCreateFieldAndHolder(receiverStructure->id(), isRead ? Slot::Kind::FieldOrInherited : Slot::Kind::FieldOrInheritedSetter, flagsOfKind, keepsOrderWordThatLeavesFieldOut ? SlotData::decode(answer.orderWords->first) : answer.field->slot, answer.holder, answer.offset, cageBase); + slot = Slot::tryCreateFieldAndHolder(receiverStructure->id(), readsField ? Slot::Kind::FieldOrInherited : Slot::Kind::FieldOrInheritedAccessor, flagsOfKind, keepsOrderWordThatLeavesFieldOut ? SlotData::decode(answer.orderWords->first) : answer.field->slot, answer.holder, answer.offset, cageBase); } return slot ? *slot : slotForField(answer, receiverStructure, Slot::Kind::Field, flags); } diff --git a/Source/JavaScriptCore/aot/runtime/AOTPropertyCachesOperations.cpp b/Source/JavaScriptCore/aot/runtime/AOTPropertyCachesOperations.cpp index d9b7eb6e43d45..9967f7d0e30fc 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTPropertyCachesOperations.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTPropertyCachesOperations.cpp @@ -483,10 +483,29 @@ JSC_DEFINE_JIT_OPERATION(operationAOTInstanceOfAndCache, size_t, (Instance* inst OPERATION_RETURN(scope, instanceOfAndCache(instance, JSValue::decode(encodedValue), JSValue::decode(encodedConstructor))); } +static bool isAcceptedBinaryData(Instance* instance, JSValue value, JSValue prototype) +{ + if (!instance->state->acceptsForeignBinaryData || !value.isCell() || !prototype.isObject()) + return false; + JSGlobalObject* realm = instance->globalObject; + JSType type = value.asCell()->type(); + JSObject* constructor = nullptr; + if (isTypedArrayType(type)) + constructor = realm->typedArrayConstructor(typedArrayType(type)); + else if (type == ArrayBufferType && !uncheckedDowncast(value.asCell())->isShared()) + constructor = realm->arrayBufferConstructor(ArrayBufferSharingMode::Default); + VM& vm = *instance->vm; + return constructor && constructor->getDirect(vm, vm.propertyNames->prototype) == prototype && constructor->getDirect(vm, vm.propertyNames->hasInstanceSymbol); +} + JSC_DEFINE_JIT_OPERATION(operationAOTDefaultHasInstance, size_t, (Instance* instance, EncodedJSValue encodedValue, EncodedJSValue encodedPrototype)) { AOT_OPERATION_BEGIN(instance, Entry::operationAOTDefaultHasInstance); - OPERATION_RETURN(scope, JSObject::defaultHasInstance(globalObject, JSValue::decode(encodedValue), JSValue::decode(encodedPrototype))); + JSValue value = JSValue::decode(encodedValue); + JSValue prototype = JSValue::decode(encodedPrototype); + bool isInstance = JSObject::defaultHasInstance(globalObject, value, prototype); + OPERATION_RETURN_IF_EXCEPTION(scope, false); + OPERATION_RETURN(scope, isInstance || isAcceptedBinaryData(instance, value, prototype)); } static bool isPlainObjectBelowNoIndexedProperties(Structure* structure) @@ -863,6 +882,10 @@ static bool hasAndCache(Instance* instance, JSObject* object, const CachedName& bool shouldFile = (isIn || object->type() != ProxyObjectType) && caches.shouldFile(object->structureID(), name.uid, SlotAccess::Read); bool starsWait = hasStarsThatWait(object); shouldFile &= !starsWait; + if (shouldFile) { + std::optional known = caches.tables().find(object->structureID(), name, SlotAccess::Read); + shouldFile = !known || known->kind != Slot::Kind::Uncacheable; + } if (shouldFile) { Object::ensureStructureForExtraPropertiesIfNeeded(vm, object); createClassObjectsOnChain(vm, object); diff --git a/Source/JavaScriptCore/aot/runtime/AOTPropertyCachesStringKeys.h b/Source/JavaScriptCore/aot/runtime/AOTPropertyCachesStringKeys.h index 442c77c9fa131..9f5d308027cc7 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTPropertyCachesStringKeys.h +++ b/Source/JavaScriptCore/aot/runtime/AOTPropertyCachesStringKeys.h @@ -15,10 +15,12 @@ #include "MarkedBlockCage.h" #include "RememberedSlot.h" #include +#include #include #include #include #include +#include #include #include #include @@ -171,11 +173,12 @@ class StringKeyCache { static constexpr unsigned keyOfPairSetShift = 6; static_assert(waysOfKeysOfPairs * sizeof(KeyOfPair) == 1 << keyOfPairSetShift); static constexpr uint32_t keyOfPairMultiplier = 0x9e3779b1u; + static constexpr uint32_t keyOfPairMultiplierOfSecond = 0x213bca7fu; static constexpr unsigned keyOfPairHashShift = 13; static constexpr uint64_t keyOfPairMaskFor(size_t tableSize) { return (tableSize / waysOfKeysOfPairs - 1) << keyOfPairSetShift; } static constexpr uint32_t hashOfPair(EncodedJSValue first, EncodedJSValue second) { - uint32_t mixed = static_cast(static_cast(first) + (static_cast(second) << 3)); + uint32_t mixed = static_cast(first) + static_cast(second) * keyOfPairMultiplierOfSecond; return mixed * keyOfPairMultiplier >> keyOfPairHashShift; } JS_EXPORT_PRIVATE static const KeyOfPair* noKeysOfPairs(); @@ -196,6 +199,143 @@ class StringKeyCache { } JSString* textOfInt32(VM&, int32_t); + struct FewCharacters { + static constexpr uint32_t bytesOfWord = sizeof(uint64_t); + static constexpr uint32_t mostCharacters = 2 * bytesOfWord; + static constexpr uint32_t limitOfIntegers = 100000000; + + ALWAYS_INLINE bool tryAppend(uint64_t text, uint32_t size) + { + ASSERT(size && size <= sizeof(uint64_t)); + uint32_t end = length + size; + if (end > mostCharacters) + return false; + if (length >= bytesOfWord) + last |= text << (8 * (length - bytesOfWord)); + else { + first |= text << (8 * length); + if (end > bytesOfWord) + last = text >> (8 * (bytesOfWord - length)); + } + length = end; + return true; + } + + ALWAYS_INLINE bool tryAppend(std::span text) + { + const Latin1Character* from = text.data(); + size_t size = text.size(); + if (size == 1) + return tryAppend(*from, 1); + if (size > mostCharacters - length) + return false; + if (size >= sizeof(uint64_t)) { + if (!tryAppend(WTF::unalignedLoad(from), sizeof(uint64_t))) + return false; + uint32_t rest = static_cast(size - sizeof(uint64_t)); + return !rest || tryAppend(WTF::unalignedLoad(from + rest) >> (8 * (sizeof(uint64_t) - rest)), rest); + } + if (size >= sizeof(uint32_t)) + return tryAppend(WTF::unalignedLoad(from) | static_cast(WTF::unalignedLoad(from + size - sizeof(uint32_t))) << (8 * (size - sizeof(uint32_t))), static_cast(size)); + return !size || tryAppend(from[0] | static_cast(from[size / 2]) << (8 * (size / 2)) | static_cast(from[size - 1]) << (8 * (size - 1)), static_cast(size)); + } + + ALWAYS_INLINE bool tryAppendInteger(uint32_t integer) + { + constexpr uint32_t limitOfTwoDigits = 100; + if (integer - limitOfTwoDigits >= limitOfIntegers - limitOfTwoDigits) { + if (integer >= limitOfIntegers) + return false; + uint32_t tens = integer * 103 >> 10; + uint32_t digits = (tens | (integer - 10 * tens) << 8) | 0x3030; + uint32_t leadingZeros = integer < 10; + return tryAppend(digits >> (8 * leadingZeros), 2 - leadingZeros); + } + if (integer < 10000) { + uint32_t pairs = integer / 100 | (integer % 100) << 16; + uint32_t tens = pairs * 103 >> 10 & 0x000f000f; + uint32_t digits = (tens | (pairs - 10 * tens) << 8) | 0x30303030; + uint32_t leadingZeros = integer < 1000; + return tryAppend(digits >> (8 * leadingZeros), 4 - leadingZeros); + } + uint64_t halves = integer / 10000 | static_cast(integer % 10000) << 32; + uint64_t hundreds = halves * 10486 >> 20 & 0x0000007f0000007full; + uint64_t pairs = hundreds | (halves - 100 * hundreds) << 16; + uint64_t tens = pairs * 103 >> 10 & 0x000f000f000f000full; + uint64_t digits = tens | (pairs - 10 * tens) << 8; + uint32_t leadingZeros = static_cast(std::countr_zero(digits)) / 8; + return tryAppend((digits | 0x3030303030303030ull) >> (8 * leadingZeros), 8 - leadingZeros); + } + + uint64_t byteAt(uint32_t offset) const { return static_cast(first >> (8 * offset)); } + std::array text() const { return std::bit_cast>(std::array { first, last }); } + ALWAYS_INLINE uint64_t lastEightBytes() const + { + ASSERT(length >= bytesOfWord); + uint32_t bytesBehindFirst = length - bytesOfWord; + return bytesBehindFirst == bytesOfWord ? last : first >> (8 * bytesBehindFirst) | (last << 1) << (8 * (bytesOfWord - bytesBehindFirst) - 1); + } + + ALWAYS_INLINE unsigned hash() const + { + ASSERT(length && length <= mostCharacters); + constexpr auto mix = [](uint64_t a, uint64_t b) ALWAYS_INLINE_LAMBDA { + UInt128 product = static_cast(a) * b; + return std::pair { static_cast(product), static_cast(product >> 64) }; + }; + constexpr std::array secret = RapidHash::secret; + constexpr auto seeds = mix(secret[0], secret[1]); + uint64_t a; + uint64_t b; + if (length >= bytesOfWord) [[likely]] { + uint64_t end = lastEightBytes(); + a = first << 32 | end >> 32; + b = first >> 32 << 32 | static_cast(end); + } else if (length >= sizeof(uint32_t)) + a = b = first << 32 | static_cast(first >> (8 * (length - sizeof(uint32_t)))); + else { + a = byteAt(0) << 56 | byteAt(length / 2) << 32 | byteAt(length - 1); + b = 0; + } + auto [low, high] = mix(a ^ secret[1], b ^ seeds.first ^ seeds.second ^ length); + auto [lowOfHash, highOfHash] = mix(low ^ secret[0] ^ length, high ^ secret[1]); + unsigned hash = static_cast(lowOfHash ^ highOfHash) & RapidHash::maskHash; + return hash ? hash : 0x80000000 >> RapidHash::flagCount; + } + + ALWAYS_INLINE bool isTextOf(const StringImpl& impl) const + { + ASSERT(length >= 2); + if (impl.length() != length || !impl.is8Bit()) + return false; + const Latin1Character* other = impl.span8().data(); + if (length >= bytesOfWord) + return WTF::unalignedLoad(other) == first && WTF::unalignedLoad(other + length - bytesOfWord) == lastEightBytes(); + if (length >= sizeof(uint32_t)) + return WTF::unalignedLoad(other) == static_cast(first) && WTF::unalignedLoad(other + length - sizeof(uint32_t)) == static_cast(first >> (8 * (length - sizeof(uint32_t)))); + return other[0] == byteAt(0) && other[length / 2] == byteAt(length / 2) && other[length - 1] == byteAt(length - 1); + } + + uint64_t first { 0 }; + uint64_t last { 0 }; + uint32_t length { 0 }; + }; + ALWAYS_INLINE JSString* tryFindCachedKey(const FewCharacters& characters, unsigned hash) const + { + size_t tableSize = m_cachedKeys.size(); + if (!tableSize) [[unlikely]] + return nullptr; + const uint64_t* pair = m_cachedKeys.span().data() + firstIndex(hash, tableSize); + for (size_t i = 0; i < 2; ++i) { + if (pair[i] >> 32 != hash) + continue; + JSString* candidate = uncheckedDowncast(MarkedBlockCage::decode(m_cageBase, static_cast(pair[i]))); + if (characters.isTextOf(*candidate->tryGetValueImpl())) + return candidate; + } + return nullptr; + } + JSString* findOrAdd(VM&, std::span); ALWAYS_INLINE JSString* tryFindCachedKey(std::span characters, unsigned hash) const { diff --git a/Source/JavaScriptCore/aot/runtime/AOTPropertyCachesWords.h b/Source/JavaScriptCore/aot/runtime/AOTPropertyCachesWords.h index 51cf8fe18d478..e3b279f67ac81 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTPropertyCachesWords.h +++ b/Source/JavaScriptCore/aot/runtime/AOTPropertyCachesWords.h @@ -152,6 +152,11 @@ struct NameOfText { UniquedStringImpl* uid; }; +struct UsesOfSubstringAsKey { + static constexpr uint8_t one = 1 << 3; + static constexpr uint8_t two = 2 * one; +}; + struct KeyedValuePlace { static constexpr unsigned numberOfPlacesPerAccess = 512; static constexpr unsigned numberOfPlaces = 2 * numberOfPlacesPerAccess; diff --git a/Source/JavaScriptCore/aot/runtime/AOTRealmStartup.h b/Source/JavaScriptCore/aot/runtime/AOTRealmStartup.h index 0caac29b2419b..a5d287f2510d8 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTRealmStartup.h +++ b/Source/JavaScriptCore/aot/runtime/AOTRealmStartup.h @@ -58,6 +58,7 @@ using BufferStructureProvider = Structure* (*)(JSGlobalObject*); JS_EXPORT_PRIVATE void setBufferStructureProvider(BufferStructureProvider); JS_EXPORT_PRIVATE void hostDidCreateConstructor(JSGlobalObject*, JSObject* constructor); JS_EXPORT_PRIVATE void hostDidCreateNodeModule(JSGlobalObject*, JSObject* constructor); +JS_EXPORT_PRIVATE void acceptForeignBinaryData(JSGlobalObject*); JS_EXPORT_PRIVATE void hostDidCreateGlobalMember(JSGlobalObject*, UniquedStringImpl* key, JSObject*); JS_EXPORT_PRIVATE void hostDidFillObjectWithFoldedMembers(JSGlobalObject*, JSObject*, ASCIILiteral holder, std::span names); diff --git a/Source/JavaScriptCore/aot/runtime/AOTRegExpOperations.cpp b/Source/JavaScriptCore/aot/runtime/AOTRegExpOperations.cpp index 45b475bc0fa61..2c365a3323c17 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTRegExpOperations.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTRegExpOperations.cpp @@ -412,10 +412,13 @@ static RegExpObject* ensureSharedRegExpLiteral(VM& vm, Instance* instance, uint3 JSC_DEFINE_JIT_OPERATION(operationAOTNewRegExpLiteral, JSCell*, (Instance* instance, uint32_t regExpRecord, uint32_t flags, uint32_t staticsOffset)) { AOT_OPERATION_BEGIN(instance, Entry::operationAOTNewRegExpLiteral); - RegExp* regExp = vm.aotState().regExpLiterals->regExpFor(vm, RegExpRecordID(regExpRecord), OptionSet::fromRaw(flags)); - ASSERT(regExp->isValid()); - if (staticsOffset != noSharedRegExpOffset) + RegExpLiterals& literals = *vm.aotState().regExpLiterals; + RegExp* regExp = literals.regExpIfCreated(RegExpRecordID(regExpRecord), OptionSet::fromRaw(flags)); + if (regExp && staticsOffset != noSharedRegExpOffset) ensureSharedRegExpLiteral(vm, instance, staticsOffset, regExp); + if (!regExp) + regExp = literals.regExpFor(vm, RegExpRecordID(regExpRecord), OptionSet::fromRaw(flags)); + ASSERT(regExp->isValid()); Structure* structure = WTF::opaque(globalObject->regExpStructure()); prepareInlineAllocation(instance, InlineAllocation::RegExpObject, structure); OPERATION_RETURN(scope, RegExpObject::create(vm, structure, regExp)); @@ -886,18 +889,37 @@ RegExpLiterals::RegExpLiterals(CacheRegistry& registry) registry.add(CacheKind::StrongRoots, "regexp literals"_s, *this, RootVolatility::GreyedByExecution); } -RegExp* RegExpLiterals::regExpFor(VM& vm, RegExpRecordID id, OptionSet flags) +uint64_t RegExpLiterals::keyOf(RegExpRecordID id, OptionSet flags) { - if (!m_records) [[unlikely]] - m_records = &RegExpRecords(*vm.aotState().image).at(id) - id.index(); - const ImageRegExp& record = m_records[id.index()]; static_assert(sizeof(flags.toRaw()) <= sizeof(uint32_t)); - uint64_t key = static_cast(id.index()) << 32 | flags.toRaw(); - if (auto known = m_cells.find(key); known != m_cells.end()) - return known->value; + return static_cast(id.index()) << 32 | flags.toRaw(); +} + +RegExp* RegExpLiterals::regExpIfCreated(RegExpRecordID id, OptionSet flags) const +{ + uint32_t firstCell = m_records ? m_firstCellOfRecord[id.index()] : 0; + if (!firstCell) + return nullptr; + if (RegExp* first = m_cellsInOrderOfCreation[firstCell - 1]; first->flags() == flags) [[likely]] + return first; + return m_cellsWithOtherFlags.get(keyOf(id, flags)); +} +RegExp* RegExpLiterals::regExpFor(VM& vm, RegExpRecordID id, OptionSet flags) +{ + if (RegExp* known = regExpIfCreated(id, flags)) + return known; + if (!m_records) [[unlikely]] { + RegExpRecords records(*vm.aotState().image); + m_records = &records.at(id) - id.index(); + m_firstCellOfRecord.fill(0, records.size()); + } + const ImageRegExp& record = m_records[id.index()]; RegExp* regExp = RegExp::createFromImage(vm, regExpPatternOf(vm, record), flags, record); - m_cells.add(key, regExp); + if (uint32_t& firstCell = m_firstCellOfRecord[id.index()]) + m_cellsWithOtherFlags.add(keyOf(id, flags), regExp); + else + firstCell = static_cast(m_cellsInOrderOfCreation.size() + 1); m_cellsInOrderOfCreation.append(regExp); return regExp; } diff --git a/Source/JavaScriptCore/aot/runtime/AOTRegExpRecords.h b/Source/JavaScriptCore/aot/runtime/AOTRegExpRecords.h index ce7d856815c4a..b98d85bf11a68 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTRegExpRecords.h +++ b/Source/JavaScriptCore/aot/runtime/AOTRegExpRecords.h @@ -66,13 +66,17 @@ class RegExpLiterals { public: explicit RegExpLiterals(CacheRegistry&); + RegExp* regExpIfCreated(RegExpRecordID, OptionSet) const; JS_EXPORT_PRIVATE RegExp* regExpFor(VM&, RegExpRecordID, OptionSet); template void visitRoots(Visitor&); private: + static uint64_t keyOf(RegExpRecordID, OptionSet); + const ImageRegExp* m_records { nullptr }; - UncheckedKeyHashMap, WTF::UnsignedWithZeroKeyHashTraits> m_cells; + Vector m_firstCellOfRecord; + UncheckedKeyHashMap, WTF::UnsignedWithZeroKeyHashTraits> m_cellsWithOtherFlags; Vector m_cellsInOrderOfCreation; size_t m_numberOfCellsVisited { 0 }; uint32_t m_lastFullVisitVersion { 0 }; diff --git a/Source/JavaScriptCore/aot/runtime/AOTRuntimeCheckFailure.cpp b/Source/JavaScriptCore/aot/runtime/AOTRuntimeCheckFailure.cpp index 55d96a2ceb34d..52168732c1e6d 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTRuntimeCheckFailure.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTRuntimeCheckFailure.cpp @@ -42,7 +42,7 @@ static String describePosition(const Image& image, const TraceFrame& frame) std::optional position = positionOf(frame); if (!position || !position->source) return { }; - return makeString(sourceURLOf(position->source), ':', position->lineColumn.line, ':', position->lineColumn.column); + return makeString(displayPathOf(position->source), ':', position->lineColumn.line, ':', position->lineColumn.column); } RuntimeCheckFailure& RuntimeCheckFailure::atReturnAddress(const void* returnAddress) diff --git a/Source/JavaScriptCore/aot/runtime/AOTRuntimeFunctions.h b/Source/JavaScriptCore/aot/runtime/AOTRuntimeFunctions.h index 3d69ea499ff96..3ebd56ddb53ee 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTRuntimeFunctions.h +++ b/Source/JavaScriptCore/aot/runtime/AOTRuntimeFunctions.h @@ -63,6 +63,8 @@ EnumeratedKey findEnumeratedKey(JSGlobalObject*, JSPropertyNameEnumerator*, JSVa void initializeIterationWords(Instance&); void appendElementsOfList(JSGlobalObject*, JSArray* target, JSArray* source); +bool tryAppendCharacters(Instance&, JSArray* target, JSString*); +JSString* stringOfCharacter(Instance&, char16_t); void appendMembersOfSet(JSGlobalObject*, JSArray* target, JSSet* source); bool tryAppendPairsOfMap(JSGlobalObject*, JSArray* target, JSMap* source); bool tryAppendPairsLeftIn(JSGlobalObject*, JSArray* target, JSMapIterator*); diff --git a/Source/JavaScriptCore/aot/runtime/AOTRuntimeHooks.cpp b/Source/JavaScriptCore/aot/runtime/AOTRuntimeHooks.cpp index 77692cb839c8a..ace78a98ed146 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTRuntimeHooks.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTRuntimeHooks.cpp @@ -16,6 +16,7 @@ #include "AOTGenerator.h" #include "AOTInstance.h" #include "AOTRuntimeCheckFailure.h" +#include "AOTState.h" #include "BuiltinExecutables.h" #include "ImmutablePropertyNames.h" #include "JSCInlines.h" @@ -72,6 +73,28 @@ bool rejectsNativeConstructorProperty(JSGlobalObject* realm, PropertyName name) return !name.isPrivateName() && Closure::isNameThatBuiltinFunctionsRefuse(realm->vm(), name.uid()) && locksBuiltins(realm); } +static bool isBinaryDataClass(JSGlobalObject* realm, JSObject* constructor) +{ + if (realm->arrayBufferConstructor(ArrayBufferSharingMode::Default) == constructor) + return true; +#define RETURN_IF_CONSTRUCTOR(name) \ + if (realm->typedArrayConstructorConcurrently(Type##name) == constructor) \ + return true; + FOR_EACH_TYPED_ARRAY_TYPE_EXCLUDING_DATA_VIEW(RETURN_IF_CONSTRUCTOR) +#undef RETURN_IF_CONSTRUCTOR + return false; +} + +bool acceptsHasInstanceOfHost(InternalFunction* constructor, PropertyName name, JSValue method) +{ + JSGlobalObject* realm = constructor->globalObject(); + Instance* instance = realm->aotInstance(); + if (name != realm->vm().propertyNames->hasInstanceSymbol || !method || !instance || !instance->state || !instance->state->acceptsForeignBinaryData) + return false; + auto* function = dynamicDowncast(method); + return function && function->isBuiltinFunction() && isBinaryDataClass(realm, constructor); +} + void crashOnHavingABadTime() { RuntimeCheckFailure failure("array prototype check failed"_s); diff --git a/Source/JavaScriptCore/aot/runtime/AOTRuntimeHooks.h b/Source/JavaScriptCore/aot/runtime/AOTRuntimeHooks.h index b9fdc9f01e3bd..76cd0713ff455 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTRuntimeHooks.h +++ b/Source/JavaScriptCore/aot/runtime/AOTRuntimeHooks.h @@ -15,7 +15,9 @@ namespace JSC { class InlineWatchpointSet; class JSArray; class JSCell; +class InternalFunction; class JSFunction; +class JSValue; class ObjectPropertyCondition; class PropertyName; @@ -40,6 +42,7 @@ using AOT::takePreallocatedStaticProperty; void willInitializeRealm(JSGlobalObject*); Structure* structureForHostFunctions(JSGlobalObject*, Structure*); bool rejectsNativeConstructorProperty(JSGlobalObject*, PropertyName); +bool acceptsHasInstanceOfHost(InternalFunction*, PropertyName, JSValue method); inline bool locksNativeConstructorPrototypes(JSGlobalObject* realm) { return locksBuiltins(realm); } String lockedBuiltinPropertyErrorMessage(PropertyName); String lockedBuiltinPrototypeErrorMessage(); diff --git a/Source/JavaScriptCore/aot/runtime/AOTRuntimeRecords.h b/Source/JavaScriptCore/aot/runtime/AOTRuntimeRecords.h index 7357ef663d0ab..3424cab0860ae 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTRuntimeRecords.h +++ b/Source/JavaScriptCore/aot/runtime/AOTRuntimeRecords.h @@ -97,7 +97,7 @@ enum class SlotKindFlag : uint8_t { v(FieldGetter, Field, HiddenWordOffset, ReadsSlot, AbsentIsUndefined) \ v(FieldCreation, Field, HiddenWordOffset, Unknown, None) \ v(FieldOrInherited, FieldAndHolder, WordIndex, Unknown, InheritsGetter) \ - v(FieldOrInheritedSetter, FieldAndHolder, WordIndex, Unknown, None) \ + v(FieldOrInheritedAccessor, FieldAndHolder, WordIndex, Unknown, None) \ v(InheritedField, Holder, None, Unknown, None) \ v(FieldOrInheritedField, FieldAndHolder, HiddenWordOffset, Unknown, None) \ v(Variable, Field, HiddenWordOffset, Present, None) \ @@ -309,7 +309,7 @@ struct alignas(16) Slot { { static_assert(OBJECT_OFFSETOF(FieldPayload, slot) == OBJECT_OFFSETOF(FieldAndHolderPayload, slot)); ASSERT(payloadOf(kind) == SlotPayload::Field || payloadOf(kind) == SlotPayload::FieldAndHolder); - ASSERT(kind != Kind::FieldOrInheritedSetter || !has(emptySlotMayBeUndefined)); + ASSERT(kind != Kind::FieldOrInheritedAccessor || !has(emptySlotMayBeUndefined)); return field.slot; } bool namesOnlyLiveCells(const Invocable auto& isMarked, uintptr_t cageBase) const; diff --git a/Source/JavaScriptCore/aot/runtime/AOTRuntimeTable.h b/Source/JavaScriptCore/aot/runtime/AOTRuntimeTable.h index 72b7459cf15f8..861d649246ce7 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTRuntimeTable.h +++ b/Source/JavaScriptCore/aot/runtime/AOTRuntimeTable.h @@ -43,6 +43,7 @@ enum class LeafPresence : size_t { TakeSlowPath, Absent, Inherited, Own }; v(VirtualCallTrampoline) \ v(ErrorConstructorClassInfo) \ v(BooleanConstructorClassInfo) \ + v(CustomGetterClassInfo) \ v(NumberConstructorClassInfo) \ v(StringConstructorClassInfo) \ v(NonConstructorHostFunction) \ @@ -294,6 +295,7 @@ constexpr ArgumentBufferAccess argumentBufferAccessOf(Entry entry) v(ArrayMutationOperations, ArraySpeciesCreate) \ v(ArrayMutationOperations, ArraySplice) \ v(ArrayMutationOperations, ArrayUnshift) \ + v(ArrayMutationOperations, SortNumbers) \ v(ArrayMutationOperations, SetArrayLength) \ v(ArrayMutationOperations, SetArrayLengthInSloppyCode) \ v(ArraySearchOperations, ArrayIndexOfDouble) \ @@ -327,11 +329,13 @@ constexpr ArgumentBufferAccess argumentBufferAccessOf(Entry entry) v(ArrayCreationOperations, ThrowIfTooLongForList) \ v(ArrayCreationOperations, ThrowNotIterable) \ v(ArrayCreationOperations, TruncateNewArray) \ + v(StringSearchOperations, StringFindLiteral16) \ v(StringSearchOperations, StringFindLiteral8) \ v(StringSearchOperations, StringIndexOfCharacter) \ v(StringSearchOperations, StringIndexOfLiteral) \ v(StringSearchOperations, StringLastIndexOf) \ v(StringSearchOperations, StringLastIndexOfLiteral) \ + v(StringSearchOperations, StringReverseFindLiteral16) \ v(StringSearchOperations, StringReverseFindLiteral8) \ v(StringCreationOperations, CharToString) \ v(StringCreationOperations, CodeURIOfAnyValue) \ @@ -349,6 +353,7 @@ constexpr ArgumentBufferAccess argumentBufferAccessOf(Entry entry) v(StringCreationOperations, ToString) \ v(StringComparisonOperations, CompareStrings) \ v(StringComparisonOperations, NarrowCharactersOfString) \ + v(StringComparisonOperations, StringCodeUnitAt) \ v(StringComparisonOperations, StringEqualsLiteral) \ v(StringComparisonOperations, StringEqualsLiteralInRegister) \ v(StringComparisonOperations, StringsAreEqual) \ @@ -518,6 +523,8 @@ class RuntimeTable { uint32_t m_compactTableIterationEntry { 0 }; }; +inline constexpr int64_t noOrderOfStringsInPlace = 2; + } } // namespace JSC::AOT #endif // ENABLE(AOT) diff --git a/Source/JavaScriptCore/aot/runtime/AOTRuntimeWords.h b/Source/JavaScriptCore/aot/runtime/AOTRuntimeWords.h index 9aad57578b1b5..cb35478429631 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTRuntimeWords.h +++ b/Source/JavaScriptCore/aot/runtime/AOTRuntimeWords.h @@ -28,10 +28,11 @@ namespace AOT { struct ExceptionWords { EncodedJSValue thrownValue; void* stackPointerOfHandler; + const void* returnAddressOfAwaitedCall; uint32_t pollLeavesOwedError; uint32_t stateOfGuessesThatFail; }; -static_assert(sizeof(ExceptionWords) == 24); +static_assert(sizeof(ExceptionWords) == 32); struct GlobalBinding { EncodedJSValue constant; diff --git a/Source/JavaScriptCore/aot/runtime/AOTStackTrace.cpp b/Source/JavaScriptCore/aot/runtime/AOTStackTrace.cpp index 37570a04a7d85..d323ddd629585 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTStackTrace.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTStackTrace.cpp @@ -147,14 +147,19 @@ String FunctionDescription::stackTraceName(VM& vm) const { if (isOfCalledBuiltin()) return nameOfBuiltin(vm, builtinOfFunctionInFrames(m_function)); - return Image::current()->isObfuscated() ? emptyString() : functionName(*this, NameForm::InFrames); + if (Image::current()->isObfuscated()) + return emptyString(); + String name = functionName(*this, NameForm::InFrames); + return name.isEmpty() ? computedNameInFrames(vm, *this) : name; } String FunctionDescription::calculatedDisplayName(VM& vm) const { if (isOfCalledBuiltin()) return nameOfBuiltin(vm, builtinOfFunctionInFrames(m_function)); - if (Image::current()->isObfuscated() || isNamedByComputedKey()) + if (Image::current()->isObfuscated()) + return emptyString(); + if (isNamedByComputedKey()) return emptyString(); return functionName(*this, NameForm::InFrames); } @@ -226,8 +231,9 @@ struct RunStart { class RunWalk { public: - RunWalk(const VMState& state, const RunStart& start) - : m_cache(*state.callSiteCache) + RunWalk(const VMState& state, const RunStart& start, const CallFrame* frameWithAwaitingCallers) + : m_frameWithAwaitingCallers(frameWithAwaitingCallers) + , m_cache(*state.callSiteCache) , m_image(*state.image) , m_stackPointer(start.stackPointer) , m_framePointer(start.framePointer) @@ -279,6 +285,7 @@ class RunWalk { m_isAtFirstReadSite = false; m_plainFunctionConstructFrame = nullptr; m_constructFrameMakesThis = false; + m_hidesFrames |= m_callFrame == m_frameWithAwaitingCallers; if (passOverFrame()) findFunctionFrame(); } @@ -336,6 +343,7 @@ class RunWalk { break; case StubFrameKind::AsyncGeneratorBody: m_isAmongFramesThatWait |= callersWaitForStep(m_callFrame); + m_hidesFrames |= m_isAmongFramesThatWait; break; case StubFrameKind::Adapter: case StubFrameKind::HandlerOfJob: @@ -380,7 +388,7 @@ class RunWalk { } m_callFrame = frameAtReturnSite(m_site->frame, m_stackPointer, m_framePointer); if (!m_site->isStubReturnSite()) [[likely]] { - if (!m_isAmongFramesThatWait) [[likely]] + if (!m_hidesFrames) [[likely]] return; if (!passOverFrame()) return; @@ -391,6 +399,7 @@ class RunWalk { } } + const CallFrame* m_frameWithAwaitingCallers; CallSiteCache& m_cache; const Image& m_image; void* m_stackPointer; @@ -402,6 +411,7 @@ class RunWalk { EntryFrame* m_entryFrame; bool m_isAtFirstReadSite; bool m_isAmongFramesThatWait { false }; + bool m_hidesFrames { false }; const CallSiteCache::Entry* m_site { nullptr }; CompiledFrameExit m_exit; CompiledFrameIterator::RunEnd m_runEnd; @@ -599,6 +609,15 @@ static std::optional findRunBelowHostFunction(VM& vm, TopFramePolicy t return RunStart { callerStackPointer(top), top->callerFrame(), top->rawReturnPC(), vm.topEntryFrame, false, true }; } +static bool isTopFrameAlone(VM& vm) +{ + CallFrame* top = vm.topCallFrame; + if (top->isZombieFrame() || top->isNativeCalleeFrame() || top->codeBlock() || top->callerFrameOrEntryFrame() != vm.topEntryFrame) + return false; + VMEntryRecord* entry = vmEntryRecord(vm.topEntryFrame); + return !entry->prevTopCallFrame() && !entry->prevTopEntryFrame(); +} + static NEVER_INLINE std::optional findRunBelowOtherFrames(VM& vm, JSCell* owner, WTF::Vector& framesOnTop, size_t maxStackSize) { countEvent(vm, RuntimeCounterID::StackWalksToCompiledFrames); @@ -615,6 +634,17 @@ static NEVER_INLINE Instance* findInstanceAmongSeveral(VM& vm, const RunStart& s return frames && !frames->isAtEnd() ? frames->instance() : nullptr; } +static ALWAYS_INLINE const CallFrame* findFrameWithAwaitingCallers(VM& vm, const RunStart& start) +{ + const Instance* instance = vm.aotState().onlyInstance; + if (!instance) [[unlikely]] { + instance = findInstanceAmongSeveral(vm, start); + if (!instance) + return nullptr; + } + return std::bit_cast(frameWithAwaitingCallers(*instance)); +} + static ALWAYS_INLINE void appendFramesBelow(VM& vm, JSCell* owner, WTF::Vector& results, const RunWalk& walk, bool hasRoom, JSC::Interpreter::StackTraceStart start, size_t maxStackSize) { auto reserve = [&] { @@ -681,7 +711,7 @@ static ALWAYS_INLINE ReturnAddressTrace* tryCaptureRun(VM& vm, JSCell* owner, si otherFrames.reserveInitialCapacity(1); otherFrames.append(StackFrame(vm, owner, vm.topCallFrame->callee().asCell())); } else - start = findRunBelowOtherFrames(vm, owner, otherFrames, maxStackSize); + start = isTopFrameAlone(vm) ? std::nullopt : findRunBelowOtherFrames(vm, owner, otherFrames, maxStackSize); if (!start) return nullptr; framesAbove = { otherFrames.size(), true }; @@ -693,7 +723,7 @@ static ALWAYS_INLINE ReturnAddressTrace* tryCaptureRun(VM& vm, JSCell* owner, si return nullptr; } - RunWalk walk(vm.aotState(), *start); + RunWalk walk(vm.aotState(), *start, std::bit_cast(frameWithAwaitingCallers(*instance))); if (walk.isAtEnd()) return nullptr; bool calleeWasEntered = start->calleeWasEntered || walk.constructFrameLeftMakesThis(); @@ -1027,7 +1057,8 @@ IterationStatus appendShownFramesOfRun(VM& vm, CompiledFrameIterator& frames, Sh } if (frames.isAmongFramesThatWait()) return IterationStatus::Continue; - RunWalk walk(vm.aotState(), { frames.stackPointer(), frames.framePointer(), const_cast(frames.callSite().address), frames.entryFrame(), frames.isAtFirstReadSite(), frames.calleeWasEntered(), frames.plainFunctionConstructFrameLeft(), frames.constructFrameLeftMakesThis() }); + RunStart start { frames.stackPointer(), frames.framePointer(), const_cast(frames.callSite().address), frames.entryFrame(), frames.isAtFirstReadSite(), frames.calleeWasEntered(), frames.plainFunctionConstructFrameLeft(), frames.constructFrameLeftMakesThis() }; + RunWalk walk(vm.aotState(), start, std::bit_cast(frameWithAwaitingCallers(*frames.instance()))); if (!appendShownFramesOfRun(vm, walk, frames.calleeWasEntered(), filter, results, listing)) return IterationStatus::Done; frames.leaveRunAt(walk.exit(), walk.runEnd()); @@ -1045,7 +1076,7 @@ void forEachShownFrameOfRunAtTop(VM& vm, HiddenFrameOnTop hiddenFrameOnTop, cons start = findRunBelowHostFunction(vm, TopFramePolicy::Skip); if (!start) return; - RunWalk walk(vm.aotState(), *start); + RunWalk walk(vm.aotState(), *start, findFrameWithAwaitingCallers(vm, *start)); if (walk.isAtEnd()) return; ShownFrameFilter filter = hiddenFrameOnTop == HiddenFrameOnTop::IsShown ? ShownFrameFilter::showingWhateverIsOnTop(isBelowHostFunction) : ShownFrameFilter(isBelowHostFunction); @@ -1067,7 +1098,7 @@ bool tryGetStackTrace(VM& vm, JSCell* owner, WTF::Vector& results, s if (!start) return false; } - RunWalk walk(vm.aotState(), *start); + RunWalk walk(vm.aotState(), *start, findFrameWithAwaitingCallers(vm, *start)); if (walk.isAtEnd()) return false; @@ -1168,6 +1199,11 @@ String sourceURLOf(ModuleID module) return module ? moduleFilePath(module) : emptyString(); } +String displayPathOf(ModuleID module) +{ + return module ? moduleDisplayPath(module) : emptyString(); +} + String declaringFilePath(JSCell* compiledFunction) { std::optional function = functionOf(compiledFunction); @@ -1232,13 +1268,11 @@ static NEVER_INLINE void appendInlinedFrames(const CallSiteCache::Entry& entry, }); } -static ALWAYS_INLINE void appendFrames(CallSiteCache& cache, const SuspendedAsyncFunction& suspended, WTF::Vector& results) +static ALWAYS_INLINE void appendFrames(CallSiteCache& cache, const void* returnAddress, WTF::Vector& results) { - if (!suspended.coroutine) - return; - const CallSiteCache::Entry* entry = cache.find(suspended.coroutine->resumeAddress()); + const CallSiteCache::Entry* entry = cache.find(returnAddress); if (!entry) [[unlikely]] - entry = cache.resolve(*Image::current(), suspended.coroutine->resumeAddress()); + entry = cache.resolve(*Image::current(), returnAddress); if (!entry || entry->isStubReturnSite() || entry->isInModuleBody) return; if (entry->site.inlineFrame) [[unlikely]] { @@ -1249,6 +1283,32 @@ static ALWAYS_INLINE void appendFrames(CallSiteCache& cache, const SuspendedAsyn results.constructAndAppend(TraceFrame { entry->code, entry->innermostFunction, entry->site.callSite, false, false }, true); } +static ALWAYS_INLINE void appendFrames(CallSiteCache& cache, const SuspendedAsyncFunction& suspended, WTF::Vector& results) +{ + if (suspended.coroutine) + appendFrames(cache, suspended.coroutine->resumeAddress(), results); +} + +using ReturnAddresses = Vector; +static ReturnAddresses findAwaitingCallsOnStack(const SavedFrame& saved) +{ + ReturnAddresses found; + if (!saved.isOnStack()) + return found; + const Instance& instance = *Instance::owning(&saved); + const uint64_t* frame = frameWithAwaitingCallers(instance); + if (!frame) [[likely]] + return found; + for (; SavedFrame::forFrame(instance, frame) != &saved; frame = std::bit_cast(frame[0])) { + const void* returnAddress = std::bit_cast(frame[1]); + const CallSiteCache::Entry* entry = instance.vm->aotState().callSiteCache->resolve(*Image::current(), returnAddress); + if (!entry || entry->isStubReturnSite()) + return { }; + found.append(returnAddress); + } + return found; +} + static ALWAYS_INLINE bool isModuleBody(VM& vm, const SavedFrame& saved) { const CallSiteCache::Entry* entry = vm.aotState().callSiteCache->resolve(*Image::current(), saved.resumeAddress()); @@ -1258,7 +1318,7 @@ static ALWAYS_INLINE bool isModuleBody(VM& vm, const SavedFrame& saved) bool isSuspendedAsyncFunction(JSCell* cell) { SavedFrame* saved = tryGetSavedFrame(cell); - return saved && saved->kind() == CoroutineKind::AsyncFunction && !isModuleBody(cell->vm(), *saved); + return saved && saved->kind() == CoroutineKind::AsyncFunction && (!isModuleBody(cell->vm(), *saved) || !findAwaitingCallsOnStack(*saved).isEmpty()); } bool isSuspendedModuleBody(VM& vm, JSValue coroutine) @@ -1315,6 +1375,17 @@ JSCell* appendDirectAsyncCallers(VM& vm, JSCell*, WTF::Vector& resul if (!current.coroutine) return coroutine; CallSiteCache& cache = *vm.aotState().callSiteCache; + if (ReturnAddresses awaitingCalls = findAwaitingCallsOnStack(*current.coroutine); !awaitingCalls.isEmpty()) [[unlikely]] { + for (const void* returnAddress : awaitingCalls) { + if (results.size() >= maxStackSize) + return nullptr; + appendFrames(cache, returnAddress, results); + } + if (results.size() > maxStackSize) + results.shrink(maxStackSize); + if (isModuleBody(vm, *current.coroutine)) + return nullptr; + } for (;;) { SuspendedAsyncFunction caller; if (current.waiter) { diff --git a/Source/JavaScriptCore/aot/runtime/AOTStackTrace.h b/Source/JavaScriptCore/aot/runtime/AOTStackTrace.h index 8c75bc1d4863f..7330ecb07d431 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTStackTrace.h +++ b/Source/JavaScriptCore/aot/runtime/AOTStackTrace.h @@ -113,6 +113,7 @@ JS_EXPORT_PRIVATE std::optional positionOf(const TraceFrame&); JS_EXPORT_PRIVATE std::optional definitionOf(const TraceFrame&); JS_EXPORT_PRIVATE LineColumn obfuscatedPositionOf(const TraceFrame&); JS_EXPORT_PRIVATE String sourceURLOf(ModuleID); +JS_EXPORT_PRIVATE String displayPathOf(ModuleID); String sourceProviderURLOf(ModuleID); String sourceProviderURLOf(const TraceFrame&, const std::optional&); JS_EXPORT_PRIVATE JSCell* livenessToken(Instance*); diff --git a/Source/JavaScriptCore/aot/runtime/AOTState.h b/Source/JavaScriptCore/aot/runtime/AOTState.h index 7a69ffad17cff..7a81f634ce5c5 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTState.h +++ b/Source/JavaScriptCore/aot/runtime/AOTState.h @@ -179,6 +179,19 @@ namespace Typed { enum class CountInEffectsNet : uint8_t; } +struct GrowingFlatText { + JSString* buffer; + const StringImpl* lastResolved; + uint32_t resolvesInTurn; +}; + +struct WideCharacterStrings { + static constexpr unsigned size = 256; + + std::array strings; + std::array characters; +}; + struct State { WTF_DEPRECATED_MAKE_STRUCT_FAST_ALLOCATED(State); WTF_MAKE_NONCOPYABLE(State); @@ -193,6 +206,8 @@ struct State { ZeroedPages pages; ZeroedTableArena tableArena; std::unique_ptr caches; + GrowingFlatText growingFlatText { }; + WideCharacterStrings wideCharacterStrings { }; std::unique_ptr immortalCells; JSObject* objectThatHoldsUndefined { nullptr }; std::unique_ptr strings; @@ -218,6 +233,7 @@ struct State { EncodedJSValue receiverOfPendingRead { 0 }; uint32_t keyOfPendingRead { 0 }; uint32_t slowPathReadCountAtPendingRead { 0 }; + bool acceptsForeignBinaryData { false }; void countWriteForEffectsNet(Typed::CountInEffectsNet count) { ++writesCountedForEffectsNet[static_cast(count)]; } private: diff --git a/Source/JavaScriptCore/aot/runtime/AOTStringBuilderStorage.cpp b/Source/JavaScriptCore/aot/runtime/AOTStringBuilderStorage.cpp index d7f4a5f4636b7..3cbe7f25be3dd 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTStringBuilderStorage.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTStringBuilderStorage.cpp @@ -137,6 +137,10 @@ void StringBuilderStorage::reset() Locker locker { cellLock() }; m_segments.shrink(0); } + if (m_units.size() > maxRetainedUnits) + m_units.clear(); + else + m_units.shrink(0); size_t longestTextInBuffer = std::max(m_position, std::exchange(m_positionWhenSegmentsWereJoined, 0)); m_position = 0; m_positionJoinedIntoFirstSegment = 0; @@ -260,6 +264,11 @@ auto StringBuilderStorage::append(JSGlobalObject* globalObject, std::span(characters.size()), [&](std::span units) { + StringImpl::copyCharacters(units, characters); + }); + } std::span copy; RefPtr impl = canGrowBy(characters.size()) ? StringImpl::tryCreateUninitialized(characters.size(), copy) : nullptr; if (!impl) { @@ -278,8 +287,16 @@ auto StringBuilderStorage::append(JSGlobalObject* globalObject, JSString* string return AppendResult::Appended; bool wasEmpty = !this->length(); uint32_t minLengthByReference = wasEmpty ? minLengthOfStringBuilderFiber : minLengthAppendedByReference; - bool isAppendedByReference = length >= minLengthByReference || !string->is8Bit(); - AppendResult result = isAppendedByReference ? addSegment(globalObject, string) : appendCharactersOf(globalObject, string); + bool is8Bit = string->is8Bit(); + AppendResult result; + if (is8Bit && length < minLengthByReference) + result = appendCharactersOf(globalObject, string); + else if (!is8Bit && hasRoomForUnits(length)) { + result = addUnits(globalObject, length, [&](std::span units) { + string->resolveToBuffer(units); + }); + } else + result = addSegment(globalObject, string); if (wasEmpty && result == AppendResult::Appended) { m_lastString.set(vm, this, string); m_lengthOfLastString = length; @@ -302,6 +319,32 @@ auto StringBuilderStorage::appendCharactersOf(JSGlobalObject* globalObject, JSSt return AppendResult::Appended; } +template +auto StringBuilderStorage::addUnits(JSGlobalObject* globalObject, uint32_t length, const CopyUnits& copyUnits) -> AppendResult +{ + if (!canGrowBy(length)) { + throwOutOfMemory(globalObject); + return AppendResult::Failed; + } + bool extendsLastSegment = !m_segments.isEmpty() && !m_segments.last().string && m_segments.last().offsetInBuffer == m_position; + if (!extendsLastSegment && m_segments.size() >= maxSegments) { + if (!collapseSegments(globalObject)) + return AppendResult::Failed; + } + uint32_t start = static_cast(m_units.size()); + m_units.grow(start + length); + copyUnits(m_units.mutableSpan().subspan(start)); + if (extendsLastSegment) + m_segments.last().length += length; + else { + Locker locker { cellLock() }; + m_segments.append(Segment { m_position, length, start, { } }); + } + m_lengthOutsideBuffer += length; + updateLimit(); + return AppendResult::Appended; +} + auto StringBuilderStorage::addSegment(JSGlobalObject* globalObject, JSString* string) -> AppendResult { VM& vm = globalObject->vm(); @@ -316,7 +359,7 @@ auto StringBuilderStorage::addSegment(JSGlobalObject* globalObject, JSString* st } { Locker locker { cellLock() }; - m_segments.append(Segment { m_position, length, WriteBarrier(vm, this, string) }); + m_segments.append(Segment { m_position, length, 0, WriteBarrier(vm, this, string) }); } m_lengthOutsideBuffer += length; updateLimit(); @@ -414,6 +457,8 @@ void StringBuilderStorage::rollBack(uint32_t length) Locker locker { cellLock() }; uint32_t remainingLength = length > start ? length - start : 0; m_lengthOutsideBuffer -= last.length - remainingLength; + if (!last.string) + m_units.shrink(last.startOfUnits + remainingLength); if (remainingLength) { last.length = remainingLength; break; @@ -432,19 +477,20 @@ void StringBuilderStorage::forEachPiece(const Functor& functor) const uint32_t taken = m_positionJoinedIntoFirstSegment; for (const Segment& segment : m_segments) { if (segment.offsetInBuffer > taken) - functor(Piece { nullptr, taken, segment.offsetInBuffer - taken }); + functor(Piece { nullptr, nullptr, taken, segment.offsetInBuffer - taken }); if (segment.length) - functor(Piece { segment.string.get(), 0, segment.length }); + functor(Piece { segment.string.get(), segment.string ? nullptr : m_units.span().subspan(segment.startOfUnits).data(), 0, segment.length }); taken = segment.offsetInBuffer; } if (m_position > taken) - functor(Piece { nullptr, taken, m_position - taken }); + functor(Piece { nullptr, nullptr, taken, m_position - taken }); } RefPtr StringBuilderStorage::tryShareOrGiveAwayBuffer(bool mayGiveBufferAway) { ASSERT(m_segments.isEmpty()); - if (m_position < minLengthAdoptedByFinish) + bool isShared = !mayGiveBufferAway && m_position >= minLengthShared && m_position >= capacity() / 2; + if (m_position < minLengthAdoptedByFinish && !isShared) return nullptr; if (!mayGiveBufferAway) { m_sharedLength = std::max(m_sharedLength, m_position); @@ -514,7 +560,7 @@ JSString* StringBuilderStorage::createFlatRun(JSGlobalObject* globalObject, std: bool is8Bit = true; for (const Piece& piece : run) { total += piece.length; - is8Bit &= !piece.string || piece.string->is8Bit(); + is8Bit &= piece.is8Bit(); } std::span output8; @@ -525,11 +571,17 @@ JSString* StringBuilderStorage::createFlatRun(JSGlobalObject* globalObject, std: return nullptr; } - auto copyPieces = [&](auto output) { + auto copyPieces = [&](std::span output) { size_t at = 0; for (const Piece& piece : run) { auto destination = output.subspan(at, piece.length); at += piece.length; + if constexpr (std::is_same_v) { + if (piece.units) { + StringImpl::copyCharacters(destination, unsafeMakeSpan(piece.units, piece.length)); + continue; + } + } if (!piece.string) { StringImpl::copyCharacters(destination, std::span { m_characters + piece.start, piece.length }); continue; @@ -563,17 +615,30 @@ JSString* StringBuilderStorage::createStringFromPieces(JSGlobalObject* globalObj if (length() < minLengthOfStringBuilderFiber) RELEASE_AND_RETURN(scope, createFlatRun(globalObject, pieces.span())); - auto is8Bit = [](const Piece& piece) { - return !piece.string || piece.string->is8Bit(); - }; + constexpr size_t minLengthOfRunOfOtherWidth = 256; auto isFiber = [&](const Piece& piece) { return piece.string && (piece.length >= minLengthOfStringBuilderFiber || (!piece.string->is8Bit() && piece.string->isRope() && !piece.string->isSubstring())); }; JSRopeString::RopeBuilder fibers(vm); size_t runStart = 0; + size_t lengthOfRun = 0; + bool runIs8Bit = true; + auto extendsRun = [&](size_t index) { + const Piece& piece = pieces[index]; + if (index == runStart) { + lengthOfRun = piece.length; + runIs8Bit = piece.is8Bit(); + return true; + } + if (piece.is8Bit() != runIs8Bit && (lengthOfRun >= minLengthOfRunOfOtherWidth || piece.length >= minLengthOfRunOfOtherWidth)) + return false; + lengthOfRun += piece.length; + runIs8Bit &= piece.is8Bit(); + return true; + }; for (size_t index = 0; index <= pieces.size(); ++index) { bool hasFiber = index < pieces.size() && isFiber(pieces[index]); - if (index < pieces.size() && !hasFiber && (index == runStart || is8Bit(pieces[index]) == is8Bit(pieces[runStart]))) + if (index < pieces.size() && !hasFiber && extendsRun(index)) continue; if (index > runStart) { JSString* run = createFlatRun(globalObject, pieces.span().subspan(runStart, index - runStart)); @@ -587,7 +652,8 @@ JSString* StringBuilderStorage::createStringFromPieces(JSGlobalObject* globalObj RETURN_IF_EXCEPTION(scope, nullptr); fibers.append(fiber); runStart = index + 1; - } + } else if (index < pieces.size()) + extendsRun(index); } if (fibers.hasOverflowed()) [[unlikely]] { throwOutOfMemoryError(globalObject, scope); @@ -607,8 +673,9 @@ bool StringBuilderStorage::collapseSegments(JSGlobalObject* globalObject) { Locker locker { cellLock() }; m_segments.shrink(0); - m_segments.append(Segment { 0, length, WriteBarrier(vm, this, all) }); + m_segments.append(Segment { 0, length, 0, WriteBarrier(vm, this, all) }); } + m_units.shrink(0); m_lengthOutsideBuffer = length; m_positionWhenSegmentsWereJoined = std::max(m_positionWhenSegmentsWereJoined, m_position); m_position = 0; @@ -625,8 +692,9 @@ void StringBuilderStorage::joinPiecesInto(VM& vm, JSString* all) { Locker locker { cellLock() }; m_segments.shrink(0); - m_segments.append(Segment { m_position, all->length(), WriteBarrier(vm, this, all) }); + m_segments.append(Segment { m_position, all->length(), 0, WriteBarrier(vm, this, all) }); } + m_units.shrink(0); m_positionJoinedIntoFirstSegment = m_position; m_lastString.set(vm, this, all); m_lengthOfLastString = all->length(); @@ -824,6 +892,21 @@ JSC_DEFINE_JIT_OPERATION(operationAOTStringBuilderAppendString, void, (Instance* AOT_MUST_TAIL_CALL return appendAnyStringToBuilder(instance, builder, string); } +JSC_DEFINE_JIT_OPERATION(operationAOTStackStringBuilderAppendCodeUnit, uint8_t*, (Instance* instance, uint8_t* builder, uint8_t* cursor, uint32_t codeUnit)) +{ + AOT_OPERATION_BEGIN(instance, Entry::operationAOTStackStringBuilderAppendCodeUnit); + auto& stackBuilder = *reinterpret_cast(builder); + StringBuilderStorage* storage = stackBuilder.moveToStorage(globalObject, *instance, byteCast(cursor)); + if (!storage) [[unlikely]] + OPERATION_RETURN(scope, nullptr); + std::array units { static_cast(codeUnit) }; + bool appended = storage->append(globalObject, std::span { units }) == StringBuilderStorage::AppendResult::Appended; + Latin1Character* next = appended ? stackBuilder.didChangeStorage(globalObject, *storage) : nullptr; + if (!next) [[unlikely]] + stackBuilder.abandonChangeOfStorage(*instance, *storage); + OPERATION_RETURN(scope, byteCast(next)); +} + struct NarrowPartOfWideString { std::span span() const LIFETIME_BOUND { return std::span { characters }.first(length); } diff --git a/Source/JavaScriptCore/aot/runtime/AOTStringBuilderStorage.h b/Source/JavaScriptCore/aot/runtime/AOTStringBuilderStorage.h index 23f5c24edef8a..5353af3b00d88 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTStringBuilderStorage.h +++ b/Source/JavaScriptCore/aot/runtime/AOTStringBuilderStorage.h @@ -32,7 +32,11 @@ class StringBuilderStorage final : public JSCell { static constexpr uint32_t maxRetainedCapacity = 262144; static constexpr uint32_t minLengthAppendedByReference = 65536; static constexpr uint32_t minLengthAdoptedByFinish = 4096; + static constexpr uint32_t minLengthShared = minCapacity / 2; static constexpr uint32_t maxSegments = 1024; + static constexpr uint32_t maxLengthOfWideStringCopied = 64; + static constexpr uint32_t maxUnits = 1 << 20; + static constexpr uint32_t maxRetainedUnits = 4096; static constexpr uint32_t maxPooledBuilders = 8; static void destroy(JSCell*); @@ -107,11 +111,15 @@ class StringBuilderStorage final : public JSCell { struct Segment { uint32_t offsetInBuffer; uint32_t length; + uint32_t startOfUnits; WriteBarrier string; }; struct Piece { + bool is8Bit() const { return !units && (!string || string->is8Bit()); } + JSString* string { nullptr }; + const char16_t* units { nullptr }; uint32_t start { 0 }; uint32_t length { 0 }; }; @@ -142,6 +150,8 @@ class StringBuilderStorage final : public JSCell { AppendResult appendCharactersOf(JSGlobalObject*, JSString*); AppendResult addSegment(JSGlobalObject*, JSString*); + template AppendResult addUnits(JSGlobalObject*, uint32_t length, const CopyUnits&); + bool hasRoomForUnits(uint32_t length) const { return length <= maxLengthOfWideStringCopied && m_units.size() + length <= maxUnits; } bool collapseSegments(JSGlobalObject*); void joinPiecesInto(VM&, JSString* all); @@ -169,6 +179,7 @@ class StringBuilderStorage final : public JSCell { static_assert(maxPooledBuilders <= std::numeric_limits::max()); RefPtr m_buffer; Vector m_segments; + Vector m_units; }; static_assert(StringBuilderStorage::offsetOfCharacters() == sizeof(JSCell), "bytes 12 to 15 of the cell are the upper half of a user-space pointer, so the compiled-object bit of the shape word is clear"); diff --git a/Source/JavaScriptCore/aot/runtime/AOTStringOperations.cpp b/Source/JavaScriptCore/aot/runtime/AOTStringOperations.cpp index 74c3b2956550d..294f454ed40c8 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTStringOperations.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTStringOperations.cpp @@ -27,6 +27,7 @@ #include "AOTState.h" #include "AOTUnwinding.h" #include "AOTVector.h" +#include "GCDeferralContextInlines.h" #include "IntlCollator.h" #include "IntlObject.h" #include "InterpreterInlines.h" @@ -78,11 +79,18 @@ static ALWAYS_INLINE bool canRepresent(uint32_t character) return !(character >> (8 * sizeof(CharacterType))); } +static constexpr uint32_t maxLatin1Character = 0xff; + +static NEVER_INLINE size_t AOT_PRESERVE_MOST_LEAF_ATTRIBUTES findLatin1CharacterFarFromStart(const Latin1Character* characters, uint32_t length, uint32_t character, uint32_t start) +{ + return WTF::find(unsafeMakeSpan(characters, length), static_cast(character), start); +} + static ALWAYS_INLINE size_t findLatin1Character(const Latin1Character* characters, size_t length, Latin1Character character, size_t start) { static_assert(std::endian::native == std::endian::little); constexpr size_t stride = SIMD::stride; - constexpr size_t stridesSearchedHere = 4; + constexpr size_t stridesSearchedHere = 8; const uint8_t* bytes = byteCast(characters); if (start >= length) return notFound; @@ -90,12 +98,12 @@ static ALWAYS_INLINE size_t findLatin1Character(const Latin1Character* character if (length - i >= stride) { auto sought = SIMD::splat(character); size_t lastStride = length - stride; - for (size_t end = i + stridesSearchedHere * stride; i <= lastStride && i < end; i += stride) { + for (size_t end = i + stridesSearchedHere * stride; i <= lastStride; i += stride) { + if (i >= end) [[unlikely]] + return findLatin1CharacterFarFromStart(characters, length, character, i); if (auto found = SIMD::findFirstNonZeroIndex(SIMD::equal(SIMD::load(bytes + i), sought))) return i + *found; } - if (i <= lastStride) - return WTF::find(unsafeMakeSpan(characters, length), character, i); if (i == length) return notFound; auto found = SIMD::findFirstNonZeroIndex(SIMD::equal(SIMD::load(bytes + lastStride), sought)); @@ -463,19 +471,102 @@ extern "C" size_t AOT_PRESERVE_MOST_LEAF_ATTRIBUTES operationAOTStringFindLongSe return findLongSeparators(characters, length, text, countAndWhetherAllAreCounted, bounds); } -extern "C" size_t AOT_PRESERVE_MOST_LEAF_ATTRIBUTES operationAOTStringFindCharacter8(const Latin1Character* characters, uint32_t length, uint32_t character, uint32_t start) +static constexpr uint32_t latin1Stride = SIMD::stride; + +static ALWAYS_INLINE auto compareLatin1Stride(const Latin1Character* characters, uint32_t character) +{ + return SIMD::equal(SIMD::load(byteCast(characters)), SIMD::splat(static_cast(character))); +} + +static NEVER_INLINE size_t AOT_PRESERVE_MOST_LEAF_ATTRIBUTES findCharacter8WithinStride(const Latin1Character* characters, uint32_t length, uint32_t character, uint32_t start) { return findCharacter(characters, length, character, start); } +static NEVER_INLINE size_t AOT_PRESERVE_MOST_LEAF_ATTRIBUTES findCharacter8AfterFirstStride(const Latin1Character* characters, uint32_t length, uint32_t character, uint32_t start) +{ + constexpr uint32_t stridesSearchedHere = 8; + uint32_t lastStride = length - latin1Stride; + uint32_t i = start + latin1Stride; + for (uint32_t strides = 0; i <= lastStride; i += latin1Stride) { + if (++strides > stridesSearchedHere) [[unlikely]] + return findLatin1CharacterFarFromStart(characters, length, character, i); + if (auto found = SIMD::findFirstNonZeroIndex(compareLatin1Stride(characters + i, character))) + return static_cast(i) + *found; + } + if (i == length) + return notFound; + auto found = SIMD::findFirstNonZeroIndex(compareLatin1Stride(characters + lastStride, character)); + return found ? static_cast(lastStride) + *found : notFound; +} + +extern "C" size_t AOT_PRESERVE_MOST_LEAF_ATTRIBUTES operationAOTStringFindCharacter8(const Latin1Character* characters, uint32_t length, uint32_t character, uint32_t start) +{ + if (start >= length || length - start < latin1Stride || character > maxLatin1Character) [[unlikely]] + [[clang::musttail]] return findCharacter8WithinStride(characters, length, character, start); + if (auto found = SIMD::findFirstNonZeroIndex(compareLatin1Stride(characters + start, character))) + return static_cast(start) + *found; + [[clang::musttail]] return findCharacter8AfterFirstStride(characters, length, character, start); +} + extern "C" size_t AOT_PRESERVE_MOST_LEAF_ATTRIBUTES operationAOTStringFindCharacter16(const char16_t* characters, uint32_t length, uint32_t character, uint32_t start) { return findCharacter(characters, length, character, start); } +static NEVER_INLINE size_t AOT_PRESERVE_MOST_LEAF_ATTRIBUTES reverseFindCharacter8WithinStride(const Latin1Character* characters, uint32_t length, uint32_t character, int32_t position) +{ + static_assert(std::endian::native == std::endian::little); + constexpr uint64_t lowBits = 0x0101010101010101ULL; + constexpr uint64_t otherBits = 0x7f * lowBits; + constexpr uint32_t wordSize = sizeof(uint64_t); + if (position < 0 || character > maxLatin1Character) + return notFound; + uint32_t end = static_cast(position) < length ? static_cast(position) + 1 : length; + if (end >= wordSize) { + for (uint32_t place : { end - wordSize, 0u }) { + uint64_t differences = WTF::unalignedLoad(characters + place) ^ lowBits * character; + if (uint64_t found = ~(((differences & otherBits) + otherBits) | differences | otherBits)) + return static_cast(place) + 7 - std::countl_zero(found) / 8; + } + return notFound; + } + while (end--) { + if (characters[end] == character) + return end; + } + return notFound; +} + +static ALWAYS_INLINE uint32_t endOfReverseSearch(uint32_t length, int32_t position) +{ + return static_cast(position) < length ? static_cast(position) + 1 : length; +} + +static NEVER_INLINE size_t AOT_PRESERVE_MOST_LEAF_ATTRIBUTES reverseFindCharacter8BeforeLastStride(const Latin1Character* characters, uint32_t length, uint32_t character, int32_t position) +{ + constexpr uint32_t stridesSearchedHere = 8; + uint32_t end = endOfReverseSearch(length, position) - latin1Stride; + for (uint32_t strides = 0; end >= latin1Stride; end -= latin1Stride) { + if (++strides > stridesSearchedHere) [[unlikely]] + return WTF::reverseFind(unsafeMakeSpan(characters, end), static_cast(character), end - 1); + if (auto found = SIMD::findLastNonZeroIndex(compareLatin1Stride(characters + end - latin1Stride, character))) + return static_cast(end - latin1Stride) + *found; + } + if (!end) + return notFound; + auto found = SIMD::findLastNonZeroIndex(compareLatin1Stride(characters, character)); + return found ? static_cast(*found) : notFound; +} + extern "C" size_t AOT_PRESERVE_MOST_LEAF_ATTRIBUTES operationAOTStringReverseFindCharacter8(const Latin1Character* characters, uint32_t length, uint32_t character, int32_t position) { - return reverseFindCharacter(characters, length, character, position); + uint32_t end = endOfReverseSearch(length, position); + if (position < 0 || end < latin1Stride || character > maxLatin1Character) [[unlikely]] + [[clang::musttail]] return reverseFindCharacter8WithinStride(characters, length, character, position); + if (auto found = SIMD::findLastNonZeroIndex(compareLatin1Stride(characters + end - latin1Stride, character))) + return static_cast(end - latin1Stride) + *found; + [[clang::musttail]] return reverseFindCharacter8BeforeLastStride(characters, length, character, position); } extern "C" size_t AOT_PRESERVE_MOST_LEAF_ATTRIBUTES operationAOTStringReverseFindCharacter16(const char16_t* characters, uint32_t length, uint32_t character, int32_t position) @@ -605,6 +696,19 @@ JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationAOTStringReverseFindLiteral8, size_t, return reverseFindString(characters, length, other.span8().data(), other.length(), position); } +JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationAOTStringFindLiteral16, size_t, (Instance*, const char16_t* characters, uint32_t length, uint32_t text, uint32_t start)) +{ + return StringView(std::span { characters, length }).find(textOf(text), start); +} + +JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationAOTStringReverseFindLiteral16, size_t, (Instance*, const char16_t* characters, uint32_t length, uint32_t text, int32_t position)) +{ + StringView other = textOf(text); + if (!other.is8Bit()) [[unlikely]] + return StringView(std::span { characters, length }).reverseFind(other, std::max(position, 0)); + return reverseFindString(characters, length, other.span8().data(), other.length(), position); +} + JSC_DEFINE_JIT_OPERATION(operationAOTConcat, JSCell*, (Instance* instance, JSCell* left, JSCell* right)) { AOT_OPERATION_BEGIN(instance, Entry::operationAOTConcat); @@ -634,14 +738,148 @@ JSC_DEFINE_JIT_OPERATION(operationAOTSubstring, JSCell*, (Instance* instance, JS OPERATION_RETURN(scope, jsSubstring(globalObject, cellArgument(string, Entry::operationAOTSubstring), from, to - from)); } +static ALWAYS_INLINE void copyShortString(std::span characters, Latin1Character* to) +{ + const Latin1Character* from = characters.data(); + size_t size = characters.size(); + if (size >= 8) { + uint64_t first = WTF::unalignedLoad(from); + uint64_t last = WTF::unalignedLoad(from + size - 8); + WTF::unalignedStore(to, first); + WTF::unalignedStore(to + size - 8, last); + return; + } + if (size >= 4) { + uint32_t first = WTF::unalignedLoad(from); + uint32_t last = WTF::unalignedLoad(from + size - 4); + WTF::unalignedStore(to, first); + WTF::unalignedStore(to + size - 4, last); + return; + } + if (size) { + Latin1Character first = from[0]; + Latin1Character middle = from[size / 2]; + Latin1Character last = from[size - 1]; + to[0] = first; + to[size / 2] = middle; + to[size - 1] = last; + } +} + +static ALWAYS_INLINE std::array fibersOf(const JSRopeString* rope) +{ + using CompactFibers = JSRopeString::CompactFibers; + static_assert(CompactFibers::widthOfFiber1LowerInLengthWord == 32 && CompactFibers::shiftOfFiber2InUpperWord == 16); + auto* bytes = reinterpret_cast(rope); + uint64_t upperWord = WTF::unalignedLoad(bytes + JSRopeString::offsetOfFiber2()); + uint64_t second = WTF::unalignedLoad(bytes + JSRopeString::offsetOfFiber1Lower()) | (upperWord & 0xffff) << 32; + return { std::bit_cast(WTF::unalignedLoad(bytes + JSRopeString::offsetOfFiber0()) & JSRopeString::stringMask), std::bit_cast(second), std::bit_cast(upperWord >> 16) }; +} + +static ALWAYS_INLINE const Latin1Character* latin1CharactersInPlace(const JSString* string, unsigned& length) +{ + if (const StringImpl* impl = string->tryGetValueImpl()) [[likely]] { + length = impl->length(); + return impl->is8Bit() ? impl->span8().data() : nullptr; + } + auto* rope = uncheckedDowncast(string); + if (!rope->isSubstring()) + return nullptr; + length = rope->length(); + const StringImpl* base = rope->substringBase()->tryGetValueImpl(); + return base->is8Bit() ? base->span8().data() + rope->substringOffset() : nullptr; +} + +static NEVER_INLINE bool tryResolveRopeOfShortRopes(VM& vm, JSRopeString* rope) +{ + constexpr unsigned maxLength = 1024; + constexpr unsigned maxLengthOfShortString = 16; + if (rope->isSubstring() || !rope->is8Bit() || rope->length() > maxLength) + return false; + std::span buffer; + RefPtr impl = StringImpl::tryCreateUninitialized(rope->length(), buffer); + if (!impl) + return false; + Latin1Character* cursor = buffer.data(); + Latin1Character* end = cursor + buffer.size(); + auto tryCopy = [&](const JSString* string) ALWAYS_INLINE_LAMBDA { + unsigned length; + const Latin1Character* characters = latin1CharactersInPlace(string, length); + if (!characters || length > static_cast(end - cursor)) + return false; + if (length <= maxLengthOfShortString) + copyShortString({ characters, length }, cursor); + else + memcpy(cursor, characters, length); + cursor += length; + return true; + }; + for (const JSString* fiber : fibersOf(rope)) { + if (!fiber) + break; + if (tryCopy(fiber)) + continue; + if (!fiber->isRope() || fiber->isSubstring()) + return false; + for (const JSString* part : fibersOf(uncheckedDowncast(fiber))) { + if (!part) + break; + if (!tryCopy(part)) + return false; + } + } + RELEASE_ASSERT(cursor == end); + size_t cost = impl->cost(); + rope->resolveBySharing(*impl); + vm.heap.reportExtraMemoryAllocated(rope, cost); + return true; +} + +JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationAOTStringCodeUnitAt, size_t, (Instance*, JSCell* cell, uint32_t index)) +{ + auto* string = cellArgument(cell, Entry::operationAOTStringCodeUnitAt); + if (string->isRope()) { + DeferGCForAWhile deferGC(string->vm()); + uncheckedDowncast(string)->resolveRope(nullptr); + } + const JSString* part = string; + while (part->isRope()) { + const auto* rope = uncheckedDowncast(part); + if (rope->isSubstring()) { + index += rope->substringOffset(); + part = rope->substringBase(); + continue; + } + for (const JSString* fiber : fibersOf(rope)) { + part = fiber; + if (index < fiber->length()) + break; + index -= fiber->length(); + } + } + return part->tryGetValueImpl()->at(index); +} + JSC_DEFINE_JIT_OPERATION(operationAOTFlatten, void*, (Instance* instance, JSCell* string)) { AOT_OPERATION_BEGIN(instance, Entry::operationAOTFlatten); JSString* cell = cellArgument(string, Entry::operationAOTFlatten); - if (cell->isRope() && uncheckedDowncast(cell)->tryResolveFlatLatin1Parts(vm)) - OPERATION_RETURN(scope, cell->tryGetValueImpl()); + bool mayGrow = false; + if (cell->isRope()) { + auto* rope = uncheckedDowncast(cell); + mayGrow = rope->length() >= minLengthOfGrowingText; + if (mayGrow && tryResolveInGrowingBuffer(*instance, rope)) + OPERATION_RETURN(scope, cell->tryGetValueImpl()); + if (rope->tryResolveFlatLatin1Parts(vm) || tryResolveRopeOfShortRopes(vm, rope)) { + if (mayGrow) + noteResolvedText(*instance, cell); + OPERATION_RETURN(scope, cell->tryGetValueImpl()); + } + } auto value = cell->value(globalObject); OPERATION_RETURN_IF_EXCEPTION(scope, nullptr); + if (mayGrow) + noteResolvedText(*instance, cell); OPERATION_RETURN(scope, value->impl()); } @@ -1061,10 +1299,41 @@ JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationAOTDoubleToString, JSCell*, (Instance OPERATION_RETURN(scope, vm.numericStrings.addJSString(vm, value)); } -JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationAOTCharToString, JSCell*, (Instance* instance, uint32_t character)) +static ALWAYS_INLINE JSString* findStringOfWideCharacter(Instance& instance, char16_t character) +{ + const WideCharacterStrings& kept = instance.state->wideCharacterStrings; + unsigned place = character % WideCharacterStrings::size; + return kept.characters[place] == character ? kept.strings[place] : nullptr; +} + +JSString* stringOfCharacter(Instance& instance, char16_t character) +{ + VM& vm = *instance.vm; + if (character <= maxSingleCharacterString) + return vm.smallStrings.singleCharacterString(static_cast(character)); + if (JSString* kept = findStringOfWideCharacter(instance, character)) + return kept; + JSString* made = jsSingleCharacterString(vm, character); + WideCharacterStrings& strings = instance.state->wideCharacterStrings; + unsigned place = character % WideCharacterStrings::size; + strings.strings[place] = made; + strings.characters[place] = character; + return made; +} + +static NEVER_INLINE JSCell* JIT_OPERATION_ATTRIBUTES makeStringOfCharacter(Instance* instance, uint32_t character) { AOT_OPERATION_BEGIN(instance, Entry::operationAOTCharToString); - OPERATION_RETURN(scope, jsSingleCharacterString(vm, static_cast(character))); + OPERATION_RETURN(scope, stringOfCharacter(*instance, static_cast(character))); +} + +JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationAOTCharToString, JSCell*, (Instance* instance, uint32_t character)) +{ + if (character > maxSingleCharacterString) { + if (JSString* kept = findStringOfWideCharacter(*instance, static_cast(character))) + return kept; + } + AOT_MUST_TAIL_CALL return makeStringOfCharacter(instance, character); } JSC_DEFINE_JIT_OPERATION(operationAOTStringOfAnyValue, EncodedJSValue, (Instance* instance, EncodedJSValue encodedValue)) @@ -1597,6 +1866,31 @@ static ALWAYS_INLINE std::optional> tryGetLatin return whole->span8().subspan(piece->substringOffset(), piece->length()); } +extern "C" int64_t AOT_PRESERVE_MOST_LEAF_ATTRIBUTES operationAOTCompareStringsInPlace(JSCell* first, JSCell* second) +{ + auto firstCharacters = tryGetLatin1Characters(uncheckedDowncast(first)); + auto secondCharacters = tryGetLatin1Characters(uncheckedDowncast(second)); + if (!firstCharacters || !secondCharacters) + return noOrderOfStringsInPlace; + const Latin1Character* left = firstCharacters->data(); + const Latin1Character* right = secondCharacters->data(); + size_t commonLength = std::min(firstCharacters->size(), secondCharacters->size()); + size_t index = 0; + for (; commonLength - index >= sizeof(uint64_t); index += sizeof(uint64_t)) { + uint64_t leftWord = WTF::unalignedLoad(left + index); + uint64_t rightWord = WTF::unalignedLoad(right + index); + if (leftWord != rightWord) + return std::byteswap(leftWord) < std::byteswap(rightWord) ? -1 : 1; + } + for (; index < commonLength; ++index) { + if (left[index] != right[index]) + return left[index] < right[index] ? -1 : 1; + } + if (firstCharacters->size() == secondCharacters->size()) + return 0; + return firstCharacters->size() < secondCharacters->size() ? -1 : 1; +} + static ALWAYS_INLINE bool containsShortText(std::span characters, std::span sought) { ASSERT(!sought.empty()); @@ -1663,7 +1957,14 @@ static JSArray* trySplitAtPlainText(JSGlobalObject* globalObject, VM& vm, JSStri StringView sought { *separatorImpl }; size_t found; unsigned position = 0; - while ((found = separatorLength == 1 ? subject->find(sought[0], position) : subject->find(vm.adaptiveStringSearcherTables(), sought, position)) != notFound) { + auto findSeparator = [&]() ALWAYS_INLINE_LAMBDA -> size_t { + if (separatorLength != 1) + return subject->find(vm.adaptiveStringSearcherTables(), sought, position); + if (!subject->is8Bit()) + return subject->find(sought[0], position); + return findCharacter(subject->span8().data(), subject->length(), sought[0], position); + }; + while ((found = findSeparator()) != notFound) { ends.append(static_cast(found)); if (ends.size() == limit) break; @@ -1675,15 +1976,6 @@ static JSArray* trySplitAtPlainText(JSGlobalObject* globalObject, VM& vm, JSStri if (ends.size() >= MIN_SPARSE_ARRAY_INDEX) return nullptr; - JSArray* result = JSArray::tryCreate(vm, globalObject->originalArrayStructureForIndexingType(ArrayWithContiguous), ends.size()); - if (!result) [[unlikely]] { - throwOutOfMemoryError(globalObject, scope); - return nullptr; - } - if (ends.size() == 1 && ends[0] == string->length()) { - result->butterfly()->contiguous().at(result, 0).set(vm, result, string); - return result; - } JSString* base = string; unsigned start = 0; if (string->isSubstring()) { @@ -1691,12 +1983,27 @@ static JSArray* trySplitAtPlainText(JSGlobalObject* globalObject, VM& vm, JSStri base = rope->substringBase(); start = rope->substringOffset(); } - unsigned offsetInBase = start; - for (unsigned i = 0; i < ends.size(); ++i) { - unsigned end = offsetInBase + ends[i]; - result->butterfly()->contiguous().at(result, i).set(vm, result, jsSubstringOfResolved(vm, base, start, end - start)); - start = end + separatorLength; + Structure* structure = globalObject->originalArrayStructureForIndexingType(ArrayWithContiguous); + JSArray* result; + { + GCDeferralContext deferralContext(vm); + ObjectInitializationScope initializationScope(vm); + result = JSArray::tryCreateUninitializedRestricted(initializationScope, &deferralContext, structure, ends.size()); + if (result) [[likely]] { + if (ends.size() == 1 && ends[0] == string->length()) + result->butterfly()->contiguous().at(result, 0).setWithoutWriteBarrier(string); + else { + unsigned offsetInBase = start; + for (unsigned i = 0; i < ends.size(); ++i) { + unsigned end = offsetInBase + ends[i]; + result->butterfly()->contiguous().at(result, i).setWithoutWriteBarrier(jsSubstringOfResolved(vm, &deferralContext, base, start, end - start)); + start = end + separatorLength; + } + } + } } + if (!result) [[unlikely]] + throwOutOfMemoryError(globalObject, scope); return result; } @@ -1711,34 +2018,6 @@ JSC_DEFINE_JIT_OPERATION(operationAOTStringSplitString, JSCell*, (Instance* inst OPERATION_RETURN(scope, stringSplitFast(globalObject, string, separator, limit)); } -static ALWAYS_INLINE void copyShortString(std::span characters, Latin1Character* to) -{ - const Latin1Character* from = characters.data(); - size_t size = characters.size(); - if (size >= 8) { - uint64_t first = WTF::unalignedLoad(from); - uint64_t last = WTF::unalignedLoad(from + size - 8); - WTF::unalignedStore(to, first); - WTF::unalignedStore(to + size - 8, last); - return; - } - if (size >= 4) { - uint32_t first = WTF::unalignedLoad(from); - uint32_t last = WTF::unalignedLoad(from + size - 4); - WTF::unalignedStore(to, first); - WTF::unalignedStore(to + size - 4, last); - return; - } - if (size) { - Latin1Character first = from[0]; - Latin1Character middle = from[size / 2]; - Latin1Character last = from[size - 1]; - to[0] = first; - to[size / 2] = middle; - to[size - 1] = last; - } -} - struct ShortKeyCharacters { std::span span() const LIFETIME_BOUND { return std::span { characters }.first(length); } @@ -1749,6 +2028,17 @@ struct ShortKeyCharacters { }; static_assert(StringKeyCache::maxCachedKeyLength >= StringKeyCache::minReadableBytes); +static ALWAYS_INLINE bool isSubstringOfManyKeys(JSString* rope) +{ + uint8_t& word = *(std::bit_cast(rope) + JSString::offsetOfValue()); + if (!(word & JSRopeString::isSubstringInPointer)) + return false; + if (word & UsesOfSubstringAsKey::two) + return true; + word += UsesOfSubstringAsKey::one; + return false; +} + static ALWAYS_INLINE bool tryWriteShortKey(std::span primitives, ShortKeyCharacters& key) { constexpr size_t mostCharactersCopiedByWords = 16; @@ -1770,7 +2060,7 @@ static ALWAYS_INLINE bool tryWriteShortKey(std::span primi key.mayBeIndex &= !size || isASCIIDigit(from.back()); } else { string->resolveToBuffer(std::span { cursor, size }); - key.hasRopePart = true; + key.hasRopePart |= !isSubstringOfManyKeys(string); } cursor += size; continue; @@ -1786,6 +2076,30 @@ static ALWAYS_INLINE bool tryWriteShortKey(std::span primi return true; } +static ALWAYS_INLINE bool tryJoinFewCharacters(std::span primitives, StringKeyCache::FewCharacters& joined) +{ + bool mayBeIndex = true; + for (EncodedJSValue encoded : primitives) { + JSValue primitive = JSValue::decode(encoded); + if (primitive.isInt32()) { + if (!joined.tryAppendInteger(static_cast(primitive.asInt32()))) + return false; + continue; + } + if (!primitive.isString()) { + std::optional integer = primitive.tryGetAsInt32(); + if (!integer || !joined.tryAppendInteger(static_cast(*integer))) + return false; + continue; + } + const StringImpl* impl = asString(primitive)->tryGetValueImpl(); + if (!impl || !impl->is8Bit() || !joined.tryAppend(impl->span8())) + return false; + mayBeIndex &= !impl->length() || isASCIIDigit(impl->span8().back()); + } + return joined.length >= 2 && (!mayBeIndex || joined.length > StringKeyCache::maxIndexLength); +} + static ALWAYS_INLINE JSValue onlyKeyPartWithCharacters(VM& vm, std::span primitives) { if (primitives.size() > 2) @@ -1908,10 +2222,39 @@ static bool textsSurelyFitInOneString(std::span primitives return length <= JSString::MaxLength; } +static ALWAYS_INLINE bool isTooLongForStringKey(std::span primitives) +{ + uint64_t length = 0; + for (EncodedJSValue primitive : primitives) { + if (JSValue value = JSValue::decode(primitive); value.isString()) + length += asString(value)->length(); + } + return length > StringKeyCache::maxCachedKeyLength; +} + static NEVER_INLINE OperationReturnType JIT_OPERATION_ATTRIBUTES concatenateShortStringKey(Instance* instance, uint32_t count, const EncodedJSValue* parts, uint32_t isLikelyKey) { - ShortKeyCharacters characters; + if (isTooLongForStringKey(unsafeMakeSpan(parts, count))) [[unlikely]] + AOT_MUST_TAIL_CALL return concatenateByContents(instance, count, parts, static_cast(UseOfJoinedText::Unknown)); StringKeyCache& keys = *instance->state->stringKeys; + StringKeyCache::FewCharacters joined; + if (tryJoinFewCharacters(unsafeMakeSpan(parts, count), joined)) [[likely]] { + unsigned hash = joined.hash(); + if (JSString* key = keys.tryFindCachedKey(joined, hash)) [[likely]] { + if (keys.lastLookupByParts().countAndKinds) + keys.cacheKeyByParts(keys.lastLookupByParts(), key); + return { key, nullptr }; + } + auto text = joined.text(); + std::span characters = std::span { text }.first(joined.length); + ASSERT(hash == RapidHash::computeHashAndMaskTop8Bits(characters)); + AOT_OPERATION_BEGIN(instance, Entry::operationAOTConcatenate); + if (JSString* made = keys.makeKey(vm, characters, hash, keys.lastLookupByParts(), useOfJoinedTextOf(isLikelyKey) == UseOfJoinedText::LikelyName)) [[likely]] + return { made, nullptr }; + throwOutOfMemoryError(globalObject, scope); + OPERATION_RETURN(scope, static_cast(nullptr)); + } + ShortKeyCharacters characters; if (tryWriteShortKey(unsafeMakeSpan(parts, count), characters) && characters.length >= 2) [[likely]] { if (characters.hasRopePart) keys.lastLookupByParts().countAndKinds = 0; diff --git a/Source/JavaScriptCore/aot/runtime/AOTTestRunFactories.cpp b/Source/JavaScriptCore/aot/runtime/AOTTestRunFactories.cpp index 569566d7971a4..f7bbd8ef8a27c 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTTestRunFactories.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTTestRunFactories.cpp @@ -133,12 +133,19 @@ std::optional Modules::failureByAbsentExport(ModuleID module) c return std::nullopt; } +std::optional Modules::moduleThatFailedToLink(ModuleID root) const +{ + if (std::optional id = failureByAbsentExport(root)) + return m_table.failure(*id).referrerModule(); + auto failure = m_linkFailures.find(root); + return failure == m_linkFailures.end() ? std::nullopt : std::optional(failure->value); +} + bool Modules::wasLinkedAheadOfFailure(ModuleID root, ModuleID module) const { - std::optional id = failureByAbsentExport(root); - if (!id) + std::optional referrer = moduleThatFailedToLink(root); + if (!referrer) return false; - ModuleID referrer = m_table.failure(*id).referrerModule(); struct Step { ModuleID importer; size_t nextRequest; @@ -155,7 +162,7 @@ bool Modules::wasLinkedAheadOfFailure(ModuleID root, ModuleID module) const Step& step = way.last(); auto requests = m_table.requests(step.importer); if (step.nextRequest == requests.size() || isMocked(step.importer)) { - if (step.importer == referrer) + if (step.importer == *referrer) return false; Step done = way.takeLast(); if (done.earliestPlaceReached == done.place) { @@ -190,7 +197,7 @@ bool Modules::wasLinkedAheadOfFailure(ModuleID root, ModuleID module) const bool Modules::wasRegisteredByFailedLoad(ModuleID module) const { - if (!m_errorsOfExportsThatBuildFoundAbsent || !data(module).isESModuleOfProgram()) + if (!hasLinkFailures() || !data(module).isESModuleOfProgram()) return false; if (status(module) == ModuleStatus::Failed) return !!failureByAbsentExport(module); @@ -199,7 +206,7 @@ bool Modules::wasRegisteredByFailedLoad(ModuleID module) const bool wasPulledIn = false; for (uint32_t index = 0; index < m_table.size(); ++index) { ModuleID root(index); - if (!failureByAbsentExport(root) || !modulesInLoadGraph(root).get(module.index())) + if (!moduleThatFailedToLink(root) || !modulesInLoadGraph(root).get(module.index())) continue; if (wasLinkedAheadOfFailure(root, module)) return false; @@ -210,11 +217,28 @@ bool Modules::wasRegisteredByFailedLoad(ModuleID module) const bool Modules::wasLinkedBeforeLinkFailure(ModuleID module) const { - if (!m_errorsOfExportsThatBuildFoundAbsent) + if (!hasLinkFailures()) return false; for (uint32_t index = 0; index < m_table.size(); ++index) { ModuleID root(index); - if (failureByAbsentExport(root) && modulesInLoadGraph(root).get(module.index()) && wasLinkedAheadOfFailure(root, module)) + if (moduleThatFailedToLink(root) && modulesInLoadGraph(root).get(module.index()) && wasLinkedAheadOfFailure(root, module)) + return true; + } + return false; +} + +bool Modules::isLinkedForImporterYetToRun(ModuleID module, std::optional asking) const +{ + if (!hasLinkFailures() || status(module) > ModuleStatus::Linked || !wasLinkedBeforeLinkFailure(module)) + return false; + for (uint32_t index = 0; index < m_table.size(); ++index) { + ModuleID importer(index); + if (importer == module || importer == asking || !data(importer).isESModuleOfProgram() || data(importer).isFacade() || status(importer) > ModuleStatus::Linked || isOfPackage(importer)) + continue; + bool importsModule = std::ranges::any_of(m_table.imports(importer), [&](const ModuleRequestData& request) { + return !request.isHoisted && moduleLoadedBy(request) == module; + }); + if (importsModule && wasLinkedBeforeLinkFailure(importer)) return true; } return false; @@ -259,6 +283,7 @@ void Modules::forgetLoadsRefusedByCheckOfNames(ModuleID mocked) void Modules::forgetFailuresOfExportsThatBuildFoundAbsent() { + m_linkFailures.clear(); JSObject* kept = std::exchange(m_errorsOfExportsThatBuildFoundAbsent, nullptr); if (!kept) return; @@ -291,8 +316,6 @@ bool Modules::evaluateLinkedModuleBeforeMock(ModuleID module, bool keepsOriginal ModuleStatus current = status(each); return current == ModuleStatus::Unlinked || current == ModuleStatus::Linked; }; - if (hasFailedWithoutMock(module) && !isMocked(module) && !m_modulesWithPendingMock.get(module.index())) - return !didCaptureExports(module); if (!data(module).isESModuleOfProgram() || data(module).isFacade() || !isUnevaluated(module) || isMocked(module) || m_modulesWithPendingMock.get(module.index())) return false; if (!isImportedByGraphUnderEvaluation(module, keepsOriginalIntact) && (keepsOriginalIntact || !wasLinkedBeforeLinkFailure(module))) @@ -1064,7 +1087,7 @@ void Modules::deferMockAfterAsyncFailure(ModuleID module) m_originalsDueUnderMock.set(module.index()); } -bool Modules::isOriginalDelegatedToHost(ModuleID mocked, const BitVector& held) +bool Modules::isOriginalDelegatedToHost(ModuleID mocked, const BitVector& held, const BitVector& isOnPath) { if (!data(mocked).isESModuleOfProgram() || data(mocked).isFacade() || !host().evaluateOriginalBeforeFactory) return true; @@ -1074,9 +1097,9 @@ bool Modules::isOriginalDelegatedToHost(ModuleID mocked, const BitVector& held) while (!work.isEmpty()) { for (uint32_t number : m_table.evaluationRequests(work.takeLast())) { ModuleID imported(number); - if (held.quickGet(number) || imported == mocked) + if (isOnPath.quickGet(number) || imported == mocked) continue; - if (hasPendingMock(fileOf(imported)) || m_modulesWithPendingMock.get(imported.index())) + if (held.quickGet(number) || hasPendingMock(fileOf(imported)) || m_modulesWithPendingMock.get(imported.index())) return true; if (!seen.quickSet(number) && status(imported) <= ModuleStatus::Linked && !isMocked(imported)) work.append(imported); @@ -1298,7 +1321,7 @@ bool Modules::sweepAheadOfFactories(ModuleID root) }; if (needsOriginalBeforeFactory(root)) { - if (isOriginalDelegatedToHost(root, held) || !noteMock(root)) + if (isOriginalDelegatedToHost(root, held, isOnPath) || !noteMock(root)) return true; OriginalInSweep original = runOriginalInSweep(root); EXCEPTION_ASSERT(!scope.exception() || original == OriginalInSweep::Failed); @@ -1339,7 +1362,7 @@ bool Modules::sweepAheadOfFactories(ModuleID root) fail(); RELEASE_AND_RETURN(scope, sweepAheadOfFactories(root)); } - if (isOriginalDelegatedToHost(imported, held)) + if (isOriginalDelegatedToHost(imported, held, isOnPath)) break; bool hasBeenEvaluated = m_importersEnteredAboveMocks.containsIf([&](const ImportersEnteredAboveMock& above) { return above.mocked == imported && above.original; }); OriginalInSweep original = hasBeenEvaluated ? OriginalInSweep::Evaluated : runOriginalInSweep(imported); @@ -1866,6 +1889,27 @@ void retryMockInDetachedLoads(JSGlobalObject* realm, ModuleID mocked) modules->retryMockInDetachedLoads(mocked); } +uint64_t Modules::resumingLoadFactoryCall(ModuleID mocked) const +{ + if (!m_resumingDetachedLoad) + return 0; + for (const DetachedLoad& left : m_detachedLoads) { + if (left.required != *m_resumingDetachedLoad || left.wait->status() != JSPromise::Status::Pending) + continue; + for (auto& [awaited, call] : left.awaitedFactoryCalls) { + if (awaited == mocked) + return call; + } + } + return 0; +} + +uint64_t resumingLoadFactoryCall(JSGlobalObject* realm, ModuleID mocked) +{ + Modules* modules = modulesOf(realm); + return modules ? modules->resumingLoadFactoryCall(mocked) : 0; +} + bool needsRetryInDetachedLoad(JSGlobalObject* realm, ModuleID mocked) { Modules* modules = modulesOf(realm); diff --git a/Source/JavaScriptCore/aot/runtime/AOTTestRunOriginals.cpp b/Source/JavaScriptCore/aot/runtime/AOTTestRunOriginals.cpp index 7e12a932658a4..938c51a4e7a0a 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTTestRunOriginals.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTTestRunOriginals.cpp @@ -481,10 +481,27 @@ void Modules::markOriginalDue(ModuleID module) record->clearTopLevelCapability(); } +bool Modules::isLeftByFailedLoad(ModuleID module) const +{ + bool wasReached = false; + for (uint32_t index = 0; index < m_table.size() && !wasReached; ++index) { + ModuleID importer(index); + ModuleRecord* record = m_records.get(importer); + wasReached = record && record->hasUnresolvableRequest() && status(importer) <= ModuleStatus::Linked && modulesInLoadGraph(importer).get(module.index()); + } + if (!wasReached || m_rootAwaitingLoader) + return false; + for (const LoadInFlight& load : m_loadsInFlight) { + if (m_loadsInFlight.isBeingLoaded(load) && modulesInLoadGraph(load.root).get(module.index())) + return false; + } + return true; +} + void Modules::retainRecordForInFlightLoads(ModuleID module) { const ModuleData& moduleData = data(module); - if (!moduleData.isESModuleOfProgram() || moduleData.isFacade() || activeMock(module)) + if (!moduleData.isESModuleOfProgram() || moduleData.isFacade() || activeMock(module) || isLeftByFailedLoad(module)) return; ModuleRecord* record = m_records.get(module); if (!record || record->hasLiveExports() || record->keepsToModule()) diff --git a/Source/JavaScriptCore/aot/runtime/AOTTextsOfFewDigits.h b/Source/JavaScriptCore/aot/runtime/AOTTextsOfFewDigits.h index 98bf51be83499..2486b2853ca8e 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTTextsOfFewDigits.h +++ b/Source/JavaScriptCore/aot/runtime/AOTTextsOfFewDigits.h @@ -8,19 +8,56 @@ #if ENABLE(AOT) #include "JSString.h" +#include +#include #include #include +#include namespace JSC { namespace AOT { +WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN + +ALWAYS_INLINE bool areDigitValues(uint64_t bytes) +{ + return !((bytes | (bytes + 0x7676767676767676ULL)) & 0x8080808080808080ULL); +} + +ALWAYS_INLINE uint32_t valueOfEightDigits(uint64_t digits) +{ + constexpr uint64_t lowByteOfEachHalf = 0x000000ff000000ffULL; + digits = digits * 10 + (digits >> 8); + return ((digits & lowByteOfEachHalf) * (100 + (1000000ULL << 32)) + ((digits >> 16) & lowByteOfEachHalf) * (1 + (10000ULL << 32))) >> 32; +} + ALWAYS_INLINE std::optional valueOfFewDigits(std::span characters) { + static_assert(std::endian::native == std::endian::little); constexpr size_t maxNumberOfDigits = 15; - if (characters.empty() || characters.size() > maxNumberOfDigits) + constexpr uint64_t zeroInEachByte = 0x3030303030303030ULL; + const Latin1Character* start = characters.data(); + size_t length = characters.size(); + if (length - 4 <= 4) { + uint64_t bytes = WTF::unalignedLoad(start) | static_cast(WTF::unalignedLoad(start + length - 4)) << 8 * (length - 4); + uint64_t digits = (bytes ^ zeroInEachByte) << 8 * (8 - length); + if (!areDigitValues(digits)) + return std::nullopt; + return static_cast(static_cast(valueOfEightDigits(digits))); + } + if (!length || length > maxNumberOfDigits) return std::nullopt; - uint64_t whole = 0; - for (Latin1Character character : characters) { - unsigned digit = character - '0'; + if (length > 8) { + static constexpr std::array powersOfTen { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000 }; + unsigned bitsOfOverlap = 8 * (16 - length); + uint64_t digits = WTF::unalignedLoad(start) ^ zeroInEachByte; + uint64_t lastDigits = (WTF::unalignedLoad(start + length - 8) ^ zeroInEachByte) >> bitsOfOverlap << bitsOfOverlap; + if (!areDigitValues(digits) || !areDigitValues(lastDigits)) + return std::nullopt; + return static_cast(static_cast(static_cast(valueOfEightDigits(digits)) * powersOfTen[length - 8] + valueOfEightDigits(lastDigits))); + } + int32_t whole = 0; + for (size_t i = 0; i < length; ++i) { + unsigned digit = start[i] - '0'; if (digit > 9) return std::nullopt; whole = whole * 10 + digit; @@ -28,6 +65,8 @@ ALWAYS_INLINE std::optional valueOfFewDigits(std::span(whole); } +WTF_ALLOW_UNSAFE_BUFFER_USAGE_END + ALWAYS_INLINE std::optional valueOfTextOfFewDigits(JSValue value) { if (!value.isString()) diff --git a/Source/JavaScriptCore/aot/runtime/AOTUnwinding.cpp b/Source/JavaScriptCore/aot/runtime/AOTUnwinding.cpp index 8a5df5fdd1f99..42eb9fed50bee 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTUnwinding.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTUnwinding.cpp @@ -136,6 +136,15 @@ static void finishInRejectingStub(CompiledUnwindResult& result, Instance& instan finishAtHandler(result, instance, { callFrame, code, CompiledHandlerKind::RejectPromise, callFrame }, frame.frameSizeInBytes()); } +static void finishInAwaitedCall(CompiledUnwindResult& result, Instance& instance, CallFrame* callFrame, const CallSiteCache::Entry& site) +{ + void* code = Image::current()->frequentStub(Image::FrequentStub::RejectAwaitedCall, [&] { + return addressOfStubInCurrentImage(rejectAwaitedCall()); + }); + instance.runtime.exceptions.returnAddressOfAwaitedCall = site.address; + finishAtHandler(result, instance, { callFrame, code, CompiledHandlerKind::RejectPromise, callFrame }, site.frame.frameSizeInBytes()); +} + static CompiledFrameExit leaveRunAt(CallFrame* outermostFrame, EntryFrame* entryFrame) { EntryFrame* entryFrameOfRun = entryFrame; @@ -209,6 +218,10 @@ static ALWAYS_INLINE CompiledUnwindResult unwindFramesOfRun(VM& vm, const Return finishInRejectingStub(result, *instance, asyncGeneratorBodyFrameLeft, asyncGeneratorBodyFrameLeft == leftFrame ? leftImageFrame : ImageFrame { }); return result; } + if (site->site.isAwaited() && leftFrame) [[unlikely]] { + finishInAwaitedCall(result, *instance, callFrame, *site); + return result; + } if (const HandlerRecord* handler = site->exceptionHandler()) { void* code = const_cast(image.codeOf(site->code.index())) + handler->codeOffset; finishAtHandler(result, *instance, { callFrame, code, kindOf(handler->type), place.framePointer }, site->frame.frameSizeInBytes()); @@ -358,7 +371,7 @@ static bool findCatchInRun(CompiledFrameIterator& frames) const HandlerRecord* handler = site.exceptionHandler(); if (handler && kindOf(handler->type) == CompiledHandlerKind::Catch) return true; - if (site.rejectsOnThrow) + if (site.rejectsOnThrow || site.site.isAwaited()) return true; instance = frames.instance(); lastFrame = frames.machineFrame(); @@ -399,6 +412,8 @@ static ALWAYS_INLINE std::optional findHandlerInRun(VM& vm, CallFra if (search == HandlerSearch::Nearest && site->stubFrameKind == StubFrameKind::ModuleStart && abandonsModuleOnThrow(static_cast(framePointer))) return HandlerType::Finally; } else { + if (site->site.isAwaited()) + return HandlerType::Catch; const HandlerRecord* handler = site->exceptionHandler(); if (handler && (search == HandlerSearch::Nearest || handler->type != HandlerType::Finally)) return handler->type; diff --git a/Source/JavaScriptCore/aot/runtime/AOTVMState.h b/Source/JavaScriptCore/aot/runtime/AOTVMState.h index f82334ca17cd6..9ab4c4fb29978 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTVMState.h +++ b/Source/JavaScriptCore/aot/runtime/AOTVMState.h @@ -9,6 +9,7 @@ #include "AOTVector.h" #include +#include #include #include #include @@ -43,9 +44,44 @@ class VMPropertyCaches; struct CyclesOfImports; struct FormatPatternState; struct FunctionTables; +struct ImportersOfModules; struct Instance; struct ReexportedNames; +class PlacesSeenByPlans { +public: + uint32_t takeMark(size_t numberOfModules) + { + if (m_numberOfModules != numberOfModules || m_nextMark > std::numeric_limits::max() - 3 - numberOfModules) { + m_numberOfModules = numberOfModules; + m_groups.clear(); + m_groups.grow((numberOfModules + modulesInGroup - 1) / modulesInGroup); + m_nextMark = 1; + } + return m_nextMark++; + } + + void setNextMark(uint32_t mark) { m_nextMark = mark; } + + uint32_t& operator[](size_t module) + { + std::unique_ptr& group = m_groups[module / modulesInGroup]; + if (!group) + group = makeUnique(); + return group->places[module % modulesInGroup]; + } + +private: + static constexpr size_t modulesInGroup = 64; + struct Group { + WTF_DEPRECATED_MAKE_STRUCT_FAST_ALLOCATED(Group); + std::array places { }; + }; + Vector> m_groups; + size_t m_numberOfModules { 0 }; + uint32_t m_nextMark { 1 }; +}; + class StructuresWithFastIndexing { public: void add(Structure* structure) { m_structures.append(structure); } @@ -103,7 +139,9 @@ struct VMState { std::unique_ptr callSiteCache; std::unique_ptr functionTables; std::unique_ptr cyclesOfImports; + std::unique_ptr importersOfModules; std::unique_ptr reexportedNames; + PlacesSeenByPlans placesSeenByPlans; std::unique_ptr regExpLiterals; std::unique_ptr formatPatternState; BitVector filesWithKnownFirstLoad; @@ -152,6 +190,13 @@ struct VMState { bool requireRunsMocks { false }; std::unique_ptr identifiers; CountedFiles countedFiles; + struct ComputedFunctionName { + Weak name; + unsigned hash { 0 }; + unsigned length { 0 }; + bool isOneOfSeveral { false }; + }; + UncheckedKeyHashMap, WTF::UnsignedWithZeroKeyHashTraits> computedFunctionNames; struct KeysComparedLastWithLongNames { UncheckedKeyHashMap byName; uint64_t versionOfHeap { 0 }; diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubArgumentThunks.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubArgumentThunks.cpp index e3cbe0d60e8df..f9b0b5085709a 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubArgumentThunks.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubArgumentThunks.cpp @@ -10,6 +10,7 @@ #include "AOTBuildContext.h" #include "AOTConvention.h" +#include "AOTInstance.h" #include "AOTStubAssembler.h" #include "AOTStubCatalog.h" #include "AOTStubDescriptor.h" @@ -18,6 +19,7 @@ #include "AOTVector.h" #include "AllowMacroScratchRegisterUsage.h" #include "Butterfly.h" +#include "JSArrayBufferView.h" #include "JSObject.h" #include "JSString.h" #include @@ -146,7 +148,7 @@ static void emitConstantsOfThunk(CCallHelpers& jit, std::span mo }); } -static void returnLengthOfListOrTextInFrontOfMoves(CCallHelpers& jit, const StubSignature& signature, std::span moves, std::optional asked) +static void returnLengthOfListOrTextInFrontOfMoves(CCallHelpers& jit, const StubSignature& signature, std::span moves, std::optional asked, bool checksWalk) { const GPRReg operand = signature.operands[0].gpr(); const GPRReg scratch = *signature.immediate; @@ -198,6 +200,17 @@ static void returnLengthOfListOrTextInFrontOfMoves(CCallHelpers& jit, const Stub jit.load8(CCallHelpers::Address(receiver, JSCell::indexingTypeAndMiscOffset()), scratch); jit.sub32(CCallHelpers::TrustedImm32(ArrayWithUndecided), scratch); jit.branchTest32(CCallHelpers::Zero, scratch, CCallHelpers::TrustedImm32(0xff & ~(6 | CopyOnWrite | MayHaveIndexedAccessors))).linkTo(hasPlainStorage, &jit); + CCallHelpers::JumpList isNoPlainBuffer; + if (!checksWalk) { + jit.load32(CCallHelpers::Address(receiver, JSCell::structureIDOffset()), scratch); + isNoPlainBuffer.append(jit.branch32(CCallHelpers::NotEqual, scratch, CCallHelpers::Address(instanceGPR, Instance::offsetOfReceiverStructureID(Receiver::Buffer)))); + isNoPlainBuffer.append(jit.branchTest8(CCallHelpers::NonZero, CCallHelpers::Address(instanceGPR, AOT_INSTANCE_OFFSET(builtins.someClassOfViewsWasWrittenAbove)))); + isNoPlainBuffer.append(jit.branchTest8(CCallHelpers::NonZero, CCallHelpers::Address(receiver, JSArrayBufferView::offsetOfMode()), CCallHelpers::TrustedImm32(resizabilityAndAutoLengthMask))); + jit.load64(CCallHelpers::Address(receiver, JSArrayBufferView::offsetOfLength()), scratch); + isNoPlainBuffer.append(jit.branch64(CCallHelpers::Above, scratch, CCallHelpers::TrustedImm32(std::numeric_limits::max()))); + answer(CCallHelpers::Address(receiver, JSArrayBufferView::offsetOfLength())); + } + isNoPlainBuffer.link(&jit); isNotCell.link(&jit); } @@ -222,9 +235,9 @@ static std::expected generateArgumentThunksOfChunk( labels.append(jit.label()); if (key.family() == StubFamily::GetLengthAtSite) - returnLengthOfListOrTextInFrontOfMoves(jit, signature, moves, std::nullopt); + returnLengthOfListOrTextInFrontOfMoves(jit, signature, moves, std::nullopt, catalog.options().checksWalk); if (key.family() == StubFamily::LengthIs) - returnLengthOfListOrTextInFrontOfMoves(jit, signature, moves, static_cast(key.count())); + returnLengthOfListOrTextInFrontOfMoves(jit, signature, moves, static_cast(key.count()), catalog.options().checksWalk); emitRegisterMovesOfThunk(jit, moves); emitLoadsOfThunk(jit, moves); emitConstantsOfThunk(jit, moves); diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubCalls.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubCalls.cpp index 0d5d98e4571ce..f322ec14a8c77 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubCalls.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubCalls.cpp @@ -112,6 +112,7 @@ StubCalls::StubCalls(const StubCatalog& catalog, RelocationRecorder& relocations , m_relocations(relocations) , m_ownRememberedKeys(rememberedKeys ? nullptr : std::make_unique()) , m_rememberedKeys(rememberedKeys ? *rememberedKeys : *m_ownRememberedKeys) + , m_unit(++m_rememberedKeys.unit) { } @@ -145,7 +146,18 @@ CodeSymbol StubCalls::prepare(StubKey key, NumberedImmediate immediate) GPRReg destination = immediateGPROfStub(m_rememberedKeys, m_catalog, finalKey); RELEASE_ASSERT(!m_registerFilledInFrontOfNextCall || (destination != GPRInfo::returnValueGPR && Reg(destination) != m_registerFilledInFrontOfNextCall)); moveNumberedImmediate(m_relocations, immediate, destination); - return { SymbolSpace::Stub, indexOf(finalKey, siblingUnderNameWithColumn(finalKey) ? std::optional(immediate.mention) : std::nullopt) }; + if (siblingUnderNameWithColumn(finalKey)) + return { SymbolSpace::Stub, indexOf(finalKey, immediate.mention) }; + return { SymbolSpace::Stub, indexOf(finalKey) }; +} + +ALWAYS_INLINE uint32_t StubCalls::indexOf(StubKey key) +{ + constexpr uint64_t multiplier = 0x9e3779b97f4a7c15ull; + CachedStubIndex& cached = m_rememberedKeys.indexCache[(key.bits() * multiplier) >> (64 - RememberedStubKeys::logOfNumberOfKeys)]; + if (cached.unit != m_unit || cached.bits != key.bits()) + cached = { key.bits(), m_unit, indexOf(key, std::nullopt) }; + return cached.index; } uint32_t StubCalls::indexOf(StubKey key, std::optional mentionOfImmediate) diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubCalls.h b/Source/JavaScriptCore/aot/stubs/AOTStubCalls.h index 2e1c248a941fa..be21008a4d2af 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubCalls.h +++ b/Source/JavaScriptCore/aot/stubs/AOTStubCalls.h @@ -163,10 +163,18 @@ struct RememberedStubKey { GPRReg immediateGPR; }; +struct CachedStubIndex { + uint64_t bits; + uint32_t unit; + uint32_t index; +}; + struct RememberedStubKeys { static constexpr unsigned logOfNumberOfKeys = 8; - std::array keys; + std::array keys { }; + std::array indexCache { }; + uint32_t unit { 0 }; }; class StubCalls { @@ -207,7 +215,8 @@ class StubCalls { ResolvedKey resolveKey(StubKey, std::optional immediate) const; StubKey prepare(CCallHelpers&, StubKey, std::optional immediate); CodeSymbol prepare(StubKey, NumberedImmediate); - uint32_t indexOf(StubKey, std::optional mentionOfImmediate = std::nullopt); + uint32_t indexOf(StubKey); + uint32_t indexOf(StubKey, std::optional mentionOfImmediate); uint32_t indexOf(Typed::CodeUnit); uint32_t emitCall(CCallHelpers&, StubKey called, CodeSymbol, const RecordedCall&); Reg m_registerFilledInFrontOfNextCall; @@ -218,6 +227,7 @@ class StubCalls { RelocationRecorder& m_relocations; std::unique_ptr m_ownRememberedKeys; RememberedStubKeys& m_rememberedKeys; + const uint32_t m_unit; Vector m_calls; Vector m_stubKeys; Vector, 16> m_mentionsOfImmediates; diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubCoreFamilies.def b/Source/JavaScriptCore/aot/stubs/AOTStubCoreFamilies.def index e78f907c73c44..e9b8cc99960eb 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubCoreFamilies.def +++ b/Source/JavaScriptCore/aot/stubs/AOTStubCoreFamilies.def @@ -15,6 +15,7 @@ v(HandleExceptionAtPrologue, CalledByCode, (), NeverReturns, 0, 0, AllISAs, NoIS v(ThrowStackOverflow, CalledByCode, (InBody, AtPrologue), FormAxis | NeverReturns, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, NoEffects, ThrowStackOverflow) v(Catch, EnteredByRuntime, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, Catch) v(RejectInFrame, EnteredByRuntime, (Plain, BelowSpilledArguments), FormAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, RejectInFrame) +v(RejectAwaitedCall, EnteredByRuntime, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, RejectAwaitedCall) v(PollAtPrologue, CalledByCode, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, NoEffects, PollAtPrologue) v(PollAtPrologueWithoutCollecting, CalledByCode, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, NoEffects, PollAtPrologueWithoutCollecting) @@ -69,11 +70,14 @@ v(StringOfAnyValueFrontEnd, FrontEnd, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice AOT_STUB_FRONT_END(StringOfAnyValue, StringOfAnyValueFrontEnd) v(MoveFewBytes, CalledByCode, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, Leaf, ClobbersOnlyScratch, reads(TypedArrayElements) | writes(TypedArrayElements), MoveFewBytes) +v(MoveElementsDown, CalledByCode, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, MoveElementsDown) v(GetByVal, CalledByCode, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, GetByVal) v(GetByValAtIndex, CalledByCode, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, GetByVal) v(GetByValKeepingRegisters, InternalEntry, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, AsOperation, GetByValKeepingRegisters) +v(GetByValAtConstantIndex, CalledByCode, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, AsOperation, GetByValKeepingRegisters) v(PutByVal, CalledByCode, (Strict, Sloppy), FormAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, PutByVal) v(PutByValAtIndex, CalledByCode, (Strict, Sloppy), FormAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, PutByVal) +v(PutByValAtConstantIndex, CalledByCode, (Strict, Sloppy), FormAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, PutByVal) v(StoreElementOfCompiledObject, InternalEntry, (Strict, Sloppy, Define), FormAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, StoreElementOfCompiledObject) v(GetByValExpecting, CalledByCode, (Int32Array, Uint8Array, Float64Array), FormAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, GetByValExpecting) v(GetByValAtIndexExpecting, CalledByCode, (Int32Array, Uint8Array, Float64Array), FormAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, GetByValExpecting) diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubHelpersValues.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubHelpersValues.cpp index 6c8a2b9fbd0d9..1bd46de1107e9 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubHelpersValues.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubHelpersValues.cpp @@ -304,14 +304,14 @@ void setFlagThatFieldHasLeft(StubAssembler& jit, GPRReg cell, GPRReg scratch) void setMarkOfCell(StubAssembler& jit, GPRReg cell, GPRReg scratch) { - const Address byteWithMark(cell, offsetOfByteWithMark); + const Address shapeWord(cell, offsetOfShapeWord); #if CPU(X86_64) UNUSED_PARAM(scratch); - jit.and8(TrustedImm32(static_cast(~byteWithMarkIsUnmarked)), byteWithMark); + jit.and32(TrustedImm32(static_cast(~shapeWordIsUnmarked)), shapeWord); #else - jit.load8(byteWithMark, scratch); - jit.and32(TrustedImm32(static_cast(~byteWithMarkIsUnmarked)), scratch); - jit.store8(scratch, byteWithMark); + jit.load32(shapeWord, scratch); + jit.and32(TrustedImm32(static_cast(~shapeWordIsUnmarked)), scratch); + jit.store32(scratch, shapeWord); #endif } diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsArithmetic.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsArithmetic.cpp index c686bc83c9378..48438258d02b3 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsArithmetic.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsArithmetic.cpp @@ -394,15 +394,16 @@ static void emitStringOrder(StubAssembler& jit, const BinaryRegisters& registers const auto [left, right, first, second, scratch, leftNumber, rightNumber] = registers; const Comparison& comparison = comparisonOf(operation); - JumpList notTwoFlatStrings; - notTwoFlatStrings.append(jit.branchIfNotCell(left)); - notTwoFlatStrings.append(jit.branchIfNotCell(right)); - notTwoFlatStrings.append(jit.branchIfNotString(left)); - notTwoFlatStrings.append(jit.branchIfNotString(right)); + JumpList isLeftToOperation; + isLeftToOperation.append(jit.branchIfNotCell(left)); + isLeftToOperation.append(jit.branchIfNotCell(right)); + isLeftToOperation.append(jit.branchIfNotString(left)); + isLeftToOperation.append(jit.branchIfNotString(right)); jit.loadPtr(Address(left, JSString::offsetOfValue()), first); jit.loadPtr(Address(right, JSString::offsetOfValue()), second); - notTwoFlatStrings.append(jit.branchIfRopeStringImpl(first)); - notTwoFlatStrings.append(jit.branchIfRopeStringImpl(second)); + JumpList oneIsRope; + oneIsRope.append(jit.branchIfRopeStringImpl(first)); + oneIsRope.append(jit.branchIfRopeStringImpl(second)); JumpList needsLeaf; JumpList isLess; @@ -426,7 +427,19 @@ static void emitStringOrder(StubAssembler& jit, const BinaryRegisters& registers jit.compare64(comparison.ofIntegers, GPRInfo::returnValueGPR, TrustedImm32(0), left); jit.returnToCaller(); - notTwoFlatStrings.link(&jit); + oneIsRope.link(&jit); + const GPRReg order = scratch; + RELEASE_ASSERT(right == secondArgument && noOverlap(order, left, right)); + jit.preserving({ left, right }, [&] { + jit.move(left, firstArgument); + jit.callLeaf(Entry::operationAOTCompareStringsInPlace, 2 * sizeof(CPURegister)); + jit.move(GPRInfo::returnValueGPR, order); + }); + isLeftToOperation.append(jit.branch64(CCallHelpers::Equal, order, TrustedImm32(noOrderOfStringsInPlace))); + jit.compare64(comparison.ofIntegers, order, TrustedImm32(0), left); + jit.returnToCaller(); + + isLeftToOperation.link(&jit); } StubSignature signatureOfBinaryOperation(StubKey) { return StubSignature::withGPRResult({ gprOf(StubRegister::R0), gprOf(StubRegister::A1) }); } diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsBoxedCalls.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsBoxedCalls.cpp index 42941f4f8401b..ba5cb3fb748c9 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsBoxedCalls.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsBoxedCalls.cpp @@ -83,10 +83,11 @@ void emitBoxedCallDispatch(StubAssembler& jit, const CallRegisters& registers, u RELEASE_ASSERT(noOverlap(executable, registers.word, entry, instanceGPR, calleeGPR, thisGPR, argumentGPR(0), argumentGPR(1), argumentGPR(2), argumentGPR(3))); constexpr unsigned numberOfForms = descriptorOf(StubFamily::CallBoxedBuiltin).numberOfForms; - jit.load8(Address(executable, NativeExecutable::offsetOfIndexOfBoxedCall()), entry); - Jump isNotLookedUp = jit.branchTest32(CCallHelpers::Zero, entry); + const Address indexOfBoxedCall(executable, NativeExecutable::offsetOfIndexOfBoxedCall()); Label isLookedUp = jit.label(); - hasNoBoxedCall.append(jit.branch32(CCallHelpers::Equal, entry, TrustedImm32(hasNoBoxedCallEntry))); + hasNoBoxedCall.append(jit.branch8(CCallHelpers::Equal, indexOfBoxedCall, TrustedImm32(hasNoBoxedCallEntry))); + jit.load8(indexOfBoxedCall, entry); + Jump isNotLookedUp = jit.branchTest32(CCallHelpers::Zero, entry); std::array handlers; for (unsigned form = 0; form < numberOfForms; ++form) handlers[form] = callBoxedBuiltin(static_cast(form), count); @@ -1164,14 +1165,17 @@ static void emitArrayShift(StubAssembler& jit, GPRReg scratch, JumpList& callsHo jit.load32(Address(butterfly, Butterfly::offsetOfPublicLength()), last); Jump isEmpty = jit.branchTest32(CCallHelpers::Zero, last); jit.sub32(TrustedImm32(1), last); - isLeftToOperation.append(jit.branch32(CCallHelpers::Above, last, TrustedImm32(mostValuesMovedByShiftInStub))); + Jump isLong = jit.branch32(CCallHelpers::Above, last, TrustedImm32(mostValuesMovedByShiftInStub)); jit.load64(Address(butterfly), element); isLeftToOperation.append(jit.branchTest64(CCallHelpers::Zero, element)); Jump isAlone = jit.branchTest32(CCallHelpers::Zero, last); - isLeftToOperation.append(branchIfNotPristine(jit, thisGPR)); - Jump holdsIntegers = jit.branch32(CCallHelpers::Equal, kind, TrustedImm32(Int32Shape)); - isLeftToOperation.append(jit.branchTest8(CCallHelpers::NonZero, Address(instanceGPR, AOT_INSTANCE_OFFSET(mutatorShouldBeFenced)))); - holdsIntegers.link(&jit); + auto leaveToOperationUnlessValuesMayMove = [&] { + isLeftToOperation.append(branchIfNotPristine(jit, thisGPR)); + Jump holdsIntegers = jit.branch32(CCallHelpers::Equal, kind, TrustedImm32(Int32Shape)); + isLeftToOperation.append(jit.branchTest8(CCallHelpers::NonZero, Address(instanceGPR, AOT_INSTANCE_OFFSET(mutatorShouldBeFenced)))); + holdsIntegers.link(&jit); + }; + leaveToOperationUnlessValuesMayMove(); jit.move(TrustedImm32(0), index); Label moveNext = jit.label(); @@ -1186,6 +1190,24 @@ static void emitArrayShift(StubAssembler& jit, GPRReg scratch, JumpList& callsHo jit.move(element, jit.result()); jit.returnToCaller(); + isLong.link(&jit); + isLeftToOperation.append(jit.branch32(CCallHelpers::AboveOrEqual, last, TrustedImm32(JSArray::shiftThreshold))); + jit.load64(Address(butterfly), element); + isLeftToOperation.append(jit.branchTest64(CCallHelpers::Zero, element)); + leaveToOperationUnlessValuesMayMove(); + const GPRReg target = jit.reuseScratch(StubRegister::A0); + RELEASE_ASSERT(target == leafArgumentGPR(0) && butterfly == leafArgumentGPR(1) && last == leafArgumentGPR(2)); + jit.store32(last, Address(butterfly, Butterfly::offsetOfPublicLength())); + jit.getEffectiveAddress(BaseIndex(butterfly, last, CCallHelpers::TimesEight), moved); + jit.pushPair(element, moved); + jit.move(butterfly, target); + jit.addPtr(TrustedImm32(sizeof(EncodedJSValue)), butterfly); + jit.lshift64(TrustedImm32(CCallHelpers::TimesEight), last); + jit.callLeaf(Entry::operationAOTMoveBytes, 2 * sizeof(CPURegister)); + jit.popPair(jit.result(), moved); + jit.store64(TrustedImm32(0), Address(moved)); + jit.returnToCaller(); + isEmpty.link(&jit); jit.move(TrustedImm64(JSValue::encode(jsUndefined())), jit.result()); jit.returnToCaller(); @@ -1697,7 +1719,7 @@ static bool isKnownAsMadeByOneCompare(const StubAssembler& jit, const BoxedCallH return flagsOfReceiverAsItsClassMakesIt(handler) && !jit.options().methodsOfBuiltinsMayBeReplaced && jit.options().hasOneInstance; } -static void branchUnlessReceiverFindsBuiltin(StubAssembler& jit, const BoxedCallHandler& handler, MethodIsRead methodIsRead, GPRReg scratch, JumpList& looksUp, std::optional entryForProvenClass = std::nullopt, GPRReg receiver = thisGPR, LooksBehindFirstLook* looksBehind = nullptr) +static void branchUnlessReceiverFindsBuiltin(StubAssembler& jit, const BoxedCallHandler& handler, MethodIsRead methodIsRead, GPRReg scratch, JumpList& looksUp, std::optional entryForProvenClass = std::nullopt, GPRReg receiver = thisGPR, LooksBehindFirstLook* looksBehind = nullptr, std::optional stubOfMethod = std::nullopt) { JumpList isOfClassAndAsMade; if (handler.receiverTypes) { @@ -1740,6 +1762,11 @@ static void branchUnlessReceiverFindsBuiltin(StubAssembler& jit, const BoxedCall jit.and32(TrustedImm32(PropertyCacheWords::numberOfStructuresBelowBuiltinClasses - 1), scratch); jit.load32(BaseIndex(instanceGPR, scratch, CCallHelpers::TimesFour, AOT_INSTANCE_OFFSET(propertyCaches.structuresBelowBuiltinClasses)), scratch); looksUp.append(jit.branch32(CCallHelpers::NotEqual, scratch, Address(receiver, JSCell::structureIDOffset()))); + if (stubOfMethod && !jit.options().checksWalk) { + jit.linkToStub(isAsMade, *stubOfMethod); + jit.linkToStub(isOfClassAndAsMade, *stubOfMethod); + return; + } isAsMade.link(&jit); isOfClassAndAsMade.link(&jit); askAgainWhatReceiverFinds(jit, handler.form, methodIsRead, receiver); @@ -2018,13 +2045,18 @@ void generateCallMethodOfTable(StubAssembler& jit) const BoxedCallHandler& handler = boxedCallHandlers[static_cast(boxedCallFormOf(method))]; const unsigned count = method == TableMethod::MapSet ? 2 : 1; const GPRReg name = gprOf(stubImmediateRegister); - const HashTableRegisters registers = HashTableRegisters::take(jit); - const GPRReg receiver = registers.owner; + const GPRReg receiver = jit.operand(0); + const GPRReg scratch = jit.allocateScratch(hashTableRegisterPlan.hash); + const StubKey stubOfMethod = stubOfTableMethod(method, keyComparisonOf(jit.key())); RELEASE_ASSERT(handler.receiverTypes && handler.emit == emitJumpToTarget && handler.firstArgument == ArgumentGuard::None && servesCallsByName(handler.form, count)); - RELEASE_ASSERT(noOverlap(receiver, name, registers.hash, argumentGPR(0), argumentGPR(1), instanceGPR)); + RELEASE_ASSERT(noOverlap(receiver, name, scratch, argumentGPR(0), argumentGPR(1), instanceGPR)); + if (method == TableMethod::MapSet) + checkStubOperands(stubOfMethod, { receiver, argumentGPR(0), argumentGPR(1) }); + else + checkStubOperands(stubOfMethod, { receiver, argumentGPR(0) }); JumpList looksUp; - branchUnlessReceiverFindsBuiltin(jit, handler, MethodIsRead::AtTheCall, registers.hash, looksUp, jit.key().withFamily(StubFamily::CallMethodOfProvenTable), receiver); + branchUnlessReceiverFindsBuiltin(jit, handler, MethodIsRead::AtTheCall, scratch, looksUp, jit.key().withFamily(StubFamily::CallMethodOfProvenTable), receiver, nullptr, stubOfMethod); jit.addOutOfLinePart([=](StubAssembler& jit) { looksUp.link(&jit); jit.move(receiver, thisGPR); @@ -2032,7 +2064,7 @@ void generateCallMethodOfTable(StubAssembler& jit) jit.move(TrustedImm32(static_cast(*number)), name); jit.jumpToStub(callMethodById(count)); }); - emitMethodOfTable(jit, method, registers); + jit.jumpToStub(stubOfMethod); } void generateCallMethodOfBuiltinClass(StubAssembler& jit) diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsBoxedCallsOfStrings.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsBoxedCallsOfStrings.cpp index 530e45417e07f..1e27e4c41b5cb 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsBoxedCallsOfStrings.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsBoxedCallsOfStrings.cpp @@ -66,9 +66,11 @@ void emitStringCodeUnitAt(StubAssembler& jit, GPRReg scratch, JumpList& callsHos RELEASE_ASSERT(noOverlap(impl, unit, trail, calleeGPR, thisGPR, instanceGPR, argumentGPR(0))); callsHostFunction.append(jit.branchIfNotInt32(argumentGPR(0))); + Label hasString = jit.label(); jit.loadPtr(Address(thisGPR, JSString::offsetOfValue()), impl); Jump isRope = jit.branchIfRopeStringImpl(impl); JumpList isOutside; + JumpList isNotSubstring; isOutside.append(jit.branch32(CCallHelpers::AboveOrEqual, argumentGPR(0), Address(impl, StringImpl::lengthMemoryOffset()))); jit.zeroExtend32ToWord(argumentGPR(0), unit); Jump is16Bit = jit.branchTest32(CCallHelpers::Zero, Address(impl, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit())); @@ -79,7 +81,7 @@ void emitStringCodeUnitAt(StubAssembler& jit, GPRReg scratch, JumpList& callsHos jit.returnToCaller(); isRope.link(&jit); - loadSubstringCharacters(jit, thisGPR, impl, trail, unit, callsHostFunction); + loadSubstringCharacters(jit, thisGPR, impl, trail, unit, isNotSubstring); isOutside.append(jit.branch32(CCallHelpers::AboveOrEqual, argumentGPR(0), unit)); jit.zeroExtend32ToWord(argumentGPR(0), unit); Jump pieceIs16Bit = jit.branchTest32(CCallHelpers::Zero, Address(impl, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit())); @@ -122,6 +124,9 @@ void emitStringCodeUnitAt(StubAssembler& jit, GPRReg scratch, JumpList& callsHos isOneUnit.link(&jit); boxInt32(jit, unit, jit.result()); jit.returnToCaller(); + + isNotSubstring.link(&jit); + flattenSubject(jit, thisGPR, { thisGPR, argumentGPR(0) }, hasString); } void emitStringCharacterAt(StubAssembler& jit, GPRReg scratch, JumpList& callsHostFunction) diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsCallAdapters.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsCallAdapters.cpp index 57958fcf1fd82..bd3c4a6735fcc 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsCallAdapters.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsCallAdapters.cpp @@ -12,6 +12,7 @@ #include "AOTCellFormat.h" #include "AOTConvention.h" #include "AOTFunctionTables.h" +#include "AOTImageFormat.h" #include "AOTInstance.h" #include "AOTProgramTables.h" #include "AOTStubAssembler.h" @@ -22,13 +23,11 @@ #include "AOTStubsClosures.h" #include "AOTStubsExceptions.h" #include "CallFrame.h" -#include "EntryFrame.h" #include "JSGlobalObject.h" #include "MicrotaskQueue.h" #include "StackAlignment.h" #include "StackManager.h" #include "VM.h" -#include "VMEntryRecord.h" #include namespace JSC { namespace AOT { @@ -103,30 +102,7 @@ static void emitSaveRegistersInAdapter(StubAssembler& jit, GPRReg instance) #endif } -static void emitFillFrameOfEntryFromNativeCode(StubAssembler& jit, GPRReg vm, AdapterCaller caller) -{ - const ptrdiff_t begin = EntryFrame::vmEntryRecordOffset() + OBJECT_OFFSETOF(VMEntryRecord, calleeSaveRegistersBuffer); - RELEASE_ASSERT(begin < 0 && !(begin % static_cast(stackAlignmentBytes()))); - Jump isCalledByInterpreter; - if (caller == AdapterCaller::NativeCodeOrInterpreter) - isCalledByInterpreter = jit.branchPtr(CCallHelpers::NotEqual, Address(vm, VM::topEntryFrameOffset()), fp); -#if CPU(ARM64) - for (ptrdiff_t offset = begin; offset < 0; offset += 2 * sizeof(CPURegister)) - jit.storePair64(ARM64Registers::zr, ARM64Registers::zr, fp, TrustedImm32(static_cast(offset))); -#else - for (ptrdiff_t offset = begin; offset < 0; offset += sizeof(CPURegister)) - jit.store64(TrustedImm32(0), Address(fp, static_cast(offset))); - if (caller == AdapterCaller::NativeCodeOrInterpreter) { - Jump argumentsFillFrame = jit.branchTest32(CCallHelpers::NonZero, incomingFrameSlot(CallFrameSlot::argumentCountIncludingThis), TrustedImm32(1)); - jit.store64(TrustedImm32(JSValue::ValueUndefined), Address(fp, static_cast(EntryFrame::vmEntryRecordOffset() - sizeof(Register)))); - argumentsFillFrame.link(&jit); - } -#endif - if (isCalledByInterpreter.isSet()) - isCalledByInterpreter.link(&jit); -} - -void emitEnterAdapter(StubAssembler& jit, AdapterCaller caller, unsigned extraBytes) +void emitEnterAdapter(StubAssembler& jit, unsigned extraBytes) { const GPRReg instance = jit.reuseScratch(StubRegister::EntryT12); const GPRReg scratch = jit.reuseScratch(StubRegister::T9); @@ -151,13 +127,10 @@ void emitEnterAdapter(StubAssembler& jit, AdapterCaller caller, unsigned extraBy jit.jump().linkTo(hasInstance, &jit); }); - emitFillFrameOfEntryFromNativeCode(jit, scratch, caller); jit.emitFunctionPrologue(); static_assert(!(AdapterFrame::saveAreaSize % stackAlignmentBytes())); jit.subPtr(TrustedImm32(AdapterFrame::saveAreaSize + extraBytes), sp); emitSaveRegistersInAdapter(jit, instance); - if constexpr (-AdapterFrame::offsetOfNewTarget < static_cast(AdapterFrame::saveAreaSize)) - jit.store64(TrustedImm32(0), Address(fp, AdapterFrame::offsetOfNewTarget - static_cast(sizeof(CPURegister)))); jit.move(instance, instanceGPR); jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(cellCageBase)), cellCageBaseGPR); #if !CPU(X86_64) @@ -382,12 +355,12 @@ void generateEnterClosure(StubAssembler& jit) { const GPRReg word = jit.reuseScratch(StubRegister::T11); JumpList hasWord; - emitEnterAdapter(jit, AdapterCaller::NativeCodeOrInterpreter); + emitEnterAdapter(jit); jit.load64(Address(calleeGPR, offsetOfClosureEntry), word); hasWord.append(jit.jump()); jit.bindEntry(enterModuleBody()); - emitEnterAdapter(jit, AdapterCaller::NativeCodeOrInterpreter); + emitEnterAdapter(jit); jit.load32(Address(fp, offsetOfFirstArgument), word); jit.lshift64(TrustedImm32(3), word); jit.add64(Address(instanceGPR, AOT_INSTANCE_OFFSET(functionTables.closureWords)), word); @@ -395,7 +368,7 @@ void generateEnterClosure(StubAssembler& jit) hasWord.append(jit.jump()); jit.bindEntry(enterSlimClosure()); - emitEnterAdapter(jit, AdapterCaller::NativeCodeOrInterpreter); + emitEnterAdapter(jit); loadSlimClosureWord(jit, calleeGPR, word); hasWord.link(&jit); @@ -502,8 +475,20 @@ void generateStartModule(StubAssembler& jit) Jump hasTarget = jit.jump(); callsBody.link(&jit); - jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(image.functionStarts)), target); - jit.load32(BaseIndex(target, operand, CCallHelpers::TimesFour), target); + jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(image.functionStartGroups)), table); + jit.urshift32(operand, TrustedImm32(FunctionStarts::groupShift), step); + jit.load32(BaseIndex(table, step, CCallHelpers::TimesFour), target); + Jump isInNarrowGroup = jit.branch32(CCallHelpers::GreaterThanOrEqual, target, TrustedImm32(0)); + jit.and32(TrustedImm32(static_cast(~FunctionStarts::wideGroup)), target); + jit.and32(TrustedImm32(FunctionStarts::functionsInGroup - 1), operand, step); + jit.add32(step, target); + jit.load32(BaseIndex(table, target, CCallHelpers::TimesFour), target); + Jump hasStart = jit.jump(); + isInNarrowGroup.link(&jit); + jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(image.functionSteps)), table); + jit.load16(BaseIndex(table, operand, CCallHelpers::TimesTwo), step); + jit.add32(step, target); + hasStart.link(&jit); jit.add64(Address(instanceGPR, AOT_INSTANCE_OFFSET(image.code)), target); jit.move(TrustedImm64(JSValue::ValueUndefined), directArgumentGPR(0)); @@ -606,6 +591,14 @@ static void emitFindConstructWord(StubAssembler& jit, GPRReg word, Jump isNeeded JumpList isNotListed; isNeeded.link(&jit); + Address rememberedEntry = emitLocateRememberedConstructWord(jit, word, half); + jit.move(bitMask(ClosureWord::isNotCallableBit), candidate); + jit.or64(word, candidate); + Jump isNotRemembered = jit.branch64(CCallHelpers::NotEqual, rememberedEntry.withOffset(OBJECT_OFFSETOF(ConstructWordCache::Entry, wordOfCall)), candidate); + jit.load64(rememberedEntry.withOffset(OBJECT_OFFSETOF(ConstructWordCache::Entry, wordOfConstruct)), word); + jit.jump().linkTo(continuation, &jit); + + isNotRemembered.link(&jit); extractBits(jit, word, ClosureWord::functionShift, ClosureWord::numberOfFunctionBits, function); jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(image.tables)), tables); jit.load32(programTableCount(tables, ProgramTableIndex::constructEntrypoints), count); @@ -629,9 +622,14 @@ static void emitFindConstructWord(StubAssembler& jit, GPRReg word, Jump isNeeded jit.load32(Address(first, OBJECT_OFFSETOF(ConstructEntrypointData, constructEntrypointAndFlag)), function); jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(functionTables.closureWords)), first); Jump allocatesThis = jit.branchTest32(CCallHelpers::NonZero, function, TrustedImm32(static_cast(ConstructEntrypointData::allocatesThis))); - jit.load64(BaseIndex(first, function, CCallHelpers::TimesEight), word); - jit.branchTest64(CCallHelpers::NonZero, word).linkTo(continuation, &jit); - isNotListed.append(jit.jump()); + jit.load64(BaseIndex(first, function, CCallHelpers::TimesEight), candidate); + isNotListed.append(jit.branchTest64(CCallHelpers::Zero, candidate)); + Address entryTakingThis = emitLocateRememberedConstructWord(jit, word, half); + jit.or64(bitMask(ClosureWord::isNotCallableBit), word); + jit.store64(word, entryTakingThis.withOffset(OBJECT_OFFSETOF(ConstructWordCache::Entry, wordOfCall))); + jit.store64(candidate, entryTakingThis.withOffset(OBJECT_OFFSETOF(ConstructWordCache::Entry, wordOfConstruct))); + jit.move(candidate, word); + jit.jump().linkTo(continuation, &jit); allocatesThis.link(&jit); Jump hasOtherNewTarget = jit.branch64(CCallHelpers::NotEqual, newTarget, calleeGPR); @@ -645,6 +643,14 @@ static void emitFindConstructWord(StubAssembler& jit, GPRReg word, Jump isNeeded jit.jump().linkTo(continuation, &jit); hasOtherNewTarget.link(&jit); + const GPRReg wordOfConstruct = candidate; + jit.and32(TrustedImm32(static_cast(~ConstructEntrypointData::allocatesThis)), function); + Address place = emitLocateRememberedConstructWord(jit, word, half); + jit.load64(BaseIndex(first, function, CCallHelpers::TimesEight), wordOfConstruct); + Jump isNotMadeYet = jit.branchTest64(CCallHelpers::Zero, wordOfConstruct); + jit.store64(word, place.withOffset(OBJECT_OFFSETOF(ConstructWordCache::Entry, wordOfCall))); + jit.store64(wordOfConstruct, place.withOffset(OBJECT_OFFSETOF(ConstructWordCache::Entry, wordOfConstruct))); + isNotMadeYet.link(&jit); jit.and64(TrustedImm64(static_cast(~(1ULL << ClosureWord::takesNewTargetBit))), word); jit.jump().linkTo(continuation, &jit); @@ -683,7 +689,7 @@ void generateEnterClosureForConstruct(StubAssembler& jit) { const GPRReg word = jit.reuseScratch(StubRegister::T11); const GPRReg base = jit.reuseScratch(constructArgumentBaseRegister); - emitEnterAdapter(jit, AdapterCaller::NativeCodeOrInterpreter, constructSlotsSize); + emitEnterAdapter(jit, constructSlotsSize); jit.load64(CCallHelpers::addressFor(CallFrameSlot::thisArgument), base); jit.store64(base, Address(fp, AdapterFrame::offsetOfNewTarget)); diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsCallAdapters.h b/Source/JavaScriptCore/aot/stubs/AOTStubsCallAdapters.h index 893264cad72c8..f16854fc2b9ee 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsCallAdapters.h +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsCallAdapters.h @@ -22,8 +22,7 @@ constexpr StubKey enterClosureForConstruct() { return StubKey(StubFamily::EnterC constexpr StubKey startModule() { return StubKey(StubFamily::StartModule); } constexpr StubKey startModuleThatRunTimeNames() { return StubKey(StubFamily::StartModuleThatRunTimeNames); } -enum class AdapterCaller : uint8_t { NativeCodeOrInterpreter, NativeCodeForJobOfCoroutine }; -void emitEnterAdapter(StubAssembler&, AdapterCaller, unsigned extraBytes = 0); +void emitEnterAdapter(StubAssembler&, unsigned extraBytes = 0); static constexpr unsigned adapterArgumentCountWithOutOfLineArguments = numberOfArgumentGPRs + 2; static constexpr unsigned outOfLineArgumentCountTagShift = 48; static constexpr uint16_t outOfLineArgumentCountTag = 1; diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsCallFamilies.def b/Source/JavaScriptCore/aot/stubs/AOTStubsCallFamilies.def index 2b5dc15bdcd69..660c386fabfd7 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsCallFamilies.def +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsCallFamilies.def @@ -36,6 +36,7 @@ v(CallHostFunction, CalledByCode, (), CountAxis, 0, N, AllISAs, NoISA, NoChoice, v(CallHostFunctionWithoutBoxedCall, CalledByCode, (), CountAxis, 0, 4, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, CallHostFunction) v(CallHostFunctionThatMayBeUnfetched, InternalEntry, (), CountAxis, 0, 4, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, CallHostFunction) v(CallHostFunctionWithCount, InternalEntry, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, CallHostFunctionWithCount) +v(CallHostGetter, CalledByCode, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, CallHostGetter) v(FindBoxedCall, InternalEntry, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, FindBoxedCall) v(CallInternalFunction, CalledByCode, (Call, Construct), FormAxis | CountAxis, 0, N, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, CallInternalFunction) v(CallDirectFunction, CalledByCode, (WithFrame, Leaf, WithHostFrame, LeafReturningNumber), FormAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersLikeCall, AsOperation, CallDirectFunction) diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsCallHost.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsCallHost.cpp index 8bdb6425122db..4547e0954c9ef 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsCallHost.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsCallHost.cpp @@ -18,6 +18,7 @@ #include "AOTStubsPropertyKeys.h" #include "FunctionRareData.h" #include "InternalFunction.h" +#include "JSCustomGetterFunction.h" #include "JSFunction.h" #include "NativeExecutable.h" @@ -28,6 +29,7 @@ StubSignature signatureOfCallHostFunctionWithoutBoxedCall(StubKey) { return call StubSignature signatureOfCallHostFunctionThatMayBeUnfetched(StubKey) { return callConventionSignature({ }); } StubSignature signatureOfCallInternalFunction(StubKey) { return callConventionSignature({ }); } StubSignature signatureOfCallHostFunctionWithCount(StubKey) { return callConventionSignatureWithCount(); } +StubSignature signatureOfCallHostGetter(StubKey) { return callConventionSignature({ }); } static void emitLoadExecutable(StubAssembler& jit, GPRReg executable) { @@ -148,6 +150,38 @@ void generateCallHostFunction(StubAssembler& jit) callNativeFunctionAndReturn(jit, CodeSpecializationKind::CodeForCall, registers.word, Address(calleeGPR, JSCallee::offsetOfScopeChain()), nativeFunction); } +void generateCallHostGetter(StubAssembler& jit) +{ + const CallRegisters registers = CallRegisters::allocate(jit); + loadStructure(jit, calleeGPR, registers.word); + jit.loadPtr(Address(registers.word, Structure::classInfoOffset()), registers.word); + jit.loadRuntimePointer(Entry::CustomGetterClassInfo, registers.scratch); + jit.linkToStub(jit.branchPtr(CCallHelpers::NotEqual, registers.word, registers.scratch), callHostFunctionWithoutBoxedCall(0)); + + jit.emitFunctionPrologue(); + const GPRReg function = registers.word; + const GPRReg realmArgument = jit.reuseScratch(StubRegister::A0); + const GPRReg receiverArgument = jit.reuseScratch(StubRegister::A1); + const GPRReg nameArgument = jit.reuseScratch(StubRegister::A2); + RELEASE_ASSERT(noOverlap(calleeGPR, realmArgument, receiverArgument, nameArgument, function, registers.scratch) && noOverlap(thisGPR, realmArgument, nameArgument, function, registers.scratch)); + jit.loadPtr(Address(calleeGPR, JSCustomGetterFunction::offsetOfGetter()), function); + jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(vm)), registers.scratch); + jit.storePtr(GPRInfo::callFrameRegister, Address(registers.scratch, VM::offsetOfTopCallFrame())); + jit.move(thisGPR, receiverArgument); + jit.loadPtr(Address(calleeGPR, JSCustomGetterFunction::offsetOfPropertyName()), nameArgument); + jit.loadPtr(Address(calleeGPR, JSCallee::offsetOfScopeChain()), realmArgument); + jit.call(function, CustomAccessorPtrTag); + jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(vm)), registers.scratch); + Jump exception = jit.branchTestPtr(CCallHelpers::NonZero, Address(registers.scratch, VM::exceptionOffset())); + jit.emitFunctionEpilogue(); + jit.pollBeforeReturningFromNativeCode(); + jit.returnToCaller(); + + exception.link(&jit); + jit.emitFunctionEpilogue(); + jit.jumpToExceptionHandler(); +} + StubSignature signatureOfCallFunctionOfBuiltinAtSite(StubKey key) { return signatureOf(callMethodAtSite(key.count())); } void generateCallFunctionOfBuiltinAtSite(StubAssembler& jit) diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsCallHost.h b/Source/JavaScriptCore/aot/stubs/AOTStubsCallHost.h index 3fcc2ca73dcf7..6435ffb6f8a2c 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsCallHost.h +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsCallHost.h @@ -19,6 +19,7 @@ constexpr StubKey callHostFunction(unsigned count) { return StubKey(StubFamily:: constexpr StubKey callHostFunctionWithoutBoxedCall(unsigned count) { return StubKey(StubFamily::CallHostFunctionWithoutBoxedCall).withCount(count); } constexpr StubKey callHostFunctionThatMayBeUnfetched(unsigned count) { return StubKey(StubFamily::CallHostFunctionThatMayBeUnfetched).withCount(count); } constexpr StubKey callHostFunctionWithCount() { return StubKey(StubFamily::CallHostFunctionWithCount); } +constexpr StubKey callHostGetter() { return StubKey(StubFamily::CallHostGetter); } enum class DirectFunctionCall : uint8_t { WithFrame, Leaf, WithHostFrame, LeafReturningNumber }; diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsCalls.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsCalls.cpp index ca003a706e8c2..fd806147613b4 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsCalls.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsCalls.cpp @@ -880,13 +880,15 @@ void generateConstructValue(StubAssembler& jit) isCallable.link(&jit); JumpList isNotRemembered; isNotRemembered.append(jit.branchTest64(CCallHelpers::Zero, registers.word, TrustedImm64(static_cast(1ULL << ClosureWord::takesNewTargetBit)))); - isNotRemembered.append(jit.branch64(CCallHelpers::NotEqual, argumentGPR(0), calleeGPR)); Address remembered = emitLocateRememberedConstructWord(jit, registers.word, registers.scratch); isNotRemembered.append(jit.branch64(CCallHelpers::NotEqual, remembered.withOffset(OBJECT_OFFSETOF(ConstructWordCache::Entry, wordOfCall)), registers.word)); + Jump hasOtherNewTarget = jit.branch64(CCallHelpers::NotEqual, argumentGPR(0), calleeGPR); jit.load64(remembered.withOffset(OBJECT_OFFSETOF(ConstructWordCache::Entry, wordOfConstruct)), registers.word); jit.move(TrustedImm64(JSValue::ValueUndefined), thisGPR); emitEnterClosure(jit, registers, std::nullopt, never); + hasOtherNewTarget.link(&jit); + jit.and64(TrustedImm64(static_cast(~(1ULL << ClosureWord::takesNewTargetBit))), registers.word); isNotRemembered.link(&jit); JumpList isNotPlainFunction; emitSpillArgumentsAndPushFrame(jit); @@ -898,6 +900,23 @@ void generateConstructValue(StubAssembler& jit) jit.jump().linkTo(hasCell, &jit); isNotRememberedForwardingFunction.link(&jit); + if (static_cast(jit.key().form()) == ConstructSyntax::NewExpression) { + JumpList isNotNativeClass; + isNotNativeClass.append(jit.branchIfNotType(calleeGPR, InternalFunctionType)); + isNotNativeClass.append(jit.branch64(CCallHelpers::NotEqual, argumentGPR(0), calleeGPR)); + jit.loadPtr(Address(calleeGPR, InternalFunction::offsetOfNativeFunctionFor(CodeSpecializationKind::CodeForConstruct)), registers.word); + jit.loadRuntimePointer(Entry::NonConstructorHostFunction, registers.scratch); + isNotNativeClass.append(jit.branchPtr(CCallHelpers::Equal, registers.word, registers.scratch)); + for (unsigned count = 0; count < numberOfArgumentGPRs; ++count) { + Jump hasOtherCount = jit.branch32(CCallHelpers::NotEqual, countGPR, TrustedImm32(count + 1)); + jit.move(argumentGPR(0), thisGPR); + for (unsigned index = 0; index < count; ++index) + jit.move(argumentGPR(index + 1), argumentGPR(index)); + jit.jumpToStub(callInternalFunction(CodeSpecializationKind::CodeForConstruct, count)); + hasOtherCount.link(&jit); + } + isNotNativeClass.link(&jit); + } isNotCell.link(&jit); emitSpillArgumentsAndPushFrame(jit); isNotPlainFunction.link(&jit); diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsClosures.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsClosures.cpp index 122f4b176722f..941556c769a6e 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsClosures.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsClosures.cpp @@ -282,18 +282,41 @@ StubSignature signatureOfClosureOfTable(StubKey) static void loadNumberOfClosureCell(StubAssembler& jit, GPRReg function, GPRReg number, GPRReg scratch) { - static_assert(sizeof(Number24) == 3 && std::endian::native == std::endian::little); + static_assert(NumberGroup::size == 32 && NumberGroup::numberOfWords * sizeof(uint32_t) == sizeof(uint64_t) && NumberGroup::isListedFlag == 1u << 31 && std::endian::native == std::endian::little); ASSERT(noOverlap(function, number, scratch)); - const GPRReg tables = scratch; - jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(image.tables)), tables); - jit.getEffectiveAddress(BaseIndex(function, function, CCallHelpers::TimesTwo), number); - jit.add32(Address(tables, offsetOfProgramTableRange(ProgramTableIndex::closureCellNumbers) + OBJECT_OFFSETOF(TableRange, offset)), number); - jit.addPtr(tables, number); - const GPRReg lowBytes = scratch; - jit.load16(Address(number), lowBytes); - jit.load8(Address(number, 2), number); - jit.lshift32(TrustedImm32(16), number); - jit.or32(lowBytes, number); + auto loadTable = [&](GPRReg table, GPRReg offset) { + jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(image.tables)), table); + jit.load32(Address(table, offsetOfProgramTableRange(ProgramTableIndex::closureCells) + OBJECT_OFFSETOF(TableRange, offset)), offset); + jit.addPtr(offset, table); + }; + const GPRReg group = number; + loadTable(scratch, group); + jit.urshift32(function, TrustedImm32(std::countr_zero(NumberGroup::size)), group); + jit.load64(BaseIndex(scratch, group, CCallHelpers::TimesEight), group); + const GPRReg bit = scratch; + jit.move(TrustedImm32(1), bit); + jit.lshift32(function, bit); + Jump hasCell = jit.branchTest32(CCallHelpers::NonZero, group, bit); + jit.move(TrustedImm32(0), number); + Jump hasNoCell = jit.jump(); + + hasCell.link(&jit); + const GPRReg rank = scratch; + jit.sub32(TrustedImm32(1), bit); + jit.and32(group, rank); + jit.countPopulation64(rank, rank, CCallHelpers::fpTempRegister); + jit.urshift64(TrustedImm32(32), group); + jit.add32(rank, number); + Jump follows = jit.branchTest32(CCallHelpers::PositiveOrZero, number, number); + + jit.and32(TrustedImm32(~NumberGroup::isListedFlag), number); + jit.lshift32(TrustedImm32(std::countr_zero(sizeof(uint32_t))), number); + jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(image.tables)), scratch); + jit.add32(Address(scratch, offsetOfProgramTableRange(ProgramTableIndex::closureCells) + OBJECT_OFFSETOF(TableRange, offset)), number); + jit.load32(BaseIndex(scratch, number, CCallHelpers::TimesOne), number); + + follows.link(&jit); + hasNoCell.link(&jit); } static void emitCreateSlimClosureOfTable(StubAssembler& jit, const ClosureRegisters& registers, GPRReg number, JumpList& slow, JumpList& isCreated) diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsCollections.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsCollections.cpp index 16f650284d31c..04a6569ec1399 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsCollections.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsCollections.cpp @@ -469,20 +469,7 @@ static void emitDelete(StubAssembler& jit, const HashTableRegisters& taken, Comp }); } -void emitMethodOfTable(StubAssembler& jit, TableMethod method, const HashTableRegisters& registers) -{ - switch (method) { - case TableMethod::MapDelete: return emitDelete(jit, registers, CompactTableOwner::Map); - case TableMethod::MapGet: return emitGet(jit, registers); - case TableMethod::MapHas: - case TableMethod::SetHas: return emitHas(jit, registers, Entry::operationAOTMapHas); - case TableMethod::MapSet: return emitInsert(jit, registers, CompactTableOwner::Map); - case TableMethod::SetAdd: return emitInsert(jit, registers, CompactTableOwner::Set); - case TableMethod::SetDelete: return emitDelete(jit, registers, CompactTableOwner::Set); - } -} - -void generateMapGet(StubAssembler& jit) { emitMethodOfTable(jit, TableMethod::MapGet, HashTableRegisters::take(jit)); } +void generateMapGet(StubAssembler& jit) { emitGet(jit, HashTableRegisters::take(jit)); } static void emitReturnPlaceOfEntry(StubAssembler& jit, const HashTableRegisters& registers, JumpList& hasNoValues) { const GPRReg values = registers.keys; @@ -549,11 +536,11 @@ void generateFindCharactersInMap(StubAssembler& jit) jit.returnToCaller(); } -void generateMapHas(StubAssembler& jit) { emitMethodOfTable(jit, TableMethod::MapHas, HashTableRegisters::take(jit)); } -void generateMapSet(StubAssembler& jit) { emitMethodOfTable(jit, TableMethod::MapSet, HashTableRegisters::take(jit)); } -void generateSetAdd(StubAssembler& jit) { emitMethodOfTable(jit, TableMethod::SetAdd, HashTableRegisters::take(jit)); } -void generateMapDelete(StubAssembler& jit) { emitMethodOfTable(jit, TableMethod::MapDelete, HashTableRegisters::take(jit)); } -void generateSetDelete(StubAssembler& jit) { emitMethodOfTable(jit, TableMethod::SetDelete, HashTableRegisters::take(jit)); } +void generateMapHas(StubAssembler& jit) { emitHas(jit, HashTableRegisters::take(jit), Entry::operationAOTMapHas); } +void generateMapSet(StubAssembler& jit) { emitInsert(jit, HashTableRegisters::take(jit), CompactTableOwner::Map); } +void generateSetAdd(StubAssembler& jit) { emitInsert(jit, HashTableRegisters::take(jit), CompactTableOwner::Set); } +void generateMapDelete(StubAssembler& jit) { emitDelete(jit, HashTableRegisters::take(jit), CompactTableOwner::Map); } +void generateSetDelete(StubAssembler& jit) { emitDelete(jit, HashTableRegisters::take(jit), CompactTableOwner::Set); } void generateWeakMapGet(StubAssembler& jit) { emitLookupInWeakTable(jit, WeakTableLookup::MapGet); } void generateWeakMapHas(StubAssembler& jit) { emitLookupInWeakTable(jit, WeakTableLookup::MapHas); } diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsCollections.h b/Source/JavaScriptCore/aot/stubs/AOTStubsCollections.h index df9305d0bbffd..e690196d32c42 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsCollections.h +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsCollections.h @@ -43,12 +43,7 @@ constexpr StubKey stubOfTableMethod(TableMethod method, KeyComparison comparison } RELEASE_ASSERT_NOT_REACHED_UNDER_CONSTEXPR_CONTEXT(); } -constexpr StubKey formForAnyValueOf(StubKey key) -{ - if (isCallOfMethodOfTable(key)) - return stubOfTableMethod(tableMethodOf(key), KeyComparison::AnyValue); - return key.withForm(static_cast(KeyComparison::AnyValue)); -} +constexpr StubKey formForAnyValueOf(StubKey key) { return key.withForm(static_cast(KeyComparison::AnyValue)); } constexpr StubKey weakMapGet() { return StubKey(StubFamily::WeakMapGet); } constexpr StubKey weakMapHas() { return StubKey(StubFamily::WeakMapHas); } constexpr StubKey weakSetHas() { return StubKey(StubFamily::WeakSetHas); } diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsConstants.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsConstants.cpp index 12d3242273326..81ff03e202b72 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsConstants.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsConstants.cpp @@ -431,6 +431,44 @@ static void returnConstantInRegisterOfKey(StubAssembler& jit, CreatedConstant co jit.returnToCaller(); } +void loadStringConstantsAndJump(StubAssembler& jit, std::span numbers, StubKey target) +{ + struct LateConstant { + Jump isNotCreated; + GPRReg number; + Label isLoaded; + }; + + const OwnedScratchRegister ownedScratch(jit); + const GPRReg word = ownedScratch; + const bool isNarrow = !jit.options().useWideReferences; + Vector lateConstants; + for (GPRReg number : numbers) { + RELEASE_ASSERT(noOverlap(number, word, instanceGPR, cellCageBaseGPR)); + Jump isNotCreated; + if (isNarrow) { + jit.load32(BaseIndex(instanceGPR, number, CCallHelpers::TimesFour, InstanceLayout::offsetOfStringCells), word); + isNotCreated = jit.branchTest32(CCallHelpers::Zero, word); + decodeCell32(jit, word, number); + } else { + jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(runtime.constants.wideStringCells)), word); + jit.load64(BaseIndex(word, number, CCallHelpers::TimesEight), word); + isNotCreated = jit.branchTest64(CCallHelpers::Zero, word); + jit.move(word, number); + } + lateConstants.append({ isNotCreated, number, jit.label() }); + } + jit.jumpToStub(target); + + for (const LateConstant& late : lateConstants) { + late.isNotCreated.link(&jit); + callMakerOfConstantAndReturn(jit, createConstant(CreatedConstant::String), late.number, [&](GPRReg number) { + jit.move(late.number, number); + }); + jit.jump().linkTo(late.isLoaded, &jit); + } +} + void generateCreateConstant(StubAssembler& jit) { if (jit.key().operandRegister()) diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsConstants.h b/Source/JavaScriptCore/aot/stubs/AOTStubsConstants.h index f11c5d57bba87..218aea7d86d54 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsConstants.h +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsConstants.h @@ -10,6 +10,7 @@ #include "AOTOperationData.h" #include "AOTStubDescriptor.h" #include "AOTStubHelpers.h" +#include namespace JSC { namespace AOT { @@ -30,6 +31,8 @@ struct LazyHeaderRegisters { GPRReg word; }; +void loadStringConstantsAndJump(StubAssembler&, std::span numbers, StubKey target); + void createLazyHeader(StubAssembler&, const LazyHeaderRegisters&, unsigned longestTextWithoutHeader, JumpList& cannot); static_assert(exists(createConstant(CreatedConstant::Key)) && !exists(createConstant(CreatedConstant::Key).withForm(3))); diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsCoroutineResumption.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsCoroutineResumption.cpp index 32a7873146bea..3b393ccd11f39 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsCoroutineResumption.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsCoroutineResumption.cpp @@ -129,7 +129,7 @@ StubSignature signatureOfEnterHandlerOfJob(StubKey) { return { }; } void generateEnterSavedFrame(StubAssembler& jit) { const GPRReg scratch = jit.reuseScratch(StubRegister::T11); - emitEnterAdapter(jit, AdapterCaller::NativeCodeForJobOfCoroutine); + emitEnterAdapter(jit); const GPRReg saved = jit.reuseScratch(StubRegister::A0); const GPRReg mode = jit.reuseScratch(StubRegister::A1); const GPRReg value = jit.reuseScratch(StubRegister::A2); @@ -315,7 +315,7 @@ void generateEnterSavedFrame(StubAssembler& jit) jit.align(); jit.bindEntry(enterHandlerOfJob()); - emitEnterAdapter(jit, AdapterCaller::NativeCodeForJobOfCoroutine); + emitEnterAdapter(jit); jit.load8(coroutineWord(AOT_INSTANCE_OFFSET(coroutines.nextEntryEndsJob)), scratch); jit.store8(TrustedImm32(0), coroutineWord(AOT_INSTANCE_OFFSET(coroutines.nextEntryEndsJob))); jit.store64(scratch, endsJob); diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsCoroutineSteps.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsCoroutineSteps.cpp index d5c8ecb944e3f..f2393c37fb66d 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsCoroutineSteps.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsCoroutineSteps.cpp @@ -107,7 +107,11 @@ void generateStepAsyncGenerator(StubAssembler& jit) Jump hasDriver = jit.jump(); hasNoSavedFrame.link(&jit); - stepsInEngine.append(jit.branchTest32(CCallHelpers::NonZero, consumer, TrustedImm32(consumerMayContinue))); + Jump hasNoRightToContinue = jit.branchTest32(CCallHelpers::Zero, consumer, TrustedImm32(consumerMayContinue)); + jit.and32(TrustedImm32(~(bodyMayContinue | consumerMayContinue)), consumer); + jit.orPtr(TrustedImm32(mustSuspendAtNextAwaitBit), fp, word); + jit.storePtr(word, Address(instanceGPR, AOT_INSTANCE_OFFSET(coroutines.frameThatMayContinue))); + hasNoRightToContinue.link(&jit); refill = emitAllocateSavedFrame(jit, CoroutineKind::AsyncFunction, siteWord, { driver, saved, word, other }, isNotAllocatedHere); jit.or32(TrustedImm32(consumerOfStepHasNewSavedFrame), consumer); hasDriver.link(&jit); @@ -189,9 +193,9 @@ static void emitGiveBackRightToContinue(StubAssembler& jit, GPRReg consumer, GPR Jump mayNotContinue = jit.branchTest32(CCallHelpers::Zero, consumer, TrustedImm32(consumerMayContinue)); Jump mustSuspendAtAwaits = jit.branchTest8(CCallHelpers::NonZero, Address(instanceGPR, AOT_INSTANCE_OFFSET(awaitMustSuspend))); jit.loadPtr(Address(fp), stored); - mayNotContinue.link(&jit); mustSuspendAtAwaits.link(&jit); jit.storePtr(stored, Address(instanceGPR, AOT_INSTANCE_OFFSET(coroutines.frameThatMayContinue))); + mayNotContinue.link(&jit); } static void emitNoteAwaitPassedOnStack(StubAssembler& jit, GPRReg consumer, GPRReg scratch) diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsCoroutineSuspension.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsCoroutineSuspension.cpp index 193974e625257..65a844ce86a7e 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsCoroutineSuspension.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsCoroutineSuspension.cpp @@ -136,12 +136,77 @@ void emitLoadCurrentAsyncContext(StubAssembler& jit, GPRReg context) jit.load64(Address(context, InternalFieldTuple::offsetOfInternalField(0)), context); } +static void emitStoreCurrentAsyncContext(StubAssembler& jit, TrustedImm64 context, GPRReg scratch) +{ + jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(globalObject)), scratch); + jit.loadPtr(Address(scratch, JSGlobalObject::offsetOfAsyncContextData()), scratch); + jit.store64(context, Address(scratch, InternalFieldTuple::offsetOfInternalField(0))); +} + void emitBranchUnlessFrameMayContinue(StubAssembler& jit, const AwaitRegisters& registers, JumpList& mustSuspend) { for (const AwaitCondition& condition : awaitConditionsOnState) emitBranchUnlessHolds(jit, condition, registers, mustSuspend); } +static constexpr Address addressOfFrameThatMayContinue(instanceGPR, AOT_INSTANCE_OFFSET(coroutines.frameThatMayContinue)); +static constexpr int32_t maxBytesOfAwaitingFrames = 128 * KB; + +static void emitBranchUnlessCallIsAwaited(StubAssembler& jit, GPRReg returnAddress, GPRReg awaitStub, GPRReg scratch, JumpList& otherwise) +{ + RELEASE_ASSERT(noOverlap(returnAddress, awaitStub, scratch, instanceGPR)); +#if CPU(X86_64) + constexpr int32_t moveIntoSiteWord = 0xba41; + constexpr int32_t call = 0xe8; + constexpr int32_t offsetOfCall = 6; + constexpr int32_t offsetOfDisplacement = offsetOfCall + 1; + constexpr int32_t offsetAfterCall = offsetOfDisplacement + sizeof(int32_t); + static_assert(gprOf(resumeModeRegister) == X86Registers::r10); + otherwise.append(jit.branch16(CCallHelpers::NotEqual, Address(returnAddress), TrustedImm32(moveIntoSiteWord))); + otherwise.append(jit.branch8(CCallHelpers::NotEqual, Address(returnAddress, offsetOfCall), TrustedImm32(call))); + jit.load32SignedExtendTo64(Address(returnAddress, offsetOfDisplacement), scratch); + jit.getEffectiveAddress(BaseIndex(returnAddress, scratch, CCallHelpers::TimesOne, offsetAfterCall), scratch); + otherwise.append(jit.branchPtr(CCallHelpers::NotEqual, scratch, awaitStub)); +#elif CPU(ARM64) + constexpr int32_t branchWithLink = static_cast(0x94000000u); + constexpr int32_t displacementMask = 0x3ffffff; + JumpList isAwaited; + jit.sub64(awaitStub, returnAddress, scratch); + jit.urshift64(TrustedImm32(2), scratch); + for (int32_t offset : { 4, 8 }) { + jit.sub32(TrustedImm32(1), scratch); + jit.and32(TrustedImm32(displacementMask), scratch); + jit.or32(TrustedImm32(branchWithLink), scratch); + if (offset == 4) + isAwaited.append(jit.branch32(CCallHelpers::Equal, scratch, Address(returnAddress, offset))); + else + otherwise.append(jit.branch32(CCallHelpers::NotEqual, scratch, Address(returnAddress, offset))); + } + isAwaited.link(&jit); +#endif +} + +struct AwaitingCallerRegisters { GPRReg frame, returnAddress, awaitStub, scratch; }; + +static void emitBranchUnlessCallersAwait(StubAssembler& jit, const AwaitingCallerRegisters& registers, JumpList& otherwise) +{ + const auto [frame, returnAddress, awaitStub, scratch] = registers; + RELEASE_ASSERT(noOverlap(frame, returnAddress, awaitStub, scratch, instanceGPR)); + const GPRReg vm = scratch; + jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(vm)), vm); + otherwise.append(jit.branchTest8(CCallHelpers::NonZero, Address(vm, static_cast(tracksNoAsyncContext.word.offset)))); + jit.loadPtr(Address(vm, VM::topEntryFrameOffset()), scratch); + jit.subPtr(fp, scratch); + otherwise.append(jit.branch64(CCallHelpers::Above, scratch, TrustedImm32(maxBytesOfAwaitingFrames))); + jit.loadStubAddress(awaitCallBySavingFrame(), awaitStub); + jit.move(fp, frame); + Label nextFrame = jit.label(); + jit.loadPtr(Address(frame, sizeof(void*)), returnAddress); + emitBranchUnlessCallIsAwaited(jit, returnAddress, awaitStub, scratch, otherwise); + jit.loadPtr(Address(frame), frame); + jit.branchPtr(CCallHelpers::NotEqual, addressOfFrameThatMayContinue, frame).linkTo(nextFrame, &jit); +} + static void emitLoadSettledValue(StubAssembler& jit, const AwaitRegisters& registers, GPRReg result, JumpList& isOtherObject, JumpList& mustSuspend) { const auto [savedFrame, awaited, word, other] = registers; @@ -575,11 +640,17 @@ void generateAwaitBySavingFrame(StubAssembler& jit) RELEASE_ASSERT(awaited == operationArgumentGPR(1) && savedFrame == operationArgumentGPR(2)); JumpList mustSuspend; + JumpList isOtherFrame; JumpList isOtherObject; + Label mayContinue; + const AwaitCondition& queueIsEmpty = awaitConditionsOnState[1]; if (awaitsCallWithoutPromise) jit.move(result, awaited); - if (form != AwaitedOperand::ValueInModuleBody) - emitBranchUnlessFrameMayContinue(jit, registers, mustSuspend); + if (form != AwaitedOperand::ValueInModuleBody) { + emitBranchUnlessHolds(jit, awaitConditionsOnState[0], registers, isOtherFrame); + mayContinue = jit.label(); + emitBranchUnlessHolds(jit, queueIsEmpty, registers, mustSuspend); + } switch (form) { case AwaitedOperand::Value: jit.move(awaited, result); @@ -607,6 +678,49 @@ void generateAwaitBySavingFrame(StubAssembler& jit) emitReturnAwaitedValue(jit, siteWord); } + if (form != AwaitedOperand::ValueInModuleBody) { + static_assert(mustSuspendAtNextAwaitBit == 1); + isOtherFrame.link(&jit); + jit.loadPtr(addressOfFrameThatMayContinue, word); + mustSuspend.append(jit.branchTestPtr(CCallHelpers::Zero, word)); + if (awaitsCallWithoutPromise) { + const GPRReg frameOfCallee = savedFrame; + jit.subPtr(sp, TrustedImm32(static_cast(prologueStackPointerDelta())), other); + jit.move(word, frameOfCallee); + jit.andPtr(TrustedImm32(~mustSuspendAtNextAwaitBit), frameOfCallee); + Jump calleeHasNotContinued = jit.branchPtr(CCallHelpers::NotEqual, frameOfCallee, other); + Jump tracksNone = emitBranchIfTracksNoAsyncContext(jit, other); + emitStoreCurrentAsyncContext(jit, TrustedImm64(JSValue::ValueUndefined), other); + tracksNone.link(&jit); + jit.andPtr(TrustedImm32(mustSuspendAtNextAwaitBit), word); + jit.orPtr(fp, word); + jit.storePtr(word, addressOfFrameThatMayContinue); + jit.branchPtr(CCallHelpers::Equal, word, fp).linkTo(mayContinue, &jit); + mustSuspend.append(jit.jump()); + calleeHasNotContinued.link(&jit); + } + + JumpList isOtherObjectInOtherFrame; + emitBranchUnlessHolds(jit, queueIsEmpty, registers, mustSuspend); + if (!awaitsCallWithoutPromise) + jit.move(awaited, result); + emitLoadSettledValue(jit, registers, result, isOtherObjectInOtherFrame, mustSuspend); + Label isSettled = jit.label(); + emitBranchUnlessCallersAwait(jit, { savedFrame, count, other, word }, mustSuspend); + jit.storePtr(fp, addressOfFrameThatMayContinue); + if (awaitsCallWithoutPromise) + jit.returnToCaller(); + else + emitReturnAwaitedValue(jit, siteWord); + + isOtherObjectInOtherFrame.link(&jit); + if (awaitsCallWithoutPromise) + mustSuspend.append(jit.branch32(CCallHelpers::Equal, word, Address(instanceGPR, AOT_INSTANCE_OFFSET(coroutines.savedFrameHeader)))); + else + emitBranchIfMayBeThenable(jit, awaited, word, other, mustSuspend); + jit.jump().linkTo(isSettled, &jit); + } + mustSuspend.link(&jit); jit.loadPtr(Address(fp, sizeof(void*)), word); Jump isFirstSave = jit.branchPtr(CCallHelpers::NotEqual, word, Address(instanceGPR, AOT_INSTANCE_OFFSET(stubs) + StubWords::offsetOfReturnAddress(StubFrameKind::SavedFrameAdapter))); diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsElements.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsElements.cpp index 44f53a6bc9c7a..c668e085762ab 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsElements.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsElements.cpp @@ -48,6 +48,24 @@ namespace JSC { namespace AOT { +StubSignature signatureOfMoveElementsDown(StubKey) +{ + return StubSignature::withoutResult({ gprOf(StubRegister::A0), gprOf(StubRegister::A2) }); +} + +void generateMoveElementsDown(StubAssembler& jit) +{ + const GPRReg butterfly = jit.operand(0); + const GPRReg count = jit.operand(1); + const GPRReg second = jit.reuseScratch(StubRegister::A1); + RELEASE_ASSERT(butterfly == leafArgumentGPR(0) && second == leafArgumentGPR(1) && count == leafArgumentGPR(2)); + jit.move(butterfly, second); + jit.addPtr(TrustedImm32(sizeof(EncodedJSValue)), second); + jit.lshift64(TrustedImm32(CCallHelpers::TimesEight), count); + jit.callLeaf(Entry::operationAOTMoveBytes); + jit.returnToCaller(); +} + StubSignature signatureOfMoveFewBytes(StubKey) { return StubSignature::withoutResult({ gprOf(StubRegister::A0), gprOf(StubRegister::A1), gprOf(StubRegister::A2) }); @@ -106,11 +124,13 @@ struct ElementRegisters { StubSignature signatureOfGetByVal(StubKey) { return StubSignature::withGPRResult({ gprOf(StubRegister::R0), gprOf(StubRegister::A1) }); } StubSignature signatureOfGetByValAtIndex(StubKey key) { return signatureOfGetByVal(key); } StubSignature signatureOfGetByValKeepingRegisters(StubKey key) { return signatureOfGetByVal(key); } +StubSignature signatureOfGetByValAtConstantIndex(StubKey key) { return signatureOfGetByVal(key); } StubSignature signatureOfGetByValExpecting(StubKey key) { return signatureOfGetByVal(key); } StubSignature signatureOfGetByValAtIndexExpecting(StubKey key) { return signatureOfGetByVal(key); } StubSignature signatureOfPutByVal(StubKey) { return StubSignature::withoutResult({ gprOf(StubRegister::R0), gprOf(StubRegister::A1), gprOf(StubRegister::A2) }); } StubSignature signatureOfPutByValAtIndex(StubKey key) { return signatureOfPutByVal(key); } +StubSignature signatureOfPutByValAtConstantIndex(StubKey key) { return signatureOfPutByVal(key); } StubSignature signatureOfPutByValExpecting(StubKey key) { return signatureOfPutByVal(key); } StubSignature signatureOfPutByValAtIndexExpecting(StubKey key) { return signatureOfPutByVal(key); } @@ -445,6 +465,30 @@ void loadKeptTextOfIndex(StubAssembler& jit, GPRReg key, GPRReg storage, GPRReg hasNoText.append(jit.branchTestPtr(CCallHelpers::Zero, text)); } +static void loadStorageOfPlainArray(StubAssembler& jit, const ElementRegisters& registers, JumpList& notHandled) +{ + const auto [base, index, storage, length, type, number] = registers; + + notHandled.append(jit.branchIfNotCell(base)); + notHandled.append(jit.branch8(CCallHelpers::NotEqual, Address(base, JSCell::indexingTypeAndMiscOffset()), TrustedImm32(ArrayWithContiguous))); + jit.loadPtr(Address(base, JSObject::butterflyOffset()), storage); + notHandled.append(jit.branch32(CCallHelpers::AboveOrEqual, index, Address(storage, Butterfly::offsetOfPublicLength()))); +} + +static Jump emitGetByValAtConstantIndexEntry(StubAssembler& jit, const ElementRegisters& registers) +{ + const auto [base, index, storage, length, type, number] = registers; + JumpList notHandled; + + jit.bindEntry(getByValAtConstantIndex()); + loadStorageOfPlainArray(jit, registers, notHandled); + returnValueUnlessHole(jit, registers, BaseIndex(storage, index, CCallHelpers::TimesEight), notHandled); + + notHandled.link(&jit); + boxInt32(jit, index, index); + return jit.jump(); +} + static void generateGetByVal(StubAssembler& jit, RegistersOfCaller registersOfCaller) { const ElementRegisters registers = allocateElementRegisters(jit); @@ -456,11 +500,16 @@ static void generateGetByVal(StubAssembler& jit, RegistersOfCaller registersOfCa JumpList absent; JumpList isNeitherStringNorCompiledObject; + Jump isConstantIndex; if (registersOfCaller == RegistersOfCaller::AreLost) emitGetByValAtIndexEntry(jit, registers, numberScratch, isNeitherStringNorCompiledObject); + else + isConstantIndex = emitGetByValAtConstantIndexEntry(jit, registers); jit.bindEntry(jit.key()); Jump notInt32 = jit.branch64(CCallHelpers::Below, key, numberTag); Label indexReady = jit.label(); + if (isConstantIndex.isSet()) + isConstantIndex.linkTo(indexReady, &jit); slow.append(jit.branchIfNotCell(base)); static_assert(JSObject::butterflyOffset() + sizeof(void*) <= MarkedBlock::atomSize); jit.loadPtr(Address(base, JSObject::butterflyOffset()), storage); @@ -568,6 +617,21 @@ static ECMAMode ecmaModeOf(ElementStoreMode mode) RELEASE_ASSERT_NOT_REACHED(); } +static Jump emitPutByValAtConstantIndexEntry(StubAssembler& jit, const ElementRegisters& registers, GPRReg value, ECMAMode mode) +{ + const auto [base, index, storage, length, type, number] = registers; + JumpList notHandled; + + jit.bindEntry(putByValAtConstantIndex(mode)); + loadStorageOfPlainArray(jit, registers, notHandled); + jit.store64(value, BaseIndex(storage, index, CCallHelpers::TimesEight)); + returnAfterStoring(jit, base, value); + + notHandled.link(&jit); + boxInt32(jit, index, index); + return jit.jump(); +} + static void emitPutByValAtIndexEntry(StubAssembler& jit, const ElementRegisters& registers, GPRReg value, ECMAMode mode) { const auto [base, index, storage, length, type, number] = registers; @@ -912,9 +976,11 @@ void generatePutByVal(StubAssembler& jit) const GPRReg shape = registers.storage; JumpList slow; + Jump isConstantIndex = emitPutByValAtConstantIndexEntry(jit, registers, value, mode); emitPutByValAtIndexEntry(jit, registers, value, mode); jit.bindEntry(jit.key()); slow.append(jit.branch64(CCallHelpers::Below, key, numberTag)); + isConstantIndex.link(&jit); slow.append(jit.branchIfNotCell(base)); jit.load8(Address(base, JSCell::indexingTypeAndMiscOffset()), shape); jit.and32(TrustedImm32(IndexingShapeMask | CopyOnWrite), shape); diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsElements.h b/Source/JavaScriptCore/aot/stubs/AOTStubsElements.h index c1615ab143448..26d24120ec314 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsElements.h +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsElements.h @@ -16,7 +16,9 @@ enum class ElementStoreMode : uint8_t { Strict, Sloppy }; constexpr StubKey getByVal(bool indexIsInteger) { return StubKey(indexIsInteger ? StubFamily::GetByValAtIndex : StubFamily::GetByVal); } constexpr StubKey getByValKeepingRegisters() { return StubKey(StubFamily::GetByValKeepingRegisters); } +constexpr StubKey getByValAtConstantIndex() { return StubKey(StubFamily::GetByValAtConstantIndex); } constexpr StubKey putByVal(bool indexIsInteger, ECMAMode mode) { return StubKey(indexIsInteger ? StubFamily::PutByValAtIndex : StubFamily::PutByVal).withForm(static_cast(mode.isStrict() ? ElementStoreMode::Strict : ElementStoreMode::Sloppy)); } +constexpr StubKey putByValAtConstantIndex(ECMAMode mode) { return StubKey(StubFamily::PutByValAtConstantIndex).withForm(static_cast(mode.isStrict() ? ElementStoreMode::Strict : ElementStoreMode::Sloppy)); } enum class ExpectedElements : uint8_t { Int32Array, Uint8Array, Float64Array }; inline constexpr uint8_t numberOfExpectedElements = 3; @@ -29,8 +31,9 @@ constexpr StubKey putByValExpecting(bool indexIsInteger, ECMAMode mode, Expected inline constexpr unsigned maxBytesMovedByStub = 16; constexpr StubKey moveFewBytes() { return StubKey(StubFamily::MoveFewBytes); } +constexpr StubKey moveElementsDown() { return StubKey(StubFamily::MoveElementsDown); } -static_assert(exists(getByVal(false)) && exists(getByVal(true)) && exists(moveFewBytes())); +static_assert(exists(getByVal(false)) && exists(getByVal(true)) && exists(getByValAtConstantIndex()) && exists(moveFewBytes()) && exists(moveElementsDown())); static_assert(exists(getByValExpecting(true, ExpectedElements::Float64Array)) && !exists(StubKey(StubFamily::GetByValExpecting).withForm(numberOfExpectedElements))); static_assert(exists(StubKey(StubFamily::PutByValAtIndexExpecting).withForm(2 * numberOfExpectedElements - 1)) && !exists(StubKey(StubFamily::PutByValExpecting).withForm(2 * numberOfExpectedElements))); static_assert(exists(StubKey(StubFamily::PutByVal).withForm(static_cast(ElementStoreMode::Sloppy))) && !exists(StubKey(StubFamily::PutByVal).withForm(2))); diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsExceptions.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsExceptions.cpp index 1507d0d279316..cfe7ea9987546 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsExceptions.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsExceptions.cpp @@ -120,6 +120,7 @@ StubSignature signatureOfHandleExceptionAtPrologue(StubKey) { return { }; } StubSignature signatureOfThrowStackOverflow(StubKey) { return { }; } StubSignature signatureOfCatch(StubKey) { return { }; } StubSignature signatureOfRejectInFrame(StubKey) { return { }; } +StubSignature signatureOfRejectAwaitedCall(StubKey) { return { }; } void generateHandleException(StubAssembler& jit) { @@ -218,6 +219,29 @@ void generateRejectInFrame(StubAssembler& jit) jit.ret(); } +void generateRejectAwaitedCall(StubAssembler& jit) +{ + constexpr GPRReg sp = CCallHelpers::stackPointerRegister; + static_assert(stackAlignmentBytes() == 2 * sizeof(void*)); + jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(runtime.exceptions.stackPointerOfHandler)), T9); + jit.move(T9, sp); +#if CPU(X86_64) + jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(runtime.exceptions.returnAddressOfAwaitedCall)), T9); + jit.push(T9); +#else + jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(runtime.exceptions.returnAddressOfAwaitedCall)), CCallHelpers::linkRegister); +#endif + jit.emitFunctionPrologue(); + jit.subPtr(TrustedImm32(stackAlignmentBytes()), sp); + jit.storePtr(GPRInfo::callFrameRegister, Address(sp)); + jit.storePtr(GPRInfo::callFrameRegister, Address(sp, sizeof(void*))); + jit.move(instanceGPR, A0); + jit.move(sp, A1); + jit.callOperation(Entry::operationAOTRejectWithException, T9); + jit.emitFunctionEpilogue(); + jit.ret(); +} + } } // namespace JSC::AOT #endif // ENABLE(AOT) diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsExceptions.h b/Source/JavaScriptCore/aot/stubs/AOTStubsExceptions.h index eb7d62c9ed0ce..6ed4b0fe38eb5 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsExceptions.h +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsExceptions.h @@ -22,6 +22,8 @@ constexpr StubKey throwStackOverflow(bool isAtPrologue = false) { return StubKey constexpr StubKey catchException() { return StubKey(StubFamily::Catch); } constexpr StubKey rejectInFrame(bool liesBelowSpilledArguments) { return StubKey(StubFamily::RejectInFrame).withForm(static_cast(liesBelowSpilledArguments ? RejectedFrameForm::BelowSpilledArguments : RejectedFrameForm::Plain)); } +constexpr StubKey rejectAwaitedCall() { return StubKey(StubFamily::RejectAwaitedCall); } + void emitUnwind(StubAssembler&); void emitThrowStackOverflow(StubAssembler&); diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsHashTables.h b/Source/JavaScriptCore/aot/stubs/AOTStubsHashTables.h index 1b2abe9643a4c..688ce98124394 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsHashTables.h +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsHashTables.h @@ -14,7 +14,6 @@ namespace JSC { namespace AOT { enum class KeyComparison : uint8_t; -enum class TableMethod : uint8_t; struct HashTableRegisterPlan { StubRegister hash; @@ -75,7 +74,6 @@ struct HashTableExits { ptrdiff_t offsetOfHashTableStorage(); void emitFindInHashTable(StubAssembler&, const HashTableRegisters&, HashTableLookup, HashTableExits&); void emitFindLatin1CharactersInHashTable(StubAssembler&, const HashTableRegisters&, GPRReg length, HashTableExits&); -void emitMethodOfTable(StubAssembler&, TableMethod, const HashTableRegisters&); void emitLoadTagOfSlot(StubAssembler&, GPRReg hash); void emitMarkThatKeyLeftHomeSlot(StubAssembler&, BaseIndex tagAtHome); void emitHashOfBits(StubAssembler&, GPRReg key, GPRReg hash, GPRReg scratch, GPRReg scratch2); diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsKeyedAccess.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsKeyedAccess.cpp index 7aa4422c1a74f..64005b2260109 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsKeyedAccess.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsKeyedAccess.cpp @@ -794,11 +794,10 @@ static void loadNameOfKeyCell(StubAssembler& jit, const KeyedProbeRegisters& reg isRope.link(&jit); hasNoName.append(jit.branchTestPtr(CCallHelpers::Zero, uid, TrustedImm32(JSRopeString::isSubstringInPointer))); - constexpr int32_t oneVisitOfPiece = 1 << 3; - constexpr int32_t twoVisitsOfPiece = 2 * oneVisitOfPiece; - static_assert(!(~JSRopeString::stringMask & (oneVisitOfPiece | twoVisitsOfPiece)) && (~JSRopeString::stringMask | oneVisitOfPiece | twoVisitsOfPiece) <= UINT8_MAX); - hasNoName.append(jit.branchTestPtr(CCallHelpers::NonZero, uid, TrustedImm32(twoVisitsOfPiece))); - jit.add8(TrustedImm32(oneVisitOfPiece), Address(key, JSString::offsetOfValue())); + using Uses = UsesOfSubstringAsKey; + static_assert(!(~JSRopeString::stringMask & (Uses::one | Uses::two)) && (~JSRopeString::stringMask | Uses::one | Uses::two) <= UINT8_MAX); + hasNoName.append(jit.branchTestPtr(CCallHelpers::NonZero, uid, TrustedImm32(Uses::two))); + jit.add8(TrustedImm32(Uses::one), Address(key, JSString::offsetOfValue())); jit.load32(Address(key, JSRopeString::offsetOfLength()), length); loadSubstringBaseAndOffset(jit, key, uid, characters); jit.loadPtr(Address(uid, JSString::offsetOfValue()), uid); diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsPropertyInstanceOf.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsPropertyInstanceOf.cpp index 5e75ed555725e..d14feb0c951ed 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsPropertyInstanceOf.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsPropertyInstanceOf.cpp @@ -67,6 +67,7 @@ void generateInstanceOf(StubAssembler& jit) const bool readsPrototypeOutOfCell = operand == InstanceOfOperand::PrototypeThatIsBuiltin; const bool givesWayAtTraps = operand == InstanceOfOperand::PrototypeWithoutTraps; const bool isGivenPrototype = operand == InstanceOfOperand::Prototype || givesWayAtTraps; + const bool mayAcceptBinaryData = readsPrototypeOutOfCell || operand == InstanceOfOperand::Prototype; const GPRReg prototype = readsPrototypeOutOfCell ? jit.clobberRegister(jit.immediate()) : isGivenPrototype ? jit.operand(1) : jit.clobberOperand(1); const GPRReg constructor = prototype; const GPRReg current = jit.allocateScratch(StubRegister::T11); @@ -74,6 +75,7 @@ void generateInstanceOf(StubAssembler& jit) const GPRReg value = jit.operand(0); JumpList isInstance; JumpList notInstance; + JumpList reachesEndOfChain; JumpList slowCases; JumpList startsOverWithPrototype; JumpList cellIsNotMade; @@ -123,13 +125,16 @@ void generateInstanceOf(StubAssembler& jit) if (!readsPrototypeOutOfCell) first.goesOn.link(&jit); Label nextLink = jit.label(); - notInstance.append(jit.branchIfNotCell(next)); + reachesEndOfChain.append(jit.branchIfNotCell(next)); jit.move(next, current); Link further = followLink(current, false); further.goesOn.linkTo(nextLink, &jit); readPolyProto(first, value); readPolyProto(further, current); + reachesEndOfChain.link(&jit); + if (mayAcceptBinaryData) + slowCases.append(jit.branchIfType(value, JSTypeRange { ArrayBufferType, static_cast(LastTypedArrayTypeExcludingDataView) })); notInstance.link(&jit); returnBoolean(false); isInstance.link(&jit); diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsPropertyReadHandlers.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsPropertyReadHandlers.cpp index 3d1344cb3c6d3..3e095094124dd 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsPropertyReadHandlers.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsPropertyReadHandlers.cpp @@ -56,7 +56,6 @@ static bool hasReadArm(SlotHandlerTableTraits traits, Slot::Kind kind) case Slot::Kind::Setter: case Slot::Kind::CustomSetter: case Slot::Kind::FieldCreation: - case Slot::Kind::FieldOrInheritedSetter: case Slot::Kind::ExtraSetter: case Slot::Kind::ExtraPropertyAddition: case Slot::Kind::Transition: @@ -70,6 +69,7 @@ static bool hasReadArm(SlotHandlerTableTraits traits, Slot::Kind kind) case Slot::Kind::FarValue: case Slot::Kind::FieldOrInherited: case Slot::Kind::FieldOrInheritedField: + case Slot::Kind::FieldOrInheritedAccessor: case Slot::Kind::Constant: return !isPacked; case Slot::Kind::Value: @@ -342,6 +342,14 @@ void generateReadSlot(StubAssembler& jit) returnUndefined(); return; } + case Slot::Kind::FieldOrInheritedAccessor: + slot.loadFirstPayloadWord(jit, T10); + jit.load32(Address(R0, offsetOfOrderWord), T11); + jit.and32(TrustedImm32(static_cast(orderAndCountMask | orderWordHasFieldOutsideSlot)), T11); + uncached.append(jit.branch32(CCallHelpers::NotEqual, T11, T10)); + loadInheritedValueBySlot(jit, slot, T10, T11, T10); + callGetter(T10); + return; case Slot::Kind::InheritedField: { if (slot.form() == SlotForm::Slot) { const GPRReg word = jit.clobberOperand(headerOperand); @@ -481,7 +489,6 @@ void generateReadSlot(StubAssembler& jit) case Slot::Kind::Setter: case Slot::Kind::CustomSetter: case Slot::Kind::FieldCreation: - case Slot::Kind::FieldOrInheritedSetter: case Slot::Kind::ExtraSetter: case Slot::Kind::ExtraPropertyAddition: case Slot::Kind::Transition: diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsPropertyStoreHandlers.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsPropertyStoreHandlers.cpp index fbfb61a3a0cce..9f944a62c383a 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsPropertyStoreHandlers.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsPropertyStoreHandlers.cpp @@ -100,7 +100,7 @@ static bool hasStoreArm(SlotHandlerTableTraits traits, Slot::Kind kind) case Slot::Kind::FieldCreation: case Slot::Kind::ExtraPropertyAddition: return !isPacked; - case Slot::Kind::FieldOrInheritedSetter: + case Slot::Kind::FieldOrInheritedAccessor: return !isDefinition && !isPacked; } RELEASE_ASSERT_NOT_REACHED(); @@ -368,7 +368,7 @@ void generateStoreSlot(StubAssembler& jit) linkToUncachedExit(arm); break; } - case Slot::Kind::FieldOrInheritedSetter: { + case Slot::Kind::FieldOrInheritedAccessor: { const FieldAddressRegisters field { .object = receiver, .slotData = T10, .extensionWordOffset = T11, .address = T11, .representation = T14 }; const GPRReg getterSetter = T10; JumpList isAbsent; @@ -454,6 +454,7 @@ void generateStoreSlot(StubAssembler& jit) isFirstStorage.link(&jit); cannot.append(jit.branchTestPtr(CCallHelpers::NonZero, butterfly)); + CCallHelpers::Label hasNoStorage = jit.label(); branchUnlessOrdinaryFunction(newStructureID); jit.move(TrustedImm32(0), oldCapacity); @@ -477,6 +478,7 @@ void generateStoreSlot(StubAssembler& jit) slot.loadSecondPayloadWord(jit, newCapacity); cannot.append(jit.branch32(CCallHelpers::NotEqual, newCapacity, TrustedImm32(initialOutOfLineCapacity))); jit.loadPtr(butterfly, distanceToButterfly); + jit.branchTestPtr(CCallHelpers::Zero, distanceToButterfly).linkTo(hasNoStorage, &jit); jit.subPtr(receiver, distanceToButterfly); cannot.append(jit.branch64(CCallHelpers::NotEqual, distanceToButterfly, TrustedImm32(static_cast(sizeOfArrayWithStorage(0, propertyRoomOfArray))))); emitAdditionInStorage(); diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsRegExpLiterals.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsRegExpLiterals.cpp index b404cb2641d1f..babf729b1a42a 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsRegExpLiterals.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsRegExpLiterals.cpp @@ -277,10 +277,12 @@ void generateRegExpMatchLiteral(StubAssembler& jit) JumpList miss; JumpList outOfLine; + const bool matchesSubstringInPlace = form == RegExpMatchForm::Test || form == RegExpMatchForm::Search; jit.loadPtr(Address(string, JSString::offsetOfValue()), impl); - outOfLine.append(jit.branchIfRopeStringImpl(impl)); + Jump isRope = jit.branchIfRopeStringImpl(impl); jit.loadPtr(Address(impl, StringImpl::dataOffset()), characters); jit.load32(Address(impl, StringImpl::lengthMemoryOffset()), subjectLength); + Label hasCharacters = jit.label(); if (lastIndexUse.isReset || lastIndexUse.isStoredAndPutBack) outOfLine.append(branchIfReadOnlyLastIndex(jit, *regExpObject)); @@ -308,6 +310,13 @@ void generateRegExpMatchLiteral(StubAssembler& jit) resetLastIndex(jit, *regExpObject); returnResult(jit, TrustedImm64(JSValue::encode(traitsOfMatchForm(form).miss))); + if (matchesSubstringInPlace) { + isRope.link(&jit); + loadSubstringCharacters(jit, string, impl, characters, subjectLength, outOfLine); + jit.jump().linkTo(hasCharacters, &jit); + } else + outOfLine.append(isRope); + outOfLine.link(&jit); matchLiteralOutOfLineAndReturn(jit, form, regExpObject, string); } diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsStringConcatenation.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsStringConcatenation.cpp index 602c7ae831455..362ecb6ae5107 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsStringConcatenation.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsStringConcatenation.cpp @@ -13,6 +13,7 @@ #include "AOTRuntimeTable.h" #include "AOTStubHelpers.h" #include "AOTStubRegisters.h" +#include "AOTStubsConstants.h" #include "AOTVector.h" #include "JSString.h" #include @@ -36,47 +37,62 @@ struct RopeRegisters { GPRReg allocator; }; -static void loadFlagsAndLength(StubAssembler& jit, GPRReg string, GPRReg flags, GPRReg length, JumpList* mayGrow = nullptr) +struct LateLengthOfRope { + Jump isRope; + Label isLoaded; + GPRReg string; + GPRReg flags; + GPRReg length; +}; + +[[nodiscard]] static LateLengthOfRope loadFlagsAndLength(StubAssembler& jit, GPRReg string, GPRReg flags, GPRReg length, JumpList& isEmpty) { jit.loadPtr(Address(string, JSString::offsetOfValue()), flags); Jump isRope = jit.branchIfRopeStringImpl(flags); jit.load32(Address(flags, StringImpl::lengthMemoryOffset()), length); jit.load32(Address(flags, StringImpl::flagsOffset()), flags); - Jump isLoaded = jit.jump(); - isRope.link(&jit); - jit.load32(Address(string, JSRopeString::offsetOfLength()), length); + isEmpty.append(jit.branchTest32(CCallHelpers::Zero, length)); + return { isRope, jit.label(), string, flags, length }; +} + +static void emitLateLengthOfRope(StubAssembler& jit, const LateLengthOfRope& late, JumpList* mayGrow = nullptr) +{ + late.isRope.link(&jit); + jit.load32(Address(late.string, JSRopeString::offsetOfLength()), late.length); if (mayGrow) { - Jump isNoSubstring = jit.branchTestPtr(CCallHelpers::Zero, flags, TrustedImm32(JSRopeString::isSubstringInPointer)); - mayGrow->append(jit.branch32(CCallHelpers::AboveOrEqual, length, TrustedImm32(minLengthOfGrowingText))); - isNoSubstring.link(&jit); + jit.branchTestPtr(CCallHelpers::Zero, late.flags, TrustedImm32(JSRopeString::isSubstringInPointer)).linkTo(late.isLoaded, &jit); + mayGrow->append(jit.branch32(CCallHelpers::AboveOrEqual, late.length, TrustedImm32(minLengthOfGrowingText))); } - isLoaded.link(&jit); + jit.jump().linkTo(late.isLoaded, &jit); +} + +[[nodiscard]] static AssemblyHelpers::FreeListRefill allocateRope(StubAssembler& jit, GPRReg rope, GPRReg allocator, GPRReg scratch, JumpList& slow) +{ + jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_PREFIX_OFFSET(ropeStringAllocator)), allocator); + return allocateCellWithLateRefill(jit, rope, allocator, scratch, slow, sizeof(JSRopeString)); } [[nodiscard]] static AssemblyHelpers::FreeListRefill allocateAndFillRope(StubAssembler& jit, const RopeRegisters& registers, JumpList& slow) { - using CompactFibers = JSRopeString::CompactFibers; const auto [first, second, third, flags, length, scratch, rope, allocator] = registers; ASSERT(noOverlap(first, second, third.value_or(InvalidGPRReg), flags, length, scratch, rope, allocator)); - jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_PREFIX_OFFSET(ropeStringAllocator)), allocator); - slow.append(jit.branchTestPtr(CCallHelpers::Zero, allocator)); - AssemblyHelpers::FreeListRefill refill = allocateCellWithLateRefill(jit, rope, allocator, scratch, slow); + AssemblyHelpers::FreeListRefill refill = allocateRope(jit, rope, allocator, scratch, slow); jit.load64(Address(instanceGPR, AOT_INSTANCE_PREFIX_OFFSET(stringHeader)), scratch); jit.and32(TrustedImm32(JSRopeString::is8BitInPointer), flags); - jit.or64(first, flags); - jit.or64(TrustedImm32(JSString::isRopeInPointer), flags); + jit.getEffectiveAddress(BaseIndex(first, flags, CCallHelpers::TimesOne, JSString::isRopeInPointer), flags); jit.storePair64(scratch, flags, rope, TrustedImm32(0)); - jit.move(second, scratch); - jit.lshift64(TrustedImm32(CompactFibers::shiftOfFiber1LowerInLengthWord), scratch); - jit.or64(length, scratch); - jit.move(second, allocator); - jit.urshift64(TrustedImm32(CompactFibers::widthOfFiber1LowerInLengthWord), allocator); + + constexpr ptrdiff_t offsetOfUpperThird = JSRopeString::offsetOfFiber2Lower() + sizeof(uint16_t); + static_assert(offsetOfUpperThird + sizeof(uint32_t) == sizeof(JSRopeString)); + jit.store32(length, Address(rope, JSRopeString::offsetOfLength())); + jit.store64(second, Address(rope, JSRopeString::offsetOfFiber1Lower())); if (third) { - jit.move(*third, flags); - jit.lshift64(TrustedImm32(CompactFibers::shiftOfFiber2InUpperWord), flags); - jit.or64(flags, allocator); - } - jit.storePair64(scratch, allocator, rope, TrustedImm32(2 * sizeof(EncodedJSValue))); + jit.store16(*third, Address(rope, JSRopeString::offsetOfFiber2Lower())); + jit.move(*third, scratch); + jit.urshift64(TrustedImm32(16), scratch); + jit.store32(scratch, Address(rope, offsetOfUpperThird)); + } else + jit.store32(TrustedImm32(0), Address(rope, offsetOfUpperThird)); fenceAfterAllocation(jit); return refill; } @@ -120,9 +136,7 @@ static void loadFlagsAndLength(StubAssembler& jit, GPRReg string, GPRReg flags, jit.add32(lengthOfPiece, lengthOfPieces); } other.append(jit.branch32(CCallHelpers::Above, lengthOfPieces, room)); - jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_PREFIX_OFFSET(ropeStringAllocator)), allocator); - other.append(jit.branchTestPtr(CCallHelpers::Zero, allocator)); - AssemblyHelpers::FreeListRefill refill = allocateCellWithLateRefill(jit, rope, allocator, scratch, other); + AssemblyHelpers::FreeListRefill refill = allocateRope(jit, rope, allocator, scratch, other); jit.load32(Address(characters, offsetOfEndOfTextInCharactersOfBuffer), scratch); jit.add32(lengthOfPieces, scratch); @@ -163,6 +177,14 @@ static void loadFlagsAndLength(StubAssembler& jit, GPRReg string, GPRReg flags, return refill; } +static void jumpToRopeOfSubstring(StubAssembler& jit, const LateLengthOfRope& late) +{ + jit.load32(Address(late.string, JSRopeString::offsetOfLength()), late.length); + jit.loadPtr(Address(late.string, JSString::offsetOfValue()), late.flags); + jit.branchIfRopeStringImpl(late.flags).linkTo(late.isLoaded, &jit); + jit.breakpoint(); +} + void makeRopeOfTwoValuesAndReturn(StubAssembler& jit, GPRReg first, GPRReg second, JumpList& slow) { const GPRReg result = jit.result(); @@ -183,12 +205,10 @@ void makeRopeOfTwoValuesAndReturn(StubAssembler& jit, GPRReg first, GPRReg secon slow.append(jit.branchIfNotCell(second)); slow.append(jit.branchIfNotString(first)); slow.append(jit.branchIfNotString(second)); - JumpList mayGrow; - loadFlagsAndLength(jit, first, registers.flags, registers.length, &mayGrow); - Label makeRope = jit.label(); - Jump firstIsEmpty = jit.branchTest32(CCallHelpers::Zero, registers.length); - loadFlagsAndLength(jit, second, registers.scratch, lengthOfSecond); - Jump secondIsEmpty = jit.branchTest32(CCallHelpers::Zero, lengthOfSecond); + JumpList firstIsEmpty; + JumpList secondIsEmpty; + LateLengthOfRope lateLengthOfFirst = loadFlagsAndLength(jit, first, registers.flags, registers.length, firstIsEmpty); + LateLengthOfRope lateLengthOfSecond = loadFlagsAndLength(jit, second, registers.scratch, lengthOfSecond, secondIsEmpty); jit.and32(registers.scratch, registers.flags); slow.append(jit.branchAdd32(CCallHelpers::Overflow, lengthOfSecond, registers.length)); AssemblyHelpers::FreeListRefill refill = allocateAndFillRope(jit, registers, slow); @@ -199,12 +219,15 @@ void makeRopeOfTwoValuesAndReturn(StubAssembler& jit, GPRReg first, GPRReg secon secondIsEmpty.link(&jit); jit.returnToCaller(); + JumpList mayGrow; + emitLateLengthOfRope(jit, lateLengthOfFirst, &mayGrow); + emitLateLengthOfRope(jit, lateLengthOfSecond); + mayGrow.link(&jit); JumpList doesNotGrowHere; AssemblyHelpers::FreeListRefill refillForSubstring = appendInBufferAndReturn(jit, registers, doesNotGrowHere); doesNotGrowHere.link(&jit); - loadFlagsAndLength(jit, first, registers.flags, registers.length); - jit.jump().linkTo(makeRope, &jit); + jumpToRopeOfSubstring(jit, lateLengthOfFirst); emitLateRefill(jit, refill, slow); emitLateRefill(jit, refillForSubstring, slow); @@ -245,25 +268,26 @@ void generateConcatenate(StubAssembler& jit) static_assert(gprOf(StubRegister::A3) == operationArgumentGPR(3)); JumpList slow; Vector refills; + Vector lateLengths; + auto loadLengthOf = [&](GPRReg string) { + JumpList isEmpty; + lateLengths.append(loadFlagsAndLength(jit, string, flags, length, isEmpty)); + return isEmpty; + }; auto addLengthOf = [&](GPRReg string) { - loadFlagsAndLength(jit, string, scratch, lengthOfNext); - Jump isEmpty = jit.branchTest32(CCallHelpers::Zero, lengthOfNext); + JumpList isEmpty; + lateLengths.append(loadFlagsAndLength(jit, string, scratch, lengthOfNext, isEmpty)); jit.and32(scratch, flags); slow.append(jit.branchAdd32(CCallHelpers::Overflow, lengthOfNext, length)); return isEmpty; }; - JumpList mayGrow; - Label makeRope; - auto joinTwoAndReturn = [&](GPRReg former, GPRReg latter, bool formerMayGrow) { + auto joinTwoAndReturn = [&](GPRReg former, GPRReg latter) { RopeRegisters both = registers; both.first = former; both.second = latter; both.third = std::nullopt; - loadFlagsAndLength(jit, former, flags, length, formerMayGrow ? &mayGrow : nullptr); - if (formerMayGrow) - makeRope = jit.label(); - Jump formerIsEmpty = jit.branchTest32(CCallHelpers::Zero, length); - Jump latterIsEmpty = addLengthOf(latter); + JumpList formerIsEmpty = loadLengthOf(former); + JumpList latterIsEmpty = addLengthOf(latter); refills.append(allocateAndFillRope(jit, both, slow)); returnResult(jit, rope); formerIsEmpty.link(&jit); @@ -272,29 +296,32 @@ void generateConcatenate(StubAssembler& jit) returnResult(jit, former); }; if (third) { - loadFlagsAndLength(jit, first, flags, length, &mayGrow); - makeRope = jit.label(); - Jump firstIsEmpty = jit.branchTest32(CCallHelpers::Zero, length); - Jump secondIsEmpty = addLengthOf(second); - Jump thirdIsEmpty = addLengthOf(*third); + JumpList firstIsEmpty = loadLengthOf(first); + JumpList secondIsEmpty = addLengthOf(second); + JumpList thirdIsEmpty = addLengthOf(*third); refills.append(allocateAndFillRope(jit, registers, slow)); returnResult(jit, rope); firstIsEmpty.link(&jit); - joinTwoAndReturn(second, *third, false); + joinTwoAndReturn(second, *third); secondIsEmpty.link(&jit); - joinTwoAndReturn(first, *third, false); + joinTwoAndReturn(first, *third); thirdIsEmpty.link(&jit); - joinTwoAndReturn(first, second, false); + joinTwoAndReturn(first, second); } else - joinTwoAndReturn(first, second, true); + joinTwoAndReturn(first, second); + + JumpList mayGrow; + const LateLengthOfRope lateLengthOfFirst = lateLengths.first(); + emitLateLengthOfRope(jit, lateLengthOfFirst, &mayGrow); + for (const LateLengthOfRope& late : lateLengths.span().subspan(1)) + emitLateLengthOfRope(jit, late); mayGrow.link(&jit); JumpList doesNotGrowHere; refills.append(appendInBufferAndReturn(jit, registers, doesNotGrowHere)); doesNotGrowHere.link(&jit); - loadFlagsAndLength(jit, first, flags, length); - jit.jump().linkTo(makeRope, &jit); + jumpToRopeOfSubstring(jit, lateLengthOfFirst); for (AssemblyHelpers::FreeListRefill& refill : refills) emitLateRefill(jit, refill, slow); @@ -306,6 +333,22 @@ void generateConcatenate(StubAssembler& jit) }); } +StubSignature signatureOfConcatenateWithConstants(StubKey key) +{ + return signatureOfConcatenate(concatenate(numberOfOperands(static_cast(key.form())))); +} + +void generateConcatenateWithConstants(StubAssembler& jit) +{ + const auto constants = static_cast(jit.key().form()); + Vector numbers; + for (unsigned operand = 0; operand < numberOfOperands(constants); ++operand) { + if (isConstantOperand(constants, operand)) + numbers.append(jit.clobberOperand(operand)); + } + loadStringConstantsAndJump(jit, numbers.span(), concatenate(numberOfOperands(constants))); +} + } } // namespace JSC::AOT #endif // ENABLE(AOT) diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsStringConversions.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsStringConversions.cpp index 699b4c9a83f3e..77db817af4fc5 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsStringConversions.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsStringConversions.cpp @@ -35,7 +35,8 @@ StubSignature signatureOfConcatenateKeyOfPair(StubKey) void locateKeysOfPair(StubAssembler& jit, GPRReg first, GPRReg second, GPRReg set, GPRReg scratch) { RELEASE_ASSERT(noOverlap(first, second, set, scratch)); - jit.getEffectiveAddress(BaseIndex(first, second, CCallHelpers::TimesEight), set); + jit.mul32(TrustedImm32(static_cast(StringKeyCache::keyOfPairMultiplierOfSecond)), second, set); + jit.add32(first, set); jit.mul32(TrustedImm32(static_cast(StringKeyCache::keyOfPairMultiplier)), set, set); jit.urshift32(TrustedImm32(StringKeyCache::keyOfPairHashShift), set); #if CPU(X86_64) @@ -48,6 +49,35 @@ void locateKeysOfPair(StubAssembler& jit, GPRReg first, GPRReg second, GPRReg se jit.addPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(realm.keysOfPairs)), set); } +static void joinLongTextAsAnyOther(StubAssembler& jit, GPRReg first, GPRReg second, GPRReg scratch) +{ + JumpList isNoLongText; + const StubKey joinOfTwo = concatenate(2); +#if CPU(ARM64) + const GPRReg firstOfJoin = jit.allocateScratch(StubRegister::T11); + const GPRReg secondOfJoin = jit.reuseScratch(StubRegister::T12); +#else + const GPRReg firstOfJoin = scratch; + const GPRReg secondOfJoin = second; +#endif + RELEASE_ASSERT(signatureOf(joinOfTwo).operands[0] == firstOfJoin && signatureOf(joinOfTwo).operands[1] == secondOfJoin); + isNoLongText.append(jit.branchIfNotCell(first)); + isNoLongText.append(jit.branchIfNotType(first, StringType)); + isNoLongText.append(jit.branchIfNotCell(second)); + isNoLongText.append(jit.branchIfNotType(second, StringType)); + jit.loadPtr(Address(first, JSString::offsetOfValue()), scratch); + Jump isRope = jit.branchIfRopeStringImpl(scratch); + isNoLongText.append(jit.branch32(CCallHelpers::BelowOrEqual, Address(scratch, StringImpl::lengthMemoryOffset()), TrustedImm32(StringKeyCache::maxCachedKeyLength))); + Jump isLong = jit.jump(); + isRope.link(&jit); + isNoLongText.append(jit.branch32(CCallHelpers::BelowOrEqual, Address(first, JSRopeString::offsetOfLength()), TrustedImm32(StringKeyCache::maxCachedKeyLength))); + isLong.link(&jit); + jit.move(second, secondOfJoin); + jit.move(first, firstOfJoin); + jit.jumpToStub(joinOfTwo); + isNoLongText.link(&jit); +} + void generateConcatenateKeyOfPair(StubAssembler& jit) { const GPRReg first = jit.operand(0); @@ -108,6 +138,7 @@ void generateConcatenateKeyOfPair(StubAssembler& jit) returnResult(jit, second); isNoTextBehindNothing.link(&jit); } + joinLongTextAsAnyOther(jit, first, second, set); isTextOfInteger.link(&jit); auto use = static_cast(jit.key().form()) == ConcatenatedKeyForm::Name ? UseOfJoinedText::LikelyName : UseOfJoinedText::LikelyKey; jit.callOperationAndReturn(Entry::operationAOTConcatenatePair, { diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsStringFamilies.def b/Source/JavaScriptCore/aot/stubs/AOTStubsStringFamilies.def index 8fd65c6819f06..085ca4df912b0 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsStringFamilies.def +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsStringFamilies.def @@ -4,12 +4,15 @@ */ v(RopeCharacters, CalledByCode, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, reads(StringStorage), RopeCharacters) +v(SubstringCharacters, CalledByCode, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, Leaf, ClobbersOnlyScratch, reads(StringStorage), SubstringCharacters) v(FlattenString, CalledByCode, (), SlowPathRecurs, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, reads(StringStorage), FlattenString) +v(StringCodeUnitAt, CalledByCode, (), SlowPathRecurs, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, reads(StringStorage) | writes(StringStorage), StringCodeUnitAt) v(StringsAreEqual, CalledByCode, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, reads(StringStorage), StringsAreEqual) v(StringBuilderAppendString, CalledByCode, (), SlowPathRecurs, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, AsOperation, StringBuilderAppendString) v(StringBuilderDiscard, CalledByCode, (), SlowPathRecurs, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, AsOperation, StringBuilderDiscard) v(StackStringBuilderAppendString, CalledByCode, (), SlowPathRecurs, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, AsOperation, StackStringBuilderAppendString) v(DistinctStringsAreEqual, CalledByCode, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, reads(StringStorage), StringsAreEqual) +v(StringEqualsValue, CalledByCode, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, reads(StringStorage), StringsAreEqual) v(CompareStrings, CalledByCode, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, reads(StringStorage), CompareStrings) v(FindCharacter8, CalledByCode, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, reads(StringStorage), FindCharacter8) v(FindText8, CalledByCode, (), NoAxis, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, reads(StringStorage), FindText8) @@ -19,6 +22,8 @@ v(IntegerToString, CalledByCode, (), SlowPathRecurs, 0, 0, AllISAs, NoISA, NoCho v(Concatenate, CalledByCode, (), CountAxis | SlowPathRecurs, 2, 3, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, reads(StringStorage, AllocatorState) | writes(AllocatorState), Concatenate) +v(ConcatenateWithConstants, CalledByCode, (FirstOfTwo, SecondOfTwo, FirstOfThree, SecondOfThree, ThirdOfThree, FirstAndThirdOfThree), FormAxis | SlowPathRecurs, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, reads(StringStorage, AllocatorState) | writes(AllocatorState), ConcatenateWithConstants) + v(Substring, CalledByCode, (), SlowPathRecurs, 0, 0, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, reads(StringStorage, AllocatorState) | writes(AllocatorState), Substring) v(StringEqualsLiteral, CalledByCode, (), CountAxis, 0, 63, AllISAs, NoISA, NoChoice, Never, MayCall, ClobbersOnlyScratch, reads(StringStorage), StringEqualsLiteral) diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsStringHelpers.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsStringHelpers.cpp index 6bc443d9428fd..73d791f26e1ab 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsStringHelpers.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsStringHelpers.cpp @@ -22,13 +22,22 @@ void loadSubstringBaseAndOffset(StubAssembler& jit, GPRReg string, GPRReg base, { using CompactFibers = JSRopeString::CompactFibers; RELEASE_ASSERT(noOverlap(string, base, offset)); - constexpr int32_t bytesInFrontOfFiber1 = sizeof(uint64_t) - 6; static_assert(CompactFibers::widthOfFiber1LowerInLengthWord == 32 && CompactFibers::shiftOfFiber2InUpperWord == 16 && JSRopeString::offsetOfFiber2() == JSRopeString::offsetOfFiber1Lower() + 4); - static_assert(std::endian::native == std::endian::little && JSRopeString::offsetOfFiber1Lower() - bytesInFrontOfFiber1 >= JSRopeString::offsetOfLength()); - jit.load64(Address(string, JSRopeString::offsetOfFiber1Lower() - bytesInFrontOfFiber1), base); - jit.urshift64(TrustedImm32(8 * bytesInFrontOfFiber1), base); + static_assert(std::endian::native == std::endian::little); +#if CPU(ARM64) + static_assert(JSRopeString::offsetOfFiber1Lower() == JSRopeString::offsetOfLength() + 4 && !(JSRopeString::offsetOfLength() % sizeof(uint64_t))); + jit.loadPair64(Address(string, JSRopeString::offsetOfLength()), base, offset); + jit.urshift64(TrustedImm32(CompactFibers::widthOfFiber1LowerInLengthWord), base); + jit.insertBitField64(offset, TrustedImm32(CompactFibers::widthOfFiber1LowerInLengthWord), TrustedImm32(CompactFibers::shiftOfFiber2InUpperWord), base); + jit.urshift64(TrustedImm32(CompactFibers::shiftOfFiber2InUpperWord), offset); +#else + jit.load32(Address(string, JSRopeString::offsetOfFiber1Lower()), base); + jit.load16(Address(string, JSRopeString::offsetOfFiber2()), offset); + jit.lshift64(TrustedImm32(CompactFibers::widthOfFiber1LowerInLengthWord), offset); + jit.or64(offset, base); jit.load64(Address(string, JSRopeString::offsetOfFiber2()), offset); jit.urshift64(TrustedImm32(CompactFibers::shiftOfFiber2InUpperWord), offset); +#endif } void packSubstringFibers(StubAssembler& jit, GPRReg base, GPRReg offset, GPRReg length, GPRReg lengthWord) diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsStringLiterals.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsStringLiterals.cpp index 0c4338bbe8b21..de5093983d2c6 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsStringLiterals.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsStringLiterals.cpp @@ -12,6 +12,7 @@ #include "AOTInstance.h" #include "AOTOperationData.h" #include "AOTRuntimeTable.h" +#include "AOTSlotRepresentation.h" #include "AOTStubAssembler.h" #include "AOTStubHelpers.h" #include "AOTStubRegisters.h" @@ -42,7 +43,13 @@ StubSignature signatureOfStringEqualsLiteral(StubKey) { return tableComparisonSi StubSignature signatureOfValueEqualsLiteral(StubKey) { return tableComparisonSignature(comparedSubjectRegister).fillingOperandInFront(0); } StubSignature signatureOfStringEqualsLiteralOfLength(StubKey) { return registerComparisonSignature().standingBehindEntryThatFillsOperandInFront(0); } StubSignature signatureOfValueEqualsLiteralOfLength(StubKey) { return registerComparisonSignature().fillingOperandInFront(0); } -StubSignature signatureOfNarrowCharactersOfString(StubKey) { return StubSignature::withGPRResult({ gprOf(StubRegister::T11) }); } +StubSignature signatureOfNarrowCharactersOfString(StubKey) +{ + StubSignature signature = StubSignature::withGPRResult({ gprOf(StubRegister::T11) }); + signature.result = StubResult::TwoGPRs; + signature.secondResult = gprOf(StubRegister::T12); + return signature; +} void generateStringEqualsLiteral(StubAssembler& jit) { @@ -81,10 +88,16 @@ void generateStringEqualsLiteral(StubAssembler& jit) returnResult(jit, TrustedImm32(0)); return; } - jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(image.stringWords)), number); - jit.load32(BaseIndex(number, stringIndex, CCallHelpers::TimesFour), number); - hasNoHeader.append(jit.branchTest32(CCallHelpers::NonZero, number, TrustedImm32(ImageString::hasNoHeaderFlag))); - jit.and32(TrustedImm32(ImageString::hashMask), number); + auto loadNumberOfHeader = [&](JumpList& hasNone) { + jit.zeroExtend32ToWord(stringIndex, number); + Jump isIdentifier = jit.branch32(CCallHelpers::Above, Address(instanceGPR, AOT_INSTANCE_OFFSET(image.endOfIdentifiers)), number); + jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(image.stringWords)), number); + jit.load32(BaseIndex(number, stringIndex, CCallHelpers::TimesFour), number); + hasNone.append(jit.branchTest32(CCallHelpers::NonZero, number, TrustedImm32(ImageString::hasNoHeaderFlag))); + jit.and32(TrustedImm32(ImageString::hashMask), number); + isIdentifier.link(&jit); + }; + loadNumberOfHeader(hasNoHeader); if (lengthOfLiteral) { jit.getEffectiveAddress(BaseIndex(number, number, CCallHelpers::TimesTwo), header); jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(image.programIdentifiers)), headers); @@ -105,13 +118,15 @@ void generateStringEqualsLiteral(StubAssembler& jit) isSameImpl.link(&jit); returnResult(jit, TrustedImm32(1)); } - isRope.link(&jit); hasNoHeader.link(&jit); + Label mayBeEqual = jit.label(); jit.jumpToStub(equalsLiteral(ComparedOperand::String)); if (lengthOfLiteral < sizeof(uint64_t)) { isSameImpl.link(&jit); returnResult(jit, TrustedImm32(1)); } + isRope.link(&jit); + jit.branch32(CCallHelpers::Equal, Address(subject, JSRopeString::offsetOfLength()), TrustedImm32(lengthOfLiteral)).linkTo(mayBeEqual, &jit); isNotEqual.link(&jit); returnResult(jit, TrustedImm32(0)); return; @@ -152,6 +167,7 @@ void generateStringEqualsLiteral(StubAssembler& jit) returnResult(jit, TrustedImm32(1)); isNotEqual.link(&jit); + Label answersNo = jit.label(); returnResult(jit, TrustedImm32(0)); hasNoHeader.link(&jit); @@ -161,6 +177,17 @@ void generateStringEqualsLiteral(StubAssembler& jit) jit.returnToCaller(); isRope.link(&jit); + if (!jit.options().isPositionIndependent) { + JumpList hasNoLength; + loadNumberOfHeader(hasNoLength); + jit.getEffectiveAddress(BaseIndex(number, number, CCallHelpers::TimesTwo), header); + jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(image.programIdentifiers)), headers); + jit.getEffectiveAddress(BaseIndex(headers, header, CCallHelpers::TimesEight), header); + jit.load32(Address(header, OBJECT_OFFSETOF(ImageStringHeader, length)), length); + hasNoLength.append(jit.branchTest32(CCallHelpers::Zero, length)); + jit.branch32(CCallHelpers::NotEqual, Address(subject, JSRopeString::offsetOfLength()), length).linkTo(answersNo, &jit); + hasNoLength.link(&jit); + } static_assert(leafArgumentGPR(2) == gprOf(StubRegister::A2)); JumpList needsOperation; load8BitCharactersOfSubstring(jit, subject, fiber, length, needsOperation); @@ -244,7 +271,7 @@ void generateStringEqualsLiteralOfLength(StubAssembler& jit) return; } isNotEqual.append(jit.branch32(CCallHelpers::NotEqual, Address(fiber, StringImpl::lengthMemoryOffset()), TrustedImm32(length))); - slowCases.append(jit.branchTest32(CCallHelpers::Zero, Address(fiber, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()))); + Jump is16Bit = jit.branchTest32(CCallHelpers::Zero, Address(fiber, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit())); jit.loadPtr(Address(fiber, StringImpl::dataOffset()), characters); Label hasCharacters = jit.label(); loadPackedCharacters(jit, length, characters, packed); @@ -256,9 +283,27 @@ void generateStringEqualsLiteralOfLength(StubAssembler& jit) isRope.link(&jit); isNotEqual.append(jit.branch32(CCallHelpers::NotEqual, Address(subject, JSRopeString::offsetOfLength()), TrustedImm32(length))); Jump hasPieces = jit.branchTestPtr(CCallHelpers::Zero, fiber, TrustedImm32(JSRopeString::isSubstringInPointer)); - load8BitCharactersOfSubstring(jit, subject, characters, offsetInBase, slowCases); + JumpList baseIs16Bit; + load8BitCharactersOfSubstring(jit, subject, characters, offsetInBase, baseIs16Bit); jit.jump().linkTo(hasCharacters, &jit); + baseIs16Bit.link(&jit); + jit.lshift64(TrustedImm32(1), offsetInBase); + jit.addPtr(Address(fiber, StringImpl::dataOffset()), offsetInBase); + jit.move(offsetInBase, characters); + Jump has16BitCharacters = jit.jump(); + is16Bit.link(&jit); + jit.loadPtr(Address(fiber, StringImpl::dataOffset()), characters); + has16BitCharacters.link(&jit); + jit.load16(Address(characters), packed); + jit.zeroExtend8To32(text, characters); + if (length == 1) + jit.returnFlags32(CCallHelpers::Equal, packed, characters); + else { + isNotEqual.append(jit.branch32(CCallHelpers::NotEqual, packed, characters)); + slowCases.append(jit.jump()); + } + hasPieces.link(&jit); const GPRReg count = gprOf(StubRegister::T10); const GPRReg unit = gprOf(StubRegister::T14); @@ -321,7 +366,57 @@ void generateStringEqualsLiteralOfLength(StubAssembler& jit) void generateNarrowCharactersOfString(StubAssembler& jit) { - jit.callOperationAndReturn(Entry::operationAOTNarrowCharactersOfString, { .saves = Saves::CallerSavedRegisters, .checksException = false, .secondArgumentIn = jit.operand(0) }); + const GPRReg string = jit.operand(0); + const GPRReg characters = jit.result(); + const GPRReg length = jit.secondResult(); + const GPRReg offset = jit.allocateScratch(StubRegister::T9); + const GPRReg index = offset; + RELEASE_ASSERT(noOverlap(string, characters, length, offset)); + + JumpList isNoSubstringOfLatin1; + JumpList isOther; + jit.loadPtr(Address(string, JSString::offsetOfValue()), characters); + load8BitCharactersOfSubstring(jit, string, characters, offset, isNoSubstringOfLatin1); + jit.load32(Address(string, JSRopeString::offsetOfLength()), length); + jit.returnToCaller(); + + isNoSubstringOfLatin1.link(&jit); + const GPRReg value = length; + const GPRReg flags = offset; + jit.loadPtr(Address(string, JSString::offsetOfValue()), value); + Jump isRope = jit.branchIfRopeStringImpl(value); + jit.loadPtr(Address(value, StringImpl::dataOffset()), characters); + jit.load32(Address(value, StringImpl::flagsOffset()), flags); + jit.load32(Address(value, StringImpl::lengthMemoryOffset()), length); + Jump is16Bit = jit.branchTest32(CCallHelpers::Zero, flags, TrustedImm32(StringImpl::flagIs8Bit())); + jit.returnToCaller(); + + isRope.link(&jit); + isOther.append(jit.branchTestPtr(CCallHelpers::Zero, value, TrustedImm32(JSRopeString::isSubstringInPointer))); + jit.load32(Address(string, JSRopeString::offsetOfLength()), length); + jit.lshift64(TrustedImm32(1), offset); + jit.loadPtr(Address(characters, StringImpl::dataOffset()), characters); + jit.addPtr(offset, characters); + + is16Bit.link(&jit); + constexpr int32_t highByte = 1; + isOther.append(jit.branch32(CCallHelpers::Above, length, TrustedImm32(std::tuple_size_v))); + jit.move(TrustedImm32(0), index); + Label next = jit.label(); + isOther.append(jit.branch32(CCallHelpers::AboveOrEqual, index, length)); + Jump isWide = jit.branchTest8(CCallHelpers::NonZero, BaseIndex(characters, index, CCallHelpers::TimesTwo, highByte)); + jit.add32(TrustedImm32(1), index); + jit.jump().linkTo(next, &jit); + + isWide.link(&jit); + jit.move(TrustedImm32(static_cast(NarrowCharactersOfString::lengthOfNoNarrowText)), length); + jit.returnToCaller(); + + isOther.link(&jit); + jit.callOperationAndReturn(Entry::operationAOTNarrowCharactersOfString, { .saves = Saves::CallerSavedRegisters, .checksException = false, .secondArgumentIn = string }, [&] { + jit.load32(Address(characters, OBJECT_OFFSETOF(NarrowCharactersOfString, length)), length); + jit.addPtr(TrustedImm32(OBJECT_OFFSETOF(NarrowCharactersOfString, characters)), characters); + }); } } } // namespace JSC::AOT diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsStringSubstrings.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsStringSubstrings.cpp index b5ff5655d9f12..5e70e252b7965 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsStringSubstrings.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsStringSubstrings.cpp @@ -171,7 +171,7 @@ void generateSubstring(StubAssembler& jit) jit.branchTestPtr(CCallHelpers::NonZero, fiber, TrustedImm32(JSRopeString::is8BitInPointer)).linkTo(isLongPartOfRope, &jit); fails.append(jit.jump()); isNotTwoOfRope.link(&jit); - fails.append(jit.branch64(CCallHelpers::Equal, span, TrustedImm32(1))); + Jump isOneOfRope = jit.branch64(CCallHelpers::Equal, span, TrustedImm32(1)); Label isEmpty = jit.label(); jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_PREFIX_OFFSET(emptyString)), result); jit.returnToCaller(); @@ -182,14 +182,39 @@ void generateSubstring(StubAssembler& jit) jit.branchTest32(CCallHelpers::NonZero, Address(fiber, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit())).linkTo(isLong, &jit); fails.append(jit.jump()); + isOneOfRope.link(&jit); + Jump isOneOfSubstring = jit.branchTestPtr(CCallHelpers::NonZero, fiber, TrustedImm32(JSRopeString::isSubstringInPointer)); + jit.move(from, start); + jit.and64(TrustedImm32(static_cast(JSRopeString::stringMask)), fiber); + jit.loadPtr(Address(fiber, JSString::offsetOfValue()), fiber); + fails.append(jit.branchIfRopeStringImpl(fiber)); + jit.load32(Address(fiber, StringImpl::lengthMemoryOffset()), flags); + Jump isInFirstFiber = jit.branch32(CCallHelpers::Below, start, flags); + jit.sub32(flags, start); + loadSubstringBaseAndOffset(jit, string, base, span); + jit.loadPtr(Address(base, JSString::offsetOfValue()), fiber); + fails.append(jit.branchIfRopeStringImpl(fiber)); + Jump isInSecondFiber = jit.branch32(CCallHelpers::Below, start, Address(fiber, StringImpl::lengthMemoryOffset())); + fails.append(jit.jump()); + + isOneOfSubstring.link(&jit); + loadSubstringBaseAndOffset(jit, string, base, start); + jit.add64(from, start); + jit.loadPtr(Address(base, JSString::offsetOfValue()), fiber); + Jump hasBase = jit.jump(); + isOne.link(&jit); + jit.move(from, start); + isInFirstFiber.link(&jit); + isInSecondFiber.link(&jit); + hasBase.link(&jit); jit.load32(Address(fiber, StringImpl::flagsOffset()), flags); jit.loadPtr(Address(fiber, StringImpl::dataOffset()), fiber); Jump isWide = jit.branchTest32(CCallHelpers::Zero, flags, TrustedImm32(StringImpl::flagIs8Bit())); - jit.load8(BaseIndex(fiber, from, CCallHelpers::TimesOne), flags); + jit.load8(BaseIndex(fiber, start, CCallHelpers::TimesOne), flags); Jump hasCodeUnit = jit.jump(); isWide.link(&jit); - jit.load16(BaseIndex(fiber, from, CCallHelpers::TimesTwo), flags); + jit.load16(BaseIndex(fiber, start, CCallHelpers::TimesTwo), flags); fails.append(jit.branch32(CCallHelpers::Above, flags, TrustedImm32(maxSingleCharacterString))); hasCodeUnit.link(&jit); jit.loadPtr(Address(instanceGPR, AOT_INSTANCE_OFFSET(singleCharacterStrings)), fiber); diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsStrings.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsStrings.cpp index 71477d24d3729..8fbba8803697f 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsStrings.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsStrings.cpp @@ -26,7 +26,7 @@ namespace JSC { namespace AOT { using Saves = OperationCall::Saves; static constexpr int32_t maxLengthComparedWithoutCall = 32; -static constexpr int32_t maxLengthSearchedByBytes = 32; +static constexpr int32_t maxLengthSearchedByBytes = 15; static StubSignature signatureOfCharactersInPlace() { @@ -36,9 +36,18 @@ static StubSignature signatureOfCharactersInPlace() } StubSignature signatureOfRopeCharacters(StubKey) { return signatureOfCharactersInPlace(); } +StubSignature signatureOfSubstringCharacters(StubKey) +{ + StubSignature signature = signatureOfCharactersInPlace(); + signature.result = StubResult::TwoGPRs; + signature.secondResult = gprOf(StubRegister::T9); + return signature; +} StubSignature signatureOfFlattenString(StubKey) { return signatureOfCharactersInPlace(); } +StubSignature signatureOfStringCodeUnitAt(StubKey) { return StubSignature::withGPRResult({ gprOf(StubRegister::T12), gprOf(StubRegister::T11) }); } StubSignature signatureOfStringsAreEqual(StubKey) { return StubSignature::withGPRResult({ gprOf(StubRegister::T11), gprOf(StubRegister::T12) }); } StubSignature signatureOfDistinctStringsAreEqual(StubKey key) { return signatureOfStringsAreEqual(key); } +StubSignature signatureOfStringEqualsValue(StubKey key) { return signatureOfStringsAreEqual(key); } StubSignature signatureOfCompareStrings(StubKey) { return StubSignature::withGPRResult({ gprOf(StubRegister::T11), gprOf(StubRegister::T12) }); } StubSignature signatureOfFindCharacter8(StubKey) { return StubSignature::withGPRResult({ gprOf(StubRegister::A0), gprOf(StubRegister::A1), gprOf(StubRegister::A2), gprOf(StubRegister::A3) }); } StubSignature signatureOfFindText8(StubKey) { return StubSignature::withGPRResult({ gprOf(StubRegister::A0), gprOf(StubRegister::A1), gprOf(StubRegister::A2), gprOf(StubRegister::A3), gprOf(StubRegister::A4) }); } @@ -303,6 +312,23 @@ void generateRopeCharacters(StubAssembler& jit) jit.callOperationAndReturn(Entry::operationAOTFlatten, { .saves = Saves::CallerSavedRegisters, .secondArgumentIn = string, .valueDestination = fiber }, loadImplCharacters); } +void generateSubstringCharacters(StubAssembler& jit) +{ + const GPRReg string = jit.clobberOperand(0); + const GPRReg characters = jit.result(); + const GPRReg impl = jit.secondResult(); + const GPRReg offset = jit.allocateScratch(StubRegister::T13); + RELEASE_ASSERT(string == characters && noOverlap(characters, impl, offset)); + loadSubstringBaseAndOffset(jit, string, impl, offset); + jit.loadPtr(Address(impl, JSString::offsetOfValue()), impl); + jit.loadPtr(Address(impl, StringImpl::dataOffset()), characters); + Jump is8Bit = jit.branchTest32(CCallHelpers::NonZero, Address(impl, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit())); + jit.lshift64(TrustedImm32(1), offset); + is8Bit.link(&jit); + jit.addPtr(offset, characters); + jit.returnToCaller(); +} + void generateFlattenString(StubAssembler& jit) { const GPRReg string = jit.operand(0); @@ -317,6 +343,31 @@ void generateFlattenString(StubAssembler& jit) jit.callOperationAndReturn(Entry::operationAOTFlatten, { .saves = Saves::CallerSavedRegisters, .secondArgumentIn = string, .valueDestination = fiber }); } +void generateStringCodeUnitAt(StubAssembler& jit) +{ + const GPRReg string = jit.operand(0); + const GPRReg index = jit.operand(1); + const GPRReg unit = jit.result(); + const GPRReg fiber = jit.allocateScratch(StubRegister::T9); + RELEASE_ASSERT(noOverlap(string, index, unit, fiber)); + + jit.loadPtr(Address(string, JSString::offsetOfValue()), fiber); + Jump isRope = jit.branchIfRopeStringImpl(fiber); + jit.zeroExtend32ToWord(index, unit); + Jump is16Bit = jit.branchTest32(CCallHelpers::Zero, Address(fiber, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit())); + jit.loadPtr(Address(fiber, StringImpl::dataOffset()), fiber); + jit.load8(BaseIndex(fiber, unit, CCallHelpers::TimesOne), unit); + jit.returnToCaller(); + + is16Bit.link(&jit); + jit.loadPtr(Address(fiber, StringImpl::dataOffset()), fiber); + jit.load16(BaseIndex(fiber, unit, CCallHelpers::TimesTwo), unit); + jit.returnToCaller(); + + isRope.link(&jit); + jit.callOperationAndReturn(Entry::operationAOTStringCodeUnitAt, { .saves = Saves::CallerSavedRegisters, .checksException = false, .secondArgumentIn = string, .thirdArgumentIn = index }); +} + static void findCodeUnitAndReturn(StubAssembler& jit, GPRReg characters, GPRReg length, GPRReg codeUnit, GPRReg position, GPRReg index, GPRReg unit) { RELEASE_ASSERT(characters == leafArgumentGPR(0) && length == leafArgumentGPR(1) && codeUnit == leafArgumentGPR(2) && position == leafArgumentGPR(3)); @@ -586,6 +637,7 @@ void generateStringsAreEqual(StubAssembler& jit) JumpList needsLeaf; areEqual.append(jit.branchPtr(CCallHelpers::Equal, first, second)); jit.bindEntry(distinctStringsAreEqual()); + Label areDistinct = jit.label(); jit.loadPtr(Address(first, JSString::offsetOfValue()), firstFiber); jit.loadPtr(Address(second, JSString::offsetOfValue()), secondFiber); jit.or64(firstFiber, secondFiber, scratch); @@ -599,6 +651,7 @@ void generateStringsAreEqual(StubAssembler& jit) branchIfCharactersDiffer(jit, { firstFiber, secondFiber, length, scratch, word }, areNotEqual, needsLeaf); areEqual.link(&jit); + Label returnTrue = jit.label(); returnResult(jit, TrustedImm32(1)); areNotEqual.link(&jit); @@ -620,6 +673,12 @@ void generateStringsAreEqual(StubAssembler& jit) differ.linkTo(returnFalse, &jit); needsOperation.link(&jit); jit.callOperationAndReturn(Entry::operationAOTStringsAreEqual, { .saves = Saves::CallerSavedRegisters, .checksException = false, .secondArgumentIn = first, .thirdArgumentIn = second }); + + jit.bindEntry(stringEqualsValue()); + jit.branchPtr(CCallHelpers::Equal, first, second).linkTo(returnTrue, &jit); + jit.branchIfNotCell(second).linkTo(returnFalse, &jit); + jit.branchIfNotString(second).linkTo(returnFalse, &jit); + jit.jump().linkTo(areDistinct, &jit); } void branchOnOrderOfFlatStrings(StubAssembler& jit, const OrderedFlatStrings& strings, JumpList& needsLeaf, JumpList& isLess, JumpList& isEqual) diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsStrings.h b/Source/JavaScriptCore/aot/stubs/AOTStubsStrings.h index 9b2d1abe3a62a..d41383c390099 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsStrings.h +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsStrings.h @@ -16,12 +16,15 @@ namespace JSC { namespace AOT { constexpr StubKey ropeCharacters() { return StubKey(StubFamily::RopeCharacters); } +constexpr StubKey substringCharacters() { return StubKey(StubFamily::SubstringCharacters); } constexpr StubKey flattenString() { return StubKey(StubFamily::FlattenString); } +constexpr StubKey stringCodeUnitAt() { return StubKey(StubFamily::StringCodeUnitAt); } constexpr StubKey stringsAreEqual() { return StubKey(StubFamily::StringsAreEqual); } constexpr StubKey stringBuilderAppendString() { return StubKey(StubFamily::StringBuilderAppendString); } constexpr StubKey stringBuilderDiscard() { return StubKey(StubFamily::StringBuilderDiscard); } constexpr StubKey stackStringBuilderAppendString() { return StubKey(StubFamily::StackStringBuilderAppendString); } constexpr StubKey distinctStringsAreEqual() { return StubKey(StubFamily::DistinctStringsAreEqual); } +constexpr StubKey stringEqualsValue() { return StubKey(StubFamily::StringEqualsValue); } constexpr StubKey compareStrings() { return StubKey(StubFamily::CompareStrings); } constexpr StubKey integerToString() { return StubKey(StubFamily::IntegerToString); } enum class ConcatenatedKeyForm : uint8_t { Key, Name }; @@ -36,12 +39,40 @@ constexpr StubKey concatenate(unsigned count) return StubKey(StubFamily::Concatenate).withCount(count); } -static_assert(exists(ropeCharacters()) && exists(flattenString())); +enum class ConstantsOfConcatenation : uint8_t { FirstOfTwo, SecondOfTwo, FirstOfThree, SecondOfThree, ThirdOfThree, FirstAndThirdOfThree }; + +constexpr unsigned numberOfOperands(ConstantsOfConcatenation constants) +{ + return constants == ConstantsOfConcatenation::FirstOfTwo || constants == ConstantsOfConcatenation::SecondOfTwo ? 2 : 3; +} + +constexpr bool isConstantOperand(ConstantsOfConcatenation constants, unsigned operand) +{ + switch (constants) { + case ConstantsOfConcatenation::FirstOfTwo: + case ConstantsOfConcatenation::FirstOfThree: + return !operand; + case ConstantsOfConcatenation::SecondOfTwo: + case ConstantsOfConcatenation::SecondOfThree: + return operand == 1; + case ConstantsOfConcatenation::ThirdOfThree: + return operand == 2; + case ConstantsOfConcatenation::FirstAndThirdOfThree: + return operand != 1; + } + return false; +} + +constexpr StubKey concatenateWithConstants(ConstantsOfConcatenation constants) { return StubKey(StubFamily::ConcatenateWithConstants).withForm(static_cast(constants)); } + +static_assert(exists(ropeCharacters()) && exists(substringCharacters()) && exists(flattenString())); static_assert(exists(stringsAreEqual()) && exists(compareStrings()) && exists(integerToString()) && exists(substring())); static_assert(exists(distinctStringsAreEqual()) && bodyOf(distinctStringsAreEqual()) == stringsAreEqual()); -static_assert(exists(findCharacter8())); +static_assert(exists(stringEqualsValue()) && bodyOf(stringEqualsValue()) == stringsAreEqual()); +static_assert(exists(findCharacter8()) && exists(stringCodeUnitAt())); static_assert(exists(concatenateKeyOfPair(ConcatenatedKeyForm::Key)) && exists(concatenateKeyOfPair(ConcatenatedKeyForm::Name))); static_assert(exists(concatenate(2)) && exists(concatenate(3))); +static_assert(exists(concatenateWithConstants(ConstantsOfConcatenation::FirstAndThirdOfThree)) && !exists(concatenateWithConstants(static_cast(6)))); static_assert(!exists(concatenate(2).withCount(1)) && !exists(concatenate(3).withCount(4))); void returnResult(StubAssembler& jit, auto value) diff --git a/Source/JavaScriptCore/aot/support/AOTLargeList.h b/Source/JavaScriptCore/aot/support/AOTLargeList.h index a7141c54c1029..26a3d4b7a5c31 100644 --- a/Source/JavaScriptCore/aot/support/AOTLargeList.h +++ b/Source/JavaScriptCore/aot/support/AOTLargeList.h @@ -62,6 +62,8 @@ class LargeList { WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN T& operator[](size_t index) LIFETIME_BOUND { return m_elements[checked(index)]; } const T& operator[](size_t index) const LIFETIME_BOUND { return m_elements[checked(index)]; } + ALWAYS_INLINE void prefetch(size_t index) const { __builtin_prefetch(m_elements + index); } + template ALWAYS_INLINE void prefetch(size_t index, Member T::* member) const { __builtin_prefetch(&(m_elements[index].*member)); } WTF_ALLOW_UNSAFE_BUFFER_USAGE_END void reserve(size_t room) diff --git a/Source/JavaScriptCore/aot/support/AOTSort.h b/Source/JavaScriptCore/aot/support/AOTSort.h index f3eb5897ac3bb..7c96018a7c582 100644 --- a/Source/JavaScriptCore/aot/support/AOTSort.h +++ b/Source/JavaScriptCore/aot/support/AOTSort.h @@ -6,7 +6,11 @@ #pragma once #include +#include +#include +#include #include +#include #include #include #include @@ -23,6 +27,8 @@ inline constexpr std::ptrdiff_t largestRangeSortedByInsertion = 32; inline constexpr std::ptrdiff_t longestRunThatMergingStartsFrom = 16; +inline constexpr std::ptrdiff_t fewestIntegersSortedByDigits = 2048; + template void insertEachWhereItBelongs(Iterator first, Iterator next, Iterator last, IsLess& isLess, Projection& projection) { @@ -81,6 +87,36 @@ NEVER_INLINE void mergeSortKeepingEqualElementsInOrder(Element* first, Element* fastFree(room); } +template +NEVER_INLINE void sortIntegersByDigits(Integer* first, Integer* last) +{ + constexpr unsigned bitsInDigit = 11; + constexpr size_t valuesOfDigit = 1 << bitsInDigit; + size_t size = last - first; + Integer bitsThatDiffer = 0; + for (size_t index = 1; index < size; ++index) + bitsThatDiffer |= first[index] ^ *first; + Integer* room = static_cast(fastMalloc(size * sizeof(Integer))); + Integer* from = first; + Integer* to = room; + while (bitsThatDiffer) { + unsigned shift = std::countr_zero(bitsThatDiffer); + bitsThatDiffer &= ~static_cast(static_cast(valuesOfDigit - 1) << shift); + std::array places { }; + for (size_t index = 0; index < size; ++index) + ++places[from[index] >> shift & (valuesOfDigit - 1)]; + size_t next = 0; + for (size_t& place : places) + next += std::exchange(place, next); + for (size_t index = 0; index < size; ++index) + to[places[from[index] >> shift & (valuesOfDigit - 1)]++] = from[index]; + std::swap(from, to); + } + if (from != first) + memcpy(first, from, size * sizeof(Integer)); + fastFree(room); +} + template void sortKeepingEqualElementsInOrder(Range&& range, IsLess isLess, Projection projection) { @@ -91,6 +127,12 @@ void sortKeepingEqualElementsInOrder(Range&& range, IsLess isLess, Projection pr return; if (last - first > largestRangeSortedByInsertion) { static_assert(std::contiguous_iterator); + if constexpr (std::unsigned_integral> && std::same_as && std::same_as) { + if (last - first >= fewestIntegersSortedByDigits) { + sortIntegersByDigits(std::to_address(first), std::to_address(last)); + return; + } + } mergeSortKeepingEqualElementsInOrder(std::to_address(first), std::to_address(first) + (last - first), isLess, projection); return; } diff --git a/Source/JavaScriptCore/aot/support/AOTThreads.h b/Source/JavaScriptCore/aot/support/AOTThreads.h index bef1315d09830..bb846cab701a2 100644 --- a/Source/JavaScriptCore/aot/support/AOTThreads.h +++ b/Source/JavaScriptCore/aot/support/AOTThreads.h @@ -137,8 +137,10 @@ Vector> collectChunksInParallel(const ParallelLoops& loops, unsigned c ChunksOfIndices division = divideIntoChunks(count); Vector> chunks(division.size()); forEachIndexInParallel(loops, division.size(), [&](unsigned chunk) { - for (unsigned index = division.firstIndexOf[chunk]; index < division.firstIndexOf[chunk + 1]; ++index) - collect(index, chunks[chunk]); + Vector& collected = chunks[chunk]; + unsigned end = division.firstIndexOf[chunk + 1]; + for (unsigned index = division.firstIndexOf[chunk]; index < end; ++index) + collect(index, collected); }); return chunks; } diff --git a/Source/JavaScriptCore/aot/tests/AOTFFIStubTests.cpp b/Source/JavaScriptCore/aot/tests/AOTFFIStubTests.cpp index 6dcc69cd531fb..d64aedcfedb38 100644 --- a/Source/JavaScriptCore/aot/tests/AOTFFIStubTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTFFIStubTests.cpp @@ -308,7 +308,7 @@ static void testOpenProgramBuiltWithoutForeignCallsHasNone() AOT_TEST_CHECK(!header.code.ffiCodeSize); AOT_TEST_CHECK(header.flags == headerOf(*withStubs).flags && header.has(ImageFlag::AdmitsUnknownJavaScript)); AOT_TEST_CHECK(header.executableSize + headerOf(*withStubs).code.ffiCodeSize <= headerOf(*withStubs).executableSize); - AOT_TEST_CHECK(static_cast(header.code.constantsOffset) + header.code.constantsSize == header.codeSize); + AOT_TEST_CHECK(std::max(header.executableSize, static_cast(header.code.constantsOffset) + header.code.constantsSize) == header.codeSize); } static void testUnseenCodeWithoutForeignCalls() @@ -332,7 +332,7 @@ static void testDirectoryIsNotExecutable() std::unique_ptr closedOutput = compileClosedProgram(); const ImageHeader& closed = headerOf(*closedOutput); - AOT_TEST_CHECK(static_cast(closed.code.constantsOffset) + closed.code.constantsSize == closed.codeSize); + AOT_TEST_CHECK(std::max(closed.executableSize, static_cast(closed.code.constantsOffset) + closed.code.constantsSize) == closed.codeSize); } static String runInShell(const CompileOutput& output, ASCIILiteral statements) diff --git a/Source/JavaScriptCore/aot/tests/AOTFactsTestCommands.cpp b/Source/JavaScriptCore/aot/tests/AOTFactsTestCommands.cpp index e679669bc4ca4..7b39a100400d6 100644 --- a/Source/JavaScriptCore/aot/tests/AOTFactsTestCommands.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTFactsTestCommands.cpp @@ -212,6 +212,8 @@ static ASCIILiteral nameInFactsText(SummaryTrait trait) return "OnlyPassesOnGatheredArguments"_s; case SummaryTrait::HoldsInt32Promise: return "HoldsInt32Promise"_s; + case SummaryTrait::ReturnsUInt32: + return "ReturnsUInt32"_s; } RELEASE_ASSERT_NOT_REACHED(); } @@ -488,7 +490,7 @@ static void printAccessOfSummary(IRPrinter& printer, const ThinSummary& summary, if (access.isToContents()) { out.print("contents "_s); printSlotOfContents(printer, access.slot); - printNamedNodeOfSummary(printer, summary, "else "_s, access.whatHoldsNothing); + printNamedNodeOfSummary(printer, summary, "else "_s, access.whatHoldsNothing()); printNamedNodeOfSummary(printer, summary, "then "_s, access.keyNode); } else if (access.key) printer.print(access.key); @@ -508,9 +510,9 @@ static void printAccessOfSummary(IRPrinter& printer, const ThinSummary& summary, case SummaryAccess::Kind::Has: break; } - if (access.beginningOfKey) { + if (StringID beginning = access.beginningOfKey()) { out.print(" begins-with "_s); - printer.print(access.beginningOfKey); + printer.print(beginning); } if (access.isPartOfConstruction) out.print(" at-construction"_s); @@ -519,18 +521,19 @@ static void printAccessOfSummary(IRPrinter& printer, const ThinSummary& summary, if (access.storesObjectMadeForIt) out.print(" made-for-it"_s); if (access.laysAccessorAsWritten) { + const SummaryWrittenAccessor& written = summary.writtenAccessorOf(index); out.print(" accessor-as-written"_s); - if (access.writtenGetter) { + if (written.getter) { out.print(" getter="_s); - printer.print(access.writtenGetter); + printer.print(written.getter); } - if (access.writtenSetter) { + if (written.setter) { out.print(" setter="_s); - printer.print(access.writtenSetter); + printer.print(written.setter); } - if (access.mapOfWrittenAccessor) { + if (written.map) { out.print(" in-map="_s); - printer.print(access.mapOfWrittenAccessor); + printer.print(written.map); } } if (access.kind == SummaryAccess::Kind::Define) @@ -1335,6 +1338,8 @@ static void printLinkedLayouts(IRPrinter& printer, const LinkTables& tables) printFlagsInFactsText(out, layout.possibleChanges - LayoutChange::GetsObjectAbove); if (!layout.reachesUnseenCode) out.print(" stays-with-compiled-code"_s); + if (layout.isAllocated && !layout.hasDirectInstances) + out.print(" has-no-direct-instances"_s); if (layout.isRedefinedByLiteralsAlone) out.print(" redefined-by-literals-alone"_s); if (layout.isFilledUnderComputedKeys) diff --git a/Source/JavaScriptCore/aot/tests/AOTGrowthOfWorkTests.cpp b/Source/JavaScriptCore/aot/tests/AOTGrowthOfWorkTests.cpp index a39b887a98967..ed406dcd7029f 100644 --- a/Source/JavaScriptCore/aot/tests/AOTGrowthOfWorkTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTGrowthOfWorkTests.cpp @@ -116,6 +116,54 @@ static void testEachOfManyMembersIsFoundByItsKey() }); } +static std::optional instructionsOfIndexingViewsForGrowthTests(unsigned numberOfViews) +{ + StringBuilder text; + for (unsigned number = 0; number < numberOfViews; ++number) { + text.append("layout L"_s, number, " name=\"I"_s, number, "\" module=m0 fields=["_s); + for (unsigned field = 0; field <= number; ++field) + text.append(field ? ", "_s : ""_s, "{key=\"b"_s, field, "\" declared={type=anything}}"_s); + text.append(']'); + if (number) + text.append(" supertypes=[L"_s, number - 1, ']'); + text.append(" flags=[IsNeverAllocated] ownFieldCount="_s, number + 1, '\n'); + } + for (unsigned number = 0; number < numberOfViews; ++number) + text.append("layout L"_s, numberOfViews + number, " module=m0 fields=[] supertypes=[L"_s, number, "]\n"_s); + text.append("module m0 path=\"main.js\" displayPath=\"main.js\" body=f0\nentry module=m0\n"_s); + text.append("function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] {\n b0():\n v0 = Builtin builtin=GlobalThis\n Return v0\n}\n"_s); + + std::optional instructions; + runOnBuildThread([&] { + BuildContext build(makeOptionsOfTestedProgram()); + std::unique_ptr program = parseProgram(withStandInLibrary(text.toString()), build); + AOT_TEST_CHECK(program); + instructions = instructionsOfCurrentThreadIn([&] { + program->indexViews(ParallelLoops(1)); + }); + LayoutID last(2 * numberOfViews - 1); + AOT_TEST_CHECK(program->viewsAbove(last).size() == numberOfViews); + PropertyKeyID middle = program->at(LayoutID(numberOfViews / 2)).fields[numberOfViews / 2].key; + unsigned numberOfFields = 0; + AOT_TEST_CHECK(program->visitFieldsWrittenAbove(last, middle, [&](Program::FieldOfView field) { + AOT_TEST_CHECK(program->at(field.view).fields[field.field].key == middle && field.view.index() == numberOfViews / 2 + numberOfFields); + ++numberOfFields; + return true; + })); + AOT_TEST_CHECK(numberOfFields == numberOfViews - numberOfViews / 2); + }); + return instructions; +} + +static void testViewIndexGrowth() +{ + constexpr unsigned numberOfViews = 100; + InstructionCounts::begin(); + std::optional smaller = instructionsOfIndexingViewsForGrowthTests(numberOfViews); + std::optional large = instructionsOfIndexingViewsForGrowthTests(4 * numberOfViews); + AOT_TEST_CHECK(!smaller || !large || *large < 28 * *smaller); +} + static std::optional instructionsOfLookupsFromFootOfRowForGrowthTests(unsigned numberOfClasses) { StringBuilder text; @@ -1846,6 +1894,7 @@ static void testFirstWalkGrowsAsJoinsThatNamesAreReadOffDo() std::span growthOfWorkTests() { static constexpr NativeTest tests[] { + { "growth-of-work/view-index-growth"_s, testViewIndexGrowth }, { "growth-of-work/each-of-many-members-of-a-class-is-found-by-its-key"_s, testEachOfManyMembersIsFoundByItsKey }, { "growth-of-work/members-of-classes-far-above-are-found-by-their-keys"_s, testMembersOfClassesFarAboveAreFoundByTheirKeys }, { "growth-of-work/asking-each-class-object-of-a-row-what-a-lookup-does-not-find-grows-as-the-row-does"_s, testAskingEachClassObjectOfRowWhatLookupDoesNotFindGrowsAsRowDoes }, diff --git a/Source/JavaScriptCore/aot/tests/AOTIDSetTests.cpp b/Source/JavaScriptCore/aot/tests/AOTIDSetTests.cpp index b0d98f02596fd..75fd09757d2ae 100644 --- a/Source/JavaScriptCore/aot/tests/AOTIDSetTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTIDSetTests.cpp @@ -908,6 +908,25 @@ static void unionsOfTypesWithEarlierStatesAreTheUnions() told.forgetEarlierStates(); } +static void rememberedAnswersDoNotShareNumbers() +{ + TypeTable table(BuiltinTable::singleton()); + table.setObjects(makeUniqueWithoutFastMallocCheck(BuiltinTable::singleton(), std::span())); + table.freeze(); + TypeArena types(table, ArenaKeeps::UntilReset); + + constexpr auto records = static_cast(AnswerKeptInArena::RecordsThatFileWrites); + constexpr auto mention = static_cast(AnswerKeptInArena::UnitMayMentionFunctions); + const uint32_t word = TypeID(TypeTable::string).bits(); + TypeArena::SetOfRound round { word, false }; + types.rememberRound(records, 2, 0, round, 0, { 5, 2 }); + AOT_TEST_CHECK(!types.rememberedAnswer(mention, word, 0)); + types.rememberAnswer(mention, word, 0, 1); + std::optional kept = types.rememberedRound(records, 2, round, 0); + AOT_TEST_CHECK(kept && (*kept)[0] == 5 && (*kept)[1] == 2); + AOT_TEST_CHECK(types.rememberedAnswer(mention, word, 0) == 1u); +} + static void freezingTypeTableFreesOutgrownSlots() { constexpr uint32_t numberMade = 64 * InternedRecords::numberOfShards; @@ -1291,6 +1310,7 @@ std::span idSetTests() { "id-sets/unions-of-types-with-earlier-states"_s, unionsOfTypesWithEarlierStatesAreTheUnions }, { "id-sets/compare-by-content"_s, idSetsCompareByContent }, { "id-sets/typed-sets"_s, typedIDSetsAnswerLikeUntypedOnes }, + { "id-sets/remembered-answers-do-not-share-numbers"_s, rememberedAnswersDoNotShareNumbers }, { "id-sets/freezing-type-table-frees-outgrown-slots"_s, freezingTypeTableFreesOutgrownSlots }, { "id-sets/list-memo"_s, idListMemoKeepsWhatIsAskedFor }, { "id-sets/shared-answers-eight-threads-number-lists"_s, eightThreadsGetOneNumberForEachSharedList }, diff --git a/Source/JavaScriptCore/aot/tests/AOTImageBuildTests.cpp b/Source/JavaScriptCore/aot/tests/AOTImageBuildTests.cpp index 7aec9ab641d2a..5b3ef6bcaa54d 100644 --- a/Source/JavaScriptCore/aot/tests/AOTImageBuildTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTImageBuildTests.cpp @@ -209,8 +209,8 @@ static void testVectorOverflow() AOT_TEST_CHECK(!writer.append(neverRead.span(), 1)); AOT_TEST_CHECK(writer.didOverflow()); AOT_TEST_CHECK(!writer.append(bytes.span(), 1)); - Vector> streams { Vector { 1 }, Vector { 2, 3 } }; - AOT_TEST_CHECK(writer.appendAll(streams.span()) == Vector({ 0, 0 })); + Vector> streams { Vector { 1 }, Vector { 2, 3 }, Vector { 1 } }; + AOT_TEST_CHECK(findFirstEqualStreams(streams.span(), context.parallelLoops()) == Vector({ 0, 1, 0 })); AOT_TEST_CHECK(writer.takeBytes().isEmpty()); AOT_TEST_CHECK(count(context, UserDiagnosticKind::LimitExceeded, LimitID::VectorBytes) == 1); AOT_TEST_CHECK(context.diagnostics().userDiagnostics().size() == 1); diff --git a/Source/JavaScriptCore/aot/tests/AOTImageCodeLayoutTests.cpp b/Source/JavaScriptCore/aot/tests/AOTImageCodeLayoutTests.cpp index fb04f7ab11544..34246371e44b1 100644 --- a/Source/JavaScriptCore/aot/tests/AOTImageCodeLayoutTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTImageCodeLayoutTests.cpp @@ -463,6 +463,21 @@ static void testRegExpAndWebAssembly() AOT_TEST_CHECK(layout.codeSize == layout.constantsOffset); } +static void testConstantsInLastExecutablePage() +{ + for (bool constantsFit : { true, false }) { + Steps steps(framedBodies(5)); + steps.findJumps(); + steps.constants.bytes.fill(1, constantsFit ? 24 : 2 * imagePageSize); + steps.layOut(); + const CodeLayout& layout = steps.layout; + size_t sizeOfConstants = WTF::roundUpToMultipleOf(steps.constants.bytes.size()); + AOT_TEST_CHECK(constantsFit == (sizeOfConstants <= layout.executableSize - layout.functionsEnd)); + AOT_TEST_CHECK(layout.constantsOffset == (constantsFit ? layout.executableSize - sizeOfConstants : layout.executableSize)); + AOT_TEST_CHECK(layout.codeSize == (constantsFit ? layout.executableSize : layout.executableSize + steps.constants.bytes.size())); + } +} + static void testNeitherRegExpNorWebAssembly() { Steps steps(framedBodies(5)); @@ -645,6 +660,7 @@ std::span codeLayoutTests() { "image/layout/far-regexp-entries"_s, testFarRegExpEntries }, #endif { "image/layout/regexp-and-wasm"_s, testRegExpAndWebAssembly }, + { "image/layout/constants-in-last-executable-page"_s, testConstantsInLastExecutablePage }, { "image/layout/neither"_s, testNeitherRegExpNorWebAssembly }, { "image/layout/no-body"_s, testNoBody }, { "image/fall-through/jump-to-next"_s, testJumpToNext }, diff --git a/Source/JavaScriptCore/aot/tests/AOTImageFaultTableTests.cpp b/Source/JavaScriptCore/aot/tests/AOTImageFaultTableTests.cpp index 84c45089da4f7..79bef61fab728 100644 --- a/Source/JavaScriptCore/aot/tests/AOTImageFaultTableTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTImageFaultTableTests.cpp @@ -226,7 +226,12 @@ static void testFaultSitesSecondWords() return { i }; return { i, 5 }; }); - for (uint32_t first : { 0x80u, 0xff00u, 0xffff00u, 0xfffffe00u }) { + for (uint32_t distinct : { 1u, 60u, 61u, 62u, 64u }) { + checkFramedBodyFaults(300, [&](unsigned i) -> FaultWords { + return { i, i % 64 < 2 ? 5u : 1000 + i % distinct }; + }); + } + for (uint32_t first : { 0x80u, 0xff00u, 0xbfff80u, 0xffff00u, 0xfffffe00u }) { checkFramedBodyFaults(300, [&](unsigned i) -> FaultWords { if (i % 5 == 4) return { i }; @@ -367,7 +372,7 @@ static void testHandlersTypes() static std::optional handlerAt(const Image& image, uint32_t function, CallSiteIndex site) { - return findHandler(SideTableParts::read(image.function(function), image.section(ImageSection::SideTables).data()).handlers, site); + return image.handlerAt(function, site); } static void testHandlersThroughImage() diff --git a/Source/JavaScriptCore/aot/tests/AOTImageFunctionTableTests.cpp b/Source/JavaScriptCore/aot/tests/AOTImageFunctionTableTests.cpp index 7eeb39758a683..34b57629041f0 100644 --- a/Source/JavaScriptCore/aot/tests/AOTImageFunctionTableTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTImageFunctionTableTests.cpp @@ -20,6 +20,7 @@ #include "AOTNumbering.h" #include "AOTOperationData.h" #include "AOTPlan.h" +#include "AOTPrefixCode.h" #include "AOTPropertyCachesFront.h" #include "AOTRepresentation.h" #include "AOTStubsBarriers.h" @@ -91,7 +92,7 @@ static void testImageFunctionOf() const Image& loaded = *image.image; auto hasOwnRecord = [&](uint32_t function) { std::optional index = loaded.imageFunctionOf(FunctionID(function)); - return index && loaded.function(*index).function() == function; + return index && loaded.rootFunctionOf(*index) == FunctionID(function); }; for (uint32_t function : { 1u, 2u, 4u, 6u }) @@ -128,7 +129,7 @@ static void testRecords() AOT_TEST_CHECK(!loaded.frameOf(frameless).frameSizeInBytes()); AOT_TEST_CHECK(&loaded.frameOf(second) == &loaded.frameOf(third)); AOT_TEST_CHECK(&loaded.frameOf(first) != &loaded.frameOf(second)); - AOT_TEST_CHECK(loaded.section(ImageSection::Functions).size() == 4); + AOT_TEST_CHECK(loaded.header().counts.functions == 4); AOT_TEST_CHECK(loaded.function(first).flags() == OptionSet({ HasCallSites, HasPositions })); AOT_TEST_CHECK(loaded.function(second).flags() == OptionSet({ HasCallSites, HasPositions })); @@ -136,7 +137,7 @@ static void testRecords() AOT_TEST_CHECK(!loaded.function(frameless).has(HasHandlers)); AOT_TEST_CHECK(!loaded.function(frameless).has(HasFrameMaps)); for (uint32_t function : { 1u, 2u, 3u, 4u }) - AOT_TEST_CHECK(loaded.function(image.indexOf(function)).function() == function); + AOT_TEST_CHECK(loaded.rootFunctionOf(image.indexOf(function)) == FunctionID(function)); } static SUPPRESS_ASAN void testSampleInNativeCode() @@ -464,7 +465,7 @@ static void testNameNumberPastFifteenBits() ++numberOfNamesPastFifteenBits; AOT_TEST_CHECK(names.at(*number).key == word.key); AOT_TEST_CHECK(*number < text.numberOfIdentifiers()); - AOT_TEST_CHECK(text.identifier(*number).text() == program.at(word.key).text); + AOT_TEST_CHECK(text.identifier(*number).text() == StringID(image.built.numbers->stringNumbers().requiredNumberOf(program.at(word.key).text))); } AOT_TEST_CHECK(numberOfNamesPastFifteenBits >= numberOfNames - firstNumberPastFifteenBits); } @@ -628,7 +629,7 @@ static void checkTablesOfVariedFunctions(const TestBuild& build, const TestImage AOT_TEST_CHECK(tables.builtinImplementedBy(index).has_value() == trace.implements().has_value()); bool isShownByName = source && source <= numberOfVariedFunctions && variedFunctionHasCode(source); - std::optional name = isShownByName && function.name && function.name != program.strings.emptyString() ? std::optional(function.name.index()) : std::nullopt; + std::optional name = isShownByName && function.name && function.name != program.strings.emptyString() ? numbers.stringNumbers().numberOf(function.name) : std::nullopt; AOT_TEST_CHECK(tables.nameOfFunction(index) == name); numberOfNames += name && index < functions.firstDerived(); @@ -642,7 +643,7 @@ static void checkTablesOfVariedFunctions(const TestBuild& build, const TestImage AOT_TEST_CHECK(tables.functionLengthOf(index) == lengthOfVariedFunction(source)); numberWithLargeCounts += lengthOfVariedFunction(source) >= countsAreListed || countsInTable.encoded() == countsAreListed; AOT_TEST_CHECK(std::bit_cast(tables.kindOfClosureNumber(closureNumber)) == std::bit_cast(record.kind)); - AOT_TEST_CHECK(tables.closureCellNumbers()[closureNumber] == numbers.closureCellNumber(unit.function)); + AOT_TEST_CHECK(SparseNumberTable(tables.closureCells()).at(closureNumber) == numbers.closureCellNumber(unit.function)); AOT_TEST_CHECK(!conventions.genericEntry(unit).constructEntry); bool onlyReturns = bodyFacts.functionsThatOnlyReturn.get(source) && record.kind.isConstructiblePlainFunction(); @@ -653,7 +654,7 @@ static void checkTablesOfVariedFunctions(const TestBuild& build, const TestImage for (unsigned capture = 0; capture < function.captures.size(); ++capture) { StringID variableName = function.captures[capture].variableName; if (variableName && Typed::holdsCells(Typed::closureCaptureRepresentationOf(program, conventions, unit.function, capture))) - namedCaptures.append({ Number24(closureNumber), static_cast(Typed::closureCaptureSlotOf(program, conventions, unit.function, capture)), variableName.index() }); + namedCaptures.append({ Number24(closureNumber), static_cast(Typed::closureCaptureSlotOf(program, conventions, unit.function, capture)), numbers.stringNumbers().requiredNumberOf(variableName) }); } std::ranges::stable_sort(namedCaptures.mutableSpan().subspan(firstOfFunction), { }, &NamedCaptureData::slot); } @@ -662,7 +663,7 @@ static void checkTablesOfVariedFunctions(const TestBuild& build, const TestImage AOT_TEST_CHECK(tables.functionsWithLargeCounts().size() == numberWithLargeCounts); AOT_TEST_CHECK(tables.constructEntrypoints().size() == numberThatOnlyReturn); Vector namedCapturesInTable; - AOT_TEST_CHECK(forEachNamedCapture(tables.namedCaptures(), [&](const NamedCaptureData& row) { + AOT_TEST_CHECK(forEachNamedCapture(decodeVarintBytes(tables.namedCaptures()).span(), [&](const NamedCaptureData& row) { namedCapturesInTable.append(row); })); AOT_TEST_CHECK(equalSpans(asBytes(namedCapturesInTable.span()), asBytes(namedCaptures.span()))); @@ -704,10 +705,13 @@ static Vector encodeTablesOfVariedFunctions(unsigned threadCount) NamesOfTables names(build.program()); nameWhatFunctionTablesName(tableInputs, GatheredMentions { }, names); const ProgramTables& tables = image.image->programTables(); + BitVector numbersOfNames; + for (size_t string : names.strings) + numbersOfNames.set(image.built.numbers->stringNumbers().requiredNumberOf(StringID(static_cast(string)))); for (Number24 name : tables.functionNames()) - AOT_TEST_CHECK(names.strings.get(name)); - AOT_TEST_CHECK(forEachNamedCapture(tables.namedCaptures(), [&](const NamedCaptureData& row) { - AOT_TEST_CHECK(names.strings.get(row.name)); + AOT_TEST_CHECK(numbersOfNames.get(name)); + AOT_TEST_CHECK(forEachNamedCapture(decodeVarintBytes(tables.namedCaptures()).span(), [&](const NamedCaptureData& row) { + AOT_TEST_CHECK(numbersOfNames.get(row.name)); })); Vector bytes(part); @@ -767,6 +771,73 @@ static void checkNamedVariablesAsBytes(const Vector>& modul } } +static void checkImageFunctions(const Vector& numbers, const Vector& sideTables) +{ + Vector bytes = ImageFunctions::encode(sideTables.span(), numbers.span()); + auto words = spanReinterpretCast(bytes.span()); + AOT_TEST_CHECK(ImageFunctions::hasSize(words, numbers.size()) && !ImageFunctions::hasSize(words, numbers.size() + 1)); + ImageFunctions section(words.data(), static_cast(numbers.size())); + for (uint32_t index = 0; index < numbers.size(); ++index) + AOT_TEST_CHECK(section.numberOf(index) == numbers[index] && section.sideTableOf(index) == sideTables[index]); +} + +static void checkImageFunctions(const Vector& numbers) +{ + auto sideTables = [&](const auto& place) { + return Vector(numbers.size(), place); + }; + checkImageFunctions(numbers, sideTables([](size_t index) { return static_cast(1000 * index); })); + checkImageFunctions(numbers, sideTables([](size_t index) { return static_cast(40000 * index); })); + checkImageFunctions(numbers, sideTables([](size_t index) { return static_cast(index % 3 ? 70 : 9 * index); })); +} + +static void testImageFunctions() +{ + checkImageFunctions({ 5 }); + checkImageFunctions(Vector(100, [](size_t index) { return static_cast(index); })); + checkImageFunctions(Vector(100, [](size_t index) { return static_cast(200 - index); })); + checkImageFunctions(Vector(40000, [](size_t index) { return static_cast(index); })); + checkImageFunctions(Vector(97, [](size_t index) { return static_cast(index + index / 5 * 1000); })); +} + +static void checkSourceFunctions(const Vector& functions, size_t maxWords) +{ + Vector words = AscendingNumberTable::create(functions.span()); + AOT_TEST_CHECK(words.size() <= maxWords); + for (uint32_t index = 0; index < functions.size(); ++index) + AOT_TEST_CHECK(AscendingNumberTable(words.span()).at(index) == functions[index]); +} + +static void testSourceFunctions() +{ + checkSourceFunctions({ }, 0); + checkSourceFunctions({ 7 }, 2); + checkSourceFunctions(Vector(40, [](size_t) { return 9u; }), 4 + 2); + checkSourceFunctions(Vector(1000, [](size_t index) { return static_cast(index); }), 64 + 63); + checkSourceFunctions(Vector(1000, [](size_t index) { return static_cast(index / 3 + index / 50 * 40); }), 64 + 66); + checkSourceFunctions(Vector(70, [](size_t index) { return static_cast(index < 40 ? index * 16 : 3000 - index); }), 6 + 70); + checkSourceFunctions(Vector(70, [](size_t index) { return static_cast(index * 17); }), 6 + 70); + checkSourceFunctions(Vector(96, [](size_t index) { return static_cast(index + (index < 50 ? 0 : 1 << 20)); }), 6 + 2 + 32 + 2); +} + +static void checkClosureCells(const Vector& cells, size_t maxWords) +{ + Vector words = SparseNumberTable::create(cells.span()); + AOT_TEST_CHECK(words.size() <= maxWords); + for (uint32_t number = 0; number < cells.size(); ++number) + AOT_TEST_CHECK(SparseNumberTable(words.span()).at(number) == cells[number]); +} + +static void testClosureCells() +{ + checkClosureCells({ }, 0); + checkClosureCells({ 0, 1, 0, 2 }, 2); + checkClosureCells(Vector(100, [](size_t index) { return static_cast(index + 1); }), 8); + checkClosureCells(Vector(100, [](size_t index) { return index % 3 ? 0 : static_cast(index / 3 + 1); }), 8); + checkClosureCells(Vector(100, [](size_t index) { return index % 2 ? 0 : static_cast(101 - index); }), 8 + 50); + checkClosureCells(Vector(64, [](size_t index) { return index < 32 ? 5 + static_cast(index) : static_cast(64 - index); }), 4 + 32); +} + static void testNamedVariablesAsBytes() { auto inStatics = [](uint32_t name, uint32_t offset, SlotRepresentation representation = SlotRepresentation::JSValue) { @@ -825,6 +896,9 @@ std::span imageFunctionTableTests() { "image/functions/name-number-past-fifteen-bits"_s, testNameNumberPastFifteenBits }, { "image/functions/tables-over-several-ranges"_s, testFunctionTablesOverSeveralRanges }, { "image/functions/named-variables-as-bytes"_s, testNamedVariablesAsBytes }, + { "image/functions/numbers-in-groups"_s, testImageFunctions }, + { "image/functions/source-functions-in-groups"_s, testSourceFunctions }, + { "image/functions/closure-cells-in-groups"_s, testClosureCells }, }; return tests; } diff --git a/Source/JavaScriptCore/aot/tests/AOTImageRegExpTableTests.cpp b/Source/JavaScriptCore/aot/tests/AOTImageRegExpTableTests.cpp index 989ca3f2d6942..9daa270c31894 100644 --- a/Source/JavaScriptCore/aot/tests/AOTImageRegExpTableTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTImageRegExpTableTests.cpp @@ -11,6 +11,7 @@ #include "AOTImageTestSupport.h" #include "AOTImageText.h" #include "AOTLimits.h" +#include "AOTNumbering.h" #include "AOTRegExpCompiler.h" #include "AOTRegExpRecords.h" #include "AOTRegExpSummary.h" @@ -84,6 +85,7 @@ class RegExpImage { RegExpRequests requests; CompiledRegExps compiled; RegExpSection section; + StringNumbers strings; RegExpSections sections; AlignedBytes bytes; std::unique_ptr image; @@ -110,6 +112,8 @@ static AlignedBytes imageWithSections(std::initializer_list(index), [](auto& entry) { return entry.first; }); size_t size = content == contents.end() ? 0 : content->second.size(); header.sections[index] = { static_cast(offset), static_cast(size) }; + if (static_cast(index) == ImageSection::Strings) + header.numbers.numberOfStringWords = static_cast(size / sizeof(ImageStringWord)); offset = roundUpToMultipleOf(offset + size); } header.codeOffset = roundUpToMultipleOf(offset); @@ -223,7 +227,11 @@ RegExpImage::RegExpImage(std::span literals, const TestOptions& o requests = RegExpRequests(WTF::move(literalRequests), makeRequests(predictedTexts, RegExpRequestOrigin::Candidate)); if (code == CodeOfRegExpImage::Compiled) compileAndLayOut(); - sections = encodeRegExpRecords({ requests, compiled, section }, context); + BitVector everyString(program->strings.size()); + for (size_t index = 0; index < program->strings.size(); ++index) + everyString.quickSet(index); + strings = StringNumbers(everyString, { }, program->strings.size()); + sections = encodeRegExpRecords({ requests, compiled, section, strings }, context); TextOfRegExpImage text = encodeNamedText(); AOT_TEST_CHECK(!context.diagnostics().hasErrors()); @@ -588,7 +596,7 @@ static void testEncodingIsDeterministic() AOT_TEST_CHECK(first.sections.details == second.sections.details); AOT_TEST_CHECK(first.sections.maxOutputSize == second.sections.maxOutputSize); - RegExpSections again = encodeRegExpRecords({ first.requests, first.compiled, first.section }, first.context); + RegExpSections again = encodeRegExpRecords({ first.requests, first.compiled, first.section, first.strings }, first.context); AOT_TEST_CHECK(again.records == first.sections.records && again.hashes == first.sections.hashes && again.details == first.sections.details); } diff --git a/Source/JavaScriptCore/aot/tests/AOTImageSiteTableTests.cpp b/Source/JavaScriptCore/aot/tests/AOTImageSiteTableTests.cpp index 91d08c4f8d643..b4fcce7c5997f 100644 --- a/Source/JavaScriptCore/aot/tests/AOTImageSiteTableTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTImageSiteTableTests.cpp @@ -8,6 +8,7 @@ #if ENABLE(AOT) #include "AOTImageSiteTestBodies.h" +#include "AOTPositionEncoder.h" #include "AOTStubsExceptions.h" #include "AOTStubsPolls.h" #include "AOTVector.h" @@ -259,13 +260,42 @@ static void testCallSitesPrologueCheckElsewhere() AOT_TEST_CHECK(!table.prologueCheckAt(table.code().data())); } -static void checkPositions(const Image& image, uint32_t imageFunction, const uint8_t* stream, OwnPosition ofFunctionInFront, const Vector& positions) +namespace { + +class CodedPositions { +public: + CodedPositions(const SiteDescription& sites, OwnPosition ofFunctionInFront) + : m_ofFunctionInFront(ofFunctionInFront) + { + PositionEncoder encoder; + encoder.encode(sites, ofFunctionInFront); + PositionCodes codes(encoder.counts()); + m_section = codes.bytes(); + m_place = m_section.size(); + encoder.appendNextCoded(m_section, codes); + } + + std::optional at(const Image& image, uint32_t function, uint32_t site) const + { + const uint8_t* section = m_section.span().data(); + return decodePosition(image, function, section, section + m_place, m_ofFunctionInFront, CallSiteIndex(site)); + } + +private: + OwnPosition m_ofFunctionInFront; + Vector m_section; + size_t m_place; +}; + +} // anonymous namespace + +static void checkPositions(const Image& image, uint32_t imageFunction, const CodedPositions& coded, const Vector& positions) { uint32_t count = static_cast(positions.size()); for (uint32_t k = 0; k < count; ++k) - AOT_TEST_CHECK(decodePosition(image, imageFunction, stream, ofFunctionInFront, CallSiteIndex(k)) == sourcePositionOf(positions[k])); - AOT_TEST_CHECK(!decodePosition(image, imageFunction, stream, ofFunctionInFront, CallSiteIndex(count))); - AOT_TEST_CHECK(!decodePosition(image, imageFunction, stream, ofFunctionInFront, CallSiteIndex(std::numeric_limits::max()))); + AOT_TEST_CHECK(coded.at(image, imageFunction, k) == sourcePositionOf(positions[k])); + AOT_TEST_CHECK(!coded.at(image, imageFunction, count)); + AOT_TEST_CHECK(!coded.at(image, imageFunction, std::numeric_limits::max())); } static void checkEncodedPositions(std::initializer_list> tables) @@ -275,9 +305,7 @@ static void checkEncodedPositions(std::initializer_list> ta SiteDescription sites; sites.positions = positions; for (OwnPosition ofFunctionInFront : { OwnPosition { }, OwnPosition::of(sites), OwnPosition { 0xffffffff, OwnPosition::of(sites).source }, OwnPosition { 7, 0xffffffff }, OwnPosition { 0, 1 } }) { - Vector stream; - encodePositions(stream, sites, nullptr, ofFunctionInFront); - checkPositions(any.table(), 0, stream.span().data(), ofFunctionInFront, positions); + checkPositions(any.table(), 0, CodedPositions(sites, ofFunctionInFront), positions); } } } @@ -313,10 +341,9 @@ static void testPositionsWithConstructionStarts() SiteConstructionStart { 4, 12, 9, true }, SiteConstructionStart { 5, 14, 3, false, true }, }; - Vector stream; - encodePositions(stream, sites, nullptr, OwnPosition { }); + CodedPositions coded(sites, OwnPosition { }); auto decoded = [&](uint32_t site) { - return decodePosition(any.table(), 0, stream.span().data(), OwnPosition { }, CallSiteIndex(site)); + return coded.at(any.table(), 0, site); }; ModuleID module(0u); AOT_TEST_CHECK(decoded(0) == SourcePosition({ { 1, 1 }, module })); @@ -329,6 +356,87 @@ static void testPositionsWithConstructionStarts() AOT_TEST_CHECK(!decoded(7)); } +static Vector positionsOnOneLine(uint32_t count) +{ + Vector positions; + for (uint32_t k = 0; k < count; ++k) + positions.append(positionAt(3, 5 + k)); + return positions; +} + +static void checkPrefixCode(const PrefixCode::Counts& counts, unsigned mostBitsOfShortWords) +{ + PrefixCode code(counts); + Vector bytesOfCode; + unsigned bitsOfShortWords = code.bitsOfShortWords(mostBitsOfShortWords); + code.append(bytesOfCode, bitsOfShortWords); + size_t numberOfBytes = std::ranges::count_if(counts, [](uint32_t count) { return count; }); + AOT_TEST_CHECK(bitsOfShortWords && bitsOfShortWords <= std::min(mostBitsOfShortWords, std::max(numberOfBytes, 1))); + AOT_TEST_CHECK(bytesOfCode.size() == PrefixCode::sizeOfShortWords(bitsOfShortWords) + PrefixCode::maxLength + numberOfBytes); + PrefixCodeView view(bytesOfCode.span().data(), 64 - bitsOfShortWords); + Vector written; + WeakRandom random(1); + for (unsigned byte = 0; byte < counts.size(); ++byte) { + for (unsigned k = 0; counts[byte] && k < 3; ++k) + written.insert(random.getUint32(written.size() + 1), static_cast(byte)); + } + for (size_t size = 0; size <= written.size(); ++size) { + Vector stream(FillWith { }, 2 * size + 1, 0); + PrefixCodeWriter writer(stream.mutableSpan().data()); + for (size_t k = 0; k < size; ++k) + writer.append(code.word(written[k])); + PrefixCodeReader reader(stream.span().data(), writer.finish()); + for (size_t k = 0; k < size; ++k) + AOT_TEST_CHECK(reader.read(view) == written[k]); + AOT_TEST_CHECK(reader.read(view) == PrefixCode::endOfStream); + } +} + +static void checkPrefixCode(const PrefixCode::Counts& counts) +{ + for (unsigned mostBitsOfShortWords : { 1u, PrefixCode::bitsOfShortWord, 10u }) + checkPrefixCode(counts, mostBitsOfShortWords); +} + +static void testPrefixCodes() +{ + PrefixCode::Counts counts { }; + checkPrefixCode(counts); + counts[7] = 1; + checkPrefixCode(counts); + counts[200] = std::numeric_limits::max(); + checkPrefixCode(counts); + counts.fill(1); + checkPrefixCode(counts); + uint32_t last = 1; + uint32_t next = 1; + for (unsigned byte = 0; byte < 40; ++byte) { + counts[byte * 5] = next; + next += std::exchange(last, next); + } + checkPrefixCode(counts); +} + +static void checkVarintBytes(std::span bytes, size_t maxSize) +{ + Vector coded = encodeVarintBytes(bytes); + AOT_TEST_CHECK(coded.size() <= maxSize); + AOT_TEST_CHECK(equalSpans(std::span(decodeVarintBytes(coded.span()).span()), bytes)); +} + +static void testVarintBytes() +{ + checkVarintBytes({ }, 0); + checkVarintBytes(std::array { 1, 2, 3 }, 7); + Vector bytes; + WeakRandom random(1); + for (unsigned count = 0; count < 100000; ++count) + appendVarint(bytes, random.getUint32(random.getUint32(8) ? 20 : 100000)); + size_t size = bytes.size(); + checkVarintBytes(bytes.span(), size); + AOT_TEST_CHECK(encodeVarintBytes(bytes.span()).size() < size); +} + static void testPositionsObfuscated() { SiteBody body = bodyWithInlineFrames(1, 1, chainedFrames, chainedFramesByCall); @@ -368,6 +476,9 @@ std::span siteTableTests() { "image/call-sites/two-prologue-checks"_s, testCallSitesTwoPrologueChecks }, { "image/call-sites/prologue-check-elsewhere"_s, testCallSitesPrologueCheckElsewhere }, { "image/positions/random"_s, [] { checkEncodedPositions({ randomPositions(2000) }); } }, + { "image/positions/of-one-kind"_s, [] { checkEncodedPositions({ positionsOnOneLine(1), positionsOnOneLine(2), positionsOnOneLine(9), positionsOnOneLine(1025) }); } }, + { "image/positions/prefix-codes"_s, testPrefixCodes }, + { "image/positions/varint-bytes"_s, testVarintBytes }, { "image/positions/construction-starts"_s, testPositionsWithConstructionStarts }, { "image/positions/obfuscated"_s, testPositionsObfuscated }, { "image/positions/function-names-kept-without-positions"_s, testFunctionNamesKeptWithoutPositions }, diff --git a/Source/JavaScriptCore/aot/tests/AOTImageTableTests.cpp b/Source/JavaScriptCore/aot/tests/AOTImageTableTests.cpp index 22924396c2d2d..3c6c4e1aefdde 100644 --- a/Source/JavaScriptCore/aot/tests/AOTImageTableTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTImageTableTests.cpp @@ -303,6 +303,23 @@ static void testSectionSizeNotWholeRecords() } } +static void testFunctionStartsInGroups() +{ + for (size_t count : { 0u, 1u, 31u, 32u, 33u, 200u }) { + for (uint32_t distance : { 1u, 2000u, 3000u, 70000u, 0x1000000u }) { + Vector starts; + for (size_t index = 0; index < count; ++index) + starts.append(static_cast(16 + index * distance + (index % 7 == 5 ? distance / 2 : 0))); + Vector bytes = FunctionStarts::encode(starts.span()); + AOT_TEST_CHECK(!(bytes.size() % sizeof(uint32_t)) && bytes.size() / sizeof(uint32_t) >= FunctionStarts::numberOfWordsWithoutLists(count)); + FunctionStarts decoded(reinterpret_cast(bytes.span().data()), count); + for (size_t index = 0; index < count; ++index) + AOT_TEST_CHECK(decoded[index] == starts[index]); + AOT_TEST_CHECK(decoded[count] == std::numeric_limits::max()); + } + } +} + static void testFunctionCount() { checkHeaderChange(Verdict::MalformedTables, [](ImageHeader& header) { header.counts.functions += 1; }); @@ -881,6 +898,7 @@ std::span imageTableTests() { "image/check/section-offset-unaligned"_s, testSectionOffsetUnaligned }, { "image/check/section-size-not-whole-records"_s, testSectionSizeNotWholeRecords }, { "image/check/function-count"_s, testFunctionCount }, + { "image/check/function-starts-in-groups"_s, testFunctionStartsInGroups }, { "image/check/functions-past-code"_s, testFunctionsPastCode }, { "image/check/too-few-granules"_s, testTooFewGranules }, { "image/check/shared-code-ends-past-functions"_s, testSharedCodeEndsPastFunctions }, diff --git a/Source/JavaScriptCore/aot/tests/AOTImageTestBodies.h b/Source/JavaScriptCore/aot/tests/AOTImageTestBodies.h index 2cb1a9d475eb9..1070a979315a5 100644 --- a/Source/JavaScriptCore/aot/tests/AOTImageTestBodies.h +++ b/Source/JavaScriptCore/aot/tests/AOTImageTestBodies.h @@ -139,7 +139,10 @@ inline void checkPartsOfCodeAddUp(TestBuild& build, const TestImage& image) uint64_t sizeOfRegExpCode = build.statistic(StatisticID::RegExpCodeBytesEmitted); AOT_TEST_CHECK(sizeOfRegExpCode <= roomOfRegExpCode && roomOfRegExpCode - sizeOfRegExpCode < RegExpSection::alignment); AOT_TEST_CHECK(build.statistic(StatisticID::RegExp16BitCodeBytes) <= sizeOfRegExpCode); - AOT_TEST_CHECK(WTF::roundUpToMultipleOf(header.executableSize + build.statistic(StatisticID::RegExpTableBytesEmitted)) == ranges.constantsOffset); + if (ranges.constantsOffset < header.executableSize) + AOT_TEST_CHECK(!roomOfRegExpCode && !ranges.ffiCodeSize && ranges.constantsOffset + WTF::roundUpToMultipleOf(ranges.constantsSize) == header.executableSize && header.codeSize == header.executableSize); + else + AOT_TEST_CHECK(WTF::roundUpToMultipleOf(header.executableSize + build.statistic(StatisticID::RegExpTableBytesEmitted)) == ranges.constantsOffset); AOT_TEST_CHECK(build.statistic(StatisticID::WasmCodeBytesEmitted) == ranges.webAssemblyCodeSize); AOT_TEST_CHECK(build.statistic(StatisticID::FFICodeBytesEmitted) == ranges.ffiCodeSize); AOT_TEST_CHECK(build.statistic(StatisticID::ConstantBytesEmitted) == ranges.constantsSize); diff --git a/Source/JavaScriptCore/aot/tests/AOTImageTextTests.cpp b/Source/JavaScriptCore/aot/tests/AOTImageTextTests.cpp index 33ffb63ef635a..1822cf22abdd0 100644 --- a/Source/JavaScriptCore/aot/tests/AOTImageTextTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTImageTextTests.cpp @@ -7,6 +7,7 @@ #if ENABLE(AOT) +#include "AOTCodedTexts.h" #include "AOTIRProgram.h" #include "AOTIdentifierNumbering.h" #include "AOTImageTestBodies.h" @@ -80,7 +81,15 @@ struct TextImage { CompiledCode bodyOfReader(); const ImageText& text() const { return image.image->text(); } const ImageNumbers& numbers() const { return *image.built.numbers; } - ImageStringWord wordOf(StringID id) const { return image.image->section(ImageSection::Strings)[id.index()]; } + StringID numberOf(StringID id) const { return StringID(numbers().stringNumbers().requiredNumberOf(id)); } + ImageStringWord wordOf(StringID id) const + { + if (numbers().stringNumbers().numberOf(id)) + return text().wordOf(numberOf(id)); + std::optional constant = numbers().stringConstants().numberOf(id); + AOT_TEST_CHECK(constant); + return text().stringConstant(*constant); + } TestBuild build; TestImage image; @@ -253,18 +262,19 @@ static void testIdentifiers() AOT_TEST_CHECK(number); AOT_TEST_CHECK(number == numbering.numberOf(image.keyOf(nameOfFreeVariable))); AOT_TEST_CHECK(image.wordOf(name).hasIdentifierNumber() && image.wordOf(name).number() == *number); - AOT_TEST_CHECK(text.isAtom(name)); + AOT_TEST_CHECK(text.isAtom(image.numberOf(name))); const ImageIdentifierWord& identifier = text.identifier(*number); - AOT_TEST_CHECK(identifier.text() == name); + AOT_TEST_CHECK(identifier.text() == image.numberOf(name)); AOT_TEST_CHECK(identifier.kind() == PropertyKeyKind::Text); AOT_TEST_CHECK(identifier.nameClass() == NameClass::Ordinary); - AOT_TEST_CHECK(text.hash(name) == String(nameOfFreeVariable).impl()->hash()); - AOT_TEST_CHECK(text.view(name) == nameOfFreeVariable); + AOT_TEST_CHECK(text.hash(image.wordOf(name)) == String(nameOfFreeVariable).impl()->hash()); + AOT_TEST_CHECK(text.view(image.wordOf(name)) == nameOfFreeVariable); AOT_TEST_CHECK(!image.wordOf(image.idOf("plain text"_s)).hasIdentifierNumber()); for (uint32_t other = 1; other < text.numberOfIdentifiers(); ++other) { const NumberedIdentifier& expected = numbering.at(other); - AOT_TEST_CHECK(text.identifier(other).text() == expected.text); + AOT_TEST_CHECK(text.identifier(other).text() == (expected.text ? image.numberOf(expected.text) : StringID())); + AOT_TEST_CHECK(text.identifier(other).kind() != PropertyKeyKind::Text || text.identifier(other).text() == StringID(other)); if (expected.key) AOT_TEST_CHECK(text.identifier(other).kind() == image.build.program().at(expected.key).kind); } @@ -289,27 +299,32 @@ static void testStringOnlyShownHasNoHeader() const ImageText& text = image.text(); unsigned hash = String(pathOfModule).impl()->hash(); - AOT_TEST_CHECK(text.hasText(path) && text.view(path) == pathOfModule && text.hash(path) == hash); - String created = text.createString(path); + ImageStringWord wordOfPath = image.wordOf(path); + AOT_TEST_CHECK(text.hasText(image.numberOf(path)) && text.view(image.numberOf(path)) == pathOfModule && text.hash(wordOfPath) == hash); + String created = text.createString(wordOfPath); AOT_TEST_CHECK(created == pathOfModule && created.impl()->hash() == hash); - AOT_TEST_CHECK(text.createString(path).impl() != created.impl()); + AOT_TEST_CHECK(text.createString(wordOfPath).impl() != created.impl()); StringID name = image.idOf(nameOfFreeVariable); - AOT_TEST_CHECK(text.createString(name).impl() == text.createString(name).impl()); - AOT_TEST_CHECK(text.createString(readByCode) == textReadThroughHeader && text.createString(readByCode).impl() == text.createString(readByCode).impl()); - AOT_TEST_CHECK(text.hasListedHash(constant) && !text.hasListedHash(path) && !text.hasListedHash(name) && !text.hasListedHash(readByCode)); - AOT_TEST_CHECK(text.view(constant) == textOfConstant && text.hash(constant) == String(textOfConstant).impl()->hash()); - AOT_TEST_CHECK(text.createString(constant) == textOfConstant && text.createString(constant).impl() != text.createString(constant).impl()); - StringID publicPath = image.idOf("main.js"_s); - AOT_TEST_CHECK(text.createString(publicPath) == "main.js"_s && text.createString(publicPath).impl() != text.createString(publicPath).impl()); + ImageStringWord wordOfName = image.wordOf(name); + AOT_TEST_CHECK(text.createString(wordOfName).impl() == text.createString(wordOfName).impl()); + ImageStringWord headerWord = image.wordOf(readByCode); + AOT_TEST_CHECK(text.createString(headerWord) == textReadThroughHeader && text.createString(headerWord).impl() == text.createString(headerWord).impl()); + ImageStringWord wordOfConstant = image.wordOf(constant); + AOT_TEST_CHECK(!numbers.stringNumbers().numberOf(constant)); + AOT_TEST_CHECK(wordOfConstant.hasListedHash() && !wordOfPath.hasListedHash() && !wordOfName.hasListedHash() && !headerWord.hasListedHash()); + AOT_TEST_CHECK(text.view(wordOfConstant) == textOfConstant && text.hash(wordOfConstant) == String(textOfConstant).impl()->hash()); + AOT_TEST_CHECK(text.createString(wordOfConstant) == textOfConstant && text.createString(wordOfConstant).impl() != text.createString(wordOfConstant).impl()); + ImageStringWord wordOfPublicPath = image.wordOf(image.idOf("main.js"_s)); + AOT_TEST_CHECK(text.createString(wordOfPublicPath) == "main.js"_s && text.createString(wordOfPublicPath).impl() != text.createString(wordOfPublicPath).impl()); AOT_TEST_CHECK(image.image.image->containsCharactersOf(*created.impl())); - String atom = text.createAtom(path); + String atom = text.createAtom(wordOfPath); AOT_TEST_CHECK(atom == pathOfModule && atom.impl()->isAtom()); } static std::span bytesOfTextInImage(StringView text) { TextImage& image = textImage(); - StringView view = image.text().view(image.idOf(text)); + StringView view = image.text().view(image.wordOf(image.idOf(text))); AOT_TEST_CHECK(view == text); return view.is8Bit() ? asBytes(view.span8()) : asBytes(view.span16()); } @@ -359,12 +374,27 @@ static Vector stringsByReaderOfRuntime() { const ImageText& text = textImage().text(); Vector strings; - for (uint32_t index = 0; index < text.numberOfStrings(); ++index) { - StringID string(index); - if (!text.hasText(string) || text.view(string).isEmpty()) - continue; - strings.append(byteCast(text.view(string).utf8().span())); + HashSet bitsOfWords; + auto append = [&](ImageStringWord word) { + if (!bitsOfWords.add(word.bits).isNewEntry || text.view(word).isEmpty()) + return; + strings.append(byteCast(text.view(word).utf8().span())); strings.append(0); + }; + uint32_t numberOfWords = text.endOfStringsWithWords(); + for (uint32_t number = 0; number < numberOfWords; ++number) { + ImageStringWord word = text.wordOf(StringID(number)); + AOT_TEST_CHECK(!word.isCoded() && (word.hasText() || number < text.numberOfIdentifiers())); + if (word.hasText()) + append(word); + } + for (uint32_t number = 0; number < text.numberOfStringConstants(); ++number) { + if (text.stringConstant(number).hasText()) + append(text.stringConstant(number)); + } + for (uint32_t number = numberOfWords; number < text.numberOfStrings(); ++number) { + AOT_TEST_CHECK(text.hasText(StringID(number)) && text.wordOf(StringID(number)).isCoded()); + append(text.wordOf(StringID(number))); } return strings; } @@ -440,6 +470,34 @@ static void testStoredStringsOfAnotherFormatAreRefusedByName() AOT_TEST_CHECK(ofNoImage.verdict == VerdictOnStoredStrings::CannotBeRead && reportOf(ofNoImage) == "the file holds no image\n"_s); } +static void checkCodedTexts(Vector&& texts) +{ + std::ranges::sort(texts, [](const String& a, const String& b) { + return std::ranges::lexicographical_compare(a.span8(), b.span8()); + }); + Vector> bytes = texts.map([](const String& text) { + return byteCast(text.span8()); + }); + Vector encoded = CodedTexts::encode(bytes.span(), textImage().build.context().parallelLoops()); + AOT_TEST_CHECK(!(encoded.size() % sizeof(uint32_t))); + CodedTexts coded(encoded.span(), static_cast(texts.size())); + AOT_TEST_CHECK(coded.count() == texts.size()); + for (uint32_t number = texts.size(); number--;) + AOT_TEST_CHECK(equalSpans(std::span(coded.at(number).span()), bytes[number])); +} + +static void testCodedTexts() +{ + checkCodedTexts({ }); + checkCodedTexts({ "a"_s }); + checkCodedTexts({ "options.first"_s, "options.second"_s, "options"_s, "options.first.name"_s }); + for (unsigned count : { 15u, 16u, 17u, 100u }) { + checkCodedTexts(Vector(count, [](size_t index) { return makeString("this.member"_s, index, "(value)"_s); })); + checkCodedTexts(Vector(count, [](size_t index) { return repeatedText('x', index + 250); })); + checkCodedTexts(Vector(count, [](size_t index) { return String(std::span(std::array { static_cast(1 + index), '~' })); })); + } +} + static void testTextsThatAddToOneAnotherLieAtFrontOfLongest() { String first = makeString(piecesOfMadeTexts[0], piecesOfMadeTexts[1]); @@ -450,7 +508,7 @@ static void testTextsThatAddToOneAnotherLieAtFrontOfLongest() TextImage& image = textImage(); for (const String& text : { first, second, third }) { AOT_TEST_CHECK(image.numbers().stringConstants().numberOf(image.idOf(text))); - AOT_TEST_CHECK(image.text().hash(image.idOf(text)) == freshCopyOf(text).impl()->hash()); + AOT_TEST_CHECK(image.text().hash(image.wordOf(image.idOf(text))) == freshCopyOf(text).impl()->hash()); } } @@ -565,9 +623,9 @@ static void testImageHoldsNoAddress() AOT_TEST_CHECK(bytesOfCopy.data() != bytes.data()); std::unique_ptr imageOfCopy = Image::create(bytesOfCopy, bytesOfCopy.data() + image.image.image->header().codeOffset); for (const ExpectedText& expected : expectedTexts()) { - StringID id = image.idOf(expected.text); - String created = imageOfCopy->text().createString(id); - AOT_TEST_CHECK(created == expected.text && imageOfCopy->text().view(id) == expected.text && imageOfCopy->text().hash(id) == image.text().hash(id)); + ImageStringWord word = image.wordOf(image.idOf(expected.text)); + String created = imageOfCopy->text().createString(word); + AOT_TEST_CHECK(created == expected.text && imageOfCopy->text().view(word) == expected.text && imageOfCopy->text().hash(word) == image.text().hash(word)); if (!expected.text.isEmpty()) AOT_TEST_CHECK(imageOfCopy->containsCharactersOf(*created.impl()) && !image.image.image->containsCharactersOf(*created.impl())); } @@ -579,6 +637,7 @@ std::span imageTextTests() { static constexpr NativeTest tests[] = { { "image/text/identifiers"_s, testIdentifiers }, + { "image/text/coded-texts"_s, testCodedTexts }, #if ENABLE(AOT_POSITION_INDEPENDENT_IMAGE) { "image/text/headers-are-created-at-first-use"_s, testHeadersAreCreatedAtFirstUse }, { "image/text/image-holds-no-address"_s, testImageHoldsNoAddress }, diff --git a/Source/JavaScriptCore/aot/tests/AOTMachineAllocationTests.cpp b/Source/JavaScriptCore/aot/tests/AOTMachineAllocationTests.cpp index d193cda13884d..55af20132f048 100644 --- a/Source/JavaScriptCore/aot/tests/AOTMachineAllocationTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTMachineAllocationTests.cpp @@ -3620,7 +3620,7 @@ static void allocatorExtendsCountOfLoop() out.setOrigin(allocationTestOriginOfExtension); LValue extended = out.zeroExtPtr(count); out.setOrigin(Origin()); - LValue nextSum = out.add(sum, extended); + LValue nextSum = out.add(sum, out.opaque(extended)); out.upsilon(nextCount, count); out.upsilon(nextSum, sum); out.branch(out.notEqual(nextCount, end), normalEdge(loop), normalEdge(behindLoop)); diff --git a/Source/JavaScriptCore/aot/tests/AOTMachineOperationTests.cpp b/Source/JavaScriptCore/aot/tests/AOTMachineOperationTests.cpp index eff52bc05cef9..7697c3b752b4c 100644 --- a/Source/JavaScriptCore/aot/tests/AOTMachineOperationTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTMachineOperationTests.cpp @@ -834,6 +834,64 @@ void checkComparisonsOf() } } +template +Vector constantsOfComparisonTest() +{ + using U = std::make_unsigned_t; + constexpr T least = std::numeric_limits::min(); + constexpr T greatest = std::numeric_limits::max(); + Vector result; + for (U distance : { U(0), U(1), U(2), U(3), U(5), U(4094), U(4095), U(4096), U(4097), U(0xffe000), U(0xfff000), U(0xfff001), U(0x1000000) }) { + result.append(static_cast(static_cast(least) + distance)); + result.append(static_cast(static_cast(greatest) - distance)); + } + for (unsigned bit = 0; bit < 8 * sizeof(T); ++bit) { + for (int distance : { -1, 0, 1 }) + result.append(static_cast((U(1) << bit) + static_cast(distance))); + result.append(static_cast(U(0) - (U(1) << bit))); + } + return result; +} + +template +void checkComparisonsWithConstantsOf() +{ + Vector constants = constantsOfComparisonTest(); + Vector values; + for (T constant : constants) { + for (int distance : { -1, 0, 1 }) + values.append(static_cast(constant) + static_cast(distance)); + } + for (const OperationTestComparison& row : operationTestComparisonsOf()) { + for (T constant : constants) { + OperationTestProcedure test; + Output& out = test.out(); + LValue argument = test.argumentGPR(0); + LValue left = sizeof(T) == sizeof(int32_t) ? out.castToInt32(argument) : argument; + auto compare = [&] { + LValue comparison = out.binary(row.opcode, out.opaque(left), sizeof(T) == sizeof(int32_t) ? out.constInt32(static_cast(constant)) : out.constInt64(static_cast(constant))); + return comparison->isConstant() ? out.opaque(comparison) : comparison; + }; + LValue result = compare(); + result = out.bitOr(result, out.shl(appendBranchOfOperationTest(out, operationTestUses[1], compare()), 1)); + result = out.bitOr(result, out.shl(out.select(compare(), out.opaque(out.constInt32(1)), out.opaque(out.constInt32(0))), 2)); + out.ret(out.zeroExt(result, Int64)); + compileOrFail(test); + for (uint64_t value : values) { + uint64_t answer = test.call(operationTestWithDirtAbove(value, sizeof(T))); + uint64_t expected = row.reference(static_cast(value), constant) ? 7 : 0; + checkOperationTest(answer == expected, test, descriptionOf(row.opcode).name, "(", RawHex(value), ", ", RawHex(static_cast(constant)), ") of ", sizeof(T), " bytes gave ", answer); + } + } + } +} + +static void testMachineOperationsThatCompareWithConstants() +{ + checkComparisonsWithConstantsOf(); + checkComparisonsWithConstantsOf(); +} + template void checkBitTestsOf() { @@ -1808,7 +1866,7 @@ static void testMachineOperationsOnLowHalfOfSumThatIsExtendedWhereTestFailed() out.appendTo(passes); LValue extendedWhereItPassed = out.zeroExtPtr(out.castToInt32(sum)); - out.ret(out.add(extendedWhereItPassed, out.constInt64(static_cast(tag)))); + out.ret(out.add(out.opaque(extendedWhereItPassed), out.constInt64(static_cast(tag)))); out.appendTo(fails); LValue wholeAsNumber = out.bitCast(out.intToDouble(sum), Int64); @@ -1818,7 +1876,7 @@ static void testMachineOperationsOnLowHalfOfSumThatIsExtendedWhereTestFailed() out.appendTo(fits); if (extendsInBlockBehind) extendedWhereItFailed = out.zeroExtPtr(low); - out.ret(out.add(extendedWhereItFailed, out.constInt64(static_cast(tag)))); + out.ret(out.add(out.opaque(extendedWhereItFailed), out.constInt64(static_cast(tag)))); out.appendTo(doesNotFit); out.ret(wholeAsNumber); compileOrFail(test); @@ -1901,6 +1959,7 @@ static std::span listOfMachineOperationTests() { "machine/operations/numbers/unary"_s, runOperationTestWhereCodeRuns }, { "machine/operations/numbers/what-keeps-the-rest-of-its-destination-does-not-wait-for-an-earlier-block"_s, runOperationTestWhereCodeRuns }, { "machine/operations/comparisons/int32"_s, runOperationTestWhereCodeRuns> }, + { "machine/operations/comparisons/constants"_s, runOperationTestWhereCodeRuns }, { "machine/operations/comparisons/double"_s, runOperationTestWhereCodeRuns> }, { "machine/operations/comparisons/bit-tests"_s, runOperationTestWhereCodeRuns }, { "machine/operations/switch"_s, runOperationTestWhereCodeRuns }, diff --git a/Source/JavaScriptCore/aot/tests/AOTMachineRandomProcedure.cpp b/Source/JavaScriptCore/aot/tests/AOTMachineRandomProcedure.cpp index ca93c866aa8a1..86baf2d0a0692 100644 --- a/Source/JavaScriptCore/aot/tests/AOTMachineRandomProcedure.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTMachineRandomProcedure.cpp @@ -35,7 +35,7 @@ struct RandomProgram { enum class Kind : uint8_t { Compute, Copy, Swap, Load, Store, Patch, Check, Split, If, Loop, Break, Continue, Switch, Try, Return }; enum class PatchEffects : uint8_t { Call, WritesLocalState, ReadsEverything, Nothing }; - enum Flag : uint8_t { + enum Flag : uint16_t { UsesImmediate = 1 << 0, ComparesWithImmediate = 1 << 1, TestsBits = 1 << 2, @@ -44,6 +44,8 @@ struct RandomProgram { KeepsScope = 1 << 5, ReturnsEverything = 1 << 6, ScalesSecondOperand = 1 << 7, + CombinesOperands = 1 << 8, + CombinesOnLeft = 1 << 9, }; struct Statement { @@ -51,7 +53,7 @@ struct RandomProgram { Opcode opcode { Nop }; Opcode relation { Equal }; Opcode secondRelation { Equal }; - uint8_t flags { 0 }; + uint16_t flags { 0 }; uint8_t destination { 0 }; std::array operands { 0, 0 }; std::array compared { 0, 0 }; @@ -1088,6 +1090,49 @@ class RandomProgramGenerator { s.relation = relationOf(Int32); } + void setCombinedOperands(Statement& s, TypeKind type) + { + static constexpr std::array readersOfShifts { Add, Sub, BitAnd, BitOr, BitXor }; + static constexpr std::array shifts { Shl, SShr, ZShr }; + s.flags |= RandomProgram::CombinesOperands; + s.operands[0] = variableOf(type); + s.operands[1] = variableOf(type); + if (chance(50)) + s.flags |= RandomProgram::CombinesOnLeft; + if (type == Int64 && m_count[Int32] && chance(35)) { + s.operands[1] = variableOf(Int32); + s.secondRelation = chance(50) ? ZExt32 : SExt32; + s.opcode = chance(50) ? Select : chance(50) ? Add : Sub; + s.relation = relationOf(Int64); + s.compared[0] = variableOf(Int64); + return; + } + unsigned roll = pick(100); + if (roll < 25) { + s.opcode = chance(50) ? Add : Sub; + s.secondRelation = Mul; + s.compared[0] = variableOf(type); + return; + } + + s.secondRelation = shifts[pick(shifts.size())]; + s.flags |= RandomProgram::ComparesWithImmediate; + s.comparedImmediate = pick(type == Int32 ? 34 : 66); + if (roll < 60) { + s.opcode = readersOfShifts[pick(readersOfShifts.size())]; + return; + } + s.flags |= RandomProgram::UsesImmediate | RandomProgram::CombinesOnLeft; + if (roll < 80) { + s.opcode = BitAnd; + s.immediate = canonicalRandomBits(type, (uint64_t { 1 } << (1 + pick(62))) - (chance(90) ? 1 : 3)); + return; + } + s.secondRelation = Shl; + s.opcode = chance(50) ? SShr : ZShr; + s.immediate = pick(type == Int32 ? 34 : 66); + } + void setDivision(Statement& s, TypeKind type) { static constexpr std::array operations { Div, UDiv, Mod, UMod }; @@ -1124,7 +1169,9 @@ class RandomProgramGenerator { } switch (type) { case Int64: - if (m_options.withScaledSums && chance(30)) + if (m_options.withCombinedOperands && chance(40)) + setCombinedOperands(s, Int64); + else if (m_options.withScaledSums && chance(30)) setScaledSum(s); else if (roll < 55) setBinary(s, integerOperations[pick(integerOperations.size())], Int64); @@ -1142,7 +1189,9 @@ class RandomProgramGenerator { setUnary(s, BitwiseCast, Double); return s; case Int32: - if (roll < 40) + if (m_options.withCombinedOperands && chance(40)) + setCombinedOperands(s, Int32); + else if (roll < 40) setBinary(s, integerOperations[pick(integerOperations.size())], Int32); else if (roll < 50) setShift(s, Int32); @@ -1747,6 +1796,22 @@ class RandomProgramEmitter { LValue sum = m_out.binary(Add, m_current[s.operands[0]], m_out.binary(Shl, index, constant(Int32, s.immediate))); return s.opcode == Select ? m_out.select(conditionThatMayFold(s), sum, index) : sum; } + if (s.flags & RandomProgram::CombinesOperands) { + bool combinesOnLeft = s.flags & RandomProgram::CombinesOnLeft; + if (s.secondRelation == ZExt32 || s.secondRelation == SExt32) { + LValue extension = m_out.unary(s.secondRelation, m_current[s.operands[1]]); + LValue left = combinesOnLeft ? extension : m_current[s.operands[0]]; + LValue right = combinesOnLeft ? m_current[s.operands[0]] : extension; + if (s.opcode == Select) + return m_out.select(m_out.binary(s.relation, left, right), m_current[s.operands[0]], m_current[s.compared[0]]); + return m_out.binary(s.opcode, left, right); + } + auto constantFor = [&](Opcode opcode, uint64_t bits) { return constant(isRandomShift(opcode) ? Int32 : result, bits); }; + LValue rightOfInner = s.flags & RandomProgram::ComparesWithImmediate ? constantFor(s.secondRelation, s.comparedImmediate) : m_current[s.compared[0]]; + LValue inner = m_out.binary(s.secondRelation, m_current[s.operands[1]], rightOfInner); + LValue other = usesImmediate ? constantFor(s.opcode, s.immediate) : m_current[s.operands[0]]; + return combinesOnLeft ? m_out.binary(s.opcode, inner, other) : m_out.binary(s.opcode, other, inner); + } if (s.opcode == Select) return m_out.select(conditionThatMayFold(s), m_current[s.operands[0]], m_current[s.operands[1]]); if (s.opcode == Opaque && usesImmediate) @@ -2317,6 +2382,23 @@ class RandomProgramInterpreter { return index; return m_values[s.operands[0]] + (index << (s.immediate & 63)); } + if (s.flags & RandomProgram::CombinesOperands) { + TypeKind type = typeOf(s.destination); + bool combinesOnLeft = s.flags & RandomProgram::CombinesOnLeft; + if (s.secondRelation == ZExt32 || s.secondRelation == SExt32) { + uint32_t word = static_cast(m_values[s.operands[1]]); + uint64_t extension = s.secondRelation == ZExt32 ? word : static_cast(static_cast(static_cast(word))); + uint64_t left = combinesOnLeft ? extension : m_values[s.operands[0]]; + uint64_t right = combinesOnLeft ? m_values[s.operands[0]] : extension; + if (s.opcode == Select) + return m_values[evaluateRandomOperation(s.relation, Int64, left, right) ? s.operands[0] : s.compared[0]]; + return evaluateRandomOperation(s.opcode, Int64, left, right); + } + uint64_t rightOfInner = s.flags & RandomProgram::ComparesWithImmediate ? s.comparedImmediate : m_values[s.compared[0]]; + uint64_t inner = canonicalRandomBits(type, evaluateRandomOperation(s.secondRelation, type, m_values[s.operands[1]], rightOfInner)); + uint64_t other = usesImmediate ? s.immediate : m_values[s.operands[0]]; + return canonicalRandomBits(type, evaluateRandomOperation(s.opcode, type, combinesOnLeft ? inner : other, combinesOnLeft ? other : inner)); + } if (s.opcode == Select) return m_values[s.operands[condition(s) ? 0 : 1]]; if (s.opcode == Opaque && usesImmediate) diff --git a/Source/JavaScriptCore/aot/tests/AOTMachineRandomTests.cpp b/Source/JavaScriptCore/aot/tests/AOTMachineRandomTests.cpp index c4276b50888b8..fdd90dc51779b 100644 --- a/Source/JavaScriptCore/aot/tests/AOTMachineRandomTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTMachineRandomTests.cpp @@ -84,6 +84,7 @@ struct RandomTestCensus { unsigned withNumbers { 0 }; unsigned withSeveralAnswers { 0 }; unsigned withScaledSumsInOneInstruction { 0 }; + unsigned withCombinedOperands { 0 }; unsigned withSelectsThatTestTheirArm { 0 }; }; @@ -92,11 +93,13 @@ static void countScaledSumsOfRandomTest(const Procedure& procedure, RandomTestCe const Liveness& liveness = procedure.liveness(); bool addsInOneInstruction = false; bool testsArm = false; + bool combinesOperands = false; for (uint32_t index = 0; index < procedure.numberOfOperations(); ++index) { const Operation* operation = procedure.operationAt(index); if (operation->hasNoPosition() || liveness.of(operation).has(ValueFlag::IsDropped)) continue; addsInOneInstruction |= addsScaledIndex(liveness, operation); + combinesOperands |= liveness.isFused(operation) && (operation->opcode() == Mul || operation->opcode() == ZShr || operation->opcode() == SShr || operation->opcode() == SExt32); if (operation->opcode() != Select || liveness.isFused(operation)) continue; forEachLeaf(liveness, operation, [&](const Operation* user, unsigned childIndex, const Operation* leaf) { @@ -105,23 +108,28 @@ static void countScaledSumsOfRandomTest(const Procedure& procedure, RandomTestCe } census.withScaledSumsInOneInstruction += addsInOneInstruction; census.withSelectsThatTestTheirArm += testsArm; + census.withCombinedOperands += combinesOperands; } static void placePinnedConstantsForRandomTest(const Procedure& procedure, TestInstance& instance) { - for (uint32_t index = 0; index < procedure.numberOfOperations(); ++index) { - const Operation* operation = procedure.operationAt(index); - if (!operation->isConstant()) - continue; - uint64_t bits = static_cast(operation->constantBits()); + auto place = [&](uint64_t bits) { std::optional pinned = findPinnedConstant(bits); if (!pinned || !pinned->offset) - continue; + return; int32_t offset = *pinned->offset; AOT_TEST_CHECK(pinned->reg == instanceGPR); AOT_TEST_CHECK(offset >= -static_cast(sizeof(instance.wordsBelow))); AOT_TEST_CHECK(offset + static_cast(sizeof(bits)) <= static_cast(sizeof(instance.words))); std::memcpy(static_cast(instance.pointer()) + offset, &bits, sizeof(bits)); + }; + constexpr uint64_t signBit = std::bit_cast(-0.0); + for (uint32_t index = 0; index < procedure.numberOfOperations(); ++index) { + const Operation* operation = procedure.operationAt(index); + if (operation->isConstant()) + place(static_cast(operation->constantBits())); + else if (operation->opcode() == Neg || operation->opcode() == Abs) + place(operation->opcode() == Neg ? signBit : ~signBit); } } @@ -172,6 +180,7 @@ static void runRandomTestCases(RandomTestCase testCase) AOT_TEST_CHECK(census.withRareEdges || !(options.withRareRegions || options.withLandingPads)); AOT_TEST_CHECK(!!census.withNumbers == options.withNumbers); AOT_TEST_CHECK(2 * census.withSeveralAnswers > numberOfSeedsInRandomTest); + AOT_TEST_CHECK(!isARM64() || !options.withCombinedOperands || 2 * census.withCombinedOperands > numberOfSeedsInRandomTest); if (options.withScaledSums) { AOT_TEST_CHECK(4 * census.withScaledSumsInOneInstruction > numberOfSeedsInRandomTest); AOT_TEST_CHECK(10 * census.withSelectsThatTestTheirArm > numberOfSeedsInRandomTest); @@ -240,6 +249,13 @@ static void testRandomProceduresWithScaledSums() runRandomTestCases({ .options = { .seed = 14001, .withScaledSums = true }, .savesFrameOnSuspend = true }); } +static void testRandomProceduresWithCombinedOperands() +{ + runRandomTestCases({ .options = { .seed = 15001, .withCombinedOperands = true } }); + runRandomTestCases({ .options = { .seed = 16001, .withCalls = false, .withLandingPads = false, .withNumbers = false, .withScaledSums = true, .withCombinedOperands = true } }); + runRandomTestCases({ .options = { .seed = 17001, .withCombinedOperands = true }, .savesFrameOnSuspend = true }); +} + static constexpr ASCIILiteral stepsOfRandomTestBuild[] = { "operations"_s, "order"_s, "liveness"_s, "allocation"_s, "frame"_s }; struct RandomTestBuild { @@ -359,6 +375,7 @@ std::span machineRandomTests() { "machine/random/functions-that-save-their-frame"_s, Machine::testRandomProceduresThatSaveTheirFrame }, { "machine/random/features-of-the-host"_s, Machine::testRandomProceduresWithFeaturesOfHost }, { "machine/random/scaled-sums"_s, Machine::testRandomProceduresWithScaledSums }, + { "machine/random/combined-operands"_s, Machine::testRandomProceduresWithCombinedOperands }, { "machine/random/short-branch-reach"_s, Machine::testRandomProceduresWithShortBranchReach }, { "machine/random/same-seed-same-bytes"_s, Machine::testRandomProceduresAreBuiltTheSameWayTwice }, { "machine/random/twice-the-operations-cost-twice-the-arena"_s, Machine::testRandomProceduresCostArenaBytesByTheOperation }, diff --git a/Source/JavaScriptCore/aot/tests/AOTModuleProgramTests.cpp b/Source/JavaScriptCore/aot/tests/AOTModuleProgramTests.cpp index 998f762317b58..29027582cfb85 100644 --- a/Source/JavaScriptCore/aot/tests/AOTModuleProgramTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTModuleProgramTests.cpp @@ -1157,7 +1157,7 @@ static uint32_t operationsInRowsOfStarts(const String& programText) std::span bytesOfTables = sectionOfImageWithRows(image, ImageSection::ProgramTables); AOT_TEST_CHECK(bytesOfTables.size() >= sizeof(ProgramTables)); const ProgramTables& tables = spanReinterpretCast(bytesOfTables.first(sizeof(ProgramTables)))[0]; - size_t numberOfFunctions = sectionOfImageWithRows(image, ImageSection::FunctionStarts).size() / sizeof(uint32_t); + size_t numberOfFunctions = spanReinterpretCast(image.first(sizeof(ImageHeader)))[0].counts.functions; std::span starts = tables.moduleStarts(); std::span steps = tables.moduleStartSteps(); AOT_TEST_CHECK(!starts.empty() && !steps.empty() && steps[0].operation() == Operation::Trap); diff --git a/Source/JavaScriptCore/aot/tests/AOTProgramTestCommands.cpp b/Source/JavaScriptCore/aot/tests/AOTProgramTestCommands.cpp index f1bd47b8bf1cb..f1aad42057cc2 100644 --- a/Source/JavaScriptCore/aot/tests/AOTProgramTestCommands.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTProgramTestCommands.cpp @@ -10,16 +10,19 @@ #include "AOTBodiesThatCodeDependsOn.h" #include "AOTChildProcess.h" +#include "AOTCodedTexts.h" #include "AOTCompileUnit.h" #include "AOTDriver.h" #include "AOTIRPrinter.h" #include "AOTImageFlagNames.h" #include "AOTImageFormat.h" #include "AOTImageText.h" +#include "AOTPrefixCode.h" #include "AOTProgramFile.h" #include "AOTProgramTables.h" #include "AOTSchemaEnums.h" #include "AOTShellProcess.h" +#include "AOTSideTableCodes.h" #include "AOTStages.h" #include "AOTTemporaryFile.h" #include "AOTTestRunner.h" @@ -312,9 +315,9 @@ static void printBytesOfImageSection(PrintStream& out, std::span } } -static void printImageFunctionRecord(PrintStream& out, const ImageFunction& function) +static void printImageFunctionRecord(PrintStream& out, uint32_t number, ImageFunction function) { - out.print("f", function.function(), " sideTables=", function.sideTables, " flags=["); + out.print("f", number, " sideTables=", function.sideTables, " flags=["); ASCIILiteral separator = ""_s; for (const NamedImageFunctionFlag& named : namedImageFunctionFlags) { if (!function.has(named.flag)) @@ -351,11 +354,10 @@ static void printImageStringWord(PrintStream& out, const ImageStringWord& word) out.print(" with listed hash"); } -static void printImageIdentifierWord(PrintStream& out, const ImageIdentifierWord& word) +static void printImageIdentifierByte(PrintStream& out, uint8_t byte) { + ImageIdentifierWord word = ImageIdentifierWord::ofByteInImage(byte); out.print(nameOf(word.kind())); - if (StringID text = word.text()) - out.print(" text=", text.index()); if (word.kind() == PropertyKeyKind::WellKnownSymbol) out.print(" symbol=", nameOf(word.wellKnownSymbol())); else @@ -446,7 +448,7 @@ static void printImageProgramTables(PrintStream& out, const TokensOfRuntimeFunct out.println(" compiledRuntimeFunction ", tokenOf(tokens, RuntimeFunctionID(row.id)), " function=f", row.function); printStartsOfModulesInImage(out, tables); size_t modulesWithNamedVariables = 0; - forEachNamedVariable(tables.namedVariables(), [&](uint32_t module, size_t numberOfVariables) { + forEachNamedVariable(decodeVarintBytes(tables.namedVariables()).span(), [&](uint32_t module, size_t numberOfVariables) { out.println(" moduleInHeapSnapshots place=", modulesWithNamedVariables, " module=m", module, " variables=", numberOfVariables); ++modulesWithNamedVariables; }, [&](const VariableData& row) { @@ -454,21 +456,26 @@ static void printImageProgramTables(PrintStream& out, const TokensOfRuntimeFunct }, [&](uint32_t first, uint32_t length) { out.println(" closuresOfModule module=", modulesWithNamedVariables - 1, " first=", first, " length=", length); }); - forEachNamedCapture(tables.namedCaptures(), [&](const NamedCaptureData& row) { + forEachNamedCapture(decodeVarintBytes(tables.namedCaptures()).span(), [&](const NamedCaptureData& row) { out.println(" namedCapture closure=", static_cast(row.closureNumber), " slot=", static_cast(row.slot), " name=s", row.name, row.isVariableOfModule ? " isVariableOfModule"_s : ""_s); }); } +static void printCodedTextsOfImage(PrintStream& out, const ImageHeader& header, std::span image) +{ + ImageSectionRange range = header.sections[static_cast(ImageSection::Strings)]; + size_t sizeOfWords = header.numbers.numberOfStringWords * sizeof(ImageStringWord); + CodedTexts texts(image.subspan(range.offset + sizeOfWords, range.size - sizeOfWords), header.numbers.numberOfCodedTexts); + for (uint32_t number = 0; number < texts.count(); ++number) + out.println(" coded ", String(byteCast(texts.at(number).span()))); +} + static void printTextOfImageSection(PrintStream& out, const TokensOfRuntimeFunctions& tokens, ImageSection section, std::span bytes) { if (section == ImageSection::ProgramTables) { printImageProgramTables(out, tokens, bytes); return; } - if (section == ImageSection::Functions) { - printElementsOfImageSection(out, bytes, printImageFunctionRecord); - return; - } if (section == ImageSection::InnerEntries) { printElementsOfImageSection(out, bytes, [](PrintStream& line, const ImageInnerEntry& entry) { line.print("f", entry.function, " codeOffset=", entry.codeOffset); @@ -484,7 +491,7 @@ static void printTextOfImageSection(PrintStream& out, const TokensOfRuntimeFunct return; } if (section == ImageSection::Identifiers) { - printElementsOfImageSection(out, bytes, printImageIdentifierWord); + printElementsOfImageSection(out, bytes, printImageIdentifierByte); return; } if (section == ImageSection::StubAddresses) { @@ -557,7 +564,22 @@ static void runImageCommand(TestCommandContext& context) context.out.println(" outside the image"); continue; } - printTextOfImageSection(context.out, tokens, named.section, image.subspan(range.offset, range.size)); + if (named.section == ImageSection::Functions) { + ImageSectionRange rangeOfSideTables = rangeOfImageSection(header, ImageSection::SideTables); + std::span sideTables = image.subspan(rangeOfSideTables.offset, rangeOfSideTables.size); + ImageFunctions functions(spanReinterpretCast(image.subspan(range.offset, range.size)).data(), header.counts.functions); + for (uint32_t index = 0; index < header.counts.functions; ++index) { + context.out.print(" ", index, ": "); + uint32_t place = functions.sideTableOf(index); + printImageFunctionRecord(context.out, functions.numberOf(index), { place, SideTableReader(sideTables, sideTables.data() + place).next(SideTableByte::Flags) }); + context.out.println(); + } + continue; + } + size_t size = named.section == ImageSection::Strings ? header.numbers.numberOfStringWords * sizeof(ImageStringWord) : named.section == ImageSection::Identifiers ? header.numbers.endOfIdentifiers : range.size; + printTextOfImageSection(context.out, tokens, named.section, image.subspan(range.offset, size)); + if (named.section == ImageSection::Text) + printCodedTextsOfImage(context.out, header, image); } } diff --git a/Source/JavaScriptCore/aot/tests/AOTPropertyLookupTableTests.cpp b/Source/JavaScriptCore/aot/tests/AOTPropertyLookupTableTests.cpp index 6ffbbf4735ce0..0df834a976b0e 100644 --- a/Source/JavaScriptCore/aot/tests/AOTPropertyLookupTableTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTPropertyLookupTableTests.cpp @@ -261,6 +261,94 @@ void absenceBesidePairDoesNotOutliveItsCount() } // namespace +void fewCharactersHashAsTheirText() +{ + auto check = [](const StringKeyCache::FewCharacters& joined, std::span text) { + AOT_TEST_CHECK(joined.length == text.size()); + auto characters = joined.text(); + AOT_TEST_CHECK(equalSpans(std::span { characters }.first(joined.length), text)); + AOT_TEST_CHECK(joined.hash() == RapidHash::computeHashAndMaskTop8Bits(text)); + if (text.size() < 2) + return; + Ref impl = StringImpl::create(text); + AOT_TEST_CHECK(joined.isTextOf(impl.get())); + for (size_t i = 0; i < text.size(); ++i) { + std::array other { }; + memcpySpan(std::span { other }, text); + other[i] = static_cast(static_cast(other[i]) ^ 1); + AOT_TEST_CHECK(!joined.isTextOf(StringImpl::create(std::span { other }.first(text.size())).get())); + } + AOT_TEST_CHECK(!joined.isTextOf(StringImpl::create(text.first(text.size() - 1)).get())); + }; + + uint64_t state = 0x9e3779b97f4a7c15ull; + auto next = [&] { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + return state; + }; + for (unsigned round = 0; round < 20000; ++round) { + std::array text; + StringKeyCache::FewCharacters joined; + size_t length = 0; + while (true) { + size_t size = next() % (text.size() + 1); + std::array part; + for (Latin1Character& character : part) + character = static_cast(next()); + bool fits = length + size <= text.size(); + AOT_TEST_CHECK(joined.tryAppend(std::span { part }.first(size)) == fits); + if (!fits) + break; + memcpySpan(std::span { text }.subspan(length), std::span { part }.first(size)); + length += size; + } + if (length) + check(joined, std::span { text }.first(length)); + } + + for (unsigned round = 0; round < 20000; ++round) { + uint32_t integer = round < 10000 ? round : static_cast(next() % StringKeyCache::FewCharacters::limitOfIntegers); + StringKeyCache::FewCharacters joined; + AOT_TEST_CHECK(joined.tryAppendInteger(integer)); + check(joined, String::number(integer).span8()); + } + StringKeyCache::FewCharacters full; + AOT_TEST_CHECK(!full.tryAppendInteger(StringKeyCache::FewCharacters::limitOfIntegers) && !full.tryAppendInteger(static_cast(-1))); + AOT_TEST_CHECK(full.tryAppendInteger(99999999) && full.tryAppendInteger(10000000) && !full.tryAppendInteger(0)); +} + +void pairKeysOfNeighborsFindPlaces() +{ + constexpr size_t tableSize = 1 << 14; + constexpr size_t numberOfSets = tableSize / StringKeyCache::waysOfKeysOfPairs; + constexpr EncodedJSValue firstCell = 0x7f0012340000; + auto check = [&](size_t firsts, EncodedJSValue first, EncodedJSValue stepOfFirst, size_t seconds, EncodedJSValue second, EncodedJSValue stepOfSecond) { + std::array pairsOfSet { }; + size_t withoutPlace = 0; + for (size_t i = 0; i < firsts; ++i) { + for (size_t j = 0; j < seconds; ++j) { + uint64_t offset = StringKeyCache::hashOfPair(first + i * stepOfFirst, second + j * stepOfSecond) & StringKeyCache::keyOfPairMaskFor(tableSize); + size_t set = offset >> StringKeyCache::keyOfPairSetShift; + AOT_TEST_CHECK(set < numberOfSets); + withoutPlace += ++pairsOfSet[set] > StringKeyCache::waysOfKeysOfPairs; + } + } + AOT_TEST_CHECK(withoutPlace * 20 <= firsts * seconds); + }; + EncodedJSValue zero = JSValue::encode(jsNumber(0)); + for (EncodedJSValue spacing : { 16, 32, 48, 64, 96 }) { + check(40, firstCell, spacing, 18, zero, 1); + check(12, firstCell, spacing, 400, zero, 1); + check(400, zero, 1, 12, firstCell, spacing); + check(40, firstCell, spacing, 40, firstCell + 0x20000, spacing); + } + check(64, zero, 1, 64, zero, 1); + check(12, zero, 1, 400, zero, 1); + check(400, zero, 1, 12, zero, 1); +} + std::span propertyLookupTableTests() { static constexpr NativeTest tests[] = { @@ -270,6 +358,8 @@ std::span propertyLookupTableTests() { "property-lookup-table/second-instance-clears-once"_s, secondInstanceClearsOnce }, { "property-lookup-table/only-instance-counts-as-run"_s, onlyInstanceCountsAsRun }, { "property-lookup-table/absence-beside-pair-does-not-outlive-its-count"_s, absenceBesidePairDoesNotOutliveItsCount }, + { "property-lookup-table/few-characters-hash-as-their-text"_s, fewCharactersHashAsTheirText }, + { "property-lookup-table/pair-keys-of-neighbors-find-places"_s, pairKeysOfNeighborsFindPlaces }, }; return tests; } diff --git a/Source/JavaScriptCore/aot/tests/AOTPropertyLookupTests.cpp b/Source/JavaScriptCore/aot/tests/AOTPropertyLookupTests.cpp index f02359eee8ffb..49a55428ae671 100644 --- a/Source/JavaScriptCore/aot/tests/AOTPropertyLookupTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTPropertyLookupTests.cpp @@ -192,7 +192,8 @@ static void testUnderscoreProtoAndIndex(LoadedProgram& program) Identifier index = Identifier::from(vm, 0u); JSObject* object = constructEmptyObject(realm); - AOT_TEST_CHECK(isLastingRefusal(readAndClassify(program, object, underscoreProto))); + AOT_TEST_CHECK(isInheritedFrom(program, readAndClassify(program, object, underscoreProto), PropertyAnswerKind::Accessor, realm->objectPrototype(), underscoreProto)); + AOT_TEST_CHECK(isLastingRefusal(readAndClassify(program, constructEmptyObject(vm, realm->nullPrototypeObjectStructure()), underscoreProto))); AOT_TEST_CHECK(isLastingRefusal(readAndClassify(program, object, index))); AOT_TEST_CHECK(presenceAfterRead(program, object, index) == Presence::Unknown); diff --git a/Source/JavaScriptCore/aot/tests/AOTRegExpCompilerTests.cpp b/Source/JavaScriptCore/aot/tests/AOTRegExpCompilerTests.cpp index 8ec1c56458193..0303f1cd03083 100644 --- a/Source/JavaScriptCore/aot/tests/AOTRegExpCompilerTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTRegExpCompilerTests.cpp @@ -259,13 +259,17 @@ static std::optional pairSearchedFor( static void testTwoPlacesOfOneUnitEachAreSearchedForAtOnce() { using Pair = Yarr::SharedRoutine::FindFirstPairOfUnits; - AOT_TEST_CHECK((pairSearchedFor("e[a-z]{2}[0-9]{3}-[a-z]+"_s, { }, 1) == Pair { Yarr::CharSize::Char16, 6, false, false })); - AOT_TEST_CHECK((pairSearchedFor("ab+c"_s, { }, 1) == Pair { Yarr::CharSize::Char16, 1, false, false })); - AOT_TEST_CHECK((pairSearchedFor("k[0-9]s[0-9]+"_s, { Yarr::Flags::IgnoreCase }, 1) == Pair { Yarr::CharSize::Char16, 2, true, true })); - AOT_TEST_CHECK((pairSearchedFor("x-[a-z]{2}:[0-9]+"_s, { Yarr::Flags::IgnoreCase }, 1) == Pair { Yarr::CharSize::Char16, 3, false, false })); + using enum Yarr::PositionUnits; + AOT_TEST_CHECK((pairSearchedFor("e[a-z]{2}[0-9]{3}-[a-z]+"_s, { }, 1) == Pair { Yarr::CharSize::Char16, 6, One, One })); + AOT_TEST_CHECK((pairSearchedFor("ab+c"_s, { }, 1) == Pair { Yarr::CharSize::Char16, 1, One, One })); + AOT_TEST_CHECK((pairSearchedFor("k[0-9]s[0-9]+"_s, { Yarr::Flags::IgnoreCase }, 1) == Pair { Yarr::CharSize::Char16, 2, IgnoreCase, IgnoreCase })); + AOT_TEST_CHECK((pairSearchedFor("x-[a-z]{2}:[0-9]+"_s, { Yarr::Flags::IgnoreCase }, 1) == Pair { Yarr::CharSize::Char16, 3, One, One })); + AOT_TEST_CHECK((pairSearchedFor("\"(?:_|\\\\u005[Ff])(?:_|\\\\u005[Ff])p"_s, { }, 1) == Pair { Yarr::CharSize::Char16, 1, One, Two })); + AOT_TEST_CHECK((pairSearchedFor("[cd]:[a-z]+e"_s, { }, 1) == Pair { Yarr::CharSize::Char16, 1, Two, One })); #if CPU(X86_64) - AOT_TEST_CHECK((pairSearchedFor("e[a-z]{2}[0-9]{3}-[a-z]+"_s, { }, 0) == Pair { Yarr::CharSize::Char8, 6, false, false })); - AOT_TEST_CHECK((pairSearchedFor("k[0-9]s[0-9]+"_s, { Yarr::Flags::IgnoreCase }, 0) == Pair { Yarr::CharSize::Char8, 2, true, true })); + AOT_TEST_CHECK((pairSearchedFor("\"(?:c|\\\\u0063)(?:o|\\\\u006[Ff])\"\\s*:"_s, { }, 0) == Pair { Yarr::CharSize::Char8, 1, One, Two })); + AOT_TEST_CHECK((pairSearchedFor("e[a-z]{2}[0-9]{3}-[a-z]+"_s, { }, 0) == Pair { Yarr::CharSize::Char8, 6, One, One })); + AOT_TEST_CHECK((pairSearchedFor("k[0-9]s[0-9]+"_s, { Yarr::Flags::IgnoreCase }, 0) == Pair { Yarr::CharSize::Char8, 2, IgnoreCase, IgnoreCase })); #endif AOT_TEST_CHECK((!pairSearchedFor("[a-z]bz?.7"_s, { }, 1))); AOT_TEST_CHECK((!pairSearchedFor("[a-z]:#?[^=]+"_s, { }, 1))); diff --git a/Source/JavaScriptCore/aot/tests/AOTRuntimeOperationTests.cpp b/Source/JavaScriptCore/aot/tests/AOTRuntimeOperationTests.cpp index 2ecdcd654c9dc..2c255a33f54a2 100644 --- a/Source/JavaScriptCore/aot/tests/AOTRuntimeOperationTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTRuntimeOperationTests.cpp @@ -349,7 +349,8 @@ static void checkLiteralsBornHoldingLists(LoadedProgram& program) { "[1, 2, 3]"_s, ArrayWithInt32, ArrayWithContiguous }, { "Array.of(1, 2, 3)"_s, ArrayWithInt32, ArrayWithContiguous }, { "['x', 1, 2, 3].slice(1)"_s, ArrayWithInt32, ArrayWithContiguous }, - { "[]"_s, std::nullopt, std::nullopt }, + { "[]"_s, ArrayWithInt32, ArrayWithContiguous }, + { "Reflect.construct(Array, [])"_s, std::nullopt, std::nullopt }, { "(a => (a.pop(), a))([1])"_s, ArrayWithInt32, ArrayWithContiguous }, { "['x'].slice(1)"_s, ArrayWithInt32, ArrayWithContiguous }, { "['x', 2, 3]"_s, ArrayWithContiguous, ArrayWithContiguous }, diff --git a/Source/JavaScriptCore/aot/tests/AOTSavedFrameTests.cpp b/Source/JavaScriptCore/aot/tests/AOTSavedFrameTests.cpp index 1e43f8fb4f099..53c6dc1c19ae2 100644 --- a/Source/JavaScriptCore/aot/tests/AOTSavedFrameTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTSavedFrameTests.cpp @@ -74,46 +74,6 @@ function f1 name="waitFor" flags=[IsAsync] parameters=[{type=anything}, {type=an } )IR"_s; -static constexpr ASCIILiteral programWithTwoAwaitsText = R"IR(module m0 path="main.js" displayPath="main.js" body=f0 -entry module=m0 -function f0 name="main" kind=Module results=[{type=anything}] { - v0 = NewClosure function=f1 - v1 = Builtin builtin=GlobalThis - v2 = PropGet v1 key="register" [isFreeName] - v3 = Call v2, -, v0 - Return -} -function f1 name="waitTwice" flags=[IsAsync] parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { - v2 = Call v0 - v3 = Call v0 - v4 = Call v0 - v5 = Call v0 - v6 = Call v0 - v7 = Await v1 - v8 = Call v0, -, v2, v3, v4, v5 - v9 = Await v1 - v10 = NewArray v6, v9 contents=anything - Return v10 -} -)IR"_s; - -static constexpr ASCIILiteral programWithAsyncGeneratorText = R"IR(module m0 path="main.js" displayPath="main.js" body=f0 -entry module=m0 -function f0 name="main" kind=Module results=[{type=anything}] { - v0 = NewClosure function=f1 - v1 = Builtin builtin=GlobalThis - v2 = PropGet v1 key="register" [isFreeName] - v3 = Call v2, -, v0 - Return -} -function f1 name="waitOnce" flags=[IsAsync, IsGenerator, TakesCallee] parameters=[{type=anything}, {type=anything}] results=[{type=AsyncGenerator}] { - InitialYield - v4 = Await v2 - v5 = Call v1, -, v4 - Return v5 -} -)IR"_s; - static Vector valuesOf(const SavedFrameMap& map) { Vector locations; @@ -659,9 +619,6 @@ static void testListedWordsSurviveACollection() } static constexpr uint64_t wordOfDeadStack = 0x5ca1ab1e5ca1ab1eULL; -static unsigned numberOfSaveAreasRead; -static const Instance* instanceOfSaveArea; - static NEVER_INLINE void fillDeadStack() { volatile uint64_t words[2048]; @@ -669,52 +626,6 @@ static NEVER_INLINE void fillDeadStack() word = wordOfDeadStack; } -static JSC_DECLARE_HOST_FUNCTION(readSaveAreaAboveResumedCaller); -JSC_DEFINE_HOST_FUNCTION(readSaveAreaAboveResumedCaller, (JSGlobalObject*, CallFrame* callFrame)) -{ - if (!callFrame->argumentCount()) - return JSValue::encode(jsNumber(1)); - constexpr ptrdiff_t instanceWord = AdapterFrame::offsetOfInstance / static_cast(sizeof(uint64_t)); - auto* frameOfAdapter = reinterpret_cast(callFrame); - unsigned numberOfFrames = 0; - do { - frameOfAdapter = *reinterpret_cast(frameOfAdapter); - AOT_TEST_CHECK(++numberOfFrames < 6); - } while (frameOfAdapter[instanceWord] != std::bit_cast(instanceOfSaveArea)); - for (ptrdiff_t word = 1; word <= static_cast(AdapterFrame::saveAreaSize / sizeof(uint64_t)); ++word) - AOT_TEST_CHECK(frameOfAdapter[-word] != wordOfDeadStack); - ++numberOfSaveAreasRead; - return JSValue::encode(jsNumber(2)); -} - -static void checkAdapterAboveResumedFrameWritesItsSaveArea(LoadedProgram& program) -{ - VM& vm = program.vm(); - JSGlobalObject* realm = program.realm(); - auto scope = DECLARE_TOP_EXCEPTION_SCOPE(vm); - program.realms().evaluate(realm, "globalThis.register = waitTwice => { globalThis.waitTwice = waitTwice; };"_s); - std::optional module = findModule("main.js"_s); - AOT_TEST_CHECK(module); - evaluateModuleSynchronously(realm, *module); - AOT_TEST_CHECK(!scope.exception()); - - MarkedArgumentBuffer arguments; - arguments.append(JSFunction::create(vm, realm, 0, "readSaveArea"_s, readSaveAreaAboveResumedCaller, ImplementationVisibility::Public)); - arguments.append(program.realms().evaluate(realm, "Promise.resolve(3)"_s)); - numberOfSaveAreasRead = 0; - instanceOfSaveArea = &program.instance(); - call(realm, program.realms().evaluate(realm, "waitTwice"_s), arguments, "waitTwice is a function"_s); - AOT_TEST_CHECK(!scope.exception() && !numberOfSaveAreasRead); - fillDeadStack(); - vm.drainMicrotasks(); - AOT_TEST_CHECK(!scope.exception() && numberOfSaveAreasRead == 1); -} - -static const uint64_t* frameOfJobCaller; -static unsigned numberOfJobEntriesRead; -static bool readsFramesOfJobLoop; -static bool bodyHoldsItsFrameInHeapAlone; - static String describeCodeAddress(const void* address) { #if OS(UNIX) @@ -727,171 +638,6 @@ static String describeCodeAddress(const void* address) return makeString("code at "_s, hex(std::bit_cast(address))); } -#if CPU(X86_64) -static bool isInJobFunction(const void* code) -{ - static constexpr size_t largestJobFunction = 4 * KB; - auto isIn = [&](auto function) { - return std::bit_cast(code) - std::bit_cast(function) < largestJobFunction; - }; - return isIn(&resumeAsyncFunction) || isIn(&resumeSavedAsyncFunction) || isIn(&runAsyncFunctionResumeJob) || isIn(&runAsyncGeneratorBodyJob); -} -#endif - -static JSC_DECLARE_HOST_FUNCTION(readFramesBelowJobCaller); -JSC_DEFINE_HOST_FUNCTION(readFramesBelowJobCaller, (JSGlobalObject*, CallFrame* callFrame)) -{ - if (!callFrame->argumentCount()) - return JSValue::encode(jsNumber(1)); - auto* frame = reinterpret_cast(callFrame); - unsigned numberOfFrames = 0; - unsigned numberOfUnwrittenWords = 0; - [[maybe_unused]] unsigned numberOfPaddingWords = 0; - while (*reinterpret_cast(frame) != frameOfJobCaller) { - auto* frameOfCaller = *reinterpret_cast(frame); - for (auto* word = frame + 2; word < frameOfCaller; ++word) { - if (*word != wordOfDeadStack) - continue; - ++numberOfUnwrittenWords; -#if CPU(X86_64) - numberOfPaddingWords += isInJobFunction(removeCodePtrTag(std::bit_cast(frame[1]))) || (bodyHoldsItsFrameInHeapAlone && frameOfCaller - frame == 4 && word == frame + 2); -#endif - dataLogLn("unwritten word ", (frameOfCaller - word) * sizeof(uint64_t), " bytes below frame record ", numberOfFrames + 1, " of ", (frameOfCaller - frame - 2) * sizeof(uint64_t), " bytes, in ", describeCodeAddress(removeCodePtrTag(std::bit_cast(frame[1])))); - } - frame = frameOfCaller; - AOT_TEST_CHECK(++numberOfFrames < 12); - } - if (numberOfUnwrittenWords) - dataLogLn(numberOfUnwrittenWords, " unwritten words in ", numberOfFrames, " frames"); -#if CPU(ARM64) - AOT_TEST_CHECK(numberOfUnwrittenWords <= numberOfFrames); -#else - if (readsFramesOfJobLoop) - AOT_TEST_CHECK(numberOfUnwrittenWords <= numberOfFrames); - else - AOT_TEST_CHECK(numberOfUnwrittenWords == numberOfPaddingWords && numberOfPaddingWords <= 1u + bodyHoldsItsFrameInHeapAlone); -#endif - ++numberOfJobEntriesRead; - return JSValue::encode(jsNumber(2)); -} - -static NEVER_INLINE void runJobOverFilledStack(JSGlobalObject* realm, VM& vm, const QueuedTask& job) -{ - NO_TAIL_CALLS(); - frameOfJobCaller = static_cast(__builtin_frame_address(0)); - fillDeadStack(); - auto arguments = job.arguments(); - runInternalMicrotask(realm, vm, job.job(), job.payload(), arguments[0], arguments[1], arguments[2], arguments[3]); -} - -static void checkFramesBetweenJobAndCompiledCodeAreWritten(LoadedProgram& program) -{ -#if HAVE(MUST_TAIL_CALL) && !ENABLE(EXCEPTION_SCOPE_VERIFICATION) && !ASAN_ENABLED - VM& vm = program.vm(); - JSGlobalObject* realm = program.realm(); - auto scope = DECLARE_TOP_EXCEPTION_SCOPE(vm); - program.realms().evaluate(realm, "globalThis.register = waitTwice => { globalThis.waitTwice = waitTwice; };"_s); - std::optional module = findModule("main.js"_s); - AOT_TEST_CHECK(module); - evaluateModuleSynchronously(realm, *module); - AOT_TEST_CHECK(!scope.exception()); - - MarkedArgumentBuffer arguments; - arguments.append(JSFunction::create(vm, realm, 0, "readFrames"_s, readFramesBelowJobCaller, ImplementationVisibility::Public)); - arguments.append(program.realms().evaluate(realm, "Promise.resolve(3)"_s)); - numberOfJobEntriesRead = 0; - call(realm, program.realms().evaluate(realm, "waitTwice"_s), arguments, "waitTwice is a function"_s); - AOT_TEST_CHECK(!scope.exception() && !numberOfJobEntriesRead); - - MicrotaskQueue& queue = vm.defaultMicrotaskQueue(); - AOT_TEST_CHECK(queue.size() == 1); - std::optional job; - queue.allOf([&](const QueuedTask& queued) { - job = queued; - return true; - }); - AOT_TEST_CHECK(job->job() == InternalMicrotask::CompiledAsyncFunctionResume); - queue.clear(); - runJobOverFilledStack(realm, vm, *job); - AOT_TEST_CHECK(!scope.exception() && numberOfJobEntriesRead == 1); - vm.drainMicrotasks(); -#else - UNUSED_PARAM(program); -#endif -} - -static NEVER_INLINE void drainOverFilledStack(VM& vm) -{ - NO_TAIL_CALLS(); - frameOfJobCaller = static_cast(__builtin_frame_address(0)); - fillDeadStack(); - vm.drainMicrotasks(); -} - -static void checkJobLoopWritesItsFrame(LoadedProgram& program) -{ -#if HAVE(MUST_TAIL_CALL) && !ENABLE(EXCEPTION_SCOPE_VERIFICATION) && !ASAN_ENABLED - VM& vm = program.vm(); - JSGlobalObject* realm = program.realm(); - auto scope = DECLARE_TOP_EXCEPTION_SCOPE(vm); - program.realms().evaluate(realm, "globalThis.register = waitTwice => { globalThis.waitTwice = waitTwice; };"_s); - std::optional module = findModule("main.js"_s); - AOT_TEST_CHECK(module); - evaluateModuleSynchronously(realm, *module); - AOT_TEST_CHECK(!scope.exception()); - - MarkedArgumentBuffer arguments; - arguments.append(JSFunction::create(vm, realm, 0, "readFrames"_s, readFramesBelowJobCaller, ImplementationVisibility::Public)); - arguments.append(program.realms().evaluate(realm, "Promise.resolve(3)"_s)); - numberOfJobEntriesRead = 0; - call(realm, program.realms().evaluate(realm, "waitTwice"_s), arguments, "waitTwice is a function"_s); - AOT_TEST_CHECK(!scope.exception() && !numberOfJobEntriesRead); - - SetForScope readsLoop { readsFramesOfJobLoop, true }; - drainOverFilledStack(vm); - AOT_TEST_CHECK(!scope.exception() && numberOfJobEntriesRead == 1); -#else - UNUSED_PARAM(program); -#endif -} - -static void checkFramesBetweenJobAndAsyncGeneratorBodyAreWritten(LoadedProgram& program) -{ -#if HAVE(MUST_TAIL_CALL) && !ENABLE(EXCEPTION_SCOPE_VERIFICATION) && !ASAN_ENABLED - VM& vm = program.vm(); - JSGlobalObject* realm = program.realm(); - auto scope = DECLARE_TOP_EXCEPTION_SCOPE(vm); - program.realms().evaluate(realm, "globalThis.register = waitOnce => { globalThis.start = (readFrames, promise) => { waitOnce(readFrames, promise).next(); }; };"_s); - std::optional module = findModule("main.js"_s); - AOT_TEST_CHECK(module); - evaluateModuleSynchronously(realm, *module); - AOT_TEST_CHECK(!scope.exception()); - - MarkedArgumentBuffer arguments; - arguments.append(JSFunction::create(vm, realm, 0, "readFrames"_s, readFramesBelowJobCaller, ImplementationVisibility::Public)); - arguments.append(program.realms().evaluate(realm, "Promise.resolve(3)"_s)); - numberOfJobEntriesRead = 0; - call(realm, program.realms().evaluate(realm, "start"_s), arguments, "start is a function"_s); - AOT_TEST_CHECK(!scope.exception() && !numberOfJobEntriesRead); - - MicrotaskQueue& queue = vm.defaultMicrotaskQueue(); - AOT_TEST_CHECK(queue.size() == 1); - std::optional job; - queue.allOf([&](const QueuedTask& queued) { - job = queued; - return true; - }); - AOT_TEST_CHECK(job->job() == InternalMicrotask::AsyncGeneratorBodyCallNormal); - queue.clear(); - SetForScope holdsFrameAlone { bodyHoldsItsFrameInHeapAlone, true }; - runJobOverFilledStack(realm, vm, *job); - AOT_TEST_CHECK(!scope.exception() && numberOfJobEntriesRead == 1); - vm.drainMicrotasks(); -#else - UNUSED_PARAM(program); -#endif -} - static const uint64_t* frameOfCollectionCaller; static const Thread* threadOfCollectionCaller; static unsigned numberOfCollectionsRead; @@ -977,11 +723,7 @@ std::span savedFrameTests() { "saved-frame/unlisted-word-is-no-root"_s, runNativeTestWithLoadedProgram }, { "saved-frame/frame-on-the-stack-is-left-alone"_s, runNativeTestWithLoadedProgram }, { "saved-frame/skipped-words"_s, runNativeTestWithLoadedProgram }, - { "saved-frame/adapter-above-a-resumed-frame-writes-its-save-area"_s, runNativeTestWithLoadedProgram }, - { "saved-frame/frames-between-job-and-compiled-code-are-written"_s, runNativeTestWithLoadedProgram }, { "saved-frame/collector-writes-the-frames-it-scans"_s, runNativeTestWithLoadedProgram }, - { "saved-frame/job-loop-writes-its-frame"_s, runNativeTestWithLoadedProgram }, - { "saved-frame/frames-between-job-and-async-generator-body-are-written"_s, runNativeTestWithLoadedProgram }, { "saved-frame/listed-words-survive-a-collection"_s, testListedWordsSurviveACollection }, }; return tests; diff --git a/Source/JavaScriptCore/aot/tests/AOTStubBlobTests.cpp b/Source/JavaScriptCore/aot/tests/AOTStubBlobTests.cpp index f668b2e548620..9a22b6be892fa 100644 --- a/Source/JavaScriptCore/aot/tests/AOTStubBlobTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTStubBlobTests.cpp @@ -309,7 +309,9 @@ static void testBodyEntries() StubCatalog catalog(StubOptions { }); checkEntries(catalog, getByVal(false), { getByVal(false), getByVal(true) }); checkEntries(catalog, getByValCached(), { getByValCached(), getByValCachedCells(), getByValCachedAtPlace(), getByValCachedBehindPlaces() }); - checkEntries(catalog, getByValKeepingRegisters(), { getByValKeepingRegisters() }); + checkEntries(catalog, getByValKeepingRegisters(), { getByValKeepingRegisters(), getByValAtConstantIndex() }); + for (ECMAMode mode : { ECMAMode::strict(), ECMAMode::sloppy() }) + checkEntries(catalog, putByVal(false, mode), { putByVal(false, mode), putByVal(true, mode), putByValAtConstantIndex(mode) }); checkEntries(catalog, getByValCached(RegistersOfCaller::AreKept), { getByValCached(RegistersOfCaller::AreKept), getByValCachedCells(RegistersOfCaller::AreKept), getByValCachedAtPlace(RegistersOfCaller::AreKept), getByValCachedBehindPlaces(RegistersOfCaller::AreKept) }); for (BaseOfKeyedRead base : { BaseOfKeyedRead::IsCell, BaseOfKeyedRead::IsAnyValue }) { RegisterSet keptByRead = registersClobberedByOperations(); diff --git a/Source/JavaScriptCore/aot/tests/AOTStubBoxedCallsOfLanguageTests.cpp b/Source/JavaScriptCore/aot/tests/AOTStubBoxedCallsOfLanguageTests.cpp index e593828bbee0b..54ccc367086d7 100644 --- a/Source/JavaScriptCore/aot/tests/AOTStubBoxedCallsOfLanguageTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTStubBoxedCallsOfLanguageTests.cpp @@ -557,17 +557,18 @@ static void testShiftOfFewValuesInLanguageCallTests() FakeObject callee; constexpr StubKey key = callBoxedBuiltin(BoxedCallForm::ArrayShift, 0); constexpr uint64_t answerOfOperation = int32InLanguageCallTests(99); + runner.install(Entry::operationAOTMoveBytes, operationAOTMoveBytes); runner.recordCallsOf(Entry::operationAOTArrayShiftGeneric); runner.setResultOfRecordedOperation(answerOfOperation); - auto isShiftedByStub = [&](ListInLanguageCallTests& list, uint64_t first) { + auto isShiftedByStub = [&](auto& list, uint64_t first) { Vector before; before.append(list.elements().first(list.length())); StubRun run = runner.runCall(key, callee.address(), list.cell.address(), { }); AOT_TEST_CHECK(!run.calledOperation && run.result == first && list.length() + 1 == before.size()); AOT_TEST_CHECK(std::ranges::equal(list.elements().first(list.length()), before.span().subspan(1)) && !list.elements()[list.length()]); }; - auto isLeftToOperation = [&](ListInLanguageCallTests& list) { + auto isLeftToOperation = [&](auto& list) { Vector before; before.append(list.elements().first(list.length())); StubRun run = runner.runCall(key, callee.address(), list.cell.address(), { }); @@ -576,11 +577,14 @@ static void testShiftOfFewValuesInLanguageCallTests() }; for (IndexingType shape : { Int32Shape, ContiguousShape }) { - for (uint32_t count : { 1u, 2u, 3u, 9u, 17u }) { + for (uint32_t count : { 1u, 2u, 3u, 9u, 17u, 18u, 40u }) { ListInLanguageCallTests list(shape, integersInLanguageCallTests(count).span()); isShiftedByStub(list, int32InLanguageCallTests(1)); } - ListInLanguageCallTests ofOneMore(shape, integersInLanguageCallTests(18).span()); + using LongList = ListOfCapacityInLanguageCallTests; + LongList longestMoved(shape, integersInLanguageCallTests(JSArray::shiftThreshold).span()); + isShiftedByStub(longestMoved, int32InLanguageCallTests(1)); + LongList ofOneMore(shape, integersInLanguageCallTests(JSArray::shiftThreshold + 1).span()); isLeftToOperation(ofOneMore); ListInLanguageCallTests empty(shape, { }); @@ -606,8 +610,10 @@ static void testShiftOfFewValuesInLanguageCallTests() isLeftToOperation(ofDoubles); runner.instance().mutatorShouldBeFenced = true; - ListInLanguageCallTests ofValuesWhileCollectorMayVisit(ContiguousShape, integersInLanguageCallTests(3).span()); - isLeftToOperation(ofValuesWhileCollectorMayVisit); + for (uint32_t count : { 3u, 18u }) { + ListInLanguageCallTests ofValuesWhileCollectorMayVisit(ContiguousShape, integersInLanguageCallTests(count).span()); + isLeftToOperation(ofValuesWhileCollectorMayVisit); + } ListInLanguageCallTests ofIntegersWhileCollectorMayVisit(Int32Shape, integersInLanguageCallTests(3).span()); isShiftedByStub(ofIntegersWhileCollectorMayVisit, int32InLanguageCallTests(1)); ListInLanguageCallTests aloneWhileCollectorMayVisit(ContiguousShape, integersInLanguageCallTests(1).span()); diff --git a/Source/JavaScriptCore/aot/tests/AOTStubPropertyStoreTests.cpp b/Source/JavaScriptCore/aot/tests/AOTStubPropertyStoreTests.cpp index a5dd5cbdae75f..61f26db843d1b 100644 --- a/Source/JavaScriptCore/aot/tests/AOTStubPropertyStoreTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTStubPropertyStoreTests.cpp @@ -726,7 +726,7 @@ static void testInheritedSettersOfAbsentFields() FakePlainObject holder(0); holder.word(Slot::wordIndexFor(offset)) = accessors.address(); const SlotData field = SlotData::create(FakeTypedObject::offsetOfField, SlotRepresentation::JSValue); - const Slot slot = *Slot::tryCreateFieldAndHolder(toStructureID(PropertyStores::structureID), Slot::Kind::FieldOrInheritedSetter, Slot::dependsOnChain, field, std::bit_cast(holder.address()), offset, stores.runner.instance().cellCageBase); + const Slot slot = *Slot::tryCreateFieldAndHolder(toStructureID(PropertyStores::structureID), Slot::Kind::FieldOrInheritedAccessor, Slot::dependsOnChain, field, std::bit_cast(holder.address()), offset, stores.runner.instance().cellCageBase); for (PutMode mode : { PutMode::Strict, PutMode::Sloppy }) { StoreCase storeCase { mode, FiledIn::HashedTable }; @@ -744,7 +744,7 @@ static void testInheritedSettersOfAbsentFields() AOT_TEST_CHECK(object.field() == JSValue::ValueUndefined); constexpr uint32_t orderWordThatLeavesFieldOut = orderWordOf(9, 3); - stores.file(storeCase, *Slot::tryCreateFieldAndHolder(toStructureID(PropertyStores::structureID), Slot::Kind::FieldOrInheritedSetter, Slot::dependsOnChain | Slot::emptySlotMayBeUndefined, SlotData::decode(orderWordThatLeavesFieldOut), std::bit_cast(holder.address()), offset, stores.runner.instance().cellCageBase)); + stores.file(storeCase, *Slot::tryCreateFieldAndHolder(toStructureID(PropertyStores::structureID), Slot::Kind::FieldOrInheritedAccessor, Slot::dependsOnChain | Slot::emptySlotMayBeUndefined, SlotData::decode(orderWordThatLeavesFieldOut), std::bit_cast(holder.address()), offset, stores.runner.instance().cellCageBase)); for (uint32_t otherBits : { 0u, orderWordHasExtraProperties, orderWordIntegrityBits }) { FakeTypedObject objectThatLeavesFieldOut(JSValue::ValueEmpty, orderWordThatLeavesFieldOut | otherBits); StubRun run = stores.store(storeCase, objectThatLeavesFieldOut.address(), value); diff --git a/Source/JavaScriptCore/aot/tests/AOTStubRuntimeTests.cpp b/Source/JavaScriptCore/aot/tests/AOTStubRuntimeTests.cpp index 91ca22b5e810e..ecf2f0a9802a6 100644 --- a/Source/JavaScriptCore/aot/tests/AOTStubRuntimeTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTStubRuntimeTests.cpp @@ -237,10 +237,16 @@ static void checkOrderLeftToOperation(StubRunner& runner, StubKey key, uint64_t } } +static int64_t AOT_PRESERVE_MOST_LEAF_ATTRIBUTES findNoOrderInPlace(JSCell*, JSCell*) +{ + return noOrderOfStringsInPlace; +} + static void testBinaryOperationStringOrder() { StubRunner runner; runner.install(Entry::operationAOTCompareStringImpls, operationAOTCompareStringImpls); + runner.install(Entry::operationAOTCompareStringsInPlace, findNoOrderInPlace); FakeString flat("order"_s, CharacterWidth::Is8Bit); FakeString wideFlat("order"_s, CharacterWidth::Is16Bit); diff --git a/Source/JavaScriptCore/aot/tests/AOTStubValueTests.cpp b/Source/JavaScriptCore/aot/tests/AOTStubValueTests.cpp index 09e3d04b4126d..f98375b3aa631 100644 --- a/Source/JavaScriptCore/aot/tests/AOTStubValueTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTStubValueTests.cpp @@ -9,6 +9,7 @@ #include "AOTStubRunner.h" #include "AOTClosureWord.h" +#include "AOTPropertyCachesStringKeys.h" #include "AOTRuntimeType.h" #include "AOTSlimClosure.h" #include "AOTStubTestSupport.h" @@ -26,6 +27,7 @@ #include #include #include +#include #include WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN @@ -144,6 +146,46 @@ static void testSubstringsFromEveryStateOfTheFreeList() AOT_TEST_CHECK(ropes.runner.run(substring(), { string.address(), 2, 5 }).calledOperation); } +static void testOneCharacterOfJoinedText() +{ + RopeAllocation ropes; + std::array singleCharacterStrings; + std::iota(singleCharacterStrings.begin(), singleCharacterStrings.end(), 1); + ropes.runner.instance().singleCharacterStrings = singleCharacterStrings.data(); + Ref firstText = StringImpl::create("ab"_span); + Ref secondText = StringImpl::create("cde"_span); + FakeFlatString first(firstText.get()); + FakeFlatString second(secondText.get()); + ropes.allocator.makeRoomAt(ropes.room.address(), sizeof(ropes.room.bytes)); + uint64_t joined = ropes.runner.run(concatenate(2), { first.address(), second.address() }).result; + constexpr std::string_view letters = "abcde"; + for (uint64_t position = 0; position < letters.size(); ++position) { + StubRun run = ropes.runner.run(substring(), { joined, position, position + 1 }); + AOT_TEST_CHECK(!run.calledOperation && run.result == singleCharacterStrings[letters[position]]); + } + Ref longText = StringImpl::create("vwxyz"_span); + FakeFlatString whole(longText.get()); + uint64_t piece = ropes.runner.run(substring(), { whole.address(), 1, 5 }).result; + StubRun run = ropes.runner.run(substring(), { piece, 2, 3 }); + AOT_TEST_CHECK(!run.calledOperation && run.result == singleCharacterStrings['y']); +} + +static void testLongKeyOfPairIsJoinedAsText() +{ + RopeAllocation ropes; + std::array letters; + letters.fill('x'); + Ref longText = StringImpl::create(std::span { letters }); + Ref shortText = StringImpl::create("ab"_span); + FakeFlatString first(longText.get()); + FakeFlatString second(shortText.get()); + StubKey stub = concatenateKeyOfPair(ConcatenatedKeyForm::Name); + ropes.allocator.makeRoomAt(ropes.room.address(), sizeof(ropes.room.bytes)); + StubRun run = ropes.runner.run(stub, { first.address(), second.address() }); + AOT_TEST_CHECK(!run.calledOperation && run.result == ropes.room.address() && ropes.word(1) == (first.address() | JSString::isRopeInPointer | JSRopeString::is8BitInPointer)); + AOT_TEST_CHECK(ropes.runner.run(stub, { second.address(), second.address() }).calledOperation); +} + static void testTruthinessOfDoubles() { StubCatalog catalog(StubOptions { }); @@ -476,11 +518,29 @@ static void testMovesOfFewBytes() } } +static void testMovesOfElementsDown() +{ + StubRunner runner; + runner.install(Entry::operationAOTMoveBytes, operationAOTMoveBytes); + for (uint64_t count : { 1u, 2u, 3u, 16u, 17u, 64u, 127u }) { + std::array words; + std::iota(words.begin(), words.end(), 1); + std::array expected = words; + std::memmove(expected.data(), expected.data() + 1, count * sizeof(uint64_t)); + StubRun run = runner.run(moveElementsDown(), { std::bit_cast(words.data()), count }); + AOT_TEST_CHECK(words == expected); + UNUSED_PARAM(run); + } +} + std::span stubValueTests() { static constexpr NativeTest tests[] = { { "stubs/move-few-bytes/every-length-and-overlap"_s, ImageTests::whenExecutable }, + { "stubs/move-elements-down/every-count"_s, ImageTests::whenExecutable }, { "stubs/concatenate/ropes-from-every-state-of-the-free-list"_s, ImageTests::whenExecutable }, + { "stubs/concatenate/long-key-of-pair"_s, ImageTests::whenExecutable }, + { "stubs/substring/one-character-of-joined-text"_s, ImageTests::whenExecutable }, { "stubs/substring/ropes-from-every-state-of-the-free-list"_s, ImageTests::whenExecutable }, { "stubs/to-boolean/doubles"_s, ImageTests::whenExecutable }, { "stubs/check-proof-of-kinds/a-value-passes-where-its-kind-is-among-them-and-nowhere-else"_s, ImageTests::whenExecutable }, diff --git a/Source/JavaScriptCore/aot/tests/AOTUnionConsecutiveSetsTests.cpp b/Source/JavaScriptCore/aot/tests/AOTUnionConsecutiveSetsTests.cpp index 7f0ae23e3fc4d..3d306c7c4557c 100644 --- a/Source/JavaScriptCore/aot/tests/AOTUnionConsecutiveSetsTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTUnionConsecutiveSetsTests.cpp @@ -91,6 +91,54 @@ static void testSetThatNoOrderKeepsTogetherIsRefusedAndChangesNothing() AOT_TEST_CHECK(standSideBySideForConsecutiveSetsTests(orderByNumberForConsecutiveSetsTests(sets, 5), { 3, 4 })); } +static void testNestedSetsMatchLeafSets() +{ + uint32_t state = 12345; + auto random = [&](uint32_t limit) { + state = state * 1664525 + 1013904223; + return (state >> 8) % limit; + }; + for (unsigned round = 0; round < 200; ++round) { + uint32_t numberOfLeaves = 2 + random(40); + Vector> groups; + for (uint32_t leaf = 0; leaf < numberOfLeaves; ++leaf) + groups.append(Vector { leaf }); + Vector> nested; + while (groups.size() > 1 && random(8)) { + Vector united; + for (unsigned count = 2 + random(3); count && !groups.isEmpty(); --count) { + size_t taken = random(groups.size()); + united.appendVector(groups[taken]); + groups.removeAt(taken); + } + nested.append(united); + groups.append(united); + } + std::ranges::stable_sort(nested, { }, &Vector::size); + ConsecutiveSets leafByLeaf(numberOfLeaves); + ConsecutiveSets byNodes(numberOfLeaves); + for (const Vector& set : nested) { + AOT_TEST_CHECK(leafByLeaf.tryKeepTogether(set.span())); + AOT_TEST_CHECK(byNodes.tryKeepNestedSetTogether(set.span())); + } + for (unsigned count = 0; count < 12; ++count) { + Vector set; + for (uint32_t leaf = 0; leaf < numberOfLeaves; ++leaf) { + if (!random(4)) + set.append(leaf); + } + AOT_TEST_CHECK(leafByLeaf.tryKeepTogether(set.span()) == byNodes.tryKeepTogether(set.span())); + } + AOT_TEST_CHECK(orderByNumberForConsecutiveSetsTests(leafByLeaf, numberOfLeaves) == orderByNumberForConsecutiveSetsTests(byNodes, numberOfLeaves)); + } + ConsecutiveSets sets(4); + Vector first { 0, 1 }; + Vector overlapping { 1, 2 }; + AOT_TEST_CHECK(sets.tryKeepNestedSetTogether(first.span())); + AOT_TEST_CHECK(!sets.tryKeepNestedSetTogether(overlapping.span())); + AOT_TEST_CHECK(sets.tryKeepTogether(overlapping.span())); +} + static void testLeafThatIsNamedTwiceCountsOnce() { ConsecutiveSets sets(4); @@ -302,6 +350,7 @@ std::span unionConsecutiveSetsTests() { "union-consecutive-sets/the-order-of-the-keys-where-nothing-is-kept-together"_s, testOrderOfKeysWhereNothingIsKeptTogether }, { "union-consecutive-sets/nested-and-separate-sets-are-all-kept-together"_s, testNestedAndSeparateSetsAreAllKeptTogether }, { "union-consecutive-sets/a-set-that-no-order-keeps-together-is-refused-and-changes-nothing"_s, testSetThatNoOrderKeepsTogetherIsRefusedAndChangesNothing }, + { "union-consecutive-sets/nested-sets-match-leaf-sets"_s, testNestedSetsMatchLeafSets }, { "union-consecutive-sets/a-leaf-that-is-named-twice-counts-once"_s, testLeafThatIsNamedTwiceCountsOnce }, { "union-consecutive-sets/the-tree-grows-as-its-leaves-do"_s, testTreeGrowsAsItsLeavesDo }, { "union-consecutive-sets/the-plan-grows-as-the-unions-do"_s, testPlanGrowsAsUnionsDo }, diff --git a/Source/JavaScriptCore/aot/tests/AOTWebAssemblyImageTests.cpp b/Source/JavaScriptCore/aot/tests/AOTWebAssemblyImageTests.cpp index 6e4a46161c7ed..bcadc730f2d78 100644 --- a/Source/JavaScriptCore/aot/tests/AOTWebAssemblyImageTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTWebAssemblyImageTests.cpp @@ -384,6 +384,7 @@ static void testPointerTableSlots() slots.append(Wasm::ImagePointerTable::wasmToJSCallee); slots.append(Wasm::ImagePointerTable::restoreFrameCallee); slots.append(Wasm::ImagePointerTable::restoreFrameReturnAddress); + slots.append(Wasm::ImagePointerTable::throwFromFaultHandler); slots.append(Wasm::ImagePointerTable::structureIDBase); #if GIGACAGE_ENABLED slots.append(Wasm::ImagePointerTable::primitiveGigacageDisableRequest); diff --git a/Source/JavaScriptCore/aot/tests/host/stand-in.builtins.ts b/Source/JavaScriptCore/aot/tests/host/stand-in.builtins.ts index b3ed5481ecba4..cb220b42a45dc 100644 --- a/Source/JavaScriptCore/aot/tests/host/stand-in.builtins.ts +++ b/Source/JavaScriptCore/aot/tests/host/stand-in.builtins.ts @@ -21,7 +21,7 @@ export default b.declarations( }, }), b.frontEndFunctions("prelude/array-search", "includes", "indexOf"), - b.frontEndFunctions("prelude/array-sort", "sort", "toSorted"), + b.frontEndFunctions("prelude/array-sort", "sort", "sortTypedArray", "toSorted"), b.frontEndFunctions( "prelude/decorators", "applyAccessorDecorators", diff --git a/Source/JavaScriptCore/aot/tests/image/name-from-outside-keeps-no-field-that-a-type-alone-spells.ir b/Source/JavaScriptCore/aot/tests/image/name-from-outside-keeps-no-field-that-a-type-alone-spells.ir index eab64a2b70f1b..12262678f08d1 100644 --- a/Source/JavaScriptCore/aot/tests/image/name-from-outside-keeps-no-field-that-a-type-alone-spells.ir +++ b/Source/JavaScriptCore/aot/tests/image/name-from-outside-keeps-no-field-that-a-type-alone-spells.ir @@ -6,6 +6,7 @@ # KEPT: 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a # RUN(GONE): image Text # GONE-NOT: 71 71 71 71 71 71 71 71 71 71 71 71 71 71 71 71 +# GONE-NOT: coded qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq # RUN: run-in-shell globalThis.nameFromOutside = () => "q".repeat(32); globalThis.hold = o => { const all = Object.keys(o); let text = ""; for (let i = 0; i < all.length; i++) text += all[i][0]; print(text, o["q".repeat(32)]); }; # CHECK: zq 2 # CHECK-NEXT: exited with 0 diff --git a/Source/JavaScriptCore/aot/tests/image/name-of-a-class-that-nothing-evaluates-is-not-in-the-image.ir b/Source/JavaScriptCore/aot/tests/image/name-of-a-class-that-nothing-evaluates-is-not-in-the-image.ir index 27ed3a74da7ae..b7382676ac5f3 100644 --- a/Source/JavaScriptCore/aot/tests/image/name-of-a-class-that-nothing-evaluates-is-not-in-the-image.ir +++ b/Source/JavaScriptCore/aot/tests/image/name-of-a-class-that-nothing-evaluates-is-not-in-the-image.ir @@ -1,13 +1,14 @@ # If every class had a record, the name of a class that stands in a branch which the build has cut away, or in a file that nobody imports, could be read out of the executable # if (false) { class qqq.. { x = 1 } } class zzz.. { constructor(x) { this.x = x; } } print(new zzz..(3).constructor.name.length); # The record of a class holds its name. A class has a record where reached code evaluates it. No NewClass of the first is left in the program. -# Each name is one letter 32 times: wherever the text begins, one line of 16 bytes holds nothing else. +# Each name is one letter 32 times: wherever the text begins, one line of 16 bytes holds nothing else. A text that is only shown is coded and printed as it reads. # LIBRARY: own # RUN(KEPT): image Text mechanisms=all -# KEPT: 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a +# KEPT: coded zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz # Link says that nothing allocates an object of the first. With Link withheld any layout may be allocated, and the record of its objects holds the name. # RUN(GONE): image Text mechanisms=all # GONE-NOT: 71 71 71 71 71 71 71 71 71 71 71 71 71 71 71 71 +# GONE-NOT: coded qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq # RUN(PLAN): plan classes mechanisms=all # PLAN-NOT: class L0 # PLAN: class L1 diff --git a/Source/JavaScriptCore/aot/tests/image/name-of-a-field-that-a-type-alone-spells-is-not-in-the-image.ir b/Source/JavaScriptCore/aot/tests/image/name-of-a-field-that-a-type-alone-spells-is-not-in-the-image.ir index 6dad71c0a551c..02145825fad92 100644 --- a/Source/JavaScriptCore/aot/tests/image/name-of-a-field-that-a-type-alone-spells-is-not-in-the-image.ir +++ b/Source/JavaScriptCore/aot/tests/image/name-of-a-field-that-a-type-alone-spells-is-not-in-the-image.ir @@ -7,6 +7,7 @@ # KEPT: 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a # RUN(GONE): image Text # GONE-NOT: 71 71 71 71 71 71 71 71 71 71 71 71 71 71 71 71 +# GONE-NOT: coded qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq # RUN: run-in-shell globalThis.hold = o => { const name = "q".repeat(32); const keys = () => { const all = Object.keys(o); let text = ""; for (let i = 0; i < all.length; i++) text += all[i][0]; return text; }; print(name in o, o[name], keys()); o[name] = 2; print(name in o, o[name], keys()); delete o[name]; print(name in o, keys()); }; # CHECK: false undefined z # CHECK-NEXT: true 2 zq diff --git a/Source/JavaScriptCore/aot/tests/image/name-of-a-variable-that-nothing-names-is-not-in-the-image.ir b/Source/JavaScriptCore/aot/tests/image/name-of-a-variable-that-nothing-names-is-not-in-the-image.ir index 80ffe37e686b4..d56dc6a6f1224 100644 --- a/Source/JavaScriptCore/aot/tests/image/name-of-a-variable-that-nothing-names-is-not-in-the-image.ir +++ b/Source/JavaScriptCore/aot/tests/image/name-of-a-variable-that-nothing-names-is-not-in-the-image.ir @@ -1,11 +1,12 @@ # If the image held the name of every variable that may hold a cell, the name of a function that only left-out code calls could be read out of the executable # function qqq..() { } function zzz..() { } hold(zzz..); # A heap snapshot shows a variable under its name, so the image has the names of the variables under which a cell can lie. Nothing names the first: it gets no word, and no table holds its name. -# Each name is one letter 32 times: wherever the text begins, one line of 16 bytes holds nothing else. +# Each name is one letter 32 times: wherever the text begins, one line of 16 bytes holds nothing else. A text that is only shown is coded and printed as it reads. # RUN(KEPT): image Text -# KEPT: 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a +# KEPT: coded zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz # RUN(GONE): image Text # GONE-NOT: 71 71 71 71 71 71 71 71 71 71 71 71 71 71 71 71 +# GONE-NOT: coded qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq # RUN(LINK): link functions # LINK: function f1 unreached diff --git a/Source/JavaScriptCore/aot/tests/image/name-of-a-variable-whose-function-the-walk-alone-rules-out-is-not-in-the-image.ir b/Source/JavaScriptCore/aot/tests/image/name-of-a-variable-whose-function-the-walk-alone-rules-out-is-not-in-the-image.ir index 57a993bab4a92..bec2e348f252d 100644 --- a/Source/JavaScriptCore/aot/tests/image/name-of-a-variable-whose-function-the-walk-alone-rules-out-is-not-in-the-image.ir +++ b/Source/JavaScriptCore/aot/tests/image/name-of-a-variable-whose-function-the-walk-alone-rules-out-is-not-in-the-image.ir @@ -1,11 +1,12 @@ # The variable of a function that nobody stores into has no word: it is the cell of the function in the closure table, wherever Link reaches the function # function qqq..() { } function zzz..() { } function keep(value: number) { if (typeof value === "string") qqq..(); return value; } hold(zzz.., keep(1)); # Link reaches the first, and the walk rules the call out: no code mentions its closure, so it has no cell, a heap snapshot has no row for its variable, and no table holds the name -# Each name is one letter 32 times: wherever the text begins, one line of 16 bytes holds nothing else. +# Each name is one letter 32 times: wherever the text begins, one line of 16 bytes holds nothing else. A text that is only shown is coded and printed as it reads. # RUN(KEPT): image Text mechanisms=all -# KEPT: 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a 7a +# KEPT: coded zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz # RUN(GONE): image Text mechanisms=all # GONE-NOT: 71 71 71 71 71 71 71 71 71 71 71 71 71 71 71 71 +# GONE-NOT: coded qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq # RUN(LINK): link functions # LINK-NOT: function f1 unreached # RUN(PLAN): plan variables mechanisms=all diff --git a/Source/JavaScriptCore/aot/tests/link/a-bound-function-passes-a-test-for-functions.ir b/Source/JavaScriptCore/aot/tests/link/a-bound-function-passes-a-test-for-functions.ir index 9182fcc46c5a7..ec71048d99b13 100644 --- a/Source/JavaScriptCore/aot/tests/link/a-bound-function-passes-a-test-for-functions.ir +++ b/Source/JavaScriptCore/aot/tests/link/a-bound-function-passes-a-test-for-functions.ir @@ -2,7 +2,7 @@ # A bound function is an object of a class of functions: a test for functions lets it pass, and the call behind the test goes by the rows of Math.floor. Where it is dropped the # call has no callee and the result is the 0 of the other arm alone. # RUN: link functions mechanisms=all -# CHECK: function f1 {{*}}result={{*}}number +# CHECK: function f1 {{*}}result=int|whole module m0 path="main.js" displayPath="main.js" body=f0 entry module=m0 diff --git a/Source/JavaScriptCore/aot/tests/link/a-call-of-a-few-functions-that-met-reaches-them-alone.ir b/Source/JavaScriptCore/aot/tests/link/a-call-of-a-few-functions-that-met-reaches-them-alone.ir index 2def4f3a45d39..d1e1f996783ed 100644 --- a/Source/JavaScriptCore/aot/tests/link/a-call-of-a-few-functions-that-met-reaches-them-alone.ir +++ b/Source/JavaScriptCore/aot/tests/link/a-call-of-a-few-functions-that-met-reaches-them-alone.ir @@ -1,4 +1,4 @@ -# Two to four closures that meet at a place are still named there. If a call of what lies at such a place counted as "lost functions are called", one callback parameter with two callers would bring in the body of every function of the program that has ever met another. +# Two to eight closures that meet at a place are still named there. If a call of what lies at such a place counted as "lost functions are called", one callback parameter with two callers would bring in the body of every function of the program that has ever met another. # function run(f) { return f(); } function look(f) { return typeof f; } run(one); run(two); look(three); look(four); # RUN: link functions mechanisms=all # CHECK: function f3 diff --git a/Source/JavaScriptCore/aot/tests/link/a-few-functions-that-met-reach-unseen-code-without-the-others.ir b/Source/JavaScriptCore/aot/tests/link/a-few-functions-that-met-reach-unseen-code-without-the-others.ir index faa4e5277263c..a5b57657b8494 100644 --- a/Source/JavaScriptCore/aot/tests/link/a-few-functions-that-met-reach-unseen-code-without-the-others.ir +++ b/Source/JavaScriptCore/aot/tests/link/a-few-functions-that-met-reach-unseen-code-without-the-others.ir @@ -1,4 +1,4 @@ -# Two to four closures that meet at a place are still named there. If such a place that leads to unseen code counted as "lost functions have reached unseen code", the first pair of getters in one descriptor would hand out every function of the program that has ever met another. +# Two to eight closures that meet at a place are still named there. If such a place that leads to unseen code counted as "lost functions have reached unseen code", the first pair of getters in one descriptor would hand out every function of the program that has ever met another. # function hand(f) { sink(f); } function look(f) { return typeof f; } hand(one); hand(two); look(three); look(four); # What `hand` is handed is one or two, and nothing else. They escape. three and four have met each other alone, and nobody calls them. # RUN: link functions mechanisms=all diff --git a/Source/JavaScriptCore/aot/tests/link/bit-operation-gives-int32.ir b/Source/JavaScriptCore/aot/tests/link/bit-operation-gives-int32.ir index e2dc4ca4bf999..3c673f72547da 100644 --- a/Source/JavaScriptCore/aot/tests/link/bit-operation-gives-int32.ir +++ b/Source/JavaScriptCore/aot/tests/link/bit-operation-gives-int32.ir @@ -4,7 +4,7 @@ # CHECK: function f1 {{*}}result=i32 returns # CHECK-NEXT: parameter 0 {{*}}proven=number # CHECK: function f2 {{*}}result=int returns -# CHECK: function f3 {{*}}result=int returns +# CHECK: function f3 {{*}}result=int[0, 4294967295] returns # CHECK: function f4 {{*}}result=i32 returns # CHECK-NEXT: parameter 0 {{*}}proven=i32 # CHECK: function f5 {{*}}result=i32 returns diff --git a/Source/JavaScriptCore/aot/tests/link/families-of-functions/a-function-that-has-met-both-unites-their-families.ir b/Source/JavaScriptCore/aot/tests/link/families-of-functions/a-function-that-has-met-both-unites-their-families.ir index c3da0341a1eb1..d8fb6ab700cf2 100644 --- a/Source/JavaScriptCore/aot/tests/link/families-of-functions/a-function-that-has-met-both-unites-their-families.ir +++ b/Source/JavaScriptCore/aot/tests/link/families-of-functions/a-function-that-has-met-both-unites-their-families.ir @@ -1,5 +1,5 @@ -# The first function has met four others at one place and five others at another: all ten are one family. If the two rows of parameters were not tied when the families are united, the text would not come to the second, with which the first may be confused where numbers are passed, nor the number to the eighth. -# callFew(, 1); callMany(, "text"); +# The first function has met eight others at one place and nine others at another: all eighteen are one family. If the two rows of parameters were not tied when the families are united, the text would not come to the second, with which the first may be confused where numbers are passed, nor the number to the eighth. +# callFew(, 1); callMany(, "text"); # RUN(HELD): link held functions mechanisms=all # HELD: function f3 # HELD-NEXT: parameter 0 expected=undefined|i32|string proven=undefined|i32|string @@ -26,19 +26,35 @@ function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anythi v11 = NewClosure function=f12 v12 = Integer integer=1 v13 = String string="text" - v14 = Call v0, -, v2, v12 calleeName="callFew" - v15 = Call v0, -, v3, v12 calleeName="callFew" - v16 = Call v0, -, v4, v12 calleeName="callFew" - v17 = Call v0, -, v5, v12 calleeName="callFew" - v18 = Call v0, -, v6, v12 calleeName="callFew" - v19 = Call v1, -, v2, v13 calleeName="callMany" - v20 = Call v1, -, v7, v13 calleeName="callMany" - v21 = Call v1, -, v8, v13 calleeName="callMany" - v22 = Call v1, -, v9, v13 calleeName="callMany" - v23 = Call v1, -, v10, v13 calleeName="callMany" - v24 = Call v1, -, v11, v13 calleeName="callMany" - v25 = Undefined - Return v25 + v14 = NewClosure function=f13 + v15 = NewClosure function=f14 + v16 = NewClosure function=f15 + v17 = NewClosure function=f16 + v18 = NewClosure function=f17 + v19 = NewClosure function=f18 + v20 = NewClosure function=f19 + v21 = NewClosure function=f20 + v22 = Call v0, -, v2, v12 calleeName="callFew" + v23 = Call v0, -, v3, v12 calleeName="callFew" + v24 = Call v0, -, v4, v12 calleeName="callFew" + v25 = Call v0, -, v5, v12 calleeName="callFew" + v26 = Call v0, -, v6, v12 calleeName="callFew" + v27 = Call v0, -, v14, v12 calleeName="callFew" + v28 = Call v0, -, v15, v12 calleeName="callFew" + v29 = Call v0, -, v16, v12 calleeName="callFew" + v30 = Call v0, -, v17, v12 calleeName="callFew" + v31 = Call v1, -, v2, v13 calleeName="callMany" + v32 = Call v1, -, v7, v13 calleeName="callMany" + v33 = Call v1, -, v8, v13 calleeName="callMany" + v34 = Call v1, -, v9, v13 calleeName="callMany" + v35 = Call v1, -, v10, v13 calleeName="callMany" + v36 = Call v1, -, v11, v13 calleeName="callMany" + v37 = Call v1, -, v18, v13 calleeName="callMany" + v38 = Call v1, -, v19, v13 calleeName="callMany" + v39 = Call v1, -, v20, v13 calleeName="callMany" + v40 = Call v1, -, v21, v13 calleeName="callMany" + v41 = Undefined + Return v41 } function f1 name="callFew" module=m0 length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { b0(v0: anything, v1: anything): @@ -90,3 +106,35 @@ function f12 module=m0 length=1 thisType={type=anything} parameters=[{type=anyth b0(v0: anything): Return v0 } +function f13 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f14 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f15 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f16 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f17 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f18 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f19 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f20 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} diff --git a/Source/JavaScriptCore/aot/tests/link/families-of-functions/functions-that-have-never-met-share-no-parameters.ir b/Source/JavaScriptCore/aot/tests/link/families-of-functions/functions-that-have-never-met-share-no-parameters.ir index b307629f3b4e9..7ecacd22d7679 100644 --- a/Source/JavaScriptCore/aot/tests/link/families-of-functions/functions-that-have-never-met-share-no-parameters.ir +++ b/Source/JavaScriptCore/aot/tests/link/families-of-functions/functions-that-have-never-met-share-no-parameters.ir @@ -1,6 +1,6 @@ -# Functions that meet are united into a family, and each family has a receiver, a row of parameters and a result of its own. If all that have ever met anybody shared one row, the number that is passed to five of them would arrive at the other five, and their text at the first five. +# Functions that meet are united into a family, and each family has a receiver, a row of parameters and a result of its own. If all that have ever met anybody shared one row, the number that is passed to nine of them would arrive at the other nine, and their text at the first nine. # A call may pass fewer arguments than a member takes, so `undefined` is among what arrives. -# function callFew(held, x) { return held(x); } function callMany(held, x) { return held(x); } callFew(, 1); callMany(, "text"); +# function callFew(held, x) { return held(x); } function callMany(held, x) { return held(x); } callFew(, 1); callMany(, "text"); # RUN(HELD): link held functions mechanisms=all # HELD: function f3 # HELD-NEXT: parameter 0 expected=undefined|i32 proven=undefined|i32 @@ -29,18 +29,34 @@ function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anythi v11 = NewClosure function=f12 v12 = Integer integer=1 v13 = String string="text" - v14 = Call v0, -, v2, v12 calleeName="callFew" - v15 = Call v0, -, v3, v12 calleeName="callFew" - v16 = Call v0, -, v4, v12 calleeName="callFew" - v17 = Call v0, -, v5, v12 calleeName="callFew" - v18 = Call v0, -, v6, v12 calleeName="callFew" - v19 = Call v1, -, v7, v13 calleeName="callMany" - v20 = Call v1, -, v8, v13 calleeName="callMany" - v21 = Call v1, -, v9, v13 calleeName="callMany" - v22 = Call v1, -, v10, v13 calleeName="callMany" - v23 = Call v1, -, v11, v13 calleeName="callMany" - v24 = Undefined - Return v24 + v14 = NewClosure function=f13 + v15 = NewClosure function=f14 + v16 = NewClosure function=f15 + v17 = NewClosure function=f16 + v18 = NewClosure function=f17 + v19 = NewClosure function=f18 + v20 = NewClosure function=f19 + v21 = NewClosure function=f20 + v22 = Call v0, -, v2, v12 calleeName="callFew" + v23 = Call v0, -, v3, v12 calleeName="callFew" + v24 = Call v0, -, v4, v12 calleeName="callFew" + v25 = Call v0, -, v5, v12 calleeName="callFew" + v26 = Call v0, -, v6, v12 calleeName="callFew" + v27 = Call v0, -, v14, v12 calleeName="callFew" + v28 = Call v0, -, v15, v12 calleeName="callFew" + v29 = Call v0, -, v16, v12 calleeName="callFew" + v30 = Call v0, -, v17, v12 calleeName="callFew" + v31 = Call v1, -, v7, v13 calleeName="callMany" + v32 = Call v1, -, v8, v13 calleeName="callMany" + v33 = Call v1, -, v9, v13 calleeName="callMany" + v34 = Call v1, -, v10, v13 calleeName="callMany" + v35 = Call v1, -, v11, v13 calleeName="callMany" + v36 = Call v1, -, v18, v13 calleeName="callMany" + v37 = Call v1, -, v19, v13 calleeName="callMany" + v38 = Call v1, -, v20, v13 calleeName="callMany" + v39 = Call v1, -, v21, v13 calleeName="callMany" + v40 = Undefined + Return v40 } function f1 name="callFew" module=m0 length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { b0(v0: anything, v1: anything): @@ -92,3 +108,35 @@ function f12 module=m0 length=1 thisType={type=anything} parameters=[{type=anyth b0(v0: anything): Return v0 } +function f13 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f14 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f15 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f16 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f17 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f18 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f19 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f20 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} diff --git a/Source/JavaScriptCore/aot/tests/link/families-of-functions/functions-that-one-helper-only-names-are-not-united-there.ir b/Source/JavaScriptCore/aot/tests/link/families-of-functions/functions-that-one-helper-only-names-are-not-united-there.ir index 8aaa0963283f4..d3a234fc990d8 100644 --- a/Source/JavaScriptCore/aot/tests/link/families-of-functions/functions-that-one-helper-only-names-are-not-united-there.ir +++ b/Source/JavaScriptCore/aot/tests/link/families-of-functions/functions-that-one-helper-only-names-are-not-united-there.ir @@ -1,4 +1,4 @@ -# `label` only names what it is handed. Five functions that take numbers have met at `few`, five that take texts at `more`; one has met nobody. All eleven are handed to `label`. +# `label` only names what it is handed. Nine functions that take numbers have met at `few`, nine that take texts at `more`; one has met nobody. All nineteen are handed to `label`. # Being named loses nobody and unites nobody: what is passed on only to be named arrives as "some function", without the name of a family. If a value that is lost already kept its name there, the two families would be united at the parameter of `label`, or not, by whether the one that has met nobody came first: one program, two answers. # function label(f, text) { Object.defineProperty(f, "name", { value: text }); } label(few(..), "few"); label(more(..), "more"); label(pass(pass(pass(pass(alone)))), "alone"); few(..)(1); more(..)("t"); # RUN(HELD): link held functions mechanisms=all @@ -36,32 +36,48 @@ function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anythi v12 = NewClosure function=f13 v13 = NewClosure function=f14 v14 = NewClosure function=f15 - v15 = Call v1, -, v3 calleeName="few" - v16 = Call v1, -, v4 calleeName="few" - v17 = Call v1, -, v5 calleeName="few" - v18 = Call v1, -, v6 calleeName="few" - v19 = Call v1, -, v7 calleeName="few" - v20 = Call v2, -, v8 calleeName="more" - v21 = Call v2, -, v9 calleeName="more" - v22 = Call v2, -, v10 calleeName="more" - v23 = Call v2, -, v11 calleeName="more" - v24 = Call v2, -, v12 calleeName="more" - v25 = String string="few" - v26 = String string="more" - v27 = String string="alone" - v28 = Call v0, -, v15, v25 calleeName="label" - v29 = Call v0, -, v20, v26 calleeName="label" - v30 = Call v14, -, v13 calleeName="pass" - v31 = Call v14, -, v30 calleeName="pass" - v32 = Call v14, -, v31 calleeName="pass" - v33 = Call v14, -, v32 calleeName="pass" - v34 = Call v0, -, v33, v27 calleeName="label" - v35 = Integer integer=1 - v36 = String string="t" - v37 = Call v15, -, v35 - v38 = Call v20, -, v36 - v39 = Undefined - Return v39 + v15 = NewClosure function=f16 + v16 = NewClosure function=f17 + v17 = NewClosure function=f18 + v18 = NewClosure function=f19 + v19 = NewClosure function=f20 + v20 = NewClosure function=f21 + v21 = NewClosure function=f22 + v22 = NewClosure function=f23 + v23 = Call v1, -, v3 calleeName="few" + v24 = Call v1, -, v4 calleeName="few" + v25 = Call v1, -, v5 calleeName="few" + v26 = Call v1, -, v6 calleeName="few" + v27 = Call v1, -, v7 calleeName="few" + v28 = Call v1, -, v15 calleeName="few" + v29 = Call v1, -, v16 calleeName="few" + v30 = Call v1, -, v17 calleeName="few" + v31 = Call v1, -, v18 calleeName="few" + v32 = Call v2, -, v8 calleeName="more" + v33 = Call v2, -, v9 calleeName="more" + v34 = Call v2, -, v10 calleeName="more" + v35 = Call v2, -, v11 calleeName="more" + v36 = Call v2, -, v12 calleeName="more" + v37 = Call v2, -, v19 calleeName="more" + v38 = Call v2, -, v20 calleeName="more" + v39 = Call v2, -, v21 calleeName="more" + v40 = Call v2, -, v22 calleeName="more" + v41 = String string="few" + v42 = String string="more" + v43 = String string="alone" + v44 = Call v0, -, v23, v41 calleeName="label" + v45 = Call v0, -, v32, v42 calleeName="label" + v46 = Call v14, -, v13 calleeName="pass" + v47 = Call v14, -, v46 calleeName="pass" + v48 = Call v14, -, v47 calleeName="pass" + v49 = Call v14, -, v48 calleeName="pass" + v50 = Call v0, -, v49, v43 calleeName="label" + v51 = Integer integer=1 + v52 = String string="t" + v53 = Call v23, -, v51 + v54 = Call v32, -, v52 + v55 = Undefined + Return v55 } function f1 name="label" module=m0 length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { b0(v0: anything, v1: anything): @@ -131,3 +147,35 @@ function f15 name="pass" module=m0 length=1 thisType={type=anything} parameters= b0(v0: anything): Return v0 } +function f16 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f17 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f18 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f19 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f20 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f21 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f22 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f23 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} diff --git a/Source/JavaScriptCore/aot/tests/link/families-of-functions/one-family-goes-out-to-unseen-code-and-the-others-stay-in.ir b/Source/JavaScriptCore/aot/tests/link/families-of-functions/one-family-goes-out-to-unseen-code-and-the-others-stay-in.ir index 540ffc07e157f..5f0c44a66e6fa 100644 --- a/Source/JavaScriptCore/aot/tests/link/families-of-functions/one-family-goes-out-to-unseen-code-and-the-others-stay-in.ir +++ b/Source/JavaScriptCore/aot/tests/link/families-of-functions/one-family-goes-out-to-unseen-code-and-the-others-stay-in.ir @@ -1,6 +1,6 @@ -# A value of which nobody says any longer who it is comes to unseen code: it is one of its family, here `K` or one of four functions. Five other functions have met one another alone: unseen code cannot have them. +# A value of which nobody says any longer who it is comes to unseen code: it is one of its family, here `K` or one of eight functions. Nine other functions have met one another alone: unseen code cannot have them. # If all that have ever met went out with it, one class beside a few functions anywhere in a program would hand every callback of the program whatever unseen code may pass. -# class K {} callFew(, 1); give(K); give(); function give(held) { sink(held); } +# class K {} callFew(, 1); give(K); give(); function give(held) { sink(held); } # RUN: link functions mechanisms=all # CHECK: function f3 {{*}}result=undefined|int[0, 2147483647] returns= # CHECK-NEXT: parameter 0 expected=undefined|i32 @@ -30,18 +30,34 @@ function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anythi v12 = NewClass v11 layout=L0 v13 = FinishClass v12 layout=L0 v14 = Integer integer=1 - v15 = Call v0, -, v2, v14 calleeName="callFew" - v16 = Call v0, -, v3, v14 calleeName="callFew" - v17 = Call v0, -, v4, v14 calleeName="callFew" - v18 = Call v0, -, v5, v14 calleeName="callFew" - v19 = Call v0, -, v6, v14 calleeName="callFew" - v20 = Call v1, -, v13 calleeName="give" - v21 = Call v1, -, v7 calleeName="give" - v22 = Call v1, -, v8 calleeName="give" - v23 = Call v1, -, v9 calleeName="give" - v24 = Call v1, -, v10 calleeName="give" - v25 = Undefined - Return v25 + v15 = NewClosure function=f13 + v16 = NewClosure function=f14 + v17 = NewClosure function=f15 + v18 = NewClosure function=f16 + v19 = NewClosure function=f17 + v20 = NewClosure function=f18 + v21 = NewClosure function=f19 + v22 = NewClosure function=f20 + v23 = Call v0, -, v2, v14 calleeName="callFew" + v24 = Call v0, -, v3, v14 calleeName="callFew" + v25 = Call v0, -, v4, v14 calleeName="callFew" + v26 = Call v0, -, v5, v14 calleeName="callFew" + v27 = Call v0, -, v6, v14 calleeName="callFew" + v28 = Call v0, -, v15, v14 calleeName="callFew" + v29 = Call v0, -, v16, v14 calleeName="callFew" + v30 = Call v0, -, v17, v14 calleeName="callFew" + v31 = Call v0, -, v18, v14 calleeName="callFew" + v32 = Call v1, -, v13 calleeName="give" + v33 = Call v1, -, v7 calleeName="give" + v34 = Call v1, -, v8 calleeName="give" + v35 = Call v1, -, v9 calleeName="give" + v36 = Call v1, -, v10 calleeName="give" + v37 = Call v1, -, v19 calleeName="give" + v38 = Call v1, -, v20 calleeName="give" + v39 = Call v1, -, v21 calleeName="give" + v40 = Call v1, -, v22 calleeName="give" + v41 = Undefined + Return v41 } function f1 name="callFew" module=m0 length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { b0(v0: anything, v1: anything): @@ -97,3 +113,35 @@ function f12 name="K" module=m0 kind=BaseConstructor flags=[HasNewTarget, HasExp v2 = NewObject v1 layout=L0 Return v2 } +function f13 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f14 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f15 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f16 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f17 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f18 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f19 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f20 module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} diff --git a/Source/JavaScriptCore/aot/tests/link/functions-that-met-are-reached-where-one-of-them-is-called.ir b/Source/JavaScriptCore/aot/tests/link/functions-that-met-are-reached-where-one-of-them-is-called.ir index 9a58df4a58023..1468331f8f984 100644 --- a/Source/JavaScriptCore/aot/tests/link/functions-that-met-are-reached-where-one-of-them-is-called.ir +++ b/Source/JavaScriptCore/aot/tests/link/functions-that-met-are-reached-where-one-of-them-is-called.ir @@ -1,5 +1,5 @@ -# More than four functions that met at one place are no longer told apart, so a call of what lies at such a place may enter any function that is lost. Two that met elsewhere are still named, and nobody has lost them. If they stayed so where what lies there flows on into the place of the many, `six` would be entered without code. -# function run(f) { return f(); } function pass(f) { return f; } run(one); run(two); run(three); run(four); run(five); pass(five); run(pass(six)); +# More than eight functions that met at one place are no longer told apart, so a call of what lies at such a place may enter any function that is lost. Two that met elsewhere are still named, and nobody has lost them. If they stayed so where what lies there flows on into the place of the many, `six` would be entered without code. +# function run(f) { return f(); } function pass(f) { return f; } run(one); run(two); run(three); run(four); run(five); run(); pass(five); run(pass(six)); # RUN: link functions mechanisms=all # CHECK: function f3 # CHECK-NOT: unreached @@ -22,16 +22,24 @@ function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anythi v5 = NewClosure function=f6 v6 = NewClosure function=f7 v7 = NewClosure function=f8 - v8 = Call v0, -, v2 calleeName="run" - v9 = Call v0, -, v3 calleeName="run" - v10 = Call v0, -, v4 calleeName="run" - v11 = Call v0, -, v5 calleeName="run" - v12 = Call v0, -, v6 calleeName="run" - v13 = Call v1, -, v6 calleeName="pass" - v14 = Call v1, -, v7 calleeName="pass" - v15 = Call v0, -, v14 calleeName="run" - v16 = Undefined - Return v16 + v8 = NewClosure function=f9 + v9 = NewClosure function=f10 + v10 = NewClosure function=f11 + v11 = NewClosure function=f12 + v12 = Call v0, -, v2 calleeName="run" + v13 = Call v0, -, v3 calleeName="run" + v14 = Call v0, -, v4 calleeName="run" + v15 = Call v0, -, v5 calleeName="run" + v16 = Call v0, -, v6 calleeName="run" + v17 = Call v0, -, v8 calleeName="run" + v18 = Call v0, -, v9 calleeName="run" + v19 = Call v0, -, v10 calleeName="run" + v20 = Call v0, -, v11 calleeName="run" + v21 = Call v1, -, v6 calleeName="pass" + v22 = Call v1, -, v7 calleeName="pass" + v23 = Call v0, -, v22 calleeName="run" + v24 = Undefined + Return v24 } function f1 name="run" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { b0(v0: anything): @@ -72,3 +80,23 @@ function f8 name="six" module=m0 thisType={type=anything} results=[{type=anythin v0 = Integer integer=6 Return v0 } +function f9 name="more9" module=m0 thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = Integer integer=5 + Return v0 +} +function f10 name="more10" module=m0 thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = Integer integer=5 + Return v0 +} +function f11 name="more11" module=m0 thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = Integer integer=5 + Return v0 +} +function f12 name="more12" module=m0 thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = Integer integer=5 + Return v0 +} diff --git a/Source/JavaScriptCore/aot/tests/link/held-contents-of-escaped-receivers.ir b/Source/JavaScriptCore/aot/tests/link/held-contents-of-escaped-receivers.ir new file mode 100644 index 0000000000000..7a6c7e9ddb4e0 --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/link/held-contents-of-escaped-receivers.ir @@ -0,0 +1,126 @@ +# class Box { constructor() { this.v = null; } } +# class Store { +# constructor() { this.values = new Map(); this.items = []; this.found = null; this.picked = null; this.front = null; } +# put(key, at) { this.values.set(key, new Box()); this.items[at] = new Box(); } +# take(key) { this.found = this.values.get(key); } +# pick(at) { this.picked = this.items[at]; } +# get best() { return this.items[0]; } +# look() { this.front = this.best; } +# } +# The store escapes. The held reading leaves out what reaches these fields through the escape alone: primitive receivers of get(), other getters under best, other keys of items. +# RUN: link held fields mechanisms=all +# CHECK: field L1.2 "found" proven=undefined|null|L0 +# CHECK: field L1.3 "picked" proven=undefined|null|L0 +# CHECK: field L1.4 "front" proven=undefined|null|L0 + +layout L0 kind=Instance name="Box" module=m0 position=20 constructor=f1 fields=[{key="v" declared={type=anything} flags=[IsRequired]}] place=20 ownFieldCount=1 +layout L1 kind=Instance name="Store" module=m0 position=67 constructor=f2 fields=[{key="values" declared={type=anything} flags=[IsRequired]}, {key="items" declared={type=anything} flags=[IsRequired]}, {key="found" declared={type=anything} flags=[IsRequired]}, {key="picked" declared={type=anything} flags=[IsRequired]}, {key="front" declared={type=anything} flags=[IsRequired]}] place=67 ownFieldCount=5 +variable g0 name="Box" module=m0 declared={type=absent|f1 flags=[IsFromCompiler]} flags=[IsImmutable] place=1 +module m0 path="main.js" displayPath="main.js" body=f0 +entry module=m0 +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = NewClosure function=f1 + v1 = NewClass v0 layout=L0 + v2 = FinishClass v1 layout=L0 + v3 = Convert v2 declared={type=f1 flags=[IsFromCompiler]} + PutModuleVariable v3 variable=g0 + v5 = NewClosure function=f2 + v6 = NewClass v5 layout=L1 + v7 = NewClosure function=f3 + DefineMethod v6, v7 layout=L1 key="put" + v9 = NewClosure function=f4 + DefineMethod v6, v9 layout=L1 index=1 key="take" + v11 = NewClosure function=f5 + DefineMethod v6, v11 layout=L1 index=2 key="pick" + v13 = NewClosure function=f6 + DefineMethod v6, v13 layout=L1 methodKind=Getter index=3 key="best" + v15 = NewClosure function=f7 + DefineMethod v6, v15 layout=L1 index=4 key="look" + v17 = FinishClass v6 layout=L1 + v18 = Convert v17 declared={type=f2 flags=[IsFromCompiler]} + v19 = Construct v18 calleeName="Store" + v20 = Integer integer=1 + v21 = PropGet v19 key="put" + v22 = Call v21, v19, v20, v20 calleeKey="put" + v23 = PropGet v19 key="take" + v24 = Call v23, v19, v20 calleeKey="take" + v25 = PropGet v19 key="pick" + v26 = Call v25, v19, v20 calleeKey="pick" + v27 = PropGet v19 key="look" + v28 = Call v27, v19 calleeKey="look" + v29 = Builtin builtin=GlobalThis + PropSet v29, v19 key="kept" + v31 = Undefined + Return v31 +} +function f1 name="Box" module=m0 position=30 kind=BaseConstructor flags=[HasNewTarget, HasExplicitName] thisType={type=anything} results=[{type=object flags=[IsFromCompiler]}] { + b0(v0: anything): + v1 = GetConstructPrototype v0 layout=L0 + v2 = NewObject v1 layout=L0 + v3 = Convert v2 declared={type=L0} + v4 = Null + PropSet v3, v4 key="v" + Return v3 +} +function f2 name="Store" module=m0 position=88 kind=BaseConstructor flags=[HasNewTarget, HasExplicitName] thisType={type=anything} results=[{type=object flags=[IsFromCompiler]}] { + b0(v0: anything): + v1 = GetConstructPrototype v0 layout=L1 + v2 = NewObject v1 layout=L1 + v3 = Convert v2 declared={type=L1} + v4 = Builtin builtin=Map + v5 = Construct v4 calleeName="Map" + PropSet v3, v5 key="values" + v7 = NewArray contents=anything + PropSet v3, v7 key="items" + v9 = Null + PropSet v3, v9 key="found" + PropSet v3, v9 key="picked" + PropSet v3, v9 key="front" + Return v3 +} +function f3 name="put" module=m0 position=134 kind=Method flags=[HasThis, HasExplicitName] length=2 thisType={type=L1 flags=[IsExpected]} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { + b0(v0: L1, v1: anything, v2: anything): + v3 = PropGet v0 key="values" + v4 = PropGet v3 key="set" + v5 = GetModuleVariable variable=g0 + v6 = CheckInitialized v5 name="Box" + v7 = Construct v6 calleeName="Box" + v8 = Call v4, v3, v1, v7 calleeKey="set" + v9 = PropGet v0 key="items" + v10 = Construct v6 calleeName="Box" + ElemSet v9, v2, v10 + v12 = Undefined + Return v12 +} +function f4 name="take" module=m0 position=176 kind=Method flags=[HasThis, HasExplicitName] length=1 thisType={type=L1 flags=[IsExpected]} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: L1, v1: anything): + v2 = PropGet v0 key="values" + v3 = PropGet v2 key="get" + v4 = Call v3, v2, v1 calleeKey="get" + PropSet v0, v4 key="found" + v6 = Undefined + Return v6 +} +function f5 name="pick" module=m0 position=226 kind=Method flags=[HasThis, HasExplicitName] length=1 thisType={type=L1 flags=[IsExpected]} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: L1, v1: anything): + v2 = PropGet v0 key="items" + v3 = ElemGet v2, v1 + PropSet v0, v3 key="picked" + v5 = Undefined + Return v5 +} +function f6 name="best" module=m0 position=266 kind=Getter flags=[HasThis, HasExplicitName] thisType={type=L1 flags=[IsExpected]} results=[{type=anything}] { + b0(v0: L1): + v1 = PropGet v0 key="items" + v2 = Integer + v3 = ElemGet v1, v2 + Return v3 +} +function f7 name="look" module=m0 position=306 kind=Method flags=[HasThis, HasExplicitName] thisType={type=L1 flags=[IsExpected]} results=[{type=anything}] { + b0(v0: L1): + v1 = PropGet v0 key="best" + PropSet v0, v1 key="front" + v3 = Undefined + Return v3 +} diff --git a/Source/JavaScriptCore/aot/tests/link/one-bound-function-alone-is-called-by-the-rows-of-its-target.ir b/Source/JavaScriptCore/aot/tests/link/one-bound-function-alone-is-called-by-the-rows-of-its-target.ir index c8a4e1d5ca5d2..553b514055ad6 100644 --- a/Source/JavaScriptCore/aot/tests/link/one-bound-function-alone-is-called-by-the-rows-of-its-target.ir +++ b/Source/JavaScriptCore/aot/tests/link/one-bound-function-alone-is-called-by-the-rows-of-its-target.ir @@ -1,7 +1,7 @@ # const act = Math.floor.bind(null); act(65.5) # The control of two-bound-functions-that-meet-at-one-call-are-called-as-unseen-code.ir: one bound function over a built-in function alone is called by the rows of its target. # RUN: link functions mechanisms=all -# CHECK: function f0 {{*}}result=number +# CHECK: function f0 {{*}}result=int|whole module m0 path="main.js" displayPath="main.js" body=f0 entry module=m0 diff --git a/Source/JavaScriptCore/aot/tests/link/regexp-texts/export-under-an-index.ir b/Source/JavaScriptCore/aot/tests/link/regexp-texts/export-under-an-index.ir new file mode 100644 index 0000000000000..b8bf0fd20e051 --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/link/regexp-texts/export-under-an-index.ir @@ -0,0 +1,31 @@ +# words.mjs: const word = "open"; export { word as "0" }; main.mjs: import * as words from "./words.mjs"; new RegExp("^" + words[0] + "$"); +# RUN: regexp-texts mechanisms=all +# CHECK-COUNT-1: text / +# RUN(TEXT): regexp-texts mechanisms=all +# TEXT: text /^open$/ + +module m0 path="words.mjs" displayPath="words.mjs" body=f0 exports=[{name="0" variable=g0}] namespaceLayout=L(m0) +module m1 path="main.mjs" displayPath="main.mjs" body=f1 requests=[{target=m0 completes=true}] namespaceLayout=L(m1) +entry module=m1 +variable g0 name="word" module=m0 declared={type=anything|absent} flags=[IsImmutable] +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] capabilities=[MayChangeSignature, MayBeCloned, MayBeInlined, HasOrdinaryFrame, IsNeverCalledByHost] { + b0(): + v0 = String string="open" @13 + PutModuleVariable v0 variable=g0 @6 + v2 = Undefined @36 + Return v2 @36 +} +function f1 module=m1 kind=Module thisType={type=anything} results=[{type=anything}] capabilities=[MayChangeSignature, MayBeCloned, MayBeInlined, HasOrdinaryFrame, IsNeverCalledByHost] { + b0(): + v0 = ModuleNamespace module=m0 @14 + v1 = Integer @46 + v2 = ElemGet v0, v1 @46 + v3 = Builtin builtin=RegExp @83 + v4 = String string="^" @90 + v5 = Add v4, v2 @90 + v6 = String string="$" @103 + v7 = Add v5, v6 @90 + v8 = Construct v3, -, v7 calleeName="RegExp" @79 + v9 = Undefined @67 + Return v9 @67 +} diff --git a/Source/JavaScriptCore/aot/tests/link/sets-of-closures/eight-closures-are-a-set.ir b/Source/JavaScriptCore/aot/tests/link/sets-of-closures/eight-closures-are-a-set.ir new file mode 100644 index 0000000000000..3d5b85173d633 --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/link/sets-of-closures/eight-closures-are-a-set.ir @@ -0,0 +1,71 @@ +# Eight is the most that a set holds: if the eighth took the set apart, a table of eight handlers would be an unknown function, and the eight would be lost. +# take(first); take(second); take(third); take(fourth); take(); +# RUN: link held functions mechanisms=all +# CHECK: function f1 +# CHECK-NEXT: parameter 0 {{*}}proven=f2|f3|f4|f5|f6|f7|f8|f9 +# RUN(KNOWN): link functions mechanisms=all +# KNOWN: function f5 callers-known +# KNOWN-NEXT: parameter 0 {{*}}proven=int[0, 2147483647] + +module m0 path="main.js" displayPath="main.js" body=f0 +entry module=m0 +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = NewClosure function=f1 + v1 = NewClosure function=f2 + v2 = NewClosure function=f3 + v3 = NewClosure function=f4 + v4 = NewClosure function=f5 + v5 = NewClosure function=f6 + v6 = NewClosure function=f7 + v7 = NewClosure function=f8 + v8 = NewClosure function=f9 + v9 = Call v0, -, v1 calleeName="take" + v10 = Call v0, -, v2 calleeName="take" + v11 = Call v0, -, v3 calleeName="take" + v12 = Call v0, -, v4 calleeName="take" + v13 = Call v0, -, v5 calleeName="take" + v14 = Call v0, -, v6 calleeName="take" + v15 = Call v0, -, v7 calleeName="take" + v16 = Call v0, -, v8 calleeName="take" + v17 = Undefined + Return v17 +} +function f1 name="take" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + v1 = Integer integer=1 + v2 = Call v0, -, v1 calleeName="f" + Return v2 +} +function f2 name="first" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f3 name="second" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f4 name="third" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f5 name="fourth" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f6 name="more6" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f7 name="more7" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f8 name="more8" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f9 name="more9" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} diff --git a/Source/JavaScriptCore/aot/tests/link/sets-of-closures/four-closures-are-a-set.ir b/Source/JavaScriptCore/aot/tests/link/sets-of-closures/four-closures-are-a-set.ir deleted file mode 100644 index dfb1b5b627e81..0000000000000 --- a/Source/JavaScriptCore/aot/tests/link/sets-of-closures/four-closures-are-a-set.ir +++ /dev/null @@ -1,47 +0,0 @@ -# Four is the most that a set holds: if the fourth took the set apart, a table of four handlers would be an unknown function, and the four would be lost. -# take(first); take(second); take(third); take(fourth); -# RUN: link held functions mechanisms=all -# CHECK: function f1 -# CHECK-NEXT: parameter 0 {{*}}proven=f2|f3|f4|f5 -# RUN(KNOWN): link functions mechanisms=all -# KNOWN: function f5 callers-known -# KNOWN-NEXT: parameter 0 {{*}}proven=int[0, 2147483647] - -module m0 path="main.js" displayPath="main.js" body=f0 -entry module=m0 -function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { - b0(): - v0 = NewClosure function=f1 - v1 = NewClosure function=f2 - v2 = NewClosure function=f3 - v3 = NewClosure function=f4 - v4 = NewClosure function=f5 - v5 = Call v0, -, v1 calleeName="take" - v6 = Call v0, -, v2 calleeName="take" - v7 = Call v0, -, v3 calleeName="take" - v8 = Call v0, -, v4 calleeName="take" - v9 = Undefined - Return v9 -} -function f1 name="take" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { - b0(v0: anything): - v1 = Integer integer=1 - v2 = Call v0, -, v1 calleeName="f" - Return v2 -} -function f2 name="first" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { - b0(v0: anything): - Return v0 -} -function f3 name="second" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { - b0(v0: anything): - Return v0 -} -function f4 name="third" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { - b0(v0: anything): - Return v0 -} -function f5 name="fourth" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { - b0(v0: anything): - Return v0 -} diff --git a/Source/JavaScriptCore/aot/tests/link/sets-of-closures/five-closures-are-lost.ir b/Source/JavaScriptCore/aot/tests/link/sets-of-closures/nine-closures-are-lost.ir similarity index 54% rename from Source/JavaScriptCore/aot/tests/link/sets-of-closures/five-closures-are-lost.ir rename to Source/JavaScriptCore/aot/tests/link/sets-of-closures/nine-closures-are-lost.ir index 7bdd2cb92fccf..aa45a376cde7c 100644 --- a/Source/JavaScriptCore/aot/tests/link/sets-of-closures/five-closures-are-lost.ir +++ b/Source/JavaScriptCore/aot/tests/link/sets-of-closures/nine-closures-are-lost.ir @@ -1,5 +1,5 @@ -# If the fifth closure were dropped in place of taking the set apart, the type would name four where five may lie, and a row of four tests would have no side for the fifth. -# take(first); take(second); take(third); take(fourth); take(fifth); +# If the ninth closure were dropped in place of taking the set apart, the type would name eight where nine may lie, and a row of eight tests would have no side for the ninth. +# take(first); take(second); take(third); take(fourth); take(fifth); take(); # RUN: link functions mechanisms=all # CHECK: function f1 callers-known # CHECK-NEXT: parameter 0 proven=function? @@ -17,13 +17,21 @@ function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anythi v3 = NewClosure function=f4 v4 = NewClosure function=f5 v5 = NewClosure function=f6 - v6 = Call v0, -, v1 calleeName="take" - v7 = Call v0, -, v2 calleeName="take" - v8 = Call v0, -, v3 calleeName="take" - v9 = Call v0, -, v4 calleeName="take" - v10 = Call v0, -, v5 calleeName="take" - v11 = Undefined - Return v11 + v6 = NewClosure function=f7 + v7 = NewClosure function=f8 + v8 = NewClosure function=f9 + v9 = NewClosure function=f10 + v10 = Call v0, -, v1 calleeName="take" + v11 = Call v0, -, v2 calleeName="take" + v12 = Call v0, -, v3 calleeName="take" + v13 = Call v0, -, v4 calleeName="take" + v14 = Call v0, -, v5 calleeName="take" + v15 = Call v0, -, v6 calleeName="take" + v16 = Call v0, -, v7 calleeName="take" + v17 = Call v0, -, v8 calleeName="take" + v18 = Call v0, -, v9 calleeName="take" + v19 = Undefined + Return v19 } function f1 name="take" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { b0(v0: anything): @@ -51,3 +59,19 @@ function f6 name="fifth" module=m0 length=1 thisType={type=anything} parameters= b0(v0: anything): Return v0 } +function f7 name="more7" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f8 name="more8" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f9 name="more9" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} +function f10 name="more10" module=m0 length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + Return v0 +} diff --git a/Source/JavaScriptCore/aot/tests/link/what-a-call-of-lost-functions-passes-comes-to-whoever-the-rest-list-is-passed-on-to.ir b/Source/JavaScriptCore/aot/tests/link/what-a-call-of-lost-functions-passes-comes-to-whoever-the-rest-list-is-passed-on-to.ir index b9edd684958b5..8c5e79124bb94 100644 --- a/Source/JavaScriptCore/aot/tests/link/what-a-call-of-lost-functions-passes-comes-to-whoever-the-rest-list-is-passed-on-to.ir +++ b/Source/JavaScriptCore/aot/tests/link/what-a-call-of-lost-functions-passes-comes-to-whoever-the-rest-list-is-passed-on-to.ir @@ -1,5 +1,5 @@ -# Five closures meet at the parameter of `run`, so its call is a call of the lost functions, and `wrap` is one of them: it is entered with one argument, `true`. If a function that only passes its rest list on heard of the calls that name it alone, the second parameter of `take` would be proven a string while `run(wrap)` leaves it `undefined`, and the first an `i32` while it is `true`. -# function take(a, b) { return b; } function wrap(...list) { return take(...list); } function run(f) { return f(true); } wrap(1, "x"); run(wrap); run(other4); run(other5); run(other6); run(other7); +# Nine closures meet at the parameter of `run`, so its call is a call of the lost functions, and `wrap` is one of them: it is entered with one argument, `true`. If a function that only passes its rest list on heard of the calls that name it alone, the second parameter of `take` would be proven a string while `run(wrap)` leaves it `undefined`, and the first an `i32` while it is `true`. +# function take(a, b) { return b; } function wrap(...list) { return take(...list); } function run(f) { return f(true); } wrap(1, "x"); run(wrap); run(other4); run(other5); run(other6); run(other7); run(); # RUN: link functions mechanisms=all # CHECK: function f2 callers-known {{*}}result=undefined|true|string returns= # CHECK-NEXT: parameter 0 expected=undefined|true|i32 proven=undefined|true|int[0, 2147483647] @@ -15,19 +15,27 @@ function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anythi v2 = NewClosure function=f1 v3 = Integer integer=1 v4 = String string="x" - v5 = Call v2, -, v3, v4 calleeName="wrap" - v6 = NewClosure function=f3 - v7 = NewClosure function=f4 - v8 = NewClosure function=f5 - v9 = NewClosure function=f6 - v10 = NewClosure function=f7 - v11 = Call v6, -, v2 calleeName="run" - v12 = Call v6, -, v7 calleeName="run" - v13 = Call v6, -, v8 calleeName="run" - v14 = Call v6, -, v9 calleeName="run" - v15 = Call v6, -, v10 calleeName="run" - v16 = Undefined - Return v16 + v5 = NewClosure function=f8 + v6 = NewClosure function=f9 + v7 = NewClosure function=f10 + v8 = NewClosure function=f11 + v9 = Call v2, -, v3, v4 calleeName="wrap" + v10 = NewClosure function=f3 + v11 = NewClosure function=f4 + v12 = NewClosure function=f5 + v13 = NewClosure function=f6 + v14 = NewClosure function=f7 + v15 = Call v10, -, v2 calleeName="run" + v16 = Call v10, -, v11 calleeName="run" + v17 = Call v10, -, v12 calleeName="run" + v18 = Call v10, -, v13 calleeName="run" + v19 = Call v10, -, v14 calleeName="run" + v20 = Call v10, -, v5 calleeName="run" + v21 = Call v10, -, v6 calleeName="run" + v22 = Call v10, -, v7 calleeName="run" + v23 = Call v10, -, v8 calleeName="run" + v24 = Undefined + Return v24 } function f1 name="wrap" module=m0 flags=[HasRestParameter] thisType={type=anything} parameters=[{type=Array flags=[IsFromCompiler]}] results=[{type=anything}] { b0(v0: Array): @@ -65,3 +73,23 @@ function f7 name="other7" module=m0 thisType={type=anything} results=[{type=anyt v0 = Integer integer=7 Return v0 } +function f8 name="more8" module=m0 thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = Integer integer=7 + Return v0 +} +function f9 name="more9" module=m0 thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = Integer integer=7 + Return v0 +} +function f10 name="more10" module=m0 thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = Integer integer=7 + Return v0 +} +function f11 name="more11" module=m0 thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = Integer integer=7 + Return v0 +} diff --git a/Source/JavaScriptCore/aot/tests/machine/every-form-of-an-access-to-elements-that-the-table-lists-has-code.ir b/Source/JavaScriptCore/aot/tests/machine/every-form-of-an-access-to-elements-that-the-table-lists-has-code.ir index c7eabddee0f34..20778a8b2195e 100644 --- a/Source/JavaScriptCore/aot/tests/machine/every-form-of-an-access-to-elements-that-the-table-lists-has-code.ir +++ b/Source/JavaScriptCore/aot/tests/machine/every-form-of-an-access-to-elements-that-the-table-lists-has-code.ir @@ -16,11 +16,11 @@ # CHECK-NEXT: HasArrayStorage arrayStorage=Double: code # CHECK-NEXT: HasArrayStorage arrayStorage=Double [admitsInt32Storage]: code # CHECK-NEXT: HasArrayStorage arrayStorage=Double [isWritable]: code -# CHECK-NEXT: HasArrayStorage arrayStorage=Double [isWritable, admitsInt32Storage]: no code +# CHECK-NEXT: HasArrayStorage arrayStorage=Double [isWritable, admitsInt32Storage]: code # CHECK-NEXT: HasArrayStorage arrayStorage=Contiguous: code # CHECK-NEXT: HasArrayStorage arrayStorage=Contiguous [admitsInt32Storage]: code # CHECK-NEXT: HasArrayStorage arrayStorage=Contiguous [isWritable]: code -# CHECK-NEXT: HasArrayStorage arrayStorage=Contiguous [isWritable, admitsInt32Storage]: no code +# CHECK-NEXT: HasArrayStorage arrayStorage=Contiguous [isWritable, admitsInt32Storage]: code # CHECK-NEXT: IsElementPresent: code # CHECK-NEXT: IsElementPresent [admitsInt32Storage]: no code # CHECK-NEXT: IsElementPresent arrayStorage=Int32: code @@ -28,7 +28,7 @@ # CHECK-NEXT: IsElementPresent arrayStorage=Double: code # CHECK-NEXT: IsElementPresent arrayStorage=Double [admitsInt32Storage]: code # CHECK-NEXT: IsElementPresent arrayStorage=Contiguous: code -# CHECK-NEXT: IsElementPresent arrayStorage=Contiguous [admitsInt32Storage]: no code +# CHECK-NEXT: IsElementPresent arrayStorage=Contiguous [admitsInt32Storage]: code # CHECK-NEXT: GetArrayElement: code # CHECK-NEXT: GetArrayElement [admitsInt32Storage]: no code # CHECK-NEXT: GetArrayElement arrayStorage=Int32: code @@ -36,7 +36,7 @@ # CHECK-NEXT: GetArrayElement arrayStorage=Double: code # CHECK-NEXT: GetArrayElement arrayStorage=Double [admitsInt32Storage]: code # CHECK-NEXT: GetArrayElement arrayStorage=Contiguous: code -# CHECK-NEXT: GetArrayElement arrayStorage=Contiguous [admitsInt32Storage]: no code +# CHECK-NEXT: GetArrayElement arrayStorage=Contiguous [admitsInt32Storage]: code # -- code # code f1:Body { # b0(v0: Array @JSValue, v1: int[0, 100] @JSValue) entry=0: diff --git a/Source/JavaScriptCore/aot/tests/machine/positions-that-the-library-vouches-for-are-tested-in-a-hardened-build.ir b/Source/JavaScriptCore/aot/tests/machine/positions-that-the-library-vouches-for-are-tested-in-a-hardened-build.ir index 1215787046d47..799522b6bf0fb 100644 --- a/Source/JavaScriptCore/aot/tests/machine/positions-that-the-library-vouches-for-are-tested-in-a-hardened-build.ir +++ b/Source/JavaScriptCore/aot/tests/machine/positions-that-the-library-vouches-for-are-tested-in-a-hardened-build.ir @@ -7,7 +7,7 @@ # CHECK: Butterfly_publicLength # CHECK: = Below( # RUN(STOPS): emit f1 mechanisms=all -# STOPS-COUNT-20: effects = Terminal|ControlDependent +# STOPS-COUNT-21: effects = Terminal|ControlDependent # STOPS-NOT: effects = Terminal|ControlDependent module m0 path="main.mjs" displayPath="main.mjs" body=f0 diff --git a/Source/JavaScriptCore/aot/tests/walk/builtin-calls/a-built-in-handed-to-a-method-of-lists-in-a-loop-is-called-in-the-copy-by-its-rows.ir b/Source/JavaScriptCore/aot/tests/walk/builtin-calls/a-built-in-handed-to-a-method-of-lists-in-a-loop-is-called-in-the-copy-by-its-rows.ir index 055b071f748f3..e2ba21debbd6d 100644 --- a/Source/JavaScriptCore/aot/tests/walk/builtin-calls/a-built-in-handed-to-a-method-of-lists-in-a-loop-is-called-in-the-copy-by-its-rows.ir +++ b/Source/JavaScriptCore/aot/tests/walk/builtin-calls/a-built-in-handed-to-a-method-of-lists-in-a-loop-is-called-in-the-copy-by-its-rows.ir @@ -16,7 +16,7 @@ # RUN(ONCE): walk f2 # ONCE-NOT: frame 1 callee=f # ONCE-NOT: TryCallInline -# ONCE: = Call {{*}}calleeKey="every" +# ONCE: = CallDirect {{*}}function=f module m0 path="x.ts" displayPath="x.ts" body=f0 namespaceLayout=L(m0) module m1 path="/$bun/js/engine/ArrayIteratorPrototype.ts" displayPath="/$bun/js/engine/ArrayIteratorPrototype.ts" flags=[IsLibrary] namespaceLayout=L(m1) module m2 path="/$bun/js/engine/ArrayPrototype.ts" displayPath="/$bun/js/engine/ArrayPrototype.ts" flags=[IsLibrary] namespaceLayout=L(m2) diff --git a/Source/JavaScriptCore/aot/tests/walk/builtin-calls/spread-push-in-checked-code.ir b/Source/JavaScriptCore/aot/tests/walk/builtin-calls/spread-push-in-checked-code.ir new file mode 100644 index 0000000000000..0f8da121c7c78 --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/walk/builtin-calls/spread-push-in-checked-code.ir @@ -0,0 +1,53 @@ +# function appendInLoop(out: number[], list: number[], turns: number): number[] { for (let i = 0; i < turns; i++) out.push(...list); return out; } +# function append(out: number[], list: number[]): number[] { out.push(...list); return out; } +# RUN: walk f1 mechanisms=all +# CHECK: AppendElements +# RUN(OUTSIDE): walk f2 mechanisms=all +# OUTSIDE-NOT: AppendElements + +module m0 path="main.ts" displayPath="main.ts" body=f0 +entry module=m0 +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = NewClosure function=f1 + v1 = NewClosure function=f2 + v2 = Integer integer=1 + v3 = NewArray v2 contents=number + v4 = NewArray v2, v2 contents=number + v5 = Call v0, -, v3, v4, v2 calleeName="appendInLoop" [isNeverInlined] + v6 = Call v1, -, v3, v4 calleeName="append" [isNeverInlined] + v7 = Builtin builtin=GlobalThis + v8 = PropGet v7 key="sink" [isFreeName] + v9 = Call v8, -, v5, v6 + Return v9 +} +function f1 name="appendInLoop" module=m0 position=10 flags=[IsChecked, HasExplicitName] length=3 thisType={type=anything} parameters=[{type=undefined|null|Array}, {type=undefined|null|Array}, {type=number}] results=[{type=undefined|null|Array}] { + b0(v0: undefined|null|Array, v1: undefined|null|Array, v2: number): + v3 = Integer + v4 = Convert v3 declared={type=number} + Jump -> next b1(v4) + b1(v5: number): + LoopHint + v7 = Less v5, v2 + Branch v7 -> taken b2(), notTaken b3() + b2(): + v8 = PropGet v0 key="push" + v9 = NewArray contents=anything + AppendSpread v9, v1 + v11 = CallList v8, v0, v9 calleeKey="push" + v12 = Convert v11 declared={type=number} + v13 = Increment v5 + v14 = Convert v13 declared={type=number} + Jump -> next b1(v14) + b3(): + Return v0 +} +function f2 name="append" module=m0 position=200 flags=[IsChecked, HasExplicitName] length=2 thisType={type=anything} parameters=[{type=undefined|null|Array}, {type=undefined|null|Array}] results=[{type=undefined|null|Array}] { + b0(v0: undefined|null|Array, v1: undefined|null|Array): + v2 = PropGet v0 key="push" + v3 = NewArray contents=anything + AppendSpread v3, v1 + v5 = CallList v2, v0, v3 calleeKey="push" + v6 = Convert v5 declared={type=number} + Return v0 +} diff --git a/Source/JavaScriptCore/aot/tests/walk/code-text/array-storage-pointer-read-after-reserve.ir b/Source/JavaScriptCore/aot/tests/walk/code-text/array-storage-pointer-read-after-reserve.ir index fdf76c1d3c0ac..3a691fef86484 100644 --- a/Source/JavaScriptCore/aot/tests/walk/code-text/array-storage-pointer-read-after-reserve.ir +++ b/Source/JavaScriptCore/aot/tests/walk/code-text/array-storage-pointer-read-after-reserve.ir @@ -8,7 +8,7 @@ # v5: opaque(ArrayStoragePointer) @RawWord = GetArrayStorage v0 traits=[mayTouchMemory] # v6 @Void = ReserveArrayCapacity v3, v2 traits=[mayCollect, mayWriteObservableState, mayTouchMemory] # v7: int[0, 4294967295] @Int64 = GetArrayLength v0 traits=[mayTouchMemory] -# v8: boolean @Boolean = IsElementPresent v0, v1, v5, v7 arrayStorage=Double [keyIsIntegral] traits=[mayTouchMemory] +# v8: boolean @Boolean = IsElementPresent v0, v1, v5, -, v7 arrayStorage=Double [keyIsIntegral] traits=[mayTouchMemory] # Return v3 # } # -- end of code diff --git a/Source/JavaScriptCore/aot/tests/walk/code-text/stale-array-storage-pointer-is-rejected.ir b/Source/JavaScriptCore/aot/tests/walk/code-text/stale-array-storage-pointer-is-rejected.ir index 4928b958a85e1..566f6e7003d80 100644 --- a/Source/JavaScriptCore/aot/tests/walk/code-text/stale-array-storage-pointer-is-rejected.ir +++ b/Source/JavaScriptCore/aot/tests/walk/code-text/stale-array-storage-pointer-is-rejected.ir @@ -7,7 +7,7 @@ # v4: opaque(ArrayStoragePointer) @RawWord = GetArrayStorage v0 traits=[mayTouchMemory] # v5 @Void = ReserveArrayCapacity v0, v2 traits=[mayCollect, mayWriteObservableState, mayTouchMemory] # v6: int[0, 4294967295] @Int64 = GetArrayLength v0 traits=[mayTouchMemory] -# v7: boolean @Boolean = IsElementPresent v0, v1, v4, v6 arrayStorage=Double [keyIsIntegral] traits=[mayTouchMemory] +# v7: boolean @Boolean = IsElementPresent v0, v1, v4, -, v6 arrayStorage=Double [keyIsIntegral] traits=[mayTouchMemory] # Return v3 # } # -- end of code diff --git a/Source/JavaScriptCore/aot/tests/walk/code-text/the-question-whether-an-element-is-present-is-held-against-the-length-of-another-list.ir b/Source/JavaScriptCore/aot/tests/walk/code-text/the-question-whether-an-element-is-present-is-held-against-the-length-of-another-list.ir index 3b0e1fec15eb4..cfd84e705c75c 100644 --- a/Source/JavaScriptCore/aot/tests/walk/code-text/the-question-whether-an-element-is-present-is-held-against-the-length-of-another-list.ir +++ b/Source/JavaScriptCore/aot/tests/walk/code-text/the-question-whether-an-element-is-present-is-held-against-the-length-of-another-list.ir @@ -6,7 +6,7 @@ # b0(v0: Array @Cell, v1: Array @Cell, v2: int[0, 100] @Int32, v3: anything @JSValue) entry=0: # v4: opaque(ArrayStoragePointer) @RawWord = GetArrayStorage v0 traits=[mayTouchMemory] # v5: int[0, 4294967295] @Int64 = GetArrayLength v1 traits=[mayTouchMemory] -# v6: boolean @Boolean = IsElementPresent v0, v2, v4, v5 arrayStorage=Double [keyIsIntegral] traits=[mayTouchMemory] +# v6: boolean @Boolean = IsElementPresent v0, v2, v4, -, v5 arrayStorage=Double [keyIsIntegral] traits=[mayTouchMemory] # Return v3 # } # -- end of code diff --git a/Source/JavaScriptCore/aot/tests/walk/direct-calls/list-callback-used-twice.ir b/Source/JavaScriptCore/aot/tests/walk/direct-calls/list-callback-used-twice.ir new file mode 100644 index 0000000000000..2effcd366e56a --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/walk/direct-calls/list-callback-used-twice.ir @@ -0,0 +1,57 @@ +# function user(k, n) { const list = [k]; let found; do { const test = x => sink(x); found = list.some(test, test); } while (n); return found; } sink(user); +# The arrow calls another function and is used twice: the method is called directly, in a loop too, and not copied. +# The library here is a stand-in. +# LIBRARY: own +# RUN: walk f1 mechanisms=all +# CHECK-LABEL: code f1:Body +# CHECK: = NewClosure +# CHECK: = CallDirect {{*}}function=f3 +# RUN(NEVER): walk f1 mechanisms=all +# NEVER-NOT: calleeKey="some" +# NEVER-NOT: callee=f3 + +module m0 path="main.js" displayPath="main.js" body=f0 +module m1 path="/$bun/js/engine/ArrayPrototype.ts" displayPath="/$bun/js/engine/ArrayPrototype.ts" flags=[IsLibrary] +entry module=m0 +runtimefunction id=ArrayPrototypeSome function=f3 +runtimefunction id=ArrayPrototypeSomeOfList function=f2 +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = Builtin builtin=GlobalThis + v1 = PropGet v0 key="sink" [isFreeName] + v2 = NewClosure function=f1 + v3 = Call v1, -, v2 + Return v3 +} +function f1 name="user" module=m0 length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { + b0(v0: anything, v1: anything): + v2 = NewArray v0 contents=anything + Jump -> next b1() + b1(): + LoopHint + v4 = PropGet v2 key="some" + v5 = NewClosure v0 function=f4 + v6 = Call v4, v2, v5, v5 calleeKey="some" calleeName="list.some" + Branch v1 -> taken b1(), notTaken b2() + b2(): + Return v6 +} +function f2 name="someOfList" module=m1 kind=Method role=Library flags=[HasThis, HasExplicitName] length=1 enclosingRuntimeFunction=ArrayPrototypeSomeOfList thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything, v1: anything): + v2 = Undefined + Return v2 +} +function f3 name="some" module=m1 kind=Method role=Library flags=[HasThis, HasExplicitName] length=1 enclosingRuntimeFunction=ArrayPrototypeSome thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything, v1: anything): + v2 = Integer integer=0 + v3 = ElemGet v0, v2 + v4 = Call v1, -, v3 + Return v4 +} +function f4 module=m0 kind=Arrow length=1 captures=[{type=anything variableName="k"}] thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything, v1: anything): + v2 = Builtin builtin=GlobalThis + v3 = PropGet v2 key="sink" [isFreeName] + v4 = Call v3, -, v1 + Return v4 +} diff --git a/Source/JavaScriptCore/aot/tests/walk/direct-calls/method-of-a-list-handed-a-function-that-calls-others-is-called-in-a-loop.ir b/Source/JavaScriptCore/aot/tests/walk/direct-calls/list-callback-with-calls.ir similarity index 86% rename from Source/JavaScriptCore/aot/tests/walk/direct-calls/method-of-a-list-handed-a-function-that-calls-others-is-called-in-a-loop.ir rename to Source/JavaScriptCore/aot/tests/walk/direct-calls/list-callback-with-calls.ir index e438eb4714ada..ff55b9af7a59c 100644 --- a/Source/JavaScriptCore/aot/tests/walk/direct-calls/method-of-a-list-handed-a-function-that-calls-others-is-called-in-a-loop.ir +++ b/Source/JavaScriptCore/aot/tests/walk/direct-calls/list-callback-with-calls.ir @@ -1,15 +1,15 @@ # function user(k, n) { const list = [k]; let found; do { found = list.some(x => sink(x)); } while (n); return found; } sink(user); -# The arrow calls another function, and goes on calling it in a copy: the copy of the method would buy one call of two a turn, for all of its text. So the method is called directly, in a loop too -# (method-of-a-list-with-two-texts-is-copied-in-a-loop.ir has the arrow that calls nobody). +# The arrow calls another function. It is written in place for this call alone, so in a loop the method is copied for it all the same: the call by value and the library's own loop fall away. +# (method-of-a-list-with-two-texts-is-copied-in-a-loop.ir has the arrow that calls nobody; list-callback-used-twice.ir has one that is used elsewhere too.) # The library here is a stand-in. # LIBRARY: own # RUN: walk f1 mechanisms=all # CHECK-LABEL: code f1:Body -# CHECK: = NewClosure -# CHECK: = CallDirect {{*}}function=f3 +# CHECK: frame {{*}}callee=f3 # RUN(NEVER): walk f1 mechanisms=all # NEVER-NOT: calleeKey="some" -# NEVER-NOT: callee=f3 +# NEVER-NOT: = CallDirect {{*}}function=f3 +# NEVER-NOT: function=f2 module m0 path="main.js" displayPath="main.js" body=f0 module m1 path="/$bun/js/engine/ArrayPrototype.ts" displayPath="/$bun/js/engine/ArrayPrototype.ts" flags=[IsLibrary] diff --git a/Source/JavaScriptCore/aot/tests/walk/direct-calls/unboxed-results-of-escaped-closures.ir b/Source/JavaScriptCore/aot/tests/walk/direct-calls/unboxed-results-of-escaped-closures.ir new file mode 100644 index 0000000000000..81c3a430b9355 --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/walk/direct-calls/unboxed-results-of-escaped-closures.ir @@ -0,0 +1,33 @@ +# const half = x => Math.sqrt(Number(x)) * 0.5; +# globalThis.kept = half; +# half(16); +# RUN: walk f0 mechanisms=all +# CHECK: @Double = CallDirect {{*}} function=f1 +# RUN(CALLEE): walk f1 mechanisms=all +# CALLEE: number @Double = Mul +# CALLEE-NEXT: Return + +module m0 path="main.js" displayPath="main.js" body=f0 +entry module=m0 +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = NewClosure function=f1 + v1 = Builtin builtin=GlobalThis + PropSet v1, v0 key="kept" + v3 = Integer integer=16 + v4 = Call v0, -, v3 calleeName="half" [isNeverInlined] + PropSet v1, v4 key="result" + v6 = Undefined + Return v6 +} +function f1 name="half" module=m0 position=27 kind=Arrow length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + v1 = Builtin builtin=Math + v2 = Builtin builtin=MathSqrt + v3 = Builtin builtin=Number + v4 = Call v3, -, v0 calleeName="Number" + v5 = Call v2, v1, v4 calleeName="Math.sqrt" + v6 = Double number=0.5 + v7 = Mul v5, v6 + Return v7 +} diff --git a/Source/JavaScriptCore/aot/tests/walk/escape-analysis/arguments-are-made-on-the-way-that-hands-them-on.ir b/Source/JavaScriptCore/aot/tests/walk/escape-analysis/arguments-are-made-on-the-way-that-hands-them-on.ir index 6ba8ba9148efb..d6279754aee69 100644 --- a/Source/JavaScriptCore/aot/tests/walk/escape-analysis/arguments-are-made-on-the-way-that-hands-them-on.ir +++ b/Source/JavaScriptCore/aot/tests/walk/escape-analysis/arguments-are-made-on-the-way-that-hands-them-on.ir @@ -22,18 +22,20 @@ # BOTH: = CreateUnmappedArguments # BOTH: Return # function twice(c) { if (c) { log(arguments); return keep(arguments); } return 0; } -# Behind the first call the object is named again: it is one cell, made where it is written +# Behind the first call the object is named again: it is one cell, made on the one way that names it # RUN(TWICE): walk f3 mechanisms=none+TypedOperations+Folding+EscapeAnalysis # TWICE-LABEL: code f3:Body -# TWICE: = CreateUnmappedArguments +# TWICE-NOT: = CreateUnmappedArguments # TWICE: Branch +# TWICE: = CreateUnmappedArguments # TWICE-NOT: = CreateUnmappedArguments # function stored(c) { if (c) { arguments[0] = 1; return keep(arguments); } return 0; } -# A store changes what the object holds +# A store changes what the object holds: the object stands in front of it # RUN(STORED): walk f4 mechanisms=none+TypedOperations+Folding+EscapeAnalysis # STORED-LABEL: code f4:Body -# STORED: = CreateUnmappedArguments +# STORED-NOT: = CreateUnmappedArguments # STORED: Branch +# STORED: = CreateUnmappedArguments # STORED-NOT: = CreateUnmappedArguments # function looped(a) { do { } while (keep(arguments)); return a; } # An object from in front of a loop is one object in every turn diff --git a/Source/JavaScriptCore/aot/tests/walk/guesses/what-a-declared-promise-settles-to-is-expected-of-the-await.ir b/Source/JavaScriptCore/aot/tests/walk/guesses/what-a-declared-promise-settles-to-is-expected-of-the-await.ir index 19b5400c08d74..0b5f1aaaa077c 100644 --- a/Source/JavaScriptCore/aot/tests/walk/guesses/what-a-declared-promise-settles-to-is-expected-of-the-await.ir +++ b/Source/JavaScriptCore/aot/tests/walk/guesses/what-a-declared-promise-settles-to-is-expected-of-the-await.ir @@ -1,9 +1,17 @@ # the source declares a promise of a Point, so a Point is expected of what the await gives -# async function read(p: Promise) { const o = await p; return o.x; } -# RUN: walk f1:CoroutineBody mechanisms=none+TypedOperations+Guesses +# async function read(p: Promise) { const o = await p; return o.x + o.y; } +# one read outside loops of an async body opens no guess +# async function readOne(p: Promise) { return (await p).x; } +# RUN: walk f1:CoroutineBody f2:CoroutineBody mechanisms=none+TypedOperations+Guesses # CHECK-LABEL: code f1:CoroutineBody # CHECK: = AwaitBySavingFrame -# CHECK: = GetSlot {{*}}key="x" receivers=[L0] level=Expected +# CHECK: = ShapeTest {{*}}layouts=[L0] +# CHECK: = GetSlot {{*}}key="x" receivers=[L0] +# CHECK: = GetSlot {{*}}key="y" receivers=[L0] +# CHECK-LABEL: code f2:CoroutineBody +# CHECK: = AwaitBySavingFrame +# CHECK-NOT: GetSlot +# CHECK: = PropGet {{*}}key="x" layout L0 name="Point" module=m0 fields=[{key="x" declared={type=anything} flags=[IsRequired]}, {key="y" declared={type=anything} flags=[IsRequired]}] ownFieldCount=2 module m0 path="main.ts" displayPath="main.ts" body=f0 @@ -19,9 +27,19 @@ function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anythi PropDefine v4, v3 key="y" v7 = FinishObject v4 v8 = Call v1, -, v7 + v9 = NewClosure function=f2 + v10 = Call v1, -, v9 Return v8 } function f1 name="read" module=m0 flags=[IsAsync] length=1 thisType={type=anything} parameters=[{type=Promise}] results=[{type=anything}] { + b0(v0: Promise): + v1 = Await v0 + v2 = PropGet v1 key="x" + v3 = PropGet v1 key="y" + v4 = Add v2, v3 + Return v4 +} +function f2 name="readOne" module=m0 flags=[IsAsync] length=1 thisType={type=anything} parameters=[{type=Promise}] results=[{type=anything}] { b0(v0: Promise): v1 = Await v0 v2 = PropGet v1 key="x" diff --git a/Source/JavaScriptCore/aot/tests/walk/inlining/a-callee-whose-loops-are-all-written-out-is-copied-into-a-loop.ir b/Source/JavaScriptCore/aot/tests/walk/inlining/a-callee-whose-loops-are-all-written-out-is-copied-into-a-loop.ir index af5d301a1b62a..a585af03d47d1 100644 --- a/Source/JavaScriptCore/aot/tests/walk/inlining/a-callee-whose-loops-are-all-written-out-is-copied-into-a-loop.ir +++ b/Source/JavaScriptCore/aot/tests/walk/inlining/a-callee-whose-loops-are-all-written-out-is-copied-into-a-loop.ir @@ -3,7 +3,7 @@ # RUN: walk f2:Body mechanisms=all # CHECK-LABEL: code f2:Body # CHECK: frame 1 callee=f1 -# Nine turns are not written out: such a callee is called as before. +# Nine turns are not written out: such a callee is called as before, where it has a second call. # RUN(NINE): walk f4:Body mechanisms=all # NINE-LABEL: code f4:Body # NINE-NOT: frame 1 callee=f3 @@ -12,7 +12,7 @@ # ANSWER-NEXT: NaN # ANSWER-NEXT: NaN # ANSWER-NEXT: 1000 -# ANSWER-NEXT: 4500 +# ANSWER-NEXT: 4545 # ANSWER-NEXT: TypeError # ANSWER-NEXT: exited with 0 @@ -104,5 +104,7 @@ function f4 name="g" module=m0 flags=[HasExplicitName] length=2 thisType={type=a v12 = Increment v6 Jump -> next b1(v11, v12) b3(): - Return v5 + v13 = Call v2, -, v0, v1 + v14 = Add v5, v13 + Return v14 } diff --git a/Source/JavaScriptCore/aot/tests/walk/inlining/leaf-copied-into-recursive-function.ir b/Source/JavaScriptCore/aot/tests/walk/inlining/leaf-copied-into-recursive-function.ir new file mode 100644 index 0000000000000..454c0d57a4147 --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/walk/inlining/leaf-copied-into-recursive-function.ir @@ -0,0 +1,77 @@ +# A leaf of 15 operations is copied into a function that calls itself, as into a loop, and stays a call in a function that is called once. +# const distance = (a, b) => (a - b) * (a - b) + (a * 3 - b * 5) * (a + b) - (a * 7 + b); +# function walk(n, x) { return n === 0 ? 0 : walk(n - 1, x + 1) + distance(n, x); } answer = walk(9, 4) + distance(1, 2); +# RUN: walk f1 mechanisms=all +# CHECK-LABEL: code f1:Body +# CHECK: callee=f2 +# RUN(NEVER): walk f1 mechanisms=all +# NEVER-NOT: function=f2 +# RUN(ONCE): walk f0 mechanisms=all +# ONCE-LABEL: code f0:Body +# ONCE-NOT: callee=f2 +# ONCE: function=f2 + +module m0 path="leaf.js" displayPath="leaf.js" body=f0 +entry module=m0 +variable g0 name="walk" module=m0 declared={type=f1 flags=[IsFromCompiler]} initialization=Function function=f1 flags=[IsImmutable] place=1 +variable g1 name="distance" module=m0 declared={type=anything|absent} flags=[IsImmutable] place=2 +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] capabilities=[MayChangeSignature, MayBeCloned, MayBeInlined, HasOrdinaryFrame, IsNeverCalledByHost] { + b0(): + v0 = NewClosure function=f2 @17 + PutModuleVariable v0 variable=g1 @6 + v2 = Builtin builtin=GlobalThis @170 + v3 = GetModuleVariable variable=g0 @190 + v4 = Integer integer=9 @195 + v5 = Integer integer=4 @198 + v6 = Call v3, -, v4, v5 calleeName="walk" @190 + v7 = GetModuleVariable variable=g1 @210 + v8 = Integer integer=1 @220 + v9 = Integer integer=2 @223 + v10 = Call v7, -, v8, v9 calleeName="distance" @210 + v11 = Add v6, v10 @190 + PropSet v2, v11 key="answer" @170 + v13 = Undefined @170 + Return v13 @170 +} +function f1 name="walk" module=m0 position=88 flags=[HasExplicitName] length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] capabilities=[MayChangeSignature, MayBeCloned, MayBeInlined, HasOrdinaryFrame, IsNeverCalledByHost] { + b0(v0: anything, v1: anything): + v2 = Integer @123 + v3 = StrictEq v0, v2 @117 + Branch v3 -> taken b1(), notTaken b2() @117 + b1(): + v4 = Integer @127 + Jump -> next b3(v4) @127 + b2(): + v5 = GetModuleVariable variable=g0 @131 + v6 = Integer integer=1 @140 + v7 = Sub v0, v6 @136 + v8 = Integer integer=1 @147 + v9 = Add v1, v8 @143 + v10 = Call v5, -, v7, v9 calleeName="walk" @131 + v11 = GetModuleVariable variable=g1 @152 + v12 = CheckInitialized v11 name="distance" @152 + v13 = Call v12, -, v0, v1 calleeName="distance" @152 + v14 = Add v10, v13 @131 + Jump -> next b3(v14) @131 + b3(v15): + Return v15 @131 +} +function f2 name="distance" module=m0 position=17 kind=Arrow length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] capabilities=[MayChangeSignature, MayBeCloned, MayBeInlined, HasOrdinaryFrame, IsNeverCalledByHost] { + b0(v0: anything, v1: anything): + v2 = Sub v0, v1 @28 + v3 = Sub v0, v1 @38 + v4 = Mul v2, v3 @27 + v5 = Integer integer=3 @52 + v6 = Mul v0, v5 @48 + v7 = Integer integer=5 @60 + v8 = Mul v1, v7 @56 + v9 = Sub v6, v8 @48 + v10 = Add v0, v1 @66 + v11 = Mul v9, v10 @47 + v12 = Add v4, v11 @27 + v13 = Integer integer=7 @80 + v14 = Mul v0, v13 @76 + v15 = Add v14, v1 @76 + v16 = Sub v12, v15 @27 + Return v16 @27 +} diff --git a/Source/JavaScriptCore/aot/tests/walk/loop-invariants/a-string-read-out-of-a-growing-text-keeps-no-builder-alive-at-an-await.ir b/Source/JavaScriptCore/aot/tests/walk/loop-invariants/a-string-read-out-of-a-growing-text-keeps-no-builder-alive-at-an-await.ir index f67baf343cb41..3308b92b935d2 100644 --- a/Source/JavaScriptCore/aot/tests/walk/loop-invariants/a-string-read-out-of-a-growing-text-keeps-no-builder-alive-at-an-await.ir +++ b/Source/JavaScriptCore/aot/tests/walk/loop-invariants/a-string-read-out-of-a-growing-text-keeps-no-builder-alive-at-an-await.ir @@ -1,5 +1,5 @@ # a string that is read out of a text while the text grows in its loop is a cell of its own: it points into no builder. So where it is read behind an await, the builder, a record in the frame, is not live at the await -# function same(value: string) { return value; } async function gather(limit: number) { const turns = limit | 0; let text = "", kept = "", at = 0; do { text += at + ","; kept = same(text); at = (at + 1) | 0; } while (at < turns); await null; return kept.length + ":" + text.length; } +# function same(value: string) { return value; } async function gather(limit: number) { const turns = limit | 0; let text = "", kept = "", at = 0; do { text += at + ","; if (at & 1) kept = same(text); at = (at + 1) | 0; } while (at < turns); await null; return kept.length + ":" + text.length; } # RUN: walk f2:CoroutineBody mechanisms=all # CHECK-LABEL: code f2:CoroutineBody # CHECK: ([[text]]: opaque(StringBuilder) @Cell @@ -44,25 +44,31 @@ function f2 name="gather" module=m0 position=57 sourceRange={start=41 end=291} f v15 = Add v12, v14 @200 v16 = Add v10, v15 @200 v17 = Convert v16 declared={type=undefined|null|string flags=[OmitsNull, OmitsUndefined]} @187 - v18 = GetModuleVariable variable=g0 @216 - v19 = Call v18, -, v17 calleeName="same" @216 - v20 = Convert v19 declared={type=undefined|null|string flags=[OmitsNull, OmitsUndefined]} @209 - v21 = Integer integer=1 @243 - v22 = Add v12, v21 @243 - v23 = Integer @248 - v24 = BitOr v22, v23 @248 - v25 = Convert v24 declared={type=number} @232 - v26 = Less v25, v3 @267 - Branch v26 -> taken b1(v17, v20, v25), notTaken b2() @267 + v18 = Integer integer=1 @216 + v19 = BitAnd v12, v18 @216 + Branch v19 -> taken b2(), notTaken b3(v11) @216 b2(): - v27 = Null @283 - v28 = Await v27, v1, v0 @283 - v29 = PropGet v20 key="length" @298 - v30 = String string=":" @312 - v31 = Add v29, v30 @312 - v32 = PropGet v17 key="length" @318 - v33 = Add v31, v32 @318 - v34 = Convert v33 declared={type=undefined|null|string|object} @318 - Return v34 @318 + v20 = GetModuleVariable variable=g0 @216 + v21 = Call v20, -, v17 calleeName="same" @216 + v22 = Convert v21 declared={type=undefined|null|string flags=[OmitsNull, OmitsUndefined]} @209 + Jump -> next b3(v22) @216 + b3(v23: undefined|null|string): + v24 = Integer integer=1 @243 + v25 = Add v12, v24 @243 + v26 = Integer @248 + v27 = BitOr v25, v26 @248 + v28 = Convert v27 declared={type=number} @232 + v29 = Less v28, v3 @267 + Branch v29 -> taken b1(v17, v23, v28), notTaken b4() @267 + b4(): + v30 = Null @283 + v31 = Await v30, v1, v0 @283 + v32 = PropGet v23 key="length" @298 + v33 = String string=":" @312 + v34 = Add v32, v33 @312 + v35 = PropGet v17 key="length" @318 + v36 = Add v34, v35 @318 + v37 = Convert v36 declared={type=undefined|null|string|object} @318 + Return v37 @318 } entry module=m0 hostReads=[{exportName="default" builtin=BunServe}, {exportName="then"}] diff --git a/Source/JavaScriptCore/aot/tests/walk/loop-invariants/growing-text-that-a-turn-reads-gets-a-string.ir b/Source/JavaScriptCore/aot/tests/walk/loop-invariants/growing-text-that-a-turn-reads-gets-a-string.ir index 5ff1377e0da1d..521b83558b82b 100644 --- a/Source/JavaScriptCore/aot/tests/walk/loop-invariants/growing-text-that-a-turn-reads-gets-a-string.ir +++ b/Source/JavaScriptCore/aot/tests/walk/loop-invariants/growing-text-that-a-turn-reads-gets-a-string.ir @@ -1,18 +1,20 @@ # whoever takes a growing text in its loop and is no growth gets a string that is made of what the builder holds, and the builder goes on. The reads of one state in one block share one string -# function told(n, part) { .. let s = ""; for (let i = 0; i < n; i++) { s += part; sink(s); } return s; } -# function twice(n, part) { .. for (..) { s += part; sink(s); sink(s); } .. } -# function handed(n, part) { .. for (..) { s += part; keep(s); } .. } a direct call looks at the kind of no argument: only the string in the place of the name keeps the builder from the callee +# function told(n, part) { .. let s = ""; for (let i = 0; i < n; i++) { s += part; if (kept) sink(s); } return s; } +# function twice(n, part) { .. for (..) { s += part; if (kept) { sink(s); sink(s); } } .. } +# function handed(n, part) { .. for (..) { s += part; if (kept) keep(s); } .. } a direct call looks at the kind of no argument: only the string in the place of the name keeps the builder from the callee # function joined(n, part) { .. for (..) { s += part; sink(s + ": " + part); } .. } a join with the text in front whose result is never the variable again reads the text # function older(n, part) { .. for (..) { const before = s; s += part; sink(before); } .. } the one property that is changed: the state that is read is not the newest, so two states are live and nothing is begun +# function always(n, part) { .. for (..) { s += part; sink(s); } .. } every iteration passes the text: a builder would copy it every time, so it stays a rope +# function either(n, part) { .. for (..) { s += part; if (kept) sink(s); else keep(s); } .. } every iteration passes the text, on one arm or the other: a rope too # RUN: walk f1 mechanisms=none+TypedOperations+Guesses+Folding+LoopInvariants # CHECK-LABEL: code f1:Body # CHECK: StringBuilderBegin # CHECK: isLoopHead=true # CHECK: StringBuilderAppendString +# CHECK: StringBuilderFinish # CHECK: [[string]]: {{*}}StringBuilderToString # CHECK: [[boxed]]: {{*}}= Box [[string]] # CHECK: = CallResolved {{*}}[[boxed]] -# CHECK: StringBuilderFinish # RUN(ALL): walk f1 mechanisms=all # ALL-LABEL: code f1:Body # ALL: isLoopHead=true @@ -41,6 +43,12 @@ # RUN(OLDERALL): walk f4 mechanisms=all # OLDERALL-LABEL: code f4:Body # OLDERALL-NOT: StringBuilder +# RUN(ALWAYS): walk f7 mechanisms=all +# ALWAYS-LABEL: code f7:Body +# ALWAYS-NOT: StringBuilder +# RUN(EITHER): walk f8 mechanisms=all +# EITHER-LABEL: code f8:Body +# EITHER-NOT: StringBuilder variable g0 name="kept" module=m0 declared={type=anything} module m0 path="main.js" displayPath="main.js" body=f0 @@ -54,13 +62,15 @@ function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anythi v4 = NewClosure function=f3 v5 = NewClosure function=f4 v6 = NewClosure function=f6 - v7 = Call v1, -, v2, v3, v4, v5, v6 - Return v7 + v7 = NewClosure function=f7 + v8 = NewClosure function=f8 + v9 = Call v1, -, v2, v3, v4, v5, v6, v7, v8 + Return v9 } function f1 name="told" module=m0 length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { b0(v0: anything, v1: anything): v2 = TypeTest v1 testedType=string - Branch v2 -> taken b1(| 0), notTaken b5() + Branch v2 -> taken b1(| 0), notTaken b7() b1(v3): v4 = Builtin builtin=GlobalThis v5 = PropGet v4 key="sink" [isFreeName] @@ -70,21 +80,26 @@ function f1 name="told" module=m0 length=2 thisType={type=anything} parameters=[ b2(v8, v9): LoopHint v11 = Less v9, v0 - Branch v11 -> taken b3(), notTaken b4() + Branch v11 -> taken b3(), notTaken b6() b3(): v12 = Add v8, v3 - v13 = Call v5, -, v12 - v14 = Increment v9 - Jump -> next b2(v12, v14) + v13 = GetModuleVariable variable=g0 + Branch v13 -> taken b4(), notTaken b5() b4(): - Return v8 + v14 = Call v5, -, v12 + Jump -> next b5() b5(): + v15 = Increment v9 + Jump -> next b2(v12, v15) + b6(): + Return v8 + b7(): Return v1 } function f2 name="twice" module=m0 length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { b0(v0: anything, v1: anything): v2 = TypeTest v1 testedType=string - Branch v2 -> taken b1(| 0), notTaken b5() + Branch v2 -> taken b1(| 0), notTaken b7() b1(v3): v4 = Builtin builtin=GlobalThis v5 = PropGet v4 key="sink" [isFreeName] @@ -94,22 +109,27 @@ function f2 name="twice" module=m0 length=2 thisType={type=anything} parameters= b2(v8, v9): LoopHint v11 = Less v9, v0 - Branch v11 -> taken b3(), notTaken b4() + Branch v11 -> taken b3(), notTaken b6() b3(): v12 = Add v8, v3 - v13 = Call v5, -, v12 - v14 = Call v5, -, v12 - v15 = Increment v9 - Jump -> next b2(v12, v15) + v13 = GetModuleVariable variable=g0 + Branch v13 -> taken b4(), notTaken b5() b4(): - Return v9 + v14 = Call v5, -, v12 + v15 = Call v5, -, v12 + Jump -> next b5() b5(): + v16 = Increment v9 + Jump -> next b2(v12, v16) + b6(): + Return v9 + b7(): Return v1 } function f3 name="handed" module=m0 length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { b0(v0: anything, v1: anything): v2 = TypeTest v1 testedType=string - Branch v2 -> taken b1(| 0), notTaken b5() + Branch v2 -> taken b1(| 0), notTaken b7() b1(v3): v4 = NewClosure function=f5 v5 = String string="" @@ -118,15 +138,20 @@ function f3 name="handed" module=m0 length=2 thisType={type=anything} parameters b2(v7, v8): LoopHint v10 = Less v8, v0 - Branch v10 -> taken b3(), notTaken b4() + Branch v10 -> taken b3(), notTaken b6() b3(): v11 = Add v7, v3 - v12 = Call v4, -, v11 calleeName="keep" - v13 = Increment v8 - Jump -> next b2(v11, v13) + v12 = GetModuleVariable variable=g0 + Branch v12 -> taken b4(), notTaken b5() b4(): - Return v7 + v13 = Call v4, -, v11 calleeName="keep" + Jump -> next b5() b5(): + v14 = Increment v8 + Jump -> next b2(v11, v14) + b6(): + Return v7 + b7(): Return v1 } function f4 name="older" module=m0 length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { @@ -186,3 +211,60 @@ function f6 name="joined" module=m0 length=2 thisType={type=anything} parameters b5(): Return v1 } +function f7 name="always" module=m0 length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { + b0(v0: anything, v1: anything): + v2 = TypeTest v1 testedType=string + Branch v2 -> taken b1(| 0), notTaken b5() + b1(v3): + v4 = Builtin builtin=GlobalThis + v5 = PropGet v4 key="sink" [isFreeName] + v6 = String string="" + v7 = Integer + Jump -> next b2(v6, v7) + b2(v8, v9): + LoopHint + v11 = Less v9, v0 + Branch v11 -> taken b3(), notTaken b4() + b3(): + v12 = Add v8, v3 + v13 = Call v5, -, v12 + v14 = Increment v9 + Jump -> next b2(v12, v14) + b4(): + Return v8 + b5(): + Return v1 +} +function f8 name="either" module=m0 length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { + b0(v0: anything, v1: anything): + v2 = TypeTest v1 testedType=string + Branch v2 -> taken b1(| 0), notTaken b8() + b1(v3): + v4 = Builtin builtin=GlobalThis + v5 = PropGet v4 key="sink" [isFreeName] + v6 = NewClosure function=f5 + v7 = String string="" + v8 = Integer + Jump -> next b2(v7, v8) + b2(v9, v10): + LoopHint + v12 = Less v10, v0 + Branch v12 -> taken b3(), notTaken b7() + b3(): + v13 = Add v9, v3 + v14 = GetModuleVariable variable=g0 + Branch v14 -> taken b4(), notTaken b5() + b4(): + v15 = Call v5, -, v13 + Jump -> next b6() + b5(): + v16 = Call v6, -, v13 calleeName="keep" + Jump -> next b6() + b6(): + v17 = Increment v10 + Jump -> next b2(v13, v17) + b7(): + Return v9 + b8(): + Return v1 +} diff --git a/Source/JavaScriptCore/aot/tests/walk/loop-invariants/length-of-a-text-in-a-builder-is-asked-of-the-builder.ir b/Source/JavaScriptCore/aot/tests/walk/loop-invariants/length-of-a-text-in-a-builder-is-asked-of-the-builder.ir index 71dd50bd57ff4..b6bfecd9d4118 100644 --- a/Source/JavaScriptCore/aot/tests/walk/loop-invariants/length-of-a-text-in-a-builder-is-asked-of-the-builder.ir +++ b/Source/JavaScriptCore/aot/tests/walk/loop-invariants/length-of-a-text-in-a-builder-is-asked-of-the-builder.ir @@ -2,7 +2,7 @@ # function rows(n) { let s = ""; for (let i = 0; i < n; i++) { if (s.length > 8) s += ","; s += i; } return s; } # function joined(n) { let s = ""; for (let i = 0; i < n; i++) { if (s) s += ","; s += i; } return s; } # function spaced(n) { let s = ""; for (let i = 0; i < n; i++) { if (s !== "") s += ","; s += i; } return s; } -# function marked(n) { let s = ""; for (let i = 0; i < n; i++) { if (s === "0") s += ","; s += i; } return s; } another text than the empty one is compared with the string +# function marked(n) { let s = ""; for (let i = 0; i < n; i++) { if (s === "0") s += ","; s += i; } return s; } the length is asked before the equality with another text: only a builder of that length makes its string # each function is handed to code that nobody has seen, so nothing is known of n # RUN: walk f1 mechanisms=none+TypedOperations+Guesses+Folding+LoopInvariants # CHECK-LABEL: code f1:Body @@ -30,9 +30,10 @@ # RUN(OTHER): walk f4 mechanisms=none+TypedOperations+Guesses+Folding+LoopInvariants # OTHER-LABEL: code f4:Body # OTHER: StringBuilderBegin +# OTHER: StringBuilderLength +# OTHER: = StrictEq # OTHER: StringBuilderToString -# RUN(OTHERONLY): walk f4 mechanisms=none+TypedOperations+Guesses+Folding+LoopInvariants -# OTHERONLY-NOT: StringBuilderLength +# OTHER: = StrictEq # RUN(ALL): walk f1 mechanisms=all # ALL-LABEL: code f1:Body # ALL: StringBuilderLength diff --git a/Source/JavaScriptCore/aot/tests/walk/loops/a-counter-that-steps-by-a-whole-number-has-no-fraction.ir b/Source/JavaScriptCore/aot/tests/walk/loops/a-counter-that-steps-by-a-whole-number-has-no-fraction.ir index 1de3c6a6ad0d9..3584d2719a648 100644 --- a/Source/JavaScriptCore/aot/tests/walk/loops/a-counter-that-steps-by-a-whole-number-has-no-fraction.ir +++ b/Source/JavaScriptCore/aot/tests/walk/loops/a-counter-that-steps-by-a-whole-number-has-no-fraction.ir @@ -16,9 +16,9 @@ # EMIT-LABEL: unit f1:Body # EMIT: Int64 @[[truncated]] = Patch({{*}}TruncatesDoubleToInt64 # EMIT-NOT: = IToD(@[[truncated]] -# EMIT: = Below(@[[truncated]], # EMIT: Int64 @[[beside]] = Add(@[[truncated]], # EMIT-NOT: = IToD(@[[beside]] +# EMIT: = Below(@[[truncated]], # EMIT: = Below(@[[beside]], # RUN(SHELL): run-in-shell globalThis.bound = () => 12; globalThis.sink = (whole, halves) => { print(String(whole) + " " + String(halves)); }; # SHELL: 0 0 diff --git a/Source/JavaScriptCore/aot/tests/walk/loops/a-read-out-of-a-view-that-goes-into-a-sum-lies-inside-or-the-turn-begins-again.ir b/Source/JavaScriptCore/aot/tests/walk/loops/a-read-out-of-a-view-that-goes-into-a-sum-lies-inside-or-the-turn-begins-again.ir index b06aa680f43fa..ac6162ddc620a 100644 --- a/Source/JavaScriptCore/aot/tests/walk/loops/a-read-out-of-a-view-that-goes-into-a-sum-lies-inside-or-the-turn-begins-again.ir +++ b/Source/JavaScriptCore/aot/tests/walk/loops/a-read-out-of-a-view-that-goes-into-a-sum-lies-inside-or-the-turn-begins-again.ir @@ -7,8 +7,9 @@ # RUN: walk f2 mechanisms=all # CHECK-LABEL: code f2:Body # CHECK: i32 @Int32, {{*}}isInLoop=true isLoopHead=true +# The header compares the counter with the lesser of its bound and the length, so the iteration checks nothing. # CHECK: [[below]]: boolean @Boolean = Less {{*}}form=Integers -# CHECK: [[below2]]: boolean @Boolean = Less {{*}}form=Integers +# CHECK-NOT: = Less # CHECK-NOT: = IntegerToNumber # CHECK: [[word]]: i32 @Int32 = GetTypedArrayElement {{*}}isInsideView # CHECK-NOT: form=Numbers diff --git a/Source/JavaScriptCore/aot/tests/walk/loops/an-element-that-is-only-compared-is-read-with-its-own-test-where-the-position-may-be-negative.ir b/Source/JavaScriptCore/aot/tests/walk/loops/an-element-that-is-only-compared-is-read-with-its-own-test-where-the-position-may-be-negative.ir index a634f875cc740..10261829ca3bf 100644 --- a/Source/JavaScriptCore/aot/tests/walk/loops/an-element-that-is-only-compared-is-read-with-its-own-test-where-the-position-may-be-negative.ir +++ b/Source/JavaScriptCore/aot/tests/walk/loops/an-element-that-is-only-compared-is-read-with-its-own-test-where-the-position-may-be-negative.ir @@ -7,7 +7,7 @@ # an-element-of-a-view-is-read-behind-its-tests-under-a-position-that-may-be-negative.ir; the read that tests for itself under such a position where it is only compared has no test of its own any more). # RUN: walk f1 f2 mechanisms=all # CHECK-LABEL: code f1:Body -# CHECK: = GreaterEq {{*}}form=Integers +# CHECK: = TypeTest {{*}} testedType=int[0, 2147483647] # CHECK: isInLoop=true isLoopHead=true # CHECK-NOT: = GreaterEq # CHECK: : i32 @Int32 = GetTypedArrayElement {{*}}isInsideView @@ -16,7 +16,7 @@ # CHECK-NOT: = GreaterEq # CHECK: = StrictEq {{*}}form=Integers # CHECK-LABEL: code f2:Body -# CHECK: = GreaterEq {{*}}form=Integers +# CHECK: = TypeTest {{*}} testedType=int[0, 2147483647] # CHECK: isInLoop=true isLoopHead=true # CHECK-NOT: = GreaterEq # CHECK: : i32 @Int32 = GetTypedArrayElement {{*}}isInsideView diff --git a/Source/JavaScriptCore/aot/tests/walk/loops/builtin-test-of-record-folds.ir b/Source/JavaScriptCore/aot/tests/walk/loops/builtin-test-of-record-folds.ir new file mode 100644 index 0000000000000..c75e0042b949d --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/walk/loops/builtin-test-of-record-folds.ir @@ -0,0 +1,40 @@ +# function total(n) { let sum = 0; for (let i = 0; i < n; i++) { const point = { x: i, y: 2 }; if (point instanceof Map) break; sum += point.x; } return sum; } +# Above a record stands Object.prototype alone: no chain is walked. +# RUN: walk f1 mechanisms=all +# CHECK-NOT: HasInPrototypeChain +# CHECK-NOT: = InstanceOf + +module m0 path="main.js" displayPath="main.js" body=f0 +entry module=m0 +layout L0 module=m0 fields=[{key="x" declared={type=anything} flags=[IsRequired]}, {key="y" declared={type=anything} flags=[IsRequired]}] +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = NewClosure function=f1 + v1 = Builtin builtin=GlobalThis + PropSet v1, v0 key="total" + v3 = Undefined + Return v3 +} +function f1 name="total" module=m0 position=14 flags=[HasExplicitName] length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + v1 = Integer + v2 = Integer integer=2 + v3 = Builtin builtin=Map + Jump -> next b1(v1, v1) + b1(v4, v5): + LoopHint + v7 = Less v5, v0 + Branch v7 -> taken b2(), notTaken b4() + b2(): + v8 = NewObject -, v5, v2 layout=L0 fields=[0, 1] + v9 = FinishObject v8 + v10 = InstanceOf v9, v3 + Branch v10 -> taken b4(), notTaken b3() + b3(): + v11 = PropGet v9 key="x" + v12 = Add v4, v11 + v13 = Increment v5 + Jump -> next b1(v12, v13) + b4(): + Return v4 +} diff --git a/Source/JavaScriptCore/aot/tests/walk/loops/capacity-of-declared-array-literal.ir b/Source/JavaScriptCore/aot/tests/walk/loops/capacity-of-declared-array-literal.ir new file mode 100644 index 0000000000000..408bbd69af55a --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/walk/loops/capacity-of-declared-array-literal.ir @@ -0,0 +1,48 @@ +# function row(first: number, count: number, above: number[]): number[] { +# const current: number[] = [first]; +# for (let j = 1; j <= count; j++) current.push(above[j] + 1); +# return current; +# } +# RUN: walk f1 mechanisms=all +# CHECK: Array @Cell = NewArray +# CHECK-NEXT: = IntegerToNumber +# CHECK-NEXT: = ReserveArrayCapacity + +module m0 path="main.ts" displayPath="main.ts" body=f0 +entry module=m0 +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = NewClosure function=f1 + v1 = Integer integer=1 + v2 = Integer integer=3 + v3 = NewArray v1, v2, v1, v2, v1 contents=number + v4 = Call v0, -, v1, v2, v3 calleeName="row" [isNeverInlined] + v5 = Call v0, -, v2, v1, v3 calleeName="row" [isNeverInlined] + v6 = Builtin builtin=GlobalThis + v7 = PropGet v6 key="sink" [isFreeName] + v8 = Call v7, -, v4, v5 + Return v8 +} +function f1 name="row" module=m0 position=10 flags=[IsChecked, HasExplicitName] length=3 thisType={type=anything} parameters=[{type=number}, {type=number}, {type=undefined|null|Array}] results=[{type=undefined|null|Array}] { + b0(v0: number, v1: number, v2: undefined|null|Array): + v3 = NewArray v0 contents=number + v4 = Convert v3 declared={type=undefined|null|Array flags=[OmitsNull, OmitsUndefined]} + v5 = Integer integer=1 + v6 = Convert v5 declared={type=number} + Jump -> next b1(v6) + b1(v7: number): + LoopHint + v9 = LessEq v7, v1 + Branch v9 -> taken b2(), notTaken b3() + b2(): + v10 = PropGet v4 key="push" + v11 = ElemGet v2, v7 + v12 = Integer integer=1 + v13 = Add v11, v12 + v14 = Call v10, v4, v13 calleeKey="push" + v15 = Increment v7 + v16 = Convert v15 declared={type=number} + Jump -> next b1(v16) + b3(): + Return v4 +} diff --git a/Source/JavaScriptCore/aot/tests/walk/loops/deleted-field-of-fresh-record.ir b/Source/JavaScriptCore/aot/tests/walk/loops/deleted-field-of-fresh-record.ir new file mode 100644 index 0000000000000..409cf1600ea10 --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/walk/loops/deleted-field-of-fresh-record.ir @@ -0,0 +1,51 @@ +# function cleared(n) { let sum = 0; for (let i = 0; i < n; i++) { const point = { x: i, y: 2, z: 3 }; delete point.y; sum += ("y" in point ? 1 : 0) + point.z; } return sum; } +# Behind the delete the slot is known to be empty, so nothing needs the record. +# RUN: walk f1 mechanisms=all +# CHECK-NOT: IsSlotFilled +# CHECK-NOT: PropDelete +# CHECK-NOT: NewObject + +module m0 path="main.js" displayPath="main.js" body=f0 +entry module=m0 +layout L0 module=m0 fields=[{key="x" declared={type=anything} flags=[IsRequired]}, {key="y" declared={type=anything} flags=[IsRequired]}, {key="z" declared={type=anything} flags=[IsRequired]}] +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = NewClosure function=f1 + v1 = Builtin builtin=GlobalThis + PropSet v1, v0 key="cleared" + v3 = Undefined + Return v3 +} +function f1 name="cleared" module=m0 position=14 flags=[HasExplicitName] length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + v1 = Integer + v2 = Integer + Jump -> next b1(v1, v2) + b1(v3, v4): + LoopHint + v6 = Less v4, v0 + Branch v6 -> taken b2(), notTaken b6() + b2(): + v7 = Integer integer=2 + v8 = Integer integer=3 + v9 = NewObject -, v4, v7, v8 layout=L0 fields=[0, 1, 2] + v10 = FinishObject v9 + v11 = PropDelete v10 key="y" + v12 = PropHas v10 key="y" + v13 = ToBoolean v12 + Branch v13 -> taken b3(| 0), notTaken b4(| 0) + b3(v14): + v15 = Integer integer=1 + Jump -> next b5(v15) + b4(v16): + v17 = Integer + Jump -> next b5(v17) + b5(v18): + v19 = PropGet v10 key="z" + v20 = Add v18, v19 + v21 = Add v3, v20 + v22 = Increment v4 + Jump -> next b1(v21, v22) + b6(): + Return v3 +} diff --git a/Source/JavaScriptCore/aot/tests/walk/loops/final-read-of-local-list.ir b/Source/JavaScriptCore/aot/tests/walk/loops/final-read-of-local-list.ir new file mode 100644 index 0000000000000..6aa0015e49090 --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/walk/loops/final-read-of-local-list.ir @@ -0,0 +1,101 @@ +# function atEnd(n, key) { const list = []; for (let i = 0; i < n; i++) list.push(i); let sum = 0; for (let i = 0; i < n; i++) sum += list[i]; if (sum < 0) return sum; return list[key]; } +# function inFront(n, key) { const list = []; for (let i = 0; i < n; i++) list.push(i); const got = list[key]; let sum = 0; for (let i = 0; i < n; i++) sum += list[i]; return got === sum; } +# A getter on Array.prototype under an unknown key gets the list as `this`. Behind the read in atEnd() this call does not use the list again. +# RUN: walk f1 mechanisms=all +# CHECK: undefined|int|whole @OptionalDouble = GetArrayElement {{*}}level=Proven +# RUN(FRONT): walk f2 mechanisms=all +# FRONT-NOT: undefined|int|whole @OptionalDouble = GetArrayElement + +module m0 path="main.js" displayPath="main.js" body=f0 +entry module=m0 +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = NewClosure function=f1 + v1 = NewClosure function=f2 + v2 = Builtin builtin=GlobalThis + PropSet v2, v0 key="atEnd" + PropSet v2, v1 key="inFront" + v5 = Undefined + Return v5 +} +function f1 name="atEnd" module=m0 position=14 flags=[HasExplicitName] length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { + b0(v0: anything, v1: anything): + v2 = NewArray contents=anything + v3 = Integer + v4 = Convert v3 declared={type=number flags=[IsExpected]} + Jump -> next b1(v4) + b1(v5): + LoopHint + v7 = Less v5, v0 + Branch v7 -> taken b2(), notTaken b3() + b2(): + v8 = PropGet v2 key="push" + v9 = Call v8, v2, v5 calleeKey="push" + v10 = Increment v5 + v11 = Convert v10 declared={type=number flags=[IsExpected]} + Jump -> next b1(v11) + b3(): + v12 = Integer + v13 = Convert v12 declared={type=number flags=[IsExpected]} + v14 = Integer + v15 = Convert v14 declared={type=i32 flags=[IsExpected]} + Jump -> next b4(v13, v15) + b4(v16, v17): + LoopHint + v19 = Less v17, v0 + Branch v19 -> taken b5(), notTaken b6() + b5(): + v20 = ElemGet v2, v17 + v21 = Add v16, v20 + v22 = Convert v21 declared={type=number flags=[IsExpected]} + v23 = Increment v17 + v24 = Convert v23 declared={type=i32 flags=[IsExpected]} + Jump -> next b4(v22, v24) + b6(): + v25 = Integer + v26 = Less v16, v25 + Branch v26 -> taken b7(), notTaken b8() + b7(): + Return v16 + b8(): + v27 = ElemGet v2, v1 + Return v27 +} +function f2 name="inFront" module=m0 position=300 flags=[HasExplicitName] length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { + b0(v0: anything, v1: anything): + v2 = NewArray contents=anything + v3 = Integer + v4 = Convert v3 declared={type=number flags=[IsExpected]} + Jump -> next b1(v4) + b1(v5): + LoopHint + v7 = Less v5, v0 + Branch v7 -> taken b2(), notTaken b3() + b2(): + v8 = PropGet v2 key="push" + v9 = Call v8, v2, v5 calleeKey="push" + v10 = Increment v5 + v11 = Convert v10 declared={type=number flags=[IsExpected]} + Jump -> next b1(v11) + b3(): + v12 = Integer + v13 = Convert v12 declared={type=number flags=[IsExpected]} + v14 = ElemGet v2, v1 + v15 = Integer + v16 = Convert v15 declared={type=i32 flags=[IsExpected]} + Jump -> next b4(v13, v16) + b4(v17, v18): + LoopHint + v20 = Less v18, v0 + Branch v20 -> taken b5(), notTaken b6() + b5(): + v21 = ElemGet v2, v18 + v22 = Add v17, v21 + v23 = Convert v22 declared={type=number flags=[IsExpected]} + v24 = Increment v18 + v25 = Convert v24 declared={type=i32 flags=[IsExpected]} + Jump -> next b4(v23, v25) + b6(): + v26 = StrictEq v14, v17 + Return v26 +} diff --git a/Source/JavaScriptCore/aot/tests/walk/loops/keys-of-two-records.ir b/Source/JavaScriptCore/aot/tests/walk/loops/keys-of-two-records.ir new file mode 100644 index 0000000000000..ad9e155573bf1 --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/walk/loops/keys-of-two-records.ir @@ -0,0 +1,85 @@ +# function same(left, right) { const others = Object.keys(right), keys = Object.keys(left); if (keys.length !== others.length) return false; for (const k of others) if (left[k] !== right[k]) return false; return true; } +# The second record may be undefined. Listing the keys of one record changes no key of another, and neither does the TypeError for undefined. +# RUN: walk f2 mechanisms=all +# CHECK: GetSlot{{*}}key="alpha" +# CHECK: GetSlot{{*}}key="beta" + +module m0 path="main.js" displayPath="main.js" body=f0 +entry module=m0 +layout L0 module=m0 fields=[{key="alpha" declared={type=anything} flags=[IsRequired]}, {key="beta" declared={type=anything} flags=[IsRequired]}] +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = NewClosure function=f1 + v1 = Builtin builtin=GlobalThis + PropSet v1, v0 key="run" + v3 = Undefined + Return v3 +} +function f1 name="run" module=m0 position=14 flags=[HasExplicitName] length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { + b0(v0: anything, v1: anything): + v2 = NewClosure function=f2 + v3 = Undefined + v4 = Integer + v5 = Undefined + v6 = Integer integer=2 + Jump -> next b1(v4, v5) + b1(v7, v8): + LoopHint + v10 = Less v7, v0 + Branch v10 -> taken b2(), notTaken b5() + b2(): + v11 = NewObject -, v7, v6 layout=L0 fields=[0, 1] + v12 = FinishObject v11 + v13 = ToBoolean v7 + Branch v13 -> taken b4(v12), notTaken b3(| 0) + b3(v14): + v15 = Undefined + Jump -> next b4(v15) + b4(v16): + v17 = NewObject -, v7, v6 layout=L0 fields=[0, 1] + v18 = FinishObject v17 + v19 = Call v2, v5, v16, v18 + v20 = Undefined + v21 = Increment v7 + Jump -> next b1(v21, v19) + b5(): + Return v8 +} +function f2 name="same" module=m0 position=24 flags=[HasExplicitName] length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { + b0(v0: anything, v1: anything): + v2 = Builtin builtin=Object + v3 = Builtin builtin=ObjectKeys + v4 = Call v3, v2, v1 calleeName="Object.keys" + v5 = Call v3, v2, v0 calleeName="Object.keys" + v6 = PropGet v4 key="length" + v7 = PropGet v5 key="length" + v8 = StrictEq v6, v7 + Branch v8 -> taken b1(), notTaken b6() + b1(): + v9 = IteratorOpen v4 + Jump -> next b2() + b2(): + LoopHint + v11 = IteratorResume v9 + v12 = ResultAt v11 + v13 = ResultAt v11 index=1 + Branch v12 -> taken b3(), notTaken b4() + b3(): + v14 = Boolean boolean=true + Return v14 + b4() handler=b7(): + v15 = ElemGet v0, v13 + v16 = ElemGet v1, v13 + v17 = StrictEq v15, v16 + Branch v17 -> taken b2(), notTaken b5() + b5(): + IteratorClose v9 + Jump -> next b6() + b6(): + v19 = Boolean + Return v19 + b7() role=IterationCleanup subject=v9: + v20 = CatchException + IteratorClose v9 completion=Throw + Rethrow v20 +} diff --git a/Source/JavaScriptCore/aot/tests/walk/loops/loop-over-keys-of-record.ir b/Source/JavaScriptCore/aot/tests/walk/loops/loop-over-keys-of-record.ir new file mode 100644 index 0000000000000..662966a97c73f --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/walk/loops/loop-over-keys-of-record.ir @@ -0,0 +1,96 @@ +# function total(record) { let sum = 0; for (const k of Object.keys(record)) sum += record[k]; return sum; } +# function changed(record) { let sum = 0; const keys = Object.keys(record); delete record.beta; for (const k of keys) sum += record[k]; return sum; } +# function run(n) { let last; for (let i = 0; i < n; i++) { last = total({ alpha: i, beta: 2 }); last = changed({ alpha: i, beta: 2 }); } return last; } +# The keys of a record are told by the order of its fields: each turn reads its field. A delete between ends that. +# RUN: walk f2 mechanisms=all +# CHECK: GetSlot{{*}}key="alpha" +# CHECK: GetSlot{{*}}key="beta" +# RUN(CHANGED): walk f3 mechanisms=all +# CHANGED-NOT: GetSlot{{*}}key="alpha" + +module m0 path="main.js" displayPath="main.js" body=f0 +entry module=m0 +layout L0 module=m0 fields=[{key="alpha" declared={type=anything} flags=[IsRequired]}, {key="beta" declared={type=anything} flags=[IsRequired]}] +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = NewClosure function=f1 + v1 = Builtin builtin=GlobalThis + PropSet v1, v0 key="run" + v3 = Undefined + Return v3 +} +function f1 name="run" module=m0 position=14 flags=[HasExplicitName] length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + v1 = NewClosure function=f2 + v2 = NewClosure function=f3 + v3 = Integer + v4 = Undefined + v5 = Integer integer=2 + Jump -> next b1(v3, v4) + b1(v6, v7): + LoopHint + v9 = Less v6, v0 + Branch v9 -> taken b2(), notTaken b3() + b2(): + v10 = NewObject -, v6, v5 layout=L0 fields=[0, 1] + v11 = FinishObject v10 + v12 = Call v1, v4, v11 + v13 = NewObject -, v6, v5 layout=L0 fields=[0, 1] + v14 = FinishObject v13 + v15 = Call v2, v4, v14 + v16 = Increment v6 + Jump -> next b1(v16, v15) + b3(): + Return v7 +} +function f2 name="total" module=m0 position=24 flags=[HasExplicitName] length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + v1 = Integer + v2 = Builtin builtin=Object + v3 = Builtin builtin=ObjectKeys + v4 = Call v3, v2, v0 calleeName="Object.keys" + v5 = IteratorOpen v4 + Jump -> next b1(v1) + b1(v6): + LoopHint + v8 = IteratorResume v5 + v9 = ResultAt v8 + v10 = ResultAt v8 index=1 + Branch v9 -> taken b2(), notTaken b3() + b2(): + Return v6 + b3() handler=b4(): + v11 = ElemGet v0, v10 + v12 = Add v6, v11 + Jump -> next b1(v12) + b4() role=IterationCleanup subject=v5: + v13 = CatchException + IteratorClose v5 completion=Throw + Rethrow v13 +} +function f3 name="changed" module=m0 position=34 flags=[HasExplicitName] length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + v1 = Integer + v2 = Builtin builtin=Object + v3 = Builtin builtin=ObjectKeys + v4 = Call v3, v2, v0 calleeName="Object.keys" + v5 = PropDelete v0 key="beta" + v6 = IteratorOpen v4 + Jump -> next b1(v1) + b1(v7): + LoopHint + v9 = IteratorResume v6 + v10 = ResultAt v9 + v11 = ResultAt v9 index=1 + Branch v10 -> taken b2(), notTaken b3() + b2(): + Return v7 + b3() handler=b4(): + v12 = ElemGet v0, v11 + v13 = Add v7, v12 + Jump -> next b1(v13) + b4() role=IterationCleanup subject=v6: + v14 = CatchException + IteratorClose v6 completion=Throw + Rethrow v14 +} diff --git a/Source/JavaScriptCore/aot/tests/walk/loops/object-prototype-method-of-record.ir b/Source/JavaScriptCore/aot/tests/walk/loops/object-prototype-method-of-record.ir new file mode 100644 index 0000000000000..c3793e6a7cba8 --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/walk/loops/object-prototype-method-of-record.ir @@ -0,0 +1,75 @@ +# function count(n) { let found = 0; for (let i = 0; i < n; i++) { const it = { kind: i, size: n }; if (it.hasOwnProperty("size")) found++; } return found; } +# function listed(n) { let found = 0; for (let i = 0; i < n; i++) { const it = { kind: i, hasOwnProperty: n }; if (it.hasOwnProperty("kind")) found++; } return found; } +# A record that lists no such name finds the method on Object.prototype, which nobody changes. +# RUN: walk f1 mechanisms=all +# CHECK-NOT: PropGet +# CHECK-NOT: = Call +# RUN(LISTED): walk f2 mechanisms=all +# LISTED: = Call + +module m0 path="main.js" displayPath="main.js" body=f0 +entry module=m0 +layout L0 module=m0 fields=[{key="kind" declared={type=anything} flags=[IsRequired]}, {key="size" declared={type=anything} flags=[IsRequired]}] +layout L1 module=m0 fields=[{key="kind" declared={type=anything} flags=[IsRequired]}, {key="hasOwnProperty" declared={type=anything} flags=[IsRequired]}] +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = NewClosure function=f1 + v1 = NewClosure function=f2 + v2 = Builtin builtin=GlobalThis + PropSet v2, v0 key="count" + PropSet v2, v1 key="listed" + v5 = Undefined + Return v5 +} +function f1 name="count" module=m0 position=14 flags=[HasExplicitName] length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + v1 = Integer + v2 = Integer + Jump -> next b1(v1, v2) + b1(v3, v4): + LoopHint + v6 = Less v4, v0 + Branch v6 -> taken b2(), notTaken b5() + b2(): + v7 = NewObject -, v4, v0 layout=L0 fields=[0, 1] + v8 = FinishObject v7 + v9 = PropGet v8 key="hasOwnProperty" + v10 = String string="size" + v11 = Call v9, v8, v10 calleeKey="hasOwnProperty" + v12 = ToBoolean v11 + Branch v12 -> taken b3(| 0), notTaken b4(v3) + b3(v13): + v14 = Increment v3 + Jump -> next b4(v14) + b4(v15): + v16 = Increment v4 + Jump -> next b1(v15, v16) + b5(): + Return v3 +} +function f2 name="listed" module=m0 position=24 flags=[HasExplicitName] length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + v1 = Integer + v2 = Integer + Jump -> next b1(v1, v2) + b1(v3, v4): + LoopHint + v6 = Less v4, v0 + Branch v6 -> taken b2(), notTaken b5() + b2(): + v7 = NewObject -, v4, v0 layout=L1 fields=[0, 1] + v8 = FinishObject v7 + v9 = PropGet v8 key="hasOwnProperty" + v10 = String string="kind" + v11 = Call v9, v8, v10 calleeKey="hasOwnProperty" + v12 = ToBoolean v11 + Branch v12 -> taken b3(| 0), notTaken b4(v3) + b3(v13): + v14 = Increment v3 + Jump -> next b4(v14) + b4(v15): + v16 = Increment v4 + Jump -> next b1(v15, v16) + b5(): + Return v3 +} diff --git a/Source/JavaScriptCore/aot/tests/walk/loops/own-property-test-of-record.ir b/Source/JavaScriptCore/aot/tests/walk/loops/own-property-test-of-record.ir new file mode 100644 index 0000000000000..0639705091511 --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/walk/loops/own-property-test-of-record.ir @@ -0,0 +1,96 @@ +# function count(n) { let found = 0; for (let i = 0; i < n; i++) { const it = { kind: i, size: n }; if (Object.hasOwn(it, "size")) found++; if (Object.hasOwn(it, "color")) found++; } return found; } +# function cleared(n) { let found = 0; for (let i = 0; i < n; i++) { const it = { kind: i, size: n }; if (i > 3) delete it.size; if (Object.hasOwn(it, "size")) found++; } return found; } +# The layout of a record answers whether a written name is its own; where the name may be deleted the slot answers. +# RUN: walk f1 mechanisms=all +# CHECK-NOT: PropHasOwn +# CHECK-NOT: IsSlotFilled +# RUN(CLEARED): walk f2 mechanisms=all +# CLEARED-NOT: PropHasOwn +# CLEARED: IsSlotFilled + +module m0 path="main.js" displayPath="main.js" body=f0 +entry module=m0 +layout L0 module=m0 fields=[{key="kind" declared={type=anything} flags=[IsRequired]}, {key="size" declared={type=anything} flags=[IsRequired]}] +layout L1 module=m0 fields=[{key="color" declared={type=anything} flags=[IsRequired]}] +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = NewClosure function=f1 + v1 = NewClosure function=f2 + v2 = Builtin builtin=GlobalThis + PropSet v2, v0 key="count" + PropSet v2, v1 key="cleared" + v5 = NewObject -, v0 layout=L1 fields=[0] + v6 = FinishObject v5 + PropSet v2, v6 key="kept" + v8 = Undefined + Return v8 +} +function f1 name="count" module=m0 position=14 flags=[HasExplicitName] length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + v1 = Integer + v2 = Integer + v3 = Builtin builtin=Object + v4 = Builtin builtin=ObjectHasOwn + Jump -> next b1(v1, v2) + b1(v5, v6): + LoopHint + v8 = Less v6, v0 + Branch v8 -> taken b2(), notTaken b7() + b2(): + v9 = NewObject -, v6, v0 layout=L0 fields=[0, 1] + v10 = FinishObject v9 + v11 = String string="size" + v12 = Call v4, v3, v10, v11 + v13 = ToBoolean v12 + Branch v13 -> taken b3(| 0), notTaken b4(v5) + b3(v14): + v15 = Increment v5 + Jump -> next b4(v15) + b4(v16): + v17 = String string="color" + v18 = Call v4, v3, v10, v17 + v19 = ToBoolean v18 + Branch v19 -> taken b5(| 0), notTaken b6(v16) + b5(v20): + v21 = Increment v16 + Jump -> next b6(v21) + b6(v22): + v23 = Increment v6 + Jump -> next b1(v22, v23) + b7(): + Return v5 +} +function f2 name="cleared" module=m0 position=24 flags=[HasExplicitName] length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + v1 = Integer + v2 = Integer + v3 = Builtin builtin=Object + v4 = Builtin builtin=ObjectHasOwn + Jump -> next b1(v1, v2) + b1(v5, v6): + LoopHint + v8 = Less v6, v0 + Branch v8 -> taken b2(), notTaken b7() + b2(): + v9 = NewObject -, v6, v0 layout=L0 fields=[0, 1] + v10 = FinishObject v9 + v11 = Integer integer=3 + v12 = Greater v6, v11 + Branch v12 -> taken b3(), notTaken b4() + b3(): + v13 = PropDelete v10 key="size" + Jump -> next b4() + b4(): + v14 = String string="size" + v15 = Call v4, v3, v10, v14 + v16 = ToBoolean v15 + Branch v16 -> taken b5(| 0), notTaken b6(v5) + b5(v17): + v18 = Increment v5 + Jump -> next b6(v18) + b6(v19): + v20 = Increment v6 + Jump -> next b1(v19, v20) + b7(): + Return v5 +} diff --git a/Source/JavaScriptCore/aot/tests/walk/loops/property-read-again-behind-test.ir b/Source/JavaScriptCore/aot/tests/walk/loops/property-read-again-behind-test.ir new file mode 100644 index 0000000000000..0fc39772eecb6 --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/walk/loops/property-read-again-behind-test.ir @@ -0,0 +1,81 @@ +# function scale(point) { if (typeof point.x === "number") return point.x * 3; } +# function moved(point) { if (typeof point.x === "number") { point.x = "moved"; return point.x * 3; } } +# function run(n) { let last; for (let i = 0; i < n; i++) { last = scale({ x: i & 1 ? i : "odd" }); last = moved({ x: i & 1 ? i : "odd" }); } return last; } +# Behind the test the second read of the property is the number that was tested, where no store stands between them. +# RUN: walk f2 mechanisms=all +# CHECK: Mul {{*}}form=Numbers +# RUN(MOVED): walk f3 mechanisms=all +# MOVED-NOT: form=Numbers + +module m0 path="main.js" displayPath="main.js" body=f0 +entry module=m0 +layout L0 module=m0 fields=[{key="x" declared={type=anything} flags=[IsRequired]}] +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = NewClosure function=f1 + v1 = Builtin builtin=GlobalThis + PropSet v1, v0 key="run" + v3 = Undefined + Return v3 +} +function f1 name="run" module=m0 position=14 flags=[HasExplicitName] length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + v1 = NewClosure function=f2 + v2 = NewClosure function=f3 + v3 = Integer + v4 = Undefined + Jump -> next b1(v3, v4) + b1(v5, v6): + LoopHint + v8 = Less v5, v0 + Branch v8 -> taken b2(), notTaken b5() + b2(): + v9 = Integer integer=1 + v10 = BitAnd v5, v9 + v11 = ToBoolean v10 + Branch v11 -> taken b4(v5), notTaken b3(| 0) + b3(v12): + v13 = String string="odd" + Jump -> next b4(v13) + b4(v14): + v15 = NewObject -, v14 layout=L0 fields=[0] + v16 = FinishObject v15 + v17 = Call v1, v4, v16 + v18 = NewObject -, v14 layout=L0 fields=[0] + v19 = FinishObject v18 + v20 = Call v2, v4, v19 + v21 = Increment v5 + Jump -> next b1(v21, v20) + b5(): + Return v6 +} +function f2 name="scale" module=m0 position=24 flags=[HasExplicitName] length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + v1 = PropGet v0 key="x" + v2 = TypeTest v1 testedType=number + Branch v2 -> taken b1(| 0), notTaken b2() + b1(v3): + v4 = PropGet v0 key="x" + v5 = Integer integer=3 + v6 = Mul v4, v5 + Return v6 + b2(): + v7 = Undefined + Return v7 +} +function f3 name="moved" module=m0 position=34 flags=[HasExplicitName] length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + v1 = PropGet v0 key="x" + v2 = TypeTest v1 testedType=number + Branch v2 -> taken b1(| 0), notTaken b2() + b1(v3): + v4 = String string="moved" + PropSet v0, v4 key="x" + v6 = PropGet v0 key="x" + v7 = Integer integer=3 + v8 = Mul v6, v7 + Return v8 + b2(): + v9 = Undefined + Return v9 +} diff --git a/Source/JavaScriptCore/aot/tests/walk/loops/pushes-onto-declared-array-literal.ir b/Source/JavaScriptCore/aot/tests/walk/loops/pushes-onto-declared-array-literal.ir new file mode 100644 index 0000000000000..7e1993d5acfff --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/walk/loops/pushes-onto-declared-array-literal.ir @@ -0,0 +1,70 @@ +# function row(text: string, above: number[], first: number): number[] { +# const current: number[] = [first]; +# for (let j = 1; j <= text.length; j++) current.push(Math.min(above[j] + 1, current[j - 1] + (text[j - 1] === "x" ? 0 : 1))); +# return current; +# } +# RUN: walk f1 mechanisms=all +# CHECK: arrayStorage=Double +# CHECK: ExtendArrayLength + +module m0 path="main.ts" displayPath="main.ts" body=f0 +entry module=m0 +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = NewClosure function=f1 + v1 = String string="axbxc" + v2 = Integer integer=1 + v3 = Integer integer=3 + v4 = NewArray v2, v3, v2, v3, v2, v3, v2 contents=number + v5 = Call v0, -, v1, v4, v3 calleeName="row" [isNeverInlined] + v6 = Builtin builtin=GlobalThis + v7 = PropGet v6 key="sink" [isFreeName] + v8 = Call v7, -, v5 + Return v8 +} +function f1 name="row" module=m0 position=10 flags=[IsChecked, HasExplicitName] length=3 thisType={type=anything} parameters=[{type=undefined|null|string}, {type=undefined|null|Array}, {type=number}] results=[{type=undefined|null|Array}] { + b0(v0: undefined|null|string, v1: undefined|null|Array, v2: number): + v3 = NewArray v2 contents=number + v4 = Convert v3 declared={type=undefined|null|Array flags=[OmitsNull, OmitsUndefined]} + v5 = Integer integer=1 + v6 = Convert v5 declared={type=number} + Jump -> next b1(v6) + b1(v7: number): + LoopHint + v9 = PropGet v0 key="length" + v10 = LessEq v7, v9 + Branch v10 -> taken b2(), notTaken b6() + b2(): + v11 = PropGet v4 key="push" + v12 = Builtin builtin=Math + v13 = Builtin builtin=MathMin + v14 = ElemGet v1, v7 + v15 = Integer integer=1 + v16 = Add v14, v15 + v17 = Integer integer=1 + v18 = Sub v7, v17 + v19 = ElemGet v4, v18 + v20 = Integer integer=1 + v21 = Sub v7, v20 + v22 = ElemGet v0, v21 + v23 = String string="x" + v24 = StrictEq v22, v23 + Branch v24 -> taken b3(), notTaken b4() + b3(): + v25 = Integer + Jump -> next b5(v25) + b4(): + v26 = Integer integer=1 + Jump -> next b5(v26) + b5(v27): + v28 = Add v19, v27 + v29 = Call v13, v12, v16, v28 calleeName="Math.min" + v30 = Convert v29 declared={type=number} + v31 = Call v11, v4, v30 calleeKey="push" + v32 = Convert v31 declared={type=number} + v33 = Increment v7 + v34 = Convert v33 declared={type=number} + Jump -> next b1(v34) + b6(): + Return v4 +} diff --git a/Source/JavaScriptCore/aot/tests/walk/typed-slots/literal-or-number-stored-to-slot.ir b/Source/JavaScriptCore/aot/tests/walk/typed-slots/literal-or-number-stored-to-slot.ir new file mode 100644 index 0000000000000..9c83d66927f5e --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/walk/typed-slots/literal-or-number-stored-to-slot.ir @@ -0,0 +1,49 @@ +# let kept; function fill(n) { kept = { x: 0 }; for (let i = 0; i < n; i++) kept = { x: i & 1 ? i : "odd" }; return kept.x; } +# What is stored is a counter or the written text: the check of the slot asks only for the bounds of the counter. +# RUN: walk f1 mechanisms=all +# CHECK: CheckProof {{*}}"odd" typeWithoutProof=int|absent|"odd" + +module m0 path="main.js" displayPath="main.js" body=f0 +entry module=m0 +layout L0 module=m0 fields=[{key="x" declared={type=anything} flags=[IsRequired]}] +variable g0 name="kept" module=m0 declared={type=anything} initialization=Undefined +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = NewClosure function=f1 + v1 = Builtin builtin=GlobalThis + PropSet v1, v0 key="fill" + v3 = Undefined + Return v3 +} +function f1 name="fill" module=m0 position=14 flags=[HasExplicitName] length=1 thisType={type=anything} parameters=[{type=anything}] results=[{type=anything}] { + b0(v0: anything): + v1 = Integer + v2 = NewObject -, v1 layout=L0 fields=[0] + v3 = FinishObject v2 + PutModuleVariable v3 variable=g0 + v5 = NewArray contents=anything + Jump -> next b1(v1) + b1(v6): + LoopHint + v8 = PropGet v5 key="length" + v9 = Less v6, v0 + Branch v9 -> taken b2(), notTaken b5() + b2(): + v10 = Integer integer=1 + v11 = BitAnd v6, v10 + v12 = ToBoolean v11 + Branch v12 -> taken b4(v6), notTaken b3(| 0) + b3(v13): + v14 = String string="odd" + Jump -> next b4(v14) + b4(v15): + v16 = NewObject -, v15 layout=L0 fields=[0] + v17 = FinishObject v16 + PutModuleVariable v17 variable=g0 + v19 = Increment v6 + Jump -> next b1(v19) + b5(): + v20 = GetModuleVariable variable=g0 + v21 = PropGet v20 key="x" + Return v21 +} diff --git a/Source/JavaScriptCore/aot/tests/walk/typed-slots/plain-heirs-with-named-members-are-guessed.ir b/Source/JavaScriptCore/aot/tests/walk/typed-slots/plain-heirs-with-named-members-are-guessed.ir new file mode 100644 index 0000000000000..5dab873279880 --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/walk/typed-slots/plain-heirs-with-named-members-are-guessed.ir @@ -0,0 +1,109 @@ +# A file of the library: +# class Base {} class Near extends Base {} class Below extends Base { [key]() {} } +# function store(object, value) { if ($isPlainHeirOf(object, Base)) { object.first = value; object.second = value; } } +# The answer proves the three classes. A store that may create a field has no typed form on Below, whose member may have any name, so a guess takes the other two. +# LIBRARY: own +# OPTIONS: mechanisms=all +# RUN(WALK): walk f1 +# WALK: IsPlainInstanceOf v0 {{*}}receivers=[L0, L1, L2] +# WALK: ShapeTest {{*}}layouts=[L0, L1] +# WALK: PutSlotOrAdd {{*}}key="first" receivers=[L0, L1] +# WALK: PutSlotOrAdd {{*}}key="second" receivers=[L0, L1] +# RUN(SHELL): run-in-shell globalThis.key = "other"; globalThis.show = objects => { for (const object of objects) print(JSON.stringify(object), object.constructor.name); }; +# SHELL: {"first":7,"second":7} Base +# SHELL-NEXT: {"first":7,"second":7} Near +# SHELL-NEXT: {"first":7,"second":7} Below +# SHELL-NEXT: exited with 0 + +module m0 path="main.js" displayPath="main.js" body=f0 namespaceLayout=L(m0) +module m1 path="/$bun/js/internal/asks.ts" displayPath="/$bun/js/internal/asks.ts" flags=[IsLibrary] +entry module=m0 +layout L0 kind=Instance name="Base" module=m0 position=60 constructor=f2 place=60 +layout L1 kind=Instance name="Near" module=m0 position=74 base=L0 constructor=f3 supertypes=[L0] flags=[HasHeritage] place=74 +layout L2 kind=Instance name="Below" module=m0 position=101 base=L0 constructor=f4 supertypes=[L0] flags=[HasHeritage] place=101 +variable g0 name="Base" module=m0 declared={type=absent|f2 flags=[IsFromCompiler]} flags=[IsImmutable] place=3 +variable g1 module=m0 declared={type=f2 flags=[IsFromCompiler]} flags=[IsImmutable] place=34 +variable g2 module=m0 declared={type=f3 flags=[IsFromCompiler]} flags=[IsImmutable] place=42 +variable g3 module=m0 declared={type=f4 flags=[IsFromCompiler]} flags=[IsImmutable] place=50 +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = Builtin builtin=GlobalThis + v1 = PropGet v0 key="key" [isFreeName] + v2 = NewClosure function=f2 @54 + v3 = NewClass v2 layout=L0 @54 + PutModuleVariable v3 variable=g1 @54 + v5 = FinishClass v3 layout=L0 @54 + v6 = Convert v5 declared={type=f2 flags=[IsFromCompiler]} @54 + PutModuleVariable v6 variable=g0 @54 + v8 = ResolveHeritage v6 @87 + v9 = NewClosure function=f3 @68 + v10 = NewClass v9, v6, v8 layout=L1 @68 + PutModuleVariable v10 variable=g2 @68 + v12 = FinishClass v10 layout=L1 @68 + v13 = Convert v12 declared={type=f3 flags=[IsFromCompiler]} @68 + v14 = ResolveHeritage v6 @115 + v15 = NewClosure function=f4 @95 + v16 = NewClass v15, v6, v14 layout=L2 @95 + PutModuleVariable v16 variable=g3 @95 + v18 = ToPropertyKey v1 @123 + v19 = NewClosure function=f5 @122 + SetFunctionName v19, v18 @122 + DefineMethod v16, v19, v18 layout=L2 @122 + v22 = FinishClass v16 layout=L2 @95 + v23 = Convert v22 declared={type=f4 flags=[IsFromCompiler]} @95 + v24 = NewClosure function=f1 @0 + v25 = Integer integer=7 @285 + v26 = Construct v6 calleeName="Base" @286 + v27 = Construct v13 calleeName="Near" @298 + v28 = Construct v23 calleeName="Below" @310 + v29 = Call v24, -, v26, v25 calleeName="store" @320 + v30 = Call v24, -, v27, v25 calleeName="store" @330 + v31 = Call v24, -, v28, v25 calleeName="store" @340 + v32 = NewArray v26, v27, v28 contents=anything @350 + v33 = PropGet v0 key="show" [isFreeName] @360 + v34 = Call v33, -, v32 calleeName="show" @360 + v35 = NewArray v24, v6, v13, v23 contents=anything @370 + PropSet v0, v35 key="published" @380 + v37 = Undefined + Return v37 +} +function f1 name="store" module=m1 position=146 role=Library flags=[HasExplicitName] length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=undefined}] { + b0(v0: anything, v1: anything): + v2 = GetModuleVariable variable=g0 @202 + v3 = CheckInitialized v2 name="Base" @202 + v4 = IsPlainHeirOf v0, v3 @202 + Branch v4 -> taken b1(), notTaken b2() @184 + b1(): + PropSet v0, v1 key="first" @214 + PropSet v0, v1 key="second" @240 + Jump -> next b2() @240 + b2(): + v7 = Undefined @240 + Return v7 @240 +} +function f2 name="Base" module=m0 position=60 kind=BaseConstructor flags=[HasNewTarget, HasExplicitName] thisType={type=anything} results=[{type=object flags=[IsFromCompiler]}] { + b0(v0: anything): + v1 = GetConstructPrototype v0 layout=L0 @60 + v2 = NewObject v1 layout=L0 @60 + v3 = Convert v2 declared={type=L0} @60 + Return v3 @60 +} +function f3 name="Near" module=m0 position=74 kind=DerivedConstructor flags=[HasRestParameter, HasNewTarget, TakesCallee, HasExplicitName] thisType={type=anything} parameters=[{type=Array flags=[IsFromCompiler]}] results=[{type=object flags=[IsFromCompiler]}] { + b0(v0: function, v1: anything, v2: Array): + v3 = GetSuperBase v0 @74 + v4 = SuperConstructList v3, v1, v2 layout=L1 @74 + v5 = Convert v4 declared={type=L1} @74 + Return v5 @74 +} +function f4 name="Below" module=m0 position=101 kind=DerivedConstructor flags=[HasRestParameter, HasNewTarget, TakesCallee, HasExplicitName] thisType={type=anything} parameters=[{type=Array flags=[IsFromCompiler]}] results=[{type=object flags=[IsFromCompiler]}] { + b0(v0: function, v1: anything, v2: Array): + v3 = GetSuperBase v0 @101 + v4 = SuperConstructList v3, v1, v2 layout=L2 @101 + v5 = Convert v4 declared={type=L2} @101 + Return v5 @101 +} +function f5 module=m1 position=122 kind=Method role=Library thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = Integer integer=1 @139 + Return v0 @139 +} diff --git a/Source/JavaScriptCore/aot/walk/AOTArrayStorageForms.h b/Source/JavaScriptCore/aot/walk/AOTArrayStorageForms.h index a192338e589f0..d1f7c72eaebf4 100644 --- a/Source/JavaScriptCore/aot/walk/AOTArrayStorageForms.h +++ b/Source/JavaScriptCore/aot/walk/AOTArrayStorageForms.h @@ -67,10 +67,10 @@ constexpr bool hasArrayStorageForm(ArrayStorage kind) { return findArrayStorageF constexpr bool accessToElementsHasCode(Opcode opcode, ArrayStorage kind, bool admitsInt32Storage, bool isWritable) { if (opcode != Opcode::HasArrayStorage) - return !isWritable && (!admitsInt32Storage || kind == ArrayStorage::Double); + return !isWritable && (!admitsInt32Storage || kind == ArrayStorage::Double || kind == ArrayStorage::Contiguous); if (!hasArrayStorageForm(kind)) return isWritable; - return !admitsInt32Storage || (kind != ArrayStorage::Int32 && !isWritable); + return !admitsInt32Storage || kind == ArrayStorage::Double || kind == ArrayStorage::Contiguous || (kind != ArrayStorage::Int32 && !isWritable); } constexpr bool isNumericStorage(ArrayStorage kind) diff --git a/Source/JavaScriptCore/aot/walk/AOTCode.cpp b/Source/JavaScriptCore/aot/walk/AOTCode.cpp index dc034ca25e2e8..18922f5fd8811 100644 --- a/Source/JavaScriptCore/aot/walk/AOTCode.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTCode.cpp @@ -59,7 +59,7 @@ bool readsNumberOrNothingOutOfBytes(const Operation& operation) { if (operation.opcode != Opcode::GetTypedArrayElement || operation.isInsideView() || operation.representation != Representation::OptionalDouble) return false; - return isFloat(operation.typedArrayType()) && hasInlineElementAccess(operation.typedArrayType()); + return !isBigInt(operation.typedArrayType()) && hasInlineElementAccess(operation.typedArrayType()); } bool handsOnBitsOfNaN(const Operation& operation) diff --git a/Source/JavaScriptCore/aot/walk/AOTCode.h b/Source/JavaScriptCore/aot/walk/AOTCode.h index 47ba56848eb35..f3a4cbcde5035 100644 --- a/Source/JavaScriptCore/aot/walk/AOTCode.h +++ b/Source/JavaScriptCore/aot/walk/AOTCode.h @@ -853,6 +853,7 @@ struct GroundsOfPosition { ValueID isBelow; bool isBelowResult { true }; ValueID isFromZeroUp; + bool isFromZeroUpResult { true }; ValueID counter; ValueID start; ValueID stepNonNegativeCheck; @@ -1376,7 +1377,7 @@ inline std::optional lengthThatListIsMadeWith(const Code& code, const } inline bool hasTestedForHole(const Operation& read) { - return read.opcode == Opcode::GetArrayElement && !read.isPresent() && !read.readsHoleAsNaN() && !testsItsOwnGuess(read); + return read.opcode == Opcode::GetArrayElement && !read.isPresent() && !read.readsHoleAsNaN(); } template ReadOfEmptySlot whatReadOfEmptySlotGives(const Read& read) diff --git a/Source/JavaScriptCore/aot/walk/AOTCodeVerifier.cpp b/Source/JavaScriptCore/aot/walk/AOTCodeVerifier.cpp index 66c8fb3c9850e..a7567c4829c89 100644 --- a/Source/JavaScriptCore/aot/walk/AOTCodeVerifier.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTCodeVerifier.cpp @@ -897,7 +897,7 @@ void CodeVerifier::verifyOperation(BlockID block, uint32_t place, ValueID id, co if (opcode == Opcode::PutHeapFrameWord && !operation.remembersFrame() && m_types.at(m_code.at(operation.valueOperand()).type).hasCells()) return fail(block, id, "what may be a cell is stored into a word of a frame in the heap, and the store does not remember the frame"_s, operation.valueOperand()); - if (opcode == Opcode::GetArrayElement && operation.holdsDoublesOperand()) { + if ((opcode == Opcode::GetArrayElement || opcode == Opcode::IsElementPresent) && operation.holdsDoublesOperand()) { const Operation& kind = m_code.at(operation.holdsDoublesOperand()); if (kind.opcode != Opcode::HasArrayStorage || kind.arrayStorage() != ArrayStorage::Double || kind.admitsInt32Storage() || !areSameCells(kind.arrayOperand(), operation.arrayOperand())) return fail(block, id, "a read takes for the kind of its list what no question of that list for doubles gives"_s, operation.holdsDoublesOperand()); @@ -1416,7 +1416,7 @@ bool CodeVerifier::treatsUndefinedAsNaN(ValueID id, const Operation& user, uint3 } if (arithmeticOnTwoNumbersInCodeVerifier.contains(user.opcode) || allRelations.contains(user.opcode)) return operandIndex < 2; - if (user.opcode == Opcode::ToInt32 || user.opcode == Opcode::ToUint32 || allBitwiseOperators.contains(user.opcode)) + if (user.opcode == Opcode::ToInt32 || user.opcode == Opcode::ToUint32 || user.opcode == Opcode::IntegerBound || allBitwiseOperators.contains(user.opcode)) return true; if (user.opcode == Opcode::TypeTest) { const Type& tested = m_types.at(user.testedType()); @@ -1663,9 +1663,7 @@ const DataNode::Array* CodeVerifier::findWrittenDataShownOf(ValueID cell, BlockI for (ValueID test : *m_testsForWrittenData) { if (underlyingCell(m_code.at(test).arrayOperand()) != asked) continue; - EdgeID passes = m_code.at(m_code.blockOf(test)).terminator.taken(); - BlockID behind = m_code.at(passes).to; - if (m_code.at(behind).firstEdgeIn() == passes && !m_code.at(passes).nextInto() && dominates(headOf(behind), headOf(block))) + if (isAnsweredAt(test, true, block, 0)) return m_program.at(m_code.at(test).data()).root.tryAs(); } return nullptr; @@ -1880,9 +1878,10 @@ void CodeVerifier::verifyKindsOfStorageNamed() return fail(block, id, "a store is taken to lay an element into a list that is not made here, or under what is no known index"_s, access.arrayOperand()); continue; } - if (!accessesAtLevels.contains(access.opcode) || access.level() != KnowledgeLevel::Proven || !hasArrayStorageForm(access.arrayStorage())) + bool isNumericStore = access.opcode == Opcode::PutContiguousIndex && access.hasNumericStorage(); + if (!isNumericStore && (!accessesAtLevels.contains(access.opcode) || access.level() != KnowledgeLevel::Proven || !hasArrayStorageForm(access.arrayStorage()))) continue; - ArrayStorage named = access.arrayStorage(); + ArrayStorage named = isNumericStore ? ArrayStorage::Other : access.arrayStorage(); bool admitsInt32Storage = access.opcode == Opcode::GetArrayElement && access.admitsInt32Storage(); Vector cells; collectUnderlyingCells(access.operand(0), cells); @@ -1895,10 +1894,10 @@ void CodeVerifier::verifyKindsOfStorageNamed() if (!everyMakerIsInSight) break; ArrayStorage born = written ? arrayStorageOfData(m_program, *written) : storageThatLiteralIsSurelyBornWith(m_program, m_types, m_code, made); - if (born != named && !(admitsInt32Storage && born == ArrayStorage::Int32)) + if (isNumericStore ? !isNumericStorage(born) : born != named && !(admitsInt32Storage && born == ArrayStorage::Int32)) bornOtherwise = cell; } - if (!everyMakerIsInSight || !bornOtherwise) + if (everyMakerIsInSight ? !bornOtherwise : !isNumericStore) continue; if (!tests) { tests.emplace(); @@ -1909,11 +1908,14 @@ void CodeVerifier::verifyKindsOfStorageNamed() } } } + auto showsKind = [&](const Operation& test) { + return isNumericStore ? test.isWritable() && isNumericStorage(test.arrayStorage()) : test.arrayStorage() == named; + }; bool hasPassedTest = std::ranges::any_of(*tests, [&](ValueID test) { - return m_code.at(test).arrayStorage() == named && areSameCells(m_code.at(test).arrayOperand(), access.operand(0)) && isAnsweredAt(test, true, block, m_places[id.index()]); + return showsKind(m_code.at(test)) && areSameCells(m_code.at(test).arrayOperand(), access.operand(0)) && isAnsweredAt(test, true, block, m_places[id.index()]); }); if (!hasPassedTest) - return fail(block, id, "an access names a kind of storage that a maker of its list is not surely born with, and no test of that kind has passed"_s, bornOtherwise, InternalErrorKind::InvalidGuard); + return fail(block, id, isNumericStore ? "a store names a numeric kind of storage that no passed test of its array shows"_s : "an access names a kind of storage that a maker of its list is not surely born with, and no test of that kind has passed"_s, isNumericStore ? access.arrayOperand() : bornOtherwise, InternalErrorKind::InvalidGuard); } } } diff --git a/Source/JavaScriptCore/aot/walk/AOTInlineBuiltinForms.cpp b/Source/JavaScriptCore/aot/walk/AOTInlineBuiltinForms.cpp index 3d4939a4a1708..e026a0023d523 100644 --- a/Source/JavaScriptCore/aot/walk/AOTInlineBuiltinForms.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTInlineBuiltinForms.cpp @@ -170,7 +170,7 @@ static InlineForm chooseFormOfReflectApply(const InlineBuiltinQuery& query) static InlineForm chooseFormOfReflectConstruct(const InlineBuiltinQuery& query) { bool takesOperands = (query.numberOfArguments() == 2 || query.numberOfArguments() == 3) && everyArgumentButThisMayBeObjectForReflect(query, std::nullopt); - return query.hasSlowPath && query.isDoneInPlace && takesOperands ? InlineForm::TwoWay : InlineForm::Call; + return query.hasSlowPath && takesOperands ? InlineForm::TwoWay : InlineForm::Call; } InlineForm chooseFormOfFunctionBuiltin(const InlineBuiltinQuery& query) diff --git a/Source/JavaScriptCore/aot/walk/AOTRepresentationRules.cpp b/Source/JavaScriptCore/aot/walk/AOTRepresentationRules.cpp index b46afccf7092f..06c334cf85c7e 100644 --- a/Source/JavaScriptCore/aot/walk/AOTRepresentationRules.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTRepresentationRules.cpp @@ -80,7 +80,7 @@ static constexpr RepresentationRule representationRuleOf(Opcode opcode) case Opcode::CheckInitialized: case Opcode::RequireObjectCoercible: case Opcode::RequireObject: case Opcode::FinishObject: case Opcode::FinishClass: case Opcode::Convert: case Opcode::Check: case Opcode::Expect: case Opcode::Upcast: case Opcode::NarrowByField: case Opcode::NarrowByTest: return RepresentationRule::Forwarding; - case Opcode::TypeTest: case Opcode::EndCellLifetime: case Opcode::CheckProof: case Opcode::AskObjectAgain: case Opcode::CheckIterationRestart: case Opcode::CheckInlineeFallback: case Opcode::CheckReloadBypass: + case Opcode::TypeTest: case Opcode::IsUnlockedInstance: case Opcode::EndCellLifetime: case Opcode::CheckProof: case Opcode::AskObjectAgain: case Opcode::CheckIterationRestart: case Opcode::CheckInlineeFallback: case Opcode::CheckReloadBypass: return RepresentationRule::AsItIs; case Opcode::Add: case Opcode::Sub: case Opcode::Mul: case Opcode::Div: case Opcode::Mod: case Opcode::Pow: case Opcode::Negate: case Opcode::Increment: case Opcode::Decrement: case Opcode::BitAnd: case Opcode::BitOr: case Opcode::BitXor: case Opcode::LShift: case Opcode::RShift: case Opcode::URShift: case Opcode::BitNot: @@ -94,7 +94,7 @@ static constexpr RepresentationRule representationRuleOf(Opcode opcode) return RepresentationRule::ElementAccess; case Opcode::ElemGet: case Opcode::ElemSet: return RepresentationRule::KeyedAccess; - case Opcode::NewArray: case Opcode::AppendElement: + case Opcode::NewArray: case Opcode::AppendElement: case Opcode::TryPutIndex: return RepresentationRule::NewElement; case Opcode::Strcat: return RepresentationRule::Concatenation; @@ -140,7 +140,7 @@ static constexpr RepresentationRule representationRuleOf(Opcode opcode) case Opcode::PropGet: case Opcode::PropSet: case Opcode::PropDefine: case Opcode::GetGlobalBinding: case Opcode::SetGlobalBinding: case Opcode::PropHas: case Opcode::ElemHas: case Opcode::PropHasOwn: case Opcode::PropDelete: case Opcode::ElemDelete: case Opcode::ElemGetByKeyParts: case Opcode::DefineGetter: case Opcode::DefineSetter: case Opcode::DefineOwnProperty: case Opcode::DefineAccessors: case Opcode::SetFunctionName: case Opcode::SetLiteralPrototype: case Opcode::ElemDefine: - case Opcode::ToObject: case Opcode::ToThis: case Opcode::GetArrayLength: case Opcode::GetArrayStorage: case Opcode::HasArrayStorage: case Opcode::DivideIntegers: case Opcode::RemainderOfIntegers: case Opcode::ReciprocalOfDivisor: case Opcode::RemainderByReciprocal: case Opcode::ToIntegerOrInfinity: + case Opcode::ToObject: case Opcode::ToThis: case Opcode::GetArrayLength: case Opcode::GetArrayStorage: case Opcode::HasArrayStorage: case Opcode::DivideIntegers: case Opcode::RemainderOfIntegers: case Opcode::ReciprocalOfDivisor: case Opcode::RemainderByReciprocal: case Opcode::WholeDivisor: case Opcode::RemainderByWholeDivisor: case Opcode::ToIntegerOrInfinity: case Opcode::IntegerBound: case Opcode::StringOrder: case Opcode::StringOrderToCharacter: case Opcode::StringEqualsLiteral: case Opcode::ValueEqualsLiteral: case Opcode::LogicalNot: case Opcode::NumberOrNaN: case Opcode::TypeOf: case Opcode::InstanceOf: case Opcode::HasInPrototypeChain: case Opcode::TryFindInPrototypeChain: case Opcode::HasDefaultHasInstance: case Opcode::ToPropertyKey: @@ -156,7 +156,7 @@ static constexpr RepresentationRule representationRuleOf(Opcode opcode) case Opcode::HasJSType: case Opcode::FindsDeclaredMember: case Opcode::FindsNoMember: case Opcode::IsPlainInstanceOf: case Opcode::IsPlainHeirOf: case Opcode::IsAsBorn: case Opcode::IsAsBornExceptForAddedNames: case Opcode::IsPrototypeChainPlain: case Opcode::GetInternalField: case Opcode::PutInternalField: case Opcode::ThrowNewError: case Opcode::IsPristineArray: case Opcode::ArrayStorageKind: case Opcode::HasContiguousIndex: case Opcode::GetContiguousIndex: case Opcode::GetDoubleIndex: case Opcode::PutContiguousIndex: case Opcode::PutDoubleIndex: case Opcode::GetOwnContiguousIndex: case Opcode::GetOwnDoubleIndex: case Opcode::GetOwnInt32Index: case Opcode::PutOwnContiguousIndex: case Opcode::PutOwnDoubleIndex: case Opcode::PutOwnInt32Index: - case Opcode::TryPutIndex: case Opcode::TryCopyFilledElements: case Opcode::TryPutElements: case Opcode::TryFillNewArray: + case Opcode::TryCopyFilledElements: case Opcode::TryPutElements: case Opcode::SortNumbers: case Opcode::TryFillNewArray: case Opcode::NewContiguousArray: case Opcode::NewArrayLike: case Opcode::TruncateNewArray: case Opcode::AppendElements: case Opcode::AppendMembers: case Opcode::AppendValuesOfMap: case Opcode::ReserveArrayCapacity: case Opcode::ConvertToDoubleStorage: case Opcode::GetTypedArrayLength: case Opcode::GetTypedArrayVector: case Opcode::RequireAttachedTypedArray: case Opcode::RequireLengthFitsList: case Opcode::GetStringLength: case Opcode::GetStringCodeUnit: case Opcode::Substring: case Opcode::FlattenString: case Opcode::StringStartsWithLiteral: case Opcode::StringEndsWithLiteral: case Opcode::StringIndexOfLiteral: case Opcode::StringIndexOfCharacter: case Opcode::StringLastIndexOfLiteral: @@ -549,7 +549,7 @@ class RepresentationRuleTable { return integerFormOf(type); if (naturalFormAt(operandIndex) == Representation::OptionalInt64) return Representation::OptionalInt64; - if (type.isNumber() && held == Representation::JSValue) + if (held == Representation::JSValue && type.hasOnlyPrimitives(Primitive::Number | Primitive::UndefinedOrNull)) return Representation::Unchosen; } if (opcode() == Opcode::ToBoolean && held == Representation::OptionalInt64) @@ -692,6 +692,8 @@ class RepresentationRuleTable { { if (opcode() == Opcode::AppendElement) return operandIndex == OperandIndex::AppendElement::value; + if (opcode() == Opcode::TryPutIndex) + return operandIndex == OperandIndex::TryPutIndex::value; return !!variadicIndexOf(operandIndex); } @@ -714,7 +716,7 @@ class RepresentationRuleTable { if (opcode() == Opcode::AppendElement) return wantedByElementAppendedTo(OperandIndex::AppendElement::array, operandIndex); Representation wanted = wantedByNewElement(operandIndex); - if (wanted != Representation::JSValue || m_subject.storage() != AllocationStorage::Heap) + if (wanted != Representation::JSValue || opcode() == Opcode::TryPutIndex || m_subject.storage() != AllocationStorage::Heap) return wanted; bool isStoredAsDouble = m_subject.arrayStorage() == ArrayStorage::Double; return isStoredAsDouble && heldFormAt(operandIndex) == Representation::Int64 ? Representation::Double : wanted; diff --git a/Source/JavaScriptCore/aot/walk/AOTSumOverRecords.cpp b/Source/JavaScriptCore/aot/walk/AOTSumOverRecords.cpp index 6cb12e26381a9..a03fafd5ade24 100644 --- a/Source/JavaScriptCore/aot/walk/AOTSumOverRecords.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTSumOverRecords.cpp @@ -237,7 +237,7 @@ bool groundsShowPositionInside(const Code& code, const TypeArena& types, ValueID bool isFromZeroUp = (isIntegerInSumOverRecords(code, types, position) && types.at(code.at(position).type).integers.minimum >= 0) || fromZero.isFromZeroUp(); if (!isFromZeroUp && grounds.counter && !(fromZero.add(grounds.counter, -1) && fromZero.add(grounds.start, 1))) return false; - isFromZeroUp = isFromZeroUp || fromZero.isFromZeroUp() || (mayBeTaken(grounds.isFromZeroUp) && fromZero.takeAwayWhatIsFromZeroUp(grounds.isFromZeroUp) && fromZero.isFromZeroUp()); + isFromZeroUp = isFromZeroUp || fromZero.isFromZeroUp() || (mayBeTaken(grounds.isFromZeroUp, grounds.isFromZeroUpResult) && fromZero.takeAwayWhatIsFromZeroUp(grounds.isFromZeroUp, grounds.isFromZeroUpResult) && fromZero.isFromZeroUp()); auto isShownBelow = [&](const SumOverRecords& sum) { SumOverRecords behindTest(sum); bool isBehindTest = mayBeTaken(grounds.isBelow, grounds.isBelowResult) && behindTest.takeAwayWhatIsFromZeroUp(grounds.isBelow, grounds.isBelowResult); diff --git a/Source/JavaScriptCore/aot/walk/AOTSweep.cpp b/Source/JavaScriptCore/aot/walk/AOTSweep.cpp index 0faf857df8d25..cd3e273be68fa 100644 --- a/Source/JavaScriptCore/aot/walk/AOTSweep.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTSweep.cpp @@ -707,6 +707,8 @@ ALWAYS_INLINE_INTO_OTHER_FILES void Sweep::visitOperationsOf(BlockID id) if (hasOwnerBarrier || !m_storesThatAwaitBirth.isEmpty()) lookForBirthOfOwner(value, operation); } + if (opcode == Opcode::CreateUnmappedArguments && m_looksForBirthsToMove) + noteIfBirthMayMove(value, operation); if (looksForCoveredFences && (m_laterFenceCovers || (row & isAllocationWithFenceInRowOfSweep))) lookForCoveredFence(value, operation); } diff --git a/Source/JavaScriptCore/aot/walk/AOTSweep.h b/Source/JavaScriptCore/aot/walk/AOTSweep.h index 5f0b40eb7e049..e03974ca82489 100644 --- a/Source/JavaScriptCore/aot/walk/AOTSweep.h +++ b/Source/JavaScriptCore/aot/walk/AOTSweep.h @@ -133,6 +133,7 @@ class Sweep { bool beginLookingForBirthsToMove(); void noteIfBirthMayMove(ValueID, const Operation&); + bool isWhereBirthMayMove(ValueID, const Operation&) const; bool moveBirthsToWhereCellsAreNeeded(); SweepTotals sweepAgainBehindMovedBirths(const SweepInputs&, Code&); @@ -332,7 +333,7 @@ inline constexpr OpcodeSet sweptOperationsWithOwnerBarriers = opcodesWhere([](Op return runsOwnerBarrierOf(opcode) && remembersOwnerOf(opcode); }); inline constexpr OpcodeSet sweptAllocationsWithFences { Opcode::NewObject, Opcode::NewClosure, Opcode::NewArray, Opcode::CreateRest }; -inline constexpr OpcodeSet birthsThatSweepMayMove { Opcode::NewObject, Opcode::NewArray, Opcode::NewClosure }; +inline constexpr OpcodeSet birthsThatSweepMayMove { Opcode::NewObject, Opcode::NewArray, Opcode::NewClosure, Opcode::CreateRest, Opcode::CreateUnmappedArguments }; inline constexpr OpcodeSet storesThatMoveWithBirthsInSweep { Opcode::PutSlot, Opcode::PutSlotOrAdd, Opcode::PutEnvironmentField, Opcode::SetMappedArgument, Opcode::FinishBirth, Opcode::TryFinishBirth }; bool newObjectFencesBeforeHandingCellOn(const Program&, const Plan&, const Operation& newObject); diff --git a/Source/JavaScriptCore/aot/walk/AOTSweepStores.cpp b/Source/JavaScriptCore/aot/walk/AOTSweepStores.cpp index 4f81d52c41842..694650199a269 100644 --- a/Source/JavaScriptCore/aot/walk/AOTSweepStores.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTSweepStores.cpp @@ -238,6 +238,7 @@ struct BirthThatMayMove { BlockID whereItComesToStand; uint32_t numberOfUsesSeen { 0 }; bool isFollowed { false }; + bool isSettledBeforeLoops { false }; Vector stores; Vector birthsThatListIt; }; @@ -253,6 +254,15 @@ struct BlockBelowBirth { uint32_t numberOfEdgesTakenInto { std::numeric_limits::max() }; uint32_t look { 0 }; bool isLookedAt { false }; + bool isPassedLoopHead { false }; + BlockID innermostLoop; + BlockID loopAround; + uint32_t numberOfWaysOutOfLoop { 0 }; +}; + +struct EdgeBackToLoopHead { + BlockID head; + BlockID from; }; struct OperationThatMoves { @@ -262,20 +272,38 @@ struct OperationThatMoves { } // anonymous namespace -static bool birthMayMoveInSweep(const Program& program, const Facts& facts, const Operation& made) +static bool entryHidesItsListInSweep(const Program& program, const Code& code) +{ + const Function& function = program.at(code.function()); + return function.has(FunctionFlags::HasHiddenRest) && !function.has(FunctionFlags::HasRestParameter); +} + +static bool isListMadeByEntryInSweep(const Code& code, ValueID list) { - if (!made.traits().isUnobservable() || made.storage() != AllocationStorage::Heap) + static constexpr OpcodeSet changesOfForm { Opcode::Box, Opcode::Unbox, Opcode::Decode }; + while (!code.isConstant(list) && changesOfForm.contains(code.at(list).opcode)) + list = code.at(list).valueOperand(); + return !code.isConstant(list) && (code.at(list).opcode == Opcode::CreateRest || (code.at(list).opcode == Opcode::Parameter && code.at(list).block() == entryBlock)); +} + +static bool birthMayMoveInSweep(const Program& program, const Facts& facts, const Code& code, const Operation& made) +{ + if (!made.traits().isUnobservable() || (PayloadWord::storage(made.opcode) && made.storage() != AllocationStorage::Heap)) return false; switch (made.opcode) { case Opcode::NewArray: return true; + case Opcode::CreateRest: + return entryHidesItsListInSweep(program, code); + case Opcode::CreateUnmappedArguments: + return entryHidesItsListInSweep(program, code) && isListMadeByEntryInSweep(code, made.restOperand()); case Opcode::NewClosure: if (made.isHeldByClosureTable() || made.isMadeOnDemand() || (made.closureForm() != ClosureForm::Full && made.closureForm() != ClosureForm::Slim)) return false; return !program.at(made.function()).constructedClass; case Opcode::NewObject: { LayoutID declared = declaredLayoutOf(program, made.layout()); - if (made.prototypeOperand() || !declared || program.at(declared).has(LayoutFlags::IsConstructorFunction)) + if (!declared || ((made.prototypeOperand() || program.at(declared).has(LayoutFlags::IsConstructorFunction)) && (!made.endsNoGuards() || !code.at(made.block()).isInLoop))) return false; switch (program.at(declared).kind) { case LayoutKind::Record: @@ -297,24 +325,29 @@ bool Sweep::beginLookingForBirthsToMove() const Code& code = *m_code; if (!code.opcodesOfValues().intersects(birthsThatSweepMayMove)) return false; - if (code.role() != CodeRole::Body || code.hasSuspensions()) + bool isEntryThatMakesArguments = code.role() == CodeRole::GenericCallEntry && code.opcodesOfValues().contains(Opcode::CreateUnmappedArguments); + if ((code.role() != CodeRole::Body && !isEntryThatMakesArguments) || code.hasSuspensions()) return false; return code.has(Mechanism::EscapeAnalysis, m_inputs->facts); } -void Sweep::noteIfBirthMayMove(ValueID allocation, const Operation& made) +bool Sweep::isWhereBirthMayMove(ValueID allocation, const Operation& made) const { const Code& code = *m_code; if (ValueID user = m_arrays.onlyUsers[allocation.index()]) { const Operation& only = code.at(user); + if (only.blockIndex == made.blockIndex && only.opcode == Opcode::Box) + return code.at(made.block()).isInLoop && isWhereBirthMayMove(user, only); if (only.blockIndex == made.blockIndex && !storesThatMoveWithBirthsInSweep.contains(only.opcode) && !birthsThatSweepMayMove.contains(only.opcode)) - return; + return false; } - const Program& program = m_inputs->program; const Block& block = code.at(made.block()); - if (block.handler || !block.terminator.numberOfEdges || (!block.isInLoop && program.at(code.function()).isModuleBody())) - return; - if (birthMayMoveInSweep(program, m_inputs->facts, made)) + return !block.handler && block.terminator.numberOfEdges && (block.isInLoop || !m_inputs->program.at(code.function()).isModuleBody()); +} + +void Sweep::noteIfBirthMayMove(ValueID allocation, const Operation& made) +{ + if (isWhereBirthMayMove(allocation, made) && birthMayMoveInSweep(m_inputs->program, m_inputs->facts, *m_code, made)) m_birthsThatMayMove.append(allocation); } @@ -385,12 +418,16 @@ bool Sweep::moveBirthsToWhereCellsAreNeeded() } }; + auto mayBeBoxThatMoves = [&](ValueID value) { + const Operation& operation = code.at(value); + return operation.opcode == Opcode::Box && operation.representation == Representation::JSValue && code.at(operation.block()).isInLoop && isWhereBirthMayMove(value, operation); + }; for (size_t firstOfBlock = 0, end = 0; firstOfBlock < m_birthsThatMayMove.size(); firstOfBlock = end) { BlockID id = code.at(m_birthsThatMayMove[firstOfBlock]).block(); while (end < m_birthsThatMayMove.size() && code.at(m_birthsThatMayMove[end]).block() == id) ++end; forEachUseOfBirthIn(id, m_birthsThatMayMove[end - 1].index() + 1, [&](ValueID birth, ValueID user, bool onlyFillsCell) { - if (onlyFillsCell || code.at(birth).block() != id || namesBirth(user)) + if (onlyFillsCell || code.at(birth).block() != id || namesBirth(user) || (user && mayBeBoxThatMoves(user))) return; mayMove.quickClear(birth.index()); --numberThatMayMove; @@ -401,6 +438,30 @@ bool Sweep::moveBirthsToWhereCellsAreNeeded() if (!numberThatMayMove) return false; + Vector boxes; + bool someBirthMovesInLoop = std::ranges::any_of(m_birthsThatMayMove, [&](ValueID allocation) { + return mayMove.quickGet(allocation.index()) && code.at(code.at(allocation).block()).isInLoop; + }); + for (BlockID id : m_reachedInPostorder) { + if (!someBirthMovesInLoop || !code.at(id).isInLoop) + continue; + for (ValueID value : code.valuesOf(id)) { + if (code.at(value).opcode == Opcode::Box && namesBirth(code.at(value).valueOperand()) && code.useCountOf(value) != usesAreNotCounted && mayBeBoxThatMoves(value)) + boxes.append(value); + } + } + for (ValueID box : boxes) + mayMove.quickSet(box.index()); + for (ValueID box : boxes) { + forEachUseOfBirthIn(code.at(box).block(), box.index() + 1, [&](ValueID used, ValueID user, bool) { + if (used == box && !namesBirth(user)) + mayMove.quickClear(box.index()); + }, [](ValueID) { }); + } + m_birthsThatMayMove.appendVector(boxes); + m_totals.operationsSwept += numberOfOperationsLookedAt; + numberOfOperationsLookedAt = 0; + auto standsOnOneSideOfBranch = [&](BlockID id) { BlockID above; for (EdgeID edge = code.at(id).firstEdgeIn(); edge; edge = code.at(edge).nextInto()) { @@ -425,7 +486,7 @@ bool Sweep::moveBirthsToWhereCellsAreNeeded() for (ValueID allocation : m_birthsThatMayMove) { if (!mayMove.quickGet(allocation.index())) continue; - births.append(BirthThatMayMove { allocation, { }, { }, 0, false, { }, { } }); + births.append(BirthThatMayMove { allocation, { }, { }, 0, false, false, { }, { } }); } AOT::sort(births, { }, &BirthThatMayMove::allocation); auto recordOf = [&](ValueID allocation) -> BirthThatMayMove& { @@ -469,6 +530,10 @@ bool Sweep::moveBirthsToWhereCellsAreNeeded() laid.depth = 0; laid.nearestFreeOfHandlers = BlockID(); laid.nearestSideOfBranch = BlockID(); + laid.isPassedLoopHead = false; + laid.innermostLoop = BlockID(); + laid.loopAround = BlockID(); + laid.numberOfWaysOutOfLoop = 0; if (above) { const BlockBelowBirth& upper = blocksBelow[above.index()]; const BlockBelowBirth& leap = blocksBelow[upper.leap.index()]; @@ -482,13 +547,26 @@ bool Sweep::moveBirthsToWhereCellsAreNeeded() laid.nearestFreeOfHandlers = id; blocksToLookAt.append(id); }; + Vector loopHeadsAwaited; + Vector loopHeadsPassed; + Vector edgesBackToLoopHeads; + bool hasOtherWayIntoPassedLoop = false; auto noteEdgeOutOf = [&](BlockID from, const Edge& edge) { BlockID to = edge.to; BlockBelowBirth& below = blocksBelow[to.index()]; + if (below.look == look && below.isPassedLoopHead) { + --below.numberOfEdgesAwaited; + hasOtherWayIntoPassedLoop |= !standsAtOrAbove(to, from); + edgesBackToLoopHeads.append({ to, from }); + return; + } if (below.isLookedAt || code.at(to).isEntry) return; if (below.look != look) { below.look = look; + below.isPassedLoopHead = false; + if (code.at(to).isLoopHead) + loopHeadsAwaited.append(to); below.aboveEdgesSeen = from; if (below.numberOfEdgesTakenInto == std::numeric_limits::max()) { below.numberOfEdgesTakenInto = 0; @@ -502,56 +580,171 @@ bool Sweep::moveBirthsToWhereCellsAreNeeded() return; layBelow(to, below.aboveEdgesSeen); }; + auto passLoopHead = [&] { + BlockID head; + for (BlockID awaited : loopHeadsAwaited) { + const BlockBelowBirth& below = blocksBelow[awaited.index()]; + if (below.numberOfEdgesAwaited && !below.isPassedLoopHead && (!head || awaited < head)) + head = awaited; + } + if (!head) + return false; + BlockBelowBirth& below = blocksBelow[head.index()]; + uint32_t numberOfEdgesAwaited = below.numberOfEdgesAwaited; + layBelow(head, below.aboveEdgesSeen); + below.numberOfEdgesAwaited = numberOfEdgesAwaited; + below.isPassedLoopHead = true; + loopHeadsPassed.append(head); + return true; + }; + auto awaitsEdgesBack = [&] { + return std::ranges::any_of(loopHeadsPassed, [&](BlockID head) { + return !!blocksBelow[head.index()].numberOfEdgesAwaited; + }); + }; + auto holdsBlock = [&](BlockID loop, BlockID block) { + BlockID around = blocksBelow[block.index()].innermostLoop; + while (around && around != loop) + around = blocksBelow[around.index()].loopAround; + return !!around; + }; + Vector blocksOfLoop; + Vector blocksInPassedLoops; + auto markPassedLoops = [&] { + blocksInPassedLoops.shrink(0); + for (size_t index = loopHeadsPassed.size(); index--;) { + BlockID head = loopHeadsPassed[index]; + auto addToLoop = [&](BlockID block) { + if (block == head) + return; + BlockBelowBirth& below = blocksBelow[block.index()]; + if (!below.innermostLoop) { + below.innermostLoop = head; + blocksOfLoop.append(block); + blocksInPassedLoops.append(block); + return; + } + BlockID loop = below.innermostLoop; + while (loop != head && blocksBelow[loop.index()].loopAround) + loop = blocksBelow[loop.index()].loopAround; + if (loop == head) + return; + blocksBelow[loop.index()].loopAround = head; + blocksOfLoop.append(loop); + }; + for (EdgeBackToLoopHead edge : edgesBackToLoopHeads) { + if (edge.head == head) + addToLoop(edge.from); + } + while (!blocksOfLoop.isEmpty()) { + BlockID block = blocksOfLoop.takeLast(); + for (EdgeID edge = code.at(block).firstEdgeIn(); edge; edge = code.at(edge).nextInto()) { + if (isSweptEdgeTaken(code, m_states, edge, code.at(edge))) + addToLoop(code.at(edge).from); + } + } + blocksBelow[head.index()].innermostLoop = head; + blocksInPassedLoops.append(head); + } + for (BlockID id : blocksInPassedLoops) { + auto noteWayOut = [&](EdgeID edge) { + if (!isSweptEdgeTaken(code, m_states, edge, code.at(edge))) + return; + for (BlockID loop = blocksBelow[id.index()].innermostLoop; loop && !holdsBlock(loop, code.at(edge).to); loop = blocksBelow[loop.index()].loopAround) + ++blocksBelow[loop.index()].numberOfWaysOutOfLoop; + }; + const Block& block = code.at(id); + for (uint32_t index = 0; index < block.terminator.numberOfEdges; ++index) + noteWayOut(block.terminator.edge(index)); + if (EdgeID handler = handlerOfSweptBlock(block, m_states[id.index()])) + noteWayOut(handler); + } + }; + auto findOutermostLoopWithout = [&](BlockID block, BlockID without) { + BlockID found; + for (BlockID loop = blocksBelow[block.index()].innermostLoop; loop; loop = blocksBelow[loop.index()].loopAround) { + if (!holdsBlock(loop, without)) + found = loop; + } + return found; + }; + auto isOnlyWayOutOfLoop = [&](BlockID side) { + EdgeID edge = code.at(side).firstEdgeIn(); + while (!isSweptEdgeTaken(code, m_states, edge, code.at(edge))) + edge = code.at(edge).nextInto(); + BlockID loop = findOutermostLoopWithout(code.at(edge).from, side); + return loop && blocksBelow[loop.index()].numberOfWaysOutOfLoop == 1; + }; const uint32_t firstBirth = births.first().allocation.index(); for (size_t indexOfTop = 0; indexOfTop < births.size(); ++indexOfTop) { if (births[indexOfTop].isFollowed || !mayMove.quickGet(births[indexOfTop].allocation.index())) continue; ++look; + loopHeadsAwaited.shrink(0); + loopHeadsPassed.shrink(0); + edgesBackToLoopHeads.shrink(0); + hasOtherWayIntoPassedLoop = false; layBelow(code.at(births[indexOfTop].allocation).block(), BlockID()); + bool hasSeenAllUses = false; do { + if (blocksToLookAt.isEmpty()) { + if (loopHeadsPassed.isEmpty()) { + for (uint32_t index : birthsFollowedInLook) + births[index].isSettledBeforeLoops = births[index].numberOfUsesSeen == code.useCountOf(births[index].allocation); + } + if (!passLoopHead()) + break; + } BlockID id = blocksToLookAt.takeLast(); - blocksBelow[id.index()].isLookedAt = true; - forEachUseOfBirthIn(id, firstBirth, [&](ValueID allocation, ValueID user, bool onlyFillsCell) { - BirthThatMayMove& birth = recordOf(allocation); - if (!birth.isFollowed && code.at(allocation).block() != id) { + if (!hasSeenAllUses) { + blocksBelow[id.index()].isLookedAt = true; + forEachUseOfBirthIn(id, firstBirth, [&](ValueID allocation, ValueID user, bool onlyFillsCell) { + BirthThatMayMove& birth = recordOf(allocation); + if (!birth.isFollowed && code.at(allocation).block() != id) { + mayMove.quickClear(allocation.index()); + return; + } + ++birth.numberOfUsesSeen; + RELEASE_ASSERT(birth.numberOfUsesSeen <= code.useCountOf(allocation)); + if (birth.isFollowed) + --numberOfUsesAwaited; + if (onlyFillsCell || namesBirth(user)) { + auto& users = onlyFillsCell ? birth.stores : birth.birthsThatListIt; + if (users.isEmpty() || users.last() != user) + users.append(user); + return; + } + birth.whereCellIsNeeded = birth.whereCellIsNeeded ? nearestBlockAboveBoth(birth.whereCellIsNeeded, id) : id; + BlockID from = code.at(allocation).block(); + BlockID side = blocksBelow[birth.whereCellIsNeeded.index()].nearestSideOfBranch; + if (side && side != from && standsAtOrAbove(from, side)) + return; mayMove.quickClear(allocation.index()); - return; - } - ++birth.numberOfUsesSeen; - RELEASE_ASSERT(birth.numberOfUsesSeen <= code.useCountOf(allocation)); - if (birth.isFollowed) - --numberOfUsesAwaited; - if (onlyFillsCell || namesBirth(user)) { - auto& users = onlyFillsCell ? birth.stores : birth.birthsThatListIt; - if (users.isEmpty() || users.last() != user) - users.append(user); - return; - } - birth.whereCellIsNeeded = birth.whereCellIsNeeded ? nearestBlockAboveBoth(birth.whereCellIsNeeded, id) : id; - BlockID from = code.at(allocation).block(); - BlockID side = blocksBelow[birth.whereCellIsNeeded.index()].nearestSideOfBranch; - if (side && side != from && standsAtOrAbove(from, side)) - return; - mayMove.quickClear(allocation.index()); - if (birth.isFollowed) - numberOfUsesAwaited -= code.useCountOf(allocation) - birth.numberOfUsesSeen; - }, [&](ValueID allocation) { - BirthThatMayMove& birth = recordOf(allocation); - birth.isFollowed = true; - numberOfUsesAwaited += code.useCountOf(allocation) - birth.numberOfUsesSeen; - birthsFollowedInLook.append(static_cast(&birth - births.begin())); - }); - if (!numberOfUsesAwaited) + if (birth.isFollowed) + numberOfUsesAwaited -= code.useCountOf(allocation) - birth.numberOfUsesSeen; + }, [&](ValueID allocation) { + BirthThatMayMove& birth = recordOf(allocation); + birth.isFollowed = true; + numberOfUsesAwaited += code.useCountOf(allocation) - birth.numberOfUsesSeen; + birthsFollowedInLook.append(static_cast(&birth - births.begin())); + }); + hasSeenAllUses = !numberOfUsesAwaited; + } + if (hasSeenAllUses && !awaitsEdgesBack()) break; const Block& block = code.at(id); for (uint32_t index = 0; index < block.terminator.numberOfEdges; ++index) noteEdgeOutOf(id, code.at(block.terminator.edge(index))); if (EdgeID handler = handlerOfSweptBlock(block, m_states[id.index()])) noteEdgeOutOf(id, code.at(handler)); - } while (!blocksToLookAt.isEmpty()); + } while (true); + bool holdsBehindLoops = !hasOtherWayIntoPassedLoop && !awaitsEdgesBack(); + if (holdsBehindLoops) + markPassedLoops(); for (uint32_t index : birthsFollowedInLook) { - if (births[index].numberOfUsesSeen != code.useCountOf(births[index].allocation)) + bool holds = holdsBehindLoops || births[index].isSettledBeforeLoops; + if (!holds || births[index].numberOfUsesSeen != code.useCountOf(births[index].allocation)) mayMove.quickClear(births[index].allocation.index()); } birthsFollowedInLook.shrink(0); @@ -569,6 +762,9 @@ bool Sweep::moveBirthsToWhereCellsAreNeeded() birth.whereItComesToStand = from; if (!mayMove.quickGet(birth.allocation.index())) continue; + bool isBox = made.opcode == Opcode::Box; + if (isBox && !mayMove.quickGet(made.valueOperand().index())) + continue; auto standsBelowBirth = [&](BlockID id) { return id && id != from && standsAtOrAbove(from, id); }; @@ -578,9 +774,19 @@ bool Sweep::moveBirthsToWhereCellsAreNeeded() BlockID ofLister = recordOf(lister).whereItComesToStand; to = to ? nearestBlockAboveBoth(to, ofLister) : ofLister; } - if (standsBelowBirth(to)) + while (standsBelowBirth(to)) { to = blocksBelow[to.index()].nearestFreeOfHandlers; - if (!standsBelowBirth(to) || !standsBelowBirth(blocksBelow[to.index()].nearestSideOfBranch)) + BlockID loop = to ? findOutermostLoopWithout(to, from) : BlockID(); + if (!loop) + break; + to = blocksBelow[loop.index()].above; + } + if (!standsBelowBirth(to)) + continue; + BlockID side = blocksBelow[to.index()].nearestSideOfBranch; + while (standsBelowBirth(side) && isOnlyWayOutOfLoop(side)) + side = blocksBelow[blocksBelow[side.index()].above.index()].nearestSideOfBranch; + if (!standsBelowBirth(side)) continue; size_t numberOfMovesBefore = moves.size(); @@ -590,7 +796,7 @@ bool Sweep::moveBirthsToWhereCellsAreNeeded() BlockID at = code.at(store).block(); if (standsAtOrAbove(to, at)) continue; - everythingHasItsPlace &= standsAtOrAbove(at, to) && !code.at(at).handler; + everythingHasItsPlace &= standsAtOrAbove(at, to) && !code.at(at).handler && !findOutermostLoopWithout(at, from); moves.append({ to, store }); } AOT::sort(moves.mutableSpan().subspan(numberOfMovesBefore), { }, &OperationThatMoves::operation); @@ -607,7 +813,7 @@ bool Sweep::moveBirthsToWhereCellsAreNeeded() moves.shrink(numberOfMovesBefore); continue; } - ++numberOfAllocationsSunk; + numberOfAllocationsSunk += !isBox; birth.whereItComesToStand = to; } if (moves.isEmpty()) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalk.cpp b/Source/JavaScriptCore/aot/walk/AOTWalk.cpp index 86cd43a198ee4..6b16aceaa8680 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalk.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalk.cpp @@ -62,7 +62,7 @@ namespace JSC { namespace AOT { namespace Typed { bool buildReadWhereSomeReceiversLackField(Walker&, const BodyValue&); -static constexpr OpcodeSet callsWithTailFormInWalk { Opcode::Call, Opcode::CallList, Opcode::CallDirect }; +static constexpr OpcodeSet callsWithTailFormInWalk { Opcode::Call, Opcode::CallList, Opcode::CallDirect, Opcode::CallTable }; namespace { @@ -70,7 +70,7 @@ enum class WhoAnswersInWalk : uint8_t { LinkedTablesUnread, LinkedTables, HeldTa static constexpr unsigned numberWhoAnswerInWalk = static_cast(WhoAnswersInWalk::Nobody) + 1; static constexpr unsigned numberOfQuestionsAtTwoLevelsInWalk = 4; static constexpr unsigned numberOfQuestionsAtOneLevelInWalk = 3; -static_assert(static_cast(AnswerKeptInArena::FirstOfWalk) + (2 * numberOfQuestionsAtTwoLevelsInWalk + numberOfQuestionsAtOneLevelInWalk) * numberWhoAnswerInWalk <= static_cast(AnswerKeptInArena::CellsOfFieldArm)); +static_assert(static_cast(AnswerKeptInArena::FirstOfWalk) + (2 * numberOfQuestionsAtTwoLevelsInWalk + numberOfQuestionsAtOneLevelInWalk) * numberWhoAnswerInWalk == static_cast(AnswerKeptInArena::LastOfWalk) + 1); static_assert(2 + static_cast(PositionTypes::QuestionOfEachLayout::ResultOfTableRow) + 1 == numberOfQuestionsAtTwoLevelsInWalk + numberOfQuestionsAtOneLevelInWalk); static_assert(2 + static_cast(PositionTypes::QuestionOfEachLayout::FoundAboveObjects) == numberOfQuestionsAtTwoLevelsInWalk && 2 + static_cast(PositionTypes::QuestionOfEachLayout::ReadByNameOffObjects) == numberOfQuestionsAtTwoLevelsInWalk + 1); @@ -1201,6 +1201,8 @@ TypeID Walker::typeThatStoreTakesOnWordOfWalk(const Draft& store) TypeID claimed = testsForItself && value && value.isValue() ? typeAsBuilt(value) : TypeID(); TypeID taken; for (LayoutID member : membersOf(m_types, store.receivers())) { + if (!plan().isInUse(member)) + continue; std::optional field = plan().slots().findField(program(), member, store.key()); TypeID type = field ? plan().slots().slot(member, *field).typeThatStoresAreHeldTo() : TypeID(); if (testsForItself && (!claimed || !field || (type && !m_types.isSubset(claimed, type)))) @@ -1247,6 +1249,8 @@ TypeID Walker::typeThatDoorsHoldOfSlotRead(const Operation& read) { TypeID held; for (LayoutID member : membersOf(m_types, read.receivers())) { + if (!plan().isInUse(member)) + continue; std::optional field = plan().slots().findField(program(), member, read.key()); TypeID type = field ? plan().slots().slot(member, *field).typeThatStoresAreHeldTo() : TypeID(); if (!type) @@ -1595,10 +1599,10 @@ void Walker::showThatLiteralHoldsWhatIsHeldFilled(const BodyValue& finishOrData) return; Reference object = referenceOf(finishOrData.operand(0)); if (object && !index.isParameter(literal) && index.opcodeOf(literal) == Opcode::NewObject) - showThatObjectHoldsWhatIsHeldFilled(object, index.value(literal).as().layout()); + showThatObjectHoldsWhatIsHeldFilled(object, index.value(literal).as().layout(), index.opcodes().contains(Opcode::CopyDataProperties)); } -void Walker::showThatObjectHoldsWhatIsHeldFilled(Reference object, LayoutID id) +void Walker::showThatObjectHoldsWhatIsHeldFilled(Reference object, LayoutID id, bool copiesMayFillIt) { Vector fieldsToAsk = fieldsHeldFilledThatAreNotShown(object, id); bool recordOfLayoutNamesThemAll = plan().layout(id).holdsCompiledObjects() && std::ranges::all_of(fieldsToAsk, [&](FieldIndex field) { @@ -1607,7 +1611,8 @@ void Walker::showThatObjectHoldsWhatIsHeldFilled(Reference object, LayoutID id) bool asksForFieldOfClassAbove = std::ranges::any_of(fieldsToAsk, [&](FieldIndex field) { return program().at(id).fields[field].flags.contains(FieldFlags::IsInherited); }); - if (fieldsToAsk.size() > 1 && asksForFieldOfClassAbove && recordOfLayoutNamesThemAll) { + bool asksAllAtOnce = asksForFieldOfClassAbove ? fieldsToAsk.size() > 1 : copiesMayFillIt && fieldsToAsk.size() > 2; + if (asksAllAtOnce && recordOfLayoutNamesThemAll) { count(StatisticID::ObjectInitializationChecksEmitted); Draft test(Opcode::AreHeldFieldsFilled, m_origin); test.setObjectOperand(object); @@ -2105,6 +2110,8 @@ TypeID Walker::typeShownAtOnce(ValueID id) const } if (record.opcode == Opcode::Narrowed) return { }; + if (record.opcode == Opcode::GetArrayElement && record.representation == Representation::OptionalDouble && record.arrayStorage() == ArrayStorage::Double && record.level() == KnowledgeLevel::Proven) + return TypeArena::fromPrimitives(Primitive::Number | Primitive::Undefined); switch (resultRuleOf(record.opcode)) { case ResultRule::None: return TypeTable::anything; @@ -2618,6 +2625,42 @@ std::optional Walker::leaveCheckOfWhatContainerHasJustGivenToTestBehind( return origin; } +BlockID Walker::splitNumberOffContainer(Reference given, Reference& number) +{ + if (!given.isValue() || !m_typesAsked.contains(given.id().index())) + return { }; + TypeID rest = m_types.withoutNullish(typeOf(given)); + TypeID nullish = m_types.differenceOf(typeAsBuilt(given), rest); + if (!m_types.at(rest).isNumber() || m_types.testablePartOf(rest) != rest || isNeverType(nullish) || !m_types.at(nullish).hasOnlyPrimitives(Primitive::UndefinedOrNull)) + return { }; + std::optional originOfCheck = leaveCheckOfWhatContainerHasJustGivenToTestBehind(given); + if (!originOfCheck) + return { }; + Draft test(Opcode::TypeTest, *originOfCheck); + test.setValueOperand(given); + test.setTestedType(rest); + Reference passes = build(WTF::move(test)); + if (!passes || !passes.isValue()) { + fail("a test of what a container has given is decided without a look"_s); + return { }; + } + m_code.makeTypeTestAskInFull(passes.id()); + BlockID isOfRest = addBlock(); + BlockID isOther = addBlock(BlockFrequency::Rare); + branch(passes, isOfRest, isOther); + continueIn(isOther); + buildCheckOfProof(given, nullish, TypeTable::anything, ProofSource::Contents, *originOfCheck, true); + BlockID whereNullish = m_block; + continueIn(isOfRest); + Draft narrowed(Opcode::Narrowed, *originOfCheck); + narrowed.setValueOperand(given); + narrowed.setTestOperand(passes); + narrowed.setOperandIndex(OperandIndex::TypeTest::value); + narrowed.setIsOnTakenEdge(true); + number = build(WTF::move(narrowed)); + return whereNullish; +} + bool Walker::splitNullishOffWhatContainerHasJustGiven(Reference given, TypeID rest, Draft&& throwsOffNullish, const Effects* ofThrow) { if (!given.isValue() || !m_typesAsked.contains(given.id().index()) || isNeverType(rest) || m_types.testablePartOf(rest) != rest) @@ -2793,6 +2836,10 @@ void Walker::buildCheckOfProof(Reference value, TypeID proven, TypeWithoutProof return; } TypeID checked = typeThatUnitMayTestFor(m_types.testablePartOf(proven), mayBringClasses); + if (hasBox && asked.isValue() && !checksProofsOf(source) && (m_types.at(checked).primitives & Primitive::LiteralString)) { + if (TypeID shown = m_types.intersectionOf(withoutProof, typeShown(asked)); !isNeverType(shown)) + withoutProof = shown; + } if (m_types.isSubset(withoutProof, checked)) { ++m_untestableProofCount; return; @@ -4115,6 +4162,12 @@ Reference Walker::build(Draft&& draft, const Effects* foundByCaller) effects.writes.add(AbstractHeap::NamedProperties); if (draft.opcode == Opcode::GetPropertyOfHeir || draft.opcode == Opcode::PutPropertyOfHeir) [[unlikely]] nameFieldInEffectsOfAccessToHeir(program(), draft.opcode, fieldAt(program(), FieldReference { draft.layout(), draft.field() }).key, effects); + const bool isStableRead = draft.opcode == Opcode::GetSlot && readsStableField(draft); + if (isStableRead) { + effects.reads.remove(AbstractHeapSet { AbstractHeap::ObjectFields }); + if (Reference kept = valueOfStableField(draft, proven)) + return kept; + } applyUnderstatedEffects(draft, effects); if ((steps & Step::storesIntoSlots) && draft.opcode == Opcode::PutSlot) [[unlikely]] { static constexpr AbstractHeapSet irreversibleHeaps { AbstractHeap::NamedProperties, AbstractHeap::ShapeWords, AbstractHeap::OrderWords, AbstractHeap::PrototypeLinks }; @@ -4158,7 +4211,7 @@ Reference Walker::build(Draft&& draft, const Effects* foundByCaller) ofCallee.merge(facts().linked().function(callee).effectsOnThrow()); Effects heard = Typed::effectsOf(typing(), draft, typesOfOperands(draft), { }, &ofCallee); recordTrialEffects(heard); - if (isCall) + if (isCall && !callIsQuietGivenArguments(callee)) noteCallOnUsualWayInTries(heard); } else if (endsNoGuard) { Effects heard = effects; @@ -4330,6 +4383,8 @@ Reference Walker::build(Draft&& draft, const Effects* foundByCaller) endAsUnreachable(m_origin); return { }; } + if (isStableRead) + noteStableField(draft, Reference::value(id)); return Reference::value(id); } @@ -4546,6 +4601,7 @@ bool Walker::copy(const BodyValue& value) count(StatisticID::ArgumentsLengthsReadInline); } bool givesResult; + TypeID libraryObjectType; if (stepsOfBuildOf(draft.opcode) & StepOfBuildInWalk::isLookedAtByCopy) { if (draft.opcode == Opcode::Call && !draft.calleeKey() && draft.calleeOperand().isValue() && draft.thisValueOperand().isRecord()) { const Operation& read = m_code.at(draft.calleeOperand().id()); @@ -4560,21 +4616,45 @@ bool Walker::copy(const BodyValue& value) sayWhatIsExpectedOfComparedOperands(*this, value, draft); if (draft.opcode == Opcode::ToBoolean && has(Mechanism::Guesses) && m_opensGuesses) sayWhatIsExpectedOfValueAskedForTruth(*this, value, draft); + if (TypeID given = draft.opcode == Opcode::Call ? asFarAsUnitMayAsk(facts().linked().nativeCallResult(currentFunction(), value.id())) : TypeID(); given && has(Mechanism::BuiltinCalls)) { + if (Reference built = buildCallAsWrittenThatGives(*this, WTF::move(draft), given)) + bind(value.id(), built); + return true; + } givesResult = resultRuleInWalk(draft.opcode) != ResultRule::None; + if (draft.opcode == Opcode::Builtin) + libraryObjectType = typeOfLibraryObject(draft.builtin()); if (draft.opcode == Opcode::Builtin && has(Mechanism::Folding)) { if (Reference cell = cellThatRunTimeKeeps(draft)) { - bind(value.id(), cell); + if (libraryObjectType) + cell = proveOrTrap(cell, libraryObjectType, ProofSource::ResultOfCall, m_origin); + if (cell) + bind(value.id(), cell); return true; } } } else givesResult = !(stepsOfBuildOf(draft.opcode) & StepOfBuildInWalk::givesNoResult); Reference built = (stepsOfBuildOf(draft.opcode) & StepOfBuildInWalk::isConstantInFormOfUses) ? constant(WTF::move(draft)) : build(WTF::move(draft)); + if (built && libraryObjectType) + built = proveOrTrap(built, libraryObjectType, ProofSource::ResultOfCall, m_origin); if (built && givesResult) bind(value.id(), built); return true; } +TypeID Walker::typeOfLibraryObject(BuiltinID builtin) +{ + ModuleID file = builtin && !program().builtins().at(builtin).holder ? compiledModuleWithObject(program(), program().builtins().canonicalID(builtin)) : ModuleID(); + if (!file || !facts().linked().modulesWhoseValueStaysIn.get(file.index()) || !objectOfLibraryFileIsResultOfBody(program(), file)) + return { }; + TypeID returned = asFarAsUnitMayAsk(m_positionsOfBuild->result(TypeLevel::Proven, program().at(file).body, 0)); + if (!returned || holdsOnlyPrimitives(returned)) + return { }; + TypeID testable = m_types.testablePartOf(returned); + return m_types.at(testable).hasOnlyLayouts() ? testable : TypeID(); +} + Reference Walker::addIndexedIteration(IndexedIteration&& iteration) { iteration.inlineFrame = m_frames.last().inlineFrame; @@ -4686,6 +4766,14 @@ void Walker::arriveAt(BlockID block, WalkState&& state) pendingBlock(block).addArrival(WTF::move(state)); } +bool Walker::isOutsideLoopsOfAsyncBody() const +{ + if (unit().role != CodeRole::CoroutineBody || isInLoop()) + return false; + const Function& function = program().at(unit().function); + return function.has(FunctionFlags::IsAsync) && !function.has(FunctionFlags::IsGenerator); +} + BlockID Walker::addBlock(BlockFrequency frequency) { BlockID block = m_code.addBlock(BlockRole::Ordinary, m_coldCopyDepth ? BlockFrequency::Rare : frequency); @@ -4948,7 +5036,12 @@ Reference Walker::tryCallInline(TerminatorDraft&& terminator, BlockID done, Bloc Representation formOfAnswer = Typed::representationOf(TypeReader(m_types), given); if (checksProofs() && formOfAnswer == Representation::Void && !isNeverType(given) && isLegalRepresentation(m_types, given, Representation::JSValue)) formOfAnswer = Representation::JSValue; + bool isTypedByContents = m_contentsBeingTyped && m_positions == m_positionsOfBuild.get() && rawFormsOfJavaScriptValuesInWalk.contains(formOfAnswer); + if (isTypedByContents) + formOfAnswer = Representation::JSValue; Reference result = addParameter(done, given, formOfAnswer); + if (m_contentsBeingTyped && m_positions == m_positionsOfBuild.get() && result && result.isValue()) + m_code.addFactUse(FactUse { .record = result.id(), .kind = FactKind::ProvenContents, .subjectKind = FactSubject::Kind::Operation, .type = m_positions->contentsBeingTyped(TypeLevel::Proven) }); settleParametersOf(done); settleParametersOf(miss); BlockID from = m_block; @@ -5038,7 +5131,7 @@ bool Walker::joinArms(std::span arms, BlockID meeting, std::span< } pendingMeeting = nullptr; Vector fieldsThatMeet; - if (unitHasMadeScopes()) { + if (unitHasMadeScopes() || !m_fieldsOfScopesOfHeads.isEmpty()) { Vector states; for (const JoinArm& arm : arms) { const PendingBlock* left = arm.last == m_block ? nullptr : findPendingBlock(arm.last); @@ -5105,6 +5198,15 @@ void Walker::listArm(const StretchInArms& stretch, ArmsThatMeet& arms) threadReturnOf(stretch.inputBlock); if (isUnreachable() || hasFailed()) return; + if (isInTrialVisitOfCurrentFrame()) { + const BodyIndex& index = currentBodyIndex(); + uint32_t firstOfBlock = index.block(stretch.inputBlock).firstValue().index(); + for (uint32_t place = stretch.begin; place < stretch.end; ++place) { + ValueID sum(firstOfBlock + place); + if (index.opcodeOf(sum) == Opcode::Add && !stretch.mentionedBehind.contains(sum)) + noteTypeOfSumLeftInArm(sum); + } + } JoinArm arm { m_block, { } }; for (ValueID mentioned : stretch.mentionedBehind) { Reference built = referenceOf(mentioned); @@ -5582,12 +5684,34 @@ void Walker::closeEdgesInto(BlockID block, BlockID inputBlock, std::span mergedStores = isHandler ? std::nullopt : mergeUndoableStores(edges); + if (mergedStores) { + for (MergedUndoableStore& field : mergedStores->stores) { + field.store.held = field.heldOnPaths[0]; + TypeID type; + forms.shrink(0); + for (Reference held : field.heldOnPaths) { + type = type ? joinTypes(type, ownTypeOf(held)) : ownTypeOf(held); + forms.append(representationOf(held)); + field.hasParameter |= held != field.store.held; + } + if (!field.hasParameter) + continue; + parameters.append(m_code.appendParameter(block, type, formOfParameter(type, forms.span()), m_origin)); + field.store.held = Reference::value(parameters.last()); + } + } + auto takeCommonStore = [&](WalkState& state) { + if (mergedStores) + state.recordUndoableStore(m_versionedLoops.last().restartPoint, mergedStores->commonStore); + }; if (parameters.isEmpty()) { RELEASE_ASSERT(m_code.at(block).parameters().empty()); bool hasTakenState = false; for (uint32_t place : edges) { OpenEdge& open = m_openEdges[place]; m_code.closeEdge(open.edge, block, { }); + takeCommonStore(open.state); if (std::exchange(hasTakenState, true)) m_state.meetWith(open.state); else @@ -5595,6 +5719,8 @@ void Walker::closeEdgesInto(BlockID block, BlockID inputBlock, std::spanstores) { + if (field.hasParameter) + arguments.append(field.heldOnPaths[way]); + } + } ++way; releaseOpenEdge(place); crossEdge(edge, block, arguments.span()); if (m_hasFailed) return; + takeCommonStore(m_state); if (std::exchange(hasMet, true)) met.meetWith(m_state); else @@ -5636,6 +5769,8 @@ void Walker::closeEdgesInto(BlockID block, BlockID inputBlock, std::span parametersOfSlots; for (ValueID parameter : parameters.span().subspan(objects.firstParameterOfSlots, objects.numberOfParametersOfSlots)) @@ -6305,6 +6440,54 @@ void Walker::noteNarrowerName(BlockID to, Reference value, Reference name) m_state.nameAnew(named, name); } +OperatorForm operatorFormFor(Walker&, const Draft&); + +static constexpr OpcodeSet stepsRestatedBehindTests { Opcode::Increment, Opcode::Decrement }; + +void Walker::restateStepsBehindTest(ValueID test, Reference tested, Reference narrowed, BlockID to, Vector& arguments) +{ + const BodyIndex& index = currentBodyIndex(); + for (BodyValue step : index.block(index.blockOf(test)).values()) { + if (step.id() == test) + return; + if (!stepsRestatedBehindTests.contains(step.opcode()) || referenceOf(step.operand(0)) != tested) + continue; + Reference stepped = referenceOf(step.id()); + if (!stepped || !stepped.isValue() || !m_types.at(ownTypeOf(stepped)).isInteger()) + continue; + Draft again(step.opcode(), m_origin); + again.setOperand(0, narrowed); + chooseConservativeFields(again); + again.setForm(operatorFormFor(*this, again)); + Reference restated = again.form() == OperatorForm::Integers ? build(WTF::move(again)) : Reference(); + if (!restated) + continue; + for (Reference& argument : arguments) { + if (argument == stepped) + argument = restated; + } + noteNarrowerName(to, stepped, restated); + count(StatisticID::StepsRestatedBehindTests); + } +} + +static bool isReadAgainInBlock(const BodyIndex& index, const BodyEdgeResult& result, BlockID block) +{ + static constexpr OpcodeSet testsOfKinds { Opcode::TypeTest, Opcode::ToBoolean }; + if (!result.test || index.isParameter(result.test) || !testsOfKinds.contains(index.opcodeOf(result.test))) + return false; + ValueID read = index.stripCasts(result.subject); + if (index.isParameter(read) || index.opcodeOf(read) != Opcode::PropGet) + return false; + auto written = index.value(read).as(); + if (written.isFreeName()) + return false; + ValueID object = index.stripCasts(written.objectOperand()); + return std::ranges::any_of(index.block(block).values(), [&](BodyValue other) { + return other.opcode() == Opcode::PropGet && other.key() == written.key() && index.stripCasts(other.objectOperand()) == object; + }); +} + static bool standUnderSameHandler(const BodyIndex& index, BlockID one, BlockID other) { BodyBlock first = index.block(one); @@ -6325,8 +6508,8 @@ void Walker::leadEdge(EdgeID edge, BlockID from, const BodyTerminator& terminato InputBlock& entered = inputBlock(frame, to); BlockID head = entered.head; uint32_t numberOfProofs = static_cast(target.proofs().size()); - bool entersAlone = !head && index.numberOfIncomingEdges(to) == 1 && !index.isLoopHead(to) - && ((frame.loopsBeingVersioned.isEmpty() && m_reversionableLoops.isEmpty()) || !exitsLoopBeingVersioned(frameIndex, to)) + bool isOnlyWayIn = !head && index.numberOfIncomingEdges(to) == 1 && !index.isLoopHead(to); + bool entersAlone = isOnlyWayIn && ((frame.loopsBeingVersioned.isEmpty() && m_reversionableLoops.isEmpty()) || !exitsLoopBeingVersioned(frameIndex, to)) && (frame.blocksBeingTailDuplicated.isEmpty() || !isTailDuplicatingBlock(frameIndex)); Vector arguments; @@ -6348,8 +6531,19 @@ void Walker::leadEdge(EdgeID edge, BlockID from, const BodyTerminator& terminato fail("an edge hands on a proof that its terminator does not give"_s); return; } - bool isRead = checksProofs() || !index.isUnused(result->parameter); - arguments.append(isRead ? narrowOnEdge(from, targetIndex, *result) : referenceOf(result->subject)); + bool isReadAgain = has(Mechanism::Folding) && isReadAgainInBlock(index, *result, to); + bool isRead = checksProofs() || isReadAgain || !index.isUnused(result->parameter); + Reference subject = referenceOf(result->subject); + if (isReadAgain) { + std::optional draft = draftOfNarrowedOnEdge(from, targetIndex, *result); + Reference narrowed = draft ? build(WTF::move(*draft)) : Reference(); + if (narrowed && entersAlone) + noteNarrowerName(to, subject, narrowed); + else if (narrowed && !m_state.checkedNameOf(subject)) + m_state.noteCheckedName(subject, narrowed); + arguments.append(narrowed ? narrowed : subject); + } else + arguments.append(isRead ? narrowOnEdge(from, targetIndex, *result) : subject); if (((isRead || namesHolderAnew) && isUnreachable()) || (namesHolderAnew && !narrowHolderOfTestedFieldOnEdge(from, targetIndex, *result, to))) { m_block = { }; return; @@ -6370,6 +6564,8 @@ void Walker::leadEdge(EdgeID edge, BlockID from, const BodyTerminator& terminato } ValueID condition = terminator.isConditionalBranch() ? terminator.as().conditionOperand() : ValueID(); bool asksRelation = condition && !index.isParameter(condition) && provesOf(index.opcodeOf(condition)) == Proof::IntegerBound; + bool restatesSteps = loopDepthInCodeOf(to) && index.holdsAnyOf(stepsRestatedBehindTests); + Vector, 2> testedNumbers; if (std::optional> bounds = asksRelation ? boundsProvenOnEdge(index, terminator, targetIndex) : std::nullopt) { for (const BodyEdgeResult& bound : *bounds) { Reference operand = referenceOf(bound.subject); @@ -6394,6 +6590,8 @@ void Walker::leadEdge(EdgeID edge, BlockID from, const BodyTerminator& terminato } if (entersAlone) noteNarrowerName(to, operand, narrowed); + if (entersAlone && restatesSteps) + testedNumbers.append({ operand, narrowed }); } for (const BodyEdgeResult& bound : *bounds) { ValueID sum = entersAlone ? index.stripCasts(bound.subject) : ValueID(); @@ -6418,6 +6616,29 @@ void Walker::leadEdge(EdgeID edge, BlockID from, const BodyTerminator& terminato } } } + ValueID truthAsked = entersAlone && !targetIndex && restatesSteps ? condition : ValueID(); + if (truthAsked && !index.isParameter(truthAsked) && index.opcodeOf(truthAsked) == Opcode::ToBoolean) { + BodyEdgeResult proven; + proven.test = truthAsked; + proven.subject = index.value(truthAsked).operand(0); + proven.operandIndex = OperandIndex::ToBoolean::value; + proven.isOnTakenEdge = true; + Reference operand = referenceOf(proven.subject); + std::optional draft = operand && operand.isValue() && m_types.at(ownTypeOf(operand)).isInteger() ? draftOfNarrowedOnEdge(from, targetIndex, proven) : std::nullopt; + if (draft) { + Reference narrowed = build(WTF::move(*draft)); + if (isUnreachable()) { + m_block = { }; + return; + } + for (Reference& argument : arguments) { + if (argument == operand) + argument = narrowed; + } + noteNarrowerName(to, operand, narrowed); + testedNumbers.append({ operand, narrowed }); + } + } ValueID cellTypeAsked = entersAlone && index.opcodes().contains(Opcode::HasJSType) ? condition : ValueID(); if (cellTypeAsked && !index.isParameter(cellTypeAsked) && index.opcodeOf(cellTypeAsked) == Opcode::ToBoolean) cellTypeAsked = index.value(cellTypeAsked).operand(0); @@ -6464,11 +6685,11 @@ void Walker::leadEdge(EdgeID edge, BlockID from, const BodyTerminator& terminato count(StatisticID::ValuesNarrowedByInstanceOf); } } - ValueID classDenied = entersAlone && targetIndex && index.opcodes().contains(Opcode::InstanceOf) ? condition : ValueID(); - while (classDenied && !index.isParameter(classDenied) && (index.opcodeOf(classDenied) == Opcode::ToBoolean || index.opcodeOf(classDenied) == Opcode::Convert)) - classDenied = index.value(classDenied).operand(0); - if (classDenied && !index.isParameter(classDenied) && index.opcodeOf(classDenied) == Opcode::InstanceOf) - noteWhatInstanceOfDenied(*this, index.value(classDenied)); + ValueID classAnswered = isOnlyWayIn && index.opcodes().contains(Opcode::InstanceOf) ? condition : ValueID(); + while (classAnswered && !index.isParameter(classAnswered) && (index.opcodeOf(classAnswered) == Opcode::ToBoolean || index.opcodeOf(classAnswered) == Opcode::Convert)) + classAnswered = index.value(classAnswered).operand(0); + if (classAnswered && !index.isParameter(classAnswered) && index.opcodeOf(classAnswered) == Opcode::InstanceOf) + noteWhatInstanceOfSaid(*this, index.value(classAnswered), !targetIndex); static constexpr OpcodeSet questionsWhoseYesTeaches { Opcode::IsPlainInstanceOf, Opcode::IsPlainHeirOf, Opcode::IsAsBorn, Opcode::IsAsBornExceptForAddedNames, Opcode::FindsDeclaredMember }; static_assert(OperandIndex::FindsDeclaredMember::object == OperandIndex::IsPlainInstanceOf::object); ValueID questionAsked = entersAlone && !targetIndex && index.holdsAnyOf(questionsWhoseYesTeaches) ? condition : ValueID(); @@ -6534,6 +6755,8 @@ void Walker::leadEdge(EdgeID edge, BlockID from, const BodyTerminator& terminato } if (entersAlone && terminator.isConditionalBranch()) nameBooleansAskedByBranch(from, terminator, targetIndex, to); + for (auto [tested, narrowed] : testedNumbers) + restateStepsBehindTest(condition, tested, narrowed, to, arguments); StringsFinishedOnEdge textsTakenOut; bool endsBlockOnEdge = !entersAlone; if (hasStringsInBuilders() && edgeExitsBuilderLoop(*this, to)) { @@ -6730,7 +6953,7 @@ ALWAYS_INLINE_INTO_OTHER_FILES void Walker::setCurrentInputBlock(Frame& frame, c frame.hasHandler = block.hasHandler(); if (frame.hasHandler && throwsPastHandlerOf(block)) frame.hasHandler = false; - frame.handlerOnlyClosesIterations = frame.hasHandler && handlerOnlyClosesIterations(*frame.index, block); + frame.handlerOnlyClosesIterations = frame.hasHandler && handlerOnlyClosesIterations(frame, block); frame.isInPreheader = false; uint32_t& first = record.firstNarrowerName; @@ -6955,7 +7178,7 @@ void Walker::enterLoopHead(BlockID inputHead) letObjectsLeaveAtSuspension(*this); else if (turnsBorrowScopes) findFieldsOfScopesInFrontOfLoop(preview, fieldsThatComeRound); - findFieldsOfScopesFromOutsideThatQuietTurnsStore(inputHead, preview.effects, fieldsThatComeRound, true); + findFieldsOfScopesFromOutsideThatQuietTurnsStore(inputHead, preview.effects, preview.slots.span(), fieldsThatComeRound, true); if (hasOpenTrialVisit(frameIndex, inputHead)) { m_trialVisits.last().assumed = preview.effects; m_trialVisits.last().knowsWhatIsAssumed = true; @@ -7034,6 +7257,8 @@ void Walker::enterLoopHead(BlockID inputHead) parameters.append(m_code.appendParameter(head, field.type, field.form, m_origin)); field.parameter = Reference::value(parameters.last()); noteWhatParameterOfFieldIsShownToBe(field); + if (field.slotKey) + count(StatisticID::SlotsCarriedAroundLoops); } m_pendingCarriedResultCount = 0; if (!m_carriedResultRequests.isEmpty() && hasOpenTrialVisit(frameIndex, inputHead) && has(Mechanism::LoopInvariants) && !preview.effects.maySuspend) { @@ -7194,7 +7419,7 @@ void Walker::walkBlock(BlockID inputBlock) const BodyIndex& index = *m_frames.last().index; BodyBlock block = index.block(inputBlock); if (!frameIndex && !m_pendingBlocks.isEmpty()) { - if (m_frames[0].loopsBeingVersioned.isEmpty()) + if (m_frames[0].loopsBeingVersioned.isEmpty() && m_reversionableLoops.isEmpty()) m_pendingBlocks.shrink(0); else releaseEndedPendingBlocks(); @@ -7390,7 +7615,7 @@ template static ALWAYS_INLINE bool isReadThatOnlyCallUsesInChain(c template bool Walker::walkThroughChainOf(const BodyValue& walked) { BodyValueOf value(walked); - if (value.opcode() == Opcode::Call && !referenceOf(value.calleeOperand()) && callMethodThroughTable(*this, value)) + if (value.opcode() == Opcode::Call && !referenceOf(value.calleeOperand()) && !callStepsGenerator(*this, value) && callMethodThroughTable(*this, value)) return false; if (hasStringsInBuilders() && bindCastOfStringInBuilder(*this, value)) return false; @@ -7416,6 +7641,7 @@ template bool Walker::walkThroughChainOf(const BodyValue& walked) } bool isBuilt = (hasURLsThatAreNeverMade() && answerFromURLThatIsNeverMade(*this, value)) || (m_loopsOverGenerators.hasStepsWrittenByHand && answerFromStepWrittenByHand(*this, value)) + || (has(Mechanism::DirectCalls) && leaveNextToStep(*this, value)) || buildDeferredDefinition(*this, value) || (has(Mechanism::Folding) && foldOperation(*this, value)) || (has(Mechanism::Folding) && foldOperationOnKnownTexts(*this, value)) @@ -7616,7 +7842,7 @@ void Walker::askObjectAgainWhereStateHasAnswered(const BodyValue& read, Property bool Walker::checkWhatIsRememberedOf(const BodyValue& read) { - if (isInLoop() && ((read.opcode() == Opcode::ElemGet && findHoistedArrayGuard(*this, read)) || readsLengthOfViewTestedInFrontOfLoop(*this, read))) { + if ((isInLoop() || guardedListOfReads()) && ((read.opcode() == Opcode::ElemGet && findHoistedArrayGuard(*this, read)) || readsLengthOfViewTestedInFrontOfLoop(*this, read))) { ++m_untestableProofCount; return false; } @@ -7863,6 +8089,9 @@ void Walker::walkOperation(const BodyValue& value) if (m_operationsBeingWalkedInFrames.size() < m_frames.size()) m_operationsBeingWalkedInFrames.grow(m_frames.size()); m_operationsBeingWalkedInFrames[m_frames.size() - 1] = value.id(); + static constexpr OpcodeSet takePositions { Opcode::ElemGet, Opcode::ElemSet, Opcode::Call }; + if (takePositions.contains(value.opcode())) + notePositions(value); } ++m_numberOfValuesWalked; ++m_numberOfValuesWalkedAndKept; @@ -8388,7 +8617,8 @@ bool Walker::staysTailCall(Reference call) const if (conventions.numberOfResults(entered) != 1) return false; returned = conventions.resultRepresentation(entered); - } + } else if (record.opcode == Opcode::CallTable) + returned = conventions.methodSelector(record.selector()).result; return returned == conventions.resultRepresentation(unit()) && record.representation == returned; } diff --git a/Source/JavaScriptCore/aot/walk/AOTWalk.h b/Source/JavaScriptCore/aot/walk/AOTWalk.h index 323eeed78b97e..cb045feef5417 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalk.h +++ b/Source/JavaScriptCore/aot/walk/AOTWalk.h @@ -132,6 +132,7 @@ enum class GuessOpened : uint8_t { WithArms, }; TypeID typeTestedInPlaceOfWrittenType(Walker&, FunctionID, uint32_t entryParameter, TypeID tested); +TypeID withoutOmittedNullish(Walker&, FunctionID, uint32_t entryParameter, TypeID); GuessOpened openGuessAt(Walker&, const BodyValue&); inline constexpr OpcodeSet opcodesThatAskWhetherHolderOfStoreInClosedArmOfLoopIsListTakes { Opcode::ElemSet }; AOT_LINK_OF_WALK(askWhetherHolderOfStoreInClosedArmOfLoopIsList, opcodesThatAskWhetherHolderOfStoreInClosedArmOfLoopIsListTakes) @@ -292,6 +293,17 @@ struct SliceOfElementsInSight { Reference buildSliceOfElementsInSight(Walker&, const SliceOfElementsInSight&); struct LoopPreview; struct HoistedArrayGuard; +struct SlotOfLoop { + std::pair order() const { return { holder, key.bits() }; } + + ValueID holder; + PropertyKeyID key; + bool isInt32Position { false }; + bool isPosition { false }; + bool hasLeftIntegers { false }; + bool isSteppedByOne { false }; + bool onlyGrows { false }; +}; void hoistInvariants(Walker&, BlockID inputHead, const LoopPreview&, std::span entering); void enterHandler(Walker&, BlockID inputHandler); Representation representationOfNumber(const Walker&, TypeID); @@ -335,7 +347,7 @@ bool askWhetherMemberOfBuiltinIsAsBuilt(Walker&, const BodyValue&); bool askCellWhetherItIsPlainInstance(Walker&, const BodyValue&); bool askCellWhetherItIsAsBorn(Walker&, const BodyValue&); bool askGuardsWhetherPrototypeChainIsPlain(Walker&, const BodyValue&); -void noteWhatInstanceOfDenied(Walker&, const BodyValue&); +void noteWhatInstanceOfSaid(Walker&, const BodyValue& test, bool passed); void beginGuardsOfClassesAtEndOfModule(Walker&, Origin); TypeID objectsExpectedOfOperandOfQuestion(Walker&, ValueID input); bool findConditionsOfOrdinaryInstanceOf(const Walker&, TypeID typeOfConstructor, Vector& conditions); @@ -363,6 +375,7 @@ bool passOnOwnArgumentsByCount(Walker&, const BodyValue& call, const Draft& writ Reference callInPlaceOfCopy(Walker&, const Draft& site, FunctionID callee, std::span entryArguments, bool isConstruction, bool entersAsCallWould = false); bool unitMayLeaveByDirectCallOf(Walker&, FunctionID callee); +bool canTailCallMethodTable(Walker&, uint32_t selector, std::span methods); bool unitTakesWhatCopyIsHanded(Walker&, FunctionID callee, std::span entryArguments); struct PlaceOfCall { @@ -397,6 +410,7 @@ bool openOverListAsMadeGoesByIndex(Walker&, const BodyIndex&, ValueID open); ValueID iteratorClosedLeavingFunction(const BodyIndex&, BodyBlock); ValueID iteratorClosedLeavingLoop(const BodyIndex&, BodyBlock, LoopID); bool mayCopyCallee(const Walker&, const Draft& site, FunctionID callee, const PlaceOfCall&); +bool canInlineSingleCallLoop(const Walker&, FunctionID callee, const PlaceOfCall&, const std::span* passedOperands); bool shouldCopyCallee(const Walker&, const Draft& site, FunctionID callee, std::span passedOperands, bool callsParameterOfCopy = false); bool copyOfCalleeCalledInLoopPays(const Walker&, FunctionID callee, size_t numberOfArguments); bool copyOfCalleePaysAt(const Walker&, FunctionID callee, size_t numberOfArguments, const PlaceOfCall&); @@ -565,6 +579,8 @@ ALWAYS_INLINE void moveWhereSanitizerWatches(Vector& list) #endif } +WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN + template class ListPutBackByLog { public: @@ -573,9 +589,10 @@ class ListPutBackByLog { const T& operator[](size_t place) const LIFETIME_BOUND { return m_list[place]; } T& operator[](size_t place) LIFETIME_BOUND { - if (m_serialOfNewestMark > m_serialsLogged[place]) [[unlikely]] + T& element = m_list[place]; + if (m_serialOfNewestMark > m_serialsLogged.span().data()[place]) [[unlikely]] log(place); - return m_list[place]; + return element; } void append(T&& value) { @@ -660,6 +677,8 @@ class ListPutBackByLog { uint32_t m_serialOfNewestMark { 0 }; }; +WTF_ALLOW_UNSAFE_BUFFER_USAGE_END + template class ListChangedOnlyByName : public ListPutBackByLog { using Base = ListPutBackByLog; @@ -863,7 +882,11 @@ class Walker { ALWAYS_INLINE uint32_t loopDepthInCodeOf(BlockID block) const { const Frame& frame = m_frames.last(); - uint32_t depth = frame.index->loopDepthOf(block); + if (frame.blockOfLoopDepth != block) { + frame.blockOfLoopDepth = block; + frame.loopDepth = frame.index->loopDepthOf(block); + } + uint32_t depth = frame.loopDepth; bool isWrittenOut = depth && frame.fullyUnrolledHeader && frame.index->isInLoop(block, frame.index->loopOf(frame.fullyUnrolledHeader)); return depth - isWrittenOut; } @@ -888,6 +911,8 @@ class Walker { bool isHolderOfCopyAtWork(Reference) const; bool isBeingWalked(FunctionID) const; bool isCurrentFrameRecursive() const { return isFrameRecursive(m_frames.last()); } + bool isOutsideLoopsOfAsyncBody() const; + bool isInRepeatedCode() const { return isInLoop() || isCurrentFrameRecursive() || (currentFunction() && facts().linked().function(currentFunction()).isIterationCallback); } TypeID backEdgeCheckType(BlockID inputHead, uint32_t parameter) const; bool needsBackEdgeRecheck(BlockID head, uint32_t parameter, Reference loopCarriedValue); TypeID typeAskedOfWhatOtherCodeBringsRound(BlockID head, TypeID ofParameter, TypeID comingRound); @@ -1230,6 +1255,7 @@ class Walker { uint32_t beginOfStretchBeingWalked() const { return m_frames.last().beginOfStretch; } bool armHoldsKeyOf(ValueID access) const { return m_accessWhoseKeyArmHolds == access && m_depthOfCopiesAroundKeyHeldByArm == depthOfCopies(); } bool isWalkingForThisWay(BlockID inputBlock) const { return m_frames.last().block == inputBlock && isTailDuplicatingBlock(m_frames.size() - 1); } + BlockID headOfTurnsWrittenOut() const { return m_frames.last().fullyUnrolledHeader; } bool turnWrittenOutHoldsKeyOf(ValueID access, LoopID loopOfKey) const { return m_valueBeingWalked == access && m_frames.last().fullyUnrolledHeader && m_frames.last().index->loopOf(m_frames.last().fullyUnrolledHeader) == loopOfKey; } bool walksStretchInsideAnother() const { return m_frames.last().numberOfStretchesBeingWalked > 1 || isTailDuplicatingBlock(m_frames.size() - 1); } bool someFrameWalksStretchInsideAnother() const @@ -1255,6 +1281,13 @@ class Walker { bool operator==(const ReadWhoseGuessIsOpen&) const = default; }; ReadWhoseGuessIsOpen& readWhoseGuessIsOpen() { return m_readWhoseGuessIsOpen; } + Reference& guardedListOfReads() { return m_guardedListOfReads; } + struct MethodChain { + size_t depthOfCopies { std::numeric_limits::max() }; + ValueID openRead; + ValueID lastCall; + }; + MethodChain& methodChainBeingWalked() { return m_methodChainBeingWalked; } void recordHoistedArrayGuard() { ++m_hoistedArrayGuardCount; } void noteLengthOfViewAskedForWrittenPlace(Reference view); void noteListFromInFrontTakenApart(Reference list); @@ -1338,6 +1371,12 @@ class Walker { uint32_t unguardedSumLoopCount() const { return m_unguardedSumLoops.size(); } bool keysAreReadOffLookInRow() const { return m_keysAreReadOffLookInRow || m_walksCopyThatAsksCountersAlone; } void noteKeyTakenForCounterAskedInFront() { m_keyWasTakenForCounterAskedInFront = true; } + struct ComputedComparison { + ValueID parameter; + ValueID relation; + ValueID bound; + }; + std::span computedComparisonsOf(ValueID parameter); uint32_t numberOfRowsWhoseKeysAreReadOffLook() const { return m_rowsWhoseKeysAreReadOffLook.size(); } uint32_t abortedLoopVisitCount() const { return m_abortedLoopVisitCount; } uint32_t predictedLoopEffectsCount() const { return m_predictedLoopEffectsCount; } @@ -1390,7 +1429,7 @@ class Walker { void stopAtMaker(Origin); void stopWhereWordOfLinkIsFalse(Origin); bool constantEndsWhereItLeaves(ValueID written, DataID) const; - void showThatObjectHoldsWhatIsHeldFilled(Reference object, LayoutID); + void showThatObjectHoldsWhatIsHeldFilled(Reference object, LayoutID, bool copiesMayFillIt = false); bool makerLacksWhatIsHeldFilled(const Draft& allocation) { return !allocationHoldsWhatIsHeldFilled(allocation); } Vector fieldsHeldFilledThatAreNotShown(Reference object, LayoutID); struct SurveyOfConstant { @@ -1459,6 +1498,7 @@ class Walker { bool shouldIgnoreAlternativeOverloads() const { return m_inputs.options.shouldIgnoreAlternativeOverloads; } bool shouldIgnoreFreshArrayEscapes() const { return m_inputs.options.shouldIgnoreFreshArrayEscapes; } bool shouldOverstateBoundArgumentCount() const { return m_inputs.options.shouldOverstateBoundArgumentCount; } + std::optional findTestThatHoldsDifferenceFromZeroUp(Reference position); std::optional findWhatHoldsPositionBelowRoomOfView(Reference view, Reference position); struct PlacesSeenToStandInStretches { const BodyIndex* index { nullptr }; @@ -1469,6 +1509,11 @@ class Walker { PlacesSeenToStandInStretches& placesSeenToStandInStretches() { return m_placesSeenToStandInStretches; } uint32_t& numberOfStretchesBehindBytesOfMembers() { return m_numberOfStretchesBehindBytesOfMembers; } bool shouldAssumeCalleesRetainNothing() const { return m_inputs.options.shouldAssumeCalleesRetainNothing; } + bool framesKeepFields() const; + bool isStableFieldOf(LayoutSetID receivers, PropertyKeyID) const; + bool readsStableField(const Draft& read) const; + Reference valueOfStableField(const Draft& read, TypeID proven) const; + void noteStableField(const Draft& read, Reference value); ALWAYS_INLINE void applyUnderstatedEffects(const Draft& draft, Effects& effects) const { if (m_inputs.options.understatedEffects) [[unlikely]] @@ -1520,6 +1565,9 @@ class Walker { bool makeObjectOfSlotReadShown(Reference value, TypeID taken); void buildCheckOfProof(Reference value, TypeID proven, TypeWithoutProof, ProofSource, Origin, bool mayBringClasses = false, bool isNotShownToFit = false, bool isConstantWrittenInBody = false, bool standsInFrontOfStoreOfValue = false); void buildCheckThatTestHolds(Reference test, ProofSource, Origin); + std::optional leaveCheckOfWhatContainerHasJustGivenToTestBehind(Reference given); + void makeTypeTestAskInFull(Reference test) { m_code.makeTypeTestAskInFull(test.id()); } + BlockID splitNumberOffContainer(Reference given, Reference& number); bool splitNullishOffWhatContainerHasJustGiven(Reference given, TypeID rest, Draft&& throwsOffNullish, const Effects* ofThrow = nullptr); bool letTestInFrontOfBranchAskWhatContainerHasJustGiven(const BodyValue& test); TypeID typeShown(Reference); @@ -1552,6 +1600,8 @@ class Walker { struct KeptForBuild { virtual ~KeptForBuild() = default; }; + void noteSlotRead(Reference object, PropertyKeyID, Reference read); + void noteSlotStore(Reference object, PropertyKeyID, Reference before, Reference stored); bool& closedArmKeepsSites() { return m_closedArmKeepsSites; } std::unique_ptr& keptByTypedAccesses() { return m_keptByTypedAccesses; } std::unique_ptr& keptByUnionTableCalls() { return m_keptByUnionTableCalls; } @@ -1582,6 +1632,8 @@ class Walker { bool isLookupLoopInvariant(std::span operands, const Effects& ofLookup) const; std::optional outermostLoopToMakeTextInFrontOf(std::span operands, const Effects& ofText) const; Reference buildLoopEntryCondition(uint32_t indexOfLoop); + bool tryBuildFirstArrivalConditions(uint32_t loopIndex, Vector& conditions); + bool isFlattenedBeforeLoop(uint32_t loopIndex, Reference string, std::span conditions) const; bool isInSightWhereWalkBuilds(Reference) const; bool executesInEverySpecializedIteration() const; bool tryPreparePreheaderForCall(); @@ -1636,6 +1688,7 @@ class Walker { bool standsUnderHandler() const { return isUnderHandler(); } void setResultOfCallBeingTyped(TypeID type) { m_resultOfCallBeingTyped = type; } TypeID asFarAsUnitMayAsk(TypeID claimed); + TypeID typeOfLibraryObject(BuiltinID); TypeID exchangeResultOfReadBeingTyped(TypeID type) { return std::exchange(m_resultOfReadBeingTyped, asFarAsUnitMayAsk(type)); } TypeID exchangeContentsBeingTyped(TypeID type) { return std::exchange(m_contentsBeingTyped, asFarAsUnitMayAsk(type)); } Reference buildReadTypedByWhatIsRemembered(Draft&& read, TypeID remembered, TypeID shownOfWhatIsRemembered) @@ -1718,6 +1771,7 @@ class Walker { bool tryHoistEntryTest(IterationPurity = IterationPurity::EffectFree); bool isHoistedDiamondParameter(Reference) const; IterationPurity arrayGuardPurity() const { return isIterationEffectFree() ? IterationPurity::EffectFree : IterationPurity::KnownEffectsOnly; } + Reference& receiverOfLargerMethod() { return m_receiverOfLargerMethod; } bool wasAskedForStorageOfNumbersInVain(Reference list) const { return list && list == m_listAskedForStorageOfNumbersInVain; } ArrayStorage requestedLiteralStorage() const { return m_requestedLiteralStorage; } ArrayStorage& requestedLiteralStorage() { return m_requestedLiteralStorage; } @@ -1741,6 +1795,8 @@ class Walker { std::optional tryHoistConditionalCheck(Reference test, bool passesWithYes, IterationPurity = IterationPurity::EffectFree); bool iterationsPreserveArrayGuards(Reference list, IterationPurity = IterationPurity::EffectFree); bool hasElementArithmeticInLoop(BlockID inputHead); + bool mayVersionLoopUnderHandler(BlockID inputHead); + bool isUnderOuterHandlersAlone() const; bool hasElementComparisonInLoop(BlockID inputHead); std::span holdersFromInFrontThatLoopReadsUnderIndexes(BlockID inputHead); bool shouldAssumeLocalArrayReceivers() const { return m_inputs.options.shouldAssumeLocalArrayReceivers; } @@ -1772,7 +1828,7 @@ class Walker { enum class IsListedAllocation : uint8_t { No, Yes }; std::optional classifyAgainstAllocations(const AllocationsUsedInLoop&, Reference) const; bool iterationsPreserveTypedArrays(IterationPurity) const; - bool areElementStoresRawDoubles(IterationPurity = IterationPurity::EffectFree) const; + bool areElementStoresNumeric(IterationPurity = IterationPurity::EffectFree) const; bool hasNoElementStores() const; bool textsOfLoopStayOutOfBuilders(BlockID inputHead) const; bool hasOpenTrialVisit(size_t frameIndex, BlockID inputHead) const; @@ -1824,8 +1880,31 @@ class Walker { Vector readsTakenBySums { }; bool isReadBySumsAlone { false }; ValueID anticipatedRead { }; + PropertyKeyID slotKey { }; + LayoutSetID receivers { }; + bool slotIsInt32Position { false }; + ValueID slotHolder { }; + bool mayStartAsInteger { false }; + bool startsAsInteger { false }; + bool slotIsSteppedByOne { false }; + bool slotOnlyGrows { false }; + }; + Reference valueKnownOf(const FieldOfScope& field) const + { + if (field.variable) + return m_state.variable(field.variable); + return field.slotKey ? m_state.slot(field.scope, field.field) : m_state.environmentField(field.scope, field.field); + } + struct SlotAccess { + ValueID object; + PropertyKeyID key; + ValueID before; + ValueID stored; + bool staysInt32 { true }; + bool isStep { true }; + bool isStepUp { true }; }; - Reference valueKnownOf(const FieldOfScope& field) const { return field.variable ? m_state.variable(field.variable) : m_state.environmentField(field.scope, field.field); } + void findSlotsOfLoop(std::span, Vector&, bool shouldReadBeforeHeader); bool mayBeInDeadZone(const FieldOfScope&) const; TypeID backEdgeTypeOfVariable(BlockID inputHead, const FieldOfScope&) const; bool namesFreshCell(const WalkState&, Reference) const; @@ -1835,13 +1914,15 @@ class Walker { for (const FieldOfScope& field : fields) { if (field.variable) m_state.noteVariable(field.variable, field.parameter); + else if (field.slotKey) + m_state.noteSlot(field.scope, field.field, field.parameter); else noteEnvironmentField(field.scope, field.field, field.parameter); } } void findFieldsOfScopesThatMeet(std::span, Vector&); void findFieldsOfScopesInFrontOfLoop(const LoopPreview&, Vector&); - void findFieldsOfScopesFromOutsideThatQuietTurnsStore(BlockID inputHead, const Effects& ofTurns, Vector&, bool shouldReadBeforeHeader); + void findFieldsOfScopesFromOutsideThatQuietTurnsStore(BlockID inputHead, const Effects& ofTurns, std::span, Vector&, bool shouldReadBeforeHeader); bool mayHandFieldOfScopeToBlock(FieldOfScope&, BlockID inputHeadOfLoopEntered = { }); void keepFieldsOfScopesThatLoopWrites(BlockID inputHead, Vector&); Draft draftCapturedVariableRead(Reference scope, FieldReference); @@ -1922,6 +2003,8 @@ class Walker { uint32_t serialLogged { 0 }; BlockID head; bool isEntered { false }; + bool handlerIsClassified { false }; + bool handlerOnlyClosesIterations { false }; bool isAsNew() const { return firstOpenEdge == noOpenEdge && firstNarrowerName == narrowerNamesOfDominator && !head && !isEntered; } }; @@ -2054,7 +2137,10 @@ class Walker { UncheckedKeyHashMap, WTF::UnsignedWithZeroKeyHashTraits> wholeExpectations; UncheckedKeyHashMap, WTF::UnsignedWithZeroKeyHashTraits> shapesOfValues; uint32_t loopDepthAtCall { 0 }; + mutable BlockID blockOfLoopDepth; + mutable uint32_t loopDepth { 0 }; mutable std::optional isRecursive; + mutable std::optional keepsFields; uint32_t numberOfValuesBuiltBefore { UINT32_MAX }; TypeID typeExpectedOfAnswer; bool answerIsDeclared { false }; @@ -2383,8 +2469,11 @@ class Walker { ValueID written; Reference left; Reference right; + bool isNegated { false }; }; std::optional findLoopEntryCondition(uint32_t indexOfLoop) const; + std::optional findConditionOnEntry(uint32_t indexOfLoop, ValueID asked) const; + Reference buildConditionOnEntry(uint32_t indexOfLoop, const LoopEntryCondition&); struct HoistedDiamond { BlockID inputJoin; uint32_t placeOfLoop { 0 }; @@ -2435,7 +2524,6 @@ class Walker { void checkProofOfRecord(ValueID record, const Draft&, TypeID proven); TypeID typeThatCallByValueIsShownToGive(const auto& call, std::span shownOfOperands); void askWhatCallByValueGives(ValueID call, const Draft&, TypeID proven); - std::optional leaveCheckOfWhatContainerHasJustGivenToTestBehind(Reference given); Reference holdResultToItsType(Reference value, Origin); void holdOperandsToWhatCalleeBelieves(const Draft&); void checkThatReadGivesValue(Reference value, ProofSource, Origin); @@ -2528,6 +2616,7 @@ class Walker { bool everyFieldWasTakenForStored { false }; Vector fields; Vector fieldsThatCallsStore; + Vector slots; }; struct CarriedResult { ValueID site; @@ -2562,6 +2651,7 @@ class Walker { Vector mentionsThatLeave; StoresIntoScopes storesIntoScopes; bool listsAreStruck { false }; + bool copyForInt32sIsStruck { false }; bool otherCodeStaysInTypedBuild { false }; bool otherCodeIsKeptOutOfFirstCopyInVain { false }; bool otherCodeStayedInVain { false }; @@ -2645,6 +2735,8 @@ class Walker { bool keepsScopesAtHead { false }; Vector fieldsOfScopesStored; Vector fieldsOfScopesEnded; + Vector slotAccesses; + Vector, 4> positions; struct TypeLeftByTurnsWrittenOut { uint32_t value { 0 }; TypeID type; @@ -2719,6 +2811,7 @@ class Walker { public: void noteTestSeenInTries(Reference value, ValueID operation = { }, TypeID tested = { }); void noteGuardOfStoresAskedInTries(const MemberConditionRecord&); + bool isAllocatedInTrialIteration(Reference object) const { return !m_trialVisits.isEmpty() && object && object.isValue() && object.id().index() >= m_trialVisits.last().firstRecord; } void noteBirthInTries(LayoutID classLayout); void noteGuardsBegunInTries(LayoutID classLayout); bool turnBeginsGuardsOf(LayoutID classLayout) const { return !m_trialVisits.isEmpty() && m_trialVisits.last().classesWhoseGuardsTurnBegins.contains(classLayout); } @@ -2728,6 +2821,9 @@ class Walker { std::span operationsThatRanOtherCodeOfRow() const LIFETIME_BOUND { return m_operationsThatRanOtherCodeOfRow.span(); } const LoopPreview* previewOutOfRow() const { return m_previewOutOfRow; } bool listsAreStruckInRow() const { return m_listsAreStruckInRow; } + bool copyForInt32sIsStruck() const { return m_copyForInt32sIsStruck; } + bool hasUnrolledInnerLoops(BlockID inputHead); + void noteCopyForInt32s() { m_hasCopyForInt32s = true; } const Effects& effectsOfQuietDraftsOfRow() const { return m_effectsOutsideArmsOfRow; } class HeardAsArm { public: @@ -2751,6 +2847,8 @@ class Walker { Vector m_guardsOfStoresAskedOfRow; const LoopPreview* m_previewOutOfRow { nullptr }; bool m_listsAreStruckInRow { false }; + bool m_copyForInt32sIsStruck { false }; + bool m_hasCopyForInt32s { false }; bool m_otherCodeStaysInTypedBuildOfRow { false }; bool m_otherCodeIsKeptOutOfFirstCopyInVainOfRow { false }; bool m_otherCodeStayedInVainOfRow { false }; @@ -2849,6 +2947,8 @@ class Walker { bool turnsCheckArithmeticOnIntegers(BlockID inputHead); void narrowSummaryToObservedTypes(LoopSummary&, const TrialVisit&); Vector findParametersSteppedOnDoubles(BlockID inputHead, const LoopSummary&); + Vector findParametersSteppedOnDoubles(BlockID inputHead, const LoopPreview&); + Vector findParametersSteppedOnDoubles(BlockID inputHead, bool isAskedBeforeVisit, const ScopedLambda& typeInRow); void letMentionsOfLoopLeave(BlockID inputHead, std::span mentionsThatLeave = { }, Vector* kept = nullptr, bool keepsScopes = true); struct BelievedAheadOfEdges { BlockID block; @@ -2872,6 +2972,9 @@ class Walker { void keepScopesAtMentions(Vector& mentions, std::span mentionsThatLeave, Vector& kept); void handScopesKeptAtHeadToPreview(const BodyIndex&, BlockID inputHead, const StoresIntoScopes&, std::span, LoopPreview&); bool learnWhatTurnsStoreIntoScopes(LoopSummary&, const TrialVisit&, bool walkIsWhole); + bool learnSlotsOfLoop(LoopSummary&, TrialVisit&); + bool canCarrySlotType(TypeID type) { return m_types.isSubset(type, m_types.withPrimitives(TypeTable::number, Primitive::Undefined)) && formOfParameter(type, { }) != Representation::JSValue; } + void notePositions(const BodyValue&); bool wordCutsWalkAtThisHearing(); ALWAYS_INLINE bool walkIsCutShortHere() { @@ -2995,6 +3098,7 @@ class Walker { BitVector blocksThatSomeQuietTurnPassesThrough(BlockID inputHead, std::span ranOtherCode); bool wayBehindOperationComesRound(BlockID inputHead, ValueID operation) const; void noteCallOnUsualWayInTries(const Effects& heard); + bool callIsQuietGivenArguments(FunctionID callee) const; bool someTextGrowsInQuietTurn(BlockID inputHead, std::span, const TrialVisit&); void keepTextsOfLoopOutOfBuilders(BlockID inputHead); void endStringsInBuildersFrom(size_t numberOfTexts); @@ -3049,9 +3153,11 @@ class Walker { BlockID block; }; HoistPoint currentHoistPoint() const; + bool standsOnMainPath(uint32_t loopIndex, const HoistPoint&) const; HoistPoint hoistPointInFrontOf(uint32_t loopIndex) const; bool executesOnEveryIteration(uint32_t loopIndex, const HoistPoint&) const; bool isGuaranteedToExecute(uint32_t loopIndex, const HoistPoint&) const; + bool precedesReturnsOfInlinedCallees(uint32_t loopIndex, const HoistPoint&) const; bool headerMayExitBefore(uint32_t loopIndex, const HoistPoint&) const; struct InvariantGuard { ValueID test; @@ -3093,6 +3199,7 @@ class Walker { }; Vector m_sharedGenericVersions; Reference m_listAskedForStorageOfNumbersInVain; + Reference m_receiverOfLargerMethod; ArrayStorage m_requestedLiteralStorage { ArrayStorage::Other }; bool m_walksCleanupBesideCopyOfLoop { false }; struct FrontOfLoopLeftOpen { @@ -3127,6 +3234,7 @@ class Walker { bool writeOutTurnsOfLoop(BlockID inputHead, std::span entering); bool writeOutTurnsOfForIn(BlockID inputHead, std::span entering); bool writeOutTurnsOfForOf(BlockID inputHead, std::span entering); + bool writeOutTurnsOverOwnKeys(BlockID inputHead, std::span entering, TurnsOfForInWrittenOut&, const ForOfByText&); void beginTurnsWrittenOut(BlockID inputHead); void walkTurnsWrittenOut(BlockID inputHead, std::span entering, uint32_t numberOfWalks, TurnsOfForInWrittenOut*); void leadIntoLoopAsWrittenBehindTurns(TurnsOfForInWrittenOut&, BlockID inputHead, std::span arriving, Reference position); @@ -3169,7 +3277,7 @@ class Walker { bool turnsThatRunOtherCodeComeRound { false }; bool isLeftAtOtherCodeOnUsualWay { false }; BlockID lastBackEdgeDominator { }; - bool hasElementArithmetic { false }; + bool mayRestartUnderHandler { false }; Vector, 1> moduleVariablesAtHeader { }; Vector, 4> undoBlocks { }; BlockID startOfCopyWithoutGuessesOfSomeTurns { }; @@ -3230,12 +3338,17 @@ class Walker { } bool walkOperationOfGeneratorInLoop(const BodyValue&); bool takeGeneratorForConsumer(const BodyValue& call); + bool loopHoldsGenerators(const BodyIndex&, LoopID); bool openGeneratorTaken(const BodyValue& open); bool closeGeneratorTaken(const BodyValue& close); bool spreadGeneratorTaken(const BodyValue& spread); bool spreadGeneratorTakenAt(size_t placeOfTaken, const BodyValue& spread); bool walkYieldOfGeneratorInConsumer(const BodyValue& yield); size_t placeOfGeneratorOnTop() const; + bool startsAtInitialYield(FunctionID generatorFunction); + ValueID onlyUserOfEntryParameter(FunctionID generatorFunction, uint32_t entryParameter); + bool passesGeneratorForIteration(const BodyIndex& caller, ValueID generatorCall, FunctionID callerID); + Reference findPassedGenerator(ValueID open, ValueID iterable, ValueID& callThatMadeIt, uint32_t& frameOfCall); size_t placeOfGeneratorTaken(ValueID taker, ValueID iterable) const; bool isBeingWalkedOtherThanAtYield(FunctionID) const; bool leavesRegionOfIteration(size_t frameIndex, BlockID to) const; @@ -3283,7 +3396,7 @@ class Walker { bool looksAtLoopsAroundHaveSeen(const Effects&) const; void leadEdge(EdgeID, BlockID from, const BodyTerminator&, uint32_t targetIndex, BodyTarget); void leadHandlerEdge(); - bool handlerOnlyClosesIterations(const BodyIndex&, BodyBlock) const; + bool handlerOnlyClosesIterations(Frame&, BodyBlock) const; bool isUnderHandlerOtherThanCleanups() const { if (m_handlerMadeOverCopies || m_numberOfFramesOverHandlersOfCode) @@ -3302,7 +3415,9 @@ class Walker { { if (!isUnderHandler()) return false; - return isUnderHandlerOtherThanCleanups() || m_versionedLoops.isEmpty() || !m_versionedLoops.last().hasElementArithmetic; + if (m_versionedLoops.isEmpty() || !m_versionedLoops.last().mayRestartUnderHandler) + return true; + return isUnderHandlerOtherThanCleanups() && !isUnderOuterHandlersAlone(); } ALWAYS_INLINE bool isUnderHandler() const { @@ -3355,6 +3470,7 @@ class Walker { Reference tagReadInFrontOfTest(const BodyValue& read); private: void noteNarrowerName(BlockID to, Reference value, Reference name); + void restateStepsBehindTest(ValueID test, Reference tested, Reference narrowed, BlockID to, Vector& arguments); bool endBlockInAnotherWay(const BodyTerminator&); bool endBlockWithSwitchOverAlternatives(const BodyTerminator&); void endWithoutEdges(TerminatorDraft&); @@ -3458,6 +3574,7 @@ class Walker { Vector m_usersOfValuesOfLoops; HashMap m_valuesTakenForKeys; HashMap m_loopsThatReckonWithElements; + HashMap> m_computedComparisons; mutable HashMap m_numbersInListsOfWrittenObjects; HashMap> m_lastStoresByKeyOfLoops; HashMap>> m_storesByKeyOfBlocks; @@ -3504,8 +3621,21 @@ class Walker { LayoutID layoutOfHeir; uint32_t fieldOfHeir { 0 }; uint32_t restartPoint { 0 }; + bool mayHoldNothing { false }; }; void recordPendingUndoableStore(UndoableStore); + struct MergedUndoableStore { + UndoableStore store; + Vector heldOnPaths; + bool hasParameter { false }; + }; + struct MergedUndoableStores { + uint32_t commonStore { 0 }; + Vector stores; + }; + std::optional mergeUndoableStores(std::span edges); + void recordMergedUndoableStores(const MergedUndoableStores&); + bool isDefinedOnPath(Reference, const WalkState&) const; void popVersionedLoop(); Vector m_undoableStores; Vector> m_undoableStoreStates; @@ -3546,6 +3676,8 @@ class Walker { FunctionID m_calleeAccountedFor; bool m_hasFailed { false }; ReadWhoseGuessIsOpen m_readWhoseGuessIsOpen; + Reference m_guardedListOfReads; + MethodChain m_methodChainBeingWalked; std::unique_ptr m_positionsOfBuild; std::unique_ptr& m_positionsOfSlowBody; @@ -3873,6 +4005,7 @@ struct LoopPreview { Vector scopesThatTurnsOnlyBorrow; Vector fieldsOfScopesThatTurnsStore; Vector fieldsOfScopesThatCallsInTurnsStore; + Vector slots; TypesOfValuesOfLoop typesOfValues; uint32_t firstValue { 0 }; }; @@ -3930,6 +4063,8 @@ TypedForm buildReadOfBuiltinMember(Walker&, const BodyValue& read, TypeID typeOf TypedForm buildStoreThroughSetterOfBuiltin(Walker&, const BodyValue& store, TypeID typeOfObject, TypeID typeOfStored, Reference object); TypeID typeOfFunctionReadOffFunctionPrototype(Walker&, PropertyKeyID); TypedForm buildReadOffFunctionPrototype(Walker&, const BodyValue& read, TypeID typeOfObject, Reference object); +TypedForm buildReadOfMemberOffObjectPrototype(Walker&, const BodyValue& read, TypeID typeOfObject, Reference object, BuiltinID location); +bool buildOwnPropertyTest(Walker&, const BodyValue& question, Reference object, PropertyKeyID); BuiltinID findMemberAboveHeirsOfBuiltinClass(const Walker&, BuiltinClassID, PropertyKeyID); TypedForm buildReadOfMemberAboveHeirOfBuiltinClass(Walker&, const BodyValue& read, TypeID typeOfObject, Reference object, BuiltinID location); bool readsOffEntryParameterThatStaysAsMade(Walker&, const BodyValue& access); @@ -3975,6 +4110,7 @@ TypeID numbersExpectedOfFieldOfHeir(Walker&, LayoutID heir, uint32_t field); bool readOffConstructorFunctionAwaitsGuard(Walker&, const BodyValue& read, TypeID typeOfObject); bool findsMethodReadOffPrototypeOfClass(Walker&, const BodyValue& readOfPrototype, TypeID typeOfFunctionObject, bool& guardsHold); bool findParentNamedByClassRecordForTypedAccess(const Walker&, LayoutID classLayout, LayoutID& parent); +bool hasOnlyNamedMembersAbove(Walker&, LayoutID classLayout); bool readOffObjectAsMadeCallsGetter(Walker&, const BodyValue& read, TypeID typeOfObject); enum class BuiltinCallAskedAbout : uint8_t { None, @@ -4029,6 +4165,11 @@ Reference buildTestOfSlot(Walker&, const BodyValue& access, Reference object); Reference buildReadThatTestsShapeAndSlot(Walker&, const BodyValue& read, Reference object, TypeID guessed); Reference buildReadThatTestsShapeAndCell(Walker&, const BodyValue& read, Reference object, TypeID guessed); bool markOfCellAnswersForRead(Walker&, const BodyValue& read, TypeID typeOfObject); +struct FilledSlot { + FieldReference place; + LayoutSetID receivers; +}; +std::optional findFilledSlot(Walker&, TypeID typeOfObject, PropertyKeyID); bool storeMaySetMarkOfCell(const Walker&, std::span receivers, PropertyKeyID, TypeID typeOfStored); bool typedStoreAheadMaySetMarkOfCell(Walker&, TypeID typeOfObject, PropertyKeyID, TypeID typeOfStored); bool buildStoreThatTestsForItself(Walker&, const BodyValue& store, Reference object, TypeID guessed); @@ -4038,7 +4179,9 @@ FunctionID findMethodAboveProvenObject(Walker&, Reference object, PropertyKeyID) FunctionID findMethodAboveProvenType(Walker&, TypeID proven, PropertyKeyID); FunctionID findMethodInSightOfProvenType(Walker&, TypeID proven, PropertyKeyID); Reference readMethodInSightOfProvenObject(Walker&, Reference object, PropertyKeyID, Origin); +Reference readMethodForCallAlone(Walker&, Reference object, PropertyKeyID, Origin); Reference readFilledFieldOfProvenObject(Walker&, Reference object, PropertyKeyID, Origin); +Reference readFilledFieldInFrontOfTurns(Walker&, const BodyValue& read, Reference object); bool findsNothingUnderNameOfProvenObject(Walker&, Reference object, PropertyKeyID); Reference valueHeldOfFilledSlot(Walker&, const BodyValue& access, Reference object); void noteAnswerOfReadOfSlotThatMayBeEmpty(Walker&, const BodyValue& access, Reference object, Reference answer); @@ -4067,6 +4210,7 @@ bool valueFitsSlot(Walker&, FieldReference, Reference value); ExpectationOfValue expectationOfAnswerOfCallBeingCopied(Walker&, const BodyValue& call); bool isEntryParameterUninformative(Walker&, FunctionID callee, uint32_t entryParameter); uint64_t entryParametersOnlyReadByTwoNamesOrMore(const BodyIndex&); +uint64_t numbersReadAfterWrites(const BodyIndex&); Walker::UseOfTextAsKey useOfAnswerOfCallBeingCopiedAsKey(Walker&, const BodyValue& call, const BodyIndex& copied); bool guessAtCalleeAsksForBuiltinWhereItFails(Walker&, const BodyValue&, ValueID operand); void forEachTypeThatGuessWouldProveAt(Walker&, const BodyValue&, const TypesOfInputs&, NOESCAPE const ScopedLambda&); @@ -4075,6 +4219,7 @@ bool rowIsToldApartFromViewsInTurn(Walker&, const BodyValue& read, TypeID expect TypeID typeThatTypeTestAsksForWhereExpected(Walker&, Reference value, TypeID expected); bool guessesMayTypeOperationInCleanTurn(Walker&, const BodyValue&, const TypesOfInputs&, bool sitesMayAnswer); bool patternMayTakeListApartInCleanTurn(Walker&, const BodyValue& open); +bool destructuresReturnedLiteral(const Walker&, const BodyIndex&, ValueID open); bool valueIsOnlyReckonedWithAsNumber(Walker&, ValueID, const TypesOfInputs&); bool someReadThatAsksItsSiteIsReckonedWith(Walker&, std::span reads, const TypesOfInputs&); bool readThatAsksItsSiteIsReckonedWithAsNumber(Walker&, ValueID read, const TypesOfInputs&); @@ -4087,6 +4232,8 @@ AOT_LINK_OF_WALK(tryReadAtSite, opcodesThatTryReadAtSiteTakes) bool shouldOpenGuess(const Walker&, Reference value, TypeID expected, bool isDeclaredInSource, uint32_t numberOfTypedOperations); void openGuess(Walker&, Reference value, TypeID expected, BlockID inputBlock, uint32_t begin, uint32_t end); uint32_t limitOfStretchOfArms(Walker&, const BodyBlock&); +bool isSmallLeaf(const Walker&, FunctionID callee, size_t numberOfArguments); +uint32_t findArmStretchEnd(Walker&, const BodyValue& call, uint32_t begin, uint32_t limit, Reference told, const ScopedLambda& mayTake); struct SlotThatFindsHolderOfSlotReadInLoop { ValueID read; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkBirths.h b/Source/JavaScriptCore/aot/walk/AOTWalkBirths.h index 341a3aff4bd90..b8056cc589b3b 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkBirths.h +++ b/Source/JavaScriptCore/aot/walk/AOTWalkBirths.h @@ -31,7 +31,7 @@ struct StoresListedAtBirth { }; StoresListedAtBirth findStoresThatAllocationMayList(Walker&, LayoutID, std::span keys, std::span stored, std::span keysStoredFurtherOn); bool looksAtCellsAskForLayoutPrototype(Walker&, LayoutID); -void noteGuardsShownByTest(Walker&, std::span, Reference test); +void noteGuardsShownByTest(Walker&, std::span, Reference test, Reference receiver = { }); void checkGuardsThatAreRemembered(Walker&, std::span, Origin); void askWholeSideWhereGuardsMayNotHaveBegun(Walker&, LayoutID classLayout, Vector& conditions); std::optional findGuardOfCallOffConstructorFunction(Walker&, FunctionID); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkBirthsListed.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkBirthsListed.cpp index 184c71ba39d50..d1f8d2639c415 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkBirthsListed.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkBirthsListed.cpp @@ -5,6 +5,7 @@ #include "config.h" #include "AOTWalkBirths.h" +#include "AOTWalkKnown.h" #include "AOTWalkKnownCalls.h" #include "AOTWalkTextBuilders.h" @@ -768,6 +769,7 @@ bool listStoresOfCopiedConstructorOfItsOpcodes(Walker& walker, const BodyValue& allocation.setFields(listed.fields.span()); allocation.setValuesOperands(stores.values.span().first(listed.fields.size())); walker.chooseConservativeFields(allocation); + allocation.setEndsNoGuards(walker.isInLoop() && writerEndsNoGuard(walker, allocation)); Reference object = walker.hasFailed() ? Reference() : walker.build(WTF::move(allocation)); if (!object) return true; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkBuiltins.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkBuiltins.cpp index f7d440624de7a..7e90116efa4f6 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkBuiltins.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkBuiltins.cpp @@ -37,6 +37,7 @@ #include "AOTWalkKnownCalls.h" #include "AOTWalkKnown.h" #include "AOTWalkListReadsInLoops.h" +#include "AOTWalkListedOwnKeys.h" #include "AOTWalkWrittenDescriptors.h" #include #include @@ -150,6 +151,13 @@ static MembersBehindWatch membersBehindWatchAboveInWalkBuiltins(Walker& walker, return hasPassed ? MembersBehindWatch::AreAsBuilt : MembersBehindWatch::AreOpen; } +static BuiltinName nameOfMemberInWalkBuiltins(const Program& program, PropertyKeyID key) +{ + if (liesInThread(key)) + return { }; + return isPrivateAliasOfBuiltinMember(program, key) ? builtinMemberNameOf(program, key).name : program.builtinNameOf(key); +} + static BuiltinID findMemberReadOffInWalkBuiltins(const Walker& walker, TypeID object, PropertyKeyID key, bool isAsItsClassMakesIt, MembersBehindWatch watch = MembersBehindWatch::AreOpen) { const Program& program = walker.program(); @@ -184,7 +192,7 @@ static BuiltinID findMemberReadOffInWalkBuiltins(const Walker& walker, TypeID ob } } } - BuiltinName name = builtinClass && !liesInThread(key) ? program.builtinNameOf(key) : BuiltinName(); + BuiltinName name = builtinClass ? nameOfMemberInWalkBuiltins(program, key) : BuiltinName(); if (!name) return { }; @@ -411,6 +419,20 @@ Reference buildTestOfObjectAsMade(Walker& walker, Reference object, Origin origi return buildTestOfInstanceAsMadeInWalkBuiltins(walker, object, classWhoseInstancesAreTestedInWalkBuiltins(walker, walker.typeOf(object)), origin, asked == TestOfObjectAsMade::AsksObjectAlone ? TestOfInstanceInWalkBuiltins::AsksStorageTooOfCellAlone : TestOfInstanceInWalkBuiltins::AsksStorageToo); } +static bool onlyNamesOfInstancesAreReadInWalkBuiltins(JSType type) +{ + switch (type) { + case ArrayBufferType: + case DataViewType: + case JSDateType: + case JSPromiseType: + case RegExpObjectType: + return true; + default: + return false; + } +} + static Draft testOfInstanceAsMadeInWalkBuiltins(Walker& walker, Reference object, BuiltinClassID builtinClass, Origin origin, TestOfInstanceInWalkBuiltins asked) { Draft test(Opcode::IsPristine, origin); @@ -418,7 +440,7 @@ static Draft testOfInstanceAsMadeInWalkBuiltins(Walker& walker, Reference object test.setValueOperand(object); test.setBuiltinClass(builtinClass); bool isViewThatNobodyGrows = isTypedView(walker.program().builtins().typedArrayTypeOf(builtinClass)) && !someThreadMayGrowSharedMemory(walker); - if (pristineTestOf(walker.program().builtins(), builtinClass) == PristineTest::InstanceStructure || (asked == TestOfInstanceInWalkBuiltins::AsksOneBitOfCell && builtinClass != BuiltinClasses::Array) || isViewThatNobodyGrows) + if (pristineTestOf(walker.program().builtins(), builtinClass) == PristineTest::InstanceStructure || (asked == TestOfInstanceInWalkBuiltins::AsksOneBitOfCell && builtinClass != BuiltinClasses::Array) || isViewThatNobodyGrows || onlyNamesOfInstancesAreReadInWalkBuiltins(walker.program().builtins().at(builtinClass).type)) test.setAspects({ PristineAspects::Names }); else test.setAspects({ PristineAspects::Storage, PristineAspects::Names }); @@ -1064,6 +1086,11 @@ TypedForm buildReadOffFunctionPrototype(Walker& walker, const BodyValue& read, T return buildReadAtLocationInWalkBuiltins(walker, read, typeOfObject, object, findConstantMemberOnceForBuild(walker, Builtins::FunctionPrototype, read.key()), TypedForm::Built); } +TypedForm buildReadOfMemberOffObjectPrototype(Walker& walker, const BodyValue& read, TypeID typeOfObject, Reference object, BuiltinID location) +{ + return buildReadAtLocationInWalkBuiltins(walker, read, typeOfObject, object, location, TypedForm::Built); +} + BuiltinID findMemberAboveHeirsOfBuiltinClass(const Walker& walker, BuiltinClassID builtinClass, PropertyKeyID key) { TypeID instances = walker.types().builtinClass(builtinClass); @@ -1297,6 +1324,7 @@ class CallInWalkBuiltins { bool planOnRowsThatEnterOneDoor(); bool planOnPartsOfLiteral(); bool planOnAnyOperands(); + std::optional effectsWhereOnlyNullishIsRefused(); bool asksForOwnPropertyUnderWrittenName(); bool mayRunCodeThatNoOperandNames() const { return mayRunCodeThatNoOperandNames(*m_overload); } @@ -1309,6 +1337,7 @@ class CallInWalkBuiltins { bool addEffectsOfFunctionsCalled(Effects&, const BuiltinParameter&, const Type&) const; void addEffectsOn(Effects&, const Operand&) const; Effects effectsOfRow(std::optional codeInLine) const; + bool holdsOnlyCompiledRecords(TypeID operand) const; bool makesFreshResult(Reference result) const; Vector freshOperandsThatStayHere() const; Vector operandsThatStayAsMade() const; @@ -1316,6 +1345,8 @@ class CallInWalkBuiltins { bool settle(Operand&, Origin); bool buildWithRegExpWrittenForCall(std::span arguments); template void fillCodeInLine(CallDraft&, std::span arguments) const; + TypeID provenContentsOfReceiver() const; + bool isOnlyCutToInt32(TypeID contents) const; Reference buildCall(std::span arguments); Reference buildCodeInLineOrCall(std::span arguments); Reference buildComparisonByContents(std::span arguments); @@ -1862,7 +1893,7 @@ bool CallInWalkBuiltins::chooseForm() AccessContext context { m_program, WalkFacts(m_walker.facts()), m_walker.plan(), m_types, m_walker.currentFunction() }; bool isHiddenInStackTraces = m_program.at(m_walker.currentFunction()).isHiddenInStackTraces(); m_inlineForm = chooseInlineForm({ context, *m_overload, *implementation.inlineBuiltin, BuiltinCallForm::Whole, received, !!implementation.otherwise, isDoneInPlace(), isHiddenInStackTraces }); - if (m_inlineForm == InlineForm::TwoWay && !m_walker.isInLoop() && runsAtMostOnce()) + if (m_inlineForm == InlineForm::TwoWay && *implementation.inlineBuiltin != InlineBuiltin::ReflectConstruct && ((!m_walker.isInLoop() && runsAtMostOnce()) || m_walker.isOutsideLoopsOfAsyncBody())) m_inlineForm = InlineForm::Call; if (m_inlineForm != InlineForm::TwoWay && hostReadsMembersFirstIn(m_table, *m_overload)) m_inlineForm = InlineForm::Call; @@ -1994,6 +2025,30 @@ static bool isTypeOnlyResult(const BuiltinResult& result) && !result.iteratedOperand && !result.prototypeSource && !result.testedOperand && !result.testedType && !result.properties.size(); } +std::optional CallInWalkBuiltins::effectsWhereOnlyNullishIsRefused() +{ + if (m_mode != BuiltinCallMode::Call || m_overload->receiver || !m_walker.opensGuesses()) + return std::nullopt; + Vector written; + for (size_t place = 0; place < m_arguments.size(); ++place) { + const BuiltinParameter* parameter = m_table.parameterFor(*m_overload, BuiltinOperand::argument(place)); + TypeID rest = m_types.withoutNullish(m_arguments[place].type); + OperandFit fit = parameter && !isNeverType(rest) ? fitOf(m_context, rest, parameter->accepted) : OperandFit(); + if (fit.fit != TypeFit::Inside || fit.unprovenPristine) + return std::nullopt; + written.append(m_arguments[place]); + written.last().parameter = parameter; + } + std::swap(m_arguments, written); + Effects effects = effectsOfRow(m_inlineBuiltin); + std::swap(m_arguments, written); + if (effects.mayRunOtherCode()) + return std::nullopt; + effects.mayThrow = true; + effects.merge(declaredBoundOf(NamedEffectSet::Allocates)); + return effects; +} + bool CallInWalkBuiltins::planOnAnyOperands() { forgetWhatIsPlanned(); @@ -2049,8 +2104,10 @@ bool CallInWalkBuiltins::planOnAnyOperands() m_receiver = std::nullopt; if (*implementation.inlineBuiltin == InlineBuiltin::ObjectHasOwn) asksForOwnPropertyUnderWrittenName(); + if (std::optional effects = effectsWhereOnlyNullishIsRefused()) + m_effects = *effects; } - if (formOnAnyOperands == InlineForm::TwoWay && (m_walker.isInLoop() || (reached.size() == 1 && !runsAtMostOnce()))) { + if (formOnAnyOperands == InlineForm::TwoWay && (m_walker.isInLoop() || (reached.size() == 1 && !runsAtMostOnce() && !m_walker.isOutsideLoopsOfAsyncBody()))) { m_inlineForm = InlineForm::TwoWay; m_inlineBuiltin = *implementation.inlineBuiltin; m_effectsOfAttempt = Effects::unknownCode(); @@ -2218,6 +2275,17 @@ void CallInWalkBuiltins::addEffectsOn(Effects& effects, const Operand& operand) effects.writes.add(AbstractHeapSet { AbstractHeap::NamedProperties }); } +bool CallInWalkBuiltins::holdsOnlyCompiledRecords(TypeID operand) const +{ + Type type = m_types.at(m_types.withoutNullish(operand)); + if (!type.isObject() || !type.hasOnlyLayouts() || type.hasOnlyClosures()) + return false; + return std::ranges::all_of(membersOf(m_types, type.layouts), [&](LayoutID member) { + LayoutKind kind = m_program.at(member).kind; + return (kind == LayoutKind::Record || kind == LayoutKind::Instance) && m_walker.plan().layout(member).holdsCompiledObjects(); + }); +} + Effects CallInWalkBuiltins::effectsOfRow(std::optional codeInLine) const { const BuiltinOverload& row = *m_overload; @@ -2258,6 +2326,8 @@ Effects CallInWalkBuiltins::effectsOfRow(std::optional codeInLine if (m_table.has(BuiltinMark::MayTerminateProcess, row.builtin)) effects.mayBeTerminated = true; + if (codeInLine == InlineBuiltin::OwnPropertyKeys && m_arguments.size() == 1 && holdsOnlyCompiledRecords(m_arguments[0].type)) + effects.writes.remove(AbstractHeapSet { AbstractHeap::ShapeWords, AbstractHeap::NamedProperties }); if (m_receiver) addEffectsOn(effects, *m_receiver); for (const Operand& argument : m_arguments) @@ -2602,9 +2672,12 @@ void CallInWalkBuiltins::build() } if (m_ownPropertyAskedFor) { + Reference object = m_overload->receiver ? m_receiver->value : arguments[0]; + if (buildOwnPropertyTest(m_walker, m_call, object, m_ownPropertyAskedFor)) + return; Draft test(Opcode::PropHasOwn, originInFrontOfCall); m_walker.chooseConservativeFields(test); - test.setObjectOperand(m_overload->receiver ? m_receiver->value : arguments[0]); + test.setObjectOperand(object); test.setKey(m_ownPropertyAskedFor); if (m_receiversAskedForOwnProperty) { test.setReceivers(m_receiversAskedForOwnProperty); @@ -2796,6 +2869,30 @@ static Reference findSameStoreIntoTableBuiltLastInWalkBuiltins(Walker& walker, I return { }; } +TypeID CallInWalkBuiltins::provenContentsOfReceiver() const +{ + if (!m_receiver || m_receiver->heirBehindLook || m_mode != BuiltinCallMode::Call) + return { }; + bool returnsContents = std::ranges::any_of(std::array { BuiltinID(Builtins::MapPrototypeGet), BuiltinID(Builtins::WeakMapPrototypeGet), BuiltinID(Builtins::ArrayPrototypePop), BuiltinID(Builtins::ArrayPrototypeShift) }, [&](BuiltinID method) { + return m_builtin == m_table.canonicalID(method); + }); + const LinkedContentsAtOperation* contents = returnsContents ? m_walker.facts().linked().contentsAtOperation(m_walker.currentFunction(), m_call.id()) : nullptr; + bool isKnown = contents && contents->value; + return isKnown && !isOnlyCutToInt32(contents->value) ? contents->value : TypeID(); +} + +bool CallInWalkBuiltins::isOnlyCutToInt32(TypeID contents) const +{ + Type type = m_types.at(contents); + if (!m_walker.isInLoop() || !m_walker.opensGuesses() || !(type.primitives & Primitive::Number) || !type.hasOnlyPrimitives(Primitive::Number | Primitive::UndefinedOrNull)) + return false; + const BodyIndex& index = m_walker.currentBodyIndex(); + ValueID user = index.onlyUsingValue(m_call.id()); + while (user && opcodesThatStateTypes.contains(index.opcodeOf(user))) + user = index.onlyUsingValue(user); + return user && allBitwiseOperators.contains(index.opcodeOf(user)); +} + Reference CallInWalkBuiltins::buildCall(std::span arguments) { if (m_form == BuiltinCallForm::Whole && m_mode != BuiltinCallMode::Get && !m_receiver && m_builtin == m_table.canonicalID(Builtins::RegExp)) { @@ -2859,12 +2956,11 @@ Reference CallInWalkBuiltins::buildCall(std::span arguments) call.resultIsReturnedAtOnce = m_resultIsReturnedAtOnce; if (m_mode != BuiltinCallMode::Set) call.inputValueGiven = m_call.id(); - const LinkedContentsAtOperation* contents = m_receiver && !m_receiver->heirBehindLook && (m_builtin == m_table.canonicalID(Builtins::MapPrototypeGet) || m_builtin == m_table.canonicalID(Builtins::WeakMapPrototypeGet)) ? m_walker.facts().linked().contentsAtOperation(m_walker.currentFunction(), m_call.id()) : nullptr; - TypeID before = m_walker.exchangeContentsBeingTyped(contents ? contents->value : TypeID()); - bool isLookupInTable = isCodeInLine && m_receiver && arguments.size() == 1 && (m_inlineBuiltin == InlineBuiltin::MapGet || m_inlineBuiltin == InlineBuiltin::MapHas || m_inlineBuiltin == InlineBuiltin::SetHas); - std::array searched { isLookupInTable ? m_receiver->value : Reference(), isLookupInTable ? arguments[0] : Reference() }; - std::optional loop = isLookupInTable ? m_walker.outermostLoopToLookUpInFrontOf(searched, m_effects) : std::nullopt; - std::optional lookedUpAgain = loop && m_walker.checksProofs() ? std::optional(call) : std::nullopt; + TypeID before = m_walker.exchangeContentsBeingTyped(provenContentsOfReceiver()); + bool looksUpInTable = isCodeInLine && m_receiver && arguments.size() == 1 && isLookupInTable(*m_inlineBuiltin); + std::array searched { looksUpInTable ? m_receiver->value : Reference(), looksUpInTable ? arguments[0] : Reference() }; + std::optional loop = looksUpInTable ? m_walker.outermostLoopToLookUpInFrontOf(searched, m_effects) : std::nullopt; + std::optional lookedUpAgain = looksUpInTable && m_walker.checksProofs() ? std::optional(call) : std::nullopt; auto isLoopInvariant = [&](Reference operand) { return m_walker.isInvariantInSpecializedVersion(operand); }; @@ -2875,18 +2971,19 @@ Reference CallInWalkBuiltins::buildCall(std::span arguments) Reference built; bool isStringCallOnInvariants = !loop && !isCalledInFront && isCodeInLine && m_receiver && m_mode == BuiltinCallMode::Call && answerHangsOnOperandsAlone && m_walker.isInvariantInInnermostLoop(m_receiver->value) && std::ranges::all_of(arguments, [&](Reference argument) { return m_walker.isInvariantInInnermostLoop(argument); }); - if (isStringCallOnInvariants || (isLookupInTable && !loop && m_mode == BuiltinCallMode::Call && m_walker.isLookupLoopInvariant(searched, m_effects))) { + if (isStringCallOnInvariants || (looksUpInTable && !loop && m_mode == BuiltinCallMode::Call && m_walker.isLookupLoopInvariant(searched, m_effects))) { built = m_walker.buildOnFirstArrival(m_call.id(), call, m_effects); if (m_walker.hasFailed()) return { }; - if (built && isLookupInTable && m_walker.checksProofs()) - lookedUpAgain = call; - else if (built && m_walker.checksHoistedCalls()) + if (built && !lookedUpAgain && m_walker.checksHoistedCalls()) calledAgain = call; } if (!built) { Walker::SpecializedPreheaderScope preheaderScope(m_walker, isCalledInFront); + uint32_t numberOfRecordsInFront = m_walker.code().numberOfValues(); built = loop ? m_walker.buildInFrontOfOpenLoop(*loop, WTF::move(call), &m_effects) : m_walker.build(WTF::move(call), &m_effects); + if (!loop && m_walker.wasBuiltInPlaceSince(numberOfRecordsInFront, built)) + lookedUpAgain = std::nullopt; } if (lookedUpAgain) m_walker.holdLookupInFrontAgainstLookupInTurn(built, WTF::move(*lookedUpAgain), m_effects); @@ -2897,7 +2994,7 @@ Reference CallInWalkBuiltins::buildCall(std::span arguments) m_walker.exchangeContentsBeingTyped(before); if (isCodeInLine) noteLengthOfViewMadeWithLengthInWalkBuiltins(m_walker, built, arguments); - if (isLookupInTable && m_inlineBuiltin == InlineBuiltin::MapGet && m_mode == BuiltinCallMode::Call) + if (looksUpInTable && m_inlineBuiltin == InlineBuiltin::MapGet && m_mode == BuiltinCallMode::Call) return guardNumericCallResult(m_walker, m_call, built, m_origin); return built; } @@ -2923,7 +3020,9 @@ Reference CallInWalkBuiltins::buildCodeInLineOrCall(std::span a BlockID done = m_walker.addBlock(); BlockID miss = m_walker.addBlock(BlockFrequency::Rare); BlockID join = m_walker.addBlock(); + TypeID before = m_walker.exchangeContentsBeingTyped(provenContentsOfReceiver()); Reference answered = m_walker.tryCallInline(WTF::move(attempt), done, miss, &m_effectsOfAttempt); + bool isTypedByContents = !!m_walker.exchangeContentsBeingTyped(before); if (!answered) return { }; @@ -2936,6 +3035,8 @@ Reference CallInWalkBuiltins::buildCodeInLineOrCall(std::span a m_walker.continueIn(done); if (m_walker.checksProofs() && answered.isValue()) m_walker.checkProof(answered, TypeTable::anything, ProofSource::ResultOfOtherCall, origin); + else if (isTypedByContents && answered.isValue() && !m_types.at(m_walker.typeOf(answered)).isAnything()) + m_walker.buildCheckOfProof(answered, m_walker.typeOf(answered), TypeTable::anything, ProofSource::Contents, origin, true); if (hostReadsMembersFirstIn(m_table, *m_overload)) { for (BuiltinID location : { BuiltinID(Builtins::DatePrototypeValueOf), BuiltinID(Builtins::DatePrototypeToISOString) }) { const Builtin& entry = m_table.at(location); @@ -3134,7 +3235,7 @@ static BuiltinID findMethodThatStubOfClassEntersInWalkBuiltins(const Walker& wal BuiltinID findMethodThatSomeClassGivesUnderKey(Walker& walker, TypeID receiver, PropertyKeyID key, unsigned numberOfArguments) { const BuiltinTable& table = walker.program().builtins(); - BuiltinName name = liesInThread(key) ? BuiltinName() : walker.program().builtinNameOf(key); + BuiltinName name = nameOfMemberInWalkBuiltins(walker.program(), key); if (!name) return { }; TypeArena& types = walker.types(); @@ -3407,7 +3508,8 @@ static bool appendListPassedToPush(Walker& walker, const BodyValue& call) return false; const BodyIndex& index = walker.currentBodyIndex(); auto operation = call.as(); - if (!index.isUnused(call.id()) || !operation.thisValueOperand()) + bool isUnused = index.isUnused(call.id()) || (walker.isInRepeatedCode() && isResultUnused(index, call.id())); + if (!isUnused || !operation.thisValueOperand()) return false; ValueID read = index.stripCasts(operation.calleeOperand()); if (!read || index.isParameter(read) || index.opcodeOf(read) != Opcode::PropGet || index.value(read).isFreeName()) @@ -3489,17 +3591,25 @@ static bool appendListPassedToPush(Walker& walker, const BodyValue& call) walker.count(StatisticID::SpreadPushesCompiledAsAppends); BlockID behind = walker.addBlock(); - if (!walker.isUnreachable()) - walker.jumpTo(behind); + Reference answer = index.isUnused(call.id()) ? Reference() : walker.addParameter(behind, { }, Representation::JSValue); + auto jumpBehind = [&](Reference result) { + if (walker.isUnreachable()) + return; + if (answer) + walker.jumpTo(behind, singleElementSpan(result)); + else + walker.jumpTo(behind); + }; + jumpBehind(answer && !walker.isUnreachable() ? buildArrayLength(walker, receiver, origin) : Reference()); walker.continueIn(asWritten); if (FunctionID listed = walker.facts().linked().calleeAt(walker.currentFunction(), call.id())) walker.leaveAnythingAt(listed); Draft closed = written; walker.chooseConservativeFields(closed); - walker.build(WTF::move(closed)); - if (!walker.isUnreachable()) - walker.jumpTo(behind); + jumpBehind(walker.build(WTF::move(closed))); walker.continueIn(behind); + if (answer) + walker.bind(call.id(), answer); return true; } @@ -3645,6 +3755,7 @@ bool callBuiltinOfItsOpcodes(Walker& walker, const BodyValue& value) return true; countBuiltinCallOfProgram(walker, call.statisticOfRow()); call.build(); + noteListedOwnKeys(walker, value.id()); return true; } diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkCalls.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkCalls.cpp index adde242219c24..fb11c7615a8a6 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkCalls.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkCalls.cpp @@ -51,6 +51,7 @@ struct CallInWalkCalls { Reference listOfUnknownLength; Reference borrowedHiddenRest; Reference borrowedHiddenRestLength; + bool listLengthIsBounded { false }; std::span writtenInFrontOfList; BuiltinID calledBuiltin; bool isConstruction { false }; @@ -151,6 +152,21 @@ bool unitMayLeaveByDirectCallOf(Walker& walker, FunctionID callee) return unitMayLeaveByCallInWalkCalls(walker, walker.plan().conventions().directEntryOf(callee)); } +bool canTailCallMethodTable(Walker& walker, uint32_t place, std::span methods) +{ + const Plan& plan = walker.plan(); + const CallingConventions& conventions = plan.conventions(); + CodeUnit unit = walker.unit(); + if (!returnsResultOfFunction(unit.role) || (plan.resultsPassNoReturn(unit.function) && plan.typeHeldOfResult(walker.program(), walker.facts().linked(), unit.function))) + return false; + const MethodSelector& selector = conventions.methodSelector(place); + if (!unitMayLeaveByCallInWalkCalls(walker, conventions.tableLookupOf(selector)) || conventions.resultRepresentation(unit) != selector.result) + return false; + return std::ranges::all_of(methods, [&](FunctionID method) { + return plan.endsItsTailCalls(method) || conventions.resultRepresentation(conventions.directEntryOf(method)) == selector.result; + }); +} + static bool findCapturesInSightInWalkCalls(const Code& code, FunctionID function, Reference closure, size_t numberOfCaptures, Vector& captures) { if (!closure || !closure.isRecord()) @@ -396,6 +412,19 @@ static std::optional restArrayFirstArgument(Walker& walker, Reference return allocation.firstArgument(); } +static bool holdsArgumentsOfUnit(Walker& walker, Reference array) +{ + if (restArrayFirstArgument(walker, array)) + return true; + std::optional index = walker.entryParameterOfUnitBoundTo(array); + if (!index) + return false; + const Function& function = walker.program().at(walker.unit().function); + EntryParameter kind = function.entryParameterKind(*index); + return kind == EntryParameter::HiddenRest + || (kind == EntryParameter::Parameter && function.has(FunctionFlags::HasRestParameter) && *index - function.firstParameterIndex() == function.numberOfPositionalParameters()); +} + static Reference sinkRestArrayAllocation(Walker& walker, Reference array) { if (!restArrayFirstArgument(walker, array)) @@ -554,7 +583,8 @@ static bool buildDirectCallInWalkCalls(Walker& walker, const BodyValue& call, co originInFrontOfCall.isTailCall = false; BlockID continuationBlock; Reference mergedResult; - if (request.listOfUnknownLength && !buildCallAsWrittenForLongListInWalkCalls(walker, written, request, originInFrontOfCall, continuationBlock, mergedResult)) + bool asksLength = request.listOfUnknownLength && !request.listLengthIsBounded; + if (asksLength && !buildCallAsWrittenForLongListInWalkCalls(walker, written, request, originInFrontOfCall, continuationBlock, mergedResult)) return true; Vector operands; @@ -950,6 +980,7 @@ static bool walkConstructOfFunctionInWalkCalls(Walker& walker, const BodyValue& if (prototype) allocation.setPrototypeOperand(prototype); walker.chooseConservativeFields(allocation); + allocation.setEndsNoGuards(walker.isInLoop() && writerEndsNoGuard(walker, allocation)); Reference object = walker.build(WTF::move(allocation)); if (!object) return true; @@ -1215,6 +1246,7 @@ static bool walkConstructionOfListInWalkCalls(Walker& walker, const BodyValue& c request.listOfUnknownLength = written->listOperand(); if (borrowed) { request.borrowedHiddenRest = borrowed; + request.listLengthIsBounded = holdsArgumentsOfUnit(walker, borrowed); request.borrowedHiddenRestLength = buildSpreadArrayLength(walker, borrowed, written->origin()); if (!request.borrowedHiddenRestLength) return true; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkChoices.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkChoices.cpp index 317bbf2dcf733..f684a56769528 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkChoices.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkChoices.cpp @@ -878,6 +878,14 @@ static bool sharesStorageOfArrayInChoices(Walker& walker, const DataNode::Array& return std::ranges::all_of(array.elements, canLieInSharedStorageInChoices); } +static bool isBornToBeFilledInChoices(const DataNode::Array& array) +{ + return array.elements.size() > 1 && std::ranges::all_of(array.elements, [](const DataNode& element) { + const auto* number = element.tryAs(); + return number && !number->value; + }); +} + static void chooseEncodingOfDataInChoices(Walker& walker, Draft& data) { RootOfConstant whereRootEnds = walker.constantEndsWhereItLeaves(data.inputValueGiven, data.data()) ? RootOfConstant::EndsWhereItLeaves : RootOfConstant::IsAsked; @@ -888,6 +896,10 @@ static void chooseEncodingOfDataInChoices(Walker& walker, Draft& data) SetForScope makingObjectsOfConstant = walker.makingObjectsOfConstant(); const DataNode& root = walker.program().at(data.data()).root; if (const auto* array = root.tryAs(); array && sharesStorageOfArrayInChoices(walker, *array)) { + if (array->elements.size() <= maxNodesOfExpandedData && isBornToBeFilledInChoices(*array)) { + becomeNodeOfDataInChoices(walker, data, root); + return; + } bool isStoredIntoInLoop = data.inputValueGiven && walker.facts().summaries().at(walker.currentFunction()).constantListIsStoredIntoInLoop(data.inputValueGiven) && surveyDataInChoices(walker, data.data()).tableOfImageHoldsIt; data.setEncoding(isStoredIntoInLoop ? DataEncoding::TreeWithOwnStorage : DataEncoding::CopyOnWriteArray); return; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkCopiesOwnKeys.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkCopiesOwnKeys.cpp index b2f66b9452a45..28d9e3962198e 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkCopiesOwnKeys.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkCopiesOwnKeys.cpp @@ -228,6 +228,16 @@ static Vector countsAtWhichMakersStopInCopiesOwnKeys(const Walker& return counts; } +static bool listsEveryRequiredFieldInCopiesOwnKeys(const Walker& walker, LayoutID layout, std::span created) +{ + const PlannedLayout& planned = walker.plan().layout(layout); + for (FieldIndex field = 0; field < planned.numberOfFields(); ++field) { + if (planned.slots[field].hasStorage() && isRequiredInCopiesOwnKeys(walker, layout, field) && !std::ranges::contains(created, field)) + return false; + } + return true; +} + static bool findArmsOfLayoutInCopiesOwnKeys(Walker& walker, LayoutID layout, unsigned mostArms, size_t mostFields, OwnKeyPlan& plan) { if (!canPlanOwnKeysOfLayoutInCopiesOwnKeys(walker, layout)) @@ -256,6 +266,7 @@ static bool findArmsOfLayoutInCopiesOwnKeys(Walker& walker, LayoutID layout, uns uint32_t numberOfOrders = keyOrders.numberOfOrdersOf(layout); if (walker.isInLoop()) { + Vector, 8> beginnings; Vector lastStopTakenOfOrders; for (bool someOrderHasStop = true; someOrderHasStop;) { someOrderHasStop = false; @@ -268,10 +279,17 @@ static bool findArmsOfLayoutInCopiesOwnKeys(Walker& walker, LayoutID layout, uns --count; lastStopTakenOfOrders[number] = count; someOrderHasStop |= !!count; - if (count && !addArmWhereThereIsRoom(keyOrders.order(layout, number).first(count))) - return false; + if (count) + beginnings.append(keyOrders.order(layout, number).first(count)); } } + std::ranges::stable_partition(beginnings, [&](std::span created) { + return listsEveryRequiredFieldInCopiesOwnKeys(walker, layout, created); + }); + for (std::span created : beginnings) { + if (!addArmWhereThereIsRoom(created)) + return false; + } } std::optional everyStopOfMakersHasArm = hasList ? std::optional(everyListHasArm) : std::nullopt; for (uint32_t number = 0; number < numberOfOrders; ++number) { diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkCoroutineAwaitedCalls.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkCoroutineAwaitedCalls.cpp index f5b0930c47809..c5cb86a00f56c 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkCoroutineAwaitedCalls.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkCoroutineAwaitedCalls.cpp @@ -112,6 +112,11 @@ static bool bodyMayThrowInAwaitedCalls(const Walker& walker, FunctionID callee) return linked.effectsOnReturn().mayThrowOtherThanStackOverflow() || linked.effectsOnThrow().mayThrowOtherThanStackOverflow(); } +static bool resultHasNoThenInAwaitedCalls(const Walker& walker, FunctionID callee) +{ + return hasNoThen(walker, walker.plan().typeHeldOfResult(walker.program(), walker.facts().linked(), CodeUnit { callee, CodeRole::CoroutineBody })); +} + static std::optional constantBooleanInAwaitedCalls(const Walker& walker, Reference reference) { if (!reference.isRecord()) @@ -306,7 +311,8 @@ bool buildAwaitedCall(Walker& walker, const BodyValue& call) bool neverSuspends = walker.plan().conventions().of(callee).coroutineNeverSuspends; if (!neverSuspends && body->kind != CoroutineKind::AsyncFunction) return false; - if (neverSuspends && !walker.isInLoop() && bodyMayThrowInAwaitedCalls(walker, callee)) + bool rejectsInStub = neverSuspends && !walker.isInLoop() && bodyMayThrowInAwaitedCalls(walker, callee); + if (rejectsInStub && (body->kind != CoroutineKind::AsyncFunction || walker.program().at(walker.unit().function).isModuleBody() || !resultHasNoThenInAwaitedCalls(walker, callee))) return false; std::optional> operands = layOutEntryOperandsOfCall(walker, call, callee, { callee, CodeRole::CoroutineBody }); @@ -318,7 +324,7 @@ bool buildAwaitedCall(Walker& walker, const BodyValue& call) Vector held; if (!walkUpToAwaitOfCall(walker, index, block, *wait, held)) return true; - Reference given = neverSuspends + Reference given = neverSuspends && !rejectsInStub ? buildAwaitOfPlainBodyInAwaitedCalls(walker, call, callee, body->kind, operands->span(), held.span(), walker.originOf(index.value(wait->await))) : buildAwaitCallInAwaitedCalls(walker, call, callee, operands->span(), body->driver, held.span(), wait->await); if (given) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkCoroutineCalls.h b/Source/JavaScriptCore/aot/walk/AOTWalkCoroutineCalls.h index b22a242e172db..0d15473421adf 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkCoroutineCalls.h +++ b/Source/JavaScriptCore/aot/walk/AOTWalkCoroutineCalls.h @@ -31,6 +31,10 @@ template ALWAYS_INLINE bool buildCoroutineOperation(Walker& walker inline constexpr OpcodeSet opcodesThatAnswerFromStepWrittenByHandTakes { Opcode::PropGet, Opcode::TypeTest }; AOT_LINK_OF_WALK(answerFromStepWrittenByHand, opcodesThatAnswerFromStepWrittenByHandTakes) +inline constexpr OpcodeSet opcodesThatLeaveNextToStepTakes { Opcode::PropGet }; +AOT_LINK_OF_WALK(leaveNextToStep, opcodesThatLeaveNextToStepTakes) +inline constexpr OpcodeSet opcodesThatCallStepsGeneratorTakes { Opcode::Call }; +AOT_LINK_OF_WALK(callStepsGenerator, opcodesThatCallStepsGeneratorTakes) std::optional> layOutEntryOperandsOfCall(Walker&, const BodyValue& call, FunctionID callee, CodeUnit entered); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkCoroutinePromises.h b/Source/JavaScriptCore/aot/walk/AOTWalkCoroutinePromises.h index a6cd0926bb038..2906c16fff8a4 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkCoroutinePromises.h +++ b/Source/JavaScriptCore/aot/walk/AOTWalkCoroutinePromises.h @@ -15,6 +15,7 @@ std::optional kindOfSuspendingBody(Walker&, Reference& driver); Reference buildSuspensionInCoroutine(Walker&, Draft&& suspension); bool isProvenToHaveNoThen(Walker&, Reference); +bool hasNoThen(const Walker&, TypeID); bool leansOnProofOfNoThen(Walker&, Reference, Origin); Reference buildTestForNoThen(Walker&, Reference value, Origin); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkCoroutineThenables.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkCoroutineThenables.cpp index ec159d7b435b8..3a1c76adf6593 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkCoroutineThenables.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkCoroutineThenables.cpp @@ -107,6 +107,11 @@ static bool isFreshListWithoutThenForThenables(Walker& walker, Reference value) }); } +bool hasNoThen(const Walker& walker, TypeID type) +{ + return isProvenToHaveNoThen(walker.program(), walker.facts(), walker.types(), type); +} + bool isProvenToHaveNoThen(Walker& walker, Reference value) { if (!value) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkCoroutines.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkCoroutines.cpp index e0095704a4f31..e826310d91338 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkCoroutines.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkCoroutines.cpp @@ -222,6 +222,13 @@ static bool generatorGoesIntoIterationAloneOfCoroutine(const BodyIndex& body, co } bool nobodyLooksAboveGeneratorsOf(const Walker& walker, FunctionID generatorFunction, uint32_t* numberOfListedCalls) +{ + return nobodyLooksAboveGeneratorsOf(walker, generatorFunction, numberOfListedCalls, scopedLambda([](const BodyIndex&, const LinkedCall&, FunctionID) { + return false; + })); +} + +bool nobodyLooksAboveGeneratorsOf(const Walker& walker, FunctionID generatorFunction, uint32_t* numberOfListedCalls, const ScopedLambda& passesForIteration) { const Function& function = walker.program().at(generatorFunction); if (!function.has(FunctionFlags::IsGenerator) || function.has(FunctionFlags::IsAsync) || !walker.facts().mechanisms().contains(Mechanism::TypedOperations)) @@ -244,7 +251,7 @@ bool nobodyLooksAboveGeneratorsOf(const Walker& walker, FunctionID generatorFunc ++*numberOfListedCalls; if (!body) body = walker.facts().bodies().find(referrer); - if (!body || !generatorGoesIntoIterationAloneOfCoroutine(*body, call)) + if (!body || (!generatorGoesIntoIterationAloneOfCoroutine(*body, call) && !passesForIteration(*body, call, referrer))) return false; } } diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkCountersOfLoops.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkCountersOfLoops.cpp index c7f42d63bf142..a565ec3f0da6e 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkCountersOfLoops.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkCountersOfLoops.cpp @@ -192,7 +192,10 @@ static bool mayBeInt32InCountersOfLoops(const Walker& walker, Reference value) if (!value || !value.isValue()) return false; Type type = walker.types().at(walker.typeOf(value)); - return type.isJavaScriptValue() && type.primitives & Primitive::Integer && !type.isInteger(); + if (!type.isJavaScriptValue() || !(type.primitives & Primitive::Integer)) + return false; + bool isWideInteger = type.isInteger() && !type.integers.isSubsetOf(Range::int32()) && walker.representationOf(value) == Representation::Double; + return !type.isInteger() || isWideInteger; } bool isNumberCarriedBesideTextOrBigInt(const Walker& walker, Reference enters, TypeID carried) @@ -317,7 +320,7 @@ static bool isSteppedAllRoundInCountersOfLoops(Walker& walker, const BodyIndex& } BodyValue operation = index.value(user); Opcode opcode = operation.opcode(); - if (opcodesThatStateTypes.contains(opcode) || opcode == Opcode::ToNumeric) { + if (opcodesThatStateTypes.contains(opcode) || opcode == Opcode::ToNumeric || opcode == Opcode::ToPropertyKey) { if (names.contains(operation.operand(0))) add(operation.id(), namesBehindSteps.contains(operation.operand(0))); continue; @@ -404,6 +407,127 @@ static bool turnsTakeCounterAsIntegerInCountersOfLoops(const BodyIndex& index, c return false; } +static bool isUsedInArithmeticAlone(Walker& walker, const BodyIndex& index, const UsersOfValuesInCountersOfLoops& users, BlockID inputHead, const LoopPreview& preview, const NamesOfCounterOfLoop& names) +{ + static constexpr OpcodeSet operators { Opcode::Sub, Opcode::Mul, Opcode::Div, Opcode::Negate }; + auto isName = [&](ValueID operand) { + return names.contains(operand); + }; + auto isNumber = [&](ValueID operand) { + TypeID type = typeByLookInCountersOfLoops(walker, index, preview, operand); + return type && !isNeverType(type) && walker.types().at(type).isNumber(); + }; + UsersOfNamesInCountersOfLoops usersLeft(users, names); + while (ValueID user = usersLeft.next()) { + if (isName(user) || index.isParameter(user)) + continue; + BodyValue operation = index.value(user); + if (operators.contains(operation.opcode())) + continue; + if (operation.opcode() != Opcode::Add || !isNumber(isName(operation.operand(0)) ? operation.operand(1) : operation.operand(0))) + return false; + } + for (BlockID member : index.membersOf(index.loopOf(inputHead))) { + bool isTerminatorOperand = false; + index.block(member).terminator().forEachOperand([&](ValueID operand, uint32_t) { + isTerminatorOperand |= isName(operand); + }); + if (isTerminatorOperand) + return false; + } + return true; +} + +static bool isIncreasingKey(const BodyIndex& index, const UsersOfValuesInCountersOfLoops& users, const NamesOfCounterOfLoop& names) +{ + static constexpr OpcodeSet accessesByKey { Opcode::ElemGet, Opcode::ElemSet }; + auto isName = [&](ValueID operand) { + return names.contains(operand); + }; + bool isKey = false; + UsersOfNamesInCountersOfLoops usersLeft(users, names); + while (ValueID user = usersLeft.next()) { + if (index.isParameter(user)) + continue; + BodyValue operation = index.value(user); + if (isName(operation.id()) && !opcodesThatStateTypes.contains(operation.opcode()) && operation.opcode() != Opcode::ToNumeric && operation.opcode() != Opcode::ToPropertyKey && operation.opcode() != Opcode::Increment) + return false; + isKey |= accessesByKey.contains(operation.opcode()) && isName(operation.propertyOperand()); + } + return isKey; +} + +struct BackEdgeInputs { + Vector elements; + bool hasOtherInputs { false }; + Vector joins; +}; + +static void collectBackEdgeInput(Walker& walker, const BodyIndex& index, BlockID inputHead, ValueID value, const NamesOfCounterOfLoop& names, BackEdgeInputs& inputs) +{ + ValueID input = index.stripCasts(value); + if (names.contains(value) || names.contains(input) || inputs.joins.contains(input)) + return; + LoopID loop = index.loopOf(inputHead); + BlockID block = index.blockOf(input); + if (!index.isParameter(input)) { + if (index.opcodeOf(input) == Opcode::ElemGet && index.isInLoop(block, loop)) + inputs.elements.append(input); + else + inputs.hasOtherInputs = true; + return; + } + if (block == inputHead || index.isLoopHead(block) || !index.isInLoop(block, loop) || inputs.joins.size() == 8) { + inputs.hasOtherInputs = true; + return; + } + inputs.joins.append(input); + uint32_t position = input.index() - index.block(block).firstParameter().index(); + for (const BodyIncomingEdge& edge : walker.incomingEdgesOf(index, block)) { + std::span arguments = edge.target.arguments(); + if (position < arguments.size()) + collectBackEdgeInput(walker, index, inputHead, arguments[position], names, inputs); + else + inputs.hasOtherInputs = true; + } +} + +static BackEdgeInputs findBackEdgeInputs(Walker& walker, const BodyIndex& index, BlockID inputHead, uint32_t position, const NamesOfCounterOfLoop& names) +{ + BackEdgeInputs inputs; + for (const BodyIncomingEdge& edge : walker.incomingEdgesOf(index, inputHead)) { + if (!index.isInLoop(edge.from, index.loopOf(inputHead))) + continue; + std::span arguments = edge.target.arguments(); + if (position < arguments.size()) + collectBackEdgeInput(walker, index, inputHead, arguments[position], names, inputs); + else + inputs.hasOtherInputs = true; + } + return inputs; +} + +static bool hasElementBackEdgeInput(Walker& walker, const BodyIndex& index, BlockID inputHead, uint32_t position) +{ + return findBackEdgeInputs(walker, index, inputHead, position, { index.block(inputHead).parameter(position) }).elements.size(); +} + +static bool isElementCarriedIndex(Walker& walker, const BodyIndex& index, const UsersOfValuesInCountersOfLoops& users, BlockID inputHead, uint32_t position, NamesOfCounterOfLoop& names, Vector* elements = nullptr) +{ + isSteppedAllRoundInCountersOfLoops(walker, index, users, inputHead, position, names); + BackEdgeInputs inputs = findBackEdgeInputs(walker, index, inputHead, position, names); + if (inputs.elements.isEmpty() || inputs.hasOtherInputs) + return false; + if (elements) + elements->appendVector(inputs.elements); + UsersOfNamesInCountersOfLoops usersLeft(users, names); + while (ValueID user = usersLeft.next()) { + if (!index.isParameter(user) && accessesUnderKeysInCountersOfLoops.contains(index.opcodeOf(user)) && names.contains(index.value(user).propertyOperand())) + return true; + } + return false; +} + enum class WhatComesRoundInCountersOfLoops : uint8_t { NotAsked, IsBeingAsked, Int32, Other }; static bool comesRoundAsInt32InCountersOfLoops(Walker& walker, const BodyIndex& index, BlockID inputHead, const LoopPreview& preview, uint32_t position, Vector& answers) @@ -442,9 +566,8 @@ static bool comesRoundAsInt32InCountersOfLoops(Walker& walker, const BodyIndex& return isInt32OnBackEdges; } -static void guessBoundsInCountersOfLoops(Walker& walker, const BodyIndex& index, const UsersOfValuesInCountersOfLoops& users, BlockID inputHead, const NamesOfCounterOfLoop& names, std::span entering, Vector& guesses) +static void guessBoundsInCountersOfLoops(Walker& walker, const BodyIndex& index, const UsersOfValuesInCountersOfLoops& users, BlockID inputHead, const NamesOfCounterOfLoop& names, NamesBeforeLoop& namesBeforeLoop, std::span entering, Vector& guesses) { - LoopID loop = index.loopOf(inputHead); uint32_t firstValueOfLoop = index.block(inputHead).firstParameter().index(); UsersOfNamesInCountersOfLoops usersLeft(users, names); while (ValueID user = usersLeft.next()) { @@ -464,7 +587,7 @@ static void guessBoundsInCountersOfLoops(Walker& walker, const BodyIndex& index, return guess.value == value; }); if (listed == notFound) - guesses.append(HoistedSpeculation { value, TypeTable::int32, { }, namesInFrontThatLoopMentions(walker, index, loop, firstValueOfLoop, value) }); + guesses.append(HoistedSpeculation { value, TypeTable::int32, { }, namesBeforeLoop.findNamesOf(value) }); else if (walker.types().at(guesses[listed].tested).isNumber()) guesses[listed].tested = TypeTable::int32; } @@ -514,9 +637,8 @@ static Reference makeAddendThatNoTurnChangesInCountersOfLoops(Walker& walker, co return made; } -static void guessAddendsOfKeysInCountersOfLoops(Walker& walker, const BodyIndex& index, const UsersOfValuesInCountersOfLoops& users, BlockID inputHead, const NamesOfCounterOfLoop& names, std::span entering, Vector& guesses) +static void guessAddendsOfKeysInCountersOfLoops(Walker& walker, const BodyIndex& index, const UsersOfValuesInCountersOfLoops& users, BlockID inputHead, const NamesOfCounterOfLoop& names, NamesBeforeLoop& namesBeforeLoop, std::span entering, Vector& guesses) { - LoopID loop = index.loopOf(inputHead); uint32_t firstValueOfLoop = index.block(inputHead).firstParameter().index(); UsersOfNamesInCountersOfLoops usersLeft(users, names); while (ValueID user = usersLeft.next()) { @@ -536,7 +658,7 @@ static void guessAddendsOfKeysInCountersOfLoops(Walker& walker, const BodyIndex& continue; if (guesses.containsIf([&](const HoistedSpeculation& guess) { return guess.value == value; })) continue; - guesses.append(HoistedSpeculation { value, TypeTable::int32, { }, namesInFrontThatLoopMentions(walker, index, loop, firstValueOfLoop, value) }); + guesses.append(HoistedSpeculation { value, TypeTable::int32, { }, namesBeforeLoop.findNamesOf(value) }); if (addend.index() >= firstValueOfLoop) guesses.last().names.append(addend); walker.count(StatisticID::InvariantNumberGuardsHoisted); @@ -544,7 +666,7 @@ static void guessAddendsOfKeysInCountersOfLoops(Walker& walker, const BodyIndex& } } -static void guessAddendsOfSumsCutToInt32InCountersOfLoops(Walker& walker, const BodyIndex& index, BlockID inputHead, std::span entering, Vector& guesses) +static void guessAddendsOfSumsCutToInt32InCountersOfLoops(Walker& walker, const BodyIndex& index, BlockID inputHead, NamesBeforeLoop& namesBeforeLoop, std::span entering, Vector& guesses) { static constexpr OpcodeSet cuts = operatorsOnBitsOfCountersOfLoops | OpcodeSet { Opcode::URShift }; static constexpr unsigned deepestSum = 4; @@ -559,7 +681,7 @@ static void guessAddendsOfSumsCutToInt32InCountersOfLoops(Walker& walker, const return; if (guesses.containsIf([&](const HoistedSpeculation& guess) { return guess.value == value; })) return; - guesses.append(HoistedSpeculation { value, TypeTable::int32, { }, namesInFrontThatLoopMentions(walker, index, loop, firstValueOfLoop, value) }); + guesses.append(HoistedSpeculation { value, TypeTable::int32, { }, namesBeforeLoop.findNamesOf(value) }); walker.count(StatisticID::InvariantNumberGuardsHoisted); }; auto follow = [&](auto& self, ValueID operand, unsigned depth) -> void { @@ -600,6 +722,63 @@ static ValueID parameterOfHeadNamedInCountersOfLoops(const BodyIndex& index, Val return { }; } +static bool isTruthAskedAtHead(const BodyIndex& index, BlockID inputHead, uint32_t position) +{ + BodyTerminator end = index.block(inputHead).terminator(); + ValueID asked = end.kind() == TerminatorKind::Branch ? end.as().conditionOperand() : ValueID(); + if (!asked || index.isParameter(asked) || index.opcodeOf(asked) != Opcode::ToBoolean || !index.isInLoop(end.targets()[0].block(), index.loopOf(inputHead))) + return false; + ValueID subject = index.stripCasts(index.value(asked).operand(0)); + while (subject && !index.isParameter(subject) && index.opcodeOf(subject) == Opcode::ToNumeric) + subject = index.stripCasts(index.value(subject).operand(0)); + return subject == index.block(inputHead).parameter(position); +} + +static bool testOfTurnHoldsNumberFromZeroUpInCountersOfLoops(const BodyIndex& index, BlockID inputHead, BlockID block, ValueID number) +{ + for (number = index.stripCasts(number); block && block != inputHead; block = index.immediateDominator(block)) { + std::optional edge = index.onlyIncomingEdge(block); + if (!edge || !index.terminator(edge->from).isConditionalBranch()) + continue; + ValueID test = index.terminator(edge->from).as().conditionOperand(); + bool holds = !edge->targetIndex; + while (test && !index.isParameter(test) && (index.opcodeOf(test) == Opcode::LogicalNot || index.opcodeOf(test) == Opcode::ToBoolean)) { + holds ^= index.opcodeOf(test) == Opcode::LogicalNot; + test = index.value(test).operand(0); + } + if (!test || index.isParameter(test) || !allRelations.contains(index.opcodeOf(test))) + continue; + BodyValue relation = index.value(test); + bool leftIsGreater = (relation.opcode() == Opcode::Greater || relation.opcode() == Opcode::GreaterEq) == holds; + bool isStrict = (relation.opcode() == Opcode::Less || relation.opcode() == Opcode::Greater) == holds; + ValueID bound = index.stripCasts(relation.operand(leftIsGreater)); + if (index.stripCasts(relation.operand(!leftIsGreater)) == number && !index.isParameter(bound) && index.opcodeOf(bound) == Opcode::Integer && index.value(bound).as().integer() >= (isStrict ? -1 : 0)) + return true; + } + return false; +} + +static ValueID findUpdatedKey(Walker& walker, const BodyIndex& index, ValueID key) +{ + ValueID value = index.stripCasts(key); + if (!index.isParameter(value) || index.valueNarrowedInto(value) || index.isLoopHead(index.blockOf(value))) + return key; + uint32_t position = value.index() - index.block(index.blockOf(value)).firstParameter().index(); + ValueID updated; + for (const BodyIncomingEdge& edge : walker.incomingEdgesOf(index, index.blockOf(value))) { + std::span arguments = edge.target.arguments(); + ValueID argument = position < arguments.size() ? index.stripCasts(arguments[position]) : ValueID(); + if (argument && !index.isParameter(argument) && index.opcodeOf(argument) == Opcode::ToPropertyKey) + argument = index.stripCasts(index.value(argument).operand(0)); + while (argument && index.isParameter(argument) && index.valueNarrowedInto(argument)) + argument = index.stripCasts(index.valueNarrowedInto(argument)); + if (!argument || (updated && updated != argument)) + return key; + updated = argument; + } + return updated ? updated : key; +} + CounterHeldBelowRoomOfViews findCounterHeldBelowRoomOfViews(Walker& walker, BlockID inputHead, NOESCAPE const ScopedLambda& typeThatEnters, NOESCAPE const ScopedLambda& typeInFront, NOESCAPE const ScopedLambda& isListOnlyReadInLoop) { static constexpr OpcodeSet relations { Opcode::Less, Opcode::LessEq }; @@ -609,7 +788,7 @@ CounterHeldBelowRoomOfViews findCounterHeldBelowRoomOfViews(Walker& walker, Bloc BodyBlock head = index.block(inputHead); uint32_t firstValueOfLoop = head.firstParameter().index(); BodyTerminator end = head.terminator(); - if (end.kind() != TerminatorKind::Branch || !head.numberOfParameters() || !index.holdsAnyOf(accessesUnderKeysInCountersOfLoops)) + if (end.kind() != TerminatorKind::Branch || !head.numberOfParameters() || !index.holdsAnyOf(accessesUnderKeysInCountersOfLoops | OpcodeSet { Opcode::Call })) return found; ValueID asked = end.as().conditionOperand(); if (!asked || index.isParameter(asked) || !head.holds(asked) || asked.index() + 1 != head.endOfValues().index() || !relations.contains(index.opcodeOf(asked))) @@ -677,9 +856,22 @@ CounterHeldBelowRoomOfViews findCounterHeldBelowRoomOfViews(Walker& walker, Bloc if (member == inputHead) continue; for (BodyValue access : index.block(member).values()) { - if (!accessesUnderKeysInCountersOfLoops.contains(access.opcode()) || index.stripCasts(access.objectOperand()).index() >= firstValueOfLoop) + if (access.opcode() == Opcode::Call && std::ranges::any_of(access.as().argumentsOperands(), [&](ValueID argument) { return argument && namesCounter(argument); })) { + forEachElementAccessInInlinedCallee(walker, access, scopedLambda([&](ValueID argument, ValueID key) { + ValueID view = index.stripCasts(argument); + if (key && namesCounter(key) && view.index() < firstValueOfLoop && isViewOfNumbersReadInLine(walker, typeInFront(view))) + found.accesses.append({ access.id(), view, { }, { }, false, false }); + })); + } + if (!accessesUnderKeysInCountersOfLoops.contains(access.opcode())) continue; - ValueID key = index.stripCasts(access.propertyOperand()); + Reference fieldView; + if (index.stripCasts(access.objectOperand()).index() >= firstValueOfLoop) { + fieldView = index.loopOf(member) == loop ? nameOfListThatEntersLoopAround(walker, index, access) : Reference(); + if (!fieldView || !fieldView.isValue() || !isViewOfNumbersReadInLine(walker, walker.typeOf(fieldView))) + continue; + } + ValueID key = index.stripCasts(findUpdatedKey(walker, index, access.propertyOperand())); ValueID addend; if (!namesCounter(key)) { if (index.isParameter(key) || index.opcodeOf(key) != Opcode::Add) @@ -690,10 +882,11 @@ CounterHeldBelowRoomOfViews findCounterHeldBelowRoomOfViews(Walker& walker, Bloc continue; } ValueID view = access.objectOperand().index() < firstValueOfLoop ? access.objectOperand() : index.stripCasts(access.objectOperand()); - bool isOfList = access.opcode() == Opcode::ElemGet && isListOnlyReadInLoop(view); - if ((!isOfList && !isViewOfNumbersReadInLine(walker, typeInFront(view))) || testOfBodyHoldsPositionBelowLengthOfText(walker, index, member, access.objectOperand(), access.propertyOperand())) + bool isOfList = !fieldView && access.opcode() == Opcode::ElemGet && isListOnlyReadInLoop(view); + if ((!isOfList && !fieldView && !isViewOfNumbersReadInLine(walker, typeInFront(view))) || testOfBodyHoldsPositionBelowLengthOfText(walker, index, member, access.objectOperand(), access.propertyOperand())) continue; - found.accesses.append({ access.id(), view, addend, isOfList }); + bool turnAsksSignOfAddend = addend && !isOfList && testOfTurnHoldsNumberFromZeroUpInCountersOfLoops(index, inputHead, member, addend); + found.accesses.append({ access.id(), view, fieldView, addend, isOfList, turnAsksSignOfAddend }); } } return found; @@ -832,6 +1025,170 @@ std::optional findNumberThatDivisionsBringRound(W return found; } +static std::optional findLoopBoundRange(Walker& walker, const BodyIndex& index, ValueID firstValueOfLoop, ValueID written, unsigned operatorsLeft = 2) +{ + ValueID value = index.stripCasts(written); + if (!index.isParameter(value) && index.opcodeOf(value) == Opcode::Integer) + return Range::one(index.value(value).as().integer()); + if (value.index() < firstValueOfLoop.index()) { + Reference built = walker.referenceOf(value); + if (!built || !built.isRecord()) + return std::nullopt; + Type type = walker.types().at(walker.typeOf(built)); + return type.isInteger() ? std::optional { type.integers } : std::nullopt; + } + if (index.isParameter(value) || !operatorsLeft || (index.opcodeOf(value) != Opcode::Add && index.opcodeOf(value) != Opcode::Sub)) + return std::nullopt; + std::optional left = findLoopBoundRange(walker, index, firstValueOfLoop, index.value(value).operand(0), operatorsLeft - 1); + std::optional right = findLoopBoundRange(walker, index, firstValueOfLoop, index.value(value).operand(1), operatorsLeft - 1); + if (!left || !right || !left->isSubsetOf(Range::exact()) || !right->isSubsetOf(Range::exact())) + return std::nullopt; + if (index.opcodeOf(value) == Opcode::Add) + return Range { left->minimum + right->minimum, left->maximum + right->maximum }; + return Range { left->minimum - right->maximum, left->maximum - right->minimum }; +} + +static std::optional findConstantAddedToCounter(const BodyIndex& index, ValueID written, ValueID counter) +{ + ValueID value = index.stripCasts(written); + if (value == counter) + return 0; + if (index.isParameter(value)) + return std::nullopt; + BodyValue sum = index.value(value); + if (sum.opcode() == Opcode::Increment) + return index.stripCasts(sum.operand(0)) == counter ? std::optional { 1 } : std::nullopt; + if (sum.opcode() != Opcode::Add) + return std::nullopt; + bool counterIsFirst = index.stripCasts(sum.operand(0)) == counter; + ValueID addend = counterIsFirst || index.stripCasts(sum.operand(1)) == counter ? index.stripCasts(sum.operand(counterIsFirst ? 1 : 0)) : ValueID(); + if (!addend || index.isParameter(addend) || index.opcodeOf(addend) != Opcode::Integer) + return std::nullopt; + int64_t constant = index.value(addend).as().integer(); + return Range::one(constant).isSubsetOf(Range::int32()) ? std::optional { constant } : std::nullopt; +} + +std::optional findCounterRange(Walker& walker, BlockID inputHead, uint32_t position, int64_t start) +{ + const BodyIndex& index = walker.currentBodyIndex(); + BodyBlock head = index.block(inputHead); + BodyTerminator end = head.terminator(); + ValueID condition = end.kind() == TerminatorKind::Branch ? end.as().conditionOperand() : ValueID(); + if (!condition || index.isParameter(condition) || !head.holds(condition) || index.isEnteredByHandlerEdge(inputHead)) + return std::nullopt; + LoopID loop = index.loopOf(inputHead); + if (index.loop(loop).numberOfLoopsWithin != 1) + return std::nullopt; + if (!index.isInLoop(end.targets()[0].block(), loop) || index.isInLoop(end.targets()[1].block(), loop)) + return std::nullopt; + + ValueID counter = head.parameter(position); + Opcode opcode = index.opcodeOf(condition); + if (opcode != Opcode::Less && opcode != Opcode::LessEq && opcode != Opcode::Greater && opcode != Opcode::GreaterEq) + return std::nullopt; + bool counterIsFirst = opcode == Opcode::Less || opcode == Opcode::LessEq; + BodyValue relation = index.value(condition); + std::optional offset = findConstantAddedToCounter(index, relation.operand(counterIsFirst ? 0 : 1), counter); + if (!offset) + return std::nullopt; + int64_t step = 1; + for (const BodyIncomingEdge& edge : walker.incomingEdgesOf(index, inputHead)) { + if (!index.isInLoop(edge.from, loop)) + continue; + std::span arguments = edge.target.arguments(); + std::optional added = position < arguments.size() && arguments[position] ? findConstantAddedToCounter(index, arguments[position], counter) : std::nullopt; + if (!added || *added < 0) + return std::nullopt; + step = std::max(step, *added); + } + + ValueID written = relation.operand(counterIsFirst ? 1 : 0); + std::optional bound = findLoopBoundRange(walker, index, head.firstParameter(), written); + if (bool isStrided = *offset || step > 1; !bound && isStrided) { + ValueID read = index.stripCasts(written); + bool isLength = !index.isParameter(read) && index.opcodeOf(read) == Opcode::PropGet && !index.value(read).isFreeName() && index.value(read).key() == walker.program().keyOf(BuiltinNames::length); + if (isLength) + bound = Range { 0, maxTypedArrayLength }; + } + if (!bound || !bound->isSubsetOf(Range::exact())) + return std::nullopt; + int64_t last = bound->maximum - !(opcode == Opcode::LessEq || opcode == Opcode::GreaterEq) - *offset + step; + if (last <= start || !Range { start, last }.isSubsetOf(Range::exact())) + return std::nullopt; + return Range { start, last }; +} + +static constexpr OpcodeSet relationsOfCountersOfLoops { Opcode::Less, Opcode::LessEq, Opcode::Greater, Opcode::GreaterEq }; +static constexpr OpcodeSet arithmeticOfCountersOfLoops { Opcode::Add, Opcode::Sub, Opcode::Mul, Opcode::Increment, Opcode::Decrement }; + +static void collectMultipliedParameters(const BodyIndex& index, ValueID written, unsigned operatorsLeft, bool isMultiplied, Vector& parameters) +{ + ValueID value = index.stripCasts(written); + if (!value) + return; + if (index.isParameter(value)) { + if (isMultiplied && index.isLoopHead(index.blockOf(value)) && !parameters.contains(value)) + parameters.append(value); + return; + } + BodyValue made = index.value(value); + bool isArithmetic = arithmeticOfCountersOfLoops.contains(made.opcode()); + if (made.opcode() != Opcode::ToNumeric && (!isArithmetic || !operatorsLeft)) + return; + made.forEachOperand([&](ValueID operand, uint32_t) { + collectMultipliedParameters(index, operand, operatorsLeft - isArithmetic, isMultiplied || made.opcode() == Opcode::Mul, parameters); + }); +} + +std::span Walker::computedComparisonsOf(ValueID parameter) +{ + static constexpr unsigned mostOperators = 4; + static constexpr OpcodeSet holdsLeftBelow { Opcode::Less, Opcode::LessEq }; + const BodyIndex& index = currentBodyIndex(); + const Vector& listed = m_computedComparisons.ensure(&index, [&] { + Vector found; + if (!index.holdsAnyOf(relationsOfCountersOfLoops) || !index.holdsAnyOf(OpcodeSet { Opcode::Mul })) + return found; + Vector parameters; + for (uint32_t block = 0; block < index.numberOfBlocks(); ++block) { + if (!index.loopOf(BlockID(block))) + continue; + for (BodyValue relation : index.block(BlockID(block)).values()) { + if (!relationsOfCountersOfLoops.contains(relation.opcode())) + continue; + uint32_t below = holdsLeftBelow.contains(relation.opcode()) ? 0 : 1; + parameters.shrink(0); + collectMultipliedParameters(index, relation.operand(below), mostOperators, false, parameters); + for (ValueID multiplied : parameters) + found.append({ multiplied, relation.id(), index.stripCasts(relation.operand(1 - below)) }); + } + } + std::ranges::stable_sort(found, { }, &ComputedComparison::parameter); + return found; + }).iterator->value; + auto range = std::ranges::equal_range(listed, parameter, { }, &ComputedComparison::parameter); + return { range.begin(), range.end() }; +} + +bool isMultipliedBelowLength(Walker& walker, BlockID inputHead, uint32_t position) +{ + const BodyIndex& index = walker.currentBodyIndex(); + LoopID loop = index.loopOf(inputHead); + uint32_t firstValueOfLoop = index.block(inputHead).firstParameter().index(); + auto isLength = [&](ValueID bound) { + if (bound.index() >= firstValueOfLoop) + return !index.isParameter(bound) && index.opcodeOf(bound) == Opcode::PropGet && !index.value(bound).isFreeName() && index.value(bound).key() == walker.program().keyOf(BuiltinNames::length); + Reference built = walker.referenceOf(bound); + if (!built || !built.isRecord()) + return false; + Type type = walker.types().at(walker.typeOf(built)); + return type.isInteger() && type.integers.maximum <= Range::uint32().maximum; + }; + return std::ranges::any_of(walker.computedComparisonsOf(index.block(inputHead).parameter(position)), [&](const Walker::ComputedComparison& comparison) { + return index.isInLoop(index.blockOf(comparison.relation), loop) && isLength(comparison.bound); + }); +} + bool operatorsOnBitsAloneBringRound(Walker& walker, BlockID inputHead, uint32_t position, std::span entering) { const BodyIndex& index = walker.currentBodyIndex(); @@ -961,22 +1318,39 @@ static void appendNumbersThatDivisionsBringRoundInCountersOfLoops(Walker& walker } } +Vector findElementCarriedIndexes(Walker& walker, BlockID inputHead) +{ + Vector found; + const BodyIndex& index = walker.currentBodyIndex(); + if (!walker.has(Mechanism::TypedOperations) || !walker.has(Mechanism::UnboxedNumbers)) + return found; + for (uint32_t position = 0; position < index.block(inputHead).numberOfParameters(); ++position) { + NamesOfCounterOfLoop names; + Vector elements; + if (hasElementBackEdgeInput(walker, index, inputHead, position) && isElementCarriedIndex(walker, index, *usersOfValuesOfLoopInCountersOfLoops(walker, index, inputHead), inputHead, position, names, &elements)) + found.appendVector(elements); + } + return found; +} + BitVector namesOfCountersAskedForInt32InFrontOfLoop(Walker& walker, BlockID inputHead, std::span entering) { BitVector found; const BodyIndex& index = walker.currentBodyIndex(); - if (!walker.has(Mechanism::TypedOperations) || !walker.has(Mechanism::UnboxedNumbers) || !index.holdsAnyOf(stepsByOneOfCountersOfLoops | stepsByNumberOfCountersOfLoops)) + if (!walker.has(Mechanism::TypedOperations) || !walker.has(Mechanism::UnboxedNumbers) || !index.holdsAnyOf(stepsByOneOfCountersOfLoops | stepsByNumberOfCountersOfLoops | OpcodeSet { Opcode::ElemGet })) return found; std::shared_ptr users; uint32_t firstValueOfLoop = index.block(inputHead).firstParameter().index(); uint32_t numberOfParameters = index.block(inputHead).numberOfParameters(); for (uint32_t position = 0; position < numberOfParameters && position < entering.size(); ++position) { - if (!isBoxOrDoubleThatMayBeInt32InCountersOfLoops(walker, entering[position]) || std::ranges::find(entering.first(position), entering[position]) != entering.first(position).end()) + bool entersAsInt32 = entering[position] && walker.types().at(walker.typeOf(entering[position])).isInt32(); + bool bringsElement = hasElementBackEdgeInput(walker, index, inputHead, position); + if (entersAsInt32 ? !bringsElement : !isBoxOrDoubleThatMayBeInt32InCountersOfLoops(walker, entering[position]) || std::ranges::find(entering.first(position), entering[position]) != entering.first(position).end()) continue; if (!users) users = usersOfValuesOfLoopInCountersOfLoops(walker, index, inputHead); NamesOfCounterOfLoop names; - if (!isSteppedAllRoundInCountersOfLoops(walker, index, *users, inputHead, position, names)) + if (bringsElement ? !isElementCarriedIndex(walker, index, *users, inputHead, position, names) : !isSteppedAllRoundInCountersOfLoops(walker, index, *users, inputHead, position, names)) continue; for (ValueID name : names) found.set(name.index() - firstValueOfLoop); @@ -984,6 +1358,87 @@ BitVector namesOfCountersAskedForInt32InFrontOfLoop(Walker& walker, BlockID inpu return found; } +static bool isComparedWithPossibleInt32(Walker& walker, const BodyIndex& index, BlockID inputHead, ValueID counter) +{ + BodyBlock head = index.block(inputHead); + BodyTerminator end = head.terminator(); + ValueID condition = end.kind() == TerminatorKind::Branch ? end.as().conditionOperand() : ValueID(); + if (!condition || index.isParameter(condition) || !head.holds(condition) || !allRelations.contains(index.opcodeOf(condition))) + return false; + BodyValue relation = index.value(condition); + if (index.stripCasts(relation.operand(0)) != counter) + return false; + ValueID bound = index.stripCasts(relation.operand(1)); + return bound.index() < head.firstParameter().index() && isBoxOrDoubleThatMayBeInt32InCountersOfLoops(walker, walker.referenceOf(bound)); +} + +static bool accessesViewUnderCounter(Walker& walker, const BodyIndex& index, const UsersOfValuesInCountersOfLoops& users, BlockID inputHead, const NamesOfCounterOfLoop& names) +{ + UsersOfNamesInCountersOfLoops usersLeft(users, names); + while (ValueID user = usersLeft.next()) { + if (index.isParameter(user) || !accessesUnderKeysInCountersOfLoops.contains(index.opcodeOf(user)) || !names.contains(index.value(user).propertyOperand())) + continue; + BodyValue access = index.value(user); + ValueID object = index.stripCasts(access.objectOperand()); + bool isBeforeLoop = object.index() < index.block(inputHead).firstParameter().index(); + Reference view = isBeforeLoop ? walker.referenceOf(object) : index.loopOf(index.blockOf(user)) == index.loopOf(inputHead) ? nameOfListThatEntersLoopAround(walker, index, access) : Reference(); + if (view && view.isValue() && isViewOfNumbersReadInLine(walker, walker.typeOf(view))) + return true; + } + return false; +} + +static Vector readsOffListsFoundInTurnInCountersOfLoops(Walker& walker, const BodyIndex& index, BlockID inputHead, const LoopPreview& preview) +{ + TypeArena& types = walker.types(); + Vector reads; + for (BlockID member : index.membersOf(index.loopOf(inputHead))) { + for (BodyValue read : index.block(member).values()) { + if (read.opcode() != Opcode::ElemGet) + continue; + ValueID list = index.stripCasts(read.objectOperand()); + if (!list || list.index() < preview.firstValue || index.isParameter(list)) + continue; + Opcode finds = index.opcodeOf(list); + if (finds != Opcode::PrivateGet && finds != Opcode::ElemGet && (finds != Opcode::PropGet || index.value(list).isFreeName())) + continue; + TypeID held = typeByLookInCountersOfLoops(walker, index, preview, list); + const ClassPart* onlyClass = held && !isNeverType(held) && types.at(types.withoutNullish(held)).isObject() ? singleBuiltinClassOf(types, types.at(types.withoutNullish(held))) : nullptr; + if (!onlyClass || walker.program().builtins().typedArrayTypeOf(onlyClass->builtinClass) == NotTypedArray) + reads.append(read.id()); + } + } + return reads; +} + +std::optional findCounterOfAnyKind(Walker& walker, BlockID inputHead, const LoopPreview& preview, std::span entering) +{ + const BodyIndex& index = walker.currentBodyIndex(); + if (!walker.has(Mechanism::TypedOperations) || !walker.has(Mechanism::UnboxedNumbers) || !index.opcodes().contains(Opcode::ElemGet) || !index.holdsAnyOf(stepsByOneOfCountersOfLoops | stepsByNumberOfCountersOfLoops)) + return std::nullopt; + std::optional> reads; + std::shared_ptr users; + uint32_t numberOfParameters = index.block(inputHead).numberOfParameters(); + for (uint32_t position = 0; position < numberOfParameters && position < entering.size(); ++position) { + Reference enters = entering[position]; + if (!isBoxOrDoubleThatMayBeInt32InCountersOfLoops(walker, enters) || walker.types().at(walker.typeOf(enters)).isNumber()) + continue; + if (!reads) + reads = readsOffListsFoundInTurnInCountersOfLoops(walker, index, inputHead, preview); + if (reads->isEmpty()) + return std::nullopt; + if (!users) + users = usersOfValuesOfLoopInCountersOfLoops(walker, index, inputHead); + NamesOfCounterOfLoop names; + bool isKey = isSteppedAllRoundInCountersOfLoops(walker, index, *users, inputHead, position, names) && std::ranges::any_of(*reads, [&](ValueID read) { + return names.contains(index.value(read).propertyOperand()); + }); + if (isKey) + return HoistedCounterCheck { position, isIncreasingKey(index, *users, names) ? walker.types().integer(Range { 0, std::numeric_limits::max() }) : TypeID(TypeTable::int32) }; + } + return std::nullopt; +} + Vector findHoistedCounterChecks(Walker& walker, BlockID inputHead, const LoopPreview& preview, std::span entering, Vector& guesses) { Vector counters; @@ -991,7 +1446,8 @@ Vector findHoistedCounterChecks(Walker& walker, BlockID return counters; const BodyIndex& index = walker.currentBodyIndex(); bool holdsOperatorsOnBits = index.holdsAnyOf(operatorsOnBitsOfCountersOfLoops | OpcodeSet { Opcode::URShift }); - if (!holdsOperatorsOnBits && !index.holdsAnyOf(stepsByOneOfCountersOfLoops | stepsByNumberOfCountersOfLoops | divisionsOfCountersOfLoops)) + bool holdsElementReads = index.holdsAnyOf(OpcodeSet { Opcode::ElemGet }); + if (!holdsOperatorsOnBits && !holdsElementReads && !index.holdsAnyOf(stepsByOneOfCountersOfLoops | stepsByNumberOfCountersOfLoops | divisionsOfCountersOfLoops)) return counters; bool holdsAccessesUnderKeys = index.holdsAnyOf(accessesUnderKeysInCountersOfLoops) && index.holdsAnyOf(stepsByNumberOfCountersOfLoops); uint32_t numberOfParameters = index.block(inputHead).numberOfParameters(); @@ -1003,15 +1459,34 @@ Vector findHoistedCounterChecks(Walker& walker, BlockID usersOnceAsked = usersOfValuesOfLoopInCountersOfLoops(walker, index, inputHead); return *usersOnceAsked; }; + NamesBeforeLoop namesBeforeLoop(walker, inputHead); for (uint32_t position = 0; position < numberOfParameters && position < entering.size(); ++position) { Reference enters = entering[position]; - if (enters && walker.types().at(walker.typeOf(enters)).isInt32() && (!guesses.isEmpty() || holdsAccessesUnderKeys)) { + bool entersAsInt32 = enters && walker.types().at(walker.typeOf(enters)).isInt32(); + TypeID inTurns = position < preview.typesOfParameters.size() ? preview.typesOfParameters[position] : TypeID(); + if (holdsElementReads && inTurns && !isNeverType(inTurns) && !walker.types().at(inTurns).isInteger() && (walker.types().at(inTurns).primitives & Primitive::Integer) && (entersAsInt32 || isBoxOrDoubleThatMayBeInt32InCountersOfLoops(walker, enters)) && hasElementBackEdgeInput(walker, index, inputHead, position)) { + bool isTested = !entersAsInt32 && (std::ranges::find(entering.first(position), enters) != entering.first(position).end() || guesses.containsIf([&](const HoistedSpeculation& guess) { + return guess.value == enters; + })); + NamesOfCounterOfLoop names; + if (!isTested && isElementCarriedIndex(walker, index, users(), inputHead, position, names)) { + counters.append({ position, TypeTable::int32, true }); + walker.count(StatisticID::CarriedIndexGuardsHoisted); + continue; + } + } + if (entersAsInt32 && index.holdsAnyOf(accessesUnderKeysInCountersOfLoops) && isComparedWithPossibleInt32(walker, index, inputHead, index.block(inputHead).parameter(position))) { + NamesOfCounterOfLoop names; + if (isSteppedAllRoundInCountersOfLoops(walker, index, users(), inputHead, position, names) && accessesViewUnderCounter(walker, index, users(), inputHead, names)) + guessBoundsInCountersOfLoops(walker, index, users(), inputHead, names, namesBeforeLoop, entering, guesses); + } + if (entersAsInt32 && (!guesses.isEmpty() || holdsAccessesUnderKeys)) { NamesOfCounterOfLoop names; if (isSteppedAllRoundInCountersOfLoops(walker, index, users(), inputHead, position, names)) { if (holdsAccessesUnderKeys) - guessAddendsOfKeysInCountersOfLoops(walker, index, users(), inputHead, names, entering, guesses); + guessAddendsOfKeysInCountersOfLoops(walker, index, users(), inputHead, names, namesBeforeLoop, entering, guesses); if (!guesses.isEmpty()) - guessBoundsInCountersOfLoops(walker, index, users(), inputHead, names, entering, guesses); + guessBoundsInCountersOfLoops(walker, index, users(), inputHead, names, namesBeforeLoop, entering, guesses); } TypeID byLook = position < preview.typesOfParameters.size() ? preview.typesOfParameters[position] : TypeID(); if (byLook && !isNeverType(byLook) && !walker.types().at(byLook).isInteger() && walker.types().at(byLook).isNumberWithoutFraction()) { @@ -1064,12 +1539,18 @@ Vector findHoistedCounterChecks(Walker& walker, BlockID } if ((isCheckedOnBackEdge || walker.types().at(walker.typeOf(enters)).isNumber()) && !turnsTakeCounterAsIntegerInCountersOfLoops(index, users(), names)) continue; + if (!isCheckedOnBackEdge && !walker.types().at(walker.typeOf(enters)).isNumber() && isUsedInArithmeticAlone(walker, index, users(), inputHead, preview, names)) { + counters.append({ position, TypeTable::number }); + walker.count(StatisticID::InvariantNumberGuardsHoisted); + continue; + } bool stepEnters = std::ranges::any_of(stepsAskedInFront, [&](ValueID step) { return std::ranges::find(entering, walker.referenceOf(step)) != entering.end(); }); if (stepEnters) continue; - counters.append({ position, TypeTable::int32, isCheckedOnBackEdge }); + bool isFromZeroUp = !isCheckedOnBackEdge && (isIncreasingKey(index, users(), names) || isTruthAskedAtHead(index, inputHead, position)); + counters.append({ position, isFromZeroUp ? walker.types().integer(Range { 0, std::numeric_limits::max() }) : TypeID(TypeTable::int32), isCheckedOnBackEdge }); bool loopInsideStepsIt = std::ranges::any_of(names, [&](ValueID name) { return index.isParameter(name) && index.blockOf(name) != inputHead && index.isLoopHead(index.blockOf(name)); }); @@ -1079,17 +1560,17 @@ Vector findHoistedCounterChecks(Walker& walker, BlockID Reference value = walker.referenceOf(step); if (guesses.containsIf([&](const HoistedSpeculation& guess) { return guess.value == value; })) continue; - guesses.append(HoistedSpeculation { value, TypeTable::int32, { }, namesInFrontThatLoopMentions(walker, index, index.loopOf(inputHead), index.block(inputHead).firstParameter().index(), value) }); + guesses.append(HoistedSpeculation { value, TypeTable::int32, { }, namesBeforeLoop.findNamesOf(value) }); walker.count(StatisticID::InvariantNumberGuardsHoisted); } - guessBoundsInCountersOfLoops(walker, index, users(), inputHead, names, entering, guesses); + guessBoundsInCountersOfLoops(walker, index, users(), inputHead, names, namesBeforeLoop, entering, guesses); if (holdsAccessesUnderKeys) - guessAddendsOfKeysInCountersOfLoops(walker, index, users(), inputHead, names, entering, guesses); + guessAddendsOfKeysInCountersOfLoops(walker, index, users(), inputHead, names, namesBeforeLoop, entering, guesses); } if (index.holdsAnyOf(divisionsOfCountersOfLoops)) appendNumbersThatDivisionsBringRoundInCountersOfLoops(walker, index, inputHead, preview, entering, guesses, counters); if (index.holdsAnyOf(stepsByNumberOfCountersOfLoops)) - guessAddendsOfSumsCutToInt32InCountersOfLoops(walker, index, inputHead, entering, guesses); + guessAddendsOfSumsCutToInt32InCountersOfLoops(walker, index, inputHead, namesBeforeLoop, entering, guesses); for (uint32_t position = 0; !walker.sumsAreNotAskedInRow() && position < numberOfParameters && position < entering.size() && position < preview.typesOfParameters.size(); ++position) { Reference enters = entering[position]; if (!isNumberCarriedBesideTextOrBigInt(walker, enters, preview.typesOfParameters[position])) @@ -1114,6 +1595,7 @@ void guessDivisorsOfIntegersInFrontOfLoop(Walker& walker, BlockID inputHead, con LoopID loop = index.loopOf(inputHead); uint32_t firstValueOfLoop = index.block(inputHead).firstParameter().index(); TypeID positiveInt32 = types.integer(Range { 1, std::numeric_limits::max() }); + NamesBeforeLoop namesBeforeLoop(walker, inputHead); for (BlockID member : index.membersOf(loop)) { for (BodyValue operation : index.block(member).values()) { if (operation.opcode() != Opcode::Mod) @@ -1141,7 +1623,7 @@ void guessDivisorsOfIntegersInFrontOfLoop(Walker& walker, BlockID inputHead, con return guess.value == value; }); if (listed == notFound) { - guesses.append(HoistedSpeculation { value, positiveInt32, { }, namesInFrontThatLoopMentions(walker, index, loop, firstValueOfLoop, value) }); + guesses.append(HoistedSpeculation { value, positiveInt32, { }, namesBeforeLoop.findNamesOf(value) }); walker.count(StatisticID::InvariantNumberGuardsHoisted); } else if (types.isSubset(positiveInt32, guesses[listed].tested)) guesses[listed].tested = positiveInt32; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkCountersOfLoops.h b/Source/JavaScriptCore/aot/walk/AOTWalkCountersOfLoops.h index 8fb3b70ecf30d..a6fce576f21f5 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkCountersOfLoops.h +++ b/Source/JavaScriptCore/aot/walk/AOTWalkCountersOfLoops.h @@ -22,8 +22,10 @@ struct HoistedCounterCheck { struct AccessUnderCounterOfLoop { ValueID access; ValueID view; + Reference fieldView; ValueID addend; bool isOfList { false }; + bool turnAsksSignOfAddend { false }; }; struct CounterHeldBelowRoomOfViews { @@ -36,8 +38,10 @@ struct CounterHeldBelowRoomOfViews { }; CounterHeldBelowRoomOfViews findCounterHeldBelowRoomOfViews(Walker&, BlockID inputHead, NOESCAPE const ScopedLambda& typeThatEnters, NOESCAPE const ScopedLambda& typeInFront, NOESCAPE const ScopedLambda& isListOnlyReadInLoop); +Vector findElementCarriedIndexes(Walker&, BlockID inputHead); BitVector namesOfCountersAskedForInt32InFrontOfLoop(Walker&, BlockID inputHead, std::span entering); Vector findHoistedCounterChecks(Walker&, BlockID inputHead, const LoopPreview&, std::span entering, Vector& guesses); +std::optional findCounterOfAnyKind(Walker&, BlockID inputHead, const LoopPreview&, std::span entering); bool isNumberCarriedBesideTextOrBigInt(const Walker&, Reference enters, TypeID carried); struct HoistedVariableCheck { @@ -55,6 +59,8 @@ struct NumberThatDivisionsBringRound { }; std::optional findNumberThatDivisionsBringRound(Walker&, BlockID inputHead, uint32_t position, std::span entering, WhatEntersHead); bool operatorsOnBitsAloneBringRound(Walker&, BlockID inputHead, uint32_t position, std::span entering); +std::optional findCounterRange(Walker&, BlockID inputHead, uint32_t position, int64_t start); +bool isMultipliedBelowLength(Walker&, BlockID inputHead, uint32_t position); struct LoopBoundComparison { ValueID relation; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkEnvironmentCells.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkEnvironmentCells.cpp index deab684ef9226..0f916789a8726 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkEnvironmentCells.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkEnvironmentCells.cpp @@ -256,22 +256,34 @@ void Walker::findFieldsOfScopesThatMeet(std::span states if (states.size() < 2) return; Vector forms; - auto comesRoundOpenLoop = [&](Reference environment, FieldReference place) { - for (const OpenLoop& loop : m_openLoops) { - auto fields = m_fieldsOfScopesOfHeads.find(loop.head.index()); - if (fields != m_fieldsOfScopesOfHeads.end() && fields->value.containsIf([&](const FieldOfScope& field) { return field.scope == environment && field.field.layout == place.layout && field.field.field == place.field; })) - return true; + Vector*, 4> fieldsOfOpenLoops; + for (const OpenLoop& loop : m_openLoops) { + if (auto fields = m_fieldsOfScopesOfHeads.find(loop.head.index()); fields != m_fieldsOfScopesOfHeads.end()) + fieldsOfOpenLoops.append(&fields->value); + } + if (fieldsOfOpenLoops.isEmpty() && !unitHasMadeScopes()) + return; + auto findCarriedField = [&](Reference holder, FieldReference place, bool isSlot) -> const FieldOfScope* { + for (const Vector* fields : fieldsOfOpenLoops) { + size_t found = fields->findIf([&](const FieldOfScope& field) { return field.scope == holder && !!field.slotKey == isSlot && field.field.layout == place.layout && field.field.field == place.field; }); + if (found != notFound) + return &fields->at(found); } - return false; + return nullptr; }; - states[0]->forEachEnvironmentFieldNotSharedWithAll(states, [&](Reference environment, FieldReference place, Reference first) { - if (!isEnvironmentAllocation(environment) && !comesRoundOpenLoop(environment, place)) + states[0]->forEachFieldNotSharedWithAll(states, [&](Reference holder, FieldReference place, Reference first, bool isSlot) { + const FieldOfScope* carried = isSlot || !isEnvironmentAllocation(holder) ? findCarriedField(holder, place, isSlot) : nullptr; + if (!carried && (isSlot || !isEnvironmentAllocation(holder))) return; - FieldOfScope field { environment, place, TypeID(), Representation::JSValue, { }, { }, { } }; + FieldOfScope field { holder, place, TypeID(), Representation::JSValue, { }, { }, { } }; + if (isSlot) { + field.slotKey = carried->slotKey; + field.receivers = carried->receivers; + } bool allBringOneValue = true; forms.shrink(0); for (const WalkState* state : states) { - Reference value = state->environmentField(environment, place); + Reference value = isSlot ? state->slot(holder, place) : state->environmentField(holder, place); TypeID type = ownTypeOf(value); if (!type || (m_types.at(type).primitives & Primitive::Absent) || representationOf(value) == Representation::Void || namesFreshCell(*state, value) || isClosureLeftUnmade(*this, value)) return; @@ -282,6 +294,8 @@ void Walker::findFieldsOfScopesThatMeet(std::span states } if (allBringOneValue) return; + if (isSlot) + count(StatisticID::CarriedSlotsMergedAtJoins); field.form = formOfParameter(field.type, forms.span()); found.append(WTF::move(field)); }); @@ -298,13 +312,235 @@ void Walker::findFieldsOfScopesInFrontOfLoop(const LoopPreview& look, Vector& known, bool shouldReadBeforeHeader) +void Walker::noteSlotRead(Reference object, PropertyKeyID key, Reference read) +{ + TypeID type = m_trialVisits.isEmpty() || !object.isValue() ? TypeID() : ownTypeOf(read); + if (!type || !canCarrySlotType(type)) + return; + ValueID cell = underlyingCellOf(m_code, object.id()); + size_t openLoop = m_openLoops.size(); + for (size_t place = m_trialVisits.size(); place--;) { + TrialVisit& visit = m_trialVisits[place]; + if (cell.index() >= visit.firstRecord) + return; + auto isVisited = [&](const OpenLoop& loop) { + return loop.frame == visit.frameIndex && m_frames[loop.frame].index->loop(loop.loop).head == visit.inputHead; + }; + while (openLoop && !isVisited(m_openLoops[openLoop - 1])) + --openLoop; + if (!openLoop) + return; + if (executesOnEveryIteration(openLoop - 1, currentHoistPoint())) + visit.slotAccesses.append(SlotAccess { cell, key, underlyingCellOf(m_code, read.id()), { } }); + } +} + +static constexpr uint32_t weightOfInt32Position = 2; +static constexpr uint32_t weightTakenByMethods = 1 << 16; + +void Walker::notePositions(const BodyValue& operation) +{ + const BodyIndex& index = currentBodyIndex(); + Vector, 4>& positions = m_trialVisits.last().positions; + auto note = [&](ValueID operand, uint32_t weight) { + Reference position = operand ? referenceOf(index.stripCasts(operand)) : Reference(); + TypeID type = position && position.isValue() && !m_code.isConstant(position.id()) ? ownTypeOf(position) : TypeID(); + if (!type || !m_types.at(type).isNumber()) + return; + ValueID taken = underlyingCellOf(m_code, position.id()); + positions.append({ taken, weight }); + const Operation& made = m_code.at(taken); + static constexpr OpcodeSet stepsAndSums { Opcode::Increment, Opcode::Decrement, Opcode::Add, Opcode::Sub }; + if (!stepsAndSums.contains(made.opcode)) + return; + for (ValueID term : made.operands()) + positions.append({ underlyingCellOf(m_code, term), weight }); + }; + if (operation.opcode() != Opcode::Call) { + Reference object = referenceOf(operation.objectOperand()); + TypeID type = object && object.isRecord() ? typeOf(object) : TypeID(); + return note(operation.propertyOperand(), type && isViewOfNumbersReadInLine(*this, type) ? 1 : weightOfInt32Position); + } + ValueID callee = index.stripCasts(operation.operand(OperandIndex::Call::callee)); + if (index.isParameter(callee) || index.opcodeOf(callee) != Opcode::PropGet) + return; + for (ValueID argument : operation.variadicOperands()) + note(argument, weightTakenByMethods); +} + +void Walker::noteSlotStore(Reference object, PropertyKeyID key, Reference before, Reference stored) +{ + TypeID type = m_trialVisits.isEmpty() || !object.isValue() || !stored.isValue() ? TypeID() : ownTypeOf(stored); + if (!type) + return; + ValueID cell = underlyingCellOf(m_code, object.id()); + auto isInt32 = [&](TypeID asked) { + return asked && m_types.isSubset(asked, TypeTable::int32); + }; + auto isOwnValue = [&](ValueID operand) { + return before && before.isValue() && underlyingCellOf(m_code, operand) == underlyingCellOf(m_code, before.id()); + }; + auto isSumWithInt32 = [&] { + if (!stored.isValue() || m_code.isConstant(stored.id())) + return false; + const Operation& sum = m_code.at(underlyingCellOf(m_code, stored.id())); + if (sum.opcode == Opcode::Increment || sum.opcode == Opcode::Decrement) + return isOwnValue(sum.operand(0)); + if (sum.opcode != Opcode::Add && sum.opcode != Opcode::Sub) + return false; + return (isOwnValue(sum.operand(0)) && isInt32(typeOf(Reference::value(sum.operand(1))))) || (sum.opcode == Opcode::Add && isOwnValue(sum.operand(1)) && isInt32(typeOf(Reference::value(sum.operand(0))))); + }; + auto isStep = [&](std::initializer_list steps) { + if (m_code.isConstant(stored.id())) + return false; + const Operation& step = m_code.at(underlyingCellOf(m_code, stored.id())); + return std::ranges::contains(steps, step.opcode) && isOwnValue(step.operand(0)); + }; + std::optional staysInt32; + bool isStepOfOne = isStep({ Opcode::Increment, Opcode::Decrement }); + bool isStepUp = isStep({ Opcode::Increment }); + for (size_t place = m_trialVisits.size(); place--;) { + TrialVisit& visit = m_trialVisits[place]; + if (cell.index() >= visit.firstRecord) + return; + if (!staysInt32) + staysInt32 = isInt32(type) || isSumWithInt32(); + visit.slotAccesses.append(SlotAccess { cell, key, before && before.isValue() ? underlyingCellOf(m_code, before.id()) : ValueID(), underlyingCellOf(m_code, stored.id()), *staysInt32, isStepOfOne, isStepUp }); + } +} + +bool Walker::learnSlotsOfLoop(LoopSummary& summary, TrialVisit& visit) +{ + Vector& accesses = visit.slotAccesses; + if (accesses.isEmpty()) + return false; + auto keyOf = [](const SlotAccess& access) { + return std::tuple { access.object, access.key.bits(), !!access.stored }; + }; + std::ranges::sort(accesses, { }, keyOf); + std::ranges::sort(visit.positions); + auto weightOfPosition = [&](ValueID value) { + uint32_t weight = 0; + for (const auto& [position, weightOfUse] : std::ranges::equal_range(visit.positions, value, { }, &std::pair::first)) + weight += weightOfUse; + return weight; + }; + Vector& slots = summary.storesIntoScopes.slots; + const BodyIndex& index = *m_frames[visit.frameIndex].index; + uint32_t firstValueOfLoop = index.block(visit.inputHead).firstParameter().index(); + HashMap, WTF::UnsignedWithZeroKeyHashTraits> holders; + bool hasNamedHolders = false; + auto typeOfHolder = [&](ValueID holder) { + return typeOf(m_frames[visit.frameIndex].references[holder.index()]); + }; + auto isObjectAlone = [&](ValueID holder) { + TypeID type = typeOfHolder(holder); + return type && !m_types.at(type).primitives; + }; + auto nameHolders = [&] { + if (std::exchange(hasNamedHolders, true)) + return; + for (BlockID member : index.membersOf(index.loopOf(visit.inputHead))) { + for (BodyValue operation : index.block(member).values()) { + operation.forEachOperand([&](ValueID operand, uint32_t) { + ValueID holder = operand ? index.stripCasts(operand) : ValueID(); + Reference named = holder && holder.index() < firstValueOfLoop ? m_frames[visit.frameIndex].references[holder.index()] : Reference(); + if (!named || !named.isValue()) + return; + auto added = holders.add(underlyingCellOf(m_code, named.id()).index(), holder); + if (!added.isNewEntry && !isObjectAlone(added.iterator->value) && isObjectAlone(holder)) + added.iterator->value = holder; + }); + } + } + }; + bool hasLearnedMore = false; + for (size_t first = 0; first < accesses.size();) { + size_t end = first; + bool isStored = false; + Vector values; + bool staysInt32 = true; + bool isSteppedByOne = true; + bool onlyGrows = true; + while (end < accesses.size() && accesses[end].object == accesses[first].object && accesses[end].key == accesses[first].key) { + isStored |= !!accesses[end].stored; + for (ValueID value : { accesses[end].before, accesses[end].stored }) { + if (value) + values.append(value); + } + staysInt32 &= accesses[end].staysInt32; + isSteppedByOne &= accesses[end].isStep; + onlyGrows &= accesses[end].isStepUp; + ++end; + } + SlotAccess access = accesses[first]; + bool isRead = !access.stored; + first = end; + std::ranges::sort(values); + values.shrink(std::ranges::unique(values).begin() - values.begin()); + uint32_t weight = 0; + for (ValueID value : values) + weight += weightOfPosition(value); + bool isInt32Position = staysInt32 && weight >= weightOfInt32Position; + if (!isStored) + continue; + nameHolders(); + auto found = holders.find(access.object.index()); + if (found == holders.end()) + continue; + auto place = std::ranges::lower_bound(slots, std::pair { found->value, access.key.bits() }, { }, &SlotOfLoop::order); + if (place != slots.end() && place->holder == found->value && place->key == access.key) { + place->isInt32Position = isInt32Position; + place->isPosition = weight; + place->isSteppedByOne = isSteppedByOne && weight < weightTakenByMethods; + place->onlyGrows = onlyGrows; + continue; + } + if (!isRead || !findFilledSlot(*this, typeOfHolder(found->value), access.key)) + continue; + slots.insert(place - slots.begin(), SlotOfLoop { found->value, access.key, isInt32Position, !!weight, false, isSteppedByOne && weight < weightTakenByMethods, onlyGrows }); + hasLearnedMore = true; + } + return hasLearnedMore; +} + +void Walker::findSlotsOfLoop(std::span slots, Vector& known, bool shouldReadBeforeHeader) +{ + for (const SlotOfLoop& slot : slots) { + Reference object = referenceOf(slot.holder); + std::optional filled = object && object.isRecord() ? findFilledSlot(*this, typeOf(object), slot.key) : std::nullopt; + if (!filled) + continue; + FieldOfScope found { object, filled->place, TypeID(), Representation::JSValue, { }, { }, { } }; + found.slotKey = slot.key; + found.receivers = filled->receivers; + found.slotIsInt32Position = slot.isInt32Position; + found.slotHolder = slot.holder; + found.slotIsSteppedByOne = slot.isSteppedByOne; + found.slotOnlyGrows = slot.onlyGrows; + found.mayStartAsInteger = slot.isPosition && !slot.hasLeftIntegers; + TypeID read = resultTypeOf(draftVariableRead(found)); + if (!read || !canCarrySlotType(read)) + continue; + Reference value = valueKnownOf(found); + if (!value && shouldReadBeforeHeader) + value = build(draftReadBeforeHeader(found, ReadBeforeHeaderUse::LoopCarriedValue)); + if (shouldReadBeforeHeader && isUnreachable()) + return; + found.values.append(value); + if (value || !shouldReadBeforeHeader) + known.append(WTF::move(found)); + } +} + +void Walker::findFieldsOfScopesFromOutsideThatQuietTurnsStore(BlockID inputHead, const Effects& ofTurns, std::span slots, Vector& known, bool shouldReadBeforeHeader) { bool isTypedBuildWithCalls = !m_versionedLoops.isEmpty() && m_versionedLoops.last().turnsThatRunOtherCodeComeRound && m_versionedLoops.last().frameIndex == m_frames.size() - 1 && m_versionedLoops.last().inputHead == inputHead; if (!has(Mechanism::LoopInvariants) || !has(Mechanism::Folding) || m_coldCopyDepth || (ofTurns.mayRunOtherCode() && !isTypedBuildWithCalls) || ofTurns.maySuspend || (shouldReadBeforeHeader && isUnreachable())) return; const BodyIndex& index = currentBodyIndex(); - if (!index.opcodes().contains(Opcode::PutEnvironmentField) && !index.opcodes().contains(Opcode::PutModuleVariable)) + findSlotsOfLoop(slots, known, shouldReadBeforeHeader); + if ((!index.opcodes().contains(Opcode::PutEnvironmentField) && !index.opcodes().contains(Opcode::PutModuleVariable)) || (shouldReadBeforeHeader && isUnreachable())) return; uint32_t firstValueOfLoop = index.block(inputHead).firstParameter().index(); LoopID loop = index.loopOf(inputHead); @@ -421,7 +657,7 @@ void Walker::findFieldsOfScopesFromOutsideThatQuietTurnsStore(BlockID inputHead, readTakenBySum = readOfPlaceTakenAlone(index.value(stored).operand(1)); } size_t listed = known.findIf([&](const FieldOfScope& field) { - return field.variable == variable && field.scope == scope && field.field.layout == place.layout && field.field.field == place.field; + return field.variable == variable && !field.slotKey && field.scope == scope && field.field.layout == place.layout && field.field.field == place.field; }); if (listed != notFound) { known[listed].isSteppedByOne &= isStep; @@ -504,7 +740,7 @@ TypeID Walker::backEdgeTypeOfVariable(BlockID inputHead, const FieldOfScope& fie if (versionedLoop.frameIndex != m_frames.size() - 1 || versionedLoop.inputHead != inputHead || !versionedLoop.entry) return { }; for (const FieldOfScope& held : versionedLoop.variableBackEdgeChecks) { - if (held.variable == field.variable && held.scope == field.scope && held.field.layout == field.field.layout && held.field.field == field.field.field) + if (held.variable == field.variable && held.slotKey == field.slotKey && held.scope == field.scope && held.field.layout == field.field.layout && held.field.field == field.field.field) return held.type; } return { }; @@ -514,12 +750,23 @@ bool Walker::mayBeInDeadZone(const FieldOfScope& field) const { if (field.variable) return program().at(field.variable).initialization == VariableInitialization::Uninitialized && !m_state.isPastDeadZone(field.variable); + if (field.slotKey) + return false; TypeID written = program().at(field.field.layout).fields[field.field.field].declared.written(); return written && (program().at(written).primitives & Primitive::Absent) && !m_state.isPastDeadZone(field.scope, field.field); } Draft Walker::draftVariableRead(const FieldOfScope& field) { + if (field.slotKey) { + Draft read(Opcode::GetSlot, m_origin); + chooseConservativeFields(read); + read.setObjectOperand(field.scope); + read.setKey(field.slotKey); + read.setReceivers(field.receivers); + read.setLevel(KnowledgeLevel::Proven); + return read; + } if (!field.variable) return draftCapturedVariableRead(field.scope, field.field); Draft read(Opcode::GetModuleVariable, m_origin); @@ -532,6 +779,8 @@ Draft Walker::draftVariableRead(const FieldOfScope& field) Draft Walker::draftReadBeforeHeader(const FieldOfScope& field, ReadBeforeHeaderUse use, bool moduleMayBeUnevaluated) { Draft read = draftVariableRead(field); + if (field.slotKey) + return read; if (!field.variable) { if (use == ReadBeforeHeaderUse::GuardOperand && !shouldAssumeSlotsInitialized()) read.setIsKnownInitialized(false); @@ -578,7 +827,7 @@ Reference Walker::fitForParameterOfField(const FieldOfScope& field, Reference kn bool Walker::mayHandFieldOfScopeToBlock(FieldOfScope& field, BlockID inputHeadOfLoopEntered) { Reference entering = field.values[0]; - if (!field.variable) { + if (!field.variable && !field.slotKey) { const Vector& fields = program().at(field.field.layout).fields; if (field.field.field >= fields.size() || fields[field.field.field].declared.isEstablished()) return false; @@ -591,9 +840,13 @@ bool Walker::mayHandFieldOfScopeToBlock(FieldOfScope& field, BlockID inputHeadOf if (TypeID held = backEdgeTypeOfVariable(inputHeadOfLoopEntered, field), own = ownTypeOf(entering); held && own && m_types.isSubset(own, held) && !m_types.isSubset(field.type, held)) { field.type = held; field.isHeld = true; + } else if (TypeID integer = m_types.integer(Range::exact()); field.mayStartAsInteger && inputHeadOfLoopEntered && own && hasOpenTrialVisit(m_frames.size() - 1, inputHeadOfLoopEntered) + && m_types.isSubset(own, integer) && m_types.isSubset(integer, field.type) && !m_types.isSubset(field.type, integer)) { + field.type = integer; + field.startsAsInteger = true; } field.form = formOfParameter(field.type, { }); - if ((field.form == Representation::JSValue || optionalRepresentations.contains(field.form)) && isEnforced(ProofSource::SlotOfScope)) + if (!field.slotKey && (field.form == Representation::JSValue || optionalRepresentations.contains(field.form)) && isEnforced(ProofSource::SlotOfScope)) field.shown = typeThatUnitMayTestFor(m_types.testablePartOf(field.type), true, true); return isFitForParameterOfField(field, entering); } @@ -613,7 +866,7 @@ void Walker::keepFieldsOfScopesThatLoopWrites(BlockID inputHead, Vectorvalue) { Reference value = fitForParameterOfField(field, valueKnownOf(field)); + if (TrialVisit* visit = !value && field.startsAsInteger ? findTryOfBlockAheadOfEdges(head) : nullptr) { + for (SlotOfLoop& slot : m_loopSummaries[visit->summaryIndex].storesIntoScopes.slots) + slot.hasLeftIntegers |= slot.holder == field.slotHolder && slot.key == field.slotKey; + visit->misfits = true; + value = field.parameter; + } if (!value && field.isHeld && !m_versionedLoops.isEmpty() && backEdgeTypeOfVariable(m_versionedLoops.last().inputHead, field) == field.type) { value = askAgainWhereItComesRound(edge, head, field.type, arguments.span().first(numberBesideFieldsOfScopes), scopedLambda([&] { if (Reference atHand = valueKnownOf(field)) @@ -1119,6 +1378,7 @@ void Walker::handScopesKeptAtHeadToPreview(const BodyIndex& index, BlockID input { Vector& scopes = preview.scopesThatTurnsOnlyBorrow; scopes.shrink(0); + preview.slots = stores.slots; for (const FreshObjectKeptAtHead& object : kept) { if (isEnvironmentAllocation(object.object)) scopes.append(object.object); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkEscapeAnalysis.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkEscapeAnalysis.cpp index 2d54fc25ec8b9..b6ac71ade386a 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkEscapeAnalysis.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkEscapeAnalysis.cpp @@ -826,7 +826,7 @@ static bool followStoreIntoFreshObject(Walker& walker, const Draft& store, const if (!cleared) return false; WalkState& state = walker.state(); - state.noteSlot(written.object, *cleared, Reference()); + state.noteSlot(written.object, *cleared, walker.constant(Draft(Opcode::Absent))); state.noteOrderWord(written.object, std::nullopt); state.noteElement(written.object, keyOfPropertiesAsListed(), Reference()); state.invalidateCellFacts(written.object, CellFact::BearsNoMark); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkFolding.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkFolding.cpp index a420ca9ef32b5..9f34704db9f7b 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkFolding.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkFolding.cpp @@ -46,7 +46,7 @@ static constexpr OpcodeSet storesThatStateFollowsForFolding { Opcode::PutModuleV static constexpr OpcodeSet placesThatStateFollowsForFolding = storesThatStateFollowsForFolding | OpcodeSet { Opcode::GetModuleVariable, Opcode::GetEnvironmentField }; static constexpr OpcodeSet conversionsToObjectsInFolding { Opcode::ToObject, Opcode::ToThis }; static constexpr OpcodeSet opcodesOfFolding = operationsOnPrimitivesInFolding | testsThatTypesDecideInFolding | castsOfFolding | readsOfFolding | storesThatStateFollowsForFolding | passThroughChecks - | conversionsToObjectsInFolding | OpcodeSet { Opcode::TypeOf, Opcode::ToLength }; + | conversionsToObjectsInFolding | OpcodeSet { Opcode::TypeOf, Opcode::ToLength, Opcode::IsUnlockedInstance }; static ALWAYS_INLINE PrimitiveKnownToFolding onlyValueOfTypeInFolding(const TypeArena& types, TypeID id) { @@ -779,11 +779,47 @@ static bool holdsAccessorInFolding(Walker& walker, FieldReference field) return fieldAt(walker.program(), field).form.contains(FieldForm::IsAccessor) || facts.formOfValue.contains(FieldForm::IsAccessor) || facts.facts.contains(FieldFact::IsAccessor); } +static const Operation* findTemplateObject(const Walker& walker, Reference object) +{ + if (!object || !object.isValue()) + return nullptr; + const Operation& made = walker.code().at(underlyingCellOf(walker.code(), object.id())); + return made.opcode == Opcode::NewTemplateObject ? &made : nullptr; +} + +static Reference partOfTemplateObject(Walker& walker, Reference object, Reference index) +{ + const Operation* made = index.isConstant() ? findTemplateObject(walker, object) : nullptr; + if (!made) + return { }; + const Operation& key = walker.code().at(index.id()); + if (key.opcode != Opcode::Integer) + return { }; + Vector parts; + for (TemplateString part : made->templateStrings()) + parts.append(part); + if (key.integer() < 0 || static_cast(key.integer()) >= parts.size()) + return { }; + StringID cooked = parts[key.integer()].cooked; + Draft constant(cooked ? Opcode::String : Opcode::Undefined); + if (cooked) + constant.setString(cooked); + return walker.constant(WTF::move(constant)); +} + static Reference answerOfNamedReadInFolding(Walker& walker, const BodyOperation& read) { if (read.isFreeName()) return { }; Reference object = walker.referenceOf(read.objectOperand()); + if (const Operation* made = walker.program().nameClassOf(read.key()) == NameClass::Length ? findTemplateObject(walker, object) : nullptr) { + int64_t numberOfParts = 0; + for ([[maybe_unused]] TemplateString part : made->templateStrings()) + ++numberOfParts; + Draft length(Opcode::Integer); + length.setInteger(numberOfParts); + return walker.constant(WTF::move(length)); + } if (!walker.state().mayHoldPlaceOf(object)) return propertyOfResultTakenInParts(walker, object, read.key()); TypeID type = typeToJudgeByInFolding(walker, object); @@ -807,6 +843,8 @@ static Reference elementAnsweredByStateInFolding(Walker& walker, const BodyValue TypeID typeOfIndex = typeToJudgeByInFolding(walker, index); if (!typeOfIndex || !walker.types().at(typeOfIndex).isNumber()) return { }; + if (Reference part = partOfTemplateObject(walker, walker.referenceOf(read.objectOperand()), index)) + return part; if (Reference part = partOfListThatIsOnlyTakenApart(walker, walker.referenceOf(read.objectOperand()), index)) return part; Reference held = unlessAbsentInFolding(walker, walker.state().element(walker.referenceOf(read.objectOperand()), index)); @@ -923,9 +961,23 @@ static bool buildCheckOfCellInFolding(Walker& walker, const BodyValue& check) return true; } +static Reference answerWhetherInstanceIsUnlocked(Walker& walker, const BodyValue& value) +{ + auto asked = value.as(); + if (instancesMayBeLocked(walker.program(), walker.facts().linked(), asked.testedType())) + return walker.constant(PrimitiveKnownToFolding::boolean(false)); + Reference object = walker.referenceOf(asked.valueOperand()); + TypeID type = typeToJudgeByInFolding(walker, object); + if (type && walker.types().isSubset(type, asked.testedType())) + return walker.constant(PrimitiveKnownToFolding::boolean(true)); + return { }; +} + static ALWAYS_INLINE Reference answerKnownToFolding(Walker& walker, const BodyValue& value) { Opcode opcode = value.opcode(); + if (opcode == Opcode::IsUnlockedInstance) + return answerWhetherInstanceIsUnlocked(walker, value); if (readsOfFolding.contains(opcode)) return answerOfReadInFolding(walker, value); if (passThroughChecks.contains(opcode)) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkFrontsOfLoops.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkFrontsOfLoops.cpp index 067c5a5160c65..ec1a4e7090b39 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkFrontsOfLoops.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkFrontsOfLoops.cpp @@ -209,6 +209,12 @@ BitVector Walker::blocksThatSomeQuietTurnPassesThrough(BlockID inputHead, std::s return reachesBackEdge; } +bool Walker::callIsQuietGivenArguments(FunctionID callee) const +{ + const LinkedEffectsGivenArguments* given = facts().linked().known.findEffectsGivenArguments(callee); + return given && !given->parametersCalled && !given->onReturn.mayRunOtherCode() && !given->onThrow.mayRunOtherCode(); +} + void Walker::noteCallOnUsualWayInTries(const Effects& heard) { if (!heard.mayRunOtherCode() || m_numberOfClosedArms || m_numberOfFormsHeardAsArms || m_code.at(m_block).frequency != BlockFrequency::Normal) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkGeneratorStepsByHand.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkGeneratorStepsByHand.cpp index 4be62a9f0633c..d19ba86003a76 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkGeneratorStepsByHand.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkGeneratorStepsByHand.cpp @@ -111,20 +111,11 @@ static bool nothingSeenHappensBetweenInStepsByHand(const BodyIndex& index, Value return true; } -static bool calleeIsNextOfEngineInStepsByHand(Walker& walker, const BodyValue& call) +static bool readFindsNextOfEngine(Walker& walker, const BodyValue& call) { const Program& program = walker.program(); const BuiltinTable& table = program.builtins(); - TypeArena& types = walker.types(); BuiltinID next = table.canonicalID(Builtins::GeneratorPrototypeNext); - - Reference callee = walker.referenceOf(call.calleeOperand()); - if (!callee) - return false; - Type typeOfCallee = types.at(walker.typeOf(callee)); - if (BuiltinID named = typeOfCallee.isObject() && typeOfCallee.isJavaScriptValue() ? singleBuiltinOf(types, typeOfCallee) : BuiltinID()) - return table.canonicalID(named) == next; - const BodyIndex& index = walker.currentBodyIndex(); ValueID read = index.stripCasts(call.calleeOperand()); if (index.isParameter(read) || index.opcodeOf(read) != Opcode::PropGet) @@ -143,6 +134,18 @@ static bool calleeIsNextOfEngineInStepsByHand(Walker& walker, const BodyValue& c return found && table.canonicalID(found) == next; } +static bool calleeIsNextOfEngineInStepsByHand(Walker& walker, const BodyValue& call) +{ + TypeArena& types = walker.types(); + Reference callee = walker.referenceOf(call.calleeOperand()); + Type typeOfCallee = callee ? types.at(walker.typeOf(callee)) : Type(); + if (BuiltinID named = callee && typeOfCallee.isObject() && typeOfCallee.isJavaScriptValue() ? singleBuiltinOf(types, typeOfCallee) : BuiltinID()) { + const BuiltinTable& table = walker.program().builtins(); + return table.canonicalID(named) == table.canonicalID(Builtins::GeneratorPrototypeNext); + } + return readFindsNextOfEngine(walker, call); +} + enum class ResultOfNextInStepsByHand : uint8_t { IsWhatCallGives, IsWhatAwaitOfCallGives }; static const BitVector& findCallsOfNextWhoseResultIsOnlyReadInStepsByHand(Walker& walker, ResultOfNextInStepsByHand result = ResultOfNextInStepsByHand::IsWhatCallGives) @@ -215,9 +218,14 @@ static const BitVector& findCallsOfNextWhoseResultIsOnlyReadInStepsByHand(Walker return onlyRead; } -static bool buildStepInStepsByHand(Walker& walker, const BodyValue& call, Reference generator, Reference sent) +static bool buildStepInStepsByHand(Walker& walker, const BodyValue& call, Reference generator, Reference sent, bool resultIsOnlyRead) { + bool readsNextBehindStep = !walker.referenceOf(call.calleeOperand()); + if (readsNextBehindStep) + walker.bind(call.calleeOperand(), generator); Draft written = walker.draftOf(call); + if (readsNextBehindStep) + walker.bind(call.calleeOperand(), Reference()); Origin origin = written.origin(); origin.isTailCall = false; Reference stepped = buildStepOfGenerator(walker, generator, ResumeMode::Next, sent, checkLeftToStepOf(walker, generator), origin); @@ -233,12 +241,14 @@ static bool buildStepInStepsByHand(Walker& walker, const BodyValue& call, Refere BlockID isStepped = walker.addBlock(); BlockID meeting = walker.addBlock(); Reference whole; - for (uint32_t parameter = 0; parameter < numberOfParametersOfMeetingInStepsByHand; ++parameter) { + uint32_t numberOfParameters = resultIsOnlyRead ? static_cast(numberOfParametersOfMeetingInStepsByHand) : 1; + for (uint32_t parameter = 0; parameter < numberOfParameters; ++parameter) { Reference added = walker.addParameter(meeting, { }, Representation::JSValue); if (parameter == Whole) whole = added; } - walker.code().markMeetingOfStepWrittenByHand(meeting); + if (resultIsOnlyRead) + walker.code().markMeetingOfStepWrittenByHand(meeting); Reference nothing = undefinedOfCoroutineCalls(walker); bool someWayArrives = false; walker.branch(isNotStepped, byCall, isStepped); @@ -247,20 +257,34 @@ static bool buildStepInStepsByHand(Walker& walker, const BodyValue& call, Refere written.setOrigin(origin); written.resultIsReturnedAtOnce = false; written.inputValueGiven = { }; - walker.chooseConservativeFields(written); - Reference answer = walker.build(WTF::move(written)); - jumpToInStepsByHand(walker, meeting, std::array { answer, answer, booleanInStepsByHand(walker, true, origin), nothing, nothing }, someWayArrives); + if (readsNextBehindStep) { + Draft read = walker.draftOf(walker.currentBodyIndex().value(call.calleeOperand())); + read.inputValueGiven = { }; + walker.chooseConservativeFields(read); + written.setCalleeOperand(buildInStepsByHand(walker, WTF::move(read))); + } + Reference answer; + if (!walker.isUnreachable()) { + walker.chooseConservativeFields(written); + answer = walker.build(WTF::move(written)); + } + std::array answered { answer, answer, booleanInStepsByHand(walker, true, origin), nothing, nothing }; + jumpToInStepsByHand(walker, meeting, std::span(answered).first(numberOfParameters), someWayArrives); walker.continueIn(isStepped); Reference isCompleted = buildTestOfStatusInStepsByHand(walker, status, GeneratorStepStatus::Completed, origin); Reference made = buildResultObjectInStepsByHand(walker, value, isCompleted, origin); - jumpToInStepsByHand(walker, meeting, std::array { made, nothing, booleanInStepsByHand(walker, false, origin), value, isCompleted }, someWayArrives); + std::array atHand { made, nothing, booleanInStepsByHand(walker, false, origin), value, isCompleted }; + jumpToInStepsByHand(walker, meeting, std::span(atHand).first(numberOfParameters), someWayArrives); if (!someWayArrives || walker.hasFailed()) return true; walker.continueIn(meeting); walker.bind(call.id(), whole); - walker.loopsOverGenerators().hasStepsWrittenByHand = true; + if (resultIsOnlyRead) + walker.loopsOverGenerators().hasStepsWrittenByHand = true; + else + walker.rememberedExpectationOf(call.id()) = { walker.types().forLayout(walker.program().layoutOf(BuiltinLayouts::IteratorResult)), false, false, true }; return true; } @@ -326,7 +350,7 @@ bool answerFromStepWrittenByHandOfItsOpcodes(Walker& walker, const BodyValue& va return true; } -bool buildStepWrittenByHand(Walker& walker, const BodyValue& call) +static bool isStepOfGenerator(Walker& walker, const BodyValue& call) { if (call.opcode() != Opcode::Call || call.isDoneInPlace() || !walker.has(Mechanism::DirectCalls) || !walker.has(Mechanism::BuiltinCalls) || !walker.opensGuesses()) return false; @@ -344,12 +368,38 @@ bool buildStepWrittenByHand(Walker& walker, const BodyValue& call) if (!expected || !types.isSubset(types.withoutNullish(expected), types.builtinClass(BuiltinClasses::Generator))) return false; } - if (!calleeIsNextOfEngineInStepsByHand(walker, call)) + return walker.isInLoop() || findCallsOfNextWhoseResultIsOnlyReadInStepsByHand(walker).quickGet(call.id().index()); +} + +bool callStepsGeneratorOfItsOpcodes(Walker& walker, const BodyValue& call) +{ + return isStepOfGenerator(walker, call) && readFindsNextOfEngine(walker, call); +} + +bool leaveNextToStepOfItsOpcodes(Walker& walker, const BodyValue& read) +{ + if (read.isFreeName() || read.key() != walker.program().keyOf(BuiltinNames::next)) + return false; + const BodyIndex& index = walker.currentBodyIndex(); + ValueID taker = index.onlyUsingValue(read.id()); + if (!taker || index.opcodeOf(taker) != Opcode::Call || index.blockOf(taker) != index.blockOf(read.id())) + return false; + BodyValue call = index.value(taker); + if (call.calleeOperand() != read.id() || taker.index() - index.block(index.blockOf(taker)).firstValue().index() >= walker.endOfStretchBeingWalked()) + return false; + if (!callStepsGenerator(walker, call)) return false; - if (!findCallsOfNextWhoseResultIsOnlyReadInStepsByHand(walker).quickGet(call.id().index())) + walker.bind(read.id(), Reference()); + return true; +} + +bool buildStepWrittenByHand(Walker& walker, const BodyValue& call) +{ + if (!isStepOfGenerator(walker, call) || !calleeIsNextOfEngineInStepsByHand(walker, call)) return false; + bool resultIsOnlyRead = findCallsOfNextWhoseResultIsOnlyReadInStepsByHand(walker).quickGet(call.id().index()); Reference sent = call.argumentsOperands().empty() ? undefinedOfCoroutineCalls(walker) : walker.referenceOf(call.argumentsOperands()[0]); - return sent && buildStepInStepsByHand(walker, call, generator, sent); + return sent && buildStepInStepsByHand(walker, call, walker.referenceOf(call.thisValueOperand()), sent, resultIsOnlyRead); } static Reference continueIfHasType(Walker& walker, Reference value, TypeID tested, BlockID otherwise, bool& someWayTurnsAway, Origin origin) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkGeneratorsInLoops.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkGeneratorsInLoops.cpp index 4c08c9e744dad..70cfaa8685d6a 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkGeneratorsInLoops.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkGeneratorsInLoops.cpp @@ -22,6 +22,7 @@ #include "AOTPlan.h" #include "AOTRepresentationRules.h" #include "AOTThinSummary.h" +#include "AOTWalkCoroutineCallsInternal.h" #include "AOTWalkLoopsOverGenerators.h" #include #include @@ -47,6 +48,8 @@ struct GeneratorFunctionDescribed { uint32_t numberOfYields { 0 }; uint64_t numberOfOperationsOnWaysOfReturn { 0 }; std::optional nobodyLooksAbove; + std::optional> onlyUsersOfEntryParameters; + bool startsAtInitialYield { false }; }; struct KeptByGeneratorsInLoops final : Walker::KeptForBuild { @@ -169,6 +172,11 @@ static bool wayOfReturnLeadsForward(const BodyIndex& body, BlockID blockOfYield, return true; } +static bool isPureInGeneratorsInLoops(const BodyValue& operation) +{ + return declaredBoundInTable(operation.opcode()).isPure() && !operandsMayRunCode(operation.opcode()); +} + static GeneratorFunctionDescribed describeGeneratorFunction(const Walker& walker, FunctionID id) { GeneratorFunctionDescribed described; @@ -190,6 +198,11 @@ static GeneratorFunctionDescribed describeGeneratorFunction(const Walker& walker if (body->numberOfIncomingEdges(entryBlock) || entry.hasHandler() || entry.numberOfParameters() != function.numberOfEntryParameters()) return described; + for (BodyValue operation : entry.values()) { + described.startsAtInitialYield = operation.opcode() == Opcode::InitialYield; + if (described.startsAtInitialYield || !isPureInGeneratorsInLoops(operation)) + break; + } for (uint32_t number = 0; number < body->numberOfBlocks(); ++number) { BodyBlock block = body->block(BlockID(number)); if (!body->isReachable(block.id())) @@ -207,6 +220,48 @@ static GeneratorFunctionDescribed describeGeneratorFunction(const Walker& walker return described; } +static Vector findOnlyUsersOfEntryParameters(const BodyIndex& body) +{ + uint32_t firstParameter = body.block(entryBlock).firstParameter().index(); + uint32_t numberOfParameters = body.block(entryBlock).numberOfParameters(); + Vector users; + Vector numbersOfUses; + users.fill(ValueID(), numberOfParameters); + numbersOfUses.fill(0, numberOfParameters); + auto noteUse = [&](ValueID operand, ValueID user) { + ValueID named = operand ? body.stripCasts(operand) : ValueID(); + if (!named || !body.isParameter(named) || named.index() < firstParameter || named.index() >= firstParameter + numberOfParameters) + return; + users[named.index() - firstParameter] = ++numbersOfUses[named.index() - firstParameter] == 1 ? user : ValueID(); + }; + for (uint32_t number = 0; number < body.numberOfBlocks(); ++number) { + BodyBlock block = body.block(BlockID(number)); + if (!body.isReachable(block.id())) + continue; + for (BodyValue operation : block.values()) { + if (opcodesThatStateTypes.contains(operation.opcode()) || operation.opcode() == Opcode::InitialYield) + continue; + if (operation.opcode() == Opcode::Yield) { + noteUse(operation.as().valueOperand(), ValueID()); + continue; + } + operation.forEachOperand([&](ValueID operand, auto...) { + noteUse(operand, operation.id()); + }); + } + auto noteUseAtEnd = [&](ValueID operand, auto...) { + noteUse(operand, ValueID()); + }; + BodyTerminator end = block.terminator(); + end.forEachOperand(noteUseAtEnd); + for (BodyTarget target : end.targets()) + std::ranges::for_each(target.arguments(), noteUseAtEnd); + if (block.hasHandler()) + std::ranges::for_each(block.handler().arguments(), noteUseAtEnd); + } + return users; +} + static bool loopHoldsYield(const BodyIndex& body, LoopID loop) { for (BlockID member : body.membersOf(loop)) { @@ -355,6 +410,116 @@ size_t Walker::placeOfGeneratorOnTop() const return notFound; } +bool Walker::loopHoldsGenerators(const BodyIndex& index, LoopID loop) +{ + if (!has(Mechanism::Handlers) || !has(Mechanism::DirectCalls)) + return false; + for (BlockID member : index.membersOf(loop)) { + for (BodyValue operation : index.block(member).values()) { + if (operation.opcode() != Opcode::IteratorOpen) + continue; + auto opened = operation.as(); + if (opened.iteratorKind() != IteratorKind::Sync || opened.isDelegating() || opened.isDestructuring()) + continue; + ValueID iterable = index.stripCasts(opened.iterableOperand()); + bool isCall = iterable && !index.isParameter(iterable) && index.opcodeOf(iterable) == Opcode::Call; + FunctionID callee = isCall ? facts().linked().calleeAt(currentFunction(), iterable) : FunctionID(); + if (!callee || !program().at(callee).has(FunctionFlags::IsGenerator)) + callee = facts().linked().calleeAt(currentFunction(), operation.id()); + if (!callee || !program().at(callee).has(FunctionFlags::IsGenerator)) + continue; + const GeneratorFunctionDescribed& described = keptByGeneratorsInLoops(m_keptByGeneratorsInLoops).functions.ensure(callee.index(), [&] { + return describeGeneratorFunction(*this, callee); + }).iterator->value; + if (described.bodyMayBeWalkedInConsumer) + return true; + } + } + return false; +} + +bool Walker::startsAtInitialYield(FunctionID generatorFunction) +{ + return keptByGeneratorsInLoops(m_keptByGeneratorsInLoops).functions.ensure(generatorFunction.index(), [&] { + return describeGeneratorFunction(*this, generatorFunction); + }).iterator->value.startsAtInitialYield; +} + +ValueID Walker::onlyUserOfEntryParameter(FunctionID generatorFunction, uint32_t entryParameter) +{ + GeneratorFunctionDescribed& described = keptByGeneratorsInLoops(m_keptByGeneratorsInLoops).functions.ensure(generatorFunction.index(), [&] { + return describeGeneratorFunction(*this, generatorFunction); + }).iterator->value; + if (!described.bodyMayBeWalkedInConsumer) + return { }; + if (!described.onlyUsersOfEntryParameters) + described.onlyUsersOfEntryParameters = findOnlyUsersOfEntryParameters(facts().bodies().at(generatorFunction)); + return entryParameter < described.onlyUsersOfEntryParameters->size() ? described.onlyUsersOfEntryParameters->at(entryParameter) : ValueID(); +} + +bool Walker::passesGeneratorForIteration(const BodyIndex& caller, ValueID generatorCall, FunctionID callerID) +{ + if (caller.isParameter(generatorCall) || caller.opcodeOf(generatorCall) != Opcode::Call) + return false; + ValueID passed = generatorCall; + ValueID taker = followCastsToOnlyTaker(caller, passed); + if (!taker || caller.opcodeOf(taker) != Opcode::Call) + return false; + auto call = caller.value(taker).as(); + std::span arguments = call.argumentsOperands(); + uint32_t position = static_cast(std::ranges::find(arguments, passed) - arguments.begin()); + FunctionID callee = facts().linked().calleeAt(callerID, taker); + if (position == arguments.size() || !callee || !facts().linked().function(callee).everyCallHasOneCallee) + return false; + const Function& function = program().at(callee); + if (!function.has(FunctionFlags::IsGenerator) || function.has(FunctionFlags::IsAsync) || position >= function.numberOfPositionalParameters() || call.calleeOperand() == passed || call.thisValueOperand() == passed) + return false; + ValueID user = onlyUserOfEntryParameter(callee, function.firstParameterIndex() + position); + if (!user) + return false; + BodyValue iteration = facts().bodies().at(callee).value(user); + if (iteration.opcode() == Opcode::AppendSpread) + return facts().bodies().at(callee).stripCasts(iteration.as().arrayOperand()).index() != facts().bodies().at(callee).block(entryBlock).firstParameter().index() + function.firstParameterIndex() + position; + return iteration.opcode() == Opcode::IteratorOpen && iteration.as().iteratorKind() == IteratorKind::Sync; +} + +Reference Walker::findPassedGenerator(ValueID open, ValueID iterable, ValueID& callThatMadeIt, uint32_t& frameOfCall) +{ + size_t place = placeOfGeneratorOnTop(); + if (place == notFound || !m_generatorsWalkedInConsumers[place].call) + return { }; + const GeneratorWalkedInConsumer& walked = m_generatorsWalkedInConsumers[place]; + const BodyIndex& body = currentBodyIndex(); + ValueID parameter = body.stripCasts(iterable); + if (!body.isParameter(parameter) || body.blockOf(parameter) != entryBlock || body.loopDepthOf(body.blockOf(open))) + return { }; + if (onlyUserOfEntryParameter(currentFunction(), parameter.index() - body.block(entryBlock).firstParameter().index()) != open) + return { }; + Reference generator = referenceBoundTo(parameter); + if (!generator || !generator.isRecord()) + return { }; + const Operation& made = m_code.at(generator.id()); + if (made.opcode == Opcode::CallDirect && !startsAtInitialYield(made.function())) + return { }; + if (made.opcode != Opcode::CallDirect || made.role() != CodeRole::Body || made.numberOfVariadicOperands() != program().at(made.function()).numberOfEntryParameters()) + return { }; + + const Frame& caller = m_frames[walked.frameOfCall]; + const BodyIndex& index = *caller.index; + for (ValueID argument : index.value(walked.call).as().argumentsOperands()) { + ValueID call = index.stripCasts(argument); + if (referenceBoundIn(caller, call) != generator) + continue; + ValueID nameInCaller = call; + if (index.isParameter(call) || index.opcodeOf(call) != Opcode::Call || index.blockOf(call) != index.blockOf(walked.call) || followCastsToOnlyTaker(index, nameInCaller) != walked.call) + return { }; + callThatMadeIt = call; + frameOfCall = walked.frameOfCall; + return generator; + } + return { }; +} + bool Walker::takeGeneratorForConsumer(const BodyValue& call) { size_t frameIndex = m_frames.size() - 1; @@ -373,12 +538,20 @@ bool Walker::takeGeneratorForConsumer(const BodyValue& call) FunctionID callee; Reference iterable; + Reference passedGenerator; + ValueID callOfGenerator = takerCallsMethodItself ? ValueID() : call.id(); + uint32_t frameOfCall = static_cast(frameIndex); PropertyKeyID keyOfMethod = types().wellKnownSymbolKey(WellKnownSymbol::Iterator); if (takerCallsMethodItself) { - iterable = referenceOf(call.opcode() == Opcode::IteratorOpen ? call.as().iterableOperand() : call.as().iterableOperand()); + ValueID iterated = call.opcode() == Opcode::IteratorOpen ? call.as().iterableOperand() : call.as().iterableOperand(); + iterable = referenceOf(iterated); + passedGenerator = has(Mechanism::DirectCalls) ? findPassedGenerator(call.id(), iterated, callOfGenerator, frameOfCall) : Reference(); TypeID present = iterable && iterable.isValue() ? m_types.withoutNullish(typeOf(iterable)) : TypeID(); - callee = has(Mechanism::DirectCalls) ? findMethodInSightOfProvenType(*this, present, keyOfMethod) : FunctionID(); - if (callee && callee != facts().linked().calleeAt(currentFunction(), call.id()) && !facts().linked().mayBeMentionedBy(unit(), callee)) + if (passedGenerator) + callee = m_code.at(passedGenerator.id()).function(); + else + callee = has(Mechanism::DirectCalls) ? findMethodInSightOfProvenType(*this, present, keyOfMethod) : FunctionID(); + if (callee && !passedGenerator && callee != facts().linked().calleeAt(currentFunction(), call.id()) && !facts().linked().mayBeMentionedBy(unit(), callee)) return false; if (m_generatorsWalkedInConsumers.containsIf([&](const GeneratorWalkedInConsumer& around) { return around.numberOfYields > 1 && around.frameOfTurn == frameIndex; })) return false; @@ -386,13 +559,18 @@ bool Walker::takeGeneratorForConsumer(const BodyValue& call) callee = findOnlyCallee(*this, call, closure); if (!callee || isBeingWalkedOtherThanAtYield(callee)) return false; - GeneratorFunctionDescribed& described = keptByGeneratorsInLoops(m_keptByGeneratorsInLoops).functions.ensure(callee.index(), [&] { + const GeneratorFunctionDescribed& found = keptByGeneratorsInLoops(m_keptByGeneratorsInLoops).functions.ensure(callee.index(), [&] { return describeGeneratorFunction(*this, callee); }).iterator->value; - if (!described.bodyMayBeWalkedInConsumer) + if (!found.bodyMayBeWalkedInConsumer) return false; - if (!described.nobodyLooksAbove) - described.nobodyLooksAbove = nobodyLooksAboveGeneratorsOf(*this, callee, nullptr); + if (!found.nobodyLooksAbove) { + bool nobodyLooksAbove = nobodyLooksAboveGeneratorsOf(*this, callee, nullptr, scopedLambda([&](const BodyIndex& caller, const LinkedCall& passed, FunctionID callerID) { + return passesGeneratorForIteration(caller, passed.site, callerID); + })); + keptByGeneratorsInLoops(m_keptByGeneratorsInLoops).functions.find(callee.index())->value.nobodyLooksAbove = nobodyLooksAbove; + } + const GeneratorFunctionDescribed& described = keptByGeneratorsInLoops(m_keptByGeneratorsInLoops).functions.find(callee.index())->value; if (!*described.nobodyLooksAbove || !engineAnswersForGenerators(*this)) return false; const LinkTables& linked = facts().linked(); @@ -405,7 +583,9 @@ bool Walker::takeGeneratorForConsumer(const BodyValue& call) taken.inlineFrame = m_frames[frameIndex].inlineFrame; taken.taker = taker; taken.function = callee; - taken.takerCallsMethodItself = takerCallsMethodItself; + taken.call = callOfGenerator; + taken.frameOfCall = frameOfCall; + taken.takerCallsMethodItself = takerCallsMethodItself && !passedGenerator; taken.numberOfYields = described.numberOfYields; uint64_t numberOfOperations = facts().summaries().at(callee).numberOfOperations; @@ -512,7 +692,7 @@ bool Walker::takeGeneratorForConsumer(const BodyValue& call) return false; Draft site = takerCallsMethodItself ? Draft(Opcode::Call, originOf(call)) : draftOf(call); - if (takerCallsMethodItself) { + if (takerCallsMethodItself && !passedGenerator) { if (m_types.at(typeOf(iterable)).primitives & Primitive::UndefinedOrNull) { Origin origin = originOf(call); origin.isTailCall = false; @@ -531,14 +711,20 @@ bool Walker::takeGeneratorForConsumer(const BodyValue& call) site.setCalleeOperand(method); site.setThisValueOperand(iterable); } - Vector passedOperands = collectPassedOperands(site, program().at(callee)); - if (hasFailed() || !callMayEnterCopyOfBody(*this, site, callee, passedOperands.span())) - return false; + if (passedGenerator) { + m_environmentCellsOfUnit.mayNameSome |= isHandedEnvironmentWithByte(*this, callee); + for (ValueID operand : m_code.at(passedGenerator.id()).variadicOperands()) + taken.entryArguments.append(Reference::value(operand)); + } else { + Vector passedOperands = collectPassedOperands(site, program().at(callee)); + if (hasFailed() || !callMayEnterCopyOfBody(*this, site, callee, passedOperands.span())) + return false; - m_environmentCellsOfUnit.mayNameSome |= isHandedEnvironmentWithByte(*this, callee); - taken.entryArguments = buildEntryArgumentsOfCopy(*this, site, callee, passedOperands.span()); - if (isUnreachable()) - return true; + m_environmentCellsOfUnit.mayNameSome |= isHandedEnvironmentWithByte(*this, callee); + taken.entryArguments = buildEntryArgumentsOfCopy(*this, site, callee, passedOperands.span()); + if (isUnreachable()) + return true; + } taken.state = addIndexedIteration(IndexedIteration { }); if (!depthOfCopies()) @@ -555,10 +741,11 @@ bool Walker::takeGeneratorForConsumer(const BodyValue& call) bind(call.id(), state); return true; } - count(StatisticID::GeneratorMethodsInlinedIntoIterations); + if (!passedGenerator) + count(StatisticID::GeneratorMethodsInlinedIntoIterations); if (call.opcode() == Opcode::AppendSpread) return spreadGeneratorTakenAt(m_generatorsTakenByConsumers.size() - 1, call); - buildFirstReadsThatOpenOfGeneratorLeavesOut(*this, originOf(call), false); + buildFirstReadsThatOpenOfGeneratorLeavesOut(*this, originOf(call), !!passedGenerator); if (!isUnreachable()) bind(call.id(), state); return true; @@ -1075,6 +1262,8 @@ bool Walker::walkGeneratorInPlaceOfLoop(BlockID inputHead, std::span(frameOfConsumer); walked.frameOfGenerator = static_cast(frameOfConsumer + 1); + walked.call = taken.call; + walked.frameOfCall = taken.frameOfCall; walked.inputHead = inputHead; walked.step = taken.step; walked.state = taken.state; @@ -1185,6 +1374,8 @@ bool Walker::spreadGeneratorTakenAt(size_t placeOfTaken, const BodyValue& spread GeneratorWalkedInConsumer walked; walked.frameOfConsumer = static_cast(frameOfConsumer); walked.frameOfGenerator = static_cast(frameOfConsumer + 1); + walked.call = taken.call; + walked.frameOfCall = taken.frameOfCall; walked.state = taken.state; walked.list = list; walked.numberOfYields = taken.numberOfYields; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkGeneratorsInLoops.h b/Source/JavaScriptCore/aot/walk/AOTWalkGeneratorsInLoops.h index 1f4d9d5f74b3f..3357a4b0f0315 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkGeneratorsInLoops.h +++ b/Source/JavaScriptCore/aot/walk/AOTWalkGeneratorsInLoops.h @@ -9,10 +9,12 @@ #include "AOTBodyIndex.h" #include "AOTCode.h" +#include namespace JSC { namespace AOT { namespace Typed { class Walker; +struct LinkedCall; inline constexpr OpcodeSet opcodesOfGeneratorsInLoops { Opcode::Call, Opcode::IteratorOpen, Opcode::IteratorResume, Opcode::IteratorClose, Opcode::AppendSpread, Opcode::InitialYield, Opcode::Yield }; @@ -25,6 +27,8 @@ struct GeneratorTakenByConsumer { uint32_t numberOfYields { 0 }; uint32_t callPosition { 0 }; uint64_t numberOfOperationsOfRegion { 0 }; + ValueID call; + uint32_t frameOfCall { 0 }; bool regionHoldsIterations { false }; bool hasBackEdge { false }; bool takerCallsMethodItself { false }; @@ -40,6 +44,8 @@ struct GeneratorWalkedInConsumer { uint32_t frameOfGenerator { 0 }; uint32_t frameOfTurn { UINT32_MAX }; uint32_t numberOfLoopsAroundTurn { 0 }; + ValueID call; + uint32_t frameOfCall { 0 }; BlockID inputHead; ValueID step; Reference state; @@ -62,6 +68,7 @@ Vector buildEntryArgumentsOfCopy(Walker&, const Draft& site, Funct bool operationsFitWindowOfCallWrittenInLoop(const Walker&, uint64_t numberOfOperations); bool calleeIsCopiedInLoopsAlone(const Walker&, FunctionID callee, size_t numberOfOperands); bool nobodyLooksAboveGeneratorsOf(const Walker&, FunctionID generatorFunction, uint32_t* numberOfListedCalls); +bool nobodyLooksAboveGeneratorsOf(const Walker&, FunctionID generatorFunction, uint32_t* numberOfListedCalls, const ScopedLambda& passesForIteration); } } } // namespace JSC::AOT::Typed diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkGuesses.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkGuesses.cpp index 03a6175a707e0..7f7b2f0052300 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkGuesses.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkGuesses.cpp @@ -9,6 +9,7 @@ #if ENABLE(AOT) #include "AOTBuildContext.h" +#include "AOTBuiltinMembers.h" #include "AOTEffectsDeclared.h" #include "AOTEffectsOfOperations.h" #include "AOTFamilies.h" @@ -33,8 +34,10 @@ #include "AOTWalkBlocksThatEveryTurnPasses.h" #include "AOTWalkBranches.h" #include "AOTWalkKnown.h" +#include "AOTWalkKnownCalls.h" #include "AOTWalkListReadsInLoops.h" #include "AOTWalkTextBuilderRows.h" +#include "AOTWalkNumberForms.h" #include "AOTWalkNumberGuesses.h" #include "AOTWalkStrings.h" #include @@ -57,6 +60,7 @@ static constexpr uint32_t minimumOperationsPerGuess = 2; static constexpr size_t fewestLooksThatOneTestOfMarkStandsFor = 2; static constexpr uint32_t minimumOperationsPerGuessInLoop = 1; static constexpr uint32_t minimumOperationsPerGuessOfShape = 1; +static constexpr uint32_t minimumOperationsPerGuessOfShapeInAsyncBody = 2; static constexpr OpcodeSet constructionsOfParentForGuesses { Opcode::SuperConstruct, Opcode::SuperConstructList }; static constexpr OpcodeSet callsOfValuesForGuesses = constructionsOfParentForGuesses | OpcodeSet { Opcode::Call, Opcode::Construct, Opcode::CallList }; @@ -579,6 +583,7 @@ struct BodySpeculationCache final : KeptWithBodyIndex { Vector literalsMadeHere; Vector loopsWithElementArithmetic; Vector loopsWithElementComparisons; + Vector loopsMovingElements; Vector holdersFromInFrontReadUnderIndexesInLoops; Vector placesOfReads; Vector readsOfTexts; @@ -603,6 +608,8 @@ struct BodySpeculationCache final : KeptWithBodyIndex { std::atomic hasHoldersFromInFrontReadUnderIndexesInLoops { false }; std::atomic hasEntryParametersOnlyReadByNames { false }; uint64_t entryParametersOnlyReadByTwoNamesOrMore { 0 }; + std::atomic hasNumbersReadAfterWrites { false }; + uint64_t numbersReadAfterWrites { 0 }; std::atomic isAskedForReadersOfCharacters { false }; std::atomic hasRootsHandedFunctionsWrittenInPlace { false }; bool callsReaderOfCharacters { false }; @@ -659,9 +666,21 @@ static bool showsValueUnderAnotherViewForGuesses(const Program& program, const B } } +static bool isPrivateCharCodeAtForGuesses(const Program& program, PropertyKeyID key) +{ + return isPrivateAliasOfBuiltinMember(program, key) && builtinMemberNameOf(program, key).name == BuiltinNames::charCodeAt; +} + +static BuiltinName builtinNameOfUseForGuesses(const Program& program, PropertyKeyID key) +{ + if (liesInThread(key)) + return { }; + return isPrivateCharCodeAtForGuesses(program, key) ? BuiltinName(BuiltinNames::charCodeAt) : program.builtinNameOf(key); +} + static bool namesReaderOfCharactersForGuesses(const Program& program, PropertyKeyID key) { - return key == program.keyOf(BuiltinNames::charCodeAt) || key == program.keyOf(BuiltinNames::charAt) || key == program.keyOf(BuiltinNames::codePointAt); + return key == program.keyOf(BuiltinNames::charCodeAt) || key == program.keyOf(BuiltinNames::charAt) || key == program.keyOf(BuiltinNames::codePointAt) || isPrivateCharCodeAtForGuesses(program, key); } static void groupRootsThatHandNamesOnForGuesses(BodySpeculationCache& kept, uint32_t numberOfValues) @@ -756,8 +775,9 @@ static void collectValueUses(Walker& walker, const BodyIndex& index, BodySpecula ListInTheMaking views; ListInTheMaking usesOfPlaces; const std::array readersOfCharacters { walker.program().keyOf(BuiltinNames::charCodeAt), walker.program().keyOf(BuiltinNames::charAt), walker.program().keyOf(BuiltinNames::codePointAt) }; + const bool isOfLibrary = isLibraryCode(walker.program(), walker.program().at(walker.currentFunction())); auto namesReaderOfCharactersByKey = [&](PropertyKeyID key) { - return key == readersOfCharacters[0] || key == readersOfCharacters[1] || key == readersOfCharacters[2]; + return key == readersOfCharacters[0] || key == readersOfCharacters[1] || key == readersOfCharacters[2] || (isOfLibrary && isPrivateCharCodeAtForGuesses(walker.program(), key)); }; static constexpr OpcodeSet relevantOpcodesWithoutLoops = inputPassThroughOpcodes | numberProducingOperators | allBitwiseOperators | OpcodeSet { Opcode::PropGet, Opcode::PropSet, Opcode::Call, Opcode::Convert, Opcode::Add }; static constexpr OpcodeSet relevantOpcodesWithLoops = relevantOpcodesWithoutLoops | OpcodeSet { Opcode::ElemSet, Opcode::NewObject, Opcode::Data, Opcode::NewArray, Opcode::ElemGet, Opcode::StrictEq, Opcode::Eq, Opcode::ResultAt, Opcode::Less, Opcode::LessEq, Opcode::Greater, Opcode::GreaterEq }; @@ -874,6 +894,8 @@ static void collectValueUses(Walker& walker, const BodyIndex& index, BodySpecula case Opcode::ElemSet: if (hasLoops) putIntoList(operation.operand(0), operation.as().valueOperand()); + if (uint32_t moved = roots[operation.as().valueOperand().index()]; std::ranges::binary_search(readsInLoops, moved) && !index.isParameter(ValueID(moved)) && index.opcodeOf(ValueID(moved)) == Opcode::ElemGet && index.loopOf(index.blockOf(ValueID(moved))) == index.loopOf(BlockID(number))) + kept.loopsMovingElements.append(index.loopOf(BlockID(number)).index()); break; case Opcode::NewObject: if (hasLoops) @@ -1020,6 +1042,7 @@ static void collectValueUses(Walker& walker, const BodyIndex& index, BodySpecula }; std::ranges::sort(kept.loopsWithElementArithmetic); std::ranges::sort(kept.loopsWithElementComparisons); + std::ranges::sort(kept.loopsMovingElements); bool someRootIsReckonedWith = !kept.rootsFromInFrontThatLoopsReckonWith.isEmpty() || !kept.rootsMadeInTurnsThatLoopsReckonWith.isEmpty() || !kept.rootsFromInFrontThatLoopsMakeWholeNumbersOf.isEmpty() || !kept.rootsReckonedWithAtWrittenPlacesInLoops.isEmpty() || !kept.rootsThatOperatorsOutsideLoopsReckonWith.isEmpty(); if (uses.isEmpty() && casts.isEmpty() && !someRootIsReckonedWith) return; @@ -1116,7 +1139,7 @@ static void collectValueUses(Walker& walker, const BodyIndex& index, BodySpecula if (walker.program().nameClassOf(name) == NameClass::Length) continue; uint32_t place = static_cast(used >> 32); - BuiltinName builtin = liesInThread(name) ? BuiltinName() : walker.program().builtinNameOf(name); + BuiltinName builtin = builtinNameOfUseForGuesses(walker.program(), name); if (!builtin || !std::ranges::contains(walker.program().builtins().classesWithInstanceMember(builtin), BuiltinClassID(BuiltinClasses::String))) placesOfOtherNames.append(place); else if (namesReaderOfCharactersByKey(name) && !kept.readersOfCharactersNotOnlyCalled.contains(name.index())) @@ -1420,6 +1443,41 @@ bool Walker::hasElementArithmeticInLoop(BlockID inputHead) return std::ranges::binary_search(kept.loopsWithElementArithmetic, index.loopOf(inputHead).index()); } +bool Walker::mayVersionLoopUnderHandler(BlockID inputHead) +{ + const BodyIndex& index = *m_frames.last().index; + if (!hasElementArithmeticInLoop(inputHead) && !std::ranges::binary_search(bodySpeculationCache(index).loopsMovingElements, index.loopOf(inputHead).index())) + return false; + if (!isUnderHandlerOtherThanCleanups()) + return true; + if (m_handlerMadeOverCopies || m_numberOfFramesOverHandlersOfCode) + return false; + if (!findValuesOfLoopReadBehindIt(inputHead) || !m_valuesReadBehindLoopFoundLast.readByHandlersOutside.isEmpty()) + return false; + LoopID loop = index.loopOf(inputHead); + return std::ranges::all_of(index.membersOf(loop), [&](BlockID member) { + BodyBlock block = index.block(member); + return !block.hasHandler() || index.isInLoop(block.handler().block(), loop) || std::ranges::none_of(block.handler().arguments(), [&](ValueID argument) { + return argument && index.isInLoop(index.blockOf(argument), loop); + }); + }); +} + +bool Walker::isUnderOuterHandlersAlone() const +{ + if (m_handlerMadeOverCopies || m_numberOfFramesOverHandlersOfCode) + return false; + const VersionedLoop& loop = m_versionedLoops.last(); + for (size_t frameIndex = loop.frameIndex; frameIndex < m_frames.size(); ++frameIndex) { + const Frame& frame = m_frames[frameIndex]; + if (!frame.hasHandler || frame.handlerOnlyClosesIterations) + continue; + if (frameIndex != loop.frameIndex || !frame.index->block(loop.inputHead).hasHandler() || frame.index->block(frame.block).handler().block() != frame.index->block(loop.inputHead).handler().block()) + return false; + } + return true; +} + bool Walker::hasElementComparisonInLoop(BlockID inputHead) { const BodyIndex& index = *m_frames.last().index; @@ -2003,7 +2061,7 @@ static DetailedExpectation expectationByNamesOfTextsForGuesses(Walker& walker, V if (!firstUsed || !turnsCallReaderOfCharactersForGuesses(walker, walker.currentBodyIndex())) return { }; auto classesThatAnswer = [&](PropertyKeyID key) { - BuiltinName name = liesInThread(key) ? BuiltinName() : program.builtinNameOf(key); + BuiltinName name = builtinNameOfUseForGuesses(program, key); return name ? program.builtins().classesWithInstanceMember(name) : std::span(); }; auto textsAnswer = [&](PropertyKeyID key) { @@ -2019,7 +2077,7 @@ static DetailedExpectation expectationByNamesOfTextsForGuesses(Walker& walker, V bool someCallReadsCharacter = std::ranges::any_of(names, [&](PropertyKeyID key) { if (classesThatAnswer(key).size() != 1 || kept.readersOfCharactersNotOnlyCalled.contains(key.index())) return false; - BuiltinMemberLookup found = table.findMember(table.canonicalID(Builtins::StringPrototype), program.builtinNameOf(key)); + BuiltinMemberLookup found = table.findMember(table.canonicalID(Builtins::StringPrototype), builtinNameOfUseForGuesses(program, key)); return found.outcome == BuiltinMemberLookup::Outcome::Found && std::ranges::any_of(table.overloadsOf(table.canonicalID(found.member)), [](const BuiltinOverload& overload) { return overload.code.implementation.inlineBuiltin && readsCharacterOfText(*overload.code.implementation.inlineBuiltin); }); @@ -2067,7 +2125,7 @@ static DetailedExpectation expectationOfWhatLoopReadsByKeyForGuesses(Walker& wal return { }; const Program& program = walker.program(); std::span names = walker.namesUsedOff(input); - if (const BodySpeculationCache& kept = bodySpeculationCache(walker.currentBodyIndex()); names.empty() && !program.at(walker.currentFunction()).has(FunctionFlags::IsChecked) && !kept.rootsOfValuesWithMarks.isEmpty() && std::ranges::binary_search(kept.rootsWhoseElementsUnderIndexesAreAskedInLoops, kept.rootOf(input.index()))) { + if (const BodySpeculationCache& kept = bodySpeculationCache(walker.currentBodyIndex()); names.empty() && !kept.rootsOfValuesWithMarks.isEmpty() && std::ranges::binary_search(kept.rootsWhoseElementsUnderIndexesAreAskedInLoops, kept.rootOf(input.index()))) { if (!hasNonArrayPropertyUse(kept, input)) { walker.count(StatisticID::ArraysSpeculatedFromElementUses); return { walker.types().builtinClass(BuiltinClasses::Array), false }; @@ -2601,6 +2659,7 @@ static DetailedExpectation findWholeExpectationForGuesses(Walker& walker, ValueI static TypeID withoutRecordsOutOfSightForGuesses(Walker& walker, TypeID expected, bool wasSeenToArrive) { static constexpr uint8_t number = static_cast(AnswerKeptInArena::RecordsThatFileWrites); + static_assert(number + 1 == static_cast(AnswerKeptInArena::WrittenRecordCoverage)); enum : uint32_t { AllStay, NoneStays, SomeStay }; TypeArena& types = walker.types(); if (!expected) @@ -3178,15 +3237,22 @@ static bool constructorIsExpectedToBeHandedItsOwnClassUnseenForGuesses(Walker& w return !seen || types.isUninformative(seen) || !types.overlap(seen, objects); } -static NEVER_INLINE DetailedExpectation lessWhatInstanceOfDeniedForGuesses(Walker& walker, ValueID input, DetailedExpectation expectation) +static NEVER_INLINE DetailedExpectation byWhatInstanceOfSaidForGuesses(Walker& walker, ValueID input, DetailedExpectation expectation) { if (!expectation.type || !walker.currentBodyIndex().opcodes().contains(Opcode::InstanceOf)) return expectation; Reference value = walker.referenceOf(input); - std::optional denied = value && value.isValue() ? findKnown(walker, KnownNote::ClassesThatInstanceOfDenied, value) : std::nullopt; - if (!denied) + if (!value || !value.isValue()) return expectation; TypeArena& types = walker.types(); + if (std::optional passed = types.at(expectation.type).hasObjects() ? findKnown(walker, KnownNote::ClassesThatInstanceOfPassed, value) : std::nullopt) { + TypeID asked = types.forLayoutSet(LayoutSetID::fromBits(static_cast(*passed))); + TypeID both = types.intersectionOf(expectation.type, asked); + return { isNeverType(both) ? asked : both, true }; + } + std::optional denied = findKnown(walker, KnownNote::ClassesThatInstanceOfDenied, value); + if (!denied) + return expectation; TypeID left = types.differenceOf(expectation.type, types.forLayoutSet(LayoutSetID::fromBits(static_cast(*denied)))); if (left == expectation.type) return expectation; @@ -3230,7 +3296,7 @@ static NEVER_INLINE DetailedExpectation findAndRememberExpectationForGuesses(Wal remembered = { expectation.type, expectation.isDeclared, expectation.isHeld, true, expectation.restsOnNamesUsedAlone }; if (Vector* rememberedForLook = walker.expectationsRememberedForLook()) rememberedForLook->append(input); - return lessWhatInstanceOfDeniedForGuesses(walker, input, expectation); + return byWhatInstanceOfSaidForGuesses(walker, input, expectation); } static DetailedExpectation findExpectationAsRememberedForGuesses(Walker& walker, ValueID input) @@ -3241,7 +3307,7 @@ static DetailedExpectation findExpectationAsRememberedForGuesses(Walker& walker, DetailedExpectation expectation { remembered.type, remembered.isDeclared, remembered.isHeld, remembered.restsOnNamesUsedAlone }; if (!expectation.type || !walker.currentBodyIndex().opcodes().contains(Opcode::InstanceOf)) return expectation; - return lessWhatInstanceOfDeniedForGuesses(walker, input, expectation); + return byWhatInstanceOfSaidForGuesses(walker, input, expectation); } static ALWAYS_INLINE DetailedExpectation unlessEveryTurnWouldAskItForGuesses(Walker& walker, ValueID input, DetailedExpectation expectation) @@ -3356,6 +3422,33 @@ uint64_t entryParametersOnlyReadByTwoNamesOrMore(const BodyIndex& body) return kept.entryParametersOnlyReadByTwoNamesOrMore; } +uint64_t numbersReadAfterWrites(const BodyIndex& body) +{ + static constexpr OpcodeSet arithmetic = numberProducingOperators | allSums; + BodySpeculationCache& kept = bodySpeculationCache(body); + ensureCachedList(kept, kept.hasNumbersReadAfterWrites, [&] { + BodyBlock entry = body.block(entryBlock); + uint32_t numberOfParameters = std::min(entry.numberOfParameters(), 64); + uint64_t readBefore = 0; + uint64_t readAfter = 0; + bool hasWritten = false; + for (uint32_t place = 0; place < entry.numberOfValues(); ++place) { + BodyValue operation = body.value(ValueID(entry.firstValue().index() + place)); + if (!arithmetic.contains(operation.opcode())) { + hasWritten |= !declaredBoundInTable(operation.opcode()).isUnobservable(); + continue; + } + operation.forEachOperand([&](ValueID operand, uint32_t) { + ValueID taken = operand ? body.stripCasts(operand) : ValueID(); + if (taken && taken.index() < numberOfParameters) + (hasWritten ? readAfter : readBefore) |= uint64_t { 1 } << taken.index(); + }); + } + kept.numbersReadAfterWrites = readAfter & ~readBefore; + }); + return kept.numbersReadAfterWrites; +} + bool isEntryParameterUninformative(Walker& walker, FunctionID callee, uint32_t entryParameter) { const LinkTables& link = walker.facts().linked(); @@ -3540,6 +3633,33 @@ static bool builtinClassGivesNameCalledOnUnknownReceiverForGuesses(Walker& walke return !!findMethodThatSomeClassGivesUnderKey(walker, walker.typeOf(receiver), index.value(read).key(), static_cast(operation.argumentsOperands().size())); } +static BuiltinID methodOfHostInstance(Walker& walker, const BodyValue& call, ValueID called) +{ + const BodyIndex& index = walker.currentBodyIndex(); + if (!walker.isInLoop() || call.opcode() != Opcode::Call || !called || index.isParameter(called) || index.opcodeOf(called) != Opcode::PropGet || index.value(called).isFreeName()) + return { }; + BodyValue read = index.value(called); + Reference receiver = walker.referenceOf(read.objectOperand()); + TypeID proven = receiver ? walker.typeOf(receiver) : TypeID(); + if (!proven || !walker.types().at(proven).isObject()) + return { }; + const BuiltinTable& table = walker.program().builtins(); + const ClassPart* part = singleBuiltinClassOf(walker.types(), walker.types().at(proven)); + BuiltinName name = part && table.at(part->builtinClass).origin == BuiltinOrigin::Host ? walker.program().builtinNameOf(read.key()) : BuiltinName(); + if (!name || table.isRedefinedBelow(part->builtinClass, name)) + return { }; + BuiltinID prototype; + for (BuiltinClassID current = part->builtinClass; current && !prototype; current = table.at(current).parent) + prototype = table.at(current).prototype; + if (!prototype) + return { }; + BuiltinMemberLookup found = table.findMember(prototype, name); + if (found.outcome != BuiltinMemberLookup::Outcome::Found || table.at(table.canonicalID(found.member)).kind != BuiltinKind::Method) + return { }; + BuiltinID method = table.canonicalID(found.member); + return isTestableBuiltin(table, method) ? method : BuiltinID(); +} + static DetailedExpectation expectationOfCalleeForGuesses(Walker& walker, const BodyValue& call, uint32_t operandIndex) { if (!walker.has(Mechanism::DirectCalls) || !call.operandUses(operandIndex).contains(OperandUse::Calls)) @@ -3585,6 +3705,8 @@ static DetailedExpectation expectationOfCalleeForGuesses(Walker& walker, const B const BodyIndex& index = walker.currentBodyIndex(); ValueID called = index.stripCasts(call.operand(operandIndex)); bool isFreeName = called && !index.isParameter(called) && index.opcodeOf(called) == Opcode::PropGet && index.value(called).isFreeName() && call.opcode() == Opcode::Call; + if (BuiltinID method = walker.has(Mechanism::BuiltinCalls) ? methodOfHostInstance(walker, call, called) : BuiltinID()) + return { walker.types().forBuiltin(method), true }; if (FunctionID written = functionWrittenForFieldThatCalleeIsReadOutOf(walker, call)) return { walker.types().forFunction(written), true }; if (FunctionID callee = walker.facts().held().calleeAt(walker.currentFunction(), call.id()); callee && !(builtinClassGivesNameCalledOnUnknownReceiverForGuesses(walker, call))) @@ -3812,7 +3934,7 @@ static std::optional chooseTestOfTypeAnewForGuesses(Walker& walker, static std::optional chooseTestOfTypeForGuesses(Walker& walker, TypeID expected) { constexpr uint8_t numberOfType = static_cast(AnswerKeptInArena::TestOfGuess); - constexpr uint8_t numberOfLayouts = numberOfType + 1; + constexpr uint8_t numberOfLayouts = static_cast(AnswerKeptInArena::LayoutsOfGuessTest); TypeArena& types = walker.types(); if (types.keeps() != ArenaKeeps::UntilReset) return chooseTestOfTypeAnewForGuesses(walker, expected); @@ -3867,9 +3989,10 @@ static ALWAYS_INLINE std::optional chooseLookAtStructureOfClassForG return chosen; } -static ALWAYS_INLINE TypeID partWhoseClassesHaveGuardsForGuesses(Walker& walker, TypeID expected) +template +static ALWAYS_INLINE TypeID filterInstancesForGuesses(Walker& walker, TypeID expected, const IsKept& isKept) { - static constexpr uint8_t number = static_cast(AnswerKeptInArena::PartWhoseClassesHaveGuards); + static constexpr uint8_t number = static_cast(answerKept); TypeArena& types = walker.types(); Type type = types.at(expected); if (!type.hasObjects() || !type.hasOnlyLayouts() || type.hasOnlyClosures() || holdsOneMember(type.layouts)) @@ -3879,7 +4002,7 @@ static ALWAYS_INLINE TypeID partWhoseClassesHaveGuardsForGuesses(Walker& walker, Vector left; bool someIsTakenOut = false; for (LayoutID layout : membersOf(types, type.layouts)) { - if (program.at(layout).kind == LayoutKind::Instance && !walker.facts().isEvaluatedAtMostOnce(declaredLayoutOf(program, layout))) + if (program.at(layout).kind == LayoutKind::Instance && !isKept(declaredLayoutOf(program, layout))) someIsTakenOut = true; else left.append(layout); @@ -3901,6 +4024,20 @@ static ALWAYS_INLINE TypeID partWhoseClassesHaveGuardsForGuesses(Walker& walker, return part ? part : expected; } +static TypeID partWhoseClassesHaveGuardsForGuesses(Walker& walker, TypeID expected) +{ + return filterInstancesForGuesses(walker, expected, [&](LayoutID classLayout) { + return walker.facts().isEvaluatedAtMostOnce(classLayout); + }); +} + +static TypeID partWithNamedMembersAboveForGuesses(Walker& walker, TypeID expected) +{ + return filterInstancesForGuesses(walker, expected, [&](LayoutID classLayout) { + return walker.facts().isEvaluatedAtMostOnce(classLayout) && hasOnlyNamedMembersAbove(walker, classLayout); + }); +} + static ALWAYS_INLINE TypeID partThatOwnAllocationMakesForGuesses(Walker& walker, const BodyIndex& index, ValueID operand, BlockID block, TypeID expected) { const Function& own = walker.program().at(walker.currentFunction()); @@ -4859,12 +4996,27 @@ static void noteWhatReadOfViewGivesForGuesses(Walker& walker, const BodyValue& r noteKnown(walker, KnownNote::NumberReadOutOfList, view, walker.referenceOf(read.propertyOperand()), 0, LocationMasks(heapsUnderNumberReadOutOfViewForGuesses), given.bits()); } -static void buildReadOfElementOfNumbersForGuesses(Walker& walker, const BodyValue& read, Reference list, Origin origin, BlockID closedBlock) +static bool isReadForBitsAlone(const BodyIndex& index, ValueID read, uint32_t endOfStretch) +{ + for (ValueID value = read; ValueID user = index.onlyUsingValue(value); value = user) { + if (user.index() >= endOfStretch) + return false; + if (allBitwiseOperators.contains(index.opcodeOf(user))) + return true; + if (!opcodesThatStateTypes.contains(index.opcodeOf(user)) || index.value(user).operand(0) != value) + return false; + } + return false; +} + +static void buildReadOfElementOfNumbersForGuesses(Walker& walker, const BodyValue& read, Reference list, Origin origin, BlockID closedBlock, Reference* bits) { Reference position = walker.referenceOf(read.propertyOperand()); std::optional inside = walker.findWhatHoldsPositionInsideList(list, position); BlockID meetingBlock = walker.addBlock(); Reference number = walker.addParameter(meetingBlock, TypeTable::number, Representation::Double); + if (bits) + *bits = walker.addParameter(meetingBlock, TypeTable::int32, Representation::Int32); for (ArrayStorage kind : { ArrayStorage::Int32, ArrayStorage::Double }) { BlockID otherKind = kind == ArrayStorage::Double ? closedBlock : walker.addBlock(); Draft hasKind(Opcode::HasArrayStorage, origin); @@ -4903,7 +5055,15 @@ static void buildReadOfElementOfNumbersForGuesses(Walker& walker, const BodyValu if (present) element.setIsPresent(*present); Reference loaded = walker.build(WTF::move(element)); - walker.jumpTo(meetingBlock, singleElementSpan(loaded)); + Vector brought { loaded }; + if (bits && loaded && kind == ArrayStorage::Double) { + Draft conversion(Opcode::ToInt32, origin); + walker.chooseConservativeFields(conversion); + conversion.setValueOperand(loaded); + brought.append(walker.build(WTF::move(conversion))); + } else if (bits) + brought.append(loaded); + walker.jumpTo(meetingBlock, brought.span()); if (walker.hasFailed()) return; if (otherKind != closedBlock) @@ -5217,6 +5377,10 @@ static void openGuessWithTest(Walker& walker, Reference value, const TestOfGuess last = checked; if (last != proven) walker.state().noteCheckedName(last, proven); + if (const Operation& narrowed = code.at(value.id()); narrowed.opcode == Opcode::Narrowed && !code.isConstant(narrowed.valueOperand())) { + if (Reference source = Reference::value(narrowed.valueOperand()); !walker.state().checkedNameOf(source) && walker.representationOf(source) == walker.representationOf(value)) + walker.state().noteCheckedName(source, value); + } } } if (Reference fails = walker.drawWhetherGuessFails(origin)) @@ -5439,6 +5603,7 @@ static void openGuessWithTest(Walker& walker, Reference value, const TestOfGuess } if (walker.hasFailed()) return; + Reference bitsOfElement; if (chosen.asksForElementOfNumbers && proven && !walker.isUnreachable()) { if (!closedBlock) closedBlock = walker.addBlock(BlockFrequency::Rare); @@ -5447,7 +5612,7 @@ static void openGuessWithTest(Walker& walker, Reference value, const TestOfGuess if (chosen.elementInsideView) noteWhatReadOfViewGivesForGuesses(walker, walker.asWalked(index.value(ValueID(first))), proven, buildReadOfElementOfViewBehindTestOfLength(walker, walker.asWalked(index.value(ValueID(first))), proven, origin, closedBlock, chosen.updatesElement)); else - buildReadOfElementOfNumbersForGuesses(walker, walker.asWalked(index.value(ValueID(first))), proven, origin, closedBlock); + buildReadOfElementOfNumbersForGuesses(walker, walker.asWalked(index.value(ValueID(first))), proven, origin, closedBlock, walker.isInLoop() && isReadForBitsAlone(index, ValueID(first), first + size) ? &bitsOfElement : nullptr); if (walker.hasFailed()) return; } @@ -5550,8 +5715,11 @@ static void openGuessWithTest(Walker& walker, Reference value, const TestOfGuess StretchOfGuess inner { end, 0, { } }; inner.orderWordAskedFor = likelyOrderWord; openGuessWithTest(walker, proven, TestOfGuess { walker.typeOf(proven), { }, true }, inputBlock, begin, inner); - } else - walker.walkStretch(inputBlock, begin, end); + } else { + if (bitsOfElement) + walker.bind(ValueID(first), bitsOfElement); + walker.walkStretch(inputBlock, bitsOfElement ? begin + 1 : begin, end); + } if (takesInCallReturnedAtOnce) walker.threadReturnOf(inputBlock); for (ValueID mentioned : valuesMentionedLater) { @@ -5620,9 +5788,9 @@ static uint32_t leastOperationsThatOpenGuessOutsideLoops(const Walker& walker, T return drawn == Walker::ChoiceOfCost::Yes; Type type = walker.types().at(expected); if (type.hasObjects() && !(type.primitives & ~Primitive::UndefinedOrNull) && type.hasOnlyLayouts() && !type.hasOnlyClosures()) - return minimumOperationsPerGuessOfShape; + return walker.isOutsideLoopsOfAsyncBody() ? minimumOperationsPerGuessOfShapeInAsyncBody : minimumOperationsPerGuessOfShape; if (isDeclaredInSource && type.isObject() && singleBuiltinOf(walker.types(), type)) - return minimumOperationsPerGuessOfShape; + return walker.isOutsideLoopsOfAsyncBody() ? minimumOperationsPerGuessOfShapeInAsyncBody : minimumOperationsPerGuessOfShape; if (!isDeclaredInSource) return 0; return leaning && oneOperationPaysForGuess(walker, expected, *leaning, leaningReadsMethodForItsCall) ? 1 : minimumOperationsPerGuess; @@ -6034,8 +6202,223 @@ static bool splitUndefinedOffKeyOfUpdateForGuesses(Walker& walker, const BodyVal return walkStretchBehindTestsOfNumbers(walker, std::span(&asked, 1), block.id(), read.id().index() - block.firstValue().index(), behindUpdate, BlockFrequency::Rare); } +static std::optional writtenIndexOfReadForGuesses(Walker& walker, const BodyIndex& index, const BodyValue& operation, Reference list) +{ + if (operation.opcode() != Opcode::ElemGet || walker.referenceOf(operation.objectOperand()) != list) + return std::nullopt; + ValueID key = index.stripCasts(operation.propertyOperand()); + if (!key || index.isParameter(key) || index.opcodeOf(key) != Opcode::Integer) + return std::nullopt; + int64_t written = index.value(key).as().integer(); + if (written < 0 || written > std::numeric_limits::max()) + return std::nullopt; + TypeID seenByLink = walker.facts().linked().resultOfReadByKey(walker.currentFunction(), operation.id()); + bool holdsNumbers = seenByLink && (walker.types().at(seenByLink).primitives & Primitive::Number) && walker.types().at(seenByLink).hasOnlyPrimitives(Primitive::Number | Primitive::Undefined); + if (!holdsNumbers && !isDeclaredToHoldNumbersAlone(walker.program(), walker.types(), walker.typeOf(list))) + return std::nullopt; + return written; +} + +static bool guardListForReads(Walker& walker, const BodyValue& read) +{ + static constexpr uint32_t fewestReads = 3; + static constexpr OpcodeSet arithmetic { Opcode::Add, Opcode::Sub, Opcode::Mul, Opcode::Div }; + static constexpr OpcodeSet takesNumbersAlone { Opcode::Sub, Opcode::Mul, Opcode::Div }; + if (walker.isInLoop() || walker.guardedListOfReads() || walker.standsInArmOfRowOfShapeTests() || walker.isNestedTooDeeply()) + return false; + const BodyIndex& index = walker.currentBodyIndex(); + Reference list = walker.referenceOf(read.objectOperand()); + if (!list || !list.isValue() || !hasKind(walker.types(), walker.typeOf(list), OperandKind::Array) || isFreshAllocation(walker, list) || !writtenIndexOfReadForGuesses(walker, index, read, list)) + return false; + BodyBlock block = index.block(walker.inputBlockBeingWalked()); + if (!block.holds(read.id())) + block = index.block(index.blockOf(read.id())); + uint32_t firstOfBlock = block.firstValue().index(); + uint32_t begin = read.id().index() - firstOfBlock; + uint32_t limit = limitOfStretchOfGuess(walker, block); + uint32_t first = read.id().index(); + + BitVector isNumber; + auto isKnownNumber = [&](ValueID operand) { + if (operand.index() >= first) + return isNumber.get(operand.index() - first); + Reference built = walker.referenceOf(operand); + TypeID type = built ? walker.typeOf(built) : TypeID(); + return type && (walker.types().at(type).primitives & Primitive::Number) && walker.types().at(type).hasOnlyPrimitives(Primitive::Number | Primitive::Undefined); + }; + Vector operandsAsked; + auto asksOperand = [&](ValueID operand, ValueID other) { + Reference built = operand.index() < first && isKnownNumber(other) ? walker.referenceOf(operand) : Reference(); + if (!built || !built.isValue() || !(walker.types().at(walker.typeOf(built)).primitives & Primitive::Number)) + return false; + if (!operandsAsked.contains(built)) + operandsAsked.append(built); + return true; + }; + auto typeInStretch = [&](ValueID input) -> TypeID { + if (input.index() >= first) + return isNumber.get(input.index() - first) ? TypeID(TypeTable::number) : TypeID(TypeTable::anything); + return provenTypeOfInputForGuesses(walker, input); + }; + uint32_t numberOfReads = 0; + uint32_t numberOfOperandsAsked = 0; + uint32_t end = begin; + int64_t greatest = 0; + BitVector isElement; + auto isElementOfStretch = [&](ValueID operand) { + return operand.index() >= first && isElement.get(operand.index() - first); + }; + for (uint32_t place = begin; place < limit; ++place) { + BodyValue operation = walker.asWalked(index.value(ValueID(firstOfBlock + place))); + Opcode opcode = operation.opcode(); + if (std::optional written = writtenIndexOfReadForGuesses(walker, index, operation, list)) { + ++numberOfReads; + greatest = std::max(greatest, *written); + numberOfOperandsAsked = operandsAsked.size(); + end = place + 1; + isElement.set(place - begin); + } else if (opcodesThatStateTypes.contains(opcode)) { + if (!isKnownNumber(operation.operand(0))) + continue; + if (isElementOfStretch(operation.operand(0))) + isElement.set(place - begin); + } else if (arithmetic.contains(opcode)) { + bool takesNumbers = isKnownNumber(operation.operand(0)) && isKnownNumber(operation.operand(1)); + if (!takesNumbers && (!takesNumbersAlone.contains(opcode) || !(asksOperand(operation.operand(0), operation.operand(1)) || asksOperand(operation.operand(1), operation.operand(0))))) + break; + if (isElementOfStretch(operation.operand(0)) || isElementOfStretch(operation.operand(1))) { + numberOfOperandsAsked = operandsAsked.size(); + end = place + 1; + } + } else if (opcode == Opcode::PropSet && operation.objectOperand().index() < first) { + TypedForm form = typedFormOf(walker, operation, typeInStretch); + walker.takeFormAskedAhead(); + if (form != TypedForm::Built) + break; + continue; + } else if (opcode != Opcode::Integer && opcode != Opcode::Double) + break; + isNumber.set(place - begin); + } + if (numberOfReads < fewestReads) + return false; + operandsAsked.shrink(numberOfOperandsAsked); + Vector, 4> namesOfOperandsAsked; + for (uint32_t place = begin; place < end && !operandsAsked.isEmpty(); ++place) { + index.value(ValueID(firstOfBlock + place)).forEachOperand([&](ValueID operand, uint32_t) { + size_t asked = operand && operand.index() < first ? operandsAsked.find(walker.referenceOf(operand)) : notFound; + if (asked != notFound && !namesOfOperandsAsked.containsIf([&](const auto& named) { return named.first == operand; })) + namesOfOperandsAsked.append({ operand, static_cast(asked) }); + }); + } + + Origin origin = originInFrontOfOperationForGuesses(walker, read); + origin.isTailCall = false; + Walker::StretchInArms stretch = walker.stretchInArms(block.id(), begin, end); + Walker::ArmsThatMeet built; + BlockID asWritten = walker.addBlock(BlockFrequency::Rare); + auto continueIfPasses = [&](Reference test) { + if (!test) + return false; + BlockID passed = walker.addBlock(); + walker.branch(test, passed, asWritten); + walker.continueIn(passed); + return !walker.isUnreachable() && !walker.hasFailed(); + }; + auto walkArm = [&] { + for (uint32_t place = begin; place < end; ++place) + walker.bind(ValueID(firstOfBlock + place), Reference()); + walker.walkStretch(block.id(), begin, end); + walker.listArm(stretch, built); + }; + SetForScope guarded(walker.guardedListOfReads(), list); + walker.recordSpeculation(); + walker.count(StatisticID::ListsGuardedForReads); + BlockID meeting = walker.addBlock(); + bool isAsked = continueIfPasses(askWhetherListIsAsMade(walker, list, origin)); + Reference length = isAsked ? buildArrayLength(walker, list, origin) : Reference(); + Reference isBelow; + if (length && length.isValue()) { + Draft test(Opcode::Less, origin); + walker.chooseConservativeFields(test); + Draft most(Opcode::Integer, origin); + most.setInteger(greatest); + test.setLeftOperand(walker.constant(WTF::move(most))); + test.setRightOperand(length); + test.setForm(operatorFormFor(walker, test)); + isBelow = walker.build(WTF::move(test)); + } + isAsked = isBelow && isBelow.isValue() && continueIfPasses(isBelow); + if (isAsked) { + Draft narrowed(Opcode::Narrowed, origin); + narrowed.setValueOperand(length); + narrowed.setTestOperand(isBelow); + narrowed.setOperandIndex(OperandIndex::Less::right); + narrowed.setIsOnTakenEdge(true); + if (Reference held = walker.build(WTF::move(narrowed)); held && held.isValue()) + noteKnown(walker, KnownNote::LengthOfList, list, { }, 0, placesUnderLengthsOfLists, held.bits()); + } + Vector operandsProven; + for (Reference operand : operandsAsked) { + Reference test = isAsked ? buildTestOfNumber(walker, operand, TypeTable::number, origin) : Reference(); + isAsked = test && test.isValue() && continueIfPasses(test); + operandsProven.append(isAsked ? buildProofOfNumber(walker, operand, test, origin) : Reference()); + isAsked = isAsked && operandsProven.last(); + } + Vector boundBefore; + for (auto [name, asked] : namesOfOperandsAsked) + boundBefore.append(walker.referenceBoundTo(name)); + bool isBehindTests = true; + for (ArrayStorage kind : { ArrayStorage::Double, ArrayStorage::Int32 }) { + Reference isOfKind = isAsked && !walker.isUnreachable() ? askWhetherStorageOfListIsOfKind(walker, list, kind, origin) : Reference(); + if (!isOfKind || !isOfKind.isValue()) + break; + BlockID passed = walker.addBlock(); + BlockID failed = kind == ArrayStorage::Int32 ? asWritten : walker.addBlock(); + walker.branch(isOfKind, passed, failed); + walker.continueIn(passed); + noteKindOfStorageThatTurnFound(walker, list, kind); + for (auto [name, asked] : namesOfOperandsAsked) + walker.bind(name, operandsProven[asked]); + walkArm(); + for (size_t name = 0; name < namesOfOperandsAsked.size(); ++name) + walker.bind(namesOfOperandsAsked[name].first, boundBefore[name]); + if (walker.hasFailed()) + return true; + isBehindTests = kind != ArrayStorage::Int32; + if (isBehindTests) + walker.continueIn(failed); + } + if (walker.hasFailed()) + return true; + if (isBehindTests && !walker.isUnreachable()) + walker.jumpTo(asWritten); + walker.continueIn(asWritten); + { + Walker::ClosedArm closedArm(walker); + walkArm(); + } + if (!walker.hasFailed()) + walker.joinArms(stretch, built, meeting); + return true; +} + +static bool isAnyNumberReadForBits(Walker& walker, const BodyValue& read, TypeID proven) +{ + Type element = walker.types().at(proven); + if ((element.primitives & Primitive::Number) != Primitive::Number || !element.hasOnlyPrimitives(Primitive::Number | Primitive::Undefined) || !element.holdsFractionsWithoutBounds()) + return false; + const BodyIndex& index = walker.currentBodyIndex(); + ValueID user = index.onlyUsingValue(read.id()); + while (user && opcodesThatStateTypes.contains(index.opcodeOf(user))) + user = index.onlyUsingValue(user); + return user && allBitwiseOperators.contains(index.opcodeOf(user)); +} + bool openGuessAtElementOfNumbersOfItsOpcodes(Walker& walker, const BodyValue& read) { + if (read.opcode() == Opcode::ElemGet && walker.has(Mechanism::TypedOperations) && walker.has(Mechanism::UnboxedNumbers) && walker.has(Mechanism::Folding) && guardListForReads(walker, read)) + return true; if (!walker.has(Mechanism::TypedOperations) || !walker.has(Mechanism::UnboxedNumbers) || !walker.has(Mechanism::Folding)) return false; Reference list = walker.referenceOf(read.objectOperand()); @@ -6069,7 +6452,7 @@ bool openGuessAtElementOfNumbersOfItsOpcodes(Walker& walker, const BodyValue& re return false; if (!elementInsideView && findHoistedArrayGuard(walker, read)) return false; - if (TypeID proven = elementInsideView ? TypeID() : walker.facts().linked().resultOfReadByKey(walker.currentFunction(), read.id()); proven && !walker.types().isUninformative(proven)) + if (TypeID proven = elementInsideView ? TypeID() : walker.facts().linked().resultOfReadByKey(walker.currentFunction(), read.id()); proven && !walker.types().isUninformative(proven) && !isAnyNumberReadForBits(walker, read, proven)) return false; ValueID cell = underlyingCellOf(walker.code(), list.id()); if (!walker.code().isConstant(cell) && walker.state().isFresh(Reference::value(cell))) @@ -6434,7 +6817,8 @@ static ALWAYS_INLINE bool nameMayOpenGuessOfKeyOrdersForGuesses(const Walker& wa return linked.keysOfAccessorsExpectedInFields.get(name.index()) || linked.keysOfStaticFields.get(name.index()); } -static constexpr size_t mostCalleesToldApartAtCall = 4; +static constexpr size_t mostCalleesToldApartAtCall = 8; +static constexpr size_t mostCalleesThatAreNoSmallLeaves = 4; static constexpr uint32_t mostOperationsWalkedBehindCallForEachCallee = 8; static constexpr OpcodeSet callsToldApartByCalleeForGuesses { Opcode::Call, Opcode::Construct }; static constexpr size_t atNoFrameOfCopies = std::numeric_limits::max(); @@ -6450,11 +6834,13 @@ TypeID fewClosuresWhoseCopiesPayInLoop(Walker& walker, TypeID held, size_t numbe return { }; size_t numberOfFunctions = 0; bool everyCopyPays = true; + bool everyCalleeIsSmallLeaf = true; forEachFunctionOf(types, named, scopedLambda([&](FunctionID function) { ++numberOfFunctions; everyCopyPays = everyCopyPays && copyOfCalleeCalledInLoopPays(walker, function, numberOfArguments); + everyCalleeIsSmallLeaf = everyCalleeIsSmallLeaf && isSmallLeaf(walker, function, numberOfArguments); })); - return everyCopyPays && numberOfFunctions >= 2 && numberOfFunctions <= mostCalleesToldApartAtCall ? closures : TypeID(); + return everyCopyPays && numberOfFunctions >= 2 && numberOfFunctions <= (everyCalleeIsSmallLeaf ? mostCalleesToldApartAtCall : mostCalleesThatAreNoSmallLeaves) ? closures : TypeID(); } static TypeID fewClosuresThatCalleeIsHeldToBeForGuesses(Walker& walker, const BodyValue& call) @@ -6485,6 +6871,42 @@ static TypeID fewClosuresThatCalleeIsHeldToBeForGuesses(Walker& walker, const Bo return expected.isHeld ? fewClosuresWhoseCopiesPayInLoop(walker, expected.type, call.argumentsOperands().size()) : TypeID(); } +uint32_t findArmStretchEnd(Walker& walker, const BodyValue& call, uint32_t begin, uint32_t limit, Reference told, const ScopedLambda& mayTake) +{ + const BodyIndex& index = walker.currentBodyIndex(); + uint32_t firstOfBlock = call.id().index() - begin; + uint32_t end = begin + 1; + if (isReturnedAtOnceForGuesses(index, call.id())) + return end; + auto callsAnotherOfSeveral = [&](const BodyValue& operation) { + if (!callsToldApartByCalleeForGuesses.contains(operation.opcode()) || !operation.calleeOperand() || operation.calleeOperand() >= call.id()) + return false; + Reference other = walker.referenceOf(operation.calleeOperand()); + if (!other || other == told || !other.isValue()) + return false; + Type type = walker.types().at(walker.typeOf(other)); + return type.isObject() && type.hasOnlyClosures() && !singleFunctionOf(walker.types(), type); + }; + BitVector leansOnResult; + leansOnResult.set(0); + uint32_t numberOfOperations = 0; + for (uint32_t place = begin + 1; place < limit && numberOfOperations < mostOperationsWalkedBehindCallForEachCallee; ++place) { + BodyValue operation = walker.asWalked(index.value(ValueID(firstOfBlock + place))); + if (standsInNoStretchOfGuess(index, operation) || isReturnedAtOnceForGuesses(index, operation.id()) || callsAnotherOfSeveral(operation) || !mayTake(operation)) + break; + numberOfOperations += !opcodesThatStateTypes.contains(operation.opcode()); + bool leans = false; + operation.forEachOperand([&](ValueID operand, uint32_t) { + leans |= operand && (operand >= call.id() ? leansOnResult.get(operand.index() - call.id().index()) : walker.referenceOf(operand) == told); + }); + if (!leans) + continue; + leansOnResult.set(place - begin); + end = place + 1; + } + return end; +} + static bool callEachFunctionOfProvenSetForGuesses(Walker& walker, const BodyValue& call) { if (walker.standsInArmOfRowOfShapeTests() || !walker.has(Mechanism::DirectCalls) || !walker.has(Mechanism::TypedOperations)) @@ -6539,6 +6961,12 @@ static bool callEachFunctionOfProvenSetForGuesses(Walker& walker, const BodyValu })); if (!areFew || functions.size() < (isProven ? 2 : 1)) return false; + size_t numberOfArguments = call.argumentsOperands().size(); + auto callsSmallLeaf = [&](FunctionID function) { + return isSmallLeaf(walker, function, numberOfArguments); + }; + if (functions.size() > mostCalleesThatAreNoSmallLeaves && (call.opcode() != Opcode::Call || !std::ranges::all_of(functions, callsSmallLeaf))) + return false; const CallingConventions& conventions = walker.plan().conventions(); for (FunctionID function : functions) { if (!conventions.exists(conventions.closureEntryOf(function)) || findOnlyCallee(walker, call, types.forFunction(function)) != function || !chooseTestOfGuess(walker, callee, types.forFunction(function))) @@ -6555,35 +6983,9 @@ static bool callEachFunctionOfProvenSetForGuesses(Walker& walker, const BodyValu if (begin >= limit || standsInNoStretchOfGuess(index, call)) return false; - auto callsAnotherOfSeveral = [&](const BodyValue& operation) { - if (!callsToldApartByCalleeForGuesses.contains(operation.opcode()) || !operation.calleeOperand() || operation.calleeOperand() >= call.id()) - return false; - Reference other = walker.referenceOf(operation.calleeOperand()); - if (!other || other == callee || !other.isValue()) - return false; - Type type = types.at(walker.typeOf(other)); - return type.isObject() && type.hasOnlyClosures() && !singleFunctionOf(types, type); - }; - uint32_t end = begin + 1; - if (!isReturnedAtOnceForGuesses(index, call.id())) { - BitVector leansOnResult; - leansOnResult.set(0); - uint32_t numberOfOperations = 0; - for (uint32_t place = begin + 1; place < limit && numberOfOperations < mostOperationsWalkedBehindCallForEachCallee; ++place) { - BodyValue operation = walker.asWalked(index.value(ValueID(firstOfBlock + place))); - if (standsInNoStretchOfGuess(index, operation) || isReturnedAtOnceForGuesses(index, operation.id()) || callsAnotherOfSeveral(operation)) - break; - numberOfOperations += !opcodesThatStateTypes.contains(operation.opcode()); - bool leans = false; - operation.forEachOperand([&](ValueID operand, uint32_t) { - leans |= operand && operand >= call.id() && leansOnResult.get(operand.index() - call.id().index()); - }); - if (!leans) - continue; - leansOnResult.set(place - begin); - end = place + 1; - } - } + uint32_t end = findArmStretchEnd(walker, call, begin, limit, callee, scopedLambda([](const BodyValue&) { + return true; + })); uint32_t first = firstOfBlock + begin; uint32_t size = end - begin; @@ -6614,10 +7016,23 @@ static bool callEachFunctionOfProvenSetForGuesses(Walker& walker, const BodyValu walker.bind(names[name], boundBefore[name]); }; + std::optional originOfCheck = walker.leaveCheckOfWhatContainerHasJustGivenToTestBehind(callee); + auto checkWhatIsLeft = [&](Reference left) { + if (originOfCheck && left && left.isValue() && !walker.isUnreachable()) + walker.buildCheckOfProof(left, walker.code().at(left.id()).type, TypeTable::anything, ProofSource::Contents, *originOfCheck, true); + }; + auto buildTest = [&](Reference value, const TestOfGuess& asked) { + Reference test = buildTestOfGuess(walker, value, asked, origin); + if (originOfCheck && test && test.isValue() && walker.code().at(test.id()).opcode == Opcode::TypeTest) + walker.makeTypeTestAskInFull(test); + return test; + }; TestOfGuess askedFirst { types.forFunction(functions[0]), { } }; - Reference test = buildTestOfGuess(walker, callee, askedFirst, origin); - if (!test || !test.isValue()) + Reference test = buildTest(callee, askedFirst); + if (!test || !test.isValue()) { + checkWhatIsLeft(callee); return false; + } SetForScope standsInArm(walker.numberOfFramesAtRowOfShapeTests(), atNoFrameOfCopies); Reference left = callee; @@ -6626,7 +7041,7 @@ static bool callEachFunctionOfProvenSetForGuesses(Walker& walker, const BodyValu for (size_t place = 0; place < numberTestedFor && !walker.isUnreachable(); ++place) { TestOfGuess asked { types.forFunction(functions[place]), { } }; if (place) - test = buildTestOfGuess(walker, left, asked, origin); + test = buildTest(left, asked); if (!test || !test.isValue()) break; if (!place) { @@ -6652,10 +7067,13 @@ static bool callEachFunctionOfProvenSetForGuesses(Walker& walker, const BodyValu if (!left || walker.hasFailed()) return true; } - if (held && !walker.isUnreachable()) - walker.restartIteration(); - else if (!walker.isUnreachable()) + checkWhatIsLeft(left); + if (!walker.isUnreachable()) { + std::optional closedArm; + if (!isProven) + closedArm.emplace(walker, true); walkArmWith(left); + } if (!walker.hasFailed()) walker.joinArms(stretch, built, meeting); return true; @@ -6891,7 +7309,7 @@ static uint32_t endOfStretchThatStoresIntoRowForGuesses(const Walker::StoresInto return standsInStretch() ? store->second + 1 : behindFirstStore; } -static bool rowChangesWithTurnsOfInnermostLoopForGuesses(const BodyIndex& index, const BodyValue& read, ValueID row) +static bool rowChangesWithTurnsOfInnermostLoopForGuesses(Walker& walker, const BodyIndex& index, const BodyValue& read, ValueID row) { LoopID innermost = index.loopOf(index.blockOf(read.id())); if (!innermost || index.isParameter(row) || index.opcodeOf(row) != Opcode::ElemGet) @@ -6899,7 +7317,7 @@ static bool rowChangesWithTurnsOfInnermostLoopForGuesses(const BodyIndex& index, auto isMadeInLoop = [&](ValueID input) { return index.isInLoop(index.blockOf(index.stripCasts(input)), innermost); }; - return isMadeInLoop(index.value(row).propertyOperand()) || isMadeInLoop(index.value(row).objectOperand()); + return isMadeInLoop(index.value(row).propertyOperand()) || (isMadeInLoop(index.value(row).objectOperand()) && !nameOfListThatEntersLoopAround(walker, index, index.value(row))); } static Reference askWhetherValueIsListItselfForGuesses(Walker& walker, Reference list, Reference value, Origin origin) @@ -6949,7 +7367,7 @@ bool askRowOnceWhereStretchStoresIntoItOfItsOpcodes(Walker& walker, const BodyVa return false; const BodyIndex& index = walker.currentBodyIndex(); ValueID source = index.stripCasts(read.objectOperand()); - if (!rowChangesWithTurnsOfInnermostLoopForGuesses(index, read, source)) + if (!rowChangesWithTurnsOfInnermostLoopForGuesses(walker, index, read, source)) return false; BodyBlock block = index.block(walker.inputBlockBeingWalked()); if (!block.holds(read.id())) @@ -6971,8 +7389,10 @@ bool askRowOnceWhereStretchStoresIntoItOfItsOpcodes(Walker& walker, const BodyVa bool isProvenList = hasKind(types, walker.typeOf(row), OperandKind::Array); TypeID expected = isProvenList ? walker.typeOf(row) : expectationOfOperandForGuesses(walker, read, read.objectOperand()).type; expected = expected ? types.withoutNullish(expected) : TypeID(); + TypeID foundByLink = walker.facts().linked().resultOfReadByKey(walker.currentFunction(), read.id()); + bool holdsWholeNumbers = foundByLink && !isNeverType(foundByLink) && types.at(foundByLink).hasOnlyPrimitives(Primitive::NumberWithoutFraction | Primitive::Undefined) && (types.at(foundByLink).primitives & Primitive::Number); + ArrayStorage kind = holdsWholeNumbers ? ArrayStorage::Int32 : ArrayStorage::Double; if (!expected || !isExpectedToBeListOfNumbers(walker, expected)) { - TypeID foundByLink = walker.facts().linked().resultOfReadByKey(walker.currentFunction(), read.id()); foundByLink = foundByLink ? types.withoutNullish(foundByLink) : TypeID(); if (!foundByLink || !types.at(foundByLink).isNumber()) return false; @@ -7039,7 +7459,7 @@ bool askRowOnceWhereStretchStoresIntoItOfItsOpcodes(Walker& walker, const BodyVa for (Reference other : others) isAsked = isAsked && continueOrRunAsWritten(askWhetherValueIsListItselfForGuesses(walker, list, other, origin)); isAsked = isAsked && continueOrRunAsWritten(askWhetherListIsAsMade(walker, list, origin)); - isAsked = isAsked && continueOrRunAsWritten(askWhetherStorageOfListIsOfKind(walker, list, ArrayStorage::Double, origin, true)); + isAsked = isAsked && continueOrRunAsWritten(askWhetherStorageOfListIsOfKind(walker, list, kind, origin, true)); if (holder) { isAsked = isAsked && continueOrRunAsWritten(askWhetherListIsAsMade(walker, holder, origin)); isAsked = isAsked && continueOrRunAsWritten(askWhetherStorageOfListIsOfKind(walker, holder, ArrayStorage::Contiguous, origin)); @@ -7066,7 +7486,7 @@ bool askRowOnceWhereStretchStoresIntoItOfItsOpcodes(Walker& walker, const BodyVa { SetForScope inStretch(walker.stretchWithArmsAsWritten(), Walker::StretchWithArmsAsWritten { asWritten, restartsIterationOnFailure ? walker.iterationRestartPoint() : walker.markRestartPoint(), walker.depthOfCopies() + 1, source, list, first, size, { }, 0, { }, restartsIterationOnFailure }); if (isList) - noteKindOfStorageThatTurnFound(walker, list, ArrayStorage::Double, true); + noteKindOfStorageThatTurnFound(walker, list, kind, true); if (holder) noteKindOfStorageThatTurnFound(walker, holder, ArrayStorage::Contiguous); walker.walkStretch(block.id(), begin, end); @@ -7344,6 +7764,7 @@ bool tryGuardInvariantBaseAsArray(Walker& walker, const BodyValue& access) } bool buildReadWhereSomeReceiversLackField(Walker&, const BodyValue&); +bool readOfTwoLayoutsBehindTestsOfShape(Walker&, const BodyValue&); static bool isFollowedByPresenceTest(const Walker& walker, const BodyIndex& index, const BodyValue& read) { @@ -7462,7 +7883,7 @@ static GuessOpened openGuessAtOperationForGuesses(Walker& walker, const BodyValu } if (expectation.type && walker.canBuildIterationGuard() && !isCall) { ExpectationOfNumber found { expectation.type, expectation.isDeclared, expectation.isHeld }; - expectation.type = (isReadOfVariableForGuesses(index, operand) ? numbersExpectedOf(walker, operand, found) : numbersExpectedOfSlotThatTurnsStep(walker, operand, found)).type; + expectation.type = (isReadOfVariableForGuesses(index, operand) ? numbersExpectedOf(walker, operand, found) : numberExpectedOfAddendOfPosition(walker, operation, operand, numbersExpectedOfSlotThatTurnsStep(walker, operand, found))).type; } if (operand == object && shouldSpeculateSoleHolder && (walker.types().isUninformative(expectation.type) || !walker.types().at(expectation.type).hasObjects())) expectation = { narrowedByNameAskedForGuesses(walker, walker.types().withOnlyObjects(walker.types().withoutNullish(provenOfObject)), operation.key()), true }; @@ -7547,7 +7968,7 @@ static GuessOpened openGuessAtOperationForGuesses(Walker& walker, const BodyValu stretch = WTF::move(stretchBehindLook); } } - if (TypeID part = lacksOnlyFilledSlot || !chosen ? expectation.type : partWhoseClassesHaveGuardsForGuesses(walker, expectation.type); part != expectation.type) { + auto choosePartWithMoreTypedOperations = [&](TypeID part) { if (std::optional testOfPart = chooseTestOfGuess(walker, value, part)) { StretchOfGuess stretchOfPart = findStretchOfGuess(walker, value, *testOfPart, block, begin, limit); if (stretchOfPart.numberOfTypedOperations > stretch.numberOfTypedOperations) { @@ -7555,6 +7976,15 @@ static GuessOpened openGuessAtOperationForGuesses(Walker& walker, const BodyValu stretch = WTF::move(stretchOfPart); } } + }; + bool provesExpectedClasses = !chosen && value && value.isValue() && walker.types().isSubset(walker.typeOf(value), expectation.type); + if (!lacksOnlyFilledSlot && (chosen || provesExpectedClasses)) { + TypeID withGuards = partWhoseClassesHaveGuardsForGuesses(walker, expectation.type); + if (chosen && withGuards != expectation.type) + choosePartWithMoreTypedOperations(withGuards); + TypeID named = partWithNamedMembersAboveForGuesses(walker, expectation.type); + if (named != withGuards && mostOperationsThatStretchCountsForGuesses(walker, value, false, block, begin, limit, stretch.numberOfTypedOperations + 1) > stretch.numberOfTypedOperations) + choosePartWithMoreTypedOperations(named); } bool asksForNameOfSomeMembers = ((expectation.isDeclared && !expectation.isHeld) || isInLoop) && !lacksOnlyFilledSlot && stretch.numberOfTypedOperations < least && !isUnderPrivateName && operand == object; if (asksForNameOfSomeMembers) { @@ -7575,7 +8005,9 @@ static GuessOpened openGuessAtOperationForGuesses(Walker& walker, const BodyValu return; if (stretch.end > begin) { uint32_t firstValue = block.firstValue().index(); - if (ValueID test = findTestOfEndThatTakesJoinApart(walker, index.value(ValueID(firstValue + stretch.end - 1)), value); test && test.index() < firstValue + limit) + BodyValue last = index.value(ValueID(firstValue + stretch.end - 1)); + ValueID test = last.opcode() == Opcode::Mod ? findTestOfRemainder(walker, last) : findTestOfEndThatTakesJoinApart(walker, last, value); + if (test && test.index() < firstValue + limit) stretch.end = test.index() + 1 - firstValue; } if (isCallbackIteration && stretch.numberOfTypedOperations < least) { @@ -7637,8 +8069,28 @@ static GuessOpened openGuessAtOperationForGuesses(Walker& walker, const BodyValu } if (testsForFunctionWrittenForField) walker.recordFieldCalleeSpeculation(); + Walker::MethodChain chain = walker.methodChainBeingWalked(); + if (isInLoop && isCalleeRead && operand == object && hasKind(walker.types(), chosen->tested, OperandKind::Array)) { + uint32_t firstValue = block.firstValue().index(); + ValueID call = index.onlyUsingValue(operation.id()); + bool isCall = call && block.holds(call) && call.index() - firstValue < limit && index.opcodeOf(call) == Opcode::Call; + ValueID last = isCall ? findLastCallInMethodChain(walker, index.value(call), firstValue + limit) : call; + bool goesOn = last != call; + for (uint32_t number = operation.id().index(); goesOn && number <= last.index(); ++number) + goesOn = !standsInNoStretchOfGuess(index, walker.asWalked(index.value(ValueID(number)))); + if (goesOn) { + stretch.end = std::max(stretch.end, last.index() + 1 - firstValue); + chain = { walker.depthOfCopies(), ValueID(operation.id().index() - 1), last }; + walker.count(StatisticID::MethodChainsKeptInArm); + } + } + SetForScope walksChain(walker.methodChainBeingWalked(), chain); if (opensForWhatStandsBehindBranch) walker.count(StatisticID::ArrayParameterChecksSpanningBranches); + if (opcode == Opcode::PropGet && operand == object && !isCalleeRead && !isInLoop && stretch.end == begin + 1 && readOfTwoLayoutsBehindTestsOfShape(walker, operation)) { + opened = GuessOpened::WithArms; + return; + } openGuessWithTest(walker, value, *chosen, block.id(), begin, stretch, &branchAskedAbout); opened = GuessOpened::WithArms; }); @@ -7671,7 +8123,7 @@ void forEachTypeThatGuessWouldProveAt(Walker& walker, const BodyValue& operation if (isReadOfVariableForGuesses(walker.currentBodyIndex(), operand)) expectation.type = numbersExpectedOf(walker, operand, { expectation.type, expectation.isDeclared, expectation.isHeld }).type; else - expectation.type = numbersExpectedOfSlotThatTurnsStep(walker, operand, { expectation.type, expectation.isDeclared, expectation.isHeld }).type; + expectation.type = numberExpectedOfAddendOfPosition(walker, operation, operand, numbersExpectedOfSlotThatTurnsStep(walker, operand, { expectation.type, expectation.isDeclared, expectation.isHeld })).type; std::optional chosen = chooseTestOfTypeForGuesses(walker, expectation.type); bool isAccessByName = (operation.opcode() == Opcode::PropGet || operation.opcode() == Opcode::PropSet) && !operation.isFreeName() && operand == operation.objectOperand(); if (LayoutID heir = isAccessByName ? layoutKnownByStructureOfClass(walker, types.withoutNullish(expectation.type)) : LayoutID()) { @@ -8232,12 +8684,13 @@ static bool listConditionsOfPlainInstanceForGuesses(const Walker& walker, Layout if (facts.hasMemberUnder(current, ClassSide::Prototype, program.keyOf(BuiltinNames::__proto__)) || !link.membersUnder(current, ClassSide::Prototype, program.keyOf(BuiltinNames::__proto__)).empty()) return false; for (uint32_t place : facts.placesOfMembersUnder(current, ClassSide::Prototype, { })) { - if (!isOfLibrary(facts.membersOf(current)[place].function)) + const auto& member = facts.membersOf(current)[place]; + if (!isOfLibrary(member.function) || (standsBelowClassAsked && member.kind != MethodKind::Method)) return false; } for (uint32_t place : link.membersUnder(current, ClassSide::Prototype, { })) { const LinkedMember& member = link.members[place]; - if (!isOfLibrary(member.everyWriterIsKnown ? member.function : member.written)) + if (!isOfLibrary(member.everyWriterIsKnown ? member.function : member.written) || (standsBelowClassAsked && member.kind != MethodKind::Method)) return false; } bool accessorIsOfProgram = false; @@ -8931,6 +9384,8 @@ Vector findHoistedSpeculations(Walker& walker, BlockID in DetailedExpectation expectation = callsOfValuesForGuesses.contains(operation.opcode()) ? expectationOfCalleeForGuesses(walker, operation, operandIndex) : findDetailedExpectation(walker, operand); + if (expectation.type && !callsOfValuesForGuesses.contains(operation.opcode())) + expectation.type = numberExpectedOfAddendOfPosition(walker, operation, operand, { expectation.type, expectation.isDeclared, expectation.isHeld }).type; if (!expectation.type && isConvertedByView) expectation.type = TypeTable::number; if (!expectation.type && allRelations.contains(operation.opcode()) && operandIndex < 2 && turnsAccessElements()) { @@ -11413,6 +11868,10 @@ static uint32_t findParametersThatBodyLeansOn(Walker& walker, std::spaninFrontOfGuard : typeInFrontOfGuard(input); }; + auto onlyFrontIsNullish = [&](ValueID input) { + Type front = types.at(typeOfNameInFrontOfGuard(input)); + return !front.hasObjects() && (front.primitives & Primitive::UndefinedOrNull) && !(types.at(typeOfNameBehindGuard(input)).primitives & Primitive::UndefinedOrNull); + }; TypesOfInputs inputsBehindGuard(typeOfNameBehindGuard); TypesOfInputs inputsInFrontOfGuard(typeOfNameInFrontOfGuard); @@ -11491,12 +11950,13 @@ static uint32_t findParametersThatBodyLeansOn(Walker& walker, std::span typesBehindParameterGuard(Walker& walker, FunctionID id) std::optional readInLoops; for (uint32_t index = 0; index < numberOfEntryParameters; ++index) { typesInFrontOfGuard.append(walker.typeArrivingAtEntryParameter(index)); - TypeID tested = isTested(index) && !singleLayoutOf(types, types.at(typesInFrontOfGuard.last())) ? typeTestedByParameterGuard(walker, findDetailedExpectation(walker, ValueID(index)).type, typesInFrontOfGuard.last()) : TypeID(); + TypeID tested; + if (isTested(index) && !singleLayoutOf(types, types.at(typesInFrontOfGuard.last()))) { + TypeID expected = findDetailedExpectation(walker, ValueID(index)).type; + if (expected && !types.at(typesInFrontOfGuard.last()).hasObjects() && types.isSubset(typesInFrontOfGuard.last(), expected)) + expected = withoutOmittedNullish(walker, id, index, expected); + tested = typeTestedByParameterGuard(walker, expected, typesInFrontOfGuard.last()); + } tested = typeTestedInPlaceOfWrittenType(walker, id, index, tested); if (tested && !function.has(FunctionFlags::IsChecked) && !types.at(tested).hasObjects()) { TypeID seen = typeHeldOfEntryParameterForGuesses(walker, index); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkInlining.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkInlining.cpp index 82c27a4d3cab8..b70fb24b3457c 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkInlining.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkInlining.cpp @@ -26,6 +26,7 @@ namespace JSC { namespace AOT { namespace Typed { static constexpr uint32_t inlineLimitOutsideLoops = 6; static constexpr uint32_t inlineLimitInLoops = 40; static constexpr uint32_t inlineLimitInLoopsWithoutObjects = 24; +static constexpr uint32_t mostOperationsThatThrow = 12; static constexpr uint32_t inlineWindowOfWrittenCall = 40; static constexpr uint32_t inlineWindowOfWrittenCallInLoops = 120; static constexpr uint32_t inlineLimitOfLeaves = 12; @@ -168,6 +169,32 @@ static bool leafReadsElementsOfHolderFromInFrontOfLoopAtInlining(const Walker& w return false; } +static bool readsParameterElements(const BodyIndex& body) +{ + BodyBlock entry = body.block(entryBlock); + for (uint32_t number = 0; number < body.numberOfBlocks(); ++number) { + for (BodyValue read : body.block(BlockID(number)).values()) { + if (read.opcode() == Opcode::ElemGet && entry.indexOfParameter(body.stripCasts(read.objectOperand()))) + return true; + } + } + return false; +} + +bool canInlineSingleCallLoop(const Walker& walker, FunctionID callee, const PlaceOfCall& place, const std::span* passedOperands) +{ + static constexpr OpcodeSet callsOfValues { Opcode::Call, Opcode::CallDirect, Opcode::CallList, Opcode::Construct }; + const ThinSummary& summary = walker.facts().summaries().at(callee); + if (!place.isInLoop || place.caller != walker.unit().function || !summary.has(SummaryTrait::HasLoops) || summary.numberOfOperations > inlineLimitInLoops || !walker.facts().linked().function(callee).hasSingleCallSite) + return false; + if (walker.standsUnderHandler() || (place.body && place.body->block(place.body->blockOf(place.call)).hasHandler())) + return false; + const BodyIndex* body = walker.facts().bodies().find(callee); + if (!body || body->numberOfLoops() != 1 || body->holdsAnyOf(callsOfValues)) + return false; + return passedOperands ? leafReadsElementsOfHolderFromInFrontOfLoopAtInlining(walker, callee, passedOperands) : readsParameterElements(*body); +} + static bool passesStringLiteralKey(const Walker& walker, FunctionID callee, const std::span* passedOperands, bool inLoopOfCallee = false) { const BodyIndex* body = walker.facts().bodies().find(callee); @@ -224,6 +251,66 @@ static bool calleeAsksForClassOfWhatItIsHandedAtInlining(const Walker& walker, F return false; } +static bool loopOfLeafStoresParameterFields(const Walker& walker, FunctionID callee) +{ + static constexpr OpcodeSet callsOfValues { Opcode::Call, Opcode::CallDirect, Opcode::CallList, Opcode::Construct }; + const BodyIndex* body = walker.facts().bodies().find(callee); + if (!body || body->numberOfLoops() != 1 || body->holdsAnyOf(callsOfValues) || !body->holdsAnyOf(OpcodeSet { Opcode::PropSet })) + return false; + BodyBlock entry = body->block(entryBlock); + for (uint32_t number = 0; number < body->numberOfBlocks(); ++number) { + LoopID loop = body->loopOf(BlockID(number)); + if (!loop) + continue; + if (body->block(body->loop(loop).head).numberOfParameters()) + return false; + for (BodyValue store : body->block(BlockID(number)).values()) { + if (store.opcode() == Opcode::PropSet && !store.isFreeName() && entry.indexOfParameter(body->stripCasts(store.objectOperand()))) + return true; + } + } + return false; +} + +static bool isInRecursionWithoutLoops(const Walker& walker) +{ + return !walker.currentBodyIndex().hasLoops() && walker.isCurrentFrameRecursive(); +} + +static uint32_t operationsThatThrowAtInlining(const BodyIndex& body) +{ + uint32_t number = 0; + for (uint32_t block = 0; block < body.numberOfBlocks(); ++block) { + if (!body.isReachable(BlockID(block)) || body.terminator(BlockID(block)).kind() != TerminatorKind::Throw || body.block(BlockID(block)).hasHandler()) + continue; + for (BodyValue operation : body.block(BlockID(block)).values()) + number += !opcodesThatStateTypes.contains(operation.opcode()); + } + return number; +} + +static bool takesListAtInlining(const Function& callee, unsigned entryParameter); + +static uint32_t readsOfRestListAtInlining(const Walker& walker, FunctionID callee, uint32_t numberOfOperations) +{ + static constexpr uint32_t mostReads = 12; + const Function& function = walker.program().at(callee); + const BodyIndex* body = function.has(FunctionFlags::HasRestParameter) && numberOfOperations > inlineLimitInLoops && numberOfOperations <= inlineLimitInLoops + mostReads ? walker.facts().bodies().find(callee) : nullptr; + if (!body) + return 0; + uint32_t number = 0; + for (uint32_t block = 0; block < body->numberOfBlocks(); ++block) { + for (BodyValue read : body->block(BlockID(block)).values()) { + if (read.opcode() != Opcode::ElemGet && (read.opcode() != Opcode::PropGet || read.isFreeName() || read.key() != walker.program().keyOf(BuiltinNames::length))) + continue; + ValueID list = body->stripCasts(read.objectOperand()); + std::optional parameter = body->isParameter(list) ? body->block(entryBlock).indexOfParameter(list) : std::nullopt; + number += parameter && takesListAtInlining(function, *parameter); + } + } + return number; +} + static bool copyPaysAtInlining(const Walker& walker, FunctionID callee, size_t numberOfOperands, const PlaceOfCall& place, const std::span* passedOperands) { if (copyIsDrawnAwayAtInlining(walker, callee, place)) @@ -233,7 +320,14 @@ static bool copyPaysAtInlining(const Walker& walker, FunctionID callee, size_t n const ThinSummary& summary = walker.facts().summaries().at(callee); uint32_t window = place.isInLoop ? inlineWindowOfWrittenCallInLoops : inlineWindowOfWrittenCall; bool isLeaf = walker.facts().linked().function(callee).weightAsLeaf != LinkedFunction::notLeaf; - if (summary.numberOfOperations - (isLeaf ? summary.numberOfOperationsThatOnlyTurnTruthRound : 0) <= inlineLimitOutsideLoops + numberOfOperands + uint32_t operationsWithoutRestList = summary.numberOfOperations - (place.isInLoop ? readsOfRestListAtInlining(walker, callee, summary.numberOfOperations) : 0); + size_t operations = summary.numberOfOperations - (isLeaf ? summary.numberOfOperationsThatOnlyTurnTruthRound : 0); + size_t mostOperations = inlineLimitOutsideLoops + numberOfOperands; + if (place.isInLoop && operations > mostOperations && operations <= mostOperations + mostOperationsThatThrow) { + if (const BodyIndex* body = walker.facts().bodies().find(callee)) + operations -= std::min(operations, operationsThatThrowAtInlining(*body)); + } + if (operations <= mostOperations && (place.caller == walker.unit().function || walker.operationsCopiedAtWrittenCall() + summary.numberOfOperations <= window || isNoLargerThanCallWithItsOwnCallsAtInlining(summary, numberOfOperands))) return true; uint32_t weight = walker.facts().linked().function(callee).weightAsLeaf; @@ -244,26 +338,33 @@ static bool copyPaysAtInlining(const Walker& walker, FunctionID callee, size_t n return true; if (place.isInLoop && place.caller == walker.unit().function && walker.facts().linked().function(callee).weightAsLeaf <= window && (passesTypedArrayElement(walker, passedOperands) || leafReadsElementsOfHolderFromInFrontOfLoopAtInlining(walker, callee, passedOperands))) return true; - if (place.isInLoop && place.caller == walker.unit().function && summary.numberOfOperations <= inlineLimitInLoops && passesStringLiteralKey(walker, callee, passedOperands)) + if (place.isInLoop && place.caller == walker.unit().function && operationsWithoutRestList <= inlineLimitInLoops && passesStringLiteralKey(walker, callee, passedOperands)) return true; if ((place.caller == walker.unit().function || walker.operationsCopiedAtWrittenCall() + summary.numberOfOperations <= inlineWindowOfWrittenCallInLoops) && summary.has(SummaryTrait::HasLoops) && summary.numberOfOperations <= inlineLimitInLoops && passesStringLiteralKey(walker, callee, passedOperands, true)) return true; + if (canInlineSingleCallLoop(walker, callee, place, passedOperands)) + return true; if (walker.program().at(callee).kind == FunctionKind::Initializer) return summary.numberOfOperations <= inlineLimitInLoops && !summary.has(SummaryTrait::HasLoops); if (place.caller != walker.unit().function && place.isInLoop && walker.canBuildIterationGuard()) return walker.operationsCopiedAtWrittenCall() + summary.numberOfOperations <= window && !summary.has(SummaryTrait::HasLoops); + if (passedOperands && place.isInLoop && place.caller == walker.unit().function && summary.has(SummaryTrait::HasLoops) && summary.numberOfOperations <= inlineLimitInLoops && loopOfLeafStoresParameterFields(walker, callee)) + return true; if (place.caller == walker.unit().function && place.isInLoop && summary.has(SummaryTrait::HasLoops) && summary.numberOfOperations <= inlineLimitInLoops) { uint32_t size = sizeOfCopyThatWritesOutLoopsOverListsHandedIn(walker, callee); return size && size <= inlineWindowOfWrittenCallInLoops; } if (place.caller != walker.unit().function && place.caller == walker.currentFunction() && summary.has(SummaryTrait::HasLoops) && walker.program().at(callee).has(FunctionFlags::IsAlwaysInlined)) return walker.operationsCopiedAtWrittenCall() + summary.numberOfOperations <= window; + if (!place.isInLoop && isLeaf && place.caller == walker.unit().function && place.caller == walker.currentFunction() && summary.numberOfOperations <= inlineLimitInLoopsWithoutObjects && !summary.has(SummaryTrait::HasLoops) + && operationsFitWindowOfCallWrittenInLoop(walker, summary.numberOfOperations) && isInRecursionWithoutLoops(walker)) + return true; if (!place.isInLoop || summary.has(SummaryTrait::HasLoops)) return false; if (place.caller != walker.unit().function && (place.caller != walker.currentFunction() || (!walker.isInLoopOfBodyAtWork() && !walker.program().at(callee).has(FunctionFlags::HasRestParameter)) || walker.operationsCopiedAtWrittenCall() + summary.numberOfOperations > window)) return false; - return summary.numberOfOperations <= inlineLimitInLoopsWithoutObjects || (summary.numberOfOperations <= inlineLimitInLoops && (copyMayLetObjectFallAwayAtInlining(walker, callee, passedOperands) || calleeAsksForClassOfWhatItIsHandedAtInlining(walker, callee))); + return summary.numberOfOperations <= inlineLimitInLoopsWithoutObjects || (operationsWithoutRestList <= inlineLimitInLoops && (copyMayLetObjectFallAwayAtInlining(walker, callee, passedOperands) || calleeAsksForClassOfWhatItIsHandedAtInlining(walker, callee))); } static bool copyFillsObjectMadeForItAtInlining(const Walker& walker, const Draft& site, FunctionID callee, const PlaceOfCall& place) @@ -712,7 +813,7 @@ static bool textOnlyFillsObjectAtInlining(const Walker& walker, FunctionID funct return true; } -static constexpr OpcodeSet operationsOfTextThatFillsObjectBehindSuperCallAtInlining = operationsOfTextThatOnlyFillsObjectAtInlining | OpcodeSet { Opcode::Absent, Opcode::GetSuperBase, Opcode::SuperConstruct, Opcode::NewRuntimeError }; +static constexpr OpcodeSet operationsOfTextThatFillsObjectBehindSuperCallAtInlining = operationsOfTextThatOnlyFillsObjectAtInlining | OpcodeSet { Opcode::Absent, Opcode::GetSuperBase, Opcode::SuperConstruct, Opcode::NewRuntimeError, Opcode::IsUnlockedInstance }; static constexpr OpcodeSet operationsThatDoMoreThanFillObjectBehindSuperCallAtInlining = opcodesWhere([](Opcode opcode) { return !operationsOfTextThatFillsObjectBehindSuperCallAtInlining.contains(opcode); }); static bool textFillsObjectBehindSuperCallAtInlining(const Walker& walker, FunctionID function) @@ -723,7 +824,7 @@ static bool textFillsObjectBehindSuperCallAtInlining(const Walker& walker, Funct for (uint32_t number = 0; number < body->numberOfBlocks(); ++number) { BodyBlock block = body->block(BlockID(number)); BodyTerminator end = block.terminator(); - bool asksForType = end.kind() == TerminatorKind::Branch && !body->isParameter(end.operand(0)) && body->opcodeOf(end.operand(0)) == Opcode::TypeTest; + bool asksForType = end.kind() == TerminatorKind::Branch && !body->isParameter(end.operand(0)) && OpcodeSet { Opcode::TypeTest, Opcode::IsUnlockedInstance }.contains(body->opcodeOf(end.operand(0))); if (block.hasHandler() || (end.kind() != TerminatorKind::Jump && end.kind() != TerminatorKind::Return && end.kind() != TerminatorKind::Throw && !asksForType)) return false; for (BodyValue operation : block.values()) { @@ -784,6 +885,32 @@ static bool copyOpensWhatWasJustMadeAtInlining(const Walker& walker, FunctionID return false; } +static bool copyKeepsReceiverLocalAtInlining(const Walker& walker, FunctionID callee, const PlaceOfCall& place, const OperandsInSightAtInlining* inSight) +{ + const ThinSummary& summary = walker.facts().summaries().at(callee); + if (!inSight || !place.isInLoop || place.caller == walker.unit().function || place.caller != walker.currentFunction() || summary.has(SummaryTrait::HasLoops) || summary.numberOfOperations > inlineLimitInLoopsWithoutObjects) + return false; + if (walker.operationsCopiedAtWrittenCall() + summary.numberOfOperations > inlineWindowOfWrittenCallInLoops) + return false; + std::optional thisParameter = walker.program().at(callee).thisParameterIndex(); + Reference receiver = thisParameter && *thisParameter < inSight->passedOperands.size() ? inSight->passedOperands[*thisParameter] : Reference(); + if (!receiver || !receiver.isValue()) + return false; + ValueID cell = underlyingCellOf(walker.code(), receiver.id()); + return !walker.code().isConstant(cell) && walker.code().at(cell).opcode == Opcode::NewObject && !walker.program().at(walker.code().at(cell).layout()).isEnvironment() && walker.state().isFresh(Reference::value(cell)); +} + +static bool writesOutTurnsAtInlining(const Walker& walker, const Draft& site, FunctionID callee, const PlaceOfCall& place) +{ + bool isInCopy = place.caller != walker.unit().function; + if (!walker.facts().summaries().at(callee).has(SummaryTrait::HasLoops) || (!place.isInLoop && (isInCopy || stepsForOfAtInlining(walker, callee)))) + return false; + if (isInCopy && !walker.program().at(callee).hasAny({ FunctionFlags::HasRestParameter, FunctionFlags::HasHiddenRest })) + return false; + std::optional turns = turnsOfCopyWrittenOutAt(walker, site, callee, place); + return turns && (!isInCopy || operationsFitWindowOfCallWrittenInLoop(walker, turns->text.numberOfOperations)); +} + static bool callAdmitsCopyAtInlining(const Walker& walker, const Draft& site, FunctionID callee, const PlaceOfCall& place, const OperandsInSightAtInlining* inSight) { auto copyPays = [&](size_t numberOfOperands) { @@ -796,8 +923,9 @@ static bool callAdmitsCopyAtInlining(const Walker& walker, const Draft& site, Fu || copyPaysForClosureThatCalleeOnlyCallsAtInlining(walker, callee, place, inSight ? &inSight->passedOperands : nullptr) || copyOfNakedConstructorPaysAtInlining(walker, callee, place, inSight ? &inSight->passedOperands : nullptr) || copyOpensWhatWasJustMadeAtInlining(walker, callee, place, inSight ? &inSight->passedOperands : nullptr) + || copyKeepsReceiverLocalAtInlining(walker, callee, place, inSight) || ((!inSight || inSight->callsParameterOfCopy) && isProfitableAsPassedClosure(walker, callee, place)) - || (place.caller == walker.unit().function && walker.facts().summaries().at(callee).has(SummaryTrait::HasLoops) && (place.isInLoop || !stepsForOfAtInlining(walker, callee)) && turnsOfCopyWrittenOutAt(walker, site, callee, place)); + || writesOutTurnsAtInlining(walker, site, callee, place); }; if (copyFillsObjectMadeForItAtInlining(walker, site, callee, place)) return true; @@ -866,6 +994,11 @@ static bool bodyOfUnitTakesCopyOfItselfAtInlining(const Walker& walker, const Dr return bodyOfUnitCallsItselfAtInlining(walker, site, callee) && walker.facts().summaries().at(callee).numberOfOperations <= inlineLimitInLoopsWithoutObjects; } +static bool isLibraryFunctionInRecursion(const Walker& walker, FunctionID callee) +{ + return isLibraryCode(walker.program(), walker.program().at(callee)) && isInRecursionWithoutLoops(walker); +} + bool mayCopyCallee(const Walker& walker, const Draft& site, FunctionID callee, const PlaceOfCall& written) { bool callsItself = bodyOfUnitTakesCopyOfItselfAtInlining(walker, site, callee); @@ -890,7 +1023,7 @@ bool shouldCopyCallee(const Walker& walker, const Draft& site, FunctionID callee { OperandsInSightAtInlining inSight { passedOperands, callsParameterOfCopy, false }; bool callsItself = bodyOfUnitTakesCopyOfItselfAtInlining(walker, site, callee); - PlaceOfCall place { walker.currentFunction(), walker.isInLoop() || callsItself }; + PlaceOfCall place { walker.currentFunction(), walker.isInLoop() || callsItself || isLibraryFunctionInRecursion(walker, callee) }; return walker.has(Mechanism::Inlining) && opcodesOfInlining.contains(site.opcode) && callAdmitsCopyAtInlining(walker, site, callee, place, &inSight) && (callsItself || !walker.isBeingWalked(callee)) && mayBeWalkedAtInlining(walker, callee, entersBodyOfCoroutineAtInlining(site)); } @@ -1339,6 +1472,14 @@ static bool copyLearnsOfThisAtInlining(Walker& walker, const BodyValue& call, Fu return !handed || !own || !walker.types().isSubset(walker.types().withoutNullish(own), handed); } +bool isSmallLeaf(const Walker& walker, FunctionID callee, size_t numberOfArguments) +{ + const LinkedFunction& linked = walker.facts().linked().function(callee); + if (linked.weightAsLeaf == LinkedFunction::notLeaf || linked.weightAsLeaf > inlineLimitOfLeaves + numberOfArguments) + return false; + return !linked.result || (walker.types().at(linked.result).primitives & ~cellPrimitives); +} + bool inlineCalleeOfItsOpcodes(Walker& walker, const BodyValue& call) { if (callsOfListsAtInlining.contains(call.opcode())) @@ -1360,14 +1501,15 @@ bool inlineCalleeOfItsOpcodes(Walker& walker, const BodyValue& call) if (!callee) return false; - if (call.opcode() == Opcode::Call && walker.standsAtFrameOfRowOfShapeTests() && !copyLearnsOfThisAtInlining(walker, call, callee)) + if (call.opcode() == Opcode::Call && walker.standsAtFrameOfRowOfShapeTests() && !copyLearnsOfThisAtInlining(walker, call, callee) + && !(walker.isInLoop() && isSmallLeaf(walker, callee, call.as().argumentsOperands().size()))) return false; Draft site = walker.draftOf(call); const BodyIndex& index = walker.currentBodyIndex(); Vector passedOperands = collectPassedOperands(site, walker.program().at(callee)); bool callsParameterOfCopy = call.opcode() == Opcode::Call && !walker.walksOwnBodyOfUnit() && index.block(entryBlock).indexOfParameter(index.stripCasts(call.as().calleeOperand())); OperandsInSightAtInlining inSight { passedOperands.span(), callsParameterOfCopy, calleeStandsBehindBuiltin }; - if (callAdmitsCopyAtInlining(walker, site, callee, { walker.currentFunction(), walker.isInLoop() || bodyOfUnitTakesCopyOfItselfAtInlining(walker, site, callee), &index, call.id() }, &inSight) && walker.walkInline(call, callee, passedOperands.span(), &site)) + if (callAdmitsCopyAtInlining(walker, site, callee, { walker.currentFunction(), walker.isInLoop() || bodyOfUnitTakesCopyOfItselfAtInlining(walker, site, callee) || isLibraryFunctionInRecursion(walker, callee), &index, call.id() }, &inSight) && walker.walkInline(call, callee, passedOperands.span(), &site)) return true; if (calleeStandsBehindBuiltin || walker.hasFailed()) return false; @@ -1603,6 +1745,49 @@ bool Walker::walkInline(const BodyValue& call, FunctionID callee, std::span> index && !isUnreachable(); ++index) { + if (!(readAfterWrites >> index & 1) || !entryArguments[index].isValue() || function.entryParameterKind(index) != EntryParameter::Parameter) + continue; + TypeID arriving = ownTypeOf(entryArguments[index]); + if (!m_types.at(arriving).isJavaScriptValue() || m_types.at(arriving).isNumber() || !m_types.overlap(arriving, TypeTable::number)) + continue; + const DeclaredType* declared = function.declaredEntryParameter(index); + TypeID expected = declared && !declared->isEstablished() ? declared->written() : TypeID(); + call.forEachOperand([&](ValueID operand, uint32_t) { + if ((!expected || m_types.isUninformative(expected)) && referenceOf(operand) == entryArguments[index]) + expected = findExpectation(*this, operand).type; + }); + if (!expected || !m_types.at(expected).isNumber()) + continue; + Draft test(Opcode::TypeTest, originOfCall); + test.setValueOperand(entryArguments[index]); + test.setTestedType(TypeTable::number); + Reference passes = build(WTF::move(test)); + if (!passes) + return true; + recordSpeculation(); + count(StatisticID::InlineeNumberGuards); + bool isAskedInFront = tryBranchInPreheader(passes, true, IterationPurity::EffectFree); + if (!isAskedInFront) { + BlockID restarts = addBlock(BlockFrequency::Rare); + BlockID passed = addBlock(); + branch(passes, passed, restarts); + continueIn(restarts); + if (!isUnreachable()) + restartIteration(); + continueIn(passed); + if (m_hasFailed) + return true; + } + SpecializedPreheaderScope preheaderScope(*this, isAskedInFront, IterationPurity::EffectFree); + Draft narrowed(Opcode::Narrowed, originOfCall); + narrowed.setValueOperand(entryArguments[index]); + narrowed.setTestOperand(passes); + narrowed.setOperandIndex(OperandIndex::TypeTest::value); + narrowed.setIsOnTakenEdge(true); + entryArguments[index] = build(WTF::move(narrowed)); + } bool secondWalkIsRareAndKeepsSites = false; if (!whereGuardFails && copyMayLeaveForCall && !m_walksCopyWhereItsGuardFailed && isInLoop() && !canRestartIteration() && !mayLeaveCopyForCall() && !inlinedHeadBlocks && !site.isTailCall && !site.resultIsReturnedAtOnce && facts().bodies().at(callee).numberOfBlocks() == 1 && !isUnreachable()) { @@ -1807,7 +1992,12 @@ bool Walker::walkInline(const BodyValue& call, FunctionID callee, std::span isArmOfDispatch; + if (standsInArmOfRowOfShapeTests()) + isArmOfDispatch.emplace(*this); + wasWalkedInLine = walkInline(call, callee, arguments, writtenOut); + } m_walksCopyWhereItsGuardFailed = false; for (uint32_t number = firstBlockOfSecondWalk; secondWalkIsRareAndKeepsSites && number < m_code.numberOfBlocks(); ++number) m_code.markRareBlockThatKeepsSites(BlockID(number)); @@ -1951,7 +2141,7 @@ bool Walker::goOnBehindReturnsOfCopy(const BodyValue& call, const Function& func } } Vector fieldsThatMeet; - if (unitHasMadeScopes()) { + if (unitHasMadeScopes() || !m_fieldsOfScopesOfHeads.isEmpty()) { Vector states; for (const ReturnOfCopy& returned : returns) states.append(&returned.state); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkIteration.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkIteration.cpp index 1a6ae518856ee..4677bc2ff8e16 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkIteration.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkIteration.cpp @@ -23,6 +23,7 @@ #include "AOTWalkLoopsOverGenerators.h" #include "AOTWalkKnown.h" #include "AOTWalkListReadsInLoops.h" +#include "AOTWalkNumberGuesses.h" #include "AOTWalkResultsInParts.h" #include "AOTWalkStrings.h" #include "AOTWalkTextBuilders.h" @@ -361,7 +362,7 @@ static bool isUnseenBetweenStepsOfPattern(Opcode opcode) return opcode != Opcode::Builtin && !operandsMayRunCode(opcode) && bound.alwaysCompletes() && !bound.mayThrowByRule() && !bound.writes.containsAny(orderedHeaps) && !bound.reads.containsAny(orderedHeaps); } -static bool stepsOfPatternStandInRow(const BodyIndex& index, const BodyValue& open) +bool stepsOfPatternStandInRow(const BodyIndex& index, const BodyValue& open) { auto written = open.as(); if (!written.isDestructuring() || written.hasRest() || !written.elementCount()) @@ -578,6 +579,58 @@ static uint32_t endOfRowOfAppendsInIteration(const BodyIndex& index, const BodyB static FunctionID functionWrittenUnderNextOfIteratorOf(Walker&, ValueID iterated); +static bool returnsIteratorMadeForCaller(const Walker& walker, FunctionID function) +{ + const BodyIndex* body = walker.facts().bodies().find(function); + if (!body || walker.program().at(function).isCoroutine()) + return false; + ValueID returned; + for (uint32_t number = 0; number < body->numberOfBlocks(); ++number) { + BodyTerminator end = body->terminator(BlockID(number)); + if (!body->isReachable(BlockID(number)) || (end.kind() != TerminatorKind::Return && end.kind() != TerminatorKind::ReturnCall)) + continue; + if (returned || !end.operand(0)) + return false; + returned = end.operand(0); + while (!body->isParameter(returned) && inputPassThroughOpcodes.contains(body->opcodeOf(returned)) && body->useCount(returned) == 1) + returned = body->value(returned).operand(0); + } + return returned && !body->isParameter(returned) && body->opcodeOf(returned) == Opcode::Call && body->useCount(returned) == 1 && body->value(returned).as().argumentsOperands().empty(); +} + +static std::optional findCollectionOfIterator(Walker& walker, Reference iterator, Reference& collection) +{ + const Code& code = walker.code(); + if (!iterator || !iterator.isValue()) + return std::nullopt; + const Operation& made = code.at(underlyingCellOf(code, iterator.id())); + if (made.opcode != Opcode::CallInline || !made.receiverOperand() || code.isConstant(made.receiverOperand())) + return std::nullopt; + ContentsStepped contents; + switch (made.inlineBuiltin()) { + case InlineBuiltin::ArrayIterator: + contents = ContentsStepped::List; + break; + case InlineBuiltin::MapIterator: + contents = ContentsStepped::Map; + break; + case InlineBuiltin::SetIterator: + contents = ContentsStepped::Set; + break; + default: + return std::nullopt; + } + const BuiltinResult& result = walker.program().builtins().at(made.overload()).result; + collection = Reference::value(made.receiverOperand()); + const ClassPart* classOfCollection = singleBuiltinClassOf(walker.types(), walker.types().at(walker.typeOf(collection))); + const ClassPart* classOfIterator = singleBuiltinClassOf(walker.types(), walker.types().at(walker.typeOf(iterator))); + if (!classOfCollection || !classOfIterator || !result.iteratedOperand || !result.iteratedOperand->isReceiver() || !nobodySeesStepsOfIterators(walker, classOfIterator->builtinClass, false)) + return std::nullopt; + if (contents == ContentsStepped::List && classOfCollection->builtinClass != BuiltinClasses::Array) + return std::nullopt; + return ClassSteppedUnseen { classOfCollection->builtinClass, contents, result.iterationKind }; +} + namespace { enum class WordingOfIteration : uint8_t { Source, Spread, Delegation }; @@ -880,7 +933,7 @@ class BuilderOfIteration { } iterable = m_walker.referenceOf(m_iteratorCallResult); } - OpenedIteration opened = open(iterable); + OpenedIteration opened = m_iteratorReturned ? openReturned(makeIteratorAgain(m_iteratorReturned)) : open(iterable); Vector openedByProtocol { undefined() }; if (keepsStateInMemory) { static_assert(!static_cast(WordOfIteration::Iterator) && static_cast(WordOfIteration::Next) == 1 && static_cast(WordOfIteration::IsDone) == 2); @@ -904,7 +957,7 @@ class BuilderOfIteration { return iteration; } - Walker::IndexedIteration openByProtocolAlone(Reference iterable) + Walker::IndexedIteration openByProtocolAlone(Reference iterable, Reference iteratorReturned = { }) { Walker::IndexedIteration iteration; iteration.canSkipDoneChecks = true; @@ -914,7 +967,7 @@ class BuilderOfIteration { iteration.position = integer(0); iteration.isSteppedByCallsAlone = !laysVerdictOnNext(iterable); bool asksWay = !iteration.isSteppedByCallsAlone && findHelper(IterationHelper::WayToStepIterator) && findHelper(IterationHelper::StepIteratorInWay); - OpenedIteration opened = open(iterable); + OpenedIteration opened = iteratorReturned ? openReturned(iteratorReturned) : open(iterable); iteration.iterator = opened.iterator; iteration.next = opened.next; iteration.goesByDriver = opened.goesByDriver; @@ -923,6 +976,43 @@ class BuilderOfIteration { return iteration; } + std::optional tryOpenOverReturnedIterator(Reference iterable, bool keepsStateInMemory, bool canSkipDoneChecks) + { + if (!goesByMethodsInSight() || !iterable || !iterable.isValue()) + return std::nullopt; + PropertyKeyID key = m_walker.types().wellKnownSymbolKey(WellKnownSymbol::Iterator); + FunctionID function = findMethodInSightOfProvenType(m_walker, m_walker.types().withoutNullish(m_walker.typeOf(iterable)), key); + if (!function || !returnsIteratorMadeForCaller(m_walker, function)) + return std::nullopt; + iterable = requireObjectCoercible(iterable); + Reference method = m_walker.isUnreachable() ? Reference() : readMethodInSightOfProvenObject(m_walker, iterable, key, m_origin); + if (!method) + return m_walker.isUnreachable() ? std::optional(Walker::IndexedIteration()) : std::nullopt; + + m_walker.count(StatisticID::IteratorMethodsCalledDirectly); + Reference iterator = call(method, iterable); + if (m_walker.isUnreachable()) + return Walker::IndexedIteration(); + Reference collection; + std::optional stepped = findCollectionOfIterator(m_walker, iterator, collection); + if (!stepped) + return openByProtocolAlone(iterable, iterator); + m_walker.count(StatisticID::IteratorAllocationsElidedInInlinees); + if (stepped->contents == ContentsStepped::List) { + m_iteratorReturned = iterator; + return openByIndexOrProtocol(collection, *stepped, keepsStateInMemory, canSkipDoneChecks); + } + Walker::IndexedIteration iteration; + iteration.list = collection; + iteration.builtinClass = stepped->builtinClass; + iteration.contents = stepped->contents; + iteration.kind = stepped->kind; + iteration.position = integer(0); + iteration.table = beginByIndex(collection, stepped->contents); + iteration.canSkipDoneChecks = canSkipDoneChecks; + return iteration; + } + Reference tableThatFirstStepBeginsIn(Reference instance, ContentsStepped contents) { return beginByIndex(instance, contents); } void takeStepsOutOfLine(Reference iterable, uint32_t numberOfSteps, Vector& taken) @@ -1159,6 +1249,12 @@ class BuilderOfIteration { RELEASE_ASSERT(stepIndex && *stepIndex < iteration.precomputedSteps.size()); return resultsOf(boolean(false), iteration.precomputedSteps[*stepIndex]); } + if (std::optional turn = iteration.precomputedSteps.isEmpty() ? std::nullopt : integerIn(iteration.position); turn && static_cast(*turn) <= iteration.precomputedSteps.size()) { + bool isDone = static_cast(*turn) == iteration.precomputedSteps.size(); + Reference name = isDone ? undefined() : iteration.precomputedSteps[static_cast(*turn)]; + iteration.position = integer(isDone ? Walker::doneIterationPosition : *turn + 1); + return resultsOf(boolean(isDone), name); + } if (iteration.isOverFlatEntries) return resumeThroughFlatEntries(iteration); if (iteration.isProtocolOnly) { @@ -1202,6 +1298,8 @@ class BuilderOfIteration { Reference keyOfPair = !iteration.isEntryPairElided ? Reference() : iteration.table ? m_walker.addParameter(after, { }, Representation::JSValue) : m_walker.addParameter(after, { }, m_walker.formOfParameter(m_walker.types().integer(Range { 0, Walker::doneIterationPosition }), { })); Reference valueOfPair = iteration.isEntryPairElided ? m_walker.addParameter(after, { }, Representation::JSValue) : Reference(); + if (iteration.hasProtocolFallback) + iteration.elementsProven = { }; stepWithProtocolFallback(iteration, isEffectFree, after); iteration.position = position; iteration.table = table; @@ -1279,7 +1377,7 @@ class BuilderOfIteration { return build(WTF::move(pair)); } - ValueID valueHandedToArithmeticAlone(const BodyValue& resume) + ValueID findStepValue(const BodyValue& resume) { const BodyIndex& index = m_walker.currentBodyIndex(); ValueID stepValue; @@ -1287,7 +1385,30 @@ class BuilderOfIteration { if (operation.opcode() == Opcode::ResultAt && operation.as().resultsOperand() == resume.id() && operation.as().index() == static_cast(IteratorResumeResult::Value)) stepValue = operation.id(); } - return stepValue && index.useCount(resume.id()) == 2 && onlyUserTakesUndefinedForNaNInTypedElements(m_walker, index.value(stepValue)) ? stepValue : ValueID(); + return index.useCount(resume.id()) == 2 ? stepValue : ValueID(); + } + + bool isAddedToNumberForPosition(ValueID stepValue) + { + const BodyIndex& index = m_walker.currentBodyIndex(); + ValueID name = stepValue; + ValueID user = index.onlyUsingValue(name); + while (user && opcodesThatStateTypes.contains(index.opcodeOf(user))) { + name = user; + user = index.onlyUsingValue(user); + } + if (!user || (index.opcodeOf(user) != Opcode::Add && index.opcodeOf(user) != Opcode::Sub)) + return false; + BodyValue sum = index.value(user); + ValueID other = sum.operand(0) == name ? sum.operand(1) : sum.operand(0); + Reference built = other.index() < stepValue.index() ? m_walker.referenceOf(other) : Reference(); + Type type = built ? m_walker.types().at(m_walker.typeOf(built)) : Type(); + return built && (type.primitives & Primitive::Integer) && type.hasOnlyPrimitives(Primitive::Number | Primitive::Undefined) && usesSpeakForInt32(m_walker, user); + } + + bool isTakenForPosition(ValueID stepValue) + { + return stepValue && m_walker.canBuildIterationGuard() && (turnTakesElementForKeyInListReadsInLoops(m_walker, stepValue) || isAddedToNumberForPosition(stepValue)); } bool isFollowedByDoneBranch(const BodyValue& resume) @@ -1349,10 +1470,15 @@ class BuilderOfIteration { m_walker.fail("a path of a threaded step did not visit the end of its block"_s); return true; } - m_valueThatStepHandsToArithmeticAlone = entered.kind == JSC::IterationKind::Values ? valueHandedToArithmeticAlone(resume) : ValueID(); + ValueID stepValue = entered.kind == JSC::IterationKind::Values ? findStepValue(resume) : ValueID(); + m_stepIsTakenForPosition = isTakenForPosition(stepValue); + m_valueThatStepHandsToArithmeticAlone = stepValue && !m_stepIsTakenForPosition && onlyUserTakesUndefinedForNaNInTypedElements(m_walker, index.value(stepValue)) ? stepValue : ValueID(); std::optional insideLength = m_walker.boundsProofAfterComparison(entered.list, inside, length, isInside); Reference given = entered.contents == ContentsStepped::View ? elementOfView(entered.list, inside, insideLength) : stepValueAt(entered, entered.list, inside, isEffectFree, insideLength, entered.isEntryPairElided); m_valueThatStepHandsToArithmeticAlone = { }; + m_stepIsTakenForPosition = false; + if (m_walker.isUnreachable() || m_walker.hasFailed()) + return true; m_walker.indexedIteration(state).position = operatorOnNumbers(Opcode::Increment, inside); if (m_walker.isUnreachable()) return true; @@ -1377,6 +1503,7 @@ class BuilderOfIteration { Walker::IndexedIteration entered = record; if (!isFollowedByDoneBranch(resume) || integerIn(entered.position) || !m_walker.canThreadBranchAfter(resume)) return false; + entered.elementsProven = { }; BlockID finished = addBlock(); BlockID giving = addBlock(); Reference given = m_walker.addParameter(giving, { }, Representation::JSValue); @@ -1828,7 +1955,7 @@ class BuilderOfIteration { Reference readMethod(Reference object, PropertyKeyID key) { - Reference method = goesByMethodsInSight() ? readMethodAboveProvenObject(m_walker, object, key, m_origin) : Reference(); + Reference method = goesByMethodsInSight() ? readMethodForCallAlone(m_walker, object, key, m_origin) : Reference(); return method ? method : read(object, key); } @@ -2235,6 +2362,10 @@ class BuilderOfIteration { m_walker.count(StatisticID::IteratorStepHoleChecksRemoved); } takeStorageAndLengthFromInFrontOfLoop(m_walker, read, *tested); + if (m_stepIsTakenForPosition && read.isInBounds() && isNumericStorage(read.arrayStorage())) { + if (!guardElementPresenceInIteration(m_walker, read, list, position, *tested)) + return { }; + } built = m_walker.build(WTF::move(read)); } else { Draft read = readOfElementAt(list, position, isEffectFree, insideLength); @@ -2807,6 +2938,20 @@ class BuilderOfIteration { OpenedIteration withNextMethod(Reference iterator) { return { iterator, readMethod(iterator, m_walker.program().keyOf(BuiltinNames::next)) }; } + OpenedIteration openReturned(Reference iterator) { return leftToSteps(withNextMethod(requireObject(iterator, RuntimeErrorKind::IteratorNotObject))); } + + Reference makeIteratorAgain(Reference iterator) + { + const Code& code = m_walker.code(); + const Operation& made = code.at(underlyingCellOf(code, iterator.id())); + Draft again = draft(Opcode::CallInline); + again.setInlineBuiltin(made.inlineBuiltin()); + again.setInlineParameters(made.inlineParameters()); + again.setOverload(made.overload()); + again.setReceiverOperand(Reference::value(made.receiverOperand())); + return build(WTF::move(again)); + } + OpenedIteration leftToSteps(OpenedIteration opened) { for (Reference named : { opened.iterator, opened.next }) { @@ -3161,10 +3306,23 @@ class BuilderOfIteration { readStepResult(call(method, iterator, singleElementSpan(sent)), readsValueWhenDone, join); } + bool callReturnInSight(Reference iterator) + { + if (!goesByMethodsInSight()) + return false; + Reference method = readMethodForCallAlone(m_walker, iterator, m_walker.program().keyOf(BuiltinNames::returnKeyword), m_origin); + if (!method || !method.isValue() || m_walker.isUnreachable() || !findOnlyCallee(m_walker, *m_site, method)) + return false; + requireObject(call(method, iterator), RuntimeErrorKind::IteratorResultNotObject); + return true; + } + void callReturn(Reference iterator, ResultOfReturnInIteration expectation) { if (goesByMethodsInSight() && findsNothingUnderNameOfProvenObject(m_walker, iterator, m_walker.program().keyOf(BuiltinNames::returnKeyword))) return; + if (expectation == ResultOfReturnInIteration::MustBeObject && callReturnInSight(iterator)) + return; if (FunctionID closes = expectation == ResultOfReturnInIteration::MustBeObject ? findHelper(IterationHelper::CloseIterator) : FunctionID()) { callHelper(closes, { iterator }); return; @@ -3238,6 +3396,7 @@ class BuilderOfIteration { BlockID m_whereNeitherTextNorListIsStepped; BlockID m_doneBlock; ValueID m_valueThatStepHandsToArithmeticAlone; + bool m_stepIsTakenForPosition { false }; bool m_takesListApart { false }; BlockFrequency m_frequencyOfBlocksAdded { BlockFrequency::Normal }; Reference m_instanceAsMadeForGood; @@ -3245,6 +3404,7 @@ class BuilderOfIteration { uint32_t m_iteratorCallBegin { 0 }; uint32_t m_iteratorCallEnd { 0 }; ValueID m_iteratorCallResult; + Reference m_iteratorReturned; }; } // anonymous namespace @@ -3759,10 +3919,15 @@ static bool openTypedIteration(Walker& walker, const BodyValue& open) if (!isSurelyAsMade && !tested) { if (!isExpectedToStepGenerator(walker, open.id())) builder.expectNoGenerator(); - iteration = builder.openByProtocolAlone(iterable); - if (FunctionID listed = walker.facts().linked().calleeAt(walker.currentFunction(), open.id())) - walker.leaveAnythingAt(listed); - walker.state().invalidateHeap(); + std::optional returned = builder.tryOpenOverReturnedIterator(iterable, way.stepsStandUnderOwnCleanup || !way.closesStandBelowOnlyStep, way.canSkipDoneChecks); + iteration = returned ? WTF::move(*returned) : builder.openByProtocolAlone(iterable); + if (!iteration.isProtocolOnly) + iteration.isEntryPairElided = canElideEntryPair(walker, index, iteration, open.id()); + if (iteration.isProtocolOnly || iteration.hasProtocolFallback) { + if (FunctionID listed = walker.facts().linked().calleeAt(walker.currentFunction(), open.id())) + walker.leaveAnythingAt(listed); + walker.state().invalidateHeap(); + } } else if (isSurelyAsMade) { walker.verifyPristineObject(iterable); if (isAsMadeForGood) @@ -4052,6 +4217,38 @@ bool patternMayTakeListApartInCleanTurn(Walker& walker, const BodyValue& open) && isDestructuredArrayLikelyNumeric(walker, open, walker.referenceOf(open.as().iterableOperand())); } +static bool returnsOnlyArrayLiterals(const Walker& walker, FunctionID function) +{ + const BodyIndex* body = walker.facts().bodies().find(function); + if (!body || !walker.program().at(function).hasBody()) + return false; + bool returnsSomething = false; + for (uint32_t number = 0; number < body->numberOfBlocks(); ++number) { + BodyTerminator end = body->terminator(BlockID(number)); + if (end.kind() == TerminatorKind::ReturnCall) + return false; + if (end.kind() != TerminatorKind::Return) + continue; + ValueID returned = body->stripCasts(end.operand(0)); + if (!returned || body->isParameter(returned) || body->opcodeOf(returned) != Opcode::NewArray) + return false; + returnsSomething = true; + } + return returnsSomething; +} + +bool destructuresReturnedLiteral(const Walker& walker, const BodyIndex& index, ValueID open) +{ + if (index.isParameter(open) || index.opcodeOf(open) != Opcode::IteratorOpen || !isOpenThatMayGoByIndex(index.value(open)) || !stepsOfPatternStandInRow(index, index.value(open))) + return false; + ValueID call = index.stripCasts(index.value(open).as().iterableOperand()); + if (!call || index.isParameter(call) || index.opcodeOf(call) != Opcode::Call || index.onlyUsingValue(call) != open) + return false; + FunctionID callee = walker.facts().linked().calleeAt(walker.currentFunction(), call); + constexpr OptionSet returnsAnotherObject { FunctionFlags::IsAsync, FunctionFlags::IsGenerator }; + return callee && !walker.program().at(callee).hasAny(returnsAnotherObject) && returnsOnlyArrayLiterals(walker, callee); +} + bool spreadAppendsElementsOfListAsMade(Walker& walker, ValueID spreadValue) { Reference iterable = walker.referenceBoundTo(spreadValue); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkKnown.h b/Source/JavaScriptCore/aot/walk/AOTWalkKnown.h index 261e5f6ad9445..7d1f9a94ef83e 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkKnown.h +++ b/Source/JavaScriptCore/aot/walk/AOTWalkKnown.h @@ -30,6 +30,7 @@ enum class KnownNote : uint8_t { PassedTestOfBeingAsMade, AnswerOfQuestionStillStands, ViewOverMemoryThatNoOtherThreadGrows, + BoundsOfPieceAreInOrder, MemberOfBuiltinIsAsBuilt, HolderOfBuiltinHasNoAddedNames, RoomOfViewIsAsInFrontOfLoop, @@ -41,6 +42,7 @@ enum class KnownNote : uint8_t { NeverLiesBelow, LengthOfViewIsFoundAbove, ElementReadOffFreshList, + ElementReadUnderNumber, StructureOfClassIsSeen, ValueOfFieldOfHeir, LoopAppendsToLiteral, @@ -51,12 +53,16 @@ enum class KnownNote : uint8_t { CapacityCoversLoop, LoopCapacityGuard, ClassesThatInstanceOfDenied, + ClassesThatInstanceOfPassed, AnswerOfReadOfSlotThatMayBeEmpty, PieceIsStoredByBodyThatCutIt, + ListedOwnKeys, }; void noteKnown(Walker&, KnownNote, Reference subject, Reference other, uint32_t word, const LocationMasks& restsOn, uint64_t noted, bool endsAtSuspension = false); std::optional findKnown(const Walker&, KnownNote, Reference subject, Reference other = { }, uint32_t word = 0); void endKnown(Walker&, KnownNote, Reference subject, Reference other = { }, uint32_t word = 0); +constexpr bool isLookupInTable(InlineBuiltin builtin) { return builtin == InlineBuiltin::MapGet || builtin == InlineBuiltin::MapHas || builtin == InlineBuiltin::SetHas; } + extern const OpcodeSet opcodesThatFindEqualOperationBuiltTakes; extern const OpcodeSet opcodesThatNoteOperationBuiltTakes; Reference findEqualOperationBuiltOfItsOpcodes(Walker&, const Draft&, const Effects&); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkKnownAnswers.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkKnownAnswers.cpp index 41ad092222599..b8630534005e3 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkKnownAnswers.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkKnownAnswers.cpp @@ -822,7 +822,7 @@ void endKnown(Walker& walker, KnownNote kind, Reference subject, Reference other } static constexpr OpcodeSet testsNumberedInKnownAnswers = allEqualities | allRelations | OpcodeSet { Opcode::ToBoolean, Opcode::HasArrayStorage }; -static constexpr OpcodeSet valuesNumberedInKnownAnswers = allNumericOperators | OpcodeSet { Opcode::ToInt32, Opcode::ToUint32, Opcode::GetStringLength, Opcode::GetTypedArrayLength, Opcode::PrivateGet, Opcode::GetValueAtSiteOrAbsent }; +static constexpr OpcodeSet valuesNumberedInKnownAnswers = allNumericOperators | OpcodeSet { Opcode::WholeDivisor, Opcode::RemainderByWholeDivisor, Opcode::ToInt32, Opcode::ToUint32, Opcode::GetStringLength, Opcode::GetTypedArrayLength, Opcode::PrivateGet, Opcode::GetValueAtSiteOrAbsent, Opcode::CallInline }; static constexpr OpcodeSet operationsThatDoNothingAgainInKnownAnswers { Opcode::GetBuiltinValue, Opcode::ModuleBuiltin }; static bool doesNothingAgainInKnownAnswers(const Walker& walker, const Draft& draft) { @@ -852,11 +852,11 @@ static bool givesNewAnswerUnaskedInKnownAnswers(const Walker& walker, const Draf static bool collectKeyOfOperationInKnownAnswers(const Code& code, const Draft& draft, AnswersOfWalk::WordsOfKey& words) { words.shrink(0); - uint32_t numberOfOperands = draft.numberOfOperands(); + Opcode opcode = draft.opcode; + uint32_t numberOfOperands = opcode == Opcode::RemainderByWholeDivisor ? OperandIndex::RemainderByWholeDivisor::wholeDivisor : draft.numberOfOperands(); if (numberOfOperands > mostValuesInKeyOfKnownAnswer) return false; - Opcode opcode = draft.opcode; words.append(headOfKeyOfKnownAnswer(KnownAnswer::Family::Operation, static_cast(opcode))); switch (opcode) { case Opcode::GetBuiltinValue: @@ -873,6 +873,9 @@ static bool collectKeyOfOperationInKnownAnswers(const Code& code, const Draft& d case Opcode::GetValueAtSiteOrAbsent: words.append(draft.key().bits()); break; + case Opcode::CallInline: + words.append(static_cast(draft.inlineBuiltin())); + break; case Opcode::ToInt32: words.append(draft.flags); break; @@ -943,6 +946,24 @@ static Reference answerOfTestBuiltInKnownAnswers(Walker& walker, const Draft& te constinit const OpcodeSet opcodesThatFindEqualOperationBuiltTakes = operationsFoundInKnownAnswers; constinit const OpcodeSet opcodesThatNoteOperationBuiltTakes = operationsFoundInKnownAnswers; +static bool isNumberedInKnownAnswers(const Draft& draft) +{ + return draft.opcode != Opcode::CallInline || isLookupInTable(draft.inlineBuiltin()); +} + +static bool makesTextOfItsKeyInKnownAnswers(const Draft& draft, const Effects& effects) +{ + return draft.opcode == Opcode::CallInline && (effects.reads.containsAny(bookkeepingHeaps) || effects.writes.containsAny(bookkeepingHeaps)); +} + +static Effects besideTextOfItsKeyInKnownAnswers(const Effects& effects) +{ + Effects besideText = effects; + besideText.reads.remove(bookkeepingHeaps); + besideText.writes.remove(bookkeepingHeaps); + return besideText; +} + static bool touchesItsSiteInKnownAnswers(const Draft& draft, const Effects& effects) { return draft.opcode == Opcode::GetValueAtSiteOrAbsent && (effects.reads.containsAny(AbstractHeapSet { AbstractHeap::RuntimeState }) || effects.writes.containsAny(AbstractHeapSet { AbstractHeap::RuntimeState })); @@ -960,6 +981,10 @@ Reference findEqualOperationBuiltOfItsOpcodes(Walker& walker, const Draft& draft { if (touchesItsSiteInKnownAnswers(draft, effects)) return findEqualOperationBuiltOfItsOpcodes(walker, draft, besideItsSiteInKnownAnswers(effects)); + if (!isNumberedInKnownAnswers(draft)) + return { }; + if (makesTextOfItsKeyInKnownAnswers(draft, effects)) + return findEqualOperationBuiltOfItsOpcodes(walker, draft, besideTextOfItsKeyInKnownAnswers(effects)); Opcode opcode = draft.opcode; if (!walker.has(Mechanism::Folding) || !walker.state().remembersPlaces()) return { }; @@ -993,6 +1018,10 @@ void noteOperationBuiltOfItsOpcodes(Walker& walker, const Draft& draft, const Ef { if (touchesItsSiteInKnownAnswers(draft, effects)) return noteOperationBuiltOfItsOpcodes(walker, draft, besideItsSiteInKnownAnswers(effects), built); + if (!isNumberedInKnownAnswers(draft)) + return; + if (makesTextOfItsKeyInKnownAnswers(draft, effects)) + return noteOperationBuiltOfItsOpcodes(walker, draft, besideTextOfItsKeyInKnownAnswers(effects), built); Opcode opcode = draft.opcode; if (!walker.has(Mechanism::Folding) || !built.isValue()) return; @@ -1152,9 +1181,18 @@ std::optional Walker::findUpperBoundProof(Reference value, Reference std::optional Walker::findNonNegativeProof(Reference value) { TypeID type = typeOf(value); - if (!type || !m_types.at(type).isInteger() || m_types.at(type).integers.minimum < 0) + if (!type || !m_types.at(type).isInteger()) + return std::nullopt; + if (m_types.at(type).integers.minimum >= 0) + return PassedTest(); + if (m_state.answers().isEmpty()) + return std::nullopt; + Draft zero(Opcode::Integer); + zero.setInteger(0); + std::optional comparison = findKnown(*this, KnownNote::Relation, constant(WTF::move(zero)), value, 1); + if (!comparison || !(*comparison & 1)) return std::nullopt; - return PassedTest(); + return findPassedTest(Reference::fromBits(static_cast(*comparison >> 32)), true); } bool isKnownToLieBelow(Walker& walker, Reference value, Reference bound) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkKnownElements.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkKnownElements.cpp index 856511fd499cc..3b2beb4f96c08 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkKnownElements.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkKnownElements.cpp @@ -47,21 +47,41 @@ std::optional Walker::findWhatHoldsPositionInsideList(Reference list return fromZeroUp ? showPositionInsideBehindTestOfSum(list, position, length) : std::nullopt; } +static bool isTermTestedForPosition(const Code& code, const Operation& narrowed, bool isWholePosition) +{ + static constexpr OpcodeSet sums { Opcode::Add, Opcode::Sub, Opcode::Increment, Opcode::Decrement }; + if (!narrowed.testOperand()) + return false; + const Operation& test = code.at(narrowed.testOperand()); + if (!allRelations.contains(test.opcode) || test.form() != OperatorForm::Integers || narrowed.operandIndex() > 1) + return false; + if (!isWholePosition) + return true; + ValueID bound = test.operand(1 - narrowed.operandIndex()); + while (valuePreservingOpcodes.contains(code.at(bound).opcode)) + bound = code.at(bound).operand(0); + return sums.contains(code.at(bound).opcode); +} + std::optional Walker::showPositionInsideBehindTestOfSum(Reference list, Reference position, Reference length) { if (!list.isRecord() || !position.isValue() || !length.isRecord()) return std::nullopt; for (ValueID term = position.id(); term && !m_code.isConstant(term);) { const Operation& record = m_code.at(term); - if (record.opcode == Opcode::Narrowed && record.narrowsTermOfOperand()) { + bool isTermTested = record.opcode == Opcode::Narrowed && !record.narrowsTermOfOperand() && isTermTestedForPosition(m_code, record, term == position.id()); + if (isTermTested || (record.opcode == Opcode::Narrowed && record.narrowsTermOfOperand())) { GroundsOfPosition grounds { .view = list.id(), .position = position.id(), .length = length.id(), .isBelow = record.testOperand(), .isBelowResult = record.isOnTakenEdge() }; std::optional shown = m_inputs.options.shouldSkipBoundsProof ? findPassedTest(referenceTo(grounds.isBelow), grounds.isBelowResult) : showPositionInside(grounds); if (shown && m_inputs.options.shouldSkipBoundsProof) shown->m_groundsOfPosition = grounds; + if (shown && isTermTested) + count(StatisticID::PositionsShownByTestedTerms); if (shown) return shown; } - if (valuePreservingOpcodes.contains(record.opcode) || record.opcode == Opcode::Increment) { + bool stepsDown = record.opcode == Opcode::Decrement || (record.opcode == Opcode::Sub && m_code.isConstant(record.operand(1))); + if (valuePreservingOpcodes.contains(record.opcode) || record.opcode == Opcode::Increment || stepsDown) { term = record.operand(0); continue; } diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkKnownRows.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkKnownRows.cpp index 045d8408cba83..e5ec93aeb6823 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkKnownRows.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkKnownRows.cpp @@ -56,15 +56,14 @@ Effects effectsOfLibraryFunctionBehindRow(const Walker& walker, const BuiltinOve std::optional effectsOfSearchForWhatIsNoString(const TypeArena& types, InlineBuiltin builtin, TypeID searchedOrNotPassed) { - bool isLookupInTable = builtin == InlineBuiltin::MapGet || builtin == InlineBuiltin::MapHas || builtin == InlineBuiltin::SetHas; - if (!isLookupInTable && builtin != InlineBuiltin::ArrayIndexOf && builtin != InlineBuiltin::ArrayIncludes) + if (!isLookupInTable(builtin) && builtin != InlineBuiltin::ArrayIndexOf && builtin != InlineBuiltin::ArrayIncludes) return std::nullopt; if (!searchedOrNotPassed) return std::nullopt; Type searched = types.at(searchedOrNotPassed); if (!searched.isJavaScriptValue() || (searched.primitives & Primitive::AnyString)) return std::nullopt; - if (isLookupInTable) + if (isLookupInTable(builtin)) return searched.primitives & Primitive::BigInt ? std::nullopt : std::optional(declaredBoundOf(InlineBuiltin::WeakMapGet)); return declaredBoundOf(InlineBuiltin::ArrayAt); } @@ -233,6 +232,13 @@ bool isLeafHandedToMethodOfLists(const Walker& walker, TypeID argumentType) return only && walker.facts().linked().function(only).weightAsLeaf != LinkedFunction::notLeaf; } +static bool handsOverOwnClosureForKnownRows(Walker& walker, const BodyValue& call) +{ + const BodyIndex& index = walker.currentBodyIndex(); + ValueID handed = index.stripCasts(call.argumentsOperands()[0]); + return handed && !index.isParameter(handed) && index.opcodeOf(handed) == Opcode::NewClosure && index.hasOneUse(handed); +} + static bool loopReadsOffResultFirstForKnownRows(Walker& walker, ValueID value) { static constexpr OpcodeSet readersOfFirstOperand { Opcode::PropGet, Opcode::PropHas, Opcode::ElemGet, Opcode::ElemHas }; @@ -398,7 +404,9 @@ bool callLibraryFunctionBehindRows(Walker& walker, const BodyValue& call) Reference firstArgument = call.argumentsOperands().empty() ? Reference() : walker.referenceOf(call.argumentsOperands()[0]); Reference receiver = call.thisValueOperand() ? walker.referenceOf(call.thisValueOperand()) : Reference(); bool isListAsItWasMade = receiver && (isKnownPristine(walker, receiver) || receiverStaysAsMadeByLink(walker, call)); - mayCopy = ((walker.isInLoop() || walker.isCurrentFrameRecursive()) ? isListAsItWasMade || walker.walksOwnBodyOfUnit() : isListAsItWasMade && isReadOutOfVariableOfFileThatStays(walker, call.thisValueOperand())) && firstArgument && isLeafHandedToMethodOfLists(walker, walker.typeOf(firstArgument)) ? MayCopyFunctionCalledDirectly::Yes : MayCopyFunctionCalledDirectly::No; + mayCopy = ((walker.isInLoop() || walker.isCurrentFrameRecursive()) ? isListAsItWasMade || walker.walksOwnBodyOfUnit() : isListAsItWasMade && isReadOutOfVariableOfFileThatStays(walker, call.thisValueOperand())) && firstArgument && (isLeafHandedToMethodOfLists(walker, walker.typeOf(firstArgument)) || (walker.isInLoop() && handsOverOwnClosureForKnownRows(walker, call))) ? MayCopyFunctionCalledDirectly::Yes : MayCopyFunctionCalledDirectly::No; + if (mayCopy == MayCopyFunctionCalledDirectly::Yes && !isLeafHandedToMethodOfLists(walker, walker.typeOf(firstArgument))) + walker.count(StatisticID::ListCallbacksCopiedWithCalls); } else if (!behind.named.isEmpty()) function = findLibraryFunctionOfRowsReachedBy(walker, call); isFoundByRowsReached = function && function != behind.enteredByValue; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkLinksOfChains.h b/Source/JavaScriptCore/aot/walk/AOTWalkLinksOfChains.h index 942f7c19cd319..4981bdc396d9b 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkLinksOfChains.h +++ b/Source/JavaScriptCore/aot/walk/AOTWalkLinksOfChains.h @@ -29,7 +29,7 @@ template ALWAYS_INLINE bool buildDeferredDefinition(Walker& walker extern const OpcodeSet opcodesThatFoldOperationTakes; inline constexpr OpcodeSet opcodesThatFoldOperationTakesInChains = operationsOnPrimitivesInFolding | castsOfFolding | passThroughChecks | OpcodeSet { Opcode::TypeTest, Opcode::IsClosureOf, Opcode::ToBoolean, Opcode::Eq, Opcode::StrictEq, Opcode::Less, Opcode::LessEq, Opcode::Greater, Opcode::GreaterEq, Opcode::GetModuleVariable, Opcode::GetEnvironmentField, Opcode::PropGet, Opcode::ElemGet, Opcode::PutModuleVariable, Opcode::PutEnvironmentField, - Opcode::ToObject, Opcode::ToThis, Opcode::TypeOf, Opcode::ToLength }; + Opcode::ToObject, Opcode::ToThis, Opcode::TypeOf, Opcode::ToLength, Opcode::IsUnlockedInstance }; AOT_LINK_OF_WALK(foldOperation, opcodesThatFoldOperationTakesInChains) inline constexpr OpcodeSet allocationsOfEscapeAnalysis = opcodesWhere([](Opcode opcode) { if (opcode == Opcode::NewClosure || opcode == Opcode::Data) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkListReadsInLoops.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkListReadsInLoops.cpp index e5ae9aa2a3d50..3fa1e53956815 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkListReadsInLoops.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkListReadsInLoops.cpp @@ -410,6 +410,72 @@ void readListsOfStableBindingsInFrontOfLoop(Walker& walker, BlockID inputHead) } } +static Reference readFieldInFrontInListReadsInLoops(Walker& walker, const BodyIndex& index, ValueID value, LoopID outermost, uint32_t firstValueOfLoop) +{ + ValueID found = valueThatEntersLoopInListReadsInLoops(index, value, firstValueOfLoop); + if (!found || found.index() < firstValueOfLoop || index.isParameter(found)) + return nameThatEntersLoopInListReadsInLoops(walker, index, value, firstValueOfLoop); + BodyValue read = index.value(found); + if (read.opcode() != Opcode::PrivateGet && (read.opcode() != Opcode::PropGet || read.isFreeName())) + return nameThatEntersLoopInListReadsInLoops(walker, index, value, firstValueOfLoop); + PropertyKeyID key = keyOfAccessToSlot(walker.program(), read); + if (walker.textOfOutermostLoopStoresUnderNameInCurrentBody(outermost, key)) + return { }; + Reference holder = readFieldInFrontInListReadsInLoops(walker, index, read.objectOperand(), outermost, firstValueOfLoop); + return readFilledFieldInFrontOfTurns(walker, read, holder); +} + +static bool loadsViewInListReadsInLoops(Walker& walker, const BodyIndex& index, ValueID value, uint32_t firstValueOfLoop) +{ + ValueID loaded = valueThatEntersLoopInListReadsInLoops(index, value, firstValueOfLoop); + if (!loaded || loaded.index() < firstValueOfLoop || index.isParameter(loaded)) + return false; + TypeID widest = widestTypeOfOperation(walker, index.value(loaded)); + return widest && isViewOfNumbersReadInLine(walker, walker.types().withoutNullish(widest)); +} + +void readListsOfFieldsInFrontOfLoop(Walker& walker, BlockID inputHead) +{ + const BodyIndex& index = walker.currentBodyIndex(); + if (!walker.has(Mechanism::Folding) || !walker.has(Mechanism::TypedOperations) || !index.holdsAnyOf(OpcodeSet { Opcode::ElemGet, Opcode::ElemSet })) + return; + bool storesElements = index.opcodes().contains(Opcode::ElemSet); + bool isToldApart = walker.standsInCopyOfLoopBuiltForEachMember(); + auto readsWhatNobodyKnows = [&](const BodyValue& access) { + if (!isToldApart) + return false; + TypeID seenByLink = access.opcode() == Opcode::ElemGet ? walker.facts().linked().resultOfReadByKey(walker.currentFunction(), access.id()) : TypeID(TypeTable::never); + return !seenByLink || walker.types().at(seenByLink).isAnything(); + }; + uint32_t firstValueOfLoop = index.block(inputHead).firstParameter().index(); + LoopID loop = index.loopOf(inputHead); + LoopID outermost = loop; + while (LoopID around = index.loop(outermost).parent) + outermost = around; + for (BlockID member : index.membersOf(loop)) { + for (BodyValue operation : index.block(member).values()) { + if (walker.isUnreachable()) + return; + if ((operation.opcode() != Opcode::ElemSet && operation.opcode() != Opcode::ElemGet) || goesUnderKeyMadeInTurnInListReadsInLoops(index, operation)) + continue; + if (storesElements || readsWhatNobodyKnows(operation) || loadsViewInListReadsInLoops(walker, index, operation.objectOperand(), firstValueOfLoop)) + readFieldInFrontInListReadsInLoops(walker, index, operation.objectOperand(), outermost, firstValueOfLoop); + } + } +} + +void forEachElementAccessInInlinedCallee(Walker& walker, const BodyValue& call, NOESCAPE const ScopedLambda& take) +{ + FunctionID callee = walker.facts().linked().calleeAt(walker.currentFunction(), call.id()); + if (!callee || !lookReadsCopyOfCalleeInListReadsInLoops(walker, call, callee)) + return; + for (Opcode access : { Opcode::ElemGet, Opcode::ElemSet }) { + forEachReadOfArgumentInCopyOfCalleeInListReadsInLoops(walker, callee, call.as().argumentsOperands(), 2, access, [&](ValueID argument, ValueID key, FunctionID, ValueID) { + take(argument, key); + }); + } +} + bool storesIntoListTestedInFrontOfLoop(Walker& walker, const BodyIndex& index, const BodyValue& store, uint32_t firstValueOfLoop, std::span lists) { Reference name = nameThatEntersLoopInListReadsInLoops(walker, index, store.objectOperand(), firstValueOfLoop); @@ -435,7 +501,7 @@ std::optional findHoistedArrayGuard(Walker& walker, const Bod return std::nullopt; const BodyIndex& index = walker.currentBodyIndex(); LoopID outermost = index.loopOf(index.blockOf(read.id())); - if (!outermost && !walker.isInLoop()) + if (!outermost && !walker.isInLoop() && !walker.guardedListOfReads()) return std::nullopt; while (LoopID around = outermost ? index.loop(outermost).parent : LoopID()) outermost = around; @@ -586,7 +652,7 @@ void takeStorageAndLengthFromInFrontOfLoop(Walker& walker, Draft& read, const Ho walker.count(StatisticID::KnownInductionsUsed); } } - if (read.opcode == Opcode::GetArrayElement && read.admitsInt32Storage()) { + if ((read.opcode == Opcode::GetArrayElement || read.opcode == Opcode::IsElementPresent) && read.admitsInt32Storage() && read.arrayStorage() == ArrayStorage::Double) { Draft kind(Opcode::HasArrayStorage, origin); walker.chooseConservativeFields(kind); kind.setArrayOperand(tested.name); @@ -912,7 +978,7 @@ const BitVector& Walker::valuesTakenForKeysInCurrentBody() }).iterator->value; } -static bool turnTakesElementForKeyInListReadsInLoops(Walker& walker, ValueID element) +bool turnTakesElementForKeyInListReadsInLoops(Walker& walker, ValueID element) { const BodyIndex& index = walker.currentBodyIndex(); return element && element.index() < index.numberOfValues() && (index.loopOf(index.blockOf(element)) || !index.hasLoops()) && walker.valuesTakenForKeysInCurrentBody().quickGet(element.index()); @@ -996,6 +1062,12 @@ static TypeID expectedOfRowThatIsNotBuiltInListReadsInLoops(Walker& walker, Valu return expected; } +bool isExpectedToBeView(Walker& walker, ValueID unbuiltList) +{ + TypeID expected = expectedOfRowThatIsNotBuiltInListReadsInLoops(walker, unbuiltList); + return expected && !!classOfViewInListReadsInLoops(walker, walker.types().withoutNullish(expected)); +} + namespace { template @@ -1044,7 +1116,7 @@ static ALWAYS_INLINE bool readsRowOfNumbersThatTurnFindsInListedList(Walker& wal if (!row || row.index() < firstValueOfLoop || index.isParameter(row) || index.opcodeOf(row) != Opcode::ElemGet) return false; ValueID holder = valueThatEntersLoopInListReadsInLoops(index, index.value(row).objectOperand(), firstValueOfLoop); - Reference name = holder && holder.index() < firstValueOfLoop ? walker.referenceOf(holder) : Reference(); + Reference name = holder ? nameThatEntersLoopInListReadsInLoops(walker, index, holder, firstValueOfLoop) : Reference(); bool isListed = name && std::ranges::any_of(lists, [&](const HoistedArrayGuard& list) { return list.name == name; }); if (!isListed && isProvenArray(walker, name) && isFreshAllocation(walker, name)) { TypeID expected = expectedOfRowThatIsNotBuiltInListReadsInLoops(walker, row); @@ -1110,6 +1182,9 @@ static bool turnMayAppendToTextInListReadsInLoops(Walker& walker, BlockID inputH } if (name && (listsByName.anyOf(name, isNumericArrayAt) || (proof && listsByName.anyOf(proof, isNumericArrayAt)))) return true; + TypeID seenByLink = walker.facts().linked().resultOfReadByKey(walker.currentFunction(), root); + if (seenByLink && !isNeverType(seenByLink) && walker.types().at(seenByLink).hasOnlyPrimitives(Primitive::Number | Primitive::Undefined)) + return true; if (readsRowOfNumbersThatTurnFindsInListedList(walker, index, index.value(root), firstValueOfLoop, lists)) return true; size_t placeOfList = entersLoop.index() - preview.firstValue; @@ -1227,6 +1302,49 @@ static bool someListedListIsReceiverOfMethodInListReadsInLoops(Walker& walker, B return false; } +template +static const LinkedEffectsGivenArguments* findEffectsGivenArgumentsInListReadsInLoops(Walker& walker, const BodyValue& operation, FunctionID callee, const TypeInLook& typeInLook) +{ + const LinkTables& linked = walker.facts().linked(); + const LinkedEffectsGivenArguments* given = linked.known.findEffectsGivenArguments(callee); + if (!given || given->parametersCalled) + return nullptr; + TypeArena& types = walker.types(); + auto call = operation.as(); + auto arguments = call.argumentsOperands(); + const Function& function = walker.program().at(callee); + std::span parameters = linked.parametersOf(callee); + for (uint32_t parameter = 0; parameter < 32; ++parameter) { + if (!(given->parametersTakenAsExpected >> parameter & 1)) + continue; + TypeID expected = parameter < parameters.size() ? parameters[parameter].writtenOrSeenToArrive() : TypeID(); + if (!expected) + return nullptr; + ValueID argument; + EntryParameter kind = function.entryParameterKind(parameter); + if (kind == EntryParameter::This) + argument = call.thisValueOperand(); + else if (uint32_t position = parameter - function.firstParameterIndex(); kind == EntryParameter::Parameter && position < arguments.size()) + argument = arguments[position]; + if (!argument) + return nullptr; + TypeID arriving = typeInLook(argument); + if (arriving && !types.overlap(arriving, expected)) + return nullptr; + } + return given; +} + +template +static bool callLeavesListsAsTheyAreGivenArguments(Walker& walker, const BodyValue& operation, const TypeInLook& typeInLook) +{ + if (operation.opcode() != Opcode::Call) + return false; + FunctionID callee = walker.facts().linked().calleeAt(walker.currentFunction(), operation.id()); + const LinkedEffectsGivenArguments* given = callee ? findEffectsGivenArgumentsInListReadsInLoops(walker, operation, callee, typeInLook) : nullptr; + return given && preservesHoistedArrayGuards(walker, given->onReturn) && preservesHoistedArrayGuards(walker, given->onThrow); +} + static bool rowFindsListsLeftAsTheyAreInListReadsInLoops(Walker& walker, BlockID inputHead, std::span lists, std::span guesses, std::span proofs) { if (walker.listsAreStruckInRow()) @@ -1251,7 +1369,7 @@ static bool rowFindsListsLeftAsTheyAreInListReadsInLoops(Walker& walker, BlockID if (!ran) return false; BodyValue operation = index.value(ran); - if (stepsViewByIndexAlone(walker, index, operation) || stepsTextByIndexAlone(walker, index, operation)) + if (stepsViewByIndexAlone(walker, index, operation) || stepsTextByIndexAlone(walker, index, operation) || callLeavesListsAsTheyAreGivenArguments(walker, operation, typeInRow)) continue; if (operation.opcode() == Opcode::ElemGet && walker.standsInCopyOfLoopBuiltForEachMember() && rowFoundUnderKeyInFrontOfLoop(walker, operation)) continue; @@ -1291,14 +1409,19 @@ static bool rowFindsListsLeftAsTheyAreInListReadsInLoops(Walker& walker, BlockID return preservesHoistedArrayGuards(walker, walker.effectsOfQuietDraftsOfRow(), lists, inputHead); } -Vector namesInFrontThatLoopMentions(const Walker& walker, const BodyIndex& index, LoopID loop, uint32_t firstValueOfLoop, Reference value) +void NamesBeforeLoop::collectNames() { - Vector names; + const BodyIndex& index = m_walker.currentBodyIndex(); + uint32_t firstValueOfLoop = index.block(m_inputHead).firstParameter().index(); + BitVector isCollected; auto mention = [&](ValueID operand) { - if (operand && operand.index() < firstValueOfLoop && !names.contains(operand) && walker.referenceOf(operand) == value) - names.append(operand); + if (!operand || operand.index() >= firstValueOfLoop || isCollected.get(operand.index())) + return; + isCollected.set(operand.index()); + if (Reference value = m_walker.referenceOf(operand)) + m_names.append({ value.bits(), operand }); }; - for (BlockID member : index.membersOf(loop)) { + for (BlockID member : index.membersOf(index.loopOf(m_inputHead))) { BodyBlock block = index.block(member); for (BodyValue operation : block.values()) { operation.forEachOperand([&](ValueID operand, uint32_t) { @@ -1318,6 +1441,18 @@ Vector namesInFrontThatLoopMentions(const Walker& walker, const Body mention(argument); } } + std::ranges::stable_sort(m_names, { }, &Name::value); +} + +Vector NamesBeforeLoop::findNamesOf(Reference value) +{ + if (!std::exchange(m_areNamesCollected, true)) + collectNames(); + Vector names; + for (const Name& name : std::ranges::equal_range(m_names, value.bits(), { }, &Name::value)) { + if (m_walker.referenceOf(name.id) == value) + names.append(name.id); + } return names; } @@ -1374,7 +1509,7 @@ static bool lookReadsCopyOfCalleeInListReadsInLoops(Walker& walker, const BodyVa return Reference::value(operand); }); PlaceOfCall place { walker.currentFunction(), true, &walker.currentBodyIndex(), call.id() }; - if (body->hasLoops() && !sizeOfCopyThatWritesOutLoopsOverListsHandedIn(walker, callee) && !turnsOfCopyWrittenOutAt(walker, site, callee, place)) + if (body->hasLoops() && !sizeOfCopyThatWritesOutLoopsOverListsHandedIn(walker, callee) && !turnsOfCopyWrittenOutAt(walker, site, callee, place) && !canInlineSingleCallLoop(walker, callee, place, nullptr)) return false; return mayCopyCallee(walker, site, callee, place); } @@ -1394,6 +1529,17 @@ static Reference buildIntegerMinimum(Walker& walker, Reference one, Reference ot return walker.joinArms(arms.span(), walker.addBlock(), singleElementSpan(lesser)) ? lesser : Reference(); } +bool isResultUnused(const BodyIndex& index, ValueID result) +{ + while (!index.isUnused(result)) { + ValueID user = index.onlyUsingValue(result); + if (!user || !opcodesThatStateTypes.contains(index.opcodeOf(user))) + return false; + result = user; + } + return true; +} + static Reference bearLiteralAgainBehindItsRoom(Walker& walker, const BodyIndex& index, ValueID written, Reference list, BlockID inputHead) { if (!walker.has(Mechanism::EscapeAnalysis) || !written || index.isParameter(written) || index.opcodeOf(written) != Opcode::NewArray || !list.isValue() || walker.referenceBoundTo(written) != list) @@ -1405,13 +1551,22 @@ static Reference bearLiteralAgainBehindItsRoom(Walker& walker, const BodyIndex& BodyTerminator end = index.terminator(home); if (home.index() >= inputHead.index() || end.kind() != TerminatorKind::Jump || end.targets()[0].block() != inputHead) return { }; - std::optional first = index.firstUser(written); + Vector casts; + ValueID named = written; + std::optional first = index.firstUser(named); + while (first && first->kind == BodyUseKind::Operand && index.hasOneUse(named) && index.blockOf(first->value) == home && opcodesThatStateTypes.contains(index.opcodeOf(first->value)) && walker.referenceBoundTo(first->value) == list) { + named = first->value; + casts.append(named); + first = index.firstUser(named); + } if (!first || (first->kind == BodyUseKind::Operand ? index.blockOf(first->value).index() : first->block.index()) < inputHead.index()) return { }; buildFreshObject(walker, index.value(written)); Reference again = walker.isUnreachable() ? Reference() : walker.referenceBoundTo(written); if (!again || !again.isValue() || again == list || walker.code().at(again.id()).opcode != Opcode::NewArray) return { }; + for (ValueID cast : casts) + walker.bind(cast, again); walker.count(StatisticID::ArrayLiteralAllocationsSunk); return again; } @@ -1643,7 +1798,7 @@ void reserveRoomOfListsThatLoopFills(Walker& walker, BlockID inputHead, std::spa listsFilledTwiceInATurn.append(list); if (!listsPushedOnto.contains(list)) listsPushedOnto.append(list); - if (index.isUnused(operation.id()) && standsBehindTestOfHead && index.loopOf(member) == loop) + if (isResultUnused(index, operation.id()) && standsBehindTestOfHead && index.loopOf(member) == loop) pushesWithUnusedResult.append({ list, operation.id() }); continue; } @@ -1765,6 +1920,7 @@ void reserveRoomOfListsThatLoopFills(Walker& walker, BlockID inputHead, std::spa continue; bool hasMergedNumberStore = false; bool hasPushesAndArithmeticUsers = false; + bool hasCapacityForSpecializedVersion = hasExactTripCount || types.at(walker.typeOf(roomByCount)).integers.maximum <= BASE_CONTIGUOUS_VECTOR_LEN_MAX || (listReadUnderCounter && listReadUnderCounter != namesOfLists[place]); if (namesAskedInFront.isEmpty() && textsJoined.isEmpty() && types.at(walker.typeOf(room)).isInteger() && !listsFilledTwiceInATurn.contains(list)) { if (hasExactTripCount && listsPushedOnto.contains(list) && born.opcode == Opcode::NewArray) { Reference turnsLeft = buildIntegerOperator(walker, Opcode::Sub, room, entering[placeOfCounter], origin); @@ -1774,15 +1930,17 @@ void reserveRoomOfListsThatLoopFills(Walker& walker, BlockID inputHead, std::spa if (filled && types.at(walker.typeOf(filled)).isInteger()) room = filled; } - size_t placeOfRead = invariantArrayReads.findIf([&](const auto& known) { return known.first == namesOfLists[place]; }); + size_t placeOfRead = invariantArrayReads.findIf([&](const auto& known) { return index.stripCasts(known.first) == index.stripCasts(namesOfLists[place]); }); TypeID seenByLink = placeOfRead != notFound && !listsPushedOnto.contains(list) && born.opcode == Opcode::NewArray && initialCounterRange == Range::one(static_cast(born.elementsOperands().size())) ? walker.facts().linked().resultOfReadByKey(walker.currentFunction(), invariantArrayReads[placeOfRead].second) : TypeID(); bool isNumeric = seenByLink && (types.at(seenByLink).primitives & Primitive::Number) && types.at(seenByLink).hasOnlyPrimitives(Primitive::Number | Primitive::Undefined | Primitive::Absent); size_t placeOfChoice = parametersStoredUnderCounter.findIf([&](const auto& known) { return known.first == lists[place]; }); - bool hasCapacityForSpecializedVersion = hasExactTripCount || types.at(walker.typeOf(roomByCount)).integers.maximum <= BASE_CONTIGUOUS_VECTOR_LEN_MAX || (listReadUnderCounter && listReadUnderCounter != namesOfLists[place]); hasMergedNumberStore = !isNumeric && hasCapacityForSpecializedVersion && placeOfChoice != notFound && !listsPushedOnto.contains(list) && born.opcode == Opcode::NewArray && initialCounterRange == Range::one(static_cast(born.elementsOperands().size())) && isMergeOfComputedNumbers(index, loop, inputHead, counter, parametersStoredUnderCounter[placeOfChoice].second); hasPushesAndArithmeticUsers = !isNumeric && hasCapacityForSpecializedVersion && placeOfRead != notFound && born.opcode == Opcode::NewArray && initialCounterRange == Range::one(static_cast(born.elementsOperands().size())) && pushesWithUnusedResult.containsIf([&](const auto& push) { return push.first == lists[place]; }) && isElementUsedInArithmetic(walker, index.value(invariantArrayReads[placeOfRead].second)) - && std::ranges::all_of(invariantArrayReads, [&](const auto& known) { return isProvenArray(walker, walker.referenceOf(known.first)); }); + && std::ranges::all_of(invariantArrayReads, [&](const auto& known) { + Reference holder = walker.referenceOf(known.first); + return isProvenArray(walker, holder) || types.at(walker.typeOf(holder)).hasOnlyPrimitives(Primitive::AnyString); + }); SetForScope requestedLiteralStorageScope(walker.requestedLiteralStorage(), hasMergedNumberStore || hasPushesAndArithmeticUsers ? ArrayStorage::Double : !isNumeric ? ArrayStorage::Other : types.at(seenByLink).isInt32OrNullish() ? ArrayStorage::Int32 : types.at(seenByLink).primitives & Primitive::FractionalNumber ? ArrayStorage::Double : ArrayStorage::Other); if (Reference again = bearLiteralAgainBehindItsRoom(walker, index, namesOfLists[place], list, inputHead)) list = again; @@ -1815,7 +1973,7 @@ void reserveRoomOfListsThatLoopFills(Walker& walker, BlockID inputHead, std::spa bool isBornForThisEntry = !index.isParameter(literal) && (index.opcodeOf(literal) == Opcode::NewArray || index.opcodeOf(literal) == Opcode::Data) && index.loopOf(index.blockOf(literal)) == index.loop(loop).parent; const Operation& filled = walker.code().at(underlyingCellOf(walker.code(), list.id())); bool startsAtLiteralLength = filled.opcode == Opcode::NewArray && filled.holes().empty() && initialCounterRange == Range::one(static_cast(filled.elementsOperands().size())); - size_t placeOfPush = pushesWithUnusedResult.findIf([&](const auto& push) { return push.first == lists[place]; }); + size_t placeOfPush = pushesWithUnusedResult.findIf([&](const auto& push) { return push.first == lists[place] && (hasCapacityForSpecializedVersion || index.isUnused(push.second)); }); bool appendsToLiteral = !walker.isUnreachable() && isBornForThisEntry && startsAtLiteralLength && (!listsPushedOnto.contains(lists[place]) || placeOfPush != notFound) && !listsFilledTwiceInATurn.contains(lists[place]); if (appendsToLiteral && placeOfPush != notFound) { noteKnown(walker, KnownNote::PushAppendsToLiteral, list, { }, inputHead.index(), LocationMasks(), pushesWithUnusedResult[placeOfPush].second.index() | static_cast(walker.currentFunction().index()) << 32); @@ -1918,8 +2076,11 @@ static bool surelyEndsWhatIsShownOfListsInListReadsInLoops(Walker& walker, const known = singleFunctionOf(types, types.at(called)); if (known && usesIterationGuards && lookReadsCopyOfCalleeInListReadsInLoops(walker, operation, known)) return false; - if (FunctionID callee = known) + if (FunctionID callee = known) { + if (const LinkedEffectsGivenArguments* given = findEffectsGivenArgumentsInListReadsInLoops(walker, operation, callee, typeInLook)) + return !preservesHoistedArrayGuards(walker, given->onReturn) || !preservesHoistedArrayGuards(walker, given->onThrow); return !preservesHoistedArrayGuards(walker, linked.function(callee).effectsOnReturn()) || !preservesHoistedArrayGuards(walker, linked.function(callee).effectsOnThrow()); + } TypeID callee = typeInLook(operation.as().calleeOperand()); return !callee || !singleBuiltinOf(types, types.at(callee)); } @@ -1996,6 +2157,44 @@ static ValueID findStepValue(const BodyIndex& index, const BodyValue& step) return { }; } +static bool isMathCall(const Walker& walker, const BodyIndex& index, const BodyValue& operation) +{ + if (operation.opcode() != Opcode::Call) + return false; + ValueID callee = operation.as().calleeOperand(); + callee = callee ? index.stripCasts(callee) : ValueID(); + if (!callee || index.isParameter(callee) || index.opcodeOf(callee) != Opcode::Builtin) + return false; + const BuiltinTable& table = walker.program().builtins(); + return std::ranges::any_of(table.overloadsOf(table.canonicalID(index.value(callee).as().builtin())), [](const BuiltinOverload& overload) { + std::optional inLine = overload.mode == BuiltinCallMode::Call ? overload.code.implementation.inlineBuiltin : std::nullopt; + if (!inLine) + return false; + switch (*inLine) { + case InlineBuiltin::MathAbs: + case InlineBuiltin::MathFloor: + case InlineBuiltin::MathCeil: + case InlineBuiltin::MathTrunc: + case InlineBuiltin::MathRound: + case InlineBuiltin::MathFround: + case InlineBuiltin::MathF16Round: + case InlineBuiltin::MathSqrt: + case InlineBuiltin::MathPow: + case InlineBuiltin::MathMin: + case InlineBuiltin::MathMax: + case InlineBuiltin::MathImul: + case InlineBuiltin::MathClz32: + case InlineBuiltin::MathSign: + case InlineBuiltin::MathFunction: + case InlineBuiltin::MathATan2: + case InlineBuiltin::MathHypot: + return true; + default: + return false; + } + }); +} + static ListsReadInLoop findListsReadInLoopInListReadsInLoops(Walker& walker, BlockID inputHead, const LoopPreview& preview, std::span entering, Vector& guesses, bool keysOfReadsTakeNotes, bool& someReadHasNotedKey, std::optional& namesOfCountersAskedInFront) { ListsReadInLoop found; @@ -2039,13 +2238,30 @@ static ListsReadInLoop findListsReadInLoopInListReadsInLoops(Walker& walker, Blo walker.noteKeyTakenForCounterAskedInFront(); return TypeTable::int32; }; + std::optional> elementCarriedIndexes; + auto isCarriedIndex = [&](ValueID read) { + if (!elementCarriedIndexes) + elementCarriedIndexes = findElementCarriedIndexes(walker, inputHead); + return elementCarriedIndexes->contains(read); + }; + BitVector numbersThatEnterInnerLoops; + auto entersLoopAsNumber = [&](ValueID value) { + ValueID made = index.stripCasts(value); + if (made && made.index() >= firstValueOfLoop && numbersThatEnterInnerLoops.get(made.index() - firstValueOfLoop)) + return true; + std::optional place = made ? index.block(inputHead).indexOfParameter(made) : std::nullopt; + TypeID entered = place && *place < entering.size() && entering[*place] ? walker.ownTypeOf(entering[*place]) : TypeID(); + if (!entered || types.at(entered).isNumber()) + return !!entered; + return !walker.sumsAreNotAskedInRow() && *place < preview.typesOfParameters.size() && isNumberCarriedBesideTextOrBigInt(walker, entering[*place], preview.typesOfParameters[*place]); + }; BitVector numbersBehindListsFound; auto isNumberBehindListsFound = [&](ValueID value) { TypeID seen = typeInLook(value); if (seen && types.at(seen).isNumber()) return true; ValueID made = index.stripCasts(value); - return made && made.index() >= firstValueOfLoop && numbersBehindListsFound.get(made.index() - firstValueOfLoop); + return (made && made.index() >= firstValueOfLoop && numbersBehindListsFound.get(made.index() - firstValueOfLoop)) || entersLoopAsNumber(value); }; BitVector wholeNumbersBehindListsFound; auto isWholeNumberBehindListsFound = [&](ValueID value) { @@ -2097,13 +2313,18 @@ static ListsReadInLoop findListsReadInLoopInListReadsInLoops(Walker& walker, Blo uint32_t edgesSeen = joinsAwaited[awaited].edgesSeen; joinsAwaited[awaited] = joinsAwaited.last(); joinsAwaited.removeLast(); - if (index.isLoopHead(join) || index.isEnteredByHandlerEdge(join) || edgesSeen != index.numberOfIncomingEdges(join) || index.numberOfIncomingEdges(join) >= UINT16_MAX) + bool isInnerHead = index.isLoopHead(join); + if (index.isEnteredByHandlerEdge(join) || (!isInnerHead && edgesSeen != index.numberOfIncomingEdges(join)) || index.numberOfIncomingEdges(join) >= UINT16_MAX) return; BodyBlock block = index.block(join); for (uint32_t position = 0; position < block.numberOfParameters(); ++position) { uint32_t place = block.parameter(position).index() - firstValueOfLoop; if (parametersThatMeetWhatIsNoNumber.get(place)) continue; + if (isInnerHead) { + numbersThatEnterInnerLoops.set(place); + continue; + } numbersBehindListsFound.set(place); if (!parametersThatMeetWhatIsNoWholeNumber.get(place)) wholeNumbersBehindListsFound.set(place); @@ -2174,19 +2395,42 @@ static ListsReadInLoop findListsReadInLoopInListReadsInLoops(Walker& walker, Blo Vector listsThatCopiesOfCalleesStoreIntoUncounted; Vector listsTakenUnderCountersAskedInFront; Vector listsReachedUnderCountersAskedInFront; - auto entersLoopAsNumber = [&](ValueID value) { - ValueID made = index.stripCasts(value); - std::optional place = made ? index.block(inputHead).indexOfParameter(made) : std::nullopt; - TypeID entered = place && *place < entering.size() && entering[*place] ? walker.ownTypeOf(entering[*place]) : TypeID(); - if (!entered || types.at(entered).isNumber()) - return !!entered; - return !walker.sumsAreNotAskedInRow() && *place < preview.typesOfParameters.size() && isNumberCarriedBesideTextOrBigInt(walker, entering[*place], preview.typesOfParameters[*place]); + static constexpr OpcodeSet stepsOfNumbers { Opcode::ToNumeric, Opcode::Increment, Opcode::Decrement }; + auto expectDoublesWhereElementIsMoved = [&](const BodyValue& operation) { + if (operation.opcode() != Opcode::ElemSet || readsOffListsOfUnknownContents.isEmpty() || index.loop(loop).numberOfLoopsWithin != 1 || walker.copyForInt32sIsStruck() || !walker.canUseIterationGuards(inputHead)) + return false; + ValueID made = index.stripCasts(operation.valueOperand()); + size_t place = readsOffListsOfUnknownContents.findIf([&](const ReadOffListOfUnknownContents& kept) { + return kept.read == made; + }); + if (place == notFound) + return false; + Reference taken = readsOffListsOfUnknownContents[place].name; + static constexpr OpcodeSet makersInSight { Opcode::NewArray, Opcode::Data }; + if (makersInSight.contains(walker.code().at(underlyingCellOf(walker.code(), taken.id())).opcode)) + return false; + for (HoistedArrayGuard& list : found.lists) { + if (list.name != taken || list.storage != ArrayStorage::Other) + continue; + list.storage = ArrayStorage::Double; + list.leavesBirthAsItIs = true; + list.hasAlternateStorage = true; + list.alternateStorage = ArrayStorage::Contiguous; + } + listsOfUnknownContents.removeAll(taken); + readsOffListsOfUnknownContents.removeAllMatching([&](const ReadOffListOfUnknownContents& kept) { + if (kept.name == taken) + numbersBehindListsFound.set(kept.read.index() - firstValueOfLoop); + return kept.name == taken; + }); + walker.noteCopyForInt32s(); + return true; }; auto expectNumbersWhereElementIsTakenForNumber = [&](const BodyValue& operation) { - static constexpr OpcodeSet operatorsThatTakeNumbers = numberProducingOperators | allBitwiseOperators; + static constexpr OpcodeSet operatorsThatTakeNumbers = numberProducingOperators | allBitwiseOperators | stepsOfNumbers; bool isEqualityInLoopThatStandsInNoOther = allEqualities.contains(operation.opcode()) && !index.loop(index.loopOf(inputHead)).parent; bool takesTextsToo = operationsThatJoinTexts.contains(operation.opcode()) || allRelations.contains(operation.opcode()) || isEqualityInLoopThatStandsInNoOther; - if (readsOffListsOfUnknownContents.isEmpty() || !(takesTextsToo || operatorsThatTakeNumbers.contains(operation.opcode()))) + if (readsOffListsOfUnknownContents.isEmpty() || !(takesTextsToo || operatorsThatTakeNumbers.contains(operation.opcode()) || isMathCall(walker, index, operation))) return false; Reference taken; operation.forEachOperand([&](ValueID operand, uint32_t position) { @@ -2197,10 +2441,10 @@ static ListsReadInLoop findListsReadInLoopInListReadsInLoops(Walker& walker, Blo if (taken || place == notFound) return; ValueID other = takesTextsToo && position < 2 ? operation.operand(1 - position) : ValueID(); - bool isHeldAgainstElementOfSuchList = other && allRelations.contains(operation.opcode()) && readsOffListsOfUnknownContents.containsIf([&](const ReadOffListOfUnknownContents& kept) { + bool isHeldAgainstElementOfSuchList = other && (allRelations.contains(operation.opcode()) || operation.opcode() == Opcode::Add) && readsOffListsOfUnknownContents.containsIf([&](const ReadOffListOfUnknownContents& kept) { return kept.read == index.stripCasts(other); }); - if (takesTextsToo && !isHeldAgainstElementOfSuchList && (!other || !(isNumberBehindListsFound(other) || entersLoopAsNumber(other)))) + if (takesTextsToo && !isHeldAgainstElementOfSuchList && (!other || !isNumberBehindListsFound(other))) return; taken = readsOffListsOfUnknownContents[place].name; }); @@ -2281,11 +2525,14 @@ static ListsReadInLoop findListsReadInLoopInListReadsInLoops(Walker& walker, Blo }); } while (expectNumbersWhereElementIsTakenForNumber(operation)) { } + bool movesElement = expectDoublesWhereElementIsMoved(operation); expectBoxedValuesWhereElementIsReadByName(operation); if (numberConstantOpcodes.contains(operation.opcode())) noteNumberBehindListsFound(operation, operation.opcode() == Opcode::Integer && operation.as().integer() == static_cast(operation.as().integer())); else if (operatorsThatMakeNumbersOfNumbers.contains(operation.opcode()) && isNumberBehindListsFound(operation.operand(0)) && isNumberBehindListsFound(operation.operand(1))) noteNumberBehindListsFound(operation, operation.opcode() != Opcode::Div && isWholeNumberBehindListsFound(operation.operand(0)) && isWholeNumberBehindListsFound(operation.operand(1))); + else if (stepsOfNumbers.contains(operation.opcode()) && isNumberBehindListsFound(operation.operand(0))) + noteNumberBehindListsFound(operation, isWholeNumberBehindListsFound(operation.operand(0))); else if (operation.opcode() == Opcode::Negate && isNumberBehindListsFound(operation.operand(0))) noteNumberBehindListsFound(operation, false); else if (asksItsSiteAndIsReckonedWith(operation)) @@ -2348,8 +2595,8 @@ static ListsReadInLoop findListsReadInLoopInListReadsInLoops(Walker& walker, Blo found.lists.append(HoistedArrayGuard { name, ArrayStorage::Int32, false }); else found.lists.append(HoistedArrayGuard { name, isNumeric ? ArrayStorage::Double : written, isNumeric }); - if (ValueID stepValue = found.lists.last().storage == ArrayStorage::Int32 ? findStepValue(index, operation) : ValueID()) - noteNumberBehindListsFound(index.value(stepValue), true); + if (ValueID stepValue = isNumericStorage(found.lists.last().storage) ? findStepValue(index, operation) : ValueID()) + noteNumberBehindListsFound(index.value(stepValue), found.lists.last().storage == ArrayStorage::Int32); if (ValueID stepValue = written == ArrayStorage::Other && (!seenByLink || types.at(seenByLink).isAnything()) ? findStepValue(index, operation) : ValueID()) { listsOfUnknownContents.append(name); readsOffListsOfUnknownContents.append({ stepValue, name }); @@ -2482,22 +2729,28 @@ static ListsReadInLoop findListsReadInLoopInListReadsInLoops(Walker& walker, Blo stored = stored ? types.withoutPrimitives(stored, Primitive::Undefined) : TypeID(); if ((!stored || isNeverType(stored) || !types.at(stored).isNumber()) && isNumberBehindListsFound(operation.valueOperand())) stored = isWholeNumberBehindListsFound(operation.valueOperand()) ? TypeTable::int32 : TypeTable::number; - if (!stored || isNeverType(stored) || !types.at(stored).isNumber()) + bool storesNumber = stored && !isNeverType(stored) && types.at(stored).isNumber(); + bool isReadForBoxedValues = isListed && found.lists[placeInList].storage == ArrayStorage::Contiguous && !found.lists[placeInList].admitsInt32Storage; + if (!storesNumber && (!stored || isNeverType(stored) || (!isReadForBoxedValues && predictedArrayStorage(walker, types.withoutNullish(walker.ownTypeOf(name))) != ArrayStorage::Contiguous))) continue; - bool takesDoublesInCopyForThem = walker.walksCopyForAnotherKindOfStorage() && !types.isSubset(stored, TypeTable::int32); - kindStoredInto = (types.at(stored).primitives & Primitive::FractionalNumber) || takesDoublesInCopyForThem ? ArrayStorage::Double : ArrayStorage::Int32; - if (!types.isSubset(stored, TypeTable::int32)) { + bool takesDoublesInCopyForThem = storesNumber && walker.walksCopyForAnotherKindOfStorage() && !types.isSubset(stored, TypeTable::int32); + kindStoredInto = !storesNumber ? ArrayStorage::Contiguous : (types.at(stored).primitives & Primitive::FractionalNumber) || takesDoublesInCopyForThem ? ArrayStorage::Double : ArrayStorage::Int32; + if (storesNumber && !types.isSubset(stored, TypeTable::int32)) { bool isWrittenToHoldNumbersAlone = isNumericStorage(predictedArrayStorage(walker, types.withoutNullish(walker.ownTypeOf(name)))); (kindStoredInto == ArrayStorage::Double && !isWrittenToHoldNumbersAlone && !walker.walksCopyForAnotherKindOfStorage() ? listsThatTakeMoreThanInt32s : arraysNeedingExactStorage).append(name); } - if (isAllocatedAsNumeric && allocationBeforeLoop->arrayStorage() == ArrayStorage::Double) { + if (storesNumber && isAllocatedAsNumeric && allocationBeforeLoop->arrayStorage() == ArrayStorage::Double) { kindStoredInto = ArrayStorage::Double; arraysNeedingExactStorage.append(name); } + if (movesElement) + arraysNeedingExactStorage.append(name); } if (isStore && isListed) { HoistedArrayGuard& list = found.lists[placeInList]; bool isOfEitherKind = list.storage == ArrayStorage::Double && list.admitsInt32Storage; + if (list.isWritable && list.storage != kindStoredInto && (list.storage == ArrayStorage::Contiguous || kindStoredInto == ArrayStorage::Contiguous)) + continue; if (list.isWritable && list.storage != kindStoredInto) kindStoredInto = ArrayStorage::Double; else if (!list.isWritable && !isOfEitherKind && list.storage != kindStoredInto) @@ -2505,6 +2758,11 @@ static ListsReadInLoop findListsReadInLoopInListReadsInLoops(Walker& walker, Blo list.storage = kindStoredInto; list.admitsInt32Storage = false; list.isWritable = true; + if (movesElement) { + list.leavesBirthAsItIs = true; + list.hasAlternateStorage = true; + list.alternateStorage = ArrayStorage::Contiguous; + } } if (isListed && operation.opcode() == Opcode::ElemGet && listsOfUnknownContents.contains(name)) readsOffListsOfUnknownContents.append({ operation.id(), name }); @@ -2583,6 +2841,19 @@ static ListsReadInLoop findListsReadInLoopInListReadsInLoops(Walker& walker, Blo if (isStore) { found.lists.append(HoistedArrayGuard { name, kindStoredInto, false, true }); + static constexpr OpcodeSet makersInSight { Opcode::NewArray, Opcode::NewArrayLike, Opcode::Data }; + bool isMadeInSight = makersInSight.contains(walker.code().at(underlyingCellOf(walker.code(), name.id())).opcode); + if (kindStoredInto == ArrayStorage::Int32 && !isMadeInSight && index.loop(loop).numberOfLoopsWithin == 1 && arraysNeedingExactStorage.contains(name) && !walker.walksCopyForAnotherKindOfStorage() && !walker.copyForInt32sIsStruck()) { + found.lists.last().hasAlternateStorage = true; + found.lists.last().alternateStorage = ArrayStorage::Other; + walker.noteCopyForInt32s(); + walker.count(StatisticID::WholeNumberListsAskedTwice); + } + if (movesElement) { + found.lists.last().leavesBirthAsItIs = true; + found.lists.last().hasAlternateStorage = true; + found.lists.last().alternateStorage = ArrayStorage::Contiguous; + } continue; } ArrayStorage written = predictedArrayStorage(walker, list); @@ -2603,13 +2874,19 @@ static ListsReadInLoop findListsReadInLoopInListReadsInLoops(Walker& walker, Blo taken = elements; std::optional guarded = takesFractions || takesWholeNumbers || mayTakeWholeNumbers ? findGuardedReadInListReadsInLoops(index) : std::nullopt; bool goesByRowOfNumbers = guarded && guarded->test == operation.id(); - bool asksForInt32sFirst = goesByRowOfNumbers && (mayTakeWholeNumbers || (!walker.walksCopyForAnotherKindOfStorage() && takesWholeNumbers && arrayStorageHolding(types, taken) != ArrayStorage::Int32)); + bool mayTakeAnyNumber = takesFractions && (types.at(taken).primitives & Primitive::NumberWithoutFraction); + bool takesAnyNumber = mayTakeAnyNumber && !walker.copyForInt32sIsStruck(); + bool asksForInt32sFirst = goesByRowOfNumbers && (mayTakeWholeNumbers || (!walker.walksCopyForAnotherKindOfStorage() && (takesAnyNumber || (takesWholeNumbers && arrayStorageHolding(types, taken) != ArrayStorage::Int32)))); if (goesByRowOfNumbers && (asksForInt32sFirst || (takesWholeNumbers && arrayStorageHolding(types, taken) == ArrayStorage::Int32))) found.lists.append(HoistedArrayGuard { name, ArrayStorage::Int32, false, false, false, true }); else if (goesByRowOfNumbers) found.lists.append(HoistedArrayGuard { name, ArrayStorage::Double, !!(types.at(taken).primitives & Primitive::NumberWithoutFraction), false, false, true }); else found.lists.append(HoistedArrayGuard { name, ArrayStorage::Contiguous, true, false, false, isNumericStorage(written) }); + if (asksForInt32sFirst && takesAnyNumber) + walker.noteCopyForInt32s(); + else if (goesByRowOfNumbers && mayTakeAnyNumber && !takesAnyNumber) + walker.count(StatisticID::CopiesForInt32sStruck); found.lists.last().hasAlternateStorage = asksForInt32sFirst; found.lists.last().alternateStorage = mayTakeWholeNumbers ? ArrayStorage::Contiguous : ArrayStorage::Double; continue; @@ -2625,7 +2902,8 @@ static ListsReadInLoop findListsReadInLoopInListReadsInLoops(Walker& walker, Blo listsTakenUnderCountersAskedInFront.append(name); bool wholeNumbersAreOnlyStoredOn = seenByLink && types.at(types.withoutNullish(seenByLink)).isNumberWithoutFraction() && onlyUserStoresWhatIsReadIntoListInListReadsInLoops(index, operation); bool holdsInt32sByLink = seenByLink && (types.at(seenByLink).isInt32OrNullish() || holdsKeysOfLoopInListReadsInLoops(walker, operation.id(), seenByLink)); - if (holdsInt32sByLink || (wholeNumbersAreOnlyStoredOn && !walker.walksCopyForAnotherKindOfStorage())) { + bool hasCarriedIndexes = !holdsInt32sByLink && (isNumeric || written == ArrayStorage::Other) && !walker.walksCopyForAnotherKindOfStorage() && isCarriedIndex(operation.id()); + if (holdsInt32sByLink || hasCarriedIndexes || (wholeNumbersAreOnlyStoredOn && !walker.walksCopyForAnotherKindOfStorage())) { found.lists.append(HoistedArrayGuard { name, ArrayStorage::Int32, false }); found.lists.last().hasAlternateStorage = !holdsInt32sByLink && !index.loop(index.loopOf(inputHead)).parent; found.lists.last().alternateStorage = ArrayStorage::Double; @@ -2633,7 +2911,13 @@ static ListsReadInLoop findListsReadInLoopInListReadsInLoops(Walker& walker, Blo noteNumberBehindListsFound(operation, true); continue; } - found.lists.append(HoistedArrayGuard { name, isNumeric ? ArrayStorage::Double : written, isNumeric }); + bool asksForDoublesFirst = isNumeric && !walker.walksCopyForAnotherKindOfStorage() && !walker.copyForInt32sIsStruck() && index.loop(loop).numberOfLoopsWithin > 1 && walker.hasUnrolledInnerLoops(inputHead); + found.lists.append(HoistedArrayGuard { name, isNumeric ? ArrayStorage::Double : written, isNumeric && !asksForDoublesFirst }); + if (asksForDoublesFirst) { + found.lists.last().hasAlternateStorage = true; + found.lists.last().alternateStorage = ArrayStorage::Double; + walker.noteCopyForInt32s(); + } if (isNumeric) noteNumberBehindListsFound(operation, false); else if (written == ArrayStorage::Other) { @@ -2666,16 +2950,35 @@ static ListsReadInLoop findListsReadInLoopInListReadsInLoops(Walker& walker, Blo walker.count(StatisticID::ArraysGuardedUnderSpeculatedCounters); found.someListIsTakenUnderCounterAskedInFront = true; } - bool turnsStoreIntoLists = found.lists.containsIf([](const HoistedArrayGuard& list) { - return list.isWritable; - }); if (found.lists.isEmpty()) return found; - + auto takesRowOfListedList = [&](BodyValue access, Opcode opcode) { + ValueID row = access.opcode() == opcode ? index.stripCasts(access.objectOperand()) : ValueID(); + if (!row || row.index() < firstValueOfLoop || index.isParameter(row) || index.opcodeOf(row) != Opcode::ElemGet) + return false; + Reference holder = nameThatEntersLoopInListReadsInLoops(walker, index, index.value(row).objectOperand(), firstValueOfLoop); + return holder && foundArrayLookup.find(found.lists.span(), holder) != notFound; + }; + auto turnsTakeRows = [&](Opcode opcode) { + return index.opcodes().contains(opcode) && std::ranges::any_of(index.membersOf(loop), [&](BlockID member) { + return std::ranges::any_of(index.block(member).values(), [&](BodyValue access) { return takesRowOfListedList(access, opcode); }); + }); + }; + bool storesIntoListsInFront = found.lists.containsIf([](const HoistedArrayGuard& list) { return list.isWritable; }); + bool turnsStoreIntoLists = storesIntoListsInFront || turnsTakeRows(Opcode::ElemSet); + bool turnsReadRows = storesIntoListsInFront && turnsTakeRows(Opcode::ElemGet); + bool holdsDoublesAlone = found.lists.containsIf([](const HoistedArrayGuard& list) { + return !list.isWritable && list.hasAlternateStorage && list.storage == ArrayStorage::Double && list.alternateStorage == ArrayStorage::Double; + }); for (HoistedArrayGuard& list : found.lists) { list.turnsStoreIntoLists = turnsStoreIntoLists; list.leavesBirthAsItIs |= walker.walksCopyForAnotherKindOfStorage(); - if (list.isWritable && isNumericStorage(list.storage) && !arraysNeedingExactStorage.contains(list.name)) { + if (list.isWritable && isNumericStorage(list.storage) && !turnsReadRows && !arraysNeedingExactStorage.contains(list.name)) { + if (holdsDoublesAlone && list.storage == ArrayStorage::Double && listsThatTakeMoreThanInt32s.contains(list.name)) { + list.hasAlternateStorage = true; + list.alternateStorage = ArrayStorage::Other; + continue; + } list.storage = ArrayStorage::Other; list.admitsInt32Storage = !listsThatTakeMoreThanInt32s.contains(list.name); } @@ -2697,13 +3000,15 @@ static ListsReadInLoop findListsReadInLoopInListReadsInLoops(Walker& walker, Blo std::ranges::sort(namesUsedEveryIteration); found.lists.removeAllMatching([&](const HoistedArrayGuard& list) { bool needsOnlyStorageCheck = !list.isWritable && walker.types().isSubset(walker.typeOf(list.name), walker.types().builtinClass(BuiltinClasses::Array)); + bool isStoredIntoAnyway = list.isWritable && hasKind(types, types.withoutNullish(walker.ownTypeOf(list.name)), OperandKind::Array); bool hasSpeculation = speculationLookup.find(guesses.span(), list.name) != notFound; bool isUsedEveryIteration = didSortNames ? std::ranges::binary_search(namesUsedEveryIteration, list.name.id().index()) : namesUsedEveryIteration.contains(list.name.id().index()); - return !isUsedEveryIteration && !needsOnlyStorageCheck && !hasSpeculation; + return !isUsedEveryIteration && !needsOnlyStorageCheck && !hasSpeculation && !isStoredIntoAnyway; }); if (found.lists.isEmpty()) return { }; + NamesBeforeLoop namesBeforeLoop(walker, inputHead); for (const HoistedArrayGuard& list : found.lists) { bool hasSpeculation = speculationLookup.find(guesses.span(), list.name) != notFound; if (hasSpeculation || isProvenArray(walker, list.name) || classOfProvenViewInListReadsInLoops(walker, list.name)) @@ -2711,7 +3016,7 @@ static ListsReadInLoop findListsReadInLoopInListReadsInLoops(Walker& walker, Blo TypeID ownType = types.withoutNullish(walker.ownTypeOf(list.name)); BuiltinClassID classOfView = classOfViewInListReadsInLoops(walker, ownType); TypeID asked = kindsOfViewsOfNumbersIn(walker.program(), types, ownType) ? ownType : types.builtinClass(classOfView ? classOfView : BuiltinClassID(BuiltinClasses::Array)); - guesses.append(HoistedSpeculation { list.name, asked, { }, namesInFrontThatLoopMentions(walker, index, loop, firstValueOfLoop, list.name) }); + guesses.append(HoistedSpeculation { list.name, asked, { }, namesBeforeLoop.findNamesOf(list.name) }); } return found; } @@ -2840,6 +3145,53 @@ std::optional tryHoistDestructuredArrayGuard(Walker& walker, return tryHoistArrayGuard(walker, HoistedArrayGuard { list, isNumeric ? ArrayStorage::Double : written, isNumeric }, walker.originOf(open), purity, Walker::GuardedIterations::Some); } +static Reference holderThatRowIsReadOffInListReadsInLoops(Walker&, Reference row); + +static ValueID holderOfRowAccess(const BodyIndex& index, const BodyValue& access) +{ + ValueID row = index.stripCasts(access.objectOperand()); + return row && !index.isParameter(row) && index.opcodeOf(row) == Opcode::ElemGet ? index.stripCasts(index.value(row).objectOperand()) : ValueID(); +} + +static bool userNeedsNumberBeside(Walker& walker, const BodyValue& read) +{ + static constexpr OpcodeSet arithmetic { Opcode::Add, Opcode::Sub, Opcode::Mul, Opcode::Div }; + const BodyIndex& index = walker.currentBodyIndex(); + ValueID name = read.id(); + ValueID user = index.onlyUsingValue(name); + while (user && opcodesThatStateTypes.contains(index.opcodeOf(user))) { + name = user; + user = index.onlyUsingValue(user); + } + if (!user || !arithmetic.contains(index.opcodeOf(user))) + return true; + BodyValue operation = index.value(user); + ValueID other = operation.operand(0) == name ? operation.operand(1) : operation.operand(0); + Reference built = other.index() < read.id().index() ? walker.referenceOf(other) : Reference(); + return built && !walker.types().at(walker.typeOf(built)).hasOnlyPrimitives(Primitive::Number | Primitive::Undefined); +} + +enum class RowStores : uint8_t { None, IntoOthers, IntoSameHolder }; + +static RowStores findRowStores(const BodyIndex& index, const BodyValue& read) +{ + LoopID loop = index.holdsAnyOf(OpcodeSet { Opcode::ElemSet }) && isReadOfBodyInListReadsInLoops(index, read) ? index.loopOf(index.blockOf(read.id())) : LoopID(); + if (!loop) + return RowStores::None; + RowStores found = RowStores::None; + ValueID holder = holderOfRowAccess(index, read); + for (BlockID member : index.membersOf(loop)) { + for (BodyValue operation : index.block(member).values()) { + if (operation.opcode() != Opcode::ElemSet) + continue; + if (holder && holderOfRowAccess(index, operation) == holder) + return RowStores::IntoSameHolder; + found = RowStores::IntoOthers; + } + } + return found; +} + std::optional tryHoistArrayGuardForAccess(Walker& walker, const BodyValue& read, Reference list, ArrayStorage kindStoredInto, bool goesByKindOfStorage) { Walker::IterationPurity purity = walker.arrayGuardPurity(); @@ -2868,11 +3220,22 @@ std::optional tryHoistArrayGuardForAccess(Walker& walker, con HoistedArrayGuard tested { list, isNumeric ? ArrayStorage::Double : written, isNumeric }; if (seenByLink && (walker.types().at(seenByLink).isInt32OrNullish() || (isReadOfBodyInListReadsInLoops(walker.currentBodyIndex(), read) && holdsKeysOfLoopInListReadsInLoops(walker, read.id(), seenByLink)))) tested = HoistedArrayGuard { list, ArrayStorage::Int32, false }; - if (kindStoredInto != ArrayStorage::Other) + if (kindStoredInto != ArrayStorage::Other && goesByKindOfStorage && holderThatRowIsReadOffInListReadsInLoops(walker, list)) + tested = HoistedArrayGuard { list, ArrayStorage::Double, kindStoredInto == ArrayStorage::Int32, true, true }; + else if (kindStoredInto != ArrayStorage::Other) tested = goesByKindOfStorage && (kindStoredInto == ArrayStorage::Int32 || !isNumeric) ? HoistedArrayGuard { list, ArrayStorage::Other, kindStoredInto == ArrayStorage::Int32, true, true } : HoistedArrayGuard { list, kindStoredInto, false, true, true }; else if (WrittenListOfVariable written = writtenListReadInListReadsInLoops(walker, walker.currentBodyIndex(), read)) { tested = HoistedArrayGuard { list, written.storage, false }; tested.written = written; + } else if (isNumericStorage(tested.storage) && holderThatRowIsReadOffInListReadsInLoops(walker, list)) { + RowStores stores = findRowStores(walker.currentBodyIndex(), read); + bool storesWholeNumbers = stores == RowStores::IntoSameHolder && seenByLink && !isNeverType(seenByLink) && walker.types().at(seenByLink).hasOnlyPrimitives(Primitive::NumberWithoutFraction | Primitive::Undefined); + if (storesWholeNumbers) + tested = HoistedArrayGuard { list, ArrayStorage::Int32, false }; + if (stores == RowStores::IntoSameHolder && seenByLink && !walker.types().at(seenByLink).isAnything() && (walker.types().at(seenByLink).primitives & Primitive::FractionalNumber)) + tested.admitsInt32Storage = false; + bool leavesAtHole = tested.storage == ArrayStorage::Int32 || !onlyUserTakesUndefinedForNaNInTypedElements(walker, read, WhatStandsBesideRead::NumberOrNothing); + tested.turnsStoreIntoLists = stores != RowStores::None && leavesAtHole && (!tested.admitsInt32Storage || userNeedsNumberBeside(walker, read)); } return tryHoistArrayGuard(walker, tested, walker.originOf(read), purity, kindStoredInto == ArrayStorage::Other ? Walker::GuardedIterations::Some : Walker::GuardedIterations::All); } @@ -2984,10 +3347,10 @@ std::optional tryGuardDestructuredArrayInIteration(Walker& wa std::optional tryHoistElementPresenceCheck(Walker& walker, const BodyValue& read, const HoistedArrayGuard& tested, Reference list, Reference position) { bool isNumber = isNumericStorage(tested.storage); - if (!walker.has(Mechanism::Folding) || walker.listsAreStruckInRow() || (tested.storage != ArrayStorage::Contiguous && !isNumber) || tested.isWritable || (isNumber && tested.turnsStoreIntoLists)) + if (!walker.has(Mechanism::Folding) || walker.listsAreStruckInRow() || (tested.storage != ArrayStorage::Contiguous && !isNumber) || holdsBoxedValuesOfEitherKind(tested) || tested.isWritable || (isNumber && tested.turnsStoreIntoLists)) return std::nullopt; Walker::IterationPurity purity = isNumber ? Walker::IterationPurity::EffectFree : walker.arrayGuardPurity(); - if (!walker.isInvariantInSpecializedVersion(list, purity) || !walker.isInvariantInSpecializedVersion(position, purity) || !walker.iterationsPreserveArrayGuards(list, purity) || !(isNumber ? walker.hasNoElementStores() : walker.areElementStoresRawDoubles(purity))) + if (!walker.isInvariantInSpecializedVersion(list, purity) || !walker.isInvariantInSpecializedVersion(position, purity) || !walker.iterationsPreserveArrayGuards(list, purity) || !(isNumber ? walker.hasNoElementStores() : walker.areElementStoresNumeric(purity))) return std::nullopt; Origin origin = walker.originOf(read); origin.isTailCall = false; @@ -3033,8 +3396,38 @@ static bool isBornToHoldDoublesInListReadsInLoops(Walker& walker, Reference list return born.opcode == Opcode::NewArrayLike && !born.sourceOperand() && born.arrayStorage() == ArrayStorage::Double; } +static void appendViewsReadOutOfVariablesInListReadsInLoops(Walker& walker, BlockID inputHead, ListsReadInLoop& found) +{ + if (walker.isUnreachable() || walker.hasFailed() || walker.effectsOfQuietDraftsOfRow().writes.contains(AbstractHeap::TypedArrayStorage)) + return; + const BodyIndex& index = walker.currentBodyIndex(); + if (!index.opcodes().contains(Opcode::CheckInitialized) || !index.opcodes().contains(Opcode::GetModuleVariable)) + return; + for (BlockID member : index.membersOf(index.loopOf(inputHead))) { + for (BodyValue operation : index.block(member).values()) { + if (operation.opcode() != Opcode::ElemGet && operation.opcode() != Opcode::ElemSet) + continue; + ValueID checked = index.stripCasts(operation.objectOperand()); + if (!checked || index.isParameter(checked) || index.opcodeOf(checked) != Opcode::CheckInitialized) + continue; + ValueID read = index.stripCasts(index.value(checked).operand(0)); + if (!read || index.isParameter(read) || index.opcodeOf(read) != Opcode::GetModuleVariable || !leavesDeadZoneForGood(walker.program(), index.value(read).variable())) + continue; + Reference name = walker.state().variable(index.value(read).variable()); + BuiltinClassID classOfView = classOfProvenViewInListReadsInLoops(walker, name); + if (!classOfView || !hasInlineElementAccess(walker.program().builtins().typedArrayTypeOf(classOfView)) || !viewMayFollowSharedMemoryThatGrows(walker, name) || findNoteInListReadsInLoops(walker, name)) + continue; + if (found.lists.containsIf([&](const HoistedArrayGuard& list) { return list.name == name; })) + continue; + found.lists.append(HoistedArrayGuard { name, ArrayStorage::Other, false }); + found.lists.last().isAskedForItsStorageAlone = true; + } + } +} + void buildTestsOfListsReadInLoop(Walker& walker, BlockID inputHead, std::span entering, ListsReadInLoop& found, std::span guesses, std::span proofs, BlockID& genericStart, Origin origin) { + appendViewsReadOutOfVariablesInListReadsInLoops(walker, inputHead, found); if (found.lists.isEmpty() || walker.isUnreachable() || walker.hasFailed()) return; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkListReadsInLoops.h b/Source/JavaScriptCore/aot/walk/AOTWalkListReadsInLoops.h index 57c0b132370e2..1391a60cd495a 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkListReadsInLoops.h +++ b/Source/JavaScriptCore/aot/walk/AOTWalkListReadsInLoops.h @@ -30,7 +30,7 @@ struct HoistedArrayGuard { inline bool isConvertedInPreheader(const HoistedArrayGuard& list) { - return (list.storage == ArrayStorage::Double || !hasArrayStorageForm(list.storage)) && list.isWritable && !list.admitsInt32Storage; + return (list.storage == ArrayStorage::Double || !hasArrayStorageForm(list.storage)) && list.isWritable && !list.admitsInt32Storage && !list.hasAlternateStorage; } BuiltinClassID classOfViewBesideInstancesBelowIt(const Walker&, TypeID); @@ -40,6 +40,7 @@ Reference buildTestOfViewOrInstanceBelow(Walker&, Reference view, Origin); void noteThatLengthOfViewIsFoundAbove(Walker&, Reference view); void askAgainWhetherLengthOfViewIsFoundAbove(Walker&, Reference view, Origin); std::optional findHoistedArrayGuard(Walker&, const BodyValue& read); +bool isExpectedToBeView(Walker&, ValueID unbuiltList); std::optional findHoistedArrayGuard(Walker&, Reference list); std::optional findListTestedForStoresInFrontOfLoop(Walker&, Reference list); std::optional tryGuardNumericArrayInIteration(Walker&, const BodyValue& read, Reference list); @@ -62,7 +63,9 @@ struct WhatReadsFromInFrontOfLoopFind { }; bool readsListFoundByReadFromInFront(const Program&, const BodyIndex&, const BodyValue& operation, uint32_t firstValueOfLoop, WhatReadsFromInFrontOfLoopFind&); void readListsOfStableBindingsInFrontOfLoop(Walker&, BlockID inputHead); +void readListsOfFieldsInFrontOfLoop(Walker&, BlockID inputHead); Reference nameOfListThatEntersLoopAround(Walker&, const BodyIndex&, const BodyValue& access); +void forEachElementAccessInInlinedCallee(Walker&, const BodyValue& call, NOESCAPE const ScopedLambda&); bool copyOfCalleeReadsUnderKeyWhatNobodyKnows(Walker&, const BodyValue& call, uint32_t firstValueOfLoop); ValueID readOfVariableThatFindsReceiverInLoop(const BodyIndex&, const BodyValue& access, uint32_t firstValueOfLoop); bool storesIntoListTestedInFrontOfLoop(Walker&, const BodyIndex&, const BodyValue& store, uint32_t firstValueOfLoop, std::span); @@ -72,14 +75,40 @@ bool positionIsKnownToLieInsideList(Walker&, Reference list, Reference position) Reference storageOfListFromInFrontOfLoop(Walker&, const HoistedArrayGuard&, Origin); bool holeLeavesTurnBehindRow(Walker&, const BodyValue& read, const HoistedArrayGuard&); +bool guardElementPresenceInIteration(Walker&, Draft& read, Reference list, Reference position, const HoistedArrayGuard&); +bool turnTakesElementForKeyInListReadsInLoops(Walker&, ValueID element); bool hasOnlyOtherArrayReads(Walker&, BlockID inputHead, ValueID made); +bool isResultUnused(const BodyIndex&, ValueID result); enum class WhatStandsBesideRead : uint8_t { Number, NumberOrNothing }; bool onlyUserTakesUndefinedForNaNInTypedElements(Walker&, const BodyValue& read, WhatStandsBesideRead = WhatStandsBesideRead::Number); bool readLeavesTurnAtHoleBehindRow(Walker&, const BodyValue& read); bool preservesHoistedArrayGuards(const Walker&, const Effects&); bool preservesHoistedArrayGuards(const Walker&, const Effects&, std::span, const Walker::AllocationsUsedInLoop&); bool preservesHoistedArrayGuards(Walker&, const Effects&, std::span, BlockID inputHead); -Vector namesInFrontThatLoopMentions(const Walker&, const BodyIndex&, LoopID, uint32_t firstValueOfLoop, Reference value); + +class NamesBeforeLoop { +public: + NamesBeforeLoop(const Walker& walker, BlockID inputHead) + : m_walker(walker) + , m_inputHead(inputHead) + { + } + + Vector findNamesOf(Reference value); + +private: + struct Name { + uint32_t value; + ValueID id; + }; + void collectNames(); + + const Walker& m_walker; + BlockID m_inputHead; + bool m_areNamesCollected { false }; + Vector m_names; +}; + bool readsLengthOfViewTestedInFrontOfLoop(Walker&, const BodyValue& read); struct ListsReadInLoop { diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkListedOwnKeys.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkListedOwnKeys.cpp index efb9f5c8d8e37..89729689ed0a2 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkListedOwnKeys.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkListedOwnKeys.cpp @@ -16,6 +16,7 @@ #include "AOTOpcodeSets.h" #include "AOTTypeArena.h" #include "AOTWalkCopies.h" +#include "AOTWalkKnown.h" #include #include #include @@ -228,6 +229,19 @@ static bool buildListBehindTestsInListedOwnKeys(Walker& walker, const CallOfOwnK return true; } +void noteListedOwnKeys(Walker& walker, ValueID id) +{ + BodyValue call = walker.currentBodyIndex().value(id); + if (call.opcode() != Opcode::Call || listAskedForInListedOwnKeys(walker, call) != OwnKeyList::Keys) + return; + Reference list = walker.isUnreachable() ? Reference() : walker.referenceBoundTo(call.id()); + Reference object = walker.referenceOf(call.argumentsOperands()[0]); + if (!list || !list.isValue() || !object || !object.isValue()) + return; + static constexpr AbstractHeapSet heapsThatTellOwnKeys { AbstractHeap::ShapeWords, AbstractHeap::OrderWords, AbstractHeap::NamedProperties }; + noteKnown(walker, KnownNote::ListedOwnKeys, object, list, 0, LocationMasks(heapsThatTellOwnKeys), 1, true); +} + bool buildOwnKeyListOfItsOpcodes(Walker& walker, const BodyValue& value) { if (!walker.has(Mechanism::BuiltinCalls) || !walker.has(Mechanism::TypedOperations)) @@ -247,6 +261,8 @@ bool buildOwnKeyListOfItsOpcodes(Walker& walker, const BodyValue& value) if (!buildListOfObjectInSightInListedOwnKeys(walker, asked, source) && !buildListBehindTestsInListedOwnKeys(walker, asked, written, source)) return false; walker.count(StatisticID::OwnKeyWalksExpanded); + if (*list == OwnKeyList::Keys) + noteListedOwnKeys(walker, value.id()); return true; } diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkListedOwnKeys.h b/Source/JavaScriptCore/aot/walk/AOTWalkListedOwnKeys.h index a2cbd973c024e..aaa018e228250 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkListedOwnKeys.h +++ b/Source/JavaScriptCore/aot/walk/AOTWalkListedOwnKeys.h @@ -14,6 +14,7 @@ namespace JSC { namespace AOT { namespace Typed { inline constexpr OpcodeSet opcodesThatBuildOwnKeyListTakes { Opcode::Call }; bool buildOwnKeyListOfItsOpcodes(Walker&, const BodyValue&); +void noteListedOwnKeys(Walker&, ValueID call); ALWAYS_INLINE bool buildOwnKeyList(Walker& walker, const BodyValue& value) { return value.opcode() == Opcode::Call && value.as().argumentsOperands().size() == 1 && buildOwnKeyListOfItsOpcodes(walker, value); } template ALWAYS_INLINE bool buildOwnKeyList(Walker& walker, const BodyValueOf& value) { return known == Opcode::Call && value.template as().argumentsOperands().size() == 1 && buildOwnKeyListOfItsOpcodes(walker, value); } diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkLoopInvariants.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkLoopInvariants.cpp index 803b0f4e3262a..7b4566bbd2473 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkLoopInvariants.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkLoopInvariants.cpp @@ -16,6 +16,7 @@ #include "AOTPlan.h" #include "AOTRepresentationRules.h" #include "AOTResultRules.h" +#include "AOTWalkKnown.h" #include "AOTWalkListReadsInLoops.h" #include "AOTWalkStrings.h" #include @@ -363,7 +364,7 @@ bool Walker::isInvariantInLoop(uint32_t indexOfLoop, Reference reference) const if (!place || place - 1 > indexOfLoop) return false; const Operation& operation = m_code.at(reference.id()); - return place - 1 < indexOfLoop || (operation.opcode != Opcode::Strcat && (operation.opcode != Opcode::FlattenString || !operation.isReadOperand())); + return place - 1 < indexOfLoop || (operation.opcode != Opcode::Strcat && (operation.opcode != Opcode::FlattenString || operation.isReadOperands().empty())); } static bool mayLeaveSomeLoopForLoopInvariants(const TypeArena& types, const Draft& draft, const Effects& effects, bool proofsHoldEverywhere) @@ -377,7 +378,8 @@ static bool mayLeaveSomeLoopForLoopInvariants(const TypeArena& types, const Draf return false; if (!effects.writes.isEmpty() || effects.mayThrow || !effects.alwaysCompletes() || effects.reads.containsAny(bookkeepingHeaps)) return false; - if (!proofsHoldEverywhere && (reliesOf(opcode).contains(Proof::SlotFilled) || reliesOf(opcode).contains(Proof::InBounds))) + bool readsStableField = opcode == Opcode::GetSlot && effects.reads.isEmpty(); + if (!proofsHoldEverywhere && !readsStableField && (reliesOf(opcode).contains(Proof::SlotFilled) || reliesOf(opcode).contains(Proof::InBounds))) return false; if (conversionsOfForm.contains(opcode)) @@ -477,8 +479,15 @@ bool Walker::isGuaranteedToExecute(uint32_t loopIndex, const HoistPoint& point) } if (loop.exitDominator && !index.dominates(blockInFrame(loop.frame), loop.exitDominator)) return false; + return precedesReturnsOfInlinedCallees(loopIndex, point); +} - for (size_t frameIndex = loop.frame + 1; frameIndex <= point.frameIndex; ++frameIndex) { +bool Walker::precedesReturnsOfInlinedCallees(uint32_t loopIndex, const HoistPoint& point) const +{ + auto blockInFrame = [&](size_t frameIndex) { + return frameIndex == point.frameIndex ? point.inputBlock : m_frames[frameIndex].block; + }; + for (size_t frameIndex = m_openLoops[loopIndex].frame + 1; frameIndex <= point.frameIndex; ++frameIndex) { const Frame& frame = m_frames[frameIndex]; const BodyIndex& callee = *frame.index; if (!frame.exitDominator) { @@ -510,6 +519,40 @@ bool Walker::headerMayExitBefore(uint32_t loopIndex, const HoistPoint& point) co return exits; } +bool Walker::standsOnMainPath(uint32_t loopIndex, const HoistPoint& point) const +{ + constexpr unsigned mostBlocksFollowed = 16; + const OpenLoop& loop = m_openLoops[loopIndex]; + bool isReachedByText = executesOnEveryIteration(loopIndex, point); + BlockID block = point.block; + for (unsigned step = 0; block && step < mostBlocksFollowed; ++step) { + EdgeID entering = m_code.at(block).firstEdgeIn(); + if (!entering || m_code.at(entering).nextInto()) + return isReachedByText; + BlockID from = m_code.at(entering).from; + const Terminator& end = m_code.at(from).terminator; + if (end.numberOfEdges > 2) + return false; + bool someOtherEdgeIsUsual = false; + for (uint32_t index = 0; index < end.numberOfEdges; ++index) { + const Edge& edge = m_code.at(end.edge(index)); + someOtherEdgeIsUsual |= edge.to != block && (!edge.to || m_code.at(edge.to).frequency == BlockFrequency::Normal); + } + if (from == loop.head) { + const BodyIndex& index = *m_frames[loop.frame].index; + bool exits = false; + index.block(index.loop(loop.loop).head).forEachOutgoingEdge([&](BodyTarget target, bool) { + exits |= !index.isInLoop(target.block(), loop.loop); + }); + return exits || !someOtherEdgeIsUsual; + } + if (someOtherEdgeIsUsual) + return false; + block = from; + } + return isReachedByText; +} + std::optional Walker::findLoopToBuildInFrontOf(const Draft& draft, const Effects& effects, TypeID proven) const { if (draft.opcode == Opcode::GetValueAtSiteOrAbsent && (effects.reads.containsAny(AbstractHeapSet { AbstractHeap::RuntimeState }) || effects.writes.containsAny(AbstractHeapSet { AbstractHeap::RuntimeState }))) { @@ -579,7 +622,9 @@ std::optional Walker::findLoopToBuildInFrontOf(const Draft& draft, con return operand && (operand.isConstant() || (operand.isValue() && writtenNumbers.contains(m_code.at(operand.id()).opcode))); }); bool mayStandWhereNotEveryTurnAsks = draft.opcode == Opcode::ResolveCallTarget || ((findsList || isSumWithWrittenNumber || neverFailOfList.contains(draft.opcode)) && m_code.at(m_block).frequency != BlockFrequency::Rare); - if (!mayStandWhereNotEveryTurnAsks && !executesOnEveryIteration(index, point)) + static constexpr OpcodeSet preparedDivisors { Opcode::WholeDivisor, Opcode::ReciprocalOfDivisor }; + bool asksMainPath = preparedDivisors.contains(draft.opcode) || (draft.opcode == Opcode::GetSlot && reads.isEmpty()); + if (!mayStandWhereNotEveryTurnAsks && !(asksMainPath ? standsOnMainPath(index, point) : executesOnEveryIteration(index, point))) return IterationStatus::Done; if (readsWhatTurnsMayWrite && loop.watchesTurns) { noteReadThatCouldStandInFront(loop, reads); @@ -684,24 +729,13 @@ std::optional Walker::findLoopEntryCondition(uint32_ return std::nullopt; if (behindYes == head.id() || !index.dominates(behindYes, frame.block)) return std::nullopt; + if (std::optional condition = findConditionOnEntry(indexOfLoop, asked)) + return condition; std::span entering = open.recordsEntering(); - auto asItEnters = [&](ValueID written) -> Reference { - ValueID value = index.stripCasts(written); - Reference reference; - if (index.isParameter(value) && index.blockOf(value) == head.id()) { - uint32_t position = value.index() - head.firstParameter().index(); - if (position < entering.size() && entering[position]) - reference = m_code.isConstant(entering[position]) ? Reference::constant(entering[position]) : Reference::value(entering[position]); - } else - reference = referenceBoundIn(frame, written); - bool isNumberInFront = reference && reference.isRecord() && isInvariantInLoop(indexOfLoop, reference) && typeOf(reference) == ownTypeOf(reference) && m_types.at(ownTypeOf(reference)).isNumber(); - return isNumberInFront ? reference : Reference(); - }; - BodyValue relation = index.value(asked); - Reference left = asItEnters(relation.operand(0)); - Reference right = asItEnters(relation.operand(1)); - if (!left || !right) { + Reference left; + Reference right; + { Reference headerTest = referenceBoundIn(frame, asked); if (!headerTest || !headerTest.isValue() || m_code.blockOf(headerTest.id()) != open.head || m_code.at(headerTest.id()).opcode != index.opcodeOf(asked)) return std::nullopt; @@ -723,21 +757,125 @@ std::optional Walker::findLoopEntryCondition(uint32_ return LoopEntryCondition { asked, left, right }; } -Reference Walker::buildLoopEntryCondition(uint32_t indexOfLoop) +std::optional Walker::findConditionOnEntry(uint32_t indexOfLoop, ValueID asked) const +{ + const OpenLoop& open = m_openLoops[indexOfLoop]; + const Frame& frame = m_frames[open.frame]; + const BodyIndex& index = *frame.index; + BodyBlock head = index.block(index.loop(open.loop).head); + std::span entering = open.recordsEntering(); + auto asItEnters = [&](ValueID written) -> Reference { + ValueID value = index.stripCasts(written); + Reference reference; + if (index.isParameter(value) && index.blockOf(value) == head.id()) { + uint32_t position = value.index() - head.firstParameter().index(); + if (position < entering.size() && entering[position]) + reference = m_code.isConstant(entering[position]) ? Reference::constant(entering[position]) : Reference::value(entering[position]); + } else + reference = referenceBoundIn(frame, written); + bool isNumberInFront = reference && reference.isRecord() && isInvariantInLoop(indexOfLoop, reference) && typeOf(reference) == ownTypeOf(reference) && m_types.at(ownTypeOf(reference)).isNumber(); + return isNumberInFront ? reference : Reference(); + }; + BodyValue relation = index.value(asked); + Reference left = asItEnters(relation.operand(0)); + Reference right = asItEnters(relation.operand(1)); + if (!left || !right) + return std::nullopt; + return LoopEntryCondition { asked, left, right }; +} + +Reference Walker::buildConditionOnEntry(uint32_t indexOfLoop, const LoopEntryCondition& asked) { - std::optional asked = findLoopEntryCondition(indexOfLoop); - if (!asked) - return { }; const OpenLoop& open = m_openLoops[indexOfLoop]; Origin origin = open.origin; - Opcode opcode = m_frames[open.frame].index->opcodeOf(asked->written); + Opcode opcode = m_frames[open.frame].index->opcodeOf(asked.written); BlockID inside = std::exchange(m_block, open.preheaderBlock); SetForScope loopsThatMayBeLeft(m_numberOfLoopsThatMayBeLeft, indexOfLoop); - Reference begins = buildOperatorForStrings(*this, opcode, asked->left, asked->right, origin); + Reference condition = buildOperatorForStrings(*this, opcode, asked.left, asked.right, origin); + if (condition && asked.isNegated && !m_hasFailed) + condition = buildNegationForStrings(*this, condition, origin); m_block = m_hasFailed ? BlockID() : inside; + return condition; +} + +Reference Walker::buildLoopEntryCondition(uint32_t indexOfLoop) +{ + std::optional asked = findLoopEntryCondition(indexOfLoop); + if (!asked) + return { }; + Reference begins = buildConditionOnEntry(indexOfLoop, *asked); return begins && begins.isValue() && representationOf(begins) == Representation::Boolean ? begins : Reference(); } +bool Walker::isFlattenedBeforeLoop(uint32_t loopIndex, Reference string, std::span conditions) const +{ + const OpenLoop& loop = m_openLoops[loopIndex]; + auto behindConversions = [&](ValueID value) { + while (conversionsOfForm.contains(m_code.at(value).opcode)) + value = m_code.at(value).operand(0); + return value; + }; + for (uint32_t place = 0; place < loop.builtInFront.size(); ++place) { + ValueID id = ValueID::fromBits(loop.builtInFront[place]); + const Operation& record = m_code.at(id); + if (record.opcode != Opcode::FlattenString || record.mayStayRope() || (!record.isReadOperands().empty() && id.index() < loop.firstRecord) || behindConversions(record.stringOperand()) != behindConversions(string.id())) + continue; + bool hasSameConditions = std::ranges::equal(record.isReadOperands(), conditions, [](ValueID held, Reference taken) { + return held == taken.id(); + }); + if (hasSameConditions) + return true; + } + return false; +} + +bool Walker::tryBuildFirstArrivalConditions(uint32_t loopIndex, Vector& conditions) +{ + if (Reference begins = buildLoopEntryCondition(loopIndex)) + conditions.append(begins); + HoistPoint point = currentHoistPoint(); + if (m_hasFailed || !executesOnEveryIteration(loopIndex, point) || !precedesReturnsOfInlinedCallees(loopIndex, point)) + return false; + const OpenLoop& loop = m_openLoops[loopIndex]; + const BodyIndex& index = *m_frames[loop.frame].index; + BlockID inputBlock = loop.frame == point.frameIndex ? point.inputBlock : m_frames[loop.frame].block; + Vector found; + for (BlockID member : index.membersOf(loop.loop)) { + if (member == index.loop(loop.loop).head || !index.isReachable(member) || index.dominates(inputBlock, member)) + continue; + BodyTerminator end = index.terminator(member); + unsigned numberOfExits = 0; + index.block(member).forEachOutgoingEdge([&](BodyTarget target, bool) { + numberOfExits += !index.isInLoop(target.block(), loop.loop); + }); + if (!numberOfExits) + continue; + ValueID asked = end.kind() == TerminatorKind::Branch ? end.as().conditionOperand() : ValueID(); + if (numberOfExits != 1 || !asked || index.isParameter(asked) || !allRelations.contains(index.opcodeOf(asked)) || !index.dominates(member, inputBlock)) + return false; + std::optional condition = findConditionOnEntry(loopIndex, asked); + if (!condition) + return false; + condition->isNegated = !index.isInLoop((*end.targets().begin()).block(), loop.loop); + found.append(*condition); + } + for (const LoopEntryCondition& condition : found) { + Reference holds = buildConditionOnEntry(loopIndex, condition); + if (m_hasFailed || !holds) + return false; + if (std::optional outcome = outcomeKnownOf(*this, holds)) { + if (!*outcome) + return false; + continue; + } + if (!holds.isValue() || representationOf(holds) != Representation::Boolean) + return false; + if (!conditions.contains(holds)) + conditions.append(holds); + } + return true; +} + void Walker::beginHoistingConstantDiamond(const BodyTerminator& branch, Reference condition) { const Frame& frame = m_frames.last(); @@ -1324,7 +1462,7 @@ static bool isSameAsRecordInFrontOfLoop(const Code& code, ValueID id, const Draf const Operation& record = code.at(id); if (record.opcode != draft.opcode || record.representation != draft.representation || record.flags != draft.flags || !std::ranges::equal(record.payload(), draft.payload())) return false; - if (record.opcode == Opcode::FlattenString && record.isReadOperand() && id.index() < firstRecordOfLoop) + if (record.opcode == Opcode::FlattenString && !record.isReadOperands().empty() && id.index() < firstRecordOfLoop) return false; return std::ranges::equal(record.operands(), draft.operands(), [](ValueID held, Reference taken) { return taken ? held == taken.id() : !held; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkLoops.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkLoops.cpp index ffa4bfac8242f..7e585ce6df0a4 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkLoops.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkLoops.cpp @@ -108,7 +108,17 @@ static bool isComparedByTestOfHeadInLoops(const BodyIndex& index, BlockID inputH return false; BodyValue relation = index.value(asked); ValueID parameter = head.parameter(position); - return index.stripCasts(relation.operand(0)) == parameter || index.stripCasts(relation.operand(1)) == parameter; + auto isParameterOrSum = [&](ValueID operand) { + ValueID value = index.stripCasts(operand); + if (value == parameter || index.isParameter(value) || index.opcodeOf(value) != Opcode::Add) + return value == parameter; + BodyValue sum = index.value(value); + ValueID left = index.stripCasts(sum.operand(0)); + ValueID right = index.stripCasts(sum.operand(1)); + ValueID addend = left == parameter ? right : right == parameter ? left : ValueID(); + return addend && !index.isParameter(addend) && index.opcodeOf(addend) == Opcode::Integer; + }; + return isParameterOrSum(relation.operand(0)) || isParameterOrSum(relation.operand(1)); } static Effects effectsOfMembersAsWrittenInLoops(const TypingContext& context, const BodyIndex& index, BlockID inputHead) @@ -373,8 +383,8 @@ bool Walker::hasOpenTrialVisit(size_t frameIndex, BlockID inputHead) const macro(returnsInPlaceOfUnit) macro(isOfUnit) macro(countAmongFramesOfFunction) macro(loopsInTrialVisits) #define FOR_EACH_FIELD_OF_FRAME_THAT_HANGS_ON_NO_STATE(macro) \ - macro(isRecursive) \ - macro(valueHandedBack) macro(hasLookedForValueHandedBack) macro(handlerBlocks) macro(numberOfHandlersThatRead) macro(parametersHandedRoundUnchangedByLoop) \ + macro(isRecursive) macro(keepsFields) \ + macro(blockOfLoopDepth) macro(loopDepth) macro(valueHandedBack) macro(hasLookedForValueHandedBack) macro(handlerBlocks) macro(numberOfHandlersThatRead) macro(parametersHandedRoundUnchangedByLoop) \ macro(outermostLoopsLookedThrough) macro(accessesUnderProvenKeys) macro(usesInLoops) macro(loopsWithBlocksTouched) macro(arithmeticInLoops) macro(operatorsFoundBehindValuesForGuesses) macro(numbersOfLongListsOfNamesOfRoots) macro(endsOfUsesOfTexts) #define FOR_EACH_FIELD_OF_FRAME_WRITTEN_OUTSIDE_LOOPS_AND_HANDLERS_ALONE(macro) \ @@ -400,7 +410,7 @@ bool Walker::hasOpenTrialVisit(size_t frameIndex, BlockID inputHead) const macro(m_isWalkingAccess) macro(m_pollInFrontOfLoopBeingEntered) macro(m_edgeOfCodeBeingLed) macro(m_callCopiedByLinkOfChain) \ macro(m_numberOfFramesAtCallCopiedByLinkOfChain) macro(m_callIsAccountedFor) macro(m_calleeAccountedFor) \ macro(m_holdsReadsOfVariablesAsConstants) macro(m_countsOfUnit) macro(m_speculationCount) macro(m_hoistedArrayGuardCount) macro(m_hoistedElementCheckCount) macro(m_hoistedNumberReadCount) macro(m_mechanisms) \ - macro(m_versionedLoops) macro(m_sharedGenericVersions) macro(m_listAskedForStorageOfNumbersInVain) macro(m_walksCleanupBesideCopyOfLoop) macro(m_copyThatMakesNewList) macro(m_runOverBoxedValuesTakesNewListOfCopyFirst) macro(m_listThatRunOverBoxedValuesTakesFirst) macro(m_receiverOfCopyThatMakesNewList) macro(m_depthOfCopyThatMakesNewList) \ + macro(m_versionedLoops) macro(m_sharedGenericVersions) macro(m_listAskedForStorageOfNumbersInVain) macro(m_receiverOfLargerMethod) macro(m_walksCleanupBesideCopyOfLoop) macro(m_copyThatMakesNewList) macro(m_runOverBoxedValuesTakesNewListOfCopyFirst) macro(m_listThatRunOverBoxedValuesTakesFirst) macro(m_receiverOfCopyThatMakesNewList) macro(m_depthOfCopyThatMakesNewList) \ macro(m_hasURLsThatAreNeverMade) \ macro(m_requestedLiteralStorage) \ macro(m_unspecifiedConstants) macro(m_valueBeingWalked) macro(m_skippedFreshCellCount) macro(m_iterationRestartCount) macro(m_undoneStoreCount) macro(m_versionedLoopCount) macro(m_versionedLoopNestCount) \ @@ -427,12 +437,12 @@ bool Walker::hasOpenTrialVisit(size_t frameIndex, BlockID inputHead) const #define FOR_EACH_FIELD_OF_WALKER_THAT_ITS_WRITER_PUTS_BACK(macro) \ macro(m_valuesStoodInFor) macro(m_restArrayAllocationBlock) macro(m_restArrayAllocator) macro(m_accessWhoseKeyIsTested) macro(m_depthOfCopiesAroundThatAccess) macro(m_accessWhoseKeyArmHolds) macro(m_depthOfCopiesAroundKeyHeldByArm) macro(m_staticFieldSeenInVariableByArm) macro(m_isMakingObjectsOfConstant) \ macro(m_trialVisitDepth) macro(m_nestingDepth) macro(m_serialOfNewestMark) macro(m_loopsInFinalBuild) macro(m_reversionableLoops) macro(m_frameLogSizes) macro(m_buysTwoBuilds) \ - macro(m_numberOfFormsHeardAsArms) macro(m_operationsThatRanOtherCodeOfRow) macro(m_guardsOfStoresAskedOfRow) macro(m_previewOutOfRow) macro(m_listsAreStruckInRow) macro(m_otherCodeStaysInTypedBuildOfRow) macro(m_otherCodeIsKeptOutOfFirstCopyInVainOfRow) macro(m_otherCodeStayedInVainOfRow) macro(m_readsAskSitesThoughCallStaysOfRow) macro(m_sumsAreNotAskedInRow) macro(m_keysAreReadOffLookInRow) macro(m_keyWasTakenForCounterAskedInFront) macro(m_walksCopyThatAsksCountersAlone) macro(m_headOfTypedBuildOverListsTestedInFront) \ + macro(m_numberOfFormsHeardAsArms) macro(m_operationsThatRanOtherCodeOfRow) macro(m_guardsOfStoresAskedOfRow) macro(m_previewOutOfRow) macro(m_listsAreStruckInRow) macro(m_copyForInt32sIsStruck) macro(m_hasCopyForInt32s) macro(m_otherCodeStaysInTypedBuildOfRow) macro(m_otherCodeIsKeptOutOfFirstCopyInVainOfRow) macro(m_otherCodeStayedInVainOfRow) macro(m_readsAskSitesThoughCallStaysOfRow) macro(m_sumsAreNotAskedInRow) macro(m_keysAreReadOffLookInRow) macro(m_keyWasTakenForCounterAskedInFront) macro(m_walksCopyThatAsksCountersAlone) macro(m_headOfTypedBuildOverListsTestedInFront) \ macro(m_frameOfTypedBuildOverListsTestedInFront) macro(m_listsMadeHereOfTypedBuildOverListsTestedInFront) macro(m_inheritedSummaryHeader) macro(m_inheritedSummaryFrame) macro(m_inheritedEntryTypes) macro(m_inheritedParameterTypes) \ macro(m_inheritedWideningRungs) macro(m_inheritedRarelyWidenedParameters) macro(m_inheritedOverreadWidenedParameters) macro(m_firstOtherCodeRanInCopyOfRow) macro(m_summaryHasUninlinedCall) macro(m_turnsOfRowAskLengthsOfViewsFromInFrontForWrittenPlaces) macro(m_turnsOfRowTakeApartListsFromInFront) macro(m_rowSaysTurnsAreQuietOutsideArms) macro(m_effectsOutsideArmsOfRow) \ macro(m_testsSeenOfRow) macro(m_isVersioningDryRun) macro(m_operatorsMadeInFrontOfLoops) macro(m_ordinalOfCopyOfLoop) macro(m_walksCopyWithoutGuessesOfSomeTurns) macro(m_walksCopyWithoutQuestionsOfEveryTurn) macro(m_walksCopyForAnotherKindOfStorage) macro(m_frontsSharedWithTypedBuilds) macro(m_restartablePreviewHeader) \ macro(m_numberOfStretchesBehindBytesOfMembers) macro(m_numbersOfFramesOfFunctions) macro(m_resultOfReadBeingTyped) macro(m_resultOfReadIsRemembered) macro(m_contentsBeingTyped) macro(m_handlerMadeOverCopies) \ - macro(m_numberOfFramesBelowHandlerMade) macro(m_isBuildingInPlace) macro(m_rowOfCopyFoundUnderKey) macro(m_numberOfLoopsAroundEdgeBeingLed) macro(m_readWhoseGuessIsOpen) macro(m_weighsWhatIsWritten) macro(m_makesLetterOfStep) + macro(m_numberOfFramesBelowHandlerMade) macro(m_isBuildingInPlace) macro(m_rowOfCopyFoundUnderKey) macro(m_numberOfLoopsAroundEdgeBeingLed) macro(m_readWhoseGuessIsOpen) macro(m_guardedListOfReads) macro(m_methodChainBeingWalked) macro(m_weighsWhatIsWritten) macro(m_makesLetterOfStep) #define FOR_EACH_FIELD_OF_WALKER_THAT_ITS_WRITER_SETS_AND_CLEARS(macro) \ macro(m_hoistedDiamond) macro(m_threadedCondition) macro(m_threadedConditionValue) macro(m_threadedConditionFrame) \ @@ -444,7 +454,7 @@ bool Walker::hasOpenTrialVisit(size_t frameIndex, BlockID inputHead) const #define FOR_EACH_FIELD_OF_WALKER_THAT_HANGS_ON_NO_STATE(macro) \ macro(m_listsThatCallsAloneTake) \ - macro(methodsGivenUnderKeys) macro(m_storesIntoRowsOfBlocks) macro(m_usersWithinBlocks) macro(m_usersOfValuesOfLoops) macro(m_valuesTakenForKeys) macro(m_loopsThatReckonWithElements) macro(m_numbersInListsOfWrittenObjects) macro(m_lastStoresByKeyOfLoops) macro(m_storesByKeyOfBlocks) macro(m_namesStoredInOutermostLoops) macro(m_whetherElementsOfLoopsMeetWhatGoesRound) macro(receiversMadeInBodies) macro(gettersExpectedInFunction) macro(m_placesSeenToStandInStretches) macro(m_keptByGeneratorsInLoops) \ + macro(methodsGivenUnderKeys) macro(m_storesIntoRowsOfBlocks) macro(m_usersWithinBlocks) macro(m_usersOfValuesOfLoops) macro(m_valuesTakenForKeys) macro(m_loopsThatReckonWithElements) macro(m_computedComparisons) macro(m_numbersInListsOfWrittenObjects) macro(m_lastStoresByKeyOfLoops) macro(m_storesByKeyOfBlocks) macro(m_namesStoredInOutermostLoops) macro(m_whetherElementsOfLoopsMeetWhatGoesRound) macro(receiversMadeInBodies) macro(gettersExpectedInFunction) macro(m_placesSeenToStandInStretches) macro(m_keptByGeneratorsInLoops) \ macro(m_listsOfMembersByTag) macro(m_membersByTagOfSets) macro(m_types) macro(m_roundsKept) macro(m_listsOfNames) macro(m_numbersOfListsOfNames) \ macro(m_functionsInReachOfUnit) macro(m_surveysOfConstants) macro(m_constantsThatListFieldsHeldFilled) macro(m_positionsOfSlowBody) macro(m_positionsKnowingNothing) \ macro(m_heldPositions) macro(m_valuesReadBehindLoopFoundLast) macro(m_accessesOfSlotsInLoopListedLast) macro(m_accessesOfScopesByFunctions) macro(m_isArrayIteratorProtocolIntact) macro(m_objectPrototypeMayGainName) macro(m_keptByTypedAccesses) \ @@ -492,7 +502,7 @@ bool Walker::hasOpenTrialVisit(size_t frameIndex, BlockID inputHead) const FOR_EACH_FIELD_OF_WALKER_THAT_NUMBERS_WHAT_STATES_OF_OPEN_TURNS_HOLD_AND_IS_EMPTIED_WHERE_NO_WAY_IS_OPEN(macro) FOR_EACH_FIELD_OF_WALKER_THAT_TRIES_LEARN(macro) FOR_EACH_FIELD_OF_WALKER_THAT_ONLY_REPORTS(macro) FOR_EACH_FIELD_OF_WALKER_WRITTEN_IN_FRONT_OF_EVERY_READ(macro) \ FOR_EACH_FIELD_OF_WALKER_WRITTEN_IN_FRONT_OF_EVERY_LOOP(macro) FOR_EACH_FIELD_OF_WALKER_THAT_ONLY_WITHHOLDS_WHEN_STALE(macro) -#define FOR_EACH_FIELD_OF_WAY_THAT_CHOOSES_COST_OR_PLACE(macro) macro(genericVersionSpeculates) macro(frameIndex) macro(inputHead) macro(entry) macro(genericStart) macro(guessesCountedOn) macro(canUseIterationGuards) macro(hasCallInIteration) macro(hasUninlinedCall) macro(relationThatAsksLesserBound) macro(turnsThatRunOtherCodeComeRound) macro(lastBackEdgeDominator) macro(hasElementArithmetic) macro(startOfCopyWithoutGuessesOfSomeTurns) macro(arrayGuardedAtHeader) macro(storageGuardedAtHeader) macro(isArrayGuardedAtHeaderWritable) +#define FOR_EACH_FIELD_OF_WAY_THAT_CHOOSES_COST_OR_PLACE(macro) macro(genericVersionSpeculates) macro(frameIndex) macro(inputHead) macro(entry) macro(genericStart) macro(guessesCountedOn) macro(canUseIterationGuards) macro(hasCallInIteration) macro(hasUninlinedCall) macro(relationThatAsksLesserBound) macro(turnsThatRunOtherCodeComeRound) macro(lastBackEdgeDominator) macro(mayRestartUnderHandler) macro(startOfCopyWithoutGuessesOfSomeTurns) macro(arrayGuardedAtHeader) macro(storageGuardedAtHeader) macro(isArrayGuardedAtHeaderWritable) #define FOR_EACH_FIELD_OF_WAY_THAT_WALK_OF_TURN_WRITES(macro) macro(frontAsksWhatNothingSpeaksFor) macro(valuesAtHeader) macro(isEntered) macro(restartPoint) macro(headAsksLesserBound) macro(headThatAsksLesserBound) macro(effectsCheckBeginning) macro(isLeftAtOtherCodeOnUsualWay) macro(moduleVariablesAtHeader) macro(undoBlocks) macro(guessOfSomeTurnsIsAskedInFront) macro(questionOfEveryTurnIsAskedInFront) macro(isLeftAtAccessOnUsualWay) #define FOR_EACH_FIELD_OF_WAY_ASKED_AGAIN_WHERE_IT_COMES_ROUND(macro) macro(backEdgeChecks) macro(variableBackEdgeChecks) #define FOR_EACH_FIELD_OF_WAY_USED_WHILE_STATE_HOLDS_ITS_NOTE(macro) macro(lesserBound) macro(boundBesideLesserBound) macro(viewWhoseLengthIsBound) macro(variableThatIsBound) macro(readOfVariableThatIsBound) macro(counterBelowRoom) @@ -1100,6 +1110,7 @@ LoopPreview Walker::makePreviewFromSummary(BlockID inputHead, std::span broughtRound = findNumberThatDivisionsBringRound(*this, inputHead, parameter, entryTypes, WhatEntersHead::IsInteger)) { summary.types[parameter] = m_types.unionOf(m_types.integer(broughtRound->integers), entering); @@ -1108,15 +1119,22 @@ LoopPreview Walker::makePreviewFromSummary(BlockID inputHead, std::span counted = record.integers.minimum == record.integers.maximum ? findCounterRange(*this, inputHead, parameter, record.integers.minimum) : std::nullopt) + summary.types[parameter] = m_types.unionOf(m_types.withIntegers(entering, *counted), entering); + else if (isMultipliedBelowLength(*this, inputHead, parameter)) { + summary.types[parameter] = m_types.withIntegers(entering, Range { record.integers.minimum, Range::uint32().maximum - 1 }); + startsAtLengthBound = true; + } else if (record.integers.minimum == record.integers.maximum) summary.types[parameter] = m_types.unionOf(widenedInLoops(m_types, entering, m_types.withIntegers(entering, Range { record.integers.minimum, record.integers.maximum + 1 })), entering); - } else { + } else if (std::optional counted = record.isInteger() && record.integers.isSubsetOf(Range::exact()) ? findCounterRange(*this, inputHead, parameter, record.integers.minimum) : std::nullopt) + summary.types[parameter] = m_types.unionOf(m_types.withIntegers(entering, *counted), entering); + else { TypeID widened = m_types.withLiteralStringsWidened(entering); if (record.primitives & Primitive::Boolean) widened = m_types.unionOf(widened, TypeTable::boolean); summary.types[parameter] = widened; } - if (summary.types[parameter] != entering) + if (summary.types[parameter] != entering && !startsAtLengthBound) summary.rungs[parameter] = 1; size_t parameterAround = 0; ValueID around = parameter < m_trialVisits.last().headParametersThatEnter.size() ? m_trialVisits.last().headParametersThatEnter[parameter] : ValueID(); @@ -1159,11 +1177,22 @@ LoopPreview Walker::makePreviewFromSummary(BlockID inputHead, std::span learned = std::exchange(summary.storesIntoScopes, m_inheritedScopeStores).slots; + for (const SlotOfLoop& slot : learned) { + Vector& slots = summary.storesIntoScopes.slots; + auto place = std::ranges::lower_bound(slots, slot.order(), { }, &SlotOfLoop::order); + if (place == slots.end() || place->order() != slot.order()) + slots.insert(place - slots.begin(), slot); + else + place->hasLeftIntegers |= slot.hasLeftIntegers; + } + } if (hasInheritedSummary && unitHasMadeScopes()) handBackScopesThatBuildLetsLeave(summary.mentionsThatLeave.span()); m_trialVisits.last().freshObjectsKept.shrink(0); @@ -1262,23 +1291,59 @@ bool Walker::narrowerTypeOfHeadPaysWalk(TypeID assumed, TypeID narrowed, bool tu Vector Walker::findParametersSteppedOnDoubles(BlockID inputHead, const LoopSummary& summary) { - const BodyIndex& index = currentBodyIndex(); - LoopID loop = index.loopOf(inputHead); - BodyBlock head = index.block(inputHead); - Vector isStepped; - if (summary.typesOfValues.isEmpty() || head.numberOfParameters() > 32) - return isStepped; + if (summary.typesOfValues.isEmpty()) + return { }; uint32_t firstValue = summary.firstValue; uint32_t endOfValues = firstValue + static_cast(summary.typesOfValues.size()); - auto typeInRow = [&](ValueID value) -> TypeID { + return findParametersSteppedOnDoubles(inputHead, false, scopedLambda([&](ValueID value) -> TypeID { if (!value) return { }; if (value.index() >= firstValue) return value.index() < endOfValues ? summary.typesOfValues[value.index() - firstValue] : TypeID(); Reference reference = referenceOf(value); return reference && reference.isRecord() ? m_code.at(reference.id()).type : TypeID(); + })); +} + +Vector Walker::findParametersSteppedOnDoubles(BlockID inputHead, const LoopPreview& preview) +{ + auto reachesLastExactInteger = [&](TypeID type) { + if (!type || !m_types.at(type).isInteger()) + return false; + Range integers = m_types.at(type).integers; + return integers.maximum >= maxExactInteger || integers.minimum <= -maxExactInteger; }; + if (std::ranges::none_of(preview.typesOfParameters, reachesLastExactInteger)) + return { }; + uint32_t firstParameter = currentBodyIndex().block(inputHead).firstParameter().index(); + uint32_t endOfParameters = firstParameter + static_cast(preview.typesOfParameters.size()); + return findParametersSteppedOnDoubles(inputHead, true, scopedLambda([&](ValueID value) -> TypeID { + if (!value) + return { }; + if (value.index() >= firstParameter) + return value.index() < endOfParameters ? preview.typesOfParameters[value.index() - firstParameter] : TypeID(); + Reference reference = referenceOf(value); + return reference && reference.isRecord() ? m_code.at(reference.id()).type : TypeID(); + })); +} + +Vector Walker::findParametersSteppedOnDoubles(BlockID inputHead, bool isAskedBeforeVisit, const ScopedLambda& typeInRow) +{ + const BodyIndex& index = currentBodyIndex(); + LoopID loop = index.loopOf(inputHead); + BodyBlock head = index.block(inputHead); + Vector isStepped; + if (head.numberOfParameters() > 32) + return isStepped; constexpr OpcodeSet steps { Opcode::Add, Opcode::Sub, Opcode::Increment, Opcode::Decrement }; + auto typeOfStep = [&](BodyValue step) -> TypeID { + if (!isAskedBeforeVisit) + return typeInRow(step.id()); + if (step.opcode() != Opcode::Increment && step.opcode() != Opcode::Decrement) + return { }; + TypeID stepped = typeInRow(index.stripCasts(step.operand(0))); + return stepped && m_types.at(stepped).isInteger() ? stepped : TypeID(); + }; auto forEachBlockInLoops = [&](const auto& functor) { for (uint32_t number = 0; number < index.numberOfBlocks(); ++number) { if (index.loopOf(BlockID(number))) @@ -1308,8 +1373,8 @@ Vector Walker::findParametersSteppedOnDoubles(BlockID inputHead, const for (uint32_t place = listed.newestStepOfLoop[loop.index()]; place != UINT32_MAX; place = listed.steps[place].second) { BodyValue step = index.value(listed.steps[place].first); uint32_t numberOfOperands = numberOfNamedOperandsOf(step.opcode()); - TypeID result = typeInRow(step.id()); - TypeID first = typeInRow(step.operand(0)); + TypeID result = typeOfStep(step); + TypeID first = isAskedBeforeVisit ? result : typeInRow(step.operand(0)); TypeID second = numberOfOperands == 2 ? typeInRow(step.operand(1)) : TypeID(); if (!result || !first || (numberOfOperands == 2 && !second)) continue; @@ -1400,8 +1465,9 @@ Vector Walker::findParametersSteppedOnDoubles(BlockID inputHead, const } BodyValue operation = index.value(ValueID(use.user)); Opcode opcode = operation.opcode(); - TypeID result = typeInRow(operation.id()); - if (opcodesThatStateTypes.contains(opcode) || (steps.contains(opcode) && result && m_types.at(result).isNumber())) + TypeID result = isAskedBeforeVisit ? TypeID() : typeInRow(operation.id()); + bool givesCounter = isAskedBeforeVisit ? opcode == Opcode::Increment || opcode == Opcode::Decrement : steps.contains(opcode) && result && m_types.at(result).isNumber(); + if (opcodesThatStateTypes.contains(opcode) || givesCounter) give(use.user, parts); else if (!allRelations.contains(opcode) || (mayBeTakenAsInteger(operation.operand(0)) && mayBeTakenAsInteger(operation.operand(1)))) takenAsIntegers |= parts; @@ -2190,6 +2256,7 @@ bool Walker::runTrialVisits(BlockID inputHead, std::span enteri Vector enteringKept; enteringKept.append(entering); readListsOfStableBindingsInFrontOfLoop(*this, inputHead); + readListsOfFieldsInFrontOfLoop(*this, inputHead); askWhatIsExpectedOfParametersThatEnterLoop(*this, inputHead); if (!m_trialVisitDepth++) m_code.keepChangesToOlderParts(true); @@ -2232,6 +2299,7 @@ bool Walker::runTrialVisits(BlockID inputHead, std::span enteri SnapshotForTry& snapshot = *ownerOfSnapshot; IterationsSteppedAtHead steppedInFrontOfLoop = takeIterationsSteppedAt(inputHead); bool stepsIteration = steppedInFrontOfLoop.asTheyEntered.containsIf([](const auto& entered) { return entered.second.otherForm == IndexedIteration::OtherForm::None; }); + bool stepsWhatCallsMayChange = steppedInFrontOfLoop.asTheyEntered.containsIf([](const auto& entered) { return entered.second.otherForm == IndexedIteration::OtherForm::None && entered.second.contents != IndexedIteration::Contents::Text; }); bool generatorIsWalkedInLoop = m_generatorsTakenByConsumers.containsIf([&](const GeneratorTakenByConsumer& taken) { return taken.inputHead == inputHead && taken.inlineFrame == m_frames[frameIndex].inlineFrame; }); @@ -2271,6 +2339,8 @@ bool Walker::runTrialVisits(BlockID inputHead, std::span enteri SetForScope inheritedOverreadWidenedScope(m_inheritedOverreadWidenedParameters, m_loopSummaries[place].overreadWidenedParameters); SetForScope buys(m_buysTwoBuilds, true); SetForScope listsStruck(m_listsAreStruckInRow, m_loopSummaries[place].listsAreStruck); + SetForScope copyStruck(m_copyForInt32sIsStruck, m_loopSummaries[place].copyForInt32sIsStruck); + SetForScope hasCopy(m_hasCopyForInt32s, false); SetForScope otherCodeStays(m_otherCodeStaysInTypedBuildOfRow, m_loopSummaries[place].otherCodeStaysInTypedBuild); SetForScope keptOutInVain(m_otherCodeIsKeptOutOfFirstCopyInVainOfRow, m_loopSummaries[place].otherCodeIsKeptOutOfFirstCopyInVain); SetForScope stayedInVain(m_otherCodeStayedInVainOfRow, m_loopSummaries[place].otherCodeStayedInVain); @@ -2304,6 +2374,16 @@ bool Walker::runTrialVisits(BlockID inputHead, std::span enteri return !std::exchange(m_typedBuildAskedNoListThatItWasBoughtFor, false) && built; }; isBuilt = isKept(isBuilt); + if (!isBuilt && m_hasCopyForInt32s && !m_hasFailed && !m_numberOfTryWhoseWalkIsCutShort) { + standInFrontAgain(); + prepareFrontOfBuilds(); + m_copyForInt32sIsStruck = true; + m_loopSummaries[place].copyForInt32sIsStruck = true; + m_typedBuildIsLeftAtOtherCodeOnUsualWay = false; + m_turnOfTypedBuildLeavesWithoutTest = false; + isBuilt = buildSeveralTimes(learned); + isBuilt = isKept(!std::exchange(m_typedBuildEndedListsTestedInFront, false) && isBuilt); + } if (!isBuilt && m_readWasKeptFromItsSiteBecauseCallStays && !m_hasFailed && !m_numberOfTryWhoseWalkIsCutShort) { standInFrontAgain(); prepareFrontOfBuilds(); @@ -2349,6 +2429,7 @@ bool Walker::runTrialVisits(BlockID inputHead, std::span enteri auto makeBuilds = [&](LoopPreview& learned, const Vector& entered, uint32_t numberOfEdgesThatCameRound, const auto& standInFrontAgain) { size_t numberOfRowsInFront = m_loopSummaries.size(); bool listsAreStruck = m_loopSummaries[place].listsAreStruck; + bool copyForInt32sIsStruck = m_loopSummaries[place].copyForInt32sIsStruck; bool otherCodeStays = m_loopSummaries[place].otherCodeStaysInTypedBuild; bool readsAskSites = m_loopSummaries[place].readsAskSitesThoughCallStays; bool otherCodeStayedInVain = m_loopSummaries[place].otherCodeStayedInVain; @@ -2365,6 +2446,7 @@ bool Walker::runTrialVisits(BlockID inputHead, std::span enteri return row.value >= numberOfRowsInFront; }); m_loopSummaries[place].listsAreStruck = listsAreStruck; + m_loopSummaries[place].copyForInt32sIsStruck = copyForInt32sIsStruck; m_loopSummaries[place].otherCodeStaysInTypedBuild = otherCodeStays; m_loopSummaries[place].readsAskSitesThoughCallStays = readsAskSites; if (keyWasTakenInVain) { @@ -2509,6 +2591,7 @@ bool Walker::runTrialVisits(BlockID inputHead, std::span enteri learned.typesOfValues = bought->typesOfValues; learned.firstValue = bought->firstValue; learned.hasLetMentionsLeave = true; + learned.slots = m_loopSummaries[place].storesIntoScopes.slots; Vector typesOfPlainWalk = m_loopSummaries[place].types; m_loopSummaries[place].types = bought->typesOfParameters; bool isBuilt = makeBuilds(learned, bought->entered, bought->numberOfEdgesThatCameRound, undoPlainWalk); @@ -2667,6 +2750,8 @@ bool Walker::runTrialVisits(BlockID inputHead, std::span enteri } bool scopesWantAnotherWalk = learnWhatTurnsStoreIntoScopes(summary, trialVisit, !hasFailedInside && !isCutShort); scopesWantAnotherWalk |= trialVisit.requestsCarriedResult; + if (!hasFailedInside && !isCutShort) + scopesWantAnotherWalk |= learnSlotsOfLoop(summary, trialVisit); bool fits = effectsFit && !trialVisit.misfits && !trialVisit.freshObjectHasLeft && !hasFailedInside; if (!effectsFit && !trialVisit.misfits && !trialVisit.freshObjectHasLeft && !hasFailedInside && !summary.isPessimistic && !isCutShort) narrowSummaryToObservedTypes(summary, trialVisit); @@ -2858,9 +2943,12 @@ bool Walker::runTrialVisits(BlockID inputHead, std::span enteri trialVisit.narrowingsThatWait.shrink(0); trialVisit.narrowingsThatWait.append({ trialVisit.firstRecord, place, trialVisit.numberOfHeadParametersWithSourcesInFront, m_headParametersWithSources.size() }); } - summary.steppedOnDoubles = findParametersSteppedOnDoubles(inputHead, summary); - for (size_t parameter = 0; parameter < summary.steppedOnDoubles.size() && parameter < summary.types.size(); ++parameter) - narrows |= summary.steppedOnDoubles[parameter] && summary.types[parameter] && representationOfNumber(*this, summary.types[parameter]) == Representation::Int64; + Vector wasOnDoubles = std::exchange(summary.steppedOnDoubles, findParametersSteppedOnDoubles(inputHead, summary)); + for (size_t parameter = 0; parameter < summary.types.size(); ++parameter) { + bool was = parameter < wasOnDoubles.size() && wasOnDoubles[parameter]; + bool is = parameter < summary.steppedOnDoubles.size() && summary.steppedOnDoubles[parameter]; + narrows |= was != is && summary.types[parameter] && representationOfNumber(*this, summary.types[parameter]) == Representation::Int64; + } if (narrows) { if (defersPurchase()) m_trialVisits.last().somethingIsWantedInside = true; @@ -2900,7 +2988,7 @@ bool Walker::runTrialVisits(BlockID inputHead, std::span enteri bool isQuietOutsideArms = !trialVisit.testsSeen.isEmpty() && !trialVisit.doneOutsideArms.mayRunOtherCode() && trialVisit.done.mayRunOtherCode(); bool isSharedGenericVersionPending = !m_sharedGenericVersions.isEmpty() && m_sharedGenericVersions.last().frameIndex == frameIndex && m_sharedGenericVersions.last().inputHead == inputHead && m_sharedGenericVersions.last().isAboutToBeTaken; if (fits && !hasAskedForTwoBuilds && !summary.severalBuildsAreStruck && !summary.isPessimistic && has(Mechanism::LoopInvariants) && (isSharedGenericVersionPending || !m_frames[frameIndex].loopsBeingVersioned.contains(loop)) && !isTooLargeToVersion(index, loop) - && !isNeverBuiltSeveralTimes && !(trialVisit.callOnUsualWayComesRound && stepsIteration) && !walkAroundIsUndone()) { + && !isNeverBuiltSeveralTimes && !(trialVisit.callOnUsualWayComesRound && stepsWhatCallsMayChange) && !walkAroundIsUndone()) { hasAskedForTwoBuilds = true; auto ownerOfLearned = makeUniqueWithoutFastMallocCheck(); LoopPreview& learned = *ownerOfLearned; @@ -2911,6 +2999,7 @@ bool Walker::runTrialVisits(BlockID inputHead, std::span enteri learned.typesOfValues = summary.typesOfValues; learned.firstValue = summary.firstValue; learned.hasLetMentionsLeave = true; + learned.slots = summary.storesIntoScopes.slots; bool mayByText; { m_rowSaysTurnsAreQuietOutsideArms = isQuietOutsideArms; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkLoopsBuiltTwice.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkLoopsBuiltTwice.cpp index 53a15bc973ede..6e8874caf497c 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkLoopsBuiltTwice.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkLoopsBuiltTwice.cpp @@ -5,6 +5,7 @@ #include "config.h" #include "AOTWalk.h" +#include "AOTWalkNumberForms.h" #if ENABLE(AOT) @@ -606,6 +607,53 @@ bool Walker::counterComesRoundNotBelowItself(BlockID head, std::span Walker::findTestThatHoldsDifferenceFromZeroUp(Reference position) +{ + struct Term { + ValueID value; + bool isTakenAway; + }; + if (!position || !position.isValue() || !m_types.at(typeOf(position)).isInteger()) + return std::nullopt; + static constexpr size_t mostTerms = 8; + Vector terms { Term { position.id(), false } }; + for (size_t place = 0; place < terms.size() && terms.size() < mostTerms;) { + ValueID root = terms[place].value; + while (valuePreservingOpcodes.contains(m_code.at(root).opcode)) + root = m_code.at(root).operand(0); + const Operation& record = m_code.at(root); + if (record.opcode != Opcode::Add && record.opcode != Opcode::Sub) { + ++place; + continue; + } + terms.append(Term { record.operand(1), terms[place].isTakenAway != (record.opcode == Opcode::Sub) }); + terms[place].value = record.operand(0); + } + auto holds = [&](Reference lesser, Reference greater, bool admitsEqual, bool answer) -> std::optional { + std::optional noted = findKnown(*this, KnownNote::Relation, answer ? lesser : greater, answer ? greater : lesser, admitsEqual); + std::optional passed = noted && static_cast(*noted & 1) == answer ? findPassedTest(Reference::fromBits(static_cast(*noted >> 32)), answer) : std::nullopt; + if (!passed || passed->sides().empty()) + return std::nullopt; + SumOverRecords sum(m_code, m_types); + return sum.add(position.id(), 1) && sum.takeAwayWhatIsFromZeroUp(passed->sides().front().test, answer) && sum.isFromZeroUp() ? passed : std::nullopt; + }; + for (const Term& lesser : terms) { + for (const Term& greater : terms) { + if (!lesser.isTakenAway || greater.isTakenAway) + continue; + for (bool admitsEqual : { true, false }) { + for (bool answer : { true, false }) { + if (std::optional found = holds(referenceTo(lesser.value), referenceTo(greater.value), admitsEqual, answer)) { + count(StatisticID::DifferencesShownFromZeroUp); + return found; + } + } + } + } + } + return std::nullopt; +} + std::optional Walker::findWhatHoldsPositionBelowRoomOfView(Reference view, Reference position) { TypeID type = view && view.isRecord() && position && position.isRecord() ? typeOf(position) : TypeID(); @@ -645,7 +693,7 @@ std::optional Walker::showPositionInside(const GroundsOfPosition& gr passed = passed.with(*one); }; restOn(grounds.isBelow, grounds.isBelowResult); - restOn(grounds.isFromZeroUp, true); + restOn(grounds.isFromZeroUp, grounds.isFromZeroUpResult); restOn(grounds.stepNonNegativeCheck, true); restOn(grounds.limitWithinLengthCheck, true); passed.m_groundsOfPosition = grounds; @@ -661,8 +709,10 @@ std::optional Walker::boundsProofFromChecks(Reference view, Referenc grounds.isBelow = below.sides().front().test; grounds.isBelowResult = below.sides().front().testResult; } - if (!fromZeroUp.sides().empty() && fromZeroUp.sides().front().testResult) + if (!fromZeroUp.sides().empty()) { grounds.isFromZeroUp = fromZeroUp.sides().front().test; + grounds.isFromZeroUpResult = fromZeroUp.sides().front().testResult; + } return showPositionInside(grounds); } @@ -734,10 +784,10 @@ void Walker::proveWhatLooksCountOnAt(const BodyValue& operation) bool Walker::areIterationsRestartable(BlockID inputHead, const LoopPreview& preview) { size_t frameIndex = m_frames.size() - 1; - if ((!preview.effects.mayRunOtherCode() && !pushOfLoopFillsLiteral(inputHead)) || hasStringsInBuilders() || m_coldCopyDepth || (isUnderHandler() && (isUnderHandlerOtherThanCleanups() || !hasElementArithmeticInLoop(inputHead))) + if ((!preview.effects.mayRunOtherCode() && !pushOfLoopFillsLiteral(inputHead)) || hasStringsInBuilders() || m_coldCopyDepth || (isUnderHandler() && !mayVersionLoopUnderHandler(inputHead)) || (m_inputHeadsOfIterations.contains(inputHeadInFrame(m_frames[frameIndex].inlineFrame, inputHead)) && !iterationsSteppedAtHandOverPositionsAlone(inputHead))) return false; - return currentBodyIndex().loop(currentBodyIndex().loopOf(inputHead)).numberOfLoopsWithin <= 1; + return currentBodyIndex().loop(currentBodyIndex().loopOf(inputHead)).numberOfLoopsWithin <= 1 || hasUnrolledInnerLoops(inputHead); } TypeID Walker::backEdgeCheckType(BlockID inputHead, uint32_t parameter) const @@ -1045,10 +1095,10 @@ bool Walker::iterationsPreserveTypedArrays(IterationPurity purity) const return !ofTurns.mayRunOtherCode() && !ofTurns.writes.overlaps(LocationMasks(AbstractHeapSet { AbstractHeap::TypedArrayStorage })); } -bool Walker::areElementStoresRawDoubles(IterationPurity purity) const +bool Walker::areElementStoresNumeric(IterationPurity purity) const { std::optional loop = specializedOpenLoop(purity); - return loop && !(m_openLoops[*loop].effects.writes.maskOf(AbstractHeap::Elements) & markOfElementsOfArraysNotStoredAsRawDoubles); + return loop && !(m_openLoops[*loop].effects.writes.maskOf(AbstractHeap::Elements) & markOfElementsThatMayHoldCells); } bool Walker::hasNoElementStores() const @@ -1169,7 +1219,7 @@ bool Walker::someOperationOfRowThatRanOtherCodeIsCall(const BodyIndex& index) co bool Walker::canUseIterationGuards(BlockID inputHead) { - if (m_coldCopyDepth || !m_opensGuesses || !has(Mechanism::Guesses) || (isUnderHandler() && (isUnderHandlerOtherThanCleanups() || !hasElementArithmeticInLoop(inputHead))) || hasStringsInBuilders()) + if (m_coldCopyDepth || !m_opensGuesses || !has(Mechanism::Guesses) || (isUnderHandler() && !mayVersionLoopUnderHandler(inputHead)) || hasStringsInBuilders()) return false; if (m_restartablePreviewHeader == inputHead) return true; @@ -1282,6 +1332,127 @@ void Walker::recordUndoableFieldStore(Reference heir, LayoutID layout, uint32_t recordPendingUndoableStore(store); } +bool Walker::isDefinedOnPath(Reference object, const WalkState& state) const +{ + const VersionedLoop& versionedLoop = m_versionedLoops.last(); + for (uint32_t place = m_openLoops.size(); place--;) { + const OpenLoop& loop = m_openLoops[place]; + if (loop.frame == versionedLoop.frameIndex && m_frames[loop.frame].index->loop(loop.loop).head == versionedLoop.inputHead && isInvariantInLoop(place, object)) + return true; + } + return !state.listSlotsOf(object).isEmpty(); +} + +std::optional Walker::mergeUndoableStores(std::span edges) +{ + if (edges.size() < 2 || m_versionedLoops.isEmpty() || !m_versionedLoops.last().isEntered) + return std::nullopt; + uint32_t restartPoint = m_versionedLoops.last().restartPoint; + MergedUndoableStores merged; + merged.commonStore = m_openEdges[edges[0]].state.newestUndoableStore(); + for (uint32_t place : edges) { + const WalkState& state = m_openEdges[place].state; + if (!state.canRestartFrom(restartPoint)) + return std::nullopt; + for (uint32_t other = state.newestUndoableStore(); other != merged.commonStore;) { + uint32_t& newer = merged.commonStore > other ? merged.commonStore : other; + newer = m_undoableStores[newer].previous; + } + } + auto forEachStoreOnPath = [&](size_t path, NOESCAPE const auto& functor) { + for (uint32_t place = m_openEdges[edges[path]].state.newestUndoableStore(); place != merged.commonStore; place = m_undoableStores[place].previous) + functor(place); + }; + Vector stores; + for (size_t path = 0; path < edges.size(); ++path) + forEachStoreOnPath(path, [&](uint32_t place) { stores.append(place); }); + std::ranges::sort(stores); + stores.shrink(std::ranges::unique(stores).begin() - stores.begin()); + if (stores.isEmpty() || std::ranges::any_of(stores, [&](uint32_t place) { return m_undoableStores[place].position || m_undoableStores[place].layoutOfHeir; })) + return std::nullopt; + + struct StoresUnderKey { + uint32_t first; + bool shareRecord { true }; + size_t path { notFound }; + size_t record { notFound }; + }; + Vector byKey(stores); + std::ranges::stable_sort(byKey, { }, [&](uint32_t place) { return m_undoableStores[place].key.bits(); }); + Vector keys; + Vector keyOfStore(stores.size()); + auto indexOfStore = [&](uint32_t place) { return std::ranges::lower_bound(stores, place) - stores.begin(); }; + for (uint32_t place : byKey) { + const UndoableStore& store = m_undoableStores[place]; + if (keys.isEmpty() || m_undoableStores[keys.last().first].key != store.key) + keys.append({ place }); + const UndoableStore& first = m_undoableStores[keys.last().first]; + keys.last().shareRecord &= store.object == first.object && store.receivers == first.receivers && store.mode == first.mode && store.mayHoldNothing == first.mayHoldNothing; + keyOfStore[indexOfStore(place)] = keys.size() - 1; + } + for (size_t path = 0; path < edges.size(); ++path) { + forEachStoreOnPath(path, [&](uint32_t place) { + StoresUnderKey& key = keys[keyOfStore[indexOfStore(place)]]; + key.shareRecord &= std::exchange(key.path, path) != path; + }); + } + Vector recordOfStore(stores.size()); + for (size_t index = 0; index < stores.size(); ++index) { + StoresUnderKey& key = keys[keyOfStore[index]]; + if (!key.shareRecord || key.record == notFound) { + key.record = merged.stores.size(); + merged.stores.append({ m_undoableStores[stores[index]], Vector(edges.size()) }); + } + recordOfStore[index] = key.record; + } + static constexpr size_t mostValuesPerStore = 4; + if (merged.stores.size() * edges.size() > mostValuesPerStore * (stores.size() + edges.size())) + return std::nullopt; + for (size_t path = 0; path < edges.size(); ++path) { + forEachStoreOnPath(path, [&](uint32_t place) { + merged.stores[recordOfStore[indexOfStore(place)]].heldOnPaths[path] = m_undoableStores[place].held; + }); + } + Reference nothing = constant(Draft(Opcode::Absent)); + for (size_t path = 0; path < edges.size(); ++path) { + Reference definedObject; + for (MergedUndoableStore& field : merged.stores) { + if (field.heldOnPaths[path]) + continue; + if (field.store.object != definedObject && !isDefinedOnPath(field.store.object, m_openEdges[edges[path]].state)) + return std::nullopt; + definedObject = field.store.object; + field.heldOnPaths[path] = nothing; + field.store.mayHoldNothing = true; + } + } + count(StatisticID::UndoableStoresMergedAtJoins); + return merged; +} + +void Walker::recordMergedUndoableStores(const MergedUndoableStores& merged) +{ + uint32_t restartPoint = m_versionedLoops.last().restartPoint; + Vector places; + for (const MergedUndoableStore& field : merged.stores) { + UndoableStore store = field.store; + store.previous = m_state.newestUndoableStore(); + store.restartPoint = restartPoint; + m_undoableStores.append(store); + places.append(m_undoableStores.size() - 1); + m_state.recordUndoableStore(restartPoint, places.last()); + } + WalkState behindStores = m_state; + Effects undone = Effects::none(); + for (const MergedUndoableStore& field : merged.stores) + undone.writes.add(program(), MemoryLocation::objectField(field.store.key)); + behindStores.invalidate(undone); + for (uint32_t place : places) { + behindStores.recordUndoableStore(restartPoint, place); + m_undoableStoreStates.append({ place, behindStores }); + } +} + void Walker::letRestOfStretchRunAsWrittenBehindFirstStore(ValueID store) { StretchWithArmsAsWritten& stretch = m_stretchWithArmsAsWritten; @@ -1371,6 +1542,26 @@ void Walker::restartIteration() ++m_undoneStoreCount; continue; } + BlockID behindStore = store.mayHoldNothing ? addBlock(BlockFrequency::Rare) : BlockID(); + if (store.mayHoldNothing) { + Draft test(Opcode::TypeTest, m_origin); + test.setValueOperand(store.held); + test.setTestedType(TypeArena::fromPrimitives(Primitive::Absent)); + Reference holdsNothing = build(WTF::move(test)); + if (m_hasFailed || isUnreachable()) + return; + BlockID hasStored = addBlock(BlockFrequency::Rare); + branch(holdsNothing, behindStore, hasStored); + continueIn(hasStored); + Draft narrowed(Opcode::Narrowed, m_origin); + narrowed.setValueOperand(store.held); + narrowed.setTestOperand(holdsNothing); + narrowed.setOperandIndex(OperandIndex::TypeTest::value); + narrowed.setIsOnTakenEdge(false); + store.held = build(WTF::move(narrowed)); + if (m_hasFailed || isUnreachable()) + return; + } Draft draft(Opcode::PutSlot, m_origin); chooseConservativeFields(draft); draft.setObjectOperand(store.object); @@ -1385,6 +1576,11 @@ void Walker::restartIteration() if (m_hasFailed || isUnreachable()) return; ++m_undoneStoreCount; + if (behindStore) { + jumpTo(behindStore, { }); + if (m_hasFailed || isUnreachable()) + return; + } } } if (Reference head = m_versionedLoops.last().effectsCheckBeginning; head && head.isValue()) [[unlikely]] { @@ -1479,6 +1675,8 @@ bool Walker::mayBuildLoopSeveralTimes(BlockID inputHead) if (isTooLargeToVersion(index, loop)) return false; + if (index.holdsAnyOf(OpcodeSet { Opcode::IteratorOpen }) && loopHoldsGenerators(index, loop)) + return false; bool yieldsWalkTurnsOfConsumer = loopCarriesValuesOfConsumerRound(inputHead); if (yieldsWalkTurnsOfConsumer && !isConsumerSmallEnoughToVersion(inputHead, maxVersionedLoopSize)) return false; @@ -1539,7 +1737,7 @@ static bool mustBeNamedByDefinitionInLoopsBuiltTwice(const BodyIndex& index, Val static BlockID findOuterCloseHandler(Walker& walker, const BodyIndex& index, LoopID loop, ValueID open) { - bool isOpenedByPattern = !index.isParameter(open) && findDestructuredEntriesIteration(walker, index, index.value(open)); + bool isOpenedByPattern = !index.isParameter(open) && (findDestructuredEntriesIteration(walker, index, index.value(open)) || destructuresReturnedLiteral(walker, index, open)); for (BlockID member : index.membersOf(loop)) { BodyBlock block = index.block(member); if (block.hasHandler() && !index.isInLoop(block.handler().block(), loop) && (isOpenedByPattern ? iteratorClosedLeavingLoop(index, block, loop) : iteratorClosedLeavingFunction(index, block)) == open) @@ -1568,6 +1766,7 @@ std::optional> Walker::findValuesOfLoopReadBehindIt(BlockID i BitVector isListed; Vector readByHandlersOutside; Vector cleanupsOutside; + bool onlyDestructuresReturnedLiterals = true; for (BlockID member : index.membersOf(loop)) { for (BodyTarget target : index.terminator(member).targets()) { if (index.isInLoop(target.block(), loop)) @@ -1596,10 +1795,12 @@ std::optional> Walker::findValuesOfLoopReadBehindIt(BlockID i if (!cleanup) return std::nullopt; cleanupsOutside.append(cleanup); + onlyDestructuresReturnedLiterals = onlyDestructuresReturnedLiterals && destructuresReturnedLiteral(*this, index, ValueID(value)); } } if (!readByHandlersOutside.isEmpty() || !cleanupsOutside.isEmpty()) { - if (turnsOfLoopCallForGuesses(index, loop)) + onlyDestructuresReturnedLiterals &= readByHandlersOutside.isEmpty(); + if (!onlyDestructuresReturnedLiterals && turnsOfLoopCallForGuesses(index, loop)) return std::nullopt; for (ValueID value : readBehindLoop) { if (mustBeNamedByDefinitionInLoopsBuiltTwice(index, value)) @@ -1858,6 +2059,12 @@ bool Walker::buildLoopForEachMember(BlockID inputHead, const LoopPreview& previe members.shrink(0); } bool isToldApart = !members.isEmpty() && (members.size() >= 2 || members[0].nothingWasExpected || members[0].restIsWalkedAsWithoutGuess); + std::optional counterOfAnyKind = isToldApart || row || standsInCopyOfLoopAround || !steppedAtHead.asTheyEntered.isEmpty() ? std::nullopt : findCounterOfAnyKind(*this, inputHead, preview, entering); + if (counterOfAnyKind) { + members.shrink(0); + members.append(HoistedSpeculation { entering[counterOfAnyKind->position], counterOfAnyKind->tested, { }, { } }); + isToldApart = true; + } if (!isToldApart || !members[0].readOfRowInLoop) goOnWithBothWays(); if (!isToldApart) @@ -2034,14 +2241,14 @@ bool Walker::buildLoopForEachMember(BlockID inputHead, const LoopPreview& previe member.value = given; } for (const HoistedSpeculation& member : members) { - Reference test = isUnreachable() ? Reference() : buildTestInFrontOfLoop(*this, member, origin); + Reference test = isUnreachable() ? Reference() : counterOfAnyKind ? buildTestOfNumber(*this, member.value, member.tested, origin) : buildTestInFrontOfLoop(*this, member, origin); if (!test) continue; BlockID passed = addBlock(); BlockID next = addBlock(); branch(test, passed, next); continueIn(passed); - Reference proof = buildProofInFrontOfLoop(*this, member, test, origin); + Reference proof = counterOfAnyKind ? buildProofOfNumber(*this, member.value, test, origin) : buildProofInFrontOfLoop(*this, member, test, origin); const ClassPart* part = proof && proof.isValue() && !isUnreachable() && m_types.at(typeOf(proof)).isObject() ? singleBuiltinClassOf(m_types, m_types.at(typeOf(proof))) : nullptr; if (part && pristineTestOf(program().builtins(), part->builtinClass) == PristineTest::InstanceStructure) { PristineAspectSet aspects { PristineAspects::Names }; @@ -2080,6 +2287,8 @@ bool Walker::buildLoopForEachMember(BlockID inputHead, const LoopPreview& previe } } Vector, 1> elementsThatEnterAsNumbers; + if (counterOfAnyKind && proof) + elementsThatEnterAsNumbers.append({ counterOfAnyKind->position, proof }); for (size_t place = 0; part && part->builtinClass == BuiltinClasses::Array && !isList && place < entering.size() && !isUnreachable(); ++place) { Reference element = entering[place]; if (!element || !element.isValue() || !(m_code.at(element.id()).opcode == Opcode::ElemGet ? m_code.at(element.id()).objectOperand() == member.value.id() : m_code.at(element.id()).opcode == Opcode::GetArrayElement && m_code.at(element.id()).arrayOperand() == member.value.id())) @@ -2285,6 +2494,9 @@ bool Walker::buildLoopForEachMember(BlockID inputHead, const LoopPreview& previe uint32_t narrowerNamesBeforeLoop = inputBlock(frameIndex, inputHead).firstNarrowerName; forgetAllButFreshObjectsAndFieldsThatBlockTakes(genericVersionEntry); + WalkState knownInFrontOfRest; + if (startOfRest) + knownInFrontOfRest = m_state; enterWithFieldsOfScopes(genericVersionEntry, entering); enterGenericVersion(genericVersionEntry, preview); { @@ -2318,8 +2530,11 @@ bool Walker::buildLoopForEachMember(BlockID inputHead, const LoopPreview& previe continue; continueIn(startOfRest); if (numberOfEdges > numberOfArrivals) { + if (numberOfArrivals) + m_state.meetWith(knownInFrontOfRest); + else + m_state = WTF::move(knownInFrontOfRest); m_block = startOfRest; - m_state.clear(); } } else { m_block = start.block; @@ -2458,7 +2673,7 @@ bool Walker::buildLoopForEachMember(BlockID inputHead, const LoopPreview& previe m_sharedGenericVersions.last().isAboutToBeTaken = true; m_versionedLoops.append(VersionedLoop { frameIndex, inputHead, genericVersionEntry, { }, whereFrontFails, { }, false }); m_versionedLoops.last().hasCallInIteration = someOperationOfRowThatRanOtherCodeIsCall(index), m_versionedLoops.last().hasUninlinedCall = m_summaryHasUninlinedCall; - m_versionedLoops.last().hasElementArithmetic = hasElementArithmeticInLoop(inputHead); + m_versionedLoops.last().mayRestartUnderHandler = mayVersionLoopUnderHandler(inputHead); m_versionedLoops.last().lastBackEdgeDominator = BackEdgeDominators(index, index.loopOf(inputHead)).lastOf(index.loopOf(inputHead)); ++m_ordinalOfCopyOfLoop; Vector enteringCopy(entering); @@ -2633,6 +2848,42 @@ static void appendLinksHeldWhereTheyComeRound(Walker& walker, BlockID inputHead, } } +static bool isElementUnderCounter(Walker& walker, const BodyIndex& index, ValueID value, const BitVector& isUnderCounter, unsigned joinsLeft = 3) +{ + for (;;) { + value = index.stripCasts(value); + if (!value) + return false; + if (index.isParameter(value)) { + ValueID narrowed = index.valueNarrowedInto(value); + if (!narrowed) + break; + value = narrowed; + continue; + } + if (index.opcodeOf(value) != Opcode::ToPropertyKey) + return index.opcodeOf(value) == Opcode::ElemGet && isUnderCounter.get(value.index()); + value = index.value(value).operand(0); + } + BlockID join = index.blockOf(value); + if (!joinsLeft || index.isLoopHead(join) || join == entryBlock) + return false; + uint32_t place = value.index() - index.block(join).firstParameter().index(); + return std::ranges::all_of(walker.incomingEdgesOf(index, join), [&](const BodyIncomingEdge& edge) { + std::span arguments = edge.target.arguments(); + return place < arguments.size() && isElementUnderCounter(walker, index, arguments[place], isUnderCounter, joinsLeft - 1); + }); +} + +static bool accessesViewUnderIntegerElement(Walker& walker, const BodyIndex& index, ValueID access, const BitVector& isUnderCounter, NOESCAPE const ScopedLambda& typeInFront) +{ + if (index.opcodeOf(access) != Opcode::ElemGet && index.opcodeOf(access) != Opcode::ElemSet) + return false; + BodyValue operation = index.value(access); + TypeID holder = typeInFront(index.stripCasts(operation.objectOperand())); + return holder && isViewOfNumbersReadInLine(walker, holder) && isElementUnderCounter(walker, index, operation.propertyOperand(), isUnderCounter); +} + bool Walker::tryVersionLoop(BlockID inputHead, const LoopPreview& preview, std::span entering) { if (!m_buysTwoBuilds && !m_isVersioningDryRun) @@ -2928,6 +3179,7 @@ bool Walker::tryVersionLoop(BlockID inputHead, const LoopPreview& preview, std:: specializedPreview->guessesCountedOn.append(GuessCountedOnByLooks { ran, receiver, typeInRow(receiver) }); } specializedPreview->hasLetMentionsLeave = true; + specializedPreview->slots = preview.slots; Vector, 2> membersAbovePrimitives; bool asksMembersByTheirBytes = false; for (const GuessCountedOnByLooks& seen : m_testsSeenOfRow) { @@ -2992,14 +3244,18 @@ bool Walker::tryVersionLoop(BlockID inputHead, const LoopPreview& preview, std:: specializedEffectsFromSummary.writes.remove(AbstractHeapSet { AbstractHeap::MemberGuards }); Vector fieldsTestedInFront; if (genericVersionHasEntry && !sharedEntry && steppedAtHead.asTheyEntered.isEmpty()) - findFieldsOfScopesFromOutsideThatQuietTurnsStore(inputHead, specializedPreview ? specializedPreview->effects : preview.effects, fieldsTestedInFront, false); + findFieldsOfScopesFromOutsideThatQuietTurnsStore(inputHead, specializedPreview ? specializedPreview->effects : preview.effects, preview.slots.span(), fieldsTestedInFront, false); auto staysInt32ByRow = [&](const FieldOfScope& field) { TypeID read = resultTypeOf(draftVariableRead(field)); + if (field.slotKey) + return field.slotIsInt32Position && !m_types.isSubset(read, TypeTable::int32) && m_types.isSubset(TypeTable::int32, read); return field.isSteppedOrAddedTo && (!field.isReadBySumsAlone || field.isSteppedByOne) && !m_types.isSubset(read, TypeTable::int32) && m_types.isSubset(TypeTable::int32, read) && std::ranges::all_of(field.addedInTurns, [&](ValueID added) { return m_types.isSubset(typeInRow(added), TypeTable::int32); }); }; fieldsTestedInFront.removeAllMatching([&](const FieldOfScope& field) { + if (field.slotKey) + return !staysInt32ByRow(field); TypeID read = resultTypeOf(draftVariableRead(field)); if (!read || !m_types.overlap(read, TypeTable::number)) return true; @@ -3013,7 +3269,7 @@ bool Walker::tryVersionLoop(BlockID inputHead, const LoopPreview& preview, std:: return !staysInt32ByRow(field) && !mayBeInDeadZone(field); }); for (FieldOfScope& field : fieldsTestedInFront) - field.type = staysInt32ByRow(field) ? TypeID(TypeTable::int32) : m_types.intersectionOf(resultTypeOf(draftVariableRead(field)), TypeTable::number); + field.type = field.slotIsSteppedByOne ? m_types.integer(Range { field.slotOnlyGrows ? 0 : -maxExactInteger, maxExactInteger }) : staysInt32ByRow(field) ? TypeID(TypeTable::int32) : m_types.intersectionOf(resultTypeOf(draftVariableRead(field)), TypeTable::number); for (const HoistedVariableCheck& asked : hoistedVariableChecks) { if (!genericVersionHasEntry || sharedEntry || !steppedAtHead.asTheyEntered.isEmpty()) break; @@ -3031,7 +3287,7 @@ bool Walker::tryVersionLoop(BlockID inputHead, const LoopPreview& preview, std:: bool storesUnderCounterAloneRanOtherCode = false; if (const Effects& ofTurns = specializedPreview ? specializedPreview->effects : preview.effects; genericVersionHasEntry && steppedAtHead.asTheyEntered.isEmpty() && !asksAtHead && !m_coldCopyDepth && has(Mechanism::TypedOperations) && has(Mechanism::UnboxedNumbers) && !isUnderHandler() && !ofTurns.maySuspend) { - bool viewsMayBeHeld = ofTurns.mayRunOtherCode() ? !specializedPreview && !m_operationsThatRanOtherCodeOfRow.isEmpty() : !ofTurns.writes.overlaps(LocationMasks(AbstractHeapSet { AbstractHeap::TypedArrayStorage })); + bool viewsMayBeHeld = ofTurns.mayRunOtherCode() ? !specializedPreview : !ofTurns.writes.overlaps(LocationMasks(AbstractHeapSet { AbstractHeap::TypedArrayStorage })); auto isListOnlyReadInLoop = [&](ValueID written) { Reference built = referenceOf(written); if (!built || !built.isValue()) @@ -3075,12 +3331,16 @@ bool Walker::tryVersionLoop(BlockID inputHead, const LoopPreview& preview, std:: for (const AccessUnderCounterOfLoop& access : heldBelowRoom.accesses) isUnderCounter.set(access.access.index()); storesUnderCounterAloneRanOtherCode = std::ranges::all_of(m_operationsThatRanOtherCodeOfRow, [&](ValueID ran) { + if (ran && accessesViewUnderIntegerElement(*this, index, ran, isUnderCounter, scopedLambda(typeInFront))) + return true; if (!ran || !isUnderCounter.get(ran.index()) || index.opcodeOf(ran) != Opcode::ElemSet) return false; ValueID stored = index.stripCasts(index.value(ran).valueOperand()); return isUnderCounter.get(stored.index()) && !index.isParameter(stored) && index.opcodeOf(stored) == Opcode::ElemGet; }); - if (!storesUnderCounterAloneRanOtherCode) + Reference boundInFront = m_operationsThatRanOtherCodeOfRow.isEmpty() && heldBelowRoom.bound ? referenceOf(heldBelowRoom.bound) : Reference(); + bool boundMayBeNoInteger = m_operationsThatRanOtherCodeOfRow.isEmpty() && !heldBelowRoom.viewWhoseLengthIsBound && !(boundInFront && boundInFront.isRecord() && m_types.at(typeOf(boundInFront)).isInteger()); + if (!storesUnderCounterAloneRanOtherCode || boundMayBeNoInteger) heldBelowRoom.accesses.shrink(0); } } @@ -3461,7 +3721,7 @@ bool Walker::tryVersionLoop(BlockID inputHead, const LoopPreview& preview, std:: count(queued == notFound ? StatisticID::LoopStoredVariableReadsHoisted : hoistedVariableChecks[queued].counted); read = build(WTF::move(inFront)); } - Reference test = read ? buildTestOfNumber(*this, read, field.type, origin) : Reference(); + Reference test = read ? buildTestOfNumber(*this, read, field.slotKey ? TypeID(TypeTable::int32) : field.type, origin) : Reference(); if (!test || !test.isValue()) continue; BlockID passed = addBlock(); @@ -3477,7 +3737,18 @@ bool Walker::tryVersionLoop(BlockID inputHead, const LoopPreview& preview, std:: m_state.noteEndOfDeadZone(field.variable); m_state.noteVariable(field.variable, proof); noteValueOfStableBinding(*this, field.variable, proof); - } else if (proof) + } else if (proof && field.slotKey) { + Reference fromZeroUp = field.slotIsSteppedByOne && field.slotOnlyGrows ? buildOperatorInNumberForms(*this, Opcode::GreaterEq, proof, integerInNumberForms(*this, 0), origin) : Reference(); + if (fromZeroUp && fromZeroUp.isValue() && !m_code.isConstant(fromZeroUp.id())) { + BlockID behind = addBlock(); + branch(fromZeroUp, behind, shouldIgnoreVariableChecks() ? behind : genericStart); + continueIn(behind); + proof = buildNarrowedInNumberForms(*this, proof, fromZeroUp, 0, true, origin); + } + if (proof) + m_state.noteSlot(field.scope, field.field, proof); + } + else if (proof) noteEnvironmentField(field.scope, field.field, proof); } bool holdsCounterBelowLengthsOfLists = heldBelowRoom.accesses.containsIf([](const AccessUnderCounterOfLoop& access) { return access.isOfList; }); @@ -3510,6 +3781,9 @@ bool Walker::tryVersionLoop(BlockID inputHead, const LoopPreview& preview, std:: } return built; }; + auto viewBeforeLoop = [&](const AccessUnderCounterOfLoop& access) -> Reference { + return access.fieldView ? access.fieldView : builtInFront(access.view); + }; auto isInteger = [&](Reference value) { return value && value.isRecord() && m_types.at(typeOf(value)).isInteger(); }; @@ -3590,16 +3864,19 @@ bool Walker::tryVersionLoop(BlockID inputHead, const LoopPreview& preview, std:: return constant(WTF::move(integer)); }; for (const AccessUnderCounterOfLoop& access : heldBelowRoom.accesses) { - Reference view = builtInFront(access.view); + Reference view = viewBeforeLoop(access); if (std::optional written = view ? writtenAddend(access) : std::nullopt) { Range& range = writtenAddendsOfViews.add(view.bits(), Range { *written, *written }).iterator->value; range = Range { std::min(range.minimum, *written), std::max(range.maximum, *written) }; } } + std::ranges::stable_partition(heldBelowRoom.accesses, [](const AccessUnderCounterOfLoop& access) { + return !access.turnAsksSignOfAddend; + }); Reference leastRoom; for (size_t place = 0; stepsUp && !m_hasFailed && place < heldBelowRoom.accesses.size(); ++place) { const AccessUnderCounterOfLoop& access = heldBelowRoom.accesses[place]; - Reference view = builtInFront(access.view); + Reference view = viewBeforeLoop(access); bool isWritten = view && writtenAddend(access); Reference addend = isWritten ? integerOf(writtenAddendsOfViews.get(view.bits()).maximum) : builtInFront(access.addend); if (!view || !view.isValue() || (addend && !isInteger(addend)) || (!isWritten && !addend)) @@ -3614,7 +3891,10 @@ bool Walker::tryVersionLoop(BlockID inputHead, const LoopPreview& preview, std:: }); if (!isAsked) { Reference leastAddend = isWritten ? integerOf(written.minimum) : addend; - std::optional leastIsNotNegative = continueIfNonNegative(leastAddend ? buildIntegerOperator(*this, Opcode::Add, enters, leastAddend, origin) : enters); + bool isSignLeftToTurn = access.turnAsksSignOfAddend && findNonNegativeProof(enters); + if (isSignLeftToTurn) + count(StatisticID::AddendsWhoseSignIsLeftToTurns); + std::optional leastIsNotNegative = isSignLeftToTurn ? std::optional(PassedTest()) : continueIfNonNegative(leastAddend ? buildIntegerOperator(*this, Opcode::Add, enters, leastAddend, origin) : enters); if (!leastIsNotNegative) continue; Reference room = lengthInFrontOfLoop(view, access.isOfList); @@ -3847,7 +4127,7 @@ bool Walker::tryVersionLoop(BlockID inputHead, const LoopPreview& preview, std:: m_sharedGenericVersions.append(SharedGenericVersion { frameIndex, inputHead, genericStart, genericVersionEntry, true }); m_versionedLoops.append(VersionedLoop { frameIndex, inputHead, genericVersionEntry, { }, genericStart, { }, false }); m_versionedLoops.last().hasCallInIteration = someOperationOfRowThatRanOtherCodeIsCall(index), m_versionedLoops.last().hasUninlinedCall = m_summaryHasUninlinedCall; - m_versionedLoops.last().hasElementArithmetic = hasElementArithmeticInLoop(inputHead); + m_versionedLoops.last().mayRestartUnderHandler = mayVersionLoopUnderHandler(inputHead); m_versionedLoops.last().lastBackEdgeDominator = BackEdgeDominators(index, index.loopOf(inputHead)).lastOf(index.loopOf(inputHead)); ++m_ordinalOfCopyOfLoop; std::pair boughtFirst { static_cast(frameIndex), inputHead }; @@ -3910,7 +4190,7 @@ bool Walker::tryVersionLoop(BlockID inputHead, const LoopPreview& preview, std:: m_versionedLoops.last().isArrayGuardedAtHeaderWritable = arrayGuardedAtHeader->isWritable; } if (genericVersionEntry) - m_versionedLoops.last().hasElementArithmetic = hasElementArithmeticInLoop(inputHead); + m_versionedLoops.last().mayRestartUnderHandler = mayVersionLoopUnderHandler(inputHead); if (genericVersionEntry) m_versionedLoops.last().lastBackEdgeDominator = BackEdgeDominators(index, index.loopOf(inputHead)).lastOf(index.loopOf(inputHead)); if (genericVersionEntry && !isCopyBeside) @@ -3996,7 +4276,7 @@ bool Walker::tryVersionLoop(BlockID inputHead, const LoopPreview& preview, std:: m_typedBuildEndedListsTestedInFront = true; else if (listsRead.areTested && !m_hasFailed && m_walkOfLoopWasKept && !preservesHoistedArrayGuards(*this, m_effectsOfLoopKeptLast)) count(StatisticID::VersionedLoopsFillingLocalArrays); - if (!m_hasFailed && m_walkOfLoopWasKept && m_hoistedElementCheckCount != elementsAskedInFront && (m_effectsOfLoopKeptLast.writes.maskOf(AbstractHeap::Elements) & markOfElementsOfArraysNotStoredAsRawDoubles)) + if (!m_hasFailed && m_walkOfLoopWasKept && m_hoistedElementCheckCount != elementsAskedInFront && (m_effectsOfLoopKeptLast.writes.maskOf(AbstractHeap::Elements) & markOfElementsThatMayHoldCells)) m_typedBuildEndedListsTestedInFront = true; bool turnsStoreElements = m_effectsOfLoopKeptLast.mayRunOtherCode() || m_effectsOfLoopKeptLast.writes.maskOf(AbstractHeap::Elements); if (!m_hasFailed && m_walkOfLoopWasKept && m_hoistedNumberReadCount != numbersReadInFront && turnsStoreElements) @@ -4042,6 +4322,10 @@ bool Walker::tryVersionLoop(BlockID inputHead, const LoopPreview& preview, std:: forgetValuesOfLoop(inputHead); switchToInputBlock(inputHead); m_frames[frameIndex].isInPreheader = true; + if (isUnderHandler()) { + for (size_t place = 0; place < entering.size(); ++place) + bind(index.block(inputHead).parameter(place), entering[place]); + } continueIn(listsRead.alternateStorageEntry); if (knownInFrontOfTestsOfLists && !isUnreachable()) m_state = WTF::move(*knownInFrontOfTestsOfLists); @@ -4102,7 +4386,7 @@ bool Walker::tryVersionLoop(BlockID inputHead, const LoopPreview& preview, std:: m_sharedGenericVersions.append(SharedGenericVersion { frameIndex, inputHead, genericStart, genericVersionEntry, true }); m_versionedLoops.append(VersionedLoop { frameIndex, inputHead, genericVersionEntry, { }, genericStart, { }, false }); m_versionedLoops.last().hasCallInIteration = someOperationOfRowThatRanOtherCodeIsCall(index), m_versionedLoops.last().hasUninlinedCall = m_summaryHasUninlinedCall; - m_versionedLoops.last().hasElementArithmetic = hasElementArithmeticInLoop(inputHead); + m_versionedLoops.last().mayRestartUnderHandler = mayVersionLoopUnderHandler(inputHead); m_versionedLoops.last().lastBackEdgeDominator = BackEdgeDominators(index, index.loopOf(inputHead)).lastOf(index.loopOf(inputHead)); ++m_ordinalOfCopyOfLoop; std::pair boughtFirst { static_cast(frameIndex), inputHead }; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkLoopsOverGenerators.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkLoopsOverGenerators.cpp index a19420c143474..7e29daad7ec83 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkLoopsOverGenerators.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkLoopsOverGenerators.cpp @@ -19,7 +19,10 @@ namespace JSC { namespace AOT { namespace Typed { +bool stepsOfPatternStandInRow(const BodyIndex&, const BodyValue& open); + static constexpr uint32_t closedOperationsOfProtocolForGenerators = 3; +static constexpr uint32_t mostStepsOfPatternOverGenerator = 4; std::optional findLoopOverIterator(const BodyIndex& index, ValueID open) { @@ -310,17 +313,8 @@ static bool openLoopOverGenerator(Walker& walker, const BodyValue& open) return true; } -static bool stepLoopOverGenerator(Walker& walker, const BodyValue& step) +static void stepGeneratorInPlace(Walker& walker, BuilderForGenerators& builder, Reference generator, const std::optional& generatorWasAsMade, const BuilderForGenerators::MeetingOfStep& after) { - Reference state = walker.referenceOf(step.operand(0)); - if (!isStateOfLoopOverGenerator(walker, state)) - return false; - - BuilderForGenerators builder(walker, walker.originOf(step)); - BuilderForGenerators::MeetingOfStep after = builder.openMeetingOfStep(); - BlockID byProtocol = builder.addBlock(BlockFrequency::Unlikely); - Reference generator = builder.continueIfTypeMatches(state, typeOfGenerators(walker), byProtocol); - std::optional generatorWasAsMade = walker.loopsOverGenerators().statesOpened.get(state.bits()).generatorWasAsMade; Reference stepped = generatorWasAsMade ? buildNextStepOfGeneratorAskedAtOpen(walker, builder, generator, *generatorWasAsMade) : buildStepOfGenerator(walker, builder, generator, ResumeMode::Next, GeneratorStepCheck::IsUnmodified); Reference value = builder.resultAt(stepped, static_cast(StepGeneratorResult::Value)); Reference status = builder.resultAt(stepped, static_cast(StepGeneratorResult::Status)); @@ -337,6 +331,20 @@ static bool stepLoopOverGenerator(Walker& walker, const BodyValue& step) builder.continueIn(isStepped); builder.endStep(builder.isInteger(status, static_cast(GeneratorStepStatus::Completed)), value, after); +} + +static bool stepLoopOverGenerator(Walker& walker, const BodyValue& step) +{ + Reference state = walker.referenceOf(step.operand(0)); + if (!isStateOfLoopOverGenerator(walker, state)) + return false; + + BuilderForGenerators builder(walker, walker.originOf(step)); + BuilderForGenerators::MeetingOfStep after = builder.openMeetingOfStep(); + BlockID byProtocol = builder.addBlock(BlockFrequency::Unlikely); + Reference generator = builder.continueIfTypeMatches(state, typeOfGenerators(walker), byProtocol); + std::optional generatorWasAsMade = walker.loopsOverGenerators().statesOpened.get(state.bits()).generatorWasAsMade; + stepGeneratorInPlace(walker, builder, generator, generatorWasAsMade, after); builder.continueIn(byProtocol); Reference words = builder.takeWordsOfProtocol(state, listOfWordsMadeForState(walker, state)); @@ -396,11 +404,83 @@ static bool closeLoopOverGenerator(Walker& walker, const BodyValue& close) return true; } +static bool takePatternOverGenerator(Walker& walker, const BodyValue& open) +{ + auto written = open.as(); + if (!written.isDestructuring() || written.iteratorKind() != IteratorKind::Sync || written.isDelegating() || !walker.isInLoop()) + return false; + Reference generator = walker.referenceOf(written.iterableOperand()); + TypeID proven = generator && generator.isValue() ? walker.typeOf(generator) : TypeID(); + if (!proven || !walker.types().isSubset(proven, typeOfGenerators(walker)) || !walker.has(Mechanism::Guesses) || !walker.opensGuesses()) + return false; + uint32_t numberOfSteps = written.elementCount().value_or(0); + if (!numberOfSteps || numberOfSteps > mostStepsOfPatternOverGenerator || !engineAnswersForGenerators(walker) || !stepsOfPatternStandInRow(walker.currentBodyIndex(), open)) + return false; + + BuilderForGenerators builder(walker, walker.originOf(open)); + BlockID byRunTime = builder.addBlock(BlockFrequency::Rare); + BlockID meeting = builder.addBlock(); + Walker::IndexedIteration pattern; + pattern.hasPrecomputedSteps = true; + for (uint32_t step = 0; step < numberOfSteps; ++step) + pattern.precomputedSteps.append(walker.addParameter(meeting, { }, Representation::JSValue)); + auto handOn = [&](std::span taken) { + if (walker.isUnreachable()) + return; + Vector arguments(taken); + while (arguments.size() < numberOfSteps) + arguments.append(builder.undefined()); + walker.jumpTo(meeting, arguments.span()); + }; + + Draft test = builder.draft(Opcode::IsUnmodifiedGenerator); + test.setValueOperand(generator); + std::optional isAsMade = builder.continueIfCheckPasses(builder.build(WTF::move(test)), byRunTime); + if (isAsMade) { + buildFirstReadsThatOpenOfGeneratorLeavesOut(walker, builder.origin()); + Vector taken; + for (uint32_t step = 0; step < numberOfSteps; ++step) { + BuilderForGenerators::MeetingOfStep after = builder.openMeetingOfStep(); + stepGeneratorInPlace(walker, builder, generator, isAsMade, after); + builder.continueIn(after.block); + BlockID last = builder.addBlock(); + BlockID further = builder.addBlock(); + builder.branch(after.done, last, further); + builder.continueIn(last); + handOn(taken.span()); + builder.continueIn(further); + taken.append(after.value); + } + closeGenerator(walker, builder, generator, IteratorCompletion::Normal); + handOn(taken.span()); + } else if (!walker.isUnreachable()) + builder.jumpTo(byRunTime); + + builder.continueIn(byRunTime); + Draft steps = builder.draft(Opcode::DestructureIterable); + steps.setIterableOperand(generator); + steps.setCount(numberOfSteps); + steps.setReadsListsByIndex(isArrayIteratorProtocolIntact(walker)); + Reference results = builder.build(WTF::move(steps)); + if (!walker.isUnreachable()) + walker.state().invalidateHeap(); + Vector taken; + for (uint32_t step = 0; step < numberOfSteps; ++step) + taken.append(builder.resultAt(results, step)); + handOn(taken.span()); + + builder.continueIn(meeting); + walker.recordSpeculation(); + if (!walker.isUnreachable()) + walker.bind(open.id(), walker.addIndexedIteration(WTF::move(pattern))); + return true; +} + bool buildLoopOverGenerator(Walker& walker, const BodyValue& value) { switch (value.opcode()) { case Opcode::IteratorOpen: - return openLoopOverGenerator(walker, value); + return openLoopOverGenerator(walker, value) || takePatternOverGenerator(walker, value); case Opcode::IteratorResume: return stepLoopOverGenerator(walker, value); case Opcode::IteratorClose: diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkNumberExpectations.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkNumberExpectations.cpp index 969dcf6abcdae..8151a839de203 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkNumberExpectations.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkNumberExpectations.cpp @@ -480,6 +480,26 @@ ExpectationOfNumber numbersExpectedOfSlotThatTurnsStep(Walker& walker, ValueID i return { cut, found.isDeclared, found.isHeld }; } +ExpectationOfNumber numberExpectedOfAddendOfPosition(Walker& walker, const BodyValue& operation, ValueID operand, ExpectationOfNumber found) +{ + if (!found.type || !sumsFollowedInNumberExpectations.contains(operation.opcode()) || !walker.has(Mechanism::TypedOperations) || !walker.has(Mechanism::UnboxedNumbers)) + return found; + const BodyIndex& index = walker.currentBodyIndex(); + ValueID written = index.stripCasts(operand); + LoopID loop = index.loopOf(index.blockOf(operation.id())); + if (!written || !loop || index.isInLoop(index.blockOf(written), loop)) + return found; + TypeArena& types = walker.types(); + Type expected = types.at(found.type); + if (!holdsNumbersBesideNullishAlone(expected) || expected.isInt32OrNullish()) + return found; + TypeID cut = withNumbersCutToInt32(types, expected); + TypeID proven = provenTypeInNumberExpectations(walker, operand); + if (!cut || (proven && !types.overlap(proven, cut)) || !usesSpeakForInt32(walker, operation.id())) + return found; + return { cut, found.isDeclared, found.isHeld }; +} + ExpectationOfNumber numbersExpectedOf(Walker& walker, ValueID input, ExpectationOfNumber found) { if (!walker.has(Mechanism::TypedOperations) || !walker.has(Mechanism::UnboxedNumbers)) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkNumberForms.h b/Source/JavaScriptCore/aot/walk/AOTWalkNumberForms.h index 23d3e5568b1e5..a8401388758d4 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkNumberForms.h +++ b/Source/JavaScriptCore/aot/walk/AOTWalkNumberForms.h @@ -57,6 +57,7 @@ struct BitsOfDividendCompared { std::optional tryCompareBitsOfDividend(Walker&, Origin, const WholeRemainder&, int64_t comparedWith); Reference buildComparisonWhereDivisorsAreNotZero(Walker&, Draft&& comparison, std::span divisorsThatMayBeZero); bool buildNumberRemainderWithItsTest(Walker&, const BodyValue& remainder); +ValueID findTestOfRemainder(Walker&, const BodyValue& remainder); inline constexpr OpcodeSet opcodesThatBuildNumberFormTakes = allBitwiseOperators | allEqualities | allRelations | OpcodeSet { Opcode::Add, Opcode::Sub, Opcode::Mul, Opcode::Div, Opcode::Mod, Opcode::Pow, Opcode::LogicalNot, Opcode::ToBoolean }; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkNumberGuesses.h b/Source/JavaScriptCore/aot/walk/AOTWalkNumberGuesses.h index 424ca8c99de52..5a54c686e7aae 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkNumberGuesses.h +++ b/Source/JavaScriptCore/aot/walk/AOTWalkNumberGuesses.h @@ -19,6 +19,7 @@ struct ExpectationOfNumber { ExpectationOfNumber numbersExpectedOf(Walker&, ValueID input, ExpectationOfNumber found); TypeID typeHeldAtPositionOf(Walker&, ValueID input); bool usesSpeakForInt32(Walker&, ValueID input); +ExpectationOfNumber numberExpectedOfAddendOfPosition(Walker&, const BodyValue& operation, ValueID operand, ExpectationOfNumber found); ExpectationOfNumber numbersExpectedOfSlotThatTurnsStep(Walker&, ValueID input, ExpectationOfNumber found); struct TestOfNumber { diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkNumberKeys.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkNumberKeys.cpp index 4d56bf6467986..71232d5e73756 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkNumberKeys.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkNumberKeys.cpp @@ -63,7 +63,7 @@ static bool keyIsBoundToRemainderInNumberKeys(Walker& walker, ValueID writtenKey ValueID record = key.id(); while (valuePreservingOpcodes.contains(code.at(record).opcode)) record = code.at(record).operand(0); - return code.at(record).opcode == Opcode::Mod; + return code.at(record).opcode == Opcode::Mod || code.at(record).opcode == Opcode::RemainderByWholeDivisor; } static WayInNumberForms endOfAccessInNumberKeys(Walker& walker, const BodyValue& value) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkNumberOperators.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkNumberOperators.cpp index b887044522e95..b8ed650218aeb 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkNumberOperators.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkNumberOperators.cpp @@ -12,6 +12,7 @@ #include "AOTOpcodeSets.h" #include "AOTRepresentationRules.h" #include "AOTTypeArena.h" +#include "AOTWalkNumberGuesses.h" #include "JSGlobalObjectFunctions.h" #include #include @@ -142,8 +143,61 @@ static Reference valueThatArithmeticGives(Walker& walker, Opcode opcode, const O } } +static Reference buildConversionInNumberOperators(Walker&, Opcode conversion, Reference number, Origin); + +static Reference buildSumWithUndefined(Walker& walker, Origin origin, ValueID written) +{ + const BodyIndex& index = walker.currentBodyIndex(); + ValueID sum = index.stripCasts(written); + static constexpr OpcodeSet sums { Opcode::Add, Opcode::Sub }; + if (!walker.has(Mechanism::Folding) || !walker.has(Mechanism::UnboxedNumbers) || index.isParameter(sum) || !sums.contains(index.opcodeOf(sum))) + return { }; + TypeArena& types = walker.types(); + std::array terms; + std::optional optionalTerm; + for (uint32_t side = 0; side < 2; ++side) { + terms[side] = walker.referenceOf(index.value(sum).operand(side)); + TypeID type = terms[side] && terms[side].isRecord() ? walker.typeOf(terms[side]) : TypeID(); + if (type && types.isSubset(type, TypeTable::int32)) + continue; + if (!type || optionalTerm || !types.overlap(type, TypeTable::int32) || !types.isSubset(type, types.withPrimitives(TypeTable::int32, Primitive::Undefined))) + return { }; + optionalTerm = side; + } + if (!optionalTerm) + return { }; + + Reference& term = terms[*optionalTerm]; + Draft test(Opcode::TypeTest, origin); + test.setValueOperand(term); + test.setTestedType(TypeArena::fromPrimitives(Primitive::Undefined)); + Reference isUndefined = walker.build(WTF::move(test)); + if (!isUndefined || !isUndefined.isValue() || walker.code().isConstant(isUndefined.id()) || walker.isUnreachable()) + return { }; + walker.count(StatisticID::SumsWithUndefinedTruncated); + BlockID undefinedBlock = walker.addBlock(BlockFrequency::Unlikely); + BlockID integerBlock = walker.addBlock(); + BlockID meeting = walker.addBlock(); + walker.branch(isUndefined, undefinedBlock, integerBlock); + + walker.continueIn(integerBlock); + term = buildNarrowedInNumberForms(walker, term, isUndefined, OperandIndex::TypeTest::value, false, origin); + Reference added = term ? buildOperatorInNumberForms(walker, index.opcodeOf(sum), terms[0], terms[1], origin) : Reference(); + Reference bits = added ? buildConversionInNumberOperators(walker, Opcode::ToInt32, added, origin) : Reference(); + if (!bits) + return { }; + std::array ways { WayInNumberForms { walker.currentBlock(), bits, { } }, WayInNumberForms { } }; + walker.continueIn(undefinedBlock); + ways[1] = WayInNumberForms { walker.currentBlock(), integerInNumberForms(walker, 0), { } }; + return meetInNumberForms(walker, meeting, ways); +} + static Reference buildSimplerOperandOfBitOperator(Walker& walker, Origin origin, ValueID written) { + if (Reference bits = buildSumWithUndefined(walker, origin, written)) + return bits; + if (walker.hasFailed() || walker.isUnreachable()) + return { }; InputsOfTypedOperation inputs(walker, nullptr); if (std::optional quotient = findQuotientOfIntegers(walker, written, inputs)) return buildDivisionOfIntegers(walker, origin, walker.referenceOf(quotient->dividend), walker.referenceOf(quotient->divisor), QuotientRounding::ToInt32); @@ -193,6 +247,35 @@ static Reference buildBitsOfNumberInNumberOperators(Walker& walker, Opcode conve return buildConversionInNumberOperators(walker, conversion, number, origin); } +static bool isBoxedNumberOrNullish(Walker& walker, Reference operand) +{ + Type type = walker.types().at(walker.typeOf(operand)); + return operand.isValue() && !type.isNumber() && (type.primitives & Primitive::Integer) && type.hasOnlyPrimitives(Primitive::Number | Primitive::UndefinedOrNull) + && walker.code().at(operand.id()).representation == Representation::JSValue; +} + +static Reference buildBitsOfNumberOrNullish(Walker& walker, Reference operand, Origin origin) +{ + Reference number; + BlockID whereNullish = walker.splitNumberOffContainer(operand, number); + if (!whereNullish && !walker.hasFailed()) { + Reference isNumber = buildTestOfNumber(walker, operand, walker.types().withoutNullish(walker.typeOf(operand)), origin); + if (!isNumber || !isNumber.isValue()) + return { }; + whereNullish = walker.addBlock(BlockFrequency::Rare); + continueIfTrue(walker, isNumber, whereNullish); + number = buildProofOfNumber(walker, operand, isNumber, origin); + } + Reference bits = number ? buildConversionInNumberOperators(walker, Opcode::ToInt32, number, origin) : Reference(); + if (!bits) + return { }; + WayInNumberForms underNumber { walker.currentBlock(), bits, { } }; + walker.continueIn(whereNullish); + WayInNumberForms underNullish { walker.currentBlock(), walker.constant(PrimitiveKnownToFolding::number(0)), { } }; + std::array ways { underNumber, underNullish }; + return meetInNumberForms(walker, walker.addBlock(), ways); +} + static bool convertOperandsOfBitOperatorOnce(Walker& walker, Opcode opcode, Origin origin, std::span operands) { TypeArena& types = walker.types(); @@ -209,7 +292,13 @@ static bool convertOperandsOfBitOperatorOnce(Walker& walker, Opcode opcode, Orig continue; if (!walker.isInLoop() && opcode == Opcode::BitAnd && isMaskOfLowestBitsOfInt32(types.at(walker.typeOf(operands[1 - operandIndex])))) continue; + if (opcode != Opcode::BitNot && !type.isNumber() && walker.representationOf(number) == Representation::JSValue && type.hasOnlyPrimitives(Primitive::Number | Primitive::UndefinedOrNull)) + continue; hasConverted = true; + if (walker.isInLoop() && walker.opensGuesses() && isBoxedNumberOrNullish(walker, number)) { + operands[operandIndex] = buildBitsOfNumberOrNullish(walker, number, origin); + continue; + } if (!type.isNumber()) number = buildOperatorInNumberForms(walker, Opcode::ToNumber, number, { }, origin); if (!number) { diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkNumberQuotients.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkNumberQuotients.cpp index 53b101f2a500a..1a9c24e4d0496 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkNumberQuotients.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkNumberQuotients.cpp @@ -310,6 +310,53 @@ static bool codeComesAgainInQuotients(const Walker& walker) return !walker.program().at(function).isModuleBody(); } +namespace { + +struct TestOfRemainder { + ValueID test; + std::optional comparedWith; +}; + +} // anonymous namespace + +static std::optional findTestInQuotients(Walker& walker, const BodyValue& remainder, std::optional divisor) +{ + if (!divisor || *divisor < 2 || *divisor > std::numeric_limits::max()) + return std::nullopt; + const BodyIndex& index = walker.currentBodyIndex(); + std::optional firstMention = index.firstUser(remainder.id()); + if (!firstMention || firstMention->kind != BodyUseKind::Operand) + return std::nullopt; + BodyValue test = index.value(firstMention->value); + BodyBlock block = index.block(index.blockOf(remainder.id())); + if (!block.holds(test.id()) || test.id().index() >= block.firstValue().index() + walker.endOfStretchBeingWalked()) + return std::nullopt; + std::optional unreadProofs = numberOfUnreadProofsBehindTestInQuotients(index, test.id(), remainder.id()); + if (!unreadProofs || 1 + *unreadProofs >= BodyIndex::maxCountedUses || index.useCount(remainder.id()) != 1 + *unreadProofs) + return std::nullopt; + + if (test.opcode() == Opcode::ToBoolean) + return TestOfRemainder { test.id(), std::nullopt }; + if (test.opcode() != Opcode::StrictEq && test.opcode() != Opcode::Eq) + return std::nullopt; + ValueID other = test.operand(0) == remainder.id() ? test.operand(1) : test.operand(0); + if (!other || index.isParameter(other) || index.opcodeOf(other) != Opcode::Integer) + return std::nullopt; + return TestOfRemainder { test.id(), index.value(other).as().integer() }; +} + +ValueID findTestOfRemainder(Walker& walker, const BodyValue& remainder) +{ + if (remainder.opcode() != Opcode::Mod || !walker.opensGuesses() || !walker.isInLoop()) + return { }; + const BodyIndex& index = walker.currentBodyIndex(); + ValueID divisor = index.stripCasts(remainder.operand(1)); + if (!divisor || index.isParameter(divisor) || index.opcodeOf(divisor) != Opcode::Integer) + return { }; + std::optional found = findTestInQuotients(walker, remainder, index.value(divisor).as().integer()); + return found ? found->test : ValueID(); +} + bool buildNumberRemainderWithItsTest(Walker& walker, const BodyValue& remainder) { if (remainder.opcode() != Opcode::Mod || !walker.opensGuesses() || !codeComesAgainInQuotients(walker)) @@ -318,35 +365,20 @@ bool buildNumberRemainderWithItsTest(Walker& walker, const BodyValue& remainder) ValueID writtenDivisor = remainder.operand(1); Reference dividend = walker.referenceOf(writtenDividend); Reference divisor = walker.referenceOf(writtenDivisor); - std::optional knownDivisor = integerKnownInNumberForms(walker, divisor); - if (!knownDivisor || *knownDivisor < 2 || *knownDivisor > std::numeric_limits::max() || !dividend.isValue()) + if (!dividend.isValue()) return false; Type typeOfDividend = typeOfBuiltInQuotients(walker, dividend); if (!typeOfDividend.isNumber() || typeOfDividend.isInteger() || !(typeOfDividend.primitives & Primitive::Integer)) return false; - const BodyIndex& index = walker.currentBodyIndex(); - std::optional firstMention = index.firstUser(remainder.id()); - if (!firstMention || firstMention->kind != BodyUseKind::Operand) + std::optional found = findTestInQuotients(walker, remainder, integerKnownInNumberForms(walker, divisor)); + if (!found) return false; - BodyValue test = index.value(firstMention->value); + const BodyIndex& index = walker.currentBodyIndex(); + BodyValue test = index.value(found->test); BodyBlock block = index.block(index.blockOf(remainder.id())); uint32_t firstValue = block.firstValue().index(); - if (!block.holds(test.id()) || test.id().index() >= firstValue + walker.endOfStretchBeingWalked()) - return false; - std::optional unreadProofs = numberOfUnreadProofsBehindTestInQuotients(index, test.id(), remainder.id()); - if (!unreadProofs || 1 + *unreadProofs >= BodyIndex::maxCountedUses || index.useCount(remainder.id()) != 1 + *unreadProofs) - return false; - - std::optional comparedWith; - if (test.opcode() != Opcode::ToBoolean) { - if (test.opcode() != Opcode::StrictEq && test.opcode() != Opcode::Eq) - return false; - ValueID other = test.operand(0) == remainder.id() ? test.operand(1) : test.operand(0); - if (!other || index.isParameter(other) || index.opcodeOf(other) != Opcode::Integer) - return false; - comparedWith = index.value(other).as().integer(); - } + std::optional comparedWith = found->comparedWith; TypeID whole = walker.types().integer(Range::exact()); if (walker.types().testablePartOf(whole) != whole) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkNumbers.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkNumbers.cpp index 3581f23cc317a..c08307c35e30c 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkNumbers.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkNumbers.cpp @@ -112,6 +112,32 @@ static Reference buildRemainderByReciprocalMadeInFrontOfLoop(Walker& walker, con return built; } +static Reference buildRemainderByWholeDivisor(Walker& walker, const Draft& remainder) +{ + std::optional loop = walker.has(Mechanism::LoopInvariants) ? walker.innermostOpenLoop() : std::nullopt; + Reference dividend = remainder.operand(OperandIndex::Mod::left); + Reference divisor = remainder.operand(OperandIndex::Mod::right); + if (!loop || !divisor.isValue() || !walker.standsInFrontOfOpenLoop(*loop, divisor) || walker.standsInFrontOfOpenLoop(*loop, dividend)) + return { }; + const Type& typeOfDivisor = walker.types().at(walker.typeOf(divisor)); + if (!walker.types().at(walker.typeOf(dividend)).isNumber() || !typeOfDivisor.isNumber() || typeOfDivisor.isInteger()) + return { }; + + Draft inFront(Opcode::WholeDivisor, remainder.origin()); + walker.chooseConservativeFields(inFront); + inFront.setDivisorOperand(divisor); + Reference wholeDivisor = walker.build(WTF::move(inFront)); + if (!wholeDivisor || !walker.standsInFrontOfOpenLoop(*loop, wholeDivisor)) + return { }; + + Draft byWholeDivisor(Opcode::RemainderByWholeDivisor, remainder.origin()); + walker.chooseConservativeFields(byWholeDivisor); + byWholeDivisor.setDividendOperand(dividend); + byWholeDivisor.setDivisorOperand(divisor); + byWholeDivisor.setWholeDivisorOperand(wholeDivisor); + return walker.build(WTF::move(byWholeDivisor)); +} + bool buildOperatorOnNumbers(Walker& walker, const BodyValue& value, const InputsOfTypedOperation& inputs) { OperatorForm form = operatorFormFor(walker, value.opcode(), value, inputs); @@ -132,8 +158,12 @@ bool buildOperatorOnNumbers(Walker& walker, const BodyValue& value, const Inputs if (sumStaysBoxed(walker, value, draft)) draft.representation = Representation::JSValue; Origin origin = draft.origin(); - Reference byReciprocal = value.opcode() == Opcode::Mod && form == OperatorForm::Integers ? buildRemainderByReciprocalMadeInFrontOfLoop(walker, draft) : Reference(); - walker.bind(value.id(), boxNumberIfNeeded(walker, byReciprocal ? byReciprocal : walker.build(WTF::move(draft)), origin)); + Reference remainder; + if (value.opcode() == Opcode::Mod && form == OperatorForm::Integers) + remainder = buildRemainderByReciprocalMadeInFrontOfLoop(walker, draft); + else if (value.opcode() == Opcode::Mod && form == OperatorForm::Numbers) + remainder = buildRemainderByWholeDivisor(walker, draft); + walker.bind(value.id(), boxNumberIfNeeded(walker, remainder ? remainder : walker.build(WTF::move(draft)), origin)); return true; } diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkReplacements.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkReplacements.cpp index 2a13e8723e2c2..bf78e43e48a97 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkReplacements.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkReplacements.cpp @@ -1009,8 +1009,10 @@ Walker::KeyOfAccess Walker::tellKeyOfAccessApartByItsTexts(const BodyValue& writ return asked.isJavaScriptValue() && !asked.hasObjects() && asked.primitives == Primitive::LiteralString && asked.stringLiterals; }; Type type = m_types.at(proven); + if (!(type.primitives & Primitive::AnyString)) + return KeyOfAccess::IsAnyValue; bool textsAreOnlyExpected = false; - if (!holdsWrittenTextsAlone(type) && (written.opcode() == Opcode::ElemGet || written.opcode() == Opcode::ElemSet) && isInLoop() && m_opensGuesses && has(Mechanism::Guesses)) { + if (!holdsWrittenTextsAlone(type) && (written.opcode() == Opcode::ElemGet || written.opcode() == Opcode::ElemSet) && isInRepeatedCode() && m_opensGuesses && has(Mechanism::Guesses)) { TypeID expected = findExpectation(*this, written.propertyOperand()).type; if (!expected || m_types.withoutNullish(expected) == TypeID(TypeTable::string)) expected = textsThatNamesOfHolderSayOfKey(*this, written); @@ -1058,7 +1060,7 @@ Walker::KeyOfAccess Walker::tellKeyOfAccessApartByItsTexts(const BodyValue& writ end = place + 1; } } - if (!key.isValue() || !has(Mechanism::Guesses) || !m_opensGuesses || !isInLoop() || end > endOfStretchBeingWalked()) + if (!key.isValue() || !has(Mechanism::Guesses) || !m_opensGuesses || !isInRepeatedCode() || end > endOfStretchBeingWalked()) return KeyOfAccess::IsAnyValue; auto typeOfInput = [&](ValueID input) { diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkStableBindings.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkStableBindings.cpp index 3d53be21daa38..18e1304f16808 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkStableBindings.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkStableBindings.cpp @@ -58,6 +58,51 @@ static bool variableStaysAsFilled(const Walker& walker, VariableID id, Definitio return !!plan.deferredDefinitions().findDeferred(id) == (definition == DefinitionOfStableVariable::IsPutOff); } +bool Walker::framesKeepFields() const +{ + const Frame& last = m_frames.last(); + if (!last.keepsFields) { + last.keepsFields = std::ranges::none_of(m_frames, [&](const Frame& frame) { + const Function& function = program().at(frame.function); + return function.isCoroutine() || function.constructedClass || function.kind == FunctionKind::Initializer; + }); + } + return *last.keepsFields; +} + +bool Walker::isStableFieldOf(LayoutSetID receivers, PropertyKeyID key) const +{ + auto isStableIn = [&](LayoutID member) { + std::optional field = plan().slots().findField(program(), member, key); + return field && plan().slots().slot(member, *field).isStable; + }; + std::span members = membersOf(m_types, receivers); + if (members.size() <= 1) + return !members.empty() && isStableIn(members.front()); + if (m_types.keeps() != ArenaKeeps::UntilReset) + return std::ranges::all_of(members, isStableIn); + constexpr uint8_t question = static_cast(AnswerKeptInArena::FieldOfLayoutsIsStable); + TypeArena::SetOfRound round = m_types.setOfRound(receivers); + if (std::optional kept = m_types.rememberedRound(question, 1, round, key.bits())) + return (*kept)[0]; + bool isStable = std::ranges::all_of(members, isStableIn); + m_types.rememberRound(question, 1, 0, round, key.bits(), { isStable }); + return isStable; +} + +bool Walker::readsStableField(const Draft& read) const +{ + if (read.level() != KnowledgeLevel::Proven || read.holdsDeclaredType() || read.receiverMayBeInBirth() || !isStableFieldOf(read.receivers(), read.key()) || testsItsOwnGuess(read)) + return false; + Reference object = read.objectOperand(); + if (!object || !object.isValue() || !framesKeepFields()) + return false; + const Operation& holder = m_code.at(underlyingCellOf(m_code, object.id())); + if (resultRuleOf(holder.opcode) == ResultRule::Allocation || interpretedOf(holder.opcode) == Interpretation::Allocation) + return false; + return holder.opcode != Opcode::Parameter || (holder.block() && m_code.at(holder.block()).isEntry); +} + bool numberReadOutOfVariableStays(const Walker& walker, VariableID variable) { return variableStaysAsFilled(walker, variable); @@ -74,6 +119,21 @@ static bool keepsNotesOfStableBindings(const Walker& walker) return walker.has(Mechanism::Folding) && walker.state().remembersPlaces(); } +Reference Walker::valueOfStableField(const Draft& read, TypeID proven) const +{ + if (!keepsNotesOfStableBindings(*this)) + return { }; + std::optional noted = findKnown(*this, KnownNote::ValueKeptByPlace, Reference::value(underlyingCellOf(m_code, read.objectOperand().id())), { }, read.key().index()); + Reference kept = noted ? Reference::fromBits(static_cast(*noted)) : Reference(); + return kept && m_types.isSubset(typeOf(kept), proven) ? kept : Reference(); +} + +void Walker::noteStableField(const Draft& read, Reference value) +{ + if (keepsNotesOfStableBindings(*this)) + noteKnown(*this, KnownNote::ValueKeptByPlace, Reference::value(underlyingCellOf(m_code, read.objectOperand().id())), { }, read.key().index(), LocationMasks(), value.bits(), true); +} + static ALWAYS_INLINE bool isValueOfFilledBinding(Walker& walker, Reference value, bool hasDeadZone) { TypeID type = value ? walker.typeOf(value) : TypeID(); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkState.h b/Source/JavaScriptCore/aot/walk/AOTWalkState.h index 2f45784ea9b5b..43722a3f88c7a 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkState.h +++ b/Source/JavaScriptCore/aot/walk/AOTWalkState.h @@ -304,7 +304,7 @@ class WalkState { functor(m_held->m_holders[holder].object); } } - void forEachEnvironmentFieldNotSharedWithAll(std::span ways, NOESCAPE const Invocable auto& functor) const + void forEachFieldNotSharedWithAll(std::span ways, NOESCAPE const Invocable auto& functor) const { if (!mayHold(Kinds::places) || !m_held->m_numberOfPlaces) return; @@ -317,7 +317,7 @@ class WalkState { m_held->m_places.forEachStretchNotSharedWith(way->m_held->m_places, [&](uint32_t begin, uint32_t end) { for (uint32_t position = begin; position < end; ++position) { const Place& place = m_held->m_places[position]; - if (place.value && place.kind == PlaceKind::EnvironmentField) + if (place.value && (place.kind == PlaceKind::EnvironmentField || place.kind == PlaceKind::Slot)) positions.append(position); } }); @@ -330,7 +330,7 @@ class WalkState { const Place& place = m_held->m_places[position]; const Holder& holder = m_held->m_holders[place.holder]; if (!holder.hasEnded && holder.object) - functor(holder.object, FieldReference { LayoutID(place.first), place.second }, place.value); + functor(holder.object, FieldReference { LayoutID(place.first), place.second }, place.value, place.kind == PlaceKind::Slot); } } void forEachBeliefOfFreshHolderLackingIn(const WalkState& arriving, NOESCAPE const Invocable fieldOfScope, Reference believed)> auto& functor) const diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkStringCharacters.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkStringCharacters.cpp index 45b2e111eeb4c..83eeff6706e8f 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkStringCharacters.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkStringCharacters.cpp @@ -312,7 +312,10 @@ bool BuilderOfCharacterRead::build(Reference position) return true; } - if (std::optional piece = pieceInSight(m_walker, m_string)) { + std::optional piece = pieceReadInLoop(m_walker, m_string, m_origin); + if (!piece) + piece = pieceInSight(m_walker, m_string); + if (piece) { m_stringRead = piece->string; m_startOfPiece = piece->start; m_length = operate(Opcode::Sub, piece->end, piece->start); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkStringComparisons.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkStringComparisons.cpp index 12decdfac55d4..6a2d2b518e346 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkStringComparisons.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkStringComparisons.cpp @@ -262,7 +262,10 @@ static bool compareSubjectWithTextForStringComparisons(Walker& walker, const Bod bool subjectIsString = isProvenString(walker, walker.typeOf(subject)); if (narrowsNothing && (subjectIsString || (value.opcode() == Opcode::StrictEq && lengthOfTextForStrings(walker, text)))) { - if (std::optional piece = pieceInSight(walker, subject, !subjectIsString)) { + std::optional piece = pieceReadInLoop(walker, subject, origin); + if (!piece) + piece = pieceInSight(walker, subject, !subjectIsString); + if (piece) { if (Reference answer = comparePieceWithTextForStringComparisons(walker, *piece, text, origin)) { walker.bind(value.id(), answer); return true; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkStringFlattening.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkStringFlattening.cpp index efa3a34dbf942..763578fe00936 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkStringFlattening.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkStringFlattening.cpp @@ -47,17 +47,26 @@ static Draft draftOfFlatStringForFlattening(Reference string, Origin origin) return flatten; } -static Reference flattenInFrontOfLoopForFlattening(Walker& walker, Reference string, Origin origin) +static std::optional loopToFlattenInFrontOfForFlattening(const Walker& walker, Reference string) { const auto& here = walker.code().at(walker.currentBlock()); if (!walker.has(Mechanism::LoopInvariants) || here.frequency == BlockFrequency::Rare || here.handler) - return { }; - std::optional loop = walker.outermostLoopToMakeTextInFrontOf(singleElementSpan(string), declaredBoundOf(Opcode::FlattenString)); + return std::nullopt; + return walker.outermostLoopToMakeTextInFrontOf(singleElementSpan(string), declaredBoundOf(Opcode::FlattenString)); +} + +static Reference flattenInFrontOfLoopForFlattening(Walker& walker, Reference string, Origin origin) +{ + std::optional loop = loopToFlattenInFrontOfForFlattening(walker, string); if (!loop) return { }; Draft flatten = draftOfFlatStringForFlattening(string, origin); - if (Reference begins = walker.buildLoopEntryCondition(*loop)) - flatten.setIsReadOperand(begins); + Vector conditions; + if (!walker.tryBuildFirstArrivalConditions(*loop, conditions) && walker.innermostOpenLoop() == loop && !walker.isFlattenedBeforeLoop(*loop, string, conditions.span())) + flatten.setMayStayRope(true); + if (walker.hasFailed()) + return { }; + flatten.setIsReadOperands(conditions.span()); return walker.buildInFrontOfOpenLoop(*loop, WTF::move(flatten)); } @@ -71,6 +80,13 @@ Reference flatStringInSight(Walker& walker, Reference string) return flat && walker.isInSightWhereWalkBuilds(flat) ? flat : Reference(); } +bool isOrCanBeFlattenedBeforeLoop(Walker& walker, Reference string) +{ + if (walker.isUnreachable() || !string.isValue()) + return false; + return flatStringInSight(walker, string) || loopToFlattenInFrontOfForFlattening(walker, string); +} + Reference flatStringMadeWhereWalkStands(Walker& walker, Reference string, Origin origin) { if (walker.isUnreachable() || !string.isValue() || !isProvenString(walker, walker.typeOf(string))) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkStringOperandsAsMade.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkStringOperandsAsMade.cpp index 97c8db0a1ce49..d8db985873318 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkStringOperandsAsMade.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkStringOperandsAsMade.cpp @@ -107,9 +107,67 @@ static ValueID findLastCallInRowOfCallsOfMethodsOfTable(Walker& walker, const Bo } } +static bool passesCallbackToCopy(Walker& walker, const BodyValue& call) +{ + const BodyIndex& index = walker.currentBodyIndex(); + auto arguments = call.as().argumentsOperands(); + ValueID callback = arguments.empty() ? ValueID() : index.stripCasts(arguments[0]); + FunctionID function; + if (callback && !index.isParameter(callback) && index.opcodeOf(callback) == Opcode::NewClosure) + function = index.value(callback).function(); + else if (callback && !index.isParameter(callback) && index.opcodeOf(callback) == Opcode::GetModuleVariable) { + const Variable& variable = walker.program().at(index.value(callback).as().variable()); + if (variable.initialization == VariableInitialization::Function && variable.flags.contains(VariableFlags::IsImmutable)) + function = variable.function; + } + return function && walker.facts().linked().function(function).weightAsLeaf != LinkedFunction::notLeaf; +} + +static bool argumentsStayInsideCall(const BodyIndex& index, ValueID read, ValueID call) +{ + for (uint32_t number = read.index() + 1; number < call.index(); ++number) { + ValueID user = index.onlyUsingValue(ValueID(number)); + if (!user || call < user) + return false; + } + return true; +} + +ValueID findLastCallInMethodChain(Walker& walker, const BodyValue& firstCall, uint32_t behindStretch) +{ + const Program& program = walker.program(); + const BuiltinTable& table = program.builtins(); + const BodyIndex& index = walker.currentBodyIndex(); + BodyBlock block = index.block(index.blockOf(firstCall.id())); + ValueID last = firstCall.id(); + if (!passesCallbackToCopy(walker, firstCall)) + return last; + for (;;) { + ValueID result = last; + uint32_t next = result.index() + 1; + while (next < behindStretch && index.opcodeOf(ValueID(next)) == Opcode::Convert && index.value(ValueID(next)).operand(0) == result && index.onlyUsingValue(result) == ValueID(next)) + result = ValueID(next++); + if (next >= behindStretch || index.opcodeOf(ValueID(next)) != Opcode::PropGet || index.useCount(result) != 2) + return last; + BodyValue read = index.value(ValueID(next)); + BuiltinID member = !read.isFreeName() && read.key() && read.objectOperand() == result ? findOwnMember(program, table.at(BuiltinClassID(BuiltinClasses::Array)).prototype, read.key()) : BuiltinID(); + bool hasLibraryImplementation = member && findLibraryFunctionsBehindBuiltin(walker, walker.types().forBuiltin(table.canonicalID(member))).enteredByValue; + ValueID callID = hasLibraryImplementation ? index.onlyUsingValue(read.id()) : ValueID(); + if (!callID || index.isParameter(callID) || !block.holds(callID) || index.opcodeOf(callID) != Opcode::Call || callID.index() >= behindStretch) + return last; + BodyValue call = index.value(callID); + if (call.calleeOperand() != read.id() || call.thisValueOperand() != result || !passesCallbackToCopy(walker, call) || !argumentsStayInsideCall(index, read.id(), callID)) + return last; + last = callID; + } +} + bool openTestOfListAtReadOfMethodOfItsOpcodes(Walker& walker, const BodyValue& read) { - if (read.isFreeName() || !walker.isInLoop() || walker.walksStretchInsideAnother() || walker.isUnreachable() || !walker.has(Mechanism::Inlining) || !walker.has(Mechanism::BuiltinCalls) || !walker.has(Mechanism::EscapeAnalysis)) + if (read.isFreeName() || !walker.isInLoop() || walker.isUnreachable() || !walker.has(Mechanism::Inlining) || !walker.has(Mechanism::BuiltinCalls) || !walker.has(Mechanism::EscapeAnalysis)) + return false; + Walker::MethodChain& chain = walker.methodChainBeingWalked(); + if (walker.walksStretchInsideAnother() && (chain.depthOfCopies != walker.depthOfCopies() || !(chain.openRead < read.id()) || chain.lastCall < read.id())) return false; const Program& program = walker.program(); const BuiltinTable& table = program.builtins(); @@ -134,36 +192,23 @@ bool openTestOfListAtReadOfMethodOfItsOpcodes(Walker& walker, const BodyValue& r return false; BodyValue call = index.value(callID); ValueID receiverID = read.objectOperand(); - auto arguments = call.as().argumentsOperands(); if (call.calleeOperand() != read.id() || call.thisValueOperand() != receiverID) return false; - ValueID callbackArgument = hasLibraryImplementation && !arguments.empty() ? index.stripCasts(arguments[0]) : ValueID(); - FunctionID callbackFunction; - if (callbackArgument && !index.isParameter(callbackArgument) && index.opcodeOf(callbackArgument) == Opcode::NewClosure) - callbackFunction = index.value(callbackArgument).function(); - else if (callbackArgument && !index.isParameter(callbackArgument) && index.opcodeOf(callbackArgument) == Opcode::GetModuleVariable) { - const Variable& variable = program.at(index.value(callbackArgument).as().variable()); - if (variable.initialization == VariableInitialization::Function && variable.flags.contains(VariableFlags::IsImmutable)) - callbackFunction = variable.function; - } - if (hasLibraryImplementation && (!callbackFunction || walker.facts().linked().function(callbackFunction).weightAsLeaf == LinkedFunction::notLeaf)) + if ((hasLibraryImplementation && !passesCallbackToCopy(walker, call)) || !argumentsStayInsideCall(index, read.id(), callID)) return false; - for (uint32_t number = read.id().index() + 1; number < callID.index(); ++number) { - ValueID user = index.onlyUsingValue(ValueID(number)); - if (!user || callID < user) - return false; - } TypeID instances = types.builtinClass(builtinClass); if (!types.isSubset(proven, instances) || isKnownPristine(walker, receiver) || receiverStaysAsMadeByLink(walker, read)) return false; Vector readsOffHolders; - ValueID last = !hasLibraryImplementation ? findLastCallInRowOfCallsOfMethodsOfTable(walker, read, call, builtinClass, readsOffHolders) : callID; + uint32_t behindStretch = block.firstValue().index() + std::min(block.numberOfValues(), walker.endOfStretchBeingWalked()); + ValueID last = hasLibraryImplementation ? findLastCallInMethodChain(walker, call, behindStretch) : findLastCallInRowOfCallsOfMethodsOfTable(walker, read, call, builtinClass, readsOffHolders); if (last == callID) readsOffHolders.shrink(0); else - walker.count(StatisticID::CollectionCallRunsGuardedOnce); - if (hasLibraryImplementation && index.opcodeOf(callbackArgument) == Opcode::GetModuleVariable) + walker.count(hasLibraryImplementation ? StatisticID::MethodChainsKeptInArm : StatisticID::CollectionCallRunsGuardedOnce); + SetForScope walksChain(chain, Walker::MethodChain { walker.depthOfCopies(), read.id(), last == callID ? read.id() : last }); + if (hasLibraryImplementation && index.opcodeOf(index.stripCasts(call.as().argumentsOperands()[0])) == Opcode::GetModuleVariable) walker.count(StatisticID::ReceiversSpeculatedForNamedCallbacks); openGuessAtOperandTakenAsMade(walker, index.value(last), receiver, instances, read.id().index() - block.firstValue().index(), readsOffHolders.span()); return true; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkStringPieces.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkStringPieces.cpp index e2e9496d2a360..525e1b0f39b98 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkStringPieces.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkStringPieces.cpp @@ -103,6 +103,20 @@ static bool bodyStoresValueForStringPieces(Walker& walker, const BodyIndex& inde return stored.get(value.index()); } +static Reference buildPositionForStringPieces(Walker& walker, Opcode opcode, Reference left, Reference right, Origin origin) +{ + Reference result = buildOperatorForStrings(walker, opcode, left, right, origin); + return buildOperatorForStrings(walker, Opcode::BitAnd, result, integerForStrings(walker, std::numeric_limits::max()), origin); +} + +static void noteThatBoundsAreInOrderForStringPieces(Walker& walker, Reference start, Reference end) +{ + start = rootForStrings(walker, start); + end = rootForStrings(walker, end); + if (start.isValue() && end.isValue()) + noteKnown(walker, KnownNote::BoundsOfPieceAreInOrder, start, end, 0, LocationMasks(), 1); +} + bool buildPieceByMethod(Walker& walker, const CallByOneRow& call) { if (!call.receiver || call.arguments.size() > 2 || !isProvenString(walker, walker.typeOf(call.receiver))) @@ -133,7 +147,8 @@ bool buildPieceByMethod(Walker& walker, const CallByOneRow& call) Origin origin = call.origin; Reference string = call.receiver; - Reference length = buildLengthOfString(walker, string, origin); + std::optional outer = pieceReadInLoop(walker, string, origin); + Reference length = outer ? buildLengthOfPiece(walker, *outer, origin) : buildLengthOfString(walker, string, origin); if (!length) return true; Reference zero = integerForStrings(walker, 0); @@ -189,6 +204,12 @@ bool buildPieceByMethod(Walker& walker, const CallByOneRow& call) else if (rootForStrings(walker, start) == rootForStrings(walker, end)) piece = walker.constant(PrimitiveKnownToFolding::text(walker.program().strings.emptyString())); else { + if (outer) { + string = outer->string; + start = buildPositionForStringPieces(walker, Opcode::Add, outer->start, start, origin); + end = buildPositionForStringPieces(walker, Opcode::Add, outer->start, end, origin); + noteThatBoundsAreInOrderForStringPieces(walker, start, end); + } piece = buildPieceOfString(walker, string, start, end, origin); if (piece && piece.isValue() && bodyStoresValueForStringPieces(walker, walker.currentBodyIndex(), call.call.id())) noteKnown(walker, KnownNote::PieceIsStoredByBodyThatCutIt, piece, { }, 0, LocationMasks(), 1); @@ -258,6 +279,10 @@ static bool endIsSumWithStartForStringPieces(const Walker& walker, const PieceOf bool isRangeOfPieceInOrder(Walker& walker, const PieceOfString& piece) { + Reference rootOfStart = rootForStrings(walker, piece.start); + Reference rootOfEnd = rootForStrings(walker, piece.end); + if (rootOfStart.isValue() && rootOfEnd.isValue() && findKnown(walker, KnownNote::BoundsOfPieceAreInOrder, rootOfStart, rootOfEnd)) + return true; if (Reference range = rangeGivingBoundForStringPieces(walker, piece.start, startOfStringRange)) { if (range == rangeGivingBoundForStringPieces(walker, piece.end, endOfStringRange)) return true; @@ -270,6 +295,44 @@ bool isRangeOfPieceInOrder(Walker& walker, const PieceOfString& piece) return relationDecidedForStrings(walker, Opcode::LessEq, piece.start, piece.end).value_or(false) || boundsAreOfOnePart(walker, piece.start, piece.end) || (piece.end.isRecord() && boundIsShownBySumsForStringPieces(walker, piece.start, piece.end, walker.code().at(piece.end.id()).origin())); } +Reference buildLengthOfPiece(Walker& walker, const PieceOfString& piece, Origin origin) +{ + return buildPositionForStringPieces(walker, Opcode::Sub, piece.end, piece.start, origin); +} + +static std::optional orderedPieceForStringPieces(Walker& walker, Reference text) +{ + Reference root = rootForStrings(walker, text); + if (!root.isValue() || walker.isUnreachable()) + return std::nullopt; + Opcode opcode = walker.code().at(root.id()).opcode; + if (opcode != Opcode::Substring && opcode != Opcode::SubstringOrUndefined) + return std::nullopt; + std::optional piece = pieceInSight(walker, text); + if (!piece || !walker.types().at(walker.typeOf(piece->start)).isInteger() || !walker.types().at(walker.typeOf(piece->end)).isInteger() || !isRangeOfPieceInOrder(walker, *piece)) + return std::nullopt; + return piece; +} + +std::optional pieceReadInLoop(Walker& walker, Reference text, Origin origin) +{ + if (walker.isUnreachable() || !walker.isInLoop() || isOrCanBeFlattenedBeforeLoop(walker, text)) + return std::nullopt; + std::optional piece = orderedPieceForStringPieces(walker, text); + while (piece && !isOrCanBeFlattenedBeforeLoop(walker, piece->string)) { + std::optional outer = orderedPieceForStringPieces(walker, piece->string); + if (!outer) + break; + Reference start = buildPositionForStringPieces(walker, Opcode::Add, outer->start, piece->start, origin); + Reference end = start ? buildPositionForStringPieces(walker, Opcode::Add, outer->start, piece->end, origin) : Reference(); + if (!end) + break; + noteThatBoundsAreInOrderForStringPieces(walker, start, end); + piece = PieceOfString { outer->string, start, end }; + } + return piece; +} + bool buildLengthOfPieceInSight(Walker& walker, const BodyValue& value, const InputsOfTypedOperation& inputs) { if (inputs.areOnlyTypes() || value.isFreeName() || value.key() != walker.program().keyOf(BuiltinNames::length) || walker.isUnreachable()) @@ -282,17 +345,18 @@ bool buildLengthOfPieceInSight(Walker& walker, const BodyValue& value, const Inp std::optional piece = pieceInSight(walker, text); if (!piece || !isRangeOfPieceInOrder(walker, *piece)) return false; - Reference difference = buildOperatorForStrings(walker, Opcode::Sub, piece->end, piece->start, origin); - Reference length = buildOperatorForStrings(walker, Opcode::BitAnd, difference, integerForStrings(walker, std::numeric_limits::max()), origin); + Reference length = buildLengthOfPiece(walker, *piece, origin); if (length) bindForStrings(walker, value.id(), length, origin); return true; } -static std::optional pieceToReadNumberOffForStringPieces(Walker& walker, Reference text) +static std::optional pieceToReadNumberOffForStringPieces(Walker& walker, Reference text, Origin origin) { if (!text || !isProvenString(walker, walker.typeOf(text))) return std::nullopt; + if (std::optional piece = pieceReadInLoop(walker, text, origin)) + return piece; std::optional piece = pieceInSight(walker, text); if (!piece || !isRangeOfPieceInOrder(walker, *piece)) return std::nullopt; @@ -303,10 +367,10 @@ bool buildNumberOfPiece(Walker& walker, const BodyValue& value, const InputsOfTy { if (inputs.areOnlyTypes() || walker.isUnreachable()) return false; - std::optional piece = pieceToReadNumberOffForStringPieces(walker, walker.referenceOf(value.operand(0))); + Origin origin = walker.originOf(value); + std::optional piece = pieceToReadNumberOffForStringPieces(walker, walker.referenceOf(value.operand(0)), origin); if (!piece) return false; - Origin origin = walker.originOf(value); Reference number = buildNumberOfRange(walker, *piece, origin); if (!number) return false; @@ -318,7 +382,7 @@ bool buildNumberOfPieceByCall(Walker& walker, const CallByOneRow& call) { if (call.arguments.empty() || call.arguments.size() > 2) return false; - std::optional piece = pieceToReadNumberOffForStringPieces(walker, call.arguments[0]); + std::optional piece = pieceToReadNumberOffForStringPieces(walker, call.arguments[0], call.origin); if (!piece) return false; @@ -391,7 +455,7 @@ static std::optional pieceThatIsKeyForStringPieces(Walker& walker { if (!walker.opensGuesses() || !walker.has(Mechanism::Folding) || call.isForwarded || !call.receiver || call.arguments.size() != numberOfArguments || call.writtenArguments().size() != numberOfArguments) return std::nullopt; - std::optional piece = pieceToReadNumberOffForStringPieces(walker, call.arguments[0]); + std::optional piece = pieceToReadNumberOffForStringPieces(walker, call.arguments[0], call.origin); if (!piece || !walker.types().at(walker.typeOf(piece->start)).isInteger() || !walker.types().at(walker.typeOf(piece->end)).isInteger()) return std::nullopt; return piece; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkStringSearches.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkStringSearches.cpp index 39f34ed34fbd2..98b8de87fb324 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkStringSearches.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkStringSearches.cpp @@ -111,11 +111,11 @@ bool buildTestOfEndOfString(Walker& walker, const CallByOneRow& call) return true; } -static Reference buildStartOfSearchForStringSearches(Walker& walker, Reference string, Reference position, Origin origin) +static Reference buildStartOfSearchForStringSearches(Walker& walker, Reference string, const std::optional& piece, Reference position, Origin origin) { if (position == integerForStrings(walker, 0)) return position; - Reference length = buildLengthOfString(walker, string, origin); + Reference length = piece ? buildLengthOfPiece(walker, *piece, origin) : buildLengthOfString(walker, string, origin); if (!length) return { }; if (Reference clamped = clampedPositionForStrings(walker, position, length, false, origin); clamped && walker.types().at(walker.typeOf(clamped)).isInt32()) @@ -129,14 +129,27 @@ static Reference buildStartOfSearchForStringSearches(Walker& walker, Reference s return buildResultOfRangeForStrings(walker, walker.build(WTF::move(range)), startOfStringRange, origin); } -static Reference buildSearchForWrittenTextForStringSearches(Walker& walker, Reference string, StringID text, Reference position, bool searchesFromEnd, Origin origin) +static std::optional pieceToSearchForStringSearches(Walker& walker, Reference string, StringID text, Origin origin) +{ + Vector buffer; + StringView view = walker.types().viewOf(text, buffer); + if (!view.is8Bit() && !WTF::charactersAreAllLatin1(view.span16())) + return std::nullopt; + return pieceReadInLoop(walker, string, origin); +} + +static Reference buildSearchForWrittenTextForStringSearches(Walker& walker, Reference string, const std::optional& piece, StringID text, Reference position, bool searchesFromEnd, Origin origin) { if (walker.isUnreachable() || !position) return { }; std::optional unit = searchesFromEnd ? std::nullopt : onlyCodeUnitOfText(walker, text); Draft search(searchesFromEnd ? Opcode::StringLastIndexOfLiteral : unit ? Opcode::StringIndexOfCharacter : Opcode::StringIndexOfLiteral, origin); - search.setStringOperand(stringReadByStringSearches(walker, string, origin)); + search.setStringOperand(stringReadByStringSearches(walker, piece ? piece->string : string, origin)); search.setPositionOperand(position); + if (piece) { + search.setStartOperand(piece->start); + search.setEndOperand(piece->end); + } if (unit) search.setCodeUnit(*unit); else @@ -172,10 +185,12 @@ bool buildSearchInString(Walker& walker, const CallByOneRow& call) Type type = walker.types().at(walker.typeOf(call.arguments[1])); if (searchesFromEnd ? !type.isInt32() : !type.isNumber()) return false; - position = searchesFromEnd ? call.arguments[1] : buildStartOfSearchForStringSearches(walker, call.receiver, call.arguments[1], origin); } + std::optional piece = pieceToSearchForStringSearches(walker, call.receiver, sought, origin); + if (hasPosition) + position = searchesFromEnd ? call.arguments[1] : buildStartOfSearchForStringSearches(walker, call.receiver, piece, call.arguments[1], origin); - Reference found = buildSearchForWrittenTextForStringSearches(walker, call.receiver, sought, position, searchesFromEnd, origin); + Reference found = buildSearchForWrittenTextForStringSearches(walker, call.receiver, piece, sought, position, searchesFromEnd, origin); if (found && asksWhetherFound) found = buildNegationForStrings(walker, buildOperatorForStrings(walker, Opcode::StrictEq, found, integerForStrings(walker, -1), origin), origin); if (found) @@ -205,7 +220,8 @@ Reference buildSearchForAtomOfRegExp(Walker& walker, const BodyValue&, BuiltinOv return { }; StringID text = pattern == StringView(summary.atom) ? regExp.pattern : walker.types().internText(encodeProgramString(StringView(summary.atom)).span()); - Reference found = buildSearchForWrittenTextForStringSearches(walker, string, text, integerForStrings(walker, 0), false, origin); + std::optional piece = pieceToSearchForStringSearches(walker, string, text, origin); + Reference found = buildSearchForWrittenTextForStringSearches(walker, string, piece, text, integerForStrings(walker, 0), false, origin); if (!found) return { }; if (*codeInLine == InlineBuiltin::StringSearch) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkStrings.h b/Source/JavaScriptCore/aot/walk/AOTWalkStrings.h index 5302963eaf784..3a54ed8c144b1 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkStrings.h +++ b/Source/JavaScriptCore/aot/walk/AOTWalkStrings.h @@ -70,6 +70,8 @@ struct PieceOfString { }; std::optional pieceInSight(Walker&, Reference text, bool alsoWhereNothingMayStand = false); bool isRangeOfPieceInOrder(Walker&, const PieceOfString&); +Reference buildLengthOfPiece(Walker&, const PieceOfString&, Origin); +std::optional pieceReadInLoop(Walker&, Reference text, Origin); Reference codePointOfCharacterInSight(Walker&, Reference character); void noteCodePointBesideCharacter(Walker&, Reference character, Reference codePoint); Reference codeUnitOfCharCodeInSight(Walker&, Reference number); @@ -100,6 +102,7 @@ AOT_LINK_OF_WALK(buildTestAgainstJoinOfTexts, opcodesThatBuildTestAgainstJoinOfT Reference buildTestOfCodeUnitBesidePartOfJoin(Walker&, Reference string, Reference lengthOfPart, Reference holdsOfPart, char16_t unit, bool testsStart, Origin); ValueID findTestOfEndThatTakesJoinApart(Walker&, const BodyValue& join, Reference takenToHaveText); bool goesByOneRowOnOperandAsMade(Walker&, const BodyValue& call, BuiltinID, Reference value, TypeID behindTest, BuiltinOverloadID* onlyRow = nullptr); +ValueID findLastCallInMethodChain(Walker&, const BodyValue& firstCall, uint32_t behindStretch); void openGuessAtOperandTakenAsMade(Walker&, const BodyValue& call, Reference value, TypeID tested, std::optional firstPlace = std::nullopt, std::span readsOffHolders = { }); struct NotesOfStrings final : Walker::KeptForBuild { @@ -152,6 +155,7 @@ bool nameProvesPositionBelowLengthForStrings(const Walker&, Reference position, Reference flatStringForReaderOfCharacters(Walker&, Reference string, Origin); Reference flatStringInSight(Walker&, Reference string); +bool isOrCanBeFlattenedBeforeLoop(Walker&, Reference string); Reference flatStringMadeWhereWalkStands(Walker&, Reference string, Origin); } } } // namespace JSC::AOT::Typed diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkTailCalls.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkTailCalls.cpp index dfc8a23740db4..6db9883c254c7 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkTailCalls.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkTailCalls.cpp @@ -17,6 +17,7 @@ #include "AOTSweep.h" #include "AOTTailCallPlan.h" #include "AOTThinSummary.h" +#include #include #include @@ -59,6 +60,13 @@ TailCallRequirement requirementOfTailCall(const Walker& walker, Reference call) if (!call || !call.isValue()) return TailCallRequirement::Required; const Operation& record = walker.code().at(call.id()); + if (record.opcode == Opcode::CallTable) { + const CallingConventions& conventions = walker.plan().conventions(); + bool everyMethodEnds = std::ranges::all_of(conventions.familyOf(conventions.methodSelector(record.selector())).methods, [&](FunctionID method) { + return walker.plan().endsItsTailCalls(method); + }); + return everyMethodEnds ? TailCallRequirement::Allowed : TailCallRequirement::Required; + } if (record.opcode != Opcode::CallDirect) return TailCallRequirement::Required; std::span arguments; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkTailCallsPastCleanups.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkTailCallsPastCleanups.cpp index 8e757115c326d..a7673f8f79c99 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkTailCallsPastCleanups.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkTailCallsPastCleanups.cpp @@ -54,16 +54,32 @@ ValueID iteratorClosedLeavingLoop(const BodyIndex& index, BodyBlock block, LoopI return iteratorClosedByHandler(index, block); } -bool Walker::handlerOnlyClosesIterations(const BodyIndex& index, BodyBlock block) const +bool Walker::handlerOnlyClosesIterations(Frame& frame, BodyBlock block) const { + const BodyIndex& index = *frame.index; + bool onlyClosesIterations = false; + Vector classified; for (BodyBlock covered = block;;) { + InputBlock& record = frame.blocks[covered.id().index()]; + if (record.handlerIsClassified) { + onlyClosesIterations = record.handlerOnlyClosesIterations; + break; + } + classified.append(covered.id()); if (!iteratorClosedByHandler(index, covered)) - return false; - BodyBlock cleanup = index.block(covered.handler().block()); - if (!cleanup.hasHandler()) - return true; - covered = cleanup; + break; + covered = index.block(covered.handler().block()); + if (!covered.hasHandler()) { + onlyClosesIterations = true; + break; + } + } + for (BlockID covered : classified) { + InputBlock& record = frame.blocks[covered.index()]; + record.handlerIsClassified = true; + record.handlerOnlyClosesIterations = onlyClosesIterations; } + return onlyClosesIterations; } bool Walker::throwsPastHandlerOf(BodyBlock block) const diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkTestsOfParameterGuards.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkTestsOfParameterGuards.cpp index 60392f7b199d4..0a5eb6935df9d 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkTestsOfParameterGuards.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkTestsOfParameterGuards.cpp @@ -83,6 +83,21 @@ static TypeID typeThatCastsOfOnlyMentionEnterForTestsOfGuards(TypeArena& types, return entered; } +TypeID withoutOmittedNullish(Walker& walker, FunctionID id, uint32_t entryParameter, TypeID type) +{ + const DeclaredType* declared = walker.program().at(id).declaredEntryParameter(entryParameter); + return type && declared ? withoutOmittedNullishForTestsOfGuards(walker.types(), type, *declared) : type; +} + +static TypeID withoutUndefinedOfElements(Walker& walker, FunctionID id, uint32_t entryParameter, TypeID seen) +{ + TypeArena& types = walker.types(); + const DeclaredType* declared = walker.program().at(id).declaredEntryParameter(entryParameter); + Type type = types.at(seen); + bool mayBeElement = !type.hasObjects() && (type.primitives & ~Primitive::UndefinedOrNull); + return mayBeElement && declared && declared->has(PositionFlags::OmitsUndefined) ? types.withoutPrimitives(seen, Primitive::Undefined) : seen; +} + static bool everyArrivalInSightFitsForTestsOfGuards(Walker& walker, FunctionID id, uint32_t entryParameter, TypeID tested, TypeID narrower) { const LinkTables& held = walker.facts().held(); @@ -98,7 +113,7 @@ static bool everyArrivalInSightFitsForTestsOfGuards(Walker& walker, FunctionID i return false; if (type.isAnything()) return true; - seen = types.intersectionOf(seen, tested); + seen = types.intersectionOf(withoutUndefinedOfElements(walker, id, entryParameter, seen), tested); if (types.withOnlyObjects(tested) == types.withOnlyObjects(narrower)) seen = types.withoutObjects(seen); return types.isSubset(seen, narrower); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkTextBuilderReads.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkTextBuilderReads.cpp index 1bba402a79610..c356259816954 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkTextBuilderReads.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkTextBuilderReads.cpp @@ -12,6 +12,8 @@ #include "AOTIRProgram.h" #include "AOTOpcodeSets.h" #include "AOTTypeArena.h" +#include "AOTWalkStrings.h" +#include namespace JSC { namespace AOT { namespace Typed { @@ -75,6 +77,34 @@ Reference buildTruthinessOfStringInBuilder(Walker& walker, StringInBuilder& text return buildTestOfLengthInTextBuilderReads(walker, WTF::move(hasCodeUnits)); } +static bool buildEqualityBehindLengthInTextBuilderReads(Walker& walker, const BodyValue& reader, StringInBuilder& text, Reference name, Reference written, Origin origin) +{ + StringID known = written ? textKnownForStrings(walker, written) : StringID(); + if (!known || operandsNarrowedBehindTestForStrings(walker, reader.id()) != std::optional(0)) + return false; + Reference length = lengthOfTextInTextBuilderReads(walker, text, name, origin); + if (!length) + return true; + Draft hasLength(Opcode::StrictEq, origin); + hasLength.setLeftOperand(length); + hasLength.setRightOperand(integerForStrings(walker, lengthOfTextForStrings(walker, known))); + Reference isOfSameLength = buildTestOfLengthInTextBuilderReads(walker, WTF::move(hasLength)); + std::array isEqual; + auto compare = [&](std::span results) { + Reference string = materializeStringInBuilder(walker, text, name, origin); + results[0] = string ? buildOperatorForStrings(walker, Opcode::StrictEq, string, written, origin) : Reference(); + }; + auto giveNo = [&](std::span results) { + results[0] = walker.constant(PrimitiveKnownToFolding::boolean(false)); + }; + bool isBuilt = buildArmsForStrings(walker, isOfSameLength, BlockFrequency::Normal, ArmForStrings(compare), ArmForStrings(giveNo), isEqual); + text.stringOfState = { }; + text.lengthOfState = { }; + if (isBuilt) + walker.bind(reader.id(), isEqual[0]); + return true; +} + bool readsTextWithoutString(Walker& walker, const BodyValue& reader, StringInBuilder& text, Reference name) { Opcode opcode = reader.opcode(); @@ -105,8 +135,11 @@ bool readsTextWithoutString(Walker& walker, const BodyValue& reader, StringInBui ValueID left = reader.operand(OperandIndex::StrictEq::left); ValueID right = reader.operand(OperandIndex::StrictEq::right); bool comparesWithEmptyText = (namesText(left) && isEmptyTextInTextBuilderReads(walker, walker.referenceOf(right))) || (namesText(right) && isEmptyTextInTextBuilderReads(walker, walker.referenceOf(left))); - if (!comparesWithEmptyText) - return false; + if (!comparesWithEmptyText) { + if (namesText(left) == namesText(right)) + return false; + return buildEqualityBehindLengthInTextBuilderReads(walker, reader, text, name, walker.referenceOf(namesText(left) ? right : left), origin); + } Reference length = lengthOfTextInTextBuilderReads(walker, text, name, origin); if (!length) return true; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkTextBuilderWebs.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkTextBuilderWebs.cpp index 61405326facd3..d8cd578c72d3e 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkTextBuilderWebs.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkTextBuilderWebs.cpp @@ -8,6 +8,7 @@ #if ENABLE(AOT) +#include "AOTBuiltinIDs.h" #include "AOTEffectsDeclared.h" #include "AOTEffectsOfOperations.h" #include "AOTIRProgram.h" @@ -19,6 +20,7 @@ #include #include #include +#include #include namespace JSC { namespace AOT { namespace Typed { @@ -27,6 +29,7 @@ static constexpr uint32_t frontOperandInTextBuilderWebs = 0; static_assert(OperandIndex::Add::left == frontOperandInTextBuilderWebs); static_assert(OperandIndex::Strcat::parts == frontOperandInTextBuilderWebs); +static constexpr OpcodeSet readersOfCharactersInTextBuilderWebs { Opcode::Call, Opcode::CallList, Opcode::Construct, Opcode::ElemGet, Opcode::ElemHas, Opcode::IteratorOpen }; static constexpr OpcodeSet castsInTextBuilderWebs = opcodesThatStateTypes | OpcodeSet { Opcode::CheckInitialized, Opcode::RequireObjectCoercible }; static bool passesOperandThroughInTextBuilderWebs(const BodyValue& operation, uint32_t operandIndex) @@ -83,6 +86,9 @@ struct CandidateInTextBuilderWeb { bool isOnlyConcatenated { true }; bool beginsAnewOnSomeEdge { false }; Vector parametersThatReadText; + bool hasReadersOfCharacters { false }; + Vector loopsOfAppends; + Vector blocksOfStores; }; class TextBuilderWebScan { @@ -126,6 +132,10 @@ class TextBuilderWebScan { void refuseTextNamedBy(ValueID); bool makesTextThatIsOnlyRead(const BodyValue& join); void noteThatStateIsSeen(const NameInTextBuilderWeb& name) { m_candidates[name.text].isOnlyConcatenated &= !m_index.isParameter(name.version); } + bool storesOrPassesOperand(const BodyValue&, ValueID operand) const; + bool runsInEveryIteration(BlockID) const; + bool coverEveryIteration(std::span, LoopID) const; + bool isCheaperAsRope(const CandidateInTextBuilderWeb&) const; Walker& m_walker; const BodyIndex& m_index; @@ -347,6 +357,72 @@ void TextBuilderWebScan::findParametersThatComeRound() } } +bool TextBuilderWebScan::storesOrPassesOperand(const BodyValue& operation, ValueID operand) const +{ + switch (operation.opcode()) { + case Opcode::PropSet: + case Opcode::PropDefine: + case Opcode::ElemSet: + case Opcode::PutModuleVariable: + case Opcode::PutEnvironmentField: + return operation.valueOperand() == operand; + case Opcode::Call: + case Opcode::Construct: + return std::ranges::contains(operation.argumentsOperands(), operand); + default: + return false; + } +} + +bool TextBuilderWebScan::runsInEveryIteration(BlockID block) const +{ + LoopID loop = m_index.loopOf(block); + return std::ranges::all_of(m_walker.incomingEdgesOf(m_index, m_index.loop(loop).head), [&](const BodyIncomingEdge& edge) { + return !m_index.isInLoop(edge.from, loop) || m_index.dominates(block, edge.from); + }); +} + +bool TextBuilderWebScan::coverEveryIteration(std::span blocks, LoopID loop) const +{ + HashSet, WTF::UnsignedWithZeroKeyHashTraits> covered; + Vector pending; + auto cover = [&](BlockID block) { + if (m_index.loopOf(block) == loop && covered.add(block.index()).isNewEntry) + pending.append(block); + }; + auto isCovered = [&](BodyTarget target) { + return !m_index.isInLoop(target.block(), loop) || covered.contains(target.block().index()); + }; + for (BlockID block : blocks) + cover(block); + for (size_t place = 0; place < pending.size(); ++place) { + if (runsInEveryIteration(pending[place])) + return true; + for (const BodyIncomingEdge& edge : m_walker.incomingEdgesOf(m_index, pending[place])) { + BodyBlock from = m_index.block(edge.from); + if (std::ranges::all_of(from.terminator().targets(), isCovered) && (!from.hasHandler() || isCovered(from.handler()))) + cover(edge.from); + } + } + return false; +} + +bool TextBuilderWebScan::isCheaperAsRope(const CandidateInTextBuilderWeb& candidate) const +{ + static constexpr unsigned maxRopesPerStore = 4; + return !candidate.hasReadersOfCharacters && std::ranges::any_of(candidate.blocksOfStores, [&](BlockID block) { + LoopID loop = m_index.loopOf(block); + unsigned appends = 0; + for (LoopID loopOfAppend : candidate.loopsOfAppends) { + if (loopOfAppend == loop) + ++appends; + else if (m_index.isInLoop(m_index.loop(loopOfAppend).head, loop)) + return false; + } + return appends && appends <= maxRopesPerStore && coverEveryIteration(candidate.blocksOfStores.span(), loop); + }); +} + bool TextBuilderWebScan::makesTextThatIsOnlyRead(const BodyValue& join) { ValueID made = join.id(); @@ -392,8 +468,14 @@ void TextBuilderWebScan::scanOperation(const BodyValue& operation) } else if (textInFront == name->text) refuse(name->text); - else + else { noteThatStateIsSeen(*name); + CandidateInTextBuilderWeb& candidate = m_candidates[name->text]; + if (!storesOrPassesOperand(operation, operand)) + candidate.hasReadersOfCharacters |= readersOfCharactersInTextBuilderWebs.contains(opcode); + else + candidate.blocksOfStores.append(m_index.blockOf(operation.id())); + } }); if (textPassedThrough != none && isOpen(textPassedThrough)) @@ -415,6 +497,7 @@ void TextBuilderWebScan::scanOperation(const BodyValue& operation) m_versions.set(textInFront, operation.id()); m_textsGrownSinceSomethingMayHaveThrown.append(textInFront); candidate.grows = true; + candidate.loopsOfAppends.append(m_index.loopOf(m_index.blockOf(operation.id()))); } void TextBuilderWebScan::arriveAt(BlockID block, uint32_t text, uint32_t parameterIndex, bool beginsAnew) @@ -720,7 +803,7 @@ void TextBuilderWebScan::run(Vector& texts) } for (uint32_t text = 0; text < m_candidates.size(); ++text) { const CandidateInTextBuilderWeb& candidate = m_candidates[text]; - if (candidate.isRefused || !candidate.grows || (candidate.beginsAnewOnSomeEdge && m_standsUnderHandlers)) + if (candidate.isRefused || !candidate.grows || (candidate.beginsAnewOnSomeEdge && m_standsUnderHandlers) || isCheaperAsRope(candidate)) continue; GrowingText grown; grown.beginsAnewOnSomeEdge = candidate.beginsAnewOnSomeEdge; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkTurnsWrittenOut.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkTurnsWrittenOut.cpp index 5951f9948136e..8248015b724d2 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkTurnsWrittenOut.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkTurnsWrittenOut.cpp @@ -28,6 +28,7 @@ static constexpr uint32_t operationsThatAccessToListOutOfSightBecomes = 14; static constexpr OpcodeSet relationsOfCountersWrittenOut { Opcode::Less, Opcode::LessEq, Opcode::Greater, Opcode::GreaterEq }; static constexpr OpcodeSet stepsOfCountersWrittenOut { Opcode::Increment, Opcode::Decrement, Opcode::Add, Opcode::Sub }; static constexpr OpcodeSet argumentsObjectsOfTurnsWrittenOut { Opcode::CreateUnmappedArguments, Opcode::CreateMappedArguments }; +static constexpr OpcodeSet callsAndClosures { Opcode::Call, Opcode::CallList, Opcode::CallDirect, Opcode::Construct, Opcode::ConstructList, Opcode::SuperConstruct, Opcode::SuperConstructList, Opcode::NewClosure }; namespace { @@ -430,7 +431,38 @@ static uint32_t numberOfOperationsOfCopiesInTurnWrittenOut(const Walker& walker, } return numberOfOperations; } -static std::optional numberOfOperationsOfLoopsWrittenOut(const Walker& walker, const IntegersByText& integers, LoopID loop, uint32_t stays, bool countsCopies) +static constexpr OpcodeSet readsThatFindInnerLists { Opcode::PropGet, Opcode::ElemGet }; + +static bool dependsOnTurn(const BodyIndex& index, LoopID loop, ValueID written, unsigned operatorsLeft = 4) +{ + ValueID value = index.stripCasts(written); + if (!index.isInLoop(index.blockOf(value), loop)) + return false; + if (index.isParameter(value) || !operatorsLeft) + return true; + bool depends = false; + index.value(value).forEachOperand([&](ValueID operand, uint32_t) { + depends |= dependsOnTurn(index, loop, operand, operatorsLeft - 1); + }); + return depends; +} + +static bool storesIntoInnerList(Walker& walker, const BodyIndex& index, LoopID loop) +{ + LoopID loopAround = index.loop(loop).parent; + if (!loopAround || index.loop(loop).numberOfLoopsWithin != 1) + return false; + for (BlockID member : index.membersOf(loop)) { + for (BodyValue operation : index.block(member).values()) { + ValueID list = operation.opcode() == Opcode::ElemSet ? index.stripCasts(operation.objectOperand()) : ValueID(); + if (list && !index.isParameter(list) && readsThatFindInnerLists.contains(index.opcodeOf(list)) && index.isInLoop(index.blockOf(list), loopAround) && !dependsOnTurn(index, loop, list) && !isExpectedToBeView(walker, list)) + return true; + } + } + return false; +} + +static std::optional numberOfOperationsOfLoopsWrittenOut(Walker& walker, const IntegersByText& integers, LoopID loop, uint32_t stays, bool countsCopies, bool isWrittenOut = true) { const BodyIndex& index = integers.index; auto walkedOnce = [&](LoopID walked) { @@ -459,7 +491,7 @@ static std::optional numberOfOperationsOfLoopsWrittenOut(const Walker& } return entering; }; - std::optional staysWithin = foreseeTurnsByText(integers, within, whatEnters); + std::optional staysWithin = !isWrittenOut && storesIntoInnerList(walker, index, within) ? std::nullopt : foreseeTurnsByText(integers, within, whatEnters); std::optional ofLoopWithin = staysWithin ? numberOfOperationsOfLoopsWrittenOut(walker, integers, within, *staysWithin, countsCopies) : std::nullopt; if (!ofLoopWithin) return std::nullopt; @@ -469,6 +501,48 @@ static std::optional numberOfOperationsOfLoopsWrittenOut(const Walker& return numberOfOperations > mostOperationsWalkedForTurnsWrittenOut ? std::nullopt : std::optional(numberOfOperations); } +static bool isOutermostUnrollingTooLarge(Walker& walker, const BodyIndex& index, BlockID inputHead, uint32_t operationsOfIteration, uint32_t numberOfWalks) +{ + LoopID loop = index.loopOf(inputHead); + uint32_t firstValue = index.block(inputHead).firstParameter().index(); + for (BlockID member : index.membersOf(loop)) { + for (BodyValue operation : index.block(member).values()) { + FunctionID callee = operation.opcode() == Opcode::Call ? walker.facts().linked().calleeAt(walker.currentFunction(), operation.id()) : operation.opcode() == Opcode::CallDirect && !operation.isConstruction() ? operation.function() : FunctionID(); + if (callee && calleeIsCopiedInLoopsAlone(walker, callee, operation.opcode() == Opcode::Call ? operation.as().argumentsOperands().size() : operation.numberOfVariadicOperands())) + return true; + if (operation.opcode() != Opcode::ElemGet && operation.opcode() != Opcode::ElemSet) + continue; + ValueID object = index.stripCasts(operation.objectOperand()); + bool isDefinedInLoop = object && object.index() >= firstValue && index.isInLoop(index.blockOf(object), loop); + Reference list = object && !isDefinedInLoop ? walker.referenceBoundTo(object) : Reference(); + ValueID cell = list && list.isValue() ? underlyingCellOf(walker.code(), list.id()) : ValueID(); + bool isInSight = (list && list.isConstant()) || (cell && (walker.code().isConstant(cell) || walker.state().isFresh(Reference::value(cell)))) || (list && findHoistedArrayGuard(walker, list)); + if (!isInSight) + operationsOfIteration += operationsThatAccessToListOutOfSightBecomes; + } + } + return operationsOfIteration * numberOfWalks > mostOperationsWalkedForTurnsWrittenOut; +} + +bool Walker::hasUnrolledInnerLoops(BlockID inputHead) +{ + size_t frameIndex = m_frames.size() - 1; + const BodyIndex& index = *m_frames[frameIndex].index; + LoopID loop = index.loopOf(inputHead); + uint32_t firstValue = index.block(inputHead).firstParameter().index(); + auto isDefinedInLoop = [&](ValueID value) { + return value.index() >= firstValue && index.isInLoop(index.blockOf(value), loop); + }; + WTF::Function(ValueID)> leaf = [&](ValueID value) -> std::optional { + return isDefinedInLoop(value) ? std::nullopt : integerBuiltForTurnsWrittenOut(*this, referenceBoundTo(value)); + }; + WTF::Function(ValueID)> lengthOf = [](ValueID) -> std::optional { + return std::nullopt; + }; + IntegersByText integers { index, program().keyOf(BuiltinNames::length), leaf, lengthOf }; + return has(Mechanism::Folding) && has(Mechanism::LoopInvariants) && numberOfOperationsOfLoopsWrittenOut(*this, integers, loop, 1, !frameIndex, false); +} + bool Walker::writeOutTurnsOfLoop(BlockID inputHead, std::span entering) { size_t frameIndex = m_frames.size() - 1; @@ -504,37 +578,23 @@ bool Walker::writeOutTurnsOfLoop(BlockID inputHead, std::span e std::optional stays = foreseeTurnsByText(integers, loop, whatEnters); if (!stays) return false; + BlockID headAround = m_frames[frameIndex].fullyUnrolledHeader; + bool isInNestWrittenOut = headAround && index.loop(loop).parent == index.loopOf(headAround); + if (!isInNestWrittenOut && storesIntoInnerList(*this, index, loop)) { + count(StatisticID::StoringLoopsLeftAsLoops); + return false; + } bool holdsLoops = index.loop(loop).numberOfLoopsWithin != 1; bool standsInLoop = m_frames[frameIndex].loopDepthAtCall || index.loopDepthOf(inputHead) >= 2; if (holdsLoops && (!standsInLoop || !numberOfOperationsOfLoopsWrittenOut(*this, integers, loop, *stays, !frameIndex))) return false; uint32_t numberOfOperationsOfTurn = numberOfOperationsOfLoopWrittenOut(index, loop) + (frameIndex ? 0 : numberOfOperationsOfCopiesInTurnWrittenOut(*this, index, loop)); - BlockID headAround = m_frames[frameIndex].fullyUnrolledHeader; - bool isReckonedWithNest = !holdsLoops && headAround && index.loop(loop).parent == index.loopOf(headAround) && numberOfOperationsOfTurn * (*stays + 1) > mostOperationsWalkedForTurnsWrittenOut; + bool isReckonedWithNest = !holdsLoops && isInNestWrittenOut && numberOfOperationsOfTurn * (*stays + 1) > mostOperationsWalkedForTurnsWrittenOut; if (!holdsLoops && !isReckonedWithNest && numberOfOperationsOfTurn * (*stays + 1) > mostOperationsWalkedForTurnsWrittenOut) return false; - if (!m_frames[frameIndex].loopDepthAtCall && index.loopDepthOf(inputHead) < 2) { - for (BlockID member : index.membersOf(loop)) { - for (BodyValue operation : index.block(member).values()) { - FunctionID callee = operation.opcode() == Opcode::Call ? facts().linked().calleeAt(currentFunction(), operation.id()) : operation.opcode() == Opcode::CallDirect && !operation.isConstruction() ? operation.function() : FunctionID(); - if (callee && calleeIsCopiedInLoopsAlone(*this, callee, operation.opcode() == Opcode::Call ? operation.as().argumentsOperands().size() : operation.numberOfVariadicOperands())) { - count(StatisticID::FullUnrollingsRejectedForSize); - return false; - } - if (operation.opcode() != Opcode::ElemGet && operation.opcode() != Opcode::ElemSet) - continue; - ValueID object = index.stripCasts(operation.objectOperand()); - Reference list = object && !isDefinedInLoop(object) ? referenceBoundTo(object) : Reference(); - ValueID cell = list && list.isValue() ? underlyingCellOf(m_code, list.id()) : ValueID(); - bool isInSight = (list && list.isConstant()) || (cell && (m_code.isConstant(cell) || m_state.isFresh(Reference::value(cell)))) || (list && findHoistedArrayGuard(*this, list)); - if (!isInSight) - numberOfOperationsOfTurn += operationsThatAccessToListOutOfSightBecomes; - } - } - if (numberOfOperationsOfTurn * (*stays + 1) > mostOperationsWalkedForTurnsWrittenOut) { - count(StatisticID::FullUnrollingsRejectedForSize); - return false; - } + if (!m_frames[frameIndex].loopDepthAtCall && index.loopDepthOf(inputHead) < 2 && isOutermostUnrollingTooLarge(*this, index, inputHead, numberOfOperationsOfTurn, *stays + 1)) { + count(StatisticID::FullUnrollingsRejectedForSize); + return false; } if (!textAdmitsTurnsWrittenOut(index, m_liveness, loop, frameIndex && !m_frames[frameIndex].returnsInPlaceOfUnit, [&] { return numbersOfHandlersThatRead(m_frames.last()); }, { }, holdsLoops)) @@ -722,7 +782,7 @@ void Walker::walkTurnsWrittenOut(BlockID inputHead, std::span e forIn->numberOfNames = static_cast(arm.names.size()); m_indexedIterations[forIn->place].position = forIn->firstPosition; m_indexedIterations[forIn->place].precomputedSteps = arm.names; - walkTurns(forIn->numberOfNames); + walkTurns(forIn->enumerator ? forIn->numberOfNames : forIn->numberOfNames + 1); if (!m_hasFailed) letWhatCameRoundWaitForLoopAsWritten(); } @@ -833,6 +893,20 @@ void Walker::standAtHeadOfTurnOfForIn(BlockID inputHead) markEnumeratorCurrent(*this, turns.enumerator, std::as_const(m_indexedIterations)[turns.place].subject); } +static bool appendNamesOfOrder(Walker& walker, const OwnKeyOrder& order, Origin origin, Vector& names) +{ + const Program& program = walker.program(); + for (FieldIndex field : order.fields) { + PropertyKeyID key = walker.plan().slots().keyOf(program, FieldReference { order.layout, field }); + if (!key || liesInThread(key) || program.at(key).kind != PropertyKeyKind::Text || program.isIndexLike(key)) + return false; + Draft name(Opcode::String, origin); + name.setString(program.at(key).text); + names.append(walker.constant(WTF::move(name))); + } + return true; +} + bool Walker::writeOutTurnsOfForIn(BlockID inputHead, std::span entering) { size_t frameIndex = m_frames.size() - 1; @@ -896,14 +970,8 @@ bool Walker::writeOutTurnsOfForIn(BlockID inputHead, std::span if (!inSight && !shouldOpenGuess(*this, object, m_types.forLayout(order.layout), planned.isDeclared, numberOfNames)) return; TurnsOfForInWrittenOut::Arm arm; - for (FieldIndex field : order.fields) { - PropertyKeyID key = plan().slots().keyOf(program(), FieldReference { order.layout, field }); - if (!key || liesInThread(key) || program().at(key).kind != PropertyKeyKind::Text || program().isIndexLike(key)) - return; - Draft name(Opcode::String, origin); - name.setString(program().at(key).text); - arm.names.append(constant(WTF::move(name))); - } + if (!appendNamesOfOrder(*this, order, origin, arm.names)) + return; namesThatStillFit -= numberOfNames; orders.append(&order); turns.arms.append(WTF::move(arm)); @@ -963,13 +1031,105 @@ static bool isStepValueUsedAsKey(const BodyIndex& index, LoopID loop, ValueID st return false; } +static ValueID listOnlyReadInBody(const Walker& walker, const BodyIndex& index, ValueID written) +{ + ValueID list = index.stripCasts(written); + if (!list || index.isParameter(list) || index.opcodeOf(list) != Opcode::Call || index.value(list).argumentsOperands().size() != 1) + return { }; + bool isStatedOnce = list != written && index.useCount(list) == 1 && index.value(written).operand(0) == list; + if (list != written && !isStatedOnce) + return { }; + return bodyOnlyReadsListOfCall(walker, index, walker.program().keyOf(BuiltinNames::length), written) ? list : ValueID(); +} + +static bool isOutermostForOfSmall(Walker& walker, const BodyIndex& index, BlockID inputHead, uint32_t length) +{ + LoopID loop = index.loopOf(inputHead); + if (index.loop(loop).numberOfLoopsWithin != 1) + return false; + uint32_t numberOfOperations = numberOfOperationsOfLoopWrittenOut(index, loop); + for (BlockID member : index.membersOf(loop)) { + for (BodyValue operation : index.block(member).values()) { + if (callsAndClosures.contains(operation.opcode())) + return false; + if (operation.opcode() != Opcode::PropGet && operation.opcode() != Opcode::PropSet) + continue; + Reference object = operation.isFreeName() ? Reference() : walker.referenceBoundTo(index.stripCasts(operation.objectOperand())); + if (!object || !singleLayoutOf(walker.types(), walker.types().at(walker.typeOf(object)))) + numberOfOperations += operationsThatAccessToListOutOfSightBecomes; + } + } + return !isOutermostUnrollingTooLarge(walker, index, inputHead, numberOfOperations, length + 1); +} + +bool Walker::writeOutTurnsOverOwnKeys(BlockID inputHead, std::span entering, TurnsOfForInWrittenOut& turns, const ForOfByText& forOf) +{ + size_t frameIndex = m_frames.size() - 1; + const BodyIndex& index = *m_frames[frameIndex].index; + LoopID loop = index.loopOf(inputHead); + ValueID list = has(Mechanism::Guesses) && m_opensGuesses ? listOnlyReadInBody(*this, index, index.value(forOf.open).as().iterableOperand()) : ValueID(); + Reference made = list ? referenceOf(list) : Reference(); + ValueID objectInBody = list ? index.value(list).argumentsOperands()[0] : ValueID(); + Reference object = objectInBody ? referenceOf(objectInBody) : Reference(); + if (!made || !made.isValue() || !object || !object.isValue() || !findKnown(*this, KnownNote::ListedOwnKeys, object, made) || !isStepValueUsedAsKey(index, loop, forOf.step)) + return false; + uint32_t mostNames = mostOperationsWalkedForTurnsWrittenOut / numberOfOperationsOfLoopWrittenOut(index, loop); + if (mostNames < 2) + return false; + --mostNames; + OwnKeyPlan planned = planOwnKeyWalk(*this, objectInBody, object, mostNames); + if (!planned || !textAdmitsTurnsWrittenOut(index, m_liveness, loop, frameIndex && !m_frames[frameIndex].returnsInPlaceOfUnit, [&] { return numbersOfHandlersThatRead(m_frames.last()); }, forOf.open)) + return false; + + Origin origin = originOf(index.value(forOf.step)); + Vector orders; + for (const OwnKeyOrder& order : planned.orders) { + uint32_t numberOfNames = static_cast(order.fields.size()); + TurnsOfForInWrittenOut::Arm arm; + if (!order.layout || !numberOfNames || numberOfNames > mostNames || !shouldOpenGuess(*this, object, m_types.forLayout(order.layout), planned.isDeclared, numberOfNames) || !appendNamesOfOrder(*this, order, origin, arm.names)) + continue; + mostNames -= numberOfNames; + orders.append(&order); + turns.arms.append(WTF::move(arm)); + } + if (orders.isEmpty()) + return false; + + beginTurnsWrittenOut(inputHead); + turns.firstPosition = std::as_const(m_indexedIterations)[turns.place].position; + for (size_t number = 0; number < orders.size() && !m_hasFailed; ++number) { + TurnsOfForInWrittenOut::Arm& arm = turns.arms[number]; + arm.object = object; + BlockID otherwise = addBlock(number + 1 == orders.size() ? BlockFrequency::Unlikely : BlockFrequency::Normal); + bool hasKeys = continueIfOwnKeys(*this, arm.object, *orders[number], planned.layoutsAreProven, origin, otherwise); + recordSpeculation(); + if (hasKeys && arm.object != object && !m_state.checkedNameOf(object)) + m_state.noteCheckedName(object, arm.object); + arm.whereTestsPassed = hasKeys ? m_block : BlockID(); + arm.stateWhereTestsPassed = WTF::move(m_state); + continueIn(otherwise); + if (isUnreachable()) + break; + if (number + 1 == orders.size()) + leadIntoLoopAsWrittenBehindTurns(turns, inputHead, entering, turns.firstPosition); + } + if (m_hasFailed) + return true; + for (size_t number = 0; number < orders.size(); ++number) + count(StatisticID::OwnKeyWalksExpanded); + walkTurnsWrittenOut(inputHead, entering, 0, &turns); + if (IndexedIteration& iteration = m_indexedIterations[turns.place]; !m_hasFailed && !iteration.head && iteration.inputHead) { + iteration.inputHead = { }; + --m_numberOfIterationsInFrontOfTheirLoops; + } + return true; +} + bool Walker::writeOutTurnsOfForOf(BlockID inputHead, std::span entering) { size_t frameIndex = m_frames.size() - 1; const BodyIndex& index = *m_frames[frameIndex].index; bool standsInLoop = frameIndex ? !!m_frames[frameIndex].loopDepthAtCall : index.loopDepthOf(inputHead) >= 2; - if (!standsInLoop && !index.holdsAnyOf(accessesUnderComputedKeys)) - return false; if (!has(Mechanism::Folding) || !has(Mechanism::TypedOperations) || !has(Mechanism::LoopInvariants) || !has(Mechanism::EscapeAnalysis) || isUnreachable() || hasStringsInBuilders()) return false; if (!index.isLoopHead(inputHead) || !m_inputHeadsOfIterations.contains(inputHeadInFrame(m_frames[frameIndex].inlineFrame, inputHead))) @@ -995,22 +1155,26 @@ bool Walker::writeOutTurnsOfForOf(BlockID inputHead, std::span bool lengthIsNoted = !length; if (lengthIsNoted) length = lengthOfListMadeInSight(*this, asItEntered.list); - if (!length || *length > mostTurnsWrittenOut || numberOfOperationsOfLoopWrittenOut(index, loop) * (*length + 1) > mostOperationsWalkedForTurnsWrittenOut) + if (!length) + return writeOutTurnsOverOwnKeys(inputHead, entering, turns, *forOf); + if (*length > mostTurnsWrittenOut || numberOfOperationsOfLoopWrittenOut(index, loop) * (*length + 1) > mostOperationsWalkedForTurnsWrittenOut) return false; + bool isConstantKeyLoop = !standsInLoop && isStepValueUsedAsKey(index, loop, forOf->step); if (!standsInLoop) { - if (!isStepValueUsedAsKey(index, loop, forOf->step)) - return false; for (uint32_t position = 0; position < *length; ++position) { Reference element = elementOfListInSight(*this, asItEntered.list, position); if (!element) element = elementWrittenInListMadeInSight(*this, asItEntered.list, position); - if (!element || !element.isConstant() || m_code.at(element.id()).opcode != Opcode::String) + if (!element) return false; + isConstantKeyLoop &= element.isConstant() && m_code.at(element.id()).opcode == Opcode::String; } + if (!isConstantKeyLoop && !isOutermostForOfSmall(*this, index, inputHead, *length)) + return false; } if (!textAdmitsTurnsWrittenOut(index, m_liveness, loop, frameIndex && !m_frames[frameIndex].returnsInPlaceOfUnit, [&] { return numbersOfHandlersThatRead(m_frames.last()); }, forOf->open)) return false; - if (!standsInLoop) + if (isConstantKeyLoop) count(StatisticID::ConstantKeyLoopsFullyUnrolled); if (lengthIsNoted) { diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkTypedAccesses.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkTypedAccesses.cpp index 697901eba6d65..7129d012d811b 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkTypedAccesses.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkTypedAccesses.cpp @@ -244,21 +244,7 @@ static ALWAYS_INLINE LinkedCell cellOfFieldForTypedAccess(const LinkTables& link static bool isLookedUpAsNameForTypedAccess(const Program& program, PropertyKeyID key) { - if (!key || liesInThread(key) || !program.types.hasStableNumber(key)) - return false; - switch (program.at(key).kind) { - case PropertyKeyKind::Text: - case PropertyKeyKind::WellKnownSymbol: - case PropertyKeyKind::CreatedSymbol: - case PropertyKeyKind::RegisteredSymbol: - case PropertyKeyKind::EnginePrivateName: - case PropertyKeyKind::ModulePrivateName: - break; - case PropertyKeyKind::PrivateName: - return false; - } - NameClass nameClass = program.nameClassOf(key); - return !mayBeElementKey(nameClass) && nameClass != NameClass::Proto; + return program.types.isLookedUpAsName(key); } static bool findsMemberWithoutCodeForTypedAccess(const Program& program, const LinkTables& link, PropertyKeyID key) @@ -846,6 +832,28 @@ static bool findsNothingForTypedAccess(const Walker& walker, const ReceiversOfTy && nothingAnswersAboveReceiversForTypedAccess(walker, receivers, key); } +static BuiltinID findMemberOfObjectPrototypeForTypedAccess(const Walker& walker, const ReceiversOfTypedAccess& receivers, const FieldOfTypedAccess& field, PropertyKeyID key) +{ + const Program& program = walker.program(); + if (field.coverage != CoverageOfTypedAccess::NoReceiver || !isOnObjectPrototype(program.nameClassOf(key)) || !walker.has(Mechanism::BuiltinCalls)) + return { }; + const BuiltinTable& table = program.builtins(); + BuiltinID location = findConstantMemberOnceForBuild(walker, Builtins::ObjectPrototype, key); + if (!location || !isObjectKind(table.at(table.canonicalID(location)).kind)) + return { }; + const LinkTables& link = linkTablesForTypedAccesses(walker); + const SlotPlan& slots = walker.plan().slots(); + bool everyReceiverInheritsIt = std::ranges::all_of(receivers.members, [&](LayoutID member) { + return nothingAnswersBelowObjectPrototypeForTypedAccess(program, link, member, key) && !slots.findField(program, member, key); + }); + return everyReceiverInheritsIt ? location : BuiltinID(); +} + +static bool changesForbidReadOffObjectPrototype(const ReceiversOfTypedAccess& receivers, OptionSet knownOfCell) +{ + return possibleChangesForbid(receivers.possibleChanges, TypedAccess::FindsNoProperty, knownOfCell) || possibleChangesForbid(receivers.possibleChanges, TypedAccess::FindsMethodAbove, knownOfCell); +} + static bool paysToLookAtSlotForTypedAccess(const Walker& walker, const FieldOfTypedAccess& field, PropertyKeyID key, bool isRead = false, bool isLastResort = false) { PropertyKeyKind kind = walker.program().at(key).kind; @@ -987,6 +995,12 @@ static Reference valueInSlotForTypedAccess(Walker& walker, Reference object, con return type && !(walker.types().at(type).primitives & Primitive::Absent) ? held : Reference(); } +static bool slotIsKnownEmptyForTypedAccess(Walker& walker, Reference object, const FieldOfTypedAccess& field) +{ + Reference noted = walker.state().slot(object, field.place); + return noted && noted.isRecord() && walker.code().at(noted.id()).opcode == Opcode::Absent; +} + static bool provesSlotFilledForTypedAccess(const Walker& walker, Reference held, PropertyKeyID key) { if (!held) @@ -1089,6 +1103,7 @@ static bool mayFindSlotEmptyBehindLooksInTypedRead(const Walker&, const Receiver static std::optional> askAheadAboutReadOfSlotThatMayBeEmpty(const Walker&, const ReceiversOfTypedAccess&, const FieldOfTypedAccess&, PropertyKeyID); static TypedForm buildReadOfMethodBehindLooksForTypedAccess(Walker&, const BodyValue&, const OperandsOfTypedAccess&, const ReceiversOfTypedAccess&, PropertyKeyID); static TypedForm buildReadThatFindsNothingBehindLooksForTypedAccess(Walker&, const BodyValue&, const OperandsOfTypedAccess&, const ReceiversOfTypedAccess&, const FieldOfTypedAccess&, PropertyKeyID); +static TypedForm buildReadOffObjectPrototypeForTypedAccess(Walker&, const BodyValue&, const OperandsOfTypedAccess&, const ReceiversOfTypedAccess&, const FieldOfTypedAccess&, PropertyKeyID); static TypedForm buildAccessToAccessorBehindLooksForTypedAccess(Walker&, const BodyValue&, const OperandsOfTypedAccess&, const ReceiversOfTypedAccess&, PropertyKeyID); static TypedForm buildReadOffConstructorFunctionBehindGuardForTypedAccess(Walker&, const BodyValue&, const OperandsOfTypedAccess&); static TypedForm buildReadOffHeirOfBuiltinForTypedAccess(Walker&, const BodyValue&, const OperandsOfTypedAccess&); @@ -1169,6 +1184,21 @@ bool markOfCellAnswersForRead(Walker& walker, const BodyValue& read, TypeID type return receivers && markOfCellAnswersForFieldInTypedAccess(walker, *receivers, describeFieldOfTypedAccess(walker, *receivers, read.key(), { }, true), read.key(), read.objectOperand()); } +std::optional findFilledSlot(Walker& walker, TypeID typeOfObject, PropertyKeyID key) +{ + if (!walker.has(Mechanism::TypedOperations) || !typeOfObject || walker.types().at(typeOfObject).primitives || !walker.types().at(typeOfObject).isJavaScriptValue()) + return std::nullopt; + std::optional receivers = findReceiversOfTypedAccess(walker, typeOfObject); + if (!receivers) + return std::nullopt; + FieldOfTypedAccess field = describeFieldOfTypedAccess(walker, *receivers, key, { }, true); + if (findsNothingForTypedAccess(walker, *receivers, field, key) || field.coverage != CoverageOfTypedAccess::EveryReceiver || !field.liesInOneSlot || slotHoldsMarkForTypedAccess(walker, *receivers, key)) + return std::nullopt; + if (!field.isFilledAtEveryRead || !field.isMovedOutByNoStore) + return std::nullopt; + return FilledSlot { field.place, receivers->set }; +} + static TypedForm buildReadForTypedAccess(Walker& walker, const BodyValue& value, const OperandsOfTypedAccess& operands, PropertyKeyID key) { std::optional receivers = findReceiversOfTypedAccess(walker, operands.typeOfObject); @@ -1204,6 +1234,8 @@ static TypedForm buildReadForTypedAccess(Walker& walker, const BodyValue& value, form = buildReadOfMethodBehindLooksForTypedAccess(walker, value, operands, *receivers, key); if (form == TypedForm::None) form = buildAccessToAccessorBehindLooksForTypedAccess(walker, value, operands, *receivers, key); + if (form == TypedForm::None) + form = buildReadOffObjectPrototypeForTypedAccess(walker, value, operands, *receivers, field, key); return form != TypedForm::None ? form : buildReadThatFindsNothingBehindLooksForTypedAccess(walker, value, operands, *receivers, field, key); } if (!field.liesInOneSlot) @@ -1254,11 +1286,7 @@ static TypedForm buildReadForTypedAccess(Walker& walker, const BodyValue& value, return TypedForm::Built; } - auto wasFoundEmpty = [&] { - Reference noted = walker.state().slot(object, field.place); - return noted && noted.isRecord() && walker.code().at(noted.id()).opcode == Opcode::Absent; - }; - if (!isFilled && walker.has(Mechanism::Folding) && (orderWordShowsThatCellLacksFieldForTypedAccess(walker, object, *receivers, key) || wasFoundEmpty())) { + if (!isFilled && walker.has(Mechanism::Folding) && (orderWordShowsThatCellLacksFieldForTypedAccess(walker, object, *receivers, key) || slotIsKnownEmptyForTypedAccess(walker, object, field))) { walker.bind(value.id(), walker.constant(Draft(Opcode::Undefined))); return TypedForm::Built; } @@ -1287,6 +1315,8 @@ static TypedForm buildReadForTypedAccess(Walker& walker, const BodyValue& value, walker.bind(value.id(), read); if (read) walker.state().noteSlot(object, field.place, read); + if (read && opcodeOfRead == Opcode::GetSlot) + walker.noteSlotRead(object, key, read); return TypedForm::Built; } @@ -1343,6 +1373,33 @@ Reference readFilledFieldOfProvenObject(Walker& walker, Reference object, Proper return read; } +static bool slotTellsWhoHasPrivateNameForTypedAccess(const Walker&, LayoutID declaring); + +Reference readFilledFieldInFrontOfTurns(Walker& walker, const BodyValue& read, Reference object) +{ + if (!object || !object.isValue() || walker.isUnreachable()) + return { }; + PropertyKeyID key = keyOfAccessToSlot(walker.program(), read); + std::optional receivers = findReceiversOfTypedAccess(walker, walker.typeOf(object)); + if (!receivers || (read.opcode() == Opcode::PrivateGet && !slotTellsWhoHasPrivateNameForTypedAccess(walker, read.layout()))) + return { }; + FieldOfTypedAccess field = describeFieldOfTypedAccess(walker, *receivers, key, { }, true); + if (field.coverage != CoverageOfTypedAccess::EveryReceiver || !field.liesInOneSlot || slotHoldsMarkForTypedAccess(walker, *receivers, key)) + return { }; + if (!isFilledForBodyBeingReadInTypedAccess(walker, *receivers, field, read.objectOperand(), key)) + return readFilledFieldOfProvenObject(walker, object, key, walker.originOf(read)); + if (Reference held = valueInSlotForTypedAccess(walker, object, field)) + return held; + Walker::ProofsHoldEverywhere filledWhereverObjectIs(walker); + Draft draft(Opcode::GetSlot, walker.originOf(read)); + walker.chooseConservativeFields(draft); + fillSlotOperationForTypedAccess(draft, object, *receivers, key); + Reference value = walker.build(WTF::move(draft)); + if (value) + walker.state().noteSlot(object, field.place, value); + return value; +} + FunctionID findMethodInSightOfProvenType(Walker& walker, TypeID proven, PropertyKeyID key) { if (FunctionID above = findMethodAboveProvenType(walker, proven, key)) @@ -1379,6 +1436,27 @@ Reference readMethodInSightOfProvenObject(Walker& walker, Reference object, Prop return held.isObject() && held.isJavaScriptValue() && singleFunctionOf(walker.types(), held) ? own : Reference(); } +Reference readMethodForCallAlone(Walker& walker, Reference object, PropertyKeyID key, Origin origin) +{ + if (Reference inSight = readMethodInSightOfProvenObject(walker, object, key, origin)) + return inSight; + if (!walker.has(Mechanism::TypedOperations) || !object || !object.isValue() || walker.isUnreachable()) + return { }; + std::optional receivers = findReceiversOfTypedAccess(walker, walker.typeOf(object)); + if (!receivers || !slotHoldsMarkForTypedAccess(walker, *receivers, key)) + return { }; + FieldOfTypedAccess field = describeFieldOfTypedAccess(walker, *receivers, key, { }, true); + if (field.coverage != CoverageOfTypedAccess::EveryReceiver || !field.liesInOneSlot || !field.isFilledAtEveryRead || !field.isMovedOutByNoStore) + return { }; + Draft draft(Opcode::GetDeferredClosureSlot, origin); + walker.chooseConservativeFields(draft); + fillSlotOperationForTypedAccess(draft, object, *receivers, key); + Effects effects = Effects::ofReadByNameThatRunsNoCode(); + Reference read = walker.build(WTF::move(draft), &effects); + readScopeBesideSlotForTypedAccess(walker, origin, read); + return read; +} + bool findsNothingUnderNameOfProvenObject(Walker& walker, Reference object, PropertyKeyID key) { if (!walker.has(Mechanism::TypedOperations) || !object || !object.isValue()) @@ -1402,6 +1480,34 @@ bool buildReadWhereSomeReceiversLackField(Walker& walker, const BodyValue& value return field.coverage == CoverageOfTypedAccess::SomeReceivers && buildReadWhereSomeReceiversLackFieldForTypedAccess(walker, value, operands, *receivers, field, key) == TypedForm::Built; } +bool buildOwnPropertyTest(Walker& walker, const BodyValue& question, Reference object, PropertyKeyID key) +{ + const Program& program = walker.program(); + if (!walker.has(Mechanism::TypedOperations) || !object || !object.isValue() || !isLookedUpAsNameForTypedAccess(program, key)) + return false; + std::optional receivers = findReceiversOfTypedAccess(walker, walker.typeOf(object)); + if (!receivers) + return false; + FieldOfTypedAccess field = describeFieldOfTypedAccess(walker, *receivers, key, { }, true); + if (field.coverage == CoverageOfTypedAccess::NoReceiver) { + const LinkTables& link = linkTablesForTypedAccesses(walker); + const SlotPlan& slots = walker.plan().slots(); + bool noReceiverHasIt = !possibleChangesForbid(receivers->possibleChanges - LayoutChange::GetsObjectAbove, TypedAccess::FindsNoProperty) && std::ranges::all_of(receivers->members, [&](LayoutID member) { + return link.findsNothing(program, member, ClassSide::Prototype, key, Of::ObjectItself) && !slots.findField(program, member, key); + }); + return noReceiverHasIt && bindBooleanForTypedAccess(walker, question, false); + } + if (!field.liesInOneSlot || !field.isMovedOutByNoStore || possibleChangesForbid(changesThatMayReachFieldForTypedAccess(*receivers, field) - LayoutChange::GetsObjectAbove, TypedAccess::ReadsSlotThatMayBeEmpty)) + return false; + bool isFilled = provesSlotFilledForTypedAccess(walker, valueInSlotForTypedAccess(walker, object, field), key) || everyReceiverKeepsFieldInSlotThatIsNeverEmpty(walker, receivers->members.span(), key); + if (walker.has(Mechanism::Folding) && isFilled) + return bindBooleanForTypedAccess(walker, question, true); + if (walker.has(Mechanism::Folding) && (orderWordShowsThatCellLacksFieldForTypedAccess(walker, object, *receivers, key) || slotIsKnownEmptyForTypedAccess(walker, object, field))) + return bindBooleanForTypedAccess(walker, question, false); + walker.bind(question.id(), walker.build(draftOfSlotOperationForTypedAccess(walker, question, Opcode::IsSlotFilled, object, *receivers, key))); + return true; +} + static TypedForm buildTestForTypedAccess(Walker& walker, const BodyValue& value, const OperandsOfTypedAccess& operands, PropertyKeyID key) { std::optional receivers = findReceiversOfTypedAccess(walker, operands.typeOfObject); @@ -1427,7 +1533,7 @@ static TypedForm buildTestForTypedAccess(Walker& walker, const BodyValue& value, if (!isFilled && !mayFindSlotEmpty) return TypedForm::None; isFilled |= everyReceiverKeepsFieldInSlotThatIsNeverEmpty(walker, receivers->members.span(), key); - if (!isFilled && walker.has(Mechanism::Folding) && orderWordShowsThatCellLacksFieldForTypedAccess(walker, object, *receivers, key)) { + if (!isFilled && walker.has(Mechanism::Folding) && (orderWordShowsThatCellLacksFieldForTypedAccess(walker, object, *receivers, key) || slotIsKnownEmptyForTypedAccess(walker, object, field))) { bindBooleanForTypedAccess(walker, value, false); return TypedForm::Built; } @@ -1559,6 +1665,11 @@ static bool rowAboveClassPassesWhateverTheNameForTypedAccess(const Walker& walke return answer == RowAbove::Passes; } +bool hasOnlyNamedMembersAbove(Walker& walker, LayoutID classLayout) +{ + return rowAboveClassPassesWhateverTheNameForTypedAccess(walker, keptByTypedAccesses(walker), classLayout); +} + static bool someClassHasMemberOfPrototypeUnderForTypedAccess(const Walker& walker, KeptByTypedAccesses& kept, PropertyKeyID key) { if (liesInThread(key)) @@ -1748,6 +1859,16 @@ static bool mayFindSlotEmptyBehindLooksInTypedRead(const Walker& walker, const R return lookupPassesAboveReceiversForTypedAccess(walker, receivers, field, key, LookupAbove::OfRead, stateAskedAboutGuards); } +static bool readAsksGuardsForTypedAccess(const ReceiversOfTypedAccess& receivers, OptionSet askedOfCell) +{ + return askedOfCell.contains(CellFact::HasLayoutPrototype) || receivers.possibleChanges.contains(LayoutChange::ChangesPrototype); +} + +static bool storeAsksGuardsForTypedAccess(const Walker& walker, const ReceiversOfTypedAccess& receivers, const FieldOfTypedAccess& field, PropertyKeyID key, OptionSet askedOfCell) +{ + return readAsksGuardsForTypedAccess(receivers, askedOfCell) || storeAsksGuardsWherePrototypeStaysForTypedAccess(walker, receivers, field, key); +} + static constexpr auto choicesAmongFactsThatLooksAtCellShow = [] { constexpr unsigned numberOfFactsShown = std::size(factsThatLooksAtCellShow); std::array, 1u << numberOfFactsShown> choices { }; @@ -1783,6 +1904,33 @@ static std::optional> askAheadAboutReadOfSlotThatMayBeEmpty( }); } +static std::optional> askAheadAboutReadOffObjectPrototype(const ReceiversOfTypedAccess& receivers) +{ + return smallestSufficientCellFacts([&](OptionSet facts) { + return !changesForbidReadOffObjectPrototype(receivers, facts); + }); +} + +static TypedForm buildReadOffObjectPrototypeForTypedAccess(Walker& walker, const BodyValue& value, const OperandsOfTypedAccess& operands, const ReceiversOfTypedAccess& receivers, const FieldOfTypedAccess& field, PropertyKeyID key) +{ + if (value.opcode() != Opcode::PropGet || value.isFreeName()) + return TypedForm::None; + BuiltinID location = findMemberOfObjectPrototypeForTypedAccess(walker, receivers, field, key); + std::optional> asked = location ? askAheadAboutReadOffObjectPrototype(receivers) : std::nullopt; + if (!asked) + return TypedForm::None; + if (operands.areOnlyTypes) { + TypedForm form = buildReadOfMemberOffObjectPrototype(walker, value, operands.typeOfObject, { }, location); + if (form == TypedForm::None || asked->isEmpty()) + return form; + return walker.isInLoop() || walker.weighsWhatIsWritten() ? TypedForm::BuiltBehindLookAtCell : TypedForm::None; + } + if (!knownCellFacts(walker, operands.object).containsAll(*asked)) + return TypedForm::None; + walker.checkFactsRememberedOfCell(operands.object, walker.originOf(value), *asked); + return buildReadOfMemberOffObjectPrototype(walker, value, operands.typeOfObject, operands.object, location); +} + static bool statementsLayNameAboveObjectsOfForTypedAccess(const Walker& walker, LayoutID member, PropertyKeyID key, bool guardOfNameAnswersForUnknownKeys = false, KeptByTypedAccesses* climbedByOtherMembers = nullptr) { const Program& program = walker.program(); @@ -2678,7 +2826,7 @@ static TypedForm buildReadOffConstructorFunctionBehindGuardForTypedAccess(Walker return bindClosureThatClosureTableHoldsForTypedAccess(walker, value, found->method, origin); } -void noteGuardsShownByTest(Walker& walker, std::span conditions, Reference test) +void noteGuardsShownByTest(Walker& walker, std::span conditions, Reference test, Reference receiver) { WalkState& state = walker.state(); for (const MemberConditionRecord& condition : conditions) { @@ -2688,7 +2836,8 @@ void noteGuardsShownByTest(Walker& walker, std::span factsLeanedOn = *askedAhead; - bool leansOnGuards = findsNothing ? receivers.possibleChanges.contains(LayoutChange::ChangesPrototype) : isProvenStore ? receivers.possibleChanges.contains(LayoutChange::ChangesPrototype) || !nothingAnswersAboveReceiversForTypedAccess(walker, receivers, key) : factsLeanedOn.contains(CellFact::HasLayoutPrototype) || (stores && storeAsksGuardsWherePrototypeStaysForTypedAccess(walker, receivers, field, key)); + bool leansOnGuards = findsNothing ? receivers.possibleChanges.contains(LayoutChange::ChangesPrototype) : isProvenStore ? receivers.possibleChanges.contains(LayoutChange::ChangesPrototype) || !nothingAnswersAboveReceiversForTypedAccess(walker, receivers, key) : stores ? storeAsksGuardsForTypedAccess(walker, receivers, field, key, factsLeanedOn) : readAsksGuardsForTypedAccess(receivers, factsLeanedOn); auto guardsAsked = lookup.conditions(); Vector guardsLeftOut; if (leansOnGuards) { @@ -3991,7 +4148,7 @@ Reference buildReadThatTestsShapeAndCell(Walker& walker, const BodyValue& read, return { }; Vector conditions; WordsListedOnce conditionsListed; - if (asked->contains(CellFact::HasLayoutPrototype)) { + if (readAsksGuardsForTypedAccess(*receivers, *asked)) { for (LayoutID member : receivers->members) { LayoutID classLayout; if (whoAnswersForWhatStandsAboveObjectsOf(walker, member, key, LookupAbove::OfRead, classLayout) != WhoAnswersForWhatStandsAbove::GuardOfClass || walker.state().guardOfReadsHolds(classLayout, key)) @@ -4055,7 +4212,7 @@ bool buildStoreThatTestsForItself(Walker& walker, const BodyValue& store, Refere ValueID written = store.id(); if (findLikelyOrderWordInFrontOfStores(walker, written, singleElementSpan(written), object, guessed)) return false; - if (ahead.askedOfCell.contains(CellFact::HasLayoutPrototype) || storeAsksGuardsWherePrototypeStaysForTypedAccess(walker, *receivers, field, key)) { + if (storeAsksGuardsForTypedAccess(walker, *receivers, field, key, ahead.askedOfCell)) { for (LayoutID member : receivers->members) { LayoutID classLayout; if (whoAnswersForWhatStandsAboveObjectsOf(walker, member, key, LookupAbove::OfStore, classLayout) != WhoAnswersForWhatStandsAbove::GuardOfClass || walker.shouldAssumeGuardsAndFacts() || walker.state().storesPassAbove(classLayout, key)) @@ -4177,9 +4334,13 @@ void lookAtCellInFrontOfStores(Walker& walker, std::span stores, if (!method && askLooksForAccessorAbove(access, key)) continue; if (!method) { + if (findMemberOfObjectPrototypeForTypedAccess(walker, *receivers, field, key)) { + asked.add(askAheadAboutReadOffObjectPrototype(*receivers).value_or(OptionSet())); + continue; + } if (std::optional> askedByRead = askAheadAboutReadThatFindsNothing(walker, *receivers, field, key)) { asked.add(*askedByRead - cellFactsForKeyedLookup(walker, object, key)); - if (askedByRead->contains(CellFact::HasLayoutPrototype)) + if (readAsksGuardsForTypedAccess(*receivers, *askedByRead)) askGuardsOfClasses(key, LookupAbove::OfRead); } continue; @@ -4202,7 +4363,7 @@ void lookAtCellInFrontOfStores(Walker& walker, std::span stores, if (!askedByRead) continue; asked.add(*askedByRead); - if (askedByRead->contains(CellFact::HasLayoutPrototype)) + if (readAsksGuardsForTypedAccess(*receivers, *askedByRead)) askGuardsOfClasses(key, LookupAbove::OfRead); continue; } @@ -4218,7 +4379,7 @@ void lookAtCellInFrontOfStores(Walker& walker, std::span stores, if (ahead.creates && !someStoreCreates) firstKeyCreated = key; someStoreCreates |= ahead.creates; - if (ahead.creates && !isDefinition && (ahead.askedOfCell.contains(CellFact::HasLayoutPrototype) || storeAsksGuardsWherePrototypeStaysForTypedAccess(walker, *receivers, field, key))) + if (ahead.creates && !isDefinition && storeAsksGuardsForTypedAccess(walker, *receivers, field, key, ahead.askedOfCell)) askGuardsOfClasses(key, LookupAbove::OfStore); } @@ -4295,7 +4456,7 @@ void lookAtCellInFrontOfStores(Walker& walker, std::span stores, Reference passed = pass(WTF::move(test)); if (!passed) return; - noteGuardsShownByTest(walker, conditions.span(), passed); + noteGuardsShownByTest(walker, conditions.span(), passed, root); } for (auto [id, method] : methodsReadOffHolder) { Draft written = walker.draftOf(walker.asWalked(index.value(id))); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkTypedElements.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkTypedElements.cpp index 69be9afda19a9..8ecf072ffc48d 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkTypedElements.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkTypedElements.cpp @@ -195,6 +195,26 @@ static bool whatIsStoredIsWrittenToBeNoNumberInTypedElements(const AccessOfTyped return arrayStorageHolding(types, written) == ArrayStorage::Contiguous && !isDeclaredToHoldNumbersAlone(access.walker.program(), types, access.typeOfBase); } +static bool storesOwnListElement(const AccessOfTypedElements& access) +{ + const BodyIndex& index = access.walker.currentBodyIndex(); + BodyValue read = index.value(index.stripCasts(access.value.valueOperand())); + if (read.isParameter() || read.opcode() != Opcode::ElemGet) + return false; + Reference list = access.walker.referenceOf(read.objectOperand()); + if (!list || !list.isValue() || !access.base || !access.base.isValue()) + return false; + return underlyingCellOf(access.walker.code(), list.id()) == underlyingCellOf(access.walker.code(), access.base.id()); +} + +static void requestStorageForStoredValue(const AccessOfTypedElements& access) +{ + if (whatIsStoredIsWrittenToBeNoNumberInTypedElements(access)) + askBirthOfListForStorage(access.walker, access.base, ArrayStorage::Contiguous); + if (!storesOwnListElement(access)) + askBirthOfListForStorageThatTakes(access.walker, access.base, access.stored); +} + static ArrayStorage storageGuessedForStoreOfTypedElements(const AccessOfTypedElements& access) { TypeArena& types = access.walker.types(); @@ -520,14 +540,20 @@ static bool everyUserTakesUndefinedForNaNInTypedElements(Walker& walker, const B const BodyIndex& index = walker.currentBodyIndex(); ValueID name = read.id(); ValueID user = index.onlyUsingValue(name); + auto standsBehindGuardedReads = [&](ValueID taken) { + if (!walker.guardedListOfReads()) + return false; + BodyBlock block = index.block(index.blockOf(read.id())); + return !block.holds(taken) || taken.index() - block.firstValue().index() >= walker.endOfStretchBeingWalked(); + }; while (user && opcodesThatStateTypes.contains(index.opcodeOf(user))) { name = user; user = index.onlyUsingValue(user); } if (user) - return userTakesUndefinedForNaNInTypedElements(walker, read, name, user, beside, false, readGivesUpItsHole); + return !standsBehindGuardedReads(user) && userTakesUndefinedForNaNInTypedElements(walker, read, name, user, beside, false, readGivesUpItsHole); return everyUseStandsInBlockOfValueAndPassesInTypedElements(walker, name, [&](ValueID userOfSeveral) { - return userTakesUndefinedForNaNInTypedElements(walker, read, name, userOfSeveral, beside, true, readGivesUpItsHole); + return !standsBehindGuardedReads(userOfSeveral) && userTakesUndefinedForNaNInTypedElements(walker, read, name, userOfSeveral, beside, true, readGivesUpItsHole); }); } @@ -542,7 +568,10 @@ static Reference numberMadeOnceOfElementInTypedElements(Walker& walker, const Bo ValueID user = index.onlyUsingValue(read.id()); while (user && opcodesThatStateTypes.contains(index.opcodeOf(user))) user = index.onlyUsingValue(user); - if (user || !everyUserTakesUndefinedForNaNInTypedElements(walker, read, WhatStandsBesideRead::NumberOrNothing, false)) + LoopID loop = index.loopOf(index.blockOf(read.id())); + LoopID loopOfUser = user ? index.loopOf(index.blockOf(user)) : LoopID(); + bool userIsInInnerLoop = loopOfUser && loopOfUser != loop && index.loop(loopOfUser).parent == loop; + if ((user && !userIsInInnerLoop) || !everyUserTakesUndefinedForNaNInTypedElements(walker, read, WhatStandsBesideRead::NumberOrNothing, false)) return element; Origin origin = walker.originOf(read); origin.isTailCall = false; @@ -609,6 +638,60 @@ static bool turnTakesWhatIsReadForKeyOrStoresItIntoViewInTypedElements(Walker& w return false; } +static bool elementMeetsWrittenNumberInTypedElements(Walker& walker, const BodyValue& read) +{ + const BodyIndex& index = walker.currentBodyIndex(); + if (read.id().index() >= index.numberOfValues() || index.value(read.id()).words() != read.words()) + return false; + LoopID loop = index.loopOf(index.blockOf(read.id())); + if (!loop) + return false; + ValueID name = read.id(); + std::optional user = index.onlyUser(name); + while (user && user->kind == BodyUseKind::Operand && opcodesThatStateTypes.contains(index.opcodeOf(user->value))) { + name = user->value; + user = index.onlyUser(name); + } + if (!user || user->kind != BodyUseKind::EdgeArgument) + return false; + BlockID join; + size_t position = 0; + for (BodyTarget target : index.terminator(user->block).targets()) { + std::span arguments = target.arguments(); + if (auto found = std::ranges::find(arguments, name); found != arguments.end()) { + join = target.block(); + position = found - arguments.begin(); + } + } + if (!join || index.isLoopHead(join) || index.loopOf(join) != loop) + return false; + for (BlockID member : index.membersOf(loop)) { + for (BodyTarget target : index.terminator(member).targets()) { + if (target.block() != join || position >= target.arguments().size()) + continue; + ValueID other = index.stripCasts(target.arguments()[position]); + if (other && !index.isParameter(other) && (index.opcodeOf(other) == Opcode::Integer || index.opcodeOf(other) == Opcode::Double)) + return true; + } + } + return false; +} + +static bool turnReckonsWithElementInTypedElements(Walker& walker, const BodyValue& read) +{ + static constexpr OpcodeSet operatorsOnNumbersAlone { Opcode::Sub, Opcode::Mul, Opcode::Div }; + const BodyIndex& index = walker.currentBodyIndex(); + if (read.id().index() >= index.numberOfValues() || index.value(read.id()).words() != read.words()) + return false; + std::optional user = index.firstUser(read.id()); + while (user && user->kind == BodyUseKind::Operand && opcodesThatStateTypes.contains(index.opcodeOf(user->value))) + user = index.firstUser(user->value); + if (!user || user->kind != BodyUseKind::Operand) + return false; + Opcode opcode = index.opcodeOf(user->value); + return operatorsOnNumbersAlone.contains(opcode) || (opcode == Opcode::Add && onlyUserTakesUndefinedForNaNInTypedElements(walker, read)); +} + static void jumpToGuardFallback(Walker& walker) { if (walker.stretchMayRunAsWrittenInstead()) @@ -637,19 +720,16 @@ static bool canGuardAccess(const AccessOfTypedElements& access) return access.isOnlyAskedAbout ? access.walker.looksTakeFormsBehindLooksAsBuilt() : access.walker.stretchMayRunAsWrittenInstead() || access.walker.canBuildIterationGuard() || access.walker.canGuardAfterStores(); } -static bool askElementInTurnOfTypedElements(const AccessOfTypedElements& access, Draft& read, const HoistedArrayGuard& tested) +bool guardElementPresenceInIteration(Walker& walker, Draft& read, Reference list, Reference position, const HoistedArrayGuard& tested) { - Walker& walker = access.walker; if (!read.keyIsIntegral() || !hasArrayStorageForm(read.arrayStorage())) return true; - Origin origin = walker.originOf(access.value); - origin.isTailCall = false; - Draft isPresent(Opcode::IsElementPresent, origin); + Draft isPresent(Opcode::IsElementPresent, read.origin()); walker.chooseConservativeFields(isPresent); - isPresent.setArrayOperand(access.base); - isPresent.setPositionOperand(access.key); + isPresent.setArrayOperand(list); + isPresent.setPositionOperand(position); isPresent.setKeyIsIntegral(true); - if (std::optional inside = walker.findWhatHoldsPositionInsideList(access.base, access.key)) + if (std::optional inside = walker.findWhatHoldsPositionInsideList(list, position)) isPresent.setIsInBounds(*inside); nameKindOfListTestedInFrontOfLoop(isPresent, tested); takeStorageAndLengthFromInFrontOfLoop(walker, isPresent, tested); @@ -664,10 +744,10 @@ static bool askElementInTurnOfTypedElements(const AccessOfTypedElements& access, jumpToGuardFallback(walker); walker.continueIn(found); walker.recordSpeculation(); - std::optional present = walker.isUnreachable() || walker.hasFailed() ? std::nullopt : walker.showElementPresentBehindTest(access.base, access.key, walker.findPassedTest(test)); + std::optional present = walker.isUnreachable() || walker.hasFailed() ? std::nullopt : walker.showElementPresentBehindTest(list, position, walker.findPassedTest(test)); if (!present) return false; - noteKnown(walker, KnownNote::PositionFoundInsideList, access.base, access.key, 0, LocationMasks(AbstractHeapSet { AbstractHeap::ArrayStorage }), test.bits()); + noteKnown(walker, KnownNote::PositionFoundInsideList, list, position, 0, LocationMasks(AbstractHeapSet { AbstractHeap::ArrayStorage }), test.bits()); read.setIsPresent(*present); return true; } @@ -823,34 +903,102 @@ static bool buildReadBehindTestsInTurnOfTypedElements(const AccessOfTypedElement return true; } -static Reference mostBelowWrittenBoundInTypedElements(Walker& walker, Reference position, Origin origin) +static bool isKeyWithWrittenMaskInTypedElements(const Code& code, ValueID key, const Type& type) +{ + static constexpr OpcodeSet masks { Opcode::BitAnd, Opcode::RShift, Opcode::URShift }; + const Operation& operation = code.at(key); + return masks.contains(operation.opcode) && (code.isConstant(operation.operand(0)) || code.isConstant(operation.operand(1))) && type.integers.maximum <= std::numeric_limits::max(); +} + +enum class WrittenMasks : bool { AreLeft, AreTaken }; + +static bool isBelowWrittenBoundInTypedElements(const Code& code, ValueID position, unsigned depth = 0) { + static constexpr OpcodeSet operators { Opcode::Add, Opcode::Sub, Opcode::Mul, Opcode::LShift }; static constexpr OpcodeSet relations { Opcode::Less, Opcode::LessEq }; + constexpr unsigned deepestOperator = 3; + while (conversionsOfForm.contains(code.at(position).opcode)) + position = code.at(position).operand(0); + const Operation& made = code.at(position); + if (code.isConstant(position)) + return made.opcode == Opcode::Integer; + if (operators.contains(made.opcode)) + return depth < deepestOperator && isBelowWrittenBoundInTypedElements(code, made.operand(0), depth + 1) && isBelowWrittenBoundInTypedElements(code, made.operand(1), depth + 1); + if (made.opcode != Opcode::Narrowed || !made.isOnTakenEdge() || made.operandIndex() != OperandIndex::Less::left) + return false; + const Operation& relation = code.at(made.testOperand()); + return relations.contains(relation.opcode) && code.isConstant(relation.operand(OperandIndex::Less::right)); +} + +static Reference mostBelowWrittenBoundInTypedElements(Walker& walker, Reference position, Origin origin, WrittenMasks masks = WrittenMasks::AreLeft) +{ Type type = walker.types().at(walker.typeOf(position)); - if (!position.isValue() || !type.isInteger() || type.integers.minimum < 0) + if (!position.isRecord() || !type.isInteger() || type.integers.minimum < 0 || (position.isConstant() && (masks == WrittenMasks::AreTaken || type.integers.maximum > std::numeric_limits::max()))) return { }; const Code& code = walker.code(); static constexpr OpcodeSet sums { Opcode::Add, Opcode::Sub }; - bool changesFromTurnToTurn = !walker.isInvariantInSpecializedVersion(position); + bool changesFromTurnToTurn = position.isValue() && !walker.isInvariantInSpecializedVersion(position); ValueID held = position.id(); while (conversionsOfForm.contains(code.at(held).opcode) || (changesFromTurnToTurn && sums.contains(code.at(held).opcode) && code.isConstant(code.at(held).operand(1)))) held = code.at(held).operand(0); - const Operation& narrowed = code.at(held); - if (narrowed.opcode != Opcode::Narrowed || !narrowed.isOnTakenEdge() || narrowed.operandIndex() != OperandIndex::Less::left) - return { }; - const Operation& relation = code.at(narrowed.testOperand()); - if (!relations.contains(relation.opcode) || !code.isConstant(relation.operand(OperandIndex::Less::right))) + if ((masks == WrittenMasks::AreLeft || !isKeyWithWrittenMaskInTypedElements(code, held, type)) && !isBelowWrittenBoundInTypedElements(code, held)) return { }; Draft most(Opcode::Integer, origin); most.setInteger(type.integers.maximum); return walker.constant(WTF::move(most)); } -static void hoistArrayBoundsCheck(const AccessOfTypedElements& access, const HoistedArrayGuard& tested) +static std::optional indexesOverTurnsInTypedElements(Walker& walker, const BodyIndex& index, ValueID written, unsigned depth = 0) +{ + static constexpr OpcodeSet operators { Opcode::Add, Opcode::Sub, Opcode::Mul }; + static constexpr OpcodeSet relations { Opcode::Less, Opcode::LessEq }; + constexpr unsigned deepestOperator = 3; + ValueID value = index.stripCasts(written); + if (!value) + return std::nullopt; + if (!index.isParameter(value) && operators.contains(index.opcodeOf(value)) && depth < deepestOperator) { + Opcode opcode = index.opcodeOf(value); + std::optional left = indexesOverTurnsInTypedElements(walker, index, index.value(value).operand(0), depth + 1); + std::optional right = left ? indexesOverTurnsInTypedElements(walker, index, index.value(value).operand(1), depth + 1) : std::nullopt; + if (!right) + return std::nullopt; + Range made; + if (opcode == Opcode::Add) + made = { left->minimum + right->minimum, left->maximum + right->maximum }; + else if (opcode == Opcode::Sub) + made = { left->minimum - right->maximum, left->maximum - right->minimum }; + else + made = { left->minimum * right->minimum, left->maximum * right->maximum }; + return made.minimum >= 0 && made.maximum <= std::numeric_limits::max() ? std::optional(made) : std::nullopt; + } + Reference held = walker.referenceOf(value); + if (!held) + return std::nullopt; + Type type = walker.types().at(walker.typeOf(held)); + if (!type.isInteger() || type.integers.minimum < 0 || type.integers.maximum > std::numeric_limits::max()) + return std::nullopt; + Range found { type.integers.minimum, type.integers.maximum }; + BlockID head = walker.headOfTurnsWrittenOut(); + if (!index.isParameter(value) || index.blockOf(value) != head) + return found; + BodyTerminator end = index.terminator(head); + ValueID relation = end.isConditionalBranch() ? end.operand(0) : ValueID(); + if (!relation || index.isParameter(relation) || !relations.contains(index.opcodeOf(relation))) + return found; + ValueID bound = index.stripCasts(index.value(relation).operand(OperandIndex::Less::right)); + if (index.stripCasts(index.value(relation).operand(OperandIndex::Less::left)) != value || !bound || index.isParameter(bound) || index.opcodeOf(bound) != Opcode::Integer) + return found; + int64_t last = index.value(bound).as().integer() - (index.opcodeOf(relation) == Opcode::Less); + if (last <= std::numeric_limits::max()) + found.maximum = std::max(found.maximum, last); + return found; +} + +static void hoistArrayBoundsCheck(const AccessOfTypedElements& access) { Walker& walker = access.walker; Walker::IterationPurity purity = walker.arrayGuardPurity(); - if (tested.isWritable || tested.turnsStoreIntoLists || !access.typeOfKey.isInteger() || !walker.isInvariantInSpecializedVersion(access.base, purity) || !walker.iterationsPreserveArrayGuards(access.base, purity)) + if (!access.typeOfKey.isInteger() || !walker.isInvariantInSpecializedVersion(access.base, purity) || !walker.iterationsPreserveArrayGuards(access.base, purity)) return; if (walker.findWhatHoldsPositionInsideList(access.base, access.key) || walker.findWhatHoldsPositionBelowRoomOfView(access.base, access.key)) return; @@ -859,6 +1007,14 @@ static void hoistArrayBoundsCheck(const AccessOfTypedElements& access, const Hoi Reference most = mostBelowWrittenBoundInTypedElements(walker, access.key, origin); if (!most) return; + const BodyIndex& index = walker.currentBodyIndex(); + bool isInBody = access.value.id().index() < index.numberOfValues() && index.value(access.value.id()).words() == access.value.words(); + if (std::optional overTurns = isInBody && walker.headOfTurnsWrittenOut() ? indexesOverTurnsInTypedElements(walker, index, access.value.propertyOperand()) : std::nullopt; overTurns && overTurns->maximum > access.typeOfKey.integers.maximum) { + Draft mostOfTurns(Opcode::Integer, origin); + mostOfTurns.setInteger(overTurns->maximum); + most = walker.constant(WTF::move(mostOfTurns)); + walker.count(StatisticID::LengthsAskedForUnrolledTurns); + } Reference length; Reference isBelow; { @@ -892,10 +1048,10 @@ static void hoistArrayBoundsCheck(const AccessOfTypedElements& access, const Hoi walker.count(StatisticID::ArrayBoundsChecksHoisted); } -static constexpr AbstractHeapSet heapsUnderElementOfFreshListInTypedElements { +static constexpr AbstractHeapSet heapsUnderElementInTypedElements { AbstractHeap::ArrayStorage, AbstractHeap::Elements, AbstractHeap::NamedProperties, AbstractHeap::PrototypeLinks, AbstractHeap::ShapeWords }; -static_assert(perObjectHeaps.containsAll(heapsUnderElementOfFreshListInTypedElements)); +static_assert(perObjectHeaps.containsAll(heapsUnderElementInTypedElements)); static bool keyGoesRoundLoopInTypedElements(const AccessOfTypedElements& access) { @@ -917,7 +1073,10 @@ static std::optional tryGuardIntegerKey(const AccessOfTyp return std::nullopt; if (!walker.has(Mechanism::Guesses) || !walker.opensGuesses() || !walker.canRestartIteration() || walker.readStandsUnderHandler()) return std::nullopt; - if (keyGoesRoundLoopInTypedElements(access) || !isDeclaredToHoldNumbersAlone(walker.program(), walker.types(), access.typeOfBase)) + TypeID linkedElementType = walker.facts().linked().resultOfReadByKey(walker.currentFunction(), access.value.id()); + bool holdsNumbersAlone = isDeclaredToHoldNumbersAlone(walker.program(), walker.types(), access.typeOfBase) + || (linkedElementType && (walker.types().at(linkedElementType).primitives & Primitive::Number) && walker.types().at(linkedElementType).hasOnlyPrimitives(Primitive::Number | Primitive::Undefined)); + if (keyGoesRoundLoopInTypedElements(access) || !holdsNumbersAlone) return std::nullopt; Origin origin = walker.originOf(access.value); origin.isTailCall = false; @@ -971,8 +1130,8 @@ static bool buildArrayReadOfTypedElements(const AccessOfTypedElements& access) read.setLevel(level); read.setAsksCellWhereNothingIsFound(restsOnLinksWordAlone && !tested); read.setKeyIsIntegral(access.typeOfKey.isNumberWithoutFraction()); - if (tested && !lengthOfListMadeInSight(walker, access.base)) - hoistArrayBoundsCheck(access, *tested); + if (tested && !lengthOfListMadeInSight(walker, access.base) && (!access.key.isConstant() || tested->turnsStoreIntoLists)) + hoistArrayBoundsCheck(access); std::optional inside = walker.findWhatHoldsPositionInsideList(access.base, access.key); if (!inside && tested && access.typeOfKey.isInteger()) inside = walker.findWhatHoldsPositionBelowRoomOfView(access.base, access.key); @@ -985,7 +1144,10 @@ static bool buildArrayReadOfTypedElements(const AccessOfTypedElements& access) nameKindOfListTestedInFrontOfLoop(read, *tested); Reference freshListReadUnderKey = namesStorage && access.key.isValue() && isArrayIndexInTypedElements(access.typeOfKey) ? freshObjectNamedInTypedElements(walker, access.base) : Reference(); bool elementIsKnownToBeThere = read.isPresent(); - if (std::optional readBefore = freshListReadUnderKey ? findKnown(walker, KnownNote::ElementReadOffFreshList, freshListReadUnderKey, access.key) : std::nullopt) { + bool isReadUnderNumber = level == KnowledgeLevel::Proven && !tested && !namesStorage && access.base.isValue() && access.key.isValue() && !isArrayIndexInTypedElements(access.typeOfKey) && !freshObjectNamedInTypedElements(walker, access.base); + std::optional readBefore = freshListReadUnderKey ? findKnown(walker, KnownNote::ElementReadOffFreshList, freshListReadUnderKey, access.key) + : isReadUnderNumber ? findKnown(walker, KnownNote::ElementReadUnderNumber, access.base, access.key) : std::nullopt; + if (readBefore && (freshListReadUnderKey || walker.types().isSubset(walker.typeOf(Reference::fromBits(static_cast(*readBefore))), walker.resultTypeOf(read)))) { Reference known = Reference::fromBits(static_cast(*readBefore)); if (walker.checksProofs()) { WalkState believed = walker.state(); @@ -1001,7 +1163,7 @@ static bool buildArrayReadOfTypedElements(const AccessOfTypedElements& access) read.setIsInBounds(*insideAsWritten); read.setIsPresent(*insideAsWritten); } - bool takesNaNForHole = tested && read.isInBounds() && read.arrayStorage() == ArrayStorage::Double && !read.admitsInt32Storage() && everyUserTakesUndefinedForNaNInTypedElements(walker, access.value, WhatStandsBesideRead::NumberOrNothing); + bool takesNaNForHole = tested && read.arrayStorage() == ArrayStorage::Double && (read.isInBounds() ? !read.admitsInt32Storage() || tested->turnsStoreIntoLists : tested->turnsStoreIntoLists && isArrayIndexInTypedElements(access.typeOfKey)) && everyUserTakesUndefinedForNaNInTypedElements(walker, access.value, WhatStandsBesideRead::NumberOrNothing); if (!takesNaNForHole && tested && tested->turnsStoreIntoLists && !tested->isWritable && isNumericStorage(read.arrayStorage()) && onlyUserHoldsItAgainstNumberInTypedElements(walker, access.value)) takesNaNForHole = true; if (takesNaNForHole && tested->turnsStoreIntoLists) @@ -1041,7 +1203,7 @@ static bool buildArrayReadOfTypedElements(const AccessOfTypedElements& access) } } if (!element || !walker.types().at(walker.typeOf(element)).hasOnlyPrimitives(Primitive::Number | Primitive::Undefined)) { - if (leavesTurnAtHole && !isReadInFront && !askElementInTurnOfTypedElements(access, read, *tested)) + if (leavesTurnAtHole && !isReadInFront && !guardElementPresenceInIteration(walker, read, access.base, access.key, *tested)) return true; if (isReadInFront && walker.checksBounds()) readAgain = read; @@ -1070,7 +1232,9 @@ static bool buildArrayReadOfTypedElements(const AccessOfTypedElements& access) walker.state().noteElementsLieOutsideViews(access.base); } if (freshListReadUnderKey && element && element.isValue()) - noteKnown(walker, KnownNote::ElementReadOffFreshList, freshListReadUnderKey, access.key, 0, elementIsKnownToBeThere ? LocationMasks(heapsUnderElementOfFreshListInTypedElements) : LocationMasks::top().without(unobservableWriteHeaps), element.bits()); + noteKnown(walker, KnownNote::ElementReadOffFreshList, freshListReadUnderKey, access.key, 0, elementIsKnownToBeThere ? LocationMasks(heapsUnderElementInTypedElements) : LocationMasks::top().without(unobservableWriteHeaps), element.bits()); + if (isReadUnderNumber && element && element.isValue()) + noteKnown(walker, KnownNote::ElementReadUnderNumber, access.base, access.key, 0, LocationMasks(heapsUnderElementInTypedElements), element.bits()); walker.bind(access.value.id(), numberMadeOnceOfElementInTypedElements(walker, access.value, element)); return true; } @@ -1091,9 +1255,7 @@ static bool buildArrayStoreUnderAnyKeyOfTypedElements(const AccessOfTypedElement put.setArrayStorage(storageGuessedForStoreOfTypedElements(access)); put.setMode(mode); put.setLevel(KnowledgeLevel::Expected); - if (whatIsStoredIsWrittenToBeNoNumberInTypedElements(access)) - askBirthOfListForStorage(walker, access.base, ArrayStorage::Contiguous); - askBirthOfListForStorageThatTakes(walker, access.base, access.stored); + requestStorageForStoredValue(access); markBaseAsPassedInTypedElements(walker, access.base); Reference store = walker.build(WTF::move(put)); noteStoredElementInTypedElements(walker, access.base, access.key, { }); @@ -1140,6 +1302,64 @@ static bool isReadOutOfDoubleStorageInTypedElements(Walker& walker, Reference st return everyArmIsSuchRead; } +static bool continueIfKeyIsNonNegative(Walker& walker, Reference key, Origin origin) +{ + Draft zero(Opcode::Integer, origin); + zero.setInteger(0); + Draft isNotNegative(Opcode::GreaterEq, origin); + walker.chooseConservativeFields(isNotNegative); + isNotNegative.setLeftOperand(key); + isNotNegative.setRightOperand(walker.constant(WTF::move(zero))); + isNotNegative.setForm(operatorFormFor(walker, isNotNegative)); + return continueOrRestartIteration(walker, walker.build(WTF::move(isNotNegative)), true); +} + +static bool testNextStoreKey(const AccessOfTypedElements& access, Origin origin) +{ + Walker& walker = access.walker; + const BodyIndex& index = walker.currentBodyIndex(); + BodyBlock block = index.block(index.blockOf(access.value.id())); + uint32_t next = access.value.id().index() + 1; + uint32_t end = block.firstValue().index() + block.numberOfValues(); + while (next < end && index.opcodeOf(ValueID(next)) != Opcode::ElemSet) + ++next; + if (next >= end) + return true; + BodyValue later = index.value(ValueID(next)); + ValueID written = index.stripCasts(later.propertyOperand()); + if (written.index() > access.value.id().index()) + return true; + Reference key = walker.referenceOf(later.propertyOperand()); + Reference list = nameOfListThatEntersLoopAround(walker, index, later); + std::optional tested = key && key.isRecord() && key != access.key && list ? findListTestedForStoresInFrontOfLoop(walker, list) : std::nullopt; + if (!tested || !tested->isWritable) + return true; + Type typeOfKey = walker.types().at(walker.typeOf(key)); + if (!typeOfKey.isInteger() || positionIsKnownToLieInsideList(walker, list, key)) + return true; + if (typeOfKey.integers.minimum < 0 && !continueIfKeyIsNonNegative(walker, key, origin)) + return false; + Draft length(Opcode::GetArrayLength, origin); + walker.chooseConservativeFields(length); + length.setArrayOperand(tested->name); + Reference held = walker.build(WTF::move(length)); + if (!held) + return false; + Draft lengthComparison(Opcode::Less, origin); + walker.chooseConservativeFields(lengthComparison); + lengthComparison.setLeftOperand(key); + lengthComparison.setRightOperand(held); + lengthComparison.setForm(operatorFormFor(walker, lengthComparison)); + Reference test = walker.build(WTF::move(lengthComparison)); + if (!test || !test.isValue()) + return !walker.isUnreachable() && !walker.hasFailed(); + if (!continueOrRestartIteration(walker, test, true)) + return false; + noteKnown(walker, KnownNote::LengthOfList, list, { }, 0, placesUnderLengthsOfLists, held.bits()); + walker.count(StatisticID::LaterStoreBoundsChecksHoisted); + return true; +} + static bool askKeyOfLaterStoresOfTypedElements(const AccessOfTypedElements& access, Origin origin) { Walker& walker = access.walker; @@ -1172,7 +1392,7 @@ static bool askKeyOfLaterStoresOfTypedElements(const AccessOfTypedElements& acce noteKnown(walker, KnownNote::LengthOfList, list, { }, 0, placesUnderLengthsOfLists, held.bits()); walker.count(StatisticID::LaterStoreBoundsChecksHoisted); } - return true; + return testNextStoreKey(access, origin); } static bool textOfLoopsReadsListByKeyInTypedElements(const BodyIndex& index, const BodyValue& store) @@ -1344,14 +1564,30 @@ static bool buildLengthExtension(const AccessOfTypedElements& access, const Hois return !walker.isUnreachable() && !walker.hasFailed(); } +static bool isPresentInListStoredInto(const AccessOfTypedElements& access) +{ + if (!access.stored || !access.stored.isValue() || !access.base.isRecord()) + return false; + const Code& code = access.walker.code(); + ValueID stored = access.stored.id(); + while (valuePreservingOpcodes.contains(code.at(stored).opcode)) + stored = code.at(stored).operand(0); + const Operation& read = code.at(stored); + return read.opcode == Opcode::GetArrayElement && read.arrayStorage() == ArrayStorage::Contiguous && read.isPresent() && underlyingCellOf(code, read.arrayOperand()) == underlyingCellOf(code, access.base.id()); +} + static bool buildStoreBehindRowOfTypedElements(const AccessOfTypedElements& access) { Walker& walker = access.walker; TypeArena& types = walker.types(); - if (!access.typeOfKey.isInteger() || !access.typeOfStored || !types.at(access.typeOfStored).isNumber()) + if (!access.typeOfKey.isInteger()) return false; + bool storesNumber = access.typeOfStored && types.at(access.typeOfStored).isNumber(); bool isPush = !!access.counterThatPushGoesBy; std::optional tested = isPush ? findListTestedForStoresInFrontOfLoop(walker, access.base) : findHoistedArrayGuard(walker, access.value); + bool hasBoxedElements = tested && tested->isWritable && tested->storage == ArrayStorage::Contiguous; + if (storesNumber ? hasBoxedElements : !hasBoxedElements || isPush || !access.typeOfStored || (types.at(access.typeOfStored).primitives & Primitive::Absent)) + return false; bool rowAsksForWritableList = tested && tested->isWritable; if (isPush && !rowAsksForWritableList) return false; @@ -1360,12 +1596,14 @@ static bool buildStoreBehindRowOfTypedElements(const AccessOfTypedElements& acce bool mayGoByKind = !tested && (asked == ArrayStorage::Double || types.isSubset(access.typeOfStored, TypeTable::int32)) && walker.isInvariantInSpecializedVersion(access.base) && !textOfLoopsReadsListByKeyInTypedElements(walker.currentBodyIndex(), access.value); tested = tryHoistArrayGuardForAccess(walker, access.value, access.base, asked, mayGoByKind); } - bool goesByKind = tested && tested->isWritable && tested->storage == ArrayStorage::Other; + bool goesByKind = tested && tested->isWritable && (tested->storage == ArrayStorage::Other || (tested->storage == ArrayStorage::Double && tested->admitsInt32Storage)); if (goesByKind && tested->admitsInt32Storage && !types.isSubset(access.typeOfStored, TypeTable::int32)) return false; - if (!tested || !tested->isWritable || (!goesByKind && (!isNumericStorage(tested->storage) || tested->admitsInt32Storage))) + if (!tested || !tested->isWritable || (!goesByKind && !hasBoxedElements && (!isNumericStorage(tested->storage) || tested->admitsInt32Storage))) return false; + if (!access.isOnlyAskedAbout && rowAsksForWritableList && !isPush && !notedCounterPosition(access, KnownNote::LoopAppendsToLiteral, tested->name)) + hoistArrayBoundsCheck(access); std::optional inside = access.isOnlyAskedAbout ? std::nullopt : walker.findWhatHoldsPositionInsideList(access.base, access.key); bool isInside = !!inside; if (!isInside && !isArrayIndexInTypedElements(access.typeOfKey) && (!rowAsksForWritableList || keyIsLengthOfListItselfInTypedElements(access))) @@ -1376,9 +1614,10 @@ static bool buildStoreBehindRowOfTypedElements(const AccessOfTypedElements& acce fits = true; if (walker.shouldAssumeNoNaNStores() && (goesByKind || tested->storage == ArrayStorage::Double) && types.at(access.typeOfStored).isNumber()) fits = true; - if ((!isInside || !fits) && !canGuardAccess(access)) + bool mayHoldInt32sAlone = hasBoxedElements && tested->admitsInt32Storage && !isPresentInListStoredInto(access); + if ((!isInside || !fits || mayHoldInt32sAlone) && !canGuardAccess(access)) return false; - Opcode opcode = tested->storage == ArrayStorage::Double ? Opcode::PutDoubleIndex : Opcode::PutContiguousIndex; + Opcode opcode = tested->storage == ArrayStorage::Double && !goesByKind ? Opcode::PutDoubleIndex : Opcode::PutContiguousIndex; bool appendsToLiteral = !isInside && notedCounterPosition(access, KnownNote::LoopAppendsToLiteral, tested->name) && literalMayHoldStorageAskedInTypedElements(walker, *tested); if (access.isOnlyAskedAbout) return answerAheadOfTypedElements(access, appendsToLiteral ? Opcode::ExtendArrayLength : opcode, KnowledgeLevel::Proven); @@ -1463,6 +1702,48 @@ static bool buildStoreBehindRowOfTypedElements(const AccessOfTypedElements& acce } } + BlockID behindKinds; + if (mayHoldInt32sAlone) { + Draft kind(Opcode::HasArrayStorage, origin); + walker.chooseConservativeFields(kind); + kind.setArrayOperand(access.base); + kind.setArrayStorage(ArrayStorage::Contiguous); + kind.setIsWritable(true); + Reference holdsAnyValues = walker.build(WTF::move(kind)); + if (!holdsAnyValues) + return true; + BlockID anyValues = walker.addBlock(); + BlockID int32sAlone = walker.addBlock(); + behindKinds = walker.addBlock(); + walker.branch(holdsAnyValues, anyValues, int32sAlone); + walker.continueIn(int32sAlone); + if (!walker.isUnreachable()) { + Reference isInt32 = buildTestOfNumber(walker, stored, TypeTable::int32, origin); + if (continueOrRestartIteration(walker, isInt32)) { + Draft whole = draftOfTypedElements(walker, access.value, opcode); + whole.setArrayOperand(access.base); + whole.setPositionOperand(access.key); + whole.setValueOperand(buildProofOfNumber(walker, stored, isInt32, origin)); + if (inside) + whole.setIsInBounds(*inside); + if (Reference pointer = storageOfListFromInFrontOfLoop(walker, *tested, origin)) + whole.setStorageOperand(pointer); + if (whole.valueOperand() && walker.build(WTF::move(whole)) && !walker.isUnreachable()) + walker.jumpTo(behindKinds); + } + } + if (walker.hasFailed()) + return true; + walker.continueIn(anyValues); + if (walker.isUnreachable()) { + walker.continueIn(behindKinds); + noteStoredElementInTypedElements(walker, access.base, access.key, stored); + walker.bind(access.value.id(), Reference()); + walker.letRestOfStretchRunAsWrittenBehindFirstStore(access.value.id()); + return true; + } + walker.count(StatisticID::BoxedStoresByKind); + } Draft put = isPush ? Draft(opcode, origin) : draftOfTypedElements(walker, access.value, opcode); if (isPush) { walker.chooseConservativeFields(put); @@ -1475,6 +1756,8 @@ static bool buildStoreBehindRowOfTypedElements(const AccessOfTypedElements& acce put.setIsInBounds(*inside); if (goesByKind) put.setGoesByKindOfStorage(true); + if (opcode == Opcode::PutContiguousIndex && isNumericStorage(tested->storage)) + put.setHasNumericStorage(true); if (valueBeforeStore && walker.canRecordUndoableStore()) walker.recordUndoableElementStore(access.base, access.key, valueBeforeStore); if (Reference pointer = storageOfListFromInFrontOfLoop(walker, *tested, origin)) @@ -1483,6 +1766,12 @@ static bool buildStoreBehindRowOfTypedElements(const AccessOfTypedElements& acce walker.count(StatisticID::DoubleElementsCopiedUnchecked); Reference store = walker.build(WTF::move(put)); walker.takePendingUndoableStore(); + if (behindKinds) { + if (!walker.isUnreachable()) + walker.jumpTo(behindKinds); + walker.continueIn(behindKinds); + store = { }; + } noteStoredElementInTypedElements(walker, access.base, access.key, stored); walker.bind(access.value.id(), store); walker.letRestOfStretchRunAsWrittenBehindFirstStore(access.value.id()); @@ -1526,9 +1815,7 @@ static bool buildArrayStoreOfTypedElements(const AccessOfTypedElements& access) put.setKeyIsIntegral(access.typeOfKey.isNumberWithoutFraction()); if (std::optional inside = level == KnowledgeLevel::Expected ? walker.findWhatHoldsPositionInsideList(access.base, access.key) : std::nullopt) put.setIsInBounds(*inside); - if (whatIsStoredIsWrittenToBeNoNumberInTypedElements(access)) - askBirthOfListForStorage(walker, access.base, ArrayStorage::Contiguous); - askBirthOfListForStorageThatTakes(walker, access.base, access.stored); + requestStorageForStoredValue(access); if (level == KnowledgeLevel::Expected) markBaseAsPassedInTypedElements(walker, access.base); Reference store = walker.build(WTF::move(put)); @@ -1565,6 +1852,8 @@ static Reference lengthOfViewInTypedElements(const AccessOfTypedElements& access Walker& walker = access.walker; if (std::optional noted = walker.state().answers().isEmpty() ? std::nullopt : findKnown(walker, KnownNote::LengthOfList, access.base)) return Reference::fromBits(static_cast(*noted)); + if (std::optional noted = walker.state().answers().isEmpty() ? std::nullopt : findKnown(walker, KnownNote::RoomOfViewIsAsInFrontOfLoop, access.base)) + return Reference::fromBits(static_cast(*noted)); if (!walker.isInLoop() || !walker.opensGuesses()) return { }; Draft length(Opcode::GetTypedArrayLength, origin); @@ -1583,7 +1872,10 @@ static Reference lengthOfViewInTypedElements(const AccessOfTypedElements& access return takeLength(); const BodyIndex& index = walker.currentBodyIndex(); ValueID written = access.value.id(); - bool isNamedAgain = written.index() < index.numberOfValues() && index.value(written).words() == access.value.words() && index.useCount(access.value.objectOperand()) > 1; + bool isOfThisBody = written.index() < index.numberOfValues() && index.value(written).words() == access.value.words(); + bool isNamedAgain = isOfThisBody && index.useCount(access.value.objectOperand()) > 1; + if (BlockID head = walker.headOfTurnsWrittenOut(); head && isOfThisBody && !isNamedAgain) + isNamedAgain = index.isInLoop(index.blockOf(written), index.loopOf(head)) && index.stripCasts(access.value.objectOperand()).index() < index.block(head).firstParameter().index(); isNamedAgain |= walker.standsInStretchWithArmsAsWritten() && walker.stretchWithArmsAsWritten().list == access.base; return isNamedAgain ? takeLength() : Reference(); } @@ -1615,11 +1907,28 @@ static bool viewIsUsedElsewhereInTypedElements(Walker& walker, const BodyValue& return isOfThisBody && !index.hasOneUse(index.stripCasts(access.objectOperand())); } -static Reference mostOfSmallRangeInTypedElements(Walker& walker, Reference position, Origin origin) +static bool viewIsWrittenNoLongerThan(const Code& code, Reference view, int64_t length) +{ + if (!view || !view.isValue()) + return false; + ValueID cell = underlyingCellOf(code, view.id()); + if (code.isConstant(cell)) + return false; + const Operation& made = code.at(cell); + if (made.opcode != Opcode::CallInline || made.inlineBuiltin() != InlineBuiltin::TypedArrayWithLength || !made.numberOfVariadicOperands()) + return false; + ValueID written = made.variadicOperands()[0]; + return code.isConstant(written) && code.at(written).opcode == Opcode::Integer && code.at(written).integer() <= length; +} + +static Reference mostOfSmallRangeInTypedElements(Walker& walker, const BodyValue& read, Reference view, Reference position, Origin origin) { Type type = walker.types().at(walker.typeOf(position)); if (!position.isValue() || !type.isInteger() || type.integers.minimum < 0 || type.integers.maximum > mostOfRangeAskedWholeInTypedElements) return { }; + uint32_t least = walker.facts().linked().leastLengthOfViewsReadAt(walker.currentFunction(), read.id()); + if (viewIsWrittenNoLongerThan(walker.code(), view, type.integers.maximum) || (least && least <= type.integers.maximum)) + return { }; Draft most(Opcode::Integer, origin); most.setInteger(type.integers.maximum); return walker.constant(WTF::move(most)); @@ -1646,8 +1955,11 @@ static std::optional tryGuardIndexInBounds(const AccessOfTypedElemen return std::nullopt; if (!access.typeOfKey.isInteger() || positionIsKnownToLieInsideList(walker, access.base, access.key)) return std::nullopt; - bool turnNeedsNumber = turnTakesWhatIsReadForKeyOrStoresItIntoViewInTypedElements(walker, access.value); + bool isReckonedWith = elementMeetsWrittenNumberInTypedElements(walker, access.value) || turnReckonsWithElementInTypedElements(walker, access.value); + bool turnNeedsNumber = isReckonedWith || turnTakesWhatIsReadForKeyOrStoresItIntoViewInTypedElements(walker, access.value); std::optional fromZeroUp = walker.findNonNegativeProof(access.key); + if (!fromZeroUp) + fromZeroUp = walker.findTestThatHoldsDifferenceFromZeroUp(access.key); if ((!fromZeroUp && !turnNeedsNumber) || (!turnNeedsNumber && !walker.stretchMayRunAsWrittenInstead() && !onlyUserTakesUndefinedForNaNInTypedElements(walker, access.value))) return std::nullopt; origin.isTailCall = false; @@ -1669,6 +1981,8 @@ static std::optional tryGuardIndexInBounds(const AccessOfTypedElemen walker.recordSpeculation(); return walker.isUnreachable() ? std::nullopt : passes; }; + if (isReckonedWith) + walker.count(StatisticID::ReadsGuardedForNumbers); Reference key = access.key; if (!fromZeroUp) { Draft zero(Opcode::Integer, origin); @@ -1696,7 +2010,7 @@ static std::optional tryGuardIndexInBounds(const AccessOfTypedElemen } } } - Reference most = mostOfSmallRangeInTypedElements(walker, key, origin); + Reference most = mostOfSmallRangeInTypedElements(walker, access.value, access.base, key, origin); if (most && !viewIsUsedElsewhereInTypedElements(walker, access.value)) most = { }; Draft test(Opcode::Less, origin); @@ -1855,7 +2169,7 @@ static void hoistTypedArrayBoundsCheck(const AccessOfTypedElements& access, Type return; Origin origin = walker.originOf(access.value); origin.isTailCall = false; - Reference most = mostBelowWrittenBoundInTypedElements(walker, access.key, origin); + Reference most = mostBelowWrittenBoundInTypedElements(walker, access.key, origin, WrittenMasks::AreTaken); if (!most && access.key.isConstant() && access.typeOfKey.integers.minimum >= 0) most = access.key; if (!most) @@ -1928,7 +2242,7 @@ Reference buildReadOfElementOfViewBehindTestOfLength(Walker& walker, const BodyV if (!fromZeroUp) return { }; } - Reference most = readIsUpdated ? Reference() : mostOfSmallRangeInTypedElements(walker, position, origin); + Reference most = readIsUpdated ? Reference() : mostOfSmallRangeInTypedElements(walker, read, view, position, origin); if (most && !viewIsUsedElsewhereInTypedElements(walker, read)) most = { }; Draft isBelow(Opcode::Less, origin); @@ -1971,6 +2285,53 @@ Reference buildReadOfElementOfViewBehindTestOfLength(Walker& walker, const BodyV return walker.build(WTF::move(element)); } +static bool hasKeyWithWrittenMaskInTypedElements(const AccessOfTypedElements& access) +{ + if (!access.key.isValue() || !access.typeOfKey.isInteger() || access.typeOfKey.integers.minimum < 0) + return false; + const Code& code = access.walker.code(); + ValueID key = access.key.id(); + while (conversionsOfForm.contains(code.at(key).opcode)) + key = code.at(key).operand(0); + return isKeyWithWrittenMaskInTypedElements(code, key, access.typeOfKey); +} + +static bool isTermOfTruncatedSumInTypedElements(const AccessOfTypedElements& access, ValueID sum) +{ + static constexpr OpcodeSet sums { Opcode::Add, Opcode::Sub }; + Walker& walker = access.walker; + const BodyIndex& index = walker.currentBodyIndex(); + if (!sums.contains(index.opcodeOf(sum))) + return false; + ValueID user = index.onlyUsingValue(sum); + while (user && opcodesThatStateTypes.contains(index.opcodeOf(user))) + user = index.onlyUsingValue(user); + if (!user || !allBitwiseOperators.contains(index.opcodeOf(user))) + return false; + ValueID other = index.value(sum).operand(index.stripCasts(index.value(sum).operand(0)) == access.value.id()); + Reference term = walker.referenceBoundTo(index.stripCasts(other)); + TypeID type = term && term.isRecord() ? walker.typeOf(term) : TypeID(); + return type && walker.types().isSubset(type, TypeTable::int32); +} + +static bool integerIsTakenAsDoubleInTypedElements(const AccessOfTypedElements& access, TypedArrayType typedArrayType) +{ + static constexpr OpcodeSet operatorsOnDoubles = OpcodeSet { Opcode::Add, Opcode::Sub, Opcode::Mul, Opcode::Div } | allRelations; + Walker& walker = access.walker; + const BodyIndex& index = walker.currentBodyIndex(); + ValueID read = access.value.id(); + if (!walker.has(Mechanism::UnboxedNumbers) || (!isInt(typedArrayType) && !isClamped(typedArrayType)) || !readsElementsInLineInTypedElements(typedArrayType)) + return false; + if (read.index() >= index.numberOfValues() || index.value(read).words() != access.value.words()) + return false; + ValueID user = index.onlyUsingValue(read); + while (user && opcodesThatStateTypes.contains(index.opcodeOf(user))) + user = index.onlyUsingValue(user); + if (!user || !operatorsOnDoubles.contains(index.opcodeOf(user)) || isTermOfTruncatedSumInTypedElements(access, user)) + return false; + return everyUserTakesUndefinedForNaNInTypedElements(walker, access.value, WhatStandsBesideRead::NumberOrNothing, false); +} + static bool buildViewReadOfTypedElements(const AccessOfTypedElements& access, TypedArrayType typedArrayType) { if (access.isOnlyAskedAbout) @@ -1980,7 +2341,7 @@ static bool buildViewReadOfTypedElements(const AccessOfTypedElements& access, Ty Walker& walker = access.walker; Draft read = draftOfTypedElements(walker, access.value, Opcode::GetTypedArrayElement); std::optional inside = access.typeOfKey.isInteger() ? walker.findWhatHoldsPositionBelowRoomOfView(access.base, access.key) : std::nullopt; - if (!inside && access.key.isConstant()) { + if (!inside && (access.key.isConstant() || hasKeyWithWrittenMaskInTypedElements(access))) { inside = walker.findWhatHoldsPositionInsideList(access.base, access.key); if (!inside) { hoistTypedArrayBoundsCheck(access, typedArrayType); @@ -2008,6 +2369,10 @@ static bool buildViewReadOfTypedElements(const AccessOfTypedElements& access, Ty walker.noteLengthOfViewAskedForWrittenPlace(access.base); read.setLevel(KnowledgeLevel::Proven); read.setKeyIsIntegral(access.typeOfKey.isNumberWithoutFraction()); + if (!inside && integerIsTakenAsDoubleInTypedElements(access, typedArrayType)) { + read.representation = Representation::OptionalDouble; + walker.count(StatisticID::IntegersOfViewsReadAsDoubles); + } nameStorageOfViewInTypedElements(access, read, typedArrayType); walker.bind(access.value.id(), walker.build(WTF::move(read))); return true; @@ -2018,7 +2383,7 @@ static KnowledgeLevel levelOfViewStoreOfTypedElements(const AccessOfTypedElement TypeArena& types = access.walker.types(); TypeID storedID = access.typeOfStored; Type stored = types.at(storedID); - bool convertsWithoutCode = !stored.isNever() && stored.isNumberLike() && (!isClamped(typedArrayType) || stored.isBoolean()); + bool convertsWithoutCode = !stored.isNever() && stored.isNumberLike(); bool convertsInLine = convertsWithoutCode && readsElementsInLineInTypedElements(typedArrayType) && access.walker.has(Mechanism::UnboxedNumbers); bool fits = isBigInt(typedArrayType) ? !stored.isNever() && stored.hasOnlyPrimitives(Primitive::BigInt) : stored.isNumber() || convertsInLine; if (fits) @@ -2041,7 +2406,8 @@ static bool buildViewStoreOfTypedElements(const AccessOfTypedElements& access, T if (level == KnowledgeLevel::Proven && !isBigInt(typedArrayType) && !walker.types().at(access.typeOfStored).isNumber()) { Origin origin = walker.originOf(access.value); origin.isTailCall = false; - if (isInt(typedArrayType) && walker.types().at(access.typeOfStored).isNumberOrNullish()) { + Type typeOfStored = walker.types().at(access.typeOfStored); + if (isInt(typedArrayType) && typeOfStored.isNumberOrNullish() && (!isClamped(typedArrayType) || typeOfStored.isInt32OrNullish())) { Draft zero(Opcode::Integer, origin); zero.setInteger(0); stored = buildIntegerOperator(walker, Opcode::BitOr, stored, walker.constant(WTF::move(zero)), origin); @@ -2063,7 +2429,7 @@ static bool buildViewStoreOfTypedElements(const AccessOfTypedElements& access, T std::optional inside = level == KnowledgeLevel::Proven ? walker.findWhatHoldsPositionInsideList(access.base, access.key) : std::nullopt; if (!inside && level == KnowledgeLevel::Proven && access.typeOfKey.isInteger()) inside = walker.findWhatHoldsPositionBelowRoomOfView(access.base, access.key); - if (!inside && level == KnowledgeLevel::Proven && access.key.isConstant()) { + if (!inside && level == KnowledgeLevel::Proven && (access.key.isConstant() || hasKeyWithWrittenMaskInTypedElements(access))) { hoistTypedArrayBoundsCheck(access, typedArrayType); inside = walker.findWhatHoldsPositionInsideList(access.base, access.key); } @@ -2441,7 +2807,15 @@ bool buildPushOntoLiteralThatLoopFillsOfItsOpcodes(Walker& walker, const BodyVal if (!key || !pushed || !walker.typeOf(key) || !walker.typeOf(pushed)) return false; AccessOfTypedElements access { walker, call, list, key, pushed, walker.typeOf(list), walker.types().at(walker.typeOf(key)), walker.typeOf(pushed), false, counter }; - return buildStoreBehindRowOfTypedElements(access); + if (!buildStoreBehindRowOfTypedElements(access)) + return false; + if (walker.isUnreachable() || index.isUnused(call.id())) + return true; + Draft one(Opcode::Integer, walker.originOf(call)); + one.setInteger(1); + if (Reference length = buildIntegerOperator(walker, Opcode::Add, key, walker.constant(WTF::move(one)), walker.originOf(call))) + walker.bind(call.id(), length); + return true; } bool buildAccessByIndexOffExpectedViewOfItsOpcodes(Walker& walker, const BodyValue& value) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkTypedTexts.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkTypedTexts.cpp index ee04b2e0f631d..96ead0339bc0e 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkTypedTexts.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkTypedTexts.cpp @@ -13,6 +13,7 @@ #include "AOTEffectsOfOperations.h" #include "AOTIRProgram.h" #include "AOTOpcodeSets.h" +#include "AOTThinSummary.h" #include "AOTTypeArena.h" #include "AOTWalkTextBuilderRows.h" #include "AOTWalkTextBuilders.h" @@ -64,6 +65,32 @@ static bool takesKeyOfCollectionInWalkTexts(Walker& walker, const BodyValue& use return expected && holdsCollections(types.withoutNullish(expected)); } +static UseOfTextAsKey useOfParameterAsKeyInWalkTexts(Walker& walker, const BodyValue& user, ValueID text) +{ + if (user.opcode() != Opcode::Call) + return UseOfTextAsKey::NotKnown; + FunctionID callee = walker.facts().linked().calleeAt(walker.currentFunction(), user.id()); + if (!callee) + return UseOfTextAsKey::NotKnown; + const ThinSummary& summary = walker.facts().summaries().at(callee); + if (!(summary.entryParametersReadAsNames | summary.entryParametersReadAsKeys)) + return UseOfTextAsKey::NotKnown; + const Function& function = walker.program().at(callee); + uint32_t position = 0; + for (ValueID argument : user.as().argumentsOperands()) { + uint32_t parameter = function.firstParameterIndex() + position; + if (position++ >= function.numberOfPositionalParameters() || parameter >= 16) + break; + if (argument != text) + continue; + if (summary.entryParametersReadAsNames >> parameter & 1) + return UseOfTextAsKey::NameOfProperty; + if (summary.entryParametersReadAsKeys >> parameter & 1) + return UseOfTextAsKey::AnyKey; + } + return UseOfTextAsKey::NotKnown; +} + static ValueID findKeyThatIsConvertedOnceInWalkTexts(const BodyIndex& index, ValueID text, const BodyValue& test) { if (test.opcode() != Opcode::TypeTest || test.as().testedType() != TypeTable::object) @@ -85,6 +112,18 @@ static ValueID findKeyThatIsConvertedOnceInWalkTexts(const BodyIndex& index, Val return place == handedOn.end() ? ValueID() : index.block(next.block()).parameter(static_cast(place - handedOn.begin())); } +static ValueID findParameterThatTakesOnJump(const BodyIndex& index, ValueID text, const BodyUser& mention) +{ + if (mention.kind != BodyUseKind::EdgeArgument) + return { }; + BodyTerminator end = index.terminator(mention.block); + if (end.kind() != TerminatorKind::Jump || end.next().block().index() <= mention.block.index()) + return { }; + std::span handedOn = end.next().arguments(); + auto place = std::ranges::find(handedOn, text); + return place == handedOn.end() ? ValueID() : index.block(end.next().block()).parameter(static_cast(place - handedOn.begin())); +} + static UseOfTextAsKey useAsKeyInWalkTexts(Walker& walker, const BodyIndex& index, ValueID text, bool goesOnThroughJoinThatAloneTakesText = false, bool isAskedOfCallBeingCopied = false) { Vector passed; @@ -104,6 +143,11 @@ static UseOfTextAsKey useAsKeyInWalkTexts(Walker& walker, const BodyIndex& index return sayAtEnd(walker.useOfAnswerOfCopyAsKey()); if (end == TerminatorKind::Return && walker.facts().resultIsReadAsKey(walker.currentFunction())) return sayAtEnd(UseOfTextAsKey::AnyKey); + if (ValueID joined = mention ? findParameterThatTakesOnJump(index, text, *mention) : ValueID()) { + passed.append(text); + text = joined; + continue; + } if (!mention || mention->kind != BodyUseKind::Operand) return sayAtEnd(UseOfTextAsKey::NotKnown); BodyValue user = index.value(mention->value); @@ -117,6 +161,8 @@ static UseOfTextAsKey useAsKeyInWalkTexts(Walker& walker, const BodyIndex& index if (!opcodesThatStateTypes.contains(user.opcode()) && user.opcode() != Opcode::ToPropertyKey && !(goesOnThroughJoinThatAloneTakesText && operationsThatJoinTexts.contains(user.opcode()) && index.onlyUsingValue(text) == user.id())) { if (takesKeyOfCollectionInWalkTexts(walker, user, text)) return sayAtEnd(UseOfTextAsKey::AnyKey); + if (UseOfTextAsKey use = useOfParameterAsKeyInWalkTexts(walker, user, text); use != UseOfTextAsKey::NotKnown) + return sayAtEnd(use); return sayAtEnd(isAskedOfCallBeingCopied && !operationsThatJoinTexts.contains(user.opcode()) ? UseOfTextAsKey::NotKnown : UseOfTextAsKey::None); } passed.append(text); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkUnionInstanceOf.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkUnionInstanceOf.cpp index 60a281d5a1d9a..a3bcde9f8f5cd 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkUnionInstanceOf.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkUnionInstanceOf.cpp @@ -345,9 +345,9 @@ bool canDecideInstanceOfByHierarchy(const Walker& walker, TypeID typeOfConstruct return tested && !tested->isConstructorFunction && walker.plan().hasClassRecord(tested->layout) && walker.facts().families().nearestFamilyOf(tested->layout); } -void noteWhatInstanceOfDenied(Walker& walker, const BodyValue& test) +void noteWhatInstanceOfSaid(Walker& walker, const BodyValue& test, bool passed) { - if (!walker.has(Mechanism::Guesses) || !walker.opensGuesses() || walker.isUnreachable()) + if (!walker.has(Mechanism::Guesses) || walker.isUnreachable()) return; Reference value = walker.referenceOf(test.valueOperand()); Reference constructor = walker.referenceOf(test.constructorOperand()); @@ -359,10 +359,11 @@ void noteWhatInstanceOfDenied(Walker& walker, const BodyValue& test) TypeArena& types = walker.types(); Vector layouts { tested->layout }; layouts.append(walker.program().layoutsExpectedBelow(tested->layout)); - TypeID denied = types.forLayouts(layouts.span()); - if (std::optional before = findKnown(walker, KnownNote::ClassesThatInstanceOfDenied, value)) - denied = types.unionOf(denied, types.forLayoutSet(LayoutSetID::fromBits(static_cast(*before)))); - noteKnown(walker, KnownNote::ClassesThatInstanceOfDenied, value, { }, 0, LocationMasks(), types.at(denied).layouts.bits()); + TypeID asked = types.forLayouts(layouts.span()); + KnownNote note = passed ? KnownNote::ClassesThatInstanceOfPassed : KnownNote::ClassesThatInstanceOfDenied; + if (std::optional before = passed ? std::nullopt : findKnown(walker, note, value)) + asked = types.unionOf(asked, types.forLayoutSet(LayoutSetID::fromBits(static_cast(*before)))); + noteKnown(walker, note, value, { }, 0, LocationMasks(), types.at(asked).layouts.bits()); } static ChainEndForInstanceOf followChainForInstanceOf(const Walker& walker, LayoutID layout, LayoutID tested) @@ -763,7 +764,11 @@ static bool buildInstanceOfClassOfProgram(Walker& walker, const BodyValue& test, return false; bool valueHasLayoutPrototype = !inputs.areOnlyTypes() && knownCellFacts(walker, walker.referenceOf(test.valueOperand())).contains(CellFact::HasLayoutPrototype); valueHasLayoutPrototype |= walker.shouldAssumeLayoutPrototype() && !inputs.areOnlyTypes(); - TypeID expectedOfValue = !inputs.areOnlyTypes() && maySpendBytesForTimeInInstanceOf(walker) ? findExpectation(walker, test.valueOperand()).type : TypeID(); + bool asksWhatIsExpected = !inputs.areOnlyTypes() && maySpendBytesForTimeInInstanceOf(walker); + TypeID expectedOfValue = asksWhatIsExpected ? findExpectation(walker, test.valueOperand()).type : TypeID(); + TypeID expectedOfAnswer = asksWhatIsExpected ? walker.typeExpectedOfAnswerOfCopy() : TypeID(); + if (expectedOfAnswer && walker.types().at(expectedOfAnswer).hasObjects() && walker.copyHandsBack(test.valueOperand())) + expectedOfValue = expectedOfValue ? walker.types().unionOf(expectedOfValue, expectedOfAnswer) : expectedOfAnswer; std::optional form = planFormOfInstanceOf(walker, typeOfValue, typeOfConstructor, valueHasLayoutPrototype, expectedOfValue); if (!form) return false; @@ -1105,6 +1110,36 @@ static bool holdsNoObjectForInstanceOf(const Walker& walker, TypeID typeOfValue) return subject.isJavaScriptValue() && !subject.mayBeUnspecified() && !subject.hasObjects(); } +static bool chainsHoldNoOtherBuiltinThanObjectPrototype(const Walker& walker, TypeID typeOfValue) +{ + TypeArena& types = walker.types(); + Type subject = types.at(typeOfValue); + Type objects = types.at(types.withOnlyObjects(typeOfValue)); + if (!subject.isJavaScriptValue() || subject.mayBeUnspecified() || !subject.hasObjects() || !objects.isObject() || !objects.hasOnlyLayouts() || objects.hasOnlyClosures() || !walker.plan().hasOneInstance()) + return false; + const Program& program = walker.program(); + const LinkTables& link = walker.facts().linked(); + return std::ranges::all_of(membersOf(types, objects.layouts), [&](LayoutID member) { + if (possibleChangesForInstanceOf(link, member).containsAny({ LayoutChange::ChangesPrototype, LayoutChange::GetsObjectAbove })) + return false; + const Layout& layout = program.at(member); + if (layout.has(LayoutFlags::IsOfCreatedObject) || layout.has(LayoutFlags::IsConstructorFunction)) + return false; + if (layout.kind == LayoutKind::Record) + return !layout.constructor; + if (layout.kind != LayoutKind::Instance || member.index() >= link.layouts.size() || !link.layout(member).listsEveryMemberAbove) + return false; + size_t classesLeft = program.layouts.size(); + LayoutID highest = member; + while (LayoutID base = program.at(highest).base) { + if (program.at(highest).writtenBuiltinBase || !classesLeft--) + return false; + highest = base; + } + return !program.at(highest).writtenBuiltinBase && !program.at(highest).has(LayoutFlags::HasHeritage); + }); +} + static bool classTestAnswersBuiltinInstanceOf(const Walker& walker, const BuiltinTestedByInstanceOf& tested, TypeID typeOfValue) { const BuiltinTable& table = walker.program().builtins(); @@ -1172,13 +1207,14 @@ static bool buildInstanceOfBuiltinByValue(Walker& walker, const BodyValue& test, return false; bool holdsNoObject = walker.has(Mechanism::Folding) && holdsNoObjectForInstanceOf(walker, typeOfValue); + bool isDecidedByLayouts = !holdsNoObject && walker.has(Mechanism::Folding) && tested->prototype && chainsHoldNoOtherBuiltinThanObjectPrototype(walker, typeOfValue); bool testsClass = false; - if (!holdsNoObject && tested->prototype && mayBuildArmsForInstanceOf(walker) && classTestAnswersBuiltinInstanceOf(walker, *tested, typeOfValue)) { + if (!holdsNoObject && !isDecidedByLayouts && tested->prototype && mayBuildArmsForInstanceOf(walker) && classTestAnswersBuiltinInstanceOf(walker, *tested, typeOfValue)) { TypeID expected = walker.isInLoop() ? TypeID() : findExpectation(walker, test.valueOperand()).type; testsClass = walker.isInLoop() || (expected && walker.types().overlap(expected, walker.types().builtinClass(tested->builtinClass))); } if (inputs.areOnlyTypes()) - return holdsNoObject || testsClass; + return holdsNoObject || isDecidedByLayouts || testsClass; Reference value = walker.referenceOf(test.valueOperand()); Reference constructor = walker.referenceOf(test.constructorOperand()); @@ -1186,9 +1222,13 @@ static bool buildInstanceOfBuiltinByValue(Walker& walker, const BodyValue& test, return false; Origin origin = walker.originOf(test); Reference answer; - if (holdsNoObject) + bool isObjectPrototype = tested->prototype == walker.program().builtins().canonicalID(Builtins::ObjectPrototype); + if (holdsNoObject || (isDecidedByLayouts && !isObjectPrototype)) answer = booleanAnswerOfInstanceOf(walker, false); - else { + else if (isDecidedByLayouts) { + TypeID objects = walker.types().withOnlyObjects(walker.typeOf(value)); + answer = walker.typeOf(value) == objects ? booleanAnswerOfInstanceOf(walker, true) : buildTypeTestOfInstanceOf(walker, value, objects, origin); + } else { bool isSeenAsMade = tested->builtinClass && walker.has(Mechanism::Folding) && walker.plan().hasOneInstance() && walker.types().isSubset(walker.typeOf(value), walker.types().builtinClass(tested->builtinClass)) && isKnownPristine(walker, value); if (isSeenAsMade) { walker.verifyPristineObject(value); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkUnionMethodCalls.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkUnionMethodCalls.cpp index 7f54be83653d7..4019c942d57dd 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkUnionMethodCalls.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkUnionMethodCalls.cpp @@ -19,7 +19,8 @@ namespace JSC { namespace AOT { namespace Typed { -static constexpr size_t mostArmsInUnionMethodCalls = 4; +static constexpr size_t mostArmsInUnionMethodCalls = 8; +static constexpr size_t mostArmsThatAreNoSmallLeaves = 4; constinit const OpcodeSet opcodesThatCallMethodsBehindTestsOfShapeTakes { Opcode::PropGet }; @@ -43,7 +44,7 @@ static ValueID findCallOfMethodReadInUnionMethodCalls(const BodyIndex& index, co return written.calleeOperand() == read.id() && written.thisValueOperand() == read.as().objectOperand() ? call : ValueID(); } -static bool walkBehindTestsOfShapeInUnionMethodCalls(Walker& walker, const BodyValue& read, ValueID call, bool takesWhatIsOnlyExpected = false) +static bool walkBehindTestsOfShapeInUnionMethodCalls(Walker& walker, const BodyValue& read, ValueID call, bool takesWhatIsOnlyExpected = false, bool standsForGuess = false) { if (walker.standsInArmOfRowOfShapeTests() || !walker.has(Mechanism::DirectCalls) || !walker.has(Mechanism::TypedOperations)) return false; @@ -88,13 +89,16 @@ static bool walkBehindTestsOfShapeInUnionMethodCalls(Walker& walker, const BodyV bool someAnswerByGetter = false; Vector arms; Vector armsThatFindNeitherGetterNorField; + bool isOfTwoLayouts = false; + LayoutID classAboveBoth; if (!call) { size_t numberInUse = 0; for (LayoutID member : membersOf(types, types.at(proven))) { if (plan.isInUse(member) && ++numberInUse > mostArmsInUnionMethodCalls) return false; } - bool someClassHasAccessor = walker.isInLoop() || std::ranges::any_of(membersOf(types, types.at(proven)), [&](LayoutID member) { + isOfTwoLayouts = standsForGuess && numberInUse == 2; + bool someClassHasAccessor = walker.isInLoop() || isOfTwoLayouts || std::ranges::any_of(membersOf(types, types.at(proven)), [&](LayoutID member) { return plan.isInUse(member) && walker.facts().linked().findMember(program, member, ClassSide::Prototype, lookup.key()).kind == MemberFound::Kind::Accessor; }); if (!someClassHasAccessor) @@ -112,6 +116,11 @@ static bool walkBehindTestsOfShapeInUnionMethodCalls(Walker& walker, const BodyV }; TypedForm form = typedFormOf(walker, read, TypesOfInputs(typeBehindTest)); someAnswerByGetter |= form == TypedForm::BuiltAsCall; + if (isOfTwoLayouts) { + LayoutID above = declaredLayoutOf(program, member) ? program.at(declaredLayoutOf(program, member)).base : LayoutID(); + isOfTwoLayouts = form == TypedForm::Built && findField(program, member, lookup.key()) && program.at(member).kind == LayoutKind::Instance && above && (!classAboveBoth || above == classAboveBoth); + classAboveBoth = above; + } if (form == TypedForm::None) { if (!call) return false; @@ -124,8 +133,27 @@ static bool walkBehindTestsOfShapeInUnionMethodCalls(Walker& walker, const BodyV arms.append(tested); } arms.appendVector(armsThatFindNeitherGetterNorField); - if (arms.isEmpty() || (!call && !someAnswerByGetter && !walker.isInLoop())) + auto callsSmallLeaves = [&](PropertyKeyID key, ValueID called) { + return index.opcodeOf(called) == Opcode::Call && std::ranges::all_of(arms, [&](TypeID arm) { + MemberFound found = walker.facts().linked().findMember(program, singleLayoutOf(types, types.at(arm)), ClassSide::Prototype, key); + return found.kind == MemberFound::Kind::Method && found.method && isSmallLeaf(walker, found.method, index.value(called).argumentsOperands().size()); + }); + }; + if (arms.size() > mostArmsThatAreNoSmallLeaves && !(call && walker.isInLoop() && callsSmallLeaves(lookup.key(), call))) + return false; + auto mayTake = [&](const BodyValue& operation) { + if (operation.opcode() == Opcode::PropGet && walker.referenceOf(operation.as().objectOperand()) == receiver) { + ValueID called = findCallOfMethodReadInUnionMethodCalls(index, operation); + return !called || callsSmallLeaves(operation.as().key(), called); + } + return operation.opcode() != Opcode::Call || std::ranges::none_of(operation.argumentsOperands(), [&](ValueID handed) { + return handed && walker.referenceOf(handed) == receiver; + }); + }; + if (arms.isEmpty() || (!call && !someAnswerByGetter && !walker.isInLoop() && !isOfTwoLayouts)) return false; + if (call && walker.isInLoop()) + end = findArmStretchEnd(walker, index.value(call), end - 1, limitOfStretchOfArms(walker, block), receiver, scopedLambda(mayTake)); if (walker.shouldDropLastCase() && !somethingIsLeft && arms.size() > 1) arms.removeLast(); @@ -158,8 +186,10 @@ static bool walkBehindTestsOfShapeInUnionMethodCalls(Walker& walker, const BodyV }; walker.count(call ? StatisticID::MethodCallsDispatchedByShape : StatisticID::PropertyReadsDispatchedByShape); - if (isOnlyExpected) + if (isOnlyExpected) { walker.count(StatisticID::SpeculatedReceiversDispatchedByShape); + walker.noteTestSeenInTries(receiver, read.id(), proven); + } Reference left = receiver; if (mayBeNothing) { Draft check(Opcode::RequireObjectCoercible, origin); @@ -217,7 +247,7 @@ static bool walkBehindTestsOfShapeInUnionMethodCalls(Walker& walker, const BodyV return true; } else if (somethingIsLeft) { walker.closedArmOfRowOfShapeTests() = { }; - Walker::ClosedArm closedArm(walker); + Walker::ClosedArm closedArm(walker, isOnlyExpected); walkArm(); } else { TypeID layoutsOfArms = arms[0]; @@ -249,7 +279,7 @@ bool callMethodsOfExpectedReceiversBehindTestsOfShape(Walker& walker, const Body return call && walkBehindTestsOfShapeInUnionMethodCalls(walker, read, call, true); } -bool readBehindTestsOfShapeOfItsOpcodes(Walker& walker, const BodyValue& read) +static bool readBehindTestsOfShapeInUnionMethodCalls(Walker& walker, const BodyValue& read, bool standsForGuess) { ValueID object = read.as().objectOperand(); TypeID proven = object ? walker.typeOf(walker.referenceOf(object)) : TypeID(); @@ -257,7 +287,17 @@ bool readBehindTestsOfShapeOfItsOpcodes(Walker& walker, const BodyValue& read) return false; if (walker.isNestedTooDeeply() || (!walker.isInLoop() && walker.program().at(walker.unit().function).isModuleBody())) return false; - return !findCallOfMethodReadInUnionMethodCalls(walker.currentBodyIndex(), read) && walkBehindTestsOfShapeInUnionMethodCalls(walker, read, { }); + return !findCallOfMethodReadInUnionMethodCalls(walker.currentBodyIndex(), read) && walkBehindTestsOfShapeInUnionMethodCalls(walker, read, { }, false, standsForGuess); +} + +bool readBehindTestsOfShapeOfItsOpcodes(Walker& walker, const BodyValue& read) +{ + return readBehindTestsOfShapeInUnionMethodCalls(walker, read, false); +} + +bool readOfTwoLayoutsBehindTestsOfShape(Walker& walker, const BodyValue& read) +{ + return readBehindTestsOfShapeInUnionMethodCalls(walker, read, true); } } } } // namespace JSC::AOT::Typed diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkUnionTableCalls.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkUnionTableCalls.cpp index e36b80fccd625..a663e402ce3ea 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkUnionTableCalls.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkUnionTableCalls.cpp @@ -27,6 +27,7 @@ namespace JSC { namespace AOT { namespace Typed { static constexpr size_t mostReceiversToldApartByArmsInUnionTableCalls = 4; +static constexpr size_t mostReceiversOfSmallLeaves = 8; namespace { @@ -34,7 +35,9 @@ struct ReceiversOfTableCallInUnionTableCalls { bool areKeptForBuild { false }; bool areRefused { false }; bool someRowIsGuarded { false }; - bool someCopyWouldLearnOfThis { false }; + bool someCopyPaysInLoop { false }; + bool someMethodIsSmallLeaf { false }; + bool everyMethodIsSmallLeaf { true }; size_t numberInUse { 0 }; Vector methods; Vector withRows; @@ -57,6 +60,8 @@ struct KeptByUnionTableCalls final : Walker::KeptForBuild { struct TableCallInUnionTableCalls { uint32_t selector { 0 }; ReceiversOfTableCallInUnionTableCalls* receivers { nullptr }; + bool isRequiredTailCall { false }; + bool needsShapeCheck { false }; }; } // anonymous namespace @@ -70,6 +75,28 @@ static ALWAYS_INLINE bool isCallOfMethodReadInUnionTableCalls(const BodyIndex& i return written.calleeOperand() == read.id() && written.thisValueOperand() == lookup.objectOperand() && written.argumentsOperands().size() <= maxArgumentsOfTableCall; } +static bool callsLargerMethodInUnionTableCalls(const Walker& walker, const BodyIndex& index, const BodyValue& read, std::span members) +{ + BodyBlock block = index.block(index.blockOf(read.id())); + ValueID receiver = index.stripCasts(read.as().objectOperand()); + for (uint32_t place = 0; place < block.numberOfValues(); ++place) { + BodyValue other = index.value(ValueID(block.firstValue().index() + place)); + if (other.opcode() != Opcode::PropGet || other.id() == read.id() || !other.as().objectOperand() || index.stripCasts(other.as().objectOperand()) != receiver) + continue; + ValueID call = index.onlyUsingValue(other.id()); + if (!isCallOfMethodReadInUnionTableCalls(index, other, call)) + continue; + size_t numberOfArguments = index.value(call).as().argumentsOperands().size(); + bool everyMethodIsSmallLeaf = std::ranges::all_of(members, [&](LayoutID member) { + MemberFound found = walker.facts().linked().findMember(walker.program(), member, ClassSide::Prototype, other.as().key()); + return found.kind == MemberFound::Kind::Method && found.method && isSmallLeaf(walker, found.method, numberOfArguments); + }); + if (!everyMethodIsSmallLeaf) + return true; + } + return false; +} + static bool readMayMoveDownToCallInUnionTableCalls(const BodyIndex& index, ValueID read, ValueID call) { for (uint32_t between = read.index() + 1; between < call.index(); ++between) { @@ -117,7 +144,10 @@ static void describeReceiversInUnionTableCalls(Walker& walker, uint32_t selector receivers.methods.append(row->method); receivers.someRowIsGuarded |= !!row->guardedClass; TypeID own = walker.program().at(row->method).thisType.written(); - receivers.someCopyWouldLearnOfThis |= !own || !types.isSubset(types.withoutNullish(own), types.forLayout(member)); + bool calleeIsSmallLeaf = isSmallLeaf(walker, row->method, conventions.methodSelector(selector).numberOfArguments); + receivers.someCopyPaysInLoop |= !own || !types.isSubset(types.withoutNullish(own), types.forLayout(member)); + receivers.someMethodIsSmallLeaf |= calleeIsSmallLeaf; + receivers.everyMethodIsSmallLeaf &= calleeIsSmallLeaf; } std::ranges::sort(receivers.methods); receivers.methods.shrink(std::ranges::unique(receivers.methods).begin() - receivers.methods.begin()); @@ -143,7 +173,7 @@ static ReceiversOfTableCallInUnionTableCalls& findReceiversInUnionTableCalls(Wal receivers.areKeptForBuild = true; if (receivers.areRefused) receivers.methods.clear(); - if (receivers.areRefused || receivers.withoutRows.isEmpty()) + if (receivers.areRefused) receivers.withRows.clear(); } return *added.iterator->value; @@ -201,24 +231,34 @@ static ALWAYS_INLINE std::optional planTableCallInUn readOfEveryOtherReceiverIsConstant &= !holdsNameItself && typedFormOf(walker, read, TypesOfInputs(typeBehindTest)) == TypedForm::Built; } bool someReceiverHasNoRow = !receivers.withoutRows.isEmpty(); - if (armsMayTakeCall == ArmsMayTakeCallInUnionTableCalls::Yes && receivers.someCopyWouldLearnOfThis && receivers.numberInUse <= mostReceiversToldApartByArmsInUnionTableCalls && walker.isInLoop() && walker.has(Mechanism::Guesses) && walker.opensGuesses() && !walker.standsInArmOfRowOfShapeTests()) - return std::nullopt; + bool armsTakeCall = armsMayTakeCall == ArmsMayTakeCallInUnionTableCalls::Yes && (receivers.someCopyPaysInLoop || receivers.someMethodIsSmallLeaf) && receivers.numberInUse <= (receivers.everyMethodIsSmallLeaf ? mostReceiversOfSmallLeaves : mostReceiversToldApartByArmsInUnionTableCalls) && walker.isInLoop() && walker.has(Mechanism::Guesses) && walker.opensGuesses() && !walker.standsInArmOfRowOfShapeTests(); + if (armsTakeCall) { + bool isLeftToTable = !receivers.someCopyPaysInLoop && (walker.receiverOfLargerMethod() == receiver || callsLargerMethodInUnionTableCalls(walker, index, read, receivers.withRows)); + if (!isLeftToTable) + return std::nullopt; + walker.receiverOfLargerMethod() = receiver; + } if ((receivers.someRowIsGuarded || (someReceiverHasNoRow && !readOfEveryOtherReceiverIsConstant)) && !readMayMoveDownToCallInUnionTableCalls(index, read.id(), call)) return std::nullopt; if (someReceiverHasNoRow && (!walker.has(Mechanism::Guesses) || !walker.opensGuesses())) return std::nullopt; + bool isRequiredTailCall = false; BodyTerminator end = index.terminator(index.blockOf(call)); if (end.kind() == TerminatorKind::ReturnCall && end.operand(0) && index.stripCasts(end.operand(0)) == call) { bool everyMethodComesBack = !receivers.someRowIsGuarded && !someReceiverHasNoRow && std::ranges::all_of(receivers.methods, [&](FunctionID method) { return plan.endsItsTailCalls(method); }); - if (!everyMethodComesBack) + isRequiredTailCall = !everyMethodComesBack; + if (isRequiredTailCall && (receivers.someRowIsGuarded || end.operand(0) != call || !canTailCallMethodTable(walker, *selector, receivers.methods.span()))) + return std::nullopt; + if (isRequiredTailCall && someReceiverHasNoRow && !walker.isInLoop() && !std::ranges::binary_search(receivers.methods, walker.currentFunction())) return std::nullopt; } - if (!someReceiverHasNoRow && !walker.isShown(receiver, proven)) + bool isShown = someReceiverHasNoRow || walker.isShown(receiver, proven); + if (!isShown && (!walker.has(Mechanism::Guesses) || !walker.opensGuesses())) return std::nullopt; - return TableCallInUnionTableCalls { *selector, &receivers }; + return TableCallInUnionTableCalls { *selector, &receivers, isRequiredTailCall, someReceiverHasNoRow || !isShown }; } constinit const OpcodeSet opcodesThatLeaveReadOfMethodToTableCallTakes { Opcode::PropGet }; @@ -249,6 +289,8 @@ bool leaveReadOfMethodToTableCallOfItsOpcodes(Walker& walker, const BodyValue& r if (!checked || walker.isUnreachable()) return true; walker.state().noteCheckedName(receiver, checked); + if (walker.receiverOfLargerMethod() == receiver) + walker.receiverOfLargerMethod() = checked; } walker.bind(read.id(), Reference()); return true; @@ -316,10 +358,28 @@ bool callMethodThroughTableOfItsOpcodes(Walker& walker, const BodyValue& call) Reference passes; Reference anyReceiver = receiver; bool someReceiverHasNoRow = !planned->receivers->withoutRows.isEmpty(); - if (someReceiverHasNoRow) { - Draft test(Opcode::ShapeTest, walker.originOf(call)); + auto emitGenericCall = [&](Reference other) { + BodyBlock block = index.block(index.blockOf(call.id())); + uint32_t first = block.firstValue().index(); + Reference boundBefore = walker.referenceBoundTo(written.thisValueOperand()); + walker.bind(written.thisValueOperand(), other); + walker.bind(read.id(), Reference()); + walker.bind(call.id(), Reference()); + walker.walkStretch(block.id(), read.id().index() - first, read.id().index() - first + 1); + if (!walker.isUnreachable() && !walker.hasFailed()) + walker.walkStretch(block.id(), call.id().index() - first, call.id().index() - first + 1); + walker.bind(written.thisValueOperand(), boundBefore); + }; + Draft returned(Opcode::Call, walker.originOf(call)); + returned.resultIsReturnedAtOnce = planned->isRequiredTailCall; + bool returnsInArm = walker.unitReturnsAtOnce(returned); + if (planned->needsShapeCheck) { + TypeID withMethod = walker.types().forLayouts(planned->receivers->withRows.span()); + Origin origin = walker.originOf(call); + origin.isTailCall = false; + Draft test(Opcode::ShapeTest, origin); test.setObjectOperand(receiver); - test.setLayouts(walker.types().at(walker.types().forLayouts(planned->receivers->withRows.span())).layouts); + test.setLayouts(walker.types().at(withMethod).layouts); test.setAsksInFull(!walker.isShown(receiver, walker.typeOf(receiver))); passes = walker.build(WTF::move(test)); if (!passes) @@ -329,8 +389,18 @@ bool callMethodThroughTableOfItsOpcodes(Walker& walker, const BodyValue& call) meeting = walker.addBlock(); walker.recordSpeculation(); walker.branch(passes, withRow, withoutRow); + if (returnsInArm) { + walker.continueIn(withoutRow); + if (Reference others = narrowedByInUnionTableCalls(walker, anyReceiver, passes, false, origin)) + emitGenericCall(others); + if (walker.hasFailed()) + return true; + if (!walker.isUnreachable()) + walker.returnAtOnce(walker.referenceOf(call.id())); + walker.bind(call.id(), Reference()); + } walker.continueIn(withRow); - receiver = narrowedByInUnionTableCalls(walker, anyReceiver, passes, true, walker.originOf(call)); + receiver = narrowedByInUnionTableCalls(walker, anyReceiver, passes, true, origin); if (!receiver) return true; walker.state().noteShape(receiver, walker.typeOf(receiver)); @@ -350,7 +420,11 @@ bool callMethodThroughTableOfItsOpcodes(Walker& walker, const BodyValue& call) table.setReceiverOperand(receiver); table.setArgumentsOperands(asWritten.argumentsOperands()); table.setKey(read.as().key()); - table.setSelector(chooseSelectorInUnionTableCalls(walker, asWritten, planned->selector)); + uint32_t selector = chooseSelectorInUnionTableCalls(walker, asWritten, planned->selector); + if (planned->isRequiredTailCall && !canTailCallMethodTable(walker, selector, planned->receivers->methods.span())) + selector = planned->selector; + table.setSelector(selector); + table.resultIsReturnedAtOnce = asWritten.resultIsReturnedAtOnce; table.expected = asWritten.expected; table.inputValueGiven = asWritten.inputValueGiven; @@ -363,23 +437,14 @@ bool callMethodThroughTableOfItsOpcodes(Walker& walker, const BodyValue& call) Reference result = walker.build(WTF::move(table), &effects); if (result) walker.bind(call.id(), result); - if (!someReceiverHasNoRow || walker.hasFailed()) + if (!planned->needsShapeCheck || returnsInArm || walker.hasFailed()) return true; if (result && !walker.isUnreachable()) arms.append({ walker.currentBlock(), { result } }); walker.continueIn(withoutRow); if (Reference others = narrowedByInUnionTableCalls(walker, anyReceiver, passes, false, walker.originOf(call))) { - BodyBlock block = index.block(index.blockOf(call.id())); - uint32_t first = block.firstValue().index(); - Reference boundBefore = walker.referenceBoundTo(written.thisValueOperand()); - walker.bind(written.thisValueOperand(), others); - walker.bind(read.id(), Reference()); - walker.bind(call.id(), Reference()); - walker.walkStretch(block.id(), read.id().index() - first, read.id().index() - first + 1); - if (!walker.isUnreachable() && !walker.hasFailed()) - walker.walkStretch(block.id(), call.id().index() - first, call.id().index() - first + 1); - walker.bind(written.thisValueOperand(), boundBefore); + emitGenericCall(others); if (walker.hasFailed()) return true; if (!walker.isUnreachable()) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkWrittenDescriptors.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkWrittenDescriptors.cpp index 272c06924e186..5c1bd2bcc4573 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkWrittenDescriptors.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkWrittenDescriptors.cpp @@ -154,6 +154,8 @@ static std::optional> propertiesDefinedBehindAl auto addPropertyIn = [&](FieldIndex field) { FieldReference reference { made.layout(), field }; Reference value = walker.state().slot(root, reference); + if (value && value.isRecord() && code.at(value.id()).opcode == Opcode::Absent) + return false; if (value && !properties.containsIf([&](const PropertyOfFreshObject& property) { return property.field == reference; })) properties.append({ reference, value }); return !!value; diff --git a/Source/JavaScriptCore/assembler/ARM64Assembler.h b/Source/JavaScriptCore/assembler/ARM64Assembler.h index 73fa3226ce6f9..1f239e1b51e92 100644 --- a/Source/JavaScriptCore/assembler/ARM64Assembler.h +++ b/Source/JavaScriptCore/assembler/ARM64Assembler.h @@ -3437,16 +3437,6 @@ class ARM64Assembler { insn(floatingPointImmediate(DATASIZE, encodeFPImm(imm), vd)); } - ALWAYS_INLINE void fmov_top(FPRegisterID vd, RegisterID rn) - { - insn(floatingPointIntegerConversions(Datasize_64, Datasize_64, FPIntConvOp_FMOV_XtoQ_top, rn, vd)); - } - - ALWAYS_INLINE void fmov_top(RegisterID rd, FPRegisterID vn) - { - insn(floatingPointIntegerConversions(Datasize_64, Datasize_64, FPIntConvOp_FMOV_QtoX_top, vn, rd)); - } - template ALWAYS_INLINE void fmsub(FPRegisterID vd, FPRegisterID vn, FPRegisterID vm, FPRegisterID va) { diff --git a/Source/JavaScriptCore/assembler/LinkBuffer.cpp b/Source/JavaScriptCore/assembler/LinkBuffer.cpp index 4cd7df3dfe7d6..79acb61619dea 100644 --- a/Source/JavaScriptCore/assembler/LinkBuffer.cpp +++ b/Source/JavaScriptCore/assembler/LinkBuffer.cpp @@ -369,7 +369,7 @@ void LinkBuffer::copyCompactAndLinkCode(MacroAssembler& macroAssembler, JITCompi else Assembler::fillNops(outData + writePtr, size); writePtr += size; - // A label at the pool's place names the code after the pool, and executableOffsetFor() reads the word before a label. + // A label here names the code after the pool, and executableOffsetFor() reads the word before a label. recordLinkOffsets(m_assemblerStorage, readPtr - sizeof(InstructionType), readPtr, readPtr - writePtr); } }; diff --git a/Source/JavaScriptCore/assembler/MacroAssembler.h b/Source/JavaScriptCore/assembler/MacroAssembler.h index 5f64d153e7951..0f782622c24e2 100644 --- a/Source/JavaScriptCore/assembler/MacroAssembler.h +++ b/Source/JavaScriptCore/assembler/MacroAssembler.h @@ -1127,7 +1127,7 @@ class MacroAssembler : public MacroAssemblerBase { void lshiftPtr(Imm32 imm, RegisterID srcDest) { - lshift64(trustedImm32ForShift(imm), srcDest); + lshift64(imm.asTrustedImm32(), srcDest); } void lshiftPtr(TrustedImm32 imm, RegisterID srcDest) @@ -1147,7 +1147,7 @@ class MacroAssembler : public MacroAssemblerBase { void rshiftPtr(Imm32 imm, RegisterID srcDest) { - rshift64(trustedImm32ForShift(imm), srcDest); + rshift64(imm.asTrustedImm32(), srcDest); } void rshiftPtr(TrustedImm32 imm, RegisterID srcDest) @@ -1162,7 +1162,7 @@ class MacroAssembler : public MacroAssemblerBase { void urshiftPtr(Imm32 imm, RegisterID srcDest) { - urshift64(trustedImm32ForShift(imm), srcDest); + urshift64(imm.asTrustedImm32(), srcDest); } void urshiftPtr(TrustedImm32 imm, RegisterID srcDest) @@ -1342,6 +1342,13 @@ class MacroAssembler : public MacroAssemblerBase { store64(TrustedImm64(imm), address); } +#if CPU(X86_64) + void storePtr(TrustedImm32 imm, BaseIndex address) + { + store64(imm, address); + } +#endif + void storePairPtr(RegisterID src1, RegisterID src2, RegisterID dest) { storePair64(src1, src2, dest); @@ -2013,11 +2020,15 @@ class MacroAssembler : public MacroAssemblerBase { // The addition immediate may be used as a pointer offset. Keep aligned based on "imm". static const uint32_t maskTable[4] = { 0xfffffffc, 0xffffffff, 0xfffffffe, 0xffffffff }; - uint32_t baseValue = imm.asTrustedImm32().m_value; - uint32_t key = keyForConstant(baseValue) & maskTable[baseValue & 3]; - if (key > baseValue) - key = key - baseValue; - return BlindedImm32(baseValue - key, key); + // Both parts have the sign of the immediate and neither exceeds it, so their sum is right at every width. + int32_t value = imm.asTrustedImm32().m_value; + uint32_t magnitude = value < 0 ? -static_cast(value) : value; + uint32_t key = keyForConstant(magnitude); + if (key > magnitude) + key %= magnitude; + key &= maskTable[magnitude & 3]; + uint32_t rest = magnitude - key; + return value < 0 ? BlindedImm32(-rest, -key) : BlindedImm32(rest, key); } BlindedImm32 andBlindedConstant(Imm32 imm) diff --git a/Source/JavaScriptCore/assembler/MacroAssemblerARM64.h b/Source/JavaScriptCore/assembler/MacroAssemblerARM64.h index 6665381923514..ce013b91d7c8a 100644 --- a/Source/JavaScriptCore/assembler/MacroAssemblerARM64.h +++ b/Source/JavaScriptCore/assembler/MacroAssemblerARM64.h @@ -1385,8 +1385,13 @@ class MacroAssemblerARM64 : public AbstractMacroAssembler { void or32(TrustedImm32 imm, Address address) { - load32(address, getCachedDataTempRegisterIDAndInvalidate()); - or32(imm, dataTempRegister, dataTempRegister); + if (!imm.m_value || imm.m_value == -1 || LogicalImmediate::create32(imm.m_value).isValid()) { + load32(address, getCachedDataTempRegisterIDAndInvalidate()); + or32(imm, dataTempRegister, dataTempRegister); + } else { + load32(address, getCachedMemoryTempRegisterIDAndInvalidate()); + or32(imm, memoryTempRegister, getCachedDataTempRegisterIDAndInvalidate()); + } store32(dataTempRegister, address); } @@ -1999,7 +2004,7 @@ class MacroAssemblerARM64 : public AbstractMacroAssembler { return; } loadFloat(Address(src, offset.m_value), dest1); - loadFloat(Address(src, offset.m_value + 8), dest2); + loadFloat(Address(src, offset.m_value + 4), dest2); } void loadPairFloat(Address src, FPRegisterID dest1, FPRegisterID dest2) @@ -2649,7 +2654,7 @@ class MacroAssemblerARM64 : public AbstractMacroAssembler { return; } storeFloat(src1, Address(dest, offset.m_value)); - storeFloat(src2, Address(dest, offset.m_value + 8)); + storeFloat(src2, Address(dest, offset.m_value + 4)); } void storePairFloat(FPRegisterID src1, FPRegisterID src2, Address dest) @@ -5110,7 +5115,7 @@ class MacroAssemblerARM64 : public AbstractMacroAssembler { if ((cond == Zero) || (cond == NonZero)) return Jump(makeCompareAndBranch<64>(static_cast(cond), reg)); m_assembler.tst<64>(reg, reg); - } else if (hasOneBitSet(mask.m_value) && ((cond == Zero) || (cond == NonZero))) + } else if (mask.m_value > 0 && hasOneBitSet(mask.m_value) && ((cond == Zero) || (cond == NonZero))) return Jump(makeTestBitAndBranch(reg, getLSBSet(mask.m_value), static_cast(cond))); else { LogicalImmediate logicalImm = LogicalImmediate::create64(mask.m_value); @@ -7433,14 +7438,10 @@ class MacroAssemblerARM64 : public AbstractMacroAssembler { // SDOT Vd.4S, Vn.16B, Vm.16B: for each i32 lane, accumulate sum of 4 signed 8-bit products. // SDOT accumulates into Vd, so we need addend in dest before the instruction. // Handle cases where dest overlaps with a or b to avoid clobbering inputs. - if (dest == a) { - moveVector(a, scratch1); - moveVector(addend, dest); - m_assembler.sdotv(dest, scratch1, b); - } else if (dest == b) { - moveVector(b, scratch1); + if (dest == a || dest == b) { + moveVector(dest, scratch1); moveVector(addend, dest); - m_assembler.sdotv(dest, a, scratch1); + m_assembler.sdotv(dest, dest == a ? scratch1 : a, dest == b ? scratch1 : b); } else { moveVector(addend, dest); m_assembler.sdotv(dest, a, b); diff --git a/Source/JavaScriptCore/assembler/MacroAssemblerX86_64.h b/Source/JavaScriptCore/assembler/MacroAssemblerX86_64.h index 2401a7a21a82d..b2fa92d2d1670 100644 --- a/Source/JavaScriptCore/assembler/MacroAssemblerX86_64.h +++ b/Source/JavaScriptCore/assembler/MacroAssemblerX86_64.h @@ -989,15 +989,15 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { void sub32(RegisterID left, RegisterID right, RegisterID dest) { + if (left == right) { + move(TrustedImm32(0), dest); + return; + } if (dest == right) { neg32(dest); add32(left, dest); return; } - if (left == right) { - move(TrustedImm32(0), dest); - return; - } move(left, dest); sub32(right, dest); } @@ -3140,7 +3140,7 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { void moveConditionallyTest32(ResultCondition cond, RegisterID testReg, TrustedImm32 mask, RegisterID src, RegisterID dest) { - test32(testReg, mask); + test32(testReg, mask, cond); cmov(x86Condition(cond), src, dest); } @@ -3149,7 +3149,7 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { ASSERT(isInvertible(cond)); ASSERT_WITH_MESSAGE(cond != Overflow, "TEST does not set the Overflow Flag."); - test32(testReg, mask); + test32(testReg, mask, cond); if (thenCase != dest && elseCase != dest) { move(elseCase, dest); @@ -3167,7 +3167,7 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { ASSERT(isInvertible(cond)); ASSERT_WITH_MESSAGE(cond != Overflow, "TEST does not set the Overflow Flag."); - test32(testReg, mask); + test32(testReg, mask, cond); if (elseCase == dest) { moveWithoutClobberingFlags(thenCase, scratchRegister()); @@ -3338,6 +3338,12 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { return Jump(m_assembler.jCC(x86Condition(cond))); } + Jump branch32(RelationalCondition cond, RegisterID left, BaseIndex right) + { + m_assembler.cmpl_mr(right.offset, right.base, right.index, right.scale, left); + return Jump(m_assembler.jCC(x86Condition(cond))); + } + Jump branch32(RelationalCondition cond, Address left, RegisterID right) { m_assembler.cmpl_rm(right, left.offset, left.base); @@ -3403,11 +3409,13 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { RELEASE_ASSERT_NOT_REACHED(); } - void test32(RegisterID reg, TrustedImm32 mask = TrustedImm32(-1)) + static constexpr bool readsSignFlag(ResultCondition cond) { return cond == Signed || cond == PositiveOrZero; } + + void test32(RegisterID reg, TrustedImm32 mask = TrustedImm32(-1), ResultCondition cond = NonZero) { if (mask.m_value == -1) m_assembler.testl_rr(reg, reg); - else if (!(mask.m_value & ~0xff) && (reg < X86Registers::esp || policy().shortensEncodings)) { + else if (!(mask.m_value & ~0xff) && !readsSignFlag(cond) && (reg < X86Registers::esp || policy().shortensEncodings)) { if (mask.m_value == 0xff) m_assembler.testb_rr(reg, reg); else @@ -3423,13 +3431,16 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { Jump branchTest32(ResultCondition cond, RegisterID reg, TrustedImm32 mask = TrustedImm32(-1)) { - test32(reg, mask); + test32(reg, mask, cond); return branch(cond); } Jump branchTest32(ResultCondition cond, Address address, TrustedImm32 mask = TrustedImm32(-1)) { - generateTest32(address, mask); + if (readsSignFlag(cond) && mask.m_value != -1) + m_assembler.testl_i32m(mask.m_value, address.offset, address.base); + else + generateTest32(address, mask); return Jump(m_assembler.jCC(x86Condition(cond))); } @@ -3784,7 +3795,10 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { void test32(ResultCondition cond, Address address, TrustedImm32 mask, RegisterID dest) { - generateTest32(address, mask); + if (readsSignFlag(cond) && mask.m_value != -1) + m_assembler.testl_i32m(mask.m_value, address.offset, address.base); + else + generateTest32(address, mask); set32(x86Condition(cond), dest); } @@ -3796,7 +3810,7 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { void test32(ResultCondition cond, RegisterID reg, TrustedImm32 mask, RegisterID dest) { - test32(reg, mask); + test32(reg, mask, cond); set32(x86Condition(cond), dest); } @@ -5518,6 +5532,10 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { void clearBits64WithMask(RegisterID mask, RegisterID dest, ClearBitsAttributes maskPreservation = ClearBitsAttributes::OKToClobberMask) { + if (mask == dest) { + move(TrustedImm32(0), dest); + return; + } not64(mask); m_assembler.andq_rr(mask, dest); if (maskPreservation == ClearBitsAttributes::MustPreserveMask) @@ -5526,6 +5544,15 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { void clearBits64WithMask(RegisterID src, RegisterID mask, RegisterID dest, ClearBitsAttributes maskPreservation = ClearBitsAttributes::OKToClobberMask) { + if (mask == dest) { + if (src == dest) + move(TrustedImm32(0), dest); + else { + not64(dest); + and64(src, dest); + } + return; + } move(src, dest); clearBits64WithMask(mask, dest, maskPreservation); } @@ -5594,14 +5621,13 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { void lshift64(TrustedImm32 imm, RegisterID shiftAmount, RegisterID dest) { - if (shiftAmount == dest) { - move(imm, scratchRegister()); - lshift64(shiftAmount, scratchRegister()); - move(scratchRegister(), dest); - } else { - move(imm, dest); - lshift64(shiftAmount, dest); - } + RegisterID target = shiftAmount == dest ? scratchRegister() : dest; + if (imm.m_value < 0) + signExtend32ToPtr(imm, target); + else + move(imm, target); + lshift64(shiftAmount, target); + move(target, dest); } void lshift64(Address src, RegisterID shiftAmount, RegisterID dest) @@ -5972,15 +5998,15 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { void sub64(RegisterID left, RegisterID right, RegisterID dest) { + if (left == right) { + move(TrustedImm32(0), dest); + return; + } if (dest == right) { neg64(dest); add64(left, dest); return; } - if (left == right) { - move(TrustedImm32(0), dest); - return; - } move(left, dest); sub64(right, dest); } @@ -6897,6 +6923,12 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { return Jump(m_assembler.jCC(x86Condition(cond))); } + Jump branch64(RelationalCondition cond, RegisterID left, BaseIndex right) + { + m_assembler.cmpq_mr(right.offset, right.base, right.index, right.scale, left); + return Jump(m_assembler.jCC(x86Condition(cond))); + } + Jump branch64(RelationalCondition cond, AbsoluteAddress left, RegisterID right) { move(TrustedImmPtr(left.m_ptr), scratchRegister()); @@ -6927,6 +6959,12 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { return Jump(m_assembler.jCC(x86Condition(cond))); } + Jump branch64(RelationalCondition cond, BaseIndex left, TrustedImm32 right) + { + m_assembler.cmpq_im(right.m_value, left.offset, left.base, left.index, left.scale); + return Jump(m_assembler.jCC(x86Condition(cond))); + } + Jump branch64(RelationalCondition cond, Address left, Address right) { load64(right, scratchRegister()); @@ -7093,7 +7131,7 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { if (op2 == dest) return branchAdd64(cond, op1, dest); if (op1.base == dest) { - load32(op1, dest); + load64(op1, dest); return branchAdd64(cond, op2, dest); } move(op2, dest); @@ -9783,6 +9821,8 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { void vectorAddSat(SIMDInfo simdInfo, FPRegisterID left, FPRegisterID right, FPRegisterID dest) { ASSERT(simdInfo.signMode != SIMDSignMode::None); + if (dest == right) + std::swap(left, right); switch (simdInfo.lane) { case SIMDLane::i8x16: @@ -9823,6 +9863,10 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { void vectorSubSat(SIMDInfo simdInfo, FPRegisterID left, FPRegisterID right, FPRegisterID dest) { ASSERT(scalarTypeIsIntegral(simdInfo.lane)); + if (!supportsAVX() && dest == right && dest != left) { + moveVector(right, fpTempRegister); + right = fpTempRegister; + } switch (simdInfo.lane) { case SIMDLane::i8x16: @@ -10040,6 +10084,8 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { void vectorAvgRound(SIMDInfo simdInfo, FPRegisterID a, FPRegisterID b, FPRegisterID dest) { + if (dest == b) + std::swap(a, b); switch (simdInfo.lane) { case SIMDLane::i8x16: if (supportsAVX()) @@ -10353,6 +10399,20 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { return start; } + Label comparePCRelative64(RegisterID left) + { + Label start = label(); + m_assembler.cmpq_rip(left); + return start; + } + + Label loadPCRelativeDouble(FPRegisterID dest) + { + m_assembler.movsd_rip(dest); + // A relocation of this kind ends with the instruction. Only a Jump makes a Label of a place other than the end of the buffer. + return Jump(m_assembler.buffer().label().labelAtOffset(-static_cast(relocationSize(RelocationKind::Load64)))).label(); + } + Label moveInstanceAddress(RegisterID instance, RegisterID dest) { RELEASE_ASSERT(instance != X86Registers::esp && instance != X86Registers::r12); diff --git a/Source/JavaScriptCore/assembler/X86Assembler.h b/Source/JavaScriptCore/assembler/X86Assembler.h index 8fb8630b8c118..f7fee092ce30d 100644 --- a/Source/JavaScriptCore/assembler/X86Assembler.h +++ b/Source/JavaScriptCore/assembler/X86Assembler.h @@ -2004,6 +2004,11 @@ class X86Assembler { m_formatter.oneByteOp(OP_CMP_GvEv, src, base, offset); } + void cmpl_mr(int offset, RegisterID base, RegisterID index, int scale, RegisterID src) + { + m_formatter.oneByteOp(OP_CMP_GvEv, src, base, index, scale, offset); + } + void cmpl_ir(int imm, RegisterID dst) { if (CAN_SIGN_EXTEND_8_32(imm)) { @@ -2084,6 +2089,11 @@ class X86Assembler { m_formatter.oneByteOp64(OP_CMP_GvEv, src, base, offset); } + void cmpq_mr(int offset, RegisterID base, RegisterID index, int scale, RegisterID src) + { + m_formatter.oneByteOp64(OP_CMP_GvEv, src, base, index, scale, offset); + } + void cmpq_ir(int imm, RegisterID dst) { if (CAN_SIGN_EXTEND_8_32(imm)) { @@ -3577,6 +3587,17 @@ class X86Assembler { m_formatter.oneByteOp64RIPRelative(OP_MOV_GvEv, dst); } + void cmpq_rip(RegisterID left) + { + m_formatter.oneByteOp64RIPRelative(OP_CMP_GvEv, left); + } + + void movsd_rip(XMMRegisterID dst) + { + m_formatter.prefix(PRE_SSE_F2); + m_formatter.twoByteOpRIPRelative(OP2_MOVSD_VsdWsd, (RegisterID)dst); + } + // Flow control: AssemblerLabel call() @@ -7380,6 +7401,7 @@ class X86Assembler { void oneByteOpAddr(OneByteOpcodeID opcode, int reg, uint32_t address) { SingleInstructionBufferWriter writer(m_buffer); + writer.emitRexIfNeeded(reg, 0, 0); writer.putByteUnchecked(opcode); writer.memoryModRMAddr(reg, address); } @@ -7644,6 +7666,16 @@ class X86Assembler { writer.putIntUnchecked(0); } + void twoByteOpRIPRelative(TwoByteOpcodeID opcode, int reg) + { + SingleInstructionBufferWriter writer(m_buffer); + writer.emitRexIfNeeded(reg, 0, 0); + writer.putByteUnchecked(OP_2BYTE_ESCAPE); + writer.putByteUnchecked(opcode); + writer.putModRm(ModRmMemoryNoDisp, reg, SingleInstructionBufferWriter::noBase); + writer.putIntUnchecked(0); + } + void twoByteOp64(TwoByteOpcodeID opcode, int reg) { SingleInstructionBufferWriter writer(m_buffer); diff --git a/Source/JavaScriptCore/assembler/testmasm.cpp b/Source/JavaScriptCore/assembler/testmasm.cpp index b3236390471f7..01d94070b6335 100644 --- a/Source/JavaScriptCore/assembler/testmasm.cpp +++ b/Source/JavaScriptCore/assembler/testmasm.cpp @@ -4684,7 +4684,7 @@ static void checkStrongCAS(unsigned width, MacroAssembler::StatusCondition condi uint64_t mask = width == 64 ? std::numeric_limits::max() : (1ull << width) - 1; uint64_t unexpected = (old ^ 1) & mask; uint64_t replacement = ~old & mask; - constexpr unsigned numberOfAttempts = 200000; + constexpr unsigned numberOfAttempts = 10000; withContendedWord([&](std::atomic& word) { for (unsigned attempt = 0; attempt < numberOfAttempts; ++attempt) { word.store(old); @@ -4694,7 +4694,7 @@ static void checkStrongCAS(unsigned width, MacroAssembler::StatusCondition condi CHECK_EQ(result.value & mask, old & mask); CHECK_EQ(!!result.status, condition == MacroAssembler::Failure); } - for (unsigned attempt = 0; attempt < numberOfAttempts / 100; ++attempt) { + for (unsigned attempt = 0; attempt < numberOfAttempts / 10; ++attempt) { word.store(old); CompareExchangeResult result; invoke(code, old & mask, replacement, &word, &result); @@ -4721,7 +4721,7 @@ static void checkBranchOnWeakCAS(unsigned width, MacroAssembler::StatusCondition uint64_t unexpected = (old ^ 1) & mask; uint64_t replacement = ~old & mask; withContendedWord([&](std::atomic& word) { - for (unsigned attempt = 0; attempt < 20000; ++attempt) { + for (unsigned attempt = 0; attempt < 2000; ++attempt) { word.store(old); CompareExchangeResult result; invoke(code, unexpected, replacement, &word, &result); @@ -6501,6 +6501,387 @@ void testProbeModifiesStackValues() CHECK_EQ(probeCallCount, 3); } + +#if CPU(X86_64) || CPU(ARM64) +struct MacroOperands { + uint64_t values[4] { }; + alignas(16) v128_t vectors[4]; + uint64_t memory[4] { }; +}; + +// Runs code on registers that are loaded from the operands and stored back. The registers of two operands may be equal. +template +static MacroOperands runOnOperands(MacroOperands operands, std::span gprs, std::span fprs, const Emit& emit, CPUFeatureSet features = CPUFeatureSet::host()) +{ + constexpr GPRReg operandsGPR = GPRInfo::argumentGPR0; + CCallHelpers jit; + AssemblerPolicy policy; + policy.features = features; + jit.setPolicy(policy); + emitFunctionPrologue(jit); + jit.pushPair(operandsGPR, operandsGPR); + for (unsigned i = 0; i < fprs.size(); ++i) + jit.loadVector(CCallHelpers::Address(operandsGPR, OBJECT_OFFSETOF(MacroOperands, vectors) + i * sizeof(v128_t)), fprs[i]); + for (unsigned i = 0; i < gprs.size(); ++i) + jit.load64(CCallHelpers::Address(operandsGPR, OBJECT_OFFSETOF(MacroOperands, values) + i * sizeof(uint64_t)), gprs[i]); + emit(jit); + constexpr GPRReg savedGPR = GPRInfo::nonPreservedNonArgumentGPR0; + jit.load64(CCallHelpers::Address(CCallHelpers::stackPointerRegister), savedGPR); + for (unsigned i = 0; i < gprs.size(); ++i) + jit.store64(gprs[i], CCallHelpers::Address(savedGPR, OBJECT_OFFSETOF(MacroOperands, values) + i * sizeof(uint64_t))); + for (unsigned i = 0; i < fprs.size(); ++i) + jit.storeVector(fprs[i], CCallHelpers::Address(savedGPR, OBJECT_OFFSETOF(MacroOperands, vectors) + i * sizeof(v128_t))); + jit.addPtr(CCallHelpers::TrustedImm32(2 * sizeof(void*)), CCallHelpers::stackPointerRegister); + emitFunctionEpilogue(jit); + jit.ret(); + LinkBuffer linkBuffer(jit, nullptr); + auto code = FINALIZE_CODE(linkBuffer, JSEntryPtrTag, nullptr, "testmasm operands"); + invoke(code, &operands); + return operands; +} + +static void testSubtractWithEqualRegisters() +{ + constexpr GPRReg registers[] = { GPRInfo::argumentGPR1, GPRInfo::argumentGPR2, GPRInfo::argumentGPR3 }; + for (GPRReg left : registers) { + for (GPRReg right : registers) { + for (GPRReg dest : registers) { + for (uint64_t value : { 0x1122334455667788ull, 1ull, 0xffffffff80000000ull }) { + GPRReg gprs[] = { left, right, dest }; + MacroOperands operands; + operands.values[0] = value; + operands.values[1] = left == right ? value : value * 3 + 7; + operands.values[2] = dest == right ? operands.values[1] : dest == left ? value : 42; + uint64_t difference = operands.values[0] - operands.values[1]; + auto result64 = runOnOperands(operands, gprs, { }, [&](CCallHelpers& jit) { + jit.sub64(left, right, dest); + }); + CHECK_EQ(result64.values[2], difference); + auto result32 = runOnOperands(operands, gprs, { }, [&](CCallHelpers& jit) { + jit.sub32(left, right, dest); + }); + CHECK_EQ(result32.values[2], static_cast(difference)); + } + } + } + } +} + +static void testClearBitsWithEqualRegisters() +{ + constexpr GPRReg registers[] = { GPRInfo::argumentGPR1, GPRInfo::argumentGPR2, GPRInfo::argumentGPR3 }; + for (GPRReg source : registers) { + for (GPRReg mask : registers) { + for (GPRReg dest : registers) { + for (auto attributes : { CCallHelpers::ClearBitsAttributes::OKToClobberMask, CCallHelpers::ClearBitsAttributes::MustPreserveMask }) { + GPRReg gprs[] = { source, mask, dest }; + MacroOperands operands; + operands.values[0] = 0x1122334455667788; + operands.values[1] = source == mask ? operands.values[0] : 0xff00ff00f0f0f0f0; + operands.values[2] = dest == mask ? operands.values[1] : dest == source ? operands.values[0] : 42; + auto result = runOnOperands(operands, gprs, { }, [&](CCallHelpers& jit) { + jit.clearBits64WithMask(source, mask, dest, attributes); + }); + CHECK_EQ(result.values[2], operands.values[0] & ~operands.values[1]); + GPRReg twoGPRs[] = { mask, dest }; + MacroOperands two; + two.values[0] = operands.values[1]; + two.values[1] = dest == mask ? operands.values[1] : operands.values[0]; + result = runOnOperands(two, twoGPRs, { }, [&](CCallHelpers& jit) { + jit.clearBits64WithMask(mask, dest, attributes); + }); + CHECK_EQ(result.values[1], two.values[1] & ~two.values[0]); + } + } + } + } +} + +static void testRelaxedDotProductWithEqualRegisters() +{ + constexpr FPRReg registers[] = { FPRInfo::fpRegT0, FPRInfo::fpRegT1, FPRInfo::fpRegT2, FPRInfo::fpRegT3 }; + const v128_t values[] = { v128_t { 0x0102030405060708, 0x7f7e7d7c01000203 }, v128_t { 0x1112131415161718, 0x0f0e0d0c0b0a0908 }, v128_t { 0x0000000100000002, 0x7fffffff00000003 }, v128_t { 42, 43 } }; + auto run = [&](CPUFeatureSet features, FPRReg a, FPRReg b, FPRReg addend, FPRReg dest) { + FPRReg fprs[] = { a, b, addend, dest }; + MacroOperands operands; + for (unsigned i = 0; i < 4; ++i) { + operands.vectors[i] = values[i]; + for (unsigned j = 0; j < i; ++j) { + if (fprs[i] == fprs[j]) + operands.vectors[i] = operands.vectors[j]; + } + } + auto result = runOnOperands(operands, { }, fprs, [&](CCallHelpers& jit) { + jit.vectorRelaxedDotI8x16I7x16Add(a, b, addend, dest, FPRInfo::fpRegT4, FPRInfo::fpRegT5); + }, features); + for (unsigned lane = 0; lane < 4; ++lane) { + uint32_t sum = operands.vectors[2].u32x4[lane]; + for (unsigned i = lane * 4; i < lane * 4 + 4; ++i) + sum += static_cast(static_cast(operands.vectors[0].u8x16[i]) * static_cast(operands.vectors[1].u8x16[i])); + CHECK_EQ(result.vectors[3].u32x4[lane], sum); + } + }; +#if CPU(X86_64) + if (!CPUFeatureSet::host().contains(CPUFeature::AVX)) + return; +#endif + for (FPRReg a : registers) { + for (FPRReg b : registers) { + for (FPRReg addend : registers) { + for (FPRReg dest : registers) { + run(CPUFeatureSet::host(), a, b, addend, dest); +#if CPU(ARM64) + run(CPUFeatureSet { }, a, b, addend, dest); +#endif + } + } + } + } +} + +static void testImmediatesOfMemoryOperations() +{ + for (uint32_t immediate : { 0u, 1u, 0x12345678u, 0x80000000u, 0xffffffffu, 0x00010203u, 0xff00u }) { + for (int32_t offset : { 0, 8, 100000 }) { + uint32_t words[] = { 0x01020304, 0x05060708, 0x090a0b0c, 0x0d0e0f10 }; + auto code = compile([=](CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.or32(CCallHelpers::TrustedImm32(immediate), CCallHelpers::Address(GPRInfo::argumentGPR0, offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + invoke(code, std::bit_cast(&words[1]) - offset); + CHECK_EQ(words[0], 0x01020304u); + CHECK_EQ(words[1], 0x05060708u | immediate); + CHECK_EQ(words[2], 0x090a0b0cu); + } + } +} + +static void testShiftedImmediates() +{ + for (int32_t amount : { 0, 1, 31, 32, 40, 63 }) { + MacroOperands operands; + operands.values[0] = 0x8877665544332211; + constexpr GPRReg value = GPRInfo::argumentGPR1; + GPRReg gprs[] = { value }; + CHECK_EQ(runOnOperands(operands, gprs, { }, [&](CCallHelpers& jit) { jit.lshiftPtr(CCallHelpers::Imm32(amount), value); }).values[0], operands.values[0] << amount); + CHECK_EQ(runOnOperands(operands, gprs, { }, [&](CCallHelpers& jit) { jit.urshiftPtr(CCallHelpers::Imm32(amount), value); }).values[0], operands.values[0] >> amount); + CHECK_EQ(runOnOperands(operands, gprs, { }, [&](CCallHelpers& jit) { jit.rshiftPtr(CCallHelpers::Imm32(amount), value); }).values[0], static_cast(static_cast(operands.values[0]) >> amount)); + for (int32_t immediate : { 1, -1, -2, 0x12345678, std::numeric_limits::min() }) { + for (GPRReg dest : { GPRInfo::argumentGPR1, GPRInfo::argumentGPR2 }) { + GPRReg two[] = { value, dest }; + MacroOperands shift; + shift.values[0] = amount; + shift.values[1] = dest == value ? amount : 42; + auto result = runOnOperands(shift, two, { }, [&](CCallHelpers& jit) { jit.lshift64(CCallHelpers::TrustedImm32(immediate), value, dest); }); + CHECK_EQ(result.values[1], static_cast(static_cast(immediate)) << amount); + } + } + } +} + +// Blinding is random, so each constant is emitted often. +static void testBlindedAdditions() +{ + constexpr GPRReg value = GPRInfo::argumentGPR1; + GPRReg gprs[] = { value }; + for (int32_t immediate : { 0x01000000, 0x40000000, 0x7fffffff, -0x01000000, -0x40000004, std::numeric_limits::min(), std::numeric_limits::min() + 2 }) { + MacroOperands operands; + operands.values[0] = 0x0000123400000010; + auto result = runOnOperands(operands, gprs, { }, [&](CCallHelpers& jit) { + for (unsigned i = 0; i < 1000; ++i) { + jit.addPtr(CCallHelpers::Imm32(immediate), value); + jit.subPtr(CCallHelpers::TrustedImm32(immediate), value); + jit.subPtr(CCallHelpers::Imm32(immediate), value); + jit.addPtr(CCallHelpers::TrustedImm32(immediate), value); + } + jit.addPtr(CCallHelpers::Imm32(immediate), value); + }); + CHECK_EQ(result.values[0], operands.values[0] + static_cast(immediate)); + } +} + +static void testSignOfMaskedWords() +{ + constexpr GPRReg value = GPRInfo::argumentGPR0; + for (uint32_t mask : { 0x1u, 0x7fu, 0x80u, 0xffu, 0x8000u, 0xff00u, 0x800000u, 0x80000000u, 0xff000000u, 0xffffffffu }) { + for (auto condition : { CCallHelpers::Zero, CCallHelpers::NonZero, CCallHelpers::Signed, CCallHelpers::PositiveOrZero }) { + auto expected = [=](uint32_t word) -> int { + uint32_t masked = word & mask; + switch (condition) { + case CCallHelpers::Zero: + return !masked; + case CCallHelpers::NonZero: + return !!masked; + case CCallHelpers::Signed: + return static_cast(masked) < 0; + default: + return static_cast(masked) >= 0; + } + }; + auto branchCode = [&](const auto& emit) { + return compile([=](CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = emit(jit); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + auto done = jit.jump(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + done.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + }; + auto branchOnRegister = branchCode([=](CCallHelpers& jit) { + jit.move(value, GPRInfo::returnValueGPR); + return jit.branchTest32(condition, GPRInfo::returnValueGPR, CCallHelpers::TrustedImm32(mask)); + }); + auto branchOnMemory = branchCode([=](CCallHelpers& jit) { + return jit.branchTest32(condition, CCallHelpers::Address(GPRInfo::argumentGPR1), CCallHelpers::TrustedImm32(mask)); + }); + auto setFromRegister = compile([=](CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(value, GPRInfo::returnValueGPR); + jit.test32(condition, GPRInfo::returnValueGPR, CCallHelpers::TrustedImm32(mask), GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + auto movedImmediate = compile([=](CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(value, GPRInfo::returnValueGPR); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::argumentGPR2); + jit.moveConditionallyTest32(condition, GPRInfo::returnValueGPR, CCallHelpers::TrustedImm32(mask), CCallHelpers::TrustedImm32(7), GPRInfo::argumentGPR2, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + auto moved = compile([=](CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.move(value, GPRInfo::returnValueGPR); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::argumentGPR1); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::argumentGPR2); + jit.moveConditionallyTest32(condition, GPRInfo::returnValueGPR, CCallHelpers::TrustedImm32(mask), GPRInfo::argumentGPR1, GPRInfo::argumentGPR2, GPRInfo::returnValueGPR); + emitFunctionEpilogue(jit); + jit.ret(); + }); + for (uint32_t word : { 0u, 1u, 0x80u, 0x7fu, 0xffu, 0x8000u, 0x800000u, 0x80000000u, 0xffffffffu, 0x7fffffffu, 0x12345678u }) { + CHECK_EQ(invoke(branchOnRegister, word, &word), expected(word)); + CHECK_EQ(invoke(branchOnMemory, word, &word), expected(word)); + CHECK_EQ(invoke(setFromRegister, word, &word), expected(word)); + CHECK_EQ(invoke(movedImmediate, word, &word), expected(word) ? 7 : 0); + CHECK_EQ(invoke(moved, word, &word), expected(word)); + } + } + } + for (uint64_t word : { 0x80000000ull, 0xffffffff00000000ull, 0x8000000000000000ull, 0x7fffffffull, 1ull << 40 }) { + auto branch64 = compile([=](CCallHelpers& jit) { + emitFunctionPrologue(jit); + auto taken = jit.branchTest64(CCallHelpers::NonZero, value, CCallHelpers::TrustedImm32(std::numeric_limits::min())); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::returnValueGPR); + auto done = jit.jump(); + taken.link(&jit); + jit.move(CCallHelpers::TrustedImm32(1), GPRInfo::returnValueGPR); + done.link(&jit); + emitFunctionEpilogue(jit); + jit.ret(); + }); + CHECK_EQ(invoke(branch64, word), !!(word & 0xffffffff80000000)); + } +} + +#if CPU(X86_64) +static void testBranchAddOfMemoryIntoItsBase() +{ + for (GPRReg dest : { GPRInfo::argumentGPR1, GPRInfo::argumentGPR2, GPRInfo::argumentGPR3 }) { + uint64_t word = 0x1122334400000001; + GPRReg gprs[] = { GPRInfo::argumentGPR1, GPRInfo::argumentGPR2, dest }; + MacroOperands operands; + operands.values[0] = std::bit_cast(&word); + operands.values[1] = 0x0000000100000002; + operands.values[2] = dest == gprs[0] ? operands.values[0] : dest == gprs[1] ? operands.values[1] : 42; + auto result = runOnOperands(operands, gprs, { }, [&](CCallHelpers& jit) { + jit.branchAdd64(CCallHelpers::Overflow, CCallHelpers::Address(GPRInfo::argumentGPR1), GPRInfo::argumentGPR2, dest).link(&jit); + }); + CHECK_EQ(result.values[2], word + operands.values[1]); + } +} + +static void testNarrowVectorOperationsWithEqualRegisters() +{ + CPUFeatureSet withoutAVX = CPUFeatureSet::fromBits(CPUFeatureSet::host().bits() & ~(CPUFeatureSet().with(CPUFeature::AVX).with(CPUFeature::AVX2).bits())); + constexpr FPRReg registers[] = { FPRInfo::fpRegT0, FPRInfo::fpRegT1, FPRInfo::fpRegT2 }; + for (CPUFeatureSet features : { CPUFeatureSet::host(), withoutAVX }) { + for (FPRReg left : registers) { + for (FPRReg right : registers) { + for (FPRReg dest : registers) { + FPRReg fprs[] = { left, right, dest }; + MacroOperands operands; + operands.vectors[0] = v128_t { 0x00102030f0e0d0c0, 0xff7f80017f80fe02 }; + operands.vectors[1] = left == right ? operands.vectors[0] : v128_t { 0x1180ff7f01020304, 0x0102030405060708 }; + operands.vectors[2] = dest == right ? operands.vectors[1] : dest == left ? operands.vectors[0] : v128_t { 42, 43 }; + SIMDInfo info { SIMDLane::i8x16, SIMDSignMode::Unsigned }; + auto sum = runOnOperands(operands, { }, fprs, [&](CCallHelpers& jit) { jit.vectorAddSat(info, left, right, dest); }, features); + auto difference = runOnOperands(operands, { }, fprs, [&](CCallHelpers& jit) { jit.vectorSubSat(info, left, right, dest); }, features); + auto average = runOnOperands(operands, { }, fprs, [&](CCallHelpers& jit) { jit.vectorAvgRound(info, left, right, dest); }, features); + for (unsigned i = 0; i < 16; ++i) { + unsigned a = operands.vectors[0].u8x16[i], b = operands.vectors[1].u8x16[i]; + CHECK_EQ(sum.vectors[2].u8x16[i], std::min(a + b, 255u)); + CHECK_EQ(difference.vectors[2].u8x16[i], a > b ? a - b : 0u); + CHECK_EQ(average.vectors[2].u8x16[i], (a + b + 1) / 2); + } + } + } + } + } +} + +static void testMovesOfAbsoluteAddressesEncodeEveryRegister() +{ + for (int reg = X86Registers::eax; reg <= X86Registers::r15; ++reg) { + X86Assembler assembler; + assembler.movl_mr(0x12345678u, static_cast(reg)); + assembler.movl_rm(static_cast(reg), 0x12345678u); + std::span code { static_cast(assembler.buffer().data()), assembler.codeSize() }; + size_t length = code.size() / 2; + CHECK_EQ(length, reg < X86Registers::r8 ? 7u : 8u); + for (unsigned i = 0; i < 2; ++i) { + std::span instruction = code.subspan(i * length, length); + if (reg >= X86Registers::r8) { + CHECK_EQ(instruction[0], 0x44); + instruction = instruction.subspan(1); + } + CHECK_EQ(instruction[0], i ? 0x89 : 0x8b); + CHECK_EQ((instruction[1] >> 3) & 7, reg & 7); + } + } +} +#endif // CPU(X86_64) + +#if CPU(ARM64) +static void testPairsOfFloats() +{ + for (int32_t offset : { 0, 8, -256, 252, 256, 1000, -1000, 100000 }) { + uint32_t words[] = { 1, 0x3f800000, 0x40490fdb, 4, 5 }; + uintptr_t base = std::bit_cast(&words[1]) - offset; + uint32_t copies[] = { 11, 12, 13, 14, 15 }; + uintptr_t baseOfCopies = std::bit_cast(&copies[1]) - offset; + auto code = compile([=](CCallHelpers& jit) { + emitFunctionPrologue(jit); + jit.loadPair32(GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(offset), FPRInfo::fpRegT0, FPRInfo::fpRegT1); + jit.storePair32(FPRInfo::fpRegT0, FPRInfo::fpRegT1, GPRInfo::argumentGPR1, CCallHelpers::TrustedImm32(offset)); + emitFunctionEpilogue(jit); + jit.ret(); + }); + invoke(code, base, baseOfCopies); + CHECK_EQ(copies[0], 11u); + CHECK_EQ(copies[1], words[1]); + CHECK_EQ(copies[2], words[2]); + CHECK_EQ(copies[3], 14u); + } +} +#endif // CPU(ARM64) +#endif // CPU(X86_64) || CPU(ARM64) + void testOrImmMem() { // FIXME: this does not test that the or does not touch beyond its width. @@ -12248,6 +12629,24 @@ void run(const char* filter) WTF_IGNORES_THREAD_SAFETY_ANALYSIS RUN(testByteRegistersEncodeEveryRegister()); #endif +#if CPU(X86_64) || CPU(ARM64) + RUN(testSubtractWithEqualRegisters()); + RUN(testClearBitsWithEqualRegisters()); + RUN(testRelaxedDotProductWithEqualRegisters()); + RUN(testImmediatesOfMemoryOperations()); + RUN(testShiftedImmediates()); + RUN(testBlindedAdditions()); + RUN(testSignOfMaskedWords()); +#endif +#if CPU(X86_64) + RUN(testBranchAddOfMemoryIntoItsBase()); + RUN(testNarrowVectorOperationsWithEqualRegisters()); + RUN(testMovesOfAbsoluteAddressesEncodeEveryRegister()); +#endif +#if CPU(ARM64) + RUN(testPairsOfFloats()); +#endif + RUN(testOrImmMem()); RUN(testAndOrDouble()); diff --git a/Source/JavaScriptCore/b3/B3EliminateCommonSubexpressions.cpp b/Source/JavaScriptCore/b3/B3EliminateCommonSubexpressions.cpp index 97e138a2932b2..8d17a3a78e82e 100644 --- a/Source/JavaScriptCore/b3/B3EliminateCommonSubexpressions.cpp +++ b/Source/JavaScriptCore/b3/B3EliminateCommonSubexpressions.cpp @@ -248,13 +248,12 @@ class CSE { continue; dataLogLnIf(B3EliminateCommonSubexpressionsInternal::verbose, "Looking at ", *m_block, ":"); - m_data = ImpureBlockData(); for (m_index = 0; m_index < m_block->size(); ++m_index) { m_value = m_block->at(m_index); process(); } m_insertionSet.execute(m_block); - m_impureBlockData[m_block] = m_data; + m_impureBlockData[m_block] = std::exchange(m_data, { }); } } diff --git a/Source/JavaScriptCore/b3/B3FixSSA.cpp b/Source/JavaScriptCore/b3/B3FixSSA.cpp index e9a7d6af84c28..eca997aadaf6e 100644 --- a/Source/JavaScriptCore/b3/B3FixSSA.cpp +++ b/Source/JavaScriptCore/b3/B3FixSSA.cpp @@ -120,6 +120,9 @@ class FrontierEdges { static constexpr unsigned noEdge = UINT_MAX; + // Up to this many edges are scanned linearly, without a tree. + static constexpr unsigned maxScannedEdges = 16; + void clear() { m_edges.shrink(0); } unsigned size() const { return m_edges.size(); } const Edge& at(unsigned position) const LIFETIME_BOUND { return m_edges[position]; } @@ -127,8 +130,13 @@ class FrontierEdges { // In ascending order of `from`. void append(const Edge& edge) { m_edges.append(edge); } + // Called for each number in order, before its edges are appended, and once after the last. + void beginEdgesFromNextNumber() { m_firstEdgeFrom.append(m_edges.size()); } + void finish() { + if (m_edges.size() <= maxScannedEdges) + return; m_leafCount = std::bit_ceil(std::max(m_edges.size(), 1)); m_leastBound.fill(UINT_MAX, 2 * m_leafCount); for (unsigned position = 0; position < m_edges.size(); ++position) @@ -139,6 +147,9 @@ class FrontierEdges { void remove(unsigned position) { + m_edges[position].bound = UINT_MAX; + if (m_edges.size() <= maxScannedEdges) + return; unsigned node = m_leafCount + position; m_leastBound[node] = UINT_MAX; while (node >>= 1) @@ -150,8 +161,24 @@ class FrontierEdges { template bool forEachJoinInFrontier(unsigned first, unsigned last, NOESCAPE const Functor& functor) { - unsigned begin = std::ranges::lower_bound(m_edges, first, { }, &Edge::from) - m_edges.begin(); - unsigned end = std::ranges::upper_bound(m_edges, last, { }, &Edge::from) - m_edges.begin(); + unsigned begin; + unsigned end; + if (m_firstEdgeFrom.isEmpty()) { + begin = std::ranges::lower_bound(m_edges, first, { }, &Edge::from) - m_edges.begin(); + end = std::ranges::upper_bound(m_edges, last, { }, &Edge::from) - m_edges.begin(); + } else if (first >= m_firstEdgeFrom.size()) + return true; // The block is unreachable. + else { + begin = m_firstEdgeFrom[first]; + end = m_firstEdgeFrom[last + 1]; + } + if (end - begin <= maxScannedEdges) { + for (unsigned position = begin; position < end; ++position) { + if (m_edges[position].bound < first && !functor(position)) + return false; + } + return true; + } return visit(1, 0, m_leafCount, begin, end, first, functor); } @@ -169,6 +196,8 @@ class FrontierEdges { } Vector m_edges; + // Indexed by number: the position of its first edge. Empty if the edges are sparse. + Vector m_firstEdgeFrom; // A binary tree over the edges: node n has the children 2 n and 2 n + 1, and the edges are its last m_leafCount nodes. Vector m_leastBound; unsigned m_leafCount { 1 }; @@ -208,9 +237,11 @@ class LivePhiFinder { auto& dominators = m_dominators; unsigned nextNumber = 1; + m_frontiers.beginEdgesFromNextNumber(); dominators.forAllBlocksInDominatorTreePreOrder( [&](BasicBlock* block) { unsigned number = nextNumber++; + m_frontiers.beginEdgesFromNextNumber(); m_firstDominated[block] = number; for (BasicBlock* join : block->successorBlocks()) { if (join->numPredecessors() < 2) @@ -224,6 +255,7 @@ class LivePhiFinder { } }, [&](BasicBlock* block) { m_lastDominated[block] = nextNumber - 1; }); + m_frontiers.beginEdgesFromNextNumber(); m_frontiers.finish(); m_numberOfBlocks = nextNumber; } @@ -794,6 +826,9 @@ void demoteValues(Procedure& proc, const IndexSet& values) dataLog("\n"); } + auto variableOf = [&](Value* value) { return values.contains(value) ? map.get(value) : nullptr; }; + auto phiVariableOf = [&](Value* value) { return values.contains(value) ? phiMap.get(value) : nullptr; }; + // Change accesses to the values to accesses to the stack slots. InsertionSet insertionSet(proc); for (BasicBlock* block : proc) { @@ -801,7 +836,7 @@ void demoteValues(Procedure& proc, const IndexSet& values) // Deal with terminals that produce values (i.e. patchpoint terminals, like the ones we // generate for the allocation fast path). Value* value = block->predecessor(0)->last(); - Variable* variable = map.get(value); + Variable* variable = variableOf(value); if (variable && block->predecessor(0)->successorBlock(0) == block) { RELEASE_ASSERT(block->numPredecessors() == 1); // Critical edges better be broken. insertionSet.insert(0, Set, value->origin(), variable, value); @@ -812,21 +847,21 @@ void demoteValues(Procedure& proc, const IndexSet& values) Value* value = block->at(valueIndex); if (value->opcode() == Phi) { - if (Variable* variable = phiMap.get(value)) { + if (Variable* variable = phiVariableOf(value)) { value->replaceWithIdentity( insertionSet.insert( valueIndex, Get, value->origin(), variable)); } } else { for (Value*& child : value->children()) { - if (Variable* variable = map.get(child)) { + if (Variable* variable = variableOf(child)) { child = insertionSet.insert( valueIndex, Get, value->origin(), variable); } } if (UpsilonValue* upsilon = value->as()) { - if (Variable* variable = phiMap.get(upsilon->phi())) { + if (Variable* variable = phiVariableOf(upsilon->phi())) { insertionSet.insert( valueIndex, Set, upsilon->origin(), variable, upsilon->child(0)); value->replaceWithNop(); @@ -834,7 +869,7 @@ void demoteValues(Procedure& proc, const IndexSet& values) } } - if (Variable* variable = map.get(value)) { + if (Variable* variable = variableOf(value)) { if (valueIndex + 1 < block->size()) { insertionSet.insert( valueIndex + 1, Set, value->origin(), variable, value); diff --git a/Source/JavaScriptCore/b3/B3LowerToAir.cpp b/Source/JavaScriptCore/b3/B3LowerToAir.cpp index 3c89b76a0c5c8..d6a548e8ff7c6 100644 --- a/Source/JavaScriptCore/b3/B3LowerToAir.cpp +++ b/Source/JavaScriptCore/b3/B3LowerToAir.cpp @@ -1055,6 +1055,12 @@ class LowerToAir { if (address->opcode() == SlotBase && m_procedure.policy().selection.extendsInstructionSelection) return Arg::stack(address->as()->slot(), offset); + if (address->opcode() == WasmAddress && address->child(0)->hasInt64() && m_procedure.policy().selection.extendsInstructionSelection) { + CheckedInt32 displacement = CheckedInt32 { offset } + address->child(0)->asInt64(); + if (!displacement.hasOverflowed() && Arg::isValidAddrForm(Air::Move, displacement.value(), width)) + return Arg::addr(Tmp(address->as()->pinnedGPR()), displacement.value()); + } + static constexpr unsigned lotsOfUses = 10; // This is arbitrary and we should tune it eventually. // Only match if the address value isn't used in some large number of places. @@ -1239,6 +1245,11 @@ class LowerToAir { return !memory || memory->accessWidth() <= width; } + static bool hasBitsOfOneByte(uint32_t mask) + { + return !(mask & ~0xffu) || !(mask & ~0xff00u) || !(mask & ~0xff0000u) || !(mask & ~0xff000000u); + } + static Width maxWidthRead(Air::Opcode opcode) { switch (opcode) { @@ -2612,6 +2623,13 @@ class LowerToAir { // Chew through any inversions. This loop isn't necessary for comparisons and branches, but // we do need at least one iteration of it for Check. for (;;) { + // ZExt32(x) and SExt32(x) are zero exactly when x is. + if (m_procedure.policy().selection.extendsInstructionSelection && (value->opcode() == ZExt32 || value->opcode() == SExt32) && value != m_value && canCommitInternal && canBeInternal(value)) { + commitInternal(value); + value = value->child(0); + continue; + } + bool shouldInvert = (value->opcode() == BitXor && value->child(1)->hasInt() && (value->child(1)->asInt() == 1) && value->child(0)->returnsBool()) || (value->opcode() == Equal && value->child(1)->isInt(0)); @@ -2627,12 +2645,6 @@ class LowerToAir { inverted = !inverted; } - // ZExt32(x) and SExt32(x) are zero exactly when x is. - if (m_procedure.policy().selection.extendsInstructionSelection && (value->opcode() == ZExt32 || value->opcode() == SExt32) && value != m_value && canCommitInternal && canBeInternal(value)) { - commitInternal(value); - value = value->child(0); - } - auto createRelCond = [&] ( MacroAssembler::RelationalCondition relationalCondition, MacroAssembler::DoubleCondition doubleCondition) { @@ -2820,6 +2832,9 @@ class LowerToAir { auto tryTestLoadImm = [&] (Width width, Arg::Signedness signedness, B3::Opcode loadOpcode) -> Inst { if (!hasRightConst || left->opcode() != loadOpcode || !canReadInPlace(left, width)) return Inst(); + // MacroAssemblerX86_64::generateTest32(Address, ..) reads the one byte that the mask has bits in. + if (isX86() && width == Width32 && left->traps() && loadPromise(left, loadOpcode).kind() == Arg::Addr && hasBitsOfOneByte(static_cast(rightConst))) + return Inst(); // Signed loads will create high bits, so if the immediate has high bits // then we cannot proceed. Consider BitAnd(Load8S(ptr), 0x101). This cannot // be turned into testb (ptr), $1, since if the high bit within that byte @@ -6706,7 +6721,17 @@ class LowerToAir { } case ZExt32: { - appendUnOp(m_value->child(0)); + Value* lowWord = m_value->child(0); + // A load that traps has to read all of its bytes. + if (m_procedure.policy().selection.extendsInstructionSelection && lowWord->opcode() == Trunc && lowWord->child(0)->type() == Int64 && !lowWord->child(0)->traps() && canBeInternal(lowWord)) { + ArgPromise address = loadPromise(lowWord->child(0)); + if (isValidForm(Move32, address.kind(), Arg::Tmp) && address.peek().isValidForm(Move32, Width32)) { + commitInternal(lowWord); + append(address.inst(Move32, m_value, address.consume(*this), tmp(m_value))); + return; + } + } + appendUnOp(lowWord); return; } @@ -6728,10 +6753,9 @@ class LowerToAir { case ArgumentReg: { if (m_valueToTmp[m_value].isReg()) return; - m_prologue.append(Inst( - moveForType(m_value->type()), m_value, - Tmp(m_value->as()->argumentReg()), - tmp(m_value))); + Reg reg = m_value->as()->argumentReg(); + bool isZeroExtended = m_procedure.policy().air.registers.zeroExtended.contains(reg, IgnoreVectors); + m_prologue.append(Inst(isZeroExtended ? Move32 : moveForType(m_value->type()), m_value, Tmp(reg), tmp(m_value))); return; } @@ -7104,7 +7128,8 @@ class LowerToAir { case ValueRep::Register: { Tmp reg = Tmp(rep.reg()); args.append(reg); - after.append(Inst(relaxedMoveForType(type), m_value, reg, tmp)); + bool isZeroExtended = type == Int32 && m_procedure.policy().air.registers.zeroExtended.contains(rep.reg(), IgnoreVectors); + after.append(Inst(isZeroExtended ? Move32 : relaxedMoveForType(type), m_value, reg, tmp)); return; } case ValueRep::StackArgument: { diff --git a/Source/JavaScriptCore/b3/B3PatchpointValue.h b/Source/JavaScriptCore/b3/B3PatchpointValue.h index 4740a6a127fcc..f2b32573c1bee 100644 --- a/Source/JavaScriptCore/b3/B3PatchpointValue.h +++ b/Source/JavaScriptCore/b3/B3PatchpointValue.h @@ -73,6 +73,10 @@ class PatchpointValue final : public StackmapValue { bool iterationsAreShort { false }; // Emits no code and has no result. It only keeps its children live up to this point. bool onlyKeepsChildrenAlive { false }; + // The result is a pinned register. + bool emitsNoCode { false }; + // x86-64: on return the zero flag reflects the first result, an Int32 in a register. + bool zeroFlagReflectsFirstResult { false }; std::optional pureFunction; ClientTag clientTag; diff --git a/Source/JavaScriptCore/b3/B3RangeReductions.cpp b/Source/JavaScriptCore/b3/B3RangeReductions.cpp index 23fb6572c381d..4c06260c8f04c 100644 --- a/Source/JavaScriptCore/b3/B3RangeReductions.cpp +++ b/Source/JavaScriptCore/b3/B3RangeReductions.cpp @@ -36,6 +36,19 @@ static std::optional extensionKindOf(const Procedure& proc, Value return value->opcode() == SExt32 ? ExtensionKind::Sign : ExtensionKind::Zero; } +static std::optional extensionKindOf(int64_t constant) +{ + bool isSignExtended = constant == static_cast(constant); + bool isZeroExtended = constant == static_cast(constant); + if (isSignExtended && isZeroExtended) + return ExtensionKind::Either; + if (isSignExtended) + return ExtensionKind::Sign; + if (isZeroExtended) + return ExtensionKind::Zero; + return std::nullopt; +} + static Opcode unsignedComparison(Opcode opcode) { switch (opcode) { @@ -56,14 +69,17 @@ static Opcode unsignedComparison(Opcode opcode) static RangeReduction compareBeforeExtension(const Procedure& proc, Value* comparison) { std::optional left = extensionKindOf(proc, comparison->child(0)); - std::optional right = extensionKindOf(proc, comparison->child(1)); + Value* rightOperand = comparison->child(1); + std::optional right = rightOperand->hasInt64() ? extensionKindOf(rightOperand->asInt64()) : extensionKindOf(proc, rightOperand); if (!left || !right) return { }; ExtensionKind kind = *left == ExtensionKind::Either ? *right : *left; if (*right != ExtensionKind::Either && *right != kind) return { }; Opcode opcode = kind == ExtensionKind::Zero ? unsignedComparison(comparison->opcode()) : comparison->opcode(); - return { RangeReduction::Kind::Comparison, 0, comparison->child(0)->child(0), comparison->child(1)->child(0), opcode }; + if (rightOperand->hasInt64()) + return { RangeReduction::Kind::ComparisonWithConstant, static_cast(rightOperand->asInt64()), comparison->child(0)->child(0), nullptr, opcode }; + return { RangeReduction::Kind::Comparison, 0, comparison->child(0)->child(0), rightOperand->child(0), opcode }; } static Value* truncatedInt64(Value* extension) diff --git a/Source/JavaScriptCore/b3/B3RangeReductions.h b/Source/JavaScriptCore/b3/B3RangeReductions.h index 622fc076d31c9..70cc1d96c5438 100644 --- a/Source/JavaScriptCore/b3/B3RangeReductions.h +++ b/Source/JavaScriptCore/b3/B3RangeReductions.h @@ -22,6 +22,7 @@ struct RangeReduction { Value, ZeroExtension, Comparison, + ComparisonWithConstant, }; Kind kind { Kind::None }; diff --git a/Source/JavaScriptCore/b3/B3ReduceStrength.cpp b/Source/JavaScriptCore/b3/B3ReduceStrength.cpp index 5ec329990e9cc..3abd43f1ff015 100644 --- a/Source/JavaScriptCore/b3/B3ReduceStrength.cpp +++ b/Source/JavaScriptCore/b3/B3ReduceStrength.cpp @@ -4501,6 +4501,9 @@ class ReduceStrength { case RangeReduction::Kind::Comparison: replaceWithNew(reduction.opcode, m_value->origin(), reduction.value, reduction.right); return true; + case RangeReduction::Kind::ComparisonWithConstant: + replaceWithNew(reduction.opcode, m_value->origin(), reduction.value, m_insertionSet.insertIntConstant(m_index, m_value->origin(), Int32, reduction.constant)); + return true; } RELEASE_ASSERT_NOT_REACHED(); } diff --git a/Source/JavaScriptCore/b3/B3ThreadJumps.cpp b/Source/JavaScriptCore/b3/B3ThreadJumps.cpp index adc903cca156a..bd00805e7e79f 100644 --- a/Source/JavaScriptCore/b3/B3ThreadJumps.cpp +++ b/Source/JavaScriptCore/b3/B3ThreadJumps.cpp @@ -654,6 +654,23 @@ class ThreadJumps { return true; } + // A block that was merged into its predecessor may have a Phi behind the Upsilon that feeds it. + void addPhiValuesFromOwnBlock(BasicBlock* block) + { + for (unsigned index = 0; index < block->size(); ++index) { + Value* phi = block->at(index); + if (phi->opcode() != Phi) + continue; + for (unsigned earlier = index; earlier--;) { + auto* upsilon = block->at(earlier)->as(); + if (upsilon && upsilon->phi() == phi) { + m_phiValuesOnPath.append({ phi, upsilon->child(0) }); + break; + } + } + } + } + void addUpsilonsAndSetsFrom(BasicBlock* block) { for (Value* value : *block) { @@ -670,6 +687,7 @@ class ThreadJumps { if (upsilon->phi()->owner == block) m_phiValuesOnPath.append({ upsilon->phi(), upsilon->child(0) }); } + addPhiValuesFromOwnBlock(block); m_path.append(block); addUpsilonsAndSetsFrom(block); } diff --git a/Source/JavaScriptCore/b3/air/AirAllocateRegistersByGreedy.cpp b/Source/JavaScriptCore/b3/air/AirAllocateRegistersByGreedy.cpp index 165cce266817d..58caf8c73fbec 100644 --- a/Source/JavaScriptCore/b3/air/AirAllocateRegistersByGreedy.cpp +++ b/Source/JavaScriptCore/b3/air/AirAllocateRegistersByGreedy.cpp @@ -3710,11 +3710,14 @@ class GreedyAllocator { // Merges the other group into this one. void merge(GreedyAllocator& allocator, AffinityGroup& other) { - if (m_liveness || other.m_liveness) { - ensureMap(allocator); - other.ensureMap(allocator); - m_liveness->merge(WTF::move(*other.m_liveness)); - other.m_liveness = nullptr; + if (m_liveness && other.m_liveness) + m_liveness->merge(WTF::move(*std::exchange(other.m_liveness, nullptr))); + else if (m_liveness) + addToMap(allocator, other.members()); + else if (other.m_liveness) { + m_liveness = WTF::move(other.m_liveness); + addToMap(allocator, members()); + m_range = LiveRange(); } else m_range = LiveRange::unionOf(allocator.m_intervalArena, m_range, other.m_range); other.m_range = LiveRange(); @@ -3760,8 +3763,7 @@ class GreedyAllocator { if (m_liveness) return; m_liveness = makeUnique>(); - for (Tmp member : members()) - m_liveness->add(member, allocator.m_map.template get(member).liveRange); + addToMap(allocator, members()); m_range = LiveRange(); } @@ -3787,6 +3789,12 @@ class GreedyAllocator { Tmp m_representative; // Only set for non-empty groups at finalization private: + void addToMap(GreedyAllocator& allocator, std::span tmps) + { + for (Tmp tmp : tmps) + m_liveness->add(tmp, allocator.m_map.template get(tmp).liveRange); + } + void makeMapIfLarge(GreedyAllocator& allocator) { if (m_size > AirLimits::maxAffinityGroupSizeWithoutMap || m_range.intervals().size() > AirLimits::maxAffinityGroupSizeWithoutMap) [[unlikely]] @@ -3890,6 +3898,20 @@ class GreedyAllocator { dataLogLnIf(verbose(), "Created group ", newIndex, ": ", groups[newIndex]); } + template + bool interferesWithMap(Tmp tmp, const AffinityGroup& group) + { + ASSERT(group.hasMap()); + const TmpData& data = m_map.get(tmp); + bool interferes = false; + group.forEachOverlap(data.liveRange, [&](const auto& tmpList) { + // Pigeonhole principle + interferes = tmpList.size() > data.coalescables.size() || std::ranges::any_of(tmpList, [&](Tmp member) { return !data.coalescables.template contains(member); }); + return interferes ? IterationStatus::Done : IterationStatus::Continue; + }); + return interferes; + } + template bool tryCoalesceSingletonWithGroup(Tmp singleton, GroupIndex groupIndex, Vector>& groups, TmpGroupMap& tmpToGroup) { @@ -3906,19 +3928,8 @@ class GreedyAllocator { } } } - } else group.forEachOverlap(singletonData.liveRange, [&](const auto& tmpList) { - if (tmpList.size() > singletonData.coalescables.size()) { - conflict = true; // Pigeonhole principle - return IterationStatus::Done; - } - for (Tmp member : tmpList) { - if (!singletonData.coalescables.contains(member)) { - conflict = true; - return IterationStatus::Done; - } - } - return IterationStatus::Continue; - }); + } else + conflict = interferesWithMap(singleton, group); if (conflict) return false; @@ -3951,11 +3962,10 @@ class GreedyAllocator { break; } } - } else { - group0.ensureMap(*this); - group1.ensureMap(*this); - } - if (group0.hasMap()) AffinityGroup::forEachPairwiseOverlap(group0, group1, [&](const auto& tmpListA, const auto& tmpListB) { + } else if (!group0.hasMap() || !group1.hasMap()) { + const auto& groupWithMap = group0.hasMap() ? group0 : group1; + conflict = std::ranges::any_of((group0.hasMap() ? group1 : group0).members(), [&](Tmp member) { return interferesWithMap(member, groupWithMap); }); + } else AffinityGroup::forEachPairwiseOverlap(group0, group1, [&](const auto& tmpListA, const auto& tmpListB) { for (Tmp tmpA : tmpListA) { const Coalescables& coalescables = m_map.get(tmpA).coalescables; if (tmpListB.size() > coalescables.size()) { @@ -4039,6 +4049,9 @@ class GreedyAllocator { return a.tmp1.tmpIndex(bank) < b.tmp1.tmpIndex(bank); }); + // Groups only grow, so two that interfere still do at the next move between them. + UncheckedKeyHashSet> interferingGroups; + for (Move& move : moves) { dataLogLnIf(verbose(), "Processing move: ", move); auto groupIndex0 = tmpToGroup[move.tmp0]; @@ -4051,9 +4064,13 @@ class GreedyAllocator { if (groupIndex0 == noGroup && groupIndex1 == noGroup) coalesceSingletons(move.tmp0, move.tmp1, groups, tmpToGroup); - else if (groupIndex0 != noGroup && groupIndex1 != noGroup) - tryCoalesceGroups(groupIndex0, groupIndex1, groups, tmpToGroup); - else { + else if (groupIndex0 != noGroup && groupIndex1 != noGroup) { + std::pair pair = std::minmax(groupIndex0, groupIndex1); + if (interferingGroups.contains(pair)) + continue; + if (!tryCoalesceGroups(groupIndex0, groupIndex1, groups, tmpToGroup) && (groups[groupIndex0].hasMap() || groups[groupIndex1].hasMap())) + interferingGroups.add(pair); + } else { auto groupIndex = groupIndex0 != noGroup ? groupIndex0 : groupIndex1; Tmp singleton = groupIndex0 != noGroup ? move.tmp1 : move.tmp0; tryCoalesceSingletonWithGroup(singleton, groupIndex, groups, tmpToGroup); @@ -7133,7 +7150,9 @@ class GreedyAllocator { imm = Arg::bigImm(value); ASSERT(inst.isValidForm(m_code.targetFeatures())); arg = imm; - if (inst.isValidForm(m_code.targetFeatures())) { + std::span args = inst.args(); + bool isInExtendedForm = args.size() == 2 ? isExtendedForm(inst.kind.opcode, args[0].kind(), args[1].kind()) : args.size() == 3 && isExtendedForm(inst.kind.opcode, args[0].kind(), args[1].kind(), args[2].kind()); + if (inst.isValidForm(m_code.targetFeatures()) && (m_usesExtendedForms || !isInExtendedForm)) { m_stats[bank].numRematerializeConst++; dataLogLnIf(verbose(), "Rematerialized (direct imm), BB", *block, " arg=", oldArg, ", inst=", inst); return; @@ -7644,7 +7663,9 @@ class GreedyAllocator { // We can coalesce a Move32 so long as either of the following holds: // - The input is already zero-filled. // - The output only cares about the low 32 bits. - if (inst.kind.opcode == Move32 && m_tmpWidth.defWidth(inst.args()[0].tmp()) > Width32) + Tmp source = inst.args()[0].tmp(); + bool isZeroFilledRegister = source.isReg() && m_registerPolicy.zeroExtended.contains(source.reg(), IgnoreVectors); + if (inst.kind.opcode == Move32 && !isZeroFilledRegister && m_tmpWidth.defWidth(source) > Width32) return false; return true; } diff --git a/Source/JavaScriptCore/b3/air/AirCheckSpillSlots.h b/Source/JavaScriptCore/b3/air/AirCheckSpillSlots.h index 95078ee633578..bd91abde92639 100644 --- a/Source/JavaScriptCore/b3/air/AirCheckSpillSlots.h +++ b/Source/JavaScriptCore/b3/air/AirCheckSpillSlots.h @@ -52,8 +52,7 @@ class PlacesOfValues { JS_EXPORT_PRIVATE void addWrite(Reg, Tmp, unsigned instIndex); JS_EXPORT_PRIVATE void addClobber(Reg, unsigned instIndex); - // False, and the procedure has failed with FailureKind::WrongValueInSpillSlot or WrongValueInRegister, if some read does not see its value. The failure's detail names the - // place, the instruction that reads it, the instruction that wrote something else, and a path. + // Returns false, with the procedure marked as failed, if a read does not see the value of its slot. JS_EXPORT_PRIVATE bool check(ASCIILiteral where); private: diff --git a/Source/JavaScriptCore/b3/air/AirCode.cpp b/Source/JavaScriptCore/b3/air/AirCode.cpp index d42526a94f8de..da8bd743f17ba 100644 --- a/Source/JavaScriptCore/b3/air/AirCode.cpp +++ b/Source/JavaScriptCore/b3/air/AirCode.cpp @@ -90,6 +90,7 @@ Code::Code(Procedure& proc) all.exclude(RegisterSet::stackRegisters()); all.exclude(RegisterSet::reservedHardwareRegisters()); auto calleeSave = RegisterSet::calleeSaveRegisters(); + calleeSave.exclude(m_policy.registers.unsavedCalleeSaves); calleeSave.merge(m_policy.registers.savedByCallee); all.forEach( [&] (Reg reg) { diff --git a/Source/JavaScriptCore/b3/air/AirExtendedForms.h b/Source/JavaScriptCore/b3/air/AirExtendedForms.h index 2d88c8c2728c4..828fb87178a6f 100644 --- a/Source/JavaScriptCore/b3/air/AirExtendedForms.h +++ b/Source/JavaScriptCore/b3/air/AirExtendedForms.h @@ -16,15 +16,21 @@ template inline bool isExtendedForm(Opcode opcode, Arguments... arguments) { constexpr size_t size = sizeof...(Arguments); - const std::array kinds { arguments..., Arg::Invalid, Arg::Invalid }; + const std::array kinds { arguments..., Arg::Invalid, Arg::Invalid, Arg::Invalid }; switch (opcode) { case Mul64: return size == 3 && kinds[0] == Arg::Imm; case SignExtend32To64: return size == 2 && kinds[0] != Arg::Tmp; + case Move: + return size == 2 && kinds[0] == Arg::Imm && kinds[1] == Arg::Index; case Test64: case MoveConditionallyTest64: return kinds[1] == Arg::Addr; + case Branch32: + return kinds[2] == Arg::Index || (kinds[1] == Arg::Index && kinds[2] == Arg::Tmp); + case Branch64: + return kinds[2] == Arg::Index || (kinds[1] == Arg::Index && kinds[2] == Arg::Imm); default: return false; } diff --git a/Source/JavaScriptCore/b3/air/AirInst.h b/Source/JavaScriptCore/b3/air/AirInst.h index b8b87e1f21d38..d6245b6598618 100644 --- a/Source/JavaScriptCore/b3/air/AirInst.h +++ b/Source/JavaScriptCore/b3/air/AirInst.h @@ -128,6 +128,8 @@ struct Inst { ~Inst() { destroyArgs(); } + bool isMoveBetweenRegisters() const { return (kind.opcode == Move || kind.opcode == Move32) && m_size == 2 && args()[0].isReg() && args()[1].isReg(); } + std::span args() LIFETIME_BOUND { return unsafeMakeSpan(argsData(), m_size); } std::span args() const LIFETIME_BOUND { return unsafeMakeSpan(argsData(), m_size); } diff --git a/Source/JavaScriptCore/b3/air/AirOpcode.opcodes b/Source/JavaScriptCore/b3/air/AirOpcode.opcodes index c197199ed52e3..631c528ae8f17 100644 --- a/Source/JavaScriptCore/b3/air/AirOpcode.opcodes +++ b/Source/JavaScriptCore/b3/air/AirOpcode.opcodes @@ -736,6 +736,7 @@ Move U:G:Ptr, D:G:Ptr /preservesFlagsOnARM64 /accessesMemoryFirst Tmp, Addr as storePtr Tmp, Index as storePtr x86_64: Imm, Addr as storePtr + x86_64: Imm, Index as storePtr arm64: ZeroReg, Tmp arm64: ZeroReg, Addr as storePtr arm64: ZeroReg, Index as storePtr @@ -1656,6 +1657,8 @@ Branch32 U:G:32, U:G:32, U:G:32 /branch /accessesMemoryFirst x86_64: RelCond, Tmp, Addr x86_64: RelCond, Addr, Tmp x86_64: RelCond, Index, Imm + x86_64: RelCond, Tmp, Index + x86_64: RelCond, Index, Tmp 64: Branch64 U:G:32, U:G:64, U:G:64 /branch /accessesMemoryFirst RelCond, Tmp, Tmp @@ -1664,6 +1667,8 @@ Branch32 U:G:32, U:G:32, U:G:32 /branch /accessesMemoryFirst x86_64: RelCond, Addr, Tmp x86_64: RelCond, Addr, Imm x86_64: RelCond, Index, Tmp + x86_64: RelCond, Tmp, Index + x86_64: RelCond, Index, Imm BranchTest8 U:G:32, U:G:8, U:G:8 /branch /accessesMemoryFirst x86_64: ResCond, Addr, BitImm diff --git a/Source/JavaScriptCore/b3/air/AirPolicy.cpp b/Source/JavaScriptCore/b3/air/AirPolicy.cpp index 2dce01ece1cb0..8914efa3fc7a7 100644 --- a/Source/JavaScriptCore/b3/air/AirPolicy.cpp +++ b/Source/JavaScriptCore/b3/air/AirPolicy.cpp @@ -82,6 +82,9 @@ void Policy::dump(PrintStream& out) const AIR_POLICY_PRINT_IF_DIFFERENT(frame, coalescesSpillSlots); AIR_POLICY_PRINT_IF_SET(frame, emitsFrame); AIR_POLICY_PRINT_IF_SET(frame, shrinkWraps); + AIR_POLICY_PRINT_IF_SET(frame, needsLikelyFramelessPath); + AIR_POLICY_PRINT_IF_SET(frame, hoistsFrameAboveSwitches); + AIR_POLICY_PRINT_IF_SET(frame, sinksFrameBelowEntry); AIR_POLICY_PRINT_IF_SET(frame, setsUpFrameAtEntry); AIR_POLICY_PRINT_IF_DIFFERENT(frame, maxBytesAllocatedBeforeStackCheck); AIR_POLICY_PRINT_IF_DIFFERENT(frame, smallestFrameTestedAtSetup); diff --git a/Source/JavaScriptCore/b3/air/AirPolicy.h b/Source/JavaScriptCore/b3/air/AirPolicy.h index cbddf146231a7..1647e821b38b6 100644 --- a/Source/JavaScriptCore/b3/air/AirPolicy.h +++ b/Source/JavaScriptCore/b3/air/AirPolicy.h @@ -39,6 +39,8 @@ struct RegisterPolicy { RegisterSet savedByCallee; RegisterSet calleeSavesClobberedAtSuspensions; RegisterSet pinned; + // A 32-bit argument or result of a patchpoint in one of these has zeros in its upper half. + RegisterSet zeroExtended; }; struct LiveRangeSplitPolicy { @@ -66,6 +68,12 @@ struct FramePolicy { AIR_POLICY_OF_CLIENT(bool, shrinkWraps, false); // With shrinkWraps: a function of which no path needs a frame has none, and a frame that some path needs is set up at the entry. AIR_POLICY_OF_CLIENT(bool, setsUpFrameAtEntry, false); + // With shrinkWraps: if every likely path needs the frame, it is set up once, at the entry. + AIR_POLICY_OF_CLIENT(bool, needsLikelyFramelessPath, false); + // With shrinkWraps: an unlikely switch with several edges into the framed region sets up the frame once, before the switch. + AIR_POLICY_OF_CLIENT(bool, hoistsFrameAboveSwitches, false); + // With shrinkWraps: an entry block that only branches, with an unlikely successor, runs before the frame is set up. + AIR_POLICY_OF_CLIENT(bool, sinksFrameBelowEntry, false); AIR_POLICY_OF_CLIENT(unsigned, maxBytesAllocatedBeforeStackCheck, 0); // A frame of at least this many bytes is tested against the end of the stack where it is set up, and one that does not fit is an error at the call in the caller. Nothing is written ahead of such a frame. AIR_POLICY_OF_CLIENT(unsigned, smallestFrameTestedAtSetup, std::numeric_limits::max()); diff --git a/Source/JavaScriptCore/b3/air/AirReuseConditionFlags.cpp b/Source/JavaScriptCore/b3/air/AirReuseConditionFlags.cpp index 450d747a78637..d5af22a3c02bf 100644 --- a/Source/JavaScriptCore/b3/air/AirReuseConditionFlags.cpp +++ b/Source/JavaScriptCore/b3/air/AirReuseConditionFlags.cpp @@ -14,6 +14,8 @@ #include "AirInstInlines.h" #include "AirOpcodeUtils.h" #include "AirPhaseScope.h" +#include "B3PatchpointValue.h" +#include "B3ValueInlines.h" #include #include #include @@ -177,6 +179,12 @@ std::optional> resultInFlagsOf(const Inst& inst) if (!isX86()) return std::nullopt; std::span args = inst.args(); + if (inst.kind.opcode == Patch) { + auto* patchpoint = inst.origin ? inst.origin->as() : nullptr; + if (!patchpoint || !patchpoint->zeroFlagReflectsFirstResult || !args[1].isReg()) + return std::nullopt; + return std::pair { args[1].tmp(), Width32 }; + } if (args.size() < 2 || args.size() > 3 || !args.back().isTmp() || !args.back().isGP()) return std::nullopt; for (const Arg& arg : args) { @@ -271,7 +279,30 @@ class ConditionFlagsReuse { return comparison; } - // The operation right in front of the branch has made the register that the branch compares with zero. + // Skips only flag-preserving moves between the operation and the branch. + std::optional findResultConditionBefore(BasicBlock* block, unsigned index) const + { + Inst& branch = block->at(index); + while (index--) { + Inst& inst = block->at(index); + if (std::optional condition = conditionOnResultOf(inst, branch)) + return condition; + if (!isFlagPreservingMove(inst) || writesOperandOf(inst, branch)) + return std::nullopt; + } + return std::nullopt; + } + + static bool writesOperandOf(Inst& move, Inst& branch) + { + bool result = false; + branch.forEachTmpFast([&](Tmp& tmp) { + result |= move.args()[1] == Arg(tmp); + }); + return result; + } + + // The branch compares the result of the operation with zero. std::optional conditionOnResultOf(const Inst& operation, Inst& branch) const { std::optional> result = resultInFlagsOf(operation); @@ -300,6 +331,12 @@ class ConditionFlagsReuse { } } + static bool isBranchOnIntegers(const Inst& inst) + { + std::optional parts = partsOf(inst.kind.opcode); + return parts && parts->consumer == FlagsConsumer::Branch; + } + static Inst definerFor(const Inst& user, const Comparison& comparison) { return Inst(comparison.definerOpcode, user.origin, comparison.left, comparison.right); @@ -333,7 +370,7 @@ class ConditionFlagsReuse { std::optional flags = flagsAtHead(block); for (unsigned index = 0; index < block->size(); ++index) { Inst& inst = block->at(index); - if (std::optional condition = index ? conditionOnResultOf(block->at(index - 1), inst) : std::nullopt) { + if (std::optional condition = isBranchOnIntegers(inst) ? findResultConditionBefore(block, index) : std::nullopt) { inst = Inst(BranchOnFlags, inst.origin, Arg::resCond(*condition)); // They are not the flags of a comparison with zero: carry and overflow may be anything. flags = std::nullopt; @@ -399,6 +436,15 @@ bool setsFlagsToResult(const Inst& inst) return !!resultInFlagsOf(inst); } +bool isFlagPreservingMove(const Inst& inst) +{ + if (!isX86() || (inst.kind.opcode != Move && inst.kind.opcode != Move32)) + return false; + std::span args = inst.args(); + // A move of zero is emitted as an xor, which sets the flags. + return args.size() == 2 && !args[0].isSomeImm(); +} + bool reuseConditionFlags(Code& code) { PhaseScope phaseScope(code, "reuseConditionFlags"_s); diff --git a/Source/JavaScriptCore/b3/air/AirReuseConditionFlags.h b/Source/JavaScriptCore/b3/air/AirReuseConditionFlags.h index 9874653058189..18a3735e60ee1 100644 --- a/Source/JavaScriptCore/b3/air/AirReuseConditionFlags.h +++ b/Source/JavaScriptCore/b3/air/AirReuseConditionFlags.h @@ -14,9 +14,12 @@ struct Inst; JS_EXPORT_PRIVATE bool reuseConditionFlags(Code&); -// x86-64: the last machine instruction of the operation leaves the zero and the sign flag as a test of its result would. The other flags are anybody's. +// x86-64: the operation sets the zero and sign flags as a test of its result would. The other flags are undefined. JS_EXPORT_PRIVATE bool setsFlagsToResult(const Inst&); +// x86-64: a move that is not emitted as an xor. +JS_EXPORT_PRIVATE bool isFlagPreservingMove(const Inst&); + } } } // namespace JSC::B3::Air #endif // ENABLE(B3_JIT) diff --git a/Source/JavaScriptCore/b3/air/AirShrinkWrapFrame.cpp b/Source/JavaScriptCore/b3/air/AirShrinkWrapFrame.cpp index 4ecbc8219a77b..edac846b3b14f 100644 --- a/Source/JavaScriptCore/b3/air/AirShrinkWrapFrame.cpp +++ b/Source/JavaScriptCore/b3/air/AirShrinkWrapFrame.cpp @@ -105,6 +105,9 @@ class FrameShrinkWrapper { void addCrowdedBlocks(const BitVector&); Vector findCyclesWithoutRareEdge(unsigned& numberOfCycles); bool closeOverCertainPaths(); + bool addUnlikelySwitches(); + bool hasLikelyFramelessPath() const; + bool canSinkFrameBelowEntry() const; bool closeOverWritesAheadOfFrame(); bool growOverLiveScratchRegisters(const RegLiveness&); void closeRegion(const RegLiveness*); @@ -116,6 +119,7 @@ class FrameShrinkWrapper { void insertOnOutgoingEdges(BasicBlock* from); bool shouldSetUpFrameAtHead(BasicBlock*) const; void insertSetups(); + static bool leavesFunction(BasicBlock*); void insertTeardowns(); Code& m_code; @@ -129,6 +133,7 @@ class FrameShrinkWrapper { BitVector m_takesSlotAddress; BitVector m_isKnownNotToWriteMemory; BitVector m_setsUpFrameAtHead; + bool m_sinksFrameBelowEntry { false }; Vector m_latestSplitByTarget; uint64_t m_savedCalleeSaves { 0 }; uint64_t m_scratchRegisters { 0 }; @@ -595,6 +600,8 @@ bool FrameShrinkWrapper::closeOverCertainPaths() bool changed = false; for (BasicBlock* block : m_code) { + if (block == m_code[0] && m_sinksFrameBelowEntry) + continue; if (!reachesExitWithoutFrame.quickGet(block->index()) || !isReachedWithoutFrame.quickGet(block->index())) changed |= addToRegion(block); } @@ -651,13 +658,76 @@ bool FrameShrinkWrapper::growOverLiveScratchRegisters(const RegLiveness& livenes return changed; } +bool FrameShrinkWrapper::addUnlikelySwitches() +{ + bool changed = false; + for (BasicBlock* block : m_code) { + if (block->numSuccessors() <= 2 || isInRegion(block) || m_isOnLikelyPath.quickGet(block->index())) + continue; + unsigned numberOfEdgesIntoRegion = 0; + for (BasicBlock* successor : block->successorBlocks()) + numberOfEdgesIntoRegion += isInRegion(successor); + if (numberOfEdgesIntoRegion > 1) + changed |= addToRegion(block); + } + return changed; +} + +bool FrameShrinkWrapper::hasLikelyFramelessPath() const +{ + BitVector isReached(m_code.size()); + Vector worklist; + auto visit = [&](BasicBlock* block) { + if (!isInRegion(block) && !isReached.quickSet(block->index())) + worklist.append(block); + }; + visit(m_code[0]); + while (!worklist.isEmpty()) { + BasicBlock* block = worklist.takeLast(); + if (leavesFunction(block)) + return true; + for (const FrequentedBlock& edge : block->successors()) { + if (edge.isLikely()) + visit(edge.block()); + } + } + return false; +} + +bool FrameShrinkWrapper::canSinkFrameBelowEntry() const +{ + BasicBlock* entry = m_code[0]; + if (!m_code.policy().frame.sinksFrameBelowEntry || isInRegion(entry) || entry->numPredecessors() || entry->numSuccessors() != 2) + return false; + if (entry->successor(0).isLikely() && entry->successor(1).isLikely()) + return false; + return std::ranges::all_of(*entry, [&](const Inst& inst) { + return inst.kind.opcode == Patch || inst.isMoveBetweenRegisters() || &inst == &entry->last(); + }); +} + void FrameShrinkWrapper::closeRegion(const RegLiveness* liveness) { + const FramePolicy& policy = m_code.policy().frame; + m_sinksFrameBelowEntry = canSinkFrameBelowEntry(); findLikelyPaths(); + bool mayCoverEntry = policy.needsLikelyFramelessPath; bool changed; do { changed = closeOverLikelyPaths(); changed |= closeOverCertainPaths(); + if (policy.hoistsFrameAboveSwitches) + changed |= addUnlikelySwitches(); + if (!changed && mayCoverEntry && !m_isInRegion.isEmpty()) { + mayCoverEntry = false; + if (!isInRegion(m_code[0]) && !hasLikelyFramelessPath()) { + if (m_sinksFrameBelowEntry) { + for (BasicBlock* successor : m_code[0]->successorBlocks()) + changed |= addToRegion(successor); + } else + changed |= addToRegion(m_code[0]); + } + } changed |= closeOverWritesAheadOfFrame(); if (liveness && m_scratchRegisters) changed |= growOverLiveScratchRegisters(*liveness); @@ -793,13 +863,17 @@ void FrameShrinkWrapper::insertSetups() } } +// A patch that ends the function and runs without a frame: a tail call. +bool FrameShrinkWrapper::leavesFunction(BasicBlock* block) +{ + return isReturn(block->last().kind.opcode) || (block->last().kind.opcode == Patch && !block->numSuccessors() && !block->last().needsFrame()); +} + void FrameShrinkWrapper::insertTeardowns() { for (size_t blockIndex : m_isInRegion) { BasicBlock* block = m_code[blockIndex]; - // A patch that ends the function and runs without a frame: a tail call. - bool leavesFunction = isReturn(block->last().kind.opcode) || (block->last().kind.opcode == Patch && !block->numSuccessors() && !block->last().needsFrame()); - if (!leavesFunction) { + if (!leavesFunction(block)) { insertOnOutgoingEdges(block); continue; } diff --git a/Source/JavaScriptCore/b3/air/AirValidate.cpp b/Source/JavaScriptCore/b3/air/AirValidate.cpp index dbd9530c69333..1cbf4e33bd760 100644 --- a/Source/JavaScriptCore/b3/air/AirValidate.cpp +++ b/Source/JavaScriptCore/b3/air/AirValidate.cpp @@ -215,7 +215,7 @@ bool anyInstUsesFlags(Code& code) bool flagsAfter(bool areDefined, const Inst& inst) { Opcode opcode = inst.kind.opcode; - return definesFlags(opcode) || setsFlagsToResult(inst) || (areDefined && preservesFlags(opcode)); + return definesFlags(opcode) || setsFlagsToResult(inst) || (areDefined && (preservesFlags(opcode) || isFlagPreservingMove(inst))); } bool flagsAtTail(bool areDefinedAtHead, BasicBlock* block) diff --git a/Source/JavaScriptCore/b3/air/testair_bun_flags.h b/Source/JavaScriptCore/b3/air/testair_bun_flags.h index 432b37c7ebed7..681d2181cb93a 100644 --- a/Source/JavaScriptCore/b3/air/testair_bun_flags.h +++ b/Source/JavaScriptCore/b3/air/testair_bun_flags.h @@ -846,7 +846,7 @@ void testReuseFlagsIsIdempotent() } enum class ResultOperation : uint8_t { ShiftLeft, ShiftLeftByNothing, AndWithMask, AndWithNegativeMask, AndWithMaskOfByte, AndWithMaskOfHalfWord, AndWithAllBits, AndWithRegister, AndIntoOtherRegister, AndOfRegisterWithItself }; -enum class ResultTestNeighbor : uint8_t { None, MoveBetween, OtherRegisterTested, OtherWidthTested }; +enum class ResultTestNeighbor : uint8_t { None, MoveBetween, MoveOfConstantBetween, MoveIntoResultBetween, OtherRegisterTested, OtherWidthTested }; enum class ResultTestForm : uint8_t { TestAgainstItself, TestAgainstAllBits, CompareWithZero }; static int64_t maskOf(ResultOperation operation) @@ -916,6 +916,10 @@ static bool appendTestOfResult(Code& code, bool is64Bit, ResultOperation operati } if (neighbor == ResultTestNeighbor::MoveBetween) appendChecked(root, Move, a3, a4); + if (neighbor == ResultTestNeighbor::MoveOfConstantBetween) + appendChecked(root, Move, Arg::imm(0), a4); + if (neighbor == ResultTestNeighbor::MoveIntoResultBetween) + appendChecked(root, Move, a3, result); Tmp tested = neighbor == ResultTestNeighbor::OtherRegisterTested ? a3 : result; switch (form) { case ResultTestForm::TestAgainstItself: @@ -943,7 +947,7 @@ static void checkBranchOnFlagsOfResult(bool is64Bit, ResultOperation operation, B3::Procedure proc; proc.setOptLevel(2); bool leavesFlags = appendTestOfResult(proc.code(), is64Bit, operation, test, form, neighbor); - bool isExpected = leavesFlags && test.conditionAgainstZero && neighbor == ResultTestNeighbor::None; + bool isExpected = leavesFlags && test.conditionAgainstZero && (neighbor == ResultTestNeighbor::None || neighbor == ResultTestNeighbor::MoveBetween); checkFlagReuse(proc.code(), isExpected); if (isExpected) CHECK(isInst(proc.code()[0]->last(), BranchOnFlags, { Arg::resCond(test.condition) })); @@ -999,7 +1003,7 @@ void testBranchOnFlagsOfResult() checkBranchOnFlagsOfResult(is64Bit, operation, test, form); } } - for (ResultTestNeighbor neighbor : { ResultTestNeighbor::MoveBetween, ResultTestNeighbor::OtherRegisterTested, ResultTestNeighbor::OtherWidthTested }) + for (ResultTestNeighbor neighbor : { ResultTestNeighbor::MoveBetween, ResultTestNeighbor::MoveOfConstantBetween, ResultTestNeighbor::MoveIntoResultBetween, ResultTestNeighbor::OtherRegisterTested, ResultTestNeighbor::OtherWidthTested }) checkBranchOnFlagsOfResult(is64Bit, ResultOperation::ShiftLeft, testConditions[0], ResultTestForm::TestAgainstItself, neighbor); } } diff --git a/Source/JavaScriptCore/b3/testb3_bun_ranges.cpp b/Source/JavaScriptCore/b3/testb3_bun_ranges.cpp index 1f3156f6f91d2..52068b3599ca6 100644 --- a/Source/JavaScriptCore/b3/testb3_bun_ranges.cpp +++ b/Source/JavaScriptCore/b3/testb3_bun_ranges.cpp @@ -669,6 +669,51 @@ static void testComparisonOfExtensionsIsNarrow() } } +static void checkComparisonOfExtensionWithConstant(B3::Opcode opcode, ExtendedOperand left, int64_t constant, bool reducesWithRanges) +{ + constexpr int32_t maxInt32 = std::numeric_limits::max(); + constexpr int32_t minInt32 = std::numeric_limits::min(); + Procedure proc(rangeReductionPolicy(reducesWithRanges)); + Graph graph { proc }; + BasicBlock* root = proc.addBlock(); + Value* pointer = cCallArgumentValues(proc, root)[0]; + Value* word = graph.load(root, Int32, pointer, 0, firstHeap); + if (left.isKnownNonNegative) + proc.setKnownRange(word, IntRange(0, maxInt32)); + Value* comparison = graph.make(root, opcode, graph.make(root, left.extension, word), graph.int64(root, constant)); + graph.ret(root, graph.make(root, ZExt32, comparison)); + + reduceStrengthOnce(proc); + bool isExtendedWord = constant == extendWord(left, static_cast(constant)); + if (reducesWithRanges && isExtendedWord) { + CHECK_EQ(numberOfValues(proc, [](Value* value) { + return isIntegerComparison(value->opcode()) && value->child(0)->type() == Int64; + }), 0u); + } + + auto code = compileProc(proc); + for (int32_t input : { 0, 1, 7, 128, maxInt32, -1, -7, -128, minInt32 }) { + if (left.isKnownNonNegative && input < 0) + continue; + int32_t memory[1] = { input }; + CHECK_EQ(invoke(*code, memory), static_cast(holds(opcode, extendWord(left, input), constant))); + } +} + +static void testComparisonOfExtensionWithConstantIsNarrow() +{ + for (B3::Opcode opcode : { Equal, NotEqual, LessThan, GreaterThan, LessEqual, GreaterEqual, Below, BelowEqual, Above, AboveEqual }) { + for (B3::Opcode extension : { SExt32, ZExt32 }) { + for (bool isNonNegative : { false, true }) { + for (int64_t constant : { 0ll, 1ll, 7ll, 128ll, 0x7fffffffll, 0x80000000ll, 0xfffffff9ll, 0xffffffffll, 0x100000000ll, 0x100000007ll, -1ll, -7ll, -128ll, -0x80000000ll, -0x80000001ll, std::numeric_limits::max(), std::numeric_limits::min() }) { + for (bool reducesWithRanges : { true, false }) + checkComparisonOfExtensionWithConstant(opcode, { extension, isNonNegative }, constant, reducesWithRanges); + } + } + } + } +} + static void checkMask(Type type, IntRange range, int64_t mask, bool isRedundant, bool reducesWithRanges) { Procedure proc(rangeReductionPolicy(reducesWithRanges)); @@ -867,6 +912,7 @@ void addBunRangeTests(const TestConfig* config, BunTestTasks& tasks) RUN(testSumOfIntegersBehindDoublesIsAddedInIntegers()); RUN(testSignExtensionOfNonNegativeIsZeroExtension()); RUN(testComparisonOfExtensionsIsNarrow()); + RUN(testComparisonOfExtensionWithConstantIsNarrow()); RUN(testMaskFoldsByRange()); RUN(testVariableShiftRanges()); RUN(testTruncOfVariableShiftLeftAndBack()); diff --git a/Source/JavaScriptCore/b3/testb3_bun_reductions.cpp b/Source/JavaScriptCore/b3/testb3_bun_reductions.cpp index b05ba51962bb8..5f8f0a84b0d6e 100644 --- a/Source/JavaScriptCore/b3/testb3_bun_reductions.cpp +++ b/Source/JavaScriptCore/b3/testb3_bun_reductions.cpp @@ -8,6 +8,7 @@ #if ENABLE(B3_JIT) +#include "AirExtendedForms.h" #include "B3BulkMemoryValue.h" #include "B3EliminateDeadCode.h" #include "B3FoldPathConstants.h" @@ -967,11 +968,16 @@ enum class ExtendedForm : uint8_t { SignExtensionOfLoad, TestOfLoad, SelectOnTestOfLoad, + Branch32OnIndexedLoad, + Branch32BeforeIndexedLoad, + Branch64BeforeIndexedLoad, + Branch64OnIndexedLoadAndImmediate, }; } // anonymous namespace static constexpr int64_t factorOfProduct = 1000003; +static constexpr int64_t boundOfLoad = 300; static Air::Opcode airOpcodeOf(ExtendedForm form) { @@ -984,11 +990,20 @@ static Air::Opcode airOpcodeOf(ExtendedForm form) return Air::Test64; case ExtendedForm::SelectOnTestOfLoad: return Air::MoveConditionallyTest64; + case ExtendedForm::Branch32OnIndexedLoad: + case ExtendedForm::Branch32BeforeIndexedLoad: + return Air::Branch32; + case ExtendedForm::Branch64BeforeIndexedLoad: + case ExtendedForm::Branch64OnIndexedLoadAndImmediate: + return Air::Branch64; } RELEASE_ASSERT_NOT_REACHED(); return Air::Oops; } +// Eight times this is in the reach of a load on arm64. +static constexpr int32_t indexOfLoad = 31; + static void buildUserOfExtendedForm(Graph& graph, ExtendedForm form) { BasicBlock* root = graph.proc.addBlock(); @@ -1012,6 +1027,23 @@ static void buildUserOfExtendedForm(Graph& graph, ExtendedForm form) case ExtendedForm::SelectOnTestOfLoad: graph.ret(root, graph.select(root, appendMaskedLoad(), operand, other)); return; + case ExtendedForm::Branch32OnIndexedLoad: + case ExtendedForm::Branch32BeforeIndexedLoad: + case ExtendedForm::Branch64BeforeIndexedLoad: + case ExtendedForm::Branch64OnIndexedLoadAndImmediate: { + Type type = airOpcodeOf(form) == Air::Branch32 ? Int32 : Int64; + Value* address = graph.make(root, Add, pointer, graph.make(root, Shl, other, graph.int32(root, 3))); + Value* loaded = graph.load(root, type, address, -indexOfLoad * 8, firstHeap); + Value* compared = form == ExtendedForm::Branch64OnIndexedLoadAndImmediate ? graph.int64(root, boundOfLoad) : type == Int32 ? graph.make(root, Trunc, operand) : operand; + bool isBefore = form == ExtendedForm::Branch32BeforeIndexedLoad || form == ExtendedForm::Branch64BeforeIndexedLoad; + Value* comparison = isBefore ? graph.make(root, LessThan, compared, loaded) : graph.make(root, LessThan, loaded, compared); + BasicBlock* whereLess = graph.proc.addBlock(); + BasicBlock* otherwise = graph.proc.addBlock(); + graph.branch(root, comparison, whereLess, otherwise); + graph.ret(whereLess, graph.int64(whereLess, 1)); + graph.ret(otherwise, graph.int64(otherwise, 0)); + return; + } } RELEASE_ASSERT_NOT_REACHED(); } @@ -1027,6 +1059,14 @@ static int64_t resultOfExtendedForm(ExtendedForm form, int64_t loaded, int64_t o return !(loaded & operand); case ExtendedForm::SelectOnTestOfLoad: return (loaded & operand) ? operand : other; + case ExtendedForm::Branch32OnIndexedLoad: + return static_cast(loaded) < static_cast(operand); + case ExtendedForm::Branch32BeforeIndexedLoad: + return static_cast(operand) < static_cast(loaded); + case ExtendedForm::Branch64BeforeIndexedLoad: + return operand < loaded; + case ExtendedForm::Branch64OnIndexedLoadAndImmediate: + return loaded < boundOfLoad; } RELEASE_ASSERT_NOT_REACHED(); return 0; @@ -1041,7 +1081,7 @@ static void testExtendedFormsAreSelectedOnRequest() }; const Case cases[] = { { Policy::fromProcessOptions(), false }, { Policy(), false }, { selectionPolicy(), true } }; for (const Case& testCase : cases) { - for (ExtendedForm form : { ExtendedForm::ProductWithImmediate, ExtendedForm::SignExtensionOfLoad, ExtendedForm::TestOfLoad, ExtendedForm::SelectOnTestOfLoad }) { + for (ExtendedForm form : { ExtendedForm::ProductWithImmediate, ExtendedForm::SignExtensionOfLoad, ExtendedForm::TestOfLoad, ExtendedForm::SelectOnTestOfLoad, ExtendedForm::Branch32OnIndexedLoad, ExtendedForm::Branch32BeforeIndexedLoad, ExtendedForm::Branch64BeforeIndexedLoad, ExtendedForm::Branch64OnIndexedLoadAndImmediate }) { bool isSelected = testCase.extendsInstructionSelection && (isX86() || form == ExtendedForm::SignExtensionOfLoad); Procedure lowered(testCase.policy); Graph loweredGraph { lowered }; @@ -1049,9 +1089,9 @@ static void testExtendedFormsAreSelectedOnRequest() lowerToAirForTesting(lowered); CHECK_EQ(numberOfInsts(lowered, airOpcodeOf(form)), 1u); CHECK_EQ(numberOfInsts(lowered, [&](Air::Inst& inst) { - return inst.kind.opcode == airOpcodeOf(form) && std::ranges::any_of(inst.args(), [](const Air::Arg& arg) { - return arg.isImm() || arg.isMemory(); - }); + std::span args = inst.args(); + bool isExtended = args.size() == 2 ? Air::isExtendedForm(inst.kind.opcode, args[0].kind(), args[1].kind()) : args.size() >= 3 && Air::isExtendedForm(inst.kind.opcode, args[0].kind(), args[1].kind(), args[2].kind()); + return inst.kind.opcode == airOpcodeOf(form) && isExtended; }), isSelected ? 1u : 0u); Procedure proc(testCase.policy); @@ -1061,8 +1101,7 @@ static void testExtendedFormsAreSelectedOnRequest() for (int64_t loaded : { 0ll, 6ll, -1ll, 0x180000001ll }) { int64_t memory[1] = { loaded }; for (int64_t operand : { 0ll, 1ll, 2ll, -5ll }) { - constexpr int64_t other = 77; - CHECK_EQ(invoke(*code, memory, operand, other), resultOfExtendedForm(form, loaded, operand, other)); + CHECK_EQ(invoke(*code, memory, operand, indexOfLoad), resultOfExtendedForm(form, loaded, operand, indexOfLoad)); } } } diff --git a/Source/JavaScriptCore/b3/testb3_bun_thread_jumps.cpp b/Source/JavaScriptCore/b3/testb3_bun_thread_jumps.cpp index 71f9fd7e0a3b7..ccb2b8e6c184d 100644 --- a/Source/JavaScriptCore/b3/testb3_bun_thread_jumps.cpp +++ b/Source/JavaScriptCore/b3/testb3_bun_thread_jumps.cpp @@ -466,6 +466,31 @@ static void testThreadJumpsDemotesPhiWithUpsilonBeforeBranch() CHECK_EQ(invoke(*code, 0, 4, memory), 20); } +static void testThreadJumpsReadsUpsilonInBlockOfPhi() +{ + Procedure proc(policyWithMaxTailSize(1)); + Graph graph { proc }; + TailWithTwoPredecessors blocks(graph); + Value* joined = graph.phi(blocks.tail, Int64); + Value* phi = proc.add(Phi, Int64, Origin()); + graph.upsilon(blocks.tail, joined, phi); + blocks.tail->append(phi); + graph.branch(blocks.root, blocks.first, blocks.left, blocks.right); + graph.upsilon(blocks.left, graph.int64(blocks.left, 0), phi); + graph.upsilon(blocks.left, graph.int64(blocks.left, 1), joined); + graph.upsilon(blocks.right, graph.int64(blocks.right, 0), phi); + graph.upsilon(blocks.right, blocks.second, joined); + graph.branch(blocks.tail, phi, blocks.taken, blocks.notTaken); + CHECK(threadJumpsMakesVariables(graph, blocks)); + + CHECK(blocks.left->successorBlock(0) == blocks.taken); + auto code = compileProc(proc); + int64_t memory[1]; + CHECK_EQ(invoke(*code, 1, 0, memory), 10); + CHECK_EQ(invoke(*code, 0, 1, memory), 10); + CHECK_EQ(invoke(*code, 0, 0, memory), 20); +} + static void testThreadJumpsKeepsUndecidedTest() { Procedure proc(policyWithMaxTailSize(1)); @@ -1573,6 +1598,7 @@ void addBunJumpThreadingTests(const TestConfig* config, BunTestTasks& tasks) RUN(testThreadJumpsKeepsBlockLocalPhi()); RUN(testThreadJumpsDemotesPhiUsedByAnotherBlock()); RUN(testThreadJumpsDemotesPhiWithUpsilonBeforeBranch()); + RUN(testThreadJumpsReadsUpsilonInBlockOfPhi()); RUN(testThreadJumpsKeepsUndecidedTest()); RUN(testThreadJumpsSkipsTailWithUncloneablePatchpoint()); RUN(testThreadJumpsSkipsLoopHeader()); diff --git a/Source/JavaScriptCore/interpreter/Interpreter.cpp b/Source/JavaScriptCore/interpreter/Interpreter.cpp index a1f429cb160e0..c51a64133d597 100644 --- a/Source/JavaScriptCore/interpreter/Interpreter.cpp +++ b/Source/JavaScriptCore/interpreter/Interpreter.cpp @@ -653,6 +653,7 @@ void Interpreter::getAsyncStackTrace(JSCell* owner, Vector& results, return context; }; + SmallSet visitedCombinators; auto getParentGenerator = [&](JSCell* gen) -> JSCell* { JSPromise* awaitedPromise = asyncFunctionPromise(gen); JSValue promiseContext = getContextValueFromPromise(awaitedPromise); @@ -664,6 +665,9 @@ void Interpreter::getAsyncStackTrace(JSCell* owner, Vector& results, if (auto* generator = asyncFunctionIn(promiseContext)) return generator; + if (!promiseContext.isCell() || !visitedCombinators.add(promiseContext.asCell()).isNewEntry) + return nullptr; + // handle `Promise.all`, `Promise.allSettled`, and `Promise.any` if (auto* globalContext = dynamicDowncast(promiseContext)) { JSValue promiseValue = globalContext->promise(); diff --git a/Source/JavaScriptCore/llint/LLIntSlowPaths.cpp b/Source/JavaScriptCore/llint/LLIntSlowPaths.cpp index 1c503d7aba9de..3fe8edf7fc149 100644 --- a/Source/JavaScriptCore/llint/LLIntSlowPaths.cpp +++ b/Source/JavaScriptCore/llint/LLIntSlowPaths.cpp @@ -1911,6 +1911,13 @@ LLINT_SLOW_PATH_DECL(slow_path_aot_new_object) LLINT_RETURN(reinterpret_cast(cell)); } +LLINT_SLOW_PATH_DECL(slow_path_aot_new_empty_array) +{ + LLINT_BEGIN(); + auto bytecode = pc->as(); + LLINT_RETURN(AOT::Interpreter::createEmptyArray(globalObject, bytecode.metadata(codeBlock).m_arrayAllocationProfile)); +} + // llint_aot_op_get_by_id already found the accessor, so there is no second lookup. extern "C" UGPRPair SYSV_ABI llint_slow_path_aot_call_custom_getter(CallFrame* callFrame, const JSInstruction* pc, const AOT::CustomAccessorRecord* record, JSCell* base) { diff --git a/Source/JavaScriptCore/llint/LLIntSlowPaths.h b/Source/JavaScriptCore/llint/LLIntSlowPaths.h index ab0dd3945cc01..0a347582d0dc9 100644 --- a/Source/JavaScriptCore/llint/LLIntSlowPaths.h +++ b/Source/JavaScriptCore/llint/LLIntSlowPaths.h @@ -97,6 +97,7 @@ LLINT_SLOW_PATH_HIDDEN_DECL(slow_path_aot_in_by_val); LLINT_SLOW_PATH_HIDDEN_DECL(slow_path_aot_instanceof); LLINT_SLOW_PATH_HIDDEN_DECL(slow_path_aot_enumerator_get_by_val); LLINT_SLOW_PATH_HIDDEN_DECL(slow_path_aot_new_object); +LLINT_SLOW_PATH_HIDDEN_DECL(slow_path_aot_new_empty_array); extern "C" UGPRPair SYSV_ABI llint_slow_path_aot_call_custom_getter(CallFrame*, const JSInstruction*, const AOT::CustomAccessorRecord*, JSCell* base) REFERENCED_FROM_ASM WTF_INTERNAL; #endif LLINT_SLOW_PATH_HIDDEN_DECL(slow_path_has_private_name); diff --git a/Source/JavaScriptCore/llint/LowLevelInterpreterAOT.asm b/Source/JavaScriptCore/llint/LowLevelInterpreterAOT.asm index 5765916820eff..516cb215ed1e5 100644 --- a/Source/JavaScriptCore/llint/LowLevelInterpreterAOT.asm +++ b/Source/JavaScriptCore/llint/LowLevelInterpreterAOT.asm @@ -40,6 +40,8 @@ const InstancePromiseAllocator = constexpr AOT::Interpreter::offsetOfPromiseAllo const InstancePromiseHeader = constexpr AOT::Interpreter::offsetOfPromiseHeader const InstanceArrayAllocator = constexpr (AOT_INSTANCE_PREFIX_OFFSET(arrayAllocator)) const InstanceNewArrayWithContiguousHeader = constexpr (AOT_INSTANCE_PREFIX_OFFSET(newArrayWithContiguousHeader)) +const InstanceNewArrayWithInt32Header = constexpr (AOT_INSTANCE_PREFIX_OFFSET(newArrayWithInt32Header)) +const InstanceEmptyArrayAllocator = constexpr (AOT_INSTANCE_OFFSET(shortArrays.allocators) + AOT::ShortArrayWords::indexOfAllocator(BASE_CONTIGUOUS_VECTOR_LEN_EMPTY) * sizeof(void*)) const InstanceRopeStringAllocator = constexpr (AOT_INSTANCE_PREFIX_OFFSET(ropeStringAllocator)) const InstanceStringHeader = constexpr (AOT_INSTANCE_PREFIX_OFFSET(stringHeader)) const InstanceClosureEntrypoint = constexpr (AOT_INSTANCE_OFFSET(stubs.closureEntrypoint)) @@ -1588,6 +1590,41 @@ aotOp(op_new_array, OpNewArray, macro (size, get, dispatch, metadata, return) getu(size, OpNewArray, m_argc, t1) btinz t1, .hasElements bia t2, constexpr ArrayWithContiguous, .slow + bieq t2, constexpr ArrayWithContiguous, .emptyInlineButterfly + bieq t2, constexpr ArrayWithInt32, .emptyInlineButterfly + bineq t2, constexpr ArrayWithUndecided, .emptySeparateButterfly + + # Like the NewEmptyArray stub: the butterfly is in the cell of the array, and an undecided array gets Int32 storage. +.emptyInlineButterfly: + loadp InstanceEmptyArrayAllocator[t5], t0 + allocateCell(t0, t1, t3, .hasNoCell) + bineq t2, constexpr ArrayWithContiguous, .holdsInt32 + loadq InstanceNewArrayWithContiguousHeader[t5], t2 + jmp .hasHeader +.holdsInt32: + loadq InstanceNewArrayWithInt32Header[t5], t2 +.hasHeader: + storeq t2, [t1] + move constexpr BASE_CONTIGUOUS_VECTOR_LEN_EMPTY, t0 + lshiftq 32, t0 + storeq t0, (constexpr (sizeof(JSArray)))[t1] + move ValueEmpty, t0 + # BASE_CONTIGUOUS_VECTOR_LEN_EMPTY words. + storeq t0, (constexpr (sizeof(JSArray) + sizeof(IndexingHeader)))[t1] + storeq t0, (constexpr (sizeof(JSArray) + sizeof(IndexingHeader) + 8))[t1] + storeq t0, (constexpr (sizeof(JSArray) + sizeof(IndexingHeader) + 16))[t1] + storeq t0, (constexpr (sizeof(JSArray) + sizeof(IndexingHeader) + 24))[t1] + storeq t0, (constexpr (sizeof(JSArray) + sizeof(IndexingHeader) + 32))[t1] + leap (constexpr (sizeof(JSArray) + sizeof(IndexingHeader)))[t1], t0 + storep t0, JSObjectWithButterfly::m_butterfly[t1] + fenceAfterAllocation(t0) + jmp .updateProfile + +.hasNoCell: + callSlowPath(_llint_slow_path_aot_new_empty_array) + dispatch() + +.emptySeparateButterfly: move constexpr BASE_CONTIGUOUS_VECTOR_LEN_EMPTY, t3 jmp .elementsFit .hasElements: @@ -1668,6 +1705,7 @@ aotOp(op_new_array, OpNewArray, macro (size, get, dispatch, metadata, return) storep t7, JSObjectWithButterfly::m_butterfly[t1] fenceAfterAllocation(t0) +.updateProfile: # ArrayAllocationProfile::updateLastAllocation(). loadq OpNewArray::Metadata::m_arrayAllocationProfile[t6], t0 urshiftq constexpr AOT::arrayAllocationProfilePointerBits, t0 diff --git a/Source/JavaScriptCore/runtime/Error.cpp b/Source/JavaScriptCore/runtime/Error.cpp index bbc62cb7e8bbb..1c681d7b41a10 100644 --- a/Source/JavaScriptCore/runtime/Error.cpp +++ b/Source/JavaScriptCore/runtime/Error.cpp @@ -223,6 +223,8 @@ bool getLineColumnAndSource(VM& vm, Vector* stackTrace, LineColumn& continue; lineColumn = position->lineColumn; sourceURL = AOT::sourceProviderURLOf(position->source); + if (!sourceURL.isEmpty()) + sourceURL = AOT::displayPathOf(position->source); return true; } #endif diff --git a/Source/JavaScriptCore/runtime/InternalFunction.cpp b/Source/JavaScriptCore/runtime/InternalFunction.cpp index 3a989a55ff2e5..3ca387882adcf 100644 --- a/Source/JavaScriptCore/runtime/InternalFunction.cpp +++ b/Source/JavaScriptCore/runtime/InternalFunction.cpp @@ -62,6 +62,11 @@ bool InternalFunction::defineOwnProperty(JSObject* object, JSGlobalObject* globa { VM& vm = globalObject->vm(); if (rejectsNewProperty(vm, uncheckedDowncast(object), propertyName)) [[unlikely]] { + if (AOT::RuntimeHooks::acceptsHasInstanceOfHost(uncheckedDowncast(object), propertyName, descriptor.value())) { + AllowLazyMaterializationOfImmutableProperties allowDefinition(vm); + object->putDirect(vm, propertyName, descriptor.value(), PropertyAttribute::ReadOnly | PropertyAttribute::DontEnum | PropertyAttribute::DontDelete); + return true; + } auto scope = DECLARE_THROW_SCOPE(vm); return typeError(globalObject, scope, shouldThrow, AOT::RuntimeHooks::lockedBuiltinPropertyErrorMessage(propertyName)); } @@ -200,6 +205,8 @@ static ALWAYS_INLINE bool retainsInternalFunctionAllocationStructure(JSFunction* if (!function->isCompiledClosure()) return false; #if ENABLE(AOT) + if (FunctionRareData* rareData = function->rareData(); rareData && rareData->internalFunctionAllocationStructure()) + return true; switch (AOT::CompiledCallee::kind(function)) { case AOT::CompiledCallee::FunctionKind::Class: case AOT::CompiledCallee::FunctionKind::Generator: diff --git a/Source/JavaScriptCore/runtime/JSCustomGetterFunction.h b/Source/JavaScriptCore/runtime/JSCustomGetterFunction.h index bd2ab18b79bd4..031dabc3a37c7 100644 --- a/Source/JavaScriptCore/runtime/JSCustomGetterFunction.h +++ b/Source/JavaScriptCore/runtime/JSCustomGetterFunction.h @@ -51,6 +51,9 @@ class JSCustomGetterFunction final : public JSFunction { DECLARE_EXPORT_INFO; + static constexpr ptrdiff_t offsetOfPropertyName() { return OBJECT_OFFSETOF(JSCustomGetterFunction, m_propertyName); } + static constexpr ptrdiff_t offsetOfGetter() { return OBJECT_OFFSETOF(JSCustomGetterFunction, m_getter); } + const Identifier& propertyName() const { return m_propertyName; } CustomFunctionPointer getter() const { return m_getter; }; CustomFunctionPointer customFunctionPointer() const { return m_getter; }; diff --git a/Source/JavaScriptCore/runtime/JSFunction.cpp b/Source/JavaScriptCore/runtime/JSFunction.cpp index 06e751692e779..57f8aee6a6901 100644 --- a/Source/JavaScriptCore/runtime/JSFunction.cpp +++ b/Source/JavaScriptCore/runtime/JSFunction.cpp @@ -283,8 +283,9 @@ const String JSFunction::calculatedDisplayName(VM& vm) #if ENABLE(AOT) if (isCompiledClosure()) { + // An anonymous function has no name, as below: the caller then names an instance after its built-in class. if (String displayName = AOT::CompiledCallee::displayName(this); !displayName.isNull()) - return displayName; + return displayName.isEmpty() ? String() : displayName; } #endif diff --git a/Source/JavaScriptCore/runtime/JSMicrotask.cpp b/Source/JavaScriptCore/runtime/JSMicrotask.cpp index 55427a3fea72d..5aef53d6806ca 100644 --- a/Source/JavaScriptCore/runtime/JSMicrotask.cpp +++ b/Source/JavaScriptCore/runtime/JSMicrotask.cpp @@ -1138,7 +1138,7 @@ static void moduleRegistryModuleSettled(JSGlobalObject* globalObject, VM& vm, st // pending-modulePromise guard used in moduleRegistryFetchSettled. if (modulePromise->status() != JSPromise::Status::Pending) { #if ENABLE(AOT) - if (status == JSPromise::Status::Rejected && vm.hasCompiledCode()) + if (status == JSPromise::Status::Rejected && modulePromise->status() == JSPromise::Status::Fulfilled && vm.hasCompiledCode()) AOT::ModuleRecord::didDiscardModuleError(globalObject, entry->key()); #endif return; diff --git a/Source/JavaScriptCore/runtime/JSModuleLoader.cpp b/Source/JavaScriptCore/runtime/JSModuleLoader.cpp index 355e18117a10b..a0afc0347be63 100644 --- a/Source/JavaScriptCore/runtime/JSModuleLoader.cpp +++ b/Source/JavaScriptCore/runtime/JSModuleLoader.cpp @@ -1216,7 +1216,12 @@ void JSModuleLoader::innerModuleLoading(JSGlobalObject* globalObject, ModuleGrap // module was loaded synchronously, which means it was already loaded before. In that case // there is no need to attach a ModuleGraphLoadingError reaction, so we skip it. bool needsErrorReaction = module->loadedModules().size() == loadedModulesCountBefore; - ASSERT(module->loadedModules().size() <= loadedModulesCountBefore + 1); +#if ENABLE(AOT) + // Making the record of a CommonJS module runs the module. Its require() may load other requests of this module inside the call. + if (vm.hasCompiledCode()) + needsErrorReaction = !module->loadedModules().contains(ModuleMapKey { request.m_specifier.impl(), request.type() }); +#endif + ASSERT(vm.hasCompiledCode() || module->loadedModules().size() <= loadedModulesCountBefore + 1); ASSERT(needsErrorReaction != module->loadedModules().contains(ModuleMapKey { request.m_specifier.impl(), request.type() })); if (needsErrorReaction) promise->performPromiseThenWithInternalMicrotask(vm, InternalMicrotask::ModuleGraphLoadingError, nullptr, state); diff --git a/Source/JavaScriptCore/runtime/JSONObject.cpp b/Source/JavaScriptCore/runtime/JSONObject.cpp index 9e3e977865141..ee7bcc99268b1 100644 --- a/Source/JavaScriptCore/runtime/JSONObject.cpp +++ b/Source/JavaScriptCore/runtime/JSONObject.cpp @@ -823,6 +823,8 @@ class FastStringifier { bool isMemberWrittenByItsFields(JSObject&); bool writeListedFieldsOfMember(Cursor&, JSObject&, std::span, unsigned reservation); void appendElementsInCompiledProgram(JSArray&); + enum class ArrayWritten : uint8_t { No, Yes, Failed }; + ArrayWritten writeArrayOfPrimitives(Cursor&, JSArray&); bool isCompiledStructureKnown(const JSObject& object) const { return (m_compiledStructuresKnown & knowsCompiledStructure) && object.structureID().bits() == m_compiledStructureWithoutToJSON; } #endif String result(); @@ -2852,6 +2854,20 @@ ALWAYS_INLINE bool FastStringifier::writeListedFields(Curs } continue; } else if constexpr (writesLatin1 && hasGap == HasGap::No) { + if (type == ArrayType) { + appendKey(); + switch (writeArrayOfPrimitives(cursor, *asArray(&cell))) { + case ArrayWritten::Yes: + if (!reserve(cursor, reservation)) [[unlikely]] + return false; + continue; + case ArrayWritten::Failed: + return false; + case ArrayWritten::No: + cursor.next = buffer() + positionOfMember; + break; + } + } // A compiled object that is fully known continues with this cursor. if (type == CompiledObjectType && !AOT::PropertyWalk::hasExtraProperties(asObject(&cell))) { if (!isMemberWrittenByItsFields(*asObject(&cell))) [[unlikely]] { @@ -2907,6 +2923,81 @@ ALWAYS_INLINE bool FastStringifier::writeListedFields(Curs return true; } +// Writes an array of strings, numbers, booleans, null and undefined. Nothing is written for an array that holds anything else, or a string that takes more than a copy. +template +NEVER_INLINE auto FastStringifier::writeArrayOfPrimitives(Cursor& cursor, JSArray& array) -> ArrayWritten +{ + if constexpr (std::same_as) { + // append() checks the prototype of arrays at the first array. + if (!m_checkedArrayPrototype || !m_globalObject.isOriginalArrayStructure(array.structure())) + return ArrayWritten::No; + IndexingType indexingType = array.indexingType(); + if (!hasInt32(indexingType) && !hasContiguous(indexingType)) + return ArrayWritten::No; + auto* butterfly = array.butterfly(); + const unsigned length = butterfly->publicLength(); + if (length > butterfly->vectorLength()) [[unlikely]] + return ArrayWritten::No; + + // A comma, a value that is no string and the closing bracket. + constexpr size_t maxShortElementLength = 1 + std::max(AOT::maxJSONNumberLength, maxInt32StringLength) + 1; + auto elements = butterfly->contiguous(); + const size_t positionOfArray = cursor.next - buffer(); + if (!enterHolder() || !reserve(cursor, 2)) [[unlikely]] + return ArrayWritten::Failed; + auto leaveUnwritten = [&] { + leaveHolder(); + cursor.next = buffer() + positionOfArray; + return ArrayWritten::No; + }; + *cursor.next++ = '['; + for (unsigned i = 0; i < length; ++i) { + if (!reserve(cursor, maxShortElementLength)) [[unlikely]] + return ArrayWritten::Failed; + if (i) + *cursor.next++ = ','; + JSValue element = elements.at(&array, i).get(); + if (element.isInt32()) { + cursor.next = AOT::appendJSONInteger(cursor.next, element.asInt32()); + continue; + } + if (!element) [[unlikely]] + return leaveUnwritten(); + if (element.isCell()) { + if (element.asCell()->type() != StringType) + return leaveUnwritten(); + const StringImpl* string = asString(element.asCell())->tryGetValueImpl(); + if (!string || !string->is8Bit()) [[unlikely]] + return leaveUnwritten(); + const size_t stringLength = string->length(); + if (!reserve(cursor, 1 + stringLength + 1 + 1)) [[unlikely]] + return ArrayWritten::Failed; + *cursor.next = '"'; + if (stringCopySameType(string->span8(), cursor.next + 1)) [[unlikely]] + return leaveUnwritten(); + cursor.next[1 + stringLength] = '"'; + cursor.next += 1 + stringLength + 1; + continue; + } + if (element.isDouble()) + cursor.next = AOT::appendJSONNumber(cursor.next, element.asDouble()); + else if (element.isBoolean()) + cursor.next = AOT::appendJSONBoolean(cursor.next, element.isTrue()); + else if (element.isUndefinedOrNull()) + cursor.next = AOT::appendJSONLiteral(cursor.next, "null"_s); + else + return leaveUnwritten(); + } + *cursor.next++ = ']'; + leaveHolder(); + return ArrayWritten::Yes; + } else { + UNUSED_PARAM(cursor); + UNUSED_PARAM(array); + return ArrayWritten::No; + } +} + // The checks that appendCompiledObject() makes on an object without extra properties before it writes the fields. template ALWAYS_INLINE bool FastStringifier::isMemberWrittenByItsFields(JSObject& object) diff --git a/Source/JavaScriptCore/runtime/LiteralParser.cpp b/Source/JavaScriptCore/runtime/LiteralParser.cpp index 1cf5359c1853e..6e7f917f08c6b 100644 --- a/Source/JavaScriptCore/runtime/LiteralParser.cpp +++ b/Source/JavaScriptCore/runtime/LiteralParser.cpp @@ -49,6 +49,10 @@ #include "KeywordLookup.h" +#if ENABLE(AOT) +#include "AOTJSONHooks.h" +#endif + WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN namespace JSC { @@ -2797,6 +2801,12 @@ JSValue JSONParse(JSGlobalObject* globalObject, std::span utf8) RELEASE_ASSERT(utf8.size() <= String::MaxLength); if (!charactersAreAllASCII(utf8)) return { }; +#if ENABLE(AOT) + if (globalObject->aotInstance()) [[unlikely]] { + if (JSValue parsed = AOT::tryParseJSON(globalObject, StringView { byteCast(utf8) })) + return parsed; + } +#endif LiteralParser parser(globalObject, byteCast(utf8), StrictJSON); return parser.tryLiteralParse(); } diff --git a/Source/JavaScriptCore/runtime/ObjectConstructor.cpp b/Source/JavaScriptCore/runtime/ObjectConstructor.cpp index e451fedf54728..42edd379ddaa5 100644 --- a/Source/JavaScriptCore/runtime/ObjectConstructor.cpp +++ b/Source/JavaScriptCore/runtime/ObjectConstructor.cpp @@ -190,7 +190,7 @@ JSC_DEFINE_HOST_FUNCTION(objectConstructorSetPrototypeOf, (JSGlobalObject* globa return JSValue::encode(objectValue); } -// Inlined into the host function, as the compiler did by itself before the test for locked properties was added. +// Inlined into the host function, as before the test for locked properties. static ALWAYS_INLINE JSValue describeOwnProperty(JSGlobalObject* globalObject, JSObject* object, const Identifier& propertyName) { VM& vm = globalObject->vm(); diff --git a/Source/JavaScriptCore/runtime/StackFrame.cpp b/Source/JavaScriptCore/runtime/StackFrame.cpp index 0d9661d59c7f9..a0a6c09cb8716 100644 --- a/Source/JavaScriptCore/runtime/StackFrame.cpp +++ b/Source/JavaScriptCore/runtime/StackFrame.cpp @@ -405,6 +405,8 @@ ALWAYS_INLINE auto StackFrame::withPartsOfString(VM& vm, const Functor& functor) return functor(functionName, "@[native code]"_s); auto position = AOT::positionOf(compiledFrame->frame); fullSourceURL = AOT::sourceProviderURLOf(compiledFrame->frame, position); + if (!fullSourceURL.isEmpty() && position && position->source) + fullSourceURL = AOT::displayPathOf(position->source); hasLineAndColumnInfo = true; if (position) lineColumn = position->lineColumn; diff --git a/Source/JavaScriptCore/wasm/WasmCallee.cpp b/Source/JavaScriptCore/wasm/WasmCallee.cpp index a4b793c28c3e3..453eae77011f7 100644 --- a/Source/JavaScriptCore/wasm/WasmCallee.cpp +++ b/Source/JavaScriptCore/wasm/WasmCallee.cpp @@ -587,6 +587,18 @@ const RegisterAtOffsetList* JSToWasmCallee::calleeSaveRegistersImpl() #if ENABLE(WEBASSEMBLY_BBQJIT) || ENABLE(WEBASSEMBLY_OMGJIT) +#if ENABLE(WEBASSEMBLY_OMGJIT) +Ref OMGCallee::createWithoutSavedRegisters(const OMGCallee& callee) +{ + Ref result = adoptRef(*new OMGCallee(callee.index(), { }, Ref { callee.profiledCallee() })); + result->setIndexOrName(callee.indexOrName()); + auto [start, end] = callee.range(); + result->setCodeInImage(std::span { static_cast(start), static_cast(end) }, { }); + NativeCalleeRegistry::singleton().registerCallee(result.ptr()); + return result; +} +#endif + void OptimizingJITCallee::linkExceptionHandlers(Vector unlinkedExceptionHandlers, Vector> exceptionHandlerLocations) { size_t count = unlinkedExceptionHandlers.size(); diff --git a/Source/JavaScriptCore/wasm/WasmCallee.h b/Source/JavaScriptCore/wasm/WasmCallee.h index 2deae3b73d27b..6087b7dbfe7ba 100644 --- a/Source/JavaScriptCore/wasm/WasmCallee.h +++ b/Source/JavaScriptCore/wasm/WasmCallee.h @@ -326,6 +326,8 @@ class OptimizingJITCallee : public JITCallee { JITCallee::setEntrypoint(WTF::move(entrypoint)); } + IPIntCallee& profiledCallee() const { return m_profiledCallee.get(); } + private: void linkExceptionHandlers(Vector, Vector>); @@ -394,6 +396,7 @@ class OMGCallee final : public OptimizingJITCallee { { return adoptRef(*new OMGCallee(index, WTF::move(name), WTF::move(profiledCallee))); } + static Ref createWithoutSavedRegisters(const OMGCallee&); using OptimizingJITCallee::setEntrypoint; diff --git a/Source/JavaScriptCore/wasm/WasmCalleeGroup.cpp b/Source/JavaScriptCore/wasm/WasmCalleeGroup.cpp index ccb33a3545bb6..de6296b328154 100644 --- a/Source/JavaScriptCore/wasm/WasmCalleeGroup.cpp +++ b/Source/JavaScriptCore/wasm/WasmCalleeGroup.cpp @@ -164,6 +164,14 @@ CodePtr CalleeGroup::javaScriptImportStubInImage(FunctionSpaceI return nullptr; } +void CalleeGroup::didFault([[maybe_unused]] CallFrame* callFrame, [[maybe_unused]] const void* pc) +{ +#if ENABLE(WEBASSEMBLY_OMGJIT) + if (m_codeInImage && CodeInImage::framelessFaultStub(pc)) + callFrame->setCallee(m_codeInImage->calleeWithoutSavedRegisters(pc).ptr()); +#endif +} + void* const* CalleeGroup::imageCodePointers() const { #if ENABLE(WEBASSEMBLY_OMGJIT) diff --git a/Source/JavaScriptCore/wasm/WasmCalleeGroup.h b/Source/JavaScriptCore/wasm/WasmCalleeGroup.h index c5e3c419273e4..3fcd994b517b3 100644 --- a/Source/JavaScriptCore/wasm/WasmCalleeGroup.h +++ b/Source/JavaScriptCore/wasm/WasmCalleeGroup.h @@ -263,6 +263,7 @@ class CalleeGroup final : public ThreadSafeRefCounted { MemoryMode mode() const { return m_mode; } void* const* imageCodePointers() const; + void didFault(CallFrame*, const void* pc); CodePtr jsEntrypointInImage(const RTT&) const; CodePtr javaScriptImportStubInImage(FunctionSpaceIndex importIndex) const; diff --git a/Source/JavaScriptCore/wasm/WasmCodeInImage.cpp b/Source/JavaScriptCore/wasm/WasmCodeInImage.cpp index c7002533341fc..ab48e203d4163 100644 --- a/Source/JavaScriptCore/wasm/WasmCodeInImage.cpp +++ b/Source/JavaScriptCore/wasm/WasmCodeInImage.cpp @@ -93,10 +93,8 @@ std::array CodeInImage::digestOf(std::span source) RegisterAtOffsetList CodeInImage::savedRegistersFrom(const SavedRegistersInImage& saved) { RegisterSet registers; - for (unsigned reg = 0; reg < 64; ++reg) { - if (saved.registers >> reg & 1) - registers.add(Reg::fromIndex(reg), IgnoreVectors); - } + for (uint64_t left = saved.registers; left; left &= left - 1) + registers.add(Reg::fromIndex(std::countr_zero(left)), IgnoreVectors); RegisterAtOffsetList list(registers); if (list.registerCount()) list.adjustOffsets(saved.offsetOfFirst - list.at(0).offset()); @@ -165,6 +163,41 @@ static std::span declinedModulesIn(const uint8_t* d return { reinterpret_cast(std::to_address(modulesIn(directory).end())), header.numberOfDeclinedModules }; } +const void* CodeInImage::framelessFaultStub(const void* pc) +{ + for (auto* image = s_lastPublished.load(std::memory_order_acquire); image; image = image->publishedBefore) { + uintptr_t place = std::bit_cast(pc) - std::bit_cast(image->code.data()); + if (place >= image->code.size()) + continue; + const uint8_t* directory = image->directory.data(); + for (auto& module : modulesIn(directory)) { + std::span ranges { reinterpret_cast(directory + module.framelessRanges), module.numberOfFramelessRanges }; + auto next = std::ranges::upper_bound(ranges, place, { }, &FramelessRangeInImage::begin); + if (next != ranges.begin() && place < std::prev(next)->end) + return image->code.data() + reinterpret_cast(directory)->faultWithoutFrame; + } + } + return nullptr; +} + +bool CodeInImage::isInFunction(const void* pc) +{ + for (auto* image = s_lastPublished.load(std::memory_order_acquire); image; image = image->publishedBefore) { + uintptr_t place = std::bit_cast(pc) - std::bit_cast(image->code.data()); + if (place >= image->code.size()) + continue; + const uint8_t* directory = image->directory.data(); + for (auto& module : modulesIn(directory)) { + for (uint32_t offset : module.functions) { + std::span functions { reinterpret_cast(directory + offset), module.numberOfFunctions }; + if (std::ranges::any_of(functions, [&](const FunctionInImage& function) { return place - function.entry < function.size; })) + return true; + } + } + } + return false; +} + const ModuleInImage* CodeInImage::findRecord(const ModuleInformation& info) { if (!info.hasSourceDigest) @@ -279,6 +312,7 @@ CodeInImage::CodeInImage(const ModuleInformation& info, const IPIntCallees& ipin m_pointers[ImagePointerTable::restoreFrameCallee] = CalleeBits::boxNativeCallee(&RestoreFrameCallee::singleton()); m_pointers[ImagePointerTable::restoreFrameReturnAddress] = reinterpret_cast(untagCodePtr(wasm_restore_frame_return)); m_pointers[ImagePointerTable::structureIDBase] = std::bit_cast(structureIDBase()); + m_pointers[ImagePointerTable::throwFromFaultHandler] = LLInt::getCodePtr(wasm_throw_from_fault_handler_trampoline_reg_instance).taggedPtr(); #if GIGACAGE_ENABLED m_pointers[ImagePointerTable::primitiveGigacageDisableRequest] = &Gigacage::disablePrimitiveGigacageRequested; m_pointers[ImagePointerTable::primitiveGigacageBase] = Gigacage::addressOfBasePtr(Gigacage::Primitive); @@ -324,6 +358,28 @@ CodeInImage::CodeInImage(const ModuleInformation& info, const IPIntCallees& ipin CodeInImage::~CodeInImage() = default; +bool CodeInImage::holds(const Callee* callee, size_t functionCodeIndex) const +{ + if (m_callees[functionCodeIndex].ptr() == callee) + return true; + Locker locker { m_faultCalleesLock }; + auto iterator = m_faultCallees.find(functionCodeIndex); + return iterator != m_faultCallees.end() && iterator->value.ptr() == callee; +} + +Ref CodeInImage::calleeWithoutSavedRegisters(const void* pc) +{ + size_t index = m_callees.findIf([&](auto& callee) { + auto [start, end] = callee->range(); + return start <= pc && pc < end; + }); + RELEASE_ASSERT(index != notFound); + Locker locker { m_faultCalleesLock }; + return m_faultCallees.ensure(index, [&] { + return OMGCallee::createWithoutSavedRegisters(m_callees[index].get()); + }).iterator->value; +} + CodePtr CodeInImage::jsEntrypoint(const RTT& rtt) const { for (auto& [type, callee] : m_jsEntrypoints) { diff --git a/Source/JavaScriptCore/wasm/WasmCodeInImage.h b/Source/JavaScriptCore/wasm/WasmCodeInImage.h index 66f07428a947f..f3ec0dfe47be1 100644 --- a/Source/JavaScriptCore/wasm/WasmCodeInImage.h +++ b/Source/JavaScriptCore/wasm/WasmCodeInImage.h @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include #include #include @@ -29,8 +31,11 @@ class CodeInImage final : public ThreadSafeRefCounted { JS_EXPORT_PRIVATE static void addImage(std::span directory, std::span code); JS_EXPORT_PRIVATE static bool isInProcess(); static bool needsSourceDigests(); - // Safe to call from a signal handler. + // These are safe to call from a signal handler. static bool contains(const void* pc); + static bool isInFunction(const void* pc); + // The stub that sets up a frame for a fault at pc. Null if the frame of the function is already set up. + static const void* framelessFaultStub(const void* pc); // Where the unwinder sends a frame whose handler lies at this place in an image. Nothing for a place outside every image. static CodePtr catchRunwayFor(const void* pc); @@ -48,7 +53,9 @@ class CodeInImage final : public ThreadSafeRefCounted { void* const* pointers() const { return m_pointers.span().data(); } CodePtr entrypoint(size_t functionCodeIndex) const { return m_callees[functionCodeIndex]->entrypoint(); } - bool holds(const Callee* callee, size_t functionCodeIndex) const { return m_callees[functionCodeIndex].ptr() == callee; } + bool holds(const Callee*, size_t functionCodeIndex) const; + // A callee for frames set up by framelessFaultStub(), which hold no saved registers. + Ref calleeWithoutSavedRegisters(const void* pc); CodePtr importStub(size_t index) const { return m_importStubs[index]; } CodePtr javaScriptImportStub(size_t index) const { return m_javaScriptImportStubs[index]; } CodePtr jsEntrypoint(const RTT&) const; @@ -62,6 +69,8 @@ class CodeInImage final : public ThreadSafeRefCounted { FixedVector> m_importStubs; FixedVector> m_javaScriptImportStubs; Vector>> m_jsEntrypoints; + mutable Lock m_faultCalleesLock; + UncheckedKeyHashMap, DefaultHash, WTF::UnsignedWithZeroKeyHashTraits> m_faultCallees WTF_GUARDED_BY_LOCK(m_faultCalleesLock); ImageCodeFlavor m_flavor; bool m_servesEveryFlavor; diff --git a/Source/JavaScriptCore/wasm/WasmCompilationContext.h b/Source/JavaScriptCore/wasm/WasmCompilationContext.h index b655e945d9c9e..c6b6258692ace 100644 --- a/Source/JavaScriptCore/wasm/WasmCompilationContext.h +++ b/Source/JavaScriptCore/wasm/WasmCompilationContext.h @@ -74,6 +74,9 @@ struct CompilationContext { Box pcToCodeOriginMap; Box pcToCodeOriginMapBuilder; Vector catchEntrypoints; + CCallHelpers::Label directEntry; + // The direct entry for callers that omit a zero argument. + Box directEntryWithZero; SegmentedVector origins; CodeForImage* codeForImage { nullptr }; }; diff --git a/Source/JavaScriptCore/wasm/WasmFaultSignalHandler.cpp b/Source/JavaScriptCore/wasm/WasmFaultSignalHandler.cpp index 386c4437ee817..662a2f6e276f3 100644 --- a/Source/JavaScriptCore/wasm/WasmFaultSignalHandler.cpp +++ b/Source/JavaScriptCore/wasm/WasmFaultSignalHandler.cpp @@ -98,9 +98,14 @@ static SignalAction trapHandler(Signal signal, SigInfo& sigInfo, PlatformRegiste if (exception) { dataLogLnIf(WasmFaultSignalHandlerInternal::verbose, "found active fast memory for faulting address"); - auto didFaultInWasm = [](void* faultingInstruction) -> std::tuple { + auto didFaultInWasm = [&](void* faultingInstruction) -> std::tuple { if (LLInt::isWasmIPIntPC(faultingInstruction)) return { true, nullptr }; +#if ENABLE(WEBASSEMBLY_OMGJIT) + // The sampling profiler holds the registry's lock until it has suspended this thread, which it cannot do while this handler is on an alternate signal stack. + if (isInCodeInImage) + return { CodeInImage::isInFunction(faultingInstruction), nullptr }; +#endif auto& calleeRegistry = NativeCalleeRegistry::singleton(); Locker locker { calleeRegistry.getLock() }; for (auto* callee : calleeRegistry.allCallees()) { @@ -126,6 +131,12 @@ static SignalAction trapHandler(Signal signal, SigInfo& sigInfo, PlatformRegiste MachineContext::setInstructionPointer(context, presignedTrampoline); return SignalAction::Handled; } +#endif +#if ENABLE(WEBASSEMBLY_OMGJIT) + if (const void* stub = isInCodeInImage ? CodeInImage::framelessFaultStub(faultingInstruction) : nullptr) { + MachineContext::setInstructionPointer(context, CodePtr::fromUntaggedPtr(const_cast(stub))); + return SignalAction::Handled; + } #endif MachineContext::setInstructionPointer(context, LLInt::getCodePtr(wasm_throw_from_fault_handler_trampoline_reg_instance)); return SignalAction::Handled; diff --git a/Source/JavaScriptCore/wasm/WasmFormat.cpp b/Source/JavaScriptCore/wasm/WasmFormat.cpp index ddcebd06845ca..3f6a3a26e9462 100644 --- a/Source/JavaScriptCore/wasm/WasmFormat.cpp +++ b/Source/JavaScriptCore/wasm/WasmFormat.cpp @@ -46,15 +46,51 @@ bool WasmCallableFunction::isJS() const return boxedCallee == CalleeBits { &WasmToJSCallee::singleton() }; } -std::unique_ptr Segment::tryCreate(std::optional offset, uint32_t sizeInBytes, Kind kind, uint32_t memoryIndex) +bool Segments::tryReserve(size_t numberOfSegments, size_t numberOfBytes, size_t numberOfIndices) { - auto result = tryFastZeroedMalloc(allocationSize(sizeInBytes)); - void* memory; - if (!result.getValue(memory)) - return nullptr; + m_hasIndices = numberOfIndices; + return m_buffer.tryReserveInitialCapacity(numberOfSegments * sizeof(Header) + numberOfBytes) && m_placeOfIndex.tryReserveInitialCapacity(numberOfIndices); +} - ASSERT(kind == Kind::Passive || !!offset); - return std::unique_ptr(new (memory) Segment(sizeInBytes, kind, WTF::move(offset), memoryIndex)); +void Segments::append(const Segment& segment) +{ + ++m_size; + if (m_hasIndices) + m_placeOfIndex.append(m_buffer.size()); + + Header header { static_cast(segment.bytes.size()), 0 }; + if (segment.isActive() && segment.offsetIfActive->isConst() && segment.offsetIfActive->bits() <= std::numeric_limits::max() && !segment.memoryIndex) [[likely]] { + header.offset = static_cast(segment.offsetIfActive->bits()); + m_buffer.append(asByteSpan(header)); + } else { + LongHeader longHeader { }; + longHeader.memoryIndex = segment.memoryIndex; + longHeader.isPassive = segment.isPassive(); + if (segment.isActive()) { + longHeader.offset = segment.offsetIfActive->bits(); + longHeader.isGlobalImport = segment.offsetIfActive->isGlobalImport(); + longHeader.isExtendedExpression = segment.offsetIfActive->isExtendedExpression(); + } + header.size |= hasLongHeaderBit; + m_buffer.append(asByteSpan(header)); + m_buffer.append(asByteSpan(longHeader)); + } + m_buffer.append(segment.bytes); +} + +void Segments::readLongHeader(size_t& place, Segment& segment) const +{ + auto header = read(place); + segment.memoryIndex = header.memoryIndex; + if (header.isPassive) { + segment.kind = Segment::Kind::Passive; + segment.offsetIfActive = std::nullopt; + } else if (header.isGlobalImport) + segment.offsetIfActive = I32InitExpr::globalImport(header.offset); + else if (header.isExtendedExpression) + segment.offsetIfActive = I32InitExpr::extendedExpression(header.offset); + else + segment.offsetIfActive = I32InitExpr::constValue(header.offset); } String makeString(const Name& characters) diff --git a/Source/JavaScriptCore/wasm/WasmFormat.h b/Source/JavaScriptCore/wasm/WasmFormat.h index 2395aaacbdaf0..de58ad63734b7 100644 --- a/Source/JavaScriptCore/wasm/WasmFormat.h +++ b/Source/JavaScriptCore/wasm/WasmFormat.h @@ -721,6 +721,7 @@ class I32InitExpr { RELEASE_ASSERT(isExtendedExpression()); return m_bits; } + uint64_t bits() const { return m_bits; } private: uint64_t m_bits; @@ -729,44 +730,77 @@ class I32InitExpr { using I64InitExpr = I32InitExpr; -class Segment final : public TrailingArray { - WTF_DEPRECATED_MAKE_FAST_ALLOCATED(Segment); - WTF_MAKE_NONCOPYABLE(Segment); - WTF_MAKE_NONMOVABLE(Segment); - using Base = TrailingArray; - friend Base; -public: +struct Segment { enum class Kind : uint8_t { Active, Passive, }; - uint8_t& byte(uint32_t pos) + bool isActive() const { return kind == Kind::Active; } + bool isPassive() const { return kind == Kind::Passive; } + + Kind kind; + uint32_t memoryIndex; + std::optional offsetIfActive; + std::span bytes; +}; + +// The data segments of a module. There may be a hundred thousand of a few bytes each, so they are in one buffer, each behind a short header. +class Segments { +public: + // Only a module with a Data Count section names its segments by index. + bool tryReserve(size_t numberOfSegments, size_t numberOfBytes, size_t numberOfIndices); + void append(const Segment&); + + size_t size() const { return m_size; } + size_t sizeInBytes() const { return m_buffer.size(); } + // Moves the place to the next segment. + Segment readAt(size_t& place) const { - return Base::at(pos); + Header header = read
(place); + uint32_t size = header.size & ~hasLongHeaderBit; + Segment segment { Segment::Kind::Active, 0, I32InitExpr::constValue(header.offset), { } }; + if (header.size & hasLongHeaderBit) [[unlikely]] + readLongHeader(place, segment); + segment.bytes = m_buffer.subspan(place, size); + place += size; + return segment; } - uint32_t sizeInBytes() const { return Base::size(); } - - Segment(size_t sizeInBytes, Kind passedKind, std::optional&& passedOffsetIfActive, uint32_t memoryIndex = 0) - : Base(sizeInBytes) - , m_kind(passedKind) - , m_offsetIfActive(WTF::move(passedOffsetIfActive)) - , m_memoryIndex(memoryIndex) + Segment operator[](size_t index) const { + size_t place = m_placeOfIndex[index]; + return readAt(place); } - static std::unique_ptr tryCreate(std::optional, uint32_t, Kind, uint32_t memoryIndex = 0); +private: + // An active segment of the first memory at a constant offset of 32 bits has this alone. + struct Header { + uint32_t size; + uint32_t offset; + }; + struct LongHeader { + uint64_t offset; + uint32_t memoryIndex; + bool isPassive; + bool isGlobalImport; + bool isExtendedExpression; + }; + static constexpr uint32_t hasLongHeaderBit = 1u << 31; - bool isActive() const { return m_kind == Kind::Active; } - bool isPassive() const { return m_kind == Kind::Passive; } - Kind kind() const { return m_kind; } - std::optional offsetIfActive() const { return m_offsetIfActive; } - uint32_t memoryIndex() const { return m_memoryIndex; } + template + T read(size_t& place) const + { + T result; + memcpySpan(asMutableByteSpan(result), m_buffer.subspan(place, sizeof(T))); + place += sizeof(T); + return result; + } + void readLongHeader(size_t& place, Segment&) const; -private: - const Kind m_kind; - const std::optional m_offsetIfActive; - const uint32_t m_memoryIndex; + Vector m_buffer; + Vector m_placeOfIndex; + size_t m_size { 0 }; + bool m_hasIndices { false }; }; struct Element { diff --git a/Source/JavaScriptCore/wasm/WasmIPIntGenerator.cpp b/Source/JavaScriptCore/wasm/WasmIPIntGenerator.cpp index bac95caae6b6f..8962da33c2348 100644 --- a/Source/JavaScriptCore/wasm/WasmIPIntGenerator.cpp +++ b/Source/JavaScriptCore/wasm/WasmIPIntGenerator.cpp @@ -2815,6 +2815,7 @@ void IPIntGenerator::addTailCallCommonData(const RTT&, const CallInformation& ca m_maxStackSize = m_stackSize + extraWasmStackInBytes / 16; ASSERT(!(stackArgumentsAndResultsInBytes % 16)); // mINT requires this for 16-bytes at a time tail-call arguments copy + m_info.m_usesTailCalls.storeRelaxed(true); } [[nodiscard]] PartialResult IPIntGenerator::addCall(unsigned callProfileIndex, FunctionSpaceIndex index, const RTT& signature, ArgumentList&, ResultList& results, CallType callType) diff --git a/Source/JavaScriptCore/wasm/WasmIPIntPlan.cpp b/Source/JavaScriptCore/wasm/WasmIPIntPlan.cpp index 5983be97538e5..994cd369660eb 100644 --- a/Source/JavaScriptCore/wasm/WasmIPIntPlan.cpp +++ b/Source/JavaScriptCore/wasm/WasmIPIntPlan.cpp @@ -122,16 +122,17 @@ void IPIntPlan::compileFunction(FunctionCodeIndex functionIndex) auto functionIndexSpace = m_moduleInformation->toSpaceIndex(functionIndex); ASSERT_UNUSED(functionIndexSpace, &m_moduleInformation->rtt(functionIndexSpace) == &m_moduleInformation->rtt(typeSignatureIndex)); + std::optional metadataOfImageFunction; #if ENABLE(WEBASSEMBLY_OMGJIT) if (m_moduleInImage) { if (CodeInImage::functionsOf(*m_moduleInImage, ImageCodeFlavor::Checking)[functionIndex.rawIndex()].usesVectors) m_moduleInformation->markUsesSIMD(functionIndex); m_moduleInformation->doneSeeingFunction(functionIndex); static constexpr std::array noBytecode { }; - m_wasmInternalFunctions[functionIndex] = makeUnique(functionIndex, function.data.isEmpty() ? std::span { noBytecode } : function.data.span()); + metadataOfImageFunction.emplace(functionIndex, function.data.isEmpty() ? std::span { noBytecode } : function.data.span()); } #endif - if (!m_wasmInternalFunctions[functionIndex]) { + if (!metadataOfImageFunction && !m_wasmInternalFunctions[functionIndex]) { beginCompilerSignpost(CompilationMode::IPIntMode, functionIndexSpace); auto parseAndCompileResult = parseAndCompileMetadata(function.data, signature, m_moduleInformation.get(), functionIndex); endCompilerSignpost(CompilationMode::IPIntMode, functionIndexSpace); @@ -146,7 +147,7 @@ void IPIntPlan::compileFunction(FunctionCodeIndex functionIndex) } { - auto callee = IPIntCallee::create(*m_wasmInternalFunctions[functionIndex], functionIndexSpace, signature, { }); + auto callee = IPIntCallee::create(metadataOfImageFunction ? *metadataOfImageFunction : *m_wasmInternalFunctions[functionIndex], functionIndexSpace, signature, { }); ASSERT(!callee->entrypoint()); bool usesSIMD = m_moduleInformation->usesSIMD(functionIndex); // Immediately tier up to BBQ for SIMD, if necesary. diff --git a/Source/JavaScriptCore/wasm/WasmIPIntPlan.h b/Source/JavaScriptCore/wasm/WasmIPIntPlan.h index 3e2f14a0c4391..55e520588a1c3 100644 --- a/Source/JavaScriptCore/wasm/WasmIPIntPlan.h +++ b/Source/JavaScriptCore/wasm/WasmIPIntPlan.h @@ -64,6 +64,7 @@ class IPIntPlan final : public EntryPlan { } void work() final; + bool hasCodeInImage() const { return m_moduleInImage; } bool didReceiveFunctionData(FunctionCodeIndex, const FunctionData&) final; diff --git a/Source/JavaScriptCore/wasm/WasmIPIntSlowPaths.cpp b/Source/JavaScriptCore/wasm/WasmIPIntSlowPaths.cpp index 9bd22929a8fe7..3d5408dd63ae0 100644 --- a/Source/JavaScriptCore/wasm/WasmIPIntSlowPaths.cpp +++ b/Source/JavaScriptCore/wasm/WasmIPIntSlowPaths.cpp @@ -1417,6 +1417,8 @@ extern "C" void SYSV_ABI wasm_log_crash(CallFrame*, JSWebAssemblyInstance* insta extern "C" UGPRPair SYSV_ABI slow_path_wasm_throw_exception(CallFrame* callFrame, JSWebAssemblyInstance* instance, Wasm::ExceptionType exceptionType) { + if (instance->faultPC()) + instance->calleeGroup()->didFault(callFrame, instance->faultPC()); // FaultPC is the exact PC causing the fault. When using it as a returnPC, we should point one next instruction instead. WasmOperationPrologueCallFrameTracer tracer(instance->vm(), callFrame, std::bit_cast(std::bit_cast(instance->faultPC()) + 1)); instance->setFaultPC(Wasm::ExceptionType::Termination, nullptr); diff --git a/Source/JavaScriptCore/wasm/WasmModule.cpp b/Source/JavaScriptCore/wasm/WasmModule.cpp index bb820dc7a0ebb..43bac4a013028 100644 --- a/Source/JavaScriptCore/wasm/WasmModule.cpp +++ b/Source/JavaScriptCore/wasm/WasmModule.cpp @@ -76,19 +76,28 @@ static Plan::CompletionTask makeValidationCallback(Name&& sourceURL, Module::Asy }); } +static void completeSync(IPIntPlan& plan) +{ + // Nothing is left to compile. + if (plan.hasCodeInImage()) { + while (plan.hasWork()) + plan.work(); + } else + Wasm::ensureWorklist().enqueue(plan); + plan.waitForCompletion(); +} + Module::ValidationResult Module::validateSync(VM& vm, Vector&& source) { Ref plan = adoptRef(*new IPIntPlan(vm, WTF::move(source), CompilerMode::Validation, Plan::dontFinalize())); - Wasm::ensureWorklist().enqueue(plan.get()); - plan->waitForCompletion(); + completeSync(plan.get()); return makeValidationResult(plan.get()); } Module::ValidationResult Module::validateSync(VM& vm, std::span source) { Ref plan = adoptRef(*new IPIntPlan(vm, source, CompilerMode::Validation, Plan::dontFinalize())); - Wasm::ensureWorklist().enqueue(plan.get()); - plan->waitForCompletion(); + completeSync(plan.get()); return makeValidationResult(plan.get()); } diff --git a/Source/JavaScriptCore/wasm/WasmModuleInformation.h b/Source/JavaScriptCore/wasm/WasmModuleInformation.h index 5c77155d10aec..915cd40b5742b 100644 --- a/Source/JavaScriptCore/wasm/WasmModuleInformation.h +++ b/Source/JavaScriptCore/wasm/WasmModuleInformation.h @@ -222,6 +222,7 @@ struct ModuleInformation final : public ThreadSafeRefCounted bool m_hasGCObjectTypes { false }; mutable Atomic m_usesLegacyExceptions { false }; mutable Atomic m_usesModernExceptions { false }; + Atomic m_usesTailCalls { false }; Vector functions; const ModuleInImage* moduleWithSkippedFunctions { nullptr }; @@ -229,7 +230,7 @@ struct ModuleInformation final : public ThreadSafeRefCounted Vector exports; std::optional startFunctionIndexSpace; - Vector> data; + Segments data; Vector elements; Vector tables; Vector globals; diff --git a/Source/JavaScriptCore/wasm/WasmOMGIRGenerator.cpp b/Source/JavaScriptCore/wasm/WasmOMGIRGenerator.cpp index fb6cc0d662417..eaa4e99478fee 100644 --- a/Source/JavaScriptCore/wasm/WasmOMGIRGenerator.cpp +++ b/Source/JavaScriptCore/wasm/WasmOMGIRGenerator.cpp @@ -29,7 +29,9 @@ #if ENABLE(WEBASSEMBLY_OMGJIT) +#include "AirArgInlines.h" #include "AirCode.h" +#include "AirInstInlines.h" #include "AllowMacroScratchRegisterUsageIf.h" #include "B3AbstractHeapRepository.h" #include "B3BasicBlockInlines.h" @@ -94,6 +96,7 @@ #include #include #include +#include #include #include #include @@ -420,6 +423,11 @@ class OMGIRGenerator { Variable* m_exception; TargetList m_tryTableTargets; bool m_hasBranchToHeader { false }; + unsigned m_numberProvenInOuterScopes { 0 }; + unsigned m_firstIndexInOuterLoop { 0 }; + BasicBlock* m_ifBranch { nullptr }; + unsigned m_callCountAtIf { 0 }; + bool m_thenArmMakesCalls { false }; }; using ControlType = ControlData; @@ -777,6 +785,7 @@ class OMGIRGenerator { [[nodiscard]] PartialResult addMemoryFill(ExpressionType dstAddress, ExpressionType targetValue, ExpressionType count, uint8_t memoryIndex); [[nodiscard]] PartialResult addMemoryCopy(ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType count, uint8_t dstMemoryIndex, uint8_t srcMemoryIndex); void emitMemoryFill(Value* dstAddress, Value* targetValue, Value* count, uint8_t memoryIndex); + Value* isRangeOutOfBounds(Value* start, Value* count, Value* memorySize); void emitMemoryCopy(Value* dstAddress, Value* srcAddress, Value* count, uint8_t dstMemoryIndex, uint8_t srcMemoryIndex); [[nodiscard]] PartialResult addMemoryInit(unsigned, ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType length, uint8_t memoryIndex); [[nodiscard]] PartialResult addDataDrop(unsigned); @@ -891,7 +900,7 @@ class OMGIRGenerator { [[nodiscard]] PartialResult emitIndirectCall(Value* calleeInstance, Value* calleeCode, Value* calleeCodeLocation, Value* boxedCalleeCallee, const RTT&, const ArgumentList& args, ValueResults&, CallType = CallType::Call); Vector createCallConstrainedArgs(BasicBlock*, const CallInformation& wasmCalleeInfo, const ArgumentList&); - auto createCallPatchpoint(BasicBlock*, const RTT&, const CallInformation&, const ArgumentList& tmpArgs) -> CallPatchpointData; + auto createCallPatchpoint(BasicBlock*, const RTT&, const CallInformation&, const ArgumentList& tmpArgs, Value* passedGlobal = nullptr) -> CallPatchpointData; auto createTailCallPatchpoint(BasicBlock*, const RTT&, const CallInformation& wasmCallerInfoAsCallee, const CallInformation& wasmCalleeInfoAsCallee, const ArgumentList& tmpArgSourceLocations, Value* boxedCalleeCallee = nullptr) -> CallPatchpointData; InliningNode* canInline(FunctionSpaceIndex functionIndexSpace, unsigned callProfileIndex) const; @@ -918,6 +927,7 @@ class OMGIRGenerator { void insertConstants(); B3::Type toB3ResultType(const RTT*); + B3::Type resultTypeWithGlobal(const RTT&); void addStackMap(unsigned callSiteIndex, StackMap&& stackmap) { @@ -959,7 +969,8 @@ class OMGIRGenerator { std::optional imagePointerSlot(ProcessPointerKind, const void*); Value* processPointer(BasicBlock*, ProcessPointerKind, const void*); void moveProcessPointer(CCallHelpers&, ProcessPointerKind, const void*, GPRReg dest); - void emitCallInImage(CCallHelpers&, uint32_t symbol); + static constexpr GPRReg passedGlobalGPR = CodeForImage::passedGlobalGPR; + void emitCallInImage(CCallHelpers&, uint32_t symbol, ImageEntry = ImageEntry::Primary); void emitCallToThrowStub(CCallHelpers&); void emitExceptionCheck(CCallHelpers&, Origin, ExceptionType); @@ -1060,6 +1071,37 @@ class OMGIRGenerator { void unify(Value* phi, const ExpressionType& source); void unifyValuesWithBlock(std::span resultStack, const ControlData& block); BasicBlock* targetBlockForBranch(ControlData&); + + // An earlier access succeeded up to this offset from the local, so accesses below it neither wrap nor trap. + struct ProvenBytes { + uint32_t end { 0 }; + unsigned index { 0 }; + }; + struct LocalAddress { + Variable* local; + Value* base; + uint32_t offset; + }; + std::optional addressInLocal(Value* pointer); + Variable* currentLocal(Value*) const; + ProvenBytes provenBytes(Variable*) const; + void foldProvenSum(LocalAddress&, Value*& pointer, uint64_t& offset); + bool isProvenInBounds(const LocalAddress&, uint64_t end) const; + void didAccess(const LocalAddress&, uint64_t end, Value* access); + void didSetLocal(Value* set); + void enterScope(ControlData&, BlockType); + void leaveScope(const ControlData&); + bool emitsShrinkWrappedFrame() const { return m_proc.policy().air.frame.emitsFrame; } + bool isPassedGlobal(uint32_t index) const { return m_passedGlobal && m_context.codeForImage->passedGlobal == index; } + Value* loadPassedGlobal(); + void reloadPassedGlobal(); + Value* passedGlobalAtReturn(); + Value* zeroExtendedPassedGlobal(); + void emitDirectEntry(CCallHelpers&, B3::Air::GenerationContext&); + void emitZeroEntry(CCallHelpers&, B3::Air::GenerationContext&, const Box& zeroSuccessor); + void storeInstanceAndCallee(CCallHelpers&, GPRReg scratch); + void finishFrameSetup(CCallHelpers&, B3::Air::GenerationContext&, const B3::Air::StackCheckSite&); + CCallHelpers::JumpList emitStackCheck(CCallHelpers&, const B3::Air::StackCheckSite&, int32_t checkSize); static SwitchValue* stateMachineDispatch(BasicBlock* loopHeader); void emitFirstStateTest(const ControlData& loop); @@ -1070,6 +1112,7 @@ class OMGIRGenerator { void restoreWasmContextInstance(BasicBlock*, Value*); void restoreWebAssemblyGlobalState(const Vector&, Value* instance, BasicBlock*); + bool callsRestorePinnedRegisters() const; void reloadMemoryRegistersFromInstance(const MemoryInformation&, Value* instance, BasicBlock*); void materializeExpressionStackIntoVariables(); @@ -1119,7 +1162,9 @@ class OMGIRGenerator { Value* set(BasicBlock* block, Variable* dst, Value* src) { - return block->appendNew(m_proc, B3::Set, origin(), dst, src); + Value* result = block->appendNew(m_proc, B3::Set, origin(), dst, src); + didSetLocal(result); + return result; } Value* set(Variable* dst, Value* src) @@ -1175,10 +1220,24 @@ class OMGIRGenerator { unsigned* m_osrEntryScratchBufferSize; UncheckedKeyHashMap m_constantPool; UncheckedKeyHashMap m_tupleMap; + UncheckedKeyHashMap m_tupleWithPassedGlobalMap; B3::Type m_int64PairTupleType { }; InsertionSet m_constantInsertionValues; + // Indexed by local. Entries are restored at the end of the scope that proved them. + Vector m_provenBytes; + Vector m_indexOfLastSet; + Vector> m_provenInScopes; + bool m_nextAccessIsInBounds { false }; + // Facts proven before a loop do not hold in its body. + unsigned m_firstIndexInLoop { 0 }; Value* m_framePointer { nullptr }; bool m_makesCalls { false }; + unsigned m_callCount { 0 }; + Variable* m_passedGlobal { nullptr }; + bool m_accessesPassedGlobal { false }; + bool m_isResumable { false }; + bool m_hasLoop { false }; + Box m_stackOverflows; bool m_makesTailCalls { false }; // This tracks the maximum stack offset for a tail call, to be used in the stack overflow check. @@ -1227,6 +1286,46 @@ class OMGIRGenerator { WTF_MAKE_TZONE_ALLOCATED_IMPL(OMGIRGenerator); +// A resumable function takes its resume point, switches on it first, and returns whether it suspended. +// Without calls it has no switch, but it still copies the stack pointer global into a local first. +bool isResumable(const ModuleInformation& info, FunctionCodeIndex functionIndex, uint32_t passedGlobal) +{ + const RTT& signature = info.rtt(info.internalFunctionTypeSignatureIndices[functionIndex]); + if (signature.argumentCount() != 1 || !signature.argumentType(0).isI32() || signature.returnCount() != 1 || !signature.returnType(0).isI32()) + return false; + + std::span body = info.functions[functionIndex].data.span(); + size_t offset = 0; + uint32_t value = 0; + uint32_t numberOfLocalGroups = 0; + if (!WTF::LEBDecoder::decodeUInt32(body, offset, numberOfLocalGroups)) + return false; + for (uint32_t group = 0; group < numberOfLocalGroups; ++group) { + constexpr uint8_t lowestNumberType = 0x7c; + constexpr uint8_t highestNumberType = 0x7f; + if (!WTF::LEBDecoder::decodeUInt32(body, offset, value) || offset >= body.size() || body[offset] < lowestNumberType || body[offset] > highestNumberType) + return false; + ++offset; + } + constexpr uint8_t emptyBlock = 0x40; + while (offset < body.size()) { + switch (static_cast(body[offset++])) { + case OpType::Block: + case OpType::Loop: + if (offset >= body.size() || body[offset++] != emptyBlock) + return false; + break; + case OpType::GetGlobal: + return WTF::LEBDecoder::decodeUInt32(body, offset, value) && value == passedGlobal && offset < body.size() && static_cast(body[offset]) == OpType::SetLocal; + case OpType::GetLocal: + return WTF::LEBDecoder::decodeUInt32(body, offset, value) && !value && offset < body.size() && static_cast(body[offset]) == OpType::BrTable; + default: + return false; + } + } + return false; +} + // Memory accesses in WebAssembly have unsigned 32-bit offsets (unsigned 64-bit offsets in memory64), whereas they have signed 32-bit offsets in B3. int32_t OMGIRGenerator::fixupPointerPlusOffset(Value*& ptr, uint64_t offset) { @@ -1386,6 +1485,8 @@ OMGIRGenerator::OMGIRGenerator(AbstractHeapRepository& heaps, CompilationContext getInstance->effects.writesPinned = false; getInstance->effects.readsPinned = true; getInstance->resultConstraints = { ValueRep::reg(GPRInfo::wasmContextInstancePointer) }; + getInstance->setRunsWithoutFrame(true); + getInstance->emitsNoCode = true; getInstance->setGenerator([=] (CCallHelpers&, const B3::StackmapGenerationParams&) { }); m_instanceValue = getInstance; m_vmValue = m_topLevelBlock->appendNew(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast(JSWebAssemblyInstance::offsetOfVM())); @@ -1408,6 +1509,8 @@ OMGIRGenerator::OMGIRGenerator(AbstractHeapRepository& heaps, CompilationContext getBaseMemory->effects.writesPinned = false; getBaseMemory->effects.readsPinned = true; getBaseMemory->resultConstraints = { ValueRep::reg(GPRInfo::wasmBaseMemoryPointer) }; + getBaseMemory->setRunsWithoutFrame(true); + getBaseMemory->emitsNoCode = true; getBaseMemory->setGenerator([=] (CCallHelpers&, const B3::StackmapGenerationParams&) { }); m_baseMemoryValue = getBaseMemory; } @@ -1426,12 +1529,23 @@ OMGIRGenerator::OMGIRGenerator(AbstractHeapRepository& heaps, CompilationContext } code.emitDefaultPrologue(jit, context); - GPRReg scratchGPR = wasmCallingConvention().prologueScratchGPRs[0]; - moveProcessPointer(jit, ProcessPointerKind::OwnCallee, CalleeBits::boxNativeCallee(m_callee), scratchGPR); - static_assert(CallFrameSlot::codeBlock + 1 == CallFrameSlot::callee); - jit.storePairPtr(GPRInfo::wasmContextInstancePointer, scratchGPR, GPRInfo::callFrameRegister, CCallHelpers::TrustedImm32(CallFrameSlot::codeBlock * sizeof(Register))); + if (m_passedGlobal) + emitDirectEntry(jit, context); + if (!emitsShrinkWrappedFrame()) + storeInstanceAndCallee(jit, wasmCallingConvention().prologueScratchGPRs[0]); }); - { + if (m_context.codeForImage && m_context.codeForImage->passedGlobal && emitsShrinkWrappedFrame()) { + m_passedGlobal = m_proc.addVariable(Int32); + m_isResumable = isResumable(m_info, FunctionCodeIndex(m_functionIndex), *m_context.codeForImage->passedGlobal); + Value* argument = m_topLevelBlock->appendNew(m_proc, Origin(), passedGlobalGPR); + m_topLevelBlock->appendNew(m_proc, Set, Origin(), m_passedGlobal, m_topLevelBlock->appendNew(m_proc, Trunc, Origin(), argument)); + } + + if (emitsShrinkWrappedFrame()) { + m_proc.code().setStackCheckGenerator(createSharedTask([this](CCallHelpers& jit, B3::Air::GenerationContext& context, const B3::Air::StackCheckSite& site) { + finishFrameSetup(jit, context, site); + })); + } else { B3::PatchpointValue* stackOverflowCheck = m_currentBlock->appendNew(m_proc, Void, Origin()); stackOverflowCheck->appendSomeRegister(instanceValue()); stackOverflowCheck->appendSomeRegister(framePointer()); @@ -1505,6 +1619,46 @@ void OMGIRGenerator::restoreWebAssemblyGlobalState(const Vectorclobber(RegisterSet { GPRInfo::wasmBoundsCheckingSizeRegister }); +#if GIGACAGE_ENABLED + patchpoint->numGPScratchRegisters = 1; +#endif +} + +static GPRReg scratchForCaging(const B3::StackmapGenerationParams& params) +{ +#if GIGACAGE_ENABLED + return params.gpScratch(0); +#else + UNUSED_PARAM(params); + return InvalidGPRReg; +#endif +} + +static void emitPinnedRegistersSwitch(CCallHelpers& jit, GPRReg calleeInstance, GPRReg scratch) +{ + emitPrepareToCageMemoryBase(jit, GPRInfo::wasmContextInstancePointer, scratch); + jit.storeWasmContextInstance(calleeInstance); + jit.loadPairPtr(calleeInstance, CCallHelpers::TrustedImm32(JSWebAssemblyInstance::offsetOfCachedMemoryBaseSizePair(0)), GPRInfo::wasmBaseMemoryPointer, GPRInfo::wasmBoundsCheckingSizeRegister); + emitCageMemoryBase(jit, scratch); +} + +static void emitPinnedRegistersRestore(CCallHelpers& jit, GPRReg scratch) +{ + jit.loadPtr(CCallHelpers::addressFor(CallFrameSlot::codeBlock), GPRInfo::wasmContextInstancePointer); + emitPrepareToCageMemoryBase(jit, GPRInfo::wasmContextInstancePointer, scratch); + jit.loadPtr(CCallHelpers::Address(GPRInfo::wasmContextInstancePointer, JSWebAssemblyInstance::offsetOfCachedMemoryBaseSizePair(0)), GPRInfo::wasmBaseMemoryPointer); + emitCageMemoryBase(jit, scratch); +} + void OMGIRGenerator::reloadMemoryRegistersFromInstance(const MemoryInformation& memory, Value* instance, BasicBlock* block) { if (!!memory) { @@ -1601,15 +1755,16 @@ void OMGIRGenerator::moveProcessPointer(CCallHelpers& jit, ProcessPointerKind ki jit.loadPtr(CCallHelpers::Address(dest, safeCast(*slot * sizeof(void*))), dest); } -void OMGIRGenerator::emitCallInImage(CCallHelpers& jit, uint32_t symbol) +void OMGIRGenerator::emitCallInImage(CCallHelpers& jit, uint32_t symbol, ImageEntry entry) { #if ENABLE(AOT) ASSERT_UNUSED(jit, &jit == m_context.wasmEntrypointJIT.get()); auto& code = *m_context.codeForImage; - code.recorder.call({ AOT::SymbolSpace::WebAssembly, symbol }); + code.recorder.call({ AOT::SymbolSpace::WebAssembly, symbol, static_cast(entry) }); #else UNUSED_PARAM(jit); UNUSED_PARAM(symbol); + UNUSED_PARAM(entry); RELEASE_ASSERT_NOT_REACHED(); #endif } @@ -1625,6 +1780,193 @@ void OMGIRGenerator::emitCallToThrowStub(CCallHelpers& jit) jit.nearCallThunk(CodeLocationLabel(Thunks::singleton().stub(throwExceptionFromOMGThunkGenerator).code())); } +static bool mayRead(B3::Air::BasicBlock* start, Reg reg) +{ + IndexSet seen; + Vector worklist { start }; + seen.add(start); + while (!worklist.isEmpty()) { + B3::Air::BasicBlock* block = worklist.takeLast(); + bool isWritten = false; + for (B3::Air::Inst& inst : *block) { + bool isRead = false; + inst.forEach([&](Reg& used, B3::Air::Arg::Role role, Bank, Width) { + if (used != reg) + return; + isRead |= B3::Air::Arg::isAnyUse(role); + isWritten |= B3::Air::Arg::isAnyDef(role); + }); + if (isRead) + return true; + isWritten |= inst.kind.opcode == B3::Air::Patch && inst.extraClobberedRegs().contains(reg, IgnoreVectors); + if (isWritten) + break; + } + if (isWritten) + continue; + for (B3::Air::BasicBlock* successor : block->successorBlocks()) { + if (seen.add(successor)) + worklist.append(successor); + } + } + return false; +} + +// The successor taken for zero, if the entry block only tests the register and nothing reads it afterward. +static B3::Air::BasicBlock* findZeroSuccessor(B3::Air::BasicBlock* entry, Reg reg) +{ + B3::Air::Inst& branch = entry->last(); + bool isReadBeforeTest = false; + for (B3::Air::Inst& inst : *entry) { + auto* patchpoint = inst.kind.opcode == B3::Air::Patch ? inst.origin->as() : nullptr; + bool isMove = inst.isMoveBetweenRegisters() && inst.args()[1] != B3::Air::Arg(B3::Air::Tmp(reg)); + if (&inst != &branch && !isMove && (!patchpoint || !patchpoint->emitsNoCode)) + return nullptr; + isReadBeforeTest |= isMove && inst.args()[0] == B3::Air::Arg(B3::Air::Tmp(reg)); + } + if (branch.kind.opcode != B3::Air::BranchTest32 || branch.args()[1] != B3::Air::Arg(B3::Air::Tmp(reg))) + return nullptr; + B3::Air::Arg mask = branch.args()[2]; + if (mask != branch.args()[1] && mask != B3::Air::Arg::bitImm(-1)) + return nullptr; + MacroAssembler::ResultCondition condition = branch.args()[0].asResultCondition(); + if (condition != MacroAssembler::Zero && condition != MacroAssembler::NonZero) + return nullptr; + B3::Air::BasicBlock* block = entry->successorBlock(condition == MacroAssembler::Zero ? 0 : 1); + return isReadBeforeTest || mayRead(block, reg) ? nullptr : block; +} + +void OMGIRGenerator::emitZeroEntry(CCallHelpers& jit, B3::Air::GenerationContext& context, const Box& zeroSuccessor) +{ + auto isMove = [](const B3::Air::Inst& inst) { + return inst.isMoveBetweenRegisters(); + }; + if (std::ranges::none_of(*context.currentBlock, isMove)) { + m_context.directEntryWithZero = zeroSuccessor; + return; + } + auto hasState = jit.jump(); + m_context.directEntryWithZero = Box::create(jit.label()); + for (B3::Air::Inst& inst : *context.currentBlock) { + if (isMove(inst)) + inst.generate(jit, context); + } + auto toBlock = jit.jump(); + context.addLatePath(Origin(), [toBlock, zeroSuccessor](CCallHelpers& jit) { + toBlock.linkTo(*zeroSuccessor, &jit); + }); + hasState.link(&jit); +} + +// Direct callers pass the global and enter at directEntry. Others enter before it, at the load from the instance. +void OMGIRGenerator::emitDirectEntry(CCallHelpers& jit, B3::Air::GenerationContext& context) +{ + jit.load32(CCallHelpers::Address(GPRInfo::wasmContextInstancePointer, safeCast(JSWebAssemblyInstance::offsetOfGlobal(m_info, *m_context.codeForImage->passedGlobal))), passedGlobalGPR); + if (m_isResumable) { + GPRReg state = wasmCallingConvention().gprArgs[0]; + if (B3::Air::BasicBlock* block = findZeroSuccessor(context.currentBlock, state)) { + auto& label = context.blockLabels[block]; + if (!label) + label = Box::create(); + emitZeroEntry(jit, context, label); + } else if (mayRead(context.currentBlock, state)) { + auto hasState = jit.jump(); + m_context.directEntryWithZero = Box::create(jit.label()); + jit.move(CCallHelpers::TrustedImm32(0), state); + hasState.link(&jit); + } + } + m_context.directEntry = jit.label(); + if (m_isResumable && !m_context.directEntryWithZero) + m_context.directEntryWithZero = Box::create(m_context.directEntry); +} + +// The instance always holds the value, so calls that leave the module's code only have to reload it. +Value* OMGIRGenerator::loadPassedGlobal() +{ + uint32_t index = *m_context.codeForImage->passedGlobal; + auto* value = m_currentBlock->appendNew(m_proc, Load, Int32, origin(), instanceValue(), safeCast(JSWebAssemblyInstance::offsetOfGlobal(m_info, index))); + m_heaps.decorateMemory(&m_heaps.JSWebAssemblyInstance_embeddedGlobals_i32[index], value); + return value; +} + +void OMGIRGenerator::reloadPassedGlobal() +{ + if (m_passedGlobal) + m_currentBlock->appendNew(m_proc, Set, origin(), m_passedGlobal, loadPassedGlobal()); +} + +Value* OMGIRGenerator::passedGlobalAtReturn() +{ + // Keeps the global out of registers across loops that do not use it. + if (m_hasLoop && !m_accessesPassedGlobal) + return m_currentBlock->appendNew(m_proc, ZExt32, origin(), loadPassedGlobal()); + return zeroExtendedPassedGlobal(); +} + +Value* OMGIRGenerator::zeroExtendedPassedGlobal() +{ + Value* global = m_currentBlock->appendNew(m_proc, B3::Get, origin(), m_passedGlobal); + return m_currentBlock->appendNew(m_proc, ZExt32, origin(), global); +} + +void OMGIRGenerator::storeInstanceAndCallee(CCallHelpers& jit, GPRReg scratch) +{ + moveProcessPointer(jit, ProcessPointerKind::OwnCallee, CalleeBits::boxNativeCallee(m_callee), scratch); + static_assert(CallFrameSlot::codeBlock + 1 == CallFrameSlot::callee); + jit.storePairPtr(GPRInfo::wasmContextInstancePointer, scratch, GPRInfo::callFrameRegister, CCallHelpers::TrustedImm32(CallFrameSlot::codeBlock * sizeof(Register))); +} + +CCallHelpers::JumpList OMGIRGenerator::emitStackCheck(CCallHelpers& jit, const B3::Air::StackCheckSite& site, int32_t checkSize) +{ +#if CPU(X86_64) + // After the frame is allocated, the extra bytes of the check may lie in the reserved zone. + constexpr int32_t maxBytesInReservedZone = 16 * KB; + bool isAllocated = site.bytesAllocatedBelowFrameRecord == site.bytesBelowFrameRecord; + if (isAllocated && checkSize - static_cast(m_proc.frameSize()) <= maxBytesInReservedZone) + return jit.branchPtr(CCallHelpers::Below, MacroAssembler::stackPointerRegister, CCallHelpers::Address(GPRInfo::wasmContextInstancePointer, JSWebAssemblyInstance::offsetOfSoftStackLimit())); +#else + UNUSED_PARAM(site); +#endif + return jit.checkWasmStackOverflow(GPRInfo::wasmContextInstancePointer, CCallHelpers::TrustedImm32(checkSize), GPRInfo::callFrameRegister); +} + +// Runs wherever Air sets up the frame, so only the macro scratch registers are free. +void OMGIRGenerator::finishFrameSetup(CCallHelpers& jit, B3::Air::GenerationContext& context, const B3::Air::StackCheckSite& site) +{ + AllowMacroScratchRegisterUsage allowScratch(jit); +#if CPU(X86_64) + storeInstanceAndCallee(jit, MacroAssembler::s_scratchRegister); +#else + storeInstanceAndCallee(jit, MacroAssembler::dataTempRegister); +#endif + int32_t checkSize = 0; + bool needsOverflowCheck = false; + computeStackCheckSize(needsOverflowCheck, checkSize); + if (!needsOverflowCheck) + return; + bool isFirstCheck = !m_stackOverflows; + if (isFirstCheck) + m_stackOverflows = Box::create(); + m_stackOverflows->append(emitStackCheck(jit, site, checkSize)); + if (!isFirstCheck) + return; + context.addLatePath(Origin(), [this](CCallHelpers& jit) { + AllowMacroScratchRegisterUsage allowScratch(jit); + m_stackOverflows->link(&jit); + B3::Air::Code& code = m_proc.code(); + RegisterAtOffsetList saves = code.calleeSaveRegisterAtOffsetList(); + int32_t stackSpace = WTF::roundUpToMultipleOf(saves.registerCount() * sizeof(Register)); + for (const RegisterAtOffset& save : saves) + RELEASE_ASSERT(save.offset() >= -stackSpace); + jit.addPtr(CCallHelpers::TrustedImm32(-stackSpace), GPRInfo::callFrameRegister, MacroAssembler::stackPointerRegister); + if (code.frameLayout().calleeSaveForm == B3::Air::CalleeSaveForm::Stored) + jit.emitSave(saves); + jit.move(CCallHelpers::TrustedImm32(static_cast(ExceptionType::StackOverflow)), GPRInfo::argumentGPR1); + emitCallToThrowStub(jit); + }); +} + void OMGIRGenerator::emitExceptionCheck(CCallHelpers& jit, Origin, ExceptionType type) { jit.move(CCallHelpers::TrustedImm32(static_cast(type)), GPRInfo::argumentGPR1); @@ -1690,6 +2032,19 @@ void OMGIRGenerator::insertConstants() m_constantInsertionValues.execute(m_proc.at(0)); } +B3::Type OMGIRGenerator::resultTypeWithGlobal(const RTT& signature) +{ + if (signature.returnsVoid()) + return B3::Int32; + return m_tupleWithPassedGlobalMap.ensure(&signature, [&] { + Vector result; + for (unsigned i = 0; i < signature.returnCount(); ++i) + result.append(toB3Type(signature.returnType(i))); + result.append(B3::Int32); + return m_proc.addTuple(WTF::move(result)); + }).iterator->value; +} + B3::Type OMGIRGenerator::toB3ResultType(const RTT* returnType) { if (returnType->returnsVoid()) @@ -2125,7 +2480,8 @@ void OMGIRGenerator::fillCallResults(Value* callResult, const RTT& signature, Va static void emitWasmCallStackResultsAndSPRestore(CCallHelpers& jit, const B3::StackmapGenerationParams& params, const RTT& signature, - const CallInformation& wasmCalleeInfo) + const CallInformation& wasmCalleeInfo, + bool needsSPRestore = true) { auto frameSize = params.code().frameSize(); @@ -2161,7 +2517,8 @@ static void emitWasmCallStackResultsAndSPRestore(CCallHelpers& jit, } } - jit.addPtr(CCallHelpers::TrustedImm32(-frameSize), GPRInfo::callFrameRegister, MacroAssembler::stackPointerRegister); + if (needsSPRestore) + jit.addPtr(CCallHelpers::TrustedImm32(-frameSize), GPRInfo::callFrameRegister, MacroAssembler::stackPointerRegister); } auto OMGIRGenerator::emitIndirectCall(Value* calleeInstance, Value* calleeCode, Value* calleeCodeLocation, Value* boxedCalleeCallee, const RTT& signature, const ArgumentList& args, ValueResults& results, CallType callType) -> PartialResult @@ -2230,6 +2587,46 @@ auto OMGIRGenerator::emitIndirectCall(Value* calleeInstance, Value* calleeCode, return { }; } + if (callsRestorePinnedRegisters()) { + auto [patchpoint, handle, prepareForCall] = createCallPatchpoint(m_currentBlock, signature, wasmCalleeInfo, args); + prepareToRestorePinnedRegisters(patchpoint); + unsigned patchArgsIndex = patchpoint->reps().size(); + patchpoint->append(calleeInstance, ValueRep::SomeRegister); + patchpoint->append(calleeCodeLocation, ValueRep::SomeRegister); + patchpoint->append(boxedCalleeCallee, ValueRep::SomeRegister); + patchArgsIndex += m_proc.resultCount(patchpoint->type()); + patchpoint->setGenerator([this, handle = handle, prepareForCall = prepareForCall, patchArgsIndex, signature = Ref(signature), wasmCalleeInfo = WTF::move(wasmCalleeInfo)](CCallHelpers& jit, const B3::StackmapGenerationParams& params) { + AllowMacroScratchRegisterUsage allowScratch(jit); + if (prepareForCall) + prepareForCall->run(jit, params); + if (handle) + handle->collectStackMap(this, params); + + GPRReg calleeInstanceGPR = params[patchArgsIndex].gpr(); + GPRReg calleeCodeLocationGPR = params[patchArgsIndex + 1].gpr(); + GPRReg scratch = scratchForCaging(params); + auto leavesInstance = jit.branchPtr(CCallHelpers::NotEqual, calleeInstanceGPR, GPRInfo::wasmContextInstancePointer); + auto pinnedRegistersAreSet = jit.label(); + params.addLatePath([calleeInstanceGPR, scratch, leavesInstance, pinnedRegistersAreSet](CCallHelpers& jit) { + AllowMacroScratchRegisterUsage allowScratch(jit); + leavesInstance.link(&jit); + emitPinnedRegistersSwitch(jit, calleeInstanceGPR, scratch); + jit.jump().linkTo(pinnedRegistersAreSet, &jit); + }); + + jit.storeWasmCalleeToCalleeCallFrame(params[patchArgsIndex + 2].gpr()); + GPRReg functionInfo = jit.scratchRegister(); + jit.addPtr(CCallHelpers::TrustedImm32(-static_cast(OBJECT_OFFSETOF(WasmOrJSImportableFunction, importFunctionStub))), calleeCodeLocationGPR, functionInfo); + jit.storePtr(functionInfo, CCallHelpers::calleeFrameCodeBlockBeforeCall()); + jit.call(CCallHelpers::Address(calleeCodeLocationGPR, WasmCallableFunction::offsetOfValueOfLoadLocation()), WasmEntryPtrTag); + emitWasmCallStackResultsAndSPRestore(jit, params, signature, wasmCalleeInfo); + emitPinnedRegistersRestore(jit, scratch); + }); + fillCallResults(patchpoint, signature, results); + reloadPassedGlobal(); + return { }; + } + // Do a context switch if needed. { BasicBlock* continuation = m_proc.addBlock(); @@ -2303,6 +2700,7 @@ auto OMGIRGenerator::emitIndirectCall(Value* calleeInstance, Value* calleeCode, // The call could have been to another WebAssembly instance, and / or could have modified our Memory. restoreWebAssemblyGlobalState(m_info.memories, instanceValue(), m_currentBlock); + reloadPassedGlobal(); return { }; } @@ -2311,7 +2709,8 @@ auto OMGIRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result, result = push(callWasmOperation(m_currentBlock, m_info.memory(memoryIndex).addressType().asB3TypeKind(), operationGrowMemory, instanceValue(), addressOperand(m_info.memory(memoryIndex).isMemory64(), delta), constant(Int32, memoryIndex))); - restoreWebAssemblyGlobalState(m_info.memories, instanceValue(), m_currentBlock); + if (!callsRestorePinnedRegisters()) + restoreWebAssemblyGlobalState(m_info.memories, instanceValue(), m_currentBlock); return { }; } @@ -2341,6 +2740,17 @@ auto OMGIRGenerator::addCurrentMemory(ExpressionType& result, uint8_t memoryInde return { }; } +// Checks a range of memory 0. +Value* OMGIRGenerator::isRangeOutOfBounds(Value* start, Value* count, Value* memorySize) +{ + auto* sum = m_currentBlock->appendNew(m_proc, Add, origin(), start, count); + // A 64-bit sum of two zero-extended words does not overflow. + if (m_context.codeForImage && !m_info.memory(0).isMemory64()) + return m_currentBlock->appendNew(m_proc, Above, origin(), sum, memorySize); + auto* sumOverflowed = m_currentBlock->appendNew(m_proc, Below, origin(), sum, start); + return m_currentBlock->appendNew(m_proc, BitOr, origin(), sumOverflowed, m_currentBlock->appendNew(m_proc, Above, origin(), sum, memorySize)); +} + auto OMGIRGenerator::addMemoryFill(ExpressionType dstAddress, ExpressionType target, ExpressionType count, uint8_t memoryIndex) -> PartialResult { bool is64Bit = m_info.memory(memoryIndex).isMemory64(); @@ -2354,10 +2764,7 @@ void OMGIRGenerator::emitMemoryFill(Value* dstAddressValue, Value* targetValue, auto* memorySize = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast(JSWebAssemblyInstance::offsetOfCachedMemory0Size())); m_heaps.decorateMemory(&m_heaps.JSWebAssemblyInstance_cachedMemory0Size, memorySize); - auto* sum = m_currentBlock->appendNew(m_proc, Add, origin(), dstAddressValue, countValue); - auto* sumOverflowed = m_currentBlock->appendNew(m_proc, Below, origin(), sum, dstAddressValue); - Value* outOfBounds = m_currentBlock->appendNew(m_proc, BitOr, origin(), sumOverflowed, m_currentBlock->appendNew(m_proc, Above, origin(), sum, memorySize)); - CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), outOfBounds); + CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), isRangeOutOfBounds(dstAddressValue, countValue, memorySize)); check->setGenerator([=, this, origin = this->origin()](CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, origin, ExceptionType::OutOfBoundsMemoryAccess); }); @@ -2418,19 +2825,13 @@ void OMGIRGenerator::emitMemoryCopy(Value* dstAddressValue, Value* srcAddressVal auto* memorySize = m_currentBlock->appendNew(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast(JSWebAssemblyInstance::offsetOfCachedMemory0Size())); m_heaps.decorateMemory(&m_heaps.JSWebAssemblyInstance_cachedMemory0Size, memorySize); { - auto* dstSum = m_currentBlock->appendNew(m_proc, Add, origin(), dstAddressValue, countValue); - auto* dstSumOverflowed = m_currentBlock->appendNew(m_proc, Below, origin(), dstSum, dstAddressValue); - Value* outOfBounds = m_currentBlock->appendNew(m_proc, BitOr, origin(), dstSumOverflowed, m_currentBlock->appendNew(m_proc, Above, origin(), dstSum, memorySize)); - CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), outOfBounds); + CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), isRangeOutOfBounds(dstAddressValue, countValue, memorySize)); check->setGenerator([=, this, origin = this->origin()](CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, origin, ExceptionType::OutOfBoundsMemoryAccess); }); } { - auto* srcSum = m_currentBlock->appendNew(m_proc, Add, origin(), srcAddressValue, countValue); - auto* srcSumOverflowed = m_currentBlock->appendNew(m_proc, Below, origin(), srcSum, srcAddressValue); - Value* outOfBounds = m_currentBlock->appendNew(m_proc, BitOr, origin(), srcSumOverflowed, m_currentBlock->appendNew(m_proc, Above, origin(), srcSum, memorySize)); - CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), outOfBounds); + CheckValue* check = m_currentBlock->appendNew(m_proc, Check, origin(), isRangeOutOfBounds(srcAddressValue, countValue, memorySize)); check->setGenerator([=, this, origin = this->origin()](CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, origin, ExceptionType::OutOfBoundsMemoryAccess); }); @@ -2548,11 +2949,109 @@ void OMGIRGenerator::traceCF(Args&&... info) return; } +// Values are only appended, so their indices follow program order. +void OMGIRGenerator::didSetLocal(Value* set) +{ + if (!m_context.codeForImage) + return; + unsigned variable = set->as()->variable()->index(); + while (variable >= m_indexOfLastSet.size()) + m_indexOfLastSet.append(0); + m_indexOfLastSet[variable] = set->index(); +} + +// The local that the value was read from, unless it was set since. +Variable* OMGIRGenerator::currentLocal(Value* value) const +{ + if (value->opcode() != B3::Get || m_locals.isEmpty()) + return nullptr; + Variable* variable = value->as()->variable(); + unsigned local = variable->index() - m_locals[0]->index(); + if (local >= m_locals.size() || m_locals[local] != variable) + return nullptr; + if (variable->index() < m_indexOfLastSet.size() && m_indexOfLastSet[variable->index()] > value->index()) + return nullptr; + return variable; +} + +auto OMGIRGenerator::provenBytes(Variable* local) const -> ProvenBytes +{ + if (local->index() >= m_provenBytes.size()) + return { }; + ProvenBytes proven = m_provenBytes[local->index()]; + bool wasSetSince = local->index() < m_indexOfLastSet.size() && m_indexOfLastSet[local->index()] > proven.index; + return wasSetSince || proven.index < m_firstIndexInLoop ? ProvenBytes() : proven; +} + +auto OMGIRGenerator::addressInLocal(Value* pointer) -> std::optional +{ + if (!m_context.codeForImage || pointer->type() != Int32) + return std::nullopt; + Value* sum = pointer->opcode() == Trunc ? pointer->child(0) : pointer; + if (Variable* local = currentLocal(sum)) + return LocalAddress { local, sum, 0 }; + + if (sum->opcode() != Add || !sum->child(1)->hasInt()) + return std::nullopt; + int64_t offset = sum->child(1)->asInt(); + if (offset <= 0 || offset > std::numeric_limits::max()) + return std::nullopt; + Value* base = sum->child(0); + if (base->opcode() == ZExt32 || base->opcode() == SExt32) + base = base->child(0); + Variable* local = currentLocal(base); + if (!local) + return std::nullopt; + return LocalAddress { local, base, static_cast(offset) }; +} + +void OMGIRGenerator::foldProvenSum(LocalAddress& address, Value*& pointer, uint64_t& offset) +{ + if (!address.offset || address.offset >= provenBytes(address.local).end || offset + address.offset > std::numeric_limits::max()) + return; + pointer = address.base->type() == Int64 ? m_currentBlock->appendNew(m_proc, Trunc, origin(), address.base) : address.base; + offset += address.offset; + address.offset = 0; +} + +bool OMGIRGenerator::isProvenInBounds(const LocalAddress& address, uint64_t end) const +{ + return !address.offset && end <= provenBytes(address.local).end; +} + +void OMGIRGenerator::didAccess(const LocalAddress& address, uint64_t end, Value* access) +{ + unsigned local = address.local->index(); + if (address.offset || end <= provenBytes(address.local).end || end > std::numeric_limits::max()) + return; + if (local >= m_provenBytes.size()) + m_provenBytes.grow(local + 1); + m_provenInScopes.append({ local, m_provenBytes[local] }); + m_provenBytes[local] = { static_cast(end), access->index() }; +} + +void OMGIRGenerator::enterScope(ControlData& data, BlockType type) +{ + data.m_numberProvenInOuterScopes = m_provenInScopes.size(); + data.m_firstIndexInOuterLoop = m_firstIndexInLoop; + if (type == BlockType::Loop) + m_firstIndexInLoop = m_proc.values().size(); +} + +void OMGIRGenerator::leaveScope(const ControlData& data) +{ + while (m_provenInScopes.size() > data.m_numberProvenInOuterScopes) { + auto [local, proven] = m_provenInScopes.takeLast(); + m_provenBytes[local] = proven; + } + m_firstIndexInLoop = data.m_firstIndexInOuterLoop; +} + auto OMGIRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult { ASSERT(m_locals[index]); m_localConstants[index] = nullptr; - m_currentBlock->appendNew(m_proc, B3::Set, origin(), m_locals[index], get(value)); + set(m_locals[index], get(value)); TRACE_VALUE(m_parser->typeOfLocal(index), get(value), "set_local ", index); return { }; } @@ -2562,7 +3061,7 @@ auto OMGIRGenerator::teeLocal(uint32_t index, ExpressionType value, ExpressionTy ASSERT(m_locals[index]); m_localConstants[index] = nullptr; Value* input = get(value); - m_currentBlock->appendNew(m_proc, B3::Set, origin(), m_locals[index], input); + set(m_locals[index], input); result = push(input); TRACE_VALUE(m_parser->typeOfLocal(index), input, "tee_local ", index); return { }; @@ -2573,6 +3072,11 @@ auto OMGIRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> Partia const Wasm::GlobalInformation& global = m_info.globals[index]; switch (global.bindingMode) { case Wasm::GlobalInformation::BindingMode::EmbeddedInInstance: { + if (isPassedGlobal(index)) { + m_accessesPassedGlobal = true; + result = push(m_currentBlock->appendNew(m_proc, B3::Get, origin(), m_passedGlobal)); + break; + } auto* value = m_currentBlock->appendNew(m_proc, Load, toB3Type(global.type), origin(), instanceValue(), safeCast(JSWebAssemblyInstance::offsetOfGlobal(m_info, index))); switch (global.type.kind()) { case TypeKind::I32: @@ -2629,6 +3133,10 @@ auto OMGIRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialR switch (global.bindingMode) { case Wasm::GlobalInformation::BindingMode::EmbeddedInInstance: { auto* store = m_currentBlock->appendNew(m_proc, Store, origin(), get(value), instanceValue(), safeCast(JSWebAssemblyInstance::offsetOfGlobal(m_info, index))); + if (isPassedGlobal(index)) { + m_accessesPassedGlobal = true; + m_currentBlock->appendNew(m_proc, Set, origin(), m_passedGlobal, get(value)); + } switch (global.type.kind()) { case TypeKind::I32: m_heaps.decorateMemory(&m_heaps.JSWebAssemblyInstance_embeddedGlobals_i32[index], store); @@ -2843,8 +3351,11 @@ inline uint32_t sizeOfLoadOp(LoadOpType op) inline B3::Kind OMGIRGenerator::memoryKind(B3::Opcode memoryOp, uint8_t memoryIndex) { - if (memoryIndex || useSignalingMemory0() || m_info.memory(0).isShared()) + if (memoryIndex || useSignalingMemory0() || m_info.memory(0).isShared()) { + if (m_nextAccessIsInBounds) + return memoryOp; return trapping(memoryOp); + } return memoryOp; } @@ -2979,8 +3490,18 @@ auto OMGIRGenerator::load(LoadOpType op, ExpressionType pointerVar, ExpressionTy break; } - } else - result = push(emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op), memoryIndex), offset, memoryIndex)); + } else { + std::optional address = memoryIndex ? std::nullopt : addressInLocal(pointer); + if (address) { + foldProvenSum(*address, pointer, offset); + m_nextAccessIsInBounds = isProvenInBounds(*address, offset + sizeOfLoadOp(op)); + } + Value* preparedPointer = emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op), memoryIndex); + result = push(emitLoadOp(op, preparedPointer, offset, memoryIndex)); + m_nextAccessIsInBounds = false; + if (address) + didAccess(*address, offset + sizeOfLoadOp(op), preparedPointer); + } return { }; } @@ -3062,8 +3583,18 @@ auto OMGIRGenerator::store(StoreOpType op, ExpressionType pointerVar, Expression throwException->setGenerator([this, origin = this->origin()] (CCallHelpers& jit, const B3::StackmapGenerationParams&) { this->emitExceptionCheck(jit, origin, ExceptionType::OutOfBoundsMemoryAccess); }); - } else - emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op), memoryIndex), value, offset, memoryIndex); + } else { + std::optional address = memoryIndex ? std::nullopt : addressInLocal(pointer); + if (address) { + foldProvenSum(*address, pointer, offset); + m_nextAccessIsInBounds = isProvenInBounds(*address, offset + sizeOfStoreOp(op)); + } + Value* preparedPointer = emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op), memoryIndex); + emitStoreOp(op, preparedPointer, value, offset, memoryIndex); + m_nextAccessIsInBounds = false; + if (address) + didAccess(*address, offset + sizeOfStoreOp(op), preparedPointer); + } return { }; } @@ -5048,6 +5579,7 @@ auto OMGIRGenerator::addLoop(BlockSignature&& signature, std::spanexpressionStack(); TRACE_CF("LOOP: entering loop index: ", loopIndex, " signature: ", signature); + m_hasLoop = true; m_localConstants.fill(nullptr); BasicBlock* body = m_proc.addBlock(); BasicBlock* continuation = m_proc.addBlock(); @@ -5060,6 +5592,7 @@ auto OMGIRGenerator::addLoop(BlockSignature&& signature, std::spansetSuccessors(FrequentedBlock(taken, takenFrequency), FrequentedBlock(notTaken, notTakenFrequency)); } + BasicBlock* branch = conditionValue->hasInt() ? nullptr : m_currentBlock; m_currentBlock = taken; TRACE_CF("IF"); result = ControlData(m_proc, origin(), WTF::move(signature), BlockType::If, continuation, notTaken); + result.m_ifBranch = branch; + result.m_callCountAtIf = m_callCount; + enterScope(result, BlockType::If); return { }; } auto OMGIRGenerator::addElse(ControlData& data, std::span currentStack) -> PartialResult { + data.m_thenArmMakesCalls = m_callCount > data.m_callCountAtIf; unifyValuesWithBlock(currentStack, data); m_currentBlock->setSuccessors(data.continuation); m_currentBlock->appendNew(m_proc, Jump, origin()); @@ -5193,6 +5732,7 @@ auto OMGIRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult { ASSERT(data.blockType() == BlockType::If); m_currentBlock = data.special; + leaveScope(data); data.convertIfToBlock(); TRACE_CF("ELSE"); return { }; @@ -5647,6 +6187,25 @@ auto OMGIRGenerator::addInlinedReturn(const auto& returnValues) -> PartialResult return { }; } +static constexpr bool resumablesSetZeroFlag = isX86(); + +// The flags reach the branch of the caller unchanged. +static void emitResumableResult(CCallHelpers& jit, GPRReg result, std::optional constant) +{ +#if CPU(X86_64) + if (constant) + jit.move(CCallHelpers::TrustedImm32(*constant), result); + // A move of zero is emitted as an xor, which sets the flags. + if (constant != 0) + jit.m_assembler.testl_rr(result, result); +#else + UNUSED_PARAM(jit); + UNUSED_PARAM(result); + UNUSED_PARAM(constant); + RELEASE_ASSERT_NOT_REACHED(); +#endif +} + auto OMGIRGenerator::addReturn(const ControlData&, std::span returnValues) -> PartialResult { TRACE_CF("RETURN"); @@ -5659,15 +6218,29 @@ auto OMGIRGenerator::addReturn(const ControlData&, std::span(B3::Void, origin()); - patch->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) { - params.code().emitEpilogue(jit); + bool tearsDownFrame = !emitsShrinkWrappedFrame(); + patch->setRunsWithoutFrame(!tearsDownFrame); + + RELEASE_ASSERT(returnValues.size() >= wasmCallInfo.results.size()); + unsigned offset = returnValues.size() - wasmCallInfo.results.size(); + bool returnsWithFlags = m_isResumable && resumablesSetZeroFlag; + std::optional constantResult; + if (returnsWithFlags && get(returnValues[offset])->hasInt32()) + constantResult = get(returnValues[offset])->asInt32(); + GPRReg resultGPR = returnsWithFlags ? B3::ValueRep(wasmCallInfo.results[0].location).gpr() : InvalidGPRReg; + patch->setGenerator([tearsDownFrame, resultGPR, constantResult] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) { + if (tearsDownFrame) { + params.code().emitEpilogue(jit); + return; + } + if (resultGPR != InvalidGPRReg) + emitResumableResult(jit, resultGPR, constantResult); + jit.ret(); }); patch->effects.terminal = true; patch->clobber(RegisterSet::macroClobberedGPRs()); - RELEASE_ASSERT(returnValues.size() >= wasmCallInfo.results.size()); - unsigned offset = returnValues.size() - wasmCallInfo.results.size(); - for (unsigned i = 0; i < wasmCallInfo.results.size(); ++i) { + for (unsigned i = 0; i < wasmCallInfo.results.size() && !constantResult; ++i) { B3::ValueRep rep = wasmCallInfo.results[i].location; if (rep.isStack()) { m_currentBlock->appendNew(m_proc, B3::Store, Origin(), get(returnValues[offset + i]), framePointer(), safeCast(rep.offsetFromFP())); @@ -5678,6 +6251,8 @@ auto OMGIRGenerator::addReturn(const ControlData&, std::spanappend(passedGlobalAtReturn(), ValueRep::reg(passedGlobalGPR)); m_currentBlock->append(patch); return { }; @@ -5735,7 +6310,7 @@ void OMGIRGenerator::emitFirstStateTest(const ControlData& loop) table->successors() = header->successors(); header->appendNew(m_proc, B3::Branch, dispatch->origin(), header->appendNew(m_proc, B3::Equal, dispatch->origin(), state, constant(Int32, 0))); - header->setSuccessors(table->successor(0), FrequentedBlock(table)); + header->setSuccessors(table->successor(0), FrequentedBlock(table, FrequencyClass::Unlikely)); } auto OMGIRGenerator::addBranch(ControlData& data, ExpressionType condition, std::span returnValues) -> PartialResult @@ -5839,6 +6414,11 @@ auto OMGIRGenerator::endBlock(ControlEntry& entry, std::span en ASSERT(enclosedStack.size() >= returnCount); auto blockResults = enclosedStack.last(returnCount); + // Without a profile, an arm that calls is unlikely if the other arm is empty. + bool hasEmptyElseArm = data.m_ifBranch && m_currentBlock == data.m_ifBranch->successorBlock(1) && !m_currentBlock->size(); + if (m_context.codeForImage && data.m_thenArmMakesCalls && hasEmptyElseArm && data.m_ifBranch->successor(0).isLikely()) + data.m_ifBranch->successor(0).frequency() = FrequencyClass::Unlikely; + if (data.blockType() != BlockType::Loop) unifyValuesWithBlock(blockResults, data); @@ -5846,6 +6426,7 @@ auto OMGIRGenerator::endBlock(ControlEntry& entry, std::span en m_currentBlock->appendNew(m_proc, Jump, origin()); m_currentBlock = data.continuation; + leaveScope(data); if (data.blockType() == BlockType::Loop) emitFirstStateTest(data); @@ -5888,6 +6469,7 @@ auto OMGIRGenerator::addEndToUnreachable(ControlEntry& entry, std::span OMGIRGenerator::createCallConstrainedArgs(BasicBlock* b } -auto OMGIRGenerator::createCallPatchpoint(BasicBlock* block, const RTT& signature, const CallInformation& wasmCalleeInfo, const ArgumentList& tmpArgs) -> CallPatchpointData +auto OMGIRGenerator::createCallPatchpoint(BasicBlock* block, const RTT& signature, const CallInformation& wasmCalleeInfo, const ArgumentList& tmpArgs, Value* passedGlobal) -> CallPatchpointData { - auto returnType = toB3ResultType(&signature); + auto returnType = passedGlobal ? resultTypeWithGlobal(signature) : toB3ResultType(&signature); auto constrainedPatchArgs = createCallConstrainedArgs(block, wasmCalleeInfo, tmpArgs); @@ -5955,6 +6537,8 @@ auto OMGIRGenerator::createCallPatchpoint(BasicBlock* block, const RTT& signatur patchpoint->clobberEarly(RegisterSet::macroClobberedGPRs()); patchpoint->clobberLate(RegisterSet::registersToSaveForJSCall(m_proc.usesSIMD() ? RegisterSet::allRegisters() : RegisterSet::allScalarRegisters())); patchpoint->appendVector(constrainedPatchArgs); + if (passedGlobal) + patchpoint->append(passedGlobal, ValueRep::reg(passedGlobalGPR)); auto handle = preparePatchpointForExceptions(block, patchpoint); const Vector& constrainedResultLocations = wasmCalleeInfo.results; if (returnType != B3::Void) { @@ -5966,6 +6550,8 @@ auto OMGIRGenerator::createCallPatchpoint(BasicBlock* block, const RTT& signatur } else resultConstraints.append(B3::ValueRep(valueLocation.location)); } + if (passedGlobal) + resultConstraints.append(B3::ValueRep::reg(passedGlobalGPR)); patchpoint->resultConstraints = WTF::move(resultConstraints); } block->append(patchpoint); @@ -6739,6 +7325,7 @@ auto OMGIRGenerator::emitDirectCall(unsigned callProfileIndex, FunctionSpaceInde Value* jumpDestination = nullptr; m_makesCalls = true; + ++m_callCount; if (callType == CallType::TailCall) m_makesTailCalls = true; @@ -6751,7 +7338,10 @@ auto OMGIRGenerator::emitDirectCall(unsigned callProfileIndex, FunctionSpaceInde // We need to clobber all potential pinned registers since we might be leaving the instance. // We pessimistically assume we could be calling to something that is bounds checking. // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181 - patchpoint->clobberLate(RegisterSet::wasmPinnedRegisters()); + if (callsRestorePinnedRegisters()) + prepareToRestorePinnedRegisters(patchpoint); + else + patchpoint->clobberLate(RegisterSet::wasmPinnedRegisters()); // emitRestoreInstanceFrameIfNeeded needs two scratches. wasmBaseMemoryPointer is // always pinned so B3 won't allocate it. wasmBoundsCheckingSizeRegister // is only pinned in BoundsChecking mode, so in Signaling mode we need B3 to give us a @@ -6784,6 +7374,8 @@ auto OMGIRGenerator::emitDirectCall(unsigned callProfileIndex, FunctionSpaceInde else { jit.call(params[patchArgsIndex].gpr(), WasmEntryPtrTag); emitWasmCallStackResultsAndSPRestore(jit, params, signature, wasmCalleeInfo); + if (callsRestorePinnedRegisters()) + emitPinnedRegistersRestore(jit, scratchForCaging(params)); } }); }; @@ -6810,14 +7402,21 @@ auto OMGIRGenerator::emitDirectCall(unsigned callProfileIndex, FunctionSpaceInde fillCallResults(patchpoint, signature, results); // The call could have been to another WebAssembly instance, and / or could have modified our Memory. - restoreWebAssemblyGlobalState(m_info.memories, instanceValue(), m_currentBlock); + if (!callsRestorePinnedRegisters()) + restoreWebAssemblyGlobalState(m_info.memories, instanceValue(), m_currentBlock); + reloadPassedGlobal(); return { }; } // isImportedFunctionFromFunctionIndexSpace Vector* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls; + ImageEntry entry = ImageEntry::Primary; + if (m_passedGlobal) + entry = args.size() && get(args[0])->isInt32(0) ? ImageEntry::DirectWithZero : ImageEntry::Direct; + bool calleeIsResumable = m_passedGlobal && isResumable(m_info, m_info.toCodeIndex(functionIndexSpace), *m_context.codeForImage->passedGlobal); + bool omitsArgument = calleeIsResumable && entry == ImageEntry::DirectWithZero; auto emitUnlinkedWasmToWasmCall = [&, this](PatchpointValue* patchpoint, RefPtr handle, RefPtr prepareForCall) -> void { - patchpoint->setGenerator([this, handle, unlinkedWasmToWasmCalls, functionIndexSpace, isTailCallRootCaller, tailCallStackOffsetFromFP, prepareForCall, signature = Ref(signature), wasmCalleeInfo](CCallHelpers& jit, const B3::StackmapGenerationParams& params) { + patchpoint->setGenerator([this, handle, unlinkedWasmToWasmCalls, functionIndexSpace, entry, isTailCallRootCaller, tailCallStackOffsetFromFP, prepareForCall, signature = Ref(signature), wasmCalleeInfo](CCallHelpers& jit, const B3::StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); if (prepareForCall) prepareForCall->run(jit, params); @@ -6835,7 +7434,7 @@ auto OMGIRGenerator::emitDirectCall(unsigned callProfileIndex, FunctionSpaceInde jit.storeWasmCalleeToCalleeCallFrame(CCallHelpers::TrustedImmPtr(CalleeBits::boxNativeCallee(callee.ptr()))); } if (m_context.codeForImage) - emitCallInImage(jit, imageSymbolOf(m_info.toCodeIndex(functionIndexSpace))); + emitCallInImage(jit, imageSymbolOf(m_info.toCodeIndex(functionIndexSpace)), entry); else { auto call = isTailCallRootCaller ? jit.threadSafePatchableNearTailCall() : jit.threadSafePatchableNearCall(); @@ -6843,8 +7442,11 @@ auto OMGIRGenerator::emitDirectCall(unsigned callProfileIndex, FunctionSpaceInde unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall(call), functionIndexSpace }); }); } - if (!isTailCallRootCaller) - emitWasmCallStackResultsAndSPRestore(jit, params, signature, wasmCalleeInfo); + if (!isTailCallRootCaller) { + // A callee in the image changes the stack pointer only by a tail call. + bool needsSPRestore = !m_context.codeForImage || m_info.m_usesTailCalls.loadRelaxed(); + emitWasmCallStackResultsAndSPRestore(jit, params, signature, wasmCalleeInfo, needsSPRestore); + } }); }; @@ -6868,14 +7470,25 @@ auto OMGIRGenerator::emitDirectCall(unsigned callProfileIndex, FunctionSpaceInde // 1. It is not tail-call. So this does not clobber the arguments of this function. // 2. We are not changing instance. Thus, |this| of this function's arguments are the same and OK. - auto [patchpoint, handle, prepareForCall] = createCallPatchpoint(m_currentBlock, signature, wasmCalleeInfo, args); + Value* passedGlobal = m_passedGlobal ? zeroExtendedPassedGlobal() : nullptr; + const ArgumentList noArguments; + auto [patchpoint, handle, prepareForCall] = createCallPatchpoint(m_currentBlock, signature, wasmCalleeInfo, omitsArgument ? noArguments : args, passedGlobal); emitUnlinkedWasmToWasmCall(patchpoint, handle, prepareForCall); + patchpoint->zeroFlagReflectsFirstResult = calleeIsResumable && resumablesSetZeroFlag && !m_info.m_usesTailCalls.loadRelaxed(); // We need to clobber the size register since the IPInt always bounds checks if (useSignalingMemory0() || (m_info.memoryCount() && m_info.memory(0).isShared())) patchpoint->clobberLate(RegisterSet { GPRInfo::wasmBoundsCheckingSizeRegister }); - fillCallResults(patchpoint, signature, results); + if (!passedGlobal) { + fillCallResults(patchpoint, signature, results); + return { }; + } + unsigned returnCount = signature.returnCount(); + for (unsigned i = 0; i < returnCount; ++i) + results.append(m_currentBlock->appendNew(m_proc, origin(), toB3Type(signature.returnType(i)), patchpoint, i)); + Value* returnedGlobal = returnCount ? m_currentBlock->appendNew(m_proc, origin(), Int32, patchpoint, returnCount) : static_cast(patchpoint); + m_currentBlock->appendNew(m_proc, Set, origin(), m_passedGlobal, returnedGlobal); return { }; } @@ -7325,7 +7938,7 @@ std::expected, String> parseAndCompileOMG(Comp Procedure& procedure = *compilationContext.procedure; procedure.setIsWasm(true); - procedure.setName(callee.nameWithHash()); + procedure.setName(compilationContext.codeForImage ? makeString(callee.indexOrName()) : callee.nameWithHash()); if (shouldDumpIRFor(functionIndex + info.importFunctionCount())) procedure.setShouldDumpIR(); diff --git a/Source/JavaScriptCore/wasm/WasmOMGIRGenerator.h b/Source/JavaScriptCore/wasm/WasmOMGIRGenerator.h index d9bd917fb3820..5aca85b1757f4 100644 --- a/Source/JavaScriptCore/wasm/WasmOMGIRGenerator.h +++ b/Source/JavaScriptCore/wasm/WasmOMGIRGenerator.h @@ -49,6 +49,8 @@ namespace JSC::Wasm { class IPIntCallee; class Module; +bool isResumable(const ModuleInformation&, FunctionCodeIndex, uint32_t passedGlobal); + std::expected, String> parseAndCompileOMG(CompilationContext&, IPIntCallee&, OptimizingJITCallee&, const FunctionData&, const RTT&, Vector&, Module&, CalleeGroup&, const ModuleInformation&, MemoryMode, CompilationMode, FunctionCodeIndex functionIndex, uint32_t loopIndexForOSREntry); } // namespace JSC::Wasm diff --git a/Source/JavaScriptCore/wasm/WasmSectionParser.cpp b/Source/JavaScriptCore/wasm/WasmSectionParser.cpp index 7dfa316e3e50d..10551322c9a0b 100644 --- a/Source/JavaScriptCore/wasm/WasmSectionParser.cpp +++ b/Source/JavaScriptCore/wasm/WasmSectionParser.cpp @@ -1351,83 +1351,33 @@ auto SectionParser::parseData() -> PartialResult WASM_PARSER_FAIL_IF(segmentCount > maxDataSegments, "Data section's count is too big "_s, segmentCount, " maximum "_s, maxDataSegments); if (m_info->numberOfDataSegments) WASM_PARSER_FAIL_IF(segmentCount != m_info->numberOfDataSegments.value(), "Data section's count "_s, segmentCount, " is different from Data Count section's count "_s, m_info->numberOfDataSegments.value()); - RELEASE_ASSERT(!m_info->data.capacity()); - WASM_ALLOCATOR_FAIL_IF(!m_info->data.tryReserveInitialCapacity(segmentCount), "can't allocate enough memory for Data section's "_s, segmentCount, " segments"_s); + RELEASE_ASSERT(!m_info->data.sizeInBytes()); + WASM_ALLOCATOR_FAIL_IF(!m_info->data.tryReserve(segmentCount, source().size() - m_offset, m_info->dataSegmentsCount()), "can't allocate enough memory for Data section's "_s, segmentCount, " segments"_s); for (uint32_t segmentNumber = 0; segmentNumber < segmentCount; ++segmentNumber) { - uint32_t memoryIndexOrDataFlag = UINT32_MAX; - WASM_PARSER_FAIL_IF(!parseVarUInt32(memoryIndexOrDataFlag), "can't get "_s, segmentNumber, "th Data segment's flag"_s); - - if (!memoryIndexOrDataFlag) { - const uint32_t memoryIndex = memoryIndexOrDataFlag; - WASM_PARSER_FAIL_IF(memoryIndex >= m_info->memoryCount(), segmentNumber, "th Data segment has index "_s, memoryIndex, " which exceeds the number of Memories "_s, m_info->memoryCount()); - - std::optional initExpr; - if (m_info->memory(memoryIndex).isMemory64()) - WASM_FAIL_IF_HELPER_FAILS(parseI64InitExprForDataSection(initExpr)); - else - WASM_FAIL_IF_HELPER_FAILS(parseI32InitExprForDataSection(initExpr)); - - uint32_t dataByteLength; - WASM_PARSER_FAIL_IF(!parseVarUInt32(dataByteLength), "can't get "_s, segmentNumber, "th Data segment's data byte length"_s); - WASM_PARSER_FAIL_IF(dataByteLength > maxModuleSize, segmentNumber, "th Data segment's data byte length is too big "_s, dataByteLength, " maximum "_s, maxModuleSize); - - auto segment = Segment::tryCreate(*initExpr, dataByteLength, Segment::Kind::Active); - WASM_PARSER_FAIL_IF(!segment, "can't allocate enough memory for "_s, segmentNumber, "th Data segment of size "_s, dataByteLength); - - WASM_PARSER_FAIL_IF(source().size() < dataByteLength || (source().size() - dataByteLength) < m_offset, "can't get data bytes from "_s, segmentNumber, "th Data segment"_s); - - memcpySpan(segment->span(), source().subspan(m_offset, dataByteLength)); - m_offset += dataByteLength; - m_info->data.append(WTF::move(segment)); - continue; - } - - const uint32_t dataFlag = memoryIndexOrDataFlag; - if (dataFlag == 0x01) { - uint32_t dataByteLength; - WASM_PARSER_FAIL_IF(!parseVarUInt32(dataByteLength), "can't get "_s, segmentNumber, "th Data segment's data byte length"_s); - WASM_PARSER_FAIL_IF(dataByteLength > maxModuleSize, segmentNumber, "th Data segment's data byte length is too big "_s, dataByteLength, " maximum "_s, maxModuleSize); - - auto segment = Segment::tryCreate(std::nullopt, dataByteLength, Segment::Kind::Passive); - WASM_PARSER_FAIL_IF(!segment, "can't allocate enough memory for "_s, segmentNumber, "th Data segment of size "_s, dataByteLength); - - WASM_PARSER_FAIL_IF(source().size() < dataByteLength || (source().size() - dataByteLength) < m_offset, "can't get data bytes from "_s, segmentNumber, "th Data segment"_s); - - memcpySpan(segment->span(), source().subspan(m_offset, dataByteLength)); - m_offset += dataByteLength; - m_info->data.append(WTF::move(segment)); - continue; - - } - - if (dataFlag == 0x02) { - uint32_t memoryIndex; - WASM_PARSER_FAIL_IF(!parseVarUInt32(memoryIndex), "can't get "_s, segmentNumber, "th Data segment's index"_s); - WASM_PARSER_FAIL_IF(memoryIndex >= m_info->memoryCount(), segmentNumber, "th Data segment has index "_s, memoryIndex, " which exceeds the number of Memories "_s, m_info->memoryCount()); - - std::optional initExpr; - if (m_info->memory(memoryIndex).isMemory64()) - WASM_FAIL_IF_HELPER_FAILS(parseI64InitExprForDataSection(initExpr)); + enum DataFlag : uint32_t { Active, Passive, ActiveWithMemoryIndex }; + uint32_t dataFlag; + WASM_PARSER_FAIL_IF(!parseVarUInt32(dataFlag), "can't get "_s, segmentNumber, "th Data segment's flag"_s); + WASM_PARSER_FAIL_IF(dataFlag > ActiveWithMemoryIndex, "unknown "_s, segmentNumber, "th Data segment's flag"_s); + + Segment segment { dataFlag == Passive ? Segment::Kind::Passive : Segment::Kind::Active, 0, std::nullopt, { } }; + if (dataFlag == ActiveWithMemoryIndex) + WASM_PARSER_FAIL_IF(!parseVarUInt32(segment.memoryIndex), "can't get "_s, segmentNumber, "th Data segment's index"_s); + if (segment.isActive()) { + WASM_PARSER_FAIL_IF(segment.memoryIndex >= m_info->memoryCount(), segmentNumber, "th Data segment has index "_s, segment.memoryIndex, " which exceeds the number of Memories "_s, m_info->memoryCount()); + if (m_info->memory(segment.memoryIndex).isMemory64()) + WASM_FAIL_IF_HELPER_FAILS(parseI64InitExprForDataSection(segment.offsetIfActive)); else - WASM_FAIL_IF_HELPER_FAILS(parseI32InitExprForDataSection(initExpr)); - - uint32_t dataByteLength; - WASM_PARSER_FAIL_IF(!parseVarUInt32(dataByteLength), "can't get "_s, segmentNumber, "th Data segment's data byte length"_s); - WASM_PARSER_FAIL_IF(dataByteLength > maxModuleSize, segmentNumber, "th Data segment's data byte length is too big "_s, dataByteLength, " maximum "_s, maxModuleSize); - - auto segment = Segment::tryCreate(*initExpr, dataByteLength, Segment::Kind::Active, memoryIndex); - WASM_PARSER_FAIL_IF(!segment, "can't allocate enough memory for "_s, segmentNumber, "th Data segment of size "_s, dataByteLength); - - WASM_PARSER_FAIL_IF(source().size() < dataByteLength || (source().size() - dataByteLength) < m_offset, "can't get data bytes from "_s, segmentNumber, "th Data segment"_s); - - memcpySpan(segment->span(), source().subspan(m_offset, dataByteLength)); - m_offset += dataByteLength; - m_info->data.append(WTF::move(segment)); - continue; + WASM_FAIL_IF_HELPER_FAILS(parseI32InitExprForDataSection(segment.offsetIfActive)); } - WASM_PARSER_FAIL_IF(true, "unknown "_s, segmentNumber, "th Data segment's flag"_s); + uint32_t dataByteLength; + WASM_PARSER_FAIL_IF(!parseVarUInt32(dataByteLength), "can't get "_s, segmentNumber, "th Data segment's data byte length"_s); + WASM_PARSER_FAIL_IF(dataByteLength > maxModuleSize, segmentNumber, "th Data segment's data byte length is too big "_s, dataByteLength, " maximum "_s, maxModuleSize); + WASM_PARSER_FAIL_IF(source().size() < dataByteLength || (source().size() - dataByteLength) < m_offset, "can't get data bytes from "_s, segmentNumber, "th Data segment"_s); + segment.bytes = source().subspan(m_offset, dataByteLength); + m_offset += dataByteLength; + m_info->data.append(segment); } return { }; diff --git a/Source/JavaScriptCore/wasm/js/JSWebAssemblyInstance.cpp b/Source/JavaScriptCore/wasm/js/JSWebAssemblyInstance.cpp index 722d83451f9dd..7a98d579b96c1 100644 --- a/Source/JavaScriptCore/wasm/js/JSWebAssemblyInstance.cpp +++ b/Source/JavaScriptCore/wasm/js/JSWebAssemblyInstance.cpp @@ -133,8 +133,7 @@ JSWebAssemblyInstance::JSWebAssemblyInstance(VM& vm, Structure* structure, JSWeb } for (unsigned dataSegmentIndex = 0; dataSegmentIndex < m_moduleInformation->dataSegmentsCount(); ++dataSegmentIndex) { - const auto& dataSegment = m_moduleInformation->data[dataSegmentIndex]; - if (dataSegment->isPassive()) + if (m_moduleInformation->data[dataSegmentIndex].isPassive()) m_passiveDataSegments.quickSet(dataSegmentIndex); } @@ -622,12 +621,12 @@ bool JSWebAssemblyInstance::memoryInit(uint64_t dstAddress, uint32_t srcAddress, if (sumOverflows(srcAddress, length)) return false; - auto& segment = module().moduleInformation().data[dataSegmentIndex]; - const uint32_t segmentSizeInBytes = m_passiveDataSegments.quickGet(dataSegmentIndex) ? segment->sizeInBytes() : 0U; + Wasm::Segment segment = module().moduleInformation().data[dataSegmentIndex]; + const uint32_t segmentSizeInBytes = m_passiveDataSegments.quickGet(dataSegmentIndex) ? segment.bytes.size() : 0U; if (srcAddress + length > segmentSizeInBytes) return false; - const uint8_t* segmentData = !length ? nullptr : &segment->byte(srcAddress); + const uint8_t* segmentData = !length ? nullptr : segment.bytes.subspan(srcAddress).data(); ASSERT(memoryIndex < m_moduleInformation->memoryCount()); return memory(memoryIndex)->memory().init(dstAddress, segmentData, length); @@ -717,8 +716,8 @@ bool JSWebAssemblyInstance::copyDataSegment(JSWebAssemblyArray* array, uint32_t // Fail if the data segment index is out of bounds RELEASE_ASSERT(segmentIndex < module().moduleInformation().dataSegmentsCount()); // Otherwise, get the `segmentIndex`th data segment - auto& segment = module().moduleInformation().data[segmentIndex]; - const uint32_t segmentSizeInBytes = m_passiveDataSegments.quickGet(segmentIndex) ? segment->sizeInBytes() : 0U; + Wasm::Segment segment = module().moduleInformation().data[segmentIndex]; + const uint32_t segmentSizeInBytes = m_passiveDataSegments.quickGet(segmentIndex) ? segment.bytes.size() : 0U; // Caller checks that the (offset + lengthInBytes) calculation doesn't overflow if ((offset + lengthInBytes) > segmentSizeInBytes) { @@ -729,7 +728,7 @@ bool JSWebAssemblyInstance::copyDataSegment(JSWebAssemblyArray* array, uint32_t if (!lengthInBytes) return true; // Cast the data segment to a pointer - const uint8_t* segmentData = &segment->byte(offset); + const uint8_t* segmentData = segment.bytes.subspan(offset).data(); // Copy the data from the segment into the out param vector if (array->elementsAreRefTypes()) { diff --git a/Source/JavaScriptCore/wasm/js/WebAssemblyModuleRecord.cpp b/Source/JavaScriptCore/wasm/js/WebAssemblyModuleRecord.cpp index 6409c173b0b06..698885032f621 100644 --- a/Source/JavaScriptCore/wasm/js/WebAssemblyModuleRecord.cpp +++ b/Source/JavaScriptCore/wasm/js/WebAssemblyModuleRecord.cpp @@ -883,8 +883,8 @@ NEVER_INLINE JSValue WebAssemblyModuleRecord::evaluate(JSGlobalObject* globalObj Wasm::Module& module = m_instance->module(); const Wasm::ModuleInformation& moduleInformation = module.moduleInformation(); - const Vector>& data = moduleInformation.data; - + const Wasm::Segments& data = moduleInformation.data; + std::optional exception; auto forEachActiveElement = [&] (auto fn) { @@ -930,29 +930,30 @@ NEVER_INLINE JSValue WebAssemblyModuleRecord::evaluate(JSGlobalObject* globalObj }; auto forEachActiveDataSegment = [&] (auto fn) { - for (const auto& segment : data) { - if (!segment->isActive()) + for (size_t place = 0; place < data.sizeInBytes();) { + Wasm::Segment segment = data.readAt(place); + if (!segment.isActive()) continue; - auto& wasmMemory = m_instance->memory(segment->memoryIndex())->memory(); + auto& wasmMemory = m_instance->memory(segment.memoryIndex)->memory(); uint8_t* memory = static_cast(wasmMemory.basePointer()); uint64_t sizeInBytes = wasmMemory.size(); - const bool isMemory64 = moduleInformation.memory(segment->memoryIndex()).isMemory64(); + const bool isMemory64 = moduleInformation.memory(segment.memoryIndex).isMemory64(); uint64_t offset = 0; - if (segment->offsetIfActive()->isGlobalImport()) { + if (segment.offsetIfActive->isGlobalImport()) { if (isMemory64) - offset = static_cast(m_instance->loadI64Global(segment->offsetIfActive()->globalImportIndex())); + offset = static_cast(m_instance->loadI64Global(segment.offsetIfActive->globalImportIndex())); else - offset = static_cast(m_instance->loadI32Global(segment->offsetIfActive()->globalImportIndex())); - } else if (segment->offsetIfActive()->isConst()) { + offset = static_cast(m_instance->loadI32Global(segment.offsetIfActive->globalImportIndex())); + } else if (segment.offsetIfActive->isConst()) { if (isMemory64) - offset = segment->offsetIfActive()->constValue(); + offset = segment.offsetIfActive->constValue(); else - offset = static_cast(segment->offsetIfActive()->constValue()); + offset = static_cast(segment.offsetIfActive->constValue()); } else { uint64_t result; - evaluateConstantExpression(globalObject, segment->offsetIfActive()->constantExpressionIndex(), result); + evaluateConstantExpression(globalObject, segment.offsetIfActive->constantExpressionIndex(), result); RETURN_IF_EXCEPTION(scope, void()); offset = isMemory64 ? result : static_cast(result); } @@ -982,20 +983,20 @@ NEVER_INLINE JSValue WebAssemblyModuleRecord::evaluate(JSGlobalObject* globalObj return exception.value(); // Validation of all segment ranges comes before all Table and Memory initialization. - forEachActiveDataSegment([&](uint8_t* memory, uint64_t sizeInBytes, const std::unique_ptr& segment, uint64_t offset) { - if (sizeInBytes < segment->sizeInBytes()) [[unlikely]] { - exception = dataSegmentFail(globalObject, vm, scope, sizeInBytes, static_cast(segment->sizeInBytes()), offset, ", segment is too big"_s); + forEachActiveDataSegment([&](uint8_t* memory, uint64_t sizeInBytes, const Wasm::Segment& segment, uint64_t offset) { + if (sizeInBytes < segment.bytes.size()) [[unlikely]] { + exception = dataSegmentFail(globalObject, vm, scope, sizeInBytes, static_cast(segment.bytes.size()), offset, ", segment is too big"_s); return IterationStatus::Done; } - if (offset > sizeInBytes - segment->sizeInBytes()) [[unlikely]] { - exception = dataSegmentFail(globalObject, vm, scope, sizeInBytes, static_cast(segment->sizeInBytes()), offset, ", segment writes outside of memory"_s); + if (offset > sizeInBytes - segment.bytes.size()) [[unlikely]] { + exception = dataSegmentFail(globalObject, vm, scope, sizeInBytes, static_cast(segment.bytes.size()), offset, ", segment writes outside of memory"_s); return IterationStatus::Done; } // Empty segments are valid, but only if memory isn't present, which would be undefined behavior in memcpy. - if (segment->sizeInBytes()) { + if (!segment.bytes.empty()) { RELEASE_ASSERT(memory); - memcpy(memory + offset, segment->span().data(), segment->sizeInBytes()); + memcpy(memory + offset, segment.bytes.data(), segment.bytes.size()); } return IterationStatus::Continue; }); diff --git a/Source/JavaScriptCore/yarr/YarrJIT.cpp b/Source/JavaScriptCore/yarr/YarrJIT.cpp index 2e53c3cbcc06e..42d888154b80f 100644 --- a/Source/JavaScriptCore/yarr/YarrJIT.cpp +++ b/Source/JavaScriptCore/yarr/YarrJIT.cpp @@ -11017,6 +11017,8 @@ class YarrGenerator final : public YarrJITInfo { struct SearchForPairOfUnits { unsigned firstPlace; std::array units; + char16_t alternateUnit; + unsigned rarity; SharedRoutine::FindFirstPairOfUnits routine; }; @@ -11025,7 +11027,8 @@ class YarrGenerator final : public YarrJITInfo { struct Place { unsigned place; char16_t unit; - bool foldsCase; + char16_t alternateUnit; + PositionUnits units; unsigned rarity; }; if (m_decodeSurrogatePairs || m_direction != Forward || m_executionMode == ExecutionMode::InlineTest) @@ -11037,31 +11040,57 @@ class YarrGenerator final : public YarrJITInfo { auto [map, exact] = info.createCandidateBitmap(place, place + 1); if (!exact.isValid() || exact.isEmpty() || exact.at(0) > 0xffff) continue; - Place found { place, static_cast(exact.at(0)), false, rarityOfUnits({ map.storage()[0], map.storage()[1] }, false) }; + Place found { place, static_cast(exact.at(0)), 0, PositionUnits::One, rarityOfUnits({ map.storage()[0], map.storage()[1] }, false) }; if (exact.size() == 2) { - if (!isASCIIAlpha(exact.at(0)) || (exact.at(0) ^ exact.at(1)) != 0x20) + if (exact.at(1) > 0xffff) continue; - found.unit = toASCIILower(found.unit); - found.foldsCase = true; + if (isASCIIAlpha(exact.at(0)) && (exact.at(0) ^ exact.at(1)) == 0x20) { + found.unit = toASCIILower(found.unit); + found.units = PositionUnits::IgnoreCase; + } else { + found.alternateUnit = static_cast(exact.at(1)); + found.units = PositionUnits::Two; + } } places.append(found); } - auto rarest = std::ranges::max_element(places, { }, &Place::rarity); - if (rarest == places.end()) - return std::nullopt; - auto next = places.end(); - for (auto place = places.begin(); place != places.end(); ++place) { - if (place->unit != rarest->unit && (next == places.end() || place->rarity > next->rarity)) - next = place; + // The routine has one extra vector: for the second unit of one position, or for the ASCII case bit. + auto matchesTwoUnits = [](const Place& place) { + return place.units == PositionUnits::Two; + }; + auto canPair = [&](const Place& a, const Place& b) { + if (!matchesTwoUnits(a) && !matchesTwoUnits(b)) + return a.unit != b.unit; + const Place& two = matchesTwoUnits(a) ? a : b; + const Place& one = matchesTwoUnits(a) ? b : a; + return one.units == PositionUnits::One && one.unit != two.unit && one.unit != two.alternateUnit; + }; + const Place* rarest = nullptr; + const Place* next = nullptr; + unsigned bestRank = 0; + for (const Place& place : places) { + for (const Place& other : places) { + if (&place == &other || place.rarity < other.rarity || !canPair(place, other)) + continue; + unsigned rank = 2 * (place.rarity + other.rarity) + (matchesTwoUnits(place) || matchesTwoUnits(other) ? 1 : 2); + if (rank > bestRank) { + rarest = &place; + next = &other; + bestRank = rank; + } + } } - if (next == places.end()) + if (!rarest) return std::nullopt; - unsigned costOfSearch = (m_charSize == CharSize::Char8 ? 6 : 8) + (320u >> std::min(rarest->rarity + next->rarity, 31u)); + bool hasTwoUnitPosition = matchesTwoUnits(*rarest) || matchesTwoUnits(*next); + unsigned rarity = rarest->rarity + next->rarity; + unsigned costOfSearch = (m_charSize == CharSize::Char8 ? 6 : 8) + (hasTwoUnitPosition ? 2 : 0) + (320u >> std::min(rarity, 31u)); if (costOfSearch >= costOtherwise) return std::nullopt; const Place& first = rarest->place < next->place ? *rarest : *next; const Place& second = rarest->place < next->place ? *next : *rarest; - SearchForPairOfUnits search { first.place, { first.unit, second.unit }, { m_charSize, static_cast(second.place - first.place), first.foldsCase, second.foldsCase } }; + char16_t alternateUnit = matchesTwoUnits(first) ? first.alternateUnit : second.alternateUnit; + SearchForPairOfUnits search { first.place, { first.unit, second.unit }, alternateUnit, rarity, { m_charSize, static_cast(second.place - first.place), first.units, second.units } }; if (!canGenerateSharedRoutine(SharedRoutine { search.routine })) return std::nullopt; return search; @@ -11092,6 +11121,8 @@ class YarrGenerator final : public YarrJITInfo { dataLogLnIf(Options::verboseRegExpCompilation(), "Searches for the units at places ", search.firstPlace, " and ", search.firstPlace + search.routine.distance, " of a match in ", m_charSize == CharSize::Char8 ? "narrow" : "wide", " texts"); m_jit.sub32(MacroAssembler::Imm32(claimed), m_regs.index); moveConstant(search.units[0] * repeats | search.units[1] * repeats << 32, m_regs.regT1); + if (search.routine.first == PositionUnits::Two || search.routine.second == PositionUnits::Two) + moveConstant(search.alternateUnit * repeats, m_regs.regT0); callRoutine(SharedRoutine::FindFirstPairOfUnits { search.routine }); m_jit.add32(MacroAssembler::Imm32(claimed), m_regs.index); matched.append(m_jit.branch32(MacroAssembler::BelowOrEqual, m_regs.index, m_regs.length)); @@ -11108,6 +11139,17 @@ class YarrGenerator final : public YarrJITInfo { return rarityOfUnits({ map.storage()[0], map.storage()[1] }, false) + std::bit_width(end - begin) - 1; } + bool hasRareUnitPair(const YarrOp& op, std::tuple range) const + { + auto [begin, end] = range; + if (isARM64() && m_charSize == CharSize::Char8) + return false; + if (rarityOfStops(*op.m_bmInfo, range) < leastRarityOfStartWorthSearchingFor || (end - begin == 1 && op.m_startPositionSearch) || startPositionSearchIsCheaperThanStride(op, end - begin)) + return false; + auto search = planSearchForPairOfUnits(*op.m_bmInfo, costOfStride(*op.m_bmInfo, range)); + return search && search->rarity >= 2 * leastRarityWorthSearchingFor; + } + bool hasSelectiveStartSearch() { for (auto& op : m_ops) { @@ -11117,7 +11159,7 @@ class YarrGenerator final : public YarrJITInfo { return true; if (op.m_bmInfo) { if (auto range = op.m_bmInfo->findWorthwhileCharacterSequenceForLookahead(m_sampler, canUseVectorScan())) - return rarityOfStops(*op.m_bmInfo, *range) >= leastRarityWorthSearchingFor; + return rarityOfStops(*op.m_bmInfo, *range) >= leastRarityWorthSearchingFor || hasRareUnitPair(op, *range); } return op.m_startPositionSearch && rarityOfUnits(op.m_startPositionSearch->asciiMembers, op.m_startPositionSearch->matchesAllNonASCII) >= leastRarityWorthSearchingFor; } diff --git a/Source/JavaScriptCore/yarr/YarrSharedRoutines.cpp b/Source/JavaScriptCore/yarr/YarrSharedRoutines.cpp index 520a5c5eb6dec..212160f5ab865 100644 --- a/Source/JavaScriptCore/yarr/YarrSharedRoutines.cpp +++ b/Source/JavaScriptCore/yarr/YarrSharedRoutines.cpp @@ -73,7 +73,7 @@ void add(Hasher& hasher, const SharedRoutine& routine) add(hasher, payload.charSize); }, [&](const SharedRoutine::FindFirstPairOfUnits& payload) { - add(hasher, payload.charSize, payload.distance, payload.foldsCaseOfFirst, payload.foldsCaseOfSecond); + add(hasher, payload.charSize, payload.distance, payload.first, payload.second); }, [&](const SharedRoutine::CanonicalEquivalence& payload) { add(hasher, payload.mode); @@ -947,7 +947,8 @@ struct FirstPairOfUnitsSearchGenerator { static constexpr MacroAssembler::RegisterID limit = YarrJITDefaultRegisters::regT3; static constexpr FPRReg firstUnits = YarrJITDefaultRegisters::vectorTemp0; static constexpr FPRReg secondUnits = YarrJITDefaultRegisters::vectorTemp1; - static constexpr FPRReg caseBits = YarrJITDefaultRegisters::vectorTemp2; + // The ASCII case bit in every lane, or the second unit of the position that matches two. + static constexpr FPRReg caseBitOrAlternate = YarrJITDefaultRegisters::vectorTemp2; static constexpr FPRReg lowerOfFirst = YarrJITDefaultRegisters::vectorInput0; static constexpr FPRReg upperOfFirst = YarrJITDefaultRegisters::vectorInput1; static constexpr FPRReg lowerOfSecond = YarrJITDefaultRegisters::vectorScratch0; @@ -960,7 +961,12 @@ struct FirstPairOfUnitsSearchGenerator { void loadVectorConstants() { - const bool foldsCase = payload.foldsCaseOfFirst || payload.foldsCaseOfSecond; + auto isOneOf = [&](PositionUnits units) { + return payload.first == units || payload.second == units; + }; + const bool foldsCase = isOneOf(PositionUnits::IgnoreCase); + const bool hasAlternateUnit = isOneOf(PositionUnits::Two); + RELEASE_ASSERT(!hasAlternateUnit || (!foldsCase && payload.first != payload.second)); if (foldsCase) jit.move(MacroAssembler::TrustedImm32(isWide ? 0x00200020 : 0x20202020), found); #if CPU(X86_64) @@ -968,68 +974,80 @@ struct FirstPairOfUnitsSearchGenerator { assembler.movq_rr(bothUnits, secondUnits); assembler.pshufd_i8rr(0x00, secondUnits, firstUnits); assembler.pshufd_i8rr(0x55, secondUnits, secondUnits); - if (foldsCase) { - assembler.movd_rr(found, caseBits); - assembler.pshufd_i8rr(0x00, caseBits, caseBits); + if (foldsCase || hasAlternateUnit) { + assembler.movd_rr(found, caseBitOrAlternate); + assembler.pshufd_i8rr(0x00, caseBitOrAlternate, caseBitOrAlternate); } #else jit.vectorSplatInt32(bothUnits, firstUnits); jit.urshift64(bothUnits, MacroAssembler::TrustedImm32(32), bothUnits); jit.vectorSplatInt32(bothUnits, secondUnits); - if (foldsCase) - jit.vectorSplatInt32(found, caseBits); + if (foldsCase || hasAlternateUnit) + jit.vectorSplatInt32(found, caseBitOrAlternate); #endif } MacroAssembler::Jump testVector() { emitVerifySubjectRead(jit, subjectAt(0), (unitsInStep + payload.distance) << unitShift, regs.input, regs.length, unitShift, __LINE__); - struct Half { - FPRReg vector; - int32_t units; - FPRReg sought; - bool foldsCase; - }; - const Half halves[] = { - { lowerOfFirst, 0, firstUnits, payload.foldsCaseOfFirst }, - { lowerOfSecond, payload.distance, secondUnits, payload.foldsCaseOfSecond }, - { upperOfFirst, 8, firstUnits, payload.foldsCaseOfFirst }, - { upperOfSecond, payload.distance + 8, secondUnits, payload.foldsCaseOfSecond }, - }; - const std::span used = std::span(halves).first(isWide ? 4 : 2); #if CPU(X86_64) auto& assembler = jit.m_assembler; - for (const Half& half : used) { - assembler.movdqu_mr(half.units << unitShift, regs.input, regs.index, isWide ? MacroAssembler::TimesTwo : MacroAssembler::TimesOne, half.vector); - if (half.foldsCase) - assembler.por_rr(caseBits, half.vector); + auto compare = [&](FPRReg sought, FPRReg vector) { if (isWide) - assembler.pcmpeqw_rr(half.sought, half.vector); + assembler.pcmpeqw_rr(sought, vector); else - assembler.pcmpeqb_rr(half.sought, half.vector); - } - assembler.pand_rr(lowerOfSecond, lowerOfFirst); - if (isWide) { - assembler.pand_rr(upperOfSecond, upperOfFirst); - assembler.packsswb_rr(upperOfFirst, lowerOfFirst); - } - assembler.pmovmskb_rr(lowerOfFirst, found); - return jit.branchTest32(MacroAssembler::NonZero, found); + assembler.pcmpeqb_rr(sought, vector); + }; + auto loadAndCompare = [&](int32_t units, PositionUnits held, FPRReg sought, FPRReg vector, FPRReg scratch) { + assembler.movdqu_mr(units << unitShift, regs.input, regs.index, isWide ? MacroAssembler::TimesTwo : MacroAssembler::TimesOne, vector); + if (held == PositionUnits::IgnoreCase) + assembler.por_rr(caseBitOrAlternate, vector); + if (held == PositionUnits::Two) { + assembler.movaps_rr(vector, scratch); + compare(caseBitOrAlternate, scratch); + } + compare(sought, vector); + if (held == PositionUnits::Two) + assembler.por_rr(scratch, vector); + }; + auto join = [&](FPRReg vector, FPRReg into) { + assembler.pand_rr(vector, into); + }; #else constexpr SIMDInfo bytes { SIMDLane::i8x16, SIMDSignMode::Unsigned }; constexpr SIMDInfo words16 { SIMDLane::i16x8, SIMDSignMode::Unsigned }; constexpr SIMDInfo whole { SIMDLane::v128, SIMDSignMode::None }; - for (const Half& half : used) { - jit.loadVector(subjectAt(half.units), half.vector); - if (half.foldsCase) - jit.vectorOr(whole, half.vector, caseBits, half.vector); - jit.compareIntegerVector(MacroAssembler::Equal, isWide ? words16 : bytes, half.vector, half.sought, half.vector); - } - jit.vectorAnd(whole, lowerOfFirst, lowerOfSecond, lowerOfFirst); + auto loadAndCompare = [&](int32_t units, PositionUnits held, FPRReg sought, FPRReg vector, FPRReg scratch) { + jit.loadVector(subjectAt(units), vector); + if (held == PositionUnits::IgnoreCase) + jit.vectorOr(whole, vector, caseBitOrAlternate, vector); + if (held == PositionUnits::Two) + jit.compareIntegerVector(MacroAssembler::Equal, isWide ? words16 : bytes, vector, caseBitOrAlternate, scratch); + jit.compareIntegerVector(MacroAssembler::Equal, isWide ? words16 : bytes, vector, sought, vector); + if (held == PositionUnits::Two) + jit.vectorOr(whole, vector, scratch, vector); + }; + auto join = [&](FPRReg vector, FPRReg into) { + jit.vectorAnd(whole, into, vector, into); + }; +#endif + loadAndCompare(0, payload.first, firstUnits, lowerOfFirst, lowerOfSecond); + if (isWide) + loadAndCompare(8, payload.first, firstUnits, upperOfFirst, upperOfSecond); + loadAndCompare(payload.distance, payload.second, secondUnits, lowerOfSecond, upperOfSecond); + join(lowerOfSecond, lowerOfFirst); if (isWide) { - jit.vectorAnd(whole, upperOfFirst, upperOfSecond, upperOfFirst); - jit.vectorUnzipEven(bytes, lowerOfFirst, upperOfFirst, lowerOfFirst); + loadAndCompare(payload.distance + 8, payload.second, secondUnits, upperOfSecond, lowerOfSecond); + join(upperOfSecond, upperOfFirst); } +#if CPU(X86_64) + if (isWide) + assembler.packsswb_rr(upperOfFirst, lowerOfFirst); + assembler.pmovmskb_rr(lowerOfFirst, found); + return jit.branchTest32(MacroAssembler::NonZero, found); +#else + if (isWide) + jit.vectorUnzipEven(bytes, lowerOfFirst, upperOfFirst, lowerOfFirst); jit.vectorShrnInt8(lowerOfFirst, 4, lowerOfFirst); jit.moveDoubleTo64(lowerOfFirst, found); return jit.branchTest64(MacroAssembler::NonZero, found); diff --git a/Source/JavaScriptCore/yarr/YarrSharedRoutines.h b/Source/JavaScriptCore/yarr/YarrSharedRoutines.h index ba30f4fe9fddf..9a0b09ff2d74a 100644 --- a/Source/JavaScriptCore/yarr/YarrSharedRoutines.h +++ b/Source/JavaScriptCore/yarr/YarrSharedRoutines.h @@ -90,6 +90,7 @@ static_assert(areCanonicallyEquivalentCharArgReg == GPRInfo::returnValueGPR); enum class SurrogatePairAdvance : uint8_t { ByCaller, ClaimsSecondUnit, ClaimsSecondUnitBeforeCallerAdvances }; enum class InputCheck : uint8_t { DoneByCaller, FailsIfIndexIsAboveLength, FailsIfIndexIsLength }; enum class ClassWords : uint8_t { Low, High, Both }; +enum class PositionUnits : uint8_t { One, IgnoreCase, Two }; #if CPU(X86_64) inline constexpr GPRReg classTableRegister = MacroAssembler::s_scratchRegister; @@ -157,8 +158,8 @@ struct SharedRoutine { struct FindFirstPairOfUnits { CharSize charSize; uint8_t distance; - bool foldsCaseOfFirst; - bool foldsCaseOfSecond; + PositionUnits first; + PositionUnits second; friend bool operator==(const FindFirstPairOfUnits&, const FindFirstPairOfUnits&) = default; }; From b7288b68eb3c35964e0563f75b64771c0849f284 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 10 Nov 2026 05:02:50 -0800 Subject: [PATCH 02/10] AOT: fix the macOS build and a build stop, plainer names outside aot/ On 7ddc7613. 23 files, +106 -118: Source/JavaScriptCore/aot 2 files +2 -14 . Source/JavaScriptCore/assembler 2 files +25 -25 . Source/JavaScriptCore/b3 11 files +33 -33 . Source/JavaScriptCore/wasm 4 files +17 -17 . Source/JavaScriptCore/runtime 4 files +29 -29. - macOS did not compile aot/machine/AOTMachineEncoderMoves.cpp: `isRepresentableAs()` of a uint64_t is ambiguous where size_t is another type. The calls pass a size_t. - A constant list of zeros that a loop walks and reads under a computed index stopped the build with an internal compiler error (BuildImage). The change that gave such lists a storage of their own, new in 7ddc7613, is out again. - Names of helpers, members, locals and native tests, and comments, in the files that oven-sh/WebKit#1028 changes outside aot/: JSONObject.cpp, the data segments in WasmFormat.h, testmasm.cpp, the b3 and Air tests, and single lines elsewhere. No logic changes with the names. --- .../aot/machine/AOTMachineEncoderMoves.cpp | 4 +- .../aot/walk/AOTWalkChoices.cpp | 12 ----- .../assembler/MacroAssemblerX86_64.h | 2 +- Source/JavaScriptCore/assembler/testmasm.cpp | 48 ++++++++--------- Source/JavaScriptCore/b3/B3LowerToAir.cpp | 4 +- Source/JavaScriptCore/b3/B3ThreadJumps.cpp | 6 +-- .../b3/air/AirAllocateRegistersByGreedy.cpp | 2 +- .../b3/air/AirCheckSpillSlots.h | 2 +- Source/JavaScriptCore/b3/air/AirPolicy.h | 2 +- .../b3/air/AirReuseConditionFlags.cpp | 4 +- .../b3/air/AirShrinkWrapFrame.cpp | 2 +- .../JavaScriptCore/b3/air/testair_bun_flags.h | 8 +-- .../JavaScriptCore/b3/testb3_bun_ranges.cpp | 12 ++--- .../b3/testb3_bun_reductions.cpp | 20 +++---- .../b3/testb3_bun_thread_jumps.cpp | 4 +- Source/JavaScriptCore/runtime/JSFunction.cpp | 2 +- .../JavaScriptCore/runtime/JSModuleLoader.cpp | 2 +- Source/JavaScriptCore/runtime/JSONObject.cpp | 52 +++++++++---------- .../runtime/ObjectConstructor.cpp | 2 +- Source/JavaScriptCore/wasm/WasmFormat.cpp | 4 +- Source/JavaScriptCore/wasm/WasmFormat.h | 10 ++-- Source/JavaScriptCore/wasm/WasmIPIntPlan.cpp | 8 +-- .../wasm/WasmOMGIRGenerator.cpp | 12 ++--- 23 files changed, 106 insertions(+), 118 deletions(-) diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineEncoderMoves.cpp b/Source/JavaScriptCore/aot/machine/AOTMachineEncoderMoves.cpp index 7674ff7459408..0b06ec3995ff6 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineEncoderMoves.cpp +++ b/Source/JavaScriptCore/aot/machine/AOTMachineEncoderMoves.cpp @@ -86,7 +86,7 @@ static bool tryMaterializeBitsBesidePinnedRegister(CCallHelpers& jit, uint64_t b static void materializeBitsInGPR(CCallHelpers& jit, uint64_t bits, Width width, GPRReg destination) { - bool hasOnlyLowHalf = !encodedMoveIsWide(width) || isRepresentableAs(bits); + bool hasOnlyLowHalf = !encodedMoveIsWide(width) || isRepresentableAs(static_cast(bits)); if (bits && hasOnlyLowHalf && tryMoveSmallConstant(jit, static_cast(static_cast(bits)), destination)) return; if (!encodedMoveIsWide(width)) { @@ -111,7 +111,7 @@ static void materializeBitsInGPR(CCallHelpers& jit, uint64_t bits, Width width, #if CPU(X86_64) static bool isLoadedIntoGPR(uint64_t bits) { - if (isRepresentableAs(bits) || isRepresentableAs(static_cast(bits))) + if (isRepresentableAs(static_cast(bits)) || isRepresentableAs(static_cast(bits))) return false; constexpr unsigned shiftOfTag = 48; constexpr uint64_t lowBytesOfLongDouble = (1ull << 40) - 1; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkChoices.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkChoices.cpp index f684a56769528..317bbf2dcf733 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkChoices.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkChoices.cpp @@ -878,14 +878,6 @@ static bool sharesStorageOfArrayInChoices(Walker& walker, const DataNode::Array& return std::ranges::all_of(array.elements, canLieInSharedStorageInChoices); } -static bool isBornToBeFilledInChoices(const DataNode::Array& array) -{ - return array.elements.size() > 1 && std::ranges::all_of(array.elements, [](const DataNode& element) { - const auto* number = element.tryAs(); - return number && !number->value; - }); -} - static void chooseEncodingOfDataInChoices(Walker& walker, Draft& data) { RootOfConstant whereRootEnds = walker.constantEndsWhereItLeaves(data.inputValueGiven, data.data()) ? RootOfConstant::EndsWhereItLeaves : RootOfConstant::IsAsked; @@ -896,10 +888,6 @@ static void chooseEncodingOfDataInChoices(Walker& walker, Draft& data) SetForScope makingObjectsOfConstant = walker.makingObjectsOfConstant(); const DataNode& root = walker.program().at(data.data()).root; if (const auto* array = root.tryAs(); array && sharesStorageOfArrayInChoices(walker, *array)) { - if (array->elements.size() <= maxNodesOfExpandedData && isBornToBeFilledInChoices(*array)) { - becomeNodeOfDataInChoices(walker, data, root); - return; - } bool isStoredIntoInLoop = data.inputValueGiven && walker.facts().summaries().at(walker.currentFunction()).constantListIsStoredIntoInLoop(data.inputValueGiven) && surveyDataInChoices(walker, data.data()).tableOfImageHoldsIt; data.setEncoding(isStoredIntoInLoop ? DataEncoding::TreeWithOwnStorage : DataEncoding::CopyOnWriteArray); return; diff --git a/Source/JavaScriptCore/assembler/MacroAssemblerX86_64.h b/Source/JavaScriptCore/assembler/MacroAssemblerX86_64.h index b2fa92d2d1670..888d3291607be 100644 --- a/Source/JavaScriptCore/assembler/MacroAssemblerX86_64.h +++ b/Source/JavaScriptCore/assembler/MacroAssemblerX86_64.h @@ -10409,7 +10409,7 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { Label loadPCRelativeDouble(FPRegisterID dest) { m_assembler.movsd_rip(dest); - // A relocation of this kind ends with the instruction. Only a Jump makes a Label of a place other than the end of the buffer. + // Relocations of this kind are the last bytes of the instruction. Only a Jump creates a Label that is not at the end of the buffer. return Jump(m_assembler.buffer().label().labelAtOffset(-static_cast(relocationSize(RelocationKind::Load64)))).label(); } diff --git a/Source/JavaScriptCore/assembler/testmasm.cpp b/Source/JavaScriptCore/assembler/testmasm.cpp index 01d94070b6335..54cfc94b98a35 100644 --- a/Source/JavaScriptCore/assembler/testmasm.cpp +++ b/Source/JavaScriptCore/assembler/testmasm.cpp @@ -6540,7 +6540,7 @@ static MacroOperands runOnOperands(MacroOperands operands, std::span(branchOnRegister, word, &word), expected(word)); CHECK_EQ(invoke(branchOnMemory, word, &word), expected(word)); - CHECK_EQ(invoke(setFromRegister, word, &word), expected(word)); + CHECK_EQ(invoke(setForRegister, word, &word), expected(word)); CHECK_EQ(invoke(movedImmediate, word, &word), expected(word) ? 7 : 0); CHECK_EQ(invoke(moved, word, &word), expected(word)); } @@ -6790,7 +6790,7 @@ static void testSignOfMaskedWords() } #if CPU(X86_64) -static void testBranchAddOfMemoryIntoItsBase() +static void testBranchAddMemoryIntoBase() { for (GPRReg dest : { GPRInfo::argumentGPR1, GPRInfo::argumentGPR2, GPRInfo::argumentGPR3 }) { uint64_t word = 0x1122334400000001; @@ -6806,7 +6806,7 @@ static void testBranchAddOfMemoryIntoItsBase() } } -static void testNarrowVectorOperationsWithEqualRegisters() +static void testNarrowVectorAliasedRegisters() { CPUFeatureSet withoutAVX = CPUFeatureSet::fromBits(CPUFeatureSet::host().bits() & ~(CPUFeatureSet().with(CPUFeature::AVX).with(CPUFeature::AVX2).bits())); constexpr FPRReg registers[] = { FPRInfo::fpRegT0, FPRInfo::fpRegT1, FPRInfo::fpRegT2 }; @@ -6835,7 +6835,7 @@ static void testNarrowVectorOperationsWithEqualRegisters() } } -static void testMovesOfAbsoluteAddressesEncodeEveryRegister() +static void testAbsoluteAddressMoveRegisters() { for (int reg = X86Registers::eax; reg <= X86Registers::r15; ++reg) { X86Assembler assembler; @@ -6858,13 +6858,13 @@ static void testMovesOfAbsoluteAddressesEncodeEveryRegister() #endif // CPU(X86_64) #if CPU(ARM64) -static void testPairsOfFloats() +static void testFloatPairs() { for (int32_t offset : { 0, 8, -256, 252, 256, 1000, -1000, 100000 }) { uint32_t words[] = { 1, 0x3f800000, 0x40490fdb, 4, 5 }; uintptr_t base = std::bit_cast(&words[1]) - offset; uint32_t copies[] = { 11, 12, 13, 14, 15 }; - uintptr_t baseOfCopies = std::bit_cast(&copies[1]) - offset; + uintptr_t copiesBase = std::bit_cast(&copies[1]) - offset; auto code = compile([=](CCallHelpers& jit) { emitFunctionPrologue(jit); jit.loadPair32(GPRInfo::argumentGPR0, CCallHelpers::TrustedImm32(offset), FPRInfo::fpRegT0, FPRInfo::fpRegT1); @@ -6872,7 +6872,7 @@ static void testPairsOfFloats() emitFunctionEpilogue(jit); jit.ret(); }); - invoke(code, base, baseOfCopies); + invoke(code, base, copiesBase); CHECK_EQ(copies[0], 11u); CHECK_EQ(copies[1], words[1]); CHECK_EQ(copies[2], words[2]); @@ -12630,21 +12630,21 @@ void run(const char* filter) WTF_IGNORES_THREAD_SAFETY_ANALYSIS #endif #if CPU(X86_64) || CPU(ARM64) - RUN(testSubtractWithEqualRegisters()); - RUN(testClearBitsWithEqualRegisters()); - RUN(testRelaxedDotProductWithEqualRegisters()); - RUN(testImmediatesOfMemoryOperations()); + RUN(testSubtractAliasedRegisters()); + RUN(testClearBitsAliasedRegisters()); + RUN(testRelaxedDotProductAliasedRegisters()); + RUN(testMemoryOperationImmediates()); RUN(testShiftedImmediates()); RUN(testBlindedAdditions()); - RUN(testSignOfMaskedWords()); + RUN(testMaskedWordSign()); #endif #if CPU(X86_64) - RUN(testBranchAddOfMemoryIntoItsBase()); - RUN(testNarrowVectorOperationsWithEqualRegisters()); - RUN(testMovesOfAbsoluteAddressesEncodeEveryRegister()); + RUN(testBranchAddMemoryIntoBase()); + RUN(testNarrowVectorAliasedRegisters()); + RUN(testAbsoluteAddressMoveRegisters()); #endif #if CPU(ARM64) - RUN(testPairsOfFloats()); + RUN(testFloatPairs()); #endif RUN(testOrImmMem()); diff --git a/Source/JavaScriptCore/b3/B3LowerToAir.cpp b/Source/JavaScriptCore/b3/B3LowerToAir.cpp index d6a548e8ff7c6..f238968882cdb 100644 --- a/Source/JavaScriptCore/b3/B3LowerToAir.cpp +++ b/Source/JavaScriptCore/b3/B3LowerToAir.cpp @@ -1245,7 +1245,7 @@ class LowerToAir { return !memory || memory->accessWidth() <= width; } - static bool hasBitsOfOneByte(uint32_t mask) + static bool isWithinOneByte(uint32_t mask) { return !(mask & ~0xffu) || !(mask & ~0xff00u) || !(mask & ~0xff0000u) || !(mask & ~0xff000000u); } @@ -2833,7 +2833,7 @@ class LowerToAir { if (!hasRightConst || left->opcode() != loadOpcode || !canReadInPlace(left, width)) return Inst(); // MacroAssemblerX86_64::generateTest32(Address, ..) reads the one byte that the mask has bits in. - if (isX86() && width == Width32 && left->traps() && loadPromise(left, loadOpcode).kind() == Arg::Addr && hasBitsOfOneByte(static_cast(rightConst))) + if (isX86() && width == Width32 && left->traps() && loadPromise(left, loadOpcode).kind() == Arg::Addr && isWithinOneByte(static_cast(rightConst))) return Inst(); // Signed loads will create high bits, so if the immediate has high bits // then we cannot proceed. Consider BitAnd(Load8S(ptr), 0x101). This cannot diff --git a/Source/JavaScriptCore/b3/B3ThreadJumps.cpp b/Source/JavaScriptCore/b3/B3ThreadJumps.cpp index bd00805e7e79f..2f6527b029727 100644 --- a/Source/JavaScriptCore/b3/B3ThreadJumps.cpp +++ b/Source/JavaScriptCore/b3/B3ThreadJumps.cpp @@ -654,8 +654,8 @@ class ThreadJumps { return true; } - // A block that was merged into its predecessor may have a Phi behind the Upsilon that feeds it. - void addPhiValuesFromOwnBlock(BasicBlock* block) + // In a block that was merged into its predecessor, a Phi may follow the Upsilon that feeds it. + void addLocalPhiValues(BasicBlock* block) { for (unsigned index = 0; index < block->size(); ++index) { Value* phi = block->at(index); @@ -687,7 +687,7 @@ class ThreadJumps { if (upsilon->phi()->owner == block) m_phiValuesOnPath.append({ upsilon->phi(), upsilon->child(0) }); } - addPhiValuesFromOwnBlock(block); + addLocalPhiValues(block); m_path.append(block); addUpsilonsAndSetsFrom(block); } diff --git a/Source/JavaScriptCore/b3/air/AirAllocateRegistersByGreedy.cpp b/Source/JavaScriptCore/b3/air/AirAllocateRegistersByGreedy.cpp index 58caf8c73fbec..a3f39211ec1cf 100644 --- a/Source/JavaScriptCore/b3/air/AirAllocateRegistersByGreedy.cpp +++ b/Source/JavaScriptCore/b3/air/AirAllocateRegistersByGreedy.cpp @@ -4049,7 +4049,7 @@ class GreedyAllocator { return a.tmp1.tmpIndex(bank) < b.tmp1.tmpIndex(bank); }); - // Groups only grow, so two that interfere still do at the next move between them. + // Groups only grow, so two groups that interfere keep interfering. UncheckedKeyHashSet> interferingGroups; for (Move& move : moves) { diff --git a/Source/JavaScriptCore/b3/air/AirCheckSpillSlots.h b/Source/JavaScriptCore/b3/air/AirCheckSpillSlots.h index bd91abde92639..f52a66dc39145 100644 --- a/Source/JavaScriptCore/b3/air/AirCheckSpillSlots.h +++ b/Source/JavaScriptCore/b3/air/AirCheckSpillSlots.h @@ -52,7 +52,7 @@ class PlacesOfValues { JS_EXPORT_PRIVATE void addWrite(Reg, Tmp, unsigned instIndex); JS_EXPORT_PRIVATE void addClobber(Reg, unsigned instIndex); - // Returns false, with the procedure marked as failed, if a read does not see the value of its slot. + // Returns false if a read does not see the value of its slot. The procedure is then marked as failed. JS_EXPORT_PRIVATE bool check(ASCIILiteral where); private: diff --git a/Source/JavaScriptCore/b3/air/AirPolicy.h b/Source/JavaScriptCore/b3/air/AirPolicy.h index 1647e821b38b6..5eb951f664b22 100644 --- a/Source/JavaScriptCore/b3/air/AirPolicy.h +++ b/Source/JavaScriptCore/b3/air/AirPolicy.h @@ -39,7 +39,7 @@ struct RegisterPolicy { RegisterSet savedByCallee; RegisterSet calleeSavesClobberedAtSuspensions; RegisterSet pinned; - // A 32-bit argument or result of a patchpoint in one of these has zeros in its upper half. + // 32-bit patchpoint arguments and results in these registers are zero-extended. RegisterSet zeroExtended; }; diff --git a/Source/JavaScriptCore/b3/air/AirReuseConditionFlags.cpp b/Source/JavaScriptCore/b3/air/AirReuseConditionFlags.cpp index d5af22a3c02bf..2e0c55b68cb82 100644 --- a/Source/JavaScriptCore/b3/air/AirReuseConditionFlags.cpp +++ b/Source/JavaScriptCore/b3/air/AirReuseConditionFlags.cpp @@ -287,13 +287,13 @@ class ConditionFlagsReuse { Inst& inst = block->at(index); if (std::optional condition = conditionOnResultOf(inst, branch)) return condition; - if (!isFlagPreservingMove(inst) || writesOperandOf(inst, branch)) + if (!isFlagPreservingMove(inst) || overwritesOperandOf(inst, branch)) return std::nullopt; } return std::nullopt; } - static bool writesOperandOf(Inst& move, Inst& branch) + static bool overwritesOperandOf(Inst& move, Inst& branch) { bool result = false; branch.forEachTmpFast([&](Tmp& tmp) { diff --git a/Source/JavaScriptCore/b3/air/AirShrinkWrapFrame.cpp b/Source/JavaScriptCore/b3/air/AirShrinkWrapFrame.cpp index edac846b3b14f..eb8d13b4a4916 100644 --- a/Source/JavaScriptCore/b3/air/AirShrinkWrapFrame.cpp +++ b/Source/JavaScriptCore/b3/air/AirShrinkWrapFrame.cpp @@ -863,7 +863,7 @@ void FrameShrinkWrapper::insertSetups() } } -// A patch that ends the function and runs without a frame: a tail call. +// A terminal patch that runs without a frame, such as a tail call. bool FrameShrinkWrapper::leavesFunction(BasicBlock* block) { return isReturn(block->last().kind.opcode) || (block->last().kind.opcode == Patch && !block->numSuccessors() && !block->last().needsFrame()); diff --git a/Source/JavaScriptCore/b3/air/testair_bun_flags.h b/Source/JavaScriptCore/b3/air/testair_bun_flags.h index 681d2181cb93a..f407727a217d7 100644 --- a/Source/JavaScriptCore/b3/air/testair_bun_flags.h +++ b/Source/JavaScriptCore/b3/air/testair_bun_flags.h @@ -846,7 +846,7 @@ void testReuseFlagsIsIdempotent() } enum class ResultOperation : uint8_t { ShiftLeft, ShiftLeftByNothing, AndWithMask, AndWithNegativeMask, AndWithMaskOfByte, AndWithMaskOfHalfWord, AndWithAllBits, AndWithRegister, AndIntoOtherRegister, AndOfRegisterWithItself }; -enum class ResultTestNeighbor : uint8_t { None, MoveBetween, MoveOfConstantBetween, MoveIntoResultBetween, OtherRegisterTested, OtherWidthTested }; +enum class ResultTestNeighbor : uint8_t { None, MoveBetween, ConstantMoveBetween, ResultMoveBetween, OtherRegisterTested, OtherWidthTested }; enum class ResultTestForm : uint8_t { TestAgainstItself, TestAgainstAllBits, CompareWithZero }; static int64_t maskOf(ResultOperation operation) @@ -916,9 +916,9 @@ static bool appendTestOfResult(Code& code, bool is64Bit, ResultOperation operati } if (neighbor == ResultTestNeighbor::MoveBetween) appendChecked(root, Move, a3, a4); - if (neighbor == ResultTestNeighbor::MoveOfConstantBetween) + if (neighbor == ResultTestNeighbor::ConstantMoveBetween) appendChecked(root, Move, Arg::imm(0), a4); - if (neighbor == ResultTestNeighbor::MoveIntoResultBetween) + if (neighbor == ResultTestNeighbor::ResultMoveBetween) appendChecked(root, Move, a3, result); Tmp tested = neighbor == ResultTestNeighbor::OtherRegisterTested ? a3 : result; switch (form) { @@ -1003,7 +1003,7 @@ void testBranchOnFlagsOfResult() checkBranchOnFlagsOfResult(is64Bit, operation, test, form); } } - for (ResultTestNeighbor neighbor : { ResultTestNeighbor::MoveBetween, ResultTestNeighbor::MoveOfConstantBetween, ResultTestNeighbor::MoveIntoResultBetween, ResultTestNeighbor::OtherRegisterTested, ResultTestNeighbor::OtherWidthTested }) + for (ResultTestNeighbor neighbor : { ResultTestNeighbor::MoveBetween, ResultTestNeighbor::ConstantMoveBetween, ResultTestNeighbor::ResultMoveBetween, ResultTestNeighbor::OtherRegisterTested, ResultTestNeighbor::OtherWidthTested }) checkBranchOnFlagsOfResult(is64Bit, ResultOperation::ShiftLeft, testConditions[0], ResultTestForm::TestAgainstItself, neighbor); } } diff --git a/Source/JavaScriptCore/b3/testb3_bun_ranges.cpp b/Source/JavaScriptCore/b3/testb3_bun_ranges.cpp index 52068b3599ca6..cba07b61a6813 100644 --- a/Source/JavaScriptCore/b3/testb3_bun_ranges.cpp +++ b/Source/JavaScriptCore/b3/testb3_bun_ranges.cpp @@ -653,7 +653,7 @@ static void checkComparisonOfExtensions(B3::Opcode opcode, ExtendedOperand left, } } -static void testComparisonOfExtensionsIsNarrow() +static void testNarrowExtensionComparison() { for (B3::Opcode opcode : { Equal, NotEqual, LessThan, GreaterThan, LessEqual, GreaterEqual, Below, BelowEqual, Above, AboveEqual }) { for (B3::Opcode leftExtension : { SExt32, ZExt32 }) { @@ -669,7 +669,7 @@ static void testComparisonOfExtensionsIsNarrow() } } -static void checkComparisonOfExtensionWithConstant(B3::Opcode opcode, ExtendedOperand left, int64_t constant, bool reducesWithRanges) +static void checkExtensionConstantComparison(B3::Opcode opcode, ExtendedOperand left, int64_t constant, bool reducesWithRanges) { constexpr int32_t maxInt32 = std::numeric_limits::max(); constexpr int32_t minInt32 = std::numeric_limits::min(); @@ -700,14 +700,14 @@ static void checkComparisonOfExtensionWithConstant(B3::Opcode opcode, ExtendedOp } } -static void testComparisonOfExtensionWithConstantIsNarrow() +static void testNarrowExtensionConstantComparison() { for (B3::Opcode opcode : { Equal, NotEqual, LessThan, GreaterThan, LessEqual, GreaterEqual, Below, BelowEqual, Above, AboveEqual }) { for (B3::Opcode extension : { SExt32, ZExt32 }) { for (bool isNonNegative : { false, true }) { for (int64_t constant : { 0ll, 1ll, 7ll, 128ll, 0x7fffffffll, 0x80000000ll, 0xfffffff9ll, 0xffffffffll, 0x100000000ll, 0x100000007ll, -1ll, -7ll, -128ll, -0x80000000ll, -0x80000001ll, std::numeric_limits::max(), std::numeric_limits::min() }) { for (bool reducesWithRanges : { true, false }) - checkComparisonOfExtensionWithConstant(opcode, { extension, isNonNegative }, constant, reducesWithRanges); + checkExtensionConstantComparison(opcode, { extension, isNonNegative }, constant, reducesWithRanges); } } } @@ -911,8 +911,8 @@ void addBunRangeTests(const TestConfig* config, BunTestTasks& tasks) RUN(testExtensionOfTruncationFoldsByRange()); RUN(testSumOfIntegersBehindDoublesIsAddedInIntegers()); RUN(testSignExtensionOfNonNegativeIsZeroExtension()); - RUN(testComparisonOfExtensionsIsNarrow()); - RUN(testComparisonOfExtensionWithConstantIsNarrow()); + RUN(testNarrowExtensionComparison()); + RUN(testNarrowExtensionConstantComparison()); RUN(testMaskFoldsByRange()); RUN(testVariableShiftRanges()); RUN(testTruncOfVariableShiftLeftAndBack()); diff --git a/Source/JavaScriptCore/b3/testb3_bun_reductions.cpp b/Source/JavaScriptCore/b3/testb3_bun_reductions.cpp index 5f8f0a84b0d6e..10c0daecc9823 100644 --- a/Source/JavaScriptCore/b3/testb3_bun_reductions.cpp +++ b/Source/JavaScriptCore/b3/testb3_bun_reductions.cpp @@ -977,7 +977,7 @@ enum class ExtendedForm : uint8_t { } // anonymous namespace static constexpr int64_t factorOfProduct = 1000003; -static constexpr int64_t boundOfLoad = 300; +static constexpr int64_t loadBound = 300; static Air::Opcode airOpcodeOf(ExtendedForm form) { @@ -1001,8 +1001,8 @@ static Air::Opcode airOpcodeOf(ExtendedForm form) return Air::Oops; } -// Eight times this is in the reach of a load on arm64. -static constexpr int32_t indexOfLoad = 31; +// Scaled by eight this is in the range of load offsets on ARM64. +static constexpr int32_t loadIndex = 31; static void buildUserOfExtendedForm(Graph& graph, ExtendedForm form) { @@ -1033,14 +1033,14 @@ static void buildUserOfExtendedForm(Graph& graph, ExtendedForm form) case ExtendedForm::Branch64OnIndexedLoadAndImmediate: { Type type = airOpcodeOf(form) == Air::Branch32 ? Int32 : Int64; Value* address = graph.make(root, Add, pointer, graph.make(root, Shl, other, graph.int32(root, 3))); - Value* loaded = graph.load(root, type, address, -indexOfLoad * 8, firstHeap); - Value* compared = form == ExtendedForm::Branch64OnIndexedLoadAndImmediate ? graph.int64(root, boundOfLoad) : type == Int32 ? graph.make(root, Trunc, operand) : operand; + Value* loaded = graph.load(root, type, address, -loadIndex * 8, firstHeap); + Value* compared = form == ExtendedForm::Branch64OnIndexedLoadAndImmediate ? graph.int64(root, loadBound) : type == Int32 ? graph.make(root, Trunc, operand) : operand; bool isBefore = form == ExtendedForm::Branch32BeforeIndexedLoad || form == ExtendedForm::Branch64BeforeIndexedLoad; Value* comparison = isBefore ? graph.make(root, LessThan, compared, loaded) : graph.make(root, LessThan, loaded, compared); - BasicBlock* whereLess = graph.proc.addBlock(); + BasicBlock* lessThanBlock = graph.proc.addBlock(); BasicBlock* otherwise = graph.proc.addBlock(); - graph.branch(root, comparison, whereLess, otherwise); - graph.ret(whereLess, graph.int64(whereLess, 1)); + graph.branch(root, comparison, lessThanBlock, otherwise); + graph.ret(lessThanBlock, graph.int64(lessThanBlock, 1)); graph.ret(otherwise, graph.int64(otherwise, 0)); return; } @@ -1066,7 +1066,7 @@ static int64_t resultOfExtendedForm(ExtendedForm form, int64_t loaded, int64_t o case ExtendedForm::Branch64BeforeIndexedLoad: return operand < loaded; case ExtendedForm::Branch64OnIndexedLoadAndImmediate: - return loaded < boundOfLoad; + return loaded < loadBound; } RELEASE_ASSERT_NOT_REACHED(); return 0; @@ -1101,7 +1101,7 @@ static void testExtendedFormsAreSelectedOnRequest() for (int64_t loaded : { 0ll, 6ll, -1ll, 0x180000001ll }) { int64_t memory[1] = { loaded }; for (int64_t operand : { 0ll, 1ll, 2ll, -5ll }) { - CHECK_EQ(invoke(*code, memory, operand, indexOfLoad), resultOfExtendedForm(form, loaded, operand, indexOfLoad)); + CHECK_EQ(invoke(*code, memory, operand, loadIndex), resultOfExtendedForm(form, loaded, operand, loadIndex)); } } } diff --git a/Source/JavaScriptCore/b3/testb3_bun_thread_jumps.cpp b/Source/JavaScriptCore/b3/testb3_bun_thread_jumps.cpp index ccb2b8e6c184d..096795b5441a8 100644 --- a/Source/JavaScriptCore/b3/testb3_bun_thread_jumps.cpp +++ b/Source/JavaScriptCore/b3/testb3_bun_thread_jumps.cpp @@ -466,7 +466,7 @@ static void testThreadJumpsDemotesPhiWithUpsilonBeforeBranch() CHECK_EQ(invoke(*code, 0, 4, memory), 20); } -static void testThreadJumpsReadsUpsilonInBlockOfPhi() +static void testThreadJumpsUpsilonBeforePhi() { Procedure proc(policyWithMaxTailSize(1)); Graph graph { proc }; @@ -1598,7 +1598,7 @@ void addBunJumpThreadingTests(const TestConfig* config, BunTestTasks& tasks) RUN(testThreadJumpsKeepsBlockLocalPhi()); RUN(testThreadJumpsDemotesPhiUsedByAnotherBlock()); RUN(testThreadJumpsDemotesPhiWithUpsilonBeforeBranch()); - RUN(testThreadJumpsReadsUpsilonInBlockOfPhi()); + RUN(testThreadJumpsUpsilonBeforePhi()); RUN(testThreadJumpsKeepsUndecidedTest()); RUN(testThreadJumpsSkipsTailWithUncloneablePatchpoint()); RUN(testThreadJumpsSkipsLoopHeader()); diff --git a/Source/JavaScriptCore/runtime/JSFunction.cpp b/Source/JavaScriptCore/runtime/JSFunction.cpp index 57f8aee6a6901..4062b7b83b4fe 100644 --- a/Source/JavaScriptCore/runtime/JSFunction.cpp +++ b/Source/JavaScriptCore/runtime/JSFunction.cpp @@ -283,7 +283,7 @@ const String JSFunction::calculatedDisplayName(VM& vm) #if ENABLE(AOT) if (isCompiledClosure()) { - // An anonymous function has no name, as below: the caller then names an instance after its built-in class. + // Anonymous functions have no name, as below. The caller then uses the name of the built-in class. if (String displayName = AOT::CompiledCallee::displayName(this); !displayName.isNull()) return displayName.isEmpty() ? String() : displayName; } diff --git a/Source/JavaScriptCore/runtime/JSModuleLoader.cpp b/Source/JavaScriptCore/runtime/JSModuleLoader.cpp index a0afc0347be63..f4a464e98c745 100644 --- a/Source/JavaScriptCore/runtime/JSModuleLoader.cpp +++ b/Source/JavaScriptCore/runtime/JSModuleLoader.cpp @@ -1217,7 +1217,7 @@ void JSModuleLoader::innerModuleLoading(JSGlobalObject* globalObject, ModuleGrap // there is no need to attach a ModuleGraphLoadingError reaction, so we skip it. bool needsErrorReaction = module->loadedModules().size() == loadedModulesCountBefore; #if ENABLE(AOT) - // Making the record of a CommonJS module runs the module. Its require() may load other requests of this module inside the call. + // Creating the record of a CommonJS module evaluates it, and its require() may load other requests of this module meanwhile. if (vm.hasCompiledCode()) needsErrorReaction = !module->loadedModules().contains(ModuleMapKey { request.m_specifier.impl(), request.type() }); #endif diff --git a/Source/JavaScriptCore/runtime/JSONObject.cpp b/Source/JavaScriptCore/runtime/JSONObject.cpp index ee7bcc99268b1..84b4a5ae0cb6b 100644 --- a/Source/JavaScriptCore/runtime/JSONObject.cpp +++ b/Source/JavaScriptCore/runtime/JSONObject.cpp @@ -823,8 +823,8 @@ class FastStringifier { bool isMemberWrittenByItsFields(JSObject&); bool writeListedFieldsOfMember(Cursor&, JSObject&, std::span, unsigned reservation); void appendElementsInCompiledProgram(JSArray&); - enum class ArrayWritten : uint8_t { No, Yes, Failed }; - ArrayWritten writeArrayOfPrimitives(Cursor&, JSArray&); + enum class ArrayResult : uint8_t { No, Yes, Failed }; + ArrayResult writePrimitiveArray(Cursor&, JSArray&); bool isCompiledStructureKnown(const JSObject& object) const { return (m_compiledStructuresKnown & knowsCompiledStructure) && object.structureID().bits() == m_compiledStructureWithoutToJSON; } #endif String result(); @@ -2856,14 +2856,14 @@ ALWAYS_INLINE bool FastStringifier::writeListedFields(Curs } else if constexpr (writesLatin1 && hasGap == HasGap::No) { if (type == ArrayType) { appendKey(); - switch (writeArrayOfPrimitives(cursor, *asArray(&cell))) { - case ArrayWritten::Yes: + switch (writePrimitiveArray(cursor, *asArray(&cell))) { + case ArrayResult::Yes: if (!reserve(cursor, reservation)) [[unlikely]] return false; continue; - case ArrayWritten::Failed: + case ArrayResult::Failed: return false; - case ArrayWritten::No: + case ArrayResult::No: cursor.next = buffer() + positionOfMember; break; } @@ -2923,37 +2923,37 @@ ALWAYS_INLINE bool FastStringifier::writeListedFields(Curs return true; } -// Writes an array of strings, numbers, booleans, null and undefined. Nothing is written for an array that holds anything else, or a string that takes more than a copy. +// Writes an array of strings, numbers, booleans, null and undefined. Writes nothing if there are other elements, or strings that cannot be copied verbatim. template -NEVER_INLINE auto FastStringifier::writeArrayOfPrimitives(Cursor& cursor, JSArray& array) -> ArrayWritten +NEVER_INLINE auto FastStringifier::writePrimitiveArray(Cursor& cursor, JSArray& array) -> ArrayResult { if constexpr (std::same_as) { // append() checks the prototype of arrays at the first array. if (!m_checkedArrayPrototype || !m_globalObject.isOriginalArrayStructure(array.structure())) - return ArrayWritten::No; + return ArrayResult::No; IndexingType indexingType = array.indexingType(); if (!hasInt32(indexingType) && !hasContiguous(indexingType)) - return ArrayWritten::No; + return ArrayResult::No; auto* butterfly = array.butterfly(); const unsigned length = butterfly->publicLength(); if (length > butterfly->vectorLength()) [[unlikely]] - return ArrayWritten::No; + return ArrayResult::No; - // A comma, a value that is no string and the closing bracket. + // A comma, a non-string value and the closing bracket. constexpr size_t maxShortElementLength = 1 + std::max(AOT::maxJSONNumberLength, maxInt32StringLength) + 1; auto elements = butterfly->contiguous(); - const size_t positionOfArray = cursor.next - buffer(); + const size_t arrayOffset = cursor.next - buffer(); if (!enterHolder() || !reserve(cursor, 2)) [[unlikely]] - return ArrayWritten::Failed; - auto leaveUnwritten = [&] { + return ArrayResult::Failed; + auto rollBack = [&] { leaveHolder(); - cursor.next = buffer() + positionOfArray; - return ArrayWritten::No; + cursor.next = buffer() + arrayOffset; + return ArrayResult::No; }; *cursor.next++ = '['; for (unsigned i = 0; i < length; ++i) { if (!reserve(cursor, maxShortElementLength)) [[unlikely]] - return ArrayWritten::Failed; + return ArrayResult::Failed; if (i) *cursor.next++ = ','; JSValue element = elements.at(&array, i).get(); @@ -2962,19 +2962,19 @@ NEVER_INLINE auto FastStringifier::writeArrayOfPrimitives( continue; } if (!element) [[unlikely]] - return leaveUnwritten(); + return rollBack(); if (element.isCell()) { if (element.asCell()->type() != StringType) - return leaveUnwritten(); + return rollBack(); const StringImpl* string = asString(element.asCell())->tryGetValueImpl(); if (!string || !string->is8Bit()) [[unlikely]] - return leaveUnwritten(); + return rollBack(); const size_t stringLength = string->length(); if (!reserve(cursor, 1 + stringLength + 1 + 1)) [[unlikely]] - return ArrayWritten::Failed; + return ArrayResult::Failed; *cursor.next = '"'; if (stringCopySameType(string->span8(), cursor.next + 1)) [[unlikely]] - return leaveUnwritten(); + return rollBack(); cursor.next[1 + stringLength] = '"'; cursor.next += 1 + stringLength + 1; continue; @@ -2986,15 +2986,15 @@ NEVER_INLINE auto FastStringifier::writeArrayOfPrimitives( else if (element.isUndefinedOrNull()) cursor.next = AOT::appendJSONLiteral(cursor.next, "null"_s); else - return leaveUnwritten(); + return rollBack(); } *cursor.next++ = ']'; leaveHolder(); - return ArrayWritten::Yes; + return ArrayResult::Yes; } else { UNUSED_PARAM(cursor); UNUSED_PARAM(array); - return ArrayWritten::No; + return ArrayResult::No; } } diff --git a/Source/JavaScriptCore/runtime/ObjectConstructor.cpp b/Source/JavaScriptCore/runtime/ObjectConstructor.cpp index 42edd379ddaa5..81578233a2b16 100644 --- a/Source/JavaScriptCore/runtime/ObjectConstructor.cpp +++ b/Source/JavaScriptCore/runtime/ObjectConstructor.cpp @@ -190,7 +190,7 @@ JSC_DEFINE_HOST_FUNCTION(objectConstructorSetPrototypeOf, (JSGlobalObject* globa return JSValue::encode(objectValue); } -// Inlined into the host function, as before the test for locked properties. +// Inlined into the host function. static ALWAYS_INLINE JSValue describeOwnProperty(JSGlobalObject* globalObject, JSObject* object, const Identifier& propertyName) { VM& vm = globalObject->vm(); diff --git a/Source/JavaScriptCore/wasm/WasmFormat.cpp b/Source/JavaScriptCore/wasm/WasmFormat.cpp index 3f6a3a26e9462..028a1c53a449f 100644 --- a/Source/JavaScriptCore/wasm/WasmFormat.cpp +++ b/Source/JavaScriptCore/wasm/WasmFormat.cpp @@ -49,14 +49,14 @@ bool WasmCallableFunction::isJS() const bool Segments::tryReserve(size_t numberOfSegments, size_t numberOfBytes, size_t numberOfIndices) { m_hasIndices = numberOfIndices; - return m_buffer.tryReserveInitialCapacity(numberOfSegments * sizeof(Header) + numberOfBytes) && m_placeOfIndex.tryReserveInitialCapacity(numberOfIndices); + return m_buffer.tryReserveInitialCapacity(numberOfSegments * sizeof(Header) + numberOfBytes) && m_segmentOffsets.tryReserveInitialCapacity(numberOfIndices); } void Segments::append(const Segment& segment) { ++m_size; if (m_hasIndices) - m_placeOfIndex.append(m_buffer.size()); + m_segmentOffsets.append(m_buffer.size()); Header header { static_cast(segment.bytes.size()), 0 }; if (segment.isActive() && segment.offsetIfActive->isConst() && segment.offsetIfActive->bits() <= std::numeric_limits::max() && !segment.memoryIndex) [[likely]] { diff --git a/Source/JavaScriptCore/wasm/WasmFormat.h b/Source/JavaScriptCore/wasm/WasmFormat.h index de58ad63734b7..de6ccc6ffee64 100644 --- a/Source/JavaScriptCore/wasm/WasmFormat.h +++ b/Source/JavaScriptCore/wasm/WasmFormat.h @@ -745,7 +745,7 @@ struct Segment { std::span bytes; }; -// The data segments of a module. There may be a hundred thousand of a few bytes each, so they are in one buffer, each behind a short header. +// The data segments of a module, in one buffer with a short header each, because there may be very many short ones. class Segments { public: // Only a module with a Data Count section names its segments by index. @@ -754,7 +754,7 @@ class Segments { size_t size() const { return m_size; } size_t sizeInBytes() const { return m_buffer.size(); } - // Moves the place to the next segment. + // Advances the offset to the next segment. Segment readAt(size_t& place) const { Header header = read
(place); @@ -768,12 +768,12 @@ class Segments { } Segment operator[](size_t index) const { - size_t place = m_placeOfIndex[index]; + size_t place = m_segmentOffsets[index]; return readAt(place); } private: - // An active segment of the first memory at a constant offset of 32 bits has this alone. + // The whole header of an active segment of memory 0 at a constant 32-bit offset. struct Header { uint32_t size; uint32_t offset; @@ -798,7 +798,7 @@ class Segments { void readLongHeader(size_t& place, Segment&) const; Vector m_buffer; - Vector m_placeOfIndex; + Vector m_segmentOffsets; size_t m_size { 0 }; bool m_hasIndices { false }; }; diff --git a/Source/JavaScriptCore/wasm/WasmIPIntPlan.cpp b/Source/JavaScriptCore/wasm/WasmIPIntPlan.cpp index 994cd369660eb..ada8f27831a57 100644 --- a/Source/JavaScriptCore/wasm/WasmIPIntPlan.cpp +++ b/Source/JavaScriptCore/wasm/WasmIPIntPlan.cpp @@ -122,17 +122,17 @@ void IPIntPlan::compileFunction(FunctionCodeIndex functionIndex) auto functionIndexSpace = m_moduleInformation->toSpaceIndex(functionIndex); ASSERT_UNUSED(functionIndexSpace, &m_moduleInformation->rtt(functionIndexSpace) == &m_moduleInformation->rtt(typeSignatureIndex)); - std::optional metadataOfImageFunction; + std::optional imageFunctionMetadata; #if ENABLE(WEBASSEMBLY_OMGJIT) if (m_moduleInImage) { if (CodeInImage::functionsOf(*m_moduleInImage, ImageCodeFlavor::Checking)[functionIndex.rawIndex()].usesVectors) m_moduleInformation->markUsesSIMD(functionIndex); m_moduleInformation->doneSeeingFunction(functionIndex); static constexpr std::array noBytecode { }; - metadataOfImageFunction.emplace(functionIndex, function.data.isEmpty() ? std::span { noBytecode } : function.data.span()); + imageFunctionMetadata.emplace(functionIndex, function.data.isEmpty() ? std::span { noBytecode } : function.data.span()); } #endif - if (!metadataOfImageFunction && !m_wasmInternalFunctions[functionIndex]) { + if (!imageFunctionMetadata && !m_wasmInternalFunctions[functionIndex]) { beginCompilerSignpost(CompilationMode::IPIntMode, functionIndexSpace); auto parseAndCompileResult = parseAndCompileMetadata(function.data, signature, m_moduleInformation.get(), functionIndex); endCompilerSignpost(CompilationMode::IPIntMode, functionIndexSpace); @@ -147,7 +147,7 @@ void IPIntPlan::compileFunction(FunctionCodeIndex functionIndex) } { - auto callee = IPIntCallee::create(metadataOfImageFunction ? *metadataOfImageFunction : *m_wasmInternalFunctions[functionIndex], functionIndexSpace, signature, { }); + auto callee = IPIntCallee::create(imageFunctionMetadata ? *imageFunctionMetadata : *m_wasmInternalFunctions[functionIndex], functionIndexSpace, signature, { }); ASSERT(!callee->entrypoint()); bool usesSIMD = m_moduleInformation->usesSIMD(functionIndex); // Immediately tier up to BBQ for SIMD, if necesary. diff --git a/Source/JavaScriptCore/wasm/WasmOMGIRGenerator.cpp b/Source/JavaScriptCore/wasm/WasmOMGIRGenerator.cpp index eaa4e99478fee..78b20070282f4 100644 --- a/Source/JavaScriptCore/wasm/WasmOMGIRGenerator.cpp +++ b/Source/JavaScriptCore/wasm/WasmOMGIRGenerator.cpp @@ -1225,7 +1225,7 @@ class OMGIRGenerator { InsertionSet m_constantInsertionValues; // Indexed by local. Entries are restored at the end of the scope that proved them. Vector m_provenBytes; - Vector m_indexOfLastSet; + Vector m_lastSetIndex; Vector> m_provenInScopes; bool m_nextAccessIsInBounds { false }; // Facts proven before a loop do not hold in its body. @@ -2955,9 +2955,9 @@ void OMGIRGenerator::didSetLocal(Value* set) if (!m_context.codeForImage) return; unsigned variable = set->as()->variable()->index(); - while (variable >= m_indexOfLastSet.size()) - m_indexOfLastSet.append(0); - m_indexOfLastSet[variable] = set->index(); + while (variable >= m_lastSetIndex.size()) + m_lastSetIndex.append(0); + m_lastSetIndex[variable] = set->index(); } // The local that the value was read from, unless it was set since. @@ -2969,7 +2969,7 @@ Variable* OMGIRGenerator::currentLocal(Value* value) const unsigned local = variable->index() - m_locals[0]->index(); if (local >= m_locals.size() || m_locals[local] != variable) return nullptr; - if (variable->index() < m_indexOfLastSet.size() && m_indexOfLastSet[variable->index()] > value->index()) + if (variable->index() < m_lastSetIndex.size() && m_lastSetIndex[variable->index()] > value->index()) return nullptr; return variable; } @@ -2979,7 +2979,7 @@ auto OMGIRGenerator::provenBytes(Variable* local) const -> ProvenBytes if (local->index() >= m_provenBytes.size()) return { }; ProvenBytes proven = m_provenBytes[local->index()]; - bool wasSetSince = local->index() < m_indexOfLastSet.size() && m_indexOfLastSet[local->index()] > proven.index; + bool wasSetSince = local->index() < m_lastSetIndex.size() && m_lastSetIndex[local->index()] > proven.index; return wasSetSince || proven.index < m_firstIndexInLoop ? ProvenBytes() : proven; } From 02c32c51c0599aff01e8e1a554a33dbf7b668ebc Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 10 Nov 2026 05:58:40 -0800 Subject: [PATCH 03/10] AOT: fixes in WebAssembly and builds, lists of zeros again, plainer loader names On b7288b68. 23 files, +343 -132: Source/JavaScriptCore/aot 14 files +82 -86 . Source/JavaScriptCore/assembler 2 files +9 -1 . Source/JavaScriptCore/b3 4 files +132 -2 . Source/JavaScriptCore/wasm 2 files +119 -42 . Source/JavaScriptCore/interpreter 1 file +1 -1. - Plain Bun too: on x64 a WebAssembly load whose low bytes alone are used returned a value at the end of the memory where it should trap, in the optimizing JIT. Fixed. - Executables: a store in a WebAssembly loop that only a trap ends could be dropped as covered by a later iteration's store to another address. Fixed. - x64: a WebAssembly function entered behind its entry test could read stale flags. Fixed; found in review. - `additionBlindedConstant(0)` no longer divides by zero. - With `--aot-harden=proof-checks`, two kinds of loop stopped the build: a loop in a loop that ends at a test for `undefined`, and a `for..of` loop that also reads the list by a remainder. Fixed. - `bun test --aot --rerun-each`: in the second pass, an import() that is refused again behind an unrelated mock is rejected with the same error object as under `bun test`. - `[0, 0, 0, 0]` is made to be filled, so it is built with a storage of its own, up to 16 elements. A list that a variable of the file or the body of a module holds shares its storage as before; built in line it stopped the build. - Two functions that Bun calls and a few members and locals of the modules' run-time state have plainer names. No logic changes with the names. Bun's side follows them, so this engine and Bun's side go together. --- .../JavaScriptCore/aot/facts/AOTThinSummary.h | 4 + .../aot/image/AOTWasmModules.cpp | 52 +----- .../aot/runtime/AOTModuleEvaluation.cpp | 2 + .../aot/runtime/AOTModuleHost.h | 8 +- .../JavaScriptCore/aot/runtime/AOTModules.cpp | 24 +-- .../JavaScriptCore/aot/runtime/AOTModules.h | 21 ++- .../aot/runtime/AOTTestRunFactories.cpp | 16 +- .../aot/runtime/AOTTestRunImportTurns.cpp | 2 +- .../aot/runtime/AOTTestRunMockViews.cpp | 4 +- .../aot/runtime/AOTTestRunMocksLetGo.cpp | 9 +- .../JavaScriptCore/aot/runtime/AOTVMState.h | 2 +- .../aot/walk/AOTCodeVerifier.cpp | 2 +- Source/JavaScriptCore/aot/walk/AOTWalk.cpp | 2 + .../aot/walk/AOTWalkChoices.cpp | 20 +++ .../JavaScriptCore/assembler/MacroAssembler.h | 2 +- Source/JavaScriptCore/assembler/testmasm.cpp | 8 + .../b3/B3EliminateCommonSubexpressions.cpp | 6 +- Source/JavaScriptCore/b3/B3LowerToAir.cpp | 2 +- .../b3/testb3_bun_selection.cpp | 58 +++++++ .../b3/testb3_bun_store_merging.cpp | 68 ++++++++ .../interpreter/Interpreter.cpp | 2 +- .../wasm/WasmCompilationContext.h | 2 +- .../wasm/WasmOMGIRGenerator.cpp | 159 +++++++++++++----- 23 files changed, 343 insertions(+), 132 deletions(-) diff --git a/Source/JavaScriptCore/aot/facts/AOTThinSummary.h b/Source/JavaScriptCore/aot/facts/AOTThinSummary.h index 9b1dfc067295f..06968ba8509ab 100644 --- a/Source/JavaScriptCore/aot/facts/AOTThinSummary.h +++ b/Source/JavaScriptCore/aot/facts/AOTThinSummary.h @@ -703,6 +703,10 @@ struct ThinSummary : SummaryWithoutLists { JS_EXPORT_PRIVATE Effects ownEffects() const; JS_EXPORT_PRIVATE std::optional ownEffectsOfOperators() const; + bool holdsWrittenList(DataID data) const + { + return std::ranges::any_of(writtenListsOfVariables, [&](const SummaryWrittenListOfVariable& written) { return written.data == data; }); + } const SummaryWrittenListOfVariable* findWrittenListOf(VariableID variable) const LIFETIME_BOUND { auto found = std::ranges::lower_bound(writtenListsOfVariables, variable.index(), { }, [](const SummaryWrittenListOfVariable& each) { return each.variable.index(); }); diff --git a/Source/JavaScriptCore/aot/image/AOTWasmModules.cpp b/Source/JavaScriptCore/aot/image/AOTWasmModules.cpp index f76cff99540af..5bd6a76fc4e4d 100644 --- a/Source/JavaScriptCore/aot/image/AOTWasmModules.cpp +++ b/Source/JavaScriptCore/aot/image/AOTWasmModules.cpp @@ -386,53 +386,6 @@ struct WasmModuleCompiler { return layout; } - static uint32_t findZeroBranchTarget(std::span code) - { -#if CPU(X86_64) - static_assert(GPRInfo::argumentGPR0 == X86Registers::edi); - auto displacementAt = [&](size_t offset, size_t size) { - int32_t displacement = static_cast(code[offset]); - if (size == sizeof(int32_t)) - memcpySpan(asMutableByteSpan(displacement), code.subspan(offset, size)); - return displacement; - }; - constexpr uint8_t shortJumpIfZero = 0x74; - constexpr uint8_t shortJumpIfNotZero = 0x75; - constexpr std::array testEDI { 0x85, 0xff }; - if (code.size() < 16 || !spanHasPrefix(code, std::span { testEDI })) - return 0; - size_t offset = testEDI.size(); - size_t size = 1; - uint8_t opcode = code[offset++]; - if (opcode == 0x0f) { - opcode = code[offset++] - 0x10; - size = sizeof(int32_t); - } - uint32_t next = static_cast(offset + size); - if (opcode == shortJumpIfNotZero) - return next; - if (opcode == shortJumpIfZero) - return next + displacementAt(offset, size); - return 0; -#elif CPU(ARM64) - uint32_t instruction; - if (code.size() < sizeof(instruction)) - return 0; - memcpySpan(asMutableByteSpan(instruction), code.first(sizeof(instruction))); - constexpr uint32_t compareAndBranchOfW0 = 0xfe00001f; - constexpr uint32_t branchIfZero = 0x34000000; - static_assert(GPRInfo::argumentGPR0 == ARM64Registers::x0); - if ((instruction & compareAndBranchOfW0 & ~0x01000000u) != branchIfZero) - return 0; - bool branchesIfNotZero = instruction & 0x01000000; - int32_t displacement = static_cast(instruction << 8) >> 13 << 2; - return branchesIfNotZero ? sizeof(instruction) : displacement; -#else - UNUSED_PARAM(code); - return 0; -#endif - } - static Vector findFramelessRanges(const B3::Air::Code& code, ImageLinkBuffer& buffer) { Vector result; @@ -538,10 +491,7 @@ struct WasmModuleCompiler { result.firstStateTests = codeForImage.firstStateTests; if (codeForImage.passedGlobal) { result.directEntry = buffer.linkBuffer().offsetOf(context.directEntry); - if (auto& withZero = context.directEntryWithZero) - result.directEntryWithZero = buffer.linkBuffer().offsetOf(*withZero); - else - result.directEntryWithZero = result.directEntry + findZeroBranchTarget(buffer.bytes().subspan(result.directEntry)); + result.directEntryWithZero = buffer.linkBuffer().offsetOf(*context.directEntryWithZero); } for (size_t directCallee : (*compiled)->outgoingJITDirectCallees) result.directCallees.set(directCallee); diff --git a/Source/JavaScriptCore/aot/runtime/AOTModuleEvaluation.cpp b/Source/JavaScriptCore/aot/runtime/AOTModuleEvaluation.cpp index 5fd4627ca9ca7..1be5d9595acfc 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTModuleEvaluation.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTModuleEvaluation.cpp @@ -4443,6 +4443,8 @@ JSValue Modules::answerRequire(ModuleID target, ModuleID referrer, RequiredBy re evaluateModulesOwedBelow(target); RETURN_IF_EXCEPTION(scope, { }); auto hearsHowRequireEnds = makeScopeExit([&] { + if (scope.exception() && isFailedESModule(target)) [[unlikely]] + m_loadRoots.set(target.index()); if (wasHeldBeforeWalk || !holdsLinksOfLoadThatMayFail(target)) [[likely]] return; if (scope.exception()) diff --git a/Source/JavaScriptCore/aot/runtime/AOTModuleHost.h b/Source/JavaScriptCore/aot/runtime/AOTModuleHost.h index 6ba036af0d5e7..83b51f3b4bcf3 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTModuleHost.h +++ b/Source/JavaScriptCore/aot/runtime/AOTModuleHost.h @@ -297,8 +297,8 @@ JS_EXPORT_PRIVATE bool wasDisplacedDuringEvaluation(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE bool wasRegisteredByFailedLoad(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE bool wasLinkedByFailedGraph(JSGlobalObject*, ModuleID, bool byEarlierTestFile = false); JS_EXPORT_PRIVATE bool wasLinkedBeforeLinkFailure(JSGlobalObject*, ModuleID); -JS_EXPORT_PRIVATE bool isLinkedForImporterYetToRun(JSGlobalObject*, ModuleID, std::optional asking); -JS_EXPORT_PRIVATE bool isFailedImportRoot(JSGlobalObject*, ModuleID); +JS_EXPORT_PRIVATE bool isLinkedForPendingImporter(JSGlobalObject*, ModuleID, std::optional asking); +JS_EXPORT_PRIVATE bool isFailedLoadRoot(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE bool wasDisplacedAfterFailure(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE bool wasLoadedBeforeTestFile(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE void clearLoadedBeforeTestFile(JSGlobalObject*, ModuleID); @@ -340,7 +340,7 @@ JS_EXPORT_PRIVATE bool hasDelayedImport(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE bool isRequestedByDetachedLoad(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE void retryMockInDetachedLoads(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE bool needsRetryInDetachedLoad(JSGlobalObject*, ModuleID); -JS_EXPORT_PRIVATE uint64_t resumingLoadFactoryCall(JSGlobalObject*, ModuleID); +JS_EXPORT_PRIVATE uint64_t factoryCallOfResumedLoad(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE void removeUnusedLingeringMocks(JSGlobalObject*); JS_EXPORT_PRIVATE bool importsStatically(JSGlobalObject*, ModuleID importer, ModuleID imported); JS_EXPORT_PRIVATE void setNeedsMockReinstall(JSGlobalObject*, ModuleID, bool waits = true); @@ -400,7 +400,7 @@ JS_EXPORT_PRIVATE AbstractModuleRecord* originalModuleRecord(JSGlobalObject*, Mo JS_EXPORT_PRIVATE void didBeginTestFile(JSGlobalObject*); JS_EXPORT_PRIVATE bool hasActiveMock(JSGlobalObject*, ModuleID); JS_EXPORT_PRIVATE bool hasUnevaluatedActiveMockImporter(JSGlobalObject*, ModuleID, std::optional asking); -JS_EXPORT_PRIVATE bool tryKeepActiveMock(JSGlobalObject*, ModuleID, bool staysTheOriginal, bool keepsCurrentTestFileLoads); +JS_EXPORT_PRIVATE bool tryKeepActiveMock(JSGlobalObject*, ModuleID, bool remainsOriginal, bool keepsCurrentTestFileLoads); struct ActiveMockExports { JSObject* namespaceObject { nullptr }; JSValue requiredValue; diff --git a/Source/JavaScriptCore/aot/runtime/AOTModules.cpp b/Source/JavaScriptCore/aot/runtime/AOTModules.cpp index 5acd6bf49bda1..0257369e7ddd0 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTModules.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTModules.cpp @@ -2434,6 +2434,8 @@ LoadedBy Modules::evaluatedBy(ModuleID module) LoadedBy Modules::fetchedBy(ModuleID module) { + if (m_refetchedFailures.get(module.index()) && isFailedESModule(module)) + return LoadedBy::ThisTestFile; LoadedBy loadedBy = evaluatedBy(module); if (loadedBy != LoadedBy::Nobody || !m_hasBegunTestFile || isEvicted(module) || m_momentsOfLoads.isEmpty() || !m_momentsOfLoads[module.index()]) return loadedBy; @@ -4530,7 +4532,7 @@ JSPromise* Modules::startImport(ModuleID module, std::optional referrer if (isNewCallAfterTests && shouldLeaveImportPending(module)) [[unlikely]] return promiseForCaller(); if (m_loadsModulesIndividually) [[unlikely]] - m_importCallRoots.set(module.index()); + m_loadRoots.set(module.index()); if (m_loadsModulesIndividually) [[unlikely]] recordVariantRequest(module); if (m_mocks) [[unlikely]] @@ -5242,7 +5244,7 @@ JSPromise* Modules::importUncompiled(const Identifier& key, ModuleID referrer, R auto scope = DECLARE_THROW_SCOPE(*m_instance.vm); VM& vm = *m_instance.vm; auto askLoader = [&] { - SetForScope frameOfImportThatAsksLoader(vm.aotState().frameOfImportThatAsksLoader, vm.topCallFrame); + SetForScope importFrameInLoader(vm.aotState().importFrameInLoader, vm.topCallFrame); return loader()->requestImportModuleForKey(realm, key, moduleKey(referrer), WTF::move(fetchParameters), nullptr, false, asyncEvaluationOrder(referrer).value_or(-1)); }; std::optional found = fetchParameters ? std::nullopt : m_table.findByPath(key.string()); @@ -5833,9 +5835,9 @@ JSObject* Modules::uncompiledNamespace(ModuleID external, ModulePhase phase, Nam RETURN_IF_EXCEPTION(scope, nullptr); if (phase != ModulePhase::Evaluation) return namespaceObject; - bool wasAskedFor = m_uncompiledNamespacesAskedFor.set(external.index()); + bool wasRequested = m_uncompiledNamespacesAskedFor.set(external.index()); if (!isShown) { - if (!wasAskedFor) + if (!wasRequested) m_uncompiledNamespacesNotShownYet.set(external.index()); return namespaceObject; } @@ -7362,7 +7364,7 @@ AbstractModuleRecord* moduleRecord(JSGlobalObject* realm, ModuleID module) modules->releaseRemovedMock(module); RETURN_IF_EXCEPTION(scope, nullptr); VMState& state = realm->vm().aotState(); - bool isFirstAskedForByImport = state.frameOfImportThatAsksLoader && (!state.stackPointerOfImport || state.stackPointerOfImport > static_cast(state.frameOfImportThatAsksLoader)); + bool isFirstAskedForByImport = state.importFrameInLoader && (!state.stackPointerOfImport || state.stackPointerOfImport > static_cast(state.importFrameInLoader)); SetForScope stackPointerOfImport(state.stackPointerOfImport, isFirstAskedForByImport ? currentStackPointer() : state.stackPointerOfImport); if (!modules->isInHostRequest()) { if (std::optional failure = modules->loadFailure(module); failure && !modules->statementThatEndsLoad(module)) [[unlikely]] { @@ -7772,12 +7774,12 @@ bool hasUnevaluatedActiveMockImporter(JSGlobalObject* realm, ModuleID module, st return modules->isGraphRunning() && modules->hasUnevaluatedLinkedImporter(module, asking); } -bool tryKeepActiveMock(JSGlobalObject* realm, ModuleID module, bool staysTheOriginal, bool keepsCurrentTestFileLoads) +bool tryKeepActiveMock(JSGlobalObject* realm, ModuleID module, bool remainsOriginal, bool keepsCurrentTestFileLoads) { if (!hasActiveMock(realm, module)) return false; Modules* modules = modulesOf(realm); - if (staysTheOriginal || (keepsCurrentTestFileLoads && modules->wasLoadedUnderMockInPlaceByThisTestFile(module))) { + if (remainsOriginal || (keepsCurrentTestFileLoads && modules->wasLoadedUnderMockInPlaceByThisTestFile(module))) { modules->noteCallOfMock(module, true); modules->releaseImportersOf(module, Modules::ReleasedImporters::AllButThoseThatThisTestFileLinkedToModule); return true; @@ -7886,10 +7888,10 @@ bool wasLinkedByFailedGraph(JSGlobalObject* realm, ModuleID module, bool byEarli return modules && modules->wasLinkedByFailedGraph(module, byEarlierTestFile); } -bool isLinkedForImporterYetToRun(JSGlobalObject* realm, ModuleID module, std::optional asking) +bool isLinkedForPendingImporter(JSGlobalObject* realm, ModuleID module, std::optional asking) { Modules* modules = modulesOf(realm); - return modules && modules->isLinkedForImporterYetToRun(module, asking); + return modules && modules->isLinkedForPendingImporter(module, asking); } bool wasLinkedBeforeLinkFailure(JSGlobalObject* realm, ModuleID module) @@ -7904,10 +7906,10 @@ bool wasDisplacedAfterFailure(JSGlobalObject* realm, ModuleID module) return modules && modules->wasDisplacedAfterFailure(module); } -bool isFailedImportRoot(JSGlobalObject* realm, ModuleID module) +bool isFailedLoadRoot(JSGlobalObject* realm, ModuleID module) { Modules* modules = modulesOf(realm); - return modules && modules->isFailedImportRoot(module); + return modules && modules->isFailedLoadRoot(module); } void clearLoadedBeforeTestFile(JSGlobalObject* realm, ModuleID module) diff --git a/Source/JavaScriptCore/aot/runtime/AOTModules.h b/Source/JavaScriptCore/aot/runtime/AOTModules.h index 1b36ce9346a15..804d23fbc9002 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTModules.h +++ b/Source/JavaScriptCore/aot/runtime/AOTModules.h @@ -217,6 +217,7 @@ class Modules { } runScheduledVariantRestores(); m_variantsFetchedByTestFile.clearAll(); + m_refetchedFailures.clearAll(); BitVector evictedFailures = m_modulesReleasedByLoader; evictedFailures.filter(m_loadedInPreload); for (BitVector* letGo : { &m_modulesReleasedByLoader, &m_importersOfMockThatEnded }) { @@ -446,7 +447,7 @@ class Modules { bool isMockLingering(ModuleID module) const { return m_modulesWithLingeringMock.get(module.index()); } bool isRequestedByDetachedLoad(ModuleID mocked) const { return m_resumingDetachedLoad && staticImportClosure(*m_resumingDetachedLoad, MockTraversal::Stop).get(mocked.index()); } void retryMockInDetachedLoads(ModuleID mocked); - uint64_t resumingLoadFactoryCall(ModuleID mocked) const; + uint64_t factoryCallOfResumedLoad(ModuleID mocked) const; bool needsRetryInDetachedLoad(ModuleID mocked) const { return m_detachedLoads.containsIf([&](const auto& left) { return left.wait->status() == JSPromise::Status::Pending && left.modulesToRetry.get(mocked.index()); }); } void didResumeAfterFactories() { m_hasImportsBlockedOnFactories = false; } bool tryMarkMockLingering(ModuleID module, bool wasInstalledOnPreloadMock) @@ -521,11 +522,18 @@ class Modules { } bool wasReleasedByLoader(ModuleID module) const { return m_modulesReleasedByLoader.get(module.index()); } void releaseFailedESModules(); + void forgetFailedLoad(ModuleID module) + { + m_refetchedFailures.clear(module.index()); + m_loadRoots.clear(module.index()); + } bool hasFailedESModule(ModuleID module) const { return isFailedESModule(module) || errorOfModuleThatRanBesideMock(module); } void releaseFailedESModule(ModuleID module) { - if (hasFailedESModule(module)) - m_modulesReleasedByLoader.set(module.index()); + if (!hasFailedESModule(module)) + return; + m_modulesReleasedByLoader.set(module.index()); + forgetFailedLoad(module); } void forgetFailuresOfExportsThatBuildFoundAbsent(); void bringBackWhatLoaderLetGoBelow(ModuleID root); @@ -740,12 +748,12 @@ class Modules { bool wasRegisteredByFailedLoad(ModuleID) const; bool wasLinkedByFailedGraph(ModuleID, bool byEarlierTestFile); bool isImportedByFailedRoot(ModuleID, bool ofEarlierTestFile = false) const; - bool isFailedImportRoot(ModuleID module) const { return m_importCallRoots.get(module.index()) && status(module) == ModuleStatus::Failed; } + bool isFailedLoadRoot(ModuleID module) const { return m_loadRoots.get(module.index()) && status(module) == ModuleStatus::Failed; } std::optional moduleThatFailedToLink(ModuleID root) const; bool hasLinkFailures() const { return m_errorsOfExportsThatBuildFoundAbsent || !m_linkFailures.isEmpty(); } bool wasLinkedAheadOfFailure(ModuleID root, ModuleID) const; bool wasLinkedBeforeLinkFailure(ModuleID) const; - bool isLinkedForImporterYetToRun(ModuleID, std::optional asking) const; + bool isLinkedForPendingImporter(ModuleID, std::optional asking) const; std::optional failureByAbsentExport(ModuleID) const; bool tryEvaluateESModuleBeforeMock(ModuleID); bool startDetachedOriginalLoad(ModuleID); @@ -2383,6 +2391,7 @@ class Modules { BitVector m_modulesWhoseMockOfPreloadIsPatched; BitVector m_mocksOfPreloadThatCameBackBeforeAnyImport; BitVector m_modulesReleasedByLoader; + BitVector m_refetchedFailures; BitVector m_importersOfMockThatEnded; BitVector m_modulesLoadedAnewWhoseNamesAreOwed; BitVector m_modulesWithNoNamesOwedBelow; @@ -2666,7 +2675,7 @@ class Modules { void takeBackFromHost(std::span files); BitVector m_keysWithoutEntryBehindMock; BitVector m_rootsOfImportsThatFailedUnderMocks; - BitVector m_importCallRoots; + BitVector m_loadRoots; BitVector m_modulesWithUnevictableFailure; BitVector m_mocksWhoseEntryProgramDeleted; BitVector m_keysWithFalsyEntryOfProgram; diff --git a/Source/JavaScriptCore/aot/runtime/AOTTestRunFactories.cpp b/Source/JavaScriptCore/aot/runtime/AOTTestRunFactories.cpp index f7bbd8ef8a27c..3367322551c7c 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTTestRunFactories.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTTestRunFactories.cpp @@ -227,7 +227,7 @@ bool Modules::wasLinkedBeforeLinkFailure(ModuleID module) const return false; } -bool Modules::isLinkedForImporterYetToRun(ModuleID module, std::optional asking) const +bool Modules::isLinkedForPendingImporter(ModuleID module, std::optional asking) const { if (!hasLinkFailures() || status(module) > ModuleStatus::Linked || !wasLinkedBeforeLinkFailure(module)) return false; @@ -522,8 +522,8 @@ void Modules::deferMock(ModuleID module, FactoryFailurePolicy failurePolicy) finishLoadsAwaitingLoader(module); keepMockForPendingImports(module, MockChange::Replaced); noteDeedOnMockUnderCallsOfImport(module, DeedOnMock::LaysAnother); - bool ranAheadOfThisTestFile = data(module).kind == ModuleKind::CommonJs && wasLoadedBeforeTestFile(module); - if ((m_modulesWithDeferredMock.get(module.index()) || ranAheadOfThisTestFile) && !activeMock(module) && !isPatchedByMock(module)) + bool wasEvaluatedBeforeTestFile = data(module).kind == ModuleKind::CommonJs && wasLoadedBeforeTestFile(module); + if ((m_modulesWithDeferredMock.get(module.index()) || wasEvaluatedBeforeTestFile) && !activeMock(module) && !isPatchedByMock(module)) releaseImportersOf(module, ReleasedImporters::All); if (hasDeferredFetchOf(fileOf(module))) m_filesToEvictOnceFetched.set(fileOf(module).index()); @@ -1009,7 +1009,7 @@ bool Modules::runMocksInFetchedGraphSlow(ModuleID module) scope.release(); if (m_modulesWithPendingMock.isEmpty()) [[likely]] return false; - bool isAskedForByImportCall = m_instance.vm->aotState().frameOfImportThatAsksLoader; + bool isAskedForByImportCall = m_instance.vm->aotState().importFrameInLoader; if (isAskedForByImportCall) clearCachedMockError(module); m_mocksWhoseFactoriesRanInsideTheirFetch.set(module.index(), m_modulesWithPendingMock.get(module.index())); @@ -1021,7 +1021,7 @@ bool Modules::runMocksInFetchedGraphSlow(ModuleID module) void Modules::willFetchAfterTick(ModuleID module) { - m_statementFetchesDelayedByTick.set(module.index(), !m_instance.vm->aotState().frameOfImportThatAsksLoader && isFetchedForImportStatement(module)); + m_statementFetchesDelayedByTick.set(module.index(), !m_instance.vm->aotState().importFrameInLoader && isFetchedForImportStatement(module)); } bool Modules::didMockFailForImportStatement(ModuleID module) @@ -1889,7 +1889,7 @@ void retryMockInDetachedLoads(JSGlobalObject* realm, ModuleID mocked) modules->retryMockInDetachedLoads(mocked); } -uint64_t Modules::resumingLoadFactoryCall(ModuleID mocked) const +uint64_t Modules::factoryCallOfResumedLoad(ModuleID mocked) const { if (!m_resumingDetachedLoad) return 0; @@ -1904,10 +1904,10 @@ uint64_t Modules::resumingLoadFactoryCall(ModuleID mocked) const return 0; } -uint64_t resumingLoadFactoryCall(JSGlobalObject* realm, ModuleID mocked) +uint64_t factoryCallOfResumedLoad(JSGlobalObject* realm, ModuleID mocked) { Modules* modules = modulesOf(realm); - return modules ? modules->resumingLoadFactoryCall(mocked) : 0; + return modules ? modules->factoryCallOfResumedLoad(mocked) : 0; } bool needsRetryInDetachedLoad(JSGlobalObject* realm, ModuleID mocked) diff --git a/Source/JavaScriptCore/aot/runtime/AOTTestRunImportTurns.cpp b/Source/JavaScriptCore/aot/runtime/AOTTestRunImportTurns.cpp index 02206b733099c..4ce1f2935ab78 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTTestRunImportTurns.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTTestRunImportTurns.cpp @@ -637,7 +637,7 @@ void didDrainDeferredFetchJobs(VM& vm) bool shouldDeferFetchFulfillment(JSGlobalObject* realm, ModuleID module) { - if (!realm->vm().aotState().frameOfImportThatAsksLoader) [[likely]] + if (!realm->vm().aotState().importFrameInLoader) [[likely]] return false; Modules* modules = modulesOf(realm); return modules && modules->isTestRun() && !modules->hasTextAtHand() && modules->textOfImportComesInLaterTurn(module, Modules::AskedBy::Loader); diff --git a/Source/JavaScriptCore/aot/runtime/AOTTestRunMockViews.cpp b/Source/JavaScriptCore/aot/runtime/AOTTestRunMockViews.cpp index e664834a2de4a..9513c0dba9196 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTTestRunMockViews.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTTestRunMockViews.cpp @@ -541,7 +541,9 @@ void Modules::bringBackLinks(const LinksBeforeLoad& before, LinksOfLoad links) } else m_momentsLoadedAnew.remove(module); } - m_modulesReleasedByLoader.merge(before.modulesReleasedByLoader); + BitVector released = before.modulesReleasedByLoader; + released.exclude(m_refetchedFailures); + m_modulesReleasedByLoader.merge(released); m_importersOfMockThatEnded.merge(before.importersOfMockThatEnded); } diff --git a/Source/JavaScriptCore/aot/runtime/AOTTestRunMocksLetGo.cpp b/Source/JavaScriptCore/aot/runtime/AOTTestRunMocksLetGo.cpp index 6b593e6e5a0c8..66d069a4a754b 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTTestRunMocksLetGo.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTTestRunMocksLetGo.cpp @@ -245,8 +245,10 @@ void Modules::releaseFailedESModules() return; for (uint32_t index = 0; index < m_table.size(); ++index) { ModuleID module(index); - if (isFailedESModule(module) && !isMocked(module) && !activeMock(module)) + if (isFailedESModule(module) && !isMocked(module) && !activeMock(module)) { m_modulesReleasedByLoader.set(index); + forgetFailedLoad(module); + } } } @@ -312,8 +314,11 @@ void Modules::bringBackWhatLoaderLetGoBelow(ModuleID root) wasLetGo |= take(m_modulesReleasedByLoader, &LinksBeforeLoad::modulesReleasedByLoader, *file); if (!wasLetGo && status(module) > ModuleStatus::Linked) continue; - if (wasLetGo) + if (wasLetGo) { tellThatFileDidNotRunAgain(module); + if (m_hasBegunTestFile && isFailedESModule(module)) + m_refetchedFailures.set(module.index()); + } bool sawMockThatEnded = wasLetGo && take(m_importersOfMockThatEnded, &LinksBeforeLoad::importersOfMockThatEnded, module); if (wasLetGo && file) sawMockThatEnded |= take(m_importersOfMockThatEnded, &LinksBeforeLoad::importersOfMockThatEnded, *file); diff --git a/Source/JavaScriptCore/aot/runtime/AOTVMState.h b/Source/JavaScriptCore/aot/runtime/AOTVMState.h index 9ab4c4fb29978..af87f497fcf85 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTVMState.h +++ b/Source/JavaScriptCore/aot/runtime/AOTVMState.h @@ -183,7 +183,7 @@ struct VMState { void* exceptionCatcherStackPointer { nullptr }; const void* returnAddressOfCallThatCollected { nullptr }; void* stackPointerOfImport { nullptr }; - CallFrame* frameOfImportThatAsksLoader { nullptr }; + CallFrame* importFrameInLoader { nullptr }; bool requesterRejectsUncompiledCode { false }; bool isResolvingImportCall { false }; bool isResolvingImportCallWithoutAttributes { false }; diff --git a/Source/JavaScriptCore/aot/walk/AOTCodeVerifier.cpp b/Source/JavaScriptCore/aot/walk/AOTCodeVerifier.cpp index a7567c4829c89..737cfd9fbe276 100644 --- a/Source/JavaScriptCore/aot/walk/AOTCodeVerifier.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTCodeVerifier.cpp @@ -1416,7 +1416,7 @@ bool CodeVerifier::treatsUndefinedAsNaN(ValueID id, const Operation& user, uint3 } if (arithmeticOnTwoNumbersInCodeVerifier.contains(user.opcode) || allRelations.contains(user.opcode)) return operandIndex < 2; - if (user.opcode == Opcode::ToInt32 || user.opcode == Opcode::ToUint32 || user.opcode == Opcode::IntegerBound || allBitwiseOperators.contains(user.opcode)) + if (user.opcode == Opcode::ToInt32 || user.opcode == Opcode::ToUint32 || user.opcode == Opcode::ToNumber || user.opcode == Opcode::IntegerBound || allBitwiseOperators.contains(user.opcode)) return true; if (user.opcode == Opcode::TypeTest) { const Type& tested = m_types.at(user.testedType()); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalk.cpp b/Source/JavaScriptCore/aot/walk/AOTWalk.cpp index 6b16aceaa8680..56806446694bc 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalk.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalk.cpp @@ -2582,6 +2582,8 @@ void Walker::checkProof(Reference value, TypeID proven, TypeWithoutProof without std::optional Walker::leaveCheckOfWhatContainerHasJustGivenToTestBehind(Reference given) { + if (Representation form = representationOf(given); form != Representation::JSValue && form != Representation::Cell) + return std::nullopt; auto isCheckOfGiven = [&](const Operation& record) { return record.opcode == Opcode::CheckProof && record.valueOperand() == given.id(); }; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkChoices.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkChoices.cpp index 317bbf2dcf733..f448d57feb1e7 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkChoices.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkChoices.cpp @@ -878,6 +878,22 @@ static bool sharesStorageOfArrayInChoices(Walker& walker, const DataNode::Array& return std::ranges::all_of(array.elements, canLieInSharedStorageInChoices); } +static bool isBornToBeFilledInChoices(const DataNode::Array& array) +{ + return array.elements.size() > 1 && std::ranges::all_of(array.elements, [](const DataNode& element) { + const auto* number = element.tryAs(); + return number && !number->value; + }); +} + +static bool variableHoldsListInChoices(Walker& walker, DataID data) +{ + const Program& program = walker.program(); + const Function& function = program.at(walker.currentFunction()); + FunctionID body = program.at(function.module).body; + return function.isModuleBody() || (body && walker.facts().summaries().at(body).holdsWrittenList(data)); +} + static void chooseEncodingOfDataInChoices(Walker& walker, Draft& data) { RootOfConstant whereRootEnds = walker.constantEndsWhereItLeaves(data.inputValueGiven, data.data()) ? RootOfConstant::EndsWhereItLeaves : RootOfConstant::IsAsked; @@ -888,6 +904,10 @@ static void chooseEncodingOfDataInChoices(Walker& walker, Draft& data) SetForScope makingObjectsOfConstant = walker.makingObjectsOfConstant(); const DataNode& root = walker.program().at(data.data()).root; if (const auto* array = root.tryAs(); array && sharesStorageOfArrayInChoices(walker, *array)) { + if (array->elements.size() <= maxNodesOfExpandedData && isBornToBeFilledInChoices(*array) && !variableHoldsListInChoices(walker, data.data())) { + becomeNodeOfDataInChoices(walker, data, root); + return; + } bool isStoredIntoInLoop = data.inputValueGiven && walker.facts().summaries().at(walker.currentFunction()).constantListIsStoredIntoInLoop(data.inputValueGiven) && surveyDataInChoices(walker, data.data()).tableOfImageHoldsIt; data.setEncoding(isStoredIntoInLoop ? DataEncoding::TreeWithOwnStorage : DataEncoding::CopyOnWriteArray); return; diff --git a/Source/JavaScriptCore/assembler/MacroAssembler.h b/Source/JavaScriptCore/assembler/MacroAssembler.h index 0f782622c24e2..b3aaad80ecbe2 100644 --- a/Source/JavaScriptCore/assembler/MacroAssembler.h +++ b/Source/JavaScriptCore/assembler/MacroAssembler.h @@ -2025,7 +2025,7 @@ class MacroAssembler : public MacroAssemblerBase { uint32_t magnitude = value < 0 ? -static_cast(value) : value; uint32_t key = keyForConstant(magnitude); if (key > magnitude) - key %= magnitude; + key = magnitude ? key % magnitude : 0; key &= maskTable[magnitude & 3]; uint32_t rest = magnitude - key; return value < 0 ? BlindedImm32(-rest, -key) : BlindedImm32(rest, key); diff --git a/Source/JavaScriptCore/assembler/testmasm.cpp b/Source/JavaScriptCore/assembler/testmasm.cpp index 54cfc94b98a35..631ada560413c 100644 --- a/Source/JavaScriptCore/assembler/testmasm.cpp +++ b/Source/JavaScriptCore/assembler/testmasm.cpp @@ -6700,6 +6700,14 @@ static void testBlindedAdditions() }); CHECK_EQ(result.values[0], operands.values[0] + static_cast(immediate)); } + + CCallHelpers jit; + for (int32_t immediate : { 0, 1, -1, 2, -2, 0x100, -0x100, std::numeric_limits::max(), std::numeric_limits::min() }) { + for (unsigned i = 0; i < 100; ++i) { + auto parts = jit.additionBlindedConstant(CCallHelpers::Imm32(immediate)); + CHECK_EQ(static_cast(parts.value1.m_value) + parts.value2.m_value, static_cast(immediate)); + } + } } static void testMaskedWordSign() diff --git a/Source/JavaScriptCore/b3/B3EliminateCommonSubexpressions.cpp b/Source/JavaScriptCore/b3/B3EliminateCommonSubexpressions.cpp index 8d17a3a78e82e..24a498a9a1964 100644 --- a/Source/JavaScriptCore/b3/B3EliminateCommonSubexpressions.cpp +++ b/Source/JavaScriptCore/b3/B3EliminateCommonSubexpressions.cpp @@ -617,7 +617,7 @@ class CSE { [&] (MemoryValue* candidate) -> bool { return candidate->opcode() == Store && candidate->offset() == offset - && candidate->accessWidth() >= clobberWidth; + && (candidate->accessWidth() == clobberWidth || (candidate->accessWidth() > clobberWidth && !candidate->traps())); }); break; } @@ -687,6 +687,10 @@ class CSE { worklist.pushAll(m_block->successorBlocks()); while (BasicBlock* block = worklist.pop()) { + // The next iteration computes another address. + if (block == ptr->owner) + return false; + ImpureBlockData& data = m_impureBlockData[block]; MemoryValue* match = data.memoryStoresAtHead.find(ptr, filter); diff --git a/Source/JavaScriptCore/b3/B3LowerToAir.cpp b/Source/JavaScriptCore/b3/B3LowerToAir.cpp index f238968882cdb..591393cde70ec 100644 --- a/Source/JavaScriptCore/b3/B3LowerToAir.cpp +++ b/Source/JavaScriptCore/b3/B3LowerToAir.cpp @@ -1239,7 +1239,7 @@ class LowerToAir { // An instruction that reads fewer bytes than the load does not trap where only the other bytes are out of bounds. bool canReadInPlace(Value* value, Width width) { - if (!m_procedure.policy().selection.extendsInstructionSelection || !value->traps()) + if (!value->traps()) return true; auto* memory = value->as(); return !memory || memory->accessWidth() <= width; diff --git a/Source/JavaScriptCore/b3/testb3_bun_selection.cpp b/Source/JavaScriptCore/b3/testb3_bun_selection.cpp index 79e1c2460fb31..4ea50153389f9 100644 --- a/Source/JavaScriptCore/b3/testb3_bun_selection.cpp +++ b/Source/JavaScriptCore/b3/testb3_bun_selection.cpp @@ -747,6 +747,63 @@ static void testBranchOnExtendedValueTestsNarrowValue() } } +static void testTrappingLoadReadsAllBytes() +{ + enum class Use { Mask, ExtendByte, ExtendHalf, BranchOnMask }; + auto build = [](Graph& graph, Type type, Use use, int64_t mask) { + Procedure& proc = graph.proc; + BasicBlock* root = proc.addBlock(); + Value* load = root->appendNew(proc, trapping(Load), type, Origin(), cCallArgumentValues(proc, root)[0], 8); + Value* masked = graph.make(root, BitAnd, load, root->appendIntConstant(proc, Origin(), type, mask)); + Value* low = type == Int64 ? graph.make(root, Trunc, load) : load; + switch (use) { + case Use::Mask: + graph.ret(root, masked); + return; + case Use::ExtendByte: + graph.ret(root, graph.make(root, SExt8, low)); + return; + case Use::ExtendHalf: + graph.ret(root, graph.make(root, SExt16, low)); + return; + case Use::BranchOnMask: { + BasicBlock* taken = proc.addBlock(); + BasicBlock* notTaken = proc.addBlock(); + graph.branch(root, masked, taken, notTaken); + graph.ret(taken, graph.int64(taken, 1)); + graph.ret(notTaken, graph.int64(notTaken, 2)); + return; + } + } + }; + for (bool usesPolicy : { false, true }) { + for (Type type : { Int32, Int64 }) { + for (Use use : { Use::Mask, Use::ExtendByte, Use::ExtendHalf, Use::BranchOnMask }) { + for (int64_t mask : { 1ll, 0xffll, 0x80ll, 0x100ll, 0xff00ll, 0xffffll, 0x7fffffffll }) { + Procedure proc(usesPolicy ? selectionPolicy() : Policy()); + Graph graph { proc }; + build(graph, type, use, mask); + proc.resetReachability(); + generateToAir(proc); + unsigned numberOfTrappingInsts = 0; + for (Air::BasicBlock* block : proc.code()) { + for (Air::Inst& inst : *block) { + inst.forEachArg([&](Air::Arg& arg, Air::Arg::Role, Bank, Width width) { + if (!arg.isMemory()) + return; + CHECK(inst.kind.effects); + CHECK_EQ(width, type == Int64 ? Width64 : Width32); + ++numberOfTrappingInsts; + }); + } + } + CHECK_EQ(numberOfTrappingInsts, 1u); + } + } + } + } +} + static void buildBranchOnLoad16(Graph& graph) { Procedure& proc = graph.proc; @@ -1413,6 +1470,7 @@ void addBunSelectionTests(const TestConfig* config, BunTestTasks& tasks) RUN(testFMinAndFMax()); RUN(testSelectAsMinOrMax()); RUN(testBranchOnExtendedValueTestsNarrowValue()); + RUN(testTrappingLoadReadsAllBytes()); RUN(testBranchOnLoad16()); RUN(testLoadsFarFromTheirUsers()); RUN(testPinnedRegisterIsNamedDirectly()); diff --git a/Source/JavaScriptCore/b3/testb3_bun_store_merging.cpp b/Source/JavaScriptCore/b3/testb3_bun_store_merging.cpp index 356931ef1c59b..5202a6772b6bf 100644 --- a/Source/JavaScriptCore/b3/testb3_bun_store_merging.cpp +++ b/Source/JavaScriptCore/b3/testb3_bun_store_merging.cpp @@ -8,6 +8,7 @@ #if ENABLE(B3_JIT) +#include "B3EliminateCommonSubexpressions.h" #include "B3MergeAdjacentConstantStores.h" #include @@ -266,6 +267,71 @@ static void testMergeZeroStoresWritesExactHeaps() CHECK(patchpoint->runsWithoutFrame()); } +static void testDeadStoreAcrossIterations() +{ + for (bool advances : { true, false }) { + Procedure proc; + Graph graph { proc }; + BasicBlock* root = proc.addBlock(); + BasicBlock* body = proc.addBlock(); + BasicBlock* latch = proc.addBlock(); + BasicBlock* exit = proc.addBlock(); + auto arguments = cCallArgumentValues(proc, root); + Value* start = arguments[0]; + Value* lastIndexAddress = arguments[1]; + Value* pointer = graph.phi(body, Int64); + Value* counter = graph.phi(body, Int32); + Value* address = advances ? pointer : start; + + graph.upsilon(root, start, pointer); + graph.upsilon(root, graph.int32(root, 0), counter); + graph.jump(root, body); + + graph.store(body, graph.int32(body, 1), address, 0, firstHeap); + Value* lastIndex = graph.load(body, Int32, lastIndexAddress, 0, firstHeap); + graph.store(body, graph.int32(body, 2), address, 0, firstHeap); + graph.branch(body, graph.make(body, Equal, counter, lastIndex), exit, latch); + + graph.upsilon(latch, graph.make(latch, Add, pointer, graph.int64(latch, sizeof(int32_t))), pointer); + graph.upsilon(latch, graph.make(latch, Add, counter, graph.int32(latch, 1)), counter); + graph.jump(latch, body); + + graph.store(exit, graph.int32(exit, 3), address, 0, firstHeap); + graph.ret(exit, graph.int32(exit, 0)); + resetAndValidate(proc); + + eliminateCommonSubexpressions(proc); + validate(proc); + CHECK_EQ(numberOfValues(proc, Store), advances ? 3u : 2u); + + std::array memory { }; + int32_t lastIndexInMemory = 3; + compileAndRun(proc, memory.data(), &lastIndexInMemory); + CHECK(memory == (advances ? std::array { 2, 2, 2, 3 } : std::array { 3, 0, 0, 0 })); + } +} + +static void testDeadStoreBeforeTrap() +{ + for (Type type : { Int32, Int64 }) { + for (bool traps : { true, false }) { + Procedure proc; + Graph graph { proc }; + BasicBlock* root = proc.addBlock(); + Value* pointer = cCallArgumentValues(proc, root)[0]; + graph.store(root, graph.int32(root, 1), pointer, 0, firstHeap); + Value* value = root->appendIntConstant(proc, Origin(), type, 2); + root->appendNew(proc, traps ? trapping(Store) : B3::Kind(Store), Origin(), value, pointer, 0, firstHeap); + graph.ret(root, graph.int32(root, 0)); + resetAndValidate(proc); + + eliminateCommonSubexpressions(proc); + validate(proc); + CHECK_EQ(numberOfValues(proc, Store), traps && type == Int64 ? 2u : 1u); + } + } +} + void addBunStoreMergingTests(const TestConfig* config, BunTestTasks& tasks) { RUN(testMergeAdjacentConstantStores()); @@ -274,6 +340,8 @@ void addBunStoreMergingTests(const TestConfig* config, BunTestTasks& tasks) RUN(testMergeZeroStoresIntoVectors()); RUN(testMergeZeroStoresKeepsOtherStores()); RUN(testMergeZeroStoresWritesExactHeaps()); + RUN(testDeadStoreAcrossIterations()); + RUN(testDeadStoreBeforeTrap()); } #endif // ENABLE(B3_JIT) diff --git a/Source/JavaScriptCore/interpreter/Interpreter.cpp b/Source/JavaScriptCore/interpreter/Interpreter.cpp index c51a64133d597..7a12f029496e8 100644 --- a/Source/JavaScriptCore/interpreter/Interpreter.cpp +++ b/Source/JavaScriptCore/interpreter/Interpreter.cpp @@ -853,7 +853,7 @@ void Interpreter::getStackTrace(JSCell* owner, Vector& results, size #if ENABLE(AOT) if (vm.hasCompiledCode()) [[unlikely]] { - if (skippedFrames < framesToSkip && vm.aotState().frameOfImportThatAsksLoader == callFrame) [[unlikely]] + if (skippedFrames < framesToSkip && vm.aotState().importFrameInLoader == callFrame) [[unlikely]] ++skippedFrames; if (caller || !AOT::tryGetStackTrace(vm, owner, results, framesToSkip - skippedFrames, maxStackSize, filter)) appendStackTrace(owner, results, { .callFrame = callFrame, .framesToSkip = framesToSkip - skippedFrames, .caller = caller, .filter = filter }, maxStackSize); diff --git a/Source/JavaScriptCore/wasm/WasmCompilationContext.h b/Source/JavaScriptCore/wasm/WasmCompilationContext.h index c6b6258692ace..654794cdb689e 100644 --- a/Source/JavaScriptCore/wasm/WasmCompilationContext.h +++ b/Source/JavaScriptCore/wasm/WasmCompilationContext.h @@ -75,7 +75,7 @@ struct CompilationContext { Box pcToCodeOriginMapBuilder; Vector catchEntrypoints; CCallHelpers::Label directEntry; - // The direct entry for callers that omit a zero argument. + // The direct entry for callers whose first argument is zero. They omit the argument of a resumable function. Box directEntryWithZero; SegmentedVector origins; CodeForImage* codeForImage { nullptr }; diff --git a/Source/JavaScriptCore/wasm/WasmOMGIRGenerator.cpp b/Source/JavaScriptCore/wasm/WasmOMGIRGenerator.cpp index 78b20070282f4..c0c06d2bee7ab 100644 --- a/Source/JavaScriptCore/wasm/WasmOMGIRGenerator.cpp +++ b/Source/JavaScriptCore/wasm/WasmOMGIRGenerator.cpp @@ -32,6 +32,8 @@ #include "AirArgInlines.h" #include "AirCode.h" #include "AirInstInlines.h" +#include "AirOpcodeUtils.h" +#include "AirReuseConditionFlags.h" #include "AllowMacroScratchRegisterUsageIf.h" #include "B3AbstractHeapRepository.h" #include "B3BasicBlockInlines.h" @@ -1236,6 +1238,7 @@ class OMGIRGenerator { Variable* m_passedGlobal { nullptr }; bool m_accessesPassedGlobal { false }; bool m_isResumable { false }; + bool m_isFirstParameterI32 { false }; bool m_hasLoop { false }; Box m_stackOverflows; bool m_makesTailCalls { false }; @@ -1537,6 +1540,8 @@ OMGIRGenerator::OMGIRGenerator(AbstractHeapRepository& heaps, CompilationContext if (m_context.codeForImage && m_context.codeForImage->passedGlobal && emitsShrinkWrappedFrame()) { m_passedGlobal = m_proc.addVariable(Int32); m_isResumable = isResumable(m_info, FunctionCodeIndex(m_functionIndex), *m_context.codeForImage->passedGlobal); + const RTT& signature = m_info.rtt(m_info.internalFunctionTypeSignatureIndices[m_functionIndex]); + m_isFirstParameterI32 = signature.argumentCount() && signature.argumentType(0).isI32(); Value* argument = m_topLevelBlock->appendNew(m_proc, Origin(), passedGlobalGPR); m_topLevelBlock->appendNew(m_proc, Set, Origin(), m_passedGlobal, m_topLevelBlock->appendNew(m_proc, Trunc, Origin(), argument)); } @@ -1780,29 +1785,25 @@ void OMGIRGenerator::emitCallToThrowStub(CCallHelpers& jit) jit.nearCallThunk(CodeLocationLabel(Thunks::singleton().stub(throwExceptionFromOMGThunkGenerator).code())); } -static bool mayRead(B3::Air::BasicBlock* start, Reg reg) +enum class AccessKind : uint8_t { None, Read, Write }; + +// Whether a path from the head of the block reads what is there before it writes it. +static bool mayRead(B3::Air::BasicBlock* start, NOESCAPE const Invocable auto& findAccess) { IndexSet seen; Vector worklist { start }; seen.add(start); while (!worklist.isEmpty()) { B3::Air::BasicBlock* block = worklist.takeLast(); - bool isWritten = false; + AccessKind access = AccessKind::None; for (B3::Air::Inst& inst : *block) { - bool isRead = false; - inst.forEach([&](Reg& used, B3::Air::Arg::Role role, Bank, Width) { - if (used != reg) - return; - isRead |= B3::Air::Arg::isAnyUse(role); - isWritten |= B3::Air::Arg::isAnyDef(role); - }); - if (isRead) - return true; - isWritten |= inst.kind.opcode == B3::Air::Patch && inst.extraClobberedRegs().contains(reg, IgnoreVectors); - if (isWritten) + access = findAccess(inst); + if (access != AccessKind::None) break; } - if (isWritten) + if (access == AccessKind::Read) + return true; + if (access == AccessKind::Write) continue; for (B3::Air::BasicBlock* successor : block->successorBlocks()) { if (seen.add(successor)) @@ -1812,28 +1813,98 @@ static bool mayRead(B3::Air::BasicBlock* start, Reg reg) return false; } -// The successor taken for zero, if the entry block only tests the register and nothing reads it afterward. -static B3::Air::BasicBlock* findZeroSuccessor(B3::Air::BasicBlock* entry, Reg reg) +static bool mayReadRegister(B3::Air::BasicBlock* start, Reg reg) { - B3::Air::Inst& branch = entry->last(); - bool isReadBeforeTest = false; - for (B3::Air::Inst& inst : *entry) { + return mayRead(start, [&](B3::Air::Inst& inst) { + bool isRead = false; + bool isWritten = inst.kind.opcode == B3::Air::Patch && inst.extraClobberedRegs().contains(reg, IgnoreVectors); + inst.forEach([&](Reg& used, B3::Air::Arg::Role role, Bank, Width) { + if (used != reg) + return; + isRead |= B3::Air::Arg::isAnyUse(role); + isWritten |= B3::Air::Arg::isAnyDef(role); + }); + return isRead ? AccessKind::Read : isWritten ? AccessKind::Write : AccessKind::None; + }); +} + +static bool mayReadFlags(B3::Air::BasicBlock* start) +{ + return mayRead(start, [](B3::Air::Inst& inst) { + if (B3::Air::usesFlags(inst.kind.opcode)) + return AccessKind::Read; + return B3::Air::preservesFlags(inst.kind.opcode) || B3::Air::isFlagPreservingMove(inst) ? AccessKind::None : AccessKind::Write; + }); +} + +struct ZeroTest { + unsigned index; + B3::Air::BasicBlock* zeroSuccessor; +}; + +// The instructions from the index to the end of the block branch on whether the register is zero and do nothing else. +static std::optional findZeroTest(B3::Air::BasicBlock* block, Reg reg) +{ + using B3::Air::Arg; + const Arg tested { B3::Air::Tmp(reg) }; + auto isTest = [&](const Arg& value, const Arg& mask) { + return value == tested && (mask == tested || mask == Arg::bitImm(-1)); + }; + auto isComparison = [&](const Arg& left, const Arg& right) { + return left == tested && right == Arg::imm(0); + }; + unsigned index = block->size() - 1; + std::span args = block->last().args(); + switch (block->last().kind.opcode) { + case B3::Air::BranchTest32: + if (!isTest(args[1], args[2])) + return std::nullopt; + break; + case B3::Air::Branch32: + if (!isComparison(args[1], args[2])) + return std::nullopt; + break; + case B3::Air::BranchOnFlags: { + if (!index) + return std::nullopt; + B3::Air::Inst& definer = block->at(--index); + bool testsRegister = definer.kind.opcode == B3::Air::TestOnFlags32 && isTest(definer.args()[0], definer.args()[1]); + bool comparesRegister = definer.kind.opcode == B3::Air::CompareOnFlags32 && isComparison(definer.args()[0], definer.args()[1]); + if (!testsRegister && !comparesRegister) + return std::nullopt; + break; + } + default: + return std::nullopt; + } + if (args[0] == Arg::resCond(MacroAssembler::Zero) || args[0] == Arg::relCond(MacroAssembler::Equal)) + return ZeroTest { index, block->successorBlock(0) }; + if (args[0] == Arg::resCond(MacroAssembler::NonZero) || args[0] == Arg::relCond(MacroAssembler::NotEqual)) + return ZeroTest { index, block->successorBlock(1) }; + return std::nullopt; +} + +// The successor taken for zero, if the block ends in a test of the register before which it only moves others, and the successor does not read the flags of the test. +static B3::Air::BasicBlock* findZeroSuccessor(B3::Air::BasicBlock* block, Reg reg) +{ + std::optional test = findZeroTest(block, reg); + if (!test || test->zeroSuccessor == block || mayReadFlags(test->zeroSuccessor)) + return nullptr; + for (unsigned index = 0; index < test->index; ++index) { + B3::Air::Inst& inst = block->at(index); auto* patchpoint = inst.kind.opcode == B3::Air::Patch ? inst.origin->as() : nullptr; bool isMove = inst.isMoveBetweenRegisters() && inst.args()[1] != B3::Air::Arg(B3::Air::Tmp(reg)); - if (&inst != &branch && !isMove && (!patchpoint || !patchpoint->emitsNoCode)) + if (!isMove && (!patchpoint || !patchpoint->emitsNoCode)) return nullptr; - isReadBeforeTest |= isMove && inst.args()[0] == B3::Air::Arg(B3::Air::Tmp(reg)); } - if (branch.kind.opcode != B3::Air::BranchTest32 || branch.args()[1] != B3::Air::Arg(B3::Air::Tmp(reg))) - return nullptr; - B3::Air::Arg mask = branch.args()[2]; - if (mask != branch.args()[1] && mask != B3::Air::Arg::bitImm(-1)) - return nullptr; - MacroAssembler::ResultCondition condition = branch.args()[0].asResultCondition(); - if (condition != MacroAssembler::Zero && condition != MacroAssembler::NonZero) - return nullptr; - B3::Air::BasicBlock* block = entry->successorBlock(condition == MacroAssembler::Zero ? 0 : 1); - return isReadBeforeTest || mayRead(block, reg) ? nullptr : block; + return test->zeroSuccessor; +} + +static bool movesRegister(B3::Air::BasicBlock* block, Reg reg) +{ + return std::ranges::any_of(*block, [&](const B3::Air::Inst& inst) { + return inst.isMoveBetweenRegisters() && inst.args()[0] == B3::Air::Arg(B3::Air::Tmp(reg)); + }); } void OMGIRGenerator::emitZeroEntry(CCallHelpers& jit, B3::Air::GenerationContext& context, const Box& zeroSuccessor) @@ -1862,22 +1933,28 @@ void OMGIRGenerator::emitZeroEntry(CCallHelpers& jit, B3::Air::GenerationContext void OMGIRGenerator::emitDirectEntry(CCallHelpers& jit, B3::Air::GenerationContext& context) { jit.load32(CCallHelpers::Address(GPRInfo::wasmContextInstancePointer, safeCast(JSWebAssemblyInstance::offsetOfGlobal(m_info, *m_context.codeForImage->passedGlobal))), passedGlobalGPR); + GPRReg firstParameter = wasmCallingConvention().gprArgs[0]; + B3::Air::BasicBlock* zeroSuccessor = m_isFirstParameterI32 ? findZeroSuccessor(context.currentBlock, firstParameter) : nullptr; + auto labelOf = [&](B3::Air::BasicBlock* block) -> const Box& { + auto& label = context.blockLabels[block]; + if (!label) + label = Box::create(); + return label; + }; if (m_isResumable) { - GPRReg state = wasmCallingConvention().gprArgs[0]; - if (B3::Air::BasicBlock* block = findZeroSuccessor(context.currentBlock, state)) { - auto& label = context.blockLabels[block]; - if (!label) - label = Box::create(); - emitZeroEntry(jit, context, label); - } else if (mayRead(context.currentBlock, state)) { + // The callers that enter there omit the argument. + if (zeroSuccessor && !movesRegister(context.currentBlock, firstParameter) && !mayReadRegister(zeroSuccessor, firstParameter)) + emitZeroEntry(jit, context, labelOf(zeroSuccessor)); + else if (mayReadRegister(context.currentBlock, firstParameter)) { auto hasState = jit.jump(); m_context.directEntryWithZero = Box::create(jit.label()); - jit.move(CCallHelpers::TrustedImm32(0), state); + jit.move(CCallHelpers::TrustedImm32(0), firstParameter); hasState.link(&jit); } - } + } else if (zeroSuccessor && std::ranges::none_of(*context.currentBlock, &B3::Air::Inst::isMoveBetweenRegisters)) + m_context.directEntryWithZero = labelOf(zeroSuccessor); m_context.directEntry = jit.label(); - if (m_isResumable && !m_context.directEntryWithZero) + if (!m_context.directEntryWithZero) m_context.directEntryWithZero = Box::create(m_context.directEntry); } From 348972fa763d1dd5767d348041a894bb10ce7670 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 10 Nov 2026 06:12:09 -0800 Subject: [PATCH 04/10] AOT: untyped fields stay stable in loops, the injected fault of the fixpoint check at super() calls On 02c32c51. 6 files, +17 -7: Source/JavaScriptCore/aot 6 files +17 -7. - A field that a loop does not store into is read in front of the loop again where it has no type, or a declared type and no nullish value. - The fault that a test injects into the check of the fixpoint stands at every super() call, so that the test sees it. --- Source/JavaScriptCore/aot/facts/AOTLinkCalls.cpp | 6 +++++- Source/JavaScriptCore/aot/ir/AOTTypeArena.h | 1 + Source/JavaScriptCore/aot/plan/AOTSlotPlan.cpp | 3 ++- Source/JavaScriptCore/aot/plan/AOTSlotPlan.h | 1 + Source/JavaScriptCore/aot/walk/AOTWalk.h | 2 +- .../JavaScriptCore/aot/walk/AOTWalkStableBindings.cpp | 11 +++++++---- 6 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Source/JavaScriptCore/aot/facts/AOTLinkCalls.cpp b/Source/JavaScriptCore/aot/facts/AOTLinkCalls.cpp index 34ef9a0f95aa1..acf4586c93b0c 100644 --- a/Source/JavaScriptCore/aot/facts/AOTLinkCalls.cpp +++ b/Source/JavaScriptCore/aot/facts/AOTLinkCalls.cpp @@ -7237,7 +7237,11 @@ void LinkFlow::callObserved(const FlowObserver& observer, const LinkType& told) return; uint32_t numberOfCallObserved = firstCallOfFunction[observer.function.index()] + observer.record; bool isEnteredNoMore = state.callsOfUnseen[numberOfCallObserved] == StateOfLinkCalls::hasGivenUpInSight || (state.callsOfUnseen[numberOfCallObserved] && !state.callsThatEnterMembersOfTable[numberOfCallObserved]); - if (isEnteredNoMore && hasSameObjectsAndFunctions(told, typeObservedBefore()) && told.hasDirectUnknownObjects() == typeObservedBefore().hasDirectUnknownObjects() && (told.hasUnknownObjects == typeObservedBefore().hasUnknownObjects || summaries.at(observer.function).calls[observer.record].newTarget == noSummaryNode || AOT::hasFlag(build.options().testing, Wire::TestingSwitch::IgnoreEscapeAfterUnknownCall))) + auto hasNewTarget = [&] { + return summaries.at(observer.function).calls[observer.record].newTarget != noSummaryNode; + }; + bool ignoresNewTarget = AOT::hasFlag(build.options().testing, Wire::TestingSwitch::IgnoreEscapeAfterUnknownCall) && hasNewTarget(); + if ((isEnteredNoMore || ignoresNewTarget) && hasSameObjectsAndFunctions(told, typeObservedBefore()) && (ignoresNewTarget || (told.hasDirectUnknownObjects() == typeObservedBefore().hasDirectUnknownObjects() && (told.hasUnknownObjects == typeObservedBefore().hasUnknownObjects || !hasNewTarget())))) return; if (!std::exchange(state.callsToResolve[firstCallOfFunction[observer.function.index()] + observer.record], 1)) state.observersToResolve[laneOfObserversOf(observer.function)].append(observer); diff --git a/Source/JavaScriptCore/aot/ir/AOTTypeArena.h b/Source/JavaScriptCore/aot/ir/AOTTypeArena.h index 82791b73f7a03..88217b751d23a 100644 --- a/Source/JavaScriptCore/aot/ir/AOTTypeArena.h +++ b/Source/JavaScriptCore/aot/ir/AOTTypeArena.h @@ -43,6 +43,7 @@ enum class AnswerKeptInArena : uint8_t { ReadFindsFieldWithoutWrittenType, LayoutsThatStayWithCompiledCode, FieldOfLayoutsIsStable, + DeclaredFieldOfLayoutsIsStable, NumberOfAnswers, }; diff --git a/Source/JavaScriptCore/aot/plan/AOTSlotPlan.cpp b/Source/JavaScriptCore/aot/plan/AOTSlotPlan.cpp index 5a6b470f57f27..1805b380ac89c 100644 --- a/Source/JavaScriptCore/aot/plan/AOTSlotPlan.cpp +++ b/Source/JavaScriptCore/aot/plan/AOTSlotPlan.cpp @@ -1613,11 +1613,12 @@ void holdDoorsToProofsInSlotPlan(Program& program, const Facts& facts, const Par if (const LinkedCell* cell = mayBeStable ? facts.linked().findCellOfField(reference) : nullptr) slot.isStable = cell->everyWriterIsKnown && !cell->isWrittenAfterConstruction && cell->isFilledAtEveryRead; TypeID proven = facts.linked().provenOfField(reference); - slot.isStable &= !!proven; + slot.isStable &= proven || !slot.type; if (!proven) continue; slot.everyStoreFits = !slot.type || types.isSubset(proven, slot.type); slot.isStable &= slot.everyStoreFits; + slot.isStableWithoutNullish = slot.isStable && !(types.at(proven).primitives & (Primitive::UndefinedOrNull | Primitive::Absent)); if (!slot.everyStoreFits || !hasDoors || !slot.indexInLayoutRecord || !slot.hasStorage() || slot.holdsAccessor || slot.deferredFunction) continue; fieldsWithDoorsInRanges[range].append(reference); diff --git a/Source/JavaScriptCore/aot/plan/AOTSlotPlan.h b/Source/JavaScriptCore/aot/plan/AOTSlotPlan.h index c4c061d933b85..1259ad9bc58e5 100644 --- a/Source/JavaScriptCore/aot/plan/AOTSlotPlan.h +++ b/Source/JavaScriptCore/aot/plan/AOTSlotPlan.h @@ -106,6 +106,7 @@ struct PlacedSlot { bool deferredClosureCapturesEnvironment { false }; bool mayTakeWordLeftInCell { false }; bool isStable { false }; + bool isStableWithoutNullish { false }; uint32_t offset { 0 }; std::optional indexInLayoutRecord; FunctionID deferredFunction; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalk.h b/Source/JavaScriptCore/aot/walk/AOTWalk.h index cb045feef5417..fc048125c2d97 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalk.h +++ b/Source/JavaScriptCore/aot/walk/AOTWalk.h @@ -1510,7 +1510,7 @@ class Walker { uint32_t& numberOfStretchesBehindBytesOfMembers() { return m_numberOfStretchesBehindBytesOfMembers; } bool shouldAssumeCalleesRetainNothing() const { return m_inputs.options.shouldAssumeCalleesRetainNothing; } bool framesKeepFields() const; - bool isStableFieldOf(LayoutSetID receivers, PropertyKeyID) const; + bool isStableFieldOf(LayoutSetID receivers, PropertyKeyID, bool holdsDeclaredType) const; bool readsStableField(const Draft& read) const; Reference valueOfStableField(const Draft& read, TypeID proven) const; void noteStableField(const Draft& read, Reference value); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkStableBindings.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkStableBindings.cpp index 18e1304f16808..cdb918941627b 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkStableBindings.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkStableBindings.cpp @@ -70,18 +70,21 @@ bool Walker::framesKeepFields() const return *last.keepsFields; } -bool Walker::isStableFieldOf(LayoutSetID receivers, PropertyKeyID key) const +bool Walker::isStableFieldOf(LayoutSetID receivers, PropertyKeyID key, bool holdsDeclaredType) const { auto isStableIn = [&](LayoutID member) { std::optional field = plan().slots().findField(program(), member, key); - return field && plan().slots().slot(member, *field).isStable; + if (!field) + return false; + const PlacedSlot& slot = plan().slots().slot(member, *field); + return holdsDeclaredType ? slot.isStableWithoutNullish : slot.isStable; }; std::span members = membersOf(m_types, receivers); if (members.size() <= 1) return !members.empty() && isStableIn(members.front()); if (m_types.keeps() != ArenaKeeps::UntilReset) return std::ranges::all_of(members, isStableIn); - constexpr uint8_t question = static_cast(AnswerKeptInArena::FieldOfLayoutsIsStable); + uint8_t question = static_cast(holdsDeclaredType ? AnswerKeptInArena::DeclaredFieldOfLayoutsIsStable : AnswerKeptInArena::FieldOfLayoutsIsStable); TypeArena::SetOfRound round = m_types.setOfRound(receivers); if (std::optional kept = m_types.rememberedRound(question, 1, round, key.bits())) return (*kept)[0]; @@ -92,7 +95,7 @@ bool Walker::isStableFieldOf(LayoutSetID receivers, PropertyKeyID key) const bool Walker::readsStableField(const Draft& read) const { - if (read.level() != KnowledgeLevel::Proven || read.holdsDeclaredType() || read.receiverMayBeInBirth() || !isStableFieldOf(read.receivers(), read.key()) || testsItsOwnGuess(read)) + if (read.level() != KnowledgeLevel::Proven || read.receiverMayBeInBirth() || !isStableFieldOf(read.receivers(), read.key(), read.holdsDeclaredType()) || testsItsOwnGuess(read)) return false; Reference object = read.objectOperand(); if (!object || !object.isValue() || !framesKeepFields()) From 83468e9234d550d318825abbbedbf1f37fe62b5f Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 10 Nov 2026 06:40:56 -0800 Subject: [PATCH 05/10] AOT: conversions between floats and data segments in WebAssembly, one length check for written indexes, a leak report On 348972fa. 9 files, +219 -14: Source/JavaScriptCore/aot 2 files +24 -1 . Source/JavaScriptCore/b3 4 files +168 -3 . Source/JavaScriptCore/wasm 3 files +27 -10. - Plain Bun too, in the optimizing JIT: `f32.demote_f64` of a converted 64-bit integer rounded once where it rounds twice, and a signaling NaN stayed signaling through `f32.demote_f64` of `f64.promote_f32`. Fixed. - A module whose data segments outgrow the buffer's first reservation fails validation when memory runs out, as before, instead of ending the process. - One check of a list's length stands for all written indexes of a block: the time of a build grew with more than the number of statements. - The decoded texts of an image are allocated as the image is, so that LeakSanitizer reports nothing. --- .../JavaScriptCore/aot/image/AOTImageText.cpp | 4 +- .../aot/walk/AOTWalkTypedElements.cpp | 21 ++++ Source/JavaScriptCore/b3/B3LowerMacros.cpp | 9 +- .../b3/B3ReduceDoubleToFloat.cpp | 21 +++- Source/JavaScriptCore/b3/B3ValueInlines.h | 24 ++++ .../b3/testb3_bun_reductions.cpp | 117 ++++++++++++++++++ Source/JavaScriptCore/wasm/WasmFormat.cpp | 30 +++-- Source/JavaScriptCore/wasm/WasmFormat.h | 5 +- .../JavaScriptCore/wasm/WasmSectionParser.cpp | 2 +- 9 files changed, 219 insertions(+), 14 deletions(-) diff --git a/Source/JavaScriptCore/aot/image/AOTImageText.cpp b/Source/JavaScriptCore/aot/image/AOTImageText.cpp index 8fcc7ebac79e9..c400d1e6f6485 100644 --- a/Source/JavaScriptCore/aot/image/AOTImageText.cpp +++ b/Source/JavaScriptCore/aot/image/AOTImageText.cpp @@ -43,6 +43,8 @@ static_assert(ImageString::hashMask == StringHasher::maskHash && ImageIdentifier static_assert(!(imageSectionAlignment % alignof(ImageStringRecord)) && !(imageSectionAlignment % alignof(ImageStringHeader))); struct ImageText::DecodedTexts { + WTF_DEPRECATED_MAKE_STRUCT_FAST_ALLOCATED(DecodedTexts); + static constexpr size_t numberOfRecentTexts = 512; struct RecentText { @@ -61,7 +63,7 @@ ImageText::ImageText(const Image& image) , m_strings(image.section(ImageSection::Strings).first(image.header().numbers.numberOfStringWords)) , m_codedTexts(image.section(ImageSection::Strings).subspan(m_strings.size_bytes())) , m_numberOfCodedTexts(image.header().numbers.numberOfCodedTexts) - , m_decodedTexts(makeUniqueWithoutFastMallocCheck()) + , m_decodedTexts(makeUnique()) , m_listedStrings(image.section(ImageSection::LongStrings)) , m_text(image.section(ImageSection::Text)) , m_identifiers(image.section(ImageSection::Identifiers).first(image.header().numbers.endOfIdentifiers)) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkTypedElements.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkTypedElements.cpp index 8ecf072ffc48d..f7e1cf0c5d72b 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkTypedElements.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkTypedElements.cpp @@ -994,6 +994,22 @@ static std::optional indexesOverTurnsInTypedElements(Walker& walker, cons return found; } +static std::optional greatestWrittenIndex(const BodyIndex& index, const BodyValue& firstAccess) +{ + static constexpr OpcodeSet accesses { Opcode::ElemGet, Opcode::ElemSet }; + ValueID list = index.stripCasts(firstAccess.objectOperand()); + std::optional greatest; + for (BodyValue access : index.block(index.blockOf(firstAccess.id())).values()) { + ValueID key = accesses.contains(access.opcode()) && index.stripCasts(access.objectOperand()) == list ? index.stripCasts(access.propertyOperand()) : ValueID(); + if (!key || index.isParameter(key) || index.opcodeOf(key) != Opcode::Integer) + continue; + if (!greatest && access.id() != firstAccess.id()) + return std::nullopt; + greatest = std::max(greatest.value_or(0), index.value(key).as().integer()); + } + return greatest && *greatest <= std::numeric_limits::max() ? greatest : std::nullopt; +} + static void hoistArrayBoundsCheck(const AccessOfTypedElements& access) { Walker& walker = access.walker; @@ -1015,6 +1031,11 @@ static void hoistArrayBoundsCheck(const AccessOfTypedElements& access) most = walker.constant(WTF::move(mostOfTurns)); walker.count(StatisticID::LengthsAskedForUnrolledTurns); } + if (std::optional greatest = isInBody && access.key.isConstant() ? greatestWrittenIndex(index, access.value) : std::nullopt; greatest && *greatest > access.typeOfKey.integers.maximum) { + Draft mostOfBlock(Opcode::Integer, origin); + mostOfBlock.setInteger(*greatest); + most = walker.constant(WTF::move(mostOfBlock)); + } Reference length; Reference isBelow; { diff --git a/Source/JavaScriptCore/b3/B3LowerMacros.cpp b/Source/JavaScriptCore/b3/B3LowerMacros.cpp index e8a0a4305491d..0097627232d22 100644 --- a/Source/JavaScriptCore/b3/B3LowerMacros.cpp +++ b/Source/JavaScriptCore/b3/B3LowerMacros.cpp @@ -37,6 +37,7 @@ #include "B3CaseCollectionInlines.h" #include "B3CheckValue.h" #include "B3ConstDoubleValue.h" +#include "B3ConstFloatValue.h" #include "B3ConstPtrValue.h" #include "B3FenceValue.h" #include "B3InsertionSetInlines.h" @@ -1179,7 +1180,13 @@ class LowerMacros { std::swap(a, b); // select(b, a) returns a when a is NaN and when both are zeros. That is only wrong when b is the zero that must win: -0 for min, +0 for max. if (isNeverNaN(b) && !isZeroWithSign(b, !isMax)) { - m_value->replaceWithIdentity(select(b, a)); + Value* result = select(b, a); + if (m_proc.isWasm() && !givesQuietNaN(a)) { + // Adding -0 makes a NaN quiet and leaves every other value unchanged. + Value* negativeZero = m_value->type() == Float ? static_cast(m_insertionSet.insert(m_index, m_origin, -0.0f)) : m_insertionSet.insert(m_index, m_origin, -0.0); + result = m_insertionSet.insert(m_index, Add, m_origin, result, negativeZero); + } + m_value->replaceWithIdentity(result); m_changed = true; return; } diff --git a/Source/JavaScriptCore/b3/B3ReduceDoubleToFloat.cpp b/Source/JavaScriptCore/b3/B3ReduceDoubleToFloat.cpp index 95e72239aaeaf..7ea8804280e9b 100644 --- a/Source/JavaScriptCore/b3/B3ReduceDoubleToFloat.cpp +++ b/Source/JavaScriptCore/b3/B3ReduceDoubleToFloat.cpp @@ -111,7 +111,7 @@ class DoubleToFloatReduction { foundConversionCandidate = true; Value* child = value->child(0); - if (child->opcode() == FloatToDouble) { + if (child->opcode() == FloatToDouble && !m_procedure.isWasm()) { // We don't really need to simplify this early but it simplifies debugging. replaceWithIdentity(value, child->child(0)); } @@ -235,6 +235,12 @@ class DoubleToFloatReduction { } } + // A double holds every 32-bit integer. It rounds a 64-bit one before the float does. + static bool isExactInDouble(Value* integer) + { + return integer->type() == Int32 || integer->opcode() == ZExt32 || integer->opcode() == SExt32; + } + bool LEGACY_NODELETE canBeTransformedToFloat(Value* value) { if (value->opcode() == FloatToDouble) @@ -262,6 +268,15 @@ class DoubleToFloatReduction { return false; } + // Each conversion of WebAssembly turns a signaling NaN into a quiet one. + Value* withQuietNaN(Value* value, unsigned valueIndex, InsertionSet& insertionSet) + { + if (!m_procedure.isWasm() || givesQuietNaN(value)) + return value; + Value* one = insertionSet.insert(valueIndex, value->origin(), 1.0f); + return insertionSet.insert(valueIndex, Mul, value->origin(), value, one); + } + Value* transformToFloat(Value* value, unsigned valueIndex, InsertionSet& insertionSet) { ASSERT(canBeTransformedToFloat(value)); @@ -370,6 +385,8 @@ class DoubleToFloatReduction { break; } case IToD: { + if (!isExactInDouble(value->child(0))) + break; Value* iToF = insertionSet.insert(index, IToF, value->origin(), value->child(0)); value->setType(Float); replaceWithIdentity(value, iToF); @@ -436,7 +453,7 @@ class DoubleToFloatReduction { for (unsigned index = 0; index < block->size(); ++index) { Value* value = block->at(index); if (value->opcode() == DoubleToFloat && value->child(0)->type() == Float) { - replaceWithIdentity(value, value->child(0)); + replaceWithIdentity(value, withQuietNaN(value->child(0), index, insertionSet)); continue; } diff --git a/Source/JavaScriptCore/b3/B3ValueInlines.h b/Source/JavaScriptCore/b3/B3ValueInlines.h index 1f8955d081fe2..d450ecd2b5c2e 100644 --- a/Source/JavaScriptCore/b3/B3ValueInlines.h +++ b/Source/JavaScriptCore/b3/B3ValueInlines.h @@ -593,6 +593,30 @@ inline SIMDValue* Value::asSIMDValue() return as(); } +// A signaling NaN among the operands comes out quiet. +inline bool givesQuietNaN(const Value* value) +{ + switch (value->foldIdentity()->opcode()) { + case Add: + case Sub: + case Mul: + case Div: + case Ceil: + case Floor: + case FTrunc: + case Sqrt: + case FMax: + case FMin: + case IToD: + case IToF: + case FloatToDouble: + case DoubleToFloat: + return true; + default: + return false; + } +} + } } // namespace JSC::B3 #endif // ENABLE(B3_JIT) diff --git a/Source/JavaScriptCore/b3/testb3_bun_reductions.cpp b/Source/JavaScriptCore/b3/testb3_bun_reductions.cpp index 10c0daecc9823..22e5a4f90104c 100644 --- a/Source/JavaScriptCore/b3/testb3_bun_reductions.cpp +++ b/Source/JavaScriptCore/b3/testb3_bun_reductions.cpp @@ -14,7 +14,9 @@ #include "B3FoldPathConstants.h" #include "B3IntRange.h" #include "B3OptimizeAssociativeExpressionTrees.h" +#include "B3ReduceDoubleToFloat.h" #include "B3ThreadJumps.h" +#include "MathCommon.h" static constexpr int64_t tag = static_cast(0xfffe000000000000ull); @@ -1533,6 +1535,118 @@ static void testDifferencePlusSubtrahend() } } +static void testFloatOfDoubleOfInteger() +{ + enum class Width { Narrow, Extended, Wide }; + for (Width width : { Width::Narrow, Width::Extended, Width::Wide }) { + for (bool joins : { false, true }) { + auto build = [&](Procedure& proc) { + Graph graph { proc }; + BasicBlock* root = proc.addBlock(); + Value* argument = cCallArgumentValues(proc, root)[0]; + Value* integer = width == Width::Wide ? argument : graph.make(root, Trunc, argument); + if (width == Width::Extended) + integer = graph.make(root, ZExt32, integer); + Value* number = graph.make(root, IToD, integer); + if (joins) { + BasicBlock* taken = proc.addBlock(); + BasicBlock* notTaken = proc.addBlock(); + BasicBlock* join = proc.addBlock(); + Value* phi = graph.phi(join, Double); + graph.branch(root, integer, taken, notTaken); + graph.upsilon(taken, number, phi); + graph.jump(taken, join); + graph.upsilon(notTaken, graph.number(notTaken, 1.5), phi); + graph.jump(notTaken, join); + graph.ret(join, graph.make(join, DoubleToFloat, phi)); + } else + graph.ret(root, graph.make(root, DoubleToFloat, number)); + resetAndValidate(proc); + }; + Procedure reduced; + build(reduced); + reduceDoubleToFloat(reduced); + validate(reduced); + CHECK_EQ(numberOfValues(reduced, IToF), width == Width::Wide ? 0u : 1u); + + Procedure proc; + build(proc); + auto code = compileProc(proc); + for (int64_t value : { 0x1000001000000001ll, -0x1000001000000001ll, 0x4000004000000001ll, 0x7fffffbfffffffffll, 0x20000010000001ll, 0x1000001000000000ll, 16777217ll, 0x7fffffffll, -1ll, 1ll }) { + int64_t integer = value; + if (width == Width::Narrow) + integer = static_cast(value); + if (width == Width::Extended) + integer = static_cast(value); + double expected = joins && !integer ? 1.5 : static_cast(integer); + CHECK_EQ(invoke(*code, value), static_cast(expected)); + } + } + } +} + +static void testFloatThroughDoubleIsQuiet() +{ + enum class Shape { Pair, Neg, Abs, Join }; + for (Shape shape : { Shape::Pair, Shape::Neg, Shape::Abs, Shape::Join }) { + Procedure proc; + proc.setIsWasm(true); + Graph graph { proc }; + BasicBlock* root = proc.addBlock(); + Value* bits = graph.make(root, Trunc, cCallArgumentValues(proc, root)[0]); + Value* number = graph.make(root, FloatToDouble, graph.make(root, BitwiseCast, bits)); + BasicBlock* last = root; + if (shape == Shape::Neg || shape == Shape::Abs) + number = graph.make(root, shape == Shape::Neg ? Neg : Abs, number); + if (shape == Shape::Join) { + BasicBlock* taken = proc.addBlock(); + BasicBlock* notTaken = proc.addBlock(); + last = proc.addBlock(); + Value* phi = graph.phi(last, Double); + graph.branch(root, bits, taken, notTaken); + graph.upsilon(taken, number, phi); + graph.jump(taken, last); + graph.upsilon(notTaken, graph.number(notTaken, 1.5), phi); + graph.jump(notTaken, last); + number = phi; + } + graph.ret(last, graph.make(last, BitwiseCast, graph.make(last, DoubleToFloat, number))); + resetAndValidate(proc); + auto code = compileProc(proc); + for (uint32_t value : { 0x7fa00000u, 0xffa12345u, 0x7fc54321u, 0x3f800001u, 0x80000001u, 0x80000000u, 0x7f7fffffu, 0xff800000u }) { + bool isNaN = (value & 0x7fffffff) > 0x7f800000; + uint32_t expected = isNaN ? value | 0x400000 : value; + if (shape == Shape::Neg) + expected ^= 0x80000000; + if (shape == Shape::Abs) + expected &= 0x7fffffff; + CHECK_EQ(static_cast(invoke(*code, static_cast(value))), expected); + } + } +} + +static void testMinWithConstantIsQuiet() +{ + for (B3::Opcode opcode : { FMin, FMax }) { + for (double constant : { 1.0, 0.0, -0.0 }) { + Procedure proc(selectionPolicy()); + proc.setIsWasm(true); + Graph graph { proc }; + BasicBlock* root = proc.addBlock(); + Value* number = graph.make(root, BitwiseCast, cCallArgumentValues(proc, root)[0]); + graph.ret(root, graph.make(root, BitwiseCast, graph.make(root, opcode, number, graph.number(root, constant)))); + resetAndValidate(proc); + auto code = compileProc(proc); + for (int64_t bits : { 0x7ff4000000000000ll, 0x7ff8000000054321ll }) + CHECK_EQ(invoke(*code, bits), bits | 0x8000000000000ll); + for (double operand : { 2.0, -2.0, 1.0, 0.0, -0.0 }) { + double expected = opcode == FMin ? Math::fMin(operand, constant) : Math::fMax(operand, constant); + CHECK_EQ(invoke(*code, std::bit_cast(operand)), std::bit_cast(expected)); + } + } + } +} + void addBunReductionTests(const TestConfig* config, BunTestTasks& tasks) { RUN(testMemoryNumberingIrreducibleCycle()); @@ -1569,6 +1683,9 @@ void addBunReductionTests(const TestConfig* config, BunTestTasks& tasks) RUN(testPartOfAddressIsSharedWithSameSumFurtherDown()); RUN(testPartOfAddressIsSharedBesideCycleThatLeavesProcedure()); RUN(testDifferencePlusSubtrahend()); + RUN(testFloatOfDoubleOfInteger()); + RUN(testFloatThroughDoubleIsQuiet()); + RUN(testMinWithConstantIsQuiet()); } #endif // ENABLE(B3_JIT) diff --git a/Source/JavaScriptCore/wasm/WasmFormat.cpp b/Source/JavaScriptCore/wasm/WasmFormat.cpp index 028a1c53a449f..1d15359770697 100644 --- a/Source/JavaScriptCore/wasm/WasmFormat.cpp +++ b/Source/JavaScriptCore/wasm/WasmFormat.cpp @@ -48,20 +48,31 @@ bool WasmCallableFunction::isJS() const bool Segments::tryReserve(size_t numberOfSegments, size_t numberOfBytes, size_t numberOfIndices) { + m_expectedSize = numberOfSegments; m_hasIndices = numberOfIndices; return m_buffer.tryReserveInitialCapacity(numberOfSegments * sizeof(Header) + numberOfBytes) && m_segmentOffsets.tryReserveInitialCapacity(numberOfIndices); } -void Segments::append(const Segment& segment) +// Most modules have no long headers. The first one reserves for all segments that follow it. +bool Segments::tryReserveLongHeaders() { - ++m_size; - if (m_hasIndices) - m_segmentOffsets.append(m_buffer.size()); + if (m_hasReservedLongHeaders) + return true; + m_hasReservedLongHeaders = m_buffer.tryReserveCapacity(m_buffer.capacity() + (m_expectedSize - m_size) * sizeof(LongHeader)); + return m_hasReservedLongHeaders; +} + +bool Segments::tryAppend(const Segment& segment) +{ + ASSERT(m_size < m_expectedSize); + if (m_hasIndices && !m_segmentOffsets.tryAppend(static_cast(m_buffer.size()))) + return false; Header header { static_cast(segment.bytes.size()), 0 }; if (segment.isActive() && segment.offsetIfActive->isConst() && segment.offsetIfActive->bits() <= std::numeric_limits::max() && !segment.memoryIndex) [[likely]] { header.offset = static_cast(segment.offsetIfActive->bits()); - m_buffer.append(asByteSpan(header)); + if (!m_buffer.tryAppend(asByteSpan(header))) + return false; } else { LongHeader longHeader { }; longHeader.memoryIndex = segment.memoryIndex; @@ -72,10 +83,13 @@ void Segments::append(const Segment& segment) longHeader.isExtendedExpression = segment.offsetIfActive->isExtendedExpression(); } header.size |= hasLongHeaderBit; - m_buffer.append(asByteSpan(header)); - m_buffer.append(asByteSpan(longHeader)); + if (!tryReserveLongHeaders() || !m_buffer.tryAppend(asByteSpan(header)) || !m_buffer.tryAppend(asByteSpan(longHeader))) + return false; } - m_buffer.append(segment.bytes); + if (!m_buffer.tryAppend(segment.bytes)) + return false; + ++m_size; + return true; } void Segments::readLongHeader(size_t& place, Segment& segment) const diff --git a/Source/JavaScriptCore/wasm/WasmFormat.h b/Source/JavaScriptCore/wasm/WasmFormat.h index de6ccc6ffee64..65636968dcf89 100644 --- a/Source/JavaScriptCore/wasm/WasmFormat.h +++ b/Source/JavaScriptCore/wasm/WasmFormat.h @@ -750,7 +750,7 @@ class Segments { public: // Only a module with a Data Count section names its segments by index. bool tryReserve(size_t numberOfSegments, size_t numberOfBytes, size_t numberOfIndices); - void append(const Segment&); + [[nodiscard]] bool tryAppend(const Segment&); size_t size() const { return m_size; } size_t sizeInBytes() const { return m_buffer.size(); } @@ -796,11 +796,14 @@ class Segments { return result; } void readLongHeader(size_t& place, Segment&) const; + bool tryReserveLongHeaders(); Vector m_buffer; Vector m_segmentOffsets; size_t m_size { 0 }; + size_t m_expectedSize { 0 }; bool m_hasIndices { false }; + bool m_hasReservedLongHeaders { false }; }; struct Element { diff --git a/Source/JavaScriptCore/wasm/WasmSectionParser.cpp b/Source/JavaScriptCore/wasm/WasmSectionParser.cpp index 10551322c9a0b..f86a43e6a101d 100644 --- a/Source/JavaScriptCore/wasm/WasmSectionParser.cpp +++ b/Source/JavaScriptCore/wasm/WasmSectionParser.cpp @@ -1377,7 +1377,7 @@ auto SectionParser::parseData() -> PartialResult WASM_PARSER_FAIL_IF(source().size() < dataByteLength || (source().size() - dataByteLength) < m_offset, "can't get data bytes from "_s, segmentNumber, "th Data segment"_s); segment.bytes = source().subspan(m_offset, dataByteLength); m_offset += dataByteLength; - m_info->data.append(segment); + WASM_ALLOCATOR_FAIL_IF(!m_info->data.tryAppend(segment), "can't allocate enough memory for "_s, segmentNumber, "th Data segment of size "_s, dataByteLength); } return { }; From 75a5b48af40d55915d2f69ab0126e4f56d621f32 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 10 Nov 2026 07:37:52 -0800 Subject: [PATCH 06/10] AOT: a hang of collections without helpers, memory of growing texts, data segments of WebAssembly, mocks below originals On 83468e92. 11 files, +102 -33: Source/JavaScriptCore/aot 6 files +92 -14 . Source/JavaScriptCore/wasm 2 files +4 -16 . Source/JavaScriptCore/assembler 1 file +2 -0 . Source/JavaScriptCore/b3 1 file +2 -2 . Source/JavaScriptCore/heap 1 file +2 -1. - A text that grows by adding elements of the list that it is stored into took gigabytes: its buffer copied what ropes share. A text grows in a buffer by short additions alone. - Plain Bun too: the headers of a module's passive data segments are reserved with the others, so a module with active segments before a passive one is not copied once more while it is compiled. - `bun test --aot`: with mocks of a file that takes its original, of a file below it, and of a file that imports that one, the last mock did nothing and the test got the real file. - A collection of compiled code that has no helper threads could wait for helpers, so a program whose heap is exhausted, or that may not start threads, could hang. The collector's thread marks there itself; plain Bun is unchanged. - `bun test --aot`: where an import() fails to link, the modules of its graph that were fetched are not fetched again by a later import(), so that a mock with an async factory settles as in `bun test`. - Small corrections in the x64 assembler and in the register allocator. Plain Bun too. --- .../aot/runtime/AOTGrowingText.cpp | 16 +++++- .../aot/runtime/AOTModuleEvaluation.cpp | 4 +- .../JavaScriptCore/aot/runtime/AOTModules.h | 1 + .../aot/runtime/AOTTestRunFactories.cpp | 12 +++++ .../aot/tests/AOTCollectorTests.cpp | 52 +++++++++++++++++-- .../aot/tests/AOTRuntimeOperationTests.cpp | 21 +++++--- .../assembler/MacroAssemblerX86_64.h | 2 + .../b3/air/AirAllocateRegistersByGreedy.cpp | 4 +- Source/JavaScriptCore/heap/SlotVisitor.cpp | 3 +- Source/JavaScriptCore/wasm/WasmFormat.cpp | 17 ++---- Source/JavaScriptCore/wasm/WasmFormat.h | 3 -- 11 files changed, 102 insertions(+), 33 deletions(-) diff --git a/Source/JavaScriptCore/aot/runtime/AOTGrowingText.cpp b/Source/JavaScriptCore/aot/runtime/AOTGrowingText.cpp index 5367544e3e1d5..19f26cfda3547 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTGrowingText.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTGrowingText.cpp @@ -212,10 +212,22 @@ JSString* tryAppendToGrowingText(Instance& instance, JSString* text, std::span lengthInAdvance) + return nullptr; uint32_t numberOfAdditions = 0; JSString* bottom = text; - for (; bottom->isNonSubstringRope(); bottom = firstPartOf(bottom)) - ++numberOfAdditions; + auto areAdditionsLong = [&] { + return length - bottom->length() + lengthOfPieces > lengthInAdvance + numberOfAdditions * maxLengthPerAddition; + }; + for (; bottom->isNonSubstringRope(); bottom = firstPartOf(bottom)) { + if (!(++numberOfAdditions % additionsPerQuestion) && areAdditionsLong()) + return nullptr; + } + if (numberOfAdditions && areAdditionsLong()) + return nullptr; if (JSString* buffer = bottom->isRope() ? bufferThatEndsWith(instance, bottom) : nullptr) { StringImpl& characters = *buffer->tryGetValueImpl(); diff --git a/Source/JavaScriptCore/aot/runtime/AOTModuleEvaluation.cpp b/Source/JavaScriptCore/aot/runtime/AOTModuleEvaluation.cpp index 1be5d9595acfc..c61defdb39da5 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTModuleEvaluation.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTModuleEvaluation.cpp @@ -2187,12 +2187,12 @@ bool Modules::isNotLoaded(ModuleID module) return false; if (ModuleRecord* record = recordIfExists(module)) return record->status() == CyclicModuleRecord::Status::New || record->status() == CyclicModuleRecord::Status::Unlinked; - return storedStatus(module) == StoredModuleStatus::Unlinked && isNotLoadedWithoutStatus(module, data(module)); + return storedStatus(module) == StoredModuleStatus::Unlinked && isNotLoadedWithoutStatus(module, data(module)) && !wasFetchedByFailedLoad(module); } void Modules::noteModulesLoadedWith(ModuleID root) { - if (!data(root).isESModuleOfProgram() || (data(root).isFacade() && !m_loadsModulesIndividually) || wasLoadedWithGraph(root) || m_rootWhoseLoadIsSwept) + if (!data(root).isESModuleOfProgram() || (data(root).isFacade() && !m_loadsModulesIndividually) || wasLoadedWithGraph(root) || (m_rootWhoseLoadIsSwept && !m_originalBeingEvaluated)) return; if (m_loadsModulesIndividually) { noteLoadOfEachModuleWith(root); diff --git a/Source/JavaScriptCore/aot/runtime/AOTModules.h b/Source/JavaScriptCore/aot/runtime/AOTModules.h index 804d23fbc9002..96cbd80afb756 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTModules.h +++ b/Source/JavaScriptCore/aot/runtime/AOTModules.h @@ -752,6 +752,7 @@ class Modules { std::optional moduleThatFailedToLink(ModuleID root) const; bool hasLinkFailures() const { return m_errorsOfExportsThatBuildFoundAbsent || !m_linkFailures.isEmpty(); } bool wasLinkedAheadOfFailure(ModuleID root, ModuleID) const; + bool wasFetchedByFailedLoad(ModuleID) const; bool wasLinkedBeforeLinkFailure(ModuleID) const; bool isLinkedForPendingImporter(ModuleID, std::optional asking) const; std::optional failureByAbsentExport(ModuleID) const; diff --git a/Source/JavaScriptCore/aot/runtime/AOTTestRunFactories.cpp b/Source/JavaScriptCore/aot/runtime/AOTTestRunFactories.cpp index 3367322551c7c..fb1e9cf3d0d96 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTTestRunFactories.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTTestRunFactories.cpp @@ -215,6 +215,18 @@ bool Modules::wasRegisteredByFailedLoad(ModuleID module) const return wasPulledIn; } +bool Modules::wasFetchedByFailedLoad(ModuleID module) const +{ + if (!hasLinkFailures()) [[likely]] + return false; + for (uint32_t index = 0; index < m_table.size(); ++index) { + ModuleID root(index); + if (root != module && moduleThatFailedToLink(root) && modulesInLoadGraph(root).get(module.index())) + return true; + } + return false; +} + bool Modules::wasLinkedBeforeLinkFailure(ModuleID module) const { if (!hasLinkFailures()) diff --git a/Source/JavaScriptCore/aot/tests/AOTCollectorTests.cpp b/Source/JavaScriptCore/aot/tests/AOTCollectorTests.cpp index 80486d3172b27..d5f0c41a824c7 100644 --- a/Source/JavaScriptCore/aot/tests/AOTCollectorTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTCollectorTests.cpp @@ -21,6 +21,7 @@ #include "JSLock.h" #include "MarkedSpace.h" #include "ObjectConstructor.h" +#include "ReleaseHeapAccessScope.h" #include "RememberedSlot.h" #include "SimpleMarkingConstraint.h" #include "VM.h" @@ -28,6 +29,7 @@ #include "WeakInlines.h" #include #include +#include #include #include #include @@ -67,12 +69,14 @@ class TestVM { } // anonymous namespace -template +enum class CollectorPause : uint8_t { Short, Unlimited }; + +template static void runInProcessWithCompiledCodeOptions() { ASCIILiteral name; for (const NativeTest& test : collectorTests()) { - if (test.run == &runInProcessWithCompiledCodeOptions) + if (test.run == &runInProcessWithCompiledCodeOptions) name = test.name; } AOT_TEST_CHECK(!name.isNull()); @@ -83,7 +87,11 @@ static void runInProcessWithCompiledCodeOptions() } ChildProcessLauncher launcher(String(programPath()), { }); - Vector arguments { String(nativeSubcommandName), String(singleProcessFlag), String(name), "--useAOT=true"_s, "--numberOfGCMarkers=4"_s, "--minimumGCPauseMS=10000"_s, "--gcPauseScale=10000"_s }; + Vector arguments { String(nativeSubcommandName), String(singleProcessFlag), String(name), "--useAOT=true"_s, "--numberOfGCMarkers=4"_s }; + if constexpr (pause == CollectorPause::Unlimited) + arguments.appendList({ "--minimumGCPauseMS=10000"_str, "--gcPauseScale=10000"_str }); + else + arguments.appendList({ "--minimumGCPauseMS=0"_str, "--gcPauseScale=0"_str }); ChildProcessResult result = launcher.run(arguments.span()); if (!result.exitedWith(0)) { dataLogLn(result.output); @@ -225,6 +233,38 @@ static void checkWideHeapStartsHelpers() ensureStillAliveHere(array); } +static void checkCollectorThreadMarksWithoutHelpers() +{ + TestVM test; + JSGlobalObject* realm = test.createRealm(); + VM& vm = test.vm(); + constexpr unsigned length = 100000; + JSArray* array = JSArray::tryCreate(vm, realm->arrayStructureForIndexingTypeDuringAllocation(ArrayWithContiguous), length); + AOT_TEST_CHECK(array); + for (unsigned index = 0; index < length; ++index) + array->putDirectIndex(realm, index, constructEmptyObject(realm)); + + unsigned numberOfConcurrentCollections = 0; + vm.heap.collector().willCollectWithoutHelpers(); + for (unsigned count = 0; count < 20; ++count) { + vm.heap.collectNow(Async, CollectionScope::Full); + if (vm.heap.objectSpace().isMarking()) + ++numberOfConcurrentCollections; + MonotonicTime deadline = MonotonicTime::now() + 10_s; + while (vm.heap.objectSpace().isMarking() && MonotonicTime::now() < deadline) { + { + ReleaseHeapAccessScope releaseAccess(vm.heap); + } + sleep(1_ms); + vm.heap.stopIfNecessary(); + } + AOT_TEST_CHECK(!vm.heap.objectSpace().isMarking()); + } + vm.heap.collector().didCollectWithoutHelpers(); + AOT_TEST_CHECK(numberOfConcurrentCollections); + ensureStillAliveHere(array); +} + static void testHelpersStartEagerlyWithoutCompiledCode() { if (Options::useAOT()) @@ -564,7 +604,10 @@ static void testTransitionChainStaysWithoutCompiledCode() } template -static constexpr void (*withCompiledCodeOptions)() = runInProcessWithCompiledCodeOptions; +static constexpr void (*withCompiledCodeOptions)() = runInProcessWithCompiledCodeOptions; + +template +static constexpr void (*withConcurrentMarking)() = runInProcessWithCompiledCodeOptions; std::span collectorTests() { @@ -576,6 +619,7 @@ std::span collectorTests() { "collector/larger-cells/off-without-compiled-code"_s, testNothingIsBorrowedWithoutCompiledCode }, { "collector/helpers/small-heap-starts-none"_s, withCompiledCodeOptions }, { "collector/helpers/wide-heap-starts-them"_s, withCompiledCodeOptions }, + { "collector/helpers/collector-thread-marks-without-them"_s, withConcurrentMarking }, { "collector/helpers/eager-without-compiled-code"_s, testHelpersStartEagerlyWithoutCompiledCode }, { "collector/roots/full-collection-defers-regions"_s, withCompiledCodeOptions }, { "collector/roots/store-after-deferral-is-seen"_s, withCompiledCodeOptions }, diff --git a/Source/JavaScriptCore/aot/tests/AOTRuntimeOperationTests.cpp b/Source/JavaScriptCore/aot/tests/AOTRuntimeOperationTests.cpp index 2c255a33f54a2..606b902b149d9 100644 --- a/Source/JavaScriptCore/aot/tests/AOTRuntimeOperationTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTRuntimeOperationTests.cpp @@ -480,13 +480,22 @@ static void checkTextThatIsAddedToGrowsInBuffer(LoadedProgram& program) AOT_TEST_CHECK(numberOfBuffers == maxNumberOfBuffers); AOT_TEST_CHECK(charactersOf(text) == repeated("ab"_s, numberOfPieces)); - JSString* longPiece = jsNontrivialString(vm, repeated("0123456789"_s, 100 * minLengthOfGrowingText)); - JSString* longText = piece; - for (unsigned additions = 0; additions < minAdditionsToGrowingText; ++additions) { - longText = joinFromNativeCode(program, longText, additions == 1 ? longPiece : piece); - AOT_TEST_CHECK(longText->isNonSubstringRope()); + for (unsigned timesOfMinLength : { 10u, 100u }) { + JSString* longPiece = jsNontrivialString(vm, repeated("0123456789"_s, timesOfMinLength * minLengthOfGrowingText)); + JSString* longText = piece; + for (unsigned additions = 0; additions < minAdditionsToGrowingText; ++additions) { + longText = joinFromNativeCode(program, longText, additions == 1 ? longPiece : piece); + AOT_TEST_CHECK(longText->isNonSubstringRope()); + } + bool additionsAreShort = timesOfMinLength == 10; + JSString* longer = joinFromNativeCode(program, longText, piece); + AOT_TEST_CHECK(canGrowInPlace(instance, longer) == additionsAreShort && longer->length() == longText->length() + piece->length()); } - AOT_TEST_CHECK(canGrowInPlace(instance, joinFromNativeCode(program, longText, piece))); + JSString* sharedPiece = jsNontrivialString(vm, repeated("0123456789"_s, 100 * minLengthOfGrowingText)); + JSString* textOfLongPieces = text; + for (unsigned additions = 0; additions < 4; ++additions) + textOfLongPieces = joinFromNativeCode(program, textOfLongPieces, sharedPiece); + AOT_TEST_CHECK(textOfLongPieces->isNonSubstringRope() && canGrowInPlace(instance, text)); AOT_TEST_CHECK(!canGrowInPlace(instance, older) && charactersOf(older) == repeated("ab"_s, numberOfPiecesOfOlderText)); JSString* other = joinFromNativeCode(program, older, jsNontrivialString(vm, "XY"_s)); diff --git a/Source/JavaScriptCore/assembler/MacroAssemblerX86_64.h b/Source/JavaScriptCore/assembler/MacroAssemblerX86_64.h index 888d3291607be..9cdf8e6ae6142 100644 --- a/Source/JavaScriptCore/assembler/MacroAssemblerX86_64.h +++ b/Source/JavaScriptCore/assembler/MacroAssemblerX86_64.h @@ -4910,6 +4910,8 @@ class MacroAssemblerX86_64 : public AbstractMacroAssembler { m_assembler.cmpl_im(0, address.offset, address.base); else if (!(mask.m_value & ~0xff)) m_assembler.testb_im(mask.m_value, address.offset, address.base); + else if (address.offset > std::numeric_limits::max() - 3) + m_assembler.testl_i32m(mask.m_value, address.offset, address.base); else if (!(mask.m_value & ~0xff00)) m_assembler.testb_im(mask.m_value >> 8, address.offset + 1, address.base); else if (!(mask.m_value & ~0xff0000)) diff --git a/Source/JavaScriptCore/b3/air/AirAllocateRegistersByGreedy.cpp b/Source/JavaScriptCore/b3/air/AirAllocateRegistersByGreedy.cpp index a3f39211ec1cf..5bad453fc1d25 100644 --- a/Source/JavaScriptCore/b3/air/AirAllocateRegistersByGreedy.cpp +++ b/Source/JavaScriptCore/b3/air/AirAllocateRegistersByGreedy.cpp @@ -6361,13 +6361,13 @@ class GreedyAllocator { // FIXME: reconsider that, see https://bugs.webkit.org/show_bug.cgi?id=288122 ASSERT(hole.begin() > positionOfHead(block)); // Model gapTmp interference with any other tmp split at this location by starting - // the gapTmp's range one position before the hole. Otherwise, the same register + // the gapTmp's range where the instruction begins. Otherwise, the same register // may be chosen for the gapTmp and another splitTmp, which wouldn't be valid // since there could be a cycle among the set of fixup Move instructions. // An alternative would be to use the Shuffle opcode (which can handle that // rotation of register assignments) but that would trigger an extra liveness // analysis (see lowerAfterRegAlloc()), and that's unlikely to be worth it. - Interval gapInterval = hole | Interval(hole.begin() - 1); + Interval gapInterval = hole | Interval(pointAtOffset(hole.begin(), PointOffsets::Pre)); if (spansStretches) gapInterval = Interval(pointAtOffset(hole.begin(), PointOffsets::Pre), pointAtOffset(hole.end() - 1, PointOffsets::Post) + 1); Tmp gapTmp = addSplitTmp(tmp, UseDefCost(freq), gapInterval); diff --git a/Source/JavaScriptCore/heap/SlotVisitor.cpp b/Source/JavaScriptCore/heap/SlotVisitor.cpp index 56f474f039e45..447dd01f0ad87 100644 --- a/Source/JavaScriptCore/heap/SlotVisitor.cpp +++ b/Source/JavaScriptCore/heap/SlotVisitor.cpp @@ -944,11 +944,12 @@ SlotVisitor::SharedDrainResult SlotVisitor::drainInParallelPassively(MonotonicTi JSC::Heap& heap = m_collector.heap(); if (Options::numberOfGCMarkers() == 1 + || Collector::startsHelpersLazily() || (heap.m_worldState.load() & Heap::mutatorWaitingBit) || !heap.hasHeapAccess() || heap.worldIsStopped()) { // This is an optimization over drainInParallel() when we have a concurrent mutator but - // otherwise it is not profitable. + // otherwise it is not profitable. Helpers that start lazily may never start. return drainInParallel(timeout); } diff --git a/Source/JavaScriptCore/wasm/WasmFormat.cpp b/Source/JavaScriptCore/wasm/WasmFormat.cpp index 1d15359770697..7edcdd62fe9a5 100644 --- a/Source/JavaScriptCore/wasm/WasmFormat.cpp +++ b/Source/JavaScriptCore/wasm/WasmFormat.cpp @@ -48,23 +48,14 @@ bool WasmCallableFunction::isJS() const bool Segments::tryReserve(size_t numberOfSegments, size_t numberOfBytes, size_t numberOfIndices) { - m_expectedSize = numberOfSegments; m_hasIndices = numberOfIndices; - return m_buffer.tryReserveInitialCapacity(numberOfSegments * sizeof(Header) + numberOfBytes) && m_segmentOffsets.tryReserveInitialCapacity(numberOfIndices); -} - -// Most modules have no long headers. The first one reserves for all segments that follow it. -bool Segments::tryReserveLongHeaders() -{ - if (m_hasReservedLongHeaders) - return true; - m_hasReservedLongHeaders = m_buffer.tryReserveCapacity(m_buffer.capacity() + (m_expectedSize - m_size) * sizeof(LongHeader)); - return m_hasReservedLongHeaders; + // Passive segments are the usual ones with a long header, and only an index can name them. + size_t sizeOfHeaders = numberOfSegments * sizeof(Header) + numberOfIndices * sizeof(LongHeader); + return m_buffer.tryReserveInitialCapacity(sizeOfHeaders + numberOfBytes) && m_segmentOffsets.tryReserveInitialCapacity(numberOfIndices); } bool Segments::tryAppend(const Segment& segment) { - ASSERT(m_size < m_expectedSize); if (m_hasIndices && !m_segmentOffsets.tryAppend(static_cast(m_buffer.size()))) return false; @@ -83,7 +74,7 @@ bool Segments::tryAppend(const Segment& segment) longHeader.isExtendedExpression = segment.offsetIfActive->isExtendedExpression(); } header.size |= hasLongHeaderBit; - if (!tryReserveLongHeaders() || !m_buffer.tryAppend(asByteSpan(header)) || !m_buffer.tryAppend(asByteSpan(longHeader))) + if (!m_buffer.tryAppend(asByteSpan(header)) || !m_buffer.tryAppend(asByteSpan(longHeader))) return false; } if (!m_buffer.tryAppend(segment.bytes)) diff --git a/Source/JavaScriptCore/wasm/WasmFormat.h b/Source/JavaScriptCore/wasm/WasmFormat.h index 65636968dcf89..e1cbcbf866ebe 100644 --- a/Source/JavaScriptCore/wasm/WasmFormat.h +++ b/Source/JavaScriptCore/wasm/WasmFormat.h @@ -796,14 +796,11 @@ class Segments { return result; } void readLongHeader(size_t& place, Segment&) const; - bool tryReserveLongHeaders(); Vector m_buffer; Vector m_segmentOffsets; size_t m_size { 0 }; - size_t m_expectedSize { 0 }; bool m_hasIndices { false }; - bool m_hasReservedLongHeaders { false }; }; struct Element { From 33c7b69470740b29e11465e123b219eaba3ed23c Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 10 Nov 2026 07:42:09 -0800 Subject: [PATCH 07/10] AOT: keep what is known of lists across stores into views, fix a test of growing texts on arm64 On 75a5b48a. 3 files, +23 -10: Source/JavaScriptCore/aot 3 files +23 -10. - With `--aot-harden=proof-checks`, a generator that stores into a `Float64Array` in a loop stopped the build: a store into a view ended what was known of the elements of fresh lists, which lie in no view. Fixed. - The native test of growing texts failed on Linux arm64, where a block is 64 KB: it added more characters than are copied into a buffer. It adds the same number on every machine now. --- .../aot/tests/AOTRuntimeOperationTests.cpp | 22 ++++++++++++------- .../aot/walk/AOTWalkEscapeAnalysis.cpp | 4 ++++ .../JavaScriptCore/aot/walk/AOTWalkState.cpp | 7 ++++-- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/Source/JavaScriptCore/aot/tests/AOTRuntimeOperationTests.cpp b/Source/JavaScriptCore/aot/tests/AOTRuntimeOperationTests.cpp index 606b902b149d9..7daeacba6acf1 100644 --- a/Source/JavaScriptCore/aot/tests/AOTRuntimeOperationTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTRuntimeOperationTests.cpp @@ -502,19 +502,25 @@ static void checkTextThatIsAddedToGrowsInBuffer(LoadedProgram& program) AOT_TEST_CHECK(charactersOf(other) == makeString(repeated("ab"_s, numberOfPiecesOfOlderText), "XY"_s)); AOT_TEST_CHECK(charactersOf(text) == repeated("ab"_s, numberOfPieces) && canGrowInPlace(instance, text)); - JSString* doubled = joinFromNativeCode(program, text, text); - AOT_TEST_CHECK(charactersOf(doubled) == repeated("ab"_s, 2 * numberOfPieces) && canGrowInPlace(instance, doubled) && canGrowInPlace(instance, text)); + AOT_TEST_CHECK(charactersOf(joinFromNativeCode(program, text, text)) == repeated("ab"_s, 2 * numberOfPieces) && canGrowInPlace(instance, text)); + + constexpr unsigned numberOfPiecesOfPart = 1000; + constexpr unsigned numberOfPiecesWithPart = numberOfPieces + numberOfPiecesOfPart; + JSString* lengthened = joinFromNativeCode(program, text, jsSubstring(program.realm(), text, 0, 2 * numberOfPiecesOfPart)); + AOT_TEST_CHECK(charactersOf(lengthened) == repeated("ab"_s, numberOfPiecesWithPart) && canGrowInPlace(instance, lengthened)); + bool isInBufferOfText = uncheckedDowncast(lengthened)->substringBase() == uncheckedDowncast(text)->substringBase(); + AOT_TEST_CHECK(canGrowInPlace(instance, text) == !isInBufferOfText); const char16_t wideCharacters[] = { 0x20ac, 0x20ac }; String wide { std::span { wideCharacters } }; - JSString* widened = joinFromNativeCode(program, doubled, jsNontrivialString(vm, wide)); + JSString* widened = joinFromNativeCode(program, lengthened, jsNontrivialString(vm, wide)); AOT_TEST_CHECK(!widened->is8Bit() && canGrowInPlace(instance, widened)); JSString* longer = joinFromNativeCode(program, widened, piece); AOT_TEST_CHECK(uncheckedDowncast(longer)->substringBase() == uncheckedDowncast(widened)->substringBase()); - AOT_TEST_CHECK(charactersOf(longer) == makeString(repeated("ab"_s, 2 * numberOfPieces), wide, "ab"_s)); - AOT_TEST_CHECK(charactersOf(doubled) == repeated("ab"_s, 2 * numberOfPieces)); + AOT_TEST_CHECK(charactersOf(longer) == makeString(repeated("ab"_s, numberOfPiecesWithPart), wide, "ab"_s)); + AOT_TEST_CHECK(charactersOf(lengthened) == repeated("ab"_s, numberOfPiecesWithPart)); - JSString* inBuffer = joinFromNativeCode(program, doubled, piece); + JSString* inBuffer = joinFromNativeCode(program, lengthened, piece); JSString* belowRopes = inBuffer; String expected = charactersOf(inBuffer); for (unsigned count = 0; count < 100; ++count) { @@ -531,14 +537,14 @@ static void checkTextThatIsAddedToGrowsInBuffer(LoadedProgram& program) constexpr unsigned numberOfPiecesBetweenCollections = 100000; constexpr size_t maxNumberOfCellsLeft = 100; - text = doubled; + text = lengthened; vm.heap.collectNow(Sync, CollectionScope::Full); size_t numberOfCellsBefore = vm.heap.objectCount(); for (unsigned count = 0; count < numberOfPiecesBetweenCollections; ++count) text = joinFromNativeCode(program, text, piece); vm.heap.collectNow(Sync, CollectionScope::Full); AOT_TEST_CHECK(vm.heap.objectCount() < numberOfCellsBefore + maxNumberOfCellsLeft); - AOT_TEST_CHECK(text->length() == 2 * (2 * numberOfPieces + numberOfPiecesBetweenCollections)); + AOT_TEST_CHECK(text->length() == 2 * (numberOfPiecesWithPart + numberOfPiecesBetweenCollections)); } static String copyIn16Bits(const String& text) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkEscapeAnalysis.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkEscapeAnalysis.cpp index b6ac71ade386a..0d9b9df054661 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkEscapeAnalysis.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkEscapeAnalysis.cpp @@ -184,6 +184,7 @@ static void noteWhatNewObjectFillsIn(Walker& walker, Reference object, const Ope } if (!isEnvironment && fields.size() == numberOfFieldsWritten && fillers.empty()) state.noteElement(object, keyOfPropertiesAsListed(), object); + state.noteElementsLieOutsideViews(object); } static Reference positionInFreshList(Walker& walker, uint32_t position) @@ -212,6 +213,7 @@ static void noteWhatNewArrayFillsIn(Walker& walker, Reference object, const Oper state.noteElement(object, key, valueHeldByFreshObject(walker.code(), element)); } state.noteElement(object, keyOfLengthOfFreshList(walker), positionInFreshList(walker, position + static_cast(holes.size()))); + state.noteElementsLieOutsideViews(object); state.noteShape(object, made.type); } @@ -245,6 +247,7 @@ static void noteWhatArgumentsObjectHolds(Walker& walker, Reference object, const if (function && walker.program().at(function).has(FunctionFlags::HasHiddenRest)) state.noteElement(object, keyOfListBehindParameters(), list); state.noteElement(object, keyOfLengthOfFreshList(walker), valueHeldByFreshObject(code, made.countOperand())); + state.noteElementsLieOutsideViews(object); state.noteShape(object, made.type); } @@ -272,6 +275,7 @@ static bool noteWhatListMadeWithLengthHolds(Walker& walker, Reference object, co return false; WalkState& state = walker.state(); state.noteElement(object, keyOfLengthOfFreshList(walker), positionInFreshList(walker, static_cast(known))); + state.noteElementsLieOutsideViews(object); state.noteShape(object, made.type); return true; } diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkState.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkState.cpp index 8cf9eabfe28fb..1b52ae0da2198 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkState.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkState.cpp @@ -1689,7 +1689,7 @@ void WalkState::Held::endWhatIsWritten(const ProgramOfState& program, const Effe Holder& record = m_holders.mutableAt(holder); record.isListedAsFreshWithElements = false; uint32_t next = std::exchange(record.nextFreshWithElements, none); - if (!record.hasEnded) + if (!record.hasEnded && !record.elementsLieOutsideViews) endPlacesOf(holder, ListOfPlaces::Elements, endsAlways); holder = next; } @@ -1903,8 +1903,11 @@ void WalkState::Held::meetWith(const Held& other, RecordsLastMet* last) if (record.shape != same.shape) endShape(holder); invalidateCellFacts(holder, factsKnownHereAlone); - if (record.elementsLieOutsideViews && !same.elementsLieOutsideViews) + if (record.elementsLieOutsideViews && !same.elementsLieOutsideViews) { m_holders.mutableAt(holder).elementsLieOutsideViews = false; + if (m_holders[holder].isFresh && m_holders[holder].firstElement != none) + listAsFreshWithElements(holder); + } if (leaves) letLeave(holder); } From 4f3f631416a7e3a8b014376f2e889dd20124e48d Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 10 Nov 2026 13:09:14 -0800 Subject: [PATCH 08/10] AOT: two tests of bounds, values on prototypes, build stops, order of originals On 33c7b694. 14 files, +206 -18: Source/JavaScriptCore/aot 13 files +126 -18 . Source/JavaScriptCore/b3 1 file +80 -0. - Two tests of bounds in a row, such as `if (v < 3) { a = 5; continue; } if (v > 9) { b = 7; continue; }`, took the first's arm for both where both arms are short. Fixed. - A value that a class lays over a member of a constructor function below it could be taken for false. Fixed. - A loop of three turns that compares an element with a number could stop the build. Fixed. - With `--aot-harden=proof-checks`: a loop that calls a function which keeps its list, an async function that passes a wider type to a parameter, and a `.cts` file with a loop stopped the build; a method that reads an element beyond the end of a list aborted. Fixed. - `bun test --aot`: the originals of two mocked files on one ring of imports ran in another order, around the first factory. Fixed. - A frame part is closed under dominance. - The stub that finds a text names every register that it writes. - A test of values that live across rare clobbers in the register allocator. --- Source/JavaScriptCore/aot/facts/AOTLink.cpp | 26 +++--- .../aot/machine/AOTMachineFrame.cpp | 6 ++ .../aot/machine/AOTMachineThreadJumps.cpp | 2 +- .../aot/runtime/AOTModuleEvaluation.cpp | 4 +- .../aot/stubs/AOTStubsStrings.cpp | 2 + .../aot/tests/AOTMachineTestsOfPhisTests.cpp | 47 +++++++++++ .../aot/tests/AOTStubBlobTests.cpp | 8 ++ .../aot/walk/AOTCodeVerifier.cpp | 9 ++- Source/JavaScriptCore/aot/walk/AOTWalk.cpp | 15 ++++ Source/JavaScriptCore/aot/walk/AOTWalk.h | 5 +- .../aot/walk/AOTWalkInlining.cpp | 4 +- .../aot/walk/AOTWalkLoopInvariants.cpp | 10 ++- .../aot/walk/AOTWalkTypedElements.cpp | 6 +- .../b3/air/testair_bun_allocation.h | 80 +++++++++++++++++++ 14 files changed, 206 insertions(+), 18 deletions(-) diff --git a/Source/JavaScriptCore/aot/facts/AOTLink.cpp b/Source/JavaScriptCore/aot/facts/AOTLink.cpp index 9779de6f6fef5..6deb97d0c868b 100644 --- a/Source/JavaScriptCore/aot/facts/AOTLink.cpp +++ b/Source/JavaScriptCore/aot/facts/AOTLink.cpp @@ -385,8 +385,10 @@ std::optional LinkTables::findsNothing(const Program& program, Lay }; for (LayoutID current = layout; current; current = of == Of::HolderAndAbove ? program.at(current).base : LayoutID()) { const Layout& written = program.at(current); + if (side == ClassSide::Prototype && (isLaidOnPrototypeOf(current, key) || mayBeLaidOnPrototypeUnderUnknownKey(program, current, key))) + return std::nullopt; if (written.has(LayoutFlags::IsConstructorFunction)) { - if (side == ClassSide::Static || isLaidOnPrototypeOf(current, key) || mayBeLaidOnPrototypeUnderUnknownKey(program, current, key)) + if (side == ClassSide::Static) return std::nullopt; continue; } @@ -406,16 +408,15 @@ bool LinkTables::findFunctionsLaidAbove(const Program& program, LayoutID layout, return std::pair { laid.layout, laid.key }; }; for (LayoutID current = layout; current; current = program.at(current).base) { - if (program.at(current).has(LayoutFlags::IsConstructorFunction)) { - if (mayBeLaidOnPrototypeUnderUnknownKey(program, current, key)) + if (mayBeLaidOnPrototypeUnderUnknownKey(program, current, key)) + return false; + for (const LinkedNameOnPrototype& laid : std::ranges::equal_range(namesOnPrototypes, std::pair { current, key }, { }, placeOf)) { + if (!laid.function) return false; - for (const LinkedNameOnPrototype& laid : std::ranges::equal_range(namesOnPrototypes, std::pair { current, key }, { }, placeOf)) { - if (!laid.function) - return false; - functions.append(laid.function); - } - continue; + functions.append(laid.function); } + if (program.at(current).has(LayoutFlags::IsConstructorFunction)) + continue; std::span places = membersUnder(current, ClassSide::Prototype, key); if (places.empty() && classHolderIsMadeWithName(program, ClassSide::Prototype, key)) return false; @@ -482,8 +483,13 @@ MemberFound LinkTables::findMember(const Program& program, LayoutID layout, Clas if (isKnown) firstClassAsked = nearest; } + auto isLaidOnPrototypeOf = [&](LayoutID holder) { + return std::ranges::binary_search(namesOnPrototypes, std::pair { holder, key }, { }, [](const LinkedNameOnPrototype& laid) { + return std::pair { laid.layout, laid.key }; + }); + }; for (LayoutID current = firstClassAsked; current; current = program.at(current).base) { - if (side == ClassSide::Prototype && program.at(current).has(LayoutFlags::IsConstructorFunction)) + if (side == ClassSide::Prototype && (program.at(current).has(LayoutFlags::IsConstructorFunction) || isLaidOnPrototypeOf(current))) return { }; MemberFound found; unsigned numberFound = 0; diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineFrame.cpp b/Source/JavaScriptCore/aot/machine/AOTMachineFrame.cpp index 3affda64f4629..8f8fb9fb65086 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineFrame.cpp +++ b/Source/JavaScriptCore/aot/machine/AOTMachineFrame.cpp @@ -793,6 +793,12 @@ void Allocator::findFramedPart() standsUnderHeadsOf(block, dominatorOf(block)); continue; } + if (const Block* dominator = dominatorOf(block); dominator && liesInFramedPart(dominator)) { + marks[block->index()] &= ~isHead; + isInFramedPart.set(block->index()); + standsUnderHeadsOf(block, dominator); + continue; + } const BlockPlace& place = placeOf(block); const Block* usualWayIn = nullptr; bool someRareWayInBringsHeads = false; diff --git a/Source/JavaScriptCore/aot/machine/AOTMachineThreadJumps.cpp b/Source/JavaScriptCore/aot/machine/AOTMachineThreadJumps.cpp index 76de7face6e61..19146b1cc4342 100644 --- a/Source/JavaScriptCore/aot/machine/AOTMachineThreadJumps.cpp +++ b/Source/JavaScriptCore/aot/machine/AOTMachineThreadJumps.cpp @@ -1272,7 +1272,7 @@ bool JumpThreader::leadToOnePlace(const Block* one, const Block* other) if (one == other) return true; auto onlyHandsOn = [](const Block* block) { - return block->numberOfIncomingEdges() == 1 && block->phis().empty() && !block->isLandingPad() && !block->isEntry() && !block->isEmpty() && block->last()->opcode() == Jump && block->size() == block->upsilons().size() + 1; + return block->numberOfIncomingEdges() == 1 && block->phis().empty() && !block->isLandingPad() && !block->isEntry() && block->size() == 1 && block->last()->opcode() == Jump; }; if (!onlyHandsOn(one) || !onlyHandsOn(other) || one->successor(0).block() != other->successor(0).block() || one->successor(0).edgeClass() != other->successor(0).edgeClass() || one->upsilons().size() != other->upsilons().size()) return false; diff --git a/Source/JavaScriptCore/aot/runtime/AOTModuleEvaluation.cpp b/Source/JavaScriptCore/aot/runtime/AOTModuleEvaluation.cpp index c61defdb39da5..29c4d9b7d2f2a 100644 --- a/Source/JavaScriptCore/aot/runtime/AOTModuleEvaluation.cpp +++ b/Source/JavaScriptCore/aot/runtime/AOTModuleEvaluation.cpp @@ -2192,7 +2192,9 @@ bool Modules::isNotLoaded(ModuleID module) void Modules::noteModulesLoadedWith(ModuleID root) { - if (!data(root).isESModuleOfProgram() || (data(root).isFacade() && !m_loadsModulesIndividually) || wasLoadedWithGraph(root) || (m_rootWhoseLoadIsSwept && !m_originalBeingEvaluated)) + if (!data(root).isESModuleOfProgram() || (data(root).isFacade() && !m_loadsModulesIndividually) || wasLoadedWithGraph(root)) + return; + if (m_rootWhoseLoadIsSwept && (!m_originalBeingEvaluated || importsStatically(root, *m_rootWhoseLoadIsSwept))) return; if (m_loadsModulesIndividually) { noteLoadOfEachModuleWith(root); diff --git a/Source/JavaScriptCore/aot/stubs/AOTStubsStrings.cpp b/Source/JavaScriptCore/aot/stubs/AOTStubsStrings.cpp index 8fbba8803697f..f6ee3c7de917b 100644 --- a/Source/JavaScriptCore/aot/stubs/AOTStubsStrings.cpp +++ b/Source/JavaScriptCore/aot/stubs/AOTStubsStrings.cpp @@ -412,6 +412,8 @@ void generateFindText8(StubAssembler& jit) { static_assert(leafArgumentGPR(0) == gprOf(StubRegister::A0) && leafArgumentGPR(1) == gprOf(StubRegister::A1) && leafArgumentGPR(2) == gprOf(StubRegister::A2)); static_assert(leafArgumentGPR(3) == gprOf(StubRegister::A3) && leafArgumentGPR(4) == gprOf(StubRegister::A4)); + for (unsigned index : { 2, 3, 4 }) + jit.clobberOperand(index); const GPRReg characters = jit.operand(0); const GPRReg length = jit.operand(1); const GPRReg sought = jit.operand(2); diff --git a/Source/JavaScriptCore/aot/tests/AOTMachineTestsOfPhisTests.cpp b/Source/JavaScriptCore/aot/tests/AOTMachineTestsOfPhisTests.cpp index f7786c80ca2c2..d3932d1390227 100644 --- a/Source/JavaScriptCore/aot/tests/AOTMachineTestsOfPhisTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTMachineTestsOfPhisTests.cpp @@ -358,6 +358,50 @@ static void testTwoBoundsOfOneValue() } } +enum class KindOfArms : uint8_t { OnlyHandOn, HoldStores, HandOnDifferentValues }; + +template +static void testArmsOfTwoBounds() +{ + TestOfPhiTest test; + Output& out = test.out; + const AbstractHeap& heap = test.testProcedure.heaps().argumentBuffer; + LBasicBlock below = out.newBlock(); + LBasicBlock asksUpperBound = out.newBlock(); + LBasicBlock above = out.newBlock(); + LBasicBlock inside = out.newBlock(); + LBasicBlock join = out.newBlock(); + LValue answer = out.phiIn(join, Int64); + LValue value = out.castToInt32(test.first); + auto arm = [&](LBasicBlock block, int32_t stored, ptrdiff_t offset, int64_t handedOn) { + out.appendTo(block); + if (kind == KindOfArms::HoldStores) + out.store32(out.constInt32(stored), out.address(heap, test.second, offset)); + out.upsilon(out.constInt64(handedOn), answer); + out.jump(join); + }; + out.branch(out.lessThan(value, out.constInt32(3)), normalEdge(below), normalEdge(asksUpperBound)); + arm(below, 5, 0, 10); + out.appendTo(asksUpperBound); + out.branch(out.greaterThan(value, out.constInt32(9)), normalEdge(above), normalEdge(inside)); + arm(above, 7, 4, kind == KindOfArms::HandOnDifferentValues ? 11 : 10); + out.appendTo(inside); + out.upsilon(out.constInt64(20), answer); + out.jump(join); + out.appendTo(join); + out.ret(answer); + const Order& order = test.compile(); + + AOT_TEST_OF_PHI_CHECK(test, order.of(asksUpperBound).isReached() == (kind != KindOfArms::OnlyHandOn)); + for (int32_t asked : { std::numeric_limits::min(), -1, 0, 2, 3, 4, 9, 10, 11, std::numeric_limits::max() }) { + std::array words { 0, 0 }; + int64_t expected = asked < 3 ? 10 : asked > 9 ? (kind == KindOfArms::HandOnDifferentValues ? 11 : 10) : 20; + AOT_TEST_OF_PHI_CHECK(test, test.run(asked, reinterpret_cast(words.data())) == expected); + AOT_TEST_OF_PHI_CHECK(test, words[0] == (kind == KindOfArms::HoldStores && asked < 3 ? 5 : 0)); + AOT_TEST_OF_PHI_CHECK(test, words[1] == (kind == KindOfArms::HoldStores && asked > 9 ? 7 : 0)); + } +} + static void testNeighboringStoresOfConstantsAreOneStore() { size_t neighbors = bytesOfStoresInTestOfPhiTest(Int32, 7, 0, 4, 1); @@ -398,6 +442,9 @@ std::span machineTestsOfPhisTests() { "machine-tests-of-phis/a-signed-lower-bound-not-below-nought-and-an-unsigned-upper-bound-are-asked-at-once"_s, Machine::testTwoBoundsOfOneValue }, { "machine-tests-of-phis/a-signed-lower-bound-below-nought-and-an-unsigned-upper-bound-stay-two-tests"_s, Machine::testTwoBoundsOfOneValue }, { "machine-tests-of-phis/an-unsigned-lower-and-a-signed-upper-bound-stay-two-tests"_s, Machine::testTwoBoundsOfOneValue }, + { "machine-tests-of-phis/arms-of-two-bounds-that-only-hand-on-one-value-are-one-arm"_s, Machine::testArmsOfTwoBounds }, + { "machine-tests-of-phis/arms-of-two-bounds-that-each-store-are-two-arms"_s, Machine::testArmsOfTwoBounds }, + { "machine-tests-of-phis/arms-of-two-bounds-that-hand-on-different-values-are-two-arms"_s, Machine::testArmsOfTwoBounds }, }; return tests; } diff --git a/Source/JavaScriptCore/aot/tests/AOTStubBlobTests.cpp b/Source/JavaScriptCore/aot/tests/AOTStubBlobTests.cpp index 9a22b6be892fa..873466a3cc7c9 100644 --- a/Source/JavaScriptCore/aot/tests/AOTStubBlobTests.cpp +++ b/Source/JavaScriptCore/aot/tests/AOTStubBlobTests.cpp @@ -532,6 +532,13 @@ static void checkSubstringSummary(const StubCatalog& catalog) AOT_TEST_CHECK(!containsRegister(summary.clobbered, reg)); } +static void checkTextSearchSummary(const StubCatalog& catalog) +{ + const StubSummary& summary = catalog.summaryOf(findText8()); + for (GPRReg reg : { A2, A3, gprOf(StubRegister::A4) }) + AOT_TEST_CHECK(containsRegister(summary.clobbered, reg)); +} + static void checkArrayCreationSummary(const StubCatalog& catalog, StubKey key, const StubSignature& largestLiteral) { const StubSummary& summary = catalog.summaryOf(key); @@ -640,6 +647,7 @@ static void testSummaries() checkCoroutineSummaries(catalog); checkNumberConversionSummary(catalog); checkSubstringSummary(catalog); + checkTextSearchSummary(catalog); checkArraySummaries(catalog); checkEqualitySummaries(catalog); } diff --git a/Source/JavaScriptCore/aot/walk/AOTCodeVerifier.cpp b/Source/JavaScriptCore/aot/walk/AOTCodeVerifier.cpp index 737cfd9fbe276..1de802bba8fb9 100644 --- a/Source/JavaScriptCore/aot/walk/AOTCodeVerifier.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTCodeVerifier.cpp @@ -1046,13 +1046,20 @@ ValueID CodeVerifier::findArrayWriteBetween(ValueID ground, ValueID access) cons BlockID home = m_code.blockOf(ground); BlockID block = m_code.blockOf(access); ValueID list = m_code.at(access).operand(0); + auto writesLists = [&](const Operation& operation) { + FunctionID callee = operation.opcode == Opcode::CallDirect && m_facts ? operation.function() : FunctionID(); + if (!callee) + return declaredBoundOf(operation.opcode).writes.containsAny(heapsOfLists); + const LinkedFunction& linked = m_facts->linked().function(callee); + return linked.effectsOnReturn().writes.containsAny(heapsOfLists) || linked.effectsOnThrow().writes.containsAny(heapsOfLists); + }; auto findIn = [&](BlockID in, uint32_t after, uint32_t before) -> ValueID { for (ValueID each : operationsOf(in)) { uint32_t place = m_places[each.index()]; const Operation& operation = m_code.at(each); if (place <= after || place >= before || operation.opcode == Opcode::ExtendArrayLength || operation.traits().mayRunOtherCode() || !operation.traits().mayWriteObservableState()) continue; - if (declaredBoundOf(operation.opcode).writes.containsAny(heapsOfLists) && operation.numberOfOperands() && operation.operand(0) && areSameCells(operation.operand(0), list)) + if (writesLists(operation) && operation.numberOfOperands() && operation.operand(0) && areSameCells(operation.operand(0), list)) return each; } return { }; diff --git a/Source/JavaScriptCore/aot/walk/AOTWalk.cpp b/Source/JavaScriptCore/aot/walk/AOTWalk.cpp index 56806446694bc..fbf06e5a633cb 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalk.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalk.cpp @@ -4034,6 +4034,19 @@ Reference Walker::build(Draft&& draft, const Effects* foundByCaller) m_valueBeforePendingStore = readFilledFieldOfProvenObject(*this, draft.objectOperand(), draft.key(), draft.origin()); if (m_hasFailed || isUnreachable()) return { }; + if (m_valueBeforePendingStore) { + Draft undoing(Opcode::PutSlot, draft.origin()); + chooseConservativeFields(undoing); + undoing.setObjectOperand(draft.objectOperand()); + undoing.setValueOperand(m_valueBeforePendingStore); + undoing.setKey(draft.key()); + undoing.setReceivers(draft.receivers()); + undoing.setLevel(KnowledgeLevel::Proven); + undoing.setMode(draft.mode()); + undoing.setIsDefinition(false); + if (findIllTypedOperand(typing(), undoing, typesOfOperands(undoing))) + m_valueBeforePendingStore = { }; + } } } if (steps & Step::isTypeTest) @@ -4685,6 +4698,8 @@ void Walker::leaveArrivals(FunctionID callee, std::span argumen Reference argument = arguments[entryParameter]; bool goesByWhatIsShown = argument && argument.isRecord() && (entryParameter >= function.numberOfEntryParameters() || function.entryParameterKind(entryParameter) != EntryParameter::Capture); TypeID proven = goesByWhatIsShown ? typeShown(argument) : ownTypeOf(argument); + if (TypeID own = goesByWhatIsShown ? ownTypeOf(argument) : TypeID(); proven && own && m_types.at(own).isJavaScriptValue() && !m_types.isSubset(own, proven)) + proven = m_types.unionOf(proven, own); if (!proven) proven = TypeTable::anything; m_arrivalsHeld.append({ callee, entryParameter, proven, ArrivalHeld::Kind::Proven }); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalk.h b/Source/JavaScriptCore/aot/walk/AOTWalk.h index fc048125c2d97..d3b679c55c09d 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalk.h +++ b/Source/JavaScriptCore/aot/walk/AOTWalk.h @@ -1573,10 +1573,13 @@ class Walker { TypeID typeShown(Reference); bool isShown(Reference value, TypeID within) { return m_types.isSubset(typeShown(value), within); } TypeID typeThatStoreTakesOnWordOfWalk(const Draft& store); - void checkProofOfResultOfCopy(Reference value, TypeID proven, Origin origin) + bool holdsNaNForUndefined(Reference returned) { return returned && returned.isValue() && m_code.at(returned.id()).opcode == Opcode::GetArrayElement && m_code.at(returned.id()).readsHoleAsNaN(); } + void checkProofOfResultOfCopy(Reference value, TypeID proven, Origin origin, bool someReturnHoldsNaNForUndefined) { if (value && value.isValue() && (m_code.at(value.id()).opcode == Opcode::AwaitCallBySavingFrame || m_code.at(value.id()).opcode == Opcode::AwaitBySavingFrame)) return; + if (someReturnHoldsNaNForUndefined && proven && (m_types.at(proven).primitives & Primitive::Undefined)) + proven = m_types.unionOf(proven, TypeTable::number); checkProofOfCopy(value, proven, ProofSource::ResultOfCall, origin); } void checkFactsRememberedOfCell(Reference object, Origin, OptionSet leanedOn = { }); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkInlining.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkInlining.cpp index b70fb24b3457c..c6d924db42328 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkInlining.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkInlining.cpp @@ -2102,7 +2102,7 @@ bool Walker::goOnBehindReturnsOfCopy(const BodyValue& call, const Function& func m_block = returns[0].block; m_state = WTF::move(returns[0].state); if (checksProofsOf(ProofSource::ResultOfCall) && !function.isCoroutine()) - checkProofOfResultOfCopy(returns[0].value, linked.result, originOfCall); + checkProofOfResultOfCopy(returns[0].value, linked.result, originOfCall, holdsNaNForUndefined(returns[0].value)); bind(call.id(), guardNumericCallResult(*this, call, returns[0].value, originOfCall)); return true; } @@ -2172,7 +2172,7 @@ bool Walker::goOnBehindReturnsOfCopy(const BodyValue& call, const Function& func if (!m_hasFailed) noteFieldsOfScopes(fieldsThatMeet.span()); if (checksProofsOf(ProofSource::ResultOfCall) && !function.isCoroutine()) - checkProofOfResultOfCopy(result, linked.result, originOfCall); + checkProofOfResultOfCopy(result, linked.result, originOfCall, std::ranges::any_of(returns, [&](const ReturnOfCopy& returned) { return holdsNaNForUndefined(returned.value); })); bind(call.id(), guardNumericCallResult(*this, call, result, originOfCall)); return true; } diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkLoopInvariants.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkLoopInvariants.cpp index 7b4566bbd2473..ccac6ae2b8210 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkLoopInvariants.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkLoopInvariants.cpp @@ -1047,8 +1047,16 @@ void Walker::appendCarriedResults(BlockID head, Vector& arguments) if (!loop || !loop->carriedResultCount) return; const Vector& requests = m_carriedResultRequests.find(loop->carriedResultsSummary)->value; + const BodyIndex& index = *m_frames[loop->frame].index; + auto isBoundByInnerLoop = [&](ValueID site) { + for (LoopID inner = index.loopOf(index.blockOf(site)); inner && inner != loop->loop; inner = index.loop(inner).parent) { + if (inputBlock(loop->frame, index.loop(inner).head).head) + return true; + } + return false; + }; for (uint32_t slot = 0; slot < loop->carriedResultCount; ++slot) { - Reference result = referenceBoundIn(m_frames[loop->frame], requests[slot].site); + Reference result = isBoundByInnerLoop(requests[slot].site) ? Reference() : referenceBoundIn(m_frames[loop->frame], requests[slot].site); BlockID block = result && result.isValue() && !m_code.isConstant(result.id()) ? m_code.blockOf(result.id()) : BlockID(); bool fits = block && block.index() > head.index() && ownTypeOf(result) && m_types.isSubset(ownTypeOf(result), requests[slot].type); arguments.append(fits ? result : constant(Draft(Opcode::Absent, m_origin))); diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkTypedElements.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkTypedElements.cpp index f7e1cf0c5d72b..8e183e175045f 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkTypedElements.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkTypedElements.cpp @@ -1207,12 +1207,12 @@ static bool buildArrayReadOfTypedElements(const AccessOfTypedElements& access) read.setReadsHoleAsNaN(false); bool givesNaNForHole = read.readsHoleAsNaN(); Reference element = leavesTurnAtHole ? walker.state().element(access.base, access.key) : Reference(); - bool isReadInFront = !!presentInFront; if (presentInFront) { if (std::exchange(read.testsPassed, std::nullopt)) read.setIsInBounds(*presentInFront); read.setIsPresent(*presentInFront); } + bool isReadInFront = presentInFront && !isNeverType(walker.resultTypeOf(read)); if (tested && !isReadInFront) takeStorageAndLengthFromInFrontOfLoop(walker, read, *tested); std::optional readAgain; @@ -1232,6 +1232,10 @@ static bool buildArrayReadOfTypedElements(const AccessOfTypedElements& access) WhatListWasMadeWith held = namesStorage && read.isPresent() && read.arrayStorage() == ArrayStorage::Contiguous ? whatListWasMadeWith(walker, access.base) : WhatListWasMadeWith(); element = held.proven ? walker.buildReadTypedByWhatIsRemembered(WTF::move(read), held.proven, held.shown) : walker.build(WTF::move(read)); } + if (isReadInFront && !element && !walker.isUnreachable()) { + walker.endAsUnreachable(walker.originOf(access.value)); + return true; + } if (readAgain && element && element.isValue()) { WalkState believed = walker.state(); walker.checkThatMemoryAnswersAsBelieved(element, WTF::move(*readAgain)); diff --git a/Source/JavaScriptCore/b3/air/testair_bun_allocation.h b/Source/JavaScriptCore/b3/air/testair_bun_allocation.h index 9a62b2c2785bc..f14183e613acb 100644 --- a/Source/JavaScriptCore/b3/air/testair_bun_allocation.h +++ b/Source/JavaScriptCore/b3/air/testair_bun_allocation.h @@ -21,6 +21,7 @@ RUN(testFirstFitSizes()); \ RUN(testNoEmptyInstAfterStackAllocation()); \ RUN(testValuesSurviveRandomClobbers()); \ + RUN(testValuesSurviveRareClobbers()); \ RUN(testValuesLiveAcross32And33Calls()); \ RUN(testValueReadByRareClobberStaysInRegisterInLoop()); \ RUN(testCursorsOfRareLoopTakeRegistersOfWordsThatPassThrough()); \ @@ -920,6 +921,85 @@ void testValuesSurviveRandomClobbers() } } +static void checkValuesSurviveRareClobbers(bool splitsEarly, unsigned seed) +{ + constexpr unsigned argumentCount = 6; + WeakRandom random(seed); + B3::Policy policy; + policy.air.splitting.splitsAroundClobbersEarly = splitsEarly; + policy.air.registers.chargesForCalleeSaves = splitsEarly; + B3::Procedure proc(policy); + proc.setOptLevel(2); + Code& code = proc.code(); + B3::Air::Special* special = patchpointSpecialOf(code); + BasicBlock* root = code.addBlock(); + BasicBlock* head = code.addBlock(8); + BasicBlock* rare = code.addBlock(0.05); + BasicBlock* tail = code.addBlock(8); + BasicBlock* exit = code.addBlock(); + + Vector values; + Vector model; + unsigned valueCount = 6 + random.getUint32(20); + for (unsigned i = 0; i < valueCount; ++i) { + uint64_t argument = 1000003ull * (i % argumentCount + 1); + values.append(code.newTmp(GP)); + model.append(argument + i); + root->append(Move, nullptr, argumentTmp(i % argumentCount), values[i]); + root->append(Add64, nullptr, Arg::imm(i), values[i]); + } + Tmp counter = code.newTmp(GP); + Tmp sum = code.newTmp(GP); + root->append(Move, nullptr, Arg::imm(3), counter); + root->append(Move, nullptr, Arg::imm(0), sum); + root->append(Jump, nullptr); + root->setSuccessors(FrequentedBlock(head)); + + Vector readInHead; + for (unsigned i = random.getUint32(valueCount); i--;) { + readInHead.append(random.getUint32(valueCount)); + head->append(Add64, nullptr, values[readInHead.last()], sum); + } + head->append(Branch32, nullptr, Arg::relCond(MacroAssembler::Equal), counter, Arg::imm(2)); + head->setSuccessors(FrequentedBlock(rare, B3::FrequencyClass::Rare), FrequentedBlock(tail)); + + for (unsigned i = 1 + random.getUint32(3); i--;) { + B3::PatchpointValue* patchpoint = addClobberingPatchpoint(proc, B3::Void, chooseRandomCallClobbers(random)); + patchpoint->clobberEarly(chooseRandomCallClobbers(random)); + rare->append(Patch, patchpoint, Arg::special(special)); + rare->append(Add64, nullptr, values[random.getUint32(valueCount)], counter); + rare->append(Move, nullptr, Arg::imm(2), counter); + } + rare->append(Jump, nullptr); + rare->setSuccessors(FrequentedBlock(tail)); + + tail->append(Add64, nullptr, Arg::imm(-1), counter); + tail->append(Branch32, nullptr, Arg::relCond(MacroAssembler::NotEqual), counter, Arg::imm(0)); + tail->setSuccessors(FrequentedBlock(head), FrequentedBlock(exit)); + + uint64_t expected = 0; + for (unsigned index : readInHead) + expected += 3 * model[index]; + for (unsigned i = 0; i < valueCount; ++i) { + exit->append(Add64, nullptr, values[i], sum); + expected += model[i]; + } + exit->append(Move, nullptr, sum, returnTmp()); + appendReturn(exit); + + auto compilation = compile(proc); + int64_t actual = invoke(*compilation, 1000003ll, 2000006ll, 3000009ll, 4000012ll, 5000015ll, 6000018ll); + CHECK(static_cast(actual) == expected); +} + +void testValuesSurviveRareClobbers() +{ + for (bool splitsEarly : { false, true }) { + for (unsigned seed = 1; seed <= 300; ++seed) + checkValuesSurviveRareClobbers(splitsEarly, seed); + } +} + void testValuesLiveAcross32And33Calls() { // Two fifths of the values have enough uses to pay for 33 splits. No CPU has a callee save for each of them. From 5473c680a77733420313d7092ce271bcfa2c91b9 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 10 Nov 2026 13:44:31 -0800 Subject: [PATCH 09/10] AOT: captured variables in loops of several versions On 4f3f6314. 1 file, +4 -1: Source/JavaScriptCore/aot 1 file +4 -1. A variable that a function captures and that a loop of several versions asks for could stop the build: `const f = make(1);` and a counted loop that sums what `f(a, b)` gives. Fixed. --- Source/JavaScriptCore/aot/walk/AOTWalkGuesses.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Source/JavaScriptCore/aot/walk/AOTWalkGuesses.cpp b/Source/JavaScriptCore/aot/walk/AOTWalkGuesses.cpp index 7f7b2f0052300..9a4e7577093ff 100644 --- a/Source/JavaScriptCore/aot/walk/AOTWalkGuesses.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTWalkGuesses.cpp @@ -2865,7 +2865,10 @@ static bool isStableAcrossIterations(Walker& walker, ValueID input) ValueID read = built.id(); while (valuePreservingOpcodes.contains(code.at(read).opcode) || code.at(read).opcode == Opcode::CheckInitialized) read = code.at(read).operand(0); - return code.at(read).opcode == Opcode::GetModuleVariable && walker.program().at(code.at(read).variable()).flags.contains(VariableFlags::IsImmutable); + if (code.at(read).opcode != Opcode::GetModuleVariable) + return false; + VariableID variable = code.at(read).variable(); + return !walker.plan().promotedEnvironments().find(variable) && walker.program().at(variable).flags.contains(VariableFlags::IsImmutable); } static bool isListThatFrontEndGuessedByNamesAloneForGuesses(Walker& walker, ValueID list) From 9ad6efd766229c3770dc0deebfb794fcb0ca99e6 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 10 Nov 2026 16:53:21 -0800 Subject: [PATCH 10/10] AOT: remainders of rounded sums; B3: a Phi that follows its own Upsilon On 5473c680. 7 files, +186 -25: Source/JavaScriptCore/aot 3 files +45 -6 . Source/JavaScriptCore/b3 4 files +141 -19. In a loop, a remainder took the integer behind a sum past 2 ** 53 in place of the rounded number: where y + i reaches 2 ** 53 + 1, (y + i) % 3 was the remainder of 2 ** 53 + 1 instead of that of 2 ** 53. Fixed. With `--aot-harden=proof-checks`, a list that is read at its older length behind a change of its storage stopped a build: the verifier took the position of the access for a ground of it. Fixed. B3, plain Bun too: a Phi that follows its own Upsilon in a block is folded, so a Phi of that kind in a loop header is not taken for one that the loop carries. Assertion builds poison Value::owner pointers that are not up to date. This is oven-sh/WebKit#986. --- .../codegen/AOTOperatorEmitterArithmetic.cpp | 6 +-- .../old-lengths-serve-as-positions.ir | 38 +++++++++++++ .../aot/walk/AOTCodeVerifier.cpp | 7 ++- .../b3/B3EliminateRedundantPhiCycles.cpp | 42 ++++++++++++++- Source/JavaScriptCore/b3/B3Generate.cpp | 21 +++++++- Source/JavaScriptCore/b3/testb3_bun_loops.cpp | 44 +++++++++------ .../b3/testb3_bun_version_loops.cpp | 53 +++++++++++++++++++ 7 files changed, 186 insertions(+), 25 deletions(-) create mode 100644 Source/JavaScriptCore/aot/tests/walk/positions-inside/old-lengths-serve-as-positions.ir diff --git a/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterArithmetic.cpp b/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterArithmetic.cpp index 579acde154cb8..b333d23e9abcf 100644 --- a/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterArithmetic.cpp +++ b/Source/JavaScriptCore/aot/codegen/AOTOperatorEmitterArithmetic.cpp @@ -586,12 +586,10 @@ class IntegerArithmeticEmitter final : public InlineMathEmitter { if (!type.isInteger() || type.integers.minimum < 0) return std::nullopt; Integer whole; - if (number.value()->opcode() != Machine::IToD) - whole = emitter().truncateUnchecked(number); - else if (LValue converted = number.value()->child(0); converted->type() == Machine::Int32) + if (LValue converted = number.value()->opcode() == Machine::IToD ? number.value()->child(0) : nullptr; converted && converted->type() == Machine::Int32) whole = zeroExtended(Word32(converted)); else - whole = Integer(converted); + whole = emitter().toInteger(number, type.integers, value.block() && generator().isInLoop(value.block())); if (checksProofs()) { Condition isInRange = logicalAnd(greaterThanOrEqual(whole, integer(0)), lessThanOrEqual(whole, integer(type.integers.maximum))); emitter().requireOrCrash(value, logicalAnd(equal(toNumber(whole), number), isInRange), std::nullopt, ProofSource::OtherOperation); diff --git a/Source/JavaScriptCore/aot/tests/walk/positions-inside/old-lengths-serve-as-positions.ir b/Source/JavaScriptCore/aot/tests/walk/positions-inside/old-lengths-serve-as-positions.ir new file mode 100644 index 0000000000000..e34d3b1caf373 --- /dev/null +++ b/Source/JavaScriptCore/aot/tests/walk/positions-inside/old-lengths-serve-as-positions.ir @@ -0,0 +1,38 @@ +# list[list.length]: the length that was read before the list's storage was converted is the POSITION of the first read. The test holds that number below a length read behind the conversion, and the +# verifier passes it. The second read rests on the old length as its BOUND, and there the verifier stops as before. +# RUN: verify-grounds access=v7 view=v0 position=v3 length=v5 below=v6 passed=v6 access=v10 view=v0 position=v1 length=v3 below=v9 passed=v9 +# CHECK: v10 (GetArrayElement): the storage or the length of a list is written between a ground of an access and the access: v4 (ConvertToDoubleStorage) +# -- code +# code f1:Body { +# constants: +# v2: undefined @JSValue = Undefined +# b0(v0: Array @Cell, v1: int[0, 2147483647] @Int32) entry=0: +# v3: int[0, 4294967295] @Int64 = GetArrayLength v0 traits=[mayTouchMemory] +# v4 @Void = ConvertToDoubleStorage v0 traits=[mayCollect, mayWriteObservableState, mayTouchMemory] +# v5: int[0, 4294967295] @Int64 = GetArrayLength v0 traits=[mayTouchMemory] +# v6: boolean @Boolean = Less v3, v5 form=Integers +# Branch v6 -> taken b1(), notTaken b3() +# b1(): +# v7: anything @JSValue = GetArrayElement v0, v3 level=Proven [keyIsIntegral, isInBounds] traits=[mayThrow, mayThrowByRule, mayCollect, mayTouchMemory] +# v8: int[0, 2147483647] @Int64 = WidenInteger v1 +# v9: boolean @Boolean = Less v8, v3 form=Integers +# Branch v9 -> taken b2(), notTaken b3() +# b2(): +# v10: anything @JSValue = GetArrayElement v0, v1 level=Proven [keyIsIntegral, isInBounds] traits=[mayThrow, mayThrowByRule, mayCollect, mayTouchMemory] +# Return v10 +# b3(): +# Return v2 +# } +# -- end of code + +module m0 path="main.js" displayPath="main.js" body=f0 +entry module=m0 +function f0 module=m0 kind=Module thisType={type=anything} results=[{type=anything}] { + b0(): + v0 = Undefined + Return v0 +} +function f1 name="asked" module=m0 length=2 thisType={type=anything} parameters=[{type=anything}, {type=anything}] results=[{type=anything}] { + b0(v0: anything, v1: anything): + Return v0 +} diff --git a/Source/JavaScriptCore/aot/walk/AOTCodeVerifier.cpp b/Source/JavaScriptCore/aot/walk/AOTCodeVerifier.cpp index 1de802bba8fb9..647b164914dc2 100644 --- a/Source/JavaScriptCore/aot/walk/AOTCodeVerifier.cpp +++ b/Source/JavaScriptCore/aot/walk/AOTCodeVerifier.cpp @@ -1791,8 +1791,11 @@ void CodeVerifier::verifyTestsPassed() const Operation& test = m_code.at(side.test); if (!writtenBy) writtenBy = findArrayWriteAfter(side.test); - for (uint32_t operand = 0; !writtenBy && operand < test.numberOfOperands(); ++operand) - writtenBy = findArrayWriteAfter(test.operand(operand)); + ValueID position = valueBeforeConversions(underlyingCell(access.operand(1))); + for (uint32_t operand = 0; !writtenBy && operand < test.numberOfOperands(); ++operand) { + if (test.operand(operand) && valueBeforeConversions(underlyingCell(test.operand(operand))) != position) + writtenBy = findArrayWriteAfter(test.operand(operand)); + } } if (writtenBy) return fail(block, id, "the storage or the length of a list is written between a ground of an access and the access"_s, writtenBy); diff --git a/Source/JavaScriptCore/b3/B3EliminateRedundantPhiCycles.cpp b/Source/JavaScriptCore/b3/B3EliminateRedundantPhiCycles.cpp index d9e1d2e6dfd1b..7cc1352610f24 100644 --- a/Source/JavaScriptCore/b3/B3EliminateRedundantPhiCycles.cpp +++ b/Source/JavaScriptCore/b3/B3EliminateRedundantPhiCycles.cpp @@ -20,9 +20,39 @@ namespace JSC { namespace B3 { -bool eliminateRedundantPhiCycles(Procedure& proc) +static bool foldPhisThatFollowTheirUpsilon(Procedure& proc) +{ + IndexSet folded; + Vector upsilonsOfBlock; + for (BasicBlock* block : proc) { + upsilonsOfBlock.shrink(0); + for (Value* value : *block) { + if (auto* upsilon = value->as()) { + upsilonsOfBlock.append(upsilon); + continue; + } + if (value->opcode() != Phi) + continue; + size_t last = upsilonsOfBlock.reverseFindIf([&](UpsilonValue* upsilon) { return upsilon->phi() == value; }); + if (last == notFound || upsilonsOfBlock[last]->child(0) == value) + continue; + value->replaceWithIdentity(upsilonsOfBlock[last]->child(0)); + folded.add(value); + } + } + if (folded.isEmpty()) + return false; + for (Value* value : proc.values()) { + if (auto* upsilon = value->as(); upsilon && folded.contains(upsilon->phi())) + upsilon->replaceWithNop(); + } + for (Value* value : proc.values()) + value->performSubstitution(); + return true; +} + +static bool eliminateCycles(Procedure& proc) { - PhaseScope phaseScope(proc, "eliminateRedundantPhiCycles"_s); PhiChildren phiChildren(proc); const auto& phis = phiChildren.phis(); if (phis.isEmpty()) @@ -154,6 +184,14 @@ bool eliminateRedundantPhiCycles(Procedure& proc) return true; } +bool eliminateRedundantPhiCycles(Procedure& proc) +{ + PhaseScope phaseScope(proc, "eliminateRedundantPhiCycles"_s); + bool changed = foldPhisThatFollowTheirUpsilon(proc); + changed |= eliminateCycles(proc); + return changed; +} + } } // namespace JSC::B3 #endif // ENABLE(B3_JIT) diff --git a/Source/JavaScriptCore/b3/B3Generate.cpp b/Source/JavaScriptCore/b3/B3Generate.cpp index ba516a3145398..baa75ffa91059 100644 --- a/Source/JavaScriptCore/b3/B3Generate.cpp +++ b/Source/JavaScriptCore/b3/B3Generate.cpp @@ -417,6 +417,18 @@ static constexpr Step steps[] = { NeededEstimate::BlockFrequencies }, }; +#if ASSERT_ENABLED +void poisonStaleValueOwners(Procedure& procedure) +{ + for (BasicBlock* block : procedure) { + for (Value* value : *block) { + if (value->owner != block) + value->owner = std::bit_cast(static_cast(0xbadbeef0)); + } + } +} +#endif + void runCleanups(std::span procedures, std::span states) { struct CleanupPass { @@ -434,8 +446,12 @@ void runCleanups(std::span procedures, std::span procedures, const StepObserver* observ procedure.updateEdgeClasses(); else if (step.neededEstimate == NeededEstimate::BlockFrequencies) procedure.updateFrequencies(); +#if ASSERT_ENABLED + poisonStaleValueOwners(procedure); +#endif state.pendingCleanups = step.run(procedure, state); hasPendingCleanups |= !state.pendingCleanups.isEmpty(); } diff --git a/Source/JavaScriptCore/b3/testb3_bun_loops.cpp b/Source/JavaScriptCore/b3/testb3_bun_loops.cpp index b32ce459aa3d3..a5f591d90db38 100644 --- a/Source/JavaScriptCore/b3/testb3_bun_loops.cpp +++ b/Source/JavaScriptCore/b3/testb3_bun_loops.cpp @@ -760,23 +760,35 @@ static void testEliminateRedundantPhiCycles() } } -static void testEliminateRedundantPhiCyclesKeepsPhiOfValueInSameBlock() +static void testEliminateRedundantPhiCyclesFoldsPhiThatFollowsItsUpsilon() { - Procedure proc; - Graph graph { proc }; - BasicBlock* root = proc.addBlock(); - Value* x = cCallArgumentValues(proc, root)[0]; - UpsilonValue* upsilon = root->appendNew(proc, Origin(), graph.make(root, Add, x, graph.int64(root, 1))); - Value* phi = graph.phi(root, Int64); - upsilon->setPhi(phi); - graph.ret(root, phi); - proc.resetReachability(); - validate(proc); + for (bool hasOtherUpsilon : { false, true }) { + Procedure proc; + Graph graph { proc }; + BasicBlock* root = proc.addBlock(); + BasicBlock* other = proc.addBlock(); + BasicBlock* rest = proc.addBlock(); + Value* x = cCallArgumentValues(proc, root)[0]; + graph.branch(root, graph.make(root, Equal, x, graph.int64(root, 7)), other, rest); + UpsilonValue* otherUpsilon = hasOtherUpsilon ? other->appendNew(proc, Origin(), graph.int64(other, 3)) : nullptr; + graph.ret(other, graph.int64(other, -1)); + UpsilonValue* upsilon = rest->appendNew(proc, Origin(), graph.make(rest, Add, x, graph.int64(rest, 1))); + Value* phi = graph.phi(rest, Int64); + upsilon->setPhi(phi); + if (otherUpsilon) + otherUpsilon->setPhi(phi); + graph.ret(rest, phi); + proc.resetReachability(); + validate(proc); - CHECK(!eliminateRedundantPhiCycles(proc)); - validate(proc); - CHECK_EQ(numberOfValues(proc, Phi), 1u); - CHECK_EQ(compileAndRun(proc, static_cast(41)), 42); + CHECK(eliminateRedundantPhiCycles(proc)); + validate(proc); + CHECK_EQ(numberOfValues(proc, Phi), 0u); + CHECK_EQ(numberOfValues(proc, Upsilon), 0u); + auto code = compileProc(proc); + CHECK_EQ(invoke(*code, static_cast(41)), 42); + CHECK_EQ(invoke(*code, static_cast(7)), -1); + } } void addBunLoopTests(const TestConfig* config, BunTestTasks& tasks) @@ -804,7 +816,7 @@ void addBunLoopTests(const TestConfig* config, BunTestTasks& tasks) RUN(testSimplifyTerminalsLooksThroughIdentity()); RUN(testSimplifyTerminalsOfSwitch()); RUN(testEliminateRedundantPhiCycles()); - RUN(testEliminateRedundantPhiCyclesKeepsPhiOfValueInSameBlock()); + RUN(testEliminateRedundantPhiCyclesFoldsPhiThatFollowsItsUpsilon()); } #endif // ENABLE(B3_JIT) diff --git a/Source/JavaScriptCore/b3/testb3_bun_version_loops.cpp b/Source/JavaScriptCore/b3/testb3_bun_version_loops.cpp index 91f8d4ec73f0d..5a531399c1b1f 100644 --- a/Source/JavaScriptCore/b3/testb3_bun_version_loops.cpp +++ b/Source/JavaScriptCore/b3/testb3_bun_version_loops.cpp @@ -8,6 +8,7 @@ #if ENABLE(B3_JIT) +#include "B3EliminateRedundantPhiCycles.h" #include "B3LoopVersionPlan.h" template @@ -769,6 +770,57 @@ static void testVersionLoopsHoistsLoadRepeatingLoadBeforeLoop() } } +static void testVersionLoopsPhiThatFollowsItsUpsilonInHeaderIsNotLoopCarried() +{ + for (bool runsPrecedingPhase : { true, false }) { + Procedure proc(optimizingPolicy()); + Graph graph { proc }; + TopTestedLoop loop(graph); + BasicBlock* special = proc.addBlock(); + BasicBlock* guarded = proc.addBlock(); + BasicBlock* unexpected = proc.addBlock(); + BasicBlock* hot = proc.addBlock(); + MemoryValue* earlierLoad = proc.add(Load, Int64, Origin(), loop.pointer, 0, firstHeap); + loop.root->appendNonTerminal(earlierLoad); + loop.root->appendNonTerminal(proc.add(Store, Origin(), earlierLoad, loop.pointer, 0, firstHeap)); + UpsilonValue* given = loop.body->appendNew(proc, Origin(), loop.extra); + Value* phi = graph.phi(loop.body, Int64); + given->setPhi(phi); + UpsilonValue* leftOver = proc.add(Origin(), proc.add(Origin(), 3)); + loop.exit->values().insert(0, leftOver->child(0)); + loop.exit->values().insert(1, leftOver); + leftOver->setPhi(phi); + graph.branch(loop.body, graph.make(loop.body, Equal, graph.make(loop.body, Add, graph.get(loop.body, loop.index), phi), earlierLoad), rarely(special), FrequentedBlock(guarded)); + graph.increase(special, loop.sum, 1000); + graph.jump(special, guarded); + Value* loaded = graph.load(guarded, Int64, loop.pointer, 0, firstHeap); + graph.branch(guarded, graph.make(guarded, Equal, loaded, graph.int64(guarded, 4)), FrequentedBlock(hot), rarely(unexpected)); + graph.increase(unexpected, loop.sum, 100); + graph.jump(unexpected, hot); + graph.increase(hot, loop.sum, loaded); + loop.endBody(hot); + loop.rotate(); + CHECK_EQ(numberOfValues(proc, [&](Value* value) { return value == phi && value->opcode() == Phi; }), 1u); + + if (runsPrecedingPhase) + CHECK(eliminateRedundantPhiCycles(proc)); + CHECK(versionLoopsAndValidate(proc)); + fixSSA(proc); + if (!runsPrecedingPhase) { + CHECK(proc.report().failure && proc.report().failure->kind == FailureKind::VariableReadBeforeSet); + continue; + } + CHECK(!proc.report().failure); + auto code = compileProc(proc); + int64_t memory[1] = { 4 }; + CHECK_EQ(run(*code, 10, memory, 0), 1040); + CHECK_EQ(run(*code, 10, memory, 4), 1040); + CHECK_EQ(run(*code, 10, memory, 5), 40); + CHECK_EQ(run(*code, 3, memory, 3), 1012); + CHECK_EQ(run(*code, 0, memory, 4), 0); + } +} + static void testVersionLoopsResolvesSingleInputPhiInUncountedLoop() { for (bool keyHasOneInput : { true, false }) { @@ -1168,6 +1220,7 @@ void addBunLoopVersioningTests(const TestConfig* config, BunTestTasks& tasks) RUN(testVersionLoopsKeepsTerminalPatchpointReadingLoopWrites()); RUN(testVersionLoopsTestsSlotBeforeReadingThroughIt()); RUN(testVersionLoopsIdentityInHeaderIsNoPhi()); + RUN(testVersionLoopsPhiThatFollowsItsUpsilonInHeaderIsNotLoopCarried()); } #endif // ENABLE(B3_JIT)