AOT: faster loops, async code and WebAssembly, smaller executables, stack traces as in bun run

Source/JavaScriptCore/aot/AOT.cmake

@@ -74,8 +74,10 @@ set(JavaScriptCore_AOT_EXPORTED_HEADERS
     aot/image/AOTImageForward.h
     aot/image/AOTImageLoading.h
     aot/image/AOTImageText.h
+    aot/image/AOTPrefixCode.h
     aot/image/AOTRegExpSummary.h
     aot/image/AOTRuntimeDescription.h
+    aot/image/AOTSideTableCodes.h
     aot/image/AOTStoredStrings.h
     aot/image/AOTVarint.h
     aot/image/AOTWasmInterface.h

Source/JavaScriptCore/aot/Sources.txt

@@ -201,6 +201,7 @@ aot/image/AOTCallSiteTable.cpp
 aot/image/AOTClassRecords.cpp
 aot/image/AOTCodeLayout.cpp
 aot/image/AOTCodeLinker.cpp
+aot/image/AOTCodedTexts.cpp
 aot/image/AOTConstantPool.cpp
 aot/image/AOTConstantTables.cpp
 aot/image/AOTDataTable.cpp
@@ -222,6 +223,7 @@ aot/image/AOTModuleTables.cpp
 aot/image/AOTNumberConstants.cpp
 aot/image/AOTNumberFunctions.cpp
 aot/image/AOTNumbering.cpp
+aot/image/AOTPrefixCode.cpp
 aot/image/AOTProgramTablesDriver.cpp
 aot/image/AOTRegExpCompiler.cpp
 aot/image/AOTRegExpInProcessLoader.cpp

Source/JavaScriptCore/aot/builtins/AOTBuiltinArrayOperations.cpp

@@ -28,9 +28,12 @@
 #include <algorithm>
 #include <array>
 #include <bit>
+#include <climits>
+#include <limits>
 #include <memory>
 #include <optional>
 #include <span>
+#include <utility>
 #include <wtf/Atomics.h>
 #include <wtf/UnalignedAccess.h>
 #include <wtf/text/StringCommon.h>
@@ -429,12 +432,39 @@ JSC_DEFINE_JIT_OPERATION(operationAOTArraySearchOfAnyValue, EncodedJSValue, (Ins
     OPERATION_RETURN(scope, encodedJSValue());
 }
 
+static bool tryUnshiftItemFast(VM& vm, JSArray* array, JSValue item)
+{
+    IndexingType mode = array->indexingMode();
+    bool itemFits = mode == ArrayWithContiguous || (mode == ArrayWithInt32 && item.isInt32()) || (mode == ArrayWithDouble && item.isNumber() && item.asNumber() == item.asNumber());
+    if (!itemFits)
+        return false;
+    Butterfly* butterfly = array->butterfly();
+    unsigned length = butterfly->publicLength();
+    if (length > JSArray::shiftThreshold || length >= butterfly->vectorLength())
+        return false;
+    butterfly->setPublicLength(length + 1);
+    WriteBarrier<Unknown>* elements = butterfly->contiguous().data();
+    if (mode == ArrayWithContiguous && vm.heap.mutatorShouldBeFenced())
+        gcSafeMemmove(elements + 1, elements, length * sizeof(JSValue));
+    else
+        memmove(static_cast<void*>(elements + 1), static_cast<const void*>(elements), length * sizeof(JSValue));
+    if (mode == ArrayWithDouble)
+        butterfly->contiguousDouble().atUnsafe(0) = item.asNumber();
+    else
+        elements[0].setWithoutWriteBarrier(item);
+    if (mode == ArrayWithContiguous)
+        vm.writeBarrier(array);
+    return true;
+}
+
 JSC_DEFINE_JIT_OPERATION(operationAOTArrayUnshift, EncodedJSValue, (Instance* instance, JSCell* cell, EncodedJSValue item))
 {
     AOT_OPERATION_BEGIN(instance, Entry::operationAOTArrayUnshift);
     JSArray* array = cellArgument<JSArray>(cell, Entry::operationAOTArrayUnshift);
     if (mayRejectOrInterceptWrites(array) || array->structure()->holesMustForwardToPrototype(array)) [[unlikely]]
         OPERATION_RETURN(scope, encodedJSValue());
+    if (tryUnshiftItemFast(vm, array, JSValue::decode(item))) [[likely]]
+        OPERATION_RETURN(scope, JSValue::encode(jsNumber(array->length())));
     uint64_t length = array->length();
     unshift(globalObject, array, 0, 0, 1, length);
     OPERATION_RETURN_IF_EXCEPTION(scope, encodedJSValue());
@@ -445,6 +475,127 @@ JSC_DEFINE_JIT_OPERATION(operationAOTArrayUnshift, EncodedJSValue, (Instance* in
     OPERATION_RETURN(scope, JSValue::encode(jsNumber(length + 1)));
 }
 
+static constexpr unsigned bitsOfSortDigit = 11;
+static constexpr unsigned numberOfSortDigits = 1 << bitsOfSortDigit;
+
+template<typename Key> static constexpr size_t minLengthSortedByDigits = sizeof(Key) > sizeof(uint32_t) ? 2048 : 512;
+
+template<typename Key>
+static std::span<Key> sortKeysByDigits(std::span<Key> keys, std::span<Key> scratch)
+{
+    constexpr unsigned numberOfPasses = (sizeof(Key) * CHAR_BIT + bitsOfSortDigit - 1) / bitsOfSortDigit;
+    using CountsOfPass = std::array<uint32_t, numberOfSortDigits>;
+    auto digitOf = [](Key key, unsigned pass) -> unsigned {
+        return static_cast<unsigned>(key >> (pass * bitsOfSortDigit)) & (numberOfSortDigits - 1);
+    };
+    void* memory = nullptr;
+    if (!tryFastZeroedMalloc(numberOfPasses * sizeof(CountsOfPass)).getValue(memory))
+        return { };
+    auto counts = std::span { static_cast<CountsOfPass*>(memory), numberOfPasses };
+    for (Key key : keys) {
+        for (unsigned pass = 0; pass < numberOfPasses; ++pass)
+            ++counts[pass][digitOf(key, pass)];
+    }
+    for (unsigned pass = 0; pass < numberOfPasses; ++pass) {
+        CountsOfPass& places = counts[pass];
+        if (places[digitOf(keys[0], pass)] == keys.size())
+            continue;
+        uint32_t place = 0;
+        for (uint32_t& count : places)
+            place += std::exchange(count, place);
+        for (Key key : keys)
+            scratch[places[digitOf(key, pass)]++] = key;
+        std::swap(keys, scratch);
+    }
+    fastFree(memory);
+    return keys;
+}
+
+template<typename Key, typename Element>
+static bool sortElementsByKey(std::span<Element> elements, NOESCAPE const auto& tryMakeKey, NOESCAPE const auto& elementOf)
+{
+    bool usesDigits = elements.size() >= minLengthSortedByDigits<Key> && elements.size() <= std::numeric_limits<uint32_t>::max();
+    void* memory = nullptr;
+    if (!tryFastMalloc((usesDigits ? 2 : 1) * elements.size() * sizeof(Key)).getValue(memory))
+        return false;
+    auto keys = std::span { static_cast<Key*>(memory), elements.size() };
+    bool hasKeys = true;
+    for (size_t i = 0; i < elements.size() && hasKeys; ++i)
+        hasKeys = tryMakeKey(elements[i], keys[i]);
+    std::span<Key> sorted;
+    if (hasKeys && usesDigits)
+        sorted = sortKeysByDigits(keys, std::span { static_cast<Key*>(memory) + elements.size(), elements.size() });
+    else if (hasKeys) {
+        std::sort(keys.begin(), keys.end());
+        sorted = keys;
+    }
+    for (size_t i = 0; i < sorted.size(); ++i)
+        elements[i] = elementOf(sorted[i]);
+    fastFree(memory);
+    return !sorted.empty();
+}
+
+static bool tryMakeSortKeyOfDouble(double number, uint64_t mask, uint64_t& key)
+{
+    constexpr uint64_t signBit = 1ULL << 63;
+    uint64_t bits = std::bit_cast<uint64_t>(number);
+    if (number != number || bits == signBit) [[unlikely]]
+        return false;
+    key = (bits ^ (static_cast<uint64_t>(static_cast<int64_t>(bits) >> 63) | signBit)) ^ mask;
+    return true;
+}
+
+static double doubleOfSortKey(uint64_t key, uint64_t mask)
+{
+    constexpr uint64_t signBit = 1ULL << 63;
+    key ^= mask;
+    return std::bit_cast<double>(key ^ (key & signBit ? signBit : ~0ULL));
+}
+
+JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationAOTSortNumbers, size_t, (Instance*, JSCell* cell, int32_t direction))
+{
+    JSArray* array = cellArgument<JSArray>(cell, Entry::operationAOTSortNumbers);
+    Butterfly& butterfly = *array->butterfly();
+    size_t length = butterfly.publicLength();
+    uint64_t mask = direction < 0 ? ~0ULL : 0;
+    switch (array->indexingMode()) {
+    case ArrayWithDouble:
+        return sortElementsByKey<uint64_t>(std::span { butterfly.contiguousDouble().data(), length }, [&](double number, uint64_t& key) {
+            return tryMakeSortKeyOfDouble(number, mask, key);
+        }, [&](uint64_t key) {
+            return doubleOfSortKey(key, mask);
+        });
+    case ArrayWithInt32:
+    case ArrayWithContiguous: {
+        auto elements = std::span { std::bit_cast<EncodedJSValue*>(butterfly.contiguous().data()), length };
+        bool holdsOnlyInt32s = std::ranges::all_of(elements, [](EncodedJSValue element) {
+            return JSValue::decode(element).isInt32();
+        });
+        if (holdsOnlyInt32s) {
+            auto flipped = [&](uint32_t bits) {
+                return bits ^ static_cast<uint32_t>(mask) ^ (1U << 31);
+            };
+            return sortElementsByKey<uint32_t>(elements, [&](EncodedJSValue element, uint32_t& key) {
+                key = flipped(static_cast<uint32_t>(JSValue::decode(element).asInt32()));
+                return true;
+            }, [&](uint32_t key) {
+                return JSValue::encode(jsNumber(static_cast<int32_t>(flipped(key))));
+            });
+        }
+        if (array->indexingMode() == ArrayWithInt32)
+            return false;
+        return sortElementsByKey<uint64_t>(elements, [&](EncodedJSValue element, uint64_t& key) {
+            JSValue number = JSValue::decode(element);
+            return number.isNumber() && tryMakeSortKeyOfDouble(number.asNumber(), mask, key);
+        }, [&](uint64_t key) {
+            return JSValue::encode(jsNumber(doubleOfSortKey(key, mask)));
+        });
+    }
+    default:
+        return false;
+    }
+}
+
 JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationAOTNewArrayIterator, JSCell*, (Instance* instance, JSCell* array, uint32_t kind))
 {
     AOT_OPERATION_BEGIN(instance, Entry::operationAOTNewArrayIterator);
@@ -479,6 +630,8 @@ JSC_DEFINE_JIT_OPERATION(operationAOTArrayUnshiftGeneric, EncodedJSValue, (Insta
     JSArray* array = cellArgument<JSArray>(cell, Entry::operationAOTArrayUnshiftGeneric);
     if (mayRejectOrInterceptWrites(array) || array->structure()->holesMustForwardToPrototype(array)) [[unlikely]]
         OPERATION_RETURN(scope, JSValue::encode(callMethodInEngine(instance, Builtins::ArrayPrototypeUnshift, array, JSValue::decode(item))));
+    if (tryUnshiftItemFast(vm, array, JSValue::decode(item))) [[likely]]
+        OPERATION_RETURN(scope, JSValue::encode(jsNumber(array->length())));
     uint64_t length = array->length();
     unshift(globalObject, array, 0, 0, 1, length);
     OPERATION_RETURN_IF_EXCEPTION(scope, encodedJSValue());

Source/JavaScriptCore/aot/builtins/AOTBuiltinCallEmitter.cpp

@@ -208,7 +208,9 @@ void BuiltinCallEmitter::generateCallBuiltin(const ValueContext& value)
         stopUnlessNativeResultIsAsWritten(value, result, overload.builtin, overload.mode == BuiltinCallMode::Construct);
         if (collectionsBefore) {
             Boxed none = boxedConstant(JSValue());
-            callVoidOperation(value, RuntimeOperation::CheckFreshResult, { result, *collectionsBefore, word32(static_cast<int32_t>(overload.builtin.index())), receiver ? boxedOperand(value, receiver) : none, arguments.empty() ? none : boxedOperand(value, arguments[0]) });
+            const Operation* receiverOperation = receiver ? &m_code.at(underlyingCellOf(m_code, receiver)) : nullptr;
+            bool hasOriginalReceiver = receiverOperation && (receiverOperation->opcode != Opcode::ToObject || receiverOperation->calledBuiltin() != overload.builtin);
+            callVoidOperation(value, RuntimeOperation::CheckFreshResult, { result, *collectionsBefore, word32(static_cast<int32_t>(overload.builtin.index())), hasOriginalReceiver ? boxedOperand(value, receiver) : none, arguments.empty() ? none : boxedOperand(value, arguments[0]) });
         }
         defineAs(value, result, typeOf(value.id()));
     };
@@ -449,6 +451,11 @@ Boxed BuiltinCallEmitter::callBuiltinFunction(const OperationContext& context, B
     bool constructs = overload.mode == BuiltinCallMode::Construct;
     unsigned count = static_cast<unsigned>(arguments.size());
     std::optional<StubKey> stub = stubThatCallsBuiltin(m_builtins, overload, count);
+    if (overload.mode == BuiltinCallMode::Get && receiver && !count && stub && stub->family() == StubFamily::CallHostFunctionWithoutBoxedCall && m_builtins.at(overload.builtin).origin == BuiltinOrigin::Host) {
+        const Operation* call = context.valueID() ? &operationAt(context.valueID()) : nullptr;
+        if (call && call->opcode == Opcode::CallBuiltin && call->overload() == id && call->builtinCallForm() == BuiltinCallForm::Whole)
+            stub = callHostGetter();
+    }
     bool leavesFunctionUnfetched = stub && stub->family() == StubFamily::CallBoxedBuiltin && fetchesFunctionOnlyToLeave(static_cast<BoxedCallForm>(stub->form()), count) && keyOfCalleeOf(overload).hasCell(m_builtins);
     Boxed callee = leavesFunctionUnfetched ? Boxed(zeroExtended(numberOf(m_mentions.builtinCell(keyOfCalleeOf(overload)))).value()) : boxCell(calleeOf(context, overload));
     Boxed thisValue = undefined();

Source/JavaScriptCore/aot/builtins/AOTBuiltinCallEmitter.h

@@ -29,6 +29,8 @@ class BuiltinCallEmitter final : public BuiltinFamilyEmitter {
     void generateRemainderOfIntegers(const ValueContext&);
     void generateReciprocalOfDivisor(const ValueContext&);
     void generateRemainderByReciprocal(const ValueContext&);
+    void generateWholeDivisor(const ValueContext&);
+    void generateRemainderByWholeDivisor(const ValueContext&);
     void generateToInt32(const ValueContext&);
     void generateToUint32(const ValueContext&);
     void generateToIntegerOrInfinity(const ValueContext&);

Source/JavaScriptCore/aot/builtins/AOTBuiltinCallEmitterNumbers.cpp

@@ -10,6 +10,7 @@
 
 #include "AOTConstantEmitter.h"
 #include "AOTConversionEmitter.h"
+#include "AOTOperatorEmitter.h"
 #include "AOTIRProgram.h"
 #include "AOTThrowEmitter.h"
 #include "AOTTypeTable.h"
@@ -299,6 +300,21 @@ void BuiltinCallEmitter::generateToInt32(const ValueContext& value)
     defineAs(value, int32BitsOf(value->valueOperand(), value->givesOnlyLowestBits()), m_program.at(TypeTable::int32));
 }
 
+void BuiltinCallEmitter::generateWholeDivisor(const ValueContext& value)
+{
+    Number divisor = numberOperand(value->divisorOperand());
+    ConversionEmitter& conversions = emitter<ConversionEmitter>();
+    Integer whole = conversions.truncateUnchecked(divisor);
+    Integer wholeOrZero = select(equal(conversions.bitsOf(toNumber(whole)), conversions.bitsOf(divisor)), whole, integer(0));
+    defineAs(value, select(below(wholeOrZero, integer(maxExactInteger + 1)), wholeOrZero, integer(0)), typeOf(value.id()));
+}
+
+void BuiltinCallEmitter::generateRemainderByWholeDivisor(const ValueContext& value)
+{
+    Number remainder = emitter<OperatorEmitter>().remainderByWholeDivisor(value, numberOperand(value->dividendOperand()), numberOperand(value->divisorOperand()), integerOperand(value->wholeDivisorOperand()));
+    defineAs(value, remainder, typeOf(value.id()));
+}
+
 void BuiltinCallEmitter::generateToUint32(const ValueContext& value)
 {
     defineAs(value, zeroExtended(int32BitsOf(value->valueOperand(), false)), m_program.at(TypeTable::uint32));

Source/JavaScriptCore/aot/builtins/AOTBuiltinResultTypes.cpp

@@ -95,6 +95,8 @@ static TypeID applyRangeRuleToBuiltinResult(const BuiltinTypeContext& context, c
     const BuiltinResult& result = overload.result;
     if (result.rangeRule == BuiltinRangeRule::None || !result.rangeOperand)
         return declared;
+    if (result.rangeRule == BuiltinRangeRule::IntegerPart)
+        declared = context.types.intersectionOf(TypeArena::fromPrimitives(Primitive::NumberWithoutFraction), declared);
 
     TypeID onlyOperand;
     std::span<const TypeID> rangeOperands;
@@ -129,8 +131,6 @@ static TypeID applyRangeRuleToBuiltinResult(const BuiltinTypeContext& context, c
         Range range = computeResultRange(result.rangeRule, ranges.span());
         return types.intersectionOf(takesOnlyIntegers ? types.integer(range) : typeOfRuledResultOfFractions(types, overload, range), declared);
     }
-    if (result.rangeRule == BuiltinRangeRule::IntegerPart && rangeOperands.front() && types.at(rangeOperands.front()).isNumber())
-        return types.intersectionOf(TypeArena::fromPrimitives(Primitive::NumberWithoutFraction), declared);
     return declared;
 }
 

Source/JavaScriptCore/aot/builtins/AOTBuiltinTable.cpp

@@ -413,6 +413,20 @@ bool BuiltinTable::ignoresReceiver(BuiltinID builtin) const
     });
 }
 
+bool BuiltinTable::ignoresArgument(BuiltinID builtin, unsigned argument) const
+{
+    BuiltinID canonical = canonicalID(builtin);
+    if (at(canonical).origin != BuiltinOrigin::Engine || argument > BuiltinOperand::maxArgumentIndex)
+        return false;
+    std::span<const BuiltinOverload> rows = overloadsOf(canonical);
+    bool noRowSpeaksOfIt = !rows.empty() && std::ranges::all_of(rows, [&](const BuiltinOverload& row) {
+        return !row.isUndeclared() && !parameterFor(row, BuiltinOperand::argument(argument));
+    });
+    return noRowSpeaksOfIt && std::ranges::all_of(overloadSummariesOf(canonical), [](const BuiltinOverloadSummary& overload) {
+        return overload.callee == BuiltinCallee::HostFunction || overload.callee == BuiltinCallee::InternalFunction;
+    });
+}
+
 bool BuiltinTable::mayDefineOnNewInstances(BuiltinClassID builtinClass, BuiltinName name) const
 {
     if (listsOwnPropertiesOfNewInstances(builtinClass))

Source/JavaScriptCore/aot/builtins/AOTBuiltinTable.h

@@ -1000,6 +1000,7 @@ class BuiltinTable {
     bool listsOwnPropertiesOfNewInstances(BuiltinClassID) const;
     bool mayDefineOnNewInstances(BuiltinClassID, BuiltinName) const;
     JS_EXPORT_PRIVATE bool ignoresReceiver(BuiltinID) const;
+    bool ignoresArgument(BuiltinID, unsigned argument) const;
     bool definesCharactersOnNewInstances(BuiltinClassID) const;
     std::span<const BuiltinClassID> classesWithInstanceMember(BuiltinName) const;
 

Source/JavaScriptCore/aot/builtins/AOTHashTableBuiltinEmitter.cpp

@@ -63,7 +63,7 @@ Boxed HashTableBuiltinEmitter::normalizedNumber(Number key)
     LBasicBlock fitsInt32 = newBlock();
     LBasicBlock otherwise = newBlock();
     Join<Boxed> join(m_out);
-    Word32 truncated = low32(conversions.toInteger(key));
+    Word32 truncated = conversions.truncateToWord(key);
     m_out.branch(equal(toNumber(truncated), key).value(), normalEdge(fitsInt32), unlikelyEdge(otherwise));
 
     m_out.appendTo(fitsInt32);

Source/JavaScriptCore/aot/builtins/AOTMathBuiltinEmitterNumbers.cpp

@@ -134,8 +134,7 @@ void MathBuiltinEmitter::generateParseInt(const InlineCall& call)
     if (ValueID argument = call.argumentOrNull(1))
         radix = int32Operand(argument);
     CellPointer string = cellOperand(call.argument(0));
-    missUnless(call, logicalNot(isRopeString(string)));
-    CharacterView view = emitter<StringEmitter>().viewOfFlatString(string);
+    CharacterView view = emitter<StringEmitter>().viewWithoutResolving(call.context(), string, rareEdge(missBlockOf(call)));
     Number result = emitter<StringEmitter>().selectByWidth<Number>(view.is8Bit, [&](bool is8Bit) {
         return callLeaf<Number>(is8Bit ? RuntimeOperation::ParseInt8 : RuntimeOperation::ParseInt16, { view.characters, view.length, radix.value_or(word32(0)) });
     }, Machine::EdgeClass::Normal);

Source/JavaScriptCore/aot/builtins/AOTMathBuiltinOperations.cpp

@@ -80,25 +80,35 @@ JSC_DEFINE_NOEXCEPT_JIT_OPERATION(operationAOTMathF16Round, double, (double x))
     return Float16 { x };
 }
 
+static NEVER_INLINE double AOT_PRESERVE_MOST_LEAF_ATTRIBUTES parseInt8Slow(const Latin1Character* characters, uint32_t length, int32_t radix)
+{
+    return parseInt(unsafeMakeSpan(characters, length), radix);
+}
+
 extern "C" double AOT_PRESERVE_MOST_LEAF_ATTRIBUTES operationAOTParseInt8(const Latin1Character* characters, uint32_t length, int32_t radix)
 {
     if (!radix || radix == 10) {
         if (std::optional<double> value = valueOfFewDigits(unsafeMakeSpan(characters, length)))
             return *value;
     }
-    return parseInt(unsafeMakeSpan(characters, length), radix);
+    [[clang::musttail]] return parseInt8Slow(characters, length, radix);
 }
 
 extern "C" double AOT_PRESERVE_MOST_LEAF_ATTRIBUTES operationAOTParseInt16(const char16_t* characters, uint32_t length, int32_t radix)
 {
     return parseInt(unsafeMakeSpan(characters, length), radix);
 }
 
+static NEVER_INLINE double AOT_PRESERVE_MOST_LEAF_ATTRIBUTES stringToNumber8Slow(const Latin1Character* characters, uint32_t length)
+{
+    return jsToNumber(StringView { unsafeMakeSpan(characters, length) });
+}
+
 extern "C" double AOT_PRESERVE_MOST_LEAF_ATTRIBUTES operationAOTStringToNumber8(const Latin1Character* characters, uint32_t length)
 {
     if (std::optional<double> value = valueOfFewDigits(unsafeMakeSpan(characters, length)))
         return *value;
-    return jsToNumber(StringView { unsafeMakeSpan(characters, length) });
+    [[clang::musttail]] return stringToNumber8Slow(characters, length);
 }
 
 extern "C" double AOT_PRESERVE_MOST_LEAF_ATTRIBUTES operationAOTStringToNumber16(const char16_t* characters, uint32_t length)