db/recsplit: AVX-512 findBijection

db/recsplit/bijection_generic.go

@@ -0,0 +1,23 @@
+// Copyright 2021 The Erigon Authors
+// This file is part of Erigon.
+//
+// Erigon is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Erigon is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with Erigon. If not, see <http://www.gnu.org/licenses/>.
+
+//go:build !(go1.27 && goexperiment.simd && amd64)
+
+package recsplit
+
+func findBijection(bucket []uint64, salt uint64) uint64 {
+	return findBijectionGeneric(bucket, salt)
+}

db/recsplit/bijection_simd.go

@@ -0,0 +1,78 @@
+// Copyright 2021 The Erigon Authors
+// This file is part of Erigon.
+//
+// Erigon is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Erigon is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with Erigon. If not, see <http://www.gnu.org/licenses/>.
+
+//go:build go1.27 && goexperiment.simd && amd64
+
+package recsplit
+
+import "simd/archsimd"
+
+var hasAVX512 = archsimd.X86.AVX512()
+
+// findBijection is findBijectionGeneric with the salt candidates held in 512-bit
+// registers, two independent chains deep: splitmix64 is serial, so a single
+// vector leaves the multiplier idle between dependent steps.
+//
+// AVX2 cannot host it: the 64x64 multiply both splitmix64 and remap16 need is
+// VPMULLQ (AVX512DQ), and emulating it costs more than the unrolled scalar form
+// saves.
+func findBijection(bucket []uint64, salt uint64) uint64 {
+	if !hasAVX512 {
+		return findBijectionGeneric(bucket, salt)
+	}
+	m := uint16(len(bucket))
+	fullMask := uint64(1)<<m - 1
+
+	lanes := [8]uint64{0, 1, 2, 3, 4, 5, 6, 7}
+	offsets := archsimd.LoadUint64x8Array(&lanes)
+	mask48v := archsimd.BroadcastUint64x8(mask48)
+	// The generic form masks the modulus to keep the shift in range; MaxLeafSize
+	// holds m far below that, but mirror it so the two cannot diverge.
+	modulus := archsimd.BroadcastUint64x8(uint64(m & 31))
+	one := archsimd.BroadcastUint64x8(1)
+	c1 := archsimd.BroadcastUint64x8(0xbf58476d1ce4e5b9)
+	c2 := archsimd.BroadcastUint64x8(0x94d049bb133111eb)
+
+	var out [8]uint64
+	for {
+		salts0 := archsimd.BroadcastUint64x8(salt).Add(offsets)
+		salts1 := archsimd.BroadcastUint64x8(salt + 8).Add(offsets)
+		acc0, acc1 := archsimd.Uint64x8{}, archsimd.Uint64x8{}
+		for _, key := range bucket {
+			k := archsimd.BroadcastUint64x8(key)
+			z0, z1 := k.Add(salts0), k.Add(salts1)
+			z0, z1 = z0.Xor(z0.ShiftAllRight(30)).Mul(c1), z1.Xor(z1.ShiftAllRight(30)).Mul(c1)
+			z0, z1 = z0.Xor(z0.ShiftAllRight(27)).Mul(c2), z1.Xor(z1.ShiftAllRight(27)).Mul(c2)
+			z0, z1 = z0.Xor(z0.ShiftAllRight(31)), z1.Xor(z1.ShiftAllRight(31))
+			// remap16: ((z & mask48) * m) >> 48, then set that bit.
+			acc0 = acc0.Or(one.ShiftLeft(z0.And(mask48v).Mul(modulus).ShiftAllRight(48)))
+			acc1 = acc1.Or(one.ShiftLeft(z1.And(mask48v).Mul(modulus).ShiftAllRight(48)))
+		}
+		acc0.StoreArray(&out)
+		for i, bits := range out {
+			if bits == fullMask {
+				return salt + uint64(i)
+			}
+		}
+		acc1.StoreArray(&out)
+		for i, bits := range out {
+			if bits == fullMask {
+				return salt + 8 + uint64(i)
+			}
+		}
+		salt += 16
+	}
+}

db/recsplit/recsplit.go

@@ -763,11 +763,12 @@ func findSplit(bucket []uint64, salt uint64, fanout, unit uint16) uint64 {
 	}
 }
 
-// findBijection finds a salt value such that all keys in bucket hash to distinct
-// positions in [0, m).
+// findBijectionGeneric finds a salt value such that all keys in bucket hash to
+// distinct positions in [0, m).
 // Uses 8-way salt parallelism with branchless OR-accumulate
 // to exploit CPU instruction-level parallelism and avoid branch mispredictions.
-func findBijection(bucket []uint64, salt uint64) uint64 {
+// findBijection dispatches here unless a vector implementation is built in.
+func findBijectionGeneric(bucket []uint64, salt uint64) uint64 {
 	m := uint16(len(bucket))
 	fullMask := uint32((1 << m) - 1)
 	for {

db/recsplit/recsplit_test.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"io"
 	"math"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"testing"
@@ -569,3 +570,22 @@ func TestResetNextSaltAfterCollision(t *testing.T) {
 		require.Equal(t, uint64(i*17), idx.OrdinalLookup(e))
 	}
 }
+
+// findBijection must agree with findBijectionGeneric whatever build tags and CPU
+// selected, so a vector implementation cannot diverge silently.
+func TestFindBijectionMatchesGeneric(t *testing.T) {
+	rnd := rand.New(rand.NewSource(42))
+	// A bijection exists for only m!/m^m of salts, so keep m at the leaf sizes
+	// the search is actually run at — m=16 already needs ~10^6 tries.
+	for _, m := range []int{1, 2, 3, 5, 8, 10, 12} {
+		for trial := range 16 {
+			bucket := make([]uint64, m)
+			for i := range bucket {
+				bucket[i] = rnd.Uint64()
+			}
+			salt := rnd.Uint64()
+			require.Equal(t, findBijectionGeneric(bucket, salt), findBijection(bucket, salt),
+				"m=%d trial=%d salt=%d", m, trial, salt)
+		}
+	}
+}