diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 249dbb0..0d9b2ad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,7 +8,6 @@ jobs: steps: - name: Install OpenCL run: | - sudo add-apt-repository ppa:intel-opencl/intel-opencl sudo apt-get update sudo apt-get install ocl-icd-opencl-dev - uses: actions/checkout@aabbfeb2ce60b5bd82389903509092c4648a9713 @@ -22,7 +21,6 @@ jobs: steps: - name: Install OpenCL run: | - sudo add-apt-repository ppa:intel-opencl/intel-opencl sudo apt-get update sudo apt-get install ocl-icd-opencl-dev - uses: actions/checkout@aabbfeb2ce60b5bd82389903509092c4648a9713 diff --git a/.github/workflows/nano-work-simd.yml b/.github/workflows/nano-work-simd.yml new file mode 100644 index 0000000..0530a8d --- /dev/null +++ b/.github/workflows/nano-work-simd.yml @@ -0,0 +1,46 @@ +# CI for the nano-work-simd crate: build, test and lint on x86-64 (AVX2 and, +# where the runner has it, AVX-512) and on Apple Silicon (NEON). The macOS job +# is the CI job that compiles and executes the NEON kernel. +name: nano-work-simd + +on: + push: + paths: + - "nano-work-simd/**" + - "src/**" + - "tests/**" + - "Cargo.toml" + - "Cargo.lock" + - ".github/workflows/nano-work-simd.yml" + pull_request: + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + test: + name: test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-14] + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - name: CPU features + shell: bash + run: | + uname -m + if [ "$(uname -s)" = "Linux" ]; then grep -m1 -o 'avx512f\|avx2' /proc/cpuinfo | sort -u; fi + if [ "$(uname -s)" = "Darwin" ]; then sysctl -n machdep.cpu.brand_string; fi + - name: Report kernels + run: cargo test --release -p nano-work-simd --lib report_kernels -- --nocapture + - name: Test + run: cargo test --release -p nano-work-simd + - name: Clippy + run: cargo clippy -p nano-work-simd --all-targets -- -D warnings diff --git a/Cargo.lock b/Cargo.lock index 08a2626..939f4fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -621,14 +621,13 @@ dependencies = [ name = "nano-work-server" version = "0.3.1" dependencies = [ - "blake2", "byteorder", "chrono", "clap", - "digest", "futures 0.3.25", "hex", "hyper", + "nano-work-simd", "ocl", "parking_lot", "rand", @@ -637,6 +636,13 @@ dependencies = [ "tokio", ] +[[package]] +name = "nano-work-simd" +version = "0.1.0" +dependencies = [ + "blake2", +] + [[package]] name = "nodrop" version = "0.1.14" diff --git a/Cargo.toml b/Cargo.toml index ef3a4cf..6852192 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,10 +15,13 @@ ocl = "0.19.4" serde_json = "1.0.87" hex = "0.4.3" rand = "0.8.5" -blake2 = "0.10.4" -digest = "0.10.5" byteorder = "1.4.3" +nano-work-simd = { path = "nano-work-simd" } parking_lot = "0.12.1" chrono = "0.4.22" tokio = { version = "1.21.2", features = ["rt-multi-thread", "macros"] } rand_xorshift = "0.3.0" + +[workspace] +members = ["nano-work-simd"] +default-members = [".", "nano-work-simd"] diff --git a/nano-work-simd/Cargo.toml b/nano-work-simd/Cargo.toml new file mode 100644 index 0000000..af1bea5 --- /dev/null +++ b/nano-work-simd/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "nano-work-simd" +version = "0.1.0" +edition = "2021" +rust-version = "1.89" +license = "BSD-2-Clause" +description = "Fast CPU proof-of-work generator for the Nano cryptocurrency (unrolled Blake2b-8 with AVX-512 / AVX2 / NEON / scalar kernels)" +repository = "https://github.com/nanocurrency/nano-work-server" +keywords = ["nano", "blake2b", "proof-of-work", "simd"] +categories = ["cryptography", "algorithms"] + +[lib] +name = "nano_work_simd" +path = "src/lib.rs" + +[dependencies] +# none: the crate is std-only + +[dev-dependencies] +blake2 = "0.10" diff --git a/nano-work-simd/README.md b/nano-work-simd/README.md new file mode 100644 index 0000000..5a673cb --- /dev/null +++ b/nano-work-simd/README.md @@ -0,0 +1,29 @@ +# nano-work-simd + +The CPU work back end of nano-work-server: Nano proof-of-work +(`u64::from_le_bytes(blake2b_8(work_le || hash)) >= threshold`) with a Blake2b +compression specialised for the fixed 40-byte message, run on 8 (AVX-512), +4 (AVX2), 2 (NEON) or 1 (scalar) nonces per instruction stream. The kernel is +picked at runtime from the CPU's feature flags (`Kernel::best()`), or forced by +the server's `--cpu-kernel` option. + +Public API: `generate`, `generate_with`, `validate`, `difficulty`, +`hash_rate_benchmark`, `expected_hashes`, `resolve_hit`, `Kernel` +(`search`, `difficulties`, `best`, `available`, `from_name`), `Precomputed`, +and the `SEND_THRESHOLD` / `RECEIVE_THRESHOLD` / `LEGACY_THRESHOLD` / `BATCH` +constants. The server uses `Kernel::search` over `BATCH`-sized nonce ranges, +`resolve_hit` to pick and re-verify the winning lane, and `validate` for +`work_validate`. + +Tests (`cargo test --release -p nano-work-simd` from the server root) cross-check +every available kernel against the `blake2` crate on 10 000 random (hash, nonce) +pairs, check nonce wrap-around at `u64::MAX`, check that `search` finds exactly +the hit set of a brute-force scan, that generated work validates for every +kernel and thread count, and the documented Nano example. `Precomputed`'s words +and the lane-level kernels are crate-private; `unsafe` is confined to the +`#[target_feature]` entry points in `src/x86.rs` / `src/neon.rs`. + +This is a vendored copy of the standalone `nano-work-simd` crate, which also +carries a CLI (`info`, `generate`, `validate`, `bench`, `timing`) and a +`hashlib`-based Python cross-check. No dependencies; `blake2` is a +dev-dependency for the tests only. MSRV 1.89. License: BSD-2-Clause. diff --git a/nano-work-simd/src/kernel.rs b/nano-work-simd/src/kernel.rs new file mode 100644 index 0000000..7e7db26 --- /dev/null +++ b/nano-work-simd/src/kernel.rs @@ -0,0 +1,440 @@ +//! Arch-independent core: the specialised Blake2b compression used by Nano PoW. +//! +//! Nano work: `hash = blake2b(digest_size = 8, work.to_le_bytes() || block_hash[32])`, +//! valid iff `u64::from_le_bytes(hash) >= threshold`. +//! +//! The 40-byte message is a single final block, so the compression has: +//! * `m0 = nonce`, `m1..m4 = block hash words (LE)`, `m5..m15 = 0` +//! * `t = 40`, final-block flag set (`v14 = !IV6`) +//! * `h0 = IV0 ^ 0x0101_0008` (digest length 8, fanout 1, depth 1) +//! +//! Only the first output word `h0' = h0 ^ v0 ^ v8` is needed. +//! +//! Three transformations on top of a straightforward unrolled implementation: +//! 1. **Zero message words**: the 12 rounds are fully unrolled with the message +//! schedule applied at compile time, so the `+ m` adds for the eleven zero words +//! disappear (a `Z` token in the [`round!`] macro emits no add at all). +//! 2. **Round-0 precompute**: three of the four round-0 column `G`s and the first +//! additions of two diagonal `G`s do not depend on the nonce. They are computed +//! once per block hash ([`Precomputed`]) and passed to the kernels as 16 words. +//! 3. **Truncated last round**: only `v0` and `v8` after round 11 are live, so the +//! parts of round 11 that cannot reach them are dropped. +//! +//! Everything here is `#[inline(always)]` and generic over [`Lane`], so each +//! `#[target_feature]` entry point in the arch modules gets its own fully inlined +//! copy compiled with the right instruction set. +//! +//! Every function taking a [`Lane`] is `unsafe fn`: the lane operations are +//! target-specific intrinsics whose precondition (the CPU feature is enabled for +//! the enclosing code) is established by the `#[target_feature]` entry points in +//! the arch modules, where every `unsafe { }` block in the crate lives (`lib.rs` +//! only calls those entry points after runtime feature detection). + +/// Blake2b IV. +pub(crate) const IV: [u64; 8] = [ + 0x6a09_e667_f3bc_c908, + 0xbb67_ae85_84ca_a73b, + 0x3c6e_f372_fe94_f82b, + 0xa54f_f53a_5f1d_36f1, + 0x510e_527f_ade6_82d1, + 0x9b05_688c_2b3e_6c1f, + 0x1f83_d9ab_fb41_bd6b, + 0x5be0_cd19_137e_2179, +]; + +/// `h0` for `digest_size = 8`, no key, no salt/personalisation. +pub(crate) const H0_NANO: u64 = IV[0] ^ 0x0101_0008; +/// `v12 = IV4 ^ t_low` with `t = 40` bytes. +pub(crate) const V12_NANO: u64 = IV[4] ^ 40; +/// `v14 = IV6 ^ !0` (final block). +pub(crate) const V14_NANO: u64 = !IV[6]; +/// `v0` before the nonce is added in round 0: `h0 + v4`. +pub(crate) const C04: u64 = H0_NANO.wrapping_add(IV[4]); + +/// Per-block-hash precomputed state: the 16 words each kernel needs besides the nonce. +/// +/// Create one with [`Precomputed::new`] and pass it to [`Kernel::search`](crate::Kernel::search), +/// [`resolve_hit`](crate::resolve_hit) or [`difficulty`](crate::difficulty). The +/// words themselves are an implementation detail of the kernels. +/// +/// Layout of `w`: +/// * `w[0..4]`: message words `m1..m4` (the block hash, little-endian) +/// * `w[4..13]`: round-0 column results `v3, v5, v6, v7, v9, v10, v11, v14, v15` +/// * `w[13]`: `a16 = v1 + v6` (first add of diagonal `G(1,6,11,12)`) +/// * `w[14]`: `a27 = v2 + v7` (first add of diagonal `G(2,7,8,13)`) +/// * `w[15]`: `d13 = rotr32(v13 ^ a27)` (second step of that same `G`) +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(C)] +pub struct Precomputed { + /// The 16 words, laid out as documented on the struct. + pub(crate) w: [u64; 16], +} + +// `w[0..4]` are the hash words `h0..h3` (read directly); the rest are indexed by name: +/// Index of `v3` in [`Precomputed::w`]. +pub(crate) const P_V3: usize = 4; +/// Index of `v5` in [`Precomputed::w`]. +pub(crate) const P_V5: usize = 5; +/// Index of `v6` in [`Precomputed::w`]. +pub(crate) const P_V6: usize = 6; +/// Index of `v7` in [`Precomputed::w`]. +pub(crate) const P_V7: usize = 7; +/// Index of `v9` in [`Precomputed::w`]. +pub(crate) const P_V9: usize = 8; +/// Index of `v10` in [`Precomputed::w`]. +pub(crate) const P_V10: usize = 9; +/// Index of `v11` in [`Precomputed::w`]. +pub(crate) const P_V11: usize = 10; +/// Index of `v14` in [`Precomputed::w`]. +pub(crate) const P_V14: usize = 11; +/// Index of `v15` in [`Precomputed::w`]. +pub(crate) const P_V15: usize = 12; +/// Index of `a16` in [`Precomputed::w`]. +pub(crate) const P_A16: usize = 13; +/// Index of `a27` in [`Precomputed::w`]. +pub(crate) const P_A27: usize = 14; +/// Index of `d13` in [`Precomputed::w`]. +pub(crate) const P_D13: usize = 15; + +#[inline(always)] +fn g_scalar(v: &mut [u64; 16], a: usize, b: usize, c: usize, d: usize, x: u64, y: u64) { + v[a] = v[a].wrapping_add(v[b]).wrapping_add(x); + v[d] = (v[d] ^ v[a]).rotate_right(32); + v[c] = v[c].wrapping_add(v[d]); + v[b] = (v[b] ^ v[c]).rotate_right(24); + v[a] = v[a].wrapping_add(v[b]).wrapping_add(y); + v[d] = (v[d] ^ v[a]).rotate_right(16); + v[c] = v[c].wrapping_add(v[d]); + v[b] = (v[b] ^ v[c]).rotate_right(63); +} + +impl Precomputed { + /// Precompute the nonce-independent part of round 0 for a block hash. + pub fn new(hash: &[u8; 32]) -> Self { + let mut h = [0u64; 4]; + for (i, word) in h.iter_mut().enumerate() { + *word = u64::from_le_bytes(hash[8 * i..8 * i + 8].try_into().unwrap()); + } + let mut v = [0u64; 16]; + v[..8].copy_from_slice(&IV); + v[8..].copy_from_slice(&IV); + v[0] = H0_NANO; + v[12] = V12_NANO; + v[14] = V14_NANO; + // Round 0 columns 1..3 (sigma_0: m2,m3 | m4,m5 | m6,m7 = h1,h2 | h3,0 | 0,0). + g_scalar(&mut v, 1, 5, 9, 13, h[1], h[2]); + g_scalar(&mut v, 2, 6, 10, 14, h[3], 0); + g_scalar(&mut v, 3, 7, 11, 15, 0, 0); + let a16 = v[1].wrapping_add(v[6]); + let a27 = v[2].wrapping_add(v[7]); + let d13 = (v[13] ^ a27).rotate_right(32); + Precomputed { + w: [ + h[0], h[1], h[2], h[3], v[3], v[5], v[6], v[7], v[9], v[10], v[11], v[14], v[15], + a16, a27, d13, + ], + } + } +} + +/// One SIMD "lane bundle": `LANES` independent 64-bit words. +/// +/// All methods are `unsafe fn` because the implementations use target-specific +/// intrinsics that are only sound to execute when the caller has verified (or +/// enabled via `#[target_feature]`) the corresponding CPU feature. +/// +/// # Safety +/// Every method has the same precondition: the CPU feature the implementing +/// type requires must be enabled for the calling code (see the arch modules). +#[allow(clippy::missing_safety_doc)] +pub(crate) trait Lane: Copy { + /// Number of 64-bit words per bundle. + const LANES: usize; + /// Broadcast `x` to all lanes. + unsafe fn splat(x: u64) -> Self; + /// `[base, base+1, ..., base+LANES-1]` (wrapping). + unsafe fn nonce(base: u64) -> Self; + /// Lane-wise wrapping add. + unsafe fn add(self, o: Self) -> Self; + /// Lane-wise xor. + unsafe fn xor(self, o: Self) -> Self; + /// Lane-wise rotate right by 32. + unsafe fn rot32(self) -> Self; + /// Lane-wise rotate right by 24. + unsafe fn rot24(self) -> Self; + /// Lane-wise rotate right by 16. + unsafe fn rot16(self) -> Self; + /// Lane-wise rotate right by 63. + unsafe fn rot63(self) -> Self; + /// Store `LANES` words to `out` (must have room for `LANES` words). + unsafe fn store(self, out: *mut u64); + /// Pre-process a threshold (`>= 1`) for [`Lane::ge_full`]. + unsafe fn thr_full(thr: u64) -> Self; + /// Pre-process a threshold whose low 32 bits are zero (`>= 1 << 32`) for [`Lane::ge_hi32`]. + unsafe fn thr_hi32(thr: u64) -> Self; + /// Non-zero iff some lane satisfies `lane >= thr` (unsigned 64-bit). + unsafe fn ge_full(self, t: Self) -> u32; + /// Non-zero iff some lane satisfies `lane >= thr`, using only the high 32 bits. + /// Only valid when `thr & 0xffff_ffff == 0`. + unsafe fn ge_hi32(self, t: Self) -> u32; +} + +/// `madd!(Z, e)` is `e`; `madd!(x, e)` is `e + x`. +macro_rules! madd { + (Z, $e:expr) => { + $e + }; + ($x:ident, $e:expr) => { + $e.add($x) + }; +} + +/// The Blake2b mixing function on state words `a,b,c,d` with message operands +/// `x, y`, where an operand written `Z` is a compile-time zero. +macro_rules! g { + ($v:ident, $a:tt, $b:tt, $c:tt, $d:tt, $x:tt, $y:tt) => { + $v[$a] = madd!($x, $v[$a].add($v[$b])); + $v[$d] = $v[$d].xor($v[$a]).rot32(); + $v[$c] = $v[$c].add($v[$d]); + $v[$b] = $v[$b].xor($v[$c]).rot24(); + $v[$a] = madd!($y, $v[$a].add($v[$b])); + $v[$d] = $v[$d].xor($v[$a]).rot16(); + $v[$c] = $v[$c].add($v[$d]); + $v[$b] = $v[$b].xor($v[$c]).rot63(); + }; +} + +/// A full Blake2b round; the 16 operands are `m[sigma[r][0..16]]` already permuted. +macro_rules! round { + ($v:ident, $m0:tt, $m1:tt, $m2:tt, $m3:tt, $m4:tt, $m5:tt, $m6:tt, $m7:tt, + $m8:tt, $m9:tt, $m10:tt, $m11:tt, $m12:tt, $m13:tt, $m14:tt, $m15:tt) => { + g!($v, 0, 4, 8, 12, $m0, $m1); + g!($v, 1, 5, 9, 13, $m2, $m3); + g!($v, 2, 6, 10, 14, $m4, $m5); + g!($v, 3, 7, 11, 15, $m6, $m7); + g!($v, 0, 5, 10, 15, $m8, $m9); + g!($v, 1, 6, 11, 12, $m10, $m11); + g!($v, 2, 7, 8, 13, $m12, $m13); + g!($v, 3, 4, 9, 14, $m14, $m15); + }; +} + +/// Splatted per-block constants, hoisted out of the search loop. +#[derive(Clone, Copy)] +pub(crate) struct Consts { + /// `m1..m4`. + pub(crate) h: [V; 4], + /// `Precomputed::w[4..16]`. + pub(crate) p: [V; 12], + /// [`C04`]. + pub(crate) c04: V, + /// `IV[0]`. + pub(crate) iv0: V, + /// `IV[4]`. + pub(crate) iv4: V, + /// [`V12_NANO`]. + pub(crate) v12: V, + /// [`H0_NANO`]. + pub(crate) h0n: V, +} + +impl Consts { + /// Splat every constant. + /// + /// # Safety + /// See [`Lane`]. + #[inline(always)] + pub(crate) unsafe fn new(pre: &Precomputed) -> Self { + let w = &pre.w; + Consts { + h: [V::splat(w[0]), V::splat(w[1]), V::splat(w[2]), V::splat(w[3])], + p: [ + V::splat(w[4]), + V::splat(w[5]), + V::splat(w[6]), + V::splat(w[7]), + V::splat(w[8]), + V::splat(w[9]), + V::splat(w[10]), + V::splat(w[11]), + V::splat(w[12]), + V::splat(w[13]), + V::splat(w[14]), + V::splat(w[15]), + ], + c04: V::splat(C04), + iv0: V::splat(IV[0]), + iv4: V::splat(IV[4]), + v12: V::splat(V12_NANO), + h0n: V::splat(H0_NANO), + } + } +} + +/// First output word `h0' = H0_NANO ^ v0 ^ v8` of the Nano compression for +/// `LANES` nonces at once. +/// +/// # Safety +/// See [`Lane`]. +#[inline(always)] +pub(crate) unsafe fn h0_lanes(c: &Consts, n: V) -> V { + let h0 = c.h[0]; + let h1 = c.h[1]; + let h2 = c.h[2]; + let h3 = c.h[3]; + let mut v: [V; 16] = [c.iv0; 16]; + + // ---- round 0 (sigma_0), with the nonce-independent parts precomputed ---- + // Column G(0,4,8,12, n, h0): everything from `v0 = c04 + n` on depends on n. + v[0] = c.c04.add(n); + v[4] = c.iv4; + v[8] = c.iv0; + v[12] = c.v12; + v[12] = v[12].xor(v[0]).rot32(); + v[8] = v[8].add(v[12]); + v[4] = v[4].xor(v[8]).rot24(); + v[0] = v[0].add(v[4]).add(h0); + v[12] = v[12].xor(v[0]).rot16(); + v[8] = v[8].add(v[12]); + v[4] = v[4].xor(v[8]).rot63(); + // Columns 1..3 are precomputed (v1, v2 and v13 are replaced by a16, a27, d13). + v[1] = c.p[P_A16 - 4]; + v[2] = c.p[P_A27 - 4]; + v[3] = c.p[P_V3 - 4]; + v[5] = c.p[P_V5 - 4]; + v[6] = c.p[P_V6 - 4]; + v[7] = c.p[P_V7 - 4]; + v[9] = c.p[P_V9 - 4]; + v[10] = c.p[P_V10 - 4]; + v[11] = c.p[P_V11 - 4]; + v[13] = c.p[P_D13 - 4]; + v[14] = c.p[P_V14 - 4]; + v[15] = c.p[P_V15 - 4]; + // Diagonals: sigma_0[8..16] selects m8..m15, which are all zero. + g!(v, 0, 5, 10, 15, Z, Z); + // G(1,6,11,12): `v1 += v6` precomputed as a16. + v[12] = v[12].xor(v[1]).rot32(); + v[11] = v[11].add(v[12]); + v[6] = v[6].xor(v[11]).rot24(); + v[1] = v[1].add(v[6]); + v[12] = v[12].xor(v[1]).rot16(); + v[11] = v[11].add(v[12]); + v[6] = v[6].xor(v[11]).rot63(); + // G(2,7,8,13): `v2 += v7` (a27) and `v13 = rotr32(v13 ^ v2)` (d13) precomputed. + v[8] = v[8].add(v[13]); + v[7] = v[7].xor(v[8]).rot24(); + v[2] = v[2].add(v[7]); + v[13] = v[13].xor(v[2]).rot16(); + v[8] = v[8].add(v[13]); + v[7] = v[7].xor(v[8]).rot63(); + g!(v, 3, 4, 9, 14, Z, Z); + + // ---- rounds 1..10, message schedule applied at compile time ---- + round!(v, Z, Z, h3, Z, Z, Z, Z, Z, h0, Z, n, h1, Z, Z, Z, h2); // sigma_1 + round!(v, Z, Z, Z, n, Z, h1, Z, Z, Z, Z, h2, Z, Z, h0, Z, h3); // sigma_2 + round!(v, Z, Z, h2, h0, Z, Z, Z, Z, h1, Z, Z, Z, h3, n, Z, Z); // sigma_3 + round!(v, Z, n, Z, Z, h1, h3, Z, Z, Z, h0, Z, Z, Z, Z, h2, Z); // sigma_4 + round!(v, h1, Z, Z, Z, n, Z, Z, h2, h3, Z, Z, Z, Z, Z, h0, Z); // sigma_5 + round!(v, Z, Z, h0, Z, Z, Z, h3, Z, n, Z, Z, h2, Z, h1, Z, Z); // sigma_6 + round!(v, Z, Z, Z, Z, Z, h0, h2, Z, Z, n, Z, h3, Z, Z, h1, Z); // sigma_7 + round!(v, Z, Z, Z, Z, Z, h2, n, Z, Z, h1, Z, Z, h0, h3, Z, Z); // sigma_8 + round!(v, Z, h1, Z, h3, Z, Z, h0, Z, Z, Z, Z, Z, h2, Z, Z, n); // sigma_9 + round!(v, n, h0, h1, h2, h3, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z, Z); // sigma_0 + + // ---- round 11 (sigma_1), truncated to what reaches v0 and v8 ---- + // Column G(0,4,8,12, Z, Z): v4's final rot63 is dead. + v[0] = v[0].add(v[4]); + v[12] = v[12].xor(v[0]).rot32(); + v[8] = v[8].add(v[12]); + v[4] = v[4].xor(v[8]).rot24(); + v[0] = v[0].add(v[4]); + v[12] = v[12].xor(v[0]).rot16(); + v[8] = v[8].add(v[12]); + // Column G(1,5,9,13, h3, Z): v5 and v13 are both needed, so it stays whole. + g!(v, 1, 5, 9, 13, h3, Z); + // Column G(2,6,10,14, Z, Z): v6's final rot63 is dead. + v[2] = v[2].add(v[6]); + v[14] = v[14].xor(v[2]).rot32(); + v[10] = v[10].add(v[14]); + v[6] = v[6].xor(v[10]).rot24(); + v[2] = v[2].add(v[6]); + v[14] = v[14].xor(v[2]).rot16(); + v[10] = v[10].add(v[14]); + // Column G(3,7,11,15, Z, Z): v7 and v15 are both needed. + g!(v, 3, 7, 11, 15, Z, Z); + // Diagonal G(0,5,10,15, h0, Z): only up to the second update of v0. + v[0] = v[0].add(v[5]).add(h0); + v[15] = v[15].xor(v[0]).rot32(); + v[10] = v[10].add(v[15]); + v[5] = v[5].xor(v[10]).rot24(); + v[0] = v[0].add(v[5]); + // Diagonal G(1,6,11,12, n, h1): dead entirely. + // Diagonal G(2,7,8,13, Z, Z): only up to the second update of v8. + v[2] = v[2].add(v[7]); + v[13] = v[13].xor(v[2]).rot32(); + v[8] = v[8].add(v[13]); + v[7] = v[7].xor(v[8]).rot24(); + v[2] = v[2].add(v[7]); + v[13] = v[13].xor(v[2]).rot16(); + v[8] = v[8].add(v[13]); + // Diagonal G(3,4,9,14, Z, h2): dead entirely. + + c.h0n.xor(v[0]).xor(v[8]) +} + +/// Search `count` nonces starting at `base` (wrapping). `count` must be a +/// multiple of `V::LANES` and `thr >= 1` (and `>= 1 << 32` if `HI32`). +/// +/// Returns the base nonce of the first lane bundle containing a hit; the caller +/// resolves which lane(s) actually satisfy the threshold. +/// +/// # Safety +/// See [`Lane`]. +#[inline(always)] +pub(crate) unsafe fn search( + pre: &Precomputed, + base: u64, + count: u64, + thr: u64, +) -> Option { + debug_assert!(count.is_multiple_of(V::LANES as u64)); + debug_assert!(thr >= 1); + debug_assert!(!HI32 || thr & 0xffff_ffff == 0); + let c = Consts::::new(pre); + let t = if HI32 { V::thr_hi32(thr) } else { V::thr_full(thr) }; + let end = base.wrapping_add(count); + let mut n = base; + while n != end { + // For the scalar lane LLVM otherwise reassociates the wrapping adds and + // hoists dozens of nonce-independent sums of the constants out of the + // loop, which do not fit in 16 GPRs and get spilled (2x slower). + // Re-materialising the constants from memory each iteration is ~20 + // loads against ~1000 ALU ops and stops that. + let c = if V::LANES == 1 { core::hint::black_box(c) } else { c }; + let r = h0_lanes(&c, V::nonce(n)); + let m = if HI32 { r.ge_hi32(t) } else { r.ge_full(t) }; + if m != 0 { + return Some(n); + } + n = n.wrapping_add(V::LANES as u64); + } + None +} + +/// Compute the difficulty (first output word) for `out.len()` consecutive nonces +/// starting at `base`. `out.len()` must be a multiple of `V::LANES`. +/// +/// # Safety +/// See [`Lane`]. +#[inline(always)] +pub(crate) unsafe fn difficulties(pre: &Precomputed, base: u64, out: &mut [u64]) { + debug_assert!(out.len().is_multiple_of(V::LANES)); + let c = Consts::::new(pre); + // clippy 1.98+ suggests as_chunks_mut, which needs a const generic; V::LANES is an + // associated const. unknown_lints covers older clippy that has no such lint. + #[allow(unknown_lints, clippy::chunks_exact_to_as_chunks)] + for (i, chunk) in out.chunks_exact_mut(V::LANES).enumerate() { + let n = base.wrapping_add((i * V::LANES) as u64); + h0_lanes(&c, V::nonce(n)).store(chunk.as_mut_ptr()); + } +} diff --git a/nano-work-simd/src/lib.rs b/nano-work-simd/src/lib.rs new file mode 100644 index 0000000..7a47712 --- /dev/null +++ b/nano-work-simd/src/lib.rs @@ -0,0 +1,415 @@ +//! Fast CPU proof-of-work generation for the Nano cryptocurrency. +//! +//! Nano work for a block is a 64-bit nonce `work` such that +//! +//! ```text +//! u64::from_le_bytes(blake2b(digest_size = 8, work.to_le_bytes() || block_hash)) >= threshold +//! ``` +//! +//! This crate implements that search with a Blake2b compression specialised for +//! the fixed 40-byte message (`src/kernel.rs`) and runs it on 8 (AVX-512), 4 (AVX2), +//! 2 (NEON) or 1 (scalar) nonces per instruction stream, across any number of +//! threads. The kernel is picked at runtime from the CPU's feature flags. +//! +//! ```no_run +//! use std::sync::atomic::AtomicBool; +//! let hash = [0u8; 32]; +//! let cancel = AtomicBool::new(false); +//! let work = nano_work_simd::generate(&hash, nano_work_simd::SEND_THRESHOLD, 4, &cancel).unwrap(); +//! assert!(nano_work_simd::validate(&hash, work) >= nano_work_simd::SEND_THRESHOLD); +//! ``` + +#![deny(missing_docs)] +#![allow(clippy::needless_range_loop)] + +mod kernel; +mod scalar; + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod x86; + +#[cfg(target_arch = "aarch64")] +mod neon; + +use std::collections::hash_map::RandomState; +use std::hash::{BuildHasher, Hash, Hasher}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +pub use kernel::Precomputed; + +/// Threshold for send/change blocks on the live network (`0xfffffff800000000`). +pub const SEND_THRESHOLD: u64 = 0xffff_fff8_0000_0000; +/// Threshold for receive/open blocks on the live network (`0xfffffe0000000000`). +pub const RECEIVE_THRESHOLD: u64 = 0xffff_fe00_0000_0000; +/// The pre-v21 base threshold (`0xffffffc000000000`). +pub const LEGACY_THRESHOLD: u64 = 0xffff_ffc0_0000_0000; + +/// Nonces handed to a kernel per call. Between calls a worker re-checks the +/// cancellation and "found" flags, so this bounds the reaction latency +/// (2^16 hashes is 1-10 ms on one core). +pub const BATCH: u64 = 1 << 16; + +/// A compute kernel. Each processes `lanes()` nonces per instruction stream. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum Kernel { + /// Portable 1-lane fallback. + Scalar, + /// x86-64 AVX2, 4 lanes. + Avx2, + /// x86-64 AVX-512F, 8 lanes. + Avx512, + /// AArch64 NEON, 2 lanes. + Neon, +} + +impl Kernel { + /// All kernels, fastest first. + pub const ALL: [Kernel; 4] = [Kernel::Avx512, Kernel::Avx2, Kernel::Neon, Kernel::Scalar]; + + /// Number of nonces processed per lane bundle. + pub const fn lanes(self) -> usize { + match self { + Kernel::Scalar => 1, + Kernel::Avx2 => 4, + Kernel::Avx512 => 8, + Kernel::Neon => 2, + } + } + + /// Short name (`scalar`, `avx2`, `avx512`, `neon`). + pub const fn name(self) -> &'static str { + match self { + Kernel::Scalar => "scalar", + Kernel::Avx2 => "avx2", + Kernel::Avx512 => "avx512", + Kernel::Neon => "neon", + } + } + + /// Parse a kernel name as produced by [`Kernel::name`]. + pub fn from_name(s: &str) -> Option { + Kernel::ALL.iter().copied().find(|k| k.name().eq_ignore_ascii_case(s)) + } + + /// Whether the running CPU can execute this kernel. + pub fn is_available(self) -> bool { + match self { + Kernel::Scalar => true, + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + Kernel::Avx2 => is_x86_feature_detected!("avx2"), + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + Kernel::Avx512 => is_x86_feature_detected!("avx512f"), + #[cfg(target_arch = "aarch64")] + Kernel::Neon => std::arch::is_aarch64_feature_detected!("neon"), + #[allow(unreachable_patterns)] + _ => false, + } + } + + /// Kernels the running CPU can execute, fastest first. + pub fn available() -> Vec { + Kernel::ALL.iter().copied().filter(|k| k.is_available()).collect() + } + + /// The fastest kernel available on the running CPU. + pub fn best() -> Kernel { + Kernel::ALL + .iter() + .copied() + .find(|k| k.is_available()) + .unwrap_or(Kernel::Scalar) + } + + /// Search `count` nonces from `base` (wrapping; `count` a multiple of + /// [`BATCH`] is convenient but any multiple of 8 works for every kernel). + /// + /// Returns the base nonce of the first lane bundle with a hit. Use + /// [`resolve_hit`] to turn it into an actual valid nonce. + /// + /// # Panics + /// If the kernel is not available on this CPU or `count % lanes() != 0`. + pub fn search(self, pre: &Precomputed, base: u64, count: u64, threshold: u64) -> Option { + assert!(self.is_available(), "kernel {} not supported by this CPU", self.name()); + assert!(count.is_multiple_of(self.lanes() as u64)); + if threshold == 0 { + return if count > 0 { Some(base) } else { None }; + } + match self { + Kernel::Scalar => scalar::search(pre, base, count, threshold), + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + // SAFETY: availability asserted above. + Kernel::Avx2 => unsafe { x86::search_avx2(pre, base, count, threshold) }, + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + // SAFETY: availability asserted above. + Kernel::Avx512 => unsafe { x86::search_avx512(pre, base, count, threshold) }, + #[cfg(target_arch = "aarch64")] + // SAFETY: availability asserted above. + Kernel::Neon => unsafe { neon::search_neon(pre, base, count, threshold) }, + #[allow(unreachable_patterns)] + _ => unreachable!(), + } + } + + /// Compute the difficulty of `out.len()` consecutive nonces starting at `base`. + /// + /// # Panics + /// If the kernel is not available on this CPU or `out.len() % lanes() != 0`. + pub fn difficulties(self, pre: &Precomputed, base: u64, out: &mut [u64]) { + assert!(self.is_available(), "kernel {} not supported by this CPU", self.name()); + assert!(out.len().is_multiple_of(self.lanes())); + match self { + Kernel::Scalar => scalar::difficulties(pre, base, out), + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + // SAFETY: availability asserted above. + Kernel::Avx2 => unsafe { x86::difficulties_avx2(pre, base, out) }, + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + // SAFETY: availability asserted above. + Kernel::Avx512 => unsafe { x86::difficulties_avx512(pre, base, out) }, + #[cfg(target_arch = "aarch64")] + // SAFETY: availability asserted above. + Kernel::Neon => unsafe { neon::difficulties_neon(pre, base, out) }, + #[allow(unreachable_patterns)] + _ => unreachable!(), + } + } +} + +impl std::fmt::Display for Kernel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.name()) + } +} + +/// Difficulty value of `work` for `hash`: `u64::from_le_bytes(blake2b_8(work_le || hash))`. +/// +/// The work is valid for a threshold `t` iff `validate(hash, work) >= t`. +#[inline] +pub fn validate(hash: &[u8; 32], work: u64) -> u64 { + scalar::difficulty(&Precomputed::new(hash), work) +} + +/// Difficulty of a nonce given a [`Precomputed`] block (cheaper than [`validate`] +/// when validating many nonces for the same hash). +#[inline] +pub fn difficulty(pre: &Precomputed, work: u64) -> u64 { + scalar::difficulty(pre, work) +} + +/// Given the lane-bundle base nonce returned by [`Kernel::search`], find the +/// first nonce in the bundle whose difficulty is `>= threshold` (verified with +/// the scalar kernel, so a SIMD kernel bug can never leak invalid work). +pub fn resolve_hit(kernel: Kernel, pre: &Precomputed, bundle_base: u64, threshold: u64) -> Option { + (0..kernel.lanes() as u64) + .map(|i| bundle_base.wrapping_add(i)) + .find(|&n| scalar::difficulty(pre, n) >= threshold) +} + +/// A reasonably random `u64` without external dependencies: the process-wide +/// randomly keyed SipHash of a monotonically increasing counter, the time and +/// the caller's address. Not for cryptographic use; it only needs to spread +/// concurrent generators over the nonce space. +/// +/// Exposed for the crate's own command-line tool; not part of the stable API. +#[doc(hidden)] +pub fn random_u64() -> u64 { + static COUNTER: AtomicU64 = AtomicU64::new(0); + thread_local! { + static STATE: RandomState = RandomState::new(); + } + let c = COUNTER.fetch_add(1, Ordering::Relaxed); + STATE.with(|s| { + let mut h = s.build_hasher(); + c.hash(&mut h); + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + .hash(&mut h); + (&COUNTER as *const AtomicU64 as usize).hash(&mut h); + std::thread::current().id().hash(&mut h); + h.finish() + }) +} + +/// Generate work for `hash` with the fastest available kernel. +/// +/// Spawns `threads` worker threads (`0` is treated as `1`); each repeatedly claims +/// a disjoint block of [`BATCH`] nonces from a shared counter that starts at a +/// random base, so no nonce is ever hashed twice. The first hit wins. Returns +/// `None` only if `cancel` was set before a hit was found. +pub fn generate(hash: &[u8; 32], threshold: u64, threads: usize, cancel: &AtomicBool) -> Option { + generate_with(Kernel::best(), hash, threshold, threads, cancel) +} + +/// [`generate`] with an explicit kernel. +/// +/// # Panics +/// If `kernel` is not available on this CPU. +pub fn generate_with( + kernel: Kernel, + hash: &[u8; 32], + threshold: u64, + threads: usize, + cancel: &AtomicBool, +) -> Option { + assert!(kernel.is_available(), "kernel {} not supported by this CPU", kernel.name()); + if threshold == 0 { + return Some(random_u64()); + } + let pre = Precomputed::new(hash); + let base = random_u64(); + let threads = threads.max(1); + let next_batch = AtomicU64::new(0); + let done = AtomicBool::new(false); + let result = AtomicU64::new(0); + + let worker = || { + while !done.load(Ordering::Acquire) && !cancel.load(Ordering::Relaxed) { + let b = next_batch.fetch_add(1, Ordering::Relaxed); + let start = base.wrapping_add(b.wrapping_mul(BATCH)); + if let Some(bundle) = kernel.search(&pre, start, BATCH, threshold) { + if let Some(nonce) = resolve_hit(kernel, &pre, bundle, threshold) { + // First writer wins; later hits are simply dropped. + if done + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + result.store(nonce, Ordering::Release); + } + return; + } + // A hit the scalar path does not confirm would be a kernel bug; + // debug builds catch it, release builds just keep searching. + debug_assert!(false, "SIMD hit not confirmed by the scalar kernel"); + } + } + }; + + if threads == 1 { + worker(); + } else { + std::thread::scope(|s| { + for _ in 0..threads { + s.spawn(worker); + } + }); + } + + if done.load(Ordering::Acquire) { + Some(result.load(Ordering::Acquire)) + } else { + None + } +} + +/// Measure the hash rate of `kernel` on `threads` threads for about `duration`. +/// +/// Returns hashes per second (sum over all threads). The search uses a threshold +/// of `u64::MAX` (a hit needs a 64-bit all-ones digest), so it measures the pure +/// search loop including per-batch overhead. +pub fn hash_rate_benchmark(kernel: Kernel, threads: usize, duration: Duration) -> f64 { + assert!(kernel.is_available(), "kernel {} not supported by this CPU", kernel.name()); + let hash: [u8; 32] = std::array::from_fn(|_| random_u64() as u8); + let pre = Precomputed::new(&hash); + let threads = threads.max(1); + let stop = AtomicBool::new(false); + let total = AtomicU64::new(0); + let start = Instant::now(); + std::thread::scope(|s| { + for t in 0..threads { + s.spawn(|| { + let mut n = random_u64(); + let mut count = 0u64; + while !stop.load(Ordering::Relaxed) { + if kernel.search(&pre, n, BATCH, u64::MAX).is_some() { + // Astronomically unlikely, but keep the count honest. + } + n = n.wrapping_add(BATCH); + count += BATCH; + } + total.fetch_add(count, Ordering::Relaxed); + }); + let _ = t; + } + std::thread::sleep(duration); + stop.store(true, Ordering::Relaxed); + }); + let elapsed = start.elapsed().as_secs_f64(); + total.load(Ordering::Relaxed) as f64 / elapsed +} + +/// Expected number of hashes to find work at `threshold`: `2^64 / (2^64 - threshold)`. +pub fn expected_hashes(threshold: u64) -> f64 { + let space = (u64::MAX - threshold) as f64 + 1.0; + 18446744073709551616.0 / space +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn docs_example() { + let hash = hex32("718CC2121C3E641059BC1C2CFC45666C99E8AE922F7A807B7D07B62C995D79E2"); + assert_eq!(validate(&hash, 0x2bf29ef00786a6bc), 0xffffffd21c3933f4); + } + + fn hex32(s: &str) -> [u8; 32] { + let mut out = [0u8; 32]; + for i in 0..32 { + out[i] = u8::from_str_radix(&s[2 * i..2 * i + 2], 16).unwrap(); + } + out + } + + #[test] + fn kernels_agree_with_scalar() { + let hash = hex32("991CF190094C00F0B68E2E5F75F6BEE95A2E0BD93CEAA4A6734DB9F19B728948"); + let pre = Precomputed::new(&hash); + let mut reference = vec![0u64; 64]; + let base = 0x1234_5678_9abc_def0u64; + scalar::difficulties(&pre, base, &mut reference); + for k in Kernel::available() { + let mut out = vec![0u64; 64]; + k.difficulties(&pre, base, &mut out); + assert_eq!(out, reference, "kernel {}", k); + } + } + + #[test] + fn generate_easy_threshold_every_kernel() { + let hash = hex32("991CF190094C00F0B68E2E5F75F6BEE95A2E0BD93CEAA4A6734DB9F19B728948"); + let cancel = AtomicBool::new(false); + for k in Kernel::available() { + for thr in [0xff00_0000_0000_0000u64, 0xffff_0000_0000_0000, 0xffff_ff00_0000_0000] { + let w = generate_with(k, &hash, thr, 2, &cancel).unwrap(); + assert!(validate(&hash, w) >= thr, "kernel {} thr {:x}", k, thr); + } + } + } + + #[test] + fn report_kernels() { + // Run with `--nocapture` to see which kernels CI actually exercised. + let best = Kernel::best(); + println!( + "available kernels: {:?}; best: {} ({} lanes)", + Kernel::available(), + best, + best.lanes() + ); + assert!(best.is_available()); + assert!(Kernel::available().contains(&Kernel::Scalar)); + #[cfg(target_arch = "aarch64")] + assert_eq!(best, Kernel::Neon); + } + + #[test] + fn cancel_is_honoured() { + let hash = [7u8; 32]; + let cancel = AtomicBool::new(true); + assert_eq!(generate(&hash, u64::MAX, 2, &cancel), None); + } +} diff --git a/nano-work-simd/src/neon.rs b/nano-work-simd/src/neon.rs new file mode 100644 index 0000000..eee96bc --- /dev/null +++ b/nano-work-simd/src/neon.rs @@ -0,0 +1,92 @@ +//! AArch64 NEON kernel (2 lanes of u64 per 128-bit register). +//! +//! NOTE: this module could only be reviewed, not executed, on the development +//! machine (no AArch64 hardware and no cross target available); see NOTES.md. + +use core::arch::aarch64::*; + +use crate::kernel::{self, Lane, Precomputed}; + +#[derive(Clone, Copy)] +pub(crate) struct Neon(uint64x2_t); + +impl Lane for Neon { + const LANES: usize = 2; + #[inline(always)] + unsafe fn splat(x: u64) -> Self { + Neon(vdupq_n_u64(x)) + } + #[inline(always)] + unsafe fn nonce(base: u64) -> Self { + let inc: [u64; 2] = [0, 1]; + Neon(vaddq_u64(vdupq_n_u64(base), vld1q_u64(inc.as_ptr()))) + } + #[inline(always)] + unsafe fn add(self, o: Self) -> Self { + Neon(vaddq_u64(self.0, o.0)) + } + #[inline(always)] + unsafe fn xor(self, o: Self) -> Self { + Neon(veorq_u64(self.0, o.0)) + } + #[inline(always)] + unsafe fn rot32(self) -> Self { + // Swap the 32-bit halves of each 64-bit lane: REV64 (32-bit elements). + Neon(vreinterpretq_u64_u32(vrev64q_u32(vreinterpretq_u32_u64(self.0)))) + } + #[inline(always)] + unsafe fn rot24(self) -> Self { + // Byte rotate via TBL: output byte i = input byte (i + 3) mod 8, per lane. + let idx: [u8; 16] = [3, 4, 5, 6, 7, 0, 1, 2, 11, 12, 13, 14, 15, 8, 9, 10]; + let t = vld1q_u8(idx.as_ptr()); + Neon(vreinterpretq_u64_u8(vqtbl1q_u8(vreinterpretq_u8_u64(self.0), t))) + } + #[inline(always)] + unsafe fn rot16(self) -> Self { + let idx: [u8; 16] = [2, 3, 4, 5, 6, 7, 0, 1, 10, 11, 12, 13, 14, 15, 8, 9]; + let t = vld1q_u8(idx.as_ptr()); + Neon(vreinterpretq_u64_u8(vqtbl1q_u8(vreinterpretq_u8_u64(self.0), t))) + } + #[inline(always)] + unsafe fn rot63(self) -> Self { + // rotr 63 == rotl 1: SHL by 1, then SRI (shift-right-insert) by 63 fills bit 0. + Neon(vsriq_n_u64::<63>(vshlq_n_u64::<1>(self.0), self.0)) + } + #[inline(always)] + unsafe fn store(self, out: *mut u64) { + vst1q_u64(out, self.0) + } + #[inline(always)] + unsafe fn thr_full(thr: u64) -> Self { + Neon(vdupq_n_u64(thr)) + } + #[inline(always)] + unsafe fn thr_hi32(thr: u64) -> Self { + // CMHS on 64-bit lanes is a single instruction; no high-32 shortcut needed. + Neon(vdupq_n_u64(thr)) + } + #[inline(always)] + unsafe fn ge_full(self, t: Self) -> u32 { + let m = vcgeq_u64(self.0, t.0); + vmaxvq_u32(vreinterpretq_u32_u64(m)) + } + #[inline(always)] + unsafe fn ge_hi32(self, t: Self) -> u32 { + self.ge_full(t) + } +} + +/// # Safety +/// The CPU must support NEON (always true on AArch64, but kept explicit so the +/// dispatcher's `is_aarch64_feature_detected!("neon")` gate is meaningful). +#[target_feature(enable = "neon")] +pub(crate) unsafe fn search_neon(pre: &Precomputed, base: u64, count: u64, thr: u64) -> Option { + kernel::search::(pre, base, count, thr) +} + +/// # Safety +/// The CPU must support NEON. +#[target_feature(enable = "neon")] +pub(crate) unsafe fn difficulties_neon(pre: &Precomputed, base: u64, out: &mut [u64]) { + kernel::difficulties::(pre, base, out) +} diff --git a/nano-work-simd/src/scalar.rs b/nano-work-simd/src/scalar.rs new file mode 100644 index 0000000..f5ee0e5 --- /dev/null +++ b/nano-work-simd/src/scalar.rs @@ -0,0 +1,84 @@ +//! Portable 1-lane kernel (also the reference used to resolve SIMD hits). + +use crate::kernel::{self, Lane, Precomputed}; + +#[derive(Clone, Copy)] +pub(crate) struct Scalar(u64); + +impl Lane for Scalar { + const LANES: usize = 1; + #[inline(always)] + unsafe fn splat(x: u64) -> Self { + Scalar(x) + } + #[inline(always)] + unsafe fn nonce(base: u64) -> Self { + Scalar(base) + } + #[inline(always)] + unsafe fn add(self, o: Self) -> Self { + Scalar(self.0.wrapping_add(o.0)) + } + #[inline(always)] + unsafe fn xor(self, o: Self) -> Self { + Scalar(self.0 ^ o.0) + } + #[inline(always)] + unsafe fn rot32(self) -> Self { + Scalar(self.0.rotate_right(32)) + } + #[inline(always)] + unsafe fn rot24(self) -> Self { + Scalar(self.0.rotate_right(24)) + } + #[inline(always)] + unsafe fn rot16(self) -> Self { + Scalar(self.0.rotate_right(16)) + } + #[inline(always)] + unsafe fn rot63(self) -> Self { + Scalar(self.0.rotate_right(63)) + } + #[inline(always)] + unsafe fn store(self, out: *mut u64) { + *out = self.0; + } + #[inline(always)] + unsafe fn thr_full(thr: u64) -> Self { + Scalar(thr) + } + #[inline(always)] + unsafe fn thr_hi32(thr: u64) -> Self { + // A 64-bit compare is a single instruction on every scalar ISA; the + // high-32 trick buys nothing here, so both variants compare in full. + Scalar(thr) + } + #[inline(always)] + unsafe fn ge_full(self, t: Self) -> u32 { + (self.0 >= t.0) as u32 + } + #[inline(always)] + unsafe fn ge_hi32(self, t: Self) -> u32 { + (self.0 >= t.0) as u32 + } +} + +/// Difficulty of a single nonce. +#[inline] +pub(crate) fn difficulty(pre: &Precomputed, nonce: u64) -> u64 { + // SAFETY: the scalar lane uses no target-specific instructions. + unsafe { + let c = kernel::Consts::::new(pre); + kernel::h0_lanes(&c, Scalar(nonce)).0 + } +} + +pub(crate) fn search(pre: &Precomputed, base: u64, count: u64, thr: u64) -> Option { + // SAFETY: as above. + unsafe { kernel::search::(pre, base, count, thr) } +} + +pub(crate) fn difficulties(pre: &Precomputed, base: u64, out: &mut [u64]) { + // SAFETY: as above. + unsafe { kernel::difficulties::(pre, base, out) } +} diff --git a/nano-work-simd/src/x86.rs b/nano-work-simd/src/x86.rs new file mode 100644 index 0000000..e72bcdf --- /dev/null +++ b/nano-work-simd/src/x86.rs @@ -0,0 +1,202 @@ +//! x86-64 kernels: AVX2 (4 lanes) and AVX-512F (8 lanes). +//! +//! All `unsafe` lives here: the intrinsics are only executed inside +//! `#[target_feature]` entry points, which the dispatcher in `lib.rs` calls only +//! after `is_x86_feature_detected!` confirmed the feature. + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +use crate::kernel::{self, Lane, Precomputed}; + +const SIGN64: i64 = i64::MIN; + +// ------------------------------------------------------------------ AVX2 + +#[derive(Clone, Copy)] +pub(crate) struct Avx2(__m256i); + +impl Lane for Avx2 { + const LANES: usize = 4; + #[inline(always)] + unsafe fn splat(x: u64) -> Self { + Avx2(_mm256_set1_epi64x(x as i64)) + } + #[inline(always)] + unsafe fn nonce(base: u64) -> Self { + Avx2(_mm256_add_epi64( + _mm256_set1_epi64x(base as i64), + _mm256_setr_epi64x(0, 1, 2, 3), + )) + } + #[inline(always)] + unsafe fn add(self, o: Self) -> Self { + Avx2(_mm256_add_epi64(self.0, o.0)) + } + #[inline(always)] + unsafe fn xor(self, o: Self) -> Self { + Avx2(_mm256_xor_si256(self.0, o.0)) + } + #[inline(always)] + unsafe fn rot32(self) -> Self { + // Swap the two 32-bit halves of each qword: one shuffle (port 5), no shifts. + Avx2(_mm256_shuffle_epi32::<0b10_11_00_01>(self.0)) + } + #[inline(always)] + unsafe fn rot24(self) -> Self { + // Byte rotate: one pshufb instead of shift/shift/or. + let m = _mm256_setr_epi8( + 3, 4, 5, 6, 7, 0, 1, 2, 11, 12, 13, 14, 15, 8, 9, 10, // + 3, 4, 5, 6, 7, 0, 1, 2, 11, 12, 13, 14, 15, 8, 9, 10, + ); + Avx2(_mm256_shuffle_epi8(self.0, m)) + } + #[inline(always)] + unsafe fn rot16(self) -> Self { + let m = _mm256_setr_epi8( + 2, 3, 4, 5, 6, 7, 0, 1, 10, 11, 12, 13, 14, 15, 8, 9, // + 2, 3, 4, 5, 6, 7, 0, 1, 10, 11, 12, 13, 14, 15, 8, 9, + ); + Avx2(_mm256_shuffle_epi8(self.0, m)) + } + #[inline(always)] + unsafe fn rot63(self) -> Self { + // rotr 63 == rotl 1: (x + x) | (x >> 63); the add can go to any ALU port. + Avx2(_mm256_or_si256( + _mm256_add_epi64(self.0, self.0), + _mm256_srli_epi64::<63>(self.0), + )) + } + #[inline(always)] + unsafe fn store(self, out: *mut u64) { + _mm256_storeu_si256(out as *mut __m256i, self.0) + } + #[inline(always)] + unsafe fn thr_full(thr: u64) -> Self { + // Unsigned x >= thr <=> signed (x ^ SIGN) > ((thr - 1) ^ SIGN), thr >= 1. + Avx2(_mm256_set1_epi64x((thr.wrapping_sub(1) as i64) ^ SIGN64)) + } + #[inline(always)] + unsafe fn thr_hi32(thr: u64) -> Self { + // Compare only the high dword of each qword (32-bit compare has 1-cycle + // latency vs 3 for vpcmpgtq). Valid when the low 32 bits of thr are zero. + let hi = (thr >> 32) as u32; + let t = (hi.wrapping_sub(1) ^ 0x8000_0000) as u64; + Avx2(_mm256_set1_epi64x((t << 32) as i64)) + } + #[inline(always)] + unsafe fn ge_full(self, t: Self) -> u32 { + let x = _mm256_xor_si256(self.0, _mm256_set1_epi64x(SIGN64)); + _mm256_movemask_pd(_mm256_castsi256_pd(_mm256_cmpgt_epi64(x, t.0))) as u32 + } + #[inline(always)] + unsafe fn ge_hi32(self, t: Self) -> u32 { + let x = _mm256_xor_si256(self.0, _mm256_set1_epi64x(SIGN64)); + let m = _mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpgt_epi32(x, t.0))) as u32; + m & 0xAA // odd bits = high dwords + } +} + +/// # Safety +/// The CPU must support AVX2. +#[target_feature(enable = "avx2")] +pub(crate) unsafe fn search_avx2(pre: &Precomputed, base: u64, count: u64, thr: u64) -> Option { + if thr & 0xffff_ffff == 0 { + kernel::search::(pre, base, count, thr) + } else { + kernel::search::(pre, base, count, thr) + } +} + +/// # Safety +/// The CPU must support AVX2. +#[target_feature(enable = "avx2")] +pub(crate) unsafe fn difficulties_avx2(pre: &Precomputed, base: u64, out: &mut [u64]) { + kernel::difficulties::(pre, base, out) +} + +// ------------------------------------------------------------------ AVX-512F + +#[derive(Clone, Copy)] +pub(crate) struct Avx512(__m512i); + +impl Lane for Avx512 { + const LANES: usize = 8; + #[inline(always)] + unsafe fn splat(x: u64) -> Self { + Avx512(_mm512_set1_epi64(x as i64)) + } + #[inline(always)] + unsafe fn nonce(base: u64) -> Self { + Avx512(_mm512_add_epi64( + _mm512_set1_epi64(base as i64), + _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7), + )) + } + #[inline(always)] + unsafe fn add(self, o: Self) -> Self { + Avx512(_mm512_add_epi64(self.0, o.0)) + } + #[inline(always)] + unsafe fn xor(self, o: Self) -> Self { + Avx512(_mm512_xor_si512(self.0, o.0)) + } + #[inline(always)] + unsafe fn rot32(self) -> Self { + Avx512(_mm512_ror_epi64::<32>(self.0)) // vprorq + } + #[inline(always)] + unsafe fn rot24(self) -> Self { + Avx512(_mm512_ror_epi64::<24>(self.0)) + } + #[inline(always)] + unsafe fn rot16(self) -> Self { + Avx512(_mm512_ror_epi64::<16>(self.0)) + } + #[inline(always)] + unsafe fn rot63(self) -> Self { + Avx512(_mm512_ror_epi64::<63>(self.0)) + } + #[inline(always)] + unsafe fn store(self, out: *mut u64) { + _mm512_storeu_si512(out as *mut _, self.0) + } + #[inline(always)] + unsafe fn thr_full(thr: u64) -> Self { + Avx512(_mm512_set1_epi64(thr as i64)) + } + #[inline(always)] + unsafe fn thr_hi32(thr: u64) -> Self { + Avx512(_mm512_set1_epi64(thr as i64)) + } + #[inline(always)] + unsafe fn ge_full(self, t: Self) -> u32 { + _mm512_cmpge_epu64_mask(self.0, t.0) as u32 + } + #[inline(always)] + unsafe fn ge_hi32(self, t: Self) -> u32 { + // Unsigned dword compare; the low dwords (thr low = 0) always compare + // true, so keep only the odd (high-dword) mask bits. + (_mm512_cmpge_epu32_mask(self.0, t.0) as u32) & 0xAAAA + } +} + +/// # Safety +/// The CPU must support AVX-512F. +#[target_feature(enable = "avx512f")] +pub(crate) unsafe fn search_avx512(pre: &Precomputed, base: u64, count: u64, thr: u64) -> Option { + if thr & 0xffff_ffff == 0 { + kernel::search::(pre, base, count, thr) + } else { + kernel::search::(pre, base, count, thr) + } +} + +/// # Safety +/// The CPU must support AVX-512F. +#[target_feature(enable = "avx512f")] +pub(crate) unsafe fn difficulties_avx512(pre: &Precomputed, base: u64, out: &mut [u64]) { + kernel::difficulties::(pre, base, out) +} diff --git a/nano-work-simd/tests/correctness.rs b/nano-work-simd/tests/correctness.rs new file mode 100644 index 0000000..7d6449c --- /dev/null +++ b/nano-work-simd/tests/correctness.rs @@ -0,0 +1,165 @@ +//! Cross-checks every kernel against the `blake2` crate (`Blake2bVar`, 8-byte output). + +use std::sync::atomic::AtomicBool; + +use blake2::digest::{Update, VariableOutput}; +use blake2::Blake2bVar; +use nano_work_simd::{ + difficulty, generate_with, resolve_hit, validate, Kernel, Precomputed, RECEIVE_THRESHOLD, + SEND_THRESHOLD, +}; + +/// Reference: blake2b(digest_size = 8, work_le || hash) as a little-endian u64. +fn reference(hash: &[u8; 32], work: u64) -> u64 { + let mut h = Blake2bVar::new(8).unwrap(); + h.update(&work.to_le_bytes()); + h.update(hash); + let mut out = [0u8; 8]; + h.finalize_variable(&mut out).unwrap(); + u64::from_le_bytes(out) +} + +/// Deterministic xorshift64* so failures are reproducible. +struct Rng(u64); +impl Rng { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + fn hash(&mut self) -> [u8; 32] { + let mut h = [0u8; 32]; + for c in h.chunks_mut(8) { + c.copy_from_slice(&self.next().to_le_bytes()); + } + h + } +} + +fn hex32(s: &str) -> [u8; 32] { + let mut out = [0u8; 32]; + for i in 0..32 { + out[i] = u8::from_str_radix(&s[2 * i..2 * i + 2], 16).unwrap(); + } + out +} + +#[test] +fn validate_matches_blake2_crate_10k_random_pairs() { + let mut rng = Rng(0x9E37_79B9_7F4A_7C15); + for _ in 0..10_000 { + let hash = rng.hash(); + let work = rng.next(); + assert_eq!(validate(&hash, work), reference(&hash, work), "hash {:02x?} work {:016x}", hash, work); + } +} + +#[test] +fn every_kernel_matches_blake2_crate_10k_random_pairs() { + // 10k random (hash, nonce) pairs per kernel present; every lane of every + // bundle is compared so lane placement bugs cannot hide. + for k in Kernel::available() { + let mut rng = Rng(0xD1B5_4A32_D192_ED03 ^ k.lanes() as u64); + let lanes = k.lanes(); + let mut out = vec![0u64; lanes]; + let mut checked = 0usize; + while checked < 10_000 { + let hash = rng.hash(); + let pre = Precomputed::new(&hash); + let base = rng.next(); + k.difficulties(&pre, base, &mut out); + for (i, &got) in out.iter().enumerate() { + let n = base.wrapping_add(i as u64); + assert_eq!(got, reference(&hash, n), "kernel {} hash {:02x?} nonce {:016x}", k, hash, n); + assert_eq!(got, difficulty(&pre, n)); + } + checked += lanes; + } + } +} + +#[test] +fn kernels_handle_nonce_wraparound() { + let hash = hex32("991CF190094C00F0B68E2E5F75F6BEE95A2E0BD93CEAA4A6734DB9F19B728948"); + let pre = Precomputed::new(&hash); + for k in Kernel::available() { + let mut out = vec![0u64; 16]; + let base = u64::MAX - 5; + k.difficulties(&pre, base, &mut out); + for (i, &got) in out.iter().enumerate() { + let n = base.wrapping_add(i as u64); + assert_eq!(got, reference(&hash, n), "kernel {} nonce {:016x}", k, n); + } + } +} + +#[test] +fn search_finds_every_hit_the_reference_finds() { + // Scan a fixed window with a low threshold and compare the set of valid + // nonces found by repeated `search` calls with a brute-force reference scan. + let hash = hex32("718CC2121C3E641059BC1C2CFC45666C99E8AE922F7A807B7D07B62C995D79E2"); + let pre = Precomputed::new(&hash); + let window = 1u64 << 14; + let base = 0x0123_4567_89ab_cdef; + for thr in [0xf000_0000_0000_0000u64, 0xf000_0000_0000_0001, 0xff00_0000_0000_0000] { + let expected: Vec = (0..window) + .map(|i| base + i) + .filter(|&n| reference(&hash, n) >= thr) + .collect(); + assert!(!expected.is_empty()); + for k in Kernel::available() { + let lanes = k.lanes() as u64; + let mut found = Vec::new(); + let mut n = base; + while n < base + window { + match k.search(&pre, n, base + window - n, thr) { + Some(bundle) => { + for i in 0..lanes { + if difficulty(&pre, bundle + i) >= thr { + found.push(bundle + i); + } + } + assert!(resolve_hit(k, &pre, bundle, thr).is_some()); + n = bundle + lanes; + } + None => break, + } + } + assert_eq!(found, expected, "kernel {} thr {:x}", k, thr); + } + } +} + +#[test] +fn generated_work_validates_for_every_kernel() { + let cancel = AtomicBool::new(false); + let mut rng = Rng(42); + for k in Kernel::available() { + for thr in [0xfff0_0000_0000_0000u64, 0xffff_0000_0000_0000, 0xffff_ff00_0000_0000] { + for threads in [1usize, 3] { + let hash = rng.hash(); + let w = generate_with(k, &hash, thr, threads, &cancel).expect("not cancelled"); + let v = reference(&hash, w); + assert!(v >= thr, "kernel {} threads {} thr {:x} got {:x}", k, threads, thr, v); + assert_eq!(v, validate(&hash, w)); + } + } + } +} + +#[test] +fn documented_example_from_nano_docs() { + // https://docs.nano.org/integration-guides/work-generation/ : + // hash 718C..79E2, work 2bf29ef00786a6bc -> difficulty ffffffd21c3933f4. + // Convention: the work hex string is the u64 value; it is hashed as its + // little-endian bytes, and the 8-byte digest is read little-endian. + let hash = hex32("718CC2121C3E641059BC1C2CFC45666C99E8AE922F7A807B7D07B62C995D79E2"); + let work = 0x2bf2_9ef0_0786_a6bc; + assert_eq!(validate(&hash, work), 0xffff_ffd2_1c39_33f4); + assert_eq!(reference(&hash, work), 0xffff_ffd2_1c39_33f4); + assert!(validate(&hash, work) >= RECEIVE_THRESHOLD); + assert!(validate(&hash, work) < SEND_THRESHOLD); +} diff --git a/src/main.rs b/src/main.rs index 61d4984..9bd5454 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,11 +21,7 @@ use rand::{Rng, SeedableRng}; use rand_xorshift::XorShiftRng; -use blake2::Blake2bVar; - -use digest::{Update, VariableOutput}; - -use byteorder::{ByteOrder, LittleEndian}; +use nano_work_simd::{Kernel, Precomputed, BATCH as CPU_BATCH}; use parking_lot::{Condvar, Mutex}; @@ -37,12 +33,8 @@ const LIVE_DIFFICULTY: u64 = 0xfffffff800000000; const LIVE_RECEIVE_DIFFICULTY: u64 = 0xfffffe0000000000; fn work_value(root: [u8; 32], work: [u8; 8]) -> u64 { - let mut buf = [0u8; 8]; - let mut hasher = Blake2bVar::new(buf.len()).expect("Unsupported hash length"); - hasher.update(&work); - hasher.update(&root); - hasher.finalize_variable(&mut buf).unwrap(); - LittleEndian::read_u64(&buf as _) + // `work` is the nonce as the little-endian bytes that are hashed. + nano_work_simd::validate(&root, u64::from_le_bytes(work)) } #[inline] @@ -540,6 +532,14 @@ async fn main() { .long("shuffle") .help("Pick a random request from the queue instead of the oldest. Increases efficiency when using multiple work servers") ) + .arg( + clap::Arg::with_name("cpu_kernel") + .long("cpu-kernel") + .value_name("KERNEL") + .default_value("auto") + .possible_values(&["auto", "avx512", "avx2", "neon", "scalar"]) + .help("Which CPU work kernel to use. 'auto' picks the fastest one this CPU supports."), + ) .get_matches(); let random_mode = args.is_present("shuffle"); let listen_addr = args @@ -552,6 +552,26 @@ async fn main() { .unwrap() .parse() .expect("Failed to parse CPU threads"); + let cpu_kernel = match args.value_of("cpu_kernel").unwrap() { + "auto" => Kernel::best(), + name => { + // clap has already restricted the value to the known kernel names. + let kernel = Kernel::from_name(name).expect("Failed to parse CPU kernel"); + if !kernel.is_available() { + eprintln!( + "CPU kernel '{}' is not supported by this CPU. Available kernels: {}", + kernel, + Kernel::available() + .iter() + .map(|k| k.name()) + .collect::>() + .join(", ") + ); + process::exit(1); + } + kernel + } + }; let gpu_local_work_size = args.value_of("gpu_local_work_size").map(|s| { s.parse() .expect("Failed to parse GPU local work size option") @@ -598,12 +618,21 @@ async fn main() { state.random_mode = random_mode; } let mut worker_handles = Vec::new(); + if cpu_threads > 0 { + println!( + "CPU work kernel: {} ({} nonces per instruction stream, {} threads)", + cpu_kernel, + cpu_kernel.lanes(), + cpu_threads + ); + } for _ in 0..cpu_threads { let work_state = work_state.clone(); let mut rng = XorShiftRng::from_rng(rand::thread_rng()).expect("Failed to create XorShiftRng"); let mut root = [0u8; 32]; let mut difficulty = 0u64; + let mut precomputed = Precomputed::new(&root); let mut task_complete = Arc::new(AtomicBool::new(true)); let handle = thread::spawn(move || loop { if task_complete.load(atomic::Ordering::Relaxed) { @@ -613,11 +642,19 @@ async fn main() { } root = state.root; difficulty = state.difficulty; + precomputed = Precomputed::new(&root); task_complete = state.task_complete.clone(); } - let mut out: [u8; 8] = rng.gen(); - for _ in 0..(1 << 18) { - if work_valid(root, out, difficulty).0 { + // Search one batch of consecutive nonces from a random start with the + // SIMD kernel, then re-check whether the task is still current. + let start: u64 = rng.gen(); + if let Some(bundle) = cpu_kernel.search(&precomputed, start, CPU_BATCH, difficulty) { + // The kernel reports a bundle of `lanes()` nonces; the scalar + // reference picks (and re-verifies) the winning one. + if let Some(nonce) = + nano_work_simd::resolve_hit(cpu_kernel, &precomputed, bundle, difficulty) + { + let out = nonce.to_le_bytes(); let mut state = work_state.0.lock(); if root == state.root { if let Some(callback) = state.callback.take() { @@ -625,14 +662,6 @@ async fn main() { state.set_task(&work_state.1); } } - break; - } - for byte in out.iter_mut() { - *byte = byte.wrapping_add(1); - if *byte != 0 { - // We did not overflow - break; - } } } });