diff --git a/Cargo.lock b/Cargo.lock index 27a84508ae..d4d9688314 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2679,7 +2679,7 @@ dependencies = [ [[package]] name = "grovedb" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=ad2492dcdc869a1452b0b10fbed8f9b0de1634c6#ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" +source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" dependencies = [ "axum 0.8.9", "bincode", @@ -2717,7 +2717,7 @@ dependencies = [ [[package]] name = "grovedb-bulk-append-tree" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=ad2492dcdc869a1452b0b10fbed8f9b0de1634c6#ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" +source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" dependencies = [ "bincode", "blake3", @@ -2733,7 +2733,7 @@ dependencies = [ [[package]] name = "grovedb-commitment-tree" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=ad2492dcdc869a1452b0b10fbed8f9b0de1634c6#ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" +source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" dependencies = [ "blake3", "grovedb-bulk-append-tree", @@ -2749,7 +2749,7 @@ dependencies = [ [[package]] name = "grovedb-costs" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=ad2492dcdc869a1452b0b10fbed8f9b0de1634c6#ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" +source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" dependencies = [ "integer-encoding", "intmap", @@ -2759,7 +2759,7 @@ dependencies = [ [[package]] name = "grovedb-dense-fixed-sized-merkle-tree" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=ad2492dcdc869a1452b0b10fbed8f9b0de1634c6#ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" +source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" dependencies = [ "bincode", "blake3", @@ -2772,7 +2772,7 @@ dependencies = [ [[package]] name = "grovedb-element" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=ad2492dcdc869a1452b0b10fbed8f9b0de1634c6#ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" +source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" dependencies = [ "bincode", "bincode_derive", @@ -2787,7 +2787,7 @@ dependencies = [ [[package]] name = "grovedb-epoch-based-storage-flags" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=ad2492dcdc869a1452b0b10fbed8f9b0de1634c6#ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" +source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" dependencies = [ "grovedb-costs", "hex", @@ -2799,7 +2799,7 @@ dependencies = [ [[package]] name = "grovedb-merk" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=ad2492dcdc869a1452b0b10fbed8f9b0de1634c6#ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" +source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" dependencies = [ "bincode", "bincode_derive", @@ -2825,7 +2825,7 @@ dependencies = [ [[package]] name = "grovedb-merkle-mountain-range" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=ad2492dcdc869a1452b0b10fbed8f9b0de1634c6#ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" +source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" dependencies = [ "bincode", "blake3", @@ -2836,7 +2836,7 @@ dependencies = [ [[package]] name = "grovedb-path" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=ad2492dcdc869a1452b0b10fbed8f9b0de1634c6#ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" +source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" dependencies = [ "hex", ] @@ -2844,7 +2844,7 @@ dependencies = [ [[package]] name = "grovedb-query" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=ad2492dcdc869a1452b0b10fbed8f9b0de1634c6#ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" +source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" dependencies = [ "bincode", "byteorder", @@ -2860,7 +2860,7 @@ dependencies = [ [[package]] name = "grovedb-storage" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=ad2492dcdc869a1452b0b10fbed8f9b0de1634c6#ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" +source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" dependencies = [ "blake3", "grovedb-costs", @@ -2879,7 +2879,7 @@ dependencies = [ [[package]] name = "grovedb-version" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=ad2492dcdc869a1452b0b10fbed8f9b0de1634c6#ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" +source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" dependencies = [ "thiserror 2.0.18", "versioned-feature-core 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2888,7 +2888,7 @@ dependencies = [ [[package]] name = "grovedb-visualize" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=ad2492dcdc869a1452b0b10fbed8f9b0de1634c6#ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" +source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" dependencies = [ "hex", "itertools 0.14.0", @@ -2897,7 +2897,7 @@ dependencies = [ [[package]] name = "grovedbg-types" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=ad2492dcdc869a1452b0b10fbed8f9b0de1634c6#ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" +source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" dependencies = [ "serde", "serde_with 3.20.0", @@ -5094,8 +5094,8 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.5.0", - "itertools 0.14.0", + "heck 0.4.1", + "itertools 0.13.0", "log", "multimap", "petgraph", @@ -5116,7 +5116,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.117", @@ -5129,7 +5129,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.117", @@ -5299,7 +5299,7 @@ dependencies = [ "once_cell", "socket2 0.6.3", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -8420,7 +8420,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -8429,16 +8429,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -8456,31 +8447,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -8489,96 +8463,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "0.5.40" diff --git a/book/src/drive/average-index-examples.md b/book/src/drive/average-index-examples.md index ad3eb4dc9a..997ab58b8a 100644 --- a/book/src/drive/average-index-examples.md +++ b/book/src/drive/average-index-examples.md @@ -1702,3 +1702,13 @@ The split closely parallels the count and sum chapters — point lookups for Q1 The chapter is grounded in the [`document_average_worst_case`](https://github.com/dashpay/platform/blob/v3.1-dev/packages/rs-drive/benches/document_average_worst_case.rs) bench's measured numbers — Q1–Q7 verify cleanly end-to-end against the shared root hash `8b15f732…ffc7`. A natural expansion follow-up (out of scope here): a worked example of "exact-precision" averages — for callers that need fractional averages (e.g. `avg = 50.7142857…` rather than `50.99`), the protocol-level approach is to return `(count, sum)` and let the client compute in its preferred numeric format (the chapter notes this in [Numerical Considerations](#numerical-considerations) above; a future expansion could walk through the fixed-point vs. floating-point trade-offs). + +### No-proof path: joint count-and-sum dispatch + +The no-proof AVG path lives in `crate::query::drive_document_count_and_sum_query`. It consumes the same `DocumentAverageRequest` the prove path uses and resolves routing through sum's versioned mode-detection table, so the `(where_clauses × mode)` → executor mapping has a single source of truth shared with the sum and count surfaces. The dispatcher splits on the resolved mode: + +- **`Total` / `PerInValue`** (no-range `Equal`/`In` on a `summable + countable` index) walks the point-lookup path query and decodes `(count, sum)` from each visited `CountSumTree` terminator in one call via `Element::count_sum_value_or_default()`. One grovedb call per `In` branch, both metrics together. +- **`RangeNoProof` distinct shapes** (`GroupByRange` / `GroupByCompound` + range on an index that declares BOTH `rangeCountable: true` AND `rangeSummable: true` — DPP exposes `rangeAverageable: true` as shorthand for the pair) walk `ProvableCountProvableSumTree` terminators once via the same `distinct_sum_path_query` builder the sum surface uses, emitting one `(count, sum)` per distinct in-range key — strictly better than the count + sum surfaces' parallel walks because both metrics come from each visited element. +- **`RangeNoProof` aggregate shapes** (`Aggregate` / `GroupByIn` + range) call grovedb's combined merk-internal accumulator directly: `query_aggregate_count_and_sum` against the PCPS path query, yielding `(u64, i64)` from a single O(log n) traversal. Compound `In + range` per-In fans out (≤100 branches per the `In::in_values()` validator cap) and issues one accumulator call per branch under a shared read transaction. Bounded regardless of how many documents the range matches — keeping the public DAPI endpoint closed against amplification. + +The no-prove combined accumulator (`query_aggregate_count_and_sum`) is the symmetric counterpart of the prove-side `AggregateCountAndSumOnRange` primitive Q5 / Q7 above use. Both sides walk the same PCPS terminator shape with `(u128, i128)` accumulators (narrowing to `(u64, i64)` at the entry point) — the only difference is that the no-prove path returns the pair directly while the prove path emits proof bytes the client verifies via `GroveDb::verify_aggregate_count_and_sum_query`. diff --git a/packages/rs-dpp/Cargo.toml b/packages/rs-dpp/Cargo.toml index 2da6961502..ebfd571587 100644 --- a/packages/rs-dpp/Cargo.toml +++ b/packages/rs-dpp/Cargo.toml @@ -71,7 +71,7 @@ strum = { version = "0.26", features = ["derive"] } json-schema-compatibility-validator = { path = '../rs-json-schema-compatibility-validator', optional = true } once_cell = "1.19.0" tracing = { version = "0.1.41" } -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "ad2492dcdc869a1452b0b10fbed8f9b0de1634c6", optional = true } +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9", optional = true } [dev-dependencies] tokio = { version = "1.40", features = ["full"] } diff --git a/packages/rs-drive-abci/Cargo.toml b/packages/rs-drive-abci/Cargo.toml index b71a2fccf0..ea3211d45a 100644 --- a/packages/rs-drive-abci/Cargo.toml +++ b/packages/rs-drive-abci/Cargo.toml @@ -82,7 +82,7 @@ derive_more = { version = "1.0", features = ["from", "deref", "deref_mut"] } async-trait = "0.1.77" console-subscriber = { version = "0.4", optional = true } bls-signatures = { git = "https://github.com/dashpay/bls-signatures", rev = "0842b17583888e8f46c252a4ee84cdfd58e0546f", optional = true } -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" } +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9" } nonempty = "0.11" [dev-dependencies] @@ -103,7 +103,7 @@ dpp = { path = "../rs-dpp", default-features = false, features = [ drive = { path = "../rs-drive", features = ["fixtures-and-mocks"] } drive-proof-verifier = { path = "../rs-drive-proof-verifier" } strategy-tests = { path = "../strategy-tests" } -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "ad2492dcdc869a1452b0b10fbed8f9b0de1634c6", features = ["client"] } +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9", features = ["client"] } assert_matches = "1.5.0" drive-abci = { path = ".", features = ["testing-config", "mocks"] } bls-signatures = { git = "https://github.com/dashpay/bls-signatures", rev = "0842b17583888e8f46c252a4ee84cdfd58e0546f" } diff --git a/packages/rs-drive/Cargo.toml b/packages/rs-drive/Cargo.toml index de8b48b700..bd4b74577e 100644 --- a/packages/rs-drive/Cargo.toml +++ b/packages/rs-drive/Cargo.toml @@ -52,12 +52,12 @@ enum-map = { version = "2.0.3", optional = true } intmap = { version = "3.0.1", features = ["serde"], optional = true } chrono = { version = "0.4.35", optional = true } itertools = { version = "0.13", optional = true } -grovedb = { git = "https://github.com/dashpay/grovedb", rev = "ad2492dcdc869a1452b0b10fbed8f9b0de1634c6", optional = true, default-features = false } -grovedb-costs = { git = "https://github.com/dashpay/grovedb", rev = "ad2492dcdc869a1452b0b10fbed8f9b0de1634c6", optional = true } -grovedb-path = { git = "https://github.com/dashpay/grovedb", rev = "ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" } -grovedb-storage = { git = "https://github.com/dashpay/grovedb", rev = "ad2492dcdc869a1452b0b10fbed8f9b0de1634c6", optional = true } -grovedb-version = { git = "https://github.com/dashpay/grovedb", rev = "ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" } -grovedb-epoch-based-storage-flags = { git = "https://github.com/dashpay/grovedb", rev = "ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" } +grovedb = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9", optional = true, default-features = false } +grovedb-costs = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9", optional = true } +grovedb-path = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9" } +grovedb-storage = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9", optional = true } +grovedb-version = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9" } +grovedb-epoch-based-storage-flags = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9" } [dev-dependencies] criterion = "0.5" diff --git a/packages/rs-drive/src/query/drive_document_average_query/drive_dispatcher.rs b/packages/rs-drive/src/query/drive_document_average_query/drive_dispatcher.rs index 6b8261ef91..4810e4ea97 100644 --- a/packages/rs-drive/src/query/drive_document_average_query/drive_dispatcher.rs +++ b/packages/rs-drive/src/query/drive_document_average_query/drive_dispatcher.rs @@ -1,34 +1,34 @@ //! Average-query dispatcher entry point. //! -//! Implementation strategy: **compose** count + sum into the -//! `(count, sum)` pair the client divides. Both executors are real -//! and live in `drive_document_count_query` / -//! `drive_document_sum_query` respectively; the average dispatcher -//! issues both requests under the same `transaction` and zips their -//! responses together by `(in_key, key)` for grouped shapes. +//! Routes a [`DocumentAverageRequest`] to one of two backends: +//! - **No-prove path** → delegates to the joint count-and-sum +//! dispatcher +//! [`Drive::execute_document_count_and_sum_request`], which walks +//! grovedb ONCE and reads both metrics from each visited +//! count-sum-bearing element via +//! [`grovedb::Element::count_sum_value_or_default`]. See its module +//! docstring for the routing / atomicity contract. +//! - **Prove path** → dispatched to +//! [`Drive::execute_document_average_prove`] (defined below), which +//! routes to one of the PCPS / direct-read prove executors based on +//! `(mode, where_clauses)`. The prove path's per-shape rules are +//! unchanged. //! -//! ## Why compose instead of using a single PCPS traversal? +//! ## Joint dispatch //! -//! grovedb's `AggregateCountAndSumOnRange` primitive returns both -//! metrics from one root-hash-committed traversal — cheaper on the -//! wire and atomic — but it only fires when the chosen index has -//! a `ProvableCountProvableSumTree` terminator (i.e. `rangeCountable -//! + rangeSummable`). For doctypes/indexes that lack PCPS-eligibility -//! (just `documentsSummable` without `rangeCountable`, for example) -//! the no-prove path has to compose two reads instead: +//! The no-prove dispatcher at +//! [`crate::query::drive_document_count_and_sum_query`] reads +//! `(count, sum)` together — via grovedb's combined +//! `query_aggregate_count_and_sum` accumulator on the aggregate range +//! branch, and via a single PCPS walk on the distinct-grouped branch. +//! Routing reuses sum's versioned mode-detection table so the +//! `(where_clauses × mode)` → executor decision has a single source +//! of truth shared with the count and sum surfaces. //! -//! - **No-prove paths**: count + sum are read within the same -//! grovedb snapshot, so they see identical state (no block- -//! boundary race, no off-by-one). When the caller passes a -//! `TransactionArg::None` (the drive-abci query path), the -//! dispatcher opens a short-lived read transaction internally and -//! reuses it across both sub-calls so the atomicity guarantee -//! holds regardless of caller plumbing. The internal transaction -//! is rolled back at the end (read-only, never commits). -//! - **Prove path**: dispatched to -//! [`Drive::execute_document_average_prove`] (defined below), -//! which routes to one of the PCPS / direct-read prove executors -//! based on `(mode, where_clauses)`: +//! ## Prove path shapes (unchanged) +//! +//! The prove-path routing table at +//! [`Self::execute_document_average_prove`] picks one of: //! - empty-where + `documentsCountable + documentsSummable` //! doctype → primary-key count-sum tree direct read //! - range AVG on a `rangeAverageable` index → PCPS @@ -49,17 +49,12 @@ use crate::drive::Drive; use crate::error::query::QuerySyntaxError; use crate::error::Error; use crate::query::drive_document_average_query::{ - AverageEntry, AverageMode, DocumentAverageRequest, DocumentAverageResponse, -}; -use crate::query::drive_document_count_query::{ - CountMode, DocumentCountRequest, DocumentCountResponse, + AverageMode, DocumentAverageRequest, DocumentAverageResponse, }; use crate::query::drive_document_sum_query::index_picker::{ find_range_summable_index_for_where_clauses, find_summable_index_for_where_clauses, }; -use crate::query::drive_document_sum_query::{ - is_range_operator, DocumentSumRequest, DocumentSumResponse, DriveDocumentSumQuery, SumMode, -}; +use crate::query::drive_document_sum_query::{is_range_operator, DriveDocumentSumQuery}; use dpp::data_contract::accessors::v0::DataContractV0Getters; use dpp::data_contract::document_type::accessors::{DocumentTypeV0Getters, DocumentTypeV2Getters}; use dpp::version::PlatformVersion; @@ -67,12 +62,14 @@ use grovedb::TransactionArg; #[cfg(feature = "server")] impl Drive { - /// Server-side entry point for the average surface. Composes the - /// count + sum executors and zips their outputs into the - /// `(count, sum)` pair the client divides. + /// Server-side entry point for the average surface. /// - /// See the module docstring for the rationale on composition vs. - /// a single PCPS traversal. + /// Splits prove vs. no-prove at the top level: + /// - `prove = true` → routes to + /// [`Self::execute_document_average_prove`]. + /// - `prove = false` → routes to + /// [`Self::execute_document_count_and_sum_request`], the joint + /// dispatcher that reads `(count, sum)` together. pub fn execute_document_average_request( &self, request: DocumentAverageRequest, @@ -82,116 +79,7 @@ impl Drive { if request.prove { return self.execute_document_average_prove(request, transaction, platform_version); } - - // Map `AverageMode` → matching `CountMode` / `SumMode`. The - // three enums are structurally identical (same four variants); - // each pair just lives in its own namespace. - let (count_mode, sum_mode) = match request.mode { - AverageMode::Aggregate => (CountMode::Aggregate, SumMode::Aggregate), - AverageMode::GroupByIn => (CountMode::GroupByIn, SumMode::GroupByIn), - AverageMode::GroupByRange => (CountMode::GroupByRange, SumMode::GroupByRange), - AverageMode::GroupByCompound => (CountMode::GroupByCompound, SumMode::GroupByCompound), - }; - - // Build parallel sub-requests. Both consume the same - // `where_clauses` + `order_clauses` + `limit` + (false) `prove` - // — the average's shape contract is "two reads of the same - // grovedb snapshot, zipped after." - // - // Architectural follow-up: tracked at - // [dashpay/platform#3687](https://github.com/dashpay/platform/issues/3687). - // The two-sub-request shape will collapse into a single - // `DocumentCountSumRequest` + a unified - // `execute_document_count_and_sum_request` that walks - // grovedb once and reads both metrics from each visited PCPS - // element via `count_sum_value_or_default()`. The prove path - // at `execute_document_average_prove` below already does - // this (one PCPS walk yields both fields); the no-proof - // path currently double-walks. The current two-request - // shape is correct (the local transaction below guarantees - // atomicity); it just does more grovedb work than strictly - // necessary, and the dual-routing requires count's and sum's - // routing tables to stay in lock-step for AVG composition to - // work (already caught one routing divergence). Issue #3687 - // captures the full scope including the four joint per-mode - // no-proof executors that need to land. - let count_request = DocumentCountRequest { - contract: request.contract, - document_type: request.document_type, - where_clauses: request.where_clauses.clone(), - order_clauses: request.order_clauses.clone(), - mode: count_mode, - limit: request.limit, - prove: false, - drive_config: request.drive_config, - }; - let sum_request = DocumentSumRequest { - contract: request.contract, - document_type: request.document_type, - sum_property: request.sum_property, - where_clauses: request.where_clauses, - order_clauses: request.order_clauses, - mode: sum_mode, - limit: request.limit, - prove: false, - drive_config: request.drive_config, - }; - - // Atomicity: both sub-reads must see the same grovedb root. If - // the caller didn't provide a transaction we open a short-lived - // read transaction here and reuse it across both executors so - // a concurrent block commit can't slip between the count and - // sum reads (the attacker-steerable race documented in the - // module-level docstring). The local transaction is read-only - // and dropped without commit at the end of this function. - let local_tx; - let effective_transaction: TransactionArg = if transaction.is_some() { - transaction - } else { - local_tx = self.grove.start_transaction(); - Some(&local_tx) - }; - - let count_response = self.execute_document_count_request( - count_request, - effective_transaction, - platform_version, - )?; - let sum_response = self.execute_document_sum_request( - sum_request, - effective_transaction, - platform_version, - )?; - - // Combine. The two executors emit either Aggregate or Entries - // (Proof is unreachable here since `prove=false` above). The - // mode-pair is symmetric so they must agree on which shape - // they emit — mismatches indicate a routing bug, surface as - // CorruptedCodeExecution. - match (count_response, sum_response) { - (DocumentCountResponse::Aggregate(count), DocumentSumResponse::Aggregate(sum)) => { - Ok(DocumentAverageResponse::Aggregate { count, sum }) - } - ( - DocumentCountResponse::Entries(count_entries), - DocumentSumResponse::Entries(sum_entries), - ) => Ok(DocumentAverageResponse::Entries(zip_entries( - count_entries, - sum_entries, - )?)), - // Mismatched shapes — count executor and sum executor - // disagreed on whether the result fits in a single row. - // Should be impossible because they share the same mode - // and `validate_and_canonicalize_where_clauses` runs the - // same checks on both. - _ => Err(Error::Drive( - crate::error::drive::DriveError::CorruptedCodeExecution( - "average composition: count and sum executors emitted disagreeing \ - response shapes — both should agree on Aggregate vs Entries given \ - identical mode + where + group_by", - ), - )), - } + self.execute_document_count_and_sum_request(request, transaction, platform_version) } /// Prove path of [`Self::execute_document_average_request`]. @@ -246,7 +134,7 @@ impl Drive { // Empty-where AVG fast path: prove the primary-key // count-sum-bearing element directly when the doctype - // declares both `documentsCountable: true` (implied by + // declares both `documents_countable: true` (implied by // having a CountSumTree primary key) and a matching // `documents_summable`. The verifier extracts `(count, // sum)` from one element. @@ -503,188 +391,9 @@ impl Drive { } } -/// Merge per-`(in_key, key)` count entries and sum entries into average -/// entries via a strict two-pointer merge keyed on `(in_key, key)`. -/// -/// Both inputs are emitted by the same executor family with identical -/// `where_clauses` / `order_clauses` / `mode` against the same grovedb -/// snapshot, so they MUST emit the same set of keys in the same -/// ascending `(in_key, key)` order. Any divergence (key on one side -/// only, or different ordering) indicates an executor bug and is -/// surfaced as `CorruptedCodeExecution` rather than silently zeroed at -/// the wire layer — the previous defensive `None`-preservation pattern -/// was indistinguishable from "this key matched zero documents but the -/// sum is nonzero" once the wire mapping flattened `Option` → -/// `u64`, which let attacker-timed inserts between the two reads -/// produce a `count=0, sum=V` bucket that crashed naive `sum / count` -/// clients with a divide-by-zero. With atomicity now enforced inside -/// `execute_document_average_request` (see module docstring), the only -/// remaining cause of divergence is a real executor bug — treating it -/// as fatal is correct. -/// -/// Output is always strictly ascending by `(in_key, key)` (same order -/// the inputs are required to be in). -#[cfg(feature = "server")] -fn zip_entries( - count_entries: Vec, - sum_entries: Vec, -) -> Result, Error> { - use crate::error::drive::DriveError; - - let mut out = Vec::with_capacity(count_entries.len().max(sum_entries.len())); - let mut c_iter = count_entries.into_iter(); - let mut s_iter = sum_entries.into_iter(); - let mut next_c = c_iter.next(); - let mut next_s = s_iter.next(); - - loop { - match (&next_c, &next_s) { - (Some(c), Some(s)) => { - let c_key = (&c.in_key, &c.key); - let s_key = (&s.in_key, &s.key); - match c_key.cmp(&s_key) { - std::cmp::Ordering::Equal => { - let c = next_c.take().expect("checked Some above"); - let s = next_s.take().expect("checked Some above"); - out.push(AverageEntry { - in_key: c.in_key, - key: c.key, - count: c.count, - sum: s.sum, - }); - next_c = c_iter.next(); - next_s = s_iter.next(); - } - std::cmp::Ordering::Less => { - return Err(Error::Drive(DriveError::CorruptedCodeExecution( - "average composition: count executor emitted a (in_key, key) the \ - sum executor didn't — both executors run identical inputs against \ - the same grovedb snapshot, so divergence indicates an executor bug", - ))); - } - std::cmp::Ordering::Greater => { - return Err(Error::Drive(DriveError::CorruptedCodeExecution( - "average composition: sum executor emitted a (in_key, key) the \ - count executor didn't — both executors run identical inputs against \ - the same grovedb snapshot, so divergence indicates an executor bug", - ))); - } - } - } - (Some(_), None) => { - return Err(Error::Drive(DriveError::CorruptedCodeExecution( - "average composition: count executor produced more entries than sum executor \ - — both executors run identical inputs against the same grovedb snapshot, \ - so divergence indicates an executor bug", - ))); - } - (None, Some(_)) => { - return Err(Error::Drive(DriveError::CorruptedCodeExecution( - "average composition: sum executor produced more entries than count executor \ - — both executors run identical inputs against the same grovedb snapshot, \ - so divergence indicates an executor bug", - ))); - } - (None, None) => break, - } - } - Ok(out) -} - #[cfg(all(test, feature = "server"))] mod tests { use super::*; - use crate::error::drive::DriveError; - use crate::query::{SplitCountEntry, SumEntry}; - - fn cc(in_key: Option<&[u8]>, key: &[u8], count: u64) -> SplitCountEntry { - SplitCountEntry { - in_key: in_key.map(|b| b.to_vec()), - key: key.to_vec(), - count: Some(count), - } - } - fn ss(in_key: Option<&[u8]>, key: &[u8], sum: i64) -> SumEntry { - SumEntry { - in_key: in_key.map(|b| b.to_vec()), - key: key.to_vec(), - sum: Some(sum), - } - } - - #[test] - fn zip_entries_merges_aligned_streams_in_ascending_order() { - let count_entries = vec![cc(None, b"a", 1), cc(None, b"b", 2), cc(None, b"c", 3)]; - let sum_entries = vec![ss(None, b"a", 10), ss(None, b"b", 20), ss(None, b"c", 30)]; - let out = zip_entries(count_entries, sum_entries).expect("aligned streams must merge"); - assert_eq!(out.len(), 3); - assert_eq!(out[0].key, b"a"); - assert_eq!(out[0].count, Some(1)); - assert_eq!(out[0].sum, Some(10)); - assert_eq!(out[2].key, b"c"); - assert_eq!(out[2].count, Some(3)); - assert_eq!(out[2].sum, Some(30)); - } - - #[test] - fn zip_entries_errors_when_count_has_an_extra_key() { - // count has `b` but sum doesn't — strict merge must reject. - let count_entries = vec![cc(None, b"a", 1), cc(None, b"b", 2)]; - let sum_entries = vec![ss(None, b"a", 10)]; - let err = zip_entries(count_entries, sum_entries) - .expect_err("divergent streams must surface as CorruptedCodeExecution"); - assert!( - matches!(err, Error::Drive(DriveError::CorruptedCodeExecution(_))), - "expected CorruptedCodeExecution, got {err:?}", - ); - } - - #[test] - fn zip_entries_errors_when_sum_has_an_extra_key() { - let count_entries = vec![cc(None, b"a", 1)]; - let sum_entries = vec![ss(None, b"a", 10), ss(None, b"b", 20)]; - let err = zip_entries(count_entries, sum_entries) - .expect_err("divergent streams must surface as CorruptedCodeExecution"); - assert!( - matches!(err, Error::Drive(DriveError::CorruptedCodeExecution(_))), - "expected CorruptedCodeExecution, got {err:?}", - ); - } - - #[test] - fn zip_entries_errors_when_streams_disagree_on_a_key_in_the_middle() { - // count has `b`, sum has `c` between the matching `a` and `d`. - let count_entries = vec![cc(None, b"a", 1), cc(None, b"b", 2), cc(None, b"d", 4)]; - let sum_entries = vec![ss(None, b"a", 10), ss(None, b"c", 30), ss(None, b"d", 40)]; - let err = zip_entries(count_entries, sum_entries) - .expect_err("middle-of-stream divergence must surface as CorruptedCodeExecution"); - assert!(matches!( - err, - Error::Drive(DriveError::CorruptedCodeExecution(_)) - )); - } - - #[test] - fn zip_entries_handles_compound_in_key_ordering() { - // (Some("X"), "a") < (Some("X"), "b") < (Some("Y"), "a") in - // lexicographic order — verify the merge follows it. - let count_entries = vec![ - cc(Some(b"X"), b"a", 1), - cc(Some(b"X"), b"b", 2), - cc(Some(b"Y"), b"a", 3), - ]; - let sum_entries = vec![ - ss(Some(b"X"), b"a", 10), - ss(Some(b"X"), b"b", 20), - ss(Some(b"Y"), b"a", 30), - ]; - let out = zip_entries(count_entries, sum_entries).expect("aligned compound merge"); - assert_eq!(out.len(), 3); - assert_eq!(out[0].in_key.as_deref(), Some(b"X".as_ref())); - assert_eq!(out[0].key, b"a"); - assert_eq!(out[2].in_key.as_deref(), Some(b"Y".as_ref())); - assert_eq!(out[2].key, b"a"); - } // ── Dispatcher limit-policy regression tests ─────────────────── // @@ -1127,4 +836,1435 @@ mod tests { let _: (GroveDBProof, _) = bincode::decode_from_slice(&proof_bytes, bincode_config) .expect("proof bytes must bincode-decode as a GroveDBProof"); } + + // ── Joint count-and-sum no-prove executor cross-checks ──────── + // + // Acceptance criterion 4 of issue #3687: "one [test] per joint + // executor confirming `(count, sum)` match what the current + // double-dispatch produces, against the same grades-contract + // fixture." + // + // Strategy: for each joint executor (Total / PerInValue / + // RangeNoProof — and RangeNoProof's distinct branch), issue the + // AVG no-prove request via `execute_document_average_request` + // AND independently issue separate count + sum requests under + // the same transaction. Assert the joint executor's + // `(count, sum)` matches the zipped pair from the independent + // count + sum surfaces — a cross-check the joint and per-surface + // dispatchers cannot silently disagree. + + use crate::query::drive_document_average_query::AverageEntry; + use crate::query::drive_document_count_query::{ + CountMode, DocumentCountRequest, DocumentCountResponse, + }; + use crate::query::drive_document_sum_query::{ + DocumentSumRequest, DocumentSumResponse, SumMode, + }; + + /// Issue an independent count + sum pair via the per-surface + /// dispatchers and return the zipped `(count, sum)` aggregate. + /// Used as the source of truth for cross-checking the joint + /// executor's output. + fn independent_count_sum_aggregate( + drive: &Drive, + contract: &dpp::data_contract::DataContract, + document_type: dpp::data_contract::document_type::DocumentTypeRef, + sum_property: &str, + where_clauses: Vec, + drive_config: &DriveConfig, + platform_version: &PlatformVersion, + ) -> (u64, i64) { + let count_request = DocumentCountRequest { + contract, + document_type, + where_clauses: where_clauses.clone(), + order_clauses: Vec::new(), + mode: CountMode::Aggregate, + limit: None, + prove: false, + drive_config, + }; + let sum_request = DocumentSumRequest { + contract, + document_type, + sum_property: sum_property.to_string(), + where_clauses, + order_clauses: Vec::new(), + mode: SumMode::Aggregate, + limit: None, + prove: false, + drive_config, + }; + let count_resp = drive + .execute_document_count_request(count_request, None, platform_version) + .expect("independent count"); + let sum_resp = drive + .execute_document_sum_request(sum_request, None, platform_version) + .expect("independent sum"); + let count = match count_resp { + DocumentCountResponse::Aggregate(c) => c, + other => panic!("expected count Aggregate, got {:?}", other), + }; + let sum = match sum_resp { + DocumentSumResponse::Aggregate(s) => s, + other => panic!("expected sum Aggregate, got {:?}", other), + }; + (count, sum) + } + + /// `execute_document_count_and_sum_total_no_proof` cross-check: + /// empty-where total on a doctype with `documents_summable + + /// documents_countable`. Goes through the primary-key fast path. + #[test] + fn joint_total_executor_matches_independent_count_plus_sum() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + + // The empty-where Total path requires the doctype's + // documents_summable + documents_countable to be set, but a + // covering `summable + countable` byColor index also works + // for the Equal-only-fully-covered sub-path. Use the latter + // since the test factory above doesn't easily produce + // doctype-level summable+countable. The Equal-only branch + // of execute_document_count_and_sum_total_no_proof still + // routes through `DocumentSumMode::Total` per sum's table. + let factory = DataContractFactory::new(PROTOCOL_VERSION_V12).expect("create factory"); + let document_schema = platform_value!({ + "type": "object", + "properties": { + "color": {"type": "string", "position": 0, "maxLength": 32}, + "amount": {"type": "integer", "position": 1, "minimum": 0, "maximum": 1000}, + }, + "required": ["color", "amount"], + "indices": [{ + "name": "byColor", + "properties": [{"color": "asc"}], + "summable": "amount", + "countable": "countable", + }], + "additionalProperties": false, + }); + let schemas = platform_value!({ "widget": document_schema }); + let data_contract = factory + .create_with_value_config( + dpp::tests::utils::generate_random_identifier_struct(), + 0, + schemas, + None, + None, + ) + .expect("create data contract") + .data_contract_owned(); + drive + .apply_contract( + &data_contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + + let docs = [ + ("red", 5u64), + ("red", 5), + ("red", 7), + ("green", 3), + ("green", 4), + ("blue", 1), + ]; + for (i, (color, amount)) in docs.iter().enumerate() { + insert_widget(&drive, &data_contract, i, color, *amount); + } + + let document_type = data_contract + .document_type_for_name("widget") + .expect("widget"); + let drive_config = DriveConfig::default(); + + // Aggregate, no where → empty-where Total path. The doctype + // doesn't declare documents_summable here so the executor + // fall-through is the picker path on the byColor index. But + // the empty-where branch requires documents_summable; if the + // doctype lacks it, the picker is invoked with empty where, + // which `find_summable_index_for_where_clauses` rejects + // (zero indexable fields). So we test Equal-only-fully- + // covered instead — same `DocumentSumMode::Total` + // resolution. + let where_clauses = vec![WhereClause { + field: "color".to_string(), + operator: WhereOperator::Equal, + value: Value::Text("red".to_string()), + }]; + + let request = DocumentAverageRequest { + contract: &data_contract, + document_type, + sum_property: "amount".to_string(), + where_clauses: where_clauses.clone(), + order_clauses: Vec::new(), + mode: AverageMode::Aggregate, + limit: None, + prove: false, + drive_config: &drive_config, + }; + + let joint_response = drive + .execute_document_average_request(request, None, platform_version) + .expect("joint total dispatch"); + let (joint_count, joint_sum) = match joint_response { + DocumentAverageResponse::Aggregate { count, sum } => (count, sum), + other => panic!("expected Aggregate, got {:?}", other), + }; + + let (indep_count, indep_sum) = independent_count_sum_aggregate( + &drive, + &data_contract, + document_type, + "amount", + where_clauses, + &drive_config, + platform_version, + ); + + assert_eq!( + (joint_count, joint_sum), + (indep_count, indep_sum), + "joint total executor must produce the same (count, sum) as \ + independent count + sum dispatch (red == 3 docs / sum 17)" + ); + // Sanity check against the fixture: red docs are 5+5+7 = 17 / count 3. + assert_eq!((joint_count, joint_sum), (3, 17)); + } + + /// `execute_document_count_and_sum_per_in_value_no_proof` + /// cross-check: In on a `summable + countable` index. + #[test] + fn joint_per_in_value_executor_matches_independent_count_plus_sum() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + + let factory = DataContractFactory::new(PROTOCOL_VERSION_V12).expect("create factory"); + let document_schema = platform_value!({ + "type": "object", + "properties": { + "color": {"type": "string", "position": 0, "maxLength": 32}, + "amount": {"type": "integer", "position": 1, "minimum": 0, "maximum": 1000}, + }, + "required": ["color", "amount"], + "indices": [{ + "name": "byColor", + "properties": [{"color": "asc"}], + "summable": "amount", + "countable": "countable", + }], + "additionalProperties": false, + }); + let schemas = platform_value!({ "widget": document_schema }); + let data_contract = factory + .create_with_value_config( + dpp::tests::utils::generate_random_identifier_struct(), + 0, + schemas, + None, + None, + ) + .expect("create data contract") + .data_contract_owned(); + drive + .apply_contract( + &data_contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + + let docs = [ + ("red", 5u64), + ("red", 7), + ("green", 3), + ("green", 4), + ("blue", 1), + ("blue", 2), + ]; + for (i, (color, amount)) in docs.iter().enumerate() { + insert_widget(&drive, &data_contract, i, color, *amount); + } + + let document_type = data_contract + .document_type_for_name("widget") + .expect("widget"); + let drive_config = DriveConfig::default(); + + let color_in = WhereClause { + field: "color".to_string(), + operator: WhereOperator::In, + value: Value::Array(vec![ + Value::Text("red".to_string()), + Value::Text("green".to_string()), + ]), + }; + + let request = DocumentAverageRequest { + contract: &data_contract, + document_type, + sum_property: "amount".to_string(), + where_clauses: vec![color_in.clone()], + order_clauses: Vec::new(), + mode: AverageMode::GroupByIn, + limit: None, + prove: false, + drive_config: &drive_config, + }; + + let joint_response = drive + .execute_document_average_request(request, None, platform_version) + .expect("joint per-in-value dispatch"); + let joint_entries = match joint_response { + DocumentAverageResponse::Entries(e) => e, + other => panic!("expected Entries, got {:?}", other), + }; + + // Cross-check via independent count + sum per-In dispatch. + let count_request = DocumentCountRequest { + contract: &data_contract, + document_type, + where_clauses: vec![color_in.clone()], + order_clauses: Vec::new(), + mode: CountMode::GroupByIn, + limit: None, + prove: false, + drive_config: &drive_config, + }; + let sum_request = DocumentSumRequest { + contract: &data_contract, + document_type, + sum_property: "amount".to_string(), + where_clauses: vec![color_in], + order_clauses: Vec::new(), + mode: SumMode::GroupByIn, + limit: None, + prove: false, + drive_config: &drive_config, + }; + let count_resp = drive + .execute_document_count_request(count_request, None, platform_version) + .expect("independent count"); + let sum_resp = drive + .execute_document_sum_request(sum_request, None, platform_version) + .expect("independent sum"); + let count_entries = match count_resp { + DocumentCountResponse::Entries(e) => e, + other => panic!("expected count Entries, got {:?}", other), + }; + let sum_entries = match sum_resp { + DocumentSumResponse::Entries(e) => e, + other => panic!("expected sum Entries, got {:?}", other), + }; + + // Zip by key and assert joint matches. + assert_eq!(joint_entries.len(), count_entries.len()); + assert_eq!(joint_entries.len(), sum_entries.len()); + for ((joint, count), sum) in joint_entries + .iter() + .zip(count_entries.iter()) + .zip(sum_entries.iter()) + { + assert_eq!(joint.key, count.key); + assert_eq!(joint.key, sum.key); + assert_eq!(joint.count, count.count); + assert_eq!(joint.sum, sum.sum); + } + // Two entries — red and green. + assert_eq!(joint_entries.len(), 2); + // red: 2 docs, sum = 12. + // green: 2 docs, sum = 7. + // BTreeMap orders by serialized key bytes (lex on string + // bytes since color is Text). "green" < "red" lex. + let mut by_key: Vec<&AverageEntry> = joint_entries.iter().collect(); + by_key.sort_by(|a, b| a.key.cmp(&b.key)); + let red_entry = by_key + .iter() + .find(|e| e.key.windows(3).any(|w| w == b"red")) + .expect("red entry"); + let green_entry = by_key + .iter() + .find(|e| e.key.windows(5).any(|w| w == b"green")) + .expect("green entry"); + assert_eq!(red_entry.count, Some(2)); + assert_eq!(red_entry.sum, Some(12)); + assert_eq!(green_entry.count, Some(2)); + assert_eq!(green_entry.sum, Some(7)); + } + + /// `execute_document_count_and_sum_range_no_proof` cross-check: + /// distinct GroupByRange on a `rangeAverageable` (PCPS) index. + #[test] + fn joint_range_no_proof_executor_matches_independent_count_plus_sum() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + let data_contract = build_widget_contract_pcps(); + drive + .apply_contract( + &data_contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + + let docs = [ + ("red", 5u64), + ("red", 7), + ("green", 3), + ("green", 4), + ("green", 6), + ("blue", 2), + ]; + for (i, (color, amount)) in docs.iter().enumerate() { + insert_widget(&drive, &data_contract, i, color, *amount); + } + + let document_type = data_contract + .document_type_for_name("widget") + .expect("widget"); + let drive_config = DriveConfig::default(); + + // `color > "blue"` on the byColor rangeAverageable index. + let color_gt_blue = WhereClause { + field: "color".to_string(), + operator: WhereOperator::GreaterThan, + value: Value::Text("blue".to_string()), + }; + + let request = DocumentAverageRequest { + contract: &data_contract, + document_type, + sum_property: "amount".to_string(), + where_clauses: vec![color_gt_blue.clone()], + order_clauses: Vec::new(), + mode: AverageMode::GroupByRange, + limit: None, + prove: false, + drive_config: &drive_config, + }; + + let joint_response = drive + .execute_document_average_request(request, None, platform_version) + .expect("joint range distinct dispatch"); + let joint_entries = match joint_response { + DocumentAverageResponse::Entries(e) => e, + other => panic!("expected Entries, got {:?}", other), + }; + + // Cross-check via independent count + sum distinct dispatch. + let count_request = DocumentCountRequest { + contract: &data_contract, + document_type, + where_clauses: vec![color_gt_blue.clone()], + order_clauses: Vec::new(), + mode: CountMode::GroupByRange, + limit: None, + prove: false, + drive_config: &drive_config, + }; + let sum_request = DocumentSumRequest { + contract: &data_contract, + document_type, + sum_property: "amount".to_string(), + where_clauses: vec![color_gt_blue], + order_clauses: Vec::new(), + mode: SumMode::GroupByRange, + limit: None, + prove: false, + drive_config: &drive_config, + }; + let count_resp = drive + .execute_document_count_request(count_request, None, platform_version) + .expect("independent count"); + let sum_resp = drive + .execute_document_sum_request(sum_request, None, platform_version) + .expect("independent sum"); + let count_entries = match count_resp { + DocumentCountResponse::Entries(e) => e, + other => panic!("expected count Entries, got {:?}", other), + }; + let sum_entries = match sum_resp { + DocumentSumResponse::Entries(e) => e, + other => panic!("expected sum Entries, got {:?}", other), + }; + + // Both executors emit per-distinct-key entries in ascending + // serialized-key order; the lengths must match and per-key + // (count, sum) must zip to the same values. + assert_eq!(joint_entries.len(), count_entries.len()); + assert_eq!(joint_entries.len(), sum_entries.len()); + for ((joint, count), sum) in joint_entries + .iter() + .zip(count_entries.iter()) + .zip(sum_entries.iter()) + { + assert_eq!(joint.key, count.key); + assert_eq!(joint.key, sum.key); + assert_eq!(joint.count, count.count); + assert_eq!(joint.sum, sum.sum); + } + // Two distinct keys (green, red); blue is filtered out by + // the range. green: 3 docs, sum=13; red: 2 docs, sum=12. + assert_eq!(joint_entries.len(), 2); + } + + /// Flat-summed range cross-check: `Aggregate + range` on a PCPS + /// index resolves to `DocumentSumMode::RangeNoProof` with + /// `return_distinct_sums_in_range = false`. The joint executor + /// folds visited PCPS elements via `count_sum_value_or_default()` + /// in Rust (no engine-side combined accumulator exists). Pin parity + /// vs. the independent count + sum aggregate dispatch — this is the + /// path where the issue's perf win lands. + #[test] + fn joint_range_aggregate_executor_matches_independent_count_plus_sum() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + let data_contract = build_widget_contract_pcps(); + drive + .apply_contract( + &data_contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + + let docs = [ + ("red", 5u64), + ("red", 7), + ("green", 3), + ("green", 4), + ("green", 6), + ("blue", 2), + ]; + for (i, (color, amount)) in docs.iter().enumerate() { + insert_widget(&drive, &data_contract, i, color, *amount); + } + + let document_type = data_contract + .document_type_for_name("widget") + .expect("widget"); + let drive_config = DriveConfig::default(); + + let color_gt_blue = WhereClause { + field: "color".to_string(), + operator: WhereOperator::GreaterThan, + value: Value::Text("blue".to_string()), + }; + + let request = DocumentAverageRequest { + contract: &data_contract, + document_type, + sum_property: "amount".to_string(), + where_clauses: vec![color_gt_blue.clone()], + order_clauses: Vec::new(), + mode: AverageMode::Aggregate, + limit: None, + prove: false, + drive_config: &drive_config, + }; + + let joint_response = drive + .execute_document_average_request(request, None, platform_version) + .expect("joint range aggregate dispatch"); + let (joint_count, joint_sum) = match joint_response { + DocumentAverageResponse::Aggregate { count, sum } => (count, sum), + other => panic!("expected Aggregate, got {:?}", other), + }; + + let (indep_count, indep_sum) = independent_count_sum_aggregate( + &drive, + &data_contract, + document_type, + "amount", + vec![color_gt_blue], + &drive_config, + platform_version, + ); + + assert_eq!( + (joint_count, joint_sum), + (indep_count, indep_sum), + "joint range-aggregate executor must produce the same (count, sum) \ + as independent count + sum range dispatch" + ); + // Sanity check: color > "blue" matches green (3,4,6 = sum 13) + // + red (5,7 = sum 12); total 5 docs / sum 25. + assert_eq!((joint_count, joint_sum), (5, 25)); + } + + /// Compound-summed range cross-check: `GroupByIn + In + range` on + /// a PCPS index resolves to `DocumentSumMode::RangeNoProof` with + /// `return_distinct_sums_in_range = false`. The joint executor's + /// distinct path query expresses the multi-In outer walk as a + /// single grovedb call (atomicity inherent) and folds each + /// In-branch's PCPS elements into one `(count, sum)` pair via + /// `count_sum_value_or_default()`. + /// + /// Pin parity vs. the independent count + sum dispatch. This is + /// the second untested-flat-summed branch the agent's three tests + /// don't cover. + #[test] + fn joint_range_group_by_in_executor_matches_independent_count_plus_sum() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + + // PCPS index keyed on (color, amount) so In on color + range + // on amount fits the rangeCountable + rangeSummable shape. + let factory = DataContractFactory::new(PROTOCOL_VERSION_V12).expect("create factory"); + let document_schema = platform_value!({ + "type": "object", + "properties": { + "color": {"type": "string", "position": 0, "maxLength": 32}, + "amount": {"type": "integer", "position": 1, "minimum": 0, "maximum": 1000}, + }, + "required": ["color", "amount"], + "indices": [{ + "name": "byColorAmount", + "properties": [{"color": "asc"}, {"amount": "asc"}], + "summable": "amount", + "rangeSummable": true, + "countable": "countable", + "rangeCountable": true, + }], + "additionalProperties": false, + }); + let schemas = platform_value!({ "widget": document_schema }); + let data_contract = factory + .create_with_value_config( + dpp::tests::utils::generate_random_identifier_struct(), + 0, + schemas, + None, + None, + ) + .expect("create data contract") + .data_contract_owned(); + drive + .apply_contract( + &data_contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + + let docs = [ + ("red", 5u64), + ("red", 7), + ("red", 9), + ("green", 3), + ("green", 4), + ("blue", 8), + ("blue", 9), + ]; + for (i, (color, amount)) in docs.iter().enumerate() { + insert_widget(&drive, &data_contract, i, color, *amount); + } + + let document_type = data_contract + .document_type_for_name("widget") + .expect("widget"); + let drive_config = DriveConfig::default(); + + // In on color (red, green) + range on amount (≥ 4). + let color_in = WhereClause { + field: "color".to_string(), + operator: WhereOperator::In, + value: Value::Array(vec![ + Value::Text("red".to_string()), + Value::Text("green".to_string()), + ]), + }; + let amount_ge_4 = WhereClause { + field: "amount".to_string(), + operator: WhereOperator::GreaterThanOrEquals, + value: Value::U64(4), + }; + + let request = DocumentAverageRequest { + contract: &data_contract, + document_type, + sum_property: "amount".to_string(), + where_clauses: vec![color_in.clone(), amount_ge_4.clone()], + order_clauses: Vec::new(), + mode: AverageMode::GroupByIn, + limit: None, + prove: false, + drive_config: &drive_config, + }; + + let joint_response = drive + .execute_document_average_request(request, None, platform_version) + .expect("joint range GroupByIn dispatch"); + let joint_entries = match joint_response { + DocumentAverageResponse::Entries(e) => e, + other => panic!("expected Entries, got {:?}", other), + }; + + // Independent count + sum GroupByIn dispatch. + let count_request = DocumentCountRequest { + contract: &data_contract, + document_type, + where_clauses: vec![color_in.clone(), amount_ge_4.clone()], + order_clauses: Vec::new(), + mode: CountMode::GroupByIn, + limit: None, + prove: false, + drive_config: &drive_config, + }; + let sum_request = DocumentSumRequest { + contract: &data_contract, + document_type, + sum_property: "amount".to_string(), + where_clauses: vec![color_in, amount_ge_4], + order_clauses: Vec::new(), + mode: SumMode::GroupByIn, + limit: None, + prove: false, + drive_config: &drive_config, + }; + let count_resp = drive + .execute_document_count_request(count_request, None, platform_version) + .expect("independent count"); + let sum_resp = drive + .execute_document_sum_request(sum_request, None, platform_version) + .expect("independent sum"); + let count_entries = match count_resp { + DocumentCountResponse::Entries(e) => e, + other => panic!("expected count Entries, got {:?}", other), + }; + let sum_entries = match sum_resp { + DocumentSumResponse::Entries(e) => e, + other => panic!("expected sum Entries, got {:?}", other), + }; + + // The independent count and sum dispatches both produce entries + // for every In branch (with `count`/`sum` reflecting the In + // branch's value); the joint executor must produce the same + // shape. Build a key-keyed map for each and assert pairwise + // equality on the (count, sum) pair. + use std::collections::BTreeMap; + let count_by_key: BTreeMap, Option> = count_entries + .iter() + .map(|e| (e.key.clone(), e.count)) + .collect(); + let sum_by_key: BTreeMap, Option> = + sum_entries.iter().map(|e| (e.key.clone(), e.sum)).collect(); + let joint_by_key: BTreeMap, (Option, Option)> = joint_entries + .iter() + .map(|e| (e.key.clone(), (e.count, e.sum))) + .collect(); + + assert_eq!( + count_by_key.keys().collect::>(), + joint_by_key.keys().collect::>(), + "joint executor must emit the same In-branch keys as independent count" + ); + for (key, (joint_count, joint_sum)) in joint_by_key.iter() { + assert_eq!(joint_count, count_by_key.get(key).unwrap()); + assert_eq!(joint_sum, sum_by_key.get(key).unwrap()); + } + } + + /// Distinct AVG no-proof MUST honor the request's `limit` — + /// `GroupByRange` over a wide range should truncate to the + /// caller's `limit` rather than enumerate every distinct in-range + /// terminator. Regression test for the joint dispatcher's + /// `RangeNoProof` distinct branch: prior to the P2 fix the + /// dispatcher hard-coded `None` into `distinct_sum_path_query`, + /// silently returning every matching key. + #[test] + fn distinct_avg_no_proof_honors_explicit_limit() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + let data_contract = build_widget_contract_pcps(); + drive + .apply_contract( + &data_contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + + // Five distinct color buckets so a `limit = 2` request must + // truncate the result set; otherwise the executor would emit + // all five. + let docs = [ + ("red", 5u64), + ("green", 7), + ("blue", 2), + ("yellow", 4), + ("purple", 9), + ]; + for (i, (color, amount)) in docs.iter().enumerate() { + insert_widget(&drive, &data_contract, i, color, *amount); + } + + let document_type = data_contract + .document_type_for_name("widget") + .expect("widget"); + let drive_config = DriveConfig::default(); + + let color_ge_a = WhereClause { + field: "color".to_string(), + operator: WhereOperator::GreaterThanOrEquals, + value: Value::Text("a".to_string()), + }; + + let request = DocumentAverageRequest { + contract: &data_contract, + document_type, + sum_property: "amount".to_string(), + where_clauses: vec![color_ge_a], + order_clauses: Vec::new(), + mode: AverageMode::GroupByRange, + limit: Some(2), + prove: false, + drive_config: &drive_config, + }; + + let response = drive + .execute_document_average_request(request, None, platform_version) + .expect("dispatcher should succeed"); + let entries = match response { + DocumentAverageResponse::Entries(e) => e, + other => panic!("expected Entries, got {:?}", other), + }; + assert_eq!( + entries.len(), + 2, + "distinct AVG no-proof must apply the request's `limit = 2` and \ + return exactly 2 entries; got {entries:?}" + ); + } + + /// Distinct AVG no-proof with `limit = None` must default to + /// `drive_config.default_query_limit`, not enumerate every + /// distinct key. Regression test for the same hard-coded `None` + /// the prior implementation passed. + #[test] + fn distinct_avg_no_proof_defaults_limit_to_operator_default_query_limit() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + let data_contract = build_widget_contract_pcps(); + drive + .apply_contract( + &data_contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + + // Five distinct buckets and an operator-tuned + // `default_query_limit = 3`. The dispatcher must honor the + // operator's runtime default on the no-proof path (this is + // explicitly documented as DIFFERENT from the prove path, + // which uses the compile-time constant for byte-stability of + // proof reconstruction). A regression that leaves limit as + // `None` would emit all 5 entries. + let docs = [ + ("red", 5u64), + ("green", 7), + ("blue", 2), + ("yellow", 4), + ("purple", 9), + ]; + for (i, (color, amount)) in docs.iter().enumerate() { + insert_widget(&drive, &data_contract, i, color, *amount); + } + + let document_type = data_contract + .document_type_for_name("widget") + .expect("widget"); + let drive_config = DriveConfig { + default_query_limit: 3, + ..Default::default() + }; + + let color_ge_a = WhereClause { + field: "color".to_string(), + operator: WhereOperator::GreaterThanOrEquals, + value: Value::Text("a".to_string()), + }; + let request = DocumentAverageRequest { + contract: &data_contract, + document_type, + sum_property: "amount".to_string(), + where_clauses: vec![color_ge_a], + order_clauses: Vec::new(), + mode: AverageMode::GroupByRange, + limit: None, + prove: false, + drive_config: &drive_config, + }; + + let response = drive + .execute_document_average_request(request, None, platform_version) + .expect("dispatcher should succeed"); + let entries = match response { + DocumentAverageResponse::Entries(e) => e, + other => panic!("expected Entries, got {:?}", other), + }; + assert_eq!( + entries.len(), + 3, + "distinct AVG no-proof with `limit = None` must default to \ + `drive_config.default_query_limit` (= 3 here) rather than \ + enumerating all 5 distinct keys; got {entries:?}" + ); + } + + /// Distinct AVG no-proof with `limit > max_query_limit` must + /// clamp to `max_query_limit`, not return an error. Mirrors + /// count's no-proof distinct-walk clamp policy (documented in + /// `DocumentAverageRequest::limit`). + #[test] + fn distinct_avg_no_proof_clamps_limit_to_max_query_limit() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + let data_contract = build_widget_contract_pcps(); + drive + .apply_contract( + &data_contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + + let docs = [ + ("red", 5u64), + ("green", 7), + ("blue", 2), + ("yellow", 4), + ("purple", 9), + ]; + for (i, (color, amount)) in docs.iter().enumerate() { + insert_widget(&drive, &data_contract, i, color, *amount); + } + + let document_type = data_contract + .document_type_for_name("widget") + .expect("widget"); + // Operator-tuned `max_query_limit = 2`. An explicit `limit = + // 4` MUST clamp to 2 (no-proof policy; the prove path errors + // on this combination instead — see the + // `range_distinct_avg_proof_rejects_limit_over_max` test + // above for the prove counterpart). + let drive_config = DriveConfig { + default_query_limit: 100, + max_query_limit: 2, + ..Default::default() + }; + + let color_ge_a = WhereClause { + field: "color".to_string(), + operator: WhereOperator::GreaterThanOrEquals, + value: Value::Text("a".to_string()), + }; + let request = DocumentAverageRequest { + contract: &data_contract, + document_type, + sum_property: "amount".to_string(), + where_clauses: vec![color_ge_a], + order_clauses: Vec::new(), + mode: AverageMode::GroupByRange, + limit: Some(4), + prove: false, + drive_config: &drive_config, + }; + + let response = drive + .execute_document_average_request(request, None, platform_version) + .expect("dispatcher should succeed (no-proof clamps, never errors)"); + let entries = match response { + DocumentAverageResponse::Entries(e) => e, + other => panic!("expected Entries, got {:?}", other), + }; + assert_eq!( + entries.len(), + 2, + "distinct AVG no-proof must clamp `limit = 4` to \ + `max_query_limit = 2`; got {entries:?}" + ); + } + + /// `execute_document_count_and_sum_request` must reject a direct + /// caller passing `prove = true`. The wrapper + /// `execute_document_average_request` is the only legitimate entry + /// that routes prove requests (to the prove-side dispatcher); + /// reaching the joint dispatcher with `prove = true` would + /// otherwise silently produce a no-proof response. Regression for + /// the CodeRabbit "enforce no-prove precondition" finding. + #[test] + fn joint_dispatcher_rejects_prove_true_request() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + let data_contract = build_widget_contract_pcps(); + drive + .apply_contract( + &data_contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + + let document_type = data_contract + .document_type_for_name("widget") + .expect("widget"); + let drive_config = DriveConfig::default(); + + let request = DocumentAverageRequest { + contract: &data_contract, + document_type, + sum_property: "amount".to_string(), + where_clauses: Vec::new(), + order_clauses: Vec::new(), + mode: AverageMode::Aggregate, + limit: None, + prove: true, + drive_config: &drive_config, + }; + + let err = drive + .execute_document_count_and_sum_request(request, None, platform_version) + .expect_err("prove=true direct call must reject"); + let msg = format!("{err:?}"); + assert!( + msg.contains("no-prove"), + "expected the prove=true guard to fire; got: {msg}" + ); + } + + /// AVG no-proof dispatcher must run + /// `validate_and_canonicalize_where_clauses` so it shares the same + /// accept/reject contract as the count and document-query + /// surfaces. Pin a representative rejection: a duplicate Equal on + /// the same field. Without the validator the executor would + /// either succeed with a silently-collapsed shape or fail + /// downstream with a less precise error. + #[test] + fn joint_dispatcher_runs_validate_and_canonicalize_where_clauses() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + let data_contract = build_widget_contract_pcps(); + drive + .apply_contract( + &data_contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + + let document_type = data_contract + .document_type_for_name("widget") + .expect("widget"); + let drive_config = DriveConfig::default(); + + // Duplicate Equal on `color` — validator rejects via + // `WhereClause::group_clauses`. + let dup_color_a = WhereClause { + field: "color".to_string(), + operator: WhereOperator::Equal, + value: Value::Text("red".to_string()), + }; + let dup_color_b = WhereClause { + field: "color".to_string(), + operator: WhereOperator::Equal, + value: Value::Text("green".to_string()), + }; + let request = DocumentAverageRequest { + contract: &data_contract, + document_type, + sum_property: "amount".to_string(), + where_clauses: vec![dup_color_a, dup_color_b], + order_clauses: Vec::new(), + mode: AverageMode::Aggregate, + limit: None, + prove: false, + drive_config: &drive_config, + }; + + let err = drive + .execute_document_average_request(request, None, platform_version) + .expect_err( + "AVG no-proof must reject duplicate Equal on the same field via \ + validate_and_canonicalize_where_clauses", + ); + // The exact error variant comes from `WhereClause::group_clauses` — + // pin only that the call returned `Err` and the error mentions + // the problematic shape rather than a generic index-picker miss. + let msg = format!("{err:?}"); + assert!( + !msg.contains("WhereClauseOnNonIndexedProperty"), + "validator should reject before the index picker would: {msg}" + ); + } + + /// `PerInValue` no-proof AVG must honor `request.limit` on the + /// returned entry list. Regression for the reviewer's "joint + /// dispatcher drops `request.limit`" finding on the PerInValue + /// arm. Count's per-In executor truncates at this same point. + #[test] + fn per_in_value_avg_no_proof_honors_explicit_limit() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + + // `byColor` index: `summable: "amount"` + `countable: + // "countable"`. No range flags — this is the no-range + // PerInValue shape. + let factory = DataContractFactory::new(PROTOCOL_VERSION_V12).expect("create factory"); + let document_schema = platform_value!({ + "type": "object", + "properties": { + "color": {"type": "string", "position": 0, "maxLength": 32}, + "amount": {"type": "integer", "position": 1, "minimum": 0, "maximum": 1000}, + }, + "required": ["color", "amount"], + "indices": [{ + "name": "byColor", + "properties": [{"color": "asc"}], + "summable": "amount", + "countable": "countable", + }], + "additionalProperties": false, + }); + let schemas = platform_value!({ "widget": document_schema }); + let data_contract = factory + .create_with_value_config( + dpp::tests::utils::generate_random_identifier_struct(), + 0, + schemas, + None, + None, + ) + .expect("create data contract") + .data_contract_owned(); + drive + .apply_contract( + &data_contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + + for (i, (color, amount)) in [("red", 5u64), ("green", 7), ("blue", 2), ("yellow", 4)] + .iter() + .enumerate() + { + insert_widget(&drive, &data_contract, i, color, *amount); + } + + let document_type = data_contract + .document_type_for_name("widget") + .expect("widget"); + let drive_config = DriveConfig::default(); + + // `In` over 4 color values, `limit = 2` — dispatcher must + // truncate the per-In entry list to 2. + let color_in = WhereClause { + field: "color".to_string(), + operator: WhereOperator::In, + value: Value::Array(vec![ + Value::Text("red".to_string()), + Value::Text("green".to_string()), + Value::Text("blue".to_string()), + Value::Text("yellow".to_string()), + ]), + }; + let request = DocumentAverageRequest { + contract: &data_contract, + document_type, + sum_property: "amount".to_string(), + where_clauses: vec![color_in], + order_clauses: Vec::new(), + mode: AverageMode::GroupByIn, + limit: Some(2), + prove: false, + drive_config: &drive_config, + }; + + let response = drive + .execute_document_average_request(request, None, platform_version) + .expect("dispatcher should succeed"); + let entries = match response { + DocumentAverageResponse::Entries(e) => e, + other => panic!("expected Entries, got {:?}", other), + }; + assert_eq!( + entries.len(), + 2, + "PerInValue AVG no-proof must apply request.limit = 2 to the per-In \ + entry list (caller asked for 4 In values, dispatcher must truncate); \ + got {entries:?}" + ); + } + + /// Empty-where `Aggregate` AVG MUST exercise the + /// [`Drive::execute_document_count_and_sum_total_no_proof`] + /// primary-key fast path when the doctype declares + /// `documentsAverageable` (= `documentsCountable: true + + /// documentsSummable: ""`). The fast path reads + /// `[contract_doc, contract_id, [1], doctype, 0]` — the PCPS + /// primary-key element — and decodes `(count, sum)` from it in one + /// grovedb call without any index. Consensus-critical: a regression + /// here would silently produce wrong `(count, sum)` for the + /// most-trafficked AVG shape (unfiltered total). + #[test] + fn empty_where_total_executor_uses_primary_key_count_sum_tree_fast_path() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + + // `documentsAverageable: "amount"` desugars to BOTH + // `documentsCountable: true` AND `documentsSummable: + // "amount"`, which is exactly what the empty-where fast path + // requires. No `indices` block — the fast path doesn't use + // an index, it reads the doctype's primary-key + // count-sum-bearing tree directly at `[..., doctype, 0]`. + let factory = DataContractFactory::new(PROTOCOL_VERSION_V12).expect("create factory"); + let document_schema = platform_value!({ + "type": "object", + "properties": { + "amount": {"type": "integer", "position": 0, "minimum": 0, "maximum": 1000}, + }, + "required": ["amount"], + "documentsAverageable": "amount", + "additionalProperties": false, + }); + let schemas = platform_value!({ "score": document_schema }); + let data_contract = factory + .create_with_value_config( + dpp::tests::utils::generate_random_identifier_struct(), + 0, + schemas, + None, + None, + ) + .expect("create data contract") + .data_contract_owned(); + drive + .apply_contract( + &data_contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + + // Insert documents directly (no need for the widget helper — + // this doctype has no color property). + let document_type = data_contract + .document_type_for_name("score") + .expect("score type"); + for (i, amount) in [10u64, 20, 30, 40].iter().enumerate() { + let mut properties = std::collections::BTreeMap::new(); + properties.insert("amount".to_string(), Value::U64(*amount)); + let document: Document = DocumentV0 { + id: Identifier::from([(i + 1) as u8; 32]), + owner_id: Identifier::from([0u8; 32]), + properties, + revision: None, + created_at: None, + updated_at: None, + transferred_at: None, + created_at_block_height: None, + updated_at_block_height: None, + transferred_at_block_height: None, + created_at_core_block_height: None, + updated_at_core_block_height: None, + transferred_at_core_block_height: None, + creator_id: None, + } + .into(); + let storage_flags = Some(std::borrow::Cow::Owned(StorageFlags::SingleEpoch(0))); + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo((&document, storage_flags)), + owner_id: None, + }, + contract: &data_contract, + document_type, + }, + false, + BlockInfo::default(), + true, + None, + platform_version, + None, + ) + .expect("insert score"); + } + + let drive_config = DriveConfig::default(); + let request = DocumentAverageRequest { + contract: &data_contract, + document_type, + sum_property: "amount".to_string(), + where_clauses: Vec::new(), + order_clauses: Vec::new(), + mode: AverageMode::Aggregate, + limit: None, + prove: false, + drive_config: &drive_config, + }; + + let response = drive + .execute_document_average_request(request, None, platform_version) + .expect("empty-where AVG no-proof must succeed via the primary-key fast path"); + match response { + DocumentAverageResponse::Aggregate { count, sum } => { + assert_eq!( + (count, sum), + (4, 100), + "primary-key count-sum tree fast path must return (4 docs, sum 10+20+30+40 = 100)" + ); + } + other => panic!("expected Aggregate, got {:?}", other), + } + } + + /// `PerInValue` no-proof AVG with `limit = None` must default to + /// `drive_config.default_query_limit` per + /// `DocumentAverageRequest::limit`'s documented contract. + /// Regression test paired with the explicit-limit case above; pins + /// the no-proof contract parity reviewers flagged after the + /// initial PerInValue fix landed. + #[test] + fn per_in_value_avg_no_proof_defaults_limit_to_operator_default_query_limit() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + + // Same `summable + countable` `byColor` index as + // `per_in_value_avg_no_proof_honors_explicit_limit`, but with + // `default_query_limit = 2` and `limit = None` on the request + // — the dispatcher must fall back to the operator's runtime + // default and truncate the per-In entry list to 2. + let factory = DataContractFactory::new(PROTOCOL_VERSION_V12).expect("create factory"); + let document_schema = platform_value!({ + "type": "object", + "properties": { + "color": {"type": "string", "position": 0, "maxLength": 32}, + "amount": {"type": "integer", "position": 1, "minimum": 0, "maximum": 1000}, + }, + "required": ["color", "amount"], + "indices": [{ + "name": "byColor", + "properties": [{"color": "asc"}], + "summable": "amount", + "countable": "countable", + }], + "additionalProperties": false, + }); + let schemas = platform_value!({ "widget": document_schema }); + let data_contract = factory + .create_with_value_config( + dpp::tests::utils::generate_random_identifier_struct(), + 0, + schemas, + None, + None, + ) + .expect("create data contract") + .data_contract_owned(); + drive + .apply_contract( + &data_contract, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_cow(), + None, + platform_version, + ) + .expect("apply contract"); + + for (i, (color, amount)) in [("red", 5u64), ("green", 7), ("blue", 2), ("yellow", 4)] + .iter() + .enumerate() + { + insert_widget(&drive, &data_contract, i, color, *amount); + } + + let document_type = data_contract + .document_type_for_name("widget") + .expect("widget"); + let drive_config = DriveConfig { + default_query_limit: 2, + ..Default::default() + }; + + let color_in = WhereClause { + field: "color".to_string(), + operator: WhereOperator::In, + value: Value::Array(vec![ + Value::Text("red".to_string()), + Value::Text("green".to_string()), + Value::Text("blue".to_string()), + Value::Text("yellow".to_string()), + ]), + }; + let request = DocumentAverageRequest { + contract: &data_contract, + document_type, + sum_property: "amount".to_string(), + where_clauses: vec![color_in], + order_clauses: Vec::new(), + mode: AverageMode::GroupByIn, + limit: None, + prove: false, + drive_config: &drive_config, + }; + + let response = drive + .execute_document_average_request(request, None, platform_version) + .expect("dispatcher should succeed"); + let entries = match response { + DocumentAverageResponse::Entries(e) => e, + other => panic!("expected Entries, got {:?}", other), + }; + assert_eq!( + entries.len(), + 2, + "PerInValue AVG no-proof with `limit = None` must default to \ + `drive_config.default_query_limit` (= 2 here) and truncate the \ + per-In entry list; got {entries:?}" + ); + } } diff --git a/packages/rs-drive/src/query/drive_document_average_query/mod.rs b/packages/rs-drive/src/query/drive_document_average_query/mod.rs index 203e603b9a..e7576ff249 100644 --- a/packages/rs-drive/src/query/drive_document_average_query/mod.rs +++ b/packages/rs-drive/src/query/drive_document_average_query/mod.rs @@ -15,15 +15,16 @@ //! [`book/src/drive/average-index-examples.md`](../../../../../book/src/drive/average-index-examples.md) //! for the design and the grades-contract worked example. //! -//! Wired end-to-end: the dispatcher composes the count + sum -//! executors on the no-proof path under a shared read-transaction -//! (see [`drive_dispatcher`] module docstring for the atomicity -//! contract), and the prove path dispatches directly to the PCPS / -//! primary-key proof executors. A planned follow-up tracked at -//! [dashpay/platform#3687](https://github.com/dashpay/platform/issues/3687) -//! will collapse the no-proof path's two-request composition into a -//! single unified executor that reads both metrics from each visited -//! PCPS element in one walk. +//! Wired end-to-end: the dispatcher routes prove-true requests to the +//! PCPS / primary-key proof executors, and prove-false requests to the +//! joint single-walk count-and-sum dispatcher at +//! [`crate::query::drive_document_count_and_sum_query`]. Both paths +//! read `(count, sum)` from each visited count-sum-bearing element in +//! a single grovedb walk — see the +//! [`drive_dispatcher`](drive_dispatcher) module docstring for the +//! routing details and +//! [`crate::query::drive_document_count_and_sum_query`] for the +//! no-prove perf / atomicity contract. #[cfg(feature = "server")] pub mod drive_dispatcher; diff --git a/packages/rs-drive/src/query/drive_document_count_and_sum_query/drive_dispatcher.rs b/packages/rs-drive/src/query/drive_document_count_and_sum_query/drive_dispatcher.rs new file mode 100644 index 0000000000..ee0eea9081 --- /dev/null +++ b/packages/rs-drive/src/query/drive_document_count_and_sum_query/drive_dispatcher.rs @@ -0,0 +1,309 @@ +//! Joint count-and-sum dispatcher entry point for the AVG no-prove +//! path. +//! +//! Consumes a [`DocumentAverageRequest`] with `prove = false` and +//! dispatches to one of three per-mode executors (`Total` / +//! `PerInValue` / `RangeNoProof`) based on sum's versioned mode- +//! detection table — see [the routing +//! table](crate::query::drive_document_sum_query::mode_detection) for +//! the per-shape contract. +//! +//! # Per-shape cost +//! +//! - `Total` / `PerInValue`: point-lookup walk against the index; +//! one grovedb read per In branch yielding `(count, sum)` together +//! via [`grovedb::Element::count_sum_value_or_default`]. +//! - `RangeNoProof` distinct (`GroupByRange` / `GroupByCompound` + +//! range): one grovedb walk against the +//! `ProvableCountProvableSumTree` terminators of +//! `distinct_sum_path_query`, bounded by the request's `limit` +//! (default `drive_config.default_query_limit`, capped at +//! `drive_config.max_query_limit`). +//! - `RangeNoProof` aggregate (`Aggregate` / `GroupByIn` + range): +//! grovedb's combined `query_aggregate_count_and_sum` accumulator +//! (O(log n), yielding `(u64, i64)` in one traversal). Compound +//! `In + range` per-In fans out (≤100 branches × 1 accumulator +//! call) under a shared read transaction. +//! +//! # Atomicity +//! +//! Executors that issue multiple grovedb reads (`PerInValue`, the +//! aggregate `RangeNoProof` branch, and its compound `In + range` +//! per-In fan-out) open a short-lived shared read transaction +//! internally when `transaction.is_none()` so the sub-reads see a +//! consistent grovedb snapshot. Executors that issue exactly one +//! read get atomicity for free. + +use super::super::drive_document_average_query::{ + AverageMode, DocumentAverageRequest, DocumentAverageResponse, +}; +use super::super::drive_document_count_query::drive_dispatcher::validate_and_canonicalize_where_clauses; +use super::super::drive_document_sum_query::mode_detection::detect_sum_mode_from_inputs; +use super::super::drive_document_sum_query::{DocumentSumMode, SumMode}; +use crate::drive::Drive; +use crate::error::Error; +use dpp::data_contract::accessors::v0::DataContractV0Getters; +use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; +use dpp::version::PlatformVersion; +use grovedb::TransactionArg; + +#[cfg(feature = "server")] +impl Drive { + /// Joint count-and-sum no-prove dispatcher. + /// + /// Consumes a [`DocumentAverageRequest`] with `prove = false` and + /// produces a [`DocumentAverageResponse`] in the appropriate shape + /// (`Aggregate { count, sum }` for non-grouped modes, + /// `Entries(Vec)` for grouped modes). + /// + /// The dispatcher returns `Proof(_)`-shape responses only when + /// invoked via [`Drive::execute_document_average_request`]'s + /// `prove = true` arm; the no-prove path here never produces proof + /// bytes. + /// + /// # Routing + /// + /// Reuses sum's versioned routing table via + /// [`detect_sum_mode_from_inputs`] with the request's + /// [`AverageMode`] converted 1:1 to [`SumMode`] (the two enums are + /// structurally identical — same four variants). `prove = false` + /// is passed unconditionally because the prove path never reaches + /// this function. The resolved [`DocumentSumMode`] is one of three + /// no-prove values: `Total`, `PerInValue`, or `RangeNoProof`; the + /// four prove-mode values are unreachable. The match arm therefore + /// returns `CorruptedCodeExecution` on those branches rather than + /// silently misbehaving — if the routing table is ever extended + /// with a new prove-only mode that accidentally fires here, we + /// want to fail loudly. + pub fn execute_document_count_and_sum_request( + &self, + request: DocumentAverageRequest, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result { + // No-prove precondition. Direct callers (i.e. anything reaching + // this method without going through + // `execute_document_average_request`'s prove/no-prove split) + // must not slip a `prove = true` request past — this dispatcher + // hard-routes with `prove = false` and would otherwise silently + // hand back a no-prove response to a caller that asked for a + // proof. Reject up front rather than honoring the routing + // mismatch. + if request.prove { + return Err(Error::Drive( + crate::error::drive::DriveError::CorruptedCodeExecution( + "execute_document_count_and_sum_request only serves the no-prove path. \ + For a proven AVG response use \ + `execute_document_average_request` (which routes prove=true to the \ + prove-side dispatcher).", + ), + )); + } + + // Validate + canonicalize the structured `where_clauses` — + // same rejections / canonicalization the count and document- + // query surfaces apply, kept here so the AVG no-prove surface + // shares the same accept/reject contract. Must run BEFORE + // `detect_sum_mode_from_inputs` because the canonicalizer + // collapses `[> A, < B]` pairs into a single `between*` + // clause whose presence changes the routing decision. See + // [`validate_and_canonicalize_where_clauses`]'s docstring for + // the catalog of rejections. + let where_clauses = validate_and_canonicalize_where_clauses(request.where_clauses)?; + + // Convert AverageMode → SumMode (1:1 by construction); sum's + // routing table is the single source of truth for the + // `(where_clauses × mode × prove=false)` triple. Forking a + // parallel "count_and_sum mode" table here is what #3687 + // explicitly warned against — PR #3661 caught one drift bug + // (count's `GroupByCompound` row had diverged from sum's), + // and the joint executor's existence is itself the reason a + // single source of truth is now mandatory. + let sum_mode = match request.mode { + AverageMode::Aggregate => SumMode::Aggregate, + AverageMode::GroupByIn => SumMode::GroupByIn, + AverageMode::GroupByRange => SumMode::GroupByRange, + AverageMode::GroupByCompound => SumMode::GroupByCompound, + }; + + let resolved_mode = + detect_sum_mode_from_inputs(&where_clauses, sum_mode, false, platform_version)?; + + let contract_id = request.contract.id().to_buffer(); + let document_type_name = request.document_type.name().to_string(); + let sum_property = request.sum_property; + let order_by_ascending = request + .order_clauses + .first() + .map(|c| c.ascending) + .unwrap_or(true); + + match resolved_mode { + DocumentSumMode::Total => self.execute_document_count_and_sum_total_no_proof( + contract_id, + request.document_type, + document_type_name, + where_clauses, + sum_property, + transaction, + platform_version, + ), + DocumentSumMode::PerInValue => { + // PerInValue's per-In fan-out is bounded by the + // `In::in_values()` 100-cap. The caller's `limit` + // applies to the returned entry list per + // [`DocumentAverageRequest::limit`]'s documented + // no-proof contract: unset → fall back to + // `drive_config.default_query_limit`, explicit > max + // clamps to `drive_config.max_query_limit`. Count's + // PerInValue arm hard-codes `MAX_LIMIT_AS_FAILSAFE` + // (which contradicts its own documented contract); + // the joint dispatcher honors the AVG contract + // instead. + let per_in_limit = request + .limit + .unwrap_or(request.drive_config.default_query_limit as u32) + .min(request.drive_config.max_query_limit as u32); + self.execute_document_count_and_sum_per_in_value_no_proof( + contract_id, + request.document_type, + document_type_name, + where_clauses, + sum_property, + order_by_ascending, + per_in_limit as u16, + transaction, + platform_version, + ) + } + DocumentSumMode::RangeNoProof => { + // Distinct flag set for the GroupByRange / + // GroupByCompound shapes (per-distinct-value entries); + // cleared for the Aggregate / GroupByIn shapes (single + // folded pair). + let return_distinct = matches!( + request.mode, + AverageMode::GroupByRange | AverageMode::GroupByCompound + ); + // Limit applies only to the distinct branches — the + // aggregate branches return a single collapsed pair + // bounded by grovedb's combined merk-internal + // accumulator (`query_aggregate_count_and_sum`) + // regardless of how many documents match. For distinct + // shapes, follow [`DocumentAverageRequest::limit`]'s + // documented no-proof contract: fall back to + // `drive_config.default_query_limit` when unset, clamp + // to `drive_config.max_query_limit` when over. Both + // `default_query_limit` and `max_query_limit` are + // `u16` in `DriveConfig`, so the `min()` keeps the + // result in `u16::MAX` regardless of caller input. + let limit_u16 = if return_distinct { + Some( + request + .limit + .unwrap_or(request.drive_config.default_query_limit as u32) + .min(request.drive_config.max_query_limit as u32) + as u16, + ) + } else { + None + }; + let response = self.execute_document_count_and_sum_range_no_proof( + contract_id, + request.document_type, + document_type_name, + where_clauses, + sum_property, + return_distinct, + order_by_ascending, + limit_u16, + transaction, + platform_version, + )?; + // Sum's dispatcher applies a second-stage shape on the + // RangeNoProof path: `Aggregate` mode collapses to a + // single value, every other mode (GroupByIn / + // GroupByRange / GroupByCompound) returns `Entries`. + // GroupByIn + range specifically returns + // `Entries(vec![one_entry])` with the In axis folded + // (sum's executor itself emits exactly one entry in + // that case — see its `execute_range_sum_no_proof` + // compound-summed branch). The joint executor returns + // `Aggregate { count, sum }` for the !distinct shape + // because it has both metrics on hand; we re-shape + // into `Entries(vec![one_entry])` here when the + // caller asked for a non-Aggregate mode so the wire + // shape matches what an independent sum + count + // GroupByIn dispatch would produce. + // Strict mode×shape pairing. Every legal combination + // is named explicitly; any other pairing surfaces as + // `CorruptedCodeExecution` rather than silently + // forwarding a wrong-shape response. A future executor + // change that emits, say, `Entries` from an + // `AverageMode::Aggregate` request would fail loudly + // here instead of leaking the wrong wire shape to the + // gRPC handler. + match (request.mode, response) { + (AverageMode::Aggregate, resp @ DocumentAverageResponse::Aggregate { .. }) => { + Ok(resp) + } + (AverageMode::GroupByIn, DocumentAverageResponse::Aggregate { count, sum }) => { + Ok(DocumentAverageResponse::Entries(vec![ + super::super::drive_document_average_query::AverageEntry { + in_key: None, + key: Vec::new(), + count: Some(count), + sum: Some(sum), + }, + ])) + } + ( + AverageMode::GroupByRange | AverageMode::GroupByCompound, + resp @ DocumentAverageResponse::Entries(_), + ) => Ok(resp), + (mode, _) => Err(Error::Drive( + crate::error::drive::DriveError::CorruptedCodeExecution(match mode { + AverageMode::Aggregate => { + "execute_document_count_and_sum_request: \ + RangeNoProof executor emitted a non-Aggregate \ + response for AverageMode::Aggregate — joint \ + range executor's shape contract violated" + } + AverageMode::GroupByIn => { + "execute_document_count_and_sum_request: \ + RangeNoProof executor emitted a non-Aggregate \ + response for AverageMode::GroupByIn — the \ + in-axis fold should yield a single (count, sum) \ + pair the dispatcher re-wraps as Entries" + } + AverageMode::GroupByRange | AverageMode::GroupByCompound => { + "execute_document_count_and_sum_request: \ + RangeNoProof executor emitted a non-Entries \ + response for a distinct grouped mode — joint \ + range executor's shape contract violated" + } + }), + )), + } + } + // The four prove-mode resolutions are unreachable here — + // the dispatcher passes `prove = false` to the routing + // table, and sum's v0 table has no row that maps a + // `prove=false` input to any of these four resolutions. + // Surface as `CorruptedCodeExecution` so a future routing- + // table change that breaks this assumption fails loudly + // rather than producing a runtime panic. + DocumentSumMode::RangeProof + | DocumentSumMode::RangeDistinctProof + | DocumentSumMode::PointLookupProof + | DocumentSumMode::RangeAggregateCarrierProof => Err(Error::Drive( + crate::error::drive::DriveError::CorruptedCodeExecution( + "execute_document_count_and_sum_request: sum's routing table \ + resolved a prove-only mode from a `prove = false` request — \ + this indicates the routing table has been extended without \ + updating the joint dispatcher's match arms", + ), + )), + } + } +} diff --git a/packages/rs-drive/src/query/drive_document_count_and_sum_query/executors/mod.rs b/packages/rs-drive/src/query/drive_document_count_and_sum_query/executors/mod.rs new file mode 100644 index 0000000000..8c5a4d11c2 --- /dev/null +++ b/packages/rs-drive/src/query/drive_document_count_and_sum_query/executors/mod.rs @@ -0,0 +1,21 @@ +//! Per-`DocumentSumMode` joint count-and-sum no-prove executors. +//! +//! Mirrors [`crate::query::drive_document_sum_query::executors`] +//! one-to-one for the no-prove subset of modes — the prove modes +//! (`RangeProof`, `RangeDistinctProof`, `PointLookupProof`, +//! `RangeAggregateCarrierProof`) are unreachable here because the AVG +//! dispatcher routes prove-true requests through +//! [`Drive::execute_document_average_prove`] before this module is +//! consulted. The three no-prove modes are: +//! +//! - [`total`] — `DocumentSumMode::Total` (empty-where + +//! `documents_summable + documents_countable` fast path AND +//! Equal/In-fully-covered point lookup). +//! - [`per_in_value`] — `DocumentSumMode::PerInValue` (Equal/In on a +//! non-range covered index, emitting one entry per In branch). +//! - [`range_no_proof`] — `DocumentSumMode::RangeNoProof` (range and +//! distinct shapes over a PCPS-eligible index). + +pub mod per_in_value; +pub mod range_no_proof; +pub mod total; diff --git a/packages/rs-drive/src/query/drive_document_count_and_sum_query/executors/per_in_value.rs b/packages/rs-drive/src/query/drive_document_count_and_sum_query/executors/per_in_value.rs new file mode 100644 index 0000000000..5a88ea33ec --- /dev/null +++ b/packages/rs-drive/src/query/drive_document_count_and_sum_query/executors/per_in_value.rs @@ -0,0 +1,210 @@ +//! Joint count-and-sum `PerInValue` executor for +//! [`DocumentSumMode::PerInValue`] dispatch on the AVG no-prove path. +//! +//! Mirrors [`crate::query::drive_document_sum_query::executors::per_in_value`] +//! with the same two substitutions documented in the sibling +//! [`super::total`] module — element decoding via +//! `count_sum_value_or_default()` and picker-filtering on +//! `idx.countable.is_countable()`. +//! +//! ## Atomicity +//! +//! The per-In fan-out issues one grovedb read per In branch. When the +//! caller didn't provide a transaction the executor opens a short- +//! lived shared read transaction internally and reuses it across every +//! per-branch read so the (count, sum) pair sees a single grovedb +//! snapshot. The compound-aggregate `range_no_proof` sibling does the +//! same on its per-In fan-out branch. + +use super::super::super::drive_document_average_query::{AverageEntry, DocumentAverageResponse}; +use super::super::super::drive_document_sum_query::index_picker::find_summable_index_for_where_clauses; +use super::super::super::drive_document_sum_query::DriveDocumentSumQuery; +use crate::drive::Drive; +use crate::error::query::QuerySyntaxError; +use crate::error::Error; +use crate::query::{WhereClause, WhereOperator}; +use dpp::data_contract::document_type::DocumentTypeRef; +use dpp::version::PlatformVersion; +use grovedb::query_result_type::{QueryResultElement, QueryResultType}; +use grovedb::TransactionArg; + +impl Drive { + /// Cartesian-forks the single `In` clause into one Equal-per-value + /// sub-query against a `summable + countable` index; reads + /// `(count, sum)` from each per-branch point-lookup walk in one + /// grovedb read each, then emits per-In-value + /// [`AverageEntry { in_key: None, key, count, sum }`] entries. + /// + /// `left_to_right` controls the entry-list ordering; `limit` + /// truncates the returned entries (callers compute it from + /// `DocumentAverageRequest::limit` with the documented no-proof + /// default/clamp policy). + /// + /// Returns [`DocumentAverageResponse::Entries`]. + #[allow(clippy::too_many_arguments)] + pub fn execute_document_count_and_sum_per_in_value_no_proof( + &self, + contract_id: [u8; 32], + document_type: DocumentTypeRef, + document_type_name: String, + where_clauses: Vec, + sum_property: String, + left_to_right: bool, + limit: u16, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result { + // Enforce exactly one `In` clause. The sum-side analog + // documents the silent-drop bug this guards against; same + // failure mode would apply here. + let in_clauses: Vec<&WhereClause> = where_clauses + .iter() + .filter(|wc| wc.operator == WhereOperator::In) + .collect(); + if in_clauses.len() != 1 { + return Err(Error::Query( + QuerySyntaxError::InvalidWhereClauseComponents( + "execute_document_count_and_sum_per_in_value_no_proof requires \ + exactly one `in` clause", + ), + )); + } + let in_clause = in_clauses[0].clone(); + let in_values = in_clause.in_values().into_data_with_error()??; + + let other_clauses: Vec = where_clauses + .iter() + .filter(|wc| wc.operator != WhereOperator::In) + .cloned() + .collect(); + + // Open a short-lived shared read transaction if the caller + // didn't provide one. Multiple per-In branches read grovedb + // separately; without a shared snapshot a concurrent block + // commit could slip between branches and produce a + // `(count, sum)` pair from inconsistent state. Read-only; the + // local transaction is dropped without commit at the end. + let local_tx; + let effective_transaction: TransactionArg = if transaction.is_some() { + transaction + } else { + local_tx = self.grove.start_transaction(); + Some(&local_tx) + }; + + use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; + use dpp::data_contract::document_type::methods::DocumentTypeV0Methods; + // BTreeMap deduplicates by canonical key bytes and orders + // ascending — matches sum-side analog. Storing the full + // (count, sum) pair per key rather than separate maps means + // we never need to zip post-walk; each grovedb read writes + // exactly one entry into the map. + let mut merged: std::collections::BTreeMap, (u64, i64)> = + std::collections::BTreeMap::new(); + for value in in_values.iter() { + let key_bytes = document_type.serialize_value_for_key( + in_clause.field.as_str(), + value, + platform_version, + )?; + if merged.contains_key(&key_bytes) { + continue; + } + + let mut clauses_for_value = other_clauses.clone(); + clauses_for_value.push(WhereClause { + field: in_clause.field.clone(), + operator: WhereOperator::Equal, + value: value.clone(), + }); + + let index = find_summable_index_for_where_clauses( + document_type.indexes(), + &clauses_for_value, + &sum_property, + ) + .filter(|idx| idx.countable.is_countable()) + .ok_or_else(|| { + Error::Query(QuerySyntaxError::WhereClauseOnNonIndexedProperty( + "average query requires an index that declares BOTH \ + `summable: \"\"` AND a countable terminator \ + (`countable: \"countable\"` or `\"countableAllowingOffset\"`) \ + matching the where clause fields" + .to_string(), + )) + })?; + + let sum_query = DriveDocumentSumQuery { + document_type, + contract_id, + document_type_name: document_type_name.clone(), + index, + where_clauses: clauses_for_value, + sum_property: sum_property.clone(), + }; + + let drive_version = &platform_version.drive; + let path_query = sum_query.point_lookup_sum_path_query(platform_version)?; + let mut drive_operations = vec![]; + let (results, _) = self.grove_get_path_query( + &path_query, + effective_transaction, + QueryResultType::QueryElementResultType, + &mut drive_operations, + drive_version, + )?; + + // For the per-In-value branch the inner walk visits at + // most one count-sum-bearing element (the Equal arm fully + // covered the index). Sum across the emitted elements to + // handle the boundary case where the picker accepts an + // index whose terminator is `In`-shaped in the same prop + // (rare for AVG since the dispatched In was lifted out, + // but cheap to do correctly). + let mut count_acc: u64 = 0; + let mut sum_acc: i64 = 0; + for elem in results.elements.iter() { + if let QueryResultElement::ElementResultItem(element) = elem { + let (c, s) = element.count_sum_value_or_default(); + count_acc = count_acc.checked_add(c).ok_or_else(|| { + Error::Query(QuerySyntaxError::Unsupported( + "per-In-value count-and-sum overflowed u64 on the count axis \ + when summing branch elements. Narrow the query or use \ + multiple queries and combine client-side." + .to_string(), + )) + })?; + sum_acc = sum_acc.checked_add(s).ok_or_else(|| { + Error::Query(QuerySyntaxError::Unsupported( + "per-In-value count-and-sum overflowed i64 on the sum axis \ + when summing branch elements. Narrow the query or use \ + multiple queries and combine client-side." + .to_string(), + )) + })?; + } + } + merged.insert(key_bytes, (count_acc, sum_acc)); + } + + let mut entries: Vec = merged + .into_iter() + .map(|(key, (count, sum))| AverageEntry { + in_key: None, + key, + count: Some(count), + sum: Some(sum), + }) + .collect(); + if !left_to_right { + entries.reverse(); + } + // Apply caller's `limit` AFTER ordering: the BTreeMap + // iteration was already ascending; an explicit descending + // order reverses the vec, then `truncate` caps from the + // front to honor the requested direction. Same shape as + // count's `execute_document_count_per_in_value_no_proof`. + entries.truncate(limit as usize); + Ok(DocumentAverageResponse::Entries(entries)) + } +} diff --git a/packages/rs-drive/src/query/drive_document_count_and_sum_query/executors/range_no_proof.rs b/packages/rs-drive/src/query/drive_document_count_and_sum_query/executors/range_no_proof.rs new file mode 100644 index 0000000000..70fbbf09ab --- /dev/null +++ b/packages/rs-drive/src/query/drive_document_count_and_sum_query/executors/range_no_proof.rs @@ -0,0 +1,360 @@ +//! Joint count-and-sum `RangeNoProof` executor for +//! [`DocumentSumMode::RangeNoProof`] dispatch on the AVG no-prove path. +//! +//! Two structurally different paths share this executor: +//! +//! ## Aggregate shapes (`Aggregate + range`, `GroupByIn + range`) +//! +//! One `grove.query_aggregate_count_and_sum` call against the index's +//! `aggregate_count_and_sum_path_query` — a single merk-internal +//! `(u128, i128)` accumulator (narrowed to `(u64, i64)` at the +//! grovedb entry point) yielding both metrics in one O(log n) +//! traversal. Bounded **regardless of how many documents the range +//! matches**, so the public DAPI surface stays closed against +//! amplification. +//! +//! For compound `(In + range)` (with `In` on a prefix property) the +//! aggregate primitive can't fork through an `In`; the executor +//! per-In fans out (≤100 branches per the `In::in_values()` validator +//! cap) and issues one combined accumulator call per branch under a +//! shared read transaction. Worst-case 100 merk-internal reads per +//! request, again independent of matched-document count. +//! +//! ## Distinct shapes (`GroupByRange + range`, `GroupByCompound + range`) +//! +//! Walk PCPS terminator elements via +//! [`DriveDocumentSumQuery::distinct_sum_path_query`] in one +//! `grove_get_raw_path_query` call — the same shape sum's distinct +//! branch uses — and decode each via +//! [`grovedb::Element::count_sum_value_or_default`] to populate +//! [`AverageEntry`] with both `count` and `sum`. One walk yields both +//! axes per visited element instead of two parallel walks zipped +//! post-hoc. The distinct walk is bounded by the request's `limit` +//! (default falls back to `drive_config.default_query_limit`, +//! explicit limits are clamped to `drive_config.max_query_limit`) so +//! the public-endpoint amplification surface stays closed on this +//! path too. + +use super::super::super::drive_document_average_query::{AverageEntry, DocumentAverageResponse}; +use super::super::super::drive_document_sum_query::index_picker::find_range_summable_index_for_where_clauses; +use super::super::super::drive_document_sum_query::DriveDocumentSumQuery; +use crate::drive::Drive; +use crate::error::query::QuerySyntaxError; +use crate::error::Error; +use crate::query::{WhereClause, WhereOperator}; +use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; +use dpp::data_contract::document_type::methods::DocumentTypeV0Methods; +use dpp::data_contract::document_type::DocumentTypeRef; +use dpp::version::PlatformVersion; +use grovedb::query_result_type::QueryResultType; +use grovedb::TransactionArg; +use grovedb_costs::CostContext; + +impl Drive { + /// Range-aware joint count-and-sum walk against a + /// `rangeSummable + rangeCountable` (PCPS-eligible) index. + /// + /// Returns either a one-pair [`DocumentAverageResponse::Aggregate`] + /// (flat / compound aggregate shapes) or per-distinct-value + /// [`DocumentAverageResponse::Entries`] (`GroupByRange` / + /// `GroupByCompound`) depending on `return_distinct`. The dispatcher + /// sets that flag based on the request's + /// [`super::super::super::drive_document_average_query::AverageMode`]. + /// + /// `limit` applies only to the distinct branch; the aggregate + /// branches return a single collapsed pair regardless. See the + /// module docstring for the per-shape cost contract. + #[allow(clippy::too_many_arguments)] + pub fn execute_document_count_and_sum_range_no_proof( + &self, + contract_id: [u8; 32], + document_type: DocumentTypeRef, + document_type_name: String, + where_clauses: Vec, + sum_property: String, + return_distinct: bool, + left_to_right: bool, + limit: Option, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result { + let index = find_range_summable_index_for_where_clauses( + document_type.indexes(), + &where_clauses, + &sum_property, + ) + .filter(|idx| idx.range_countable) + .ok_or_else(|| { + Error::Query(QuerySyntaxError::WhereClauseOnNonIndexedProperty( + "average range query requires an index that declares BOTH \ + `rangeSummable: true` AND `rangeCountable: true` (a \ + `rangeAverageable: true` index is the shorthand) whose last \ + property matches the range field" + .to_string(), + )) + })?; + + let drive_version = &platform_version.drive; + let has_in_on_prefix = where_clauses + .iter() + .any(|wc| wc.operator == WhereOperator::In); + + if !return_distinct { + // Aggregate shape: one combined merk-internal accumulator + // call (`query_aggregate_count_and_sum`) yielding + // `(u64, i64)` in O(log n) — strictly bounded regardless + // of how many documents the range matches. Compound + // `In + range` per-In fans out to one accumulator call + // per branch under a shared read transaction (see + // `aggregate_range_count_and_sum` below). + return self.aggregate_range_count_and_sum( + contract_id, + document_type, + document_type_name, + index, + where_clauses, + sum_property, + has_in_on_prefix, + transaction, + platform_version, + ); + } + + // Distinct shape: walk PCPS terminator elements via the + // distinct path query, bounded by the caller's `limit`. + let sum_query = DriveDocumentSumQuery { + document_type, + contract_id, + document_type_name, + index, + where_clauses, + sum_property, + }; + let path_query = + sum_query.distinct_sum_path_query(limit, left_to_right, platform_version)?; + let base_path_len = path_query.path.len(); + + let mut drive_operations = vec![]; + let result = self.grove_get_raw_path_query( + &path_query, + transaction, + QueryResultType::QueryPathKeyElementTrioResultType, + &mut drive_operations, + drive_version, + ); + let elements = match result { + Ok((elements, _)) => elements, + Err(Error::GroveDB(e)) + if matches!( + e.as_ref(), + grovedb::Error::PathNotFound(_) + | grovedb::Error::PathParentLayerNotFound(_) + | grovedb::Error::PathKeyNotFound(_) + ) => + { + return Ok(DocumentAverageResponse::Entries(Vec::new())); + } + Err(e) => return Err(e), + }; + + let mut entries: Vec = Vec::new(); + for triple in elements.to_path_key_elements() { + let (path, key, element) = triple; + let (count, sum) = element.count_sum_value_or_default(); + // Drop fully-empty rows only. Sum's standalone distinct + // executor drops on `sum == 0` regardless of count, but + // that's overly aggressive for AVG: a row with + // `count = N, sum = 0` is informative — it means the + // group has N documents whose averageable values sum to + // zero (e.g. all zero, or signed values that cancel). The + // joint executor preserves such rows and only drops + // grovedb-absent rows that decode as `(0, 0)`. Diverges + // intentionally from sum's drop predicate. + if count == 0 && sum == 0 { + continue; + } + let in_key = if has_in_on_prefix && path.len() > base_path_len { + Some(path[base_path_len].clone()) + } else { + None + }; + entries.push(AverageEntry { + in_key, + key, + count: Some(count), + sum: Some(sum), + }); + } + + Ok(DocumentAverageResponse::Entries(entries)) + } + + /// Aggregate-range branch: returns `(count, sum)` via grovedb's + /// combined `query_aggregate_count_and_sum` accumulator. One call + /// for the flat shape; compound `In + range` per-In fans out and + /// sums per-branch totals (one accumulator call per branch). All + /// branches share a read transaction (opened internally when the + /// caller didn't supply one) so per-In sub-reads see the same + /// grovedb snapshot. + #[allow(clippy::too_many_arguments)] + fn aggregate_range_count_and_sum( + &self, + contract_id: [u8; 32], + document_type: DocumentTypeRef, + document_type_name: String, + index: &dpp::data_contract::document_type::Index, + where_clauses: Vec, + sum_property: String, + has_in_on_prefix: bool, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result { + let drive_version = &platform_version.drive; + + // Open a shared read transaction across per-In branches in + // the compound shape so each branch's accumulator call sees + // the same grovedb snapshot. The flat path issues a single + // read and gets atomicity for free; the transaction is + // harmless there. Read-only; dropped without commit at scope + // end. + let local_tx; + let effective_transaction: TransactionArg = if transaction.is_some() { + transaction + } else { + local_tx = self.grove.start_transaction(); + Some(&local_tx) + }; + + if !has_in_on_prefix { + let (count, sum) = self.flat_aggregate_count_and_sum( + contract_id, + document_type, + document_type_name, + index, + where_clauses, + sum_property, + effective_transaction, + drive_version, + platform_version, + )?; + return Ok(DocumentAverageResponse::Aggregate { count, sum }); + } + + // Compound: per-In fan-out with the In replaced by Equal-per- + // value. Exactly one In clause allowed — sum's existing + // executor documents the silent-drop bug this guards. + let in_clauses: Vec<&WhereClause> = where_clauses + .iter() + .filter(|wc| wc.operator == WhereOperator::In) + .collect(); + if in_clauses.len() != 1 { + return Err(Error::Query( + QuerySyntaxError::InvalidWhereClauseComponents( + "compound range count-and-sum requires exactly one `in` clause", + ), + )); + } + let in_clause = in_clauses[0]; + let in_values = in_clause.in_values().into_data_with_error()??; + let other_clauses: Vec = where_clauses + .iter() + .filter(|wc| wc.operator != WhereOperator::In) + .cloned() + .collect(); + + let mut count_total: u64 = 0; + let mut sum_total: i64 = 0; + let mut seen_keys: std::collections::BTreeSet> = std::collections::BTreeSet::new(); + for value in in_values.iter() { + // Dedupe by canonical serialized bytes — DPP `in_values()` + // already rejects raw-Value duplicates, but defense-in- + // depth for future Value variants that serialize identically. + let key_bytes = document_type.serialize_value_for_key( + in_clause.field.as_str(), + value, + platform_version, + )?; + if !seen_keys.insert(key_bytes) { + continue; + } + + let mut clauses_for_value = other_clauses.clone(); + clauses_for_value.push(WhereClause { + field: in_clause.field.clone(), + operator: WhereOperator::Equal, + value: value.clone(), + }); + + let (branch_count, branch_sum) = self.flat_aggregate_count_and_sum( + contract_id, + document_type, + document_type_name.clone(), + index, + clauses_for_value, + sum_property.clone(), + effective_transaction, + drive_version, + platform_version, + )?; + // `checked_add` rather than `saturating_add` so an + // overflow fails deterministically on both axes — + // matches the pattern in the sibling `total` / + // `per_in_value` executors. + count_total = count_total.checked_add(branch_count).ok_or_else(|| { + Error::Query(QuerySyntaxError::Unsupported( + "compound In-on-prefix range count-and-sum overflowed u64 on the count \ + axis when summing per-In aggregates. Narrow the query (smaller In set \ + or narrower range) or use multiple queries and combine client-side." + .to_string(), + )) + })?; + sum_total = sum_total.checked_add(branch_sum).ok_or_else(|| { + Error::Query(QuerySyntaxError::Unsupported( + "compound In-on-prefix range count-and-sum overflowed i64 on the sum \ + axis when summing per-In aggregates. Narrow the query (smaller In set \ + or narrower range) or use multiple queries and combine client-side." + .to_string(), + )) + })?; + } + + Ok(DocumentAverageResponse::Aggregate { + count: count_total, + sum: sum_total, + }) + } + + /// Flat (no In on prefix) aggregate count + sum: one + /// `query_aggregate_count_and_sum` call against the PCPS path + /// query — a single O(log n) merk-internal accumulator yielding + /// both metrics from one traversal. + #[allow(clippy::too_many_arguments)] + fn flat_aggregate_count_and_sum( + &self, + contract_id: [u8; 32], + document_type: DocumentTypeRef, + document_type_name: String, + index: &dpp::data_contract::document_type::Index, + where_clauses: Vec, + sum_property: String, + transaction: TransactionArg, + drive_version: &dpp::version::drive_versions::DriveVersion, + platform_version: &PlatformVersion, + ) -> Result<(u64, i64), Error> { + let sum_query = DriveDocumentSumQuery { + document_type, + contract_id, + document_type_name, + index, + where_clauses, + sum_property, + }; + let path_query = sum_query.aggregate_count_and_sum_path_query(platform_version)?; + let CostContext { value, cost: _ } = self.grove.query_aggregate_count_and_sum( + &path_query, + transaction, + &drive_version.grove_version, + ); + value.map_err(|e| Error::GroveDB(Box::new(e))) + } +} diff --git a/packages/rs-drive/src/query/drive_document_count_and_sum_query/executors/total.rs b/packages/rs-drive/src/query/drive_document_count_and_sum_query/executors/total.rs new file mode 100644 index 0000000000..1ea08baea6 --- /dev/null +++ b/packages/rs-drive/src/query/drive_document_count_and_sum_query/executors/total.rs @@ -0,0 +1,218 @@ +//! Joint count-and-sum Total executor for [`DocumentSumMode::Total`] +//! dispatch on the AVG no-prove path. +//! +//! Mirrors [`crate::query::drive_document_sum_query::executors::total`] +//! one-to-one with two substitutions: +//! +//! 1. **Element decoding**: the sum-side executor reads +//! [`Element::sum_value_or_default()`]; this one reads +//! [`Element::count_sum_value_or_default()`] so a single visited +//! element yields both metrics. The terminator's value tree is a +//! `CountSumTree` / `ProvableCountSumTree` / +//! `ProvableCountProvableSumTree` (the count-sum-bearing family), +//! so the joint decoder is well-defined on every shape the picker +//! accepts. +//! 2. **Index selection**: the picker is +//! [`find_summable_index_for_where_clauses`] (sum's), but with an +//! additional `.filter(|idx| idx.countable.is_countable())` so the +//! chosen index also carries the `countable` declaration the count +//! side needs. The AVG prove path's point-lookup arm does the same +//! filter (see `drive_document_average_query::drive_dispatcher:: +//! execute_document_average_prove`'s no-range arm). +//! +//! Routing semantics match sum's: this executor handles both the +//! empty-where case (doctype's `documents_summable + documents_countable` +//! primary-key fast path) AND the Equal-only fully-covered Equal/In +//! point-lookup case. The branch on `where_clauses.is_empty()` inside +//! the executor body is the same one sum's `total.rs` carries. + +use super::super::super::drive_document_average_query::DocumentAverageResponse; +use super::super::super::drive_document_sum_query::index_picker::find_summable_index_for_where_clauses; +use super::super::super::drive_document_sum_query::DriveDocumentSumQuery; +use crate::drive::Drive; +use crate::error::query::QuerySyntaxError; +use crate::error::Error; +use crate::query::WhereClause; +use dpp::data_contract::document_type::DocumentTypeRef; +use dpp::version::PlatformVersion; +use grovedb::query_result_type::{QueryResultElement, QueryResultType}; +use grovedb::TransactionArg; + +impl Drive { + /// Joint count-and-sum total for the empty-where / + /// Equal-only-fully-covered point-lookup shape — the AVG no-prove + /// analog of [`Drive::execute_document_sum_total_no_proof`]. + /// + /// Returns [`DocumentAverageResponse::Aggregate { count, sum }`] + /// directly (collapsed to a single pair across all matched + /// documents). The PerInValue branch is the sibling executor that + /// emits one entry per In branch. + #[allow(clippy::too_many_arguments)] + pub fn execute_document_count_and_sum_total_no_proof( + &self, + contract_id: [u8; 32], + document_type: DocumentTypeRef, + document_type_name: String, + where_clauses: Vec, + sum_property: String, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result { + use dpp::data_contract::document_type::accessors::{ + DocumentTypeV0Getters, DocumentTypeV2Getters, + }; + + // Empty-where fast path: unfiltered (count, sum) on a doctype + // that declares BOTH `documents_summable: Some(matching_prop)` + // AND `documents_countable: true` reads the primary-key + // count-sum-bearing tree directly (O(1)). Mirrors the sum-side + // analog with the additional `documents_countable` predicate; + // the AVG prove path's empty-where fast path enforces the same + // pair. + if where_clauses.is_empty() + && document_type.documents_countable() + && document_type + .documents_summable() + .map(|p| p == sum_property) + .unwrap_or(false) + { + let (count, sum) = self.read_primary_key_count_sum_tree( + &contract_id, + &document_type_name, + transaction, + platform_version, + )?; + return Ok(DocumentAverageResponse::Aggregate { count, sum }); + } + + let index = find_summable_index_for_where_clauses( + document_type.indexes(), + &where_clauses, + &sum_property, + ) + .filter(|idx| idx.countable.is_countable()) + .ok_or_else(|| { + Error::Query(QuerySyntaxError::WhereClauseOnNonIndexedProperty( + "average query requires an index that declares BOTH \ + `summable: \"\"` AND a countable terminator \ + (`countable: \"countable\"` or `\"countableAllowingOffset\"`) \ + whose properties exactly match the where clause fields, \ + OR `documentsSummable: \"\"` AND `documentsCountable: true` \ + on the document type for the unfiltered total case" + .to_string(), + )) + })?; + let sum_query = DriveDocumentSumQuery { + document_type, + contract_id, + document_type_name, + index, + where_clauses, + sum_property, + }; + + // Reuse sum's `point_lookup_sum_path_query` builder — the path + // query shape is identical for count-and-sum point lookups; the + // only difference is that we decode each emitted element via + // `count_sum_value_or_default()` rather than + // `sum_value_or_default()`. The terminator's value tree on a + // `summable + countable` index is a `CountSumTree` / + // `ProvableCountSumTree`, so both fields are present on every + // emitted element. + let drive_version = &platform_version.drive; + let path_query = sum_query.point_lookup_sum_path_query(platform_version)?; + let mut drive_operations = vec![]; + let (results, _) = self.grove_get_path_query( + &path_query, + transaction, + QueryResultType::QueryElementResultType, + &mut drive_operations, + drive_version, + )?; + + // Fold across emitted count-sum-bearing elements: + // - Equal-only: 0 or 1 element (0 when the branch is absent). + // - In at any position: one element per In branch that has at + // least one doc; missing branches contribute (0, 0). + // + // `checked_add` rather than `saturating_add` / wrapping add so + // an overflowed aggregate fails deterministically with a typed + // query error rather than silently clamping. The sum-side + // executor uses the same pattern for its i64 fold; we + // additionally guard u64 overflow on the count axis. + let mut count_acc: u64 = 0; + let mut sum_acc: i64 = 0; + for elem in results.elements.iter() { + if let QueryResultElement::ElementResultItem(element) = elem { + let (c, s) = element.count_sum_value_or_default(); + count_acc = count_acc.checked_add(c).ok_or_else(|| { + Error::Query(QuerySyntaxError::Unsupported( + "point-lookup count-and-sum overflowed u64 on the count axis \ + when summing per-In branches. Narrow the query (smaller In set) \ + or use multiple queries and combine client-side." + .to_string(), + )) + })?; + sum_acc = sum_acc.checked_add(s).ok_or_else(|| { + Error::Query(QuerySyntaxError::Unsupported( + "point-lookup count-and-sum overflowed i64 on the sum axis \ + when summing per-In branches. Narrow the query (smaller In set) \ + or use multiple queries and combine client-side." + .to_string(), + )) + })?; + } + } + Ok(DocumentAverageResponse::Aggregate { + count: count_acc, + sum: sum_acc, + }) + } + + /// Reads the document-type primary-key tree's count-sum-bearing + /// element (`[contract_doc, contract_id, [1], doctype, 0]`) and + /// returns `count_sum_value_or_default()`. Used by the + /// `documents_summable + documents_countable` empty-where fast path + /// on the joint count-and-sum flow. + /// + /// `insert_contract_operations_v0` unconditionally creates a + /// count-sum-bearing tree at `[..., doctype, 0]` for every applied + /// document type whose `documents_summable` is set AND whose + /// `documents_countable` is true, so a missing element here + /// indicates contract-state corruption or a mis-applied contract — + /// fail fast rather than silently returning `(0, 0)`. Mirrors + /// `read_primary_key_sum_tree`'s contract. + pub(super) fn read_primary_key_count_sum_tree( + &self, + contract_id: &[u8; 32], + document_type_name: &str, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(u64, i64), Error> { + let drive_version = &platform_version.drive; + let path = [ + &[crate::drive::RootTree::DataContractDocuments as u8] as &[u8], + contract_id, + &[1u8], + document_type_name.as_bytes(), + ]; + let mut drive_operations = vec![]; + let element = self + .grove_get_raw_optional( + grovedb_path::SubtreePath::from(path.as_slice()), + &[0], + crate::util::grove_operations::DirectQueryType::StatefulDirectQuery, + transaction, + &mut drive_operations, + drive_version, + )? + .ok_or_else(|| { + Error::Drive(crate::error::drive::DriveError::CorruptedCodeExecution( + "missing primary-key count-sum tree for an applied document type — \ + insert_contract_operations_v0 must have created it when both \ + documents_summable and documents_countable are set", + )) + })?; + Ok(element.count_sum_value_or_default()) + } +} diff --git a/packages/rs-drive/src/query/drive_document_count_and_sum_query/mod.rs b/packages/rs-drive/src/query/drive_document_count_and_sum_query/mod.rs new file mode 100644 index 0000000000..78ae8def49 --- /dev/null +++ b/packages/rs-drive/src/query/drive_document_count_and_sum_query/mod.rs @@ -0,0 +1,60 @@ +//! Joint count-and-sum executor surface for the AVG no-prove path. +//! +//! [`Drive::execute_document_count_and_sum_request`] consumes the same +//! [`DocumentAverageRequest`] / [`DocumentAverageResponse`] pair used +//! by the prove path and dispatches to one of three per-mode executors +//! depending on the resolved [`DocumentSumMode`]. +//! +//! ## Per-shape execution +//! +//! - **`Total` / `PerInValue`** (empty-where, or Equal/`In` on a +//! `summable + countable` index): point-lookup walk against the +//! index, decoding `(count, sum)` from each visited `CountSumTree` / +//! `ProvableCountSumTree` terminator in one call via +//! [`Element::count_sum_value_or_default`]. One grovedb read per +//! In branch yields both metrics together. +//! - **`RangeNoProof` distinct shapes** (`GroupByRange` / +//! `GroupByCompound` + range on a `rangeAverageable` index): one +//! grovedb walk against +//! [`crate::query::drive_document_sum_query::DriveDocumentSumQuery::distinct_sum_path_query`]'s +//! `ProvableCountProvableSumTree` terminators, emitting one +//! `(count, sum)` per visited distinct in-range key. Bounded by the +//! request's `limit` (default falls back to +//! `drive_config.default_query_limit`, explicit limits are clamped to +//! `drive_config.max_query_limit`). +//! - **`RangeNoProof` aggregate shapes** (`Aggregate` / `GroupByIn` + +//! range): grovedb's combined merk-internal accumulator — +//! `query_aggregate_count_and_sum` against the PCPS path query — +//! yielding `(u64, i64)` from a single O(log n) traversal. Compound +//! `In + range` per-In fans out (≤100 branches per the +//! `In::in_values()` validator cap) and issues one combined +//! accumulator call per branch under a shared read transaction. +//! +//! ## Routing +//! +//! Mode resolution reuses sum's versioned routing table +//! ([`crate::query::drive_document_sum_query::mode_detection::detect_sum_mode_from_inputs`]) +//! — the routing decision is consensus-relevant and identical for +//! count, sum, and joint count+sum on the no-prove path, so forking +//! the table here would invite the drift bug PR #3661 caught (count's +//! `GroupByCompound` row had diverged from sum's). A trivial +//! [`AverageMode`] → [`SumMode`] adapter feeds sum's mode detector; +//! `prove = false` is passed unconditionally because the prove path +//! never enters this module. +//! +//! ## Engine-side primitive note +//! +//! grovedb exposes `query_aggregate_count_and_sum` as the combined +//! no-prove accumulator (proof-side analog +//! `AggregateCountAndSumOnRange` is what the prove path uses for the +//! same shape). The aggregate `RangeNoProof` branch routes through it +//! directly — one merk-internal `(u128, i128)` walk per request, +//! narrowed to `(u64, i64)` at the grovedb entry. Same cost class +//! and the same shape the prove path uses, just without the proof +//! envelope. + +#[cfg(feature = "server")] +pub mod drive_dispatcher; + +#[cfg(feature = "server")] +pub mod executors; diff --git a/packages/rs-drive/src/query/mod.rs b/packages/rs-drive/src/query/mod.rs index e250dd6be6..21895f8e16 100644 --- a/packages/rs-drive/src/query/mod.rs +++ b/packages/rs-drive/src/query/mod.rs @@ -222,6 +222,13 @@ pub mod drive_document_sum_query; #[cfg(any(feature = "server", feature = "verify"))] pub mod drive_document_average_query; +/// Joint count-and-sum no-prove executor surface — backs the AVG +/// no-prove path's unified single-walk dispatch. See its module +/// docstring for the perf / atomicity contract. Server-only because +/// the surface only fires on the no-prove (server-materialized) path. +#[cfg(feature = "server")] +pub mod drive_document_count_and_sum_query; + /// A Query Syntax Validation Result that contains data pub type QuerySyntaxValidationResult = ValidationResult; diff --git a/packages/rs-platform-version/Cargo.toml b/packages/rs-platform-version/Cargo.toml index 17a5a8e90d..febc2c1e94 100644 --- a/packages/rs-platform-version/Cargo.toml +++ b/packages/rs-platform-version/Cargo.toml @@ -11,7 +11,7 @@ license = "MIT" thiserror = { version = "2.0.12" } bincode = { version = "=2.0.1" } versioned-feature-core = { git = "https://github.com/dashpay/versioned-feature-core", version = "1.0.0" } -grovedb-version = { git = "https://github.com/dashpay/grovedb", rev = "ad2492dcdc869a1452b0b10fbed8f9b0de1634c6" } +grovedb-version = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9" } [features] mock-versions = [] diff --git a/packages/rs-platform-wallet/Cargo.toml b/packages/rs-platform-wallet/Cargo.toml index 75a11215b2..846e736e94 100644 --- a/packages/rs-platform-wallet/Cargo.toml +++ b/packages/rs-platform-wallet/Cargo.toml @@ -48,7 +48,7 @@ image = { version = "0.25", default-features = false, features = ["png", "jpeg", zeroize = "1" # Shielded pool (optional, behind `shielded` feature) -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "ad2492dcdc869a1452b0b10fbed8f9b0de1634c6", optional = true } +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9", optional = true } zip32 = { version = "0.2.0", default-features = false, optional = true } [dev-dependencies] diff --git a/packages/rs-sdk/Cargo.toml b/packages/rs-sdk/Cargo.toml index 19d450ec32..d31165a691 100644 --- a/packages/rs-sdk/Cargo.toml +++ b/packages/rs-sdk/Cargo.toml @@ -18,7 +18,7 @@ drive = { path = "../rs-drive", default-features = false, features = [ ] } drive-proof-verifier = { path = "../rs-drive-proof-verifier", default-features = false } -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "ad2492dcdc869a1452b0b10fbed8f9b0de1634c6", features = ["client", "sqlite"], optional = true } +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9", features = ["client", "sqlite"], optional = true } dash-async = { path = "../rs-dash-async" } dash-context-provider = { path = "../rs-context-provider", default-features = false } dash-platform-macros = { path = "../rs-dash-platform-macros" }