diff --git a/banderwagon/src/element.rs b/banderwagon/src/element.rs index af271541..7263498e 100644 --- a/banderwagon/src/element.rs +++ b/banderwagon/src/element.rs @@ -1,4 +1,4 @@ -use ark_ec::{twisted_edwards::TECurveConfig, CurveGroup, PrimeGroup, ScalarMul, VariableBaseMSM}; +use ark_ec::{twisted_edwards::TECurveConfig, CurveGroup, PrimeGroup, ScalarMul}; use ark_ed_on_bls12_381_bandersnatch::{BandersnatchConfig, EdwardsAffine, EdwardsProjective, Fq}; use ark_ff::{serial_batch_inversion_and_mul, Field, One, Zero}; use ark_serialize::{CanonicalDeserialize, CanonicalSerialize, SerializationError}; @@ -413,15 +413,26 @@ fn subgroup_check(point: &EdwardsProjective) -> bool { } pub fn multi_scalar_mul(bases: &[Element], scalars: &[Fr]) -> Element { - let bases_inner: Vec<_> = bases.iter().map(|element| element.0).collect(); - - // XXX: Converting all of these to affine hurts performance - let bases = EdwardsProjective::batch_convert_to_mul_base(&bases_inner); - - let result = EdwardsProjective::msm(&bases, scalars) - .expect("number of bases should equal number of scalars"); + assert_eq!( + bases.len(), + scalars.len(), + "number of bases should equal number of scalars" + ); + + // The MSM needs affine bases. Points loaded from storage or deserialized + // proofs already have `z = 1`, so skip the batch inversion entirely in + // that common case. + let bases: Vec = if bases.iter().all(|element| element.0.z.is_one()) { + bases + .iter() + .map(|element| EdwardsAffine::new_unchecked(element.0.x, element.0.y)) + .collect() + } else { + let bases_inner: Vec<_> = bases.iter().map(|element| element.0).collect(); + EdwardsProjective::batch_convert_to_mul_base(&bases_inner) + }; - Element(result) + Element(crate::msm::msm_windowed(&bases, scalars)) } /// Multiplies an `Element` by a scalar field element. diff --git a/banderwagon/src/msm.rs b/banderwagon/src/msm.rs index 0013776b..ecaac227 100644 --- a/banderwagon/src/msm.rs +++ b/banderwagon/src/msm.rs @@ -1,11 +1,154 @@ use crate::Element; use ark_ec::scalar_mul::wnaf::WnafContext; -use ark_ed_on_bls12_381_bandersnatch::{EdwardsProjective, Fr}; -use ark_ff::Zero; +use ark_ed_on_bls12_381_bandersnatch::{EdwardsAffine, EdwardsProjective, Fr}; +use ark_ff::{AdditiveGroup, PrimeField, Zero}; +use core::cmp::Ordering; -use salt_macros::iter; use salt_macros::prelude::*; -use std::vec::Vec; +use salt_macros::{into_iter, iter}; +use std::{vec, vec::Vec}; + +/// Window-parallel Pippenger multi-scalar multiplication over affine bases. +/// +/// Mirrors the signed-digit bucket algorithm of arkworks' `msm_bigint_wnaf`, +/// but converts scalars and builds the digit matrix serially: at the MSM +/// sizes SALT handles (tens to a few thousand points), the parallel +/// conversion collects inside arkworks cost more in thread-pool traffic than +/// the entire digit pass. Windows are processed in parallel only when each +/// carries enough work to pay for the dispatch. +pub(crate) fn msm_windowed(bases: &[EdwardsAffine], scalars: &[Fr]) -> EdwardsProjective { + let size = bases.len(); + let c = if size < 32 { + 3 + } else { + // ln_without_floats: log2(size) * ln(2), plus 2 + (usize::BITS - (size - 1).leading_zeros()) as usize * 69 / 100 + 2 + }; + msm_windowed_with_c(bases, scalars, c) +} + +/// [`msm_windowed`] with the window width given, so the tests can exercise every +/// width — and with it every bucket-count branch of [`bucket_len`] — at a size +/// that keeps a debug-mode run cheap. +fn msm_windowed_with_c(bases: &[EdwardsAffine], scalars: &[Fr], c: usize) -> EdwardsProjective { + debug_assert_eq!(bases.len(), scalars.len()); + let size = bases.len(); + if size == 0 { + return EdwardsProjective::zero(); + } + + let num_bits = ::MODULUS_BIT_SIZE as usize; + let digits_count = num_bits.div_ceil(c); + + // Digit matrix: scalar `i` owns scalar_digits[i * digits_count..][..digits_count]. + let mut scalar_digits = vec![0i64; size * digits_count]; + for (scalar, digits) in scalars + .iter() + .zip(scalar_digits.chunks_exact_mut(digits_count)) + { + make_digits(&scalar.into_bigint().0, c, digits); + } + + let window_sum = |window: usize| -> EdwardsProjective { + let mut buckets = + vec![EdwardsProjective::zero(); bucket_len(c, num_bits, digits_count, window)]; + for (digits, base) in scalar_digits.chunks_exact(digits_count).zip(bases) { + let digit = digits[window]; + match 0.cmp(&digit) { + Ordering::Less => buckets[(digit - 1) as usize] += base, + Ordering::Greater => buckets[(-digit - 1) as usize] -= base, + Ordering::Equal => (), + } + } + + let mut running_sum = EdwardsProjective::zero(); + let mut res = EdwardsProjective::zero(); + buckets.into_iter().rev().for_each(|b| { + running_sum += &b; + res += &running_sum; + }); + res + }; + + // Below this size the per-window work is too small to farm out. + const MIN_PARALLEL_SIZE: usize = 64; + let window_sums: Vec = if size >= MIN_PARALLEL_SIZE { + into_iter!(0..digits_count).map(window_sum).collect() + } else { + (0..digits_count).map(window_sum).collect() + }; + + // Traverse windows from high to low, doubling `c` times between windows. + let lowest = *window_sums.first().expect("digits_count >= 1"); + lowest + + window_sums[1..] + .iter() + .rev() + .fold(EdwardsProjective::zero(), |mut total, sum_i| { + total += sum_i; + for _ in 0..c { + total.double_in_place(); + } + total + }) +} + +/// Buckets one window of [`msm_windowed_with_c`] needs, which is half of the +/// window's `2^c` digit values for every window but the last. +/// +/// [`make_digits`] recenters each digit to `[-2^(c-1), 2^(c-1) - 1]`, so a +/// bucket index `|digit| - 1` stays below `2^(c-1)` — the upper half of a +/// `2^c` array is never written, and reducing over it costs one full point +/// addition per empty bucket. The last window is the exception: it absorbs the +/// outstanding carry instead of being recentered, so its digits run +/// `0..=2^last_bits`, where `last_bits` is what the modulus leaves for it. +/// `last_bits == c` when `c` divides the modulus width (c = 11 and c = 23 at +/// 253 bits), and then the last window does need all `2^c` buckets. +fn bucket_len(c: usize, num_bits: usize, digits_count: usize, window: usize) -> usize { + if window == digits_count - 1 { + 1 << (num_bits - (digits_count - 1) * c) + } else { + 1 << (c - 1) + } +} + +/// Decomposes a scalar into signed `w`-bit digits in `[-2^(w-1), 2^(w-1)]`, +/// written into `digits`. Identical digit semantics to arkworks' +/// `make_digits` (the final digit absorbs the outstanding carry). +fn make_digits(scalar: &[u64], w: usize, digits: &mut [i64]) { + let radix: u64 = 1 << w; + let window_mask: u64 = radix - 1; + + let mut carry = 0u64; + let digits_count = digits.len(); + for (i, digit_out) in digits.iter_mut().enumerate() { + // Construct a buffer of bits of the scalar, starting at `bit_offset`. + let bit_offset = i * w; + let u64_idx = bit_offset / 64; + let bit_idx = bit_offset % 64; + // Read the bits from the scalar + let bit_buf = if bit_idx < 64 - w || u64_idx == scalar.len() - 1 { + // This window's bits are contained in a single u64, + // or it's the last u64 anyway. + scalar[u64_idx] >> bit_idx + } else { + // Combine the current u64's bits with the bits from the next u64 + (scalar[u64_idx] >> bit_idx) | (scalar[1 + u64_idx] << (64 - bit_idx)) + }; + + // Read the actual coefficient value from the window + let coef = carry + (bit_buf & window_mask); // coef = [0, 2^w) + + // Recenter coefficients from [0,2^w) to [-2^w/2, 2^w/2) + carry = (coef + radix / 2) >> w; + let mut digit = (coef as i64) - (carry << w) as i64; + + if i == digits_count - 1 { + digit += (carry << w) as i64; + } + *digit_out = digit; + } +} #[derive(Clone, Debug)] pub struct MSMPrecompWnaf { @@ -68,6 +211,100 @@ mod tests { use crate::{multi_scalar_mul, Element}; use std::vec; + /// `size` random affine points and scalars, with the edge cases `0`, `r - 1` (which + /// maximizes the last window's digit, the one the last bucket count has to cover) and `1` + /// patched into the first scalars. + fn msm_input(seed: u8, size: usize) -> (Vec, Vec) { + use ark_ec::CurveGroup; + use ark_ff::UniformRand; + use rand_chacha::rand_core::SeedableRng; + use rand_chacha::ChaCha20Rng; + + let mut rng = ChaCha20Rng::from_seed([seed; 32]); + let bases_proj: Vec = (0..size) + .map(|_| EdwardsProjective::rand(&mut rng)) + .collect(); + let bases = EdwardsProjective::normalize_batch(&bases_proj); + let mut scalars: Vec = (0..size).map(|_| Fr::rand(&mut rng)).collect(); + for (scalar, edge) in scalars + .iter_mut() + .zip([Fr::zero(), -Fr::from(1u64), Fr::from(1u64)]) + { + *scalar = edge; + } + (bases, scalars) + } + + /// The hand-rolled windowed MSM must agree with arkworks' generic MSM for + /// every size class (serial windows, parallel windows, small-c) and for + /// edge-case scalars. + #[test] + fn msm_windowed_matches_arkworks() { + use ark_ec::VariableBaseMSM; + + for size in [1usize, 2, 16, 31, 32, 33, 129, 256, 1000] { + let (bases, scalars) = msm_input(9, size); + let expected = EdwardsProjective::msm(&bases, &scalars).unwrap(); + let got = msm_windowed(&bases, &scalars); + assert_eq!(got, expected, "size {size}"); + } + + // Empty input yields the identity. + assert!(msm_windowed(&[], &[]).is_zero()); + } + + /// Every window width must agree with arkworks, not only the ones the size + /// heuristic happens to pick: [`bucket_len`] sizes the last window from the + /// bits the modulus leaves it, and `c = 11` — the width chosen for 8,193 to + /// 16,384 points — is the one where that is the full `2^c`. + #[test] + fn msm_windowed_matches_arkworks_for_every_window_width() { + use ark_ec::VariableBaseMSM; + + let (bases, scalars) = msm_input(21, 200); + let expected = EdwardsProjective::msm(&bases, &scalars).unwrap(); + for c in 3..=13 { + assert_eq!( + msm_windowed_with_c(&bases, &scalars, c), + expected, + "c = {c}" + ); + } + } + + /// The bucket counts [`bucket_len`] hands out must cover every digit + /// [`make_digits`] can produce: a bucket index is `|digit| - 1`, so every + /// window needs `|digit| <= bucket_len`. + #[test] + fn make_digits_stays_within_the_bucket_counts() { + use ark_ff::UniformRand; + use rand_chacha::rand_core::SeedableRng; + use rand_chacha::ChaCha20Rng; + + let mut rng = ChaCha20Rng::from_seed([7u8; 32]); + let num_bits = ::MODULUS_BIT_SIZE as usize; + let mut scalars: Vec = (0..64).map(|_| Fr::rand(&mut rng)).collect(); + scalars.extend([Fr::zero(), Fr::from(1u64), -Fr::from(1u64)]); + + for c in 3..=16 { + let digits_count = num_bits.div_ceil(c); + let mut digits = vec![0i64; digits_count]; + for scalar in &scalars { + make_digits(&scalar.into_bigint().0, c, &mut digits); + for (window, &digit) in digits.iter().enumerate() { + let len = bucket_len(c, num_bits, digits_count, window) as i64; + assert!( + digit.abs() <= len, + "c = {c}, window {window}: digit {digit} needs more than {len} buckets" + ); + if window == digits_count - 1 { + assert!(digit >= 0, "c = {c}: the last digit absorbs the carry"); + } + } + } + } + } + #[test] fn correctness_smoke_test() { let mut crs = Vec::with_capacity(256); diff --git a/banderwagon/src/salt_committer.rs b/banderwagon/src/salt_committer.rs index 41bfa2ec..0409d519 100644 --- a/banderwagon/src/salt_committer.rs +++ b/banderwagon/src/salt_committer.rs @@ -31,8 +31,8 @@ use ark_ed_on_bls12_381_bandersnatch::{EdwardsAffine, EdwardsProjective, Fq, Fr} use ark_ff::PrimeField; use ark_ff::Zero; -use salt_macros::iter; use salt_macros::prelude::*; +use salt_macros::{chunks, iter, thread_chunk_size}; use std::{vec, vec::Vec}; /// Precomputed Multi-Scalar Multiplication engine for fixed base points. @@ -187,6 +187,11 @@ impl Committer { } } + /// Returns the number of base points covered by the precomputed tables. + pub fn num_bases(&self) -> usize { + self.tables.len() + } + /// Multiplies a precomputed base point by a scalar using windowed NAF. /// /// This x86_64-optimized version uses CPU prefetching instructions to @@ -333,6 +338,43 @@ impl Committer { .for_each(|p| add_affine_point(&mut result, &p.x, &p.y)); Element(result) } + + /// Sums `scalar · G[index]` over `terms`, splitting them across threads. + pub fn msm(&self, terms: &[(usize, Fr)]) -> Element { + chunks!(terms, thread_chunk_size!(terms.len())) + .map(|chunk| { + let mut acc = Element::zero(); + for (index, scalar) in chunk { + if !scalar.is_zero() { + acc += self.mul_index(scalar, *index); + } + } + acc + }) + .sum() + } + + /// Sums `scalars[i] · G[i]`, splitting the scalars across threads: [`Committer::msm`] for + /// a dense vector over the first `scalars.len()` bases, without a list of terms to build. + /// + /// # Panics + /// + /// If there are more scalars than bases. + pub fn msm_dense(&self, scalars: &[Fr]) -> Element { + let chunk_size = thread_chunk_size!(scalars.len()); + chunks!(scalars, chunk_size) + .enumerate() + .map(|(chunk, scalars)| { + let mut acc = Element::zero(); + for (offset, scalar) in scalars.iter().enumerate() { + if !scalar.is_zero() { + acc += self.mul_index(scalar, chunk * chunk_size + offset); + } + } + acc + }) + .sum() + } } /// Adds an affine point to a projective point using extended Edwards coordinates. @@ -483,6 +525,34 @@ mod tests { use std::str::FromStr; use std::string::ToString; + /// `msm_dense` is `msm` over the scalars' own positions: for a full vector, a shorter + /// one, a single scalar and none, it gives what `msm` gives for the same terms and what + /// the reference MSM gives, zero scalars included. + #[test] + fn msm_dense_matches_msm_and_the_reference() { + let crs: Vec = (0..256) + .map(|i| Element::prime_subgroup_generator() * Fr::from((i + 1) as u64)) + .collect(); + let committer = Committer::new(&crs, platform::DEFAULT_PRECOMP_WINDOW_SIZE); + + let mut rng = ChaCha20Rng::seed_from_u64(7); + let mut scalars: Vec = (0..256).map(|_| Fr::rand(&mut rng)).collect(); + scalars[3] = Fr::zero(); + scalars[255] = Fr::zero(); + + for len in [256, 100, 1, 0] { + let scalars = &scalars[..len]; + let terms: Vec<(usize, Fr)> = scalars.iter().copied().enumerate().collect(); + let dense = committer.msm_dense(scalars); + assert_eq!(dense, committer.msm(&terms), "{len} scalars"); + assert_eq!( + dense, + multi_scalar_mul(&crs[..len], scalars), + "{len} scalars" + ); + } + } + /// Tests the correctness of precomputed MSM against the reference implementation. /// /// This comprehensive test verifies that the `Committer`'s optimized scalar multiplication diff --git a/ipa-multipoint/benches/benchmarks/multipoint_prove.rs b/ipa-multipoint/benches/benchmarks/multipoint_prove.rs index 5c83e643..bc5a232f 100644 --- a/ipa-multipoint/benches/benchmarks/multipoint_prove.rs +++ b/ipa-multipoint/benches/benchmarks/multipoint_prove.rs @@ -35,7 +35,7 @@ pub fn criterion_benchmark(c: &mut Criterion) { let prover_query = ProverQuery { commitment: poly_comm, - poly, + poly: poly.into(), point, result: y_i, }; diff --git a/ipa-multipoint/benches/benchmarks/multipoint_verify.rs b/ipa-multipoint/benches/benchmarks/multipoint_verify.rs index a2a89a45..06dcc5e3 100644 --- a/ipa-multipoint/benches/benchmarks/multipoint_verify.rs +++ b/ipa-multipoint/benches/benchmarks/multipoint_verify.rs @@ -36,7 +36,7 @@ pub fn criterion_benchmark(c: &mut Criterion) { let prover_query = ProverQuery { commitment: poly_comm, - poly, + poly: poly.into(), point, result: y_i, }; diff --git a/ipa-multipoint/src/crs.rs b/ipa-multipoint/src/crs.rs index 6971a663..23350e11 100644 --- a/ipa-multipoint/src/crs.rs +++ b/ipa-multipoint/src/crs.rs @@ -10,7 +10,7 @@ //! approach, ensuring reproducibility and verifiability of the setup. use crate::{default_crs, ipa::slow_vartime_multiscalar_mul, lagrange_basis::LagrangeBasis}; -use banderwagon::{try_reduce_to_element, Element, SerializationError}; +use banderwagon::{salt_committer::Committer, try_reduce_to_element, Element, SerializationError}; use std::{string::String, vec::Vec}; use thiserror::Error; @@ -180,6 +180,19 @@ impl CRS { pub fn commit_lagrange_poly(&self, polynomial: &LagrangeBasis) -> Element { slow_vartime_multiscalar_mul(polynomial.values().iter(), self.G.iter()) } + + /// Fixed-base tables over this CRS: `G` in order, then `Q` as base `n`. This is the shape + /// [`crate::multiproof::MultiPoint::open_with_committer`] and [`crate::ipa::create_with_precomp`] + /// take, so every MSM they compute — the IPA's blinding terms included — is fixed-base. + pub fn committer(&self, window_size: usize) -> Committer { + let bases: Vec = self + .G + .iter() + .copied() + .chain(core::iter::once(self.Q)) + .collect(); + Committer::new(&bases, window_size) + } } impl core::ops::Index for CRS { diff --git a/ipa-multipoint/src/ipa.rs b/ipa-multipoint/src/ipa.rs index bf8ed379..ef655db7 100644 --- a/ipa-multipoint/src/ipa.rs +++ b/ipa-multipoint/src/ipa.rs @@ -4,13 +4,13 @@ use crate::math_utils::inner_product; use crate::transcript::{Transcript, TranscriptProtocol}; use crate::{IOResult, SerdeError}; -use banderwagon::{multi_scalar_mul, trait_defs::*, Element, Fr}; +use banderwagon::{multi_scalar_mul, salt_committer::Committer, trait_defs::*, Element, Fr}; use core::iter; use itertools::Itertools; use salt_macros::prelude::*; -use salt_macros::{chunks, chunks_mut, join, num_threads}; -use std::vec::Vec; +use salt_macros::{chunks, chunks_mut, join, thread_chunk_size}; +use std::{vec, vec::Vec}; #[derive(Debug, Clone, PartialEq)] pub struct IPAProof { @@ -152,7 +152,7 @@ pub fn create( let x = transcript.challenge_scalar(b"x"); let x_inv = x.inverse().unwrap(); - let chunk_size = G_L.len().div_ceil(num_threads!()); + let chunk_size = thread_chunk_size!(G_L.len()); chunks_mut!(G_L, chunk_size) .zip(chunks!(G_R, chunk_size)) .for_each(|(g_l_chunk, g_r_chunk)| { @@ -177,6 +177,137 @@ pub fn create( } } +/// Creates the same IPA proof as [`create`], but computes every `L`/`R` point +/// as a fixed-base multi-scalar multiplication over the *original* CRS +/// generators using precomputed wNAF tables, instead of folding the generator +/// vector with one variable-base scalar multiplication per element. +/// +/// # How it works +/// +/// After `k` folding rounds, each folded generator is a known linear +/// combination of the original generators: +/// +/// ```text +/// G_k[i] = Σ_m coeff[m] · G[m] over m with m mod len == i +/// ``` +/// +/// where `coeff[m]` is the product of the inverse challenges of every past +/// round in which index `m` sat in the right half. Substituting this into +/// `L_k = + z_L·Q` (and symmetrically for `R_k`) turns each round's +/// two MSMs into MSMs over the fixed generator set, where each original index +/// contributes to exactly one of `L`/`R` per round. The scalars fold exactly +/// as before, so the resulting proof is byte-identical to [`create`]'s. +/// +/// # Arguments +/// +/// * `precomp` - The CRS's fixed-base tables ([`CRS::committer`](crate::crs::CRS::committer)): +/// `G` in order, then `Q` as base `a_vec.len()`, so the blinding terms are fixed-base too +pub fn create_with_precomp( + transcript: &mut Transcript, + precomp: &Committer, + mut a_vec: Vec, + a_comm: Element, + mut b_vec: Vec, + // This is the z in f(z) + input_point: Fr, +) -> IPAProof { + transcript.domain_sep(b"ipa"); + + let mut a = &mut a_vec[..]; + let mut b = &mut b_vec[..]; + + let n = a.len(); + assert_eq!(b.len(), n); + assert!(n.is_power_of_two()); + assert_eq!( + precomp.num_bases(), + n + 1, + "the committer must hold the CRS `G` vector followed by `Q`" + ); + + let output_point = inner_product(a, b); + transcript.append_point(b"C", &a_comm); + transcript.append_scalar(b"input point", &input_point); + transcript.append_scalar(b"output point", &output_point); + + let w = transcript.challenge_scalar(b"w"); + // `Q` is base `n` of the tables, so the blinding term `z·(w·Q) = (w·z)·Q` is fixed-base too. + let blind = |z: Fr| precomp.mul_index(&(w * z), n); + + let num_rounds = log2(n); + + let mut L_vec: Vec = Vec::with_capacity(num_rounds as usize); + let mut R_vec: Vec = Vec::with_capacity(num_rounds as usize); + + // coeff[m] tracks the coefficient of original generator G[m] inside the + // implicitly folded generator vector. + let mut coeff = vec![Fr::one(); n]; + let mut len = n; + // This round's L/R terms over the original generators; each index lands in one of them. + let mut l_terms: Vec<(usize, Fr)> = Vec::with_capacity(n / 2); + let mut r_terms: Vec<(usize, Fr)> = Vec::with_capacity(n / 2); + + for _ in 0..num_rounds { + let half = len / 2; + let (a_L, a_R) = halve(a); + let (b_L, b_R) = halve(b); + + // Express this round's L/R in terms of the original generators. Every + // original index lands in exactly one of the two point sets. + l_terms.clear(); + r_terms.clear(); + for (m, c) in coeff.iter().enumerate() { + let i = m % len; + if i < half { + l_terms.push((m, a_R[i] * c)); + } else { + r_terms.push((m, a_L[i - half] * c)); + } + } + + let left_compute = || -> Element { + let z_L = inner_product(a_R, b_L); + precomp.msm(&l_terms) + blind(z_L) + }; + let right_compute = || -> Element { + let z_R = inner_product(a_L, b_R); + precomp.msm(&r_terms) + blind(z_R) + }; + + let (L, R) = join!(left_compute, right_compute); + L_vec.push(L); + R_vec.push(R); + + transcript.append_point(b"L", &L); + transcript.append_point(b"R", &R); + + let x = transcript.challenge_scalar(b"x"); + let x_inv = x.inverse().unwrap(); + + for i in 0..a_L.len() { + a_L[i] += x * a_R[i]; + b_L[i] += x_inv * b_R[i]; + } + // Indices in the right half of the current fold pick up the inverse + // challenge, mirroring `G_L += G_R * x_inv` on the original bases. + for (m, c) in coeff.iter_mut().enumerate() { + if m % len >= half { + *c *= x_inv; + } + } + + a = a_L; + b = b_L; + len = half; + } + + IPAProof { + L_vec, + R_vec, + a: a[0], + } +} + // Halves the slice that is passed in // Assumes that the slice has an even length fn halve(scalars: &mut [T]) -> (&mut [T], &mut [T]) { @@ -217,9 +348,10 @@ impl IPAProof { let w = transcript.challenge_scalar(b"w"); // Generate all of the necessary challenges and their inverses + // (log n elements — invert serially) let challenges = generate_challenges(self, transcript); let mut challenges_inv = challenges.clone(); - batch_inversion(&mut challenges_inv); + serial_batch_inversion_and_mul(&mut challenges_inv, &Fr::one()); // Generate the coefficients for the `G` vector and the `b` vector // {-g_i}{-b_i} @@ -276,15 +408,6 @@ pub fn slow_vartime_multiscalar_mul<'a>( multi_scalar_mul(&points, &scalars) } -pub fn multi_scalar_mul_par(bases: &[Element], scalars: &[Fr]) -> Element { - let chunk_size = bases.len().div_ceil(num_threads!()); - - chunks!(bases, chunk_size) - .zip(chunks!(scalars, chunk_size)) - .map(|(bases, scalars)| multi_scalar_mul(bases, scalars)) - .sum() -} - fn generate_challenges(proof: &IPAProof, transcript: &mut Transcript) -> Vec { let mut challenges: Vec = Vec::with_capacity(proof.L_vec.len()); @@ -307,6 +430,50 @@ mod tests { use ark_std::{rand::SeedableRng, UniformRand}; use rand_chacha::ChaCha20Rng; + /// `create_with_precomp` must produce exactly the proof `create` does — + /// same transcript interaction, same L/R points, same final scalar. + #[test] + fn create_with_precomp_matches_create() { + let n = 256; + let crs = CRS::new(n, b"random seed"); + let committer = crs.committer(6); + + let mut rng = ChaCha20Rng::from_seed([7u8; 32]); + let a: Vec = (0..n).map(|_| Fr::rand(&mut rng)).collect(); + let input_point = Fr::rand(&mut rng); + let b = powers_of(input_point, n); + let a_comm = slow_vartime_multiscalar_mul(a.iter(), crs.G.iter()); + + let folded = create( + &mut Transcript::new(b"ip_no_zk"), + crs.clone(), + a.clone(), + a_comm, + b.clone(), + input_point, + ); + let precomputed = create_with_precomp( + &mut Transcript::new(b"ip_no_zk"), + &committer, + a.clone(), + a_comm, + b.clone(), + input_point, + ); + + assert_eq!(folded.to_bytes().unwrap(), precomputed.to_bytes().unwrap()); + + let output_point = inner_product(&a, &b); + assert!(precomputed.verify_multiexp( + &mut Transcript::new(b"ip_no_zk"), + &crs, + b, + a_comm, + input_point, + output_point + )); + } + #[test] fn test_create_IPAProof_proof() { let n = 8; diff --git a/ipa-multipoint/src/lagrange_basis.rs b/ipa-multipoint/src/lagrange_basis.rs index 6ace3c7f..6edd4a6d 100644 --- a/ipa-multipoint/src/lagrange_basis.rs +++ b/ipa-multipoint/src/lagrange_basis.rs @@ -236,7 +236,8 @@ impl LagrangeBasis { self.values[index] } - pub(crate) fn values(&self) -> &[Fr] { + /// The polynomial's evaluations over the domain, in order. + pub fn values(&self) -> &[Fr] { &self.values } diff --git a/ipa-multipoint/src/math_utils.rs b/ipa-multipoint/src/math_utils.rs index 69c8b82d..da3d76c0 100644 --- a/ipa-multipoint/src/math_utils.rs +++ b/ipa-multipoint/src/math_utils.rs @@ -1,6 +1,6 @@ use banderwagon::{trait_defs::*, Fr}; use salt_macros::prelude::*; -use salt_macros::{chunks_mut, num_threads}; +use salt_macros::{chunks_mut, thread_chunk_size}; use std::{vec, vec::Vec}; /// Computes the inner product between two scalar vectors @@ -22,10 +22,17 @@ pub fn powers_of(point: Fr, n: usize) -> Vec { #[inline(always)] pub fn powers_of_par(point: Fr, n: usize) -> Vec { + // A field multiplication is ~20ns, so even tens of thousands of powers + // compute faster serially than the cost of waking the thread pool. + const MIN_PARALLEL_LEN: usize = 1 << 16; + if n < MIN_PARALLEL_LEN { + return powers_of(point, n); + } + let mut powers = vec![Fr::zero(); n]; // Compute base powers for each chunk - let chunk_size = n.div_ceil(num_threads!()); + let chunk_size = thread_chunk_size!(n); // to handle the case where n is not a multiple of chunk_size let len = n.div_ceil(chunk_size); diff --git a/ipa-multipoint/src/multiproof.rs b/ipa-multipoint/src/multiproof.rs index b0d6c0aa..51f59d65 100644 --- a/ipa-multipoint/src/multiproof.rs +++ b/ipa-multipoint/src/multiproof.rs @@ -2,28 +2,31 @@ #![allow(non_snake_case)] use crate::crs::CRS; -use crate::ipa::{multi_scalar_mul_par, IPAProof}; +use crate::ipa::IPAProof; use crate::lagrange_basis::{LagrangeBasis, PrecomputedWeights}; use crate::math_utils::powers_of_par; use crate::transcript::Transcript; use crate::transcript::TranscriptProtocol; -use banderwagon::{trait_defs::*, Element, Fr}; +use banderwagon::{multi_scalar_mul, salt_committer::Committer, trait_defs::*, Element, Fr}; use hashbrown::HashMap; use rustc_hash::FxBuildHasher; type FxHashMap = HashMap; use salt_macros::prelude::*; -use salt_macros::{chunks, chunks_mut, into_iter, iter, num_threads, reduce}; -use std::vec::Vec; +use salt_macros::{chunks, chunks_mut, into_iter, iter, reduce, thread_chunk_size}; +use std::{borrow::Cow, sync::Arc, vec, vec::Vec}; pub struct MultiPoint; #[derive(Clone, Debug)] pub struct ProverQuery { pub commitment: Element, - pub poly: LagrangeBasis, // TODO: Make this a reference so that upstream libraries do not need to clone + /// Shared so that opening one polynomial at several points does not clone + /// its 256 coefficients per point. A caller that owns a `LagrangeBasis` + /// passes `poly.into()`. + pub poly: Arc, // Given a function f, we use z_i to denote the input point and y_i to denote the output, ie f(z_i) = y_i pub point: usize, pub result: Fr, @@ -73,6 +76,47 @@ impl MultiPoint { precomp: &PrecomputedWeights, transcript: &mut Transcript, queries: Vec, + ) -> MultiPointProof { + Self::open_inner(Cow::Owned(crs), None, precomp, transcript, queries) + } + + /// Same proof as [`MultiPoint::open`] (byte-identical output), but every + /// multi-scalar multiplication over the CRS generators — the two + /// polynomial commitments and all IPA `L`/`R` points — is computed with + /// the given precomputed fixed-base tables. This removes both the + /// per-round generator folding and the small variable-base MSMs, which + /// dominate proving time. + /// + /// `committer` is `crs`'s fixed-base tables, [`CRS::committer`]: `G` in order, then `Q`. + pub fn open_with_committer( + crs: &CRS, + committer: &Committer, + precomp: &PrecomputedWeights, + transcript: &mut Transcript, + queries: Vec, + ) -> MultiPointProof { + assert_eq!( + committer.num_bases(), + crs.n + 1, + "the committer must hold the CRS `G` vector followed by `Q`" + ); + Self::open_inner( + Cow::Borrowed(crs), + Some(committer), + precomp, + transcript, + queries, + ) + } + + /// `crs` is owned on the path without a committer, whose IPA folds the generators in + /// place and so takes the CRS by value; borrowed, that path would have to clone it. + fn open_inner( + crs: Cow<'_, CRS>, + committer: Option<&Committer>, + precomp: &PrecomputedWeights, + transcript: &mut Transcript, + queries: Vec, ) -> MultiPointProof { transcript.domain_sep(b"multiproof"); @@ -88,21 +132,30 @@ impl MultiPoint { let grouped_queries: Vec<_> = into_iter!(grouped_queries).collect(); - let chunk_size = grouped_queries.len().div_ceil(num_threads!()); + let chunk_size = thread_chunk_size!(grouped_queries.len()); - // aggregate all of the queries evaluated at the same point + // aggregate all of the queries evaluated at the same point; + // accumulate scaled polynomials in place instead of cloning each + // query's polynomial (one 8 KB allocation plus an extra add pass per + // query otherwise) let aggregated_queries: Vec<_> = chunks!(grouped_queries, chunk_size) .flat_map(|chunk| { chunk .iter() .map(|(point, queries_challenges)| { - let aggregated_polynomial = queries_challenges - .iter() - .map(|(query, challenge)| query.poly.clone() * *challenge) - .reduce(|acc, x| acc + x) - .expect("Failed to aggregate polynomial"); - - (*point, aggregated_polynomial) + let mut aggregated = vec![Fr::zero(); crs.n]; + for (query, challenge) in queries_challenges.iter() { + assert_eq!( + query.poly.values().len(), + crs.n, + "a query polynomial must have one coefficient per CRS generator" + ); + for (acc, coeff) in aggregated.iter_mut().zip(query.poly.values()) { + *acc += **challenge * coeff; + } + } + + (*point, LagrangeBasis::new(aggregated)) }) .collect::>() }) @@ -117,7 +170,7 @@ impl MultiPoint { |a, b| a + b ); - let g_x_comm = crs.commit_lagrange_poly(&g_x); + let g_x_comm = commit_dense(&crs, committer, &g_x); transcript.append_point(b"D", &g_x_comm); @@ -148,7 +201,7 @@ impl MultiPoint { |a, b| a + b ); - let g1_comm = crs.commit_lagrange_poly(&g1_x); + let g1_comm = commit_dense(&crs, committer, &g1_x); transcript.append_point(b"E", &g1_comm); @@ -158,7 +211,14 @@ impl MultiPoint { let g_3_x_comm = g1_comm - g_x_comm; // 4. Compute the IPA for g_3 - let g_3_ipa = open_point_outside_of_domain(crs, precomp, transcript, g_3_x, g_3_x_comm, t); + let a = g_3_x.values().to_vec(); + let b = LagrangeBasis::evaluate_lagrange_coefficients(precomp, crs.n, t); + let g_3_ipa = match committer { + Some(committer) => { + crate::ipa::create_with_precomp(transcript, committer, a, g_3_x_comm, b, t) + } + None => crate::ipa::create(transcript, crs.into_owned(), a, g_3_x_comm, b, t), + }; MultiPointProof { open_proof: g_3_ipa, @@ -167,6 +227,15 @@ impl MultiPoint { } } +/// Commits to a dense polynomial in Lagrange basis: with precomputed tables +/// when available, otherwise via the generic variable-base MSM. +fn commit_dense(crs: &CRS, committer: Option<&Committer>, poly: &LagrangeBasis) -> Element { + match committer { + Some(committer) => committer.msm_dense(poly.values()), + None => crs.commit_lagrange_poly(poly), + } +} + /// This trait is used to abstract over the query data that is used to record the transcript /// in the prover and verifier. /// @@ -289,24 +358,43 @@ impl MultiPointProof { // 3. Compute g_2(t) // - let mut g2_den: Vec<_> = iter!(queries).map(|query| t - query.point).collect(); + // A handful of field operations per query: serial execution beats + // waking the thread pool even for tens of thousands of queries. + let mut g2_den: Vec<_> = queries.iter().map(|query| t - query.point).collect(); - batch_inversion(&mut g2_den); + serial_batch_inversion_and_mul(&mut g2_den, &Fr::one()); - let helper_scalars: Vec<_> = into_iter!(powers_of_r) + let helper_scalars: Vec<_> = powers_of_r + .into_iter() .zip(g2_den) .map(|(r_i, den_inv)| den_inv * r_i) .collect(); - let g2_t: Fr = iter!(helper_scalars) - .zip(iter!(queries)) + let g2_t: Fr = helper_scalars + .iter() + .zip(queries.iter()) .map(|(r_i_den_inv, query)| *r_i_den_inv * query.result) .sum(); //4. Compute [g_1(X)] = E - let comms: Vec<_> = iter!(queries).map(|query| query.commitment).collect(); + // + // Queries arrive grouped per trie node, so runs of consecutive queries + // share one commitment. Summing the scalars of each run first shrinks + // the multiexp from one point per query to one point per node, which + // is where verification time is spent for large witnesses. + let mut comms: Vec = Vec::with_capacity(queries.len()); + let mut comm_scalars: Vec = Vec::with_capacity(queries.len()); + for (query, scalar) in queries.iter().zip(helper_scalars.iter()) { + match (comms.last(), comm_scalars.last_mut()) { + (Some(last), Some(acc)) if *last == query.commitment => *acc += scalar, + _ => { + comms.push(query.commitment); + comm_scalars.push(*scalar); + } + } + } - let g1_comm = multi_scalar_mul_par(&comms, &helper_scalars); + let g1_comm = multi_scalar_mul(&comms, &comm_scalars); transcript.append_point(b"E", &g1_comm); @@ -321,24 +409,6 @@ impl MultiPointProof { } } -// TODO: we could probably get rid of this method altogether and just do this in the multiproof -// TODO method -// TODO: check that the point is actually not in the domain -pub(crate) fn open_point_outside_of_domain( - crs: CRS, - precomp: &PrecomputedWeights, - transcript: &mut Transcript, - polynomial: LagrangeBasis, - commitment: Element, - z_i: Fr, -) -> IPAProof { - let a = polynomial.values().to_vec(); - - let b = LagrangeBasis::evaluate_lagrange_coefficients(precomp, crs.n, z_i); - - crate::ipa::create(transcript, crs, a, commitment, b, z_i) -} - #[cfg(test)] mod tests { use super::*; @@ -365,7 +435,7 @@ mod tests { let prover_query = ProverQuery { commitment: poly_comm, - poly, + poly: poly.into(), point, result: y_i, }; @@ -407,13 +477,13 @@ mod tests { let prover_query_i = ProverQuery { commitment: poly_comm, - poly: poly.clone(), + poly: poly.clone().into(), point: z_i, result: y_i, }; let prover_query_j = ProverQuery { commitment: poly_comm, - poly, + poly: poly.into(), point: x_j, result: y_j, }; @@ -456,13 +526,13 @@ mod tests { ); let mut prover_transcript = Transcript::new(b"test"); - - let proof = open_point_outside_of_domain( - crs.clone(), - &precomp, + let b = LagrangeBasis::evaluate_lagrange_coefficients(&precomp, crs.n, input_point); + let proof = crate::ipa::create( &mut prover_transcript, - polynomial, + crs.clone(), + poly.clone(), commitment, + b.clone(), input_point, ); @@ -475,7 +545,6 @@ mod tests { ); let mut verifier_transcript = Transcript::new(b"test"); - let b = LagrangeBasis::evaluate_lagrange_coefficients(&precomp, crs.n, input_point); let output_point = inner_product(&poly, &b); let mut bytes = [0u8; 32]; output_point.serialize_compressed(&mut bytes[..]).unwrap(); @@ -535,13 +604,13 @@ mod tests { let prover_query_a = ProverQuery { commitment: poly_comm_a, - poly: polynomial_a, + poly: polynomial_a.into(), point: point_a, result: y_a, }; let prover_query_b = ProverQuery { commitment: poly_comm_b, - poly: polynomial_b, + poly: polynomial_b.into(), point: point_b, result: y_b, }; @@ -583,6 +652,60 @@ mod tests { assert_eq!(got, expected) } + /// The committer-accelerated prover must emit byte-identical proofs to + /// the generic prover: it computes the same group elements, only through + /// fixed-base precomputation instead of generator folding. + #[test] + fn open_with_committer_matches_open() { + let n = 256; + let crs = CRS::new(n, b"random seed"); + let committer = crs.committer(6); + let precomp = PrecomputedWeights::new(n); + + let mut rng = test_rng(); + let queries: Vec = (0..7) + .map(|i| { + let poly = LagrangeBasis::new((0..n).map(|_| Fr::rand(&mut rng)).collect()); + let commitment = crs.commit_lagrange_poly(&poly); + let point = (i * 41) % n; + ProverQuery { + commitment, + result: poly.evaluate_in_domain(point), + poly: poly.into(), + point, + } + }) + .collect(); + + let generic = MultiPoint::open( + crs.clone(), + &precomp, + &mut Transcript::new(b"st"), + queries.clone(), + ); + let accelerated = MultiPoint::open_with_committer( + &crs, + &committer, + &precomp, + &mut Transcript::new(b"st"), + queries.clone(), + ); + + assert_eq!( + generic.to_bytes().unwrap(), + accelerated.to_bytes().unwrap(), + "committer-accelerated proof must be byte-identical" + ); + + let verifier_queries: Vec = queries.into_iter().map(Into::into).collect(); + assert!(accelerated.check( + &crs, + &precomp, + &verifier_queries, + &mut Transcript::new(b"st") + )); + } + #[test] fn record_query_transcript_consistency() { let (prover_queries, verifier_queries) = generate_test_queries(100); @@ -623,7 +746,7 @@ mod tests { ProverQuery { commitment: random_element, - poly: poly.clone(), + poly: poly.clone().into(), point: i % 7, result: random_scalar, } diff --git a/mutants/suppressions.toml b/mutants/suppressions.toml index 3301f45d..17560892 100644 --- a/mutants/suppressions.toml +++ b/mutants/suppressions.toml @@ -110,25 +110,9 @@ reviewer = "full-run triage 2026-07-08 (issue #144)" kind = "line" category = "equivalent" file = "salt/src/proof/subtrie.rs" -mutant = "replace * with + in multi_commitments_to_scalars" -justification = "total_capacity only sizes Vec::with_capacity, a performance hint with no behavioral effect." -reviewer = "full-run triage 2026-07-08 (issue #144)" - -[[suppress]] -kind = "line" -category = "equivalent" -file = "salt/src/proof/subtrie.rs" -mutant = "replace * with / in multi_commitments_to_scalars" -justification = "total_capacity only sizes Vec::with_capacity, a performance hint with no behavioral effect." -reviewer = "full-run triage 2026-07-08 (issue #144)" - -[[suppress]] -kind = "line" -category = "equivalent" -file = "salt/src/proof/subtrie.rs" -mutant = "replace < with <= in process_leaf_node" -justification = "parent == BUCKET_SLOT_ID_MASK is unreachable for the same reason as in create_leaf_node_queries: neither addressing scheme can produce that value." -reviewer = "full-run triage 2026-07-08 (issue #144)" +mutant = "replace < with <= in leaf_polynomial" +justification = "parent == BUCKET_SLOT_ID_MASK is unreachable for the same reason as in create_leaf_node_queries: neither addressing scheme can produce that value. (process_leaf_node became leaf_polynomial in #152; the two multi_commitments_to_scalars entries went with the total_capacity arithmetic that PR removed.)" +reviewer = "full-run triage 2026-07-08 (issue #144); re-pointed in #152" [[suppress]] kind = "line" @@ -269,7 +253,7 @@ reviewer = "full-run triage 2026-07-08 (issue #144)" kind = "line" category = "timeout" file = "salt/src/trie/trie.rs" -line = 894 +line = 905 mutant = "replace < with <= in StateRoot<'a, Store>::create_node_aligned_chunks" justification = "The outer-loop variant re-enters with next_boundary == len and never advances (observed hang). Pinned to the outer-loop site; the boundary-check twin terminates and is killed by the exact-chunks test." reviewer = "full-run triage 2026-07-08 (issue #144)" @@ -286,7 +270,7 @@ reviewer = "full-run triage 2026-07-08 (issue #144)" kind = "line" category = "timeout" file = "salt/src/trie/trie.rs" -line = 903 +line = 914 mutant = "replace < with == in StateRoot<'a, Store>::create_node_aligned_chunks" justification = "Observed non-terminating: the boundary check never fires, so next_boundary stops advancing inside the outer loop. Pinned to the boundary-check site." reviewer = "full-run triage 2026-07-08 (issue #144)" @@ -295,7 +279,7 @@ reviewer = "full-run triage 2026-07-08 (issue #144)" kind = "line" category = "timeout" file = "salt/src/trie/trie.rs" -line = 900 +line = 911 mutant = "replace += with *= in StateRoot<'a, Store>::create_node_aligned_chunks" justification = "The inner-scan variant multiplies by one and never advances past an equal-node run (observed hang). Pinned to the inner-scan site; the stride twin terminates and is killed by the exact-chunks test." reviewer = "full-run triage 2026-07-08 (issue #144)" @@ -304,7 +288,7 @@ reviewer = "full-run triage 2026-07-08 (issue #144)" kind = "line" category = "timeout" file = "salt/src/trie/trie.rs" -line = 905 +line = 916 mutant = "replace += with -= in StateRoot<'a, Store>::create_node_aligned_chunks" justification = "Observed non-terminating: the boundary marches backwards and the loop never completes. Pinned to the stride site." reviewer = "full-run triage 2026-07-08 (issue #144)" diff --git a/salt-macros/src/lib.rs b/salt-macros/src/lib.rs index 5e0466ac..40901ef2 100644 --- a/salt-macros/src/lib.rs +++ b/salt-macros/src/lib.rs @@ -26,6 +26,15 @@ macro_rules! num_threads { }}; } +/// The chunk size that spreads `len` items over the threads [`num_threads!`] reports, never +/// zero: `chunks!`/`chunks_mut!` panic on a zero size, and an empty slice still has to chunk. +#[macro_export] +macro_rules! thread_chunk_size { + ($len: expr) => { + ($len).div_ceil($crate::num_threads!()).max(1) + }; +} + /// Chooses between parallel and sequential `into_iter`. /// Uses `into_par_iter()` if the "parallel" feature is enabled, otherwise uses `into_iter()`. #[macro_export] diff --git a/salt/Cargo.toml b/salt/Cargo.toml index c35b7a41..4b6eec63 100644 --- a/salt/Cargo.toml +++ b/salt/Cargo.toml @@ -47,6 +47,10 @@ enable-hugepages = ["std", "banderwagon/enable-hugepages"] name = "salt_trie" harness = false +[[bench]] +name = "proof" +harness = false + [[bench]] name = "parents_commitments_deser" harness = false diff --git a/salt/benches/proof.rs b/salt/benches/proof.rs new file mode 100644 index 00000000..dcd0af9b --- /dev/null +++ b/salt/benches/proof.rs @@ -0,0 +1,230 @@ +//! SALT Proof & Witness Performance Benchmarks +//! +//! Measures the four hot paths of the proof layer: +//! +//! - `SaltProof::create` — proof generation for a set of salt keys +//! - `SaltProof::check` — proof verification against the state root +//! - `Witness::create` — block witness generation (lookups + updates) +//! - `Witness::verify` — block witness verification (stateless validator) +//! +//! Creation is timed twice. The plain groups repeat one input, so after the first iteration +//! every node polynomial is served from the cache; the `-cold` groups clear the cache before +//! each iteration, so every node is rebuilt from the store and checked before it is cached. +//! A caller sees something in between. +//! +//! ```bash +//! cargo bench --package salt --bench proof +//! ``` + +use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion}; +use rand::{rngs::StdRng, Rng, SeedableRng}; +use salt::{ + types::*, EphemeralSaltState, MemStore, NodePolyCache, SaltProof, SaltWitness, StateRoot, + Witness, +}; +use std::collections::BTreeMap; +use std::hint::black_box; +use std::sync::OnceLock; +use std::time::Duration; + +/// Number of key-value pairs pre-inserted into the store. +const INITIAL_KEYS: usize = 50_000; + +struct Setup { + store: MemStore, + /// Plain keys that exist in the store. + plain_keys: Vec>, + /// Salt keys of all existing entries. + salt_keys: Vec, + /// State root after setup. + root: ScalarBytes, +} + +fn mock_data(rng: &mut StdRng, len: usize) -> Vec { + (0..len).map(|_| rng.random()).collect() +} + +fn setup() -> &'static Setup { + static SETUP: OnceLock = OnceLock::new(); + SETUP.get_or_init(|| { + let mut rng = StdRng::seed_from_u64(42); + + let kvs: hashbrown::HashMap, Option>> = (0..INITIAL_KEYS) + .map(|_| (mock_data(&mut rng, 32), Some(mock_data(&mut rng, 32)))) + .collect(); + + let store = MemStore::new(); + let updates = EphemeralSaltState::new(&store).update_fin(&kvs).unwrap(); + store.update_state(updates.clone()); + + let (root, trie_updates) = StateRoot::new(&store).update_fin(&updates).unwrap(); + store.update_trie(trie_updates); + + let plain_keys: Vec> = kvs.keys().cloned().collect(); + let salt_keys: Vec = updates + .data + .keys() + .copied() + .filter(|k| !k.is_in_meta_bucket()) + .collect(); + + Setup { + store, + plain_keys, + salt_keys, + root, + } + }) +} + +/// Selects `n` existing salt keys plus a few non-existent ones (sorted, deduped). +fn proof_keys(s: &Setup, n: usize) -> Vec { + let mut keys: Vec = s + .salt_keys + .iter() + .step_by(s.salt_keys.len() / n) + .copied() + .collect(); + keys.truncate(n); + // A few keys proven absent (bucket untouched by setup with default capacity). + keys.push(SaltKey::from((16_777_215, 1))); + keys.push(SaltKey::from((16_777_215, 254))); + keys.sort_unstable(); + keys.dedup(); + keys +} + +fn bench_salt_proof(c: &mut Criterion) { + let s = setup(); + + let mut group = c.benchmark_group("salt_proof/create"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(10)); + for n in [16usize, 256, 2048] { + let keys = proof_keys(s, n); + group.bench_with_input(BenchmarkId::from_parameter(n), &keys, |b, keys| { + b.iter(|| black_box(SaltProof::create(keys.iter().copied(), &s.store).unwrap())); + }); + } + group.finish(); + + let mut group = c.benchmark_group("salt_proof/create-cold"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(10)); + for n in [16usize, 256, 2048] { + let keys = proof_keys(s, n); + group.bench_with_input(BenchmarkId::from_parameter(n), &keys, |b, keys| { + b.iter_batched( + || NodePolyCache::global().clear(), + |()| black_box(SaltProof::create(keys.iter().copied(), &s.store).unwrap()), + BatchSize::PerIteration, + ); + }); + } + group.finish(); + + let mut group = c.benchmark_group("salt_proof/check"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(10)); + for n in [16usize, 256, 2048] { + let keys = proof_keys(s, n); + let SaltWitness { kvs, proof } = SaltWitness::create(&keys, &s.store).unwrap(); + group.bench_with_input( + BenchmarkId::from_parameter(n), + &(proof, kvs), + |b, (proof, kvs)| { + b.iter(|| proof.check(black_box(kvs), s.root).unwrap()); + }, + ); + } + group.finish(); +} + +/// Plain-key updates: key -> new value (None = delete). +type PlainUpdates = BTreeMap, Option>>; + +/// A block-like workload: half the size is lookups (2/3 hits, 1/3 misses), +/// half is updates (1/2 overwrites of existing keys, 1/2 fresh inserts). +fn witness_workload(s: &Setup, n: usize, rng: &mut StdRng) -> (Vec>, PlainUpdates) { + let half = n / 2; + let hits = half * 2 / 3; + let step = (s.plain_keys.len() / half.max(1)).max(1); + + let mut lookups: Vec> = s + .plain_keys + .iter() + .step_by(step) + .take(hits) + .cloned() + .collect(); + for _ in 0..(half - hits) { + lookups.push(mock_data(rng, 32)); // misses + } + + let mut updates = BTreeMap::new(); + for key in s.plain_keys.iter().skip(1).step_by(step).take(half / 2) { + updates.insert(key.clone(), Some(mock_data(rng, 32))); // overwrite + } + while updates.len() < half { + updates.insert(mock_data(rng, 32), Some(mock_data(rng, 32))); // insert + } + + (lookups, updates) +} + +fn bench_witness(c: &mut Criterion) { + let s = setup(); + let mut rng = StdRng::seed_from_u64(7); + + let mut group = c.benchmark_group("witness/create"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(15)); + for n in [256usize, 1024] { + let (lookups, updates) = witness_workload(s, n, &mut rng); + group.bench_with_input( + BenchmarkId::from_parameter(n), + &(lookups, updates), + |b, (lookups, updates)| { + b.iter(|| { + black_box(Witness::create([], lookups.iter(), updates, &s.store).unwrap()) + }); + }, + ); + } + group.finish(); + + let mut group = c.benchmark_group("witness/create-cold"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(15)); + for n in [256usize, 1024] { + let (lookups, updates) = witness_workload(s, n, &mut rng); + group.bench_with_input( + BenchmarkId::from_parameter(n), + &(lookups, updates), + |b, (lookups, updates)| { + b.iter_batched( + || NodePolyCache::global().clear(), + |()| black_box(Witness::create([], lookups.iter(), updates, &s.store).unwrap()), + BatchSize::PerIteration, + ); + }, + ); + } + group.finish(); + + let mut group = c.benchmark_group("witness/verify"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(15)); + for n in [256usize, 1024] { + let (lookups, updates) = witness_workload(s, n, &mut rng); + let witness = Witness::create([], lookups.iter(), &updates, &s.store).unwrap(); + assert_eq!(witness.state_root().unwrap(), s.root); + group.bench_with_input(BenchmarkId::from_parameter(n), &witness, |b, witness| { + b.iter(|| witness.verify().unwrap()); + }); + } + group.finish(); +} + +criterion_group!(benches, bench_salt_proof, bench_witness); +criterion_main!(benches); diff --git a/salt/src/lib.rs b/salt/src/lib.rs index 0ef061f6..312b3fb0 100644 --- a/salt/src/lib.rs +++ b/salt/src/lib.rs @@ -15,10 +15,40 @@ pub(crate) type Lazy = std::sync::LazyLock; #[cfg(not(feature = "std"))] pub(crate) type Lazy = spin::Lazy; +/// Reader-writer lock for process-wide state, under the same policy as [`Lazy`]: waiters +/// park under `std`, and spin only on `no_std` targets, which have nothing to park on. +/// +/// The `std` lock's poisoning is ignored: what these locks guard is written one whole +/// entry at a time, so the data a panicking holder leaves behind is still valid. +#[cfg(feature = "std")] +pub(crate) struct RwLock(std::sync::RwLock); + +#[cfg(feature = "std")] +impl RwLock { + pub(crate) fn new(value: T) -> Self { + Self(std::sync::RwLock::new(value)) + } + + pub(crate) fn read(&self) -> std::sync::RwLockReadGuard<'_, T> { + self.0 + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + pub(crate) fn write(&self) -> std::sync::RwLockWriteGuard<'_, T> { + self.0 + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +#[cfg(not(feature = "std"))] +pub(crate) type RwLock = spin::RwLock; + pub mod constant; pub mod empty_salt; pub mod proof; -pub use proof::{fx_hashmap_serde, ProofError, SaltProof, SaltWitness, Witness}; +pub use proof::{fx_hashmap_serde, NodePolyCache, ProofError, SaltProof, SaltWitness, Witness}; pub mod state; pub use state::{ hasher, state::EphemeralSaltState, state::PlainStateProvider, updates::StateUpdates, diff --git a/salt/src/proof/mod.rs b/salt/src/proof/mod.rs index bff27f0c..6e143f1b 100644 --- a/salt/src/proof/mod.rs +++ b/salt/src/proof/mod.rs @@ -78,6 +78,7 @@ mod test_utils; pub use prover::{fx_hashmap_serde, SaltProof, SerdeCommitment, SerdeMultiPointProof}; pub use salt_witness::SaltWitness; +pub use subtrie::NodePolyCache; pub use witness::Witness; /// Error type for proof operations diff --git a/salt/src/proof/prover.rs b/salt/src/proof/prover.rs index 83b2cc14..1f7251af 100644 --- a/salt/src/proof/prover.rs +++ b/salt/src/proof/prover.rs @@ -11,14 +11,13 @@ use crate::{ traits::{StateReader, TrieReader}, trie::{ node_utils::{get_child_node, subtree_leaf_start_key}, - trie::kv_hash, + trie::{kv_hash, shared_committer, DEFAULT_CRS}, }, types::{hash_commitment, CommitmentBytes, NodeId, SaltKey, SaltValue}, BucketId, ScalarBytes, }; use banderwagon::{Element, Fr}; use ipa_multipoint::{ - crs::CRS, lagrange_basis::PrecomputedWeights, multiproof::{MultiPoint, MultiPointProof, VerifierQuery}, transcript::Transcript, @@ -26,7 +25,7 @@ use ipa_multipoint::{ use crate::Lazy; use salt_macros::prelude::*; -use salt_macros::{chunks, into_iter, iter, num_threads, sort_unstable}; +use salt_macros::{chunks, into_iter, iter, sort_unstable, thread_chunk_size}; use serde::{ de::{Error as _, MapAccess, Visitor}, ser::SerializeMap, @@ -258,11 +257,17 @@ impl SaltProof { let (prover_queries, parents_commitments, levels) = create_sub_trie(store, &keys)?; - let crs = CRS::default(); - let mut transcript = Transcript::new(b"st"); - let proof = MultiPoint::open(crs, &PRECOMPUTED_WEIGHTS, &mut transcript, prover_queries); + // Reuse the shared CRS (deriving it decompresses 257 points) and the + // trie's fixed-base tables for all MSMs over the CRS generators. + let proof = MultiPoint::open_with_committer( + &DEFAULT_CRS, + shared_committer(), + &PRECOMPUTED_WEIGHTS, + &mut transcript, + prover_queries, + ); Ok(SaltProof { parents_commitments, @@ -295,14 +300,13 @@ impl SaltProof { let mut transcript = Transcript::new(b"st"); - let crs = CRS::default(); - // call MultiPointProof::check to verify the proof - if self - .proof - .0 - .check(&crs, &PRECOMPUTED_WEIGHTS, &queries, &mut transcript) - { + if self.proof.0.check( + &DEFAULT_CRS, + &PRECOMPUTED_WEIGHTS, + &queries, + &mut transcript, + ) { Ok(()) } else { Err(ProofError::MultiPointProofFailed) @@ -419,7 +423,7 @@ fn create_internal_node_queries( ) -> ProofResult> { // Distribute internal nodes across CPU threads for parallel processing let in_nodes: Vec<_> = internal_nodes.iter().collect(); - let chunk_size = in_nodes.len().div_ceil(num_threads!()); + let chunk_size = thread_chunk_size!(in_nodes.len()); let queries = chunks!(in_nodes, chunk_size) .map(|nodes| { // Step 1: Collect all child commitments needed by this thread's nodes @@ -617,7 +621,7 @@ mod tests { BucketMeta, }; use banderwagon::{CanonicalSerialize, PrimeField}; - use ipa_multipoint::lagrange_basis::LagrangeBasis; + use ipa_multipoint::{crs::CRS, lagrange_basis::LagrangeBasis}; use rand::{rngs::StdRng, SeedableRng}; use std::vec; diff --git a/salt/src/proof/shape.rs b/salt/src/proof/shape.rs index c4aa6aee..bf24cf5a 100644 --- a/salt/src/proof/shape.rs +++ b/salt/src/proof/shape.rs @@ -11,8 +11,9 @@ //! - **Bucket Subtrees**: Dynamic trees within buckets that can expand from 1-5 levels use salt_macros::prelude::*; -use salt_macros::{iter, reduce}; +use salt_macros::{chunks, thread_chunk_size}; use std::collections::{BTreeMap, BTreeSet}; +use std::vec::Vec; use hashbrown::HashMap; use rustc_hash::FxBuildHasher; @@ -63,103 +64,128 @@ pub(crate) fn parents_and_points( BTreeMap>, BTreeMap>, ) { - reduce!( - iter!(salt_keys).map(|salt_key| { + // One accumulator pair per thread chunk, merged once at the end. (A + // per-key map-reduce would allocate and merge two BTreeMaps per key.) + let chunk_size = thread_chunk_size!(salt_keys.len()); + let partials: Vec<_> = chunks!(salt_keys, chunk_size) + .map(|chunk| { let mut internal_nodes: BTreeMap> = BTreeMap::new(); let mut slot_position_nodes: BTreeMap> = BTreeMap::new(); - let bucket_id = salt_key.bucket_id(); - let level = levels[&bucket_id]; - - // ============================================================================ - // Phase 1: Main Trie Traversal - // ============================================================================ - // Walk from the bucket root up to the main trie root, recording each - // parent-child relationship. The main trie has a fixed depth, so the walk - // is exactly `MAIN_TRIE_LEVELS - 1` steps; bounding it by that count rather - // than by reaching the root keeps a corrupt parent id from spinning forever. - let mut node = bucket_root_node_id(salt_key.bucket_id()); - for _ in 0..MAIN_TRIE_LEVELS - 1 { - let parent_node = get_parent_node(&node); - // Record that this parent needs to prove the child at this position - internal_nodes - .entry(parent_node) - .or_default() - .insert(vc_position_in_parent(&node)); - - node = parent_node; - } - debug_assert_eq!(node, ROOT_NODE_ID); - - // ============================================================================ - // Phase 2: Bucket Tree Traversal - // ============================================================================ - // For expanded buckets (>256 slots), traverse the internal bucket tree - // structure from the key's leaf segment up toward the bucket root. - // This only applies to buckets with level > 2 (multi-level bucket trees). - let mut node = subtree_leaf_for_key(salt_key); - - let mut count = level; - while count > 2 { - let parent_node = get_parent_node(&node); - // Record parent-child relationships within the bucket subtree - internal_nodes - .entry(parent_node) - .or_default() - .insert(vc_position_in_parent(&node)); - - node = parent_node; - count -= 1; + for salt_key in chunk { + record_key_paths( + salt_key, + levels, + &mut internal_nodes, + &mut slot_position_nodes, + ); } + (internal_nodes, slot_position_nodes) + }) + .collect(); - // ============================================================================ - // Phase 3: Bridge Connection (Expanded buckets) - // ============================================================================ - // For bucket trees with exactly 2 levels, create the bridge connection - // between the bucket subtree and the main trie. The encode_parent function - // embeds level information in the node ID to distinguish different tree levels. - if count == 2 { - let main_trie_node = bucket_root_node_id(salt_key.bucket_id()); - // Use encoded parent to bridge bucket tree to main trie - internal_nodes - .entry(encode_parent(main_trie_node, level)) - .or_default() - .insert(vc_position_in_parent(&node)); - } + let merge = |into: &mut BTreeMap>, from| { + for (node_id, positions) in from { + into.entry(node_id).or_default().extend(positions); + } + }; + let (mut internal_nodes, mut slot_position_nodes) = (BTreeMap::new(), BTreeMap::new()); + for (internal_map, slot_map) in partials { + merge(&mut internal_nodes, internal_map); + merge(&mut slot_position_nodes, slot_map); + } + (internal_nodes, slot_position_nodes) +} + +/// Records the authentication-path nodes and slot positions for one key into +/// the given accumulators. See [`parents_and_points`] for the algorithm. +fn record_key_paths( + salt_key: &SaltKey, + levels: &FxHashMap, + internal_nodes: &mut BTreeMap>, + slot_position_nodes: &mut BTreeMap>, +) { + let bucket_id = salt_key.bucket_id(); + let level = levels[&bucket_id]; + + // ============================================================================ + // Phase 1: Main Trie Traversal + // ============================================================================ + // Walk from the bucket root up to the main trie root, recording each + // parent-child relationship. The main trie has a fixed depth, so the walk + // is exactly `MAIN_TRIE_LEVELS - 1` steps; bounding it by that count rather + // than by reaching the root keeps a corrupt parent id from spinning forever. + let mut node = bucket_root_node_id(salt_key.bucket_id()); + for _ in 0..MAIN_TRIE_LEVELS - 1 { + let parent_node = get_parent_node(&node); + // Record that this parent needs to prove the child at this position + internal_nodes + .entry(parent_node) + .or_default() + .insert(vc_position_in_parent(&node)); + + node = parent_node; + } + debug_assert_eq!(node, ROOT_NODE_ID); + + // ============================================================================ + // Phase 2: Bucket Tree Traversal + // ============================================================================ + // For expanded buckets (>256 slots), traverse the internal bucket tree + // structure from the key's leaf segment up toward the bucket root. + // This only applies to buckets with level > 2 (multi-level bucket trees). + let mut node = subtree_leaf_for_key(salt_key); + + let mut count = level; + while count > 2 { + let parent_node = get_parent_node(&node); + // Record parent-child relationships within the bucket subtree + internal_nodes + .entry(parent_node) + .or_default() + .insert(vc_position_in_parent(&node)); + + node = parent_node; + count -= 1; + } - // ============================================================================ - // Phase 4: Key-Value Slot Position - // ============================================================================ - // Record the direct parent of the key-value pair itself. This determines - // which node contains the actual data slot and what position within that - // node's 256-slot array the key occupies. - let node = if level == 1 { - // Level 1: Key stored directly in bucket root (single 256-slot segment) - bucket_root_node_id(salt_key.bucket_id()) - } else { - // Level 2+: Key stored in a leaf segment of the bucket tree - subtree_leaf_for_key(salt_key) - }; - - // Record which slot position within the segment contains this key - // Use lowest 8 bits of slot_id as position within 256-slot segment - slot_position_nodes - .entry(node) - .or_default() - .insert((salt_key.slot_id() & 0xFF) as usize); + // ============================================================================ + // Phase 3: Bridge Connection (Expanded buckets) + // ============================================================================ + // For bucket trees with exactly 2 levels, create the bridge connection + // between the bucket subtree and the main trie. The encode_parent function + // embeds level information in the node ID to distinguish different tree levels. + if count == 2 { + let main_trie_node = bucket_root_node_id(salt_key.bucket_id()); + // Use encoded parent to bridge bucket tree to main trie + internal_nodes + .entry(encode_parent(main_trie_node, level)) + .or_default() + .insert(vc_position_in_parent(&node)); + } - (internal_nodes, slot_position_nodes) - }), - || (BTreeMap::new(), BTreeMap::new()), - |mut acc, (internal_map, slot_map)| { - for (node_id, positions) in internal_map { - acc.0.entry(node_id).or_default().extend(positions); - } - for (node_id, positions) in slot_map { - acc.1.entry(node_id).or_default().extend(positions); - } - acc - } - ) + // ============================================================================ + // Phase 4: Key-Value Slot Position + // ============================================================================ + // Record the direct parent of the key-value pair itself. This determines + // which node contains the actual data slot and what position within that + // node's 256-slot array the key occupies. + let node = if level == 1 { + // Level 1: Key stored directly in bucket root (single 256-slot segment) + bucket_root_node_id(salt_key.bucket_id()) + } else { + // Level 2+: Key stored in a leaf segment of the bucket tree + subtree_leaf_for_key(salt_key) + }; + + slot_position_nodes + .entry(node) + .or_default() + .insert(slot_position(salt_key)); +} + +/// The position of `key`'s slot in its leaf's 256-slot polynomial (the slot id's low 8 bits). +pub(super) fn slot_position(key: &SaltKey) -> usize { + (key.slot_id() & 0xFF) as usize } /// Encodes bucket tree level information into a main trie node ID. diff --git a/salt/src/proof/subtrie.rs b/salt/src/proof/subtrie.rs index cf5d027d..a150e1bb 100644 --- a/salt/src/proof/subtrie.rs +++ b/salt/src/proof/subtrie.rs @@ -11,8 +11,9 @@ //! 1. Extract and deduplicate bucket IDs from input keys //! 2. Determine trie levels for each bucket (metadata vs dynamic data buckets) //! 3. Build minimal node hierarchy using [`parents_and_points`] -//! 4. Collect cryptographic commitments for all parent nodes -//! 5. Generate IPA prover queries for leaf nodes (bucket contents) and internal nodes (child commitments) +//! 4. Read the commitment of every parent node +//! 5. Resolve every parent's polynomial, from the process-wide node-polynomial cache or storage +//! 6. Generate IPA prover queries for leaf nodes (bucket contents) and internal nodes (child commitments) use crate::{ constant::{ default_commitment, BUCKET_SLOT_BITS, BUCKET_SLOT_ID_MASK, DOMAIN_SIZE, MAX_SUBTREE_LEVELS, @@ -20,29 +21,227 @@ use crate::{ }, proof::{ prover::slot_to_field, - shape::{connect_parent_id, logic_parent_id, parents_and_points}, + shape::{connect_parent_id, logic_parent_id, parents_and_points, slot_position}, ProofError, ProofResult, SerdeCommitment, }, traits::{StateReader, TrieReader}, - trie::node_utils::{get_child_node, subtree_leaf_start_key, subtree_root_level}, - types::{BucketId, BucketMeta, NodeId, SaltKey}, + trie::{ + node_utils::{get_child_node, subtree_leaf_start_key, subtree_root_level}, + trie::shared_committer, + }, + types::{BucketId, BucketMeta, CommitmentBytes, NodeId, SaltKey}, SlotId, }; -use banderwagon::{Element, Fr}; +use banderwagon::{multi_scalar_mul, Element, Fr, PrimeField, Zero}; use ipa_multipoint::{lagrange_basis::LagrangeBasis, multiproof::ProverQuery}; use salt_macros::prelude::*; -use salt_macros::{chunks, into_iter, num_threads}; +use salt_macros::{chunks, iter, reduce, thread_chunk_size}; use std::collections::{BTreeMap, BTreeSet}; -use std::{format, string::ToString, vec, vec::Vec}; +use std::{format, string::ToString, sync::Arc, vec, vec::Vec}; use hashbrown::HashMap; use rustc_hash::FxBuildHasher; type FxHashMap = HashMap; +/// Process-wide cache of node polynomials, keyed by node id and validated by the node's +/// commitment. +/// +/// A commitment binds the node's children, so a node whose commitment is unchanged since its +/// polynomial was last built has the same 256 child scalars (internal node) or slot scalars +/// (leaf node), and the store reads plus scalar conversions that produce them can be skipped. +/// Consecutive blocks share most of their authentication paths, which is what makes the hit +/// rate worth the memory: by default at most 32768 polynomials of 8 KiB each, 256 MiB +/// ([`NodePolyCache::set_capacity`]). A full shard evicts one polynomial per new node and +/// passes over those used since it last considered them, so the paths that proof after +/// proof walks outlive the leaves a single proof touched. +/// +/// The commitment key only holds if the polynomial really opens to that commitment, which a +/// store that is not a consistent snapshot (commitments and children or slots from different +/// states) does not guarantee. Callers should still keep the store unchanged while a proof is +/// created, since such a proof will not verify; but an inconsistent store must not poison the +/// cache for consistent ones, so a rebuilt polynomial is cached only once it is shown to open +/// to the commitment it was read at: one check for all the polynomials a call rebuilds +/// ([`batch_opens`]), and one per polynomial only when that fails. One that does not open is +/// used for this proof and dropped. +mod node_poly_cache { + use super::*; + use crate::{Lazy, RwLock}; + use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + pub(super) const SHARDS: usize = 64; + + /// One cached polynomial, with the node it belongs to and the commitment it was built + /// for. + struct Slot { + node: NodeId, + commitment: CommitmentBytes, + poly: Arc, + /// Set when the polynomial is served or the node is cached again, cleared when the + /// eviction hand passes: a polynomial is evicted only when the hand comes round and + /// finds it unused since. + used: AtomicBool, + } + + /// One shard: its polynomials, which slot holds which node, and the slot the next + /// eviction considers first (taken modulo the number of slots). + #[derive(Default)] + struct Shard { + slots: Vec, + index: FxHashMap, + hand: usize, + } + + /// The cache of node polynomials that proof and witness creation share across calls: see + /// [`NodePolyCache::global`] for the one instance a process has. + /// + /// A cached polynomial saves the store reads and scalar conversions that rebuild it, and + /// costs 8 KiB while it is kept. The default bound suits a process that proves block + /// after block; one that creates a single proof, or cannot spare the memory, can lower it + /// or turn the cache off with [`set_capacity`](Self::set_capacity). + pub struct NodePolyCache { + shards: Vec>, + max_per_shard: AtomicUsize, + } + + static CACHE: Lazy = + Lazy::new(|| NodePolyCache::with_capacity(NodePolyCache::DEFAULT_CAPACITY)); + + impl NodePolyCache { + /// The bound a cache starts with: 32768 polynomials, 256 MiB. + pub const DEFAULT_CAPACITY: usize = SHARDS * 512; + + /// The cache every proof and witness creation in this process reads and fills. + pub fn global() -> &'static Self { + &CACHE + } + + /// The most polynomials the cache holds, at 8 KiB each. + pub fn capacity(&self) -> usize { + self.max_per_shard.load(Ordering::Relaxed) * SHARDS + } + + /// Bounds the cache to `polynomials` entries, rounded up to a multiple of its 64 + /// shards. Zero turns it off: nothing is cached, and proof creation skips the check + /// that caching a polynomial needs. A shard that holds more than its new share is + /// emptied. + pub fn set_capacity(&self, polynomials: usize) { + let max = polynomials.div_ceil(SHARDS); + // Stored before the shards are trimmed: `insert` reads it under the shard lock, + // so an insert that follows a shard's trim sees the new bound. + self.max_per_shard.store(max, Ordering::Relaxed); + for shard in &self.shards { + let mut guard = shard.write(); + let trimmed = (guard.slots.len() > max).then(|| core::mem::take(&mut *guard)); + drop(guard); + drop(trimmed); + } + } + + /// Drops every cached polynomial, each shard's after its lock is released. + pub fn clear(&self) { + for shard in &self.shards { + let mut guard = shard.write(); + let dropped = core::mem::take(&mut *guard); + drop(guard); + drop(dropped); + } + } + + pub(super) fn with_capacity(polynomials: usize) -> Self { + Self { + shards: (0..SHARDS).map(|_| RwLock::new(Shard::default())).collect(), + max_per_shard: AtomicUsize::new(polynomials.div_ceil(SHARDS)), + } + } + + fn shard(&self, node: NodeId) -> &RwLock { + &self.shards[(node % SHARDS as NodeId) as usize] + } + + /// The nodes the cache holds, in ascending order. Unlike `get`, looking does not + /// count as a use. + #[cfg(test)] + pub(super) fn nodes(&self) -> Vec { + let mut nodes: Vec = self + .shards + .iter() + .flat_map(|shard| shard.read().index.keys().copied().collect::>()) + .collect(); + nodes.sort_unstable(); + nodes + } + + /// The cached polynomial of `node`, if it was built for exactly this commitment. + pub(super) fn get( + &self, + node: NodeId, + commitment: &CommitmentBytes, + ) -> Option> { + let shard = self.shard(node).read(); + let slot = &shard.slots[*shard.index.get(&node)?]; + if slot.commitment != *commitment { + return None; + } + // Written only when it changes, so the readers of a polynomial every proof uses + // do not pass its cache line back and forth. + if !slot.used.load(Ordering::Relaxed) { + slot.used.store(true, Ordering::Relaxed); + } + Some(Arc::clone(&slot.poly)) + } + + pub(super) fn insert( + &self, + node: NodeId, + commitment: CommitmentBytes, + poly: Arc, + ) { + let mut guard = self.shard(node).write(); + let max = self.max_per_shard.load(Ordering::Relaxed); + if max == 0 { + return; + } + let shard = &mut *guard; + let slot = |used| Slot { + node, + commitment, + poly, + used: AtomicBool::new(used), + }; + // Whatever the insert replaces or evicts is dropped after the lock is released, + // so readers do not wait on the free. + let dropped = if let Some(&at) = shard.index.get(&node) { + // A node cached again was resolved again, which is a use. + Some(core::mem::replace(&mut shard.slots[at], slot(true))) + } else if shard.slots.len() < max { + shard.index.insert(node, shard.slots.len()); + shard.slots.push(slot(false)); + None + } else { + // The hand passes over the polynomials used since its last visit, clearing + // their mark, and takes the first one it finds unused: the one it started + // at, after a full turn, if every mark was set. + let len = shard.slots.len(); + let at = (0..=len) + .map(|step| (shard.hand + step) % len) + .find(|&at| !core::mem::take(shard.slots[at].used.get_mut())) + .expect("a full turn clears every mark"); + shard.hand = at + 1; + let evicted = core::mem::replace(&mut shard.slots[at], slot(false)); + shard.index.remove(&evicted.node); + shard.index.insert(node, at); + Some(evicted) + }; + drop(guard); + drop(dropped); + } + } +} +pub use node_poly_cache::NodePolyCache; + // Constants for improved code readability pub(crate) const METADATA_BUCKET_LEVEL: u8 = 1; -const SLOT_INDEX_MASK: u64 = 0xff; const ROOT_LEVEL_CHILD_START: NodeId = 1; /// Information returned by the subtrie creation process @@ -52,49 +251,38 @@ type SubTrieInfo = ( FxHashMap, ); -/// Converts cryptographic commitments from multiple internal trie nodes into scalar field elements. -/// -/// This function processes internal nodes in the trie hierarchy and converts their child node -/// commitments into scalar field elements suitable for IPA (Inner Product Argument) proof generation. -/// For each internal node, it loads commitments for all 256 possible child positions, using default -/// commitments where actual child nodes don't exist (sparse trie optimization). -/// -/// # Parameters -/// -/// * `store` - Storage backend providing access to trie node commitments -/// * `nodes` - Internal nodes with their evaluation points (child indices to prove) -/// -/// # Returns -/// -/// A vector of scalar field elements (`Fr`) representing all child commitments for the given nodes. -/// The elements are ordered by node, then by child index (0-255 per node). -/// -/// # Implementation Details -/// -/// - Handles sparse child nodes by filling missing positions with appropriate default commitments -/// - Special handling for root level nodes (different default commitment for first child) -/// - Uses parallel processing for performance when dealing with multiple nodes -/// - Converts Element commitments to scalar field using `serial_batch_map_to_scalar_field` +/// The polynomial of every internal node in `nodes` (logical ids), in order: its 256 child +/// commitments mapped to the scalar field, with the level's default commitment where a child +/// does not exist in storage (the root level's first child has its own default). /// /// # Errors /// /// Returns `ProofError::StateReadError` if unable to read child node commitments from storage. -fn multi_commitments_to_scalars( - store: &Store, - nodes: &[(NodeId, BTreeSet)], -) -> ProofResult> +fn internal_polynomials(store: &Store, nodes: &[NodeId]) -> ProofResult> where Store: TrieReader, { - // Helper closures for concise element conversion + // Helper closure for concise element conversion let to_element = |bytes| Element::from_bytes_unchecked_uncompressed(bytes); - let default_element = |node_id| to_element(default_commitment(node_id)); - let total_capacity = nodes.len() * DOMAIN_SIZE; - let mut all_child_commitments = Vec::with_capacity(total_capacity); + // The trie is sparse, so most of a node's 256 children carry a default + // commitment, of which only a handful of distinct values exist (one or two + // per level). Mapping a commitment to the scalar field costs a field + // inversion, so defaults are converted once and cached by value while only + // the children that actually exist in storage are batch-converted. + let mut default_scalars: FxHashMap = FxHashMap::default(); + let mut cached_default_scalar = |bytes: CommitmentBytes| { + *default_scalars + .entry(bytes) + .or_insert_with(|| to_element(bytes).map_to_scalar_field()) + }; - // Load child commitments for each internal node - for (node_id, _) in nodes { + let mut polys: Vec> = Vec::with_capacity(nodes.len()); + // Children present in storage: `(node index, child index)`, and their commitments + let mut real_positions: Vec<(usize, usize)> = Vec::new(); + let mut real_children: Vec = Vec::new(); + + for (i, node_id) in nodes.iter().enumerate() { // Calculate starting index for this node's 256 children let child_idx = get_child_node(&logic_parent_id(*node_id), 0); @@ -111,26 +299,42 @@ where } else { child_idx // Non-root levels: all use child_idx as default }; - let mut child_commitments = vec![default_element(default_idx); DOMAIN_SIZE]; + let mut node_scalars = + vec![cached_default_scalar(default_commitment(default_idx)); DOMAIN_SIZE]; // Special case: root level first child uses different default if child_idx == ROOT_LEVEL_CHILD_START { - child_commitments[0] = default_element(child_idx); + node_scalars[0] = cached_default_scalar(default_commitment(child_idx)); } - // Replace defaults with actual commitments where they exist + // Record actual commitments to overwrite the defaults where they exist for (absolute_node_id, commitment_bytes) in children { // Subtract in u64 before narrowing: a 256-child range near the top of // a level-4 subtree straddles 2^32, so casting each id to a 32-bit // `usize` first would underflow. - let relative_index = (absolute_node_id - child_idx) as usize; - child_commitments[relative_index] = to_element(commitment_bytes); + real_positions.push((i, (absolute_node_id - child_idx) as usize)); + real_children.push(to_element(commitment_bytes)); } + polys.push(node_scalars); + } - all_child_commitments.extend(child_commitments); + // Batch-convert the existing children and scatter them into place. + let real_scalars = Element::batch_map_to_scalar_field(&real_children); + for ((node, position), scalar) in real_positions.into_iter().zip(real_scalars) { + polys[node][position] = scalar; } - Ok(Element::batch_map_to_scalar_field(&all_child_commitments)) + Ok(polys.into_iter().map(LagrangeBasis::new).collect()) +} + +/// One parent node of the subtrie: the logical id [`parents_and_points`] reports (which +/// encodes a bucket root's level), the physical id the store and the polynomial cache key by, +/// and its commitment as read for this proof. +struct ParentNode { + logical: NodeId, + physical: NodeId, + bytes: CommitmentBytes, + commitment: Element, } /// Creates IPA prover queries for a given commitment and evaluation points. @@ -144,21 +348,21 @@ where /// /// # Parameters /// * `commitment` - The cryptographic commitment to the polynomial -/// * `poly` - The polynomial in Lagrange basis form (256 coefficients) +/// * `poly` - The polynomial in Lagrange basis form (256 coefficients), shared by every query /// * `points` - Set of evaluation points (child indices) to create queries for /// /// # Returns /// A vector of `ProverQuery` objects, one for each evaluation point fn create_prover_queries( commitment: Element, - poly: LagrangeBasis, - points: BTreeSet, + poly: Arc, + points: &BTreeSet, ) -> Vec { points .iter() .map(|&i| ProverQuery { commitment, - poly: poly.clone(), + poly: Arc::clone(&poly), point: i, result: poly.evaluate_in_domain(i), }) @@ -201,144 +405,232 @@ where }); } - // Validate salt keys - salt_keys.iter().try_for_each(|key| { - let capacity = if key.is_in_meta_bucket() { - META_BUCKET_SIZE as u64 + // Steps 1 & 2: Validate every key against its bucket capacity and record the trie level + // of each bucket. Keys arrive sorted, so each bucket's keys form one run and its metadata + // is read once. + let mut buckets_level: FxHashMap = FxHashMap::default(); + for bucket_keys in salt_keys.chunk_by(|a, b| a.bucket_id() == b.bucket_id()) { + let bucket_id = bucket_keys[0].bucket_id(); + let (capacity, level) = if bucket_id < NUM_META_BUCKETS as BucketId { + // Metadata buckets are always at level 1 (never expand into subtrees) + (META_BUCKET_SIZE as u64, METADATA_BUCKET_LEVEL) } else { - store - .metadata(key.bucket_id()) + // Data buckets: read metadata to determine capacity and + // subtree structure (higher capacity = higher level root) + let meta = store + .metadata(bucket_id) .map_err(|e| ProofError::StateReadError { - reason: format!( - "Failed to read metadata for bucket {}: {e:?}", - key.bucket_id() - ), - })? - .capacity + reason: format!("Failed to read metadata for bucket {bucket_id}: {e:?}"), + })?; + let level = MAX_SUBTREE_LEVELS - subtree_root_level(meta.capacity); + (meta.capacity, level as u8) }; - - (key.slot_id() < capacity) - .then_some(()) - .ok_or(ProofError::InvalidSaltKey { + buckets_level.insert(bucket_id, level); + if let Some(key) = bucket_keys.iter().find(|key| key.slot_id() >= capacity) { + return Err(ProofError::InvalidSaltKey { key: *key, capacity, - }) - })?; + }); + } + } - // Step 1: Extract and deduplicate bucket IDs from the input keys - let mut bucket_ids = salt_keys.iter().map(|k| k.bucket_id()).collect::>(); - bucket_ids.dedup(); + // Step 3: Build the minimal node hierarchy needed for authentication + let (internal_nodes, leaf_nodes) = parents_and_points(salt_keys, &buckets_level); - // Step 2: Determine the trie level for each bucket - let buckets_level: FxHashMap = bucket_ids - .into_iter() - .map(|bucket_id| { - if bucket_id < NUM_META_BUCKETS as BucketId { - // Metadata buckets are always at level 1 (never expand into subtrees) - Ok((bucket_id, METADATA_BUCKET_LEVEL)) - } else { - // Data buckets: read metadata to determine subtree structure - let meta = store - .metadata(bucket_id) + // Step 4: Read every parent's commitment in parallel, internal nodes first, then leaves; + // steps 5 and 6 index this list instead of re-reading the store. + let parents: Vec = { + let logical_ids: Vec = internal_nodes + .keys() + .chain(leaf_nodes.keys()) + .copied() + .collect(); + // A store read is too short a task to split below one chunk per thread. + iter!(logical_ids, thread_chunk_size!(logical_ids.len())) + .map(|&logical| { + let physical = connect_parent_id(logical); + let bytes = store + .commitment(physical) .map_err(|e| ProofError::StateReadError { - reason: format!("Failed to read metadata for bucket {bucket_id}: {e:?}"), + reason: format!("Failed to load commitment for node {physical}: {e:?}"), })?; - // Convert capacity to subtree root level (higher capacity = higher level root) - let level = MAX_SUBTREE_LEVELS - subtree_root_level(meta.capacity); - Ok((bucket_id, level as u8)) - } - }) - .collect::>()?; + Ok(ParentNode { + logical, + physical, + bytes, + commitment: Element::from_bytes_unchecked_uncompressed(bytes), + }) + }) + .collect::>()? + }; + let parents_commitments: BTreeMap = parents + .iter() + .map(|parent| (parent.physical, SerdeCommitment(parent.commitment))) + .collect(); + + // Step 5: Resolve the polynomial of every parent, internal nodes in parallel chunks and + // leaves one per task. + let (internal_parents, leaf_parents) = parents.split_at(internal_nodes.len()); + let cache = NodePolyCache::global(); + let internal_polys = resolve_polys(cache, internal_parents, |missing| { + let chunk_size = thread_chunk_size!(missing.len()); + let chunks = chunks!(missing, chunk_size) + .map(|nodes| internal_polynomials(store, nodes)) + .collect::>>()?; + Ok(chunks.into_iter().flatten().collect()) + })?; + let leaf_polys = resolve_polys(cache, leaf_parents, |missing| { + iter!(missing) + .map(|&node| leaf_polynomial(store, node)) + .collect() + })?; - // Step 3: Build the minimal node hierarchy needed for authentication - let (internal_nodes, leaf_nodes) = parents_and_points(salt_keys, &buckets_level); + // Step 6: One IPA prover query per proven point, in node order. + let points = internal_nodes.into_values().chain(leaf_nodes.into_values()); + let polys = internal_polys.into_iter().chain(leaf_polys); + let mut queries: Vec = Vec::new(); + for ((parent, points), poly) in parents.iter().zip(points).zip(polys) { + queries.extend(create_prover_queries(parent.commitment, poly, &points)); + } + + Ok((queries, parents_commitments, buckets_level)) +} - // Step 4: Collect cryptographic commitments for all parent nodes - let parents_commitments: BTreeMap = internal_nodes +/// The polynomial of every parent in `parents`, in order: query order is node order, which +/// the transcript hashes, so a cache hit must not reorder the proof. A parent whose polynomial +/// `cache` holds for the commitment it was read at skips `rebuild`; the misses are rebuilt +/// in one call (by logical id, one polynomial per miss) and cached under that commitment, but +/// only if the polynomial opens to it. +fn resolve_polys( + cache: &NodePolyCache, + parents: &[ParentNode], + rebuild: impl FnOnce(&[NodeId]) -> ProofResult>, +) -> ProofResult>> { + let hits: Vec>> = parents .iter() - .chain(leaf_nodes.iter()) - .map(|(&parent, _)| { - let physical_parent = connect_parent_id(parent); - let commitment = Element::from_bytes_unchecked_uncompressed( - store - .commitment(physical_parent) - .map_err(|e| ProofError::StateReadError { - reason: format!( - "Failed to load commitment for node {physical_parent}: {e:?}" - ), - })?, - ); + .map(|parent| cache.get(parent.physical, &parent.bytes)) + .collect(); + let misses: Vec<&ParentNode> = parents + .iter() + .zip(&hits) + .filter(|(_, hit)| hit.is_none()) + .map(|(parent, _)| parent) + .collect(); + let missing: Vec = misses.iter().map(|parent| parent.logical).collect(); + let rebuilt: Vec> = rebuild(&missing)?.into_iter().map(Arc::new).collect(); + + // A polynomial rebuilt from a store that does not match the commitment must not be cached + // under it: the cache is process-wide, so every later proof reading that commitment would + // reuse it. A consistent store passes the one check for the whole call; only a call that + // fails it is gone through node by node, to cache the polynomials that do open. A cache + // that is turned off takes nothing, so there is nothing to check. + if cache.capacity() != 0 { + let consistent: Vec = if batch_opens(&misses, &rebuilt) { + vec![true; misses.len()] + } else { + iter!(misses) + .zip(iter!(rebuilt)) + .map(|(parent, poly)| opens_to(poly, &parent.commitment)) + .collect() + }; + for ((parent, poly), consistent) in misses.iter().zip(&rebuilt).zip(consistent) { + if consistent { + cache.insert(parent.physical, parent.bytes, Arc::clone(poly)); + } + } + } - Ok((physical_parent, SerdeCommitment(commitment))) - }) - .collect::>()?; + let mut rebuilt = rebuilt.into_iter(); + Ok(hits + .into_iter() + .map(|hit| hit.unwrap_or_else(|| rebuilt.next().expect("one rebuilt polynomial per miss"))) + .collect()) +} - // Step 5: Generate IPA prover queries for each node in internal nodes - let in_nodes: Vec<_> = internal_nodes.into_iter().collect(); - let chunk_size = in_nodes.len().div_ceil(num_threads!()); - let mut queries = chunks!(in_nodes, chunk_size) - .map(|nodes| { - let children_scalars = multi_commitments_to_scalars(store, nodes)?; +/// Whether every polynomial in `rebuilt` opens to the commitment its node was read at, +/// decided for all of them at once. +/// +/// Checking a node on its own costs a 256-term fixed-base MSM ([`opens_to`]), and one per +/// miss is most of the work of a cold proof. Here every node gets a challenge `r_i`, and +/// `Σ r_i·poly_i` is committed once and compared with `Σ r_i·C_i`. The two are equal when +/// every polynomial opens to its commitment. When one does not, they are equal only if the +/// challenges cancel the differences, and the challenges are 128 bits drawn from a hash of +/// the very commitments and polynomials being checked ([`batch_challenges`]), so that happens +/// with probability about 2^-128. +fn batch_opens(misses: &[&ParentNode], rebuilt: &[Arc]) -> bool { + let challenges = batch_challenges(misses, rebuilt); + + // Σ r_i·poly_i, one partial sum per thread. + let chunk_size = thread_chunk_size!(rebuilt.len()); + let combined = reduce!( + chunks!(rebuilt, chunk_size) + .zip(chunks!(challenges, chunk_size)) + .map(|(polys, challenges)| { + let mut sum = [Fr::zero(); DOMAIN_SIZE]; + for (poly, challenge) in polys.iter().zip(challenges) { + for (term, value) in sum.iter_mut().zip(poly.values()) { + *term += *value * challenge; + } + } + sum + }), + || [Fr::zero(); DOMAIN_SIZE], + |mut sum, part| { + for (term, part) in sum.iter_mut().zip(part) { + *term += part; + } + sum + } + ); - let res = nodes - .iter() - .zip(children_scalars.chunks(DOMAIN_SIZE)) - .map(|((parent, points), children_scalars)| { - let commitment = store - .commitment(connect_parent_id(*parent)) - .map(Element::from_bytes_unchecked_uncompressed) - .map_err(|e| ProofError::StateReadError { - reason: format!("Failed to load commitment for node {parent}: {e:?}"), - })?; - - Ok(create_prover_queries( - commitment, - LagrangeBasis::new(children_scalars.to_vec()), - points.clone(), - )) - }) - .collect::>>()? - .into_iter() - .flatten() - .collect::>(); + let commitments: Vec = misses.iter().map(|parent| parent.commitment).collect(); + shared_committer().msm_dense(&combined) == multi_scalar_mul(&commitments, &challenges) +} - Ok(res) - }) - .collect::>>()? - .into_iter() - .flatten() - .collect::>(); - - // Step 6: Generate IPA prover queries for each node in leaf nodes - let leaf_queries = into_iter!(leaf_nodes) - .map(|(parent, points)| { - let physical_parent = connect_parent_id(parent); - let parent_commitment = store - .commitment(physical_parent) - .map(Element::from_bytes_unchecked_uncompressed) - .map_err(|e| ProofError::StateReadError { - reason: format!( - "Failed to load element commitment for node {physical_parent}: {e:?}" - ), - })?; - process_leaf_node(store, parent, parent_commitment, points) +/// One 128-bit challenge per miss, drawn from a hash of the whole batch: each node's +/// commitment bytes and the 256 evaluations of its polynomial, in node order. +fn batch_challenges(misses: &[&ParentNode], rebuilt: &[Arc]) -> Vec { + const SCALAR_BYTES: usize = 32; + + // Each node is hashed on its own, so the polynomials (8 KiB apiece) are absorbed in + // parallel and the batch hash below only reads a digest per node. + let digests: Vec<[u8; blake3::OUT_LEN]> = iter!(misses) + .zip(iter!(rebuilt)) + .map(|(parent, poly)| { + let mut evaluations = [0u8; DOMAIN_SIZE * SCALAR_BYTES]; + let limbs = poly.values().iter().flat_map(|value| value.into_bigint().0); + for (bytes, limb) in evaluations.chunks_exact_mut(8).zip(limbs) { + bytes.copy_from_slice(&limb.to_le_bytes()); + } + let mut hasher = blake3::Hasher::new(); + hasher.update(&parent.bytes).update(&evaluations); + *hasher.finalize().as_bytes() }) - .collect::>>()? - .into_iter() - .flatten(); + .collect(); - queries.extend(leaf_queries); + let mut hasher = blake3::Hasher::new(); + for digest in &digests { + hasher.update(digest); + } + let mut output = hasher.finalize_xof(); + digests + .iter() + .map(|_| { + let mut challenge = [0u8; 16]; + output.fill(&mut challenge); + Fr::from(u128::from_le_bytes(challenge)) + }) + .collect() +} - Ok((queries, parents_commitments, buckets_level)) +/// Whether `poly` commits to `commitment` under the shared CRS. +fn opens_to(poly: &LagrangeBasis, commitment: &Element) -> bool { + shared_committer().msm_dense(poly.values()) == *commitment } -/// Processes a leaf node to create prover queries. -fn process_leaf_node( - store: &Store, - parent: NodeId, - parent_commitment: Element, - points: BTreeSet, -) -> ProofResult> +/// The 256-slot polynomial of a leaf node: every slot's value hashed to the field, with the +/// bucket kind's default for empty slots. +fn leaf_polynomial(store: &Store, parent: NodeId) -> ProofResult where Store: StateReader, { @@ -368,27 +660,24 @@ where })?; // Initialize polynomial coefficients with appropriate default values - let mut default_coefficients = if bucket_id < NUM_META_BUCKETS as BucketId { - // Metadata buckets: initialize with default metadata hash - vec![slot_to_field(&Some(BucketMeta::default().into())); DOMAIN_SIZE] - } else { - // Data buckets: initialize with empty slot hash - vec![slot_to_field(&None); DOMAIN_SIZE] - }; + let mut default_coefficients = vec![empty_slot_scalar(bucket_id); DOMAIN_SIZE]; // Replace default values with actual key-value hashes where data exists for (key, value) in entries { - // Map slot ID to polynomial coefficient index (last 8 bits) - let index = (key.slot_id() & SLOT_INDEX_MASK) as usize; - default_coefficients[index] = slot_to_field(&Some(value)); + default_coefficients[slot_position(&key)] = slot_to_field(&Some(value)); } - // Create IPA prover queries for the specified evaluation points - Ok(create_prover_queries( - parent_commitment, - LagrangeBasis::new(default_coefficients), - points, - )) + Ok(LagrangeBasis::new(default_coefficients)) +} + +/// The scalar of an empty slot in `bucket_id`: the default metadata's hash for a metadata +/// bucket, the empty slot's hash for a data bucket. +fn empty_slot_scalar(bucket_id: BucketId) -> Fr { + if bucket_id < NUM_META_BUCKETS as BucketId { + slot_to_field(&Some(BucketMeta::default().into())) + } else { + slot_to_field(&None) + } } #[cfg(test)] @@ -397,8 +686,10 @@ mod tests { use crate::constant::{META_BUCKET_SIZE, ROOT_NODE_ID}; use crate::proof::test_utils::*; use crate::{ - mem_store::MemStore, proof::prover::PRECOMPUTED_WEIGHTS, state::state::EphemeralSaltState, - trie::trie::StateRoot, + mem_store::MemStore, + proof::prover::PRECOMPUTED_WEIGHTS, + state::state::EphemeralSaltState, + trie::trie::{StateRoot, DEFAULT_CRS}, }; use ark_ff::BigInt; use banderwagon::{Fr, Zero}; @@ -467,7 +758,7 @@ mod tests { let poly = LagrangeBasis::new(coeffs); let query = ProverQuery { commitment: crs.commit_lagrange_poly(&poly), - poly: poly.clone(), + poly: poly.clone().into(), point: 208, result: poly.evaluate_in_domain(208), }; @@ -520,25 +811,27 @@ mod tests { } #[test] - fn process_leaf_node_with_real_commitment() { + fn leaf_polynomial_with_real_commitment() { let (store, salt_key) = setup_test_store(); let parent_node = STARTING_NODE_ID[3] + salt_key.bucket_id() as u64; let commitment = Element::from_bytes_unchecked_uncompressed(store.commitment(parent_node).unwrap()); - let points = [0, salt_key.slot_id() as usize & 0xff].into(); + let points = [0, slot_position(&salt_key)].into(); - let queries = process_leaf_node(&store, parent_node, commitment, points).unwrap(); + let poly = Arc::new(leaf_polynomial(&store, parent_node).unwrap()); + let queries = create_prover_queries(commitment, poly, &points); assert_eq!(queries.len(), 2); assert!(verify_ipa_proof(queries)); } #[test] - fn process_leaf_node_uses_metadata_default_for_meta_bucket() { + fn leaf_polynomial_uses_metadata_default_for_meta_bucket() { let store = MemStore::new(); let parent_node = STARTING_NODE_ID[3]; let commitment = Element::from_bytes_unchecked_uncompressed(default_commitment(parent_node)); - let queries = process_leaf_node(&store, parent_node, commitment, [0, 255].into()).unwrap(); + let poly = Arc::new(leaf_polynomial(&store, parent_node).unwrap()); + let queries = create_prover_queries(commitment, poly, &[0, 255].into()); let metadata_default = slot_to_field(&Some(BucketMeta::default().into())); assert_eq!( @@ -549,12 +842,13 @@ mod tests { } #[test] - fn process_leaf_node_uses_empty_default_for_data_bucket() { + fn leaf_polynomial_uses_empty_default_for_data_bucket() { let store = MemStore::new(); let parent_node = STARTING_NODE_ID[3] + NUM_META_BUCKETS as NodeId; let commitment = Element::from_bytes_unchecked_uncompressed(default_commitment(parent_node)); - let queries = process_leaf_node(&store, parent_node, commitment, [0, 255].into()).unwrap(); + let poly = Arc::new(leaf_polynomial(&store, parent_node).unwrap()); + let queries = create_prover_queries(commitment, poly, &[0, 255].into()); let empty_default = slot_to_field(&None); assert_eq!( @@ -564,24 +858,383 @@ mod tests { assert!(queries.iter().all(|q| q.result == empty_default)); } + /// A cache bounded to four polynomials per shard, with the pieces the cache tests share: + /// node `i` of one shard, a commitment made of one byte, and a polynomial to store. + fn small_cache() -> ( + NodePolyCache, + impl Fn(usize) -> NodeId, + impl Fn(u8) -> CommitmentBytes, + Arc, + ) { + use node_poly_cache::SHARDS; + ( + NodePolyCache::with_capacity(4 * SHARDS), + // Nodes `7 + i * SHARDS` share one shard. + |i| (7 + i * SHARDS) as NodeId, + |byte| [byte; 64], + Arc::new(LagrangeBasis::new(vec![Fr::zero(); DOMAIN_SIZE])), + ) + } + + /// A full shard evicts one polynomial per new node: the first the hand finds unused since + /// its last visit. A polynomial served, or a node cached again, counts as used; a lookup + /// at another commitment does not, and neither is a use that the hand has already passed. + #[test] + fn node_poly_cache_evicts_what_was_not_used_since_the_hand_passed() { + let (cache, node, commitment, poly) = small_cache(); + let nodes = |held: &[usize]| held.iter().map(|&i| node(i)).collect::>(); + let insert = |i: usize, byte: u8| cache.insert(node(i), commitment(byte), poly.clone()); + let hit = |i: usize, byte: u8| cache.get(node(i), &commitment(byte)).is_some(); + + // A node of another shard, which nothing below disturbs. + cache.insert(8, commitment(1), poly.clone()); + let with_other_shard = |held: &[usize]| { + let mut all = nodes(held); + all.push(8); + all.sort_unstable(); + all + }; + + (0..4).for_each(|i| insert(i, 1)); + assert_eq!(cache.nodes(), with_other_shard(&[0, 1, 2, 3])); + assert!(hit(0, 1) && hit(2, 1)); + assert!(!hit(1, 2), "another commitment is a miss"); + assert!(!hit(9, 1), "a node the cache does not hold is a miss"); + + // The hand passes node 0 (used) and evicts node 1, then passes node 2 and evicts 3. + insert(4, 1); + assert_eq!(cache.nodes(), with_other_shard(&[0, 2, 3, 4])); + insert(5, 1); + assert_eq!(cache.nodes(), with_other_shard(&[0, 2, 4, 5])); + // Back at node 0, whose use it has already passed over once. + insert(6, 1); + assert_eq!(cache.nodes(), with_other_shard(&[2, 4, 5, 6])); + + // Caching node 4 again replaces its polynomial in place and counts as a use, so the + // hand passes it and evicts node 2. + insert(4, 2); + assert_eq!(cache.nodes(), with_other_shard(&[2, 4, 5, 6])); + insert(7, 1); + assert_eq!(cache.nodes(), with_other_shard(&[4, 5, 6, 7])); + assert!(hit(4, 2) && !hit(4, 1)); + + // With every polynomial used, the hand makes a full turn and evicts where it began. + assert!(hit(5, 1) && hit(6, 1) && hit(7, 1)); + insert(9, 1); + assert_eq!(cache.nodes(), with_other_shard(&[4, 6, 7, 9])); + } + + /// The capacity is a bound per shard, rounded up. Lowering it empties the shards that hold + /// more than their new share and leaves the rest; zero turns the cache off, and raising + /// it again turns it back on. Clearing drops everything and changes no bound. #[test] - fn multi_commitments_to_scalars_empty_and_single_node() { + fn node_poly_cache_capacity_bounds_trims_and_turns_the_cache_off() { + use node_poly_cache::SHARDS; + + let (cache, node, commitment, poly) = small_cache(); + assert_eq!(cache.capacity(), 4 * SHARDS); + assert_eq!(NodePolyCache::DEFAULT_CAPACITY, 32768); + assert_eq!( + NodePolyCache::global().capacity(), + NodePolyCache::DEFAULT_CAPACITY + ); + + // Three nodes in one shard, one in another. + (0..3).for_each(|i| cache.insert(node(i), commitment(1), poly.clone())); + cache.insert(8, commitment(1), poly.clone()); + let everything = vec![node(0), 8, node(1), node(2)]; + assert_eq!(cache.nodes(), everything); + + cache.set_capacity(2 * SHARDS + 1); + assert_eq!(cache.capacity(), 3 * SHARDS); + assert_eq!(cache.nodes(), everything, "no shard is over three"); + cache.set_capacity(2 * SHARDS); + assert_eq!(cache.nodes(), [8], "the shard holding three is emptied"); + + cache.set_capacity(0); + assert_eq!(cache.capacity(), 0); + assert!(cache.nodes().is_empty()); + cache.insert(node(0), commitment(1), poly.clone()); + assert!( + cache.nodes().is_empty(), + "a cache that is off takes nothing" + ); + + cache.set_capacity(1); + assert_eq!(cache.capacity(), SHARDS); + cache.insert(node(0), commitment(1), poly.clone()); + cache.insert(8, commitment(1), poly.clone()); + assert_eq!(cache.nodes(), [node(0), 8]); + // One polynomial per shard: a second node of a shard takes the first one's place. + cache.insert(node(1), commitment(1), poly.clone()); + assert_eq!(cache.nodes(), [8, node(1)]); + + cache.clear(); + assert!(cache.nodes().is_empty()); + assert_eq!(cache.capacity(), SHARDS); + cache.insert(node(0), commitment(1), poly); + assert_eq!(cache.nodes(), [node(0)]); + } + + /// Regression test for PR #152 review comment r4080228453: a store whose trie and state + /// disagree must not poison the process-wide node-polynomial cache for consistent stores. + /// + /// The cache is keyed by (node id, commitment). On a miss, `resolve_polys` reads the + /// node's commitment and rebuilds its polynomial from the store's slots or children; it + /// files the result only if the polynomial opens to that commitment. + /// + /// Here `inconsistent` has block 2's state but block 1's trie (`update_state` without + /// `update_trie`), so the leaf holding `key` reports block 1's commitment while its slots + /// hold block 2's value. Without the check, creating a witness on it files block 2's leaf + /// polynomial under block 1's commitment, `consistent` (a correct store of block 1) is + /// then served that polynomial, and its witness no longer verifies. + /// + /// The `inconsistent` witness is created first on purpose: a cache entry for block 1's + /// commitment that `consistent` had already filed would hide the problem. The key and + /// values come from a seed no other test uses, so the only entries this test could poison + /// are ones no other test in the process reads. + #[test] + fn inconsistent_store_does_not_poison_witnesses_of_a_consistent_store() { + use crate::proof::salt_witness::SaltWitness; + + let mut rng = StdRng::seed_from_u64(0x5A17_0152); + let key = mock_data(&mut rng, 52); + let (block_1_value, block_2_value) = (mock_data(&mut rng, 32), mock_data(&mut rng, 32)); + let write = |value: &Vec| -> HashMap, Option>> { + [(key.clone(), Some(value.clone()))].into_iter().collect() + }; + + // Block 1, fully applied: state and trie agree. + let consistent = MemStore::new(); + let block_1 = EphemeralSaltState::new(&consistent) + .update_fin(&write(&block_1_value)) + .unwrap(); + consistent.update_state(block_1.clone()); + let (_, block_1_trie) = StateRoot::new(&consistent).update_fin(&block_1).unwrap(); + consistent.update_trie(block_1_trie); + let salt_key = *block_1.data.keys().next().unwrap(); + + // Block 2 on top of it, with only its state applied: the commitments are block 1's. + let inconsistent = consistent.clone(); + let block_2 = EphemeralSaltState::new(&inconsistent) + .update_fin(&write(&block_2_value)) + .unwrap(); + assert_eq!(*block_2.data.keys().next().unwrap(), salt_key); + inconsistent.update_state(block_2); + + // A witness from the inconsistent store cannot verify: its value is block 2's, its + // commitments are block 1's. That is the expected cost of the bad store itself. + let bad = SaltWitness::create(&[salt_key], &inconsistent).unwrap(); + assert!(bad.verify_proof().is_err()); + + // The consistent store must be unaffected by what the other store did. + let good = SaltWitness::create(&[salt_key], &consistent).unwrap(); + good.verify_proof().expect( + "a witness from a consistent store must verify, \ + whatever other stores the process has proven before", + ); + } + + /// `count` nodes with random polynomials, each read at the commitment its polynomial + /// opens to. The commitments come from the CRS directly, not from the committer the + /// checks under test use, and a node's physical id differs from its logical one. + fn committed_nodes(seed: u64, count: usize) -> (Vec, Vec>) { + let mut rng = StdRng::seed_from_u64(seed); + (0..count as NodeId) + .map(|logical| { + let poly = LagrangeBasis::new( + (0..DOMAIN_SIZE) + .map(|_| Fr::from_le_bytes_mod_order(&mock_data(&mut rng, 32))) + .collect(), + ); + let bytes = DEFAULT_CRS + .commit_lagrange_poly(&poly) + .to_bytes_uncompressed(); + let node = ParentNode { + logical, + physical: logical + 1000, + bytes, + commitment: Element::from_bytes_unchecked_uncompressed(bytes), + }; + (node, Arc::new(poly)) + }) + .unzip() + } + + /// `poly` with the evaluation at `index` zeroed, which no longer opens to `poly`'s + /// commitment: a random evaluation is not zero. + fn zeroed_at(poly: &LagrangeBasis, index: usize) -> LagrangeBasis { + let mut evaluations: Vec = (0..DOMAIN_SIZE) + .map(|i| poly.evaluate_in_domain(i)) + .collect(); + assert!(!evaluations[index].is_zero()); + evaluations[index] = Fr::zero(); + LagrangeBasis::new(evaluations) + } + + /// The batch check passes when every polynomial opens to its node's commitment, and fails + /// when any does not, however small the difference and wherever it sits in the batch. + #[test] + fn batch_check_passes_only_when_every_polynomial_opens_to_its_commitment() { + let (nodes, polys) = committed_nodes(0xBA7C, 5); + let misses: Vec<&ParentNode> = nodes.iter().collect(); + + assert!(batch_opens(&misses, &polys)); + assert!(batch_opens(&misses[..1], &polys[..1])); + assert!(batch_opens(&[], &[])); + + // One evaluation of one polynomial differs. + for (node, index) in [(0, 0), (3, DOMAIN_SIZE - 1), (4, 17)] { + let mut off = polys.clone(); + off[node] = Arc::new(zeroed_at(&polys[node], index)); + assert!(!batch_opens(&misses, &off), "node {node}, index {index}"); + assert!(!batch_opens(&misses[node..=node], &off[node..=node])); + } + + // Two polynomials trade places. Each still opens to a commitment of the batch, so + // the plain sums of the two sides agree; it is the per-node challenges that tell. + let mut swapped = polys.clone(); + swapped.swap(1, 2); + assert!(!batch_opens(&misses, &swapped)); + } + + /// The challenges are drawn from the batch itself: one per node, 128 bits each, and any + /// change to a commitment, a polynomial or the node order moves every one of them. + #[test] + fn batch_challenges_depend_on_every_commitment_and_polynomial() { + let (nodes, polys) = committed_nodes(0xC4A1, 3); + let misses: Vec<&ParentNode> = nodes.iter().collect(); + let challenges = batch_challenges(&misses, &polys); + + assert_eq!(challenges.len(), 3); + assert_eq!(challenges, batch_challenges(&misses, &polys)); + for (i, challenge) in challenges.iter().enumerate() { + assert!(!challenge.is_zero()); + assert_eq!(challenge.into_bigint().0[2..], [0, 0], "128 bits"); + assert!(!challenges[..i].contains(challenge)); + } + let all_moved = + |other: Vec| other.len() == 3 && other.iter().zip(&challenges).all(|(a, b)| a != b); + + // The first and the last evaluation of a polynomial. + for (node, index) in [(0, 0), (2, DOMAIN_SIZE - 1)] { + let mut changed = polys.clone(); + changed[node] = Arc::new(zeroed_at(&polys[node], index)); + assert!(all_moved(batch_challenges(&misses, &changed))); + } + + // One bit of a commitment's bytes. + let mut bytes = nodes[1].bytes; + bytes[63] ^= 1; + let rewritten = ParentNode { + logical: nodes[1].logical, + physical: nodes[1].physical, + bytes, + commitment: nodes[1].commitment, + }; + assert!(all_moved(batch_challenges( + &[&nodes[0], &rewritten, &nodes[2]], + &polys + ))); + + // The order of the nodes. + let reversed: Vec> = polys.iter().rev().cloned().collect(); + assert!(all_moved(batch_challenges( + &[&nodes[2], &nodes[1], &nodes[0]], + &reversed + ))); + } + + /// `resolve_polys` hands the proof every polynomial it rebuilt, but caches only those that + /// open to the commitment their node was read at: one that does not is rebuilt again by + /// the next call, and the ones that do are served from the cache from then on. + #[test] + fn resolve_polys_caches_only_the_rebuilds_that_open_to_their_commitment() { + let cache = NodePolyCache::with_capacity(NodePolyCache::DEFAULT_CAPACITY); + let (nodes, polys) = committed_nodes(0x0152, 4); + let rebuild = |poly: &Arc| LagrangeBasis::clone(poly); + // What an inconsistent store rebuilds for node 2. + let stale = zeroed_at(&polys[2], 0); + + let logical_ids: Vec = nodes.iter().map(|node| node.logical).collect(); + let first = resolve_polys(&cache, &nodes, |missing| { + assert_eq!(missing, logical_ids); + Ok(vec![ + rebuild(&polys[0]), + rebuild(&polys[1]), + stale.clone(), + rebuild(&polys[3]), + ]) + }) + .unwrap(); + assert_eq!(*first[2], stale); + for (i, node) in nodes.iter().enumerate() { + let cached = cache.get(node.physical, &node.bytes); + if i == 2 { + assert!(cached.is_none()); + } else { + assert_eq!(first[i], polys[i]); + assert_eq!(cached.as_ref(), Some(&polys[i])); + } + assert!(cache.get(node.logical, &node.bytes).is_none()); + } + + // Node 2 alone is rebuilt again, this time from a consistent store. + let second = resolve_polys(&cache, &nodes, |missing| { + assert_eq!(missing, [nodes[2].logical]); + Ok(vec![rebuild(&polys[2])]) + }) + .unwrap(); + assert_eq!(second, polys); + + let third = resolve_polys(&cache, &nodes, |missing| { + assert!(missing.is_empty()); + Ok(Vec::new()) + }) + .unwrap(); + assert_eq!(third, polys); + } + + /// A cache that is turned off serves and takes nothing: every call rebuilds every node. + #[test] + fn resolve_polys_rebuilds_everything_when_the_cache_is_off() { + let cache = NodePolyCache::with_capacity(0); + let (nodes, polys) = committed_nodes(0x0FF, 3); + let logical_ids: Vec = nodes.iter().map(|node| node.logical).collect(); + + for _ in 0..2 { + let resolved = resolve_polys(&cache, &nodes, |missing| { + assert_eq!(missing, logical_ids); + Ok(polys + .iter() + .map(|poly| LagrangeBasis::clone(poly)) + .collect()) + }) + .unwrap(); + assert_eq!(resolved, polys); + assert!(cache.nodes().is_empty()); + } + } + + #[test] + fn internal_polynomials_empty_and_single_node() { let store = MemStore::new(); // Empty input - assert_eq!(multi_commitments_to_scalars(&store, &[]).unwrap().len(), 0); + assert!(internal_polynomials(&store, &[]).unwrap().is_empty()); // Single internal node - let node_points = vec![(STARTING_NODE_ID[2], [0, 1].into())]; - let scalars = multi_commitments_to_scalars(&store, &node_points).unwrap(); - assert_eq!(scalars.len(), DOMAIN_SIZE); + let polys = internal_polynomials(&store, &[STARTING_NODE_ID[2]]).unwrap(); + assert_eq!(polys.len(), 1); } #[test] - fn multi_commitments_to_scalars_root_defaults_are_pinned() { + fn internal_polynomials_root_defaults_are_pinned() { let store = MemStore::new(); - let scalars = - multi_commitments_to_scalars(&store, &[(ROOT_NODE_ID, [0, 1, 255].into())]).unwrap(); + let polys = internal_polynomials(&store, &[ROOT_NODE_ID]).unwrap(); + let scalars: Vec = (0..DOMAIN_SIZE) + .map(|i| polys[0].evaluate_in_domain(i)) + .collect(); let expected = Element::batch_map_to_scalar_field(&[ Element::from_bytes_unchecked_uncompressed(default_commitment(STARTING_NODE_ID[1])), Element::from_bytes_unchecked_uncompressed(default_commitment(STARTING_NODE_ID[1] + 1)), diff --git a/salt/src/proof/test_utils.rs b/salt/src/proof/test_utils.rs index 38eb35a8..19207ebb 100644 --- a/salt/src/proof/test_utils.rs +++ b/salt/src/proof/test_utils.rs @@ -2,7 +2,10 @@ //! //! This module provides common mock data generation functions used across proof tests. +use crate::mem_store::MemStore; use crate::proof::SerdeCommitment; +use crate::state::updates::StateUpdates; +use crate::trie::trie::StateRoot; use crate::types::SaltValue; use banderwagon::{Element, Fr}; use rand::{rngs::StdRng, Rng}; @@ -38,3 +41,10 @@ pub(crate) fn mock_commitment() -> SerdeCommitment { pub(crate) fn mock_salt_value() -> SaltValue { SaltValue::new(&[1u8; 32], &[2u8; 32]) } + +/// Applies one block's transition to `store`. +pub(crate) fn apply_block(store: &MemStore, updates: StateUpdates) { + let (_, trie_updates) = StateRoot::new(store).update_fin(&updates).unwrap(); + store.update_state(updates); + store.update_trie(trie_updates); +} diff --git a/salt/src/proof/witness.rs b/salt/src/proof/witness.rs index 8ded614d..d7b0aafc 100644 --- a/salt/src/proof/witness.rs +++ b/salt/src/proof/witness.rs @@ -460,9 +460,7 @@ mod tests { let witness = Witness::create([], kvs.keys(), &BTreeMap::new(), &store).unwrap(); // Test serialization round-trip of the underlying SaltWitness - let serialized = - bincode::serde::encode_to_vec(&witness.salt_witness, bincode::config::legacy()) - .unwrap(); + let serialized = encode_salt_witness(&witness); let (deserialized, _): (SaltWitness, _) = bincode::serde::decode_from_slice(&serialized, bincode::config::legacy()).unwrap(); @@ -1173,4 +1171,31 @@ mod tests { .iter() .all(|&id| witness.metadata(id).unwrap() == new_metadata)); } + + /// The bytes of `witness`'s SALT witness, as the proof byte-identity tests compare them. + fn encode_salt_witness(witness: &Witness) -> Vec { + bincode::serde::encode_to_vec(&witness.salt_witness, bincode::config::legacy()).unwrap() + } + + /// A node-polynomial cache hit must reproduce the miss byte for byte: the same inputs + /// witnessed twice serialize identically (the second pass is served from the cache), and + /// the witness still verifies. + #[test] + fn cached_node_polynomials_reproduce_the_witness_byte_for_byte() { + let mut rng = StdRng::seed_from_u64(7); + let kvs: HashMap<_, _> = (0..64) + .map(|_| (mock_data(&mut rng, 20), Some(mock_data(&mut rng, 40)))) + .collect(); + let store = MemStore::new(); + apply_block( + &store, + EphemeralSaltState::new(&store).update_fin(&kvs).unwrap(), + ); + + let build = || Witness::create([], kvs.keys(), &BTreeMap::new(), &store).unwrap(); + let first = encode_salt_witness(&build()); + let second = encode_salt_witness(&build()); + assert_eq!(first, second, "a cache hit must not change the proof"); + build().verify().unwrap(); + } } diff --git a/salt/src/trie/trie.rs b/salt/src/trie/trie.rs index 6843c204..b64a4a1c 100644 --- a/salt/src/trie/trie.rs +++ b/salt/src/trie/trie.rs @@ -43,9 +43,20 @@ use std::{sync::Arc, vec}; use core::{cmp::Ordering, ops::Range}; +/// Shared default CRS. Constructing `CRS::default()` decompresses 257 points, +/// each costing a modular square root plus a subgroup check (several +/// milliseconds total), so it is done once per process and reused by the +/// shared committer and by every proof creation and verification. +pub(crate) static DEFAULT_CRS: Lazy = Lazy::new(CRS::default); + /// Global shared instance of the Committer to avoid repeated expensive initialization static SHARED_COMMITTER: Lazy> = Lazy::new(|| Arc::new(build_shared_committer())); +/// The process-wide committer: fixed-base tables over the shared CRS. +pub(crate) fn shared_committer() -> &'static Committer { + SHARED_COMMITTER.as_ref() +} + /// Builds the shared committer's precomputation tables. /// /// With `parallel` enabled, the construction is fully isolated from the @@ -59,7 +70,7 @@ static SHARED_COMMITTER: Lazy> = Lazy::new(|| Arc::new(build_shar /// touches `SHARED_COMMITTER` would re-enter this initialization on the /// same stack and deadlock. fn build_shared_committer() -> Committer { - let build = || Committer::new(&CRS::default().G, platform::DEFAULT_PRECOMP_WINDOW_SIZE); + let build = || DEFAULT_CRS.committer(platform::DEFAULT_PRECOMP_WINDOW_SIZE); #[cfg(feature = "parallel")] { // Never run `build` inline here: under `parallel` it would par_iter