Generalize ClarityStore related traits

changelog.d/clarity_backing_store_trait.changed

@@ -0,0 +1 @@
+Updated the ClarityBackingStore trait to be no more dependent on sqlite

changelog.d/clarity_store_trait.changed

@@ -0,0 +1 @@
+Updated ClarityStore and related traits to be non MARF-centric

clarity/src/vm/database/clarity_db.rs

@@ -628,7 +628,7 @@ impl<'a> ClarityDatabase<'a> {
     pub fn get_data_with_proof<T>(
         &mut self,
         key: &str,
-    ) -> Result<Option<(T, Vec<u8>)>, VmExecutionError>
+    ) -> Result<Option<(T, Option<Vec<u8>>)>, VmExecutionError>
     where
         T: ClarityDeserializable<T>,
     {
@@ -638,7 +638,7 @@ impl<'a> ClarityDatabase<'a> {
     pub fn get_data_with_proof_by_hash<T>(
         &mut self,
         hash: &TrieHash,
-    ) -> Result<Option<(T, Vec<u8>)>, VmExecutionError>
+    ) -> Result<Option<(T, Option<Vec<u8>>)>, VmExecutionError>
     where
         T: ClarityDeserializable<T>,
     {

clarity/src/vm/database/clarity_store.rs

@@ -57,16 +57,20 @@ pub trait ClarityBackingStore {
     fn get_data(&mut self, key: &str) -> Result<Option<String>, VmExecutionError>;
     /// fetch Hash(K)-V out of the commmitted datastore
     fn get_data_from_path(&mut self, hash: &TrieHash) -> Result<Option<String>, VmExecutionError>;
-    /// fetch K-V out of the committed datastore, along with the byte representation
-    ///  of the Merkle proof for that key-value pair
+    /// Fetch K-V out of the committed datastore, along with the byte representation of the
+    /// Merkle proof for that key-value pair -- if this backend is able to produce one. The
+    /// outer `Option` is `None` if the key doesn't exist; the inner `Option` is `None` if the
+    /// key exists but this backend has no proof to offer for it (e.g. a non-Merkleized
+    /// backend). Callers that were asked for a proof must treat that as "no proof available",
+    /// not as a valid empty proof.
     fn get_data_with_proof(
         &mut self,
         key: &str,
-    ) -> Result<Option<(String, Vec<u8>)>, VmExecutionError>;
+    ) -> Result<Option<(String, Option<Vec<u8>>)>, VmExecutionError>;
     fn get_data_with_proof_from_path(
         &mut self,
         hash: &TrieHash,
-    ) -> Result<Option<(String, Vec<u8>)>, VmExecutionError>;
+    ) -> Result<Option<(String, Option<Vec<u8>>)>, VmExecutionError>;
     fn has_entry(&mut self, key: &str) -> Result<bool, VmExecutionError> {
         Ok(self.get_data(key)?.is_some())
     }
@@ -87,9 +91,6 @@ pub trait ClarityBackingStore {
     fn get_open_chain_tip_height(&mut self) -> u32;
     fn get_open_chain_tip(&mut self) -> StacksBlockId;
 
-    #[cfg(feature = "rusqlite")]
-    fn get_side_store(&mut self) -> &Connection;
-
     fn get_cc_special_cases_handler(&self) -> Option<SpecialCaseHandler> {
         None
     }
@@ -144,6 +145,18 @@ pub trait ClarityBackingStore {
     }
 }
 
+/// Opt-in capability for backends that happen to store their data in sqlite.
+///
+/// This is *not* part of [`ClarityBackingStore`]: nothing about the Clarity VM's execution
+/// semantics requires sqlite, and a non-sqlite backend (e.g. a plain hashmap) has no reason to
+/// implement it. It exists only so that sqlite-backed implementations can share the free-function
+/// helpers in [`crate::vm::database::sqlite`], and so that tests/tools with a concrete sqlite-backed
+/// store in hand can still reach the raw connection for inspection.
+#[cfg(feature = "rusqlite")]
+pub trait SqliteBackingStore: ClarityBackingStore {
+    fn get_side_store(&mut self) -> &Connection;
+}
+
 // TODO: Figure out where this belongs
 pub fn make_contract_hash_key(contract: &QualifiedContractIdentifier) -> String {
     format!("clarity-contract::{contract}")
@@ -216,22 +229,17 @@ impl ClarityBackingStore for NullBackingStore {
     fn get_data_with_proof(
         &mut self,
         _key: &str,
-    ) -> Result<Option<(String, Vec<u8>)>, VmExecutionError> {
+    ) -> Result<Option<(String, Option<Vec<u8>>)>, VmExecutionError> {
         panic!("NullBackingStore can't retrieve data")
     }
 
     fn get_data_with_proof_from_path(
         &mut self,
         _hash: &TrieHash,
-    ) -> Result<Option<(String, Vec<u8>)>, VmExecutionError> {
+    ) -> Result<Option<(String, Option<Vec<u8>>)>, VmExecutionError> {
         panic!("NullBackingStore can't retrieve data")
     }
 
-    #[cfg(feature = "rusqlite")]
-    fn get_side_store(&mut self) -> &Connection {
-        panic!("NullBackingStore has no side store")
-    }
-
     fn get_block_at_height(&mut self, _height: u32) -> Option<StacksBlockId> {
         panic!("NullBackingStore can't get block at height")
     }

clarity/src/vm/database/key_value_wrapper.rs

@@ -359,11 +359,13 @@ impl RollbackWrapper<'_> {
     }
 
     /// this function will only return commitment proofs for values _already_ materialized
-    ///  in the underlying store. otherwise it returns None.
+    ///  in the underlying store. otherwise it returns None. The inner `Option<Vec<u8>>` is
+    ///  `None` if the backing store has no proof to offer for this value at all (e.g. a
+    ///  non-Merkleized backend), as distinct from the value simply not existing.
     pub fn get_data_with_proof<T>(
         &mut self,
         key: &str,
-    ) -> Result<Option<(T, Vec<u8>)>, VmExecutionError>
+    ) -> Result<Option<(T, Option<Vec<u8>>)>, VmExecutionError>
     where
         T: ClarityDeserializable<T>,
     {
@@ -374,11 +376,13 @@ impl RollbackWrapper<'_> {
     }
 
     /// this function will only return commitment proofs for values _already_ materialized
-    ///  in the underlying store. otherwise it returns None.
+    ///  in the underlying store. otherwise it returns None. The inner `Option<Vec<u8>>` is
+    ///  `None` if the backing store has no proof to offer for this value at all (e.g. a
+    ///  non-Merkleized backend), as distinct from the value simply not existing.
     pub fn get_data_with_proof_by_hash<T>(
         &mut self,
         hash: &TrieHash,
-    ) -> Result<Option<(T, Vec<u8>)>, VmExecutionError>
+    ) -> Result<Option<(T, Option<Vec<u8>>)>, VmExecutionError>
     where
         T: ClarityDeserializable<T>,
     {

clarity/src/vm/database/mod.rs

@@ -21,7 +21,10 @@
     BurnStateDB, ClarityDatabase, HeadersDB, NULL_BURN_STATE_DB, NULL_HEADER_DB,
     STORE_CONTRACT_SRC_INTERFACE, StoreType,
 };
+#[cfg(feature = "rusqlite")]
+pub use self::clarity_store::SqliteBackingStore;
 pub use self::clarity_store::{ClarityBackingStore, SpecialCaseHandler};
+pub use self::hashmap_store::HashMapBackingStore;
 pub use self::key_value_wrapper::{RollbackWrapper, RollbackWrapperPersistedLog};
 #[cfg(feature = "rusqlite")]
 pub use self::sqlite::{DATA_TABLE_NAME, METADATA_TABLE_NAME, MetadataRow, SqliteConnection};
@@ -33,6 +36,7 @@
 mod caching;
 pub mod clarity_db;
 pub mod clarity_store;
+pub mod hashmap_store;
 mod key_value_wrapper;
 #[cfg(feature = "rusqlite")]
 pub mod sqlite;

clarity/src/vm/database/sqlite.rs

@@ -21,7 +21,7 @@ use stacks_common::types::sqlite::NO_PARAMS;
 use stacks_common::util::db::tx_busy_handler;
 use stacks_common::util::hash::Sha512Trunc256Sum;
 
-use super::clarity_store::{ContractCommitment, make_contract_hash_key};
+use super::clarity_store::{ContractCommitment, SqliteBackingStore, make_contract_hash_key};
 use super::{
     ClarityBackingStore, ClarityDatabase, ClarityDeserializable, NULL_BURN_STATE_DB,
     NULL_HEADER_DB, SpecialCaseHandler,
@@ -108,7 +108,7 @@ pub fn sqlite_get_contract_hash(
 }
 
 pub fn sqlite_insert_metadata(
-    store: &mut dyn ClarityBackingStore,
+    store: &mut dyn SqliteBackingStore,
     contract: &QualifiedContractIdentifier,
     key: &str,
     value: &str,
@@ -124,7 +124,7 @@ pub fn sqlite_insert_metadata(
 }
 
 pub fn sqlite_get_metadata(
-    store: &mut dyn ClarityBackingStore,
+    store: &mut dyn SqliteBackingStore,
     contract: &QualifiedContractIdentifier,
     key: &str,
 ) -> Result<Option<String>, VmExecutionError> {
@@ -133,7 +133,7 @@ pub fn sqlite_get_metadata(
 }
 
 pub fn sqlite_get_metadata_manual(
-    store: &mut dyn ClarityBackingStore,
+    store: &mut dyn SqliteBackingStore,
     at_height: u32,
     contract: &QualifiedContractIdentifier,
     key: &str,
@@ -408,21 +408,19 @@ impl ClarityBackingStore for MemoryBackingStore {
     fn get_data_with_proof(
         &mut self,
         key: &str,
-    ) -> Result<Option<(String, Vec<u8>)>, VmExecutionError> {
-        Ok(SqliteConnection::get(self.get_side_store(), key)?.map(|x| (x, vec![])))
+    ) -> Result<Option<(String, Option<Vec<u8>>)>, VmExecutionError> {
+        // This backend has no MARF trie, so it can't produce a real Merkle proof -- report that
+        // honestly instead of fabricating one.
+        Ok(SqliteConnection::get(self.get_side_store(), key)?.map(|x| (x, None)))
     }
 
     fn get_data_with_proof_from_path(
         &mut self,
         hash: &TrieHash,
-    ) -> Result<Option<(String, Vec<u8>)>, VmExecutionError> {
+    ) -> Result<Option<(String, Option<Vec<u8>>)>, VmExecutionError> {
         self.get_data_with_proof(&hash.to_string())
     }
 
-    fn get_side_store(&mut self) -> &Connection {
-        &self.side_store
-    }
-
     fn get_block_at_height(&mut self, height: u32) -> Option<StacksBlockId> {
         if height == 0 {
             Some(StacksBlockId([255; 32]))
@@ -488,6 +486,12 @@ impl ClarityBackingStore for MemoryBackingStore {
     }
 }
 
+impl SqliteBackingStore for MemoryBackingStore {
+    fn get_side_store(&mut self) -> &Connection {
+        &self.side_store
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

contrib/clarity-cli/src/lib.rs

@@ -56,7 +56,7 @@ use stackslib::chainstate::stacks::boot::{
     POX_2_MAINNET_CODE, POX_2_TESTNET_CODE,
 };
 use stackslib::chainstate::stacks::index::ClarityMarfTrieId;
-use stackslib::clarity_vm::clarity::{ClarityMarfStore, ClarityMarfStoreTransaction};
+use stackslib::clarity_vm::clarity::{ClarityStore, ClarityStoreTransaction};
 use stackslib::clarity_vm::database::MemoryBackingStore;
 use stackslib::clarity_vm::database::marf::{MarfedKV, PersistentWritableMarfStore};
 use stackslib::core::{BLOCK_LIMIT_MAINNET_205, HELIUM_BLOCK_LIMIT_20, StacksEpochId};
@@ -406,7 +406,7 @@ where
 
     let marf_tx = marf_kv.begin(&from, &to);
     let (marf_return, result) = f(marf_tx);
-    marf_return.drop_current_trie();
+    marf_return.drop_current_block();
     result
 }
 
@@ -421,7 +421,7 @@ where
 
     let marf_tx = marf_kv.begin(&from, &to);
     let (marf_return, result) = f(marf_tx);
-    marf_return.drop_current_trie();
+    marf_return.drop_current_block();
     result
 }
 

stackslib/src/chainstate/stacks/boot/contract_tests.rs

@@ -46,8 +46,8 @@ use crate::chainstate::stacks::boot::{
 use crate::chainstate::stacks::index::ClarityMarfTrieId;
 use crate::chainstate::stacks::{C32_ADDRESS_VERSION_TESTNET_SINGLESIG, *};
 use crate::clarity_vm::clarity::{
-    ClarityBlockConnection, ClarityError, ClarityMarfStore, ClarityMarfStoreTransaction,
-    WritableMarfStore,
+    ClarityBlockConnection, ClarityError, ClarityStore, ClarityStoreTransaction,
+    WritableClarityStore,
 };
 use crate::clarity_vm::database::marf::MarfedKV;
 use crate::core::{
@@ -160,12 +160,12 @@ impl ClarityTestSim {
         &'_ mut self,
         new_tenure: bool,
     ) -> (
-        Box<dyn WritableMarfStore + '_>,
+        Box<dyn WritableClarityStore + '_>,
         TestSimHeadersDB,
         TestSimBurnStateDB,
         StacksEpochId,
     ) {
-        let mut store: Box<dyn WritableMarfStore> = Box::new(self.marf.begin(
+        let mut store: Box<dyn WritableClarityStore> = Box::new(self.marf.begin(
             &StacksBlockId(test_sim_height_to_hash(self.block_height, self.fork)),
             &StacksBlockId(test_sim_height_to_hash(self.block_height + 1, self.fork)),
         ));
@@ -253,7 +253,7 @@ impl ClarityTestSim {
     }
 
     fn check_and_bump_epoch<'a>(
-        store: &mut Box<dyn WritableMarfStore + 'a>,
+        store: &mut Box<dyn WritableClarityStore + 'a>,
         headers_db: &TestSimHeadersDB,
         burn_db: &dyn BurnStateDB,
     ) -> StacksEpochId {
@@ -280,7 +280,7 @@ impl ClarityTestSim {
     where
         F: FnOnce(&mut OwnedEnvironment) -> R,
     {
-        let mut store: Box<dyn WritableMarfStore> = Box::new(self.marf.begin(
+        let mut store: Box<dyn WritableClarityStore> = Box::new(self.marf.begin(
             &StacksBlockId(test_sim_height_to_hash(parent_height, self.fork)),
             &StacksBlockId(test_sim_height_to_hash(parent_height + 1, self.fork + 1)),
         ));

stackslib/src/chainstate/stacks/db/mod.rs

@@ -520,7 +520,7 @@ impl<'a, 'b> ClarityTx<'a, 'b> {
     /// by `inner_clarity_tx_begin`. Paired with
     /// [`ClarityBlockConnection::from_writable_store`], it lets a caller run the
     /// whole consensus transaction engine (`process_transaction`, `finish_block`,
-    /// `seal`, …) against a custom [`WritableMarfStore`] backend rather than the
+    /// `seal`, …) against a custom [`WritableClarityStore`] backend rather than the
     /// datastore owned by a `ClarityInstance`.
     ///
     /// `DBConfig` is derived from the block connection so mainnet/chain_id cannot
@@ -1364,7 +1364,7 @@ impl StacksChainState {
     /// final `commit_to_block`. Extracted from [`Self::install_boot_code`] (which
     /// now calls it against a `MarfedKV`-backed tx) so the same boot can be run
     /// against *any* [`ClarityTx`] — e.g. one built over a custom
-    /// [`WritableMarfStore`] via [`ClarityBlockConnection::from_writable_store_genesis`]
+    /// [`WritableClarityStore`] via [`ClarityBlockConnection::from_writable_store_genesis`]
     /// — yielding a byte-identical genesis state root regardless of backend.
     ///
     /// Network mode (`mainnet` vs testnet) is taken from `clarity_tx.config` so it

stackslib/src/chainstate/stacks/db/snapshot/tests/clarity.rs

@@ -29,7 +29,7 @@ use super::super::copy_clarity_side_tables;
 use crate::chainstate::stacks::index::marf::{MARFOpenOpts, MARF};
 use crate::chainstate::stacks::index::storage::TrieHashCalculationMode;
 use crate::chainstate::stacks::index::{ClarityMarfTrieId as _, Error, MARFValue};
-use crate::clarity_vm::clarity::ClarityMarfStoreTransaction as _;
+use crate::clarity_vm::clarity::ClarityStoreTransaction as _;
 use crate::clarity_vm::database::marf::MarfedKV;
 
 /// Build a Clarity MARF with N blocks of data and a single contract.