From b2a7abc24cb470f59ddc76fd0d93188c8dd1f9ee Mon Sep 17 00:00:00 2001 From: Tony Giorgio Date: Thu, 18 Jan 2024 12:57:18 -0600 Subject: [PATCH 1/7] Pull out payment info from ldk storage --- mutiny-core/src/event.rs | 46 ++++++---- mutiny-core/src/ldkstorage.rs | 106 ++++++---------------- mutiny-core/src/node.rs | 164 ++++++++++++++++++++++------------ mutiny-core/src/storage.rs | 89 +++++++++++++++++- 4 files changed, 248 insertions(+), 157 deletions(-) diff --git a/mutiny-core/src/event.rs b/mutiny-core/src/event.rs index 998d07adc..9c5f2657e 100644 --- a/mutiny-core/src/event.rs +++ b/mutiny-core/src/event.rs @@ -1,5 +1,3 @@ -use crate::fees::MutinyFeeEstimator; -use crate::keymanager::PhantomKeysManager; use crate::ldkstorage::{MutinyNodePersister, PhantomChannelManager}; use crate::logging::MutinyLogger; use crate::lsp::{AnyLsp, Lsp}; @@ -8,6 +6,8 @@ use crate::nodemanager::ChannelClosure; use crate::onchain::OnChainWallet; use crate::storage::MutinyStorage; use crate::utils::sleep; +use crate::{fees::MutinyFeeEstimator, storage::read_payment_info}; +use crate::{keymanager::PhantomKeysManager, storage::persist_payment_info}; use anyhow::anyhow; use bitcoin::hashes::hex::ToHex; use bitcoin::secp256k1::PublicKey; @@ -282,10 +282,12 @@ impl EventHandler { } => (payment_preimage, Some(payment_secret)), PaymentPurpose::SpontaneousPayment(preimage) => (Some(preimage), None), }; - match self - .persister - .read_payment_info(&payment_hash.0, true, &self.logger) - { + match read_payment_info( + &self.persister.storage, + &payment_hash.0, + true, + &self.logger, + ) { Some(mut saved_payment_info) => { let payment_preimage = payment_preimage.map(|p| p.0); let payment_secret = payment_secret.map(|p| p.0); @@ -294,7 +296,8 @@ impl EventHandler { saved_payment_info.secret = payment_secret; saved_payment_info.amt_msat = MillisatAmount(Some(amount_msat)); saved_payment_info.last_update = crate::utils::now().as_secs(); - match self.persister.persist_payment_info( + match persist_payment_info( + self.persister.storage.clone(), &payment_hash.0, &saved_payment_info, true, @@ -321,7 +324,8 @@ impl EventHandler { bolt11: None, last_update, }; - match self.persister.persist_payment_info( + match persist_payment_info( + self.persister.storage.clone(), &payment_hash.0, &payment_info, true, @@ -347,16 +351,19 @@ impl EventHandler { payment_hash.0.to_hex() ); - match self - .persister - .read_payment_info(&payment_hash.0, false, &self.logger) - { + match read_payment_info( + &self.persister.storage, + &payment_hash.0, + false, + &self.logger, + ) { Some(mut saved_payment_info) => { saved_payment_info.status = HTLCStatus::Succeeded; saved_payment_info.preimage = Some(payment_preimage.0); saved_payment_info.fee_paid_msat = fee_paid_msat; saved_payment_info.last_update = crate::utils::now().as_secs(); - match self.persister.persist_payment_info( + match persist_payment_info( + self.persister.storage.clone(), &payment_hash.0, &saved_payment_info, false, @@ -445,14 +452,17 @@ impl EventHandler { payment_hash.0.to_hex() ); - match self - .persister - .read_payment_info(&payment_hash.0, false, &self.logger) - { + match read_payment_info( + &self.persister.storage, + &payment_hash.0, + false, + &self.logger, + ) { Some(mut saved_payment_info) => { saved_payment_info.status = HTLCStatus::Failed; saved_payment_info.last_update = crate::utils::now().as_secs(); - match self.persister.persist_payment_info( + match persist_payment_info( + self.persister.storage.clone(), &payment_hash.0, &saved_payment_info, false, diff --git a/mutiny-core/src/ldkstorage.rs b/mutiny-core/src/ldkstorage.rs index 22585f540..7c5189d7e 100644 --- a/mutiny-core/src/ldkstorage.rs +++ b/mutiny-core/src/ldkstorage.rs @@ -1,5 +1,4 @@ use crate::error::{MutinyError, MutinyStorageError}; -use crate::event::PaymentInfo; use crate::fees::MutinyFeeEstimator; use crate::gossip::PROB_SCORER_KEY; use crate::keymanager::PhantomKeysManager; @@ -26,12 +25,11 @@ use lightning::io::Cursor; use lightning::ln::channelmanager::{ self, ChainParameters, ChannelManager as LdkChannelManager, ChannelManagerReadArgs, }; -use lightning::ln::PaymentHash; use lightning::sign::{InMemorySigner, SpendableOutputDescriptor, WriteableEcdsaChannelSigner}; use lightning::util::logger::Logger; use lightning::util::persist::Persister; use lightning::util::ser::{Readable, ReadableArgs, Writeable}; -use lightning::{log_debug, log_error, log_trace}; +use lightning::{log_debug, log_error}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::io; @@ -40,8 +38,6 @@ use std::sync::Arc; pub const CHANNEL_MANAGER_KEY: &str = "manager"; pub const MONITORS_PREFIX_KEY: &str = "monitors/"; -const PAYMENT_INBOUND_PREFIX_KEY: &str = "payment_inbound/"; -const PAYMENT_OUTBOUND_PREFIX_KEY: &str = "payment_outbound/"; const CHANNEL_OPENING_PARAMS_PREFIX: &str = "chan_open_params/"; const CHANNEL_CLOSURE_PREFIX: &str = "channel_closure/"; const FAILED_SPENDABLE_OUTPUT_DESCRIPTOR_KEY: &str = "failed_spendable_outputs"; @@ -362,54 +358,6 @@ impl MutinyNodePersister { }) } - pub(crate) fn persist_payment_info( - &self, - payment_hash: &[u8; 32], - payment_info: &PaymentInfo, - inbound: bool, - ) -> io::Result<()> { - let key = self.get_key(payment_key(inbound, payment_hash).as_str()); - self.storage - .set_data(key, payment_info, None) - .map_err(io::Error::other) - } - - pub(crate) fn read_payment_info( - &self, - payment_hash: &[u8; 32], - inbound: bool, - logger: &MutinyLogger, - ) -> Option { - let key = self.get_key(payment_key(inbound, payment_hash).as_str()); - log_trace!(logger, "Trace: checking payment key: {key}"); - let deserialized_value: Result, MutinyError> = - self.storage.get_data(key); - deserialized_value.ok().flatten() - } - - pub(crate) fn list_payment_info( - &self, - inbound: bool, - ) -> Result, MutinyError> { - let prefix = match inbound { - true => PAYMENT_INBOUND_PREFIX_KEY, - false => PAYMENT_OUTBOUND_PREFIX_KEY, - }; - let suffix = format!("_{}", self.node_id); - let map: HashMap = self.storage.scan(prefix, Some(&suffix))?; - - // convert keys to PaymentHash - Ok(map - .into_iter() - .map(|(key, value)| { - let payment_hash_str = key.trim_start_matches(prefix).trim_end_matches(&suffix); - let hash: [u8; 32] = - FromHex::from_hex(payment_hash_str).expect("key should be a sha256 hash"); - (PaymentHash(hash), value) - }) - .collect()) - } - pub(crate) fn persist_channel_closure( &self, user_channel_id: u128, @@ -600,22 +548,6 @@ impl ChannelOpenParams { } } -fn payment_key(inbound: bool, payment_hash: &[u8; 32]) -> String { - if inbound { - format!( - "{}{}", - PAYMENT_INBOUND_PREFIX_KEY, - payment_hash.to_hex().as_str() - ) - } else { - format!( - "{}{}", - PAYMENT_OUTBOUND_PREFIX_KEY, - payment_hash.to_hex().as_str() - ) - } -} - impl Persister< '_, @@ -749,23 +681,26 @@ pub(crate) async fn persist_monitor( #[cfg(test)] mod test { - use crate::node::scoring_params; - use crate::onchain::OnChainWallet; - use crate::storage::MemoryStorage; + use crate::{ + event::PaymentInfo, + storage::{list_payment_info, MemoryStorage}, + }; use crate::{ event::{HTLCStatus, MillisatAmount}, scorer::HubPreferentialScorer, }; use crate::{keymanager::create_keys_manager, scorer::ProbScorer}; + use crate::{node::scoring_params, storage::persist_payment_info}; + use crate::{onchain::OnChainWallet, storage::read_payment_info}; use bip39::Mnemonic; use bitcoin::hashes::Hash; use bitcoin::secp256k1::PublicKey; use bitcoin::util::bip32::ExtendedPrivKey; use bitcoin::Txid; use esplora_client::Builder; - use lightning::routing::router::DefaultRouter; use lightning::routing::scoring::ProbabilisticScoringDecayParameters; use lightning::sign::EntropySource; + use lightning::{ln::PaymentHash, routing::router::DefaultRouter}; use lightning_transaction_sync::EsploraSyncClient; use std::str::FromStr; use std::sync::atomic::AtomicBool; @@ -807,27 +742,42 @@ mod test { secret: None, last_update: utils::now().as_secs(), }; - let result = persister.persist_payment_info(&payment_hash.0, &payment_info, true); + let result = persist_payment_info( + persister.storage.clone(), + &payment_hash.0, + &payment_info, + true, + ); assert!(result.is_ok()); - let result = persister.read_payment_info(&payment_hash.0, true, &MutinyLogger::default()); + let result = read_payment_info( + &persister.storage, + &payment_hash.0, + true, + &MutinyLogger::default(), + ); assert!(result.is_some()); assert_eq!(result.clone().unwrap().preimage, Some(preimage)); assert_eq!(result.unwrap().status, HTLCStatus::Succeeded); - let list = persister.list_payment_info(true).unwrap(); + let list = list_payment_info(&persister.storage, true).unwrap(); assert_eq!(list.len(), 1); assert_eq!(list[0].0, payment_hash); assert_eq!(list[0].1.preimage, Some(preimage)); - let result = persister.read_payment_info(&payment_hash.0, true, &MutinyLogger::default()); + let result = read_payment_info( + &persister.storage, + &payment_hash.0, + true, + &MutinyLogger::default(), + ); assert!(result.is_some()); assert_eq!(result.clone().unwrap().preimage, Some(preimage)); assert_eq!(result.unwrap().status, HTLCStatus::Succeeded); - let list = persister.list_payment_info(true).unwrap(); + let list = list_payment_info(&persister.storage, true).unwrap(); assert_eq!(list.len(), 1); assert_eq!(list[0].0, payment_hash); assert_eq!(list[0].1.preimage, Some(preimage)); diff --git a/mutiny-core/src/node.rs b/mutiny-core/src/node.rs index a783d7a51..bb70d7235 100644 --- a/mutiny-core/src/node.rs +++ b/mutiny-core/src/node.rs @@ -1,6 +1,3 @@ -use crate::ldkstorage::{persist_monitor, ChannelOpenParams}; -use crate::lsp::{InvoiceRequest, LspConfig}; -use crate::messagehandler::MutinyMessageHandler; use crate::nodemanager::ChannelClosure; use crate::peermanager::LspMessageRouter; use crate::storage::MutinyStorage; @@ -23,6 +20,15 @@ use crate::{ use crate::{fees::P2WSH_OUTPUT_SIZE, peermanager::connect_peer_if_necessary}; use crate::{keymanager::PhantomKeysManager, scorer::HubPreferentialScorer}; use crate::{labels::LabelStorage, DEFAULT_PAYMENT_TIMEOUT}; +use crate::{ + ldkstorage::{persist_monitor, ChannelOpenParams}, + storage::persist_payment_info, +}; +use crate::{ + lsp::{InvoiceRequest, LspConfig}, + storage::list_payment_info, +}; +use crate::{messagehandler::MutinyMessageHandler, storage::read_payment_info}; use anyhow::{anyhow, Context}; use bdk::FeeRate; use bitcoin::hashes::{hex::ToHex, sha256::Hash as Sha256}; @@ -1183,12 +1189,16 @@ impl Node { payee_pubkey: None, last_update, }; - self.persister - .persist_payment_info(&payment_hash.0, &payment_info, true) - .map_err(|e| { - log_error!(self.logger, "ERROR: could not persist payment info: {e}"); - MutinyError::InvoiceCreationFailed - })?; + persist_payment_info( + self.persister.storage.clone(), + &payment_hash.0, + &payment_info, + true, + ) + .map_err(|e| { + log_error!(self.logger, "ERROR: could not persist payment info: {e}"); + MutinyError::InvoiceCreationFailed + })?; Ok(()) } @@ -1224,9 +1234,7 @@ impl Node { let now = utils::now(); let labels_map = self.persister.storage.get_invoice_labels()?; - Ok(self - .persister - .list_payment_info(inbound)? + Ok(list_payment_info(&self.persister.storage, inbound)? .into_iter() .filter_map(|(h, i)| { let labels = match i.bolt11.clone() { @@ -1274,18 +1282,22 @@ impl Node { payment_hash: &bitcoin::hashes::sha256::Hash, ) -> Result<(PaymentInfo, bool), MutinyError> { // try inbound first - if let Some(payment_info) = - self.persister - .read_payment_info(payment_hash.as_inner(), true, &self.logger) - { + if let Some(payment_info) = read_payment_info( + &self.persister.storage, + payment_hash.as_inner(), + true, + &self.logger, + ) { return Ok((payment_info, true)); } // if no inbound check outbound - match self - .persister - .read_payment_info(payment_hash.as_inner(), false, &self.logger) - { + match read_payment_info( + &self.persister.storage, + payment_hash.as_inner(), + false, + &self.logger, + ) { Some(payment_info) => Ok((payment_info, false)), None => Err(MutinyError::NotFound), } @@ -1304,17 +1316,13 @@ impl Node { ) -> Result<(PaymentId, PaymentHash), MutinyError> { let payment_hash = invoice.payment_hash().as_inner(); - if self - .persister - .read_payment_info(payment_hash, false, &self.logger) + if read_payment_info(&self.persister.storage, payment_hash, false, &self.logger) .is_some_and(|p| p.status != HTLCStatus::Failed) { return Err(MutinyError::NonUniquePaymentHash); } - if self - .persister - .read_payment_info(payment_hash, true, &self.logger) + if read_payment_info(&self.persister.storage, payment_hash, true, &self.logger) .is_some_and(|p| p.status != HTLCStatus::Failed) { return Err(MutinyError::NonUniquePaymentHash); @@ -1385,8 +1393,12 @@ impl Node { last_update, }; - self.persister - .persist_payment_info(payment_hash, &payment_info, false)?; + persist_payment_info( + self.persister.storage.clone(), + payment_hash, + &payment_info, + false, + )?; match pay_result { Ok(id) => Ok((id, PaymentHash(payment_hash.to_owned()))), @@ -1401,8 +1413,12 @@ impl Node { ); payment_info.status = HTLCStatus::Failed; - self.persister - .persist_payment_info(payment_hash, &payment_info, false)?; + persist_payment_info( + self.persister.storage.clone(), + payment_hash, + &payment_info, + false, + )?; Err(map_sending_failure(error, amt_msat, ¤t_channels)) } @@ -1466,9 +1482,12 @@ impl Node { return Err(MutinyError::PaymentTimeout); } - let payment_info = - self.persister - .read_payment_info(&payment_hash.0, false, &self.logger); + let payment_info = read_payment_info( + &self.persister.storage, + &payment_hash.0, + false, + &self.logger, + ); if let Some(info) = payment_info { match info.status { @@ -1562,8 +1581,12 @@ impl Node { last_update, }; - self.persister - .persist_payment_info(&payment_hash.0, &payment_info, false)?; + persist_payment_info( + self.persister.storage.clone(), + &payment_hash.0, + &payment_info, + false, + )?; match pay_result { Ok(_) => { @@ -1573,8 +1596,12 @@ impl Node { } Err(error) => { payment_info.status = HTLCStatus::Failed; - self.persister - .persist_payment_info(&payment_hash.0, &payment_info, false)?; + persist_payment_info( + self.persister.storage.clone(), + &payment_hash.0, + &payment_info, + false, + )?; let current_channels = self.channel_manager.list_channels(); Err(map_sending_failure( PaymentError::Sending(error), @@ -2495,9 +2522,13 @@ mod tests { // check that it still fails if it is inflight - node.persister - .persist_payment_info(&payment_hash.0, &payment_info, false) - .unwrap(); + persist_payment_info( + node.persister.storage.clone(), + &payment_hash.0, + &payment_info, + false, + ) + .unwrap(); let result = node .await_payment(payment_id, payment_hash, 1, vec![]) @@ -2508,9 +2539,13 @@ mod tests { // check that we get proper error if it fails payment_info.status = HTLCStatus::Failed; - node.persister - .persist_payment_info(&payment_hash.0, &payment_info, false) - .unwrap(); + persist_payment_info( + node.persister.storage.clone(), + &payment_hash.0, + &payment_info, + false, + ) + .unwrap(); let result = node .await_payment(payment_id, payment_hash, 1, vec![]) @@ -2521,9 +2556,13 @@ mod tests { // check that we get success payment_info.status = HTLCStatus::Succeeded; - node.persister - .persist_payment_info(&payment_hash.0, &payment_info, false) - .unwrap(); + persist_payment_info( + node.persister.storage.clone(), + &payment_hash.0, + &payment_info, + false, + ) + .unwrap(); let result = node .await_payment(payment_id, payment_hash, 1, vec![]) @@ -2536,11 +2575,11 @@ mod tests { #[cfg(test)] #[cfg(target_arch = "wasm32")] mod wasm_test { - use crate::error::MutinyError; use crate::event::{MillisatAmount, PaymentInfo}; use crate::storage::MemoryStorage; use crate::test_utils::create_node; use crate::HTLCStatus; + use crate::{error::MutinyError, storage::persist_payment_info}; use bitcoin::hashes::hex::ToHex; use lightning::ln::channelmanager::PaymentId; use lightning::ln::PaymentHash; @@ -2636,10 +2675,13 @@ mod wasm_test { }; // check that it still fails if it is inflight - - node.persister - .persist_payment_info(&payment_hash.0, &payment_info, false) - .unwrap(); + persist_payment_info( + node.persister.storage.clone(), + &payment_hash.0, + &payment_info, + false, + ) + .unwrap(); let result = node .await_payment(payment_id, payment_hash, 1, vec![]) @@ -2650,9 +2692,13 @@ mod wasm_test { // check that we get proper error if it fails payment_info.status = HTLCStatus::Failed; - node.persister - .persist_payment_info(&payment_hash.0, &payment_info, false) - .unwrap(); + persist_payment_info( + node.persister.storage.clone(), + &payment_hash.0, + &payment_info, + false, + ) + .unwrap(); let result = node .await_payment(payment_id, payment_hash, 1, vec![]) @@ -2663,9 +2709,13 @@ mod wasm_test { // check that we get success payment_info.status = HTLCStatus::Succeeded; - node.persister - .persist_payment_info(&payment_hash.0, &payment_info, false) - .unwrap(); + persist_payment_info( + node.persister.storage.clone(), + &payment_hash.0, + &payment_info, + false, + ) + .unwrap(); let result = node .await_payment(payment_id, payment_hash, 1, vec![]) diff --git a/mutiny-core/src/storage.rs b/mutiny-core/src/storage.rs index fc8249e6f..bb6543f58 100644 --- a/mutiny-core/src/storage.rs +++ b/mutiny-core/src/storage.rs @@ -1,5 +1,3 @@ -use crate::error::{MutinyError, MutinyStorageError}; -use crate::ldkstorage::CHANNEL_MANAGER_KEY; use crate::nodemanager::{NodeStorage, DEVICE_LOCK_INTERVAL_SECS}; use crate::utils::{now, spawn}; use crate::vss::{MutinyVssClient, VssKeyValueItem}; @@ -7,11 +5,18 @@ use crate::{ encrypt::{decrypt_with_password, encrypt, encryption_key_from_pass, Cipher}, federation::FederationStorage, }; +use crate::{ + error::{MutinyError, MutinyStorageError}, + event::PaymentInfo, +}; +use crate::{ldkstorage::CHANNEL_MANAGER_KEY, logging::MutinyLogger}; use async_trait::async_trait; use bdk::chain::{Append, PersistBackend}; use bip39::Mnemonic; -use lightning::log_error; -use lightning::util::logger::Logger; +use bitcoin::hashes::hex::ToHex; +use hex::FromHex; +use lightning::{ln::PaymentHash, util::logger::Logger}; +use lightning::{log_error, log_trace}; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::HashMap; @@ -29,6 +34,8 @@ const FIRST_SYNC_KEY: &str = "first_sync"; pub(crate) const DEVICE_ID_KEY: &str = "device_id"; pub const DEVICE_LOCK_KEY: &str = "device_lock"; pub(crate) const EXPECTED_NETWORK_KEY: &str = "network"; +const PAYMENT_INBOUND_PREFIX_KEY: &str = "payment_inbound/"; +const PAYMENT_OUTBOUND_PREFIX_KEY: &str = "payment_outbound/"; fn needs_encryption(key: &str) -> bool { match key { @@ -653,6 +660,80 @@ impl MutinyStorage for () { } } +fn payment_key(inbound: bool, payment_hash: &[u8; 32]) -> String { + if inbound { + format!( + "{}{}", + PAYMENT_INBOUND_PREFIX_KEY, + payment_hash.to_hex().as_str() + ) + } else { + format!( + "{}{}", + PAYMENT_OUTBOUND_PREFIX_KEY, + payment_hash.to_hex().as_str() + ) + } +} + +pub(crate) fn persist_payment_info( + storage: S, + payment_hash: &[u8; 32], + payment_info: &PaymentInfo, + inbound: bool, +) -> std::io::Result<()> { + let key = payment_key(inbound, payment_hash); + storage + .set_data(key, payment_info, None) + .map_err(std::io::Error::other) +} + +pub(crate) fn read_payment_info( + storage: &S, + payment_hash: &[u8; 32], + inbound: bool, + logger: &MutinyLogger, +) -> Option { + let key = payment_key(inbound, payment_hash); + log_trace!(logger, "Trace: checking payment key: {key}"); + match storage.get_data(&key).transpose() { + Some(Ok(v)) => Some(v), + _ => { + // To scan for the old format that had `_{node_id}` at the end + if let Ok(map) = storage.scan(&key, None) { + map.into_values().next() + } else { + None + } + } + } +} + +pub(crate) fn list_payment_info( + storage: &S, + inbound: bool, +) -> Result, MutinyError> { + let prefix = match inbound { + true => PAYMENT_INBOUND_PREFIX_KEY, + false => PAYMENT_OUTBOUND_PREFIX_KEY, + }; + let map: HashMap = storage.scan(prefix, None)?; + + // convert keys to PaymentHash + Ok(map + .into_iter() + .map(|(key, value)| { + let payment_hash_str = key + .trim_start_matches(prefix) + .splitn(2, '_') // To support the old format that had `_{node_id}` at the end + .collect::>()[0]; + let hash: [u8; 32] = + FromHex::from_hex(payment_hash_str).expect("key should be a sha256 hash"); + (PaymentHash(hash), value) + }) + .collect()) +} + #[derive(Clone)] pub struct OnChainStorage(pub(crate) S); From 8fdef01bb1e75bf1b6f698b3acfeb63db69eb113 Mon Sep 17 00:00:00 2001 From: Tony Giorgio Date: Thu, 18 Jan 2024 10:00:31 -0600 Subject: [PATCH 2/7] Switch fedimint to use our indexeddb store --- mutiny-core/src/event.rs | 8 +- mutiny-core/src/federation.rs | 405 ++++++++++++++++++++++++--------- mutiny-core/src/ldkstorage.rs | 7 +- mutiny-core/src/lib.rs | 142 +++++------- mutiny-core/src/node.rs | 32 +-- mutiny-core/src/nodemanager.rs | 36 ++- mutiny-core/src/storage.rs | 23 +- mutiny-wasm/src/lib.rs | 1 - 8 files changed, 427 insertions(+), 227 deletions(-) diff --git a/mutiny-core/src/event.rs b/mutiny-core/src/event.rs index 9c5f2657e..06b9a495b 100644 --- a/mutiny-core/src/event.rs +++ b/mutiny-core/src/event.rs @@ -297,7 +297,7 @@ impl EventHandler { saved_payment_info.amt_msat = MillisatAmount(Some(amount_msat)); saved_payment_info.last_update = crate::utils::now().as_secs(); match persist_payment_info( - self.persister.storage.clone(), + &self.persister.storage, &payment_hash.0, &saved_payment_info, true, @@ -325,7 +325,7 @@ impl EventHandler { last_update, }; match persist_payment_info( - self.persister.storage.clone(), + &self.persister.storage, &payment_hash.0, &payment_info, true, @@ -363,7 +363,7 @@ impl EventHandler { saved_payment_info.fee_paid_msat = fee_paid_msat; saved_payment_info.last_update = crate::utils::now().as_secs(); match persist_payment_info( - self.persister.storage.clone(), + &self.persister.storage, &payment_hash.0, &saved_payment_info, false, @@ -462,7 +462,7 @@ impl EventHandler { saved_payment_info.status = HTLCStatus::Failed; saved_payment_info.last_update = crate::utils::now().as_secs(); match persist_payment_info( - self.persister.storage.clone(), + &self.persister.storage, &payment_hash.0, &saved_payment_info, false, diff --git a/mutiny-core/src/federation.rs b/mutiny-core/src/federation.rs index b3635fd17..e10e6fa16 100644 --- a/mutiny-core/src/federation.rs +++ b/mutiny-core/src/federation.rs @@ -1,13 +1,17 @@ use crate::{ - error::MutinyError, + error::{MutinyError, MutinyStorageError}, + event::PaymentInfo, key::{create_root_child_key, ChildKey}, logging::MutinyLogger, nodemanager::MutinyInvoice, onchain::coin_type_from_network, - sql::{glue::GlueDB, ApplicationStore}, - utils::{self, sleep}, - ActivityItem, HTLCStatus, DEFAULT_PAYMENT_TIMEOUT, + storage::{ + get_payment_info, list_payment_info, persist_payment_info, MutinyStorage, VersionedValue, + }, + utils::sleep, + HTLCStatus, DEFAULT_PAYMENT_TIMEOUT, }; +use async_trait::async_trait; use bip39::Mnemonic; use bitcoin::{ hashes::{hex::ToHex, sha256}, @@ -16,6 +20,7 @@ use bitcoin::{ util::bip32::{ChildNumber, DerivationPath}, Network, }; +use core::fmt; use fedimint_bip39::Bip39RootSecretStrategy; use fedimint_client::{ db::ChronologicalOperationLogKey, @@ -33,6 +38,14 @@ use fedimint_core::{ task::{MaybeSend, MaybeSync}, Amount, }; +use fedimint_core::{ + db::{ + mem_impl::{MemDatabase, MemTransaction}, + IDatabaseTransactionOps, IDatabaseTransactionOpsCore, IRawDatabase, + IRawDatabaseTransaction, PrefixStream, + }, + BitcoinHash, +}; use fedimint_ln_client::{ InternalPayState, LightningClientInit, LightningClientModule, LightningOperationMeta, LightningOperationMetaVariant, LnPayState, LnReceiveState, @@ -42,9 +55,13 @@ use fedimint_mint_client::MintClientInit; use fedimint_wallet_client::{WalletClientInit, WalletClientModule}; use futures::future::{self}; use futures_util::{pin_mut, StreamExt}; -use lightning::{log_debug, log_error, log_info, log_trace, log_warn, util::logger::Logger}; +use hex::FromHex; +use lightning::{ + ln::PaymentHash, log_debug, log_error, log_info, log_trace, log_warn, util::logger::Logger, +}; use lightning_invoice::Bolt11Invoice; use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use std::sync::atomic::{AtomicU32, Ordering}; use std::{collections::HashMap, fmt::Debug, sync::Arc}; // The amount of time in milliseconds to wait for @@ -128,20 +145,20 @@ pub struct FedimintBalance { pub amount: u64, } -pub(crate) struct FederationClient { +pub(crate) struct FederationClient { pub(crate) uuid: String, pub(crate) fedimint_client: ClientArc, - g: GlueDB, + storage: S, pub(crate) logger: Arc, } -impl FederationClient { +impl FederationClient { #[allow(clippy::too_many_arguments)] pub(crate) async fn new( uuid: String, federation_code: InviteCode, xprivkey: ExtendedPrivKey, - g: GlueDB, + storage: &S, network: Network, logger: Arc, ) -> Result { @@ -166,10 +183,14 @@ impl FederationClient { client_builder.with_module(LightningClientInit); log_trace!(logger, "Building fedimint client db"); - let db = g - .new_fedimint_client_db(federation_info.federation_id().to_string()) - .await? - .into(); + let db = FedimintStorage::new( + storage.clone(), + federation_info.federation_id().to_string(), + logger.clone(), + ) + .await? + .into(); + if get_config_from_db(&db).await.is_none() { client_builder.with_federation_info(federation_info.clone()); } @@ -204,7 +225,7 @@ impl FederationClient { Ok(FederationClient { uuid, fedimint_client, - g, + storage: storage.clone(), logger, }) } @@ -227,7 +248,9 @@ impl FederationClient { stored_payment.labels = labels; log_trace!(self.logger, "Persiting payment"); - self.g.save_payment(stored_payment).await?; + let hash = *stored_payment.payment_hash.as_inner(); + let payment_info = PaymentInfo::from(stored_payment); + persist_payment_info(&self.storage, &hash, &payment_info, true)?; log_trace!(self.logger, "Persisted payment"); Ok(invoice.into()) @@ -238,19 +261,36 @@ impl FederationClient { Ok(self.fedimint_client.get_balance().await.msats / 1_000) } - pub async fn get_activity(&self) -> Result, MutinyError> { + pub async fn check_activity(&self) -> Result<(), MutinyError> { log_trace!(self.logger, "Getting activity"); - let payments = self.g.list_payments().await?; - - let mut payments_map: HashMap = HashMap::new(); - let mut pending_invoices: Vec<&MutinyInvoice> = Vec::new(); - for payment in payments.iter() { - payments_map.insert(payment.payment_hash, payment.clone()); - if matches!(payment.status, HTLCStatus::InFlight | HTLCStatus::Pending) { - pending_invoices.push(payment); - } - } + let mut pending_invoices = Vec::new(); + // inbound + pending_invoices.extend( + list_payment_info(&self.storage, true)? + .into_iter() + .filter_map(|(h, i)| { + let mutiny_invoice = MutinyInvoice::from(i.clone(), h, true, vec![]).ok(); + + // filter out finalized invoices + mutiny_invoice.filter(|invoice| { + matches!(invoice.status, HTLCStatus::InFlight | HTLCStatus::Pending) + }) + }), + ); + // outbound + pending_invoices.extend( + list_payment_info(&self.storage, false)? + .into_iter() + .filter_map(|(h, i)| { + let mutiny_invoice = MutinyInvoice::from(i.clone(), h, false, vec![]).ok(); + + // filter out finalized invoices + mutiny_invoice.filter(|invoice| { + matches!(invoice.status, HTLCStatus::InFlight | HTLCStatus::Pending) + }) + }), + ); let operations = if !pending_invoices.is_empty() { log_trace!(self.logger, "pending invoices, going to list operations"); @@ -309,30 +349,11 @@ impl FederationClient { { self.maybe_update_after_checking_fedimint(updated_invoice.clone()) .await?; - payments_map.insert(hash, updated_invoice); } } } - let updated_payments = payments_map.into_values().collect::>(); - - let activity_items = updated_payments - .into_iter() - .filter_map(|invoice| { - if !invoice - .bolt11 - .as_ref() - .is_some_and(|b| b.would_expire(utils::now())) - && matches!(invoice.status, HTLCStatus::Succeeded | HTLCStatus::InFlight) - { - Some(ActivityItem::Lightning(Box::new(invoice))) - } else { - None - } - }) - .collect::>(); - - Ok(activity_items) + Ok(()) } async fn maybe_update_after_checking_fedimint( @@ -344,21 +365,10 @@ impl FederationClient { HTLCStatus::Succeeded | HTLCStatus::Failed ) { log_debug!(self.logger, "Saving updated payment"); - self.g - .update_payment_status( - &updated_invoice.payment_hash, - updated_invoice.status.clone(), - ) - .await?; - self.g - .update_payment_fee(&updated_invoice.payment_hash, updated_invoice.fees_paid) - .await?; - self.g - .update_payment_preimage( - &updated_invoice.payment_hash, - updated_invoice.preimage.clone(), - ) - .await?; + let hash = *updated_invoice.payment_hash.as_inner(); + let inbound = updated_invoice.inbound; + let payment_info = PaymentInfo::from(updated_invoice); + persist_payment_info(&self.storage, &hash, &payment_info, inbound)?; } Ok(()) } @@ -370,7 +380,7 @@ impl FederationClient { log_trace!(self.logger, "get_invoice_by_hash"); // Try to get the invoice from storage first - let invoice = match self.g.get_payment(hash).await { + let (invoice, inbound) = match get_payment_info(&self.storage, hash, &self.logger) { Ok(i) => i, Err(e) => { log_error!(self.logger, "could not get invoice by hash: {e}"); @@ -378,55 +388,54 @@ impl FederationClient { } }; - if let Some(invoice) = invoice { - log_trace!(self.logger, "retrieved invoice by hash"); - - if matches!(invoice.status, HTLCStatus::InFlight | HTLCStatus::Pending) { - log_trace!(self.logger, "invoice still in flight, getting operations"); - // If the invoice is InFlight or Pending, check the operation log for updates - let lightning_module = self - .fedimint_client - .get_first_module::(); - - let operations = self - .fedimint_client - .operation_log() - .list_operations(FEDIMINT_OPERATIONS_LIST_MAX, None) - .await; - - log_trace!( - self.logger, - "going to go through {} operations", - operations.len() - ); - for (key, entry) in operations { - if entry.operation_module_kind() == LightningCommonInit::KIND.as_str() { - if let Some(updated_invoice) = extract_invoice_from_entry( - self.logger.clone(), - &entry, - hash, - key.operation_id, - &lightning_module, - ) - .await - { - self.maybe_update_after_checking_fedimint(updated_invoice.clone()) - .await?; - return Ok(updated_invoice); - } - } else { - log_warn!( - self.logger, - "Unsupported module: {}", - entry.operation_module_kind() - ); + log_trace!(self.logger, "retrieved invoice by hash"); + + if matches!(invoice.status, HTLCStatus::InFlight | HTLCStatus::Pending) { + log_trace!(self.logger, "invoice still in flight, getting operations"); + // If the invoice is InFlight or Pending, check the operation log for updates + let lightning_module = self + .fedimint_client + .get_first_module::(); + + let operations = self + .fedimint_client + .operation_log() + .list_operations(FEDIMINT_OPERATIONS_LIST_MAX, None) + .await; + + log_trace!( + self.logger, + "going to go through {} operations", + operations.len() + ); + for (key, entry) in operations { + if entry.operation_module_kind() == LightningCommonInit::KIND.as_str() { + if let Some(updated_invoice) = extract_invoice_from_entry( + self.logger.clone(), + &entry, + hash, + key.operation_id, + &lightning_module, + ) + .await + { + self.maybe_update_after_checking_fedimint(updated_invoice.clone()) + .await?; + return Ok(updated_invoice); } + } else { + log_warn!( + self.logger, + "Unsupported module: {}", + entry.operation_module_kind() + ); } - } else { - // If the invoice is not InFlight or Pending, return it directly - log_trace!(self.logger, "returning final invoice"); - return Ok(invoice); } + } else { + // If the invoice is not InFlight or Pending, return it directly + log_trace!(self.logger, "returning final invoice"); + // TODO labels + return MutinyInvoice::from(invoice, PaymentHash(hash.into_inner()), inbound, vec![]); } log_debug!(self.logger, "could not find invoice"); @@ -450,7 +459,9 @@ impl FederationClient { let mut stored_payment: MutinyInvoice = invoice.clone().into(); stored_payment.inbound = false; stored_payment.labels = labels; - self.g.save_payment(stored_payment.clone()).await?; + let hash = *stored_payment.payment_hash.as_inner(); + let payment_info = PaymentInfo::from(stored_payment); + persist_payment_info(&self.storage, &hash, &payment_info, true)?; // Subscribe and process outcome based on payment type let mut inv = match outgoing_payment.payment_type { @@ -683,6 +694,182 @@ where invoice } +#[derive(Clone)] +pub struct FedimintStorage { + pub(crate) storage: S, + fedimint_memory: Arc, + federation_id: String, + federation_version: Arc, +} + +impl fmt::Debug for FedimintStorage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FedimintDB").finish() + } +} + +impl FedimintStorage { + pub async fn new( + storage: S, + federation_id: String, + logger: Arc, + ) -> Result { + log_debug!(logger, "initializing fedimint storage"); + + let fedimint_memory = MemDatabase::new(); + + let key = key_id(&federation_id); + + let federation_version = match storage.get_data::(&key) { + Ok(Some(versioned_value)) => { + // get the value/version and load it into fedimint memory + let hex: String = serde_json::from_value(versioned_value.value.clone())?; + if !hex.is_empty() { + let bytes: Vec = + FromHex::from_hex(&hex).map_err(|e| MutinyError::ReadError { + source: MutinyStorageError::Other(anyhow::Error::new(e)), + })?; + let key_value_pairs: Vec<(Vec, Vec)> = bincode::deserialize(&bytes) + .map_err(|e| MutinyError::ReadError { + source: MutinyStorageError::Other(e.into()), + })?; + + let mut mem_db_tx = fedimint_memory.begin_transaction().await; + for (key, value) in key_value_pairs { + mem_db_tx + .raw_insert_bytes(&key, &value) + .await + .map_err(|_| { + MutinyError::write_err(MutinyStorageError::IndexedDBError) + })?; + } + mem_db_tx + .commit_tx() + .await + .map_err(|_| MutinyError::write_err(MutinyStorageError::IndexedDBError))?; + } + versioned_value.version + } + Ok(None) => 0, + Err(e) => { + panic!("unparsable value in federation storage: {e}") + } + }; + + log_debug!(logger, "done setting up FedimintDB for fedimint"); + + Ok(Self { + storage, + federation_id, + federation_version: Arc::new(federation_version.into()), + fedimint_memory: Arc::new(fedimint_memory), + }) + } +} + +fn key_id(federation_id: &str) -> String { + format!("fedimint_key_{}", federation_id) +} + +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +impl IRawDatabase for FedimintStorage { + type Transaction<'a> = IndexedDBPseudoTransaction<'a, S>; + + async fn begin_transaction<'a>(&'a self) -> IndexedDBPseudoTransaction { + IndexedDBPseudoTransaction { + storage: self.storage.clone(), + federation_id: self.federation_id.clone(), + federation_version: self.federation_version.clone(), + mem: self.fedimint_memory.begin_transaction().await, + } + } +} + +pub struct IndexedDBPseudoTransaction<'a, S: MutinyStorage> { + pub(crate) storage: S, + federation_version: Arc, + federation_id: String, + mem: MemTransaction<'a>, +} + +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +impl<'a, S: MutinyStorage> IRawDatabaseTransaction for IndexedDBPseudoTransaction<'a, S> { + async fn commit_tx(mut self) -> anyhow::Result<()> { + let key_value_pairs = self + .mem + .raw_find_by_prefix(&[]) + .await? + .collect::, Vec)>>() + .await; + self.mem.commit_tx().await?; + + let serialized_data = bincode::serialize(&key_value_pairs).map_err(anyhow::Error::new)?; + let hex_serialized_data = hex::encode(serialized_data); + + let old = self.federation_version.fetch_add(1, Ordering::SeqCst); + let version = old + 1; + let value = VersionedValue { + version, + value: serde_json::to_value(hex_serialized_data).unwrap(), + }; + self.storage + .set_data(key_id(&self.federation_id), value, None)?; + + Ok(()) + } +} + +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +impl<'a, S: MutinyStorage> IDatabaseTransactionOpsCore for IndexedDBPseudoTransaction<'a, S> { + async fn raw_insert_bytes( + &mut self, + key: &[u8], + value: &[u8], + ) -> anyhow::Result>> { + self.mem.raw_insert_bytes(key, value).await + } + + async fn raw_get_bytes(&mut self, key: &[u8]) -> anyhow::Result>> { + self.mem.raw_get_bytes(key).await + } + + async fn raw_remove_entry(&mut self, key: &[u8]) -> anyhow::Result>> { + self.mem.raw_remove_entry(key).await + } + + async fn raw_find_by_prefix(&mut self, key_prefix: &[u8]) -> anyhow::Result> { + self.mem.raw_find_by_prefix(key_prefix).await + } + + async fn raw_remove_by_prefix(&mut self, key_prefix: &[u8]) -> anyhow::Result<()> { + self.mem.raw_remove_by_prefix(key_prefix).await + } + + async fn raw_find_by_prefix_sorted_descending( + &mut self, + key_prefix: &[u8], + ) -> anyhow::Result> { + self.mem + .raw_find_by_prefix_sorted_descending(key_prefix) + .await + } +} + +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +impl<'a, S: MutinyStorage> IDatabaseTransactionOps for IndexedDBPseudoTransaction<'a, S> { + async fn rollback_tx_to_savepoint(&mut self) -> anyhow::Result<()> { + self.mem.rollback_tx_to_savepoint().await + } + + async fn set_tx_savepoint(&mut self) -> anyhow::Result<()> { + self.mem.set_tx_savepoint().await + } +} + #[cfg(test)] fn fedimint_seed_generation() { use crate::generate_seed; diff --git a/mutiny-core/src/ldkstorage.rs b/mutiny-core/src/ldkstorage.rs index 7c5189d7e..f3a9786a7 100644 --- a/mutiny-core/src/ldkstorage.rs +++ b/mutiny-core/src/ldkstorage.rs @@ -742,12 +742,7 @@ mod test { secret: None, last_update: utils::now().as_secs(), }; - let result = persist_payment_info( - persister.storage.clone(), - &payment_hash.0, - &payment_info, - true, - ); + let result = persist_payment_info(&persister.storage, &payment_hash.0, &payment_info, true); assert!(result.is_ok()); let result = read_payment_info( diff --git a/mutiny-core/src/lib.rs b/mutiny-core/src/lib.rs index 739dc1891..bbf588a48 100644 --- a/mutiny-core/src/lib.rs +++ b/mutiny-core/src/lib.rs @@ -53,7 +53,6 @@ use crate::{ federation::{FederationClient, FederationIdentity, FederationIndex, FederationStorage}, labels::{get_contact_key, Contact, LabelStorage}, nodemanager::NodeBalance, - sql::glue::GlueDB, }; use crate::{ lnurlauth::make_lnurl_auth_connection, @@ -400,7 +399,6 @@ pub struct MutinyWalletConfig { pub struct MutinyWalletBuilder { xprivkey: ExtendedPrivKey, storage: S, - glue_db: Option, config: Option, session_id: Option, network: Option, @@ -417,7 +415,6 @@ impl MutinyWalletBuilder { MutinyWalletBuilder:: { xprivkey, storage, - glue_db: None, config: None, session_id: None, network: None, @@ -442,10 +439,6 @@ impl MutinyWalletBuilder { self } - pub fn with_glue_db(&mut self, glue_db: GlueDB) { - self.glue_db = Some(glue_db); - } - pub fn with_session_id(&mut self, session_id: String) { self.session_id = Some(session_id); } @@ -526,25 +519,10 @@ impl MutinyWalletBuilder { // create federation module if any exist let federation_storage = self.storage.get_federations()?; - let (federations, glue_db) = if !federation_storage.federations.is_empty() { - // create gluedb storage - let glue_db = GlueDB::new( - #[cfg(target_arch = "wasm32")] - None, - logger.clone(), - ) - .await?; - - let federations = create_federations( - federation_storage.clone(), - &config, - glue_db.clone(), - &logger, - ) - .await?; - (federations, Some(glue_db)) + let federations = if !federation_storage.federations.is_empty() { + create_federations(federation_storage.clone(), &config, &self.storage, &logger).await? } else { - (Arc::new(RwLock::new(HashMap::new())), None) + Arc::new(RwLock::new(HashMap::new())) }; if !self.skip_hodl_invoices { @@ -581,7 +559,6 @@ impl MutinyWalletBuilder { xprivkey: self.xprivkey, config, storage: self.storage, - glue_db, node_manager, nostr, federation_storage: Arc::new(RwLock::new(federation_storage)), @@ -622,6 +599,9 @@ impl MutinyWalletBuilder { // start the nostr wallet connect background process mw.start_nostr_wallet_connect().await; + // start the federation background processor + mw.start_fedimint_background_checker().await; + Ok(mw) } } @@ -634,11 +614,10 @@ pub struct MutinyWallet { xprivkey: ExtendedPrivKey, config: MutinyWalletConfig, pub(crate) storage: S, - glue_db: Option, pub node_manager: Arc>, pub nostr: Arc>, pub federation_storage: Arc>, - pub(crate) federations: Arc>>>, + pub(crate) federations: Arc>>>>, lnurl_client: Arc, auth: AuthManager, subscription_client: Option>, @@ -977,13 +956,6 @@ impl MutinyWallet { // Get activities from node manager let mut activities = self.node_manager.get_activity().await?; - // Directly iterate over federation clients to get their activities - let federations = self.federations.read().await; - for (_fed_id, federation) in federations.iter() { - let federation_activities = federation.get_activity().await?; - activities.extend(federation_activities); - } - // Sort all activities, newest first activities.sort_by(|a, b| b.cmp(a)); @@ -1298,12 +1270,9 @@ impl MutinyWallet { Ok(()) } - /// Deletes all the storage and gluedb data + /// Deletes all the storage pub async fn delete_all(&self) -> Result<(), MutinyError> { self.storage.delete_all().await?; - if let Some(glue_db) = self.glue_db.as_ref() { - glue_db.delete_all().await?; - } Ok(()) } @@ -1311,11 +1280,7 @@ impl MutinyWallet { /// /// Backup the state beforehand. Does not restore lightning data. /// Should refresh or restart afterwards. Wallet should be stopped. - pub async fn restore_mnemonic( - mut storage: S, - glue_db: Option, - m: Mnemonic, - ) -> Result<(), MutinyError> { + pub async fn restore_mnemonic(mut storage: S, m: Mnemonic) -> Result<(), MutinyError> { // Delete our storage but insert some device specific data let device_id = storage.get_device_id()?; let logs: Option> = storage.get_data(LOGGING_KEY)?; @@ -1327,12 +1292,6 @@ impl MutinyWallet { storage.set_data(DEVICE_ID_KEY.to_string(), device_id, None)?; storage.set_data(LOGGING_KEY.to_string(), logs, None)?; - // Delete all of glue_db storage - // FIXME: on Safari this will clash with previous storage setup - if let Some(glue_db) = glue_db.as_ref() { - glue_db.delete_all().await?; - } - Ok(()) } @@ -1355,21 +1314,9 @@ impl MutinyWallet { &mut self, federation_code: InviteCode, ) -> Result { - // if we have not init glue_db yet, do it now - if self.glue_db.is_none() { - let glue_db = GlueDB::new( - #[cfg(target_arch = "wasm32")] - None, - self.logger.clone(), - ) - .await?; - self.glue_db = Some(glue_db); - } - create_new_federation( self.xprivkey, self.storage.clone(), - self.glue_db.as_ref().expect("just created this").clone(), self.network, self.logger.clone(), self.federation_storage.clone(), @@ -1458,6 +1405,47 @@ impl MutinyWallet { Ok(FederationBalances { balances }) } + /// Starts a background process that will check pending fedimint operations + pub(crate) async fn start_fedimint_background_checker(&self) { + let logger = self.logger.clone(); + let stop = self.stop.clone(); + let self_clone = self.clone(); + utils::spawn(async move { + loop { + if stop.load(Ordering::Relaxed) { + break; + }; + + sleep(1000).await; + let federation_lock = self_clone.federations.read().await; + + match self_clone.list_federation_ids().await { + Ok(federation_ids) => { + for fed_id in federation_ids { + match federation_lock.get(&fed_id) { + Some(fedimint_client) => { + let _ = fedimint_client.check_activity().await.map_err(|e| { + log_error!(logger, "error checking activity: {e}") + }); + } + None => { + log_error!( + logger, + "could not get a federation from the lock: {}", + fed_id + ) + } + } + } + } + Err(e) => { + log_error!(logger, "could not list federations: {e}") + } + } + } + }); + } + /// Calls upon a LNURL to get the parameters for it. /// This contains what kind of LNURL it is (pay, withdrawal, auth, etc). // todo revamp LnUrlParams to be well designed @@ -1642,12 +1630,12 @@ impl InvoiceHandler for MutinyWallet { } } -async fn create_federations( +async fn create_federations( federation_storage: FederationStorage, c: &MutinyWalletConfig, - g: GlueDB, + storage: &S, logger: &Arc, -) -> Result>>>, MutinyError> { +) -> Result>>>>, MutinyError> { let federations = federation_storage.federations.into_iter(); let mut federation_map = HashMap::new(); for federation_item in federations { @@ -1655,7 +1643,7 @@ async fn create_federations( federation_item.0, federation_item.1.federation_code.clone(), c.xprivkey, - g.clone(), + storage, c.network, logger.clone(), ) @@ -1674,11 +1662,10 @@ async fn create_federations( pub(crate) async fn create_new_federation( xprivkey: ExtendedPrivKey, storage: S, - g: GlueDB, network: Network, logger: Arc, federation_storage: Arc>, - federations: Arc>>>, + federations: Arc>>>>, federation_code: InviteCode, ) -> Result { // Begin with a mutex lock so that nothing else can @@ -1717,7 +1704,7 @@ pub(crate) async fn create_new_federation( next_federation_uuid.clone(), federation_code, xprivkey, - g.clone(), + &storage, network, logger.clone(), ) @@ -1745,12 +1732,9 @@ pub(crate) async fn create_new_federation( #[cfg(test)] mod tests { - use std::sync::Arc; - use crate::{ - encrypt::encryption_key_from_pass, generate_seed, logging::MutinyLogger, - nodemanager::NodeManager, sql::glue::GlueDB, MutinyWallet, MutinyWalletBuilder, - MutinyWalletConfigBuilder, + encrypt::encryption_key_from_pass, generate_seed, nodemanager::NodeManager, MutinyWallet, + MutinyWalletBuilder, MutinyWalletConfigBuilder, }; use bitcoin::util::bip32::ExtendedPrivKey; use bitcoin::Network; @@ -1895,15 +1879,7 @@ mod tests { let cipher = encryption_key_from_pass(&pass).unwrap(); let storage3 = MemoryStorage::new(Some(pass), Some(cipher), None); - let logger = Arc::new(MutinyLogger::default()); - let glue_db = GlueDB::new( - #[cfg(target_arch = "wasm32")] - Some(test_name.to_string()), - logger.clone(), - ) - .await - .unwrap(); - MutinyWallet::restore_mnemonic(storage3.clone(), Some(glue_db), mnemonic.clone()) + MutinyWallet::restore_mnemonic(storage3.clone(), mnemonic.clone()) .await .expect("mutiny wallet should restore"); diff --git a/mutiny-core/src/node.rs b/mutiny-core/src/node.rs index bb70d7235..d81a62377 100644 --- a/mutiny-core/src/node.rs +++ b/mutiny-core/src/node.rs @@ -1190,7 +1190,7 @@ impl Node { last_update, }; persist_payment_info( - self.persister.storage.clone(), + &self.persister.storage, &payment_hash.0, &payment_info, true, @@ -1393,12 +1393,7 @@ impl Node { last_update, }; - persist_payment_info( - self.persister.storage.clone(), - payment_hash, - &payment_info, - false, - )?; + persist_payment_info(&self.persister.storage, payment_hash, &payment_info, false)?; match pay_result { Ok(id) => Ok((id, PaymentHash(payment_hash.to_owned()))), @@ -1413,12 +1408,7 @@ impl Node { ); payment_info.status = HTLCStatus::Failed; - persist_payment_info( - self.persister.storage.clone(), - payment_hash, - &payment_info, - false, - )?; + persist_payment_info(&self.persister.storage, payment_hash, &payment_info, false)?; Err(map_sending_failure(error, amt_msat, ¤t_channels)) } @@ -1582,7 +1572,7 @@ impl Node { }; persist_payment_info( - self.persister.storage.clone(), + &self.persister.storage, &payment_hash.0, &payment_info, false, @@ -1597,7 +1587,7 @@ impl Node { Err(error) => { payment_info.status = HTLCStatus::Failed; persist_payment_info( - self.persister.storage.clone(), + &self.persister.storage, &payment_hash.0, &payment_info, false, @@ -2523,7 +2513,7 @@ mod tests { // check that it still fails if it is inflight persist_payment_info( - node.persister.storage.clone(), + &node.persister.storage, &payment_hash.0, &payment_info, false, @@ -2540,7 +2530,7 @@ mod tests { payment_info.status = HTLCStatus::Failed; persist_payment_info( - node.persister.storage.clone(), + &node.persister.storage, &payment_hash.0, &payment_info, false, @@ -2557,7 +2547,7 @@ mod tests { payment_info.status = HTLCStatus::Succeeded; persist_payment_info( - node.persister.storage.clone(), + &node.persister.storage, &payment_hash.0, &payment_info, false, @@ -2676,7 +2666,7 @@ mod wasm_test { // check that it still fails if it is inflight persist_payment_info( - node.persister.storage.clone(), + &node.persister.storage, &payment_hash.0, &payment_info, false, @@ -2693,7 +2683,7 @@ mod wasm_test { payment_info.status = HTLCStatus::Failed; persist_payment_info( - node.persister.storage.clone(), + &node.persister.storage, &payment_hash.0, &payment_info, false, @@ -2710,7 +2700,7 @@ mod wasm_test { payment_info.status = HTLCStatus::Succeeded; persist_payment_info( - node.persister.storage.clone(), + &node.persister.storage, &payment_hash.0, &payment_info, false, diff --git a/mutiny-core/src/nodemanager.rs b/mutiny-core/src/nodemanager.rs index 607e56a63..cdcee8a27 100644 --- a/mutiny-core/src/nodemanager.rs +++ b/mutiny-core/src/nodemanager.rs @@ -1,4 +1,4 @@ -use crate::event::{HTLCStatus, PaymentInfo}; +use crate::event::{HTLCStatus, MillisatAmount, PaymentInfo}; use crate::labels::LabelStorage; use crate::logging::LOGGING_KEY; use crate::utils::{sleep, spawn}; @@ -157,6 +157,40 @@ impl From for MutinyInvoice { } } +impl From for PaymentInfo { + fn from(invoice: MutinyInvoice) -> Self { + let preimage = invoice + .preimage + .map(|s| hex::decode(s).expect("preimage should decode")) + .map(|v| { + let mut arr = [0; 32]; + arr[..].copy_from_slice(&v); + arr + }); + let secret = None; + let status = invoice.status; + let amt_msat = invoice + .amount_sats + .map(|s| MillisatAmount(Some(s))) + .unwrap_or(MillisatAmount(None)); + let fee_paid_msat = invoice.fees_paid; + let bolt11 = invoice.bolt11; + let payee_pubkey = invoice.payee_pubkey; + let last_update = invoice.last_updated; + + PaymentInfo { + preimage, + secret, + status, + amt_msat, + fee_paid_msat, + bolt11, + payee_pubkey, + last_update, + } + } +} + impl MutinyInvoice { pub(crate) fn from( i: PaymentInfo, diff --git a/mutiny-core/src/storage.rs b/mutiny-core/src/storage.rs index bb6543f58..68213bd65 100644 --- a/mutiny-core/src/storage.rs +++ b/mutiny-core/src/storage.rs @@ -1,3 +1,5 @@ +use crate::ldkstorage::CHANNEL_MANAGER_KEY; +use crate::logging::MutinyLogger; use crate::nodemanager::{NodeStorage, DEVICE_LOCK_INTERVAL_SECS}; use crate::utils::{now, spawn}; use crate::vss::{MutinyVssClient, VssKeyValueItem}; @@ -9,11 +11,11 @@ use crate::{ error::{MutinyError, MutinyStorageError}, event::PaymentInfo, }; -use crate::{ldkstorage::CHANNEL_MANAGER_KEY, logging::MutinyLogger}; use async_trait::async_trait; use bdk::chain::{Append, PersistBackend}; use bip39::Mnemonic; use bitcoin::hashes::hex::ToHex; +use bitcoin::hashes::Hash; use hex::FromHex; use lightning::{ln::PaymentHash, util::logger::Logger}; use lightning::{log_error, log_trace}; @@ -677,7 +679,7 @@ fn payment_key(inbound: bool, payment_hash: &[u8; 32]) -> String { } pub(crate) fn persist_payment_info( - storage: S, + storage: &S, payment_hash: &[u8; 32], payment_info: &PaymentInfo, inbound: bool, @@ -688,6 +690,23 @@ pub(crate) fn persist_payment_info( .map_err(std::io::Error::other) } +pub(crate) fn get_payment_info( + storage: &S, + payment_hash: &bitcoin::hashes::sha256::Hash, + logger: &MutinyLogger, +) -> Result<(PaymentInfo, bool), MutinyError> { + // try inbound first + if let Some(payment_info) = read_payment_info(storage, payment_hash.as_inner(), true, logger) { + return Ok((payment_info, true)); + } + + // if no inbound check outbound + match read_payment_info(storage, payment_hash.as_inner(), false, logger) { + Some(payment_info) => Ok((payment_info, false)), + None => Err(MutinyError::NotFound), + } +} + pub(crate) fn read_payment_info( storage: &S, payment_hash: &[u8; 32], diff --git a/mutiny-wasm/src/lib.rs b/mutiny-wasm/src/lib.rs index 45cc1701f..d0711e93b 100644 --- a/mutiny-wasm/src/lib.rs +++ b/mutiny-wasm/src/lib.rs @@ -1655,7 +1655,6 @@ impl MutinyWallet { let storage = IndexedDbStorage::new(password, cipher, None, logger.clone()).await?; mutiny_core::MutinyWallet::::restore_mnemonic( storage, - None, // FIXME: We dont currently support deleting glue_db yet due to safari bug Mnemonic::from_str(&m).map_err(|_| MutinyJsError::InvalidMnemonic)?, ) .await?; From 18bd2ef6e34b05f081fab68faa0925e324025164 Mon Sep 17 00:00:00 2001 From: Tony Giorgio Date: Thu, 18 Jan 2024 19:46:12 -0600 Subject: [PATCH 3/7] Remove sql stuff --- Cargo.lock | 14 - mutiny-core/Cargo.toml | 2 - mutiny-core/src/lib.rs | 1 - mutiny-core/src/sql/glue.rs | 1319 ----------------------------------- mutiny-core/src/sql/mod.rs | 31 - 5 files changed, 1367 deletions(-) delete mode 100644 mutiny-core/src/sql/glue.rs delete mode 100644 mutiny-core/src/sql/mod.rs diff --git a/Cargo.lock b/Cargo.lock index d4645e0af..365d94223 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1663,7 +1663,6 @@ checksum = "45dfe66dc370c1d234bf8d0a22b7e62062af674f304ed3e347e3ad4e8f8224d3" dependencies = [ "gluesql-core", "gluesql-idb-storage", - "gluesql_memory_storage", ] [[package]] @@ -1726,18 +1725,6 @@ dependencies = [ "pin-project", ] -[[package]] -name = "gluesql_memory_storage" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ae7ac5bf854a6c966f26e58dda1bc5408260c772499731596428d12492b245" -dependencies = [ - "async-trait", - "futures", - "gluesql-core", - "serde", -] - [[package]] name = "group" version = "0.12.1" @@ -2554,7 +2541,6 @@ dependencies = [ "getrandom", "gloo-net", "gluesql", - "gluesql-core", "hex", "instant", "itertools 0.11.0", diff --git a/mutiny-core/Cargo.toml b/mutiny-core/Cargo.toml index 507beabfc..a93f34643 100644 --- a/mutiny-core/Cargo.toml +++ b/mutiny-core/Cargo.toml @@ -45,8 +45,6 @@ aes = { version = "0.8" } jwt-compact = { version = "0.8.0-beta.1", features = ["es256k"] } argon2 = { version = "0.5.0", features = ["password-hash", "alloc"] } payjoin = { version = "0.13.0", features = ["send", "base64"] } -gluesql = { version = "0.15", default-features = false, features = ["memory-storage"] } -gluesql-core = "0.15.0" bincode = "1.3.3" hex = "0.4.3" async-lock = "3.2.0" diff --git a/mutiny-core/src/lib.rs b/mutiny-core/src/lib.rs index bbf588a48..61bee717d 100644 --- a/mutiny-core/src/lib.rs +++ b/mutiny-core/src/lib.rs @@ -32,7 +32,6 @@ pub mod nostr; mod onchain; mod peermanager; pub mod scorer; -pub mod sql; pub mod storage; mod subscription; pub mod utils; diff --git a/mutiny-core/src/sql/glue.rs b/mutiny-core/src/sql/glue.rs deleted file mode 100644 index fd6be27c6..000000000 --- a/mutiny-core/src/sql/glue.rs +++ /dev/null @@ -1,1319 +0,0 @@ -use crate::HTLCStatus; -use crate::{ - error::{MutinyError, MutinyStorageError}, - logging::MutinyLogger, - nodemanager::MutinyInvoice, - sql::ApplicationStore, -}; -use async_trait::async_trait; -use bitcoin::hashes::sha256; -use bitcoin::secp256k1::PublicKey; -use fedimint_core::db::{ - mem_impl::{MemDatabase, MemTransaction}, - IDatabaseTransactionOps, IDatabaseTransactionOpsCore, IRawDatabase, IRawDatabaseTransaction, - PrefixStream, -}; -use gluesql::prelude::{Glue, Payload, Value}; -use lightning::{log_debug, log_error, log_info, log_trace, util::logger::Logger}; -use lightning_invoice::Bolt11Invoice; -use std::str::FromStr; - -use std::{fmt, sync::Arc}; - -#[cfg(not(target_arch = "wasm32"))] -use tokio::sync::Mutex; - -#[cfg(target_arch = "wasm32")] -use gluesql::core::executor::ValidateError; - -#[cfg(target_arch = "wasm32")] -use crate::utils; - -#[cfg(target_arch = "wasm32")] -use futures::lock::Mutex; - -#[cfg(target_arch = "wasm32")] -use futures::StreamExt; - -#[cfg(not(target_arch = "wasm32"))] -use gluesql::prelude::MemoryStorage; - -#[cfg(not(target_arch = "wasm32"))] -#[derive(Clone)] -pub struct FedimintDB { - pub(crate) db: Arc>>, // TODO eventually use sled for server version - fedimint_memory: Arc, - federation_id: String, -} - -#[cfg(not(target_arch = "wasm32"))] -#[derive(Clone)] -pub struct GlueDB { - pub(crate) db: Arc>>, // TODO eventually use sled for server version - logger: Arc, -} - -#[cfg(target_arch = "wasm32")] -use gluesql::prelude::IdbStorage; - -#[cfg(target_arch = "wasm32")] -#[derive(Clone)] -pub struct GlueDB { - pub(crate) db: Arc>>, - logger: Arc, -} - -impl GlueDB { - pub async fn new( - #[cfg(target_arch = "wasm32")] namespace: Option, - logger: Arc, - ) -> Result { - log_debug!(logger, "initializing glue storage"); - - #[cfg(target_arch = "wasm32")] - let storage = IdbStorage::new(namespace) - .await - .map_err(|_| MutinyError::ReadError { - source: MutinyStorageError::IndexedDBError, - })?; - - #[cfg(not(target_arch = "wasm32"))] - let storage = MemoryStorage::default(); - - let mut glue_db = Glue::new(storage); - glue_db - .execute( - "CREATE TABLE IF NOT EXISTS mutiny_kv (key TEXT PRIMARY KEY, val BYTEA NOT NULL, version INTEGER NOT NULL)", - ) - .await - .map_err(|_| MutinyError::write_err(MutinyStorageError::IndexedDBError))?; - - glue_db - .execute( - "CREATE TABLE IF NOT EXISTS mutiny_invoice ( - bolt11 TEXT, - description TEXT NULL, - payment_hash TEXT PRIMARY KEY, - preimage TEXT NULL, - payee_pubkey TEXT NULL, - amount_sats INTEGER NULL, - expire INTEGER NOT NULL, - status TEXT NOT NULL, - fees_paid INTEGER NULL, - inbound BOOLEAN, - labels TEXT, - last_updated INTEGER - )", - ) - .await - .map_err(|_| MutinyError::write_err(MutinyStorageError::IndexedDBError))?; - - log_debug!(logger, "done setting up GlueDB"); - - Ok(Self { - db: Arc::new(Mutex::new(glue_db)), - logger, - }) - } - - pub async fn new_fedimint_client_db( - &self, - federation_id: String, - ) -> Result { - FedimintDB::new(self.db.clone(), federation_id, self.logger.clone()).await - } - - pub async fn delete_all(&self) -> Result<(), MutinyError> { - let mut db = self.db.lock().await; - - db.execute("DROP TABLE IF EXISTS mutiny_kv") - .await - .map_err(|_| MutinyError::write_err(MutinyStorageError::IndexedDBError))?; - - db.execute("DROP TABLE IF EXISTS mutiny_invoice") - .await - .map_err(|_| MutinyError::write_err(MutinyStorageError::IndexedDBError))?; - - log_info!(self.logger, "All data deleted from GlueDB"); - - Ok(()) - } -} - -impl ApplicationStore for GlueDB { - async fn save_payment(&self, invoice: MutinyInvoice) -> Result<(), MutinyError> { - #[cfg(not(target_arch = "wasm32"))] - unimplemented!("can't run on servers until Send is supported in Glue"); - - #[cfg(target_arch = "wasm32")] - { - let labels_json = serde_json::to_string(&invoice.labels).map_err(|_| { - MutinyError::PersistenceFailed { - source: MutinyStorageError::IndexedDBError, - } - })?; - let payment_hash = invoice.payment_hash.to_string(); - let payee_pubkey = invoice.payee_pubkey.map(|k| k.to_string()); - let bolt11 = invoice.bolt11.map(|b| b.to_string()); - let preimage = invoice.preimage.clone(); - let status = invoice.status.to_string(); - - let sql = format!( - "INSERT INTO mutiny_invoice (bolt11, description, payment_hash, preimage, payee_pubkey, amount_sats, expire, status, fees_paid, inbound, labels, last_updated) - VALUES ({}, {}, '{}', {}, {}, {}, {}, '{}', {}, {}, '{}', {})", - bolt11.as_ref().map(|s| format!("'{}'", s)).unwrap_or("NULL".to_string()), - invoice.description.as_ref().map(|s| format!("'{}'", s)).unwrap_or("NULL".to_string()), - payment_hash, - preimage.as_ref().map(|s| format!("'{}'", s)).unwrap_or("NULL".to_string()), - payee_pubkey.as_ref().map(|s| format!("'{}'", s)).unwrap_or("NULL".to_string()), - invoice.amount_sats.as_ref().map(|s| format!("{}", s)).unwrap_or("NULL".to_string()), - invoice.expire, - status, - invoice.fees_paid.as_ref().map(|s| format!("{}", s)).unwrap_or("NULL".to_string()), - invoice.inbound, - labels_json, - invoice.last_updated, - ); - - let mut glue = self.db.lock().await; - match glue.execute(&sql).await { - Ok(_) => Ok(()), - Err(gluesql_core::error::Error::Validate( - ValidateError::DuplicateEntryOnPrimaryKeyField(_), - )) => { - // Define the UPDATE query - let update_sql = format!( - "UPDATE mutiny_invoice - SET bolt11 = {}, description = {}, preimage = {}, payee_pubkey = {}, - amount_sats = {}, expire = {}, status = '{}', fees_paid = {}, inbound = {}, - labels = '{}', last_updated = {} - WHERE payment_hash = '{}'", - bolt11.as_ref().map(|s| format!("'{}'", s)).unwrap_or("NULL".to_string()), - invoice.description.as_ref().map(|s| format!("'{}'", s)).unwrap_or("NULL".to_string()), - preimage.as_ref().map(|s| format!("'{}'", s)).unwrap_or("NULL".to_string()), - payee_pubkey.as_ref().map(|s| format!("'{}'", s)).unwrap_or("NULL".to_string()), - invoice.amount_sats.as_ref().map(|s| format!("{}", s)).unwrap_or("NULL".to_string()), - invoice.expire, - status, - invoice.fees_paid.as_ref().map(|s| format!("{}", s)).unwrap_or("NULL".to_string()), - invoice.inbound, - labels_json, - invoice.last_updated, - payment_hash, - ); - - glue.execute(&update_sql).await.map_err(|_| { - MutinyError::PersistenceFailed { - source: MutinyStorageError::IndexedDBError, - } - })?; - Ok(()) - } - _ => Err(MutinyError::PersistenceFailed { - source: MutinyStorageError::IndexedDBError, - }), - } - } - } - - async fn get_payment( - &self, - payment_hash: &bitcoin::hashes::sha256::Hash, - ) -> Result, MutinyError> { - #[cfg(not(target_arch = "wasm32"))] - unimplemented!("can't run on servers until Send is supported in Glue"); - - #[cfg(target_arch = "wasm32")] - { - log_trace!(self.logger, "calling get_payment"); - - let payment_hash_str = payment_hash.to_string(); - let select_query = format!( - "SELECT bolt11, description, payment_hash, preimage, payee_pubkey, amount_sats, expire, status, fees_paid, inbound, labels, last_updated FROM mutiny_invoice WHERE payment_hash = '{}'", - payment_hash_str - ); - - log_trace!(self.logger, "locking db"); - let mut glue = self.db.lock().await; - log_trace!(self.logger, "running query: {select_query}"); - let mut result = glue.execute(&select_query).await.map_err(|e| { - log_error!( - self.logger, - "failed to execute query ({}): {e}", - select_query - ); - MutinyError::PersistenceFailed { - source: MutinyStorageError::IndexedDBError, - } - })?; - - log_trace!(self.logger, "going through rows"); - if let Payload::Select { rows, .. } = result.pop().unwrap() { - if let Some(row) = rows.first() { - log_trace!(self.logger, "parsing first row"); - let invoice = parse_row_to_invoice(row.to_vec(), self.logger.clone())?; - Ok(Some(invoice)) - } else { - Ok(None) - } - } else { - log_error!( - self.logger, - "could not find a row when executing query ({})", - select_query - ); - Err(MutinyError::PersistenceFailed { - source: MutinyStorageError::IndexedDBError, - }) - } - } - } - - async fn update_payment_status( - &self, - payment_hash: &bitcoin::hashes::sha256::Hash, - status: HTLCStatus, - ) -> Result<(), MutinyError> { - #[cfg(not(target_arch = "wasm32"))] - unimplemented!("can't run on servers until Send is supported in Glue"); - - #[cfg(target_arch = "wasm32")] - { - let status_str = status.to_string(); - let payment_hash_str = payment_hash.to_string(); - let now = utils::now().as_secs(); - let sql = format!( - "UPDATE mutiny_invoice SET status = '{}', last_updated = {} WHERE payment_hash = '{}'", - status_str, now, payment_hash_str - ); - - let mut glue = self.db.lock().await; - glue.execute(&sql) - .await - .map_err(|_| MutinyError::PersistenceFailed { - source: MutinyStorageError::IndexedDBError, - })?; - Ok(()) - } - } - - async fn update_payment_fee( - &self, - payment_hash: &bitcoin::hashes::sha256::Hash, - fee: Option, - ) -> Result<(), MutinyError> { - #[cfg(not(target_arch = "wasm32"))] - unimplemented!("can't run on servers until Send is supported in Glue"); - - #[cfg(target_arch = "wasm32")] - { - let fee_str = match fee { - Some(fee_value) => fee_value.to_string(), - None => "NULL".to_string(), - }; - let payment_hash_str = payment_hash.to_string(); - let now = utils::now().as_secs(); - let sql = format!( - "UPDATE mutiny_invoice SET fees_paid = {}, last_updated = {} WHERE payment_hash = '{}'", - fee_str, now, payment_hash_str - ); - - let mut glue = self.db.lock().await; - glue.execute(&sql) - .await - .map_err(|_| MutinyError::PersistenceFailed { - source: MutinyStorageError::IndexedDBError, - })?; - Ok(()) - } - } - - async fn update_payment_preimage( - &self, - payment_hash: &bitcoin::hashes::sha256::Hash, - preimage: Option, - ) -> Result<(), MutinyError> { - #[cfg(not(target_arch = "wasm32"))] - unimplemented!("can't run on servers until Send is supported in Glue"); - - #[cfg(target_arch = "wasm32")] - { - let preimage_str = match preimage { - Some(ref img) => format!("'{}'", img), - None => "NULL".to_string(), - }; - let payment_hash_str = payment_hash.to_string(); - let now = utils::now().as_secs(); - let sql = format!( - "UPDATE mutiny_invoice SET preimage = {}, last_updated = {} WHERE payment_hash = '{}'", - preimage_str, now, payment_hash_str - ); - - let mut glue = self.db.lock().await; - glue.execute(&sql) - .await - .map_err(|_| MutinyError::PersistenceFailed { - source: MutinyStorageError::IndexedDBError, - })?; - Ok(()) - } - } - - async fn list_payments(&self) -> Result, MutinyError> { - #[cfg(not(target_arch = "wasm32"))] - unimplemented!("can't run on servers until Send is supported in Glue"); - - #[cfg(target_arch = "wasm32")] - { - let select_query = "SELECT bolt11, description, payment_hash, preimage, payee_pubkey, amount_sats, expire, status, fees_paid, inbound, labels, last_updated FROM mutiny_invoice"; - - let mut glue = self.db.lock().await; - let mut result = - glue.execute(&select_query) - .await - .map_err(|_| MutinyError::PersistenceFailed { - source: MutinyStorageError::IndexedDBError, - })?; - - let mut invoices = Vec::new(); - if let Payload::Select { rows, .. } = result.pop().unwrap() { - for row in rows { - let invoice = parse_row_to_invoice(row.to_vec(), self.logger.clone())?; - invoices.push(invoice); - } - } else { - return Err(MutinyError::PersistenceFailed { - source: MutinyStorageError::IndexedDBError, - }); - } - - Ok(invoices) - } - } -} - -fn parse_row_to_invoice( - row: Vec, - logger: Arc, -) -> Result { - let bolt11 = match &row[0] { - Value::Str(val) => { - let b = Bolt11Invoice::from_str(val).map_err(|e| { - log_error!(logger, "failed to parse invoice ({}): {e}", val); - e - })?; - Some(b) - } - _ => None, - }; - let description = match &row[1] { - Value::Str(val) => Some(val.clone()), - _ => None, - }; - let payment_hash = match &row[2] { - Value::Str(val) => sha256::Hash::from_str(val).map_err(|e| { - log_error!(logger, "failed to parse hash ({}): {e}", val); - e - })?, - _ => panic!("Expected String for preimage"), - }; - let preimage = match &row[3] { - Value::Str(val) => Some(val.clone()), - _ => None, - }; - let payee_pubkey = match &row[4] { - Value::Str(val) => { - let b = PublicKey::from_str(val).map_err(|e| { - log_error!(logger, "failed to parse pubkey ({}): {e}", val); - MutinyError::PersistenceFailed { - source: MutinyStorageError::IndexedDBError, - } - })?; - Some(b) - } - _ => None, - }; - let amount_sats = match &row[5] { - Value::I64(val) => Some(*val as u64), - _ => None, - }; - let expire = match &row[6] { - Value::I64(val) => *val as u64, - _ => panic!("Expected i64 for expire"), - }; - let status = match &row[7] { - Value::Str(val) => HTLCStatus::from_str(val).map_err(|e| { - log_error!(logger, "failed to parse status ({}): {e}", val); - MutinyError::PersistenceFailed { - source: MutinyStorageError::IndexedDBError, - } - })?, - _ => HTLCStatus::Pending, - }; - let fees_paid = match &row[8] { - Value::I64(val) => Some(*val as u64), - _ => None, - }; - let inbound = match &row[9] { - Value::Bool(val) => *val, - _ => false, - }; - let labels = match &row[10] { - Value::Str(val) => { - let labels_vec: Vec = serde_json::from_str(val).map_err(|e| { - log_error!(logger, "failed to parse labels ({}): {e}", val); - MutinyError::PersistenceFailed { - source: MutinyStorageError::IndexedDBError, - } - })?; - labels_vec - } - _ => vec![], - }; - let last_updated = match &row[11] { - Value::I64(val) => *val as u64, - _ => panic!("Expected i64 for last_updated"), - }; - - Ok(MutinyInvoice { - bolt11, - description, - payment_hash, - preimage, - payee_pubkey, - amount_sats, - expire, - status, - fees_paid, - inbound, - labels, - last_updated, - }) -} - -#[cfg(target_arch = "wasm32")] -#[derive(Clone)] -pub struct FedimintDB { - pub(crate) db: Arc>>, - fedimint_memory: Arc, - federation_id: String, -} - -impl fmt::Debug for FedimintDB { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("FedimintDB").finish() - } -} - -impl FedimintDB { - pub async fn new( - #[cfg(target_arch = "wasm32")] db: Arc>>, - #[cfg(not(target_arch = "wasm32"))] db: Arc>>, - federation_id: String, - logger: Arc, - ) -> Result { - log_debug!(logger, "initializing glue storage"); - - let fedimint_memory = MemDatabase::new(); - - { - let mut glue_db = db.lock().await; - - let select_query = format!( - "SELECT val FROM mutiny_kv WHERE key = '{}'", - key_id(&federation_id) - ); - let mut result = - glue_db - .execute(select_query) - .await - .map_err(|_| MutinyError::ReadError { - source: MutinyStorageError::IndexedDBError, - })?; - - if let Payload::Select { rows, .. } = result.pop().expect("should get something") { - if rows.is_empty() { - let stmt = format!( - "INSERT INTO mutiny_kv (key, val, version) VALUES ('{}', X'', 0)", - key_id(&federation_id) - ); - glue_db - .execute(stmt) - .await - .map_err(|_| MutinyError::ReadError { - source: MutinyStorageError::IndexedDBError, - })?; - } else if let Some(row) = rows.first() { - if let Value::Bytea(binary_data) = &row[0] { - if !binary_data.is_empty() { - let key_value_pairs: Vec<(Vec, Vec)> = - bincode::deserialize(binary_data).map_err(|e| { - MutinyError::ReadError { - source: MutinyStorageError::Other(e.into()), - } - })?; - - let mut mem_db_tx = fedimint_memory.begin_transaction().await; - for (key, value) in key_value_pairs { - mem_db_tx - .raw_insert_bytes(&key, &value) - .await - .map_err(|_| { - MutinyError::write_err(MutinyStorageError::IndexedDBError) - })?; - } - mem_db_tx.commit_tx().await.map_err(|_| { - MutinyError::write_err(MutinyStorageError::IndexedDBError) - })?; - } - } else { - return Err(MutinyError::ReadError { - source: MutinyStorageError::IndexedDBError, - }); - } - } - } else { - return Err(MutinyError::ReadError { - source: MutinyStorageError::IndexedDBError, - }); - } - } - - log_debug!(logger, "done setting up FedimintDB for fedimint"); - - Ok(Self { - db, - federation_id, - fedimint_memory: Arc::new(fedimint_memory), - }) - } -} - -fn key_id(federation_id: &str) -> String { - format!("fedimint_key_{}", federation_id) -} - -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -impl IRawDatabase for FedimintDB { - type Transaction<'a> = GluePseudoTransaction<'a>; - - async fn begin_transaction<'a>(&'a self) -> GluePseudoTransaction { - GluePseudoTransaction { - db: self.db.clone(), - federation_id: self.federation_id.clone(), - mem: self.fedimint_memory.begin_transaction().await, - } - } -} - -pub struct GluePseudoTransaction<'a> { - #[cfg(not(target_arch = "wasm32"))] - pub(crate) db: Arc>>, - #[cfg(target_arch = "wasm32")] - pub(crate) db: Arc>>, - federation_id: String, - mem: MemTransaction<'a>, -} - -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -impl<'a> IRawDatabaseTransaction for GluePseudoTransaction<'a> { - async fn commit_tx(mut self) -> anyhow::Result<()> { - #[cfg(not(target_arch = "wasm32"))] - unimplemented!("can't run on servers until Send is supported in Glue"); - - #[cfg(target_arch = "wasm32")] - { - let key_value_pairs = self - .mem - .raw_find_by_prefix(&[]) - .await? - .collect::, Vec)>>() - .await; - self.mem.commit_tx().await?; - - let serialized_data = - bincode::serialize(&key_value_pairs).map_err(anyhow::Error::new)?; - let hex_serialized_data = hex::encode(serialized_data); - - let update_query = format!( - "UPDATE mutiny_kv SET val = X'{}' WHERE key = '{}'", - hex_serialized_data, - key_id(&self.federation_id) - ); - - let mut db = self.db.lock().await; - - db.execute(&update_query) - .await - .map_err(anyhow::Error::new)?; - - Ok(()) - } - } -} - -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -impl<'a> IDatabaseTransactionOpsCore for GluePseudoTransaction<'a> { - async fn raw_insert_bytes( - &mut self, - key: &[u8], - value: &[u8], - ) -> anyhow::Result>> { - self.mem.raw_insert_bytes(key, value).await - } - - async fn raw_get_bytes(&mut self, key: &[u8]) -> anyhow::Result>> { - self.mem.raw_get_bytes(key).await - } - - async fn raw_remove_entry(&mut self, key: &[u8]) -> anyhow::Result>> { - self.mem.raw_remove_entry(key).await - } - - async fn raw_find_by_prefix(&mut self, key_prefix: &[u8]) -> anyhow::Result> { - self.mem.raw_find_by_prefix(key_prefix).await - } - - async fn raw_remove_by_prefix(&mut self, key_prefix: &[u8]) -> anyhow::Result<()> { - self.mem.raw_remove_by_prefix(key_prefix).await - } - - async fn raw_find_by_prefix_sorted_descending( - &mut self, - key_prefix: &[u8], - ) -> anyhow::Result> { - self.mem - .raw_find_by_prefix_sorted_descending(key_prefix) - .await - } -} - -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -impl<'a> IDatabaseTransactionOps for GluePseudoTransaction<'a> { - async fn rollback_tx_to_savepoint(&mut self) -> anyhow::Result<()> { - self.mem.rollback_tx_to_savepoint().await - } - - async fn set_tx_savepoint(&mut self) -> anyhow::Result<()> { - self.mem.set_tx_savepoint().await - } -} - -#[cfg(test)] -use gluesql::core::store::GStore; -#[cfg(test)] -use gluesql::core::store::GStoreMut; - -#[cfg(test)] -async fn create_glue_and_fedimint_storage() { - let db = GlueDB::new( - #[cfg(target_arch = "wasm32")] - Some("create_glue_and_fedimint_storage".to_string()), - Arc::new(MutinyLogger::default()), - ) - .await - .unwrap(); - - db.new_fedimint_client_db("create_glue_storage".to_string()) - .await - .unwrap(); -} - -#[cfg(test)] -async fn create_glue_and_payments_storage() { - use bitcoin::hashes::hex::{FromHex, ToHex}; - - const INVOICE: &str = "lnbc923720n1pj9nr6zpp5xmvlq2u5253htn52mflh2e6gn7pk5ht0d4qyhc62fadytccxw7hqhp5l4s6qwh57a7cwr7zrcz706qx0qy4eykcpr8m8dwz08hqf362egfscqzzsxqzfvsp5pr7yjvcn4ggrf6fq090zey0yvf8nqvdh2kq7fue0s0gnm69evy6s9qyyssqjyq0fwjr22eeg08xvmz88307yqu8tqqdjpycmermks822fpqyxgshj8hvnl9mkh6srclnxx0uf4ugfq43d66ak3rrz4dqcqd23vxwpsqf7dmhm"; - - let db = GlueDB::new( - #[cfg(target_arch = "wasm32")] - Some("create_glue_and_payments_storage".to_string()), - Arc::new(MutinyLogger::default()), - ) - .await - .unwrap(); - - let preimage: [u8; 32] = - FromHex::from_hex("7600f5a9ad72452dea7ad86dabbc9cb46be96a1a2fcd961e041d066b38d93008") - .unwrap(); - - let payment_hash = - sha256::Hash::from_hex("55ecf9169a6fa07e8ba181fdddf5b0bcc7860176659fa22a7cca9da2a359a33b") - .unwrap(); - - let pubkey = - PublicKey::from_str("02465ed5be53d04fde66c9418ff14a5f2267723810176c9212b722e542dc1afb1b") - .unwrap(); - - let i = Bolt11Invoice::from_str(INVOICE).unwrap(); - - let label = "test".to_string(); - let labels = vec![label.clone()]; - - let invoice1: MutinyInvoice = MutinyInvoice { - bolt11: Some(i), - description: Some("dest".to_string()), - payment_hash, - preimage: Some(preimage.to_hex()), - payee_pubkey: Some(pubkey), - amount_sats: Some(100), - expire: 1681781585, - status: HTLCStatus::Succeeded, - fees_paid: Some(1), - inbound: false, - labels: labels.clone(), - last_updated: 1681781585, - }; - db.save_payment(invoice1.clone()).await.unwrap(); - - let db_invoice = db.get_payment(&payment_hash).await.unwrap().unwrap(); - assert_eq!(invoice1, db_invoice); - - let db_invoices = db.list_payments().await.unwrap(); - assert_eq!(db_invoices.len(), 1); - assert_eq!(invoice1, db_invoices[0]); - - let payment_hash2 = - sha256::Hash::from_hex("05ecf9169a6fa07e8ba181fdddf5b0bcc7860176659fa22a7cca9da2a359a33b") - .unwrap(); - - let i = Bolt11Invoice::from_str(INVOICE).unwrap(); - - let mut invoice2: MutinyInvoice = MutinyInvoice { - bolt11: Some(i.clone()), - description: Some("dest".to_string()), - payment_hash: payment_hash2, - preimage: Some(preimage.to_hex()), - payee_pubkey: Some(pubkey), - amount_sats: Some(1000), - expire: 1681781585, - status: HTLCStatus::Pending, - fees_paid: Some(1), - inbound: false, - labels: labels.clone(), - last_updated: 1681781585, - }; - db.save_payment(invoice2.clone()).await.unwrap(); - - let db_invoice2 = db.get_payment(&payment_hash2).await.unwrap().unwrap(); - assert_eq!(invoice2, db_invoice2); - assert_ne!(invoice1, db_invoice2); - - let db_invoices = db.list_payments().await.unwrap(); - assert_eq!(db_invoices.len(), 2); - assert_eq!(invoice1, db_invoices[1]); - assert_eq!(invoice2, db_invoices[0]); - - invoice2.status = HTLCStatus::Succeeded; - db.save_payment(invoice2.clone()).await.unwrap(); - let db_invoice2_updated = db.get_payment(&payment_hash2).await.unwrap().unwrap(); - assert_eq!(invoice2, db_invoice2_updated); - assert_ne!(db_invoice2, db_invoice2_updated); - - // allow nullable values - let payment_hash_nullable = - sha256::Hash::from_hex("44ecf9169a6fa07e8ba181fdddf5b0bcc7860176659fa22a7cca9da2a359a33b") - .unwrap(); - let invoice_nullable: MutinyInvoice = MutinyInvoice { - bolt11: Some(i), - description: None, - payment_hash: payment_hash_nullable, - preimage: None, - payee_pubkey: None, - amount_sats: None, - expire: 1681781585, - status: HTLCStatus::Succeeded, - fees_paid: None, - inbound: false, - labels: labels.clone(), - last_updated: 1681781585, - }; - db.save_payment(invoice_nullable.clone()).await.unwrap(); - - let db_invoice_nullable = db - .get_payment(&payment_hash_nullable) - .await - .unwrap() - .unwrap(); - assert_eq!(invoice_nullable, db_invoice_nullable); -} - -#[cfg(test)] -async fn update_payments() { - use bitcoin::hashes::hex::{FromHex, ToHex}; - - const INVOICE: &str = "lnbc923720n1pj9nr6zpp5xmvlq2u5253htn52mflh2e6gn7pk5ht0d4qyhc62fadytccxw7hqhp5l4s6qwh57a7cwr7zrcz706qx0qy4eykcpr8m8dwz08hqf362egfscqzzsxqzfvsp5pr7yjvcn4ggrf6fq090zey0yvf8nqvdh2kq7fue0s0gnm69evy6s9qyyssqjyq0fwjr22eeg08xvmz88307yqu8tqqdjpycmermks822fpqyxgshj8hvnl9mkh6srclnxx0uf4ugfq43d66ak3rrz4dqcqd23vxwpsqf7dmhm"; - - let db = GlueDB::new( - #[cfg(target_arch = "wasm32")] - Some("update_payments".to_string()), - Arc::new(MutinyLogger::default()), - ) - .await - .unwrap(); - - let preimage: [u8; 32] = - FromHex::from_hex("7600f5a9ad72452dea7ad86dabbc9cb46be96a1a2fcd961e041d066b38d93008") - .unwrap(); - - let payment_hash = - sha256::Hash::from_hex("55ecf9169a6fa07e8ba181fdddf5b0bcc7860176659fa22a7cca9da2a359a33b") - .unwrap(); - - let pubkey = - PublicKey::from_str("02465ed5be53d04fde66c9418ff14a5f2267723810176c9212b722e542dc1afb1b") - .unwrap(); - - let i = Bolt11Invoice::from_str(INVOICE).unwrap(); - - let label = "test".to_string(); - let labels = vec![label.clone()]; - - let invoice1: MutinyInvoice = MutinyInvoice { - bolt11: Some(i), - description: Some("dest".to_string()), - payment_hash, - preimage: Some(preimage.to_hex()), - payee_pubkey: Some(pubkey), - amount_sats: Some(100), - expire: 1681781585, - status: HTLCStatus::Pending, - fees_paid: Some(1), - inbound: false, - labels: labels.clone(), - last_updated: 1681781585, - }; - db.save_payment(invoice1.clone()).await.unwrap(); - - let db_invoice = db.get_payment(&payment_hash).await.unwrap().unwrap(); - assert_eq!(invoice1, db_invoice); - - let db_invoices = db.list_payments().await.unwrap(); - assert_eq!(db_invoices.len(), 1); - assert_eq!(invoice1, db_invoices[0]); - - // Test update_payment_status - let new_status = HTLCStatus::Succeeded; - db.update_payment_status(&payment_hash, new_status.clone()) - .await - .unwrap(); - let updated_invoice = db.get_payment(&payment_hash).await.unwrap().unwrap(); - assert_eq!(updated_invoice.status, new_status); - - // Test update_payment_fee - let new_fee = Some(10u64); - db.update_payment_fee(&payment_hash, new_fee).await.unwrap(); - let updated_invoice_fee = db.get_payment(&payment_hash).await.unwrap().unwrap(); - assert_eq!(updated_invoice_fee.fees_paid, new_fee); - - // Test update_payment_preimage - let new_preimage: Option = - Some("0600f5a9ad72452dea7ad86dabbc9cb46be96a1a2fcd961e041d066b38d93008".to_string()); - db.update_payment_preimage(&payment_hash, new_preimage.clone()) - .await - .unwrap(); - let updated_invoice_preimage = db.get_payment(&payment_hash).await.unwrap().unwrap(); - assert_eq!(updated_invoice_preimage.preimage, new_preimage); - - // Test to make the timestamp was updated - assert!(updated_invoice.last_updated <= updated_invoice_fee.last_updated); - assert!(updated_invoice_fee.last_updated <= updated_invoice_preimage.last_updated); -} - -#[cfg(test)] -async fn delete_all() { - use bitcoin::hashes::hex::{FromHex, ToHex}; - - const INVOICE: &str = "lnbc923720n1pj9nr6zpp5xmvlq2u5253htn52mflh2e6gn7pk5ht0d4qyhc62fadytccxw7hqhp5l4s6qwh57a7cwr7zrcz706qx0qy4eykcpr8m8dwz08hqf362egfscqzzsxqzfvsp5pr7yjvcn4ggrf6fq090zey0yvf8nqvdh2kq7fue0s0gnm69evy6s9qyyssqjyq0fwjr22eeg08xvmz88307yqu8tqqdjpycmermks822fpqyxgshj8hvnl9mkh6srclnxx0uf4ugfq43d66ak3rrz4dqcqd23vxwpsqf7dmhm"; - - let db = GlueDB::new( - #[cfg(target_arch = "wasm32")] - Some("delete_all".to_string()), - Arc::new(MutinyLogger::default()), - ) - .await - .unwrap(); - - // Insert something into the payments table - let preimage: [u8; 32] = - FromHex::from_hex("7600f5a9ad72452dea7ad86dabbc9cb46be96a1a2fcd961e041d066b38d93008") - .unwrap(); - let payment_hash = - sha256::Hash::from_hex("55ecf9169a6fa07e8ba181fdddf5b0bcc7860176659fa22a7cca9da2a359a33b") - .unwrap(); - let pubkey = - PublicKey::from_str("02465ed5be53d04fde66c9418ff14a5f2267723810176c9212b722e542dc1afb1b") - .unwrap(); - let i = Bolt11Invoice::from_str(INVOICE).unwrap(); - let label = "test".to_string(); - let labels = vec![label.clone()]; - let invoice1: MutinyInvoice = MutinyInvoice { - bolt11: Some(i), - description: Some("dest".to_string()), - payment_hash, - preimage: Some(preimage.to_hex()), - payee_pubkey: Some(pubkey), - amount_sats: Some(100), - expire: 1681781585, - status: HTLCStatus::Pending, - fees_paid: Some(1), - inbound: false, - labels: labels.clone(), - last_updated: 1681781585, - }; - db.save_payment(invoice1.clone()).await.unwrap(); - let db_invoices = db.list_payments().await.unwrap(); - assert_eq!(1, db_invoices.len()); - - // now insert something into the kv table too - db.db - .lock() - .await - .execute("INSERT INTO mutiny_kv (key, val, version) VALUES ('storage', X'', 0)") - .await - .unwrap(); - let select_query = "SELECT val FROM mutiny_kv WHERE key = 'storage'"; - let mut result = db.db.lock().await.execute(select_query).await.unwrap(); - if let Payload::Select { rows, .. } = result.pop().expect("should get something") { - if let Some(row) = rows.first() { - if let Value::Bytea(hex_string) = &row[0] { - let serialized_data = hex::decode(hex_string).unwrap(); - assert!(serialized_data.is_empty()); - } else { - panic!("Expected bytea"); - } - } - } else { - panic!("Expected Payload::Select"); - } - - // Now delete it all - db.delete_all().await.unwrap(); - drop(db); - - // Start up the DB again - let db = GlueDB::new( - #[cfg(target_arch = "wasm32")] - Some("delete_all".to_string()), - Arc::new(MutinyLogger::default()), - ) - .await - .unwrap(); - - // Test to make sure it's not there - let db_invoices = db.list_payments().await.unwrap(); - assert_eq!(0, db_invoices.len()); - - let mut result = db.db.lock().await.execute(select_query).await.unwrap(); - if let Payload::Select { rows, .. } = result.pop().expect("should get something") { - assert_eq!(0, rows.len()); - } else { - panic!("Expected Payload::Select"); - } -} - -#[cfg(test)] -async fn create_glue_storage_value() { - let db = GlueDB::new( - #[cfg(target_arch = "wasm32")] - Some("create_glue_storage_value".to_string()), - Arc::new(MutinyLogger::default()), - ) - .await - .unwrap(); - - let f = db - .new_fedimint_client_db("create_glue_storage".to_string()) - .await - .unwrap(); - - let mut tx = f.begin_transaction().await; - let previous_bytes = tx - .raw_insert_bytes("k".as_bytes(), "v".as_bytes()) - .await - .unwrap(); - assert!(previous_bytes.is_none()); - - let tx_get_bytes = tx.raw_get_bytes("k".as_bytes()).await.unwrap(); - assert_eq!(tx_get_bytes, Some("v".as_bytes().to_vec())); - - tx.commit_tx().await.unwrap(); - - let mut tx_after_commit = f.begin_transaction().await; - let bytes_after_commit = tx_after_commit.raw_get_bytes("k".as_bytes()).await.unwrap(); - assert_eq!(bytes_after_commit, Some("v".as_bytes().to_vec())); - - // now reinit the DB and see if it loads the same values from the table - let same_f = db - .new_fedimint_client_db("create_glue_storage".to_string()) - .await - .unwrap(); - let mut tx = same_f.begin_transaction().await; - let tx_get_bytes = tx.raw_get_bytes("k".as_bytes()).await.unwrap(); - assert_eq!(tx_get_bytes, Some("v".as_bytes().to_vec())); -} - -#[cfg(test)] -async fn run_basic_glue_tests(glue: &mut Glue) { - use std::borrow::Cow; - - use gluesql::core::{ - ast::{AstLiteral, Expr}, - ast_builder::{num, table, Execute, ExprNode}, - }; - - glue.execute("DROP TABLE IF EXISTS api_test").await.unwrap(); - glue.execute( - "CREATE TABLE api_test ( - id INTEGER, - name TEXT, - nullable TEXT NULL, - is BOOLEAN - )", - ) - .await - .unwrap(); - - glue.execute( - "INSERT INTO api_test ( - id, - name, - nullable, - is - ) VALUES - (1, 'test1', 'not null', TRUE), - (2, 'test2', NULL, FALSE)", - ) - .await - .unwrap(); - - let select_query = "SELECT id, name, nullable, is FROM api_test"; - let mut result = glue.execute(select_query).await.unwrap(); - - assert_eq!(result.len(), 1); - - if let Payload::Select { rows, .. } = result.pop().unwrap() { - assert_eq!(rows.len(), 2); - - let row1 = &rows[0]; - assert_eq!(row1[0], Value::I64(1)); - assert_eq!(row1[1], Value::Str("test1".to_string())); - assert_eq!(row1[2], Value::Str("not null".to_string())); - assert_eq!(row1[3], Value::Bool(true)); - - let row2 = &rows[1]; - assert_eq!(row2[0], Value::I64(2)); - assert_eq!(row2[1], Value::Str("test2".to_string())); - assert_eq!(row2[2], Value::Null); - assert_eq!(row2[3], Value::Bool(false)); - } else { - panic!("Expected Payload::Select"); - } - - // Use AST Builder to insert another row - table("api_test") - .insert() - .columns("id, name, nullable, is") - .values(vec![vec![ - num(3), - ExprNode::QuotedString(Cow::Owned("test3".to_string())), - ExprNode::Expr(Cow::Owned(Expr::Literal(AstLiteral::Null))), - ExprNode::Expr(Cow::Owned(Expr::Literal(AstLiteral::Boolean(true)))), - ]]) - .execute(glue) - .await - .unwrap(); - - // Use AST Builder to select all rows - let ast_select_all = table("api_test").select().execute(glue).await.unwrap(); - - if let Payload::Select { rows, .. } = ast_select_all { - assert_eq!(rows.len(), 3); - - struct ApiTestRow { - id: i64, - name: String, - nullable: Option, - is: bool, - } - - let api_test_rows = rows - .into_iter() - .map(|row| ApiTestRow { - id: match row[0] { - Value::I64(val) => val, - _ => panic!("Expected I64 for id"), - }, - name: match &row[1] { - Value::Str(val) => val.clone(), - _ => panic!("Expected Str for name"), - }, - nullable: match &row[2] { - Value::Str(val) => Some(val.clone()), - Value::Null => None, - _ => panic!("Expected Str or Null for nullable"), - }, - is: match row[3] { - Value::Bool(val) => val, - _ => panic!("Expected Bool for is"), - }, - }) - .collect::>(); - - assert_eq!(api_test_rows[2].id, 3); - assert_eq!(api_test_rows[2].name, "test3"); - assert!(api_test_rows[2].nullable.is_none()); - assert!(api_test_rows[2].is); - } else { - panic!("Expected Payload::Select"); - } - - // do some bytea tests - glue.execute("CREATE TABLE IF NOT EXISTS mutiny_kv (key TEXT PRIMARY KEY, val BYTEA NOT NULL)") - .await - .unwrap(); - - let key_value_pairs = vec![ - (vec![1, 2, 3], vec![4, 5, 6]), - (vec![7, 8, 9], vec![10, 11, 12]), - ]; - let serialized_data = bincode::serialize(&key_value_pairs).unwrap(); - let hex_serialized_data = hex::encode(serialized_data); - - glue.execute("INSERT INTO mutiny_kv (key, val) VALUES ('storage', X'')") - .await - .unwrap(); - let select_query = "SELECT val FROM mutiny_kv WHERE key = 'storage'"; - let mut result = glue.execute(select_query).await.unwrap(); - - if let Payload::Select { rows, .. } = result.pop().expect("should get something") { - if let Some(row) = rows.first() { - if let Value::Bytea(hex_string) = &row[0] { - let serialized_data = hex::decode(hex_string).unwrap(); - assert!(serialized_data.is_empty()); - } else { - panic!("Expected bytea"); - } - } - } else { - panic!("Expected Payload::Select"); - } - - let update_query = format!( - "UPDATE mutiny_kv SET val = X'{}' WHERE key = 'fedimint_storage'", - hex_serialized_data - ); - glue.execute(&update_query).await.unwrap(); - - let select_query = "SELECT val FROM mutiny_kv WHERE key = 'fedimint_storage'"; - let mut result = glue.execute(select_query).await.unwrap(); - - if let Payload::Select { rows, .. } = result.pop().expect("should get something") { - if let Some(row) = rows.first() { - if let Value::Bytea(binary_data) = &row[0] { - let retrieved_pairs: Vec<(Vec, Vec)> = - bincode::deserialize(binary_data).unwrap(); - - assert_eq!(retrieved_pairs, key_value_pairs); - } else { - panic!("Expected bytea"); - } - } - } else { - panic!("Expected Payload::Select"); - } -} - -#[cfg(test)] -#[cfg(not(target_arch = "wasm32"))] -mod tests { - use super::*; - use gluesql::prelude::MemoryStorage; - use tokio; - - #[tokio::test] - async fn basic_glue_tests() { - let storage = MemoryStorage::default(); - let mut glue = Glue::new(storage); - run_basic_glue_tests(&mut glue).await; - } - - #[cfg(feature = "ignored_tests")] - #[tokio::test] - async fn create_glue_storage_tests() { - create_glue_and_fedimint_storage().await; - } - - #[cfg(feature = "ignored_tests")] - #[tokio::test] - async fn create_glue_storage_value_tests() { - create_glue_storage_value().await; - } - - #[cfg(feature = "ignored_tests")] - #[tokio::test] - async fn create_glue_and_payments_storage_tests() { - create_glue_and_payments_storage().await; - } - - #[cfg(feature = "ignored_tests")] - #[tokio::test] - async fn update_payments_tests() { - update_payments().await; - } - - #[cfg(feature = "ignored_tests")] - #[tokio::test] - async fn delete_all_tests() { - delete_all().await; - } -} - -#[cfg(test)] -#[cfg(target_arch = "wasm32")] -mod wasm_tests { - use gluesql::prelude::{Glue, IdbStorage}; - use wasm_bindgen_test::{wasm_bindgen_test as test, wasm_bindgen_test_configure}; - - use super::*; - - wasm_bindgen_test_configure!(run_in_browser); - - #[test] - async fn basic_wasm32_glue_tests() { - let storage = IdbStorage::new(Some("basic_wasm32_glue_tests".to_string())) - .await - .unwrap(); - let mut glue = Glue::new(storage); - run_basic_glue_tests(&mut glue).await; - } - - #[test] - async fn create_glue_storage_tests() { - create_glue_and_fedimint_storage().await; - } - - #[test] - async fn create_glue_storage_value_tests() { - create_glue_storage_value().await; - } - - #[test] - async fn create_glue_and_payments_storage_tests() { - create_glue_and_payments_storage().await; - } - - #[test] - async fn update_payments_tests() { - update_payments().await; - } - - #[test] - async fn delete_all_tests() { - delete_all().await; - } -} diff --git a/mutiny-core/src/sql/mod.rs b/mutiny-core/src/sql/mod.rs deleted file mode 100644 index cc92094f8..000000000 --- a/mutiny-core/src/sql/mod.rs +++ /dev/null @@ -1,31 +0,0 @@ -#![cfg_attr( - not(target_arch = "wasm32"), - allow(unused_variables, dead_code, unused_imports) -)] -use crate::{error::MutinyError, nodemanager::MutinyInvoice, HTLCStatus}; - -pub mod glue; - -pub(crate) trait ApplicationStore { - async fn save_payment(&self, i: MutinyInvoice) -> Result<(), MutinyError>; - async fn get_payment( - &self, - payment_hash: &bitcoin::hashes::sha256::Hash, - ) -> Result, MutinyError>; - async fn update_payment_status( - &self, - payment_hash: &bitcoin::hashes::sha256::Hash, - status: HTLCStatus, - ) -> Result<(), MutinyError>; - async fn update_payment_fee( - &self, - payment_hash: &bitcoin::hashes::sha256::Hash, - fee: Option, - ) -> Result<(), MutinyError>; - async fn update_payment_preimage( - &self, - payment_hash: &bitcoin::hashes::sha256::Hash, - preimage: Option, - ) -> Result<(), MutinyError>; - async fn list_payments(&self) -> Result, MutinyError>; -} From 58dbbcf83ac5e538a9f5b9ab31ad7b6d8cf9c2ab Mon Sep 17 00:00:00 2001 From: Tony Giorgio Date: Thu, 18 Jan 2024 21:08:29 -0600 Subject: [PATCH 4/7] Save fedimint data to vss Storage reference --- mutiny-core/src/federation.rs | 7 ++-- mutiny-core/src/lib.rs | 7 ++-- mutiny-core/src/storage.rs | 5 +-- mutiny-wasm/src/indexed_db.rs | 61 +++++++++++++++++++++++++++++++++-- 4 files changed, 71 insertions(+), 9 deletions(-) diff --git a/mutiny-core/src/federation.rs b/mutiny-core/src/federation.rs index e10e6fa16..bc0832040 100644 --- a/mutiny-core/src/federation.rs +++ b/mutiny-core/src/federation.rs @@ -75,6 +75,8 @@ const FEDIMINT_STATUS_TIMEOUT_CHECK_MS: u64 = 30; // their internal list. const FEDIMINT_OPERATIONS_LIST_MAX: usize = 100; +pub const FEDIMINTS_PREFIX_KEY: &str = "fedimints/"; + impl From for HTLCStatus { fn from(state: LnReceiveState) -> Self { match state { @@ -768,7 +770,7 @@ impl FedimintStorage { } fn key_id(federation_id: &str) -> String { - format!("fedimint_key_{}", federation_id) + format!("{}{}", FEDIMINTS_PREFIX_KEY, federation_id) } #[cfg_attr(target_arch = "wasm32", async_trait(?Send))] @@ -815,7 +817,8 @@ impl<'a, S: MutinyStorage> IRawDatabaseTransaction for IndexedDBPseudoTransactio value: serde_json::to_value(hex_serialized_data).unwrap(), }; self.storage - .set_data(key_id(&self.federation_id), value, None)?; + .set_data_async(key_id(&self.federation_id), value, Some(version)) + .await?; Ok(()) } diff --git a/mutiny-core/src/lib.rs b/mutiny-core/src/lib.rs index 61bee717d..f2aba2d4a 100644 --- a/mutiny-core/src/lib.rs +++ b/mutiny-core/src/lib.rs @@ -1357,7 +1357,8 @@ impl MutinyWallet { if federation_storage_guard.federations.contains_key(uuid) { federation_storage_guard.federations.remove(uuid); self.storage - .insert_federations(federation_storage_guard.clone())?; + .insert_federations(federation_storage_guard.clone()) + .await?; federations_guard.remove(&federation_id); } else { return Err(MutinyError::NotFound); @@ -1695,7 +1696,9 @@ pub(crate) async fn create_new_federation( .federations .insert(next_federation_uuid.clone(), next_federation.clone()); - storage.insert_federations(existing_federations.clone())?; + storage + .insert_federations(existing_federations.clone()) + .await?; federation_mutex.federations = existing_federations.federations.clone(); // now create the federation process and init it diff --git a/mutiny-core/src/storage.rs b/mutiny-core/src/storage.rs index 68213bd65..00df95a44 100644 --- a/mutiny-core/src/storage.rs +++ b/mutiny-core/src/storage.rs @@ -372,9 +372,10 @@ pub trait MutinyStorage: Clone + Sized + Send + Sync + 'static { } /// Inserts the federation indexes into storage - fn insert_federations(&self, federations: FederationStorage) -> Result<(), MutinyError> { + async fn insert_federations(&self, federations: FederationStorage) -> Result<(), MutinyError> { let version = Some(federations.version); - self.set_data(FEDERATIONS_KEY.to_string(), federations, version) + self.set_data_async(FEDERATIONS_KEY.to_string(), federations, version) + .await } /// Get the current fee estimates from storage diff --git a/mutiny-wasm/src/indexed_db.rs b/mutiny-wasm/src/indexed_db.rs index 054bb494c..b9f1208e4 100644 --- a/mutiny-wasm/src/indexed_db.rs +++ b/mutiny-wasm/src/indexed_db.rs @@ -5,8 +5,6 @@ use gloo_utils::format::JsValueSerdeExt; use lightning::util::logger::Logger; use lightning::{log_debug, log_error}; use log::error; -use mutiny_core::logging::MutinyLogger; -use mutiny_core::nodemanager::NodeStorage; use mutiny_core::storage::*; use mutiny_core::vss::*; use mutiny_core::*; @@ -14,6 +12,8 @@ use mutiny_core::{ encrypt::Cipher, error::{MutinyError, MutinyStorageError}, }; +use mutiny_core::{federation::FederationStorage, logging::MutinyLogger}; +use mutiny_core::{federation::FEDIMINTS_PREFIX_KEY, nodemanager::NodeStorage}; use rexie::{ObjectStore, Rexie, TransactionMode}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -294,11 +294,18 @@ impl IndexedDbStorage { futs.push(Self::handle_vss_key(kv, vss, &map, logger)); } let results = futures::future::join_all(futs).await; + let mut items_vector = Vec::new(); for result in results { if let Some((key, value)) = result? { - map.set_data(key, value, None)?; + // save to memory and batch the write to local storage + map.set_data(key.clone(), value.clone(), None)?; + items_vector.push((key, value)); } } + if !items_vector.is_empty() { + // write them so we don't have to pull them down again + Self::save_to_indexed_db(indexed_db, &items_vector).await?; + } let final_map = map.memory.read().unwrap(); Ok(final_map.clone()) } @@ -336,6 +343,25 @@ impl IndexedDbStorage { } } } + FEDERATIONS_KEY => { + // we can get version from federation storage, so we should compare + match current.get_data::(&kv.key)? { + Some(local) => { + if local.version < kv.version { + let obj = vss.get_object(&kv.key).await?; + if serde_json::from_value::(obj.value.clone()) + .is_ok() + { + return Ok(Some((kv.key, obj.value))); + } + } + } + None => { + let obj = vss.get_object(&kv.key).await?; + return Ok(Some((kv.key, obj.value))); + } + } + } DEVICE_LOCK_KEY => { // we can get version from device lock, so we should compare match current.get_data::(&kv.key)? { @@ -409,6 +435,35 @@ impl IndexedDbStorage { } } } + } else if key.starts_with(FEDIMINTS_PREFIX_KEY) { + // we can get versions from each fedimint, so we should compare + match current.get_data::(&kv.key)? { + Some(local) => { + if local.version < kv.version { + let obj = vss.get_object(&kv.key).await?; + if serde_json::from_value::(obj.value.clone()) + .is_ok() + { + return Ok(Some((kv.key, obj.value))); + } + } else { + log_debug!( + logger, + "Skipping vss key {} with version {}, current version is {}", + kv.key, + kv.version, + local.version + ); + return Ok(None); + } + } + None => { + let obj = vss.get_object(&kv.key).await?; + if serde_json::from_value::(obj.value.clone()).is_ok() { + return Ok(Some((kv.key, obj.value))); + } + } + } } } } From 8aed050889fe5c0e9491b92de42b41fd8c84cbeb Mon Sep 17 00:00:00 2001 From: Tony Giorgio Date: Fri, 19 Jan 2024 04:31:22 -0600 Subject: [PATCH 5/7] Refactor listing invoices --- mutiny-core/src/federation.rs | 3 +- mutiny-core/src/lib.rs | 229 +++++++++++++++++++++++++++++++-- mutiny-core/src/node.rs | 40 +----- mutiny-core/src/nodemanager.rs | 206 ++--------------------------- mutiny-core/src/nostr/nwc.rs | 2 +- mutiny-wasm/src/lib.rs | 4 +- mutiny-wasm/src/models.rs | 5 +- 7 files changed, 241 insertions(+), 248 deletions(-) diff --git a/mutiny-core/src/federation.rs b/mutiny-core/src/federation.rs index bc0832040..825a92ff9 100644 --- a/mutiny-core/src/federation.rs +++ b/mutiny-core/src/federation.rs @@ -3,13 +3,12 @@ use crate::{ event::PaymentInfo, key::{create_root_child_key, ChildKey}, logging::MutinyLogger, - nodemanager::MutinyInvoice, onchain::coin_type_from_network, storage::{ get_payment_info, list_payment_info, persist_payment_info, MutinyStorage, VersionedValue, }, utils::sleep, - HTLCStatus, DEFAULT_PAYMENT_TIMEOUT, + HTLCStatus, MutinyInvoice, DEFAULT_PAYMENT_TIMEOUT, }; use async_trait::async_trait; use bip39::Mnemonic; diff --git a/mutiny-core/src/lib.rs b/mutiny-core/src/lib.rs index f2aba2d4a..0ec4a2224 100644 --- a/mutiny-core/src/lib.rs +++ b/mutiny-core/src/lib.rs @@ -13,7 +13,7 @@ pub mod auth; mod chain; pub mod encrypt; pub mod error; -mod event; +pub mod event; pub mod federation; mod fees; mod gossip; @@ -40,12 +40,13 @@ pub mod vss; #[cfg(test)] mod test_utils; -pub use crate::event::HTLCStatus; +use crate::event::{HTLCStatus, MillisatAmount, PaymentInfo}; pub use crate::gossip::{GOSSIP_SYNC_TIME_KEY, NETWORK_GRAPH_KEY, PROB_SCORER_KEY}; pub use crate::keymanager::generate_seed; pub use crate::ldkstorage::{CHANNEL_MANAGER_KEY, MONITORS_PREFIX_KEY}; - -use crate::storage::{MutinyStorage, DEVICE_ID_KEY, EXPECTED_NETWORK_KEY, NEED_FULL_SYNC_KEY}; +use crate::storage::{ + list_payment_info, MutinyStorage, DEVICE_ID_KEY, EXPECTED_NETWORK_KEY, NEED_FULL_SYNC_KEY, +}; use crate::{auth::MutinyAuthClient, logging::MutinyLogger}; use crate::{error::MutinyError, nostr::ReservedProfile}; use crate::{ @@ -55,7 +56,7 @@ use crate::{ }; use crate::{ lnurlauth::make_lnurl_auth_connection, - nodemanager::{ChannelClosure, MutinyBip21RawMaterials, MutinyInvoice, TransactionDetails}, + nodemanager::{ChannelClosure, MutinyBip21RawMaterials, TransactionDetails}, }; use crate::{lnurlauth::AuthManager, nostr::MUTINY_PLUS_SUBSCRIPTION_LABEL}; use crate::{logging::LOGGING_KEY, nodemanager::NodeManagerBuilder}; @@ -73,13 +74,16 @@ use async_lock::RwLock; use bdk_chain::ConfirmationTime; use bip39::Mnemonic; use bitcoin::hashes::hex::ToHex; +use bitcoin::hashes::{sha256, Hash}; +use bitcoin::secp256k1::PublicKey; use bitcoin::util::bip32::ExtendedPrivKey; -use bitcoin::{hashes::sha256, Network}; -use fedimint_core::{api::InviteCode, config::FederationId, BitcoinHash}; +use bitcoin::Network; +use fedimint_core::{api::InviteCode, config::FederationId}; use futures::{pin_mut, select, FutureExt}; +use lightning::ln::PaymentHash; use lightning::{log_debug, util::logger::Logger}; use lightning::{log_error, log_info, log_warn}; -use lightning_invoice::Bolt11Invoice; +use lightning_invoice::{Bolt11Invoice, Bolt11InvoiceDescription}; use lnurl::{lnurl::LnUrl, AsyncClient as LnUrlClient, LnUrlResponse, Response}; use nostr_sdk::{Client, RelayPoolNotification}; use serde::{Deserialize, Serialize}; @@ -250,6 +254,155 @@ impl Ord for ActivityItem { } } +#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)] +pub struct MutinyInvoice { + pub bolt11: Option, + pub description: Option, + pub payment_hash: sha256::Hash, + pub preimage: Option, + pub payee_pubkey: Option, + pub amount_sats: Option, + pub expire: u64, + pub status: HTLCStatus, + pub fees_paid: Option, + pub inbound: bool, + pub labels: Vec, + pub last_updated: u64, +} + +impl MutinyInvoice { + pub fn paid(&self) -> bool { + self.status == HTLCStatus::Succeeded + } +} + +impl From for MutinyInvoice { + fn from(value: Bolt11Invoice) -> Self { + let description = match value.description() { + Bolt11InvoiceDescription::Direct(a) => { + if a.is_empty() { + None + } else { + Some(a.to_string()) + } + } + Bolt11InvoiceDescription::Hash(_) => None, + }; + + let timestamp = value.duration_since_epoch().as_secs(); + let expiry = timestamp + value.expiry_time().as_secs(); + + let payment_hash = value.payment_hash().to_owned(); + let payee_pubkey = value.payee_pub_key().map(|p| p.to_owned()); + let amount_sats = value.amount_milli_satoshis().map(|m| m / 1000); + + MutinyInvoice { + bolt11: Some(value), + description, + payment_hash, + preimage: None, + payee_pubkey, + amount_sats, + expire: expiry, + status: HTLCStatus::Pending, + fees_paid: None, + inbound: true, + labels: vec![], + last_updated: timestamp, + } + } +} + +impl From for PaymentInfo { + fn from(invoice: MutinyInvoice) -> Self { + let preimage = invoice + .preimage + .map(|s| hex::decode(s).expect("preimage should decode")) + .map(|v| { + let mut arr = [0; 32]; + arr[..].copy_from_slice(&v); + arr + }); + let secret = None; + let status = invoice.status; + let amt_msat = invoice + .amount_sats + .map(|s| MillisatAmount(Some(s))) + .unwrap_or(MillisatAmount(None)); + let fee_paid_msat = invoice.fees_paid; + let bolt11 = invoice.bolt11; + let payee_pubkey = invoice.payee_pubkey; + let last_update = invoice.last_updated; + + PaymentInfo { + preimage, + secret, + status, + amt_msat, + fee_paid_msat, + bolt11, + payee_pubkey, + last_update, + } + } +} + +impl MutinyInvoice { + pub(crate) fn from( + i: PaymentInfo, + payment_hash: PaymentHash, + inbound: bool, + labels: Vec, + ) -> Result { + match i.bolt11 { + Some(invoice) => { + // Construct an invoice from a bolt11, easy + let amount_sats = if let Some(inv_amt) = invoice.amount_milli_satoshis() { + if inv_amt == 0 { + i.amt_msat.0.map(|a| a / 1_000) + } else { + Some(inv_amt / 1_000) + } + } else { + i.amt_msat.0.map(|a| a / 1_000) + }; + Ok(MutinyInvoice { + inbound, + last_updated: i.last_update, + status: i.status, + labels, + amount_sats, + payee_pubkey: i.payee_pubkey, + preimage: i.preimage.map(|p| p.to_hex()), + fees_paid: i.fee_paid_msat.map(|f| f / 1_000), + ..invoice.into() + }) + } + None => { + let amount_sats: Option = i.amt_msat.0.map(|s| s / 1_000); + let fees_paid = i.fee_paid_msat.map(|f| f / 1_000); + let preimage = i.preimage.map(|p| p.to_hex()); + let payment_hash = sha256::Hash::from_inner(payment_hash.0); + let invoice = MutinyInvoice { + bolt11: None, + description: None, + payment_hash, + preimage, + payee_pubkey: i.payee_pubkey, + amount_sats, + expire: i.last_update, + status: i.status, + fees_paid, + inbound, + labels, + last_updated: i.last_update, + }; + Ok(invoice) + } + } + } +} + pub struct MutinyWalletConfigBuilder { xprivkey: ExtendedPrivKey, #[cfg(target_arch = "wasm32")] @@ -952,8 +1105,34 @@ impl MutinyWallet { /// Get the sorted activity list for lightning payments, channels, and txs. pub async fn get_activity(&self) -> Result, MutinyError> { + // Get activity for lightning invoices + let lightning = self + .list_invoices() + .map_err(|e| { + log_warn!(self.logger, "Failed to get lightning activity: {e}"); + e + }) + .unwrap_or_default(); + // Get activities from node manager - let mut activities = self.node_manager.get_activity().await?; + let (closures, onchain) = self.node_manager.get_activity().await?; + + let mut activities = Vec::with_capacity(lightning.len() + onchain.len() + closures.len()); + for ln in lightning { + // Only show paid and in-flight invoices + match ln.status { + HTLCStatus::Succeeded | HTLCStatus::InFlight => { + activities.push(ActivityItem::Lightning(Box::new(ln))); + } + HTLCStatus::Pending | HTLCStatus::Failed => {} + } + } + for on in onchain { + activities.push(ActivityItem::OnChain(on)); + } + for chan in closures { + activities.push(ActivityItem::ChannelClosed(chan)); + } // Sort all activities, newest first activities.sort_by(|a, b| b.cmp(a)); @@ -961,6 +1140,38 @@ impl MutinyWallet { Ok(activities) } + pub fn list_invoices(&self) -> Result, MutinyError> { + let mut inbound_invoices = self.list_payment_info_from_persisters(true)?; + let mut outbound_invoices = self.list_payment_info_from_persisters(false)?; + inbound_invoices.append(&mut outbound_invoices); + Ok(inbound_invoices) + } + + fn list_payment_info_from_persisters( + &self, + inbound: bool, + ) -> Result, MutinyError> { + let now = utils::now(); + let labels_map = self.storage.get_invoice_labels()?; + + Ok(list_payment_info(&self.storage, inbound)? + .into_iter() + .filter_map(|(h, i)| { + let labels = match i.bolt11.clone() { + None => vec![], + Some(i) => labels_map.get(&i).cloned().unwrap_or_default(), + }; + let mutiny_invoice = MutinyInvoice::from(i.clone(), h, inbound, labels).ok(); + + // filter out expired invoices + mutiny_invoice.filter(|invoice| { + !invoice.bolt11.as_ref().is_some_and(|b| b.would_expire(now)) + || matches!(invoice.status, HTLCStatus::Succeeded | HTLCStatus::InFlight) + }) + }) + .collect()) + } + /// Gets an invoice. /// This includes sent and received invoices. pub async fn get_invoice(&self, invoice: &Bolt11Invoice) -> Result { diff --git a/mutiny-core/src/node.rs b/mutiny-core/src/node.rs index d81a62377..4ef2c8f17 100644 --- a/mutiny-core/src/node.rs +++ b/mutiny-core/src/node.rs @@ -1,3 +1,4 @@ +use crate::lsp::{InvoiceRequest, LspConfig}; use crate::nodemanager::ChannelClosure; use crate::peermanager::LspMessageRouter; use crate::storage::MutinyStorage; @@ -12,10 +13,11 @@ use crate::{ ldkstorage::{MutinyNodePersister, PhantomChannelManager}, logging::MutinyLogger, lsp::{AnyLsp, FeeRequest, Lsp}, - nodemanager::{MutinyInvoice, NodeIndex}, + nodemanager::NodeIndex, onchain::OnChainWallet, peermanager::{GossipMessageHandler, PeerManagerImpl}, utils::{self, sleep}, + MutinyInvoice, }; use crate::{fees::P2WSH_OUTPUT_SIZE, peermanager::connect_peer_if_necessary}; use crate::{keymanager::PhantomKeysManager, scorer::HubPreferentialScorer}; @@ -24,10 +26,6 @@ use crate::{ ldkstorage::{persist_monitor, ChannelOpenParams}, storage::persist_payment_info, }; -use crate::{ - lsp::{InvoiceRequest, LspConfig}, - storage::list_payment_info, -}; use crate::{messagehandler::MutinyMessageHandler, storage::read_payment_info}; use anyhow::{anyhow, Context}; use bdk::FeeRate; @@ -1220,38 +1218,6 @@ impl Node { ) } - pub fn list_invoices(&self) -> Result, MutinyError> { - let mut inbound_invoices = self.list_payment_info_from_persisters(true)?; - let mut outbound_invoices = self.list_payment_info_from_persisters(false)?; - inbound_invoices.append(&mut outbound_invoices); - Ok(inbound_invoices) - } - - fn list_payment_info_from_persisters( - &self, - inbound: bool, - ) -> Result, MutinyError> { - let now = utils::now(); - let labels_map = self.persister.storage.get_invoice_labels()?; - - Ok(list_payment_info(&self.persister.storage, inbound)? - .into_iter() - .filter_map(|(h, i)| { - let labels = match i.bolt11.clone() { - None => vec![], - Some(i) => labels_map.get(&i).cloned().unwrap_or_default(), - }; - let mutiny_invoice = MutinyInvoice::from(i.clone(), h, inbound, labels).ok(); - - // filter out expired invoices - mutiny_invoice.filter(|invoice| { - !invoice.bolt11.as_ref().is_some_and(|b| b.would_expire(now)) - || matches!(i.status, HTLCStatus::Succeeded | HTLCStatus::InFlight) - }) - }) - .collect()) - } - /// Gets all the closed channels for this node pub fn get_channel_closure( &self, diff --git a/mutiny-core/src/nodemanager.rs b/mutiny-core/src/nodemanager.rs index cdcee8a27..e0a15ccb1 100644 --- a/mutiny-core/src/nodemanager.rs +++ b/mutiny-core/src/nodemanager.rs @@ -1,8 +1,9 @@ -use crate::event::{HTLCStatus, MillisatAmount, PaymentInfo}; +use crate::event::HTLCStatus; use crate::labels::LabelStorage; use crate::logging::LOGGING_KEY; use crate::utils::{sleep, spawn}; use crate::ActivityItem; +use crate::MutinyInvoice; use crate::MutinyWalletConfig; use crate::{ chain::MutinyChain, @@ -27,7 +28,7 @@ use bdk::chain::{BlockId, ConfirmationTime}; use bdk::{wallet::AddressIndex, FeeRate, LocalUtxo}; use bitcoin::blockdata::script; use bitcoin::hashes::hex::ToHex; -use bitcoin::hashes::{sha256, Hash}; +use bitcoin::hashes::sha256; use bitcoin::psbt::PartiallySignedTransaction; use bitcoin::secp256k1::PublicKey; use bitcoin::util::bip32::ExtendedPrivKey; @@ -39,12 +40,12 @@ use lightning::chain::Confirm; use lightning::events::ClosureReason; use lightning::ln::channelmanager::{ChannelDetails, PhantomRouteHints}; use lightning::ln::script::ShutdownScript; -use lightning::ln::{ChannelId, PaymentHash}; +use lightning::ln::ChannelId; use lightning::routing::gossip::NodeId; use lightning::sign::{NodeSigner, Recipient}; use lightning::util::logger::*; use lightning::{log_debug, log_error, log_info, log_warn}; -use lightning_invoice::{Bolt11Invoice, Bolt11InvoiceDescription}; +use lightning_invoice::Bolt11Invoice; use lightning_transaction_sync::EsploraSyncClient; use payjoin::Uri; use reqwest::Client; @@ -98,155 +99,6 @@ pub struct MutinyBip21RawMaterials { pub labels: Vec, } -#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)] -pub struct MutinyInvoice { - pub bolt11: Option, - pub description: Option, - pub payment_hash: sha256::Hash, - pub preimage: Option, - pub payee_pubkey: Option, - pub amount_sats: Option, - pub expire: u64, - pub status: HTLCStatus, - pub fees_paid: Option, - pub inbound: bool, - pub labels: Vec, - pub last_updated: u64, -} - -impl MutinyInvoice { - pub fn paid(&self) -> bool { - self.status == HTLCStatus::Succeeded - } -} - -impl From for MutinyInvoice { - fn from(value: Bolt11Invoice) -> Self { - let description = match value.description() { - Bolt11InvoiceDescription::Direct(a) => { - if a.is_empty() { - None - } else { - Some(a.to_string()) - } - } - Bolt11InvoiceDescription::Hash(_) => None, - }; - - let timestamp = value.duration_since_epoch().as_secs(); - let expiry = timestamp + value.expiry_time().as_secs(); - - let payment_hash = value.payment_hash().to_owned(); - let payee_pubkey = value.payee_pub_key().map(|p| p.to_owned()); - let amount_sats = value.amount_milli_satoshis().map(|m| m / 1000); - - MutinyInvoice { - bolt11: Some(value), - description, - payment_hash, - preimage: None, - payee_pubkey, - amount_sats, - expire: expiry, - status: HTLCStatus::Pending, - fees_paid: None, - inbound: true, - labels: vec![], - last_updated: timestamp, - } - } -} - -impl From for PaymentInfo { - fn from(invoice: MutinyInvoice) -> Self { - let preimage = invoice - .preimage - .map(|s| hex::decode(s).expect("preimage should decode")) - .map(|v| { - let mut arr = [0; 32]; - arr[..].copy_from_slice(&v); - arr - }); - let secret = None; - let status = invoice.status; - let amt_msat = invoice - .amount_sats - .map(|s| MillisatAmount(Some(s))) - .unwrap_or(MillisatAmount(None)); - let fee_paid_msat = invoice.fees_paid; - let bolt11 = invoice.bolt11; - let payee_pubkey = invoice.payee_pubkey; - let last_update = invoice.last_updated; - - PaymentInfo { - preimage, - secret, - status, - amt_msat, - fee_paid_msat, - bolt11, - payee_pubkey, - last_update, - } - } -} - -impl MutinyInvoice { - pub(crate) fn from( - i: PaymentInfo, - payment_hash: PaymentHash, - inbound: bool, - labels: Vec, - ) -> Result { - match i.bolt11 { - Some(invoice) => { - // Construct an invoice from a bolt11, easy - let amount_sats = if let Some(inv_amt) = invoice.amount_milli_satoshis() { - if inv_amt == 0 { - i.amt_msat.0.map(|a| a / 1_000) - } else { - Some(inv_amt / 1_000) - } - } else { - i.amt_msat.0.map(|a| a / 1_000) - }; - Ok(MutinyInvoice { - inbound, - last_updated: i.last_update, - status: i.status, - labels, - amount_sats, - payee_pubkey: i.payee_pubkey, - preimage: i.preimage.map(|p| p.to_hex()), - fees_paid: i.fee_paid_msat.map(|f| f / 1_000), - ..invoice.into() - }) - } - None => { - let amount_sats: Option = i.amt_msat.0.map(|s| s / 1_000); - let fees_paid = i.fee_paid_msat.map(|f| f / 1_000); - let preimage = i.preimage.map(|p| p.to_hex()); - let payment_hash = sha256::Hash::from_inner(payment_hash.0); - let invoice = MutinyInvoice { - bolt11: None, - description: None, - payment_hash, - preimage, - payee_pubkey: i.payee_pubkey, - amount_sats, - expire: i.last_update, - status: i.status, - fees_paid, - inbound, - labels, - last_updated: i.last_update, - }; - Ok(invoice) - } - } - } -} - #[derive(Serialize, Deserialize, Clone, Eq, PartialEq)] pub struct MutinyPeer { pub pubkey: PublicKey, @@ -1066,17 +918,13 @@ impl NodeManager { } /// Returns all the on-chain and lightning activity from the wallet. - pub(crate) async fn get_activity(&self) -> Result, MutinyError> { + pub(crate) async fn get_activity( + &self, + ) -> Result<(Vec, Vec), MutinyError> { // todo add contacts to the activity - let (lightning, closures) = - futures_util::join!(self.list_invoices(), self.list_channel_closures()); - let lightning = lightning - .map_err(|e| { - log_warn!(self.logger, "Failed to get lightning activity: {e}"); - e - }) - .unwrap_or_default(); - let closures = closures + let closures = self + .list_channel_closures() + .await .map_err(|e| { log_warn!(self.logger, "Failed to get channel closures: {e}"); e @@ -1090,24 +938,7 @@ impl NodeManager { }) .unwrap_or_default(); - let mut activity = Vec::with_capacity(lightning.len() + onchain.len() + closures.len()); - for ln in lightning { - // Only show paid and in-flight invoices - match ln.status { - HTLCStatus::Succeeded | HTLCStatus::InFlight => { - activity.push(ActivityItem::Lightning(Box::new(ln))); - } - HTLCStatus::Pending | HTLCStatus::Failed => {} - } - } - for on in onchain { - activity.push(ActivityItem::OnChain(on)); - } - for chan in closures { - activity.push(ActivityItem::ChannelClosed(chan)); - } - - Ok(activity) + Ok((closures, onchain)) } /// Returns all the on-chain and lightning activity for a given label @@ -1614,19 +1445,6 @@ impl NodeManager { Err(MutinyError::NotFound) } - /// Gets an invoice from the node manager. - /// This includes sent and received invoices. - pub async fn list_invoices(&self) -> Result, MutinyError> { - let mut invoices: Vec = vec![]; - let nodes = self.nodes.lock().await; - for (_, node) in nodes.iter() { - if let Ok(mut invs) = node.list_invoices() { - invoices.append(&mut invs) - } - } - Ok(invoices) - } - pub async fn get_channel_closure( &self, user_channel_id: u128, diff --git a/mutiny-core/src/nostr/nwc.rs b/mutiny-core/src/nostr/nwc.rs index 5dac64a6c..27fba70c6 100644 --- a/mutiny-core/src/nostr/nwc.rs +++ b/mutiny-core/src/nostr/nwc.rs @@ -1190,11 +1190,11 @@ mod test { mod wasm_test { use super::*; use crate::logging::MutinyLogger; - use crate::nodemanager::MutinyInvoice; use crate::nostr::ProfileType; use crate::storage::MemoryStorage; use crate::test_utils::{create_dummy_invoice, create_mutiny_wallet, create_nwc_request}; use crate::MockInvoiceHandler; + use crate::MutinyInvoice; use bitcoin::secp256k1::ONE_KEY; use bitcoin::Network; use mockall::predicate::eq; diff --git a/mutiny-wasm/src/lib.rs b/mutiny-wasm/src/lib.rs index d0711e93b..63bfde369 100644 --- a/mutiny-wasm/src/lib.rs +++ b/mutiny-wasm/src/lib.rs @@ -860,9 +860,7 @@ impl MutinyWallet { /// This includes sent and received invoices. #[wasm_bindgen] pub async fn list_invoices(&self) -> Result */, MutinyJsError> { - Ok(JsValue::from_serde( - &self.inner.node_manager.list_invoices().await?, - )?) + Ok(JsValue::from_serde(&self.inner.list_invoices()?)?) } /// Gets an channel closure from the node manager. diff --git a/mutiny-wasm/src/models.rs b/mutiny-wasm/src/models.rs index db055085b..79d76bcde 100644 --- a/mutiny-wasm/src/models.rs +++ b/mutiny-wasm/src/models.rs @@ -6,6 +6,7 @@ use gloo_utils::format::JsValueSerdeExt; use lightning_invoice::{Bolt11Invoice, Bolt11InvoiceDescription}; use lnurl::lightning_address::LightningAddress; use lnurl::lnurl::LnUrl; +use mutiny_core::event::HTLCStatus; use mutiny_core::labels::Contact as MutinyContact; use mutiny_core::nostr::nwc::SpendingConditions; use mutiny_core::*; @@ -172,8 +173,8 @@ impl MutinyInvoice { } } -impl From for MutinyInvoice { - fn from(m: nodemanager::MutinyInvoice) -> Self { +impl From for MutinyInvoice { + fn from(m: mutiny_core::MutinyInvoice) -> Self { let potential_hodl_invoice = match m.bolt11 { Some(ref b) => { utils::HODL_INVOICE_NODES.contains(&b.recover_payee_pub_key().to_hex().as_str()) From 83aa41dc7089d317db24dcdf718b69cc88bb1ba7 Mon Sep 17 00:00:00 2001 From: Tony Giorgio Date: Fri, 19 Jan 2024 06:50:25 -0600 Subject: [PATCH 6/7] Fix fee & inbound for fedimint --- mutiny-core/src/federation.rs | 16 ++++++++++------ mutiny-core/src/lib.rs | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/mutiny-core/src/federation.rs b/mutiny-core/src/federation.rs index 825a92ff9..12505239a 100644 --- a/mutiny-core/src/federation.rs +++ b/mutiny-core/src/federation.rs @@ -236,6 +236,8 @@ impl FederationClient { amount: u64, labels: Vec, ) -> Result { + let inbound = true; + let lightning_module = self .fedimint_client .get_first_module::(); @@ -245,13 +247,13 @@ impl FederationClient { // persist the invoice let mut stored_payment: MutinyInvoice = invoice.clone().into(); - stored_payment.inbound = true; + stored_payment.inbound = inbound; stored_payment.labels = labels; log_trace!(self.logger, "Persiting payment"); let hash = *stored_payment.payment_hash.as_inner(); let payment_info = PaymentInfo::from(stored_payment); - persist_payment_info(&self.storage, &hash, &payment_info, true)?; + persist_payment_info(&self.storage, &hash, &payment_info, inbound)?; log_trace!(self.logger, "Persisted payment"); Ok(invoice.into()) @@ -448,6 +450,8 @@ impl FederationClient { invoice: Bolt11Invoice, labels: Vec, ) -> Result { + let inbound = false; + let lightning_module = self .fedimint_client .get_first_module::(); @@ -458,11 +462,11 @@ impl FederationClient { // Save after payment was initiated successfully let mut stored_payment: MutinyInvoice = invoice.clone().into(); - stored_payment.inbound = false; + stored_payment.inbound = inbound; stored_payment.labels = labels; let hash = *stored_payment.payment_hash.as_inner(); let payment_info = PaymentInfo::from(stored_payment); - persist_payment_info(&self.storage, &hash, &payment_info, true)?; + persist_payment_info(&self.storage, &hash, &payment_info, inbound)?; // Subscribe and process outcome based on payment type let mut inv = match outgoing_payment.payment_type { @@ -473,7 +477,7 @@ impl FederationClient { o, process_pay_state_internal, invoice.clone(), - true, + inbound, DEFAULT_PAYMENT_TIMEOUT * 1_000, Arc::clone(&self.logger), ) @@ -489,7 +493,7 @@ impl FederationClient { o, process_pay_state_ln, invoice.clone(), - false, + inbound, DEFAULT_PAYMENT_TIMEOUT * 1_000, Arc::clone(&self.logger), ) diff --git a/mutiny-core/src/lib.rs b/mutiny-core/src/lib.rs index 0ec4a2224..8b34562be 100644 --- a/mutiny-core/src/lib.rs +++ b/mutiny-core/src/lib.rs @@ -329,7 +329,7 @@ impl From for PaymentInfo { .amount_sats .map(|s| MillisatAmount(Some(s))) .unwrap_or(MillisatAmount(None)); - let fee_paid_msat = invoice.fees_paid; + let fee_paid_msat = invoice.fees_paid.map(|f| f * 1_000); let bolt11 = invoice.bolt11; let payee_pubkey = invoice.payee_pubkey; let last_update = invoice.last_updated; From 89a0de4f671593f17f756a0f5919ddee16f9ae27 Mon Sep 17 00:00:00 2001 From: Tony Giorgio Date: Fri, 19 Jan 2024 06:52:04 -0600 Subject: [PATCH 7/7] Don't await on persisting fedimint data --- mutiny-core/src/federation.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mutiny-core/src/federation.rs b/mutiny-core/src/federation.rs index 12505239a..733c1905a 100644 --- a/mutiny-core/src/federation.rs +++ b/mutiny-core/src/federation.rs @@ -819,9 +819,9 @@ impl<'a, S: MutinyStorage> IRawDatabaseTransaction for IndexedDBPseudoTransactio version, value: serde_json::to_value(hex_serialized_data).unwrap(), }; + // TODO await on persisting remotely self.storage - .set_data_async(key_id(&self.federation_id), value, Some(version)) - .await?; + .set_data(key_id(&self.federation_id), value, Some(version))?; Ok(()) }