Merge pull request lightningdevkit#19 from TheBlueMatt/main

Update LDK to 0.0.112 and workaround Postgres being broken

Author: TheBlueMatt

.github/workflows/build.yml

@@ -36,6 +36,7 @@ jobs:
           cargo update -p tokio-postgres --precise "0.7.5" --verbose
           cargo update -p postgres-types --precise "0.2.3" --verbose
           cargo update -p tokio --precise "1.14.1" --verbose
+          cargo update -p once_cell --precise "1.14.0" --verbose
       - name: Build on Rust ${{ matrix.toolchain }}
         run: |
           cargo build --verbose --color always

Cargo.toml

@@ -5,9 +5,9 @@ edition = "2018"
 
 [dependencies]
 bitcoin = "0.29"
-lightning = { version = "0.0.111" }
-lightning-block-sync = { version = "0.0.111", features=["rest-client"] }
-lightning-net-tokio = { version = "0.0.111" }
+lightning = { version = "0.0.112" }
+lightning-block-sync = { version = "0.0.112", features=["rest-client"] }
+lightning-net-tokio = { version = "0.0.112" }
 tokio = { version = "1.14.1", features = ["full"] }
 tokio-postgres = { version="0.7.5" }
 futures = "0.3"

src/config.rs

@@ -200,6 +200,13 @@ pub(crate) async fn upgrade_db(schema: i32, client: &mut tokio_postgres::Client)
 	if schema <= 1 || schema > SCHEMA_VERSION {
 		panic!("Unknown schema in db: {}, we support up to {}", schema, SCHEMA_VERSION);
 	}
+	// PostgreSQL (at least v13, but likely later versions as well) handles insert-only tables
+	// *very* poorly. After some number of inserts, it refuses to rely on indexes, assuming them to
+	// be possibly-stale, until a VACUUM happens. Thus, we set the vacuum factor really low here,
+	// pushing PostgreSQL to vacuum often.
+	// See https://www.cybertec-postgresql.com/en/postgresql-autovacuum-insert-only-tables/
+	let _ = client.execute("ALTER TABLE channel_updates SET ( autovacuum_vacuum_insert_scale_factor = 0.005 );", &[]).await;
+	let _ = client.execute("ALTER TABLE channel_announcements SET ( autovacuum_vacuum_insert_scale_factor = 0.005 );", &[]).await;
 }
 
 /// EDIT ME

src/downloader.rs

@@ -106,7 +106,7 @@ impl RoutingMessageHandler for GossipRouter {
 		self.native_router.get_next_node_announcement(starting_point)
 	}
 
-	fn peer_connected(&self, their_node_id: &PublicKey, init: &Init) {
+	fn peer_connected(&self, their_node_id: &PublicKey, init: &Init) -> Result<(), ()> {
 		self.native_router.peer_connected(their_node_id, init)
 	}
 

src/lib.rs

@@ -60,7 +60,7 @@ impl RapidSyncProcessor {
 			let mut buffered_reader = BufReader::new(file);
 			let network_graph_result = NetworkGraph::read(&mut buffered_reader, logger);
 			if let Ok(network_graph) = network_graph_result {
-				network_graph.remove_stale_channels();
+				network_graph.remove_stale_channels_and_tracking();
 				println!("Initialized from cached network graph!");
 				network_graph
 			} else {
@@ -106,6 +106,8 @@ impl RapidSyncProcessor {
 async fn serialize_delta(network_graph: Arc<NetworkGraph<TestLogger>>, last_sync_timestamp: u32, consider_intermediate_updates: bool) -> SerializedResponse {
 	let (client, connection) = lookup::connect_to_db().await;
 
+	network_graph.remove_stale_channels_and_tracking();
+
 	tokio::spawn(async move {
 		if let Err(e) = connection.await {
 			panic!("connection error: {}", e);

src/persistence.rs

@@ -189,7 +189,7 @@ impl GossipPersister {
 			.truncate(true)
 			.open(&cache_path)
 			.unwrap();
-		self.network_graph.remove_stale_channels();
+		self.network_graph.remove_stale_channels_and_tracking();
 		let mut writer = BufWriter::new(file);
 		self.network_graph.write(&mut writer).unwrap();
 		writer.flush().unwrap();

src/verifier.rs

@@ -5,7 +5,7 @@ use bitcoin::blockdata::block::Block;
 use bitcoin::hashes::Hash;
 use lightning::chain;
 use lightning::chain::AccessError;
-use lightning_block_sync::BlockSource;
+use lightning_block_sync::{BlockData, BlockSource};
 use lightning_block_sync::http::BinaryResponse;
 use lightning_block_sync::rest::RestClient;
 
@@ -36,11 +36,16 @@ impl ChainVerifier {
 			let block_hash = BlockHash::from_slice(&block_hash).unwrap();
 
 			let block_result = self.rest_client.get_block(&block_hash).await;
-			let block = block_result.map_err(|error| {
-				eprintln!("Couldn't retrieve block {}: {:?} ({})", block_height, error, block_hash);
-				AccessError::UnknownChain
-			})?;
-			Ok(block)
+			match block_result {
+				Ok(BlockData::FullBlock(block)) => {
+					Ok(block)
+				},
+				Ok(_) => unreachable!(),
+				Err(error) => {
+					eprintln!("Couldn't retrieve block {}: {:?} ({})", block_height, error, block_hash);
+					Err(AccessError::UnknownChain)
+				}
+			}
 		}) })
 	}
 }