Remove the need for bitcoin full node

.github/workflows/middleware.yml

@@ -0,0 +1,61 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  build-linux:
+    name: Build Backend & Publish (Linux)
+    runs-on: ubuntu-22.04
+    container:
+      image: "ubuntu:22.04"
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      # beware about using v2 because https://github.com/actions/checkout/issues/100
+      - uses: actions/checkout@v1
+        with:
+          submodules: recursive
+      - name: Install .NET6
+        run: |
+          apt update && apt install --yes sudo
+          sudo apt install --yes curl dotnet6
+      - name: Build middleware
+        run: |
+          dotnet restore RgsToElectrumServerMiddleware/RgsToElectrumServerMiddleware.fsproj
+          dotnet build RgsToElectrumServerMiddleware/RgsToElectrumServerMiddleware.fsproj
+      - name: Publish for linux
+        run: |
+          mkdir publish
+          # Create a platform-specific self-contained executable (no need to install .NET runtime on the target machine)
+          # More info: https://docs.microsoft.com/en-us/dotnet/core/deploying/
+          dotnet publish RgsToElectrumServerMiddleware/RgsToElectrumServerMiddleware.fsproj -p:PublishSingleFile=true -r linux-x64 -c Release -o publish --self-contained true
+      - name: Create version.txt file
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          echo "$GITHUB_REF_NAME" > publish/version.txt
+      - name: Archive publish
+        run: |
+          sudo apt install --yes zip
+          cd publish
+          zip -r ../middleware-server-linux-amd64.zip *
+          # is this needed?
+          cd ..
+      - name: Create Release
+        id: create_release
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: actions/create-release@v1  
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: Release ${{ github.ref }}
+          body: Backend
+          draft: false
+          prerelease: false
+      - name: Upload Server As Release Asset
+        id: upload-server-asset 
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: actions/upload-release-asset@v1
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./middleware-server-linux-amd64.zip
+          asset_name: middleware-server-linux-amd64.zip
+          asset_content_type: application/zip

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "geewallet"]
+	path = geewallet
+	url = https://github.com/nblockchain/geewallet.git

Cargo.toml

@@ -11,6 +11,7 @@ lightning-net-tokio = { version = "0.0.112" }
 tokio = { version = "1.14.1", features = ["full"] }
 tokio-postgres = { version="0.7.5" }
 futures = "0.3"
+hex = "0.4.3"
 
 [profile.release]
 opt-level = 3

RgsToElectrumServerMiddleware/.gitignore

@@ -0,0 +1,37 @@
+*.swp
+*.*~
+project.lock.json
+.DS_Store
+*.pyc
+nupkg/
+
+# Visual Studio Code
+.vscode
+
+# Rider
+.idea
+
+# User-specific files
+*.suo
+*.user
+*.userosscache
+*.sln.docstates
+
+# Build results
+[Dd]ebug/
+[Dd]ebugPublic/
+[Rr]elease/
+[Rr]eleases/
+x64/
+x86/
+build/
+bld/
+[Bb]in/
+[Oo]bj/
+[Oo]ut/
+msbuild.log
+msbuild.err
+msbuild.wrn
+
+# Visual Studio 2015
+.vs/

RgsToElectrumServerMiddleware/Program.fs

@@ -0,0 +1,44 @@
+open System
+open System.Threading.Tasks
+open Microsoft.AspNetCore.Builder
+open Microsoft.Extensions.Hosting
+
+open GWallet.Backend
+open GWallet.Backend.UtxoCoin
+
+[<EntryPoint>]
+let main args =
+    let builder = WebApplication.CreateBuilder args
+    let app = builder.Build()
+
+    app.Urls.Add "http://0.0.0.0:5108"
+
+    app.MapGet("/getTransaction/{height}/{txPos}",
+        Func<string, string, Task<string>>(
+            fun height txPos ->
+                async {
+                    Console.WriteLine(sprintf "Request: looking for transaction #%s at block height #%s" txPos height)
+
+                    let height = Convert.ToUInt32 height
+                    let txPos = Convert.ToUInt32 txPos
+
+                    let querySettings =
+                        QuerySettings.Default ServerSelectionMode.Fast
+
+                    let getTransactionFromPosIdJob = ElectrumClient.GetBlockchainTransactionIdFromPos height txPos
+                    let! txId =
+                        Server.Query Currency.BTC querySettings getTransactionFromPosIdJob None
+
+                    let getTransactionJob = ElectrumClient.GetBlockchainTransaction txId
+                    let! tx =
+                        Server.Query Currency.BTC querySettings getTransactionJob None
+
+                    return tx
+                } |> Async.StartAsTask
+        )
+    ) |> ignore
+
+    app.Run()
+
+    0 // Exit code
+

RgsToElectrumServerMiddleware/RgsToElectrumServerMiddleware.fsproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="Program.fs" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="NBitcoin" Version="6.0.17" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\geewallet\src\GWallet.Backend\GWallet.Backend.fsproj" />
+  </ItemGroup>
+
+</Project>

RgsToElectrumServerMiddleware/appsettings.Development.json

@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}

RgsToElectrumServerMiddleware/appsettings.json

@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}

src/config.rs

@@ -35,13 +35,13 @@ pub(crate) fn db_connection_config() -> Config {
 	config
 }
 
-pub(crate) fn bitcoin_rest_endpoint() -> HttpEndpoint {
-	let host = env::var("BITCOIN_REST_DOMAIN").unwrap_or("127.0.0.1".to_string());
-	let port = env::var("BITCOIN_REST_PORT")
-		.unwrap_or("8332".to_string())
+pub(crate) fn middleware_rest_endpoint() -> HttpEndpoint {
+	let host = env::var("MIDDLEWARE_REST_DOMAIN").unwrap_or("127.0.0.1".to_string());
+	let port = env::var("MIDDLEWARE_REST_PORT")
+		.unwrap_or("5108".to_string())
 		.parse::<u16>()
-		.expect("BITCOIN_REST_PORT env variable must be a u16.");
-	let path = env::var("BITCOIN_REST_PATH").unwrap_or("/rest/".to_string());
+		.expect("MIDDLEWARE_REST_PORT env variable must be a u16.");
+	let path = env::var("MIDDLEWARE_REST_PATH").unwrap_or("/".to_string());
 	HttpEndpoint::for_host(host).with_port(port).with_path(path)
 }
 

src/downloader.rs

@@ -63,6 +63,7 @@ impl RoutingMessageHandler for GossipRouter {
 		{
 			let mut counter = self.counter.write().unwrap();
 			output_value = native_result.map_err(|error| {
+				eprintln!("Error happened on handle_channel_announcement:{}", error.err);
 				if error.err.contains("didn't match on-chain script") {
 					counter.channel_announcements_with_mismatched_scripts += 1;
 				}

src/tracking.rs

@@ -75,7 +75,7 @@ pub(crate) async fn download_gossip(persistence_sender: mpsc::Sender<GossipMessa
 
 		loop {
 			i += 1; // count the background activity
-			let sleep = tokio::time::sleep(Duration::from_secs(5));
+			let sleep = tokio::time::sleep(Duration::from_secs(20));
 			sleep.await;
 
 			{
@@ -85,7 +85,7 @@ pub(crate) async fn download_gossip(persistence_sender: mpsc::Sender<GossipMessa
 
 				let was_previously_caught_up_with_gossip = is_caught_up_with_gossip;
 				// TODO: make new message threshold (20) adjust based on connected peer count
-				is_caught_up_with_gossip = new_message_count < 20 && previous_announcement_count > 0 && previous_update_count > 0;
+				is_caught_up_with_gossip = new_message_count < 1 && previous_announcement_count > 0 && previous_update_count > 0;
 				if new_message_count > 0 {
 					latest_new_gossip_time = Instant::now();
 				}

src/verifier.rs

@@ -1,11 +1,10 @@
+use hex;
 use std::convert::TryInto;
 
-use bitcoin::{BlockHash, TxOut};
-use bitcoin::blockdata::block::Block;
-use bitcoin::hashes::Hash;
+use bitcoin::{BlockHash, TxOut, Transaction};
+use bitcoin::consensus::deserialize;
 use lightning::chain;
 use lightning::chain::AccessError;
-use lightning_block_sync::{BlockData, BlockSource};
 use lightning_block_sync::http::BinaryResponse;
 use lightning_block_sync::rest::RestClient;
 
@@ -20,32 +19,42 @@ struct RestBinaryResponse(Vec<u8>);
 impl ChainVerifier {
 	pub(crate) fn new() -> Self {
 		ChainVerifier {
-			rest_client: RestClient::new(config::bitcoin_rest_endpoint()).unwrap(),
+			rest_client: RestClient::new(config::middleware_rest_endpoint()).unwrap(),
 		}
 	}
 
-	fn retrieve_block(&self, block_height: u32) -> Result<Block, AccessError> {
+	fn retrieve_tx(&self, block_height: u32, transaction_index: u32) -> Result<Transaction, AccessError> {
 		tokio::task::block_in_place(move || { tokio::runtime::Handle::current().block_on(async move {
-			let uri = format!("blockhashbyheight/{}.bin", block_height);
-			let block_hash_result =
+			let uri = format!("getTransaction/{}/{}", block_height, transaction_index);
+			let tx_result =
 				self.rest_client.request_resource::<BinaryResponse, RestBinaryResponse>(&uri).await;
-			let block_hash: Vec<u8> = block_hash_result.map_err(|error| {
-				eprintln!("Could't find block hash at height {}: {}", block_height, error.to_string());
+			let tx_hex_in_bytes: Vec<u8> = tx_result.map_err(|error| {
+				eprintln!("Could't find transaction at height {} and pos {}: {}", block_height, transaction_index, error.to_string());
 				AccessError::UnknownChain
 			})?.0;
-			let block_hash = BlockHash::from_slice(&block_hash).unwrap();
 
-			let block_result = self.rest_client.get_block(&block_hash).await;
-			match block_result {
-				Ok(BlockData::FullBlock(block)) => {
-					Ok(block)
-				},
-				Ok(_) => unreachable!(),
-				Err(error) => {
-					eprintln!("Couldn't retrieve block {}: {:?} ({})", block_height, error, block_hash);
-					Err(AccessError::UnknownChain)
-				}
-			}
+			let tx_hex_in_string =
+				String::from_utf8(tx_hex_in_bytes)
+					.map_err(|non_utf8| String::from_utf8_lossy(non_utf8.as_bytes()).into_owned())
+					.unwrap();
+
+			let tx_bytes =
+				hex::decode(tx_hex_in_string)
+					.map_err(|error| {
+						eprintln!("Could't find transaction at height {} and pos {}: {}", block_height, transaction_index, error.to_string());
+						AccessError::UnknownChain
+					})
+					.unwrap();
+
+			let transaction =
+				deserialize::<Transaction>(tx_bytes.as_slice())
+					.map_err(|error| {
+						eprintln!("Could't find transaction at height {} and pos {}: {}", block_height, transaction_index, error.to_string());
+						AccessError::UnknownChain
+					})
+					.unwrap();
+
+			Ok(transaction)
 		}) })
 	}
 }
@@ -56,11 +65,8 @@ impl chain::Access for ChainVerifier {
 		let transaction_index = ((short_channel_id >> 2 * 8) & 0xffffff) as u32;
 		let output_index = (short_channel_id & 0xffff) as u16;
 
-		let block = self.retrieve_block(block_height)?;
-		let transaction = block.txdata.get(transaction_index as usize).ok_or_else(|| {
-			eprintln!("Transaction index {} out of bounds in block {} ({})", transaction_index, block_height, block.block_hash().to_string());
-			AccessError::UnknownTx
-		})?;
+		let transaction = self.retrieve_tx(block_height, transaction_index)?;
+
 		let output = transaction.output.get(output_index as usize).ok_or_else(|| {
 			eprintln!("Output index {} out of bounds in transaction {}", output_index, transaction.txid().to_string());
 			AccessError::UnknownTx