From 869d6807b5e9137c747bce238a12a3565a64319b Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Tue, 5 Dec 2023 17:46:28 +0900 Subject: [PATCH] More flexible wallets API --- NBXplorer.Client/ExplorerClient.cs | 150 ++++++--- .../Models/GenerateWalletRequest.cs | 2 + NBXplorer.Client/Models/ImportUTXORequest.cs | 20 ++ NBXplorer.Client/Models/TrackWalletRequest.cs | 1 + NBXplorer.Client/Models/TrackedSource.cs | 40 +++ .../Models/TrackedSourceRequest.cs | 6 + NBXplorer.Tests/UnitTest1.cs | 310 ++++++++++++++++++ NBXplorer.Tests/xunit.runner.json | 3 +- .../Backends/Postgres/PostgresRepository.cs | 168 ++++++++-- NBXplorer/Controllers/MainController.PSBT.cs | 11 +- NBXplorer/Controllers/MainController.cs | 13 +- .../Controllers/PostgresMainController.cs | 257 ++++++++++++++- NBXplorer/Controllers/TrackedSourceContext.cs | 28 +- .../021.KeyPathInfoReturnsWalletId.sql | 13 + NBXplorer/DBScripts/FullSchema.sql | 4 +- NBXplorer/Extensions.cs | 7 + NBXplorer/TrackedTransaction.cs | 2 +- 17 files changed, 938 insertions(+), 97 deletions(-) create mode 100644 NBXplorer.Client/Models/ImportUTXORequest.cs create mode 100644 NBXplorer.Client/Models/TrackedSourceRequest.cs create mode 100644 NBXplorer/DBScripts/021.KeyPathInfoReturnsWalletId.sql diff --git a/NBXplorer.Client/ExplorerClient.cs b/NBXplorer.Client/ExplorerClient.cs index ddc2d9f07..ae61c5c6b 100644 --- a/NBXplorer.Client/ExplorerClient.cs +++ b/NBXplorer.Client/ExplorerClient.cs @@ -15,6 +15,7 @@ using NBitcoin.RPC; using System.Runtime.CompilerServices; using System.Linq; +using System.Diagnostics; namespace NBXplorer { @@ -172,7 +173,8 @@ public PruneResponse Prune(DerivationStrategyBase extKey, PruneRequest pruneRequ return PruneAsync(extKey, pruneRequest, cancellation).GetAwaiter().GetResult(); } - internal class RawStr { + internal class RawStr + { private string str; public RawStr(string str) { @@ -269,7 +271,7 @@ public void Track(DerivationStrategyBase strategy, CancellationToken cancellatio } public Task TrackAsync(DerivationStrategyBase strategy, CancellationToken cancellation = default) { - return TrackAsync(TrackedSource.Create(strategy), cancellation); + return TrackAsync(TrackedSource.Create(strategy), cancellation: cancellation); } public void Track(DerivationStrategyBase strategy, TrackWalletRequest trackDerivationRequest, CancellationToken cancellation = default) @@ -285,14 +287,13 @@ public async Task TrackAsync(DerivationStrategyBase strategy, TrackWalletRequest public void Track(TrackedSource trackedSource, CancellationToken cancellation = default) { - TrackAsync(trackedSource, cancellation).GetAwaiter().GetResult(); + TrackAsync(trackedSource, cancellation: cancellation).GetAwaiter().GetResult(); } - public Task TrackAsync(TrackedSource trackedSource, CancellationToken cancellation = default) + public Task TrackAsync(TrackedSource trackedSource, TrackWalletRequest trackDerivationRequest = null, CancellationToken cancellation = default) { if (trackedSource == null) throw new ArgumentNullException(nameof(trackedSource)); - - return SendAsync(HttpMethod.Post, null, GetBasePath(trackedSource), cancellation); + return SendAsync(HttpMethod.Post, trackDerivationRequest, GetBasePath(trackedSource), cancellation); } private Exception UnSupported(TrackedSource trackedSource) @@ -311,7 +312,7 @@ public GetBalanceResponse GetBalance(DerivationStrategyBase userDerivationScheme } public Task GetBalanceAsync(DerivationStrategyBase userDerivationScheme, CancellationToken cancellation = default) { - return SendAsync(HttpMethod.Get, null, $"v1/cryptos/{CryptoCode}/derivations/{userDerivationScheme}/balance", cancellation); + return GetBalanceAsync(TrackedSource.Create(userDerivationScheme), cancellation); } @@ -321,9 +322,12 @@ public GetBalanceResponse GetBalance(BitcoinAddress address, CancellationToken c } public Task GetBalanceAsync(BitcoinAddress address, CancellationToken cancellation = default) { - return SendAsync(HttpMethod.Get, null, $"v1/cryptos/{CryptoCode}/addresses/{address}/balance", cancellation); + return GetBalanceAsync(TrackedSource.Create(address), cancellation); + } + public Task GetBalanceAsync(TrackedSource trackedSource, CancellationToken cancellation = default) + { + return SendAsync(HttpMethod.Get, null, $"{GetBasePath(trackedSource)}/balance", cancellation); } - public Task CancelReservationAsync(DerivationStrategyBase strategy, KeyPath[] keyPaths, CancellationToken cancellation = default) { return SendAsync(HttpMethod.Post, keyPaths, $"v1/cryptos/{CryptoCode}/derivations/{strategy}/addresses/cancelreservation", cancellation); @@ -363,18 +367,7 @@ public Task GetTransactionsAsync(DerivationStrategyBase } public Task GetTransactionsAsync(TrackedSource trackedSource, CancellationToken cancellation = default) { - if (trackedSource == null) - throw new ArgumentNullException(nameof(trackedSource)); - if (trackedSource is DerivationSchemeTrackedSource dsts) - { - return SendAsync(HttpMethod.Get, null, $"v1/cryptos/{CryptoCode}/derivations/{dsts.DerivationStrategy}/transactions", cancellation); - } - else if (trackedSource is AddressTrackedSource asts) - { - return SendAsync(HttpMethod.Get, null, $"v1/cryptos/{CryptoCode}/addresses/{asts.Address}/transactions", cancellation); - } - else - throw UnSupported(trackedSource); + return SendAsync(HttpMethod.Get, null, $"{GetBasePath(trackedSource)}/transactions", cancellation); } @@ -399,16 +392,25 @@ public Task GetTransactionAsync(TrackedSource trackedSou throw new ArgumentNullException(nameof(txId)); if (trackedSource == null) throw new ArgumentNullException(nameof(trackedSource)); - if (trackedSource is DerivationSchemeTrackedSource dsts) - { - return SendAsync(HttpMethod.Get, null, $"v1/cryptos/{CryptoCode}/derivations/{dsts.DerivationStrategy}/transactions/{txId}", cancellation); - } - else if (trackedSource is AddressTrackedSource asts) - { - return SendAsync(HttpMethod.Get, null, $"v1/cryptos/{CryptoCode}/addresses/{asts.Address}/transactions/{txId}", cancellation); - } - else - throw UnSupported(trackedSource); + return SendAsync(HttpMethod.Get, null, $"{GetBasePath(trackedSource)}/transactions/{txId}", cancellation); + } + + public async Task AssociateScriptsAsync(TrackedSource trackedSource, AssociateScriptRequest[] scripts, CancellationToken cancellation = default) + { + if (scripts == null) + throw new ArgumentNullException(nameof(scripts)); + if (trackedSource == null) + throw new ArgumentNullException(nameof(trackedSource)); + await SendAsync(HttpMethod.Post, scripts, $"{GetBasePath(trackedSource)}/associate", cancellation); + } + + public async Task ImportUTXOs(TrackedSource trackedSource, ImportUTXORequest request, CancellationToken cancellation = default) + { + if (request == null) + throw new ArgumentNullException(nameof(request)); + if (trackedSource == null) + throw new ArgumentNullException(nameof(trackedSource)); + await SendAsync(HttpMethod.Post, request, $"{GetBasePath(trackedSource)}/import-utxos", cancellation); } public Task RescanAsync(RescanRequest rescanRequest, CancellationToken cancellation = default) @@ -447,16 +449,17 @@ public KeyPathInformation GetKeyInformation(DerivationStrategyBase strategy, Scr public async Task GetKeyInformationAsync(DerivationStrategyBase strategy, Script script, CancellationToken cancellation = default) { - return await SendAsync(HttpMethod.Get, null, $"v1/cryptos/{CryptoCode}/derivations/{strategy}/scripts/{script.ToHex()}", cancellation).ConfigureAwait(false); + return await GetKeyInformationAsync(new DerivationSchemeTrackedSource(strategy), script, cancellation).ConfigureAwait(false); + } + public async Task GetKeyInformationAsync(TrackedSource trackedSource, Script script, CancellationToken cancellation = default) + { + return await SendAsync(HttpMethod.Get, null, $"{GetBasePath(trackedSource)}/scripts/{script.ToHex()}", cancellation).ConfigureAwait(false); } - - [Obsolete("Use GetKeyInformationAsync(DerivationStrategyBase strategy, Script script) instead")] public async Task GetKeyInformationsAsync(Script script, CancellationToken cancellation = default) { return await SendAsync(HttpMethod.Get, null, $"v1/cryptos/{CryptoCode}/scripts/{script.ToHex()}", cancellation).ConfigureAwait(false); } - [Obsolete("Use GetKeyInformation(DerivationStrategyBase strategy, Script script) instead")] public KeyPathInformation[] GetKeyInformations(Script script, CancellationToken cancellation = default) { return GetKeyInformationsAsync(script, cancellation).GetAwaiter().GetResult(); @@ -563,6 +566,53 @@ public GenerateWalletResponse GenerateWallet(GenerateWalletRequest request = nul return GenerateWalletAsync(request, cancellationToken).GetAwaiter().GetResult(); } + public async Task GetChildWallets(TrackedSource trackedSource, + CancellationToken cancellation = default) + { + return await GetAsync($"{GetBasePath(trackedSource)}/children", cancellation); + } + public async Task GetParentWallets(TrackedSource trackedSource, + CancellationToken cancellation = default) + { + return await GetAsync($"{GetBasePath(trackedSource)}/parents", cancellation); + } + public async Task AddChildWallet(TrackedSource trackedSource, TrackedSource childWallet, CancellationToken cancellation = default) + { + var request = new TrackedSourceRequest() + { + TrackedSource = childWallet + }; + await SendAsync(HttpMethod.Post, request, $"{GetBasePath(trackedSource)}/children", cancellation); + } + + public async Task AddParentWallet(TrackedSource trackedSource, TrackedSource parentWallet, + CancellationToken cancellation = default) + { + var request = new TrackedSourceRequest() + { + TrackedSource = parentWallet + }; + await SendAsync(HttpMethod.Post, request, $"{GetBasePath(trackedSource)}/parents", cancellation); + } + public async Task RemoveChildWallet(TrackedSource trackedSource, TrackedSource childWallet, CancellationToken cancellation = default) + { + var request = new TrackedSourceRequest() + { + TrackedSource = childWallet + }; + await SendAsync(HttpMethod.Delete, request, $"{GetBasePath(trackedSource)}/children", cancellation); + } + + public async Task RemoveParentWallet(TrackedSource trackedSource, TrackedSource parentWallet, + CancellationToken cancellation = default) + { + var request = new TrackedSourceRequest() + { + TrackedSource = parentWallet + }; + await SendAsync(HttpMethod.Delete, request, $"{GetBasePath(trackedSource)}/parents", cancellation); + } + private static readonly HttpClient SharedClient = new HttpClient(); internal HttpClient Client = SharedClient; @@ -611,8 +661,8 @@ static FormattableString EncodeUrlParameters(FormattableString url) return FormattableStringFactory.Create( url.Format, url.GetArguments() - .Select(a => - a is RawStr ? a : + .Select(a => + a is RawStr ? a : a is FormattableString o ? EncodeUrlParameters(o) : Uri.EscapeDataString(a?.ToString() ?? "")) .ToArray()); @@ -653,11 +703,30 @@ internal async Task SendAsync(HttpMethod method, object body, FormattableS if (Auth.RefreshCache()) { message = CreateMessage(method, body, relativePath); - result = await Client.SendAsync(message).ConfigureAwait(false); + result = await Client.SendAsync(message, cancellation).ConfigureAwait(false); } } return await ParseResponse(result).ConfigureAwait(false); } + internal async Task SendAsync(HttpMethod method, object body, FormattableString relativePath, CancellationToken cancellation) + { + var message = CreateMessage(method, body, relativePath); + var result = await Client.SendAsync(message, cancellation).ConfigureAwait(false); + + if (result.StatusCode == HttpStatusCode.GatewayTimeout || result.StatusCode == HttpStatusCode.RequestTimeout) + { + throw new HttpRequestException($"HTTP error {(int)result.StatusCode}", new TimeoutException()); + } + if ((int)result.StatusCode == 401) + { + if (Auth.RefreshCache()) + { + message = CreateMessage(method, body, relativePath); + result = await Client.SendAsync(message, cancellation).ConfigureAwait(false); + } + } + await ParseResponse(result).ConfigureAwait(false); + } internal HttpRequestMessage CreateMessage(HttpMethod method, object body, FormattableString relativePath) { @@ -717,11 +786,14 @@ private async Task ParseResponse(HttpResponseMessage response) private FormattableString GetBasePath(TrackedSource trackedSource) { + if (trackedSource is null) + throw new ArgumentNullException(nameof(trackedSource)); return trackedSource switch { DerivationSchemeTrackedSource dsts => $"v1/cryptos/{CryptoCode}/derivations/{dsts.DerivationStrategy}", AddressTrackedSource asts => $"v1/cryptos/{CryptoCode}/addresses/{asts.Address}", - _ => throw UnSupported(trackedSource) + WalletTrackedSource wts => $"v1/cryptos/{CryptoCode}/wallets/{wts.WalletId}", + _ => $"v1/cryptos/{CryptoCode}/tracked-sources/{trackedSource}" }; } } diff --git a/NBXplorer.Client/Models/GenerateWalletRequest.cs b/NBXplorer.Client/Models/GenerateWalletRequest.cs index 666987fb0..baaea9658 100644 --- a/NBXplorer.Client/Models/GenerateWalletRequest.cs +++ b/NBXplorer.Client/Models/GenerateWalletRequest.cs @@ -17,5 +17,7 @@ public class GenerateWalletRequest public bool ImportKeysToRPC { get; set; } public bool SavePrivateKeys { get; set; } public Dictionary AdditionalOptions { get; set; } + + public TrackedSource ParentWallet { get; set; } } } diff --git a/NBXplorer.Client/Models/ImportUTXORequest.cs b/NBXplorer.Client/Models/ImportUTXORequest.cs new file mode 100644 index 000000000..d7723151a --- /dev/null +++ b/NBXplorer.Client/Models/ImportUTXORequest.cs @@ -0,0 +1,20 @@ +using NBitcoin; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace NBXplorer.Models; + +public class ImportUTXORequest +{ + [JsonProperty("UTXOs")] + public OutPoint[] Utxos { get; set; } + + public MerkleBlock[] Proofs { get; set; } +} + +public class AssociateScriptRequest +{ + public IDestination Destination { get; set; } + public bool Used { get; set; } + public JObject Metadata { get; set; } +} \ No newline at end of file diff --git a/NBXplorer.Client/Models/TrackWalletRequest.cs b/NBXplorer.Client/Models/TrackWalletRequest.cs index 19ed4f196..977d0a40c 100644 --- a/NBXplorer.Client/Models/TrackWalletRequest.cs +++ b/NBXplorer.Client/Models/TrackWalletRequest.cs @@ -7,6 +7,7 @@ public class TrackWalletRequest { public TrackDerivationOption[] DerivationOptions { get; set; } public bool Wait { get; set; } = false; + public TrackedSource ParentWallet { get; set; } } public class TrackDerivationOption diff --git a/NBXplorer.Client/Models/TrackedSource.cs b/NBXplorer.Client/Models/TrackedSource.cs index c170a95df..5dcebe50b 100644 --- a/NBXplorer.Client/Models/TrackedSource.cs +++ b/NBXplorer.Client/Models/TrackedSource.cs @@ -26,6 +26,12 @@ public static bool TryParse(string str, out TrackedSource trackedSource, NBXplor return false; trackedSource = addressTrackedSource; } + else if (strSpan.StartsWith("WALLET:".AsSpan(), StringComparison.Ordinal)) + { + if (!WalletTrackedSource.TryParse(strSpan, out var walletTrackedSource)) + return false; + trackedSource = walletTrackedSource; + } else { return false; @@ -97,6 +103,40 @@ public static TrackedSource Parse(string str, NBXplorerNetwork network) } } + public class WalletTrackedSource : TrackedSource + { + public string WalletId { get; } + + public WalletTrackedSource(string walletId) + { + WalletId = walletId; + } + + public static bool TryParse(ReadOnlySpan strSpan, out WalletTrackedSource walletTrackedSource) + { + if (strSpan == null) + throw new ArgumentNullException(nameof(strSpan)); + walletTrackedSource = null; + if (!strSpan.StartsWith("WALLET:".AsSpan(), StringComparison.Ordinal)) + return false; + try + { + walletTrackedSource = new WalletTrackedSource(strSpan.Slice("WALLET:".Length).ToString()); + return true; + } + catch { return false; } + } + + public override string ToString() + { + return "WALLET:" + WalletId; + } + public override string ToPrettyString() + { + return WalletId; + } + } + public class AddressTrackedSource : TrackedSource, IDestination { // Note that we should in theory access BitcoinAddress. But parsing BitcoinAddress is very expensive, so we keep storing plain strings diff --git a/NBXplorer.Client/Models/TrackedSourceRequest.cs b/NBXplorer.Client/Models/TrackedSourceRequest.cs new file mode 100644 index 000000000..da3837a74 --- /dev/null +++ b/NBXplorer.Client/Models/TrackedSourceRequest.cs @@ -0,0 +1,6 @@ +namespace NBXplorer.Models; + +public class TrackedSourceRequest +{ + public TrackedSource TrackedSource { get; set; } +} \ No newline at end of file diff --git a/NBXplorer.Tests/UnitTest1.cs b/NBXplorer.Tests/UnitTest1.cs index e1fca3cb1..c64b4b3d2 100644 --- a/NBXplorer.Tests/UnitTest1.cs +++ b/NBXplorer.Tests/UnitTest1.cs @@ -28,6 +28,7 @@ using System.Net; using NBXplorer.HostedServices; using System.Reflection; +using NBitcoin.DataEncoders; namespace NBXplorer.Tests { @@ -4523,5 +4524,314 @@ public async Task CanUseRPCProxy(Backend backend) await tester.Client.RPCClient.GetTxOutAsync(uint256.One, 0); } } + private async Task Eventually(Func tsk) + { + var i = 0; + while (i < 10) + { + try + { + await tsk.Invoke(); + break; + } + catch (Exception) + { + await Task.Delay(500); + } + + i++; + } + } + + [Theory] + [InlineData(Backend.Postgres)] + public async Task CanAssociateIndependentScripts(Backend backend) + { + using var tester = ServerTester.Create(backend); + + var wallet1 = Guid.NewGuid().ToString(); + var wallet1TS = new WalletTrackedSource(wallet1); + var parentWallet = Guid.NewGuid().ToString(); + var parentWalletTS = new WalletTrackedSource(parentWallet); + await tester.Client.TrackAsync(wallet1TS, new TrackWalletRequest() + { + ParentWallet = parentWalletTS + }, CancellationToken.None); + + GenerateWalletResponse derivationWallet = null; + + derivationWallet = await tester.Client.GenerateWalletAsync(new GenerateWalletRequest() + { + ParentWallet = parentWalletTS + }); + + var derivationWalletTS = TrackedSource.Create(derivationWallet.DerivationScheme); + + var address = await tester.Client.GetUnusedAsync(derivationWallet.DerivationScheme, + DerivationFeature.Deposit, 0, true); + await tester.RPC.SendToAddressAsync(address.Address, Money.FromUnit(0.1m, MoneyUnit.BTC)); + var b1 = await tester.Client.GetBalanceAsync(derivationWalletTS); + var b2 = await tester.Client.GetBalanceAsync(parentWalletTS); + Assert.Equal(b1.Total, b2.Total); + + var derivationUtxos = await tester.Client.GetUTXOsAsync(derivationWalletTS); + var parentWalletUtxos = await tester.Client.GetUTXOsAsync(parentWalletTS); + Assert.Equal(derivationUtxos.GetUnspentUTXOs().Count(), parentWalletUtxos.GetUnspentUTXOs().Count()); + + var newAddr = await tester.RPC.GetNewAddressAsync(); + var newAddr2 = await tester.RPC.GetNewAddressAsync(); + var udetectedTxId = await tester.RPC.SendToAddressAsync(newAddr, Money.FromUnit(0.1m, MoneyUnit.BTC)); + await Task.Delay(3000); + var utxos = Assert.Single(await tester.RPC.ListUnspentAsync(0, 0, newAddr)); + await tester.Client.AssociateScriptsAsync(wallet1TS, new[] + { + new AssociateScriptRequest() + { + Destination = newAddr2, Used = true + } + }); + + + await tester.RPC.SendToAddressAsync(newAddr2, Money.FromUnit(0.2m, MoneyUnit.BTC)); + + derivationUtxos = await tester.Client.GetUTXOsAsync(derivationWalletTS); + UTXOChanges scriptBagUtxos = null; + //1 utxo was before we started tracking + + await Eventually(async () => + { + parentWalletUtxos = await tester.Client.GetUTXOsAsync(parentWalletTS); + scriptBagUtxos = await tester.Client.GetUTXOsAsync(wallet1TS); + Assert.Single(scriptBagUtxos.GetUnspentUTXOs()); + Assert.Equal(2, parentWalletUtxos.GetUnspentUTXOs().Length); + Assert.Equal(derivationUtxos.GetUnspentUTXOs().Count() + scriptBagUtxos.GetUnspentUTXOs().Length, + parentWalletUtxos.GetUnspentUTXOs().Count()); + }); + + + await tester.Client.AssociateScriptsAsync(wallet1TS, new[] + { + new AssociateScriptRequest() + { + Destination = newAddr, Used = true + } + }); + + await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest() + { + Utxos = new[] + { + utxos.OutPoint, + } + }); + + await Eventually(async () => + { + scriptBagUtxos = await tester.Client.GetUTXOsAsync(wallet1TS); + Assert.Equal(2, scriptBagUtxos.GetUnspentUTXOs().Length); + }); + + //create wallet A + //create wallet b using generate and make it child of A + // create address using unused on B + // creat wallet C tracking address from B, make it child of A, B + var walletA = new WalletTrackedSource(Guid.NewGuid().ToString()); + await tester.Client.TrackAsync(walletA); + var generatResponse = await tester.Client.GenerateWalletAsync(new GenerateWalletRequest() + { + ParentWallet = walletA + }); + var walletB = TrackedSource.Create(generatResponse.DerivationScheme); + var addressA = await tester.Client.GetUnusedAsync(generatResponse.DerivationScheme, DerivationFeature.Deposit, 0, true); + var walletC = AddressTrackedSource.Create(addressA.Address); + await tester.Client.TrackAsync(walletC, new TrackWalletRequest() + { + ParentWallet = walletB + }); + await tester.Client.TrackAsync(walletC, new TrackWalletRequest() + { + ParentWallet = walletA + }); + + var kpi = await tester.Client.GetKeyInformationsAsync(addressA.ScriptPubKey, CancellationToken.None); + var tss = kpi.Select(information => information.TrackedSource); + Assert.True(tss.Distinct().Count() == tss.Count(), "The result should only distinct tracked source matches. While this endpoint is marked obsolete, the same logic is used to trigger events, which means there will be duplicated events when the script is matched against"); + + var parentsOfC = await tester.Client.GetParentWallets(walletC); + Assert.Equal(2, parentsOfC.Length); + Assert.Contains(parentsOfC, w => w == walletA); + Assert.Contains(parentsOfC, w => w == walletB); + + var parentsOfB = await tester.Client.GetParentWallets(walletB); + Assert.Single(parentsOfB); + Assert.Contains(parentsOfB, w => w == walletA); + + var parentsOfA = await tester.Client.GetParentWallets(walletA); + Assert.Empty(parentsOfA); + + var childrenOfA = await tester.Client.GetChildWallets(walletA); + Assert.Equal(2, childrenOfA.Length); + + Assert.Contains(childrenOfA, w => w == walletB); + Assert.Contains(childrenOfA, w => w == walletC); + + var childrenOfB = await tester.Client.GetChildWallets(walletB); + Assert.Single(childrenOfB); + Assert.Contains(childrenOfB, w => w == walletC); + + var childrenOfC = await tester.Client.GetChildWallets(walletC); + Assert.Empty(childrenOfC); + + await tester.Client.RemoveParentWallet(walletB, walletA); + await tester.Client.RemoveChildWallet(walletB, walletC); + + parentsOfB = await tester.Client.GetParentWallets(walletB); + Assert.Empty(parentsOfB); + + childrenOfB = await tester.Client.GetChildWallets(walletB); + Assert.Empty(childrenOfB); + + + await tester.Client.AddParentWallet(walletB, walletA); + await tester.Client.AddChildWallet(walletB, walletC); + + + childrenOfB = await tester.Client.GetChildWallets(walletB); + Assert.Single(childrenOfB); + Assert.Contains(childrenOfB, w => w == walletC); + + parentsOfB = await tester.Client.GetParentWallets(walletB); + Assert.Single(parentsOfB); + Assert.Contains(parentsOfB, w => w == walletA); + + } + + [Theory] + [InlineData(Backend.Postgres)] + public async Task CanImportUTXOs(Backend backend) + { + using var tester = ServerTester.Create(backend); + + var wallet1 = Guid.NewGuid().ToString(); + var wallet1TS = new WalletTrackedSource(wallet1); + + var k = new Key(); + var kAddress = k.GetAddress(ScriptPubKeyType.Segwit, tester.Network); + var kScript = kAddress.ScriptPubKey; + + // test 1: create a script and send 2 utxos to it(from diff txs), without confirming + // import the first one, verify it is unconfirmed, confirm, then the second one and see it is confirmed + + var tx = await tester.RPC.SendToAddressAsync(kAddress, Money.Coins(1.0m)); + var tx2 = await tester.RPC.SendToAddressAsync(kAddress, Money.Coins(1.0m)); + var rawTx = await tester.RPC.GetRawTransactionAsync(tx); + var rawTx2 = await tester.RPC.GetRawTransactionAsync(tx2); + var utxo = rawTx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == kScript); + var utxo2 = rawTx2.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == kScript); + + await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest() + { + Utxos = new[] + { + utxo.ToCoin().Outpoint, + } + }); + + var utxos = await tester.Client.GetUTXOsAsync(wallet1TS); + var matched = Assert.Single(utxos.Unconfirmed.UTXOs); + Assert.Equal(kAddress, matched.Address); + + await tester.RPC.GenerateAsync(1); + await Eventually(async () => + { + var utxos = await tester.Client.GetUTXOsAsync(wallet1TS); + Assert.Single(utxos.Confirmed.UTXOs); + }); + + await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest() + { + Utxos = new[] + { + utxo2.ToCoin().Outpoint, + } + }); + + await Eventually(async () => + { + var utxos = await tester.Client.GetUTXOsAsync(wallet1TS); + Assert.Equal(2, utxos.Confirmed.UTXOs.Count); + + //utxo2 may be confirmed but we dont know much about it I guess? + var utxoInfo = utxos.Confirmed.UTXOs.First(u => u.ScriptPubKey == utxo2.TxOut.ScriptPubKey); + Assert.Equal(NBitcoin.Utils.UnixTimeToDateTime(0), utxoInfo.Timestamp); + }); + + //test2: try adding in fake utxos or spent ones + var fakescript = new Key().PubKey.GetAddress(ScriptPubKeyType.Segwit, tester.Network).ScriptPubKey; + var fakeUtxo = new Coin(new OutPoint(uint256.One, 1), new TxOut(Money.Coins(1.0m), fakescript)); + var kToSpend = new Key(); + var kToSpendAddress = kToSpend.GetAddress(ScriptPubKeyType.Segwit, tester.Network); + var tospendtx = await tester.RPC.SendToAddressAsync(kToSpendAddress, Money.Coins(1.0m)); + var tospendrawtx = await tester.RPC.GetRawTransactionAsync(tospendtx); + var tospendutxo = tospendrawtx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == kToSpendAddress.ScriptPubKey); + var validScript = new Key().PubKey.GetAddress(ScriptPubKeyType.Segwit, tester.Network).ScriptPubKey; + var spendingtx = tester.Network.CreateTransactionBuilder() + .AddKeys(kToSpend) + .AddCoins(new Coin(tospendutxo)) + .SendEstimatedFees(new FeeRate(100m)) + .SendAll(validScript).BuildTransaction(true); + await tester.RPC.SendRawTransactionAsync(spendingtx); + + var validScriptUtxo = spendingtx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == validScript); + + await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest() + { + Utxos = new[] + { + fakeUtxo.Outpoint, + new Coin(tospendutxo).Outpoint, + new Coin(validScriptUtxo).Outpoint + } + }); + + await Eventually(async () => + { + var utxos = await tester.Client.GetUTXOsAsync(wallet1TS); + Assert.Equal(validScript, Assert.Single(utxos.Unconfirmed.UTXOs).ScriptPubKey); + + }); + + + // let's test out proofs + + var yoScript = new Key().PubKey.GetAddress(ScriptPubKeyType.Segwit, tester.Network).ScriptPubKey; + var yoTxId = await tester.SendToAddressAsync(yoScript, Money.Coins(1.0m)); + var yoTx = await tester.RPC.GetRawTransactionAsync(yoTxId); + var yoUtxo = yoTx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == yoScript); + var blockHash = await tester.RPC.GenerateAsync(1); + var proofResult = await tester.RPC.SendCommandAsync(RPCOperations.gettxoutproof, new[] { yoTxId.ToString() }, blockHash[0].ToString()); + + + var merkleBLockProofBytes = Encoders.Hex.DecodeData(proofResult.ResultString); + var mb = new MerkleBlock(); + mb.FromBytes(merkleBLockProofBytes); + + await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest() + { + Utxos = new[] + { + new Coin(yoUtxo).Outpoint + } + }); + + await Eventually(async () => + { + var utxos = await tester.Client.GetUTXOsAsync(wallet1TS); + var importedUtxoWithProof = utxos.Confirmed.UTXOs.Single(utxo1 => utxo1.ScriptPubKey == yoScript); + + Assert.NotEqual(NBitcoin.Utils.UnixTimeToDateTime(0), importedUtxoWithProof.Timestamp); + + }); + } } } diff --git a/NBXplorer.Tests/xunit.runner.json b/NBXplorer.Tests/xunit.runner.json index dc14f82b1..6dae53c9c 100644 --- a/NBXplorer.Tests/xunit.runner.json +++ b/NBXplorer.Tests/xunit.runner.json @@ -1,3 +1,4 @@ { - "parallelizeTestCollections": false + "parallelizeTestCollections": false, + "methodDisplay": "method" } \ No newline at end of file diff --git a/NBXplorer/Backends/Postgres/PostgresRepository.cs b/NBXplorer/Backends/Postgres/PostgresRepository.cs index fdca826d0..738a1aed9 100644 --- a/NBXplorer/Backends/Postgres/PostgresRepository.cs +++ b/NBXplorer/Backends/Postgres/PostgresRepository.cs @@ -207,7 +207,12 @@ internal WalletKey GetWalletKey(DerivationStrategyBase strategy) m.Add(new JProperty("derivation", new JValue(strategy.ToString()))); return new WalletKey(hash, m.ToString(Formatting.None)); } - + WalletKey GetWalletKey(WalletTrackedSource walletTrackedSource) + { + var m = new JObject { new JProperty("type", new JValue("Wallet")) }; + var res = new WalletKey(walletTrackedSource.WalletId, m.ToString(Formatting.None)); + return res; + } WalletKey GetWalletKey(IDestination destination) { var address = destination.ScriptPubKey.GetDestinationAddress(Network.NBitcoinNetwork); @@ -226,10 +231,72 @@ internal WalletKey GetWalletKey(TrackedSource source) { DerivationSchemeTrackedSource derivation => GetWalletKey(derivation.DerivationStrategy), AddressTrackedSource addr => GetWalletKey(addr.Address), + WalletTrackedSource wallet => GetWalletKey(wallet), _ => throw new NotSupportedException(source.GetType().ToString()) }; } + internal TrackedSource GetTrackedSource(WalletKey walletKey) + { + var metadata = JObject.Parse(walletKey.metadata); + if (metadata.TryGetValue("type", StringComparison.OrdinalIgnoreCase, out JToken typeJToken) && + typeJToken.Value() is { } type) + { + if ((metadata.TryGetValue("code", StringComparison.OrdinalIgnoreCase, out JToken codeJToken) && + codeJToken.Value() is { } code) && !code.Equals(Network.CryptoCode, + StringComparison.InvariantCultureIgnoreCase)) + { + return null; + } + + switch (type) + { + case "NBXv1-Derivation": + var derivation = metadata["derivation"].Value(); + return new DerivationSchemeTrackedSource(Network.DerivationStrategyFactory.Parse(derivation)); + case "NBXv1-Address": + var address = metadata["address"].Value(); + return new AddressTrackedSource(BitcoinAddress.Create(address, Network.NBitcoinNetwork)); + case "Wallet": + return new WalletTrackedSource(walletKey.wid); + } + } + + return null; + } + + public async Task AssociateScriptsToWalletExplicitly(TrackedSource trackedSource, + AssociateScriptRequest[] scripts) + { + var walletKey = GetWalletKey(trackedSource); + await using var conn = await GetConnection(); + + var scriptsRecords = scripts.Select(pair => new ScriptInsert(this.Network.CryptoCode, walletKey.wid, + pair.Destination.ScriptPubKey.ToHex(), pair.Destination.ToString(), pair.Used)).ToArray(); + { + await conn.Connection.ExecuteAsync(WalletInsertQuery, new { walletKey.wid, walletKey.metadata }); + await conn.Connection.ExecuteAsync( + "INSERT INTO scripts (code, script, addr, used) VALUES(@code, @script, @addr, @used) ON CONFLICT DO NOTHING;" + + "INSERT INTO wallets_scripts (code, wallet_id, script) VALUES(@code, @wallet_id, @script) ON CONFLICT DO NOTHING;" + , scriptsRecords); + var descriptScriptInsert = scripts.Where(request => request.Metadata is not null).Select(request => + new DescriptorScriptInsert(request.Destination.ScriptPubKey.ToHex(), 0, + request.Destination.ScriptPubKey.ToHex(), request.Metadata.ToString(Formatting.None), + request.Destination.ToString(), request.Used)).ToList(); + await InsertDescriptorsScripts(conn.Connection, descriptScriptInsert); + + if (ImportRPCMode.Legacy == await GetImportRPCMode(conn, walletKey)) + { + foreach (var scriptsRecord in scriptsRecords) + { + await ImportAddressToRPC(null, + BitcoinAddress.Create(scriptsRecord.addr, Network.NBitcoinNetwork), null); + } + } + } + } + + internal record ScriptInsert(string code, string wallet_id, string script, string addr, bool used); internal record DescriptorScriptInsert(string descriptor, int idx, string script, string metadata, string addr, bool used); public async Task GenerateAddresses(DerivationStrategyBase strategy, DerivationFeature derivationFeature, GenerateAddressQuery query = null) { @@ -450,12 +517,22 @@ async Task> GetKeyInformations( if (scripts.Count == 0) return result; string additionalColumn = Network.IsElement ? ", ts.blinded_addr" : ""; - var rows = await connection.QueryAsync($"SELECT ts.script, ts.addr, ts.derivation, ts.keypath, ts.redeem {additionalColumn} FROM " + - "unnest(@records) AS r (script)," + - " LATERAL (" + - " SELECT script, addr, descriptor_metadata->>'derivation' derivation, keypath, descriptors_scripts_metadata->>'redeem' redeem, descriptors_scripts_metadata->>'blindedAddress' blinded_addr " + - " FROM nbxv1_keypath_info ki " + - " WHERE ki.code=@code AND ki.script=r.script) ts;", new { code = Network.CryptoCode, records = scripts.Select(s => s.ToHex()).ToArray() }); + var rows = await connection.QueryAsync($@" + SELECT ts.code, ts.script, ts.addr, ts.derivation, ts.keypath, ts.redeem{additionalColumn}, + ts.wallet_id, + w.metadata->>'type' AS wallet_type + FROM unnest(@records) AS r (script), + LATERAL ( + SELECT code, script, wallet_id, addr, descriptor_metadata->>'derivation' derivation, + keypath, descriptors_scripts_metadata->>'redeem' redeem, + descriptors_scripts_metadata->>'blindedAddress' blinded_addr, + descriptors_scripts_metadata->>'blindingKey' blindingKey, + descriptor_metadata->>'descriptor' descriptor + FROM nbxv1_keypath_info ki + WHERE ki.code=@code AND ki.script=r.script + ) ts + JOIN wallets w USING(wallet_id)", + new { code = Network.CryptoCode, records = scripts.Select(s => s.ToHex()).ToArray() }); foreach (var r in rows) { // This might be the case for a derivation added by a different indexer @@ -465,24 +542,40 @@ async Task> GetKeyInformations( bool isExplicit = r.derivation is null; bool isDescriptor = !isExplicit; var script = Script.FromHex(r.script); - var derivationStrategy = isDescriptor ? Network.DerivationStrategyFactory.Parse(r.derivation) : null; - var keypath = isDescriptor ? KeyPath.Parse(r.keypath) : null; + var derivationStrategy = r.derivation is not null ? Network.DerivationStrategyFactory.Parse(r.derivation) : null; + var keypath = r.keypath is not null ? KeyPath.Parse(r.keypath) : null; var redeem = (string)r.redeem; - result.Add(script, new KeyPathInformation() - { - Address = addr, - DerivationStrategy = isDescriptor ? derivationStrategy : null, - KeyPath = isDescriptor ? keypath : null, - ScriptPubKey = script, - TrackedSource = isDescriptor && derivationStrategy is not null ? new DerivationSchemeTrackedSource(derivationStrategy) : - isExplicit ? new AddressTrackedSource(addr) : null, - Feature = keypath is null ? DerivationFeature.Deposit : KeyPathTemplates.GetDerivationFeature(keypath), - Redeem = redeem is null ? null : Script.FromHex(redeem) - }); + string walletType = r.wallet_type; + string walletId = r.wallet_id; + + var trackedSource = derivationStrategy is not null + ? new DerivationSchemeTrackedSource(derivationStrategy) + : walletType == "Wallet" + ? walletId is null ? (TrackedSource)null : new WalletTrackedSource(walletId) + : new AddressTrackedSource(addr); + var ki = Network.IsElement && r.blindingKey is not null + ? new LiquidKeyPathInformation() + { + BlindingKey = Key.Parse(r.blindingKey, Network.NBitcoinNetwork) + } + : new KeyPathInformation(); + ki.Address = addr; + ki.DerivationStrategy = r.derivation is not null ? derivationStrategy : null; + ki.KeyPath = keypath; + ki.ScriptPubKey = script; + ki.TrackedSource = trackedSource; + ki.Feature = keypath is null ? DerivationFeature.Deposit : KeyPathTemplates.GetDerivationFeature(keypath); + ki.Redeem = redeem is null ? null : Script.FromHex(redeem); + result.Add(script, ki); } return result; } + public class LiquidKeyPathInformation : KeyPathInformation + { + public Key BlindingKey { get; set; } + } + private BitcoinAddress GetAddress(dynamic r) { if (Network.IsElement && r.blinded_addr is not null) @@ -952,11 +1045,14 @@ await connection.ExecuteAsync( "INSERT INTO scripts VALUES (@code, @script, @address) ON CONFLICT DO NOTHING;" + "INSERT INTO wallets_scripts VALUES (@code, @script, @walletid) ON CONFLICT DO NOTHING;", inserts); } - + private async Task GetImportRPCMode(DbConnectionHelper connection, WalletKey walletKey) + { + return ImportRPCMode.Parse((await connection.GetMetadata(walletKey.wid, WellknownMetadataKeys.ImportAddressToRPC))); + } private async Task ImportAddressToRPC(DbConnectionHelper connection, TrackedSource trackedSource, BitcoinAddress address, KeyPath keyPath) { var k = GetWalletKey(trackedSource); - var shouldImportRPC = ImportRPCMode.Parse((await connection.GetMetadata(k.wid, WellknownMetadataKeys.ImportAddressToRPC))); + var shouldImportRPC = await GetImportRPCMode(connection, k); if (shouldImportRPC != ImportRPCMode.Legacy) return; var accountKey = await connection.GetMetadata(k.wid, WellknownMetadataKeys.AccountHDKey); @@ -1175,7 +1271,7 @@ public async Task Track(IDestination address) await using var conn = await GetConnection(); var walletKey = GetWalletKey(address); await conn.Connection.ExecuteAsync( - "INSERT INTO wallets VALUES (@wid, @metadata::JSONB) ON CONFLICT DO NOTHING;" + + WalletInsertQuery + "INSERT INTO scripts VALUES (@code, @script, @addr) ON CONFLICT DO NOTHING;" + "INSERT INTO wallets_scripts VALUES (@code, @script, @wid) ON CONFLICT DO NOTHING" , new { code = Network.CryptoCode, script = address.ScriptPubKey.ToHex(), addr = address.ScriptPubKey.GetDestinationAddress(Network.NBitcoinNetwork).ToString(), walletKey.wid, walletKey.metadata }); @@ -1263,9 +1359,31 @@ public async Task SaveBlocks(IList slimBlocks) public async Task EnsureWalletCreated(DerivationStrategyBase strategy) { - using var connection = await ConnectionFactory.CreateConnection(); - await connection.ExecuteAsync("INSERT INTO wallets VALUES (@wid, @metadata::JSONB) ON CONFLICT DO NOTHING", GetWalletKey(strategy)); + await EnsureWalletCreated(GetWalletKey(strategy)); + } + + public async Task EnsureWalletCreated(TrackedSource trackedSource, params TrackedSource[] parentTrackedSource) + { + parentTrackedSource = parentTrackedSource.Where(source => source is not null).ToArray(); + await EnsureWalletCreated(GetWalletKey(trackedSource), parentTrackedSource?.Select(GetWalletKey).ToArray()); } + + record WalletHierarchyInsert(string child, string parent); + public async Task EnsureWalletCreated(WalletKey walletKey, WalletKey[] parentWallets = null) + { + await using var connection = await ConnectionFactory.CreateConnection(); + parentWallets ??= Array.Empty(); + var walletRecords = new[] { walletKey }.Concat(parentWallets).ToArray(); + var parentsRecords = parentWallets.Select(key => new WalletHierarchyInsert(walletKey.wid, key.wid)).ToArray(); + + + await connection.ExecuteAsync(WalletInsertQuery, walletRecords); + await connection.ExecuteAsync( + "INSERT INTO wallets_wallets (wallet_id, parent_id) VALUES (@child, @parent)ON CONFLICT DO NOTHING" + , parentsRecords); + } + + private readonly string WalletInsertQuery = "INSERT INTO wallets (wallet_id, metadata) VALUES (@wid, @metadata::JSONB) ON CONFLICT DO NOTHING;"; } public class LegacyDescriptorMetadata diff --git a/NBXplorer/Controllers/MainController.PSBT.cs b/NBXplorer/Controllers/MainController.PSBT.cs index beab9945d..4314bf1fd 100644 --- a/NBXplorer/Controllers/MainController.PSBT.cs +++ b/NBXplorer/Controllers/MainController.PSBT.cs @@ -29,7 +29,7 @@ public async Task CreatePSBT( if (body == null) throw new ArgumentNullException(nameof(body)); var network = trackedSourceContext.Network; - CreatePSBTRequest request = ParseJObject(body, network); + CreatePSBTRequest request = network.ParseJObject(body); var repo = RepositoryProvider.GetRepository(network); var txBuilder = request.Seed is int s ? network.NBitcoinNetwork.CreateTransactionBuilder(s) @@ -390,7 +390,7 @@ public async Task UpdatePSBT( [FromBody] JObject body) { - var update = ParseJObject(body, network); + var update = network.ParseJObject(body); if (update.PSBT == null) throw new NBXplorerException(new NBXplorerError(400, "missing-parameter", "'psbt' is missing")); await UpdatePSBTCore(update, network); @@ -587,12 +587,5 @@ await Task.WhenAll(update.PSBT.Inputs await getTransactions; } } - - protected T ParseJObject(JObject requestObj, NBXplorerNetwork network) - { - if (requestObj == null) - return default; - return network.Serializer.ToObject(requestObj); - } } } diff --git a/NBXplorer/Controllers/MainController.cs b/NBXplorer/Controllers/MainController.cs index 1b0bf18ab..a3b772ea9 100644 --- a/NBXplorer/Controllers/MainController.cs +++ b/NBXplorer/Controllers/MainController.cs @@ -75,7 +75,9 @@ public AddressPoolService AddressPoolService "getrawtransaction", "gettxout", "estimatesmartfee", - "getmempoolinfo" + "getmempoolinfo", + "gettxoutproof", + "verifytxoutproof" }; private Exception JsonRPCNotExposed() { @@ -512,7 +514,7 @@ public async Task TrackWallet( { var network = trackedSourceContext.Network; var trackedSource = trackedSourceContext.TrackedSource; - var request = ParseJObject(rawRequest ?? new JObject(), network); + var request = network.ParseJObject(rawRequest ?? new JObject()); if (trackedSource is DerivationSchemeTrackedSource dts) { if (request.Wait) @@ -554,7 +556,8 @@ private GenerateAddressQuery GenerateAddressQuery(TrackWalletRequest request, De } return null; } - + + [HttpGet] [Route($"{CommonRoutes.DerivationEndpoint}/{CommonRoutes.TransactionsPath}")] [Route($"{CommonRoutes.AddressEndpoint}/{CommonRoutes.TransactionsPath}")] [Route($"{CommonRoutes.WalletEndpoint}/{CommonRoutes.TransactionsPath}")] @@ -658,7 +661,7 @@ public async Task Rescan(TrackedSourceContext trackedSourceContex if (body == null) throw new ArgumentNullException(nameof(body)); var network = trackedSourceContext.Network; - var rescanRequest = ParseJObject(body, network); + var rescanRequest = network.ParseJObject(body); if (rescanRequest == null) throw new ArgumentNullException(nameof(rescanRequest)); if (rescanRequest?.Transactions == null) @@ -1024,7 +1027,7 @@ public async Task Broadcast( public async Task GenerateWallet(TrackedSourceContext trackedSourceContext, [FromBody] JObject rawRequest = null) { var network = trackedSourceContext.Network; - var request = ParseJObject(rawRequest, network) ?? new GenerateWalletRequest(); + var request = network.ParseJObject(rawRequest) ?? new GenerateWalletRequest(); if (request.ImportKeysToRPC && trackedSourceContext.RpcClient is null) { diff --git a/NBXplorer/Controllers/PostgresMainController.cs b/NBXplorer/Controllers/PostgresMainController.cs index 70417fc18..aadc68da6 100644 --- a/NBXplorer/Controllers/PostgresMainController.cs +++ b/NBXplorer/Controllers/PostgresMainController.cs @@ -8,6 +8,7 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using NBitcoin; +using NBitcoin.DataEncoders; using NBitcoin.RPC; using NBXplorer.Backends.Postgres; using NBXplorer.DerivationStrategy; @@ -23,11 +24,11 @@ namespace NBXplorer.Controllers [Route($"v1/{CommonRoutes.WalletEndpoint}")] [Route($"v1/{CommonRoutes.TrackedSourceEndpoint}")] [Authorize] - public class PostgresMainController :Controller, IUTXOService + public class PostgresMainController : Controller, IUTXOService { public PostgresMainController( DbConnectionFactory connectionFactory, - KeyPathTemplates keyPathTemplates) + KeyPathTemplates keyPathTemplates) { ConnectionFactory = connectionFactory; KeyPathTemplates = keyPathTemplates; @@ -83,6 +84,199 @@ public async Task GetBalance(TrackedSourceContext trackedSourceCo return Json(balance, network.JsonSerializerSettings); } + [HttpPost("associate")] + [PostgresImplementationActionConstraint(true)] + public async Task AssociateScripts(TrackedSourceContext trackedSourceContext, [FromBody] JArray rawRequest) + { + var repo = (PostgresRepository)trackedSourceContext.Repository; + var jsonSerializer = JsonSerializer.Create(trackedSourceContext.Network.JsonSerializerSettings); + var requests = rawRequest.ToObject(jsonSerializer); + await repo.AssociateScriptsToWalletExplicitly(trackedSourceContext.TrackedSource, requests); + return Ok(); + } + + [HttpPost("import-utxos")] + [TrackedSourceContext.TrackedSourceContextRequirement(true)] + public async Task ImportUTXOs(TrackedSourceContext trackedSourceContext, [FromBody] JObject rawRequest) + { + var repo = (PostgresRepository)trackedSourceContext.Repository; + var jsonSerializer = JsonSerializer.Create(trackedSourceContext.Network.JsonSerializerSettings); + var request = rawRequest.ToObject(jsonSerializer); + + if (request.Utxos?.Any() is not true) + return Ok(); + + var rpc = trackedSourceContext.RpcClient; + + var clientBatch = rpc.PrepareBatch(); + var coinToTxOut = new Dictionary>(); + var txsToBlockHash = new ConcurrentDictionary(); + var blockHeaders = new ConcurrentDictionary>(); + var allUtxoTransactionHashes = request.Utxos.Select(u => u.Hash).Distinct().ToArray(); + foreach (var importUtxoRequest in request.Utxos) + { + coinToTxOut.TryAdd(importUtxoRequest, clientBatch.GetTxOutAsync(importUtxoRequest.Hash, (int)importUtxoRequest.N)); + } + request.Proofs ??= Array.Empty(); + var verifyTasks = request.Proofs + .Where(p => p is not null && p.PartialMerkleTree.Hashes.Any(uint256 => allUtxoTransactionHashes.Contains(uint256))) + .Select(async proof => + { + var txoutproofResult = await clientBatch.SendCommandAsync("verifytxoutproof", + Encoders.Hex.EncodeData(proof.ToBytes())); + if (txoutproofResult.Error is not null && txoutproofResult.Result is JArray prooftxs) + { + foreach (var txProof in prooftxs) + { + var txId = uint256.Parse(txProof.Value()); + blockHeaders.TryAdd(proof.Header.GetHash(), Task.FromResult((proof.Header.GetHash(), proof.Header.BlockTime))); + txsToBlockHash.TryAdd(txId, proof.Header.GetHash()); + } + } + }); + + await clientBatch.SendBatchAsync(); + await Task.WhenAll(verifyTasks.Concat(coinToTxOut.Values)); + + + coinToTxOut = coinToTxOut.Where(c => c.Value.Result is not null).ToDictionary(pair => pair.Key, pair => pair.Value); + + await using var conn = await repo.ConnectionFactory.CreateConnection(); + + var blockTasks = new ConcurrentDictionary>(); + var blocksToRequest = new HashSet(); + foreach (var cTxOut in coinToTxOut) + { + var result = await cTxOut.Value; + if (result.Confirmations == 1) + { + txsToBlockHash.TryAdd(cTxOut.Key.Hash, result.BestBlock); + continue; + } + blocksToRequest.Add(result.BestBlock); + } + + var res = await conn.QueryAsync( + $"SELECT blk_id, height FROM blks WHERE code=@code AND blk_id IN (SELECT unnest(@blkIds)) ", + new + { + code = trackedSourceContext.Network.CryptoCode, + blkIds = blocksToRequest.Select(uint256 => uint256.ToString()).ToArray() + }); + + foreach (var r in res) + { + var blockHash = uint256.Parse((string)r.blk_id); + var height = (int)r.height; + blockTasks.TryAdd(blockHash, Task.FromResult(height)); + blocksToRequest.Remove(blockHash); + } + + clientBatch = rpc.PrepareBatch(); + foreach (var bh in blocksToRequest) + { + blockTasks.TryAdd(bh, clientBatch.GetBlockHeaderAsyncEx(bh).ContinueWith(task => task.Result.Height)); + } + await clientBatch.SendBatchAsync(); + await Task.WhenAll(blockTasks.Values); + var heightToBlockHash = new ConcurrentDictionary>(); + var heightsToFetch = new HashSet(); + foreach (var cTxOut in coinToTxOut) + { + var result = await cTxOut.Value; + + if (result.Confirmations <= 1) + continue; + + blockTasks.TryGetValue(result.BestBlock, out var blockTask); + var b = await blockTask; + + var heightToFetch = b - result.Confirmations - 1; + heightsToFetch.Add(heightToFetch); + } + + res = await conn.QueryAsync( + $"SELECT blk_id, height, indexed_at FROM blks WHERE code=@code AND height IN (SELECT unnest(@heights)) ", + new + { + code = trackedSourceContext.Network.CryptoCode, + heights = heightsToFetch.ToArray() + }); + + foreach (var r in res) + { + var blockHash = uint256.Parse((string)r.blk_id); + var height = (int)r.height; + var blockTime = (DateTimeOffset)r.indexed_at; + blockTasks.TryAdd(blockHash, Task.FromResult(height)); + heightToBlockHash.TryAdd(height, Task.FromResult((blockHash, blockTime))); + heightsToFetch.Remove((int)r.height); + } + + foreach (var heightToFetch in heightsToFetch) + { + heightToBlockHash.TryAdd(heightToFetch, clientBatch.GetBlockHeaderAsync(heightToFetch).ContinueWith(task => (task.Result.GetHash(), task.Result.BlockTime))); + } + + clientBatch = rpc.PrepareBatch(); + + await clientBatch.SendBatchAsync(); + foreach (var htbh in heightToBlockHash.Values) + { + var result = await htbh; + blockHeaders.TryAdd(result.hash, Task.FromResult(result)); + + foreach (var cto in coinToTxOut) + { + var result2 = await cto.Value; + if (result2.Confirmations <= 1) + continue; + + txsToBlockHash.TryAdd(cto.Key.Hash, result.hash); + } + } + + var now = DateTimeOffset.UtcNow; + + var scripts = coinToTxOut + .Select(pair => ( + pair.Value.Result.TxOut.ScriptPubKey.GetDestinationAddress(repo.Network.NBitcoinNetwork), pair)) + .Where(pair => pair.Item1 is not null).Select(tuple => new AssociateScriptRequest() + { + Destination = tuple.Item1, + Used = tuple.pair.Value is not null, + Metadata = null + }).ToArray(); + + await repo.AssociateScriptsToWalletExplicitly(trackedSourceContext.TrackedSource, scripts); + + + var trackedTransactions = coinToTxOut.Select(async pair => + { + var txOutResult = await pair.Value; + txsToBlockHash.TryGetValue(pair.Key.Hash, out var blockHash); + (uint256 hash, DateTimeOffset time)? blockHeader = null; + if (blockHash is not null && blockHeaders.TryGetValue(blockHash, out var blockHeaderT)) + { + blockHeader = await blockHeaderT; + }; + + var coin = new Coin(pair.Key, txOutResult.TxOut); + + var ttx = repo.CreateTrackedTransaction(trackedSourceContext.TrackedSource, + new TrackedTransactionKey(pair.Key.Hash, blockHash, true) { }, + new[] { coin }, null); + ttx.Inserted = now; + ttx.Immature = txOutResult.IsCoinBase && txOutResult.Confirmations <= 100; + ttx.FirstSeen = blockHeader?.time ?? NBitcoin.Utils.UnixTimeToDateTime(0); ; + return ttx; + }); + + await repo.SaveMatches(await Task.WhenAll(trackedTransactions)); + + return Ok(); + } + private IMoney Format(NBXplorerNetwork network, MoneyBag bag) { if (network.IsElement) @@ -103,7 +297,7 @@ private static MoneyBag RemoveZeros(MoneyBag bag) [HttpGet("utxos")] - public async Task GetUTXOs( TrackedSourceContext trackedSourceContext) + public async Task GetUTXOs(TrackedSourceContext trackedSourceContext) { var trackedSource = trackedSourceContext.TrackedSource; var repo = (PostgresRepository)trackedSourceContext.Repository; @@ -183,5 +377,62 @@ public async Task GetUTXOs( TrackedSourceContext trackedSourceCon } return Json(changes, network.JsonSerializerSettings); } + + [HttpGet("children")] + public async Task GetWalletChildren(TrackedSourceContext trackedSourceContext) + { + var repo = (PostgresRepository)trackedSourceContext.Repository; + await using var conn = await ConnectionFactory.CreateConnection(); + var children = await conn.QueryAsync($"SELECT w.wallet_id, w.metadata FROM wallets_wallets ww JOIN wallets w ON ww.wallet_id = w.wallet_id WHERE ww.parent_id=@walletId", new { walletId = repo.GetWalletKey(trackedSourceContext.TrackedSource).wid }); + + return Json(children.Select(c => repo.GetTrackedSource(new PostgresRepository.WalletKey(c.wallet_id, c.metadata))).ToArray(), trackedSourceContext.Network.JsonSerializerSettings); + } + [HttpGet("parents")] + public async Task GetWalletParents(TrackedSourceContext trackedSourceContext) + { + var repo = (PostgresRepository)trackedSourceContext.Repository; + await using var conn = await ConnectionFactory.CreateConnection(); + var children = await conn.QueryAsync($"SELECT w.wallet_id, w.metadata FROM wallets_wallets ww JOIN wallets w ON ww.parent_id = w.wallet_id WHERE ww.wallet_id=@walletId", new { walletId = repo.GetWalletKey(trackedSourceContext.TrackedSource).wid }); + + return Json(children.Select(c => repo.GetTrackedSource(new PostgresRepository.WalletKey(c.wallet_id, c.metadata))).ToArray(), trackedSourceContext.Network.JsonSerializerSettings); + } + [HttpPost("children")] + public async Task AddWalletChild(TrackedSourceContext trackedSourceContext, [FromBody] JObject request) + { + var trackedSource = trackedSourceContext.Network.ParseJObject(request).TrackedSource; + var repo = (PostgresRepository)trackedSourceContext.Repository; + await repo.EnsureWalletCreated(trackedSource, trackedSourceContext.TrackedSource); + return Ok(); + } + [HttpPost("parents")] + public async Task AddWalletParent(TrackedSourceContext trackedSourceContext, [FromBody] JObject request) + { + var trackedSource = trackedSourceContext.Network.ParseJObject(request).TrackedSource; + var repo = (PostgresRepository)trackedSourceContext.Repository; + await repo.EnsureWalletCreated(trackedSourceContext.TrackedSource, trackedSource); + return Ok(); + } + [HttpDelete("children")] + public async Task RemoveWalletChild(TrackedSourceContext trackedSourceContext, [FromBody] JObject request) + { + var repo = (PostgresRepository)trackedSourceContext.Repository; + + var trackedSource = repo.GetWalletKey(trackedSourceContext.Network + .ParseJObject(request).TrackedSource); + var conn = await ConnectionFactory.CreateConnection(); + await conn.ExecuteAsync($"DELETE FROM wallets_wallets WHERE wallet_id=@walletId AND parent_id=@parentId", new { walletId = trackedSource.wid, parentId = repo.GetWalletKey(trackedSourceContext.TrackedSource).wid }); + return Ok(); + } + [HttpDelete("parents")] + public async Task RemoveWalletParent(TrackedSourceContext trackedSourceContext, [FromBody] JObject request) + { + var repo = (PostgresRepository)trackedSourceContext.Repository; + + var trackedSource = repo.GetWalletKey(trackedSourceContext.Network + .ParseJObject(request).TrackedSource); + var conn = await ConnectionFactory.CreateConnection(); + await conn.ExecuteAsync($"DELETE FROM wallets_wallets WHERE wallet_id=@walletId AND parent_id=@parentId", new { walletId = repo.GetWalletKey(trackedSourceContext.TrackedSource).wid, parentId = trackedSource.wid }); + return Ok(); + } } } diff --git a/NBXplorer/Controllers/TrackedSourceContext.cs b/NBXplorer/Controllers/TrackedSourceContext.cs index fa757adad..035a06883 100644 --- a/NBXplorer/Controllers/TrackedSourceContext.cs +++ b/NBXplorer/Controllers/TrackedSourceContext.cs @@ -13,7 +13,7 @@ namespace NBXplorer.Controllers; -[ModelBinder] +[ModelBinder] public class TrackedSourceContext { public TrackedSource TrackedSource { get; set; } @@ -21,21 +21,21 @@ public class TrackedSourceContext public RPCClient RpcClient { get; set; } public IIndexer Indexer { get; set; } public IRepository Repository { get; set; } - - public class TrackedSourceContextRequirementAttribute: Attribute + + public class TrackedSourceContextRequirementAttribute : Attribute { public bool RequireRpc { get; } public bool RequireTrackedSource { get; } public bool DisallowTrackedSource { get; } public Type[] AllowedTrackedSourceTypes { get; } - public TrackedSourceContextRequirementAttribute(bool requireRPC = false, bool requireTrackedSource = true, bool disallowTrackedSource = false, params Type[] allowedTrackedSourceTypes) + public TrackedSourceContextRequirementAttribute(bool requireRPC = false, bool requireTrackedSource = true, bool disallowTrackedSource = false, params Type[] allowedTrackedSourceTypes) { RequireRpc = requireRPC; RequireTrackedSource = requireTrackedSource; DisallowTrackedSource = disallowTrackedSource; AllowedTrackedSourceTypes = allowedTrackedSourceTypes; - + } } @@ -50,7 +50,7 @@ public Task BindModelAsync(ModelBindingContext bindingContext) var addressValue = bindingContext.ValueProvider.GetValue("address").FirstValue; var derivationSchemeValue = bindingContext.ValueProvider.GetValue("derivationScheme").FirstValue; - derivationSchemeValue??= bindingContext.ValueProvider.GetValue("extPubKey").FirstValue; + derivationSchemeValue ??= bindingContext.ValueProvider.GetValue("extPubKey").FirstValue; var walletIdValue = bindingContext.ValueProvider.GetValue("walletId").FirstValue; var trackedSourceValue = bindingContext.ValueProvider.GetValue("trackedSource").FirstValue; @@ -67,10 +67,10 @@ public Task BindModelAsync(ModelBindingContext bindingContext) $"{cryptoCode} is not supported")); } - var requirements = ((ControllerActionDescriptor) bindingContext.ActionContext.ActionDescriptor) + var requirements = ((ControllerActionDescriptor)bindingContext.ActionContext.ActionDescriptor) .MethodInfo.GetCustomAttributes().FirstOrDefault(); - - + + var rpcClient = indexer.GetConnectedClient(); if (rpcClient?.Capabilities == null) { @@ -87,16 +87,16 @@ public Task BindModelAsync(ModelBindingContext bindingContext) network); if (ts is null && requirements?.RequireTrackedSource is true) { - + throw new NBXplorerException(new NBXplorerError(400, "tracked-source-required", $"A tracked source is required for this endpoint.")); } - if ( ts is not null && requirements?.DisallowTrackedSource is true) + if (ts is not null && requirements?.DisallowTrackedSource is true) { throw new NBXplorerException(new NBXplorerError(400, "tracked-source-unwanted", $"This endpoint does not tracked sources..")); } - if(ts is not null && requirements?.AllowedTrackedSourceTypes?.Any() is true && !requirements.AllowedTrackedSourceTypes.Any(t => t.IsInstanceOfType(ts))) + if (ts is not null && requirements?.AllowedTrackedSourceTypes?.Any() is true && !requirements.AllowedTrackedSourceTypes.Any(t => t.IsInstanceOfType(ts))) { throw new NBXplorerException(new NBXplorerError(400, "tracked-source-invalid", $"The tracked source provided is not valid for this endpoint.")); @@ -106,7 +106,7 @@ public Task BindModelAsync(ModelBindingContext bindingContext) { Indexer = indexer, Network = network, - TrackedSource = ts , + TrackedSource = ts, RpcClient = rpcClient, Repository = repositoryProvider.GetRepository(network) }); @@ -126,6 +126,8 @@ public static TrackedSource GetTrackedSource(string derivationScheme, string add return new AddressTrackedSource(BitcoinAddress.Create(address, network.NBitcoinNetwork)); if (derivationScheme != null) return new DerivationSchemeTrackedSource(network.DerivationStrategyFactory.Parse(derivationScheme)); + if (walletId != null) + return new WalletTrackedSource(walletId); return null; } } diff --git a/NBXplorer/DBScripts/021.KeyPathInfoReturnsWalletId.sql b/NBXplorer/DBScripts/021.KeyPathInfoReturnsWalletId.sql new file mode 100644 index 000000000..f8d6563a2 --- /dev/null +++ b/NBXplorer/DBScripts/021.KeyPathInfoReturnsWalletId.sql @@ -0,0 +1,13 @@ +CREATE OR REPLACE VIEW nbxv1_keypath_info AS + SELECT ws.code, + ws.script, + s.addr, + d.metadata AS descriptor_metadata, + nbxv1_get_keypath(d.metadata, ds.idx) AS keypath, + ds.metadata AS descriptors_scripts_metadata, + ws.wallet_id + FROM ((wallets_scripts ws + JOIN scripts s ON (((s.code = ws.code) AND (s.script = ws.script)))) + LEFT JOIN ((wallets_descriptors wd + JOIN descriptors_scripts ds ON (((ds.code = wd.code) AND (ds.descriptor = wd.descriptor)))) + JOIN descriptors d ON (((d.code = ds.code) AND (d.descriptor = ds.descriptor)))) ON (((wd.wallet_id = ws.wallet_id) AND (wd.code = ws.code) AND (ds.script = ws.script)))); \ No newline at end of file diff --git a/NBXplorer/DBScripts/FullSchema.sql b/NBXplorer/DBScripts/FullSchema.sql index 040bfd826..c2c736fb0 100644 --- a/NBXplorer/DBScripts/FullSchema.sql +++ b/NBXplorer/DBScripts/FullSchema.sql @@ -958,7 +958,8 @@ CREATE VIEW nbxv1_keypath_info AS s.addr, d.metadata AS descriptor_metadata, nbxv1_get_keypath(d.metadata, ds.idx) AS keypath, - ds.metadata AS descriptors_scripts_metadata + ds.metadata AS descriptors_scripts_metadata, + ws.wallet_id FROM ((wallets_scripts ws JOIN scripts s ON (((s.code = ws.code) AND (s.script = ws.script)))) LEFT JOIN ((wallets_descriptors wd @@ -1362,6 +1363,7 @@ INSERT INTO nbxv1_migrations VALUES ('017.FixDoubleSpendDetection'); INSERT INTO nbxv1_migrations VALUES ('018.FastWalletRecent'); INSERT INTO nbxv1_migrations VALUES ('019.FixDoubleSpendDetection2'); INSERT INTO nbxv1_migrations VALUES ('020.ReplacingShouldBeIdempotent'); +INSERT INTO nbxv1_migrations VALUES ('021.KeyPathInfoReturnsWalletId'); ALTER TABLE ONLY nbxv1_migrations ADD CONSTRAINT nbxv1_migrations_pkey PRIMARY KEY (script_name); \ No newline at end of file diff --git a/NBXplorer/Extensions.cs b/NBXplorer/Extensions.cs index dca02252a..69e4301be 100644 --- a/NBXplorer/Extensions.cs +++ b/NBXplorer/Extensions.cs @@ -31,11 +31,18 @@ using Npgsql; using NBitcoin.Altcoins; using System.Threading; +using Newtonsoft.Json.Linq; namespace NBXplorer { public static class Extensions { + public static T ParseJObject(this NBXplorerNetwork network, JObject requestObj) + { + if (requestObj == null) + return default; + return network.Serializer.ToObject(requestObj); + } public static async Task ReliableOpenConnectionAsync(this NpgsqlDataSource ds, CancellationToken cancellationToken = default) { int maxRetries = 10; diff --git a/NBXplorer/TrackedTransaction.cs b/NBXplorer/TrackedTransaction.cs index 9a4589c1d..a079747ba 100644 --- a/NBXplorer/TrackedTransaction.cs +++ b/NBXplorer/TrackedTransaction.cs @@ -151,7 +151,7 @@ public IEnumerable GetReceivedOutputs() Output: o, KeyPath: KnownKeyPathMapping.TryGet(o.TxOut.ScriptPubKey), Address: KnownKeyPathInformation.TryGet(o.TxOut.ScriptPubKey)?.Address)) - .Where(o => o.KeyPath != null || o.Output.TxOut.ScriptPubKey == (TrackedSource as IDestination)?.ScriptPubKey) + .Where(o => o.KeyPath != null || o.Output.TxOut.ScriptPubKey == (TrackedSource as IDestination)?.ScriptPubKey || TrackedSource is WalletTrackedSource) .Select(o => new MatchedOutput() { Index = o.Index,