diff --git a/.gitignore b/.gitignore index cfa290e..a66e9c4 100644 --- a/.gitignore +++ b/.gitignore @@ -277,4 +277,6 @@ __pycache__/ errorlogs.txt version.txt *launchSettings.json -AuditZips/ \ No newline at end of file +AuditZips/ + +.vscode diff --git a/CheckerApi/Jobs/ForkWatchJob.cs b/CheckerApi/Jobs/ForkWatchJob.cs new file mode 100644 index 0000000..f593c76 --- /dev/null +++ b/CheckerApi/Jobs/ForkWatchJob.cs @@ -0,0 +1,21 @@ +using CheckerApi.Services.Interfaces; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Quartz; +using System; + +namespace CheckerApi.Jobs +{ + [DisallowConcurrentExecution] + public class ForkWatchJob : Job + { + public override void Execute(JobDataMap data, IServiceProvider serviceProvider) + { + var config = serviceProvider.GetService(); + var watchService = serviceProvider.GetService(); + + var rpcConfig = this.GetNodeRpcConfig(config); + watchService.Execute(rpcConfig); + } + } +} \ No newline at end of file diff --git a/CheckerApi/Jobs/Job.cs b/CheckerApi/Jobs/Job.cs index 1358857..c4cb7d1 100644 --- a/CheckerApi/Jobs/Job.cs +++ b/CheckerApi/Jobs/Job.cs @@ -1,7 +1,10 @@ using System; using System.Diagnostics; +using System.Net; using System.Threading.Tasks; +using CheckerApi.Models.Config; using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Quartz; @@ -40,5 +43,19 @@ public Task Execute(IJobExecutionContext context) } }); } + + public RpcConfig GetNodeRpcConfig(IConfiguration config) + { + return new RpcConfig() + { + Url = config.GetValue("Node:Url"), + Port = config.GetValue("Node:RpcPort"), + Credentials = new NetworkCredential() + { + UserName = config.GetValue("Node:RpcUser"), + Password = config.GetValue("Node:RpcPass") + } + }; + } } } diff --git a/CheckerApi/Jobs/NodeJob.cs b/CheckerApi/Jobs/NodeJob.cs index f7f53cb..d8e9b2b 100644 --- a/CheckerApi/Jobs/NodeJob.cs +++ b/CheckerApi/Jobs/NodeJob.cs @@ -28,16 +28,7 @@ public override void Execute(JobDataMap data, IServiceProvider serviceProvider) var logger = serviceProvider.GetService>(); var mapper = serviceProvider.GetService(); - var rpcConfig = new RpcConfig() - { - Url = config.GetValue("Node:Url"), - Port = config.GetValue("Node:RpcPort"), - Credentials = new NetworkCredential() - { - UserName = config.GetValue("Node:RpcUser"), - Password = config.GetValue("Node:RpcPass") - } - }; + var rpcConfig = GetNodeRpcConfig(config); var cache = serviceProvider.GetService(); var difficultyResult = dataExtractor.RpcCall(rpcConfig, "getdifficulty"); diff --git a/CheckerApi/Models/DTO/VirtualCheckpointDTO.cs b/CheckerApi/Models/DTO/VirtualCheckpointDTO.cs new file mode 100644 index 0000000..752dd7c --- /dev/null +++ b/CheckerApi/Models/DTO/VirtualCheckpointDTO.cs @@ -0,0 +1,8 @@ +namespace CheckerApi.Models.DTO +{ + public class VirtualCheckpointDTO + { + public int Height { get; set; } + public string Hash { get; set; } + } +} diff --git a/CheckerApi/Models/Rpc/GetChainTipsResult.cs b/CheckerApi/Models/Rpc/GetChainTipsResult.cs new file mode 100644 index 0000000..f6fcf4d --- /dev/null +++ b/CheckerApi/Models/Rpc/GetChainTipsResult.cs @@ -0,0 +1,25 @@ +using Newtonsoft.Json; + +namespace CheckerApi.Models.Rpc +{ + public class GetChainTipsResult + { + [JsonProperty("result")] + public ChainTip[] Result { get; set; } + } + + public class ChainTip + { + [JsonProperty("height")] + public int Height { get; set; } + + [JsonProperty("hash")] + public string Hash { get; set; } + + [JsonProperty("branchlen")] + public int BranchLen { get; set; } + + [JsonProperty("status")] + public string Status { get; set; } + } +} diff --git a/CheckerApi/Models/Rpc/RpcBlockInfo.cs b/CheckerApi/Models/Rpc/RpcBlockInfoBase.cs similarity index 52% rename from CheckerApi/Models/Rpc/RpcBlockInfo.cs rename to CheckerApi/Models/Rpc/RpcBlockInfoBase.cs index 7523be6..37a7137 100644 --- a/CheckerApi/Models/Rpc/RpcBlockInfo.cs +++ b/CheckerApi/Models/Rpc/RpcBlockInfoBase.cs @@ -2,7 +2,7 @@ namespace CheckerApi.Models.Rpc { - public class RpcBlockInfo + public class RpcBlockInfoBase { [JsonProperty("hash")] public string Hash { get; set; } @@ -15,6 +15,24 @@ public class RpcBlockInfo [JsonProperty("previousblockhash")] public string PreviousBlockHash { get; set; } + + [JsonProperty("chainwork")] + public string ChainWork { get; set; } + + [JsonProperty("confirmations")] + public int Confirmations { get; set; } + } + + public class RpcBlockInfo : RpcBlockInfoBase + { + [JsonProperty("tx")] + public string[] Tx { get; set; } + } + + public class RpcBlockInfoVerbose : RpcBlockInfoBase + { + [JsonProperty("tx")] + public Transaction[] Tx { get; set; } } public class RpcBlockResult diff --git a/CheckerApi/Models/Rpc/Transaction.cs b/CheckerApi/Models/Rpc/Transaction.cs new file mode 100644 index 0000000..6fed36e --- /dev/null +++ b/CheckerApi/Models/Rpc/Transaction.cs @@ -0,0 +1,10 @@ +using Newtonsoft.Json; + +namespace CheckerApi.Models.Rpc +{ + public class Transaction + { + [JsonProperty("txid")] + public string TxId { get; set; } + } +} diff --git a/CheckerApi/Program.cs b/CheckerApi/Program.cs index 9a55375..e9eae19 100644 --- a/CheckerApi/Program.cs +++ b/CheckerApi/Program.cs @@ -106,6 +106,19 @@ public static void Main(string[] args) startAt: DateTimeOffset.UtcNow.AddSeconds(2) ); } + + var forkWatchEnabled = config.GetValue("ForkWatch:Enable"); + if (forkWatchEnabled) + { + scheduler.AddJob( + host, + tb => tb.WithSimpleSchedule(x => x + .WithIntervalInSeconds(10) + .RepeatForever() + ), + startAt: DateTimeOffset.UtcNow.AddSeconds(3) + ); + } } }) .Run(); diff --git a/CheckerApi/Services/DataExtractorService.cs b/CheckerApi/Services/DataExtractorService.cs index 9ed9e6e..bcef7f2 100644 --- a/CheckerApi/Services/DataExtractorService.cs +++ b/CheckerApi/Services/DataExtractorService.cs @@ -3,6 +3,7 @@ using CheckerApi.Models.Rpc; using CheckerApi.Services.Interfaces; using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using RestSharp; using System; using System.Collections.Generic; @@ -41,7 +42,7 @@ public Result> GetData(string url, string req, string patter return Result>.Ok(match.Groups.Select(g => g.Value)); } - public Result RpcCall(RpcConfig config, string method, params string[] parameters) + public Result RpcCall(RpcConfig config, string method, params object[] parameters) { var result = this.RpcCall(config, method, parameters); if (result.IsSuccess()) @@ -52,7 +53,7 @@ public Result RpcCall(RpcConfig config, string method, params string[] p return Result.Fail(result.Messages.ToArray()); } - public Result RpcCall(RpcConfig config, string method, params string[] parameters) where T : class + public Result RpcCall(RpcConfig config, string method, params object[] parameters) where T : class { var client = new RestClient($"{config.Url}:{config.Port}"); var request = new RestRequest(string.Empty, Method.POST) @@ -60,8 +61,8 @@ public Result RpcCall(RpcConfig config, string method, params string[] par Credentials = config.Credentials }; - var pars = string.Join(",", parameters.Select(p => $"\"{p}\"")); - request.AddParameter("text/xml", $"{{\"jsonrpc\":\"1.0\",\"id\":\"alert-bot\",\"method\":\"{method}\",\"params\":[{pars}]}}", ParameterType.RequestBody); + var pars = ObjectToJArray(parameters).ToString(); + request.AddParameter("text/xml", $"{{\"jsonrpc\":\"1.0\",\"id\":\"alert-bot\",\"method\":\"{method}\",\"params\":{pars}}}", ParameterType.RequestBody); var response = client.Execute(request); if (response.StatusCode != HttpStatusCode.OK) @@ -79,5 +80,30 @@ public Result RpcCall(RpcConfig config, string method, params string[] par return Result.Fail($"RPC serialization fail '{method}' at '{config.Url}/{config.Port}'", $"object type: '{nameof(T)}'", $"ex: '{ex}'"); } } + + private JArray ObjectToJArray(object[] objs) + { + var a = new JArray(); + foreach (var o in objs) + { + JValue val; + if (o is int) + { + val = new JValue((int)o); + } + else if (o is string) + { + val = new JValue((string)o); + } + else + { + val = new JValue(o.ToString()); + } + + a.Add(val); + } + + return a; + } } } diff --git a/CheckerApi/Services/ForkWatchService.cs b/CheckerApi/Services/ForkWatchService.cs new file mode 100644 index 0000000..ecd0030 --- /dev/null +++ b/CheckerApi/Services/ForkWatchService.cs @@ -0,0 +1,167 @@ +using CheckerApi.Extensions; +using CheckerApi.Models; +using CheckerApi.Models.Config; +using CheckerApi.Models.DTO; +using CheckerApi.Models.Rpc; +using CheckerApi.Services.Interfaces; +using CheckerApi.Utils; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace CheckerApi.Services +{ + public class ForkWatchService: IForkWatchService + { + private const int VIRTUALFINALIZEBLOCKS = 3; + + private IDataExtractorService _dataExtractor; + private ILogger _logger; + private IMemoryCache _cache; + private INotificationManager _notificationManager; + + public ForkWatchService(IDataExtractorService dataExtractor, IMemoryCache cache, INotificationManager notificationManager) + { + _dataExtractor = dataExtractor; + _cache = cache; + _notificationManager = notificationManager; + } + + public void Execute(RpcConfig rpcConfig) + { + // Compare chain tips + var lastSeenTips = _cache.GetOrCreate(Constants.LastSeenTipKey, entry => null); + ChainTip[] tips = null; + Handle(_dataExtractor.RpcCall(rpcConfig, "getchaintips"), r => + { + tips = r.Result; + var shouldSend = true; + var desc = "ForkWatch started"; + if (lastSeenTips != null) + { + (shouldSend, desc) = DiffTip(lastSeenTips, tips); + } + + if (shouldSend) + { + var url = string.Empty; // TODO: upload to pastbin and get url + var message = $"{desc}\n{url}"; + _notificationManager.TriggerHook(message); + } + + _cache.Set(Constants.LastSeenTipKey, tips); + }); + + if (tips == null || tips.Length == 0) + { + _logger.LogWarning("ForkWatch: Got malformed tip from RPC"); + } + + if (lastSeenTips != null && lastSeenTips[0].Hash == tips[0].Hash) + { + // Short-circurit: no new block found + return; + } + + // Check virtual checkpoint rolled-back + var lastCheckpoint = _cache.GetOrCreate(Constants.VirtualCheckpointKey, entry => null); + if (lastCheckpoint != null) + { + bool foundReorg = false; + + // Check if the checkpoint is still in the main chain + Handle(_dataExtractor.RpcCall(rpcConfig, "getblockhash", lastCheckpoint.Height), r => + { + var hash = r.Result; + foundReorg = hash != lastCheckpoint.Hash; + var message = $"ForkWatch: Virtual checkpoint {lastCheckpoint.Hash} at height {lastCheckpoint.Height} replaced by {hash}"; + _notificationManager.TriggerHook(message); + }); + + // TODO: if foundReorg, move to "PREPARE" + } + + // Update virtual checkpoint + var height = tips[0].Height; + var toFinalize = height - VIRTUALFINALIZEBLOCKS; + + _logger.LogWarning("getblockhash({0})", toFinalize); + + Handle(_dataExtractor.RpcCall(rpcConfig, "getblockhash", toFinalize), r => + { + var hash = r.Result; + _cache.Set(Constants.VirtualCheckpointKey, new VirtualCheckpointDTO() + { + Hash = hash, + Height = toFinalize + }); + + _notificationManager.TriggerHook($"ForkWatch: new checkpoint {hash} at {toFinalize} ({-VIRTUALFINALIZEBLOCKS}) tip: {height}"); + }); + } + + private void Handle(Result result, Action action) + { + if (result.HasFailed()) + { + _logger.LogError(result.Messages.ToCommaSeparated()); + return; + } + + action(result.Value); + } + + private (bool found, string payload) DiffTip(IEnumerable a, IEnumerable b) + { + var hashesA = a.Where(t => t.Status != "active").Select(t => t.Hash).ToHashSet(); + var dictB = b.Where(t => t.Status != "active" && t.Status != "headers-only").ToDictionary(k => k.Hash, v => v); + var hashesB = dictB.Select(t => t.Key).ToHashSet(); + var added = new HashSet(hashesB); + added.ExceptWith(hashesA); + + if (!added.Any()) + { + return (false, null); + } + + var branches = SimpleJson.SimpleJson.SerializeObject( + (from h in added select dictB[h]).ToList()); + return (true, $"New branch: {branches}"); + } + + // n == 0: backtrace until meet main chain + // n > 0: backtrace n blocks + private List BacktraceBlocks(RpcConfig rpcConfig, string tipHash, int n = 0) + { + var blocks = new List(); + var h = tipHash; + while (true) + { + var blockInfoResult = _dataExtractor.RpcCall(rpcConfig, "getblock", h); + if (blockInfoResult.HasFailed()) + { + _logger.LogError(blockInfoResult.Messages.ToCommaSeparated()); + break; + } + + var blockInfo = blockInfoResult.Value; + _logger.LogInformation("Backtrace: {0} {1}", h, blockInfo.Confirmations); + if (n == 0 && blockInfo.Confirmations >= 0) + { + break; + } + + blocks.Append(blockInfo); + h = blockInfo.PreviousBlockHash; + if (blocks.Count == n) + { + break; + } + } + + return blocks; + } + } +} diff --git a/CheckerApi/Services/Interfaces/IDataExtractorService.cs b/CheckerApi/Services/Interfaces/IDataExtractorService.cs index af76e90..5d65513 100644 --- a/CheckerApi/Services/Interfaces/IDataExtractorService.cs +++ b/CheckerApi/Services/Interfaces/IDataExtractorService.cs @@ -7,7 +7,7 @@ namespace CheckerApi.Services.Interfaces public interface IDataExtractorService { Result> GetData(string url, string req, string pattern); - Result RpcCall(RpcConfig config, string method, params string[] parameters) where T : class; - Result RpcCall(RpcConfig config, string method, params string[] parameters); + Result RpcCall(RpcConfig config, string method, params object[] parameters) where T : class; + Result RpcCall(RpcConfig config, string method, params object[] parameters); } } diff --git a/CheckerApi/Services/Interfaces/IForkWatchService.cs b/CheckerApi/Services/Interfaces/IForkWatchService.cs new file mode 100644 index 0000000..5707ea5 --- /dev/null +++ b/CheckerApi/Services/Interfaces/IForkWatchService.cs @@ -0,0 +1,9 @@ +using CheckerApi.Models.Config; + +namespace CheckerApi.Services.Interfaces +{ + public interface IForkWatchService + { + void Execute(RpcConfig rpcConfig); + } +} diff --git a/CheckerApi/Services/SyncService.cs b/CheckerApi/Services/SyncService.cs index eccdeae..8e9955d 100644 --- a/CheckerApi/Services/SyncService.cs +++ b/CheckerApi/Services/SyncService.cs @@ -140,7 +140,8 @@ public IEnumerable> GetTotalOrders(bool enableAudit) page++; List orders = data?.Orders?.Select(o => CreateDTO(o, location)).ToList() ?? new List(); pagedList.AddRange(orders); - } while (true); + } + while (true); totalOrders.Add(pagedList); if (enableAudit) @@ -165,7 +166,8 @@ private BidEntry CreateDTO(BidDTO order, string location) else if (int.TryParse(location, out int l)) { loc = l; - } else + } + else { loc = 0; } diff --git a/CheckerApi/Startup.cs b/CheckerApi/Startup.cs index da0451b..e1f9894 100644 --- a/CheckerApi/Startup.cs +++ b/CheckerApi/Startup.cs @@ -42,6 +42,7 @@ public void ConfigureServices(IServiceCollection services) services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); services.AddMvc(); } diff --git a/CheckerApi/Utils/Constants.cs b/CheckerApi/Utils/Constants.cs index aab5265..2ba89f0 100644 --- a/CheckerApi/Utils/Constants.cs +++ b/CheckerApi/Utils/Constants.cs @@ -6,5 +6,7 @@ public static class Constants public static string BtcBtgPriceKey { get; } = "BtcBtgPriceKey"; public static string DifficultyKey { get; } = "DifficultyKey"; public static string BlocksInfoKey { get; } = "BlocksInfoKey"; + public static string LastSeenTipKey { get; } = "LastSeenTipKey"; + public static string VirtualCheckpointKey { get; } = "VirtualCheckpointKey"; } }