Improving check command responsiveness

Author: benbierens

Tools/BiblioTech/CodexChecking/CheckRepo.cs

@@ -1,37 +1,53 @@
-using Newtonsoft.Json;
+using Logging;
+using Newtonsoft.Json;
+using Utils;
 
 namespace BiblioTech.CodexChecking
 {
     public class CheckRepo
     {
         private const string modelFilename = "model.json";
+        private readonly ILog log;
         private readonly Configuration config;
         private readonly object _lock = new object();
         private CheckRepoModel? model = null;
 
-        public CheckRepo(Configuration config)
+        public CheckRepo(ILog log, Configuration config)
         {
+            this.log = log;
             this.config = config;
         }
 
         public CheckReport GetOrCreate(ulong userId)
         {
             lock (_lock)
             {
-                if (model == null) LoadModel();
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                try
+                {
+                    if (model == null) LoadModel();
 
-                var existing = model.Reports.SingleOrDefault(r  => r.UserId == userId);
-                if (existing == null)
+                    var existing = model.Reports.SingleOrDefault(r => r.UserId == userId);
+                    if (existing == null)
+                    {
+                        var newEntry = new CheckReport
+                        {
+                            UserId = userId,
+                        };
+                        model.Reports.Add(newEntry);
+                        SaveChanges();
+                        return newEntry;
+                    }
+                    return existing;
+                }
+                finally
                 {
-                    var newEntry = new CheckReport
+                    var elapsed = sw.Elapsed;
+                    if (elapsed > TimeSpan.FromMilliseconds(500))
                     {
-                        UserId = userId,
-                    };
-                    model.Reports.Add(newEntry);
-                    SaveChanges();
-                    return newEntry;
+                        log.Log($"Warning {nameof(GetOrCreate)} took {Time.FormatDuration(elapsed)}");
+                    }
                 }
-                return existing;
             }
         }
 

Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs

@@ -43,7 +43,7 @@ public async Task StartDownloadCheck(ICheckResponseHandler handler, ulong userId
                 repo.SaveChanges();
             }
 
-            var cid = UploadData(check.UniqueData);
+            var cid = await UploadData(check.UniqueData);
             await handler.GiveCidToUser(cid);
         }
 
@@ -87,7 +87,7 @@ public async Task VerifyUploadCheck(ICheckResponseHandler handler, ulong userId,
                 return;
             }
 
-            var manifest = GetManifest(receivedCid);
+            var manifest = await GetManifest(receivedCid);
             if (manifest == null)
             {
                 await handler.CouldNotDownloadCid();
@@ -96,7 +96,7 @@ public async Task VerifyUploadCheck(ICheckResponseHandler handler, ulong userId,
 
             if (IsManifestLengthCompatible(handler, check, manifest))
             {
-                if (IsContentCorrect(handler, check, receivedCid))
+                if (await IsContentCorrect(handler, check, receivedCid))
                 {
                     await CheckNowCompleted(handler, check, userId, "UploadCheck");
                     return;
@@ -120,7 +120,7 @@ private bool IsUniqueDataStale(TransferCheck check)
                 check.CompletedUtc < expiry;
         }
 
-        private string UploadData(string uniqueData)
+        private async Task<string> UploadData(string uniqueData)
         {
             var filePath = Path.Combine(config.ChecksDataPath, Guid.NewGuid().ToString());
 
@@ -129,7 +129,7 @@ private string UploadData(string uniqueData)
                 File.WriteAllText(filePath, uniqueData);
                 var file = new TrackedFile(log, filePath, "checkData");
 
-                return codexWrapper.OnCodex(node =>
+                return await codexWrapper.OnCodex(node =>
                 {
                     return node.UploadFile(file).Id;
                 });
@@ -145,11 +145,11 @@ private string UploadData(string uniqueData)
             }
         }
 
-        private Manifest? GetManifest(string receivedCid)
+        private async Task<Manifest?> GetManifest(string receivedCid)
         {
             try
             {
-                return codexWrapper.OnCodex(node =>
+                return await codexWrapper.OnCodex(node =>
                 {
                     return node.DownloadManifestOnly(new ContentId(receivedCid)).Manifest;
                 });
@@ -172,11 +172,11 @@ private bool IsManifestLengthCompatible(ICheckResponseHandler handler, TransferC
                 manifestLength < (dataLength + 1);
         }
 
-        private bool IsContentCorrect(ICheckResponseHandler handler, TransferCheck check, string receivedCid)
+        private async Task<bool> IsContentCorrect(ICheckResponseHandler handler, TransferCheck check, string receivedCid)
         {
             try
             {
-                var content = codexWrapper.OnCodex(node =>
+                var content = await codexWrapper.OnCodex(node =>
                 {
                     var file = node.DownloadContent(new ContentId(receivedCid));
                     if (file == null) return string.Empty;

Tools/BiblioTech/CodexChecking/CodexWrapper.cs

@@ -23,20 +23,26 @@ public CodexWrapper(ILog log, Configuration config)
             factory = new CodexNodeFactory(log, httpFactory, dataDir: config.DataPath);
         }
 
-        public void OnCodex(Action<ICodexNode> action)
+        public async Task OnCodex(Action<ICodexNode> action)
         {
-            lock (codexLock)
+            await Task.Run(() =>
             {
-                action(Get());
-            }
+                lock (codexLock)
+                {
+                    action(Get());
+                }
+            });
         }
 
-        public T OnCodex<T>(Func<ICodexNode, T> func)
+        public async Task<T> OnCodex<T>(Func<ICodexNode, T> func)
         {
-            lock (codexLock)
+            return await Task<T>.Run(() =>
             {
-                return func(Get());
-            }
+                lock (codexLock)
+                {
+                    return func(Get());
+                }
+            });            
         }
 
         private ICodexNode Get()
@@ -76,10 +82,28 @@ private HttpFactory CreateHttpFactory()
             var tokens = config.CodexEndpointAuth.Split(':');
             if (tokens.Length != 2) throw new Exception("Expected '<username>:<password>' in CodexEndpointAuth parameter.");
 
-            return new HttpFactory(log, onClientCreated: client =>
+            return new HttpFactory(log, new SnappyTimeSet(), onClientCreated: client =>
             {
                 client.SetBasicAuthentication(tokens[0], tokens[1]);
             });
         }
+
+        public class SnappyTimeSet : IWebCallTimeSet
+        {
+            public TimeSpan HttpCallRetryDelay()
+            {
+                return TimeSpan.FromSeconds(1.0);
+            }
+
+            public TimeSpan HttpCallTimeout()
+            {
+                return TimeSpan.FromSeconds(3.0);
+            }
+
+            public TimeSpan HttpRetryTimeout()
+            {
+                return TimeSpan.FromSeconds(12.0);
+            }
+        }
     }
 }

Tools/BiblioTech/Commands/CheckDownloadCommand.cs

@@ -18,7 +18,7 @@ public CheckDownloadCommand(CodexTwoWayChecker checker)
         }
 
         public override string Name => "checkdownload";
-        public override string StartingMessage => RandomBusyMessage.Get();
+        public override string StartingMessage => "Connecting to the testnet... Please be patient... " + RandomBusyMessage.Get();
         public override string Description => "Checks the download connectivity of your Codex node.";
         public override CommandOption[] Options => [contentOption];
 

Tools/BiblioTech/Commands/CheckUploadCommand.cs

@@ -18,7 +18,7 @@ public CheckUploadCommand(CodexTwoWayChecker checker)
         }
 
         public override string Name => "checkupload";
-        public override string StartingMessage => RandomBusyMessage.Get();
+        public override string StartingMessage => "Connecting to the testnet... Please be patient... " + RandomBusyMessage.Get();
         public override string Description => "Checks the upload connectivity of your Codex node.";
         public override CommandOption[] Options => [cidOption];
 

Tools/BiblioTech/Program.cs

@@ -88,7 +88,7 @@ private async Task StartDiscordBot()
             client = new DiscordSocketClient();
             client.Log += ClientLog;
 
-            var checkRepo = new CheckRepo(Config);
+            var checkRepo = new CheckRepo(Log, Config);
             var codexWrapper = new CodexWrapper(Log, Config);
             var checker = new CodexTwoWayChecker(Log, Config, checkRepo, codexWrapper);
             var notifyCommand = new NotifyCommand();