Merge branch 'feature/multi-codex-folder-saver'

Author: benbierens

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -62,7 +62,6 @@ public decimal GetEthBalance(string address)
             log.Debug(typeof(TFunction).ToString());
             var handler = web3.Eth.GetContractQueryHandler<TFunction>();
             var result = Time.Wait(handler.QueryRawAsync(contractAddress, function, new BlockParameter(blockNumber)));
-            var aaaa = 0;
         }
 
         public string SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()

Tools/AutoClient/App.cs

@@ -20,18 +20,13 @@ public App(Configuration config)
                 new FileLog(Path.Combine(config.LogPath, "performance")),
                 new ConsoleLog()
             ));
-
-            var httpFactory = new HttpFactory(Log, new AutoClientWebTimeSet());
-
-            CodexNodeFactory = new CodexNodeFactory(log: Log, httpFactory: httpFactory, dataDir: Config.DataPath);
         }
 
         public Configuration Config { get; }
         public ILog Log { get; }
         public IFileGenerator Generator { get; }
         public CancellationTokenSource Cts { get; } = new CancellationTokenSource();
         public Performance Performance { get; }
-        public CodexNodeFactory CodexNodeFactory { get; }
 
         private IFileGenerator CreateGenerator()
         {

Tools/AutoClient/AutomaticPurchaser.cs

@@ -6,7 +6,14 @@ public class Configuration
     {
         [Uniform("codex-endpoints", "ce", "CODEXENDPOINTS", false, "Codex endpoints. Semi-colon separated. (default 'http://localhost:8080')")]
         public string CodexEndpoints { get; set; } =
-            "http://localhost:8080";
+            "http://localhost:8080" + ";" +
+            "http://localhost:8081" + ";" +
+            "http://localhost:8082" + ";" +
+            "http://localhost:8083" + ";" +
+            "http://localhost:8084" + ";" +
+            "http://localhost:8085" + ";" +
+            "http://localhost:8086" + ";" +
+            "http://localhost:8087";
 
         [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
         public string DataPath { get; set; } = "datapath";
@@ -21,7 +28,7 @@ public class Configuration
         // Cluster nodes configured for max 7-day storage.
 
         [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15 minutes)")]
-        public int ContractExpiryMinutes { get; set; } = 60;
+        public int ContractExpiryMinutes { get; set; } = 15;
 
         [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 10)")]
         public int NumHosts { get; set; } = 5;
@@ -41,8 +48,16 @@ public class Configuration
         [Uniform("folderToStore", "fts", "FOLDERTOSTORE", false, "When set, autoclient will attempt to upload and purchase storage for every non-JSON file in the provided folder.")]
         public string FolderToStore { get; set; } = "/data/EthereumMainnetPreMergeEraFiles";
 
-        [Uniform("ethAddressFile", "eaf", "ETHADDRESSFILE", false, "File with eth address used by codex node. Used for balance checking if geth/contracts information is provided.")]
-        public string EthAddressFile { get; set; } = "/root/codex-testnet-starter/scripts/eth.address";
+        [Uniform("ethAddressFile", "eaf", "ETHADDRESSFILE", false, "File(s) with eth address used by codex node. Used for balance checking if geth/contracts information is provided. Semi-colon separated.")]
+        public string EthAddressFile { get; set; } = 
+            "/root/codex-testnet-starter/scripts/eth.address" + ";" +
+            "/root/codex-testnet-starter/scripts/eth_2.address" + ";" +
+            "/root/codex-testnet-starter/scripts/eth_3.address" + ";" +
+            "/root/codex-testnet-starter/scripts/eth_4.address" + ";" +
+            "/root/codex-testnet-starter/scripts/eth_5.address" + ";" +
+            "/root/codex-testnet-starter/scripts/eth_6.address" + ";" +
+            "/root/codex-testnet-starter/scripts/eth_7.address" + ";" +
+            "/root/codex-testnet-starter/scripts/eth_8.address";
 
         public string LogPath
         {

Tools/AutoClient/Configuration.cs

@@ -0,0 +1,131 @@
+using Logging;
+
+namespace AutoClient
+{
+    public class LoadBalancer
+    {
+        private readonly List<Cdx> instances;
+        private readonly object instanceLock = new object();
+
+        private class Cdx
+        {
+            private readonly ILog log;
+            private readonly CodexWrapper instance;
+            private readonly List<Action<CodexWrapper>> queue = new List<Action<CodexWrapper>>();
+            private readonly object queueLock = new object();
+            private bool running = true;
+            private Task worker = Task.CompletedTask;
+
+            public Cdx(App app, CodexWrapper instance)
+            {
+                Id = instance.Node.GetName();
+                log = new LogPrefixer(app.Log, $"[Queue-{Id}]");
+                this.instance = instance;
+            }
+
+            public string Id { get; }
+
+            public void Start()
+            {
+                worker = Task.Run(Worker);
+            }
+
+            public void Stop()
+            {
+                running = false;
+                worker.Wait();
+            }
+
+            public void CheckErrors()
+            {
+                if (worker.IsFaulted) throw worker.Exception;
+            }
+
+            public void Queue(Action<CodexWrapper> action)
+            {
+                if (queue.Count > 2) log.Log("Queue full. Waiting...");
+                while (queue.Count > 2)
+                {
+                    Thread.Sleep(TimeSpan.FromSeconds(5.0));
+                }
+
+                lock (queueLock)
+                {
+                    queue.Add(action);
+                }
+            }
+
+            private void Worker()
+            {
+                try
+                {
+                    while (running)
+                    {
+                        while (queue.Count == 0) Thread.Sleep(TimeSpan.FromSeconds(5.0));
+
+                        Action<CodexWrapper> action = w => { };
+                        lock (queueLock)
+                        {
+                            action = queue[0];
+                            queue.RemoveAt(0);
+                        }
+
+                        action(instance);
+                    }
+                }
+                catch (Exception ex)
+                {
+                    log.Error("Exception in worker: " + ex);
+                    throw;
+                }
+            }
+        }
+
+        public LoadBalancer(App app, CodexWrapper[] instances)
+        {
+            this.instances = instances.Select(i => new Cdx(app, i)).ToList();
+        }
+
+        public void Start()
+        {
+            foreach (var i in instances) i.Start();
+        }
+
+        public void Stop()
+        {
+            foreach (var i in instances) i.Stop();
+        }
+
+        public void DispatchOnCodex(Action<CodexWrapper> action)
+        {
+            lock (instanceLock)
+            {
+                var i = instances.First();
+                instances.RemoveAt(0);
+                instances.Add(i);
+
+                i.Queue(action);
+            }
+        }
+
+        public void DispatchOnSpecificCodex(Action<CodexWrapper> action, string id)
+        {
+            lock (instanceLock)
+            {
+                var i = instances.Single(a => a.Id == id);
+                instances.Remove(i);
+                instances.Add(i);
+
+                i.Queue(action);
+            }
+        }
+
+        public void CheckErrors()
+        {
+            lock (instanceLock)
+            {
+                foreach (var i in instances) i.CheckErrors();
+            }
+        }
+    }
+}