Applies purchase-paramters type to ensure slot sizes

Author: benbierens

Framework/Utils/ByteSize.cs

@@ -25,6 +25,21 @@ public ByteSize Multiply(double factor)
             return new ByteSize(Convert.ToInt64(result));
         }
 
+        public int DivUp(ByteSize div)
+        {
+            var d = div.SizeInBytes;
+            var remaining = SizeInBytes;
+            var result = 0;
+            while (remaining > d)
+            {
+                remaining -= d;
+                result++;
+            }
+
+            if (remaining > 0) result++;
+            return result;
+        }
+
         public override bool Equals(object? obj)
         {
             return obj is ByteSize size && SizeInBytes == size.SizeInBytes;

ProjectPlugins/CodexPlugin/CodexDockerImage.cs

@@ -2,7 +2,7 @@
 {
     public class CodexDockerImage
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-28a83db-dist-tests";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:0.2.3-dist-tests";
 
         public static string Override { get; set; } = string.Empty;
 

Tests/CodexReleaseTests/MarketTests/FinishTest.cs

@@ -12,19 +12,16 @@ public class FinishTest : MarketplaceAutoBootstrapDistTest
         public FinishTest(int hosts, int slots, int tolerance)
         {
             this.hosts = hosts;
-            this.slots = slots;
-            this.tolerance = tolerance;
+            purchaseParams = new PurchaseParams(slots, tolerance, uploadFilesize: 10.MB());
         }
 
-        private const int FilesizeMb = 10;
         private readonly TestToken pricePerBytePerSecond = 10.TstWei();
         private readonly int hosts;
-        private readonly int slots;
-        private readonly int tolerance;
+        private readonly PurchaseParams purchaseParams;
 
         protected override int NumberOfHosts => hosts;
         protected override int NumberOfClients => 1;
-        protected override ByteSize HostAvailabilitySize => (5 * FilesizeMb).MB();
+        protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(5.1);
         protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration() * 12;
 
         [Test]
@@ -53,14 +50,14 @@ public void Finish(
 
         private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
         {
-            var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB()));
+            var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize));
             var config = GetContracts().Deployment.Config;
             return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
             {
                 Duration = GetContractDuration(),
                 Expiry = GetContractExpiry(),
-                MinRequiredNumberOfNodes = (uint)slots,
-                NodeFailureTolerance = (uint)tolerance,
+                MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes,
+                NodeFailureTolerance = (uint)purchaseParams.Tolerance,
                 PricePerBytePerSecond = pricePerBytePerSecond,
                 ProofProbability = 20,
                 CollateralPerByte = 100.TstWei()

Tests/CodexReleaseTests/MarketTests/RepairTest.cs

@@ -11,35 +11,22 @@ public class RepairTest : MarketplaceAutoBootstrapDistTest
     {
         #region Setup
 
-        private readonly ByteSize Filesize;
-        private readonly uint Slots;
-        private readonly uint Tolerance;
-        private readonly ByteSize EncodedFilesize;
-        private readonly ByteSize SlotSize;
+        private readonly PurchaseParams purchaseParams = new PurchaseParams(
+            nodes: 4,
+            tolerance: 2,
+            uploadFilesize: 32.MB()
+        );
 
         public RepairTest()
         {
-            Filesize = 32.MB();
-            Slots = 4;
-            Tolerance = 2;
-
-            EncodedFilesize = new ByteSize(Filesize.SizeInBytes * (Slots / Tolerance));
-            SlotSize = new ByteSize(EncodedFilesize.SizeInBytes / Slots);
-            Assert.That(IsPowerOfTwo(SlotSize));
-            Assert.That(Slots, Is.LessThan(NumberOfHosts));
+            Assert.That(purchaseParams.Nodes, Is.LessThan(NumberOfHosts));
         }
 
         protected override int NumberOfHosts => 5;
         protected override int NumberOfClients => 1;
-        protected override ByteSize HostAvailabilitySize => SlotSize.Multiply(1.1); // Each host can hold 1 slot.
+        protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(1.1); // Each host can hold 1 slot.
         protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(5.0);
 
-        private static bool IsPowerOfTwo(ByteSize size)
-        {
-            var x = size.SizeInBytes;
-            return (x != 0) && ((x & (x - 1)) == 0);
-        }
-
         #endregion
 
         [Ignore("Test is ready. Waiting for repair implementation. " +
@@ -173,14 +160,14 @@ private SlotFill GetSlotFillByOldestHost(List<ICodexNode> hosts)
 
         private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
         {
-            var cid = client.UploadFile(GenerateTestFile(Filesize));
+            var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize));
             var config = GetContracts().Deployment.Config;
             return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
             {
                 Duration = HostAvailabilityMaxDuration / 2,
                 Expiry = TimeSpan.FromMinutes(10.0),
-                MinRequiredNumberOfNodes = Slots,
-                NodeFailureTolerance = Tolerance,
+                MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes,
+                NodeFailureTolerance = (uint)purchaseParams.Tolerance,
                 PricePerBytePerSecond = 10.TstWei(),
                 ProofProbability = 1, // One proof every period. Free slot as quickly as possible.
                 CollateralPerByte = 1.TstWei()

Tests/CodexReleaseTests/MarketTests/SequentialContracts.cs

@@ -12,18 +12,15 @@ public class SequentialContracts : MarketplaceAutoBootstrapDistTest
         public SequentialContracts(int hosts, int slots, int tolerance)
         {
             this.hosts = hosts;
-            this.slots = slots;
-            this.tolerance = tolerance;
+            purchaseParams = new PurchaseParams(slots, tolerance, 10.MB());
         }
 
-        private const int FilesizeMb = 10;
         private readonly int hosts;
-        private readonly int slots;
-        private readonly int tolerance;
+        private readonly PurchaseParams purchaseParams;
 
         protected override int NumberOfHosts => hosts;
         protected override int NumberOfClients => 8;
-        protected override ByteSize HostAvailabilitySize => (1000 * FilesizeMb).MB();
+        protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(100.0);
         protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration() * 12;
         private readonly TestToken pricePerBytePerSecond = 10.TstWei();
 
@@ -80,14 +77,14 @@ private TResult[] All<T, TResult>(T[] items, Func<T, TResult> action)
 
         private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
         {
-            var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB()));
+            var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize));
             var config = GetContracts().Deployment.Config;
             return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
             {
                 Duration = GetContractDuration(),
                 Expiry = GetContractExpiry(),
-                MinRequiredNumberOfNodes = (uint)slots,
-                NodeFailureTolerance = (uint)tolerance,
+                MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes,
+                NodeFailureTolerance = (uint)purchaseParams.Tolerance,
                 PricePerBytePerSecond = pricePerBytePerSecond,
                 ProofProbability = 10000,
                 CollateralPerByte = 1.TstWei()
@@ -107,7 +104,7 @@ private TimeSpan GetContractDuration()
         private TimeSpan Get8TimesConfiguredPeriodDuration()
         {
             var config = GetContracts().Deployment.Config;
-            return TimeSpan.FromSeconds(((double)config.Proofs.Period) * 8.0);
+            return TimeSpan.FromSeconds(config.Proofs.Period * 8.0);
         }
     }
 }

Tests/CodexReleaseTests/MarketTests/StartTest.cs

@@ -8,12 +8,16 @@ namespace CodexReleaseTests.MarketTests
     [TestFixture]
     public class StartTest : MarketplaceAutoBootstrapDistTest
     {
-        private const int FilesizeMb = 10;
+        private readonly PurchaseParams purchaseParams = new PurchaseParams(
+            nodes: 3,
+            tolerance: 1,
+            uploadFilesize: 10.MB()
+        );
         private readonly TestToken pricePerBytePerSecond = 10.TstWei();
 
         protected override int NumberOfHosts => 5;
         protected override int NumberOfClients => 1;
-        protected override ByteSize HostAvailabilitySize => (5 * FilesizeMb).MB();
+        protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(10.0);
         protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration() * 12;
 
         [Test]
@@ -36,14 +40,14 @@ public void Start(
 
         private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
         {
-            var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB()));
+            var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize));
             var config = GetContracts().Deployment.Config;
             return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
             {
                 Duration = GetContractDuration(),
                 Expiry = GetContractExpiry(),
-                MinRequiredNumberOfNodes = 3,
-                NodeFailureTolerance = 1,
+                MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes,
+                NodeFailureTolerance = (uint)purchaseParams.Tolerance,
                 PricePerBytePerSecond = pricePerBytePerSecond,
                 ProofProbability = 20,
                 CollateralPerByte = 100.TstWei()

Tests/CodexReleaseTests/Utils/PurchaseParams.cs

@@ -0,0 +1,67 @@
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.Utils
+{
+    public class PurchaseParams
+    {
+        private readonly ByteSize blockSize = 64.KB();
+
+        public PurchaseParams(int nodes, int tolerance, ByteSize uploadFilesize)
+        {
+            Nodes = nodes;
+            Tolerance = tolerance;
+            UploadFilesize = uploadFilesize;
+
+            EncodedDatasetSize = CalculateEncodedDatasetSize();
+            SlotSize = CalculateSlotSize();
+
+            Assert.That(IsPowerOfTwo(SlotSize));
+        }
+
+        public int Nodes { get; }
+        public int Tolerance { get; }
+        public ByteSize UploadFilesize { get; }
+        public ByteSize EncodedDatasetSize { get; }
+        public ByteSize SlotSize { get; }
+
+        private ByteSize CalculateSlotSize()
+        {
+            // encoded dataset is divided over the nodes.
+            // then each slot is rounded up to the nearest power-of-two blocks.
+            var numBlocks = EncodedDatasetSize.DivUp(blockSize);
+            var numSlotBlocks = 1 + ((numBlocks - 1) / Nodes); // round-up div.
+
+            // Next power of two:
+            var numSlotBlocksPow2 = NextPowerOf2(numSlotBlocks);
+            return new ByteSize(blockSize.SizeInBytes * numSlotBlocksPow2);
+        }
+
+        private ByteSize CalculateEncodedDatasetSize()
+        {
+            var numBlocks = UploadFilesize.DivUp(blockSize);
+
+            var ecK = Nodes - Tolerance;
+            var ecM = Tolerance;
+
+            // for each K blocks, we generate M parity blocks
+            var numParityBlocks = (numBlocks / ecK) * ecM;
+            var totalBlocks = numBlocks + numParityBlocks;
+
+            return new ByteSize(blockSize.SizeInBytes * totalBlocks);
+        }
+
+        private int NextPowerOf2(int n)
+        {
+            n = n - 1;
+            var lg = Convert.ToInt32(Math.Round(Math.Log2(Convert.ToDouble(n))));
+            return 1 << (lg + 1);
+        }
+
+        private static bool IsPowerOfTwo(ByteSize size)
+        {
+            var x = size.SizeInBytes;
+            return (x != 0) && ((x & (x - 1)) == 0);
+        }
+    }
+}