Updates to latest codex api

Author: benbierens

Framework/Utils/Retry.cs

@@ -6,18 +6,20 @@ public class Retry
         private readonly TimeSpan maxTimeout;
         private readonly TimeSpan sleepAfterFail;
         private readonly Action<Failure> onFail;
+        private readonly bool failFast;
 
-        public Retry(string description, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action<Failure> onFail)
+        public Retry(string description, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action<Failure> onFail, bool failFast)
         {
             this.description = description;
             this.maxTimeout = maxTimeout;
             this.sleepAfterFail = sleepAfterFail;
             this.onFail = onFail;
+            this.failFast = failFast;
         }
 
         public void Run(Action task)
         {
-            var run = new RetryRun(description, task, maxTimeout, sleepAfterFail, onFail);
+            var run = new RetryRun(description, task, maxTimeout, sleepAfterFail, onFail, failFast);
             run.Run();
         }
 
@@ -28,7 +30,7 @@ public T Run<T>(Func<T> task)
             var run = new RetryRun(description, () =>
             {
                 result = task();
-            }, maxTimeout, sleepAfterFail, onFail);
+            }, maxTimeout, sleepAfterFail, onFail, failFast);
             run.Run();
 
             return result!;
@@ -43,16 +45,18 @@ private class RetryRun
             private readonly Action<Failure> onFail;
             private readonly DateTime start = DateTime.UtcNow;
             private readonly List<Failure> failures = new List<Failure>();
+            private readonly bool failFast;
             private int tryNumber;
             private DateTime tryStart;
 
-            public RetryRun(string description, Action task, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action<Failure> onFail)
+            public RetryRun(string description, Action task, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action<Failure> onFail, bool failFast)
             {
                 this.description = description;
                 this.task = task;
                 this.maxTimeout = maxTimeout;
                 this.sleepAfterFail = sleepAfterFail;
                 this.onFail = onFail;
+                this.failFast = failFast;
 
                 tryNumber = 0;
                 tryStart = DateTime.UtcNow;
@@ -97,7 +101,7 @@ private void CheckMaximums()
 
                 // If we have a few very fast failures, retrying won't help us. There's probably something wrong with our operation.
                 // In this case, don't wait the full duration and fail quickly.
-                if (failures.Count > 5 && failures.All(f => f.Duration < TimeSpan.FromSeconds(1.0))) Fail();
+                if (failFast && failures.Count > 5 && failures.All(f => f.Duration < TimeSpan.FromSeconds(1.0))) Fail();
             }
 
             private void Fail()

Framework/Utils/Time.cs

@@ -124,13 +124,13 @@ public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime
 
         public static void Retry(Action action, TimeSpan maxTimeout, TimeSpan retryTime, string description, Action<Failure> onFail)
         {
-            var r = new Retry(description, maxTimeout, retryTime, onFail);
+            var r = new Retry(description, maxTimeout, retryTime, onFail, failFast: true);
             r.Run(action);
         }
 
         public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime, string description, Action<Failure> onFail)
         {
-            var r = new Retry(description, maxTimeout, retryTime, onFail);
+            var r = new Retry(description, maxTimeout, retryTime, onFail, failFast: true);
             return r.Run(action);
         }
     }

Framework/WebUtils/Http.cs

@@ -40,7 +40,7 @@ public T OnClient<T>(Func<HttpClient, T> action)
 
         public T OnClient<T>(Func<HttpClient, T> action, string description)
         {
-            var retry = new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), f => { });
+            var retry = new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), f => { }, failFast: true);
             return OnClient(action, retry);
         }
 

ProjectPlugins/CodexClient/CodexAccess.cs

@@ -170,27 +170,8 @@ public CodexSpace Space()
 
         public StoragePurchase? GetPurchaseStatus(string purchaseId)
         {
-            return CrashCheck(() =>
-            {
-                var endpoint = GetEndpoint();
-                try
-                {
-                    return Time.Retry(() =>
-                    {
-                        var str = endpoint.HttpGetString($"storage/purchases/{purchaseId}");
-                        if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
-                        return JsonConvert.DeserializeObject<StoragePurchase>(str)!;
-                    }, nameof(GetPurchaseStatus));
-                }
-                catch (Exception exc)
-                {
-                    log.Error($"Failed to fetch purchase information for id: '{purchaseId}'. Exception: {exc.Message}");
-                    return null;
-                }
-            });
-
-            // TODO: current getpurchase api does not line up with its openapi spec.
-            // return mapper.Map(OnCodex(api => api.GetPurchaseAsync(purchaseId)));
+            var purchase = OnCodex(api => api.GetPurchaseAsync(purchaseId));
+            return mapper.Map(purchase);
         }
 
         public string GetName()

ProjectPlugins/CodexClient/CodexNode.cs

@@ -380,8 +380,9 @@ public void WaitUntilQuotaUsedIncreased(
 
             var retry = new Retry($"Checking local space for quotaUsed increase of {expectedIncreaseOfQuotaUsed}",
             maxTimeout: maxTimeout,
-            sleepAfterFail: TimeSpan.FromSeconds(3),
-            onFail: f => { });
+            sleepAfterFail: TimeSpan.FromSeconds(10),
+            onFail: f => { },
+            failFast: false);
 
             retry.Run(() =>
             {

ProjectPlugins/CodexClient/Mapper.cs

@@ -86,23 +86,21 @@ public StoragePurchase Map(CodexOpenApi.Purchase purchase)
             return new StoragePurchase
             {
                 Request = Map(purchase.Request),
-                State = purchase.State.ToString(), //Map(purchase.State),
+                State = Map(purchase.State),
                 Error = purchase.Error
             };
         }
 
         public StoragePurchaseState Map(CodexOpenApi.PurchaseState purchaseState)
         {
-            // TODO: to be re-enabled when marketplace api lines up with openapi.yaml.
-
             // Explicit mapping: If the API changes, we will get compile errors here.
             // That's what we want.
             switch (purchaseState)
             {
                 case CodexOpenApi.PurchaseState.Cancelled:
                     return StoragePurchaseState.Cancelled;
-                case CodexOpenApi.PurchaseState.Error:
-                    return StoragePurchaseState.Error;
+                case CodexOpenApi.PurchaseState.Errored:
+                    return StoragePurchaseState.Errored;
                 case CodexOpenApi.PurchaseState.Failed:
                     return StoragePurchaseState.Failed;
                 case CodexOpenApi.PurchaseState.Finished:

ProjectPlugins/CodexClient/MarketplaceTypes.cs

@@ -34,21 +34,21 @@ public void Log(ILog log)
 
     public class StoragePurchase
     {
-        public string State { get; set; } = string.Empty;
+        public StoragePurchaseState State { get; set; } = StoragePurchaseState.Unknown;
         public string Error { get; set; } = string.Empty;
         public StorageRequest Request { get; set; } = null!;
 
-        public bool IsCancelled => State.ToLowerInvariant().Contains("cancel");
-        public bool IsError => State.ToLowerInvariant().Contains("error");
-        public bool IsFinished => State.ToLowerInvariant().Contains("finished");
-        public bool IsStarted => State.ToLowerInvariant().Contains("started");
-        public bool IsSubmitted => State.ToLowerInvariant().Contains("submitted");
+        public bool IsCancelled => State == StoragePurchaseState.Cancelled;
+        public bool IsError => State == StoragePurchaseState.Errored;
+        public bool IsFinished => State == StoragePurchaseState.Finished;
+        public bool IsStarted => State == StoragePurchaseState.Started;
+        public bool IsSubmitted => State == StoragePurchaseState.Submitted;
     }
 
     public enum StoragePurchaseState
     {
         Cancelled = 0,
-        Error = 1,
+        Errored = 1,
         Failed = 2,
         Finished = 3,
         Pending = 4,

ProjectPlugins/CodexClient/StoragePurchaseContract.cs

@@ -27,7 +27,7 @@ public class StoragePurchaseContract : IStoragePurchaseContract
         private DateTime? contractSubmittedUtc = DateTime.UtcNow;
         private DateTime? contractStartedUtc;
         private DateTime? contractFinishedUtc;
-        private string lastState = string.Empty;
+        private StoragePurchaseState lastState = StoragePurchaseState.Unknown;
         private ContentId encodedContentId = new ContentId();
 
         public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase, ICodexNodeHooks hooks)
@@ -67,8 +67,8 @@ public ContentId ContentId
         public void WaitForStorageContractSubmitted()
         {
             var timeout = Purchase.Expiry + gracePeriod;
-            var raiseHook = lastState != "submitted";
-            WaitForStorageContractState(timeout, "submitted", sleep: 200);
+            var raiseHook = lastState != StoragePurchaseState.Submitted;
+            WaitForStorageContractState(timeout, StoragePurchaseState.Submitted, sleep: 200);
             contractSubmittedUtc = DateTime.UtcNow;
             if (raiseHook) hooks.OnStorageContractSubmitted(this);
             LogSubmittedDuration();
@@ -79,7 +79,7 @@ public void WaitForStorageContractStarted()
         {
             var timeout = Purchase.Expiry + gracePeriod;
 
-            WaitForStorageContractState(timeout, "started");
+            WaitForStorageContractState(timeout, StoragePurchaseState.Started);
             contractStartedUtc = DateTime.UtcNow;
             LogStartedDuration();
             AssertDuration(SubmittedToStarted, timeout, nameof(SubmittedToStarted));
@@ -93,7 +93,7 @@ public void WaitForStorageContractFinished()
             }
             var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value;
             var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
-            WaitForStorageContractState(timeout, "finished");
+            WaitForStorageContractState(timeout, StoragePurchaseState.Finished);
             contractFinishedUtc = DateTime.UtcNow;
             LogFinishedDuration();
             AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished));
@@ -107,10 +107,10 @@ public void WaitForContractFailed()
             }
             var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value;
             var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
-            WaitForStorageContractState(timeout, "failed");
+            WaitForStorageContractState(timeout, StoragePurchaseState.Failed);
         }
 
-        private void WaitForStorageContractState(TimeSpan timeout, string desiredState, int sleep = 1000)
+        private void WaitForStorageContractState(TimeSpan timeout, StoragePurchaseState desiredState, int sleep = 1000)
         {
             var waitStart = DateTime.UtcNow;
 
@@ -128,7 +128,7 @@ private void WaitForStorageContractState(TimeSpan timeout, string desiredState,
                     hooks.OnStorageContractUpdated(purchaseStatus);
                 }
 
-                if (lastState == "errored")
+                if (lastState == StoragePurchaseState.Errored)
                 {
                     FrameworkAssert.Fail("Contract errored: " + statusJson);
                 }

ProjectPlugins/CodexClient/openapi.yaml

@@ -50,6 +50,10 @@ components:
       type: string
       description: The amount of tokens paid per byte per second per slot to hosts the client is willing to pay
 
+    CollateralPerByte:
+      type: string
+      description: Number as decimal string that represents how much collateral per byte is asked from hosts that wants to fill a slots
+
     Duration:
       type: integer
       format: int64
@@ -320,8 +324,7 @@ components:
           default: 1
           minimum: 1
         collateralPerByte:
-          type: string
-          description: Number as decimal string that represents how much collateral per byte is asked from hosts that wants to fill a slots
+          $ref: "#/components/schemas/CollateralPerByte"
         expiry:
           type: integer
           format: int64
@@ -351,6 +354,8 @@ components:
           $ref: "#/components/schemas/ProofProbability"
         pricePerBytePerSecond:
           $ref: "#/components/schemas/PricePerBytePerSecond"
+        collateralPerByte:
+          $ref: "#/components/schemas/CollateralPerByte"
         maxSlotLoss:
           type: integer
           format: int64
@@ -392,7 +397,7 @@ components:
           description: Description of the Request's state
           enum:
             - cancelled
-            - error
+            - errored
             - failed
             - finished
             - pending
@@ -586,6 +591,8 @@ paths:
             text/plain:
               schema:
                 type: string
+        "422":
+          description: The mimetype of the filename is invalid
         "500":
           description: Well it was bad-bad and the upload did not work out
 

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -10,7 +10,7 @@ namespace CodexPlugin
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "EE-A5-6C-F9-F2-81-21-63-AB-F0-8D-63-0C-30-E8-55-F0-CC-7A-B0-69-6E-7F-77-C1-88-B0-31-F3-64-40-1A";
+        private const string OpenApiYamlHash = "1A-F7-DF-C3-E1-C6-98-FF-32-20-21-9B-26-40-B0-51-08-35-C2-E7-DB-41-49-93-60-A9-CE-47-B5-AD-3D-A3";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 

Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs

@@ -118,7 +118,8 @@ private Retry GetBalanceAssertRetry()
             return new Retry("AssertBalance",
                 maxTimeout: TimeSpan.FromMinutes(10.0),
                 sleepAfterFail: TimeSpan.FromSeconds(10.0),
-                onFail: f => { });
+                onFail: f => { },
+                failFast: false);
         }
 
         private TestToken GetTstBalance(ICodexNode node)