Merge pull request #104 from codex-storage/feature/bot-checks-cids

Feature/bot checks cids

Author: benbierens

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -10,7 +10,7 @@ namespace CodexPlugin
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "AC-19-7F-3A-88-07-CB-43-53-60-4F-21-3D-A6-B1-53-47-65-07-3B-91-C6-88-B9-76-B2-7E-33-6A-1C-69-F4";
+        private const string OpenApiYamlHash = "D5-C3-18-71-E8-FF-8F-89-9C-6B-98-3C-F2-C2-D2-37-0A-9F-27-23-35-67-EA-F6-1F-F9-D5-C6-63-34-5A-92";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -73,7 +73,7 @@ public string UploadFile(FileStream fileStream, Action<Failure> onFailure)
         public Stream DownloadFile(string contentId, Action<Failure> onFailure)
         {
             var fileResponse = OnCodex(
-                api => api.DownloadNetworkAsync(contentId),
+                api => api.DownloadNetworkStreamAsync(contentId),
                 CreateRetryConfig(nameof(DownloadFile), onFailure));
 
             if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
@@ -88,25 +88,25 @@ public LocalDatasetList LocalFiles()
         public StorageAvailability SalesAvailability(StorageAvailability request)
         {
             var body = mapper.Map(request);
-            var read = OnCodex<SalesAvailabilityREAD>(api => api.OfferStorageAsync(body));
+            var read = OnCodex(api => api.OfferStorageAsync(body));
             return mapper.Map(read);
         }
 
         public StorageAvailability[] GetAvailabilities()
         {
-            var collection = OnCodex<ICollection<SalesAvailabilityREAD>>(api => api.GetAvailabilitiesAsync());
+            var collection = OnCodex(api => api.GetAvailabilitiesAsync());
             return mapper.Map(collection);
         }
 
         public string RequestStorage(StoragePurchaseRequest request)
         {
             var body = mapper.Map(request);
-            return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
+            return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
         }
 
         public CodexSpace Space()
         {
-            var space = OnCodex<Space>(api => api.SpaceAsync());
+            var space = OnCodex(api => api.SpaceAsync());
             return mapper.Map(space);
         }
 

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -456,9 +456,35 @@ paths:
 
   "/data/{cid}/network":
     get:
-      summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
+      summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed."
       tags: [ Data ]
       operationId: downloadNetwork
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: "File to be downloaded."
+      responses:
+        "200":
+          description: Manifest information for download that has been started.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DataItem"
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Failed to download dataset manifest
+        "500":
+          description: Well it was bad-bad
+
+  "/data/{cid}/network/stream":
+    get:
+      summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
+      tags: [ Data ]
+      operationId: downloadNetworkStream
       parameters:
         - in: path
           name: cid
@@ -481,6 +507,32 @@ paths:
         "500":
           description: Well it was bad-bad
 
+  "/data/{cid}/network/manifest":
+    get:
+      summary: "Download only the dataset manifest from the network to the local node if it's not available locally."
+      tags: [ Data ]
+      operationId: downloadNetworkManifest
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: "File for which the manifest is to be downloaded."
+      responses:
+        "200":
+          description: Manifest information.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DataItem"
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Failed to download dataset manifest
+        "500":
+          description: Well it was bad-bad
+
   "/space":
     get:
       summary: "Gets a summary of the storage space allocation of the node."
@@ -726,7 +778,7 @@ paths:
         "503":
           description: Persistence is not enabled
 
-  "/node/spr":
+  "/spr":
     get:
       summary: "Get Node's SPR"
       operationId: getSPR
@@ -744,7 +796,7 @@ paths:
         "503":
           description: Node SPR not ready, try again later
 
-  "/node/peerid":
+  "/peerid":
     get:
       summary: "Get Node's PeerID"
       operationId: getPeerId
@@ -792,4 +844,4 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/DebugInfo"
+                $ref: "#/components/schemas/DebugInfo"

Tools/AutoClient/Purchaser.cs

@@ -68,7 +68,7 @@ private async Task DownloadForeignCid()
                 var filename = Guid.NewGuid().ToString().ToLowerInvariant();
                 {
                     using var fileStream = File.OpenWrite(filename);
-                    var fileResponse = await codex.DownloadNetworkAsync(cid);
+                    var fileResponse = await codex.DownloadNetworkStreamAsync(cid);
                     fileResponse.Stream.CopyTo(fileStream);
                 }
                 var time = sw.Elapsed;

Tools/BiblioTech/BaseCommand.cs

@@ -1,7 +1,6 @@
 using Discord.WebSocket;
 using BiblioTech.Options;
 using Discord;
-using k8s.KubeConfigModels;
 
 namespace BiblioTech
 {

Tools/BiblioTech/BiblioTech.csproj

@@ -12,6 +12,7 @@
     <ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
     <ProjectReference Include="..\..\Framework\DiscordRewards\DiscordRewards.csproj" />
     <ProjectReference Include="..\..\Framework\GethConnector\GethConnector.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
   </ItemGroup>
 
 </Project>

Tools/BiblioTech/CodexCidChecker.cs

@@ -0,0 +1,198 @@
+using CodexOpenApi;
+using IdentityModel.Client;
+using Utils;
+
+namespace BiblioTech
+{
+    public class CodexCidChecker
+    {
+        private static readonly string nl = Environment.NewLine;
+        private readonly Configuration config;
+        private CodexApi? currentCodexNode;
+
+        public CodexCidChecker(Configuration config)
+        {
+            this.config = config;
+        }
+
+        public async Task<CheckResponse> PerformCheck(string cid)
+        {
+            if (string.IsNullOrEmpty(config.CodexEndpoint))
+            {
+                return new CheckResponse(false, "Codex CID checker is not (yet) available.", "");
+            }
+
+            try
+            {
+                var codex = GetCodex();
+                var nodeCheck = await CheckCodex(codex);
+                if (!nodeCheck) return new CheckResponse(false, "Codex node is not available. Cannot perform check.", $"Codex node at '{config.CodexEndpoint}' did not respond correctly to debug/info.");
+
+                return await PerformCheck(codex, cid);
+            }
+            catch (Exception ex)
+            {
+                return new CheckResponse(false, "Internal server error", ex.ToString());
+            }
+        }
+
+        private async Task<CheckResponse> PerformCheck(CodexApi codex, string cid)
+        {
+            try
+            {
+                var manifest = await codex.DownloadNetworkManifestAsync(cid);
+                return SuccessMessage(manifest);
+            }
+            catch (ApiException apiEx)
+            {
+                if (apiEx.StatusCode == 400) return CidFormatInvalid(apiEx.Response);
+                if (apiEx.StatusCode == 404) return FailedToFetch(apiEx.Response);
+                return UnexpectedReturnCode(apiEx.Response);
+            }
+            catch (Exception ex)
+            {
+                return UnexpectedException(ex);
+            }
+        }
+
+        #region Response formatting
+
+        private CheckResponse SuccessMessage(DataItem content)
+        {
+            return FormatResponse(
+                success: true,
+                title: $"Success: '{content.Cid}'",
+                error: "",
+                $"size: {content.Manifest.OriginalBytes} bytes",
+                $"blockSize: {content.Manifest.BlockSize} bytes",
+                $"protected: {content.Manifest.Protected}"
+            );
+        }
+
+        private CheckResponse UnexpectedException(Exception ex)
+        {
+            return FormatResponse(
+                success: false,
+                title: "Unexpected error",
+                error: ex.ToString(),
+                content: "Details will be sent to the bot-admin channel."
+            );
+        }
+
+        private CheckResponse UnexpectedReturnCode(string response)
+        {
+            var msg = "Unexpected return code. Response: " + response;
+            return FormatResponse(
+                success: false,
+                title: "Unexpected return code",
+                error: msg,
+                content: msg
+            );
+        }
+
+        private CheckResponse FailedToFetch(string response)
+        {
+            var msg = "Failed to download content. Response: " + response;
+            return FormatResponse(
+                success: false,
+                title: "Could not download content",
+                error: msg,
+                msg,
+                $"Connection trouble? See 'https://docs.codex.storage/learn/troubleshoot'"
+            );
+        }
+
+        private CheckResponse CidFormatInvalid(string response)
+        {
+            return FormatResponse(
+                success: false,
+                title: "Invalid format",
+                error: "",
+                content: "Provided CID is not formatted correctly."
+            );
+        }
+
+        private CheckResponse FormatResponse(bool success, string title, string error, params string[] content)
+        {
+            var msg = string.Join(nl,
+                new string[]
+                {
+                    title,
+                    "```"
+                }
+                .Concat(content)
+                .Concat(new string[]
+                {
+                    "```"
+                })
+            ) + nl + nl;
+
+            return new CheckResponse(success, msg, error);
+        }
+
+        #endregion
+
+        #region Codex Node API 
+
+        private CodexApi GetCodex()
+        {
+            if (currentCodexNode == null) currentCodexNode = CreateCodex();
+            return currentCodexNode;
+        }
+
+        private async Task<bool> CheckCodex(CodexApi codex)
+        {
+            try
+            {
+                var info = await currentCodexNode!.GetDebugInfoAsync();
+                if (info == null || string.IsNullOrEmpty(info.Id)) return false;
+                return true;
+            }
+            catch (Exception e)
+            {
+                return false;
+            }
+        }
+
+        private CodexApi CreateCodex()
+        {
+            var endpoint = config.CodexEndpoint;
+            var splitIndex = endpoint.LastIndexOf(':');
+            var host = endpoint.Substring(0, splitIndex);
+            var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1));
+
+            var address = new Address(
+                host: host,
+                port: port
+            );
+
+            var client = new HttpClient();
+            if (!string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":"))
+            {
+                var tokens = config.CodexEndpointAuth.Split(':');
+                if (tokens.Length != 2) throw new Exception("Expected '<username>:<password>' in CodexEndpointAuth parameter.");
+                client.SetBasicAuthentication(tokens[0], tokens[1]);    
+            }
+
+            var codex = new CodexApi(client);
+            codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
+            return codex;
+        }
+
+        #endregion
+    }
+
+    public class CheckResponse
+    {
+        public CheckResponse(bool success, string message, string error)
+        {
+            Success = success;
+            Message = message;
+            Error = error;
+        }
+
+        public bool Success { get; }
+        public string Message { get; }
+        public string Error { get; }
+    }
+}