nice csv outputs

Author: benbierens

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -10,7 +10,7 @@ namespace CodexPlugin
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "AC-19-7F-3A-88-07-CB-43-53-60-4F-21-3D-A6-B1-53-47-65-07-3B-91-C6-88-B9-76-B2-7E-33-6A-1C-69-F4";
+        private const string OpenApiYamlHash = "2E-7C-A2-F3-67-D9-F2-A6-4E-D5-FF-A2-EC-65-ED-59-CE-89-A8-92-57-5E-CF-40-9A-83-49-0B-49-42-5D-EC";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -726,7 +726,7 @@ paths:
         "503":
           description: Persistence is not enabled
 
-  "/node/spr":
+  "/spr":
     get:
       summary: "Get Node's SPR"
       operationId: getSPR
@@ -744,7 +744,7 @@ paths:
         "503":
           description: Node SPR not ready, try again later
 
-  "/node/peerid":
+  "/peerid":
     get:
       summary: "Get Node's PeerID"
       operationId: getPeerId

Tools/CsvCombiner/CsvCombiner.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Logging\Logging.csproj" />
+  </ItemGroup>
+
+</Project>

Tools/CsvCombiner/Program.cs

@@ -0,0 +1,71 @@
+using Logging;
+
+public class Program
+{
+    public static void Main(string[] args)
+    {
+        args = ["d:\\CodexTestLogs\\BlockExchange\\experiment2-fetchbatched"];
+        var p = new Program(args[0]);
+        p.Run();
+    }
+
+    private static readonly ILog log = new ConsoleLog();
+    private string path;
+
+    private readonly Dictionary<string, List<string>> combine = new Dictionary<string, List<string>>();
+
+    public Program(string path)
+    {
+        this.path = path;
+    }
+
+    private void Run()
+    {
+        Log("Starting in " + path);
+
+        var files = Directory.GetFiles(path)
+            .Where(f => f.ToLowerInvariant().EndsWith(".csv")).ToArray();
+
+        foreach (var file in files)
+        {
+            AddToMap(file);
+        }
+
+        var i = 0;
+        foreach (var pair in combine)
+        {
+            var list = pair.Value;
+            list.Insert(0, pair.Key);
+
+            File.WriteAllLines(Path.Combine(path, "combine_" + i + ".csv"), list.ToArray());
+            i++;
+        }
+
+        Log("done");
+    }
+
+    private void AddToMap(string file)
+    {
+        var lines = File.ReadAllLines(file);
+        if (lines.Length > 1)
+        {
+            var header = lines[0];
+            var list = GetList(header);
+            list.AddRange(lines.Skip(1));
+        }
+    }
+
+    private List<string> GetList(string header)
+    {
+        if (!combine.ContainsKey(header))
+        {
+            combine.Add(header, new List<string>());
+        }
+        return combine[header];
+    }
+
+    private void Log(string msg)
+    {
+        log.Log(msg);
+    }
+}

Tools/TranscriptAnalysis/CsvWriter.cs

@@ -1,7 +1,16 @@
-namespace TranscriptAnalysis
+using Logging;
+
+namespace TranscriptAnalysis
 {
     public class CsvWriter
     {
+        private readonly ILog log;
+
+        public CsvWriter(ILog log)
+        {
+            this.log = log;
+        }
+
         public ICsv CreateNew()
         {
             return new Csv();
@@ -14,6 +23,8 @@ public void Write(ICsv csv, string filename)
             using var file = File.OpenWrite(filename);
             using var writer = new StreamWriter(file);
             c.CreateLines(writer.WriteLine);
+
+            log.Log($"CSV written to: '{filename}'");
         }
     }
 

Tools/TranscriptAnalysis/Receivers/BaseReceiver.cs

@@ -8,13 +8,18 @@ public abstract class BaseReceiver<T> : IEventReceiver<T>
     {
         protected ILog log { get; private set; } = new NullLog();
         protected OverwatchCodexHeader Header { get; private set; } = null!;
-        protected CsvWriter CsvWriter { get; private set; } = new CsvWriter();
+        protected CsvWriter CsvWriter { get; private set; }
         protected string SourceFilename { get; private set; } = string.Empty;
 
         public abstract string Name { get; }
         public abstract void Receive(ActivateEvent<T> @event);
         public abstract void Finish();
 
+        protected BaseReceiver()
+        {
+            CsvWriter = new CsvWriter(log);
+        }
+
         public void Init(string sourceFilename, ILog log, OverwatchCodexHeader header)
         {
             this.log = new LogPrefixer(log, $"({Name}) ");

Tools/TranscriptAnalysis/Receivers/NodesDegree.cs

@@ -46,6 +46,7 @@ public Node(string peerId)
 
         private readonly Dictionary<string, Node> dialingNodes = new Dictionary<string, Node>();
         private readonly Dictionary<string, Dial> dials = new Dictionary<string, Dial>();
+        private long uploadSize;
 
         public override string Name => "NodesDegree";
 
@@ -57,6 +58,11 @@ public override void Receive(ActivateEvent<OverwatchCodexEvent> @event)
                 if (peerId == null) return;
                 AddDial(peerId, @event.Payload.DialSuccessful.TargetPeerId);
             }
+            if (@event.Payload.FileUploaded != null)
+            {
+                var uploadEvent = @event.Payload.FileUploaded;
+                uploadSize = uploadEvent.ByteSize;
+            }
         }
 
         public override void Finish()
@@ -84,6 +90,7 @@ public override void Finish()
 
             float tot = numNodes;
             csv.GetColumn("numNodes", Header.Nodes.Length);
+            csv.GetColumn("filesize", uploadSize.ToString());
             var degreeColumn = csv.GetColumn("degree", 0.0f);
             var occuranceColumn = csv.GetColumn("occurance", 0.0f);
             degreeOccurances.Print((i, count) =>

cs-codex-dist-testing.sln

@@ -76,6 +76,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TranscriptAnalysis", "Tools
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MarketInsights", "Tools\MarketInsights\MarketInsights.csproj", "{004614DF-1C65-45E3-882D-59AE44282573}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CsvCombiner", "Tools\CsvCombiner\CsvCombiner.csproj", "{6230347F-5045-4E25-8E7A-13D7221B7444}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -202,6 +204,10 @@ Global
 		{004614DF-1C65-45E3-882D-59AE44282573}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{004614DF-1C65-45E3-882D-59AE44282573}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{004614DF-1C65-45E3-882D-59AE44282573}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6230347F-5045-4E25-8E7A-13D7221B7444}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6230347F-5045-4E25-8E7A-13D7221B7444}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6230347F-5045-4E25-8E7A-13D7221B7444}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6230347F-5045-4E25-8E7A-13D7221B7444}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -237,6 +243,7 @@ Global
 		{870DDFBE-D7ED-4196-9681-13CA947BDEA6} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7}
 		{C0EEBD32-23CB-45EC-A863-79FB948508C8} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 		{004614DF-1C65-45E3-882D-59AE44282573} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
+		{6230347F-5045-4E25-8E7A-13D7221B7444} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}