Skip to content

Commit 8694bfe

Browse files
benbierensbenbierens
committed
Merge branch 'feature/auto-client'
2 parents 637b5a4 + 7eed616 commit 8694bfe

File tree

10 files changed

+399
-6
lines changed

10 files changed

+399
-6
lines changed

‎.github/workflows/docker-autoclient.yml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: Docker - AutoClient
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
tags:
8+
- 'v*.*.*'
9+
paths:
10+
- 'Tools/AutoClient/**'
11+
- '!Tools/AutoClient/docker/docker-compose.yaml'
12+
- 'Framework/**'
13+
- 'ProjectPlugins/**'
14+
- .github/workflows/docker-autoclient.yml
15+
- .github/workflows/docker-reusable.yml
16+
workflow_dispatch:
17+
18+
jobs:
19+
build-and-push:
20+
name: Build and Push
21+
uses: ./.github/workflows/docker-reusable.yml
22+
with:
23+
docker_file: Tools/AutoClient/docker/Dockerfile
24+
docker_repo: codexstorage/codex-autoclient
25+
secrets: inherit
26+

‎.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
.vs
22
obj
33
bin
4-
.vscode
4+
.vscode
5+
Tools/AutoClient/datapath

‎Framework/FileUtils/FileManager.cs

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -70,16 +70,27 @@ public void DeleteAllFiles()
7070
public void ScopedFiles(Action action)
7171
{
7272
PushFileSet();
73-
action();
74-
PopFileSet();
73+
try
74+
{
75+
action();
76+
}
77+
finally
78+
{
79+
PopFileSet();
80+
}
7581
}
7682

7783
public T ScopedFiles<T>(Func<T> action)
7884
{
7985
PushFileSet();
80-
var result = action();
81-
PopFileSet();
82-
return result;
86+
try
87+
{
88+
return action();
89+
}
90+
finally
91+
{
92+
PopFileSet();
93+
}
8394
}
8495

8596
private void PushFileSet()

‎Tools/AutoClient/AutoClient.csproj

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net7.0</TargetFramework>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
</PropertyGroup>
9+
10+
<ItemGroup>
11+
<ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
12+
<ProjectReference Include="..\..\Framework\Logging\Logging.csproj" />
13+
<ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
14+
</ItemGroup>
15+
16+
</Project>

‎Tools/AutoClient/Configuration.cs

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
using ArgsUniform;
2+
3+
namespace AutoClient
4+
{
5+
public class Configuration
6+
{
7+
[Uniform("codex-host", "ch", "CODEXHOST", false, "Codex Host address. (default 'http://localhost')")]
8+
public string CodexHost { get; set; } = "http://localhost";
9+
10+
[Uniform("codex-port", "cp", "CODEXPORT", false, "port number of Codex API. (8080 by default)")]
11+
public int CodexPort { get; set; } = 8080;
12+
13+
[Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
14+
public string DataPath { get; set; } = "datapath";
15+
16+
[Uniform("purchases", "np", "PURCHASES", false, "Number of concurrent purchases.")]
17+
public int NumConcurrentPurchases { get; set; } = 10;
18+
19+
[Uniform("contract-duration", "cd", "CONTRACTDURATION", false, "contract duration in minutes. (default 30)")]
20+
public int ContractDurationMinutes { get; set; } = 30;
21+
22+
[Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15)")]
23+
public int ContractExpiryMinutes { get; set; } = 15;
24+
25+
[Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 5)")]
26+
public int NumHosts { get; set; } = 5;
27+
28+
[Uniform("num-hosts-tolerance", "nt", "NUMTOL", false, "Number of host tolerance for contract. (default 2)")]
29+
public int HostTolerance { get; set; } = 2;
30+
31+
[Uniform("price","p", "PRICE", false, "Price of contract. (default 10)")]
32+
public int Price { get; set; } = 10;
33+
34+
[Uniform("collateral", "c", "COLLATERAL", false, "Required collateral. (default 1)")]
35+
public int RequiredCollateral { get; set; } = 1;
36+
37+
public string LogPath
38+
{
39+
get
40+
{
41+
return Path.Combine(DataPath, "logs");
42+
}
43+
}
44+
}
45+
}

‎Tools/AutoClient/ImageGenerator.cs

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
namespace AutoClient
2+
{
3+
public class ImageGenerator
4+
{
5+
public async Task<string> GenerateImage()
6+
{
7+
var httpClient = new HttpClient();
8+
var thing = await httpClient.GetStreamAsync("https://picsum.photos/3840/2160");
9+
10+
var filename = $"{Guid.NewGuid().ToString().ToLowerInvariant()}.jpg";
11+
using var file = File.OpenWrite(filename);
12+
await thing.CopyToAsync(file);
13+
14+
return filename;
15+
}
16+
}
17+
}

‎Tools/AutoClient/Program.cs

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
using ArgsUniform;
2+
using AutoClient;
3+
using CodexOpenApi;
4+
using Core;
5+
using Logging;
6+
7+
public static class Program
8+
{
9+
public static async Task Main(string[] args)
10+
{
11+
var cts = new CancellationTokenSource();
12+
var cancellationToken = cts.Token;
13+
Console.CancelKeyPress += (sender, args) => cts.Cancel();
14+
15+
var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
16+
var config = uniformArgs.Parse(true);
17+
18+
if (config.NumConcurrentPurchases < 1)
19+
{
20+
throw new Exception("Number of concurrent purchases must be > 0");
21+
}
22+
23+
var log = new LogSplitter(
24+
new FileLog(Path.Combine(config.LogPath, "autoclient")),
25+
new ConsoleLog()
26+
);
27+
28+
var address = new Utils.Address(
29+
host: config.CodexHost,
30+
port: config.CodexPort
31+
);
32+
33+
log.Log($"Start. Address: {address}");
34+
35+
var imgGenerator = new ImageGenerator();
36+
37+
var client = new HttpClient();
38+
var codex = new CodexApi(client);
39+
codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
40+
41+
await CheckCodex(codex, log);
42+
43+
var purchasers = new List<Purchaser>();
44+
for (var i = 0; i < config.NumConcurrentPurchases; i++)
45+
{
46+
purchasers.Add(
47+
new Purchaser(new LogPrefixer(log, $"({i}) "), client, address, codex, cancellationToken, config, imgGenerator)
48+
);
49+
}
50+
51+
var delayPerPurchaser = TimeSpan.FromMinutes(config.ContractDurationMinutes) / config.NumConcurrentPurchases;
52+
foreach (var purchaser in purchasers)
53+
{
54+
purchaser.Start();
55+
await Task.Delay(delayPerPurchaser);
56+
}
57+
58+
log.Log("Done.");
59+
}
60+
61+
private static async Task CheckCodex(CodexApi codex, ILog log)
62+
{
63+
log.Log("Checking Codex...");
64+
try
65+
{
66+
var info = await codex.GetDebugInfoAsync();
67+
if (string.IsNullOrEmpty(info.Id)) throw new Exception("Failed to fetch Codex node id");
68+
}
69+
catch (Exception ex)
70+
{
71+
log.Log($"Codex not OK: {ex}");
72+
throw;
73+
}
74+
}
75+
76+
private static void PrintHelp()
77+
{
78+
Console.WriteLine("Generates fake data and creates Codex storage contracts for it.");
79+
}
80+
}

‎Tools/AutoClient/Purchaser.cs

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
using CodexOpenApi;
2+
using CodexPlugin;
3+
using Logging;
4+
using Newtonsoft.Json;
5+
using Utils;
6+
7+
namespace AutoClient
8+
{
9+
public class Purchaser
10+
{
11+
private readonly ILog log;
12+
private readonly HttpClient client;
13+
private readonly Address address;
14+
private readonly CodexApi codex;
15+
private readonly CancellationToken ct;
16+
private readonly Configuration config;
17+
private readonly ImageGenerator generator;
18+
19+
public Purchaser(ILog log, HttpClient client, Address address, CodexApi codex, CancellationToken ct, Configuration config, ImageGenerator generator)
20+
{
21+
this.log = log;
22+
this.client = client;
23+
this.address = address;
24+
this.codex = codex;
25+
this.ct = ct;
26+
this.config = config;
27+
this.generator = generator;
28+
}
29+
30+
public void Start()
31+
{
32+
Task.Run(Worker);
33+
}
34+
35+
private async Task Worker()
36+
{
37+
while (!ct.IsCancellationRequested)
38+
{
39+
var pid = await StartNewPurchase();
40+
await WaitTillFinished(pid);
41+
}
42+
}
43+
44+
private async Task<string> StartNewPurchase()
45+
{
46+
var file = await CreateFile();
47+
var cid = await UploadFile(file);
48+
return await RequestStorage(cid);
49+
}
50+
51+
private async Task<string> CreateFile()
52+
{
53+
return await generator.GenerateImage();
54+
}
55+
56+
private async Task<ContentId> UploadFile(string filename)
57+
{
58+
// Copied from CodexNode :/
59+
using var fileStream = File.OpenRead(filename);
60+
61+
log.Log($"Uploading file {filename}...");
62+
var response = await codex.UploadAsync(fileStream, ct);
63+
64+
if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
65+
if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
66+
67+
log.Log($"Uploaded file. Received contentId: '{response}'.");
68+
return new ContentId(response);
69+
}
70+
71+
private async Task<string> RequestStorage(ContentId cid)
72+
{
73+
log.Log("Requesting storage for " + cid.Id);
74+
var result = await codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
75+
{
76+
Collateral = config.RequiredCollateral.ToString(),
77+
Duration = (config.ContractDurationMinutes * 60).ToString(),
78+
Expiry = (config.ContractExpiryMinutes * 60).ToString(),
79+
Nodes = config.NumHosts,
80+
Reward = config.Price.ToString(),
81+
ProofProbability = "15",
82+
Tolerance = config.HostTolerance
83+
}, ct);
84+
85+
log.Log("Purchase ID: " + result);
86+
87+
return result;
88+
}
89+
90+
private async Task<string?> GetPurchaseState(string pid)
91+
{
92+
try
93+
{
94+
// openapi still don't match code.
95+
var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
96+
if (string.IsNullOrEmpty(str)) return null;
97+
var sp = JsonConvert.DeserializeObject<StoragePurchase>(str)!;
98+
log.Log($"Purchase {pid} is {sp.State}");
99+
if (!string.IsNullOrEmpty(sp.Error)) log.Log($"Purchase {pid} error is {sp.Error}");
100+
return sp.State;
101+
}
102+
catch
103+
{
104+
return null;
105+
}
106+
}
107+
108+
private async Task WaitTillFinished(string pid)
109+
{
110+
log.Log("Waiting...");
111+
try
112+
{
113+
var emptyResponseTolerance = 10;
114+
while (true)
115+
{
116+
var status = (await GetPurchaseState(pid))?.ToLowerInvariant();
117+
if (string.IsNullOrEmpty(status))
118+
{
119+
emptyResponseTolerance--;
120+
if (emptyResponseTolerance == 0)
121+
{
122+
log.Log("Received 10 empty responses. Stop tracking this purchase.");
123+
await ExpiryTimeDelay();
124+
return;
125+
}
126+
}
127+
else
128+
{
129+
if (status.Contains("cancel") ||
130+
status.Contains("error") ||
131+
status.Contains("finished"))
132+
{
133+
return;
134+
}
135+
if (status.Contains("started"))
136+
{
137+
await FixedDurationDelay();
138+
}
139+
}
140+
141+
await FixedShortDelay();
142+
}
143+
}
144+
catch (Exception ex)
145+
{
146+
log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time.");
147+
await ExpiryTimeDelay();
148+
}
149+
}
150+
151+
private async Task FixedDurationDelay()
152+
{
153+
await Task.Delay(config.ContractDurationMinutes * 60 * 1000, ct);
154+
}
155+
156+
private async Task ExpiryTimeDelay()
157+
{
158+
await Task.Delay(config.ContractExpiryMinutes * 60 * 1000, ct);
159+
}
160+
161+
private async Task FixedShortDelay()
162+
{
163+
await Task.Delay(15 * 1000, ct);
164+
}
165+
}
166+
}

0 commit comments

Comments
 (0)