Skip to content

Commit 95aa6fd

Browse files
benbierensbenbierens
committed
Merge branch 'feature/folder-saver'
2 parents c24861e + 1d9f22c commit 95aa6fd

21 files changed

+1222
-298
lines changed

‎ProjectPlugins/CodexPlugin/CodexAccess.cs

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,20 @@ public LocalDataset DownloadManifestOnly(ContentId cid)
110110

111111
public LocalDatasetList LocalFiles()
112112
{
113-
return mapper.Map(OnCodex(api => api.ListDataAsync()));
113+
// API for listData mismatches.
114+
//return mapper.Map(OnCodex(api => api.ListDataAsync()));
115+
116+
return mapper.Map(CrashCheck(() =>
117+
{
118+
var endpoint = GetEndpoint();
119+
return Time.Retry(() =>
120+
{
121+
var str = endpoint.HttpGetString("data");
122+
if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
123+
return JsonConvert.DeserializeObject<LocalDatasetListJson>(str)!;
124+
}, nameof(LocalFiles));
125+
}));
126+
114127
}
115128

116129
public StorageAvailability SalesAvailability(StorageAvailability request)

‎ProjectPlugins/CodexPlugin/CodexNode.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ public ContentId UploadFile(TrackedFile file)
151151

152152
public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
153153
{
154-
return UploadFile(file, "application/x-binary", $"attachment; filename=\"{file.Filename}\"", onFailure);
154+
return UploadFile(file, "application/octet-stream", $"attachment; filename=\"{Path.GetFileName(file.Filename)}\"", onFailure);
155155
}
156156

157157
public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure)

‎ProjectPlugins/CodexPlugin/Mapper.cs

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,14 @@ public DebugInfo Map(CodexOpenApi.DebugInfo debugInfo)
2121
};
2222
}
2323

24+
public LocalDatasetList Map(LocalDatasetListJson json)
25+
{
26+
return new LocalDatasetList
27+
{
28+
Content = json.Content.Select(Map).ToArray()
29+
};
30+
}
31+
2432
public LocalDatasetList Map(CodexOpenApi.DataList dataList)
2533
{
2634
return new LocalDatasetList
@@ -38,6 +46,15 @@ public LocalDataset Map(CodexOpenApi.DataItem dataItem)
3846
};
3947
}
4048

49+
public LocalDataset Map(LocalDatasetListJsonItem item)
50+
{
51+
return new LocalDataset
52+
{
53+
Cid = new ContentId(item.Cid),
54+
Manifest = MapManifest(item.Manifest)
55+
};
56+
}
57+
4158
public CodexOpenApi.SalesAvailabilityCREATE Map(StorageAvailability availability)
4259
{
4360
return new CodexOpenApi.SalesAvailabilityCREATE
@@ -188,6 +205,18 @@ private Manifest MapManifest(CodexOpenApi.ManifestItem manifest)
188205
};
189206
}
190207

208+
public Manifest MapManifest(LocalDatasetListJsonItemManifest manifest)
209+
{
210+
return new Manifest
211+
{
212+
// needs update
213+
BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
214+
OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
215+
RootHash = manifest.TreeCid,
216+
Protected = manifest.Protected
217+
};
218+
}
219+
191220
private JArray JArray(IDictionary<string, object> map, string name)
192221
{
193222
return (JArray)map[name];
@@ -243,4 +272,42 @@ private ByteSize ToByteSize(string size)
243272
return new ByteSize(Convert.ToInt64(size));
244273
}
245274
}
275+
276+
277+
//"content": [
278+
// {
279+
// "cid": "zDvZRwzkxLxVaGces3kpkHjo8EcTPXudvYMfNxdoH21Ask1Js5fJ",
280+
// "manifest": {
281+
// "treeCid": "zDzSvJTf8GBRyEDNuAzXS9VnRfh8cNuYuRPwTLW6RUQReSgKnhCt",
282+
// "datasetSize": 5242880,
283+
// "blockSize": 65536,
284+
// "filename": null,
285+
// "mimetype": "application/octet-stream",
286+
// "uploadedAt": 1731426230,
287+
// "protected": false
288+
// }
289+
// }
290+
// ]
291+
292+
public class LocalDatasetListJson
293+
{
294+
public LocalDatasetListJsonItem[] Content { get; set; } = Array.Empty<LocalDatasetListJsonItem>();
295+
}
296+
297+
public class LocalDatasetListJsonItem
298+
{
299+
public string Cid { get; set; } = string.Empty;
300+
public LocalDatasetListJsonItemManifest Manifest { get; set; } = new();
301+
}
302+
303+
public class LocalDatasetListJsonItemManifest
304+
{
305+
public string TreeCid { get; set; } = string.Empty;
306+
public int DatasetSize { get; set; }
307+
public int BlockSize { get; set; }
308+
public string? Filename { get; set; } = string.Empty;
309+
public string? MimeType { get; set; } = string.Empty;
310+
public int? UploadedAt { get; set; }
311+
public bool Protected { get; set; }
312+
}
246313
}

‎ProjectPlugins/CodexPlugin/MarketplaceTypes.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,12 @@ public class StoragePurchase
3838
public string State { get; set; } = string.Empty;
3939
public string Error { get; set; } = string.Empty;
4040
public StorageRequest Request { get; set; } = null!;
41+
42+
public bool IsCancelled => State.ToLowerInvariant().Contains("cancel");
43+
public bool IsError => State.ToLowerInvariant().Contains("error");
44+
public bool IsFinished => State.ToLowerInvariant().Contains("finished");
45+
public bool IsStarted => State.ToLowerInvariant().Contains("started");
46+
public bool IsSubmitted => State.ToLowerInvariant().Contains("submitted");
4147
}
4248

4349
public class StorageRequest

‎Tools/AutoClient/App.cs

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using Logging;
1+
using AutoClient.Modes.FolderStore;
2+
using Logging;
23

34
namespace AutoClient
45
{
@@ -19,6 +20,15 @@ public App(Configuration config)
1920
new FileLog(Path.Combine(config.LogPath, "performance")),
2021
new ConsoleLog()
2122
));
23+
24+
if (!string.IsNullOrEmpty(config.FolderToStore))
25+
{
26+
FolderWorkDispatcher = new FolderWorkDispatcher(Log, config.FolderToStore);
27+
}
28+
else
29+
{
30+
FolderWorkDispatcher = null!;
31+
}
2232
}
2333

2434
public Configuration Config { get; }
@@ -27,6 +37,7 @@ public App(Configuration config)
2737
public CancellationTokenSource Cts { get; } = new CancellationTokenSource();
2838
public CidRepo CidRepo { get; }
2939
public Performance Performance { get; }
40+
public FolderWorkDispatcher FolderWorkDispatcher { get; }
3041

3142
private IFileGenerator CreateGenerator()
3243
{

‎Tools/AutoClient/AutomaticPurchaser.cs

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
using CodexOpenApi;
2+
using CodexPlugin;
3+
using Logging;
4+
using Newtonsoft.Json;
5+
using Utils;
6+
7+
namespace AutoClient
8+
{
9+
public class AutomaticPurchaser
10+
{
11+
private readonly ILog log;
12+
private readonly ICodexInstance instance;
13+
private readonly CodexNode codex;
14+
private Task workerTask = Task.CompletedTask;
15+
private App app => instance.App;
16+
17+
public AutomaticPurchaser(ILog log, ICodexInstance instance, CodexNode codex)
18+
{
19+
this.log = log;
20+
this.instance = instance;
21+
this.codex = codex;
22+
}
23+
24+
public void Start()
25+
{
26+
workerTask = Task.Run(Worker);
27+
}
28+
29+
public void Stop()
30+
{
31+
workerTask.Wait();
32+
}
33+
34+
private async Task Worker()
35+
{
36+
log.Log("Worker started.");
37+
while (!app.Cts.Token.IsCancellationRequested)
38+
{
39+
try
40+
{
41+
var pid = await StartNewPurchase();
42+
await WaitTillFinished(pid);
43+
await DownloadForeignCid();
44+
}
45+
catch (Exception ex)
46+
{
47+
log.Error("Worker failed with: " + ex);
48+
await Task.Delay(TimeSpan.FromHours(6));
49+
}
50+
}
51+
}
52+
53+
private async Task DownloadForeignCid()
54+
{
55+
var cid = app.CidRepo.GetForeignCid(instance.NodeId);
56+
if (cid == null) return;
57+
58+
var size = app.CidRepo.GetSizeForCid(cid);
59+
if (size == null) return;
60+
61+
var filename = Guid.NewGuid().ToString().ToLowerInvariant();
62+
await codex.DownloadCid(filename, cid, size);
63+
64+
DeleteFile(filename);
65+
}
66+
67+
private async Task<string> StartNewPurchase()
68+
{
69+
var file = await CreateFile();
70+
try
71+
{
72+
var cid = await codex.UploadFile(file);
73+
var response = await codex.RequestStorage(cid);
74+
return response.PurchaseId;
75+
}
76+
finally
77+
{
78+
DeleteFile(file);
79+
}
80+
}
81+
82+
private async Task<string> CreateFile()
83+
{
84+
return await app.Generator.Generate();
85+
}
86+
87+
private void DeleteFile(string file)
88+
{
89+
try
90+
{
91+
File.Delete(file);
92+
}
93+
catch (Exception exc)
94+
{
95+
app.Log.Error($"Failed to delete file '{file}': {exc}");
96+
}
97+
}
98+
99+
private async Task WaitTillFinished(string pid)
100+
{
101+
try
102+
{
103+
var emptyResponseTolerance = 10;
104+
while (!app.Cts.Token.IsCancellationRequested)
105+
{
106+
var purchase = await codex.GetStoragePurchase(pid);
107+
if (purchase == null)
108+
{
109+
await FixedShortDelay();
110+
emptyResponseTolerance--;
111+
if (emptyResponseTolerance == 0)
112+
{
113+
log.Log("Received 10 empty responses. Stop tracking this purchase.");
114+
await ExpiryTimeDelay();
115+
return;
116+
}
117+
continue;
118+
}
119+
if (purchase.IsCancelled)
120+
{
121+
app.Performance.StorageContractCancelled();
122+
return;
123+
}
124+
if (purchase.IsError)
125+
{
126+
app.Performance.StorageContractErrored(purchase.Error);
127+
return;
128+
}
129+
if (purchase.IsFinished)
130+
{
131+
app.Performance.StorageContractFinished();
132+
return;
133+
}
134+
if (purchase.IsStarted)
135+
{
136+
app.Performance.StorageContractStarted();
137+
await FixedDurationDelay();
138+
}
139+
140+
await FixedShortDelay();
141+
}
142+
}
143+
catch (Exception ex)
144+
{
145+
log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time.");
146+
await ExpiryTimeDelay();
147+
}
148+
}
149+
150+
private async Task FixedDurationDelay()
151+
{
152+
await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
153+
}
154+
155+
private async Task ExpiryTimeDelay()
156+
{
157+
await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
158+
}
159+
160+
private async Task FixedShortDelay()
161+
{
162+
await Task.Delay(15 * 1000, app.Cts.Token);
163+
}
164+
}
165+
}

‎Tools/AutoClient/CidRepo.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ public void Add(string nodeId, string cid, long knownSize)
1717
lock (_lock)
1818
{
1919
entries.Add(new CidEntry(nodeId, cid, knownSize));
20+
if (entries.Count > 1000) entries.Clear();
2021
}
2122
}
2223

0 commit comments

Comments
 (0)