Skip to content

Commit f7de11e

Browse files
benbierensbenbierens
committed
Cleanup foldersaver mode
1 parent 278b6c9 commit f7de11e

File tree

13 files changed

+359
-728
lines changed

13 files changed

+359
-728
lines changed

‎ProjectPlugins/CodexClient/StoragePurchaseContract.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ public class StoragePurchaseContract : IStoragePurchaseContract
2323
private readonly ILog log;
2424
private readonly CodexAccess codexAccess;
2525
private readonly ICodexNodeHooks hooks;
26-
private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(30);
26+
private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(60);
2727
private readonly DateTime contractPendingUtc = DateTime.UtcNow;
2828
private DateTime? contractSubmittedUtc = DateTime.UtcNow;
2929
private DateTime? contractStartedUtc;

‎Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs

Lines changed: 57 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,9 @@ public ICodexNodeGroup StartHosts()
6565
var config = GetContracts().Deployment.Config;
6666
foreach (var host in hosts)
6767
{
68-
Assert.That(GetTstBalance(host).TstWei, Is.EqualTo(StartingBalanceTST.Tst().TstWei));
69-
Assert.That(GetEthBalance(host).Wei, Is.EqualTo(StartingBalanceEth.Eth().Wei));
70-
68+
AssertTstBalance(host, StartingBalanceTST.Tst(), nameof(StartHosts));
69+
AssertEthBalance(host, StartingBalanceEth.Eth(), nameof(StartHosts));
70+
7171
host.Marketplace.MakeStorageAvailable(new StorageAvailability(
7272
totalSpace: HostAvailabilitySize,
7373
maxDuration: HostAvailabilityMaxDuration,
@@ -78,22 +78,63 @@ public ICodexNodeGroup StartHosts()
7878
return hosts;
7979
}
8080

81-
public TestToken GetTstBalance(ICodexNode node)
81+
public void AssertTstBalance(EthAddress address, TestToken expectedBalance, string message)
82+
{
83+
var retry = GetBalanceAssertRetry();
84+
retry.Run(() =>
85+
{
86+
var balance = GetTstBalance(address);
87+
88+
Assert.That(balance, Is.EqualTo(expectedBalance), message);
89+
});
90+
}
91+
92+
public void AssertTstBalance(ICodexNode node, TestToken expectedBalance, string message)
93+
{
94+
var retry = GetBalanceAssertRetry();
95+
retry.Run(() =>
96+
{
97+
var balance = GetTstBalance(node);
98+
99+
Assert.That(balance, Is.EqualTo(expectedBalance), message);
100+
});
101+
}
102+
103+
public void AssertEthBalance(ICodexNode node, Ether expectedBalance, string message)
104+
{
105+
var retry = GetBalanceAssertRetry();
106+
retry.Run(() =>
107+
{
108+
var balance = GetEthBalance(node);
109+
110+
Assert.That(balance, Is.EqualTo(expectedBalance), message);
111+
});
112+
}
113+
114+
private Retry GetBalanceAssertRetry()
115+
{
116+
return new Retry("AssertBalance",
117+
maxTimeout: TimeSpan.FromMinutes(30.0),
118+
sleepAfterFail: TimeSpan.FromSeconds(10.0),
119+
onFail: f => { });
120+
}
121+
122+
private TestToken GetTstBalance(ICodexNode node)
82123
{
83124
return GetContracts().GetTestTokenBalance(node);
84125
}
85126

86-
public TestToken GetTstBalance(EthAddress address)
127+
private TestToken GetTstBalance(EthAddress address)
87128
{
88129
return GetContracts().GetTestTokenBalance(address);
89130
}
90131

91-
public Ether GetEthBalance(ICodexNode node)
132+
private Ether GetEthBalance(ICodexNode node)
92133
{
93134
return GetGeth().GetEthBalance(node);
94135
}
95136

96-
public Ether GetEthBalance(EthAddress address)
137+
private Ether GetEthBalance(EthAddress address)
97138
{
98139
return GetGeth().GetEthBalance(address);
99140
}
@@ -141,10 +182,9 @@ public SlotFill[] GetOnChainSlotFills(ICodexNodeGroup possibleHosts)
141182

142183
protected void AssertClientHasPaidForContract(TestToken pricePerBytePerSecond, ICodexNode client, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
143184
{
144-
var balance = GetTstBalance(client);
145185
var expectedBalance = StartingBalanceTST.Tst() - GetContractFinalCost(pricePerBytePerSecond, contract, hosts);
146186

147-
Assert.That(balance, Is.EqualTo(expectedBalance), "Client balance incorrect.");
187+
AssertTstBalance(client, expectedBalance, "Client balance incorrect.");
148188
}
149189

150190
protected void AssertHostsWerePaidForContract(TestToken pricePerBytePerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
@@ -162,20 +202,10 @@ protected void AssertHostsWerePaidForContract(TestToken pricePerBytePerSecond, I
162202
expectedBalances[fill.Host.EthAddress] += GetContractCostPerSlot(pricePerBytePerSecond, slotSize, slotDuration);
163203
}
164204

165-
var retry = new Retry(nameof(AssertHostsWerePaidForContract),
166-
maxTimeout: TimeSpan.FromMinutes(30),
167-
sleepAfterFail: TimeSpan.FromSeconds(10),
168-
onFail: f => { }
169-
);
170-
171-
retry.Run(() =>
205+
foreach (var pair in expectedBalances)
172206
{
173-
foreach (var pair in expectedBalances)
174-
{
175-
var balance = GetTstBalance(pair.Key);
176-
Assert.That(balance, Is.EqualTo(pair.Value), "Host was not paid for storage.");
177-
}
178-
});
207+
AssertTstBalance(pair.Key, pair.Value, "Host was not paid for storage.");
208+
}
179209
}
180210

181211
protected void AssertHostsCollateralsAreUnchanged(ICodexNodeGroup hosts)
@@ -184,7 +214,11 @@ protected void AssertHostsCollateralsAreUnchanged(ICodexNodeGroup hosts)
184214
// All host balances should be equal to or greater than the starting balance.
185215
foreach (var host in hosts)
186216
{
187-
Assert.That(GetTstBalance(host), Is.GreaterThanOrEqualTo(StartingBalanceTST.Tst()));
217+
var retry = GetBalanceAssertRetry();
218+
retry.Run(() =>
219+
{
220+
Assert.That(GetTstBalance(host), Is.GreaterThanOrEqualTo(StartingBalanceTST.Tst()));
221+
});
188222
}
189223
}
190224

‎Tools/AutoClient/App.cs

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,6 @@ public App(Configuration config)
2222
new ConsoleLog()
2323
));
2424

25-
if (!string.IsNullOrEmpty(config.FolderToStore))
26-
{
27-
FolderWorkDispatcher = new FolderWorkDispatcher(Log, config.FolderToStore);
28-
}
29-
else
30-
{
31-
FolderWorkDispatcher = null!;
32-
}
33-
3425
var httpFactory = new HttpFactory(Log, new AutoClientWebTimeSet());
3526

3627
CodexNodeFactory = new CodexNodeFactory(log: Log, httpFactory: httpFactory, dataDir: Config.DataPath);
@@ -41,7 +32,6 @@ public App(Configuration config)
4132
public IFileGenerator Generator { get; }
4233
public CancellationTokenSource Cts { get; } = new CancellationTokenSource();
4334
public Performance Performance { get; }
44-
public FolderWorkDispatcher FolderWorkDispatcher { get; }
4535
public CodexNodeFactory CodexNodeFactory { get; }
4636

4737
private IFileGenerator CreateGenerator()

‎Tools/AutoClient/Modes/FolderStore/FileSaver.cs

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
using CodexClient;
2+
using Logging;
3+
using Utils;
4+
5+
namespace AutoClient.Modes.FolderStore
6+
{
7+
public class FileSaver
8+
{
9+
private readonly ILog log;
10+
private readonly CodexWrapper instance;
11+
private readonly string folderFile;
12+
private readonly FileStatus entry;
13+
14+
public FileSaver(ILog log, CodexWrapper instance, string folderFile, FileStatus entry)
15+
{
16+
this.log = log;
17+
this.instance = instance;
18+
this.folderFile = folderFile;
19+
this.entry = entry;
20+
}
21+
22+
public bool HasFailed { get; private set; }
23+
24+
public void Process()
25+
{
26+
HasFailed = false;
27+
if (HasRecentPurchase(entry))
28+
{
29+
Log($"Purchase running: '{entry.PurchaseId}'");
30+
return;
31+
}
32+
33+
EnsureBasicCid();
34+
CreateNewPurchase();
35+
}
36+
37+
private void EnsureBasicCid()
38+
{
39+
if (IsBasicCidAvailable()) return;
40+
UploadFile();
41+
}
42+
43+
private bool IsBasicCidAvailable()
44+
{
45+
if (string.IsNullOrEmpty(entry.BasicCid)) return false;
46+
return NodeContainsBasicCid();
47+
}
48+
49+
private bool HasRecentPurchase(FileStatus entry)
50+
{
51+
if (string.IsNullOrEmpty(entry.PurchaseId)) return false;
52+
var purchase = GetPurchase(entry.PurchaseId);
53+
if (purchase == null) return false;
54+
if (!purchase.IsStarted) return false;
55+
56+
// Purchase is started. But, if it finishes soon, we will treat it as already finished.
57+
var threshold = DateTime.UtcNow + TimeSpan.FromHours(3.0);
58+
if (entry.PurchaseFinishedUtc < threshold)
59+
{
60+
Log($"Running purchase will expire soon.");
61+
return false;
62+
}
63+
return true;
64+
}
65+
66+
private StoragePurchase? GetPurchase(string purchaseId)
67+
{
68+
return instance.GetStoragePurchase(purchaseId);
69+
}
70+
71+
private bool NodeContainsBasicCid()
72+
{
73+
try
74+
{
75+
var result = instance.Node.DownloadManifestOnly(new ContentId(entry.BasicCid));
76+
return !string.IsNullOrEmpty(result.Cid.Id);
77+
}
78+
catch
79+
{
80+
Log("Failed to download manifest for basicCid");
81+
return false;
82+
}
83+
}
84+
85+
private void UploadFile()
86+
{
87+
try
88+
{
89+
entry.BasicCid = instance.UploadFile(folderFile).Id;
90+
Log($"Successfully uploaded. BasicCid: '{entry.BasicCid}'");
91+
}
92+
catch (Exception exc)
93+
{
94+
entry.BasicCid = string.Empty;
95+
log.Error("Failed to upload: " + exc);
96+
HasFailed = true;
97+
}
98+
}
99+
100+
private void CreateNewPurchase()
101+
{
102+
if (string.IsNullOrEmpty(entry.BasicCid)) return;
103+
104+
try
105+
{
106+
var request = instance.RequestStorage(new ContentId(entry.BasicCid));
107+
entry.EncodedCid = request.Purchase.ContentId.Id;
108+
entry.PurchaseId = request.PurchaseId;
109+
110+
request.WaitForStorageContractSubmitted();
111+
request.WaitForStorageContractStarted();
112+
113+
entry.PurchaseFinishedUtc = DateTime.UtcNow + request.Purchase.Duration;
114+
Log($"Successfully started new purchase: '{entry.PurchaseId}' for {Time.FormatDuration(request.Purchase.Duration)} ");
115+
}
116+
catch (Exception exc)
117+
{
118+
entry.EncodedCid = string.Empty;
119+
entry.PurchaseId = string.Empty;
120+
log.Error("Failed to start new purchase: " + exc);
121+
HasFailed = true;
122+
}
123+
}
124+
125+
private void Log(string msg)
126+
{
127+
log.Log(msg);
128+
}
129+
}
130+
}

‎Tools/AutoClient/Modes/FolderStore/FileStatus.cs

Lines changed: 0 additions & 63 deletions
This file was deleted.

0 commit comments

Comments
 (0)