Skip to content

Commit 6a128fd

Browse files
benbierensbenbierens
committed
Merge branch 'feature/proofs-and-frees'
2 parents 6ad001c + 44d14e5 commit 6a128fd

File tree

23 files changed

+283
-44
lines changed

23 files changed

+283
-44
lines changed

‎Framework/KubernetesWorkflow/LogHandler.cs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,9 @@ public WriteToFileLogHandler(ILog sourceLog, string description, string addFileN
4040

4141
protected override void ProcessLine(string line)
4242
{
43-
if (line.Contains("Received JSON-RPC response")) return;
43+
// This line is not useful and has no topic so we can't filter it with
44+
// normal log-level controls.
45+
if (line.Contains("Received JSON-RPC response") && !line.Contains("topics=")) return;
4446
if (line.Contains("object field not marked with serialize, skipping")) return;
4547

4648
LogFile.Write(line);

‎ProjectPlugins/CodexClient/CodexAccess.cs

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ public class CodexAccess
1111
private readonly ILog log;
1212
private readonly IHttpFactory httpFactory;
1313
private readonly IProcessControl processControl;
14-
private ICodexInstance instance;
14+
private readonly ICodexInstance instance;
1515
private readonly Mapper mapper = new Mapper();
1616

1717
public CodexAccess(ILog log, IHttpFactory httpFactory, IProcessControl processControl, ICodexInstance instance)
@@ -25,8 +25,6 @@ public CodexAccess(ILog log, IHttpFactory httpFactory, IProcessControl processCo
2525
public void Stop(bool waitTillStopped)
2626
{
2727
processControl.Stop(waitTillStopped);
28-
// Prevents accidental use after stop:
29-
instance = null!;
3028
}
3129

3230
public IDownloadedLog DownloadLog(string additionalName = "")
@@ -143,7 +141,7 @@ public LocalDatasetList LocalFiles()
143141
return mapper.Map(OnCodex(api => api.ListDataAsync()));
144142
}
145143

146-
public StorageAvailability SalesAvailability(StorageAvailability request)
144+
public StorageAvailability SalesAvailability(CreateStorageAvailability request)
147145
{
148146
var body = mapper.Map(request);
149147
var read = OnCodex(api => api.OfferStorageAsync(body));

‎ProjectPlugins/CodexClient/Mapper.cs

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ public LocalDataset Map(CodexOpenApi.DataItem dataItem)
3636
};
3737
}
3838

39-
public CodexOpenApi.SalesAvailability Map(StorageAvailability availability)
39+
public CodexOpenApi.SalesAvailability Map(CreateStorageAvailability availability)
4040
{
4141
return new CodexOpenApi.SalesAvailability
4242
{
@@ -70,15 +70,13 @@ public StorageAvailability Map(CodexOpenApi.SalesAvailabilityREAD availability)
7070
{
7171
return new StorageAvailability
7272
(
73+
availability.Id,
7374
ToByteSize(availability.TotalSize),
7475
ToTimespan(availability.Duration),
7576
new TestToken(ToBigInt(availability.MinPricePerBytePerSecond)),
76-
new TestToken(ToBigInt(availability.TotalCollateral))
77-
)
78-
{
79-
Id = availability.Id,
80-
FreeSpace = ToByteSize(availability.FreeSize),
81-
};
77+
new TestToken(ToBigInt(availability.TotalCollateral)),
78+
ToByteSize(availability.FreeSize)
79+
);
8280
}
8381

8482
public StoragePurchase Map(CodexOpenApi.Purchase purchase)

‎ProjectPlugins/CodexClient/MarketplaceAccess.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ namespace CodexClient
66
{
77
public interface IMarketplaceAccess
88
{
9-
string MakeStorageAvailable(StorageAvailability availability);
9+
string MakeStorageAvailable(CreateStorageAvailability availability);
1010
StorageAvailability[] GetAvailabilities();
1111
IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase);
1212
}
@@ -49,7 +49,7 @@ public IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
4949
return new StoragePurchaseContract(log, codexAccess, response, purchase, hooks);
5050
}
5151

52-
public string MakeStorageAvailable(StorageAvailability availability)
52+
public string MakeStorageAvailable(CreateStorageAvailability availability)
5353
{
5454
availability.Log(log);
5555

@@ -77,7 +77,7 @@ private void Log(string msg)
7777

7878
public class MarketplaceUnavailable : IMarketplaceAccess
7979
{
80-
public string MakeStorageAvailable(StorageAvailability availability)
80+
public string MakeStorageAvailable(CreateStorageAvailability availability)
8181
{
8282
Unavailable();
8383
throw new NotImplementedException();

‎ProjectPlugins/CodexClient/MarketplaceTypes.cs

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -84,30 +84,59 @@ public class StorageContent
8484
//public PoRParameters Por { get; set; }
8585
}
8686

87+
public class CreateStorageAvailability
88+
{
89+
public CreateStorageAvailability(ByteSize totalSpace, TimeSpan maxDuration, TestToken minPricePerBytePerSecond, TestToken totalCollateral)
90+
{
91+
TotalSpace = totalSpace;
92+
MaxDuration = maxDuration;
93+
MinPricePerBytePerSecond = minPricePerBytePerSecond;
94+
TotalCollateral = totalCollateral;
95+
}
96+
97+
public ByteSize TotalSpace { get; }
98+
public TimeSpan MaxDuration { get; }
99+
public TestToken MinPricePerBytePerSecond { get; }
100+
public TestToken TotalCollateral { get; }
101+
102+
public void Log(ILog log)
103+
{
104+
log.Log($"Create storage Availability: (" +
105+
$"totalSize: {TotalSpace}, " +
106+
$"maxDuration: {Time.FormatDuration(MaxDuration)}, " +
107+
$"minPricePerBytePerSecond: {MinPricePerBytePerSecond}, " +
108+
$"totalCollateral: {TotalCollateral})");
109+
}
110+
}
111+
87112
public class StorageAvailability
88113
{
89-
public StorageAvailability(ByteSize totalSpace, TimeSpan maxDuration, TestToken minPricePerBytePerSecond, TestToken totalCollateral)
114+
public StorageAvailability(string id, ByteSize totalSpace, TimeSpan maxDuration, TestToken minPricePerBytePerSecond, TestToken totalCollateral, ByteSize freeSpace)
90115
{
116+
Id = id;
91117
TotalSpace = totalSpace;
92118
MaxDuration = maxDuration;
93119
MinPricePerBytePerSecond = minPricePerBytePerSecond;
94120
TotalCollateral = totalCollateral;
121+
FreeSpace = freeSpace;
95122
}
96123

97-
public string Id { get; set; } = string.Empty;
124+
public string Id { get; }
98125
public ByteSize TotalSpace { get; }
99126
public TimeSpan MaxDuration { get; }
100127
public TestToken MinPricePerBytePerSecond { get; }
101128
public TestToken TotalCollateral { get; }
102-
public ByteSize FreeSpace { get; set; } = ByteSize.Zero;
129+
public ByteSize FreeSpace { get; }
103130

104131
public void Log(ILog log)
105132
{
106133
log.Log($"Storage Availability: (" +
134+
$"id: {Id}, " +
107135
$"totalSize: {TotalSpace}, " +
108136
$"maxDuration: {Time.FormatDuration(MaxDuration)}, " +
109137
$"minPricePerBytePerSecond: {MinPricePerBytePerSecond}, " +
110-
$"totalCollateral: {TotalCollateral})");
138+
$"totalCollateral: {TotalCollateral}, " +
139+
$"freeSpace: {FreeSpace})");
111140
}
112141
}
113142
}

‎ProjectPlugins/CodexClient/StoragePurchaseContract.cs

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,13 @@ public interface IStoragePurchaseContract
1414
void WaitForStorageContractSubmitted();
1515
void WaitForStorageContractStarted();
1616
void WaitForStorageContractFinished();
17-
void WaitForContractFailed();
17+
void WaitForContractFailed(IMarketplaceConfigInput config);
18+
}
19+
20+
public interface IMarketplaceConfigInput
21+
{
22+
int MaxNumberOfSlashes { get; }
23+
TimeSpan PeriodDuration { get; }
1824
}
1925

2026
public class StoragePurchaseContract : IStoragePurchaseContract
@@ -99,17 +105,40 @@ public void WaitForStorageContractFinished()
99105
AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished));
100106
}
101107

102-
public void WaitForContractFailed()
108+
public void WaitForContractFailed(IMarketplaceConfigInput config)
103109
{
104110
if (!contractStartedUtc.HasValue)
105111
{
106112
WaitForStorageContractStarted();
107113
}
108114
var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value;
109115
var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
116+
var minTimeout = TimeNeededToFailEnoughProofsToFreeASlot(config);
117+
118+
if (timeout < minTimeout)
119+
{
120+
throw new ArgumentOutOfRangeException(
121+
$"Test is misconfigured. Assuming a proof is required every period, it will take {Time.FormatDuration(minTimeout)} " +
122+
$"to fail enough proofs for a slot to be freed. But, the storage contract will complete in {Time.FormatDuration(timeout)}. " +
123+
$"Increase the duration."
124+
);
125+
}
126+
110127
WaitForStorageContractState(timeout, StoragePurchaseState.Failed);
111128
}
112129

130+
private TimeSpan TimeNeededToFailEnoughProofsToFreeASlot(IMarketplaceConfigInput config)
131+
{
132+
var numMissedProofsRequiredForFree = config.MaxNumberOfSlashes;
133+
var timePerProof = config.PeriodDuration;
134+
var result = timePerProof * (numMissedProofsRequiredForFree + 1);
135+
136+
// Times 2!
137+
// Because of pointer-downtime it's possible that some periods even though there's a probability of 100%
138+
// will not require any proof. To be safe we take twice the required time.
139+
return result * 2;
140+
}
141+
113142
private void WaitForStorageContractState(TimeSpan timeout, StoragePurchaseState desiredState, int sleep = 1000)
114143
{
115144
var waitStart = DateTime.UtcNow;

‎ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public ChainState(ILog log, ICodexContracts contracts, IChainStateChangeHandler
4747
handler = changeHandler;
4848
this.doProofPeriodMonitoring = doProofPeriodMonitoring;
4949
TotalSpan = new TimeRange(startUtc, startUtc);
50-
PeriodMonitor = new PeriodMonitor(contracts);
50+
PeriodMonitor = new PeriodMonitor(log, contracts);
5151
}
5252

5353
public TimeRange TotalSpan { get; private set; }

‎ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
1-
using Utils;
1+
using Logging;
2+
using Utils;
23

34
namespace CodexContractsPlugin.ChainMonitor
45
{
56
public class PeriodMonitor
67
{
8+
private readonly ILog log;
79
private readonly ICodexContracts contracts;
810
private readonly List<PeriodReport> reports = new List<PeriodReport>();
911
private ulong? currentPeriod = null;
1012

11-
public PeriodMonitor(ICodexContracts contracts)
13+
public PeriodMonitor(ILog log, ICodexContracts contracts)
1214
{
15+
this.log = log;
1316
this.contracts = contracts;
1417
}
1518

@@ -58,7 +61,9 @@ private void CreateReportForPeriod(ulong lastBlockInPeriod, ulong periodNumber,
5861
}
5962
}
6063
}
61-
reports.Add(new PeriodReport(periodNumber, total, required, missed.ToArray()));
64+
var report = new PeriodReport(periodNumber, total, required, missed.ToArray());
65+
log.Log($"Period report: {report}");
66+
reports.Add(report);
6267
}
6368
}
6469

@@ -105,6 +110,16 @@ public PeriodReport(ulong periodNumber, ulong totalNumSlots, ulong totalProofsRe
105110
public ulong TotalNumSlots { get; }
106111
public ulong TotalProofsRequired { get; }
107112
public PeriodProofMissed[] MissedProofs { get; }
113+
114+
public override string ToString()
115+
{
116+
var missed = "None";
117+
if (MissedProofs.Length > 0)
118+
{
119+
missed = string.Join("+", MissedProofs.Select(p => $"{p.FormatHost()} missed {p.Request.Request.Id} slot {p.SlotIndex}"));
120+
}
121+
return $"Period:{PeriodNumber}=[Slots:{TotalNumSlots},ProofsRequired:{TotalProofsRequired},ProofsMissed:{missed}]";
122+
}
108123
}
109124

110125
public class PeriodProofMissed
@@ -119,5 +134,11 @@ public PeriodProofMissed(EthAddress? host, IChainStateRequest request, int slotI
119134
public EthAddress? Host { get; }
120135
public IChainStateRequest Request { get; }
121136
public int SlotIndex { get; }
137+
138+
public string FormatHost()
139+
{
140+
if (Host == null) return "Unknown host";
141+
return Host.Address;
142+
}
122143
}
123144
}

‎ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,6 @@ public ulong GetPeriodNumber(DateTime utc)
146146

147147
public void WaitUntilNextPeriod()
148148
{
149-
log.Log("Waiting until next proof period...");
150149
var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
151150
var periodSeconds = (int)Deployment.Config.Proofs.Period;
152151
var secondsLeft = now % periodSeconds;

‎ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using BlockchainUtils;
33
using Nethereum.Hex.HexConvertors.Extensions;
44
using Newtonsoft.Json;
5+
using CodexClient;
56
using Utils;
67

78
namespace CodexContractsPlugin.Marketplace
@@ -92,5 +93,11 @@ public partial class ReserveSlotFunction : IHasBlock, IHasRequestId, IHasSlotInd
9293
[JsonIgnore]
9394
public BlockTimeEntry Block { get; set; }
9495
}
96+
97+
public partial class MarketplaceConfig : IMarketplaceConfigInput
98+
{
99+
public int MaxNumberOfSlashes => this.Collateral.MaxNumberOfSlashes;
100+
public TimeSpan PeriodDuration => TimeSpan.FromSeconds(this.Proofs.Period);
101+
}
95102
}
96103
#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

‎ProjectPlugins/CodexContractsPlugin/Marketplace/Marketplace.cs

Lines changed: 12 additions & 1 deletion
Large diffs are not rendered by default.

‎ProjectPlugins/CodexPlugin/CodexDockerImage.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
{
33
public class CodexDockerImage
44
{
5-
private const string DefaultDockerImage = "codexstorage/nim-codex:0.2.3-dist-tests";
5+
private const string DefaultDockerImage = "codexstorage/nim-codex:sha-2e1306a-dist-tests";
66

77
public static string Override { get; set; } = string.Empty;
88

‎Tests/CodexReleaseTests/MarketTests/ChainMonitor.cs

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
using CodexContractsPlugin;
2+
using CodexContractsPlugin.ChainMonitor;
3+
using Logging;
4+
5+
namespace CodexReleaseTests.MarketTests
6+
{
7+
public class ChainMonitor
8+
{
9+
private readonly ChainState chainMonitor;
10+
private readonly TimeSpan interval;
11+
private CancellationTokenSource cts = new CancellationTokenSource();
12+
private Task worker = null!;
13+
14+
public ChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc, TimeSpan interval)
15+
{
16+
chainMonitor = new ChainState(log, contracts, new DoNothingChainEventHandler(), startUtc, true);
17+
this.interval = interval;
18+
}
19+
20+
public void Start()
21+
{
22+
cts = new CancellationTokenSource();
23+
worker = Task.Run(Worker);
24+
}
25+
26+
public void Stop()
27+
{
28+
cts.Cancel();
29+
worker.Wait();
30+
31+
worker = null!;
32+
cts = null!;
33+
}
34+
35+
public PeriodMonitorResult GetPeriodReports()
36+
{
37+
return chainMonitor.PeriodMonitor.GetAndClearReports();
38+
}
39+
40+
private void Worker()
41+
{
42+
while (!cts.IsCancellationRequested)
43+
{
44+
Thread.Sleep(interval);
45+
chainMonitor.Update();
46+
}
47+
}
48+
}
49+
}

0 commit comments

Comments
 (0)