Skip to content

Commit a0abea4

Browse files
benbierensbenbierens
committed
Merge branch 'master' into feature/block-retransmit
# Conflicts: # ProjectPlugins/CodexPlugin/ApiChecker.cs
2 parents 3e245b7 + 2f39327 commit a0abea4

32 files changed

+1262
-297
lines changed

‎Framework/DiscordRewards/GiveRewardsCommand.cs

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
public class GiveRewardsCommand
44
{
55
public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
6-
public string[] EventsOverview { get; set; } = Array.Empty<string>();
6+
public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty<ChainEventMessage>();
7+
public string[] Errors { get; set; } = Array.Empty<string>();
78

89
public bool HasAny()
910
{
@@ -16,4 +17,10 @@ public class RewardUsersCommand
1617
public ulong RewardId { get; set; }
1718
public string[] UserAddresses { get; set; } = Array.Empty<string>();
1819
}
20+
21+
public class ChainEventMessage
22+
{
23+
public ulong BlockNumber { get; set; }
24+
public string Message { get; set; } = string.Empty;
25+
}
1926
}

‎Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ public BlockTimeEntry Get(ulong blockNumber)
2929
public ulong? GetHighestBlockNumberBefore(DateTime moment)
3030
{
3131
bounds.Initialize();
32-
if (moment <= bounds.Genesis.Utc) return null;
32+
if (moment < bounds.Genesis.Utc) return null;
33+
if (moment == bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
3334
if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber;
3435

3536
return Log(() => Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector));
@@ -38,7 +39,8 @@ public BlockTimeEntry Get(ulong blockNumber)
3839
public ulong? GetLowestBlockNumberAfter(DateTime moment)
3940
{
4041
bounds.Initialize();
41-
if (moment >= bounds.Current.Utc) return null;
42+
if (moment > bounds.Current.Utc) return null;
43+
if (moment == bounds.Current.Utc) return bounds.Current.BlockNumber;
4244
if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
4345

4446
return Log(()=> Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector)); ;

‎ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ public interface IChainStateChangeHandler
1717
void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex);
1818
void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex);
1919
void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex);
20+
21+
void OnError(string msg);
2022
}
2123

2224
public class RequestEvent
@@ -67,7 +69,11 @@ public int Update(DateTime toUtc)
6769
private void Apply(ChainEvents events)
6870
{
6971
if (events.BlockInterval.TimeRange.From < TotalSpan.From)
70-
throw new Exception("Attempt to update ChainState with set of events from before its current record.");
72+
{
73+
var msg = "Attempt to update ChainState with set of events from before its current record.";
74+
handler.OnError(msg);
75+
throw new Exception(msg);
76+
}
7177

7278
log.Log($"ChainState updating: {events.BlockInterval}");
7379

@@ -110,31 +116,31 @@ private void ApplyEvent(Request request)
110116

111117
private void ApplyEvent(RequestFulfilledEventDTO @event)
112118
{
113-
var r = FindRequest(@event.RequestId);
119+
var r = FindRequest(@event);
114120
if (r == null) return;
115121
r.UpdateState(@event.Block.BlockNumber, RequestState.Started);
116122
handler.OnRequestFulfilled(new RequestEvent(@event.Block, r));
117123
}
118124

119125
private void ApplyEvent(RequestCancelledEventDTO @event)
120126
{
121-
var r = FindRequest(@event.RequestId);
127+
var r = FindRequest(@event);
122128
if (r == null) return;
123129
r.UpdateState(@event.Block.BlockNumber, RequestState.Cancelled);
124130
handler.OnRequestCancelled(new RequestEvent(@event.Block, r));
125131
}
126132

127133
private void ApplyEvent(RequestFailedEventDTO @event)
128134
{
129-
var r = FindRequest(@event.RequestId);
135+
var r = FindRequest(@event);
130136
if (r == null) return;
131137
r.UpdateState(@event.Block.BlockNumber, RequestState.Failed);
132138
handler.OnRequestFailed(new RequestEvent(@event.Block, r));
133139
}
134140

135141
private void ApplyEvent(SlotFilledEventDTO @event)
136142
{
137-
var r = FindRequest(@event.RequestId);
143+
var r = FindRequest(@event);
138144
if (r == null) return;
139145
r.Hosts.Add(@event.Host, (int)@event.SlotIndex);
140146
r.Log($"[{@event.Block.BlockNumber}] SlotFilled (host:'{@event.Host}', slotIndex:{@event.SlotIndex})");
@@ -143,7 +149,7 @@ private void ApplyEvent(SlotFilledEventDTO @event)
143149

144150
private void ApplyEvent(SlotFreedEventDTO @event)
145151
{
146-
var r = FindRequest(@event.RequestId);
152+
var r = FindRequest(@event);
147153
if (r == null) return;
148154
r.Hosts.RemoveHost((int)@event.SlotIndex);
149155
r.Log($"[{@event.Block.BlockNumber}] SlotFreed (slotIndex:{@event.SlotIndex})");
@@ -152,7 +158,7 @@ private void ApplyEvent(SlotFreedEventDTO @event)
152158

153159
private void ApplyEvent(SlotReservationsFullEventDTO @event)
154160
{
155-
var r = FindRequest(@event.RequestId);
161+
var r = FindRequest(@event);
156162
if (r == null) return;
157163
r.Log($"[{@event.Block.BlockNumber}] SlotReservationsFull (slotIndex:{@event.SlotIndex})");
158164
handler.OnSlotReservationsFull(new RequestEvent(@event.Block, r), @event.SlotIndex);
@@ -171,10 +177,23 @@ private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
171177
}
172178
}
173179

174-
private ChainStateRequest? FindRequest(byte[] requestId)
180+
private ChainStateRequest? FindRequest(IHasRequestId request)
175181
{
176-
var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, requestId));
177-
if (r == null) log.Log("Unable to find request by ID!");
182+
var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, request.RequestId));
183+
if (r == null)
184+
{
185+
var blockNumber = "unknown";
186+
if (request is IHasBlock blk)
187+
{
188+
blockNumber = blk.Block.BlockNumber.ToString();
189+
}
190+
191+
var msg = $"Received event of type '{request.GetType()}' in block '{blockNumber}' for request by Id: '{request.RequestId}'. " +
192+
$"Failed to find request. Request creation event not seen! (Tracker start time: {TotalSpan.From})";
193+
194+
log.Error(msg);
195+
handler.OnError(msg);
196+
}
178197
return r;
179198
}
180199

‎ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,5 @@
11
using GethPlugin;
2-
using System;
3-
using System.Collections.Generic;
4-
using System.Linq;
52
using System.Numerics;
6-
using System.Text;
7-
using System.Threading.Tasks;
83

94
namespace CodexContractsPlugin.ChainMonitor
105
{
@@ -56,5 +51,10 @@ public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotInd
5651
{
5752
foreach (var handler in Handlers) handler.OnSlotReservationsFull(requestEvent, slotIndex);
5853
}
54+
55+
public void OnError(string msg)
56+
{
57+
foreach (var handler in Handlers) handler.OnError(msg);
58+
}
5959
}
6060
}

‎ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,5 +36,9 @@ public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
3636
public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
3737
{
3838
}
39+
40+
public void OnError(string msg)
41+
{
42+
}
3943
}
4044
}

‎ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,12 @@ public interface IHasBlock
1010
BlockTimeEntry Block { get; set; }
1111
}
1212

13-
public partial class Request : RequestBase, IHasBlock
13+
public interface IHasRequestId
14+
{
15+
byte[] RequestId { get; set; }
16+
}
17+
18+
public partial class Request : RequestBase, IHasBlock, IHasRequestId
1419
{
1520
[JsonIgnore]
1621
public BlockTimeEntry Block { get; set; }
@@ -28,38 +33,38 @@ public string Id
2833
}
2934
}
3035

31-
public partial class RequestFulfilledEventDTO : IHasBlock
36+
public partial class RequestFulfilledEventDTO : IHasBlock, IHasRequestId
3237
{
3338
[JsonIgnore]
3439
public BlockTimeEntry Block { get; set; }
3540
}
3641

37-
public partial class RequestCancelledEventDTO : IHasBlock
42+
public partial class RequestCancelledEventDTO : IHasBlock, IHasRequestId
3843
{
3944
[JsonIgnore]
4045
public BlockTimeEntry Block { get; set; }
4146
}
4247

43-
public partial class RequestFailedEventDTO : IHasBlock
48+
public partial class RequestFailedEventDTO : IHasBlock, IHasRequestId
4449
{
4550
[JsonIgnore]
4651
public BlockTimeEntry Block { get; set; }
4752
}
4853

49-
public partial class SlotFilledEventDTO : IHasBlock
54+
public partial class SlotFilledEventDTO : IHasBlock, IHasRequestId
5055
{
5156
[JsonIgnore]
5257
public BlockTimeEntry Block { get; set; }
5358
public EthAddress Host { get; set; }
5459
}
5560

56-
public partial class SlotFreedEventDTO : IHasBlock
61+
public partial class SlotFreedEventDTO : IHasBlock, IHasRequestId
5762
{
5863
[JsonIgnore]
5964
public BlockTimeEntry Block { get; set; }
6065
}
6166

62-
public partial class SlotReservationsFullEventDTO : IHasBlock
67+
public partial class SlotReservationsFullEventDTO : IHasBlock, IHasRequestId
6368
{
6469
[JsonIgnore]
6570
public BlockTimeEntry Block { get; set; }

‎ProjectPlugins/CodexPlugin/ApiChecker.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ namespace CodexPlugin
1010
public class ApiChecker
1111
{
1212
// <INSERT-OPENAPI-YAML-HASH>
13-
private const string OpenApiYamlHash = "2E-7C-A2-F3-67-D9-F2-A6-4E-D5-FF-A2-EC-65-ED-59-CE-89-A8-92-57-5E-CF-40-9A-83-49-0B-49-42-5D-EC";
13+
private const string OpenApiYamlHash = "D5-C3-18-71-E8-FF-8F-89-9C-6B-98-3C-F2-C2-D2-37-0A-9F-27-23-35-67-EA-F6-1F-F9-D5-C6-63-34-5A-92";
1414
private const string OpenApiFilePath = "/codex/openapi.yaml";
1515
private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
1616

‎ProjectPlugins/CodexPlugin/CodexAccess.cs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ public string UploadFile(FileStream fileStream, Action<Failure> onFailure)
7373
public Stream DownloadFile(string contentId, Action<Failure> onFailure)
7474
{
7575
var fileResponse = OnCodex(
76-
api => api.DownloadNetworkAsync(contentId),
76+
api => api.DownloadNetworkStreamAsync(contentId),
7777
CreateRetryConfig(nameof(DownloadFile), onFailure));
7878

7979
if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
@@ -88,25 +88,25 @@ public LocalDatasetList LocalFiles()
8888
public StorageAvailability SalesAvailability(StorageAvailability request)
8989
{
9090
var body = mapper.Map(request);
91-
var read = OnCodex<SalesAvailabilityREAD>(api => api.OfferStorageAsync(body));
91+
var read = OnCodex(api => api.OfferStorageAsync(body));
9292
return mapper.Map(read);
9393
}
9494

9595
public StorageAvailability[] GetAvailabilities()
9696
{
97-
var collection = OnCodex<ICollection<SalesAvailabilityREAD>>(api => api.GetAvailabilitiesAsync());
97+
var collection = OnCodex(api => api.GetAvailabilitiesAsync());
9898
return mapper.Map(collection);
9999
}
100100

101101
public string RequestStorage(StoragePurchaseRequest request)
102102
{
103103
var body = mapper.Map(request);
104-
return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
104+
return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
105105
}
106106

107107
public CodexSpace Space()
108108
{
109-
var space = OnCodex<Space>(api => api.SpaceAsync());
109+
var space = OnCodex(api => api.SpaceAsync());
110110
return mapper.Map(space);
111111
}
112112

‎ProjectPlugins/CodexPlugin/openapi.yaml

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -456,9 +456,35 @@ paths:
456456

457457
"/data/{cid}/network":
458458
get:
459-
summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
459+
summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed."
460460
tags: [ Data ]
461461
operationId: downloadNetwork
462+
parameters:
463+
- in: path
464+
name: cid
465+
required: true
466+
schema:
467+
$ref: "#/components/schemas/Cid"
468+
description: "File to be downloaded."
469+
responses:
470+
"200":
471+
description: Manifest information for download that has been started.
472+
content:
473+
application/json:
474+
schema:
475+
$ref: "#/components/schemas/DataItem"
476+
"400":
477+
description: Invalid CID is specified
478+
"404":
479+
description: Failed to download dataset manifest
480+
"500":
481+
description: Well it was bad-bad
482+
483+
"/data/{cid}/network/stream":
484+
get:
485+
summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
486+
tags: [ Data ]
487+
operationId: downloadNetworkStream
462488
parameters:
463489
- in: path
464490
name: cid
@@ -481,6 +507,32 @@ paths:
481507
"500":
482508
description: Well it was bad-bad
483509

510+
"/data/{cid}/network/manifest":
511+
get:
512+
summary: "Download only the dataset manifest from the network to the local node if it's not available locally."
513+
tags: [ Data ]
514+
operationId: downloadNetworkManifest
515+
parameters:
516+
- in: path
517+
name: cid
518+
required: true
519+
schema:
520+
$ref: "#/components/schemas/Cid"
521+
description: "File for which the manifest is to be downloaded."
522+
responses:
523+
"200":
524+
description: Manifest information.
525+
content:
526+
application/json:
527+
schema:
528+
$ref: "#/components/schemas/DataItem"
529+
"400":
530+
description: Invalid CID is specified
531+
"404":
532+
description: Failed to download dataset manifest
533+
"500":
534+
description: Well it was bad-bad
535+
484536
"/space":
485537
get:
486538
summary: "Gets a summary of the storage space allocation of the node."
@@ -792,4 +844,4 @@ paths:
792844
content:
793845
application/json:
794846
schema:
795-
$ref: "#/components/schemas/DebugInfo"
847+
$ref: "#/components/schemas/DebugInfo"

0 commit comments

Comments
 (0)