Skip to content
This repository was archived by the owner on Jan 24, 2025. It is now read-only.

Commit 698bfd4

Browse files
authored
Merge pull request #908 from iotaledger/feat/add-snapshots-api
Add snapshots API
2 parents bea8598 + 47e63ee commit 698bfd4

File tree

10 files changed

+246
-23
lines changed

10 files changed

+246
-23
lines changed

components/protocol/component.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ func initConfigParams(c *dig.Container) error {
9090
DatabaseEngine db.Engine `name:"databaseEngine"`
9191
BaseToken *BaseToken
9292
ProtocolParameters []iotago.ProtocolParameters
93+
SnapshotFilePath string `name:"snapshotFilePath"`
9394
}
9495

9596
if err := c.Provide(func() cfgResult {
@@ -102,6 +103,7 @@ func initConfigParams(c *dig.Container) error {
102103
DatabaseEngine: dbEngine,
103104
BaseToken: &ParamsProtocol.BaseToken,
104105
ProtocolParameters: readProtocolParameters(),
106+
SnapshotFilePath: ParamsProtocol.Snapshot.Path,
105107
}
106108
}); err != nil {
107109
Component.LogPanic(err.Error())

components/restapi/management/component.go

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ type dependencies struct {
3535
Protocol *protocol.Protocol
3636
PeeringConfigManager *p2p.ConfigManager
3737
NetworkManager network.Manager
38+
SnapshotFilePath string `name:"snapshotFilePath"`
3839
}
3940

4041
func configure() error {
@@ -79,14 +80,14 @@ func configure() error {
7980
return responseByHeader(c, resp, http.StatusOK)
8081
})
8182

82-
// routeGroup.POST(api.ManagementEndpointSnapshotsCreate, func(c echo.Context) error {
83-
// resp, err := createSnapshots(c)
84-
// if err != nil {
85-
// return err
86-
// }
87-
//
88-
// return responseByHeader(c, resp, http.StatusOK)
89-
// })
83+
routeGroup.POST(api.ManagementEndpointSnapshotsCreate, func(c echo.Context) error {
84+
resp, err := createSnapshots(c)
85+
if err != nil {
86+
return err
87+
}
88+
89+
return responseByHeader(c, resp, http.StatusOK)
90+
})
9091

9192
return nil
9293
}

components/restapi/management/pruning.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ import (
1010
)
1111

1212
func pruneDatabase(c echo.Context) (*api.PruneDatabaseResponse, error) {
13-
if deps.Protocol.Engines.Main.Get().Storage.IsPruning() {
14-
return nil, ierrors.WithMessage(echo.ErrServiceUnavailable, "node is already pruning")
13+
if deps.Protocol.Engines.Main.Get().IsSnapshotting() || deps.Protocol.Engines.Main.Get().Storage.IsPruning() {
14+
return nil, ierrors.WithMessage(echo.ErrServiceUnavailable, "node is already creating a snapshot or pruning is running")
1515
}
1616

1717
request := &api.PruneDatabaseRequest{}
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package management
2+
3+
import (
4+
"fmt"
5+
"path/filepath"
6+
"strings"
7+
8+
"github.com/labstack/echo/v4"
9+
10+
"github.com/iotaledger/hive.go/ierrors"
11+
"github.com/iotaledger/inx-app/pkg/httpserver"
12+
"github.com/iotaledger/iota.go/v4/api"
13+
)
14+
15+
func createSnapshots(c echo.Context) (*api.CreateSnapshotResponse, error) {
16+
if deps.Protocol.Engines.Main.Get().IsSnapshotting() || deps.Protocol.Engines.Main.Get().Storage.IsPruning() {
17+
return nil, ierrors.WithMessage(echo.ErrServiceUnavailable, "node is already creating a snapshot or pruning is running")
18+
}
19+
20+
request := &api.CreateSnapshotRequest{}
21+
if err := c.Bind(request); err != nil {
22+
return nil, ierrors.WithMessagef(httpserver.ErrInvalidParameter, "invalid request: %w", err)
23+
}
24+
if request.Slot == 0 {
25+
return nil, ierrors.WithMessage(httpserver.ErrInvalidParameter, "slot needs to be specified")
26+
}
27+
28+
directory := filepath.Dir(deps.SnapshotFilePath)
29+
fileName := filepath.Base(deps.SnapshotFilePath)
30+
fileExt := filepath.Ext(fileName)
31+
fileNameWithoutExt := strings.TrimSuffix(fileName, fileExt)
32+
filePath := filepath.Join(directory, fmt.Sprintf("%s_%d%s", fileNameWithoutExt, request.Slot, fileExt))
33+
34+
if err := deps.Protocol.Engines.Main.Get().WriteSnapshot(filePath, request.Slot); err != nil {
35+
return nil, ierrors.WithMessagef(echo.ErrInternalServerError, "creating snapshot failed: %w", err)
36+
}
37+
38+
return &api.CreateSnapshotResponse{
39+
Slot: request.Slot,
40+
FilePath: filePath,
41+
}, nil
42+
}

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ require (
2929
github.com/iotaledger/hive.go/stringify v0.0.0-20240326102522-2e37ab3611a3
3030
github.com/iotaledger/inx-app v1.0.0-rc.3.0.20240307101848-db58eb9353ec
3131
github.com/iotaledger/inx/go v1.0.0-rc.2.0.20240307100839-48553e1d2022
32-
github.com/iotaledger/iota.go/v4 v4.0.0-20240325092426-32979eef3205
32+
github.com/iotaledger/iota.go/v4 v4.0.0-20240415115618-57e9e887bf49
3333
github.com/labstack/echo/v4 v4.11.4
3434
github.com/labstack/gommon v0.4.2
3535
github.com/libp2p/go-libp2p v0.33.1

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -326,8 +326,8 @@ github.com/iotaledger/inx/go v1.0.0-rc.2.0.20240307100839-48553e1d2022 h1:I178Sa
326326
github.com/iotaledger/inx/go v1.0.0-rc.2.0.20240307100839-48553e1d2022/go.mod h1:jTFxIWiMUdAwO263jlJCSWcNLqEkgYEVOFXfjp5aNJM=
327327
github.com/iotaledger/iota-crypto-demo v0.0.0-20240320124000-d02f37a4fdff h1:Do8fakxvFaj7dLckoo/z+mRyBdZo8QvT8HcgnQlG2Sg=
328328
github.com/iotaledger/iota-crypto-demo v0.0.0-20240320124000-d02f37a4fdff/go.mod h1:aVEutEWFnhDNJBxtVuzy2BeTN+8FAlnR83k7hKV0CFE=
329-
github.com/iotaledger/iota.go/v4 v4.0.0-20240325092426-32979eef3205 h1:nn7nCEezVLmFExiONAJ2XAgZKOJJ+iWrwfDBFdYTKSY=
330-
github.com/iotaledger/iota.go/v4 v4.0.0-20240325092426-32979eef3205/go.mod h1:qn/63CB0/jE1em6ewqDSiz+ovS+E/os7K5b7g2pmJFg=
329+
github.com/iotaledger/iota.go/v4 v4.0.0-20240415115618-57e9e887bf49 h1:1uYaqFeokRrmgkX813vYdn1KTLUGMa97OxJVcOfHm7c=
330+
github.com/iotaledger/iota.go/v4 v4.0.0-20240415115618-57e9e887bf49/go.mod h1:qn/63CB0/jE1em6ewqDSiz+ovS+E/os7K5b7g2pmJFg=
331331
github.com/ipfs/boxo v0.18.0 h1:MOL9/AgoV3e7jlVMInicaSdbgralfqSsbkc31dZ9tmw=
332332
github.com/ipfs/boxo v0.18.0/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80=
333333
github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s=

pkg/protocol/engine/engine.go

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"io"
55
"os"
66
"path/filepath"
7+
"sync/atomic"
78
"time"
89

910
"github.com/libp2p/go-libp2p/core/peer"
@@ -43,6 +44,10 @@ import (
4344
iotago "github.com/iotaledger/iota.go/v4"
4445
)
4546

47+
var (
48+
ErrSnapshottingInProgress = ierrors.New("snapshotting is already in progress")
49+
)
50+
4651
// region Engine /////////////////////////////////////////////////////////////////////////////////////////////////////
4752

4853
type Engine struct {
@@ -84,6 +89,8 @@ type Engine struct {
8489
chainID iotago.CommitmentID
8590
mutex syncutils.RWMutex
8691

92+
isSnapshotting atomic.Bool
93+
8794
optsSnapshotPath string
8895
optsEntryPointsDepth int
8996
optsSnapshotDepth int
@@ -351,10 +358,27 @@ func (e *Engine) CommitmentAPI(commitmentID iotago.CommitmentID) (*CommitmentAPI
351358
return NewCommitmentAPI(e, commitmentID), nil
352359
}
353360

354-
func (e *Engine) WriteSnapshot(filePath string, targetSlot ...iotago.SlotIndex) (err error) {
361+
func (e *Engine) IsSnapshotting() bool {
362+
return e.isSnapshotting.Load()
363+
}
364+
365+
func (e *Engine) WriteSnapshot(filePath string, targetSlot ...iotago.SlotIndex) error {
366+
if e.isSnapshotting.Swap(true) {
367+
return ErrSnapshottingInProgress
368+
}
369+
defer e.isSnapshotting.Store(false)
370+
371+
latestCommittedSlot := e.Storage.Settings().LatestCommitment().Slot()
372+
355373
if len(targetSlot) == 0 {
356-
targetSlot = append(targetSlot, e.Storage.Settings().LatestCommitment().Slot())
357-
} else if lastPrunedEpoch, hasPruned := e.Storage.LastPrunedEpoch(); hasPruned && e.APIForSlot(targetSlot[0]).TimeProvider().EpochFromSlot(targetSlot[0]) <= lastPrunedEpoch {
374+
targetSlot = append(targetSlot, latestCommittedSlot)
375+
}
376+
377+
if targetSlot[0] > latestCommittedSlot {
378+
return ierrors.Errorf("impossible to create a snapshot for slot %d because it is not committed yet (latest committed slot %d)", targetSlot[0], latestCommittedSlot)
379+
}
380+
381+
if lastPrunedEpoch, hasPruned := e.Storage.LastPrunedEpoch(); hasPruned && e.APIForSlot(targetSlot[0]).TimeProvider().EpochFromSlot(targetSlot[0]) <= lastPrunedEpoch {
358382
return ierrors.Errorf("impossible to create a snapshot for slot %d because it is pruned (last pruned slot %d)", targetSlot[0], lo.Return1(e.Storage.LastPrunedEpoch()))
359383
}
360384

@@ -366,7 +390,7 @@ func (e *Engine) WriteSnapshot(filePath string, targetSlot ...iotago.SlotIndex)
366390
return ierrors.Wrap(err, "failed to close snapshot file")
367391
}
368392

369-
return
393+
return nil
370394
}
371395

372396
func (e *Engine) ImportSettings(reader io.ReadSeeker) (err error) {

tools/docker-network/tests/api_management_test.go

Lines changed: 157 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,14 @@ func getContextWithTimeout(duration time.Duration) context.Context {
2020
return ctx
2121
}
2222

23-
// Test_PeerManagementAPI tests if the peer management API returns the expected results.
23+
// Test_ManagementAPI_Peers tests if the peer management API returns the expected results.
2424
// 1. Run docker network.
2525
// 2. List all peers of node 1.
2626
// 3. Delete a peer from node 1.
2727
// 4. List all peers of node 1 again and check if the peer was deleted.
2828
// 5. Re-Add the peer to node 1.
2929
// 6. List all peers of node 1 again and check if the peer was added.
30-
func Test_PeerManagementAPI(t *testing.T) {
30+
func Test_ManagementAPI_Peers(t *testing.T) {
3131
d := NewDockerTestFramework(t,
3232
WithProtocolParametersOptions(
3333
iotago.WithTimeProviderOptions(5, time.Now().Unix(), 10, 4),
@@ -116,7 +116,7 @@ func Test_PeerManagementAPI(t *testing.T) {
116116
}
117117
}
118118

119-
func Test_PeerManagementAPI_BadRequests(t *testing.T) {
119+
func Test_ManagementAPI_Peers_BadRequests(t *testing.T) {
120120
d := NewDockerTestFramework(t,
121121
WithProtocolParametersOptions(
122122
iotago.WithTimeProviderOptions(5, time.Now().Unix(), 10, 4),
@@ -179,3 +179,157 @@ func Test_PeerManagementAPI_BadRequests(t *testing.T) {
179179
t.Run(test.name, test.testFunc)
180180
}
181181
}
182+
183+
func Test_ManagementAPI_Pruning(t *testing.T) {
184+
d := NewDockerTestFramework(t,
185+
WithProtocolParametersOptions(
186+
iotago.WithSupplyOptions(1813620509061365, 63, 1, 4, 0, 0, 0),
187+
iotago.WithTimeProviderOptions(0, time.Now().Unix(), 3, 4),
188+
iotago.WithLivenessOptions(3, 4, 2, 4, 8),
189+
iotago.WithCongestionControlOptions(1, 1, 1, 400_000, 250_000, 50_000_000, 1000, 100),
190+
iotago.WithRewardsOptions(8, 10, 2, 384),
191+
iotago.WithTargetCommitteeSize(4),
192+
))
193+
defer d.Stop()
194+
195+
d.AddValidatorNode("V1", "docker-network-inx-validator-1-1", "http://localhost:8050", "rms1pzg8cqhfxqhq7pt37y8cs4v5u4kcc48lquy2k73ehsdhf5ukhya3y5rx2w6")
196+
d.AddValidatorNode("V2", "docker-network-inx-validator-2-1", "http://localhost:8060", "rms1pqm4xk8e9ny5w5rxjkvtp249tfhlwvcshyr3pc0665jvp7g3hc875k538hl")
197+
d.AddValidatorNode("V3", "docker-network-inx-validator-3-1", "http://localhost:8070", "rms1pp4wuuz0y42caz48vv876qfpmffswsvg40zz8v79sy8cp0jfxm4kunflcgt")
198+
d.AddValidatorNode("V4", "docker-network-inx-validator-4-1", "http://localhost:8040", "rms1pr8cxs3dzu9xh4cduff4dd4cxdthpjkpwmz2244f75m0urslrsvtsshrrjw")
199+
d.AddNode("node5", "docker-network-node-5-1", "http://localhost:8090")
200+
201+
runErr := d.Run()
202+
require.NoError(t, runErr)
203+
204+
d.WaitUntilNetworkReady()
205+
206+
nodeClientV1 := d.Client("V1")
207+
208+
// get the management client
209+
managementClient, err := nodeClientV1.Management(getContextWithTimeout(5 * time.Second))
210+
require.NoError(t, err)
211+
212+
awaitNextEpoch := func() {
213+
info, err := nodeClientV1.Info(getContextWithTimeout(5 * time.Second))
214+
require.NoError(t, err)
215+
216+
currentEpoch := nodeClientV1.CommittedAPI().TimeProvider().EpochFromSlot(info.Status.LatestCommitmentID.Slot())
217+
218+
// await the start slot of the next epoch
219+
d.AwaitCommitment(nodeClientV1.CommittedAPI().TimeProvider().EpochStart(currentEpoch + 1))
220+
}
221+
222+
type test struct {
223+
name string
224+
testFunc func(t *testing.T)
225+
}
226+
227+
tests := []*test{
228+
{
229+
name: "Test_PruneDatabase_ByEpoch",
230+
testFunc: func(t *testing.T) {
231+
// we need to wait until epoch 3 to be able to prune epoch 1
232+
awaitNextEpoch()
233+
awaitNextEpoch()
234+
awaitNextEpoch()
235+
236+
// prune database by epoch
237+
pruneDatabaseResponse, err := managementClient.PruneDatabaseByEpoch(getContextWithTimeout(5*time.Second), 1)
238+
require.NoError(t, err)
239+
require.NotNil(t, pruneDatabaseResponse)
240+
},
241+
},
242+
{
243+
name: "Test_PruneDatabase_ByDepth",
244+
testFunc: func(t *testing.T) {
245+
// wait for the next epoch to start
246+
awaitNextEpoch()
247+
248+
// prune database by depth
249+
pruneDatabaseResponse, err := managementClient.PruneDatabaseByDepth(getContextWithTimeout(5*time.Second), 1)
250+
require.NoError(t, err)
251+
require.NotNil(t, pruneDatabaseResponse)
252+
},
253+
},
254+
{
255+
name: "Test_PruneDatabase_BySize",
256+
testFunc: func(t *testing.T) {
257+
// wait for the next epoch to start
258+
awaitNextEpoch()
259+
260+
// prune database by size
261+
pruneDatabaseResponse, err := managementClient.PruneDatabaseBySize(getContextWithTimeout(5*time.Second), "1M")
262+
require.NoError(t, err)
263+
require.NotNil(t, pruneDatabaseResponse)
264+
},
265+
},
266+
}
267+
268+
for _, test := range tests {
269+
t.Run(test.name, test.testFunc)
270+
}
271+
}
272+
273+
func Test_ManagementAPI_Snapshots(t *testing.T) {
274+
d := NewDockerTestFramework(t,
275+
WithProtocolParametersOptions(
276+
iotago.WithSupplyOptions(1813620509061365, 63, 1, 4, 0, 0, 0),
277+
iotago.WithTimeProviderOptions(0, time.Now().Unix(), 3, 4),
278+
iotago.WithLivenessOptions(3, 4, 2, 4, 8),
279+
iotago.WithCongestionControlOptions(1, 1, 1, 400_000, 250_000, 50_000_000, 1000, 100),
280+
iotago.WithRewardsOptions(8, 10, 2, 384),
281+
iotago.WithTargetCommitteeSize(4),
282+
))
283+
defer d.Stop()
284+
285+
d.AddValidatorNode("V1", "docker-network-inx-validator-1-1", "http://localhost:8050", "rms1pzg8cqhfxqhq7pt37y8cs4v5u4kcc48lquy2k73ehsdhf5ukhya3y5rx2w6")
286+
d.AddValidatorNode("V2", "docker-network-inx-validator-2-1", "http://localhost:8060", "rms1pqm4xk8e9ny5w5rxjkvtp249tfhlwvcshyr3pc0665jvp7g3hc875k538hl")
287+
d.AddValidatorNode("V3", "docker-network-inx-validator-3-1", "http://localhost:8070", "rms1pp4wuuz0y42caz48vv876qfpmffswsvg40zz8v79sy8cp0jfxm4kunflcgt")
288+
d.AddValidatorNode("V4", "docker-network-inx-validator-4-1", "http://localhost:8040", "rms1pr8cxs3dzu9xh4cduff4dd4cxdthpjkpwmz2244f75m0urslrsvtsshrrjw")
289+
d.AddNode("node5", "docker-network-node-5-1", "http://localhost:8090")
290+
291+
runErr := d.Run()
292+
require.NoError(t, runErr)
293+
294+
d.WaitUntilNetworkReady()
295+
296+
nodeClientV1 := d.Client("V1")
297+
298+
// get the management client
299+
managementClient, err := nodeClientV1.Management(getContextWithTimeout(5 * time.Second))
300+
require.NoError(t, err)
301+
302+
awaitNextEpoch := func() {
303+
info, err := nodeClientV1.Info(getContextWithTimeout(5 * time.Second))
304+
require.NoError(t, err)
305+
306+
currentEpoch := nodeClientV1.CommittedAPI().TimeProvider().EpochFromSlot(info.Status.LatestCommitmentID.Slot())
307+
308+
// await the start slot of the next epoch
309+
d.AwaitCommitment(nodeClientV1.CommittedAPI().TimeProvider().EpochStart(currentEpoch + 1))
310+
}
311+
312+
type test struct {
313+
name string
314+
testFunc func(t *testing.T)
315+
}
316+
317+
tests := []*test{
318+
{
319+
name: "Test_CreateSnapshot",
320+
testFunc: func(t *testing.T) {
321+
// wait for the next epoch to start
322+
awaitNextEpoch()
323+
324+
// create snapshot
325+
snapshotResponse, err := managementClient.CreateSnapshot(getContextWithTimeout(5*time.Second), 1)
326+
require.NoError(t, err)
327+
require.NotNil(t, snapshotResponse)
328+
},
329+
},
330+
}
331+
332+
for _, test := range tests {
333+
t.Run(test.name, test.testFunc)
334+
}
335+
}

tools/gendoc/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ require (
7575
github.com/iotaledger/inx-app v1.0.0-rc.3.0.20240307101848-db58eb9353ec // indirect
7676
github.com/iotaledger/inx/go v1.0.0-rc.2.0.20240307100839-48553e1d2022 // indirect
7777
github.com/iotaledger/iota-crypto-demo v0.0.0-20240320124000-d02f37a4fdff // indirect
78-
github.com/iotaledger/iota.go/v4 v4.0.0-20240325092426-32979eef3205 // indirect
78+
github.com/iotaledger/iota.go/v4 v4.0.0-20240415115618-57e9e887bf49 // indirect
7979
github.com/ipfs/boxo v0.18.0 // indirect
8080
github.com/ipfs/go-cid v0.4.1 // indirect
8181
github.com/ipfs/go-datastore v0.6.0 // indirect

tools/gendoc/go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -330,8 +330,8 @@ github.com/iotaledger/inx/go v1.0.0-rc.2.0.20240307100839-48553e1d2022 h1:I178Sa
330330
github.com/iotaledger/inx/go v1.0.0-rc.2.0.20240307100839-48553e1d2022/go.mod h1:jTFxIWiMUdAwO263jlJCSWcNLqEkgYEVOFXfjp5aNJM=
331331
github.com/iotaledger/iota-crypto-demo v0.0.0-20240320124000-d02f37a4fdff h1:Do8fakxvFaj7dLckoo/z+mRyBdZo8QvT8HcgnQlG2Sg=
332332
github.com/iotaledger/iota-crypto-demo v0.0.0-20240320124000-d02f37a4fdff/go.mod h1:aVEutEWFnhDNJBxtVuzy2BeTN+8FAlnR83k7hKV0CFE=
333-
github.com/iotaledger/iota.go/v4 v4.0.0-20240325092426-32979eef3205 h1:nn7nCEezVLmFExiONAJ2XAgZKOJJ+iWrwfDBFdYTKSY=
334-
github.com/iotaledger/iota.go/v4 v4.0.0-20240325092426-32979eef3205/go.mod h1:qn/63CB0/jE1em6ewqDSiz+ovS+E/os7K5b7g2pmJFg=
333+
github.com/iotaledger/iota.go/v4 v4.0.0-20240415115618-57e9e887bf49 h1:1uYaqFeokRrmgkX813vYdn1KTLUGMa97OxJVcOfHm7c=
334+
github.com/iotaledger/iota.go/v4 v4.0.0-20240415115618-57e9e887bf49/go.mod h1:qn/63CB0/jE1em6ewqDSiz+ovS+E/os7K5b7g2pmJFg=
335335
github.com/ipfs/boxo v0.18.0 h1:MOL9/AgoV3e7jlVMInicaSdbgralfqSsbkc31dZ9tmw=
336336
github.com/ipfs/boxo v0.18.0/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80=
337337
github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s=

0 commit comments

Comments
 (0)