From 14761931ea487f1178182ed91f003c7eb49305ff Mon Sep 17 00:00:00 2001 From: VolodymyrBg Date: Wed, 5 Feb 2025 17:58:06 +0200 Subject: [PATCH 01/15] eds: make Size method return error --- store/file/ods_q4.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/store/file/ods_q4.go b/store/file/ods_q4.go index f0ca686094..6c4b962956 100644 --- a/store/file/ods_q4.go +++ b/store/file/ods_q4.go @@ -110,7 +110,7 @@ func (odsq4 *ODSQ4) tryLoadQ4() *q4 { return q4 } -func (odsq4 *ODSQ4) Size(ctx context.Context) int { +func (odsq4 *ODSQ4) Size(ctx context.Context) (int, error) { return odsq4.ods.Size(ctx) } @@ -137,7 +137,10 @@ func (odsq4 *ODSQ4) Sample(ctx context.Context, idx shwap.SampleCoords) (shwap.S } func (odsq4 *ODSQ4) AxisHalf(ctx context.Context, axisType rsmt2d.Axis, axisIdx int) (eds.AxisHalf, error) { - size := odsq4.Size(ctx) // TODO(@Wondertan): Should return error. + size, err := odsq4.Size(ctx) + if err != nil { + return nil, fmt.Errorf("getting size: %w", err) + } if axisIdx >= size/2 { // lazy load Q4 file and read axis from it if loaded From 2c5b292e61ce3c57423af4ef74bbd67f7711876d Mon Sep 17 00:00:00 2001 From: VolodymyrBg Date: Wed, 5 Feb 2025 17:58:37 +0200 Subject: [PATCH 02/15] Update ods.go --- store/file/ods.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/store/file/ods.go b/store/file/ods.go index e9b14e8a80..d24e8d7745 100644 --- a/store/file/ods.go +++ b/store/file/ods.go @@ -183,8 +183,11 @@ func OpenODS(path string) (*ODS, error) { } // Size returns EDS size stored in file's header. -func (o *ODS) Size(context.Context) int { - return o.size() +func (o *ODS) Size(context.Context) (int, error) { + if o.hdr == nil { + return 0, fmt.Errorf("header is not initialized") + } + return o.size(), nil } func (o *ODS) size() int { @@ -238,8 +241,13 @@ func (o *ODS) Sample(ctx context.Context, idx shwap.SampleCoords) (shwap.Sample, // to calculate the sample rowIdx, colIdx := idx.Row, idx.Col + size, err := o.Size(ctx) + if err != nil { + return shwap.Sample{}, fmt.Errorf("getting size: %w", err) + } + axisType, axisIdx, shrIdx := rsmt2d.Row, rowIdx, colIdx - if colIdx < o.size()/2 && rowIdx >= o.size()/2 { + if colIdx < size/2 && rowIdx >= size/2 { axisType, axisIdx, shrIdx = rsmt2d.Col, colIdx, rowIdx } From 8139e70bf9efd3a36ecac9b8a906495d3440db6d Mon Sep 17 00:00:00 2001 From: VolodymyrBg Date: Wed, 5 Feb 2025 17:59:08 +0200 Subject: [PATCH 03/15] Update accessor.go --- share/eds/accessor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/eds/accessor.go b/share/eds/accessor.go index eb3fea6e41..c909b1e22d 100644 --- a/share/eds/accessor.go +++ b/share/eds/accessor.go @@ -17,7 +17,7 @@ var EmptyAccessor = &Rsmt2D{ExtendedDataSquare: share.EmptyEDS()} // Accessor is an interface for accessing extended data square data. type Accessor interface { // Size returns square size of the Accessor. - Size(ctx context.Context) int + Size(ctx context.Context) (int, error) // DataHash returns data hash of the Accessor. DataHash(ctx context.Context) (share.DataHash, error) // AxisRoots returns share.AxisRoots (DataAvailabilityHeader) of the Accessor. From 70ea05cb018e6c8d981603771c78a4ec0cd5f767 Mon Sep 17 00:00:00 2001 From: VolodymyrBg Date: Wed, 5 Feb 2025 18:00:18 +0200 Subject: [PATCH 04/15] Update rsmt2d.go --- share/eds/rsmt2d.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/share/eds/rsmt2d.go b/share/eds/rsmt2d.go index 1d806324b5..ae6ce6e168 100644 --- a/share/eds/rsmt2d.go +++ b/share/eds/rsmt2d.go @@ -21,8 +21,11 @@ type Rsmt2D struct { } // Size returns the size of the Extended Data Square. -func (eds *Rsmt2D) Size(context.Context) int { - return int(eds.Width()) +func (eds *Rsmt2D) Size(context.Context) (int, error) { + if eds.ExtendedDataSquare == nil { + return 0, fmt.Errorf("extended data square is not initialized") + } + return int(eds.Width()), nil } // DataHash returns data hash of the Accessor. From 155167b3e8299442259aa357275b34a232db5eff Mon Sep 17 00:00:00 2001 From: VolodymyrBg Date: Wed, 5 Feb 2025 18:00:41 +0200 Subject: [PATCH 05/15] Update noop.go --- store/cache/noop.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/store/cache/noop.go b/store/cache/noop.go index 27b33e2dd6..4261df4020 100644 --- a/store/cache/noop.go +++ b/store/cache/noop.go @@ -47,8 +47,8 @@ func (n NoopFile) Reader() (io.Reader, error) { return noopReader{}, nil } -func (n NoopFile) Size(context.Context) int { - return 0 +func (n NoopFile) Size(context.Context) (int, error) { + return 0, nil } func (n NoopFile) DataHash(context.Context) (share.DataHash, error) { From 4bab67df52d817eccf37cc3518db73d683acf84b Mon Sep 17 00:00:00 2001 From: VolodymyrBg Date: Wed, 5 Feb 2025 18:01:05 +0200 Subject: [PATCH 06/15] Update validation.go --- share/eds/validation.go | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/share/eds/validation.go b/share/eds/validation.go index 4f6cf0aa85..4e86b980fd 100644 --- a/share/eds/validation.go +++ b/share/eds/validation.go @@ -24,18 +24,25 @@ func WithValidation(f Accessor) Accessor { return &validation{Accessor: f, size: new(atomic.Int32)} } -func (f validation) Size(ctx context.Context) int { +func (f validation) Size(ctx context.Context) (int, error) { size := f.size.Load() if size == 0 { - loaded := f.Accessor.Size(ctx) + loaded, err := f.Accessor.Size(ctx) + if err != nil { + return 0, fmt.Errorf("loading size: %w", err) + } f.size.Store(int32(loaded)) - return loaded + return loaded, nil } - return int(size) + return int(size), nil } func (f validation) Sample(ctx context.Context, idx shwap.SampleCoords) (shwap.Sample, error) { - _, err := shwap.NewSampleID(1, idx, f.Size(ctx)) + size, err := f.Size(ctx) + if err != nil { + return shwap.Sample{}, fmt.Errorf("getting size: %w", err) + } + _, err = shwap.NewSampleID(1, idx, size) if err != nil { return shwap.Sample{}, fmt.Errorf("sample validation: %w", err) } @@ -43,7 +50,11 @@ func (f validation) Sample(ctx context.Context, idx shwap.SampleCoords) (shwap.S } func (f validation) AxisHalf(ctx context.Context, axisType rsmt2d.Axis, axisIdx int) (AxisHalf, error) { - _, err := shwap.NewRowID(1, axisIdx, f.Size(ctx)) + size, err := f.Size(ctx) + if err != nil { + return AxisHalf{}, fmt.Errorf("getting size: %w", err) + } + _, err = shwap.NewRowID(1, axisIdx, size) if err != nil { return AxisHalf{}, fmt.Errorf("axis half validation: %w", err) } @@ -55,7 +66,11 @@ func (f validation) RowNamespaceData( namespace libshare.Namespace, rowIdx int, ) (shwap.RowNamespaceData, error) { - _, err := shwap.NewRowNamespaceDataID(1, rowIdx, namespace, f.Size(ctx)) + size, err := f.Size(ctx) + if err != nil { + return shwap.RowNamespaceData{}, fmt.Errorf("getting size: %w", err) + } + _, err = shwap.NewRowNamespaceDataID(1, rowIdx, namespace, size) if err != nil { return shwap.RowNamespaceData{}, fmt.Errorf("row namespace data validation: %w", err) } From ce91a66aa8c16805e0247e92521f74c9a7eccc90 Mon Sep 17 00:00:00 2001 From: VolodymyrBg Date: Wed, 5 Feb 2025 18:02:06 +0200 Subject: [PATCH 07/15] Update proofs_cache.go --- share/eds/proofs_cache.go | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/share/eds/proofs_cache.go b/share/eds/proofs_cache.go index f73ebcdf80..de17cb9605 100644 --- a/share/eds/proofs_cache.go +++ b/share/eds/proofs_cache.go @@ -75,13 +75,17 @@ func WithProofsCache(ac AccessorStreamer) AccessorStreamer { } } -func (c *proofsCache) Size(ctx context.Context) int { +func (c *proofsCache) Size(ctx context.Context) (int, error) { size := c.size.Load() if size == 0 { - size = int32(c.inner.Size(ctx)) - c.size.Store(size) + loaded, err := c.inner.Size(ctx) + if err != nil { + return 0, fmt.Errorf("loading size from inner accessor: %w", err) + } + c.size.Store(int32(loaded)) + return loaded, nil } - return int(size) + return int(size), nil } func (c *proofsCache) DataHash(ctx context.Context) (share.DataHash, error) { @@ -121,7 +125,11 @@ func (c *proofsCache) Sample(ctx context.Context, idx shwap.SampleCoords) (shwap // build share proof from proofs cached for given axis share := ax.shares[shrIdx] - proofs, err := ipld.GetProof(ctx, ax.proofs, ax.root, shrIdx, c.Size(ctx)) + size, err := c.Size(ctx) + if err != nil { + return shwap.Sample{}, fmt.Errorf("getting size: %w", err) + } + proofs, err := ipld.GetProof(ctx, ax.proofs, ax.root, shrIdx, size) if err != nil { return shwap.Sample{}, fmt.Errorf("building proof from cache: %w", err) } @@ -159,9 +167,13 @@ func (c *proofsCache) axisWithProofs(ctx context.Context, axisType rsmt2d.Axis, } // build proofs from Shares and cache them - adder := ipld.NewProofsAdder(c.Size(ctx), true) + size, err := c.Size(ctx) + if err != nil { + return axisWithProofs{}, fmt.Errorf("getting size: %w", err) + } + adder := ipld.NewProofsAdder(size, true) tree := wrapper.NewErasuredNamespacedMerkleTree( - uint64(c.Size(ctx)/2), + uint64(size/2), uint(axisIdx), nmt.NodeVisitor(adder.VisitFn()), ) @@ -233,9 +245,13 @@ func (c *proofsCache) RowNamespaceData( } func (c *proofsCache) Shares(ctx context.Context) ([]libshare.Share, error) { - odsSize := c.Size(ctx) / 2 + size, err := c.Size(ctx) + if err != nil { + return nil, fmt.Errorf("getting size: %w", err) + } + odsSize := size / 2 shares := make([]libshare.Share, 0, odsSize*odsSize) - for i := 0; i < c.Size(ctx)/2; i++ { + for i := 0; i < size/2; i++ { ax, err := c.AxisHalf(ctx, rsmt2d.Row, i) if err != nil { return nil, err From 235a0b5bc4367e19cd990a2e5b6da820841b7ac0 Mon Sep 17 00:00:00 2001 From: VolodymyrBg Date: Wed, 5 Feb 2025 18:02:20 +0200 Subject: [PATCH 08/15] Update close_once.go --- share/eds/close_once.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/share/eds/close_once.go b/share/eds/close_once.go index 579b4b1e20..805ccb1b05 100644 --- a/share/eds/close_once.go +++ b/share/eds/close_once.go @@ -36,9 +36,9 @@ func (c *closeOnce) Close() error { return err } -func (c *closeOnce) Size(ctx context.Context) int { +func (c *closeOnce) Size(ctx context.Context) (int, error) { if c.closed.Load() { - return 0 + return 0, errAccessorClosed } return c.f.Size(ctx) } From 3df730c0c8d27bf69e7b9fc97674c7d01cbb880b Mon Sep 17 00:00:00 2001 From: VolodymyrBg Date: Wed, 5 Feb 2025 18:02:35 +0200 Subject: [PATCH 09/15] Update testing.go --- share/eds/testing.go | 52 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/share/eds/testing.go b/share/eds/testing.go index c8859dca5a..5804373635 100644 --- a/share/eds/testing.go +++ b/share/eds/testing.go @@ -248,8 +248,7 @@ func testAccessorRowNamespaceData( // check that the amount of shares in the namespace is equal to the expected amount require.Equal(t, amount, actualSharesAmount) - } - }) + }) t.Run("not included", func(t *testing.T) { t.Parallel() @@ -279,7 +278,7 @@ func testAccessorRowNamespaceData( err = rowData.Verify(roots, absentNs, i) require.NoError(t, err) - } + }) }) } @@ -472,9 +471,19 @@ func (q quadrantIdx) String() string { } func (q quadrantIdx) coordinates(edsSize int) (rowIdx, colIdx int) { - colIdx = edsSize/2*(int(q-1)%2) + 1 - rowIdx = edsSize/2*(int(q-1)/2) + 1 - return rowIdx, colIdx + half := edsSize / 2 + switch q { + case q1: + return rand.IntN(half), rand.IntN(half) + case q2: + return rand.IntN(half), half + rand.IntN(half) + case q3: + return half + rand.IntN(half), rand.IntN(half) + case q4: + return half + rand.IntN(half), half + rand.IntN(half) + default: + panic("invalid quadrant") + } } func checkPowerOfTwo(n int) bool { @@ -484,3 +493,34 @@ func checkPowerOfTwo(n int) bool { } return n&(n-1) == 0 } + +func TestAccessorSampling(t *testing.T) { + ctx := context.Background() + acc := NewRandAccessor(t, 8) + defer acc.Close() + + size, err := acc.Size(ctx) + require.NoError(t, err) + + for squareHalf := 0; squareHalf < 2; squareHalf++ { + for axisType := range []rsmt2d.Axis{rsmt2d.Row, rsmt2d.Col} { + _, err := acc.AxisHalf(ctx, axisType, size/2*(squareHalf)) + require.NoError(t, err) + } + } +} + +func TestAccessorSamplingQuadrants(t *testing.T) { + ctx := context.Background() + acc := NewRandAccessor(t, 8) + defer acc.Close() + + size, err := acc.Size(ctx) + require.NoError(t, err) + + for q := range []quadrant{q1, q2, q3, q4} { + rowIdx, colIdx := q.coordinates(size) + _, err := acc.Sample(ctx, shwap.SampleCoords{Row: rowIdx, Col: colIdx}) + require.NoError(t, err) + } +} From 290e752d1ff56bae1fdf18606cf0ce93d1212ecf Mon Sep 17 00:00:00 2001 From: VolodymyrBg Date: Wed, 5 Feb 2025 18:02:54 +0200 Subject: [PATCH 10/15] Update accessor_cache_test.go --- store/cache/accessor_cache_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/store/cache/accessor_cache_test.go b/store/cache/accessor_cache_test.go index 8b537049e1..5865a8d60b 100644 --- a/store/cache/accessor_cache_test.go +++ b/store/cache/accessor_cache_test.go @@ -301,10 +301,11 @@ type mockAccessor struct { m sync.Mutex data []byte isClosed bool + size int } -func (m *mockAccessor) Size(context.Context) int { - panic("implement me") +func (m *mockAccessor) Size(context.Context) (int, error) { + return m.size, nil } func (m *mockAccessor) DataHash(context.Context) (share.DataHash, error) { From f7e5ef6986b9bba806de549b6e83463c369f61f2 Mon Sep 17 00:00:00 2001 From: VolodymyrBg Date: Wed, 5 Feb 2025 18:03:16 +0200 Subject: [PATCH 11/15] Update close_once_test.go --- share/eds/close_once_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/share/eds/close_once_test.go b/share/eds/close_once_test.go index e34299f805..fea6c9b660 100644 --- a/share/eds/close_once_test.go +++ b/share/eds/close_once_test.go @@ -47,8 +47,8 @@ type stubEdsAccessorCloser struct { closed bool } -func (s *stubEdsAccessorCloser) Size(context.Context) int { - return 0 +func (s *stubEdsAccessorCloser) Size(context.Context) (int, error) { + return 0, nil } func (s *stubEdsAccessorCloser) DataHash(context.Context) (share.DataHash, error) { From 01e000595ffc97ad9c18a21879ef9efe427ecfa9 Mon Sep 17 00:00:00 2001 From: VolodymyrBg Date: Thu, 6 Feb 2025 20:17:38 +0200 Subject: [PATCH 12/15] Update share/eds/proofs_cache.go Co-authored-by: Vlad <13818348+walldiss@users.noreply.github.com> --- share/eds/proofs_cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/eds/proofs_cache.go b/share/eds/proofs_cache.go index de17cb9605..dbf07dcde6 100644 --- a/share/eds/proofs_cache.go +++ b/share/eds/proofs_cache.go @@ -251,7 +251,7 @@ func (c *proofsCache) Shares(ctx context.Context) ([]libshare.Share, error) { } odsSize := size / 2 shares := make([]libshare.Share, 0, odsSize*odsSize) - for i := 0; i < size/2; i++ { + for i := 0; i < odsSize; i++ { ax, err := c.AxisHalf(ctx, rsmt2d.Row, i) if err != nil { return nil, err From 693c7da5679ea49e852c8293d114b1771794aaf2 Mon Sep 17 00:00:00 2001 From: VolodymyrBg Date: Tue, 11 Feb 2025 21:41:58 +0200 Subject: [PATCH 13/15] Update rsmt2d.go --- share/eds/rsmt2d.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/share/eds/rsmt2d.go b/share/eds/rsmt2d.go index ae6ce6e168..f428ae49f7 100644 --- a/share/eds/rsmt2d.go +++ b/share/eds/rsmt2d.go @@ -22,9 +22,6 @@ type Rsmt2D struct { // Size returns the size of the Extended Data Square. func (eds *Rsmt2D) Size(context.Context) (int, error) { - if eds.ExtendedDataSquare == nil { - return 0, fmt.Errorf("extended data square is not initialized") - } return int(eds.Width()), nil } From afc02efb893829dd1eb9a8e98fb05f28180c54a6 Mon Sep 17 00:00:00 2001 From: VolodymyrBg Date: Tue, 11 Feb 2025 21:53:53 +0200 Subject: [PATCH 14/15] Update testing.go --- share/eds/testing.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/share/eds/testing.go b/share/eds/testing.go index 5804373635..5fe77614ba 100644 --- a/share/eds/testing.go +++ b/share/eds/testing.go @@ -248,7 +248,8 @@ func testAccessorRowNamespaceData( // check that the amount of shares in the namespace is equal to the expected amount require.Equal(t, amount, actualSharesAmount) - }) + } + }) t.Run("not included", func(t *testing.T) { t.Parallel() From 1269882ead9b333fb6c9ab068f42a02b2d6bf182 Mon Sep 17 00:00:00 2001 From: VolodymyrBg Date: Tue, 11 Feb 2025 21:57:13 +0200 Subject: [PATCH 15/15] Update testing.go --- share/eds/testing.go | 57 ++++++++++++++++---------------------------- 1 file changed, 20 insertions(+), 37 deletions(-) diff --git a/share/eds/testing.go b/share/eds/testing.go index 5fe77614ba..1bb88bfe41 100644 --- a/share/eds/testing.go +++ b/share/eds/testing.go @@ -248,7 +248,7 @@ func testAccessorRowNamespaceData( // check that the amount of shares in the namespace is equal to the expected amount require.Equal(t, amount, actualSharesAmount) - } + }) }) t.Run("not included", func(t *testing.T) { @@ -465,22 +465,36 @@ func BenchGetSampleFromAccessor( type quadrantIdx int -var quadrants = []quadrantIdx{1, 2, 3, 4} +const ( + q1 quadrantIdx = iota + 1 // top-left: original data quadrant + q2 // top-right: row parity quadrant + q3 // bottom-left: column parity quadrant + q4 // bottom-right: combined parity quadrant +) + +var quadrants = []quadrantIdx{q1, q2, q3, q4} func (q quadrantIdx) String() string { return strconv.Itoa(int(q)) } +// coordinates returns random row and column indices within the specified quadrant. +// The EDS is divided into 4 quadrants, each with size (edsSize/2 x edsSize/2): +// +-----+-----+ +// | Q1 | Q2 | +// +-----+-----+ +// | Q3 | Q4 | +// +-----+-----+ func (q quadrantIdx) coordinates(edsSize int) (rowIdx, colIdx int) { half := edsSize / 2 switch q { - case q1: + case q1: // top-left quadrant (original data) return rand.IntN(half), rand.IntN(half) - case q2: + case q2: // top-right quadrant (row parity) return rand.IntN(half), half + rand.IntN(half) - case q3: + case q3: // bottom-left quadrant (column parity) return half + rand.IntN(half), rand.IntN(half) - case q4: + case q4: // bottom-right quadrant (combined parity) return half + rand.IntN(half), half + rand.IntN(half) default: panic("invalid quadrant") @@ -494,34 +508,3 @@ func checkPowerOfTwo(n int) bool { } return n&(n-1) == 0 } - -func TestAccessorSampling(t *testing.T) { - ctx := context.Background() - acc := NewRandAccessor(t, 8) - defer acc.Close() - - size, err := acc.Size(ctx) - require.NoError(t, err) - - for squareHalf := 0; squareHalf < 2; squareHalf++ { - for axisType := range []rsmt2d.Axis{rsmt2d.Row, rsmt2d.Col} { - _, err := acc.AxisHalf(ctx, axisType, size/2*(squareHalf)) - require.NoError(t, err) - } - } -} - -func TestAccessorSamplingQuadrants(t *testing.T) { - ctx := context.Background() - acc := NewRandAccessor(t, 8) - defer acc.Close() - - size, err := acc.Size(ctx) - require.NoError(t, err) - - for q := range []quadrant{q1, q2, q3, q4} { - rowIdx, colIdx := q.coordinates(size) - _, err := acc.Sample(ctx, shwap.SampleCoords{Row: rowIdx, Col: colIdx}) - require.NoError(t, err) - } -}