Skip to content

Commit 9b966f4

Browse files
rvaggrvagg
committed
feat(miner): make precommit HAMT & AMT CIDs nullable
1 parent 5f30872 commit 9b966f4

File tree

5 files changed

+238
-54
lines changed

5 files changed

+238
-54
lines changed

‎builtin/v16/migration/miner.go

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
package migration
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"sync/atomic"
7+
8+
miner15 "github.com/filecoin-project/go-state-types/builtin/v15/miner"
9+
miner16 "github.com/filecoin-project/go-state-types/builtin/v16/miner"
10+
"github.com/filecoin-project/go-state-types/migration"
11+
"github.com/ipfs/go-cid"
12+
cbor "github.com/ipfs/go-ipld-cbor"
13+
"golang.org/x/xerrors"
14+
)
15+
16+
type minerMigrator struct {
17+
OutCodeCID cid.Cid
18+
EmptyPreCommittedSectorsHamtCid cid.Cid
19+
EmptyPrecommitCleanUpAmtCid cid.Cid
20+
}
21+
22+
func newMinerMigrator(
23+
_ context.Context,
24+
_ cbor.IpldStore,
25+
outCode,
26+
emptyPreCommittedSectorsHamtCid,
27+
emptyPrecommitCleanUpAmtCid cid.Cid,
28+
) (*minerMigrator, error) {
29+
return &minerMigrator{
30+
OutCodeCID: outCode,
31+
EmptyPreCommittedSectorsHamtCid: emptyPreCommittedSectorsHamtCid,
32+
EmptyPrecommitCleanUpAmtCid: emptyPrecommitCleanUpAmtCid,
33+
}, nil
34+
}
35+
36+
// TOOD: remove these, they're just for debugging
37+
var minerCount, withEmptyPrecommitCleanUpAmtCidCount, withEmptyPreCommittedSectorsHamtCidCount, migratedCount uint64
38+
39+
func (m *minerMigrator) MigrateState(ctx context.Context, store cbor.IpldStore, in migration.ActorMigrationInput) (result *migration.ActorMigrationResult, err error) {
40+
var inState miner15.State
41+
if err := store.Get(ctx, in.Head, &inState); err != nil {
42+
return nil, xerrors.Errorf("failed to load miner state for %s: %w", in.Address, err)
43+
}
44+
45+
newHead := in.Head
46+
47+
var epccuacc, epcshcc uint64
48+
var preCommittedSectors, preCommittedSectorsCleanUp *cid.Cid
49+
if !m.EmptyPrecommitCleanUpAmtCid.Equals(inState.PreCommittedSectorsCleanUp) {
50+
preCommittedSectorsCleanUp = &inState.PreCommittedSectorsCleanUp
51+
} else {
52+
epccuacc = atomic.AddUint64(&withEmptyPrecommitCleanUpAmtCidCount, 1)
53+
}
54+
if !m.EmptyPreCommittedSectorsHamtCid.Equals(inState.PreCommittedSectors) {
55+
preCommittedSectors = &inState.PreCommittedSectors
56+
} else {
57+
epcshcc = atomic.AddUint64(&withEmptyPreCommittedSectorsHamtCidCount, 1)
58+
}
59+
60+
// we only need to update the state if one of these need to be nullable, otherwise the existing
61+
// state, with a valid CID, will decode correctly even though they're now pointer fields
62+
if preCommittedSectors == nil || preCommittedSectorsCleanUp == nil {
63+
outState := miner16.State{
64+
Info: inState.Info,
65+
PreCommitDeposits: inState.PreCommitDeposits,
66+
LockedFunds: inState.LockedFunds,
67+
VestingFunds: inState.VestingFunds,
68+
FeeDebt: inState.FeeDebt,
69+
InitialPledge: inState.InitialPledge,
70+
PreCommittedSectors: preCommittedSectors,
71+
PreCommittedSectorsCleanUp: preCommittedSectorsCleanUp,
72+
AllocatedSectors: inState.AllocatedSectors,
73+
Sectors: inState.Sectors,
74+
ProvingPeriodStart: inState.ProvingPeriodStart,
75+
CurrentDeadline: inState.CurrentDeadline,
76+
Deadlines: inState.Deadlines,
77+
EarlyTerminations: inState.EarlyTerminations,
78+
DeadlineCronActive: inState.DeadlineCronActive,
79+
}
80+
81+
if newHead, err = store.Put(ctx, &outState); err != nil {
82+
return nil, xerrors.Errorf("failed to put new state: %w", err)
83+
}
84+
85+
atomic.AddUint64(&migratedCount, 1)
86+
}
87+
88+
if nc := atomic.AddUint64(&minerCount, 1); nc%10000 == 0 {
89+
fmt.Printf("Checked %d miners, %d with empty PreCommittedSectors, %d with empty PreCommittedSectorsCleanUp, %d migrated\n", nc, epcshcc, epccuacc, atomic.LoadUint64(&migratedCount))
90+
}
91+
92+
return &migration.ActorMigrationResult{
93+
NewCodeCID: m.MigratedCodeCID(),
94+
NewHead: newHead,
95+
}, nil
96+
}
97+
98+
func (m *minerMigrator) MigratedCodeCID() cid.Cid {
99+
return m.OutCodeCID
100+
}
101+
102+
func (m *minerMigrator) Deferred() bool {
103+
return false
104+
}
105+
106+
var _ migration.ActorMigration = (*minerMigrator)(nil)

‎builtin/v16/migration/top.go

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,10 @@ package migration
33
import (
44
"context"
55

6-
adt16 "github.com/filecoin-project/go-state-types/builtin/v16/util/adt"
6+
"github.com/filecoin-project/go-state-types/builtin/v16/util/adt"
77

88
system15 "github.com/filecoin-project/go-state-types/builtin/v13/system"
9+
miner16 "github.com/filecoin-project/go-state-types/builtin/v16/miner"
910

1011
"github.com/filecoin-project/go-state-types/abi"
1112
"github.com/filecoin-project/go-state-types/builtin"
@@ -24,7 +25,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID
2425
return cid.Undef, xerrors.Errorf("invalid migration config with %d workers", cfg.MaxWorkers)
2526
}
2627

27-
adtStore := adt16.WrapStore(ctx, store)
28+
adtStore := adt.WrapStore(ctx, store)
2829

2930
// Load input and output state trees
3031
actorsIn, err := builtin.LoadTree(adtStore, actorsRootIn)
@@ -67,14 +68,22 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID
6768
// Set of prior version code CIDs for actors to defer during iteration, for explicit migration afterwards.
6869
deferredCodeIDs := make(map[cid.Cid]struct{})
6970

71+
miner15Cid := cid.Undef
72+
7073
for _, oldEntry := range oldManifestData.Entries {
7174
newCodeCID, ok := newManifest.Get(oldEntry.Name)
7275
if !ok {
7376
return cid.Undef, xerrors.Errorf("code cid for %s actor not found in new manifest", oldEntry.Name)
7477
}
78+
if oldEntry.Name == manifest.MinerKey {
79+
miner15Cid = oldEntry.Code
80+
}
7581
migrations[oldEntry.Code] = migration.CachedMigration(cache, migration.CodeMigrator{OutCodeCID: newCodeCID})
7682
}
7783

84+
if miner15Cid == cid.Undef {
85+
return cid.Undef, xerrors.Errorf("could not find miner actor in old manifest")
86+
}
7887
// migrations that migrate both code and state, override entries in `migrations`
7988

8089
// The System Actor
@@ -90,6 +99,35 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID
9099
return cid.Undef, xerrors.Errorf("incomplete migration specification with %d code CIDs, need %d", len(migrations)+len(deferredCodeIDs), len(oldManifestData.Entries))
91100
}
92101

102+
var emptyPreCommittedSectorsHamtCid cid.Cid
103+
if hamt, err := adt.MakeEmptyMap(adtStore, builtin.DefaultHamtBitwidth); err != nil {
104+
return cid.Undef, xerrors.Errorf("failed to create empty precommit clean up amount array: %w", err)
105+
} else {
106+
if emptyPreCommittedSectorsHamtCid, err = hamt.Root(); err != nil {
107+
return cid.Undef, xerrors.Errorf("failed to get root of empty precommit clean up amount array: %w", err)
108+
}
109+
}
110+
var emptyPrecommitCleanUpAmtCid cid.Cid
111+
if amt, err := adt.MakeEmptyArray(adtStore, miner16.PrecommitCleanUpAmtBitwidth); err != nil {
112+
return cid.Undef, xerrors.Errorf("failed to create empty precommit clean up amount array: %w", err)
113+
} else {
114+
if emptyPrecommitCleanUpAmtCid, err = amt.Root(); err != nil {
115+
return cid.Undef, xerrors.Errorf("failed to get root of empty precommit clean up amount array: %w", err)
116+
}
117+
}
118+
119+
miner16Cid, ok := newManifest.Get(manifest.MinerKey)
120+
if !ok {
121+
return cid.Undef, xerrors.Errorf("code cid for miner actor not found in new manifest")
122+
}
123+
124+
minerMig, err := newMinerMigrator(ctx, store, miner16Cid, emptyPreCommittedSectorsHamtCid, emptyPrecommitCleanUpAmtCid)
125+
if err != nil {
126+
return cid.Undef, xerrors.Errorf("failed to create miner migrator: %w", err)
127+
}
128+
129+
migrations[miner15Cid] = migration.CachedMigration(cache, minerMig)
130+
93131
actorsOut, err := migration.RunMigration(ctx, cfg, cache, store, log, actorsIn, migrations)
94132
if err != nil {
95133
return cid.Undef, xerrors.Errorf("failed to run migration: %w", err)

‎builtin/v16/miner/cbor_gen.go

Lines changed: 42 additions & 10 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎builtin/v16/miner/invariants.go

Lines changed: 43 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -778,54 +778,58 @@ func CheckPreCommits(st *State, store adt.Store, allocatedSectorsMap map[uint64]
778778

779779
// invert pre-commit clean up queue into a lookup by sector number
780780
cleanUpEpochs := make(map[uint64]abi.ChainEpoch)
781-
if cleanUpQ, err := util.LoadBitfieldQueue(store, st.PreCommittedSectorsCleanUp, st.QuantSpecEveryDeadline(), PrecommitCleanUpAmtBitwidth); err != nil {
782-
acc.Addf("error loading pre-commit clean up queue: %v", err)
783-
} else {
784-
err = cleanUpQ.ForEach(func(epoch abi.ChainEpoch, bf bitfield.BitField) error {
785-
quantized := quant.QuantizeUp(epoch)
786-
acc.Require(quantized == epoch, "precommit expiration %d is not quantized", epoch)
787-
if err = bf.ForEach(func(secNum uint64) error {
788-
cleanUpEpochs[secNum] = epoch
781+
if st.PreCommittedSectorsCleanUp != nil {
782+
if cleanUpQ, err := util.LoadBitfieldQueue(store, *st.PreCommittedSectorsCleanUp, st.QuantSpecEveryDeadline(), PrecommitCleanUpAmtBitwidth); err != nil {
783+
acc.Addf("error loading pre-commit clean up queue: %v", err)
784+
} else {
785+
err = cleanUpQ.ForEach(func(epoch abi.ChainEpoch, bf bitfield.BitField) error {
786+
quantized := quant.QuantizeUp(epoch)
787+
acc.Require(quantized == epoch, "precommit expiration %d is not quantized", epoch)
788+
if err = bf.ForEach(func(secNum uint64) error {
789+
cleanUpEpochs[secNum] = epoch
790+
return nil
791+
}); err != nil {
792+
acc.Addf("error iteration pre-commit expiration bitfield: %v", err)
793+
}
789794
return nil
790-
}); err != nil {
791-
acc.Addf("error iteration pre-commit expiration bitfield: %v", err)
792-
}
793-
return nil
794-
})
795-
acc.RequireNoError(err, "error iterating pre-commit clean up queue")
795+
})
796+
acc.RequireNoError(err, "error iterating pre-commit clean up queue")
797+
}
796798
}
797799

798800
precommitTotal := big.Zero()
799-
if precommitted, err := adt.AsMap(store, st.PreCommittedSectors, builtin.DefaultHamtBitwidth); err != nil {
800-
acc.Addf("error loading precommitted sectors: %v", err)
801-
} else {
802-
var precommit SectorPreCommitOnChainInfo
803-
err = precommitted.ForEach(&precommit, func(key string) error {
804-
secNum, err := abi.ParseUIntKey(key)
805-
if err != nil {
806-
acc.Addf("error parsing pre-commit key as uint: %v", err)
807-
return nil
808-
}
809-
810-
allocated := false
811-
if allocatedSectorsMap != nil {
812-
allocated = allocatedSectorsMap[secNum]
813-
} else {
814-
allocated, err = allocatedSectorsBf.IsSet(secNum)
801+
if st.PreCommittedSectors != nil {
802+
if precommitted, err := adt.AsMap(store, *st.PreCommittedSectors, builtin.DefaultHamtBitwidth); err != nil {
803+
acc.Addf("error loading precommitted sectors: %v", err)
804+
} else {
805+
var precommit SectorPreCommitOnChainInfo
806+
err = precommitted.ForEach(&precommit, func(key string) error {
807+
secNum, err := abi.ParseUIntKey(key)
815808
if err != nil {
816-
acc.Addf("error checking allocated sectors: %v", err)
809+
acc.Addf("error parsing pre-commit key as uint: %v", err)
817810
return nil
818811
}
819-
}
820-
acc.Require(allocated, "pre-committed sector number has not been allocated %d", secNum)
821812

822-
_, found := cleanUpEpochs[secNum]
823-
acc.Require(found, "no clean up epoch for pre-commit at %d", precommit.PreCommitEpoch)
813+
allocated := false
814+
if allocatedSectorsMap != nil {
815+
allocated = allocatedSectorsMap[secNum]
816+
} else {
817+
allocated, err = allocatedSectorsBf.IsSet(secNum)
818+
if err != nil {
819+
acc.Addf("error checking allocated sectors: %v", err)
820+
return nil
821+
}
822+
}
823+
acc.Require(allocated, "pre-committed sector number has not been allocated %d", secNum)
824824

825-
precommitTotal = big.Add(precommitTotal, precommit.PreCommitDeposit)
826-
return nil
827-
})
828-
acc.RequireNoError(err, "error iterating pre-committed sectors")
825+
_, found := cleanUpEpochs[secNum]
826+
acc.Require(found, "no clean up epoch for pre-commit at %d", precommit.PreCommitEpoch)
827+
828+
precommitTotal = big.Add(precommitTotal, precommit.PreCommitDeposit)
829+
return nil
830+
})
831+
acc.RequireNoError(err, "error iterating pre-committed sectors")
832+
}
829833
}
830834

831835
acc.Require(st.PreCommitDeposits.Equals(precommitTotal),

‎builtin/v16/miner/miner_state.go

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,10 @@ type State struct {
3535
InitialPledge abi.TokenAmount // Sum of initial pledge requirements of all active sectors
3636

3737
// Sectors that have been pre-committed but not yet proven.
38-
PreCommittedSectors cid.Cid // Map, HAMT[SectorNumber]SectorPreCommitOnChainInfo
38+
PreCommittedSectors *cid.Cid // Map, HAMT[SectorNumber]SectorPreCommitOnChainInfo
3939

4040
// PreCommittedSectorsCleanUp maintains the state required to cleanup expired PreCommittedSectors.
41-
PreCommittedSectorsCleanUp cid.Cid // BitFieldQueue (AMT[Epoch]*BitField)
41+
PreCommittedSectorsCleanUp *cid.Cid // BitFieldQueue (AMT[Epoch]*BitField)
4242

4343
// Allocated sector IDs. Sector IDs can never be reused once allocated.
4444
AllocatedSectors cid.Cid // BitField
@@ -217,7 +217,11 @@ func (st *State) QuantSpecForDeadline(dlIdx uint64) builtin.QuantSpec {
217217
}
218218

219219
func (st *State) GetPrecommittedSector(store adt.Store, sectorNo abi.SectorNumber) (*SectorPreCommitOnChainInfo, bool, error) {
220-
precommitted, err := adt.AsMap(store, st.PreCommittedSectors, builtin.DefaultHamtBitwidth)
220+
if st.PreCommittedSectors == nil {
221+
return nil, false, nil
222+
}
223+
224+
precommitted, err := adt.AsMap(store, *st.PreCommittedSectors, builtin.DefaultHamtBitwidth)
221225
if err != nil {
222226
return nil, false, err
223227
}

0 commit comments

Comments
 (0)