Merge pull request #834 from iotaledger/fix/orphanage2

Fix: Orphanage

pkg/protocol/engine/blocks/block.go

@@ -2,6 +2,7 @@ package blocks
 
 import (
 	"fmt"
+	"sync/atomic"
 	"time"
 
 	"github.com/iotaledger/hive.go/ds"
@@ -11,10 +12,20 @@ import (
 	"github.com/iotaledger/hive.go/stringify"
 	"github.com/iotaledger/iota-core/pkg/core/account"
 	"github.com/iotaledger/iota-core/pkg/model"
+	"github.com/iotaledger/iota-core/pkg/protocol/engine/mempool"
 	iotago "github.com/iotaledger/iota.go/v4"
 )
 
 type Block struct {
+	// ParentsBooked is triggered when all parents of the block are booked.
+	ParentsBooked reactive.Event
+
+	// PayloadDependenciesAvailable is triggered when the dependencies of the block's payload are available.
+	PayloadDependenciesAvailable reactive.Event
+
+	// SignedTransactionMetadata contains the signed transaction metadata of the block.
+	SignedTransactionMetadata reactive.Variable[mempool.SignedTransactionMetadata]
+
 	// BlockDAG block
 	missing             bool
 	missingBlockID      iotago.BlockID
@@ -76,6 +87,10 @@ func (r *rootBlock) String() string {
 
 func newEmptyBlock() *Block {
 	return &Block{
+		ParentsBooked:                reactive.NewEvent(),
+		PayloadDependenciesAvailable: reactive.NewEvent(),
+		SignedTransactionMetadata:    reactive.NewVariable[mempool.SignedTransactionMetadata](),
+
 		witnesses:             ds.NewSet[account.SeatIndex](),
 		spenderIDs:            ds.NewSet[iotago.TransactionID](),
 		payloadSpenderIDs:     ds.NewSet[iotago.TransactionID](),
@@ -112,6 +127,8 @@ func NewRootBlock(blockID iotago.BlockID, commitmentID iotago.CommitmentID, issu
 	b.scheduled = true
 
 	// This should be true since we commit and evict on acceptance.
+	b.ParentsBooked.Set(true)
+	b.PayloadDependenciesAvailable.Set(true)
 	b.solid.Init(true)
 	b.booked.Init(true)
 	b.weightPropagated.Init(true)
@@ -663,3 +680,30 @@ func (b *Block) ModelBlock() *model.Block {
 func (b *Block) WorkScore() iotago.WorkScore {
 	return b.workScore
 }
+
+func (b *Block) WaitForPayloadDependencies(dependencies ds.Set[mempool.StateMetadata]) {
+	if dependencies == nil || dependencies.Size() == 0 {
+		b.PayloadDependenciesAvailable.Trigger()
+
+		return
+	}
+
+	var unreferencedOutputCount atomic.Int32
+	unreferencedOutputCount.Store(int32(dependencies.Size()))
+
+	dependencies.Range(func(dependency mempool.StateMetadata) {
+		dependencyReady := false
+
+		dependency.OnAccepted(func() {
+			dependency.OnInclusionSlotUpdated(func(_ iotago.SlotIndex, inclusionSlot iotago.SlotIndex) {
+				if !dependencyReady && inclusionSlot <= b.ID().Slot() {
+					dependencyReady = true
+
+					if unreferencedOutputCount.Add(-1) == 0 {
+						b.PayloadDependenciesAvailable.Trigger()
+					}
+				}
+			})
+		})
+	})
+}

pkg/protocol/engine/booker/inmemorybooker/booker.go

@@ -96,15 +96,14 @@ func (b *Booker) Init(ledger ledger.Ledger, loadBlockFromStorage func(iotago.Blo
 // Queue checks if payload is solid and then sets up the block to react to its parents.
 func (b *Booker) Queue(block *blocks.Block) error {
 	signedTransactionMetadata, containsTransaction := b.ledger.AttachTransaction(block)
-
 	if !containsTransaction {
 		b.setupBlock(block)
-		return nil
-	}
 
-	if signedTransactionMetadata == nil {
+		return nil
+	} else if signedTransactionMetadata == nil {
 		return ierrors.Errorf("transaction in block %s was not attached", block.ID())
 	}
+	block.SignedTransactionMetadata.Set(signedTransactionMetadata)
 
 	// Based on the assumption that we always fork and the UTXO and Tangle past cones are always fully known.
 	signedTransactionMetadata.OnSignaturesValid(func() {
@@ -137,6 +136,12 @@ func (b *Booker) Queue(block *blocks.Block) error {
 func (b *Booker) Reset() { /* nothing to reset but comply with interface */ }
 
 func (b *Booker) setupBlock(block *blocks.Block) {
+	var payloadDependencies, directlyReferencedPayloadDependencies ds.Set[mempool.StateMetadata]
+	if signedTransactionMetadata := block.SignedTransactionMetadata.Get(); signedTransactionMetadata != nil && signedTransactionMetadata.SignaturesInvalid() == nil && !signedTransactionMetadata.TransactionMetadata().IsInvalid() {
+		payloadDependencies = signedTransactionMetadata.TransactionMetadata().Inputs()
+		directlyReferencedPayloadDependencies = ds.NewSet[mempool.StateMetadata]()
+	}
+
 	var unbookedParentsCount atomic.Int32
 	unbookedParentsCount.Store(int32(len(block.Parents())))
 
@@ -149,13 +154,15 @@ func (b *Booker) setupBlock(block *blocks.Block) {
 		}
 
 		parentBlock.Booked().OnUpdateOnce(func(_ bool, _ bool) {
-			if unbookedParentsCount.Add(-1) == 0 {
-				if err := b.book(block); err != nil {
-					if block.SetInvalid() {
-						b.events.BlockInvalid.Trigger(block, ierrors.Wrap(err, "failed to book block"))
-					}
+			if directlyReferencedPayloadDependencies != nil {
+				if parentTransactionMetadata := parentBlock.SignedTransactionMetadata.Get(); parentTransactionMetadata != nil {
+					directlyReferencedPayloadDependencies.AddAll(parentTransactionMetadata.TransactionMetadata().Outputs())
 				}
 			}
+
+			if unbookedParentsCount.Add(-1) == 0 {
+				block.ParentsBooked.Trigger()
+			}
 		})
 
 		parentBlock.Invalid().OnUpdateOnce(func(_ bool, _ bool) {
@@ -164,6 +171,22 @@ func (b *Booker) setupBlock(block *blocks.Block) {
 			}
 		})
 	})
+
+	block.ParentsBooked.OnTrigger(func() {
+		if directlyReferencedPayloadDependencies != nil {
+			payloadDependencies.DeleteAll(directlyReferencedPayloadDependencies)
+		}
+
+		block.WaitForPayloadDependencies(payloadDependencies)
+	})
+
+	block.PayloadDependenciesAvailable.OnTrigger(func() {
+		if err := b.book(block); err != nil {
+			if block.SetInvalid() {
+				b.events.BlockInvalid.Trigger(block, ierrors.Wrap(err, "failed to book block"))
+			}
+		}
+	})
 }
 
 func (b *Booker) book(block *blocks.Block) error {

pkg/protocol/engine/ledger/tests/state.go

@@ -10,13 +10,15 @@ type MockedState struct {
 	id           iotago.OutputID
 	output       *MockedOutput
 	creationSlot iotago.SlotIndex
+	slotBooked   iotago.SlotIndex
 }
 
 func NewMockedState(transactionID iotago.TransactionID, index uint16) *MockedState {
 	return &MockedState{
 		id:           iotago.OutputIDFromTransactionIDAndIndex(transactionID, index),
 		output:       &MockedOutput{},
 		creationSlot: iotago.SlotIndex(0),
+		slotBooked:   iotago.SlotIndex(0),
 	}
 }
 
@@ -44,6 +46,10 @@ func (m *MockedState) SlotCreated() iotago.SlotIndex {
 	return m.creationSlot
 }
 
+func (m *MockedState) SlotBooked() iotago.SlotIndex {
+	return m.slotBooked
+}
+
 func (m *MockedState) String() string {
 	return "MockedOutput(" + m.id.ToHex() + ")"
 }

pkg/protocol/engine/mempool/signed_transaction_metadata.go

@@ -11,6 +11,8 @@ type SignedTransactionMetadata interface {
 
 	OnSignaturesInvalid(callback func(err error)) (unsubscribe func())
 
+	SignaturesInvalid() error
+
 	TransactionMetadata() TransactionMetadata
 
 	Attachments() []iotago.BlockID

pkg/protocol/engine/mempool/state.go

@@ -15,6 +15,9 @@ type State interface {
 
 	// Whether the state is read only.
 	IsReadOnly() bool
+
+	// SlotBooked returns the slot index of the state if it is booked.
+	SlotBooked() iotago.SlotIndex
 }
 
 // A thin wrapper around a resolved commitment.
@@ -34,6 +37,10 @@ func (s CommitmentInputState) IsReadOnly() bool {
 	return true
 }
 
+func (s CommitmentInputState) SlotBooked() iotago.SlotIndex {
+	return s.Commitment.Slot
+}
+
 func CommitmentInputStateFromCommitment(commitment *iotago.Commitment) CommitmentInputState {
 	return CommitmentInputState{
 		Commitment: commitment,

pkg/protocol/engine/mempool/state_metadata.go

@@ -6,6 +6,8 @@ import (
 )
 
 type StateMetadata interface {
+	CreatingTransaction() TransactionMetadata
+
 	State() State
 
 	SpenderIDs() reactive.Set[iotago.TransactionID]
@@ -16,5 +18,9 @@ type StateMetadata interface {
 
 	OnAcceptedSpenderUpdated(callback func(spender TransactionMetadata))
 
+	InclusionSlot() iotago.SlotIndex
+
+	OnInclusionSlotUpdated(callback func(prevSlot iotago.SlotIndex, newSlot iotago.SlotIndex))
+
 	inclusionFlags
 }

pkg/protocol/engine/mempool/v1/mempool.go

@@ -466,9 +466,12 @@ func (m *MemPool[VoteRank]) requestState(stateRef mempool.StateReference, waitIf
 		request := m.resolveState(stateRef)
 
 		request.OnSuccess(func(state mempool.State) {
+			inclusionSlot := state.SlotBooked()
+
 			// The output was resolved from the ledger, meaning it was actually persisted as it was accepted and
 			// committed: otherwise we would have found it in cache or the request would have never resolved.
 			stateMetadata := NewStateMetadata(state)
+			stateMetadata.inclusionSlot.Set(&inclusionSlot)
 			stateMetadata.accepted.Set(true)
 
 			p.Resolve(stateMetadata)

pkg/protocol/engine/mempool/v1/signed_transaction_metadata.go

@@ -50,6 +50,10 @@ func (s *SignedTransactionMetadata) OnSignaturesInvalid(callback func(error)) (u
 	})
 }
 
+func (s *SignedTransactionMetadata) SignaturesInvalid() error {
+	return s.signaturesInvalid.Get()
+}
+
 func (s *SignedTransactionMetadata) OnSignaturesValid(callback func()) (unsubscribe func()) {
 	return s.signaturesValid.OnTrigger(callback)
 }

pkg/protocol/engine/mempool/v1/state_metadata.go

@@ -12,14 +12,16 @@ import (
 )
 
 type StateMetadata struct {
-	state mempool.State
+	state  mempool.State
+	source *TransactionMetadata
 
 	// lifecycle
 	spenderCount       uint64
 	spent              *promise.Event
 	doubleSpent        *promise.Event
 	spendAccepted      reactive.Variable[*TransactionMetadata]
 	spendCommitted     reactive.Variable[*TransactionMetadata]
+	inclusionSlot      reactive.Variable[*iotago.SlotIndex]
 	allSpendersRemoved *event.Event
 
 	spenderIDs reactive.DerivedSet[iotago.TransactionID]
@@ -29,12 +31,14 @@ type StateMetadata struct {
 
 func NewStateMetadata(state mempool.State, optSource ...*TransactionMetadata) *StateMetadata {
 	return (&StateMetadata{
-		state: state,
+		state:  state,
+		source: lo.First(optSource),
 
 		spent:              promise.NewEvent(),
 		doubleSpent:        promise.NewEvent(),
 		spendAccepted:      reactive.NewVariable[*TransactionMetadata](),
 		spendCommitted:     reactive.NewVariable[*TransactionMetadata](),
+		inclusionSlot:      reactive.NewVariable[*iotago.SlotIndex](),
 		allSpendersRemoved: event.New(),
 
 		spenderIDs: reactive.NewDerivedSet[iotago.TransactionID](),
@@ -51,6 +55,16 @@ func (s *StateMetadata) setup(optSource ...*TransactionMetadata) *StateMetadata
 
 	s.spenderIDs.InheritFrom(source.spenderIDs)
 
+	source.earliestIncludedValidAttachment.OnUpdate(func(_, newValue iotago.BlockID) {
+		s.inclusionSlot.Compute(func(currentValue *iotago.SlotIndex) *iotago.SlotIndex {
+			if newSlot := newValue.Slot(); currentValue == nil || newSlot < *currentValue {
+				return &newSlot
+			}
+
+			return currentValue
+		})
+	})
+
 	source.OnAccepted(func() { s.accepted.Set(true) })
 	source.OnRejected(func() { s.rejected.Trigger() })
 	source.OnCommittedSlotUpdated(lo.Void(s.committedSlot.Set))
@@ -59,6 +73,14 @@ func (s *StateMetadata) setup(optSource ...*TransactionMetadata) *StateMetadata
 	return s
 }
 
+func (s *StateMetadata) CreatingTransaction() mempool.TransactionMetadata {
+	if s.source == nil {
+		return nil
+	}
+
+	return s.source
+}
+
 func (s *StateMetadata) State() mempool.State {
 	return s.state
 }
@@ -113,6 +135,23 @@ func (s *StateMetadata) HasNoSpenders() bool {
 	return atomic.LoadUint64(&s.spenderCount) == 0
 }
 
+func (s *StateMetadata) InclusionSlot() iotago.SlotIndex {
+	return *s.inclusionSlot.Get()
+}
+
+func (s *StateMetadata) OnInclusionSlotUpdated(callback func(prevID iotago.SlotIndex, newID iotago.SlotIndex)) {
+	s.inclusionSlot.OnUpdate(func(oldValue *iotago.SlotIndex, newValue *iotago.SlotIndex) {
+		switch {
+		case oldValue == nil:
+			callback(iotago.SlotIndex(0), *newValue)
+		case newValue == nil:
+			callback(*oldValue, iotago.SlotIndex(0))
+		default:
+			callback(*oldValue, *newValue)
+		}
+	})
+}
+
 func (s *StateMetadata) increaseSpenderCount() {
 	if spenderCount := atomic.AddUint64(&s.spenderCount, 1); spenderCount == 1 {
 		s.spent.Trigger()