Fix: Orphanage

pkg/protocol/engine/blocks/block.go

@@ -2,6 +2,7 @@ package blocks
 
 import (
 	"fmt"
+	"sync/atomic"
 	"time"
 
 	"github.com/iotaledger/hive.go/ds"
@@ -12,10 +13,15 @@ import (
 	"github.com/iotaledger/hive.go/stringify"
 	"github.com/iotaledger/iota-core/pkg/core/account"
 	"github.com/iotaledger/iota-core/pkg/model"
+	"github.com/iotaledger/iota-core/pkg/protocol/engine/mempool"
 	iotago "github.com/iotaledger/iota.go/v4"
 )
 
 type Block struct {
+	AllParentsBooked          reactive.Event
+	AllDependenciesReady      reactive.Event
+	SignedTransactionMetadata reactive.Variable[mempool.SignedTransactionMetadata]
+
 	// BlockDAG block
 	missing             bool
 	missingBlockID      iotago.BlockID
@@ -77,6 +83,10 @@ func (r *rootBlock) String() string {
 // NewBlock creates a new Block with the given options.
 func NewBlock(modelBlock *model.Block) *Block {
 	return &Block{
+		AllParentsBooked:          reactive.NewEvent(),
+		AllDependenciesReady:      reactive.NewEvent(),
+		SignedTransactionMetadata: reactive.NewVariable[mempool.SignedTransactionMetadata](),
+
 		witnesses:             ds.NewSet[account.SeatIndex](),
 		spenderIDs:            ds.NewSet[iotago.TransactionID](),
 		payloadSpenderIDs:     ds.NewSet[iotago.TransactionID](),
@@ -95,6 +105,10 @@ func NewBlock(modelBlock *model.Block) *Block {
 
 func NewRootBlock(blockID iotago.BlockID, commitmentID iotago.CommitmentID, issuingTime time.Time) *Block {
 	b := &Block{
+		AllParentsBooked:          reactive.NewEvent(),
+		AllDependenciesReady:      reactive.NewEvent(),
+		SignedTransactionMetadata: reactive.NewVariable[mempool.SignedTransactionMetadata](),
+
 		witnesses:             ds.NewSet[account.SeatIndex](),
 		spenderIDs:            ds.NewSet[iotago.TransactionID](),
 		payloadSpenderIDs:     ds.NewSet[iotago.TransactionID](),
@@ -117,6 +131,8 @@ func NewRootBlock(blockID iotago.BlockID, commitmentID iotago.CommitmentID, issu
 	}
 
 	// This should be true since we commit and evict on acceptance.
+	b.AllParentsBooked.Set(true)
+	b.AllDependenciesReady.Set(true)
 	b.solid.Set(true)
 	b.booked.Set(true)
 	b.weightPropagated.Set(true)
@@ -128,6 +144,10 @@ func NewRootBlock(blockID iotago.BlockID, commitmentID iotago.CommitmentID, issu
 
 func NewMissingBlock(blockID iotago.BlockID) *Block {
 	return &Block{
+		AllParentsBooked:          reactive.NewEvent(),
+		AllDependenciesReady:      reactive.NewEvent(),
+		SignedTransactionMetadata: reactive.NewVariable[mempool.SignedTransactionMetadata](),
+
 		missing:               true,
 		missingBlockID:        blockID,
 		witnesses:             ds.NewSet[account.SeatIndex](),
@@ -690,3 +710,30 @@ func (b *Block) ModelBlock() *model.Block {
 func (b *Block) WorkScore() iotago.WorkScore {
 	return b.workScore
 }
+
+func (b *Block) WaitForUTXODependencies(dependencies ds.Set[mempool.StateMetadata]) {
+	if dependencies == nil || dependencies.Size() == 0 {
+		b.AllDependenciesReady.Trigger()
+
+		return
+	}
+
+	var unreferencedOutputCount atomic.Int32
+	unreferencedOutputCount.Store(int32(dependencies.Size()))
+
+	dependencies.Range(func(dependency mempool.StateMetadata) {
+		dependencyReady := false
+
+		dependency.OnAccepted(func() {
+			dependency.OnInclusionSlotUpdated(func(_ iotago.SlotIndex, inclusionSlot iotago.SlotIndex) {
+				if !dependencyReady && inclusionSlot <= b.ID().Slot() {
+					dependencyReady = true
+
+					if unreferencedOutputCount.Add(-1) == 0 {
+						b.AllDependenciesReady.Trigger()
+					}
+				}
+			})
+		})
+	})
+}

pkg/protocol/engine/booker/inmemorybooker/booker.go

@@ -94,26 +94,19 @@ func (b *Booker) Init(ledger ledger.Ledger, loadBlockFromStorage func(iotago.Blo
 // Queue checks if payload is solid and then sets up the block to react to its parents.
 func (b *Booker) Queue(block *blocks.Block) error {
 	signedTransactionMetadata, containsTransaction := b.ledger.AttachTransaction(block)
-
 	if !containsTransaction {
 		b.setupBlock(block)
-		return nil
-	}
 
-	if signedTransactionMetadata == nil {
+		return nil
+	} else if signedTransactionMetadata == nil {
 		return ierrors.Errorf("transaction in block %s was not attached", block.ID())
 	}
+	block.SignedTransactionMetadata.Set(signedTransactionMetadata)
 
 	// Based on the assumption that we always fork and the UTXO and Tangle past cones are always fully known.
 	signedTransactionMetadata.OnSignaturesValid(func() {
 		transactionMetadata := signedTransactionMetadata.TransactionMetadata()
 
-		if orphanedSlot, isOrphaned := transactionMetadata.OrphanedSlot(); isOrphaned && orphanedSlot <= block.SlotCommitmentID().Slot() {
-			block.SetInvalid()
-
-			return
-		}
-
 		transactionMetadata.OnBooked(func() {
 			block.SetPayloadSpenderIDs(transactionMetadata.SpenderIDs())
 			b.setupBlock(block)
@@ -135,6 +128,12 @@ func (b *Booker) Queue(block *blocks.Block) error {
 func (b *Booker) Reset() { /* nothing to reset but comply with interface */ }
 
 func (b *Booker) setupBlock(block *blocks.Block) {
+	var utxoDependencies, directlyReferencedUTXODependencies ds.Set[mempool.StateMetadata]
+	if signedTransactionMetadata := block.SignedTransactionMetadata.Get(); signedTransactionMetadata != nil && signedTransactionMetadata.SignaturesInvalid() == nil && !signedTransactionMetadata.TransactionMetadata().IsInvalid() {
+		utxoDependencies = signedTransactionMetadata.TransactionMetadata().Inputs()
+		directlyReferencedUTXODependencies = ds.NewSet[mempool.StateMetadata]()
+	}
+
 	var unbookedParentsCount atomic.Int32
 	unbookedParentsCount.Store(int32(len(block.Parents())))
 
@@ -147,13 +146,15 @@ func (b *Booker) setupBlock(block *blocks.Block) {
 		}
 
 		parentBlock.Booked().OnUpdateOnce(func(_ bool, _ bool) {
-			if unbookedParentsCount.Add(-1) == 0 {
-				if err := b.book(block); err != nil {
-					if block.SetInvalid() {
-						b.events.BlockInvalid.Trigger(block, ierrors.Wrap(err, "failed to book block"))
-					}
+			if directlyReferencedUTXODependencies != nil {
+				if parentTransactionMetadata := parentBlock.SignedTransactionMetadata.Get(); parentTransactionMetadata != nil {
+					directlyReferencedUTXODependencies.AddAll(parentTransactionMetadata.TransactionMetadata().Outputs())
 				}
 			}
+
+			if unbookedParentsCount.Add(-1) == 0 {
+				block.AllParentsBooked.Trigger()
+			}
 		})
 
 		parentBlock.Invalid().OnUpdateOnce(func(_ bool, _ bool) {
@@ -162,6 +163,22 @@ func (b *Booker) setupBlock(block *blocks.Block) {
 			}
 		})
 	})
+
+	block.AllParentsBooked.OnTrigger(func() {
+		if directlyReferencedUTXODependencies != nil {
+			utxoDependencies.DeleteAll(directlyReferencedUTXODependencies)
+		}
+
+		block.WaitForUTXODependencies(utxoDependencies)
+	})
+
+	block.AllDependenciesReady.OnTrigger(func() {
+		if err := b.book(block); err != nil {
+			if block.SetInvalid() {
+				b.events.BlockInvalid.Trigger(block, ierrors.Wrap(err, "failed to book block"))
+			}
+		}
+	})
 }
 
 func (b *Booker) book(block *blocks.Block) error {

pkg/protocol/engine/ledger/tests/state.go

@@ -10,13 +10,15 @@ type MockedState struct {
 	id           iotago.OutputID
 	output       *MockedOutput
 	creationSlot iotago.SlotIndex
+	slotBooked   iotago.SlotIndex
 }
 
 func NewMockedState(transactionID iotago.TransactionID, index uint16) *MockedState {
 	return &MockedState{
 		id:           iotago.OutputIDFromTransactionIDAndIndex(transactionID, index),
 		output:       &MockedOutput{},
 		creationSlot: iotago.SlotIndex(0),
+		slotBooked:   iotago.SlotIndex(0),
 	}
 }
 
@@ -44,6 +46,10 @@ func (m *MockedState) SlotCreated() iotago.SlotIndex {
 	return m.creationSlot
 }
 
+func (m *MockedState) SlotBooked() iotago.SlotIndex {
+	return m.slotBooked
+}
+
 func (m *MockedState) String() string {
 	return "MockedOutput(" + m.id.ToHex() + ")"
 }

pkg/protocol/engine/mempool/signed_transaction_metadata.go

@@ -11,6 +11,8 @@ type SignedTransactionMetadata interface {
 
 	OnSignaturesInvalid(func(err error)) (unsubscribe func())
 
+	SignaturesInvalid() error
+
 	TransactionMetadata() TransactionMetadata
 
 	Attachments() []iotago.BlockID

pkg/protocol/engine/mempool/state.go

@@ -15,6 +15,9 @@ type State interface {
 
 	// Whether the state is read only.
 	IsReadOnly() bool
+
+	// SlotBooked returns the slot index of the state if it is booked, otherwise -1.
+	SlotBooked() iotago.SlotIndex
 }
 
 // A thin wrapper around a resolved commitment.
@@ -34,6 +37,10 @@ func (s CommitmentInputState) IsReadOnly() bool {
 	return true
 }
 
+func (s CommitmentInputState) SlotBooked() iotago.SlotIndex {
+	return s.Commitment.Slot
+}
+
 func CommitmentInputStateFromCommitment(commitment *iotago.Commitment) CommitmentInputState {
 	return CommitmentInputState{
 		Commitment: commitment,

pkg/protocol/engine/mempool/state_metadata.go

@@ -16,5 +16,9 @@ type StateMetadata interface {
 
 	OnAcceptedSpenderUpdated(callback func(spender TransactionMetadata))
 
+	InclusionSlot() iotago.SlotIndex
+
+	OnInclusionSlotUpdated(func(prevSlot iotago.SlotIndex, newSlot iotago.SlotIndex))
+
 	inclusionFlags
 }

pkg/protocol/engine/mempool/v1/mempool.go

@@ -463,9 +463,12 @@ func (m *MemPool[VoteRank]) requestState(stateRef mempool.StateReference, waitIf
 		request := m.resolveState(stateRef)
 
 		request.OnSuccess(func(state mempool.State) {
+			inclusionSlot := state.SlotBooked()
+
 			// The output was resolved from the ledger, meaning it was actually persisted as it was accepted and
 			// committed: otherwise we would have found it in cache or the request would have never resolved.
 			stateMetadata := NewStateMetadata(state)
+			stateMetadata.inclusionSlot.Set(&inclusionSlot)
 			stateMetadata.accepted.Set(true)
 
 			p.Resolve(stateMetadata)

pkg/protocol/engine/mempool/v1/signed_transaction_metadata.go

@@ -50,6 +50,10 @@ func (s *SignedTransactionMetadata) OnSignaturesInvalid(callback func(error)) (u
 	})
 }
 
+func (s *SignedTransactionMetadata) SignaturesInvalid() error {
+	return s.signaturesInvalid.Get()
+}
+
 func (s *SignedTransactionMetadata) OnSignaturesValid(callback func()) (unsubscribe func()) {
 	return s.signaturesValid.OnTrigger(callback)
 }

pkg/protocol/engine/mempool/v1/state_metadata.go

@@ -20,6 +20,7 @@ type StateMetadata struct {
 	doubleSpent        *promise.Event
 	spendAccepted      reactive.Variable[*TransactionMetadata]
 	spendCommitted     reactive.Variable[*TransactionMetadata]
+	inclusionSlot      reactive.Variable[*iotago.SlotIndex]
 	allSpendersRemoved *event.Event
 
 	spenderIDs reactive.DerivedSet[iotago.TransactionID]
@@ -35,6 +36,7 @@ func NewStateMetadata(state mempool.State, optSource ...*TransactionMetadata) *S
 		doubleSpent:        promise.NewEvent(),
 		spendAccepted:      reactive.NewVariable[*TransactionMetadata](),
 		spendCommitted:     reactive.NewVariable[*TransactionMetadata](),
+		inclusionSlot:      reactive.NewVariable[*iotago.SlotIndex](),
 		allSpendersRemoved: event.New(),
 
 		spenderIDs: reactive.NewDerivedSet[iotago.TransactionID](),
@@ -51,6 +53,16 @@ func (s *StateMetadata) setup(optSource ...*TransactionMetadata) *StateMetadata
 
 	s.spenderIDs.InheritFrom(source.spenderIDs)
 
+	source.earliestIncludedValidAttachment.OnUpdate(func(_, newValue iotago.BlockID) {
+		s.inclusionSlot.Compute(func(currentValue *iotago.SlotIndex) *iotago.SlotIndex {
+			if newSlot := newValue.Slot(); currentValue == nil || newSlot < *currentValue {
+				return &newSlot
+			}
+
+			return currentValue
+		})
+	})
+
 	source.OnPending(func() { s.accepted.Set(false) })
 	source.OnAccepted(func() { s.accepted.Set(true) })
 	source.OnRejected(func() { s.rejected.Trigger() })
@@ -114,6 +126,22 @@ func (s *StateMetadata) HasNoSpenders() bool {
 	return atomic.LoadUint64(&s.spenderCount) == 0
 }
 
+func (s *StateMetadata) InclusionSlot() iotago.SlotIndex {
+	return *s.inclusionSlot.Get()
+}
+
+func (s *StateMetadata) OnInclusionSlotUpdated(callback func(prevID iotago.SlotIndex, newID iotago.SlotIndex)) {
+	s.inclusionSlot.OnUpdate(func(oldValue *iotago.SlotIndex, newValue *iotago.SlotIndex) {
+		if oldValue == nil {
+			callback(iotago.SlotIndex(0), *newValue)
+		} else if newValue == nil {
+			callback(*oldValue, iotago.SlotIndex(0))
+		} else {
+			callback(*oldValue, *newValue)
+		}
+	})
+}
+
 func (s *StateMetadata) increaseSpenderCount() {
 	if spenderCount := atomic.AddUint64(&s.spenderCount, 1); spenderCount == 1 {
 		s.spent.Trigger()