Remove BlockHeaderProofType union type (#725)

* Remove union type

* fix tests and utils

* fix more tests

* remove unused method

* Some test updates

* comment out invalid test

* Fix broken test

* fix moar test

* Fix integration test

* address feedback

Author: acolytec3

packages/cli/src/util.ts

@@ -1,14 +1,3 @@
-import { BlockHeader } from '@ethereumjs/block'
-import { RLP } from '@ethereumjs/rlp'
-import { TransactionFactory } from '@ethereumjs/tx'
-import { bytesToHex } from '@ethereumjs/util'
-import {
-  BlockBodyContentType,
-  BlockHeaderWithProof,
-  EpochAccumulator,
-  sszReceiptType,
-  sszUnclesType,
-} from 'portalnetwork'
 
 import type { BaseNetwork, NetworkId } from 'portalnetwork'
 import type { Enr } from './rpc/schema/types.js'
@@ -62,52 +51,4 @@ export const addBootNode = async (networkId: NetworkId, baseNetwork: BaseNetwork
     throw new Error(`Error adding bootnode ${enr} to network \
       ${networkId}: ${error.message ?? error}`)
   }
-}
-
-export const toJSON = (contentKey: Uint8Array, res: Uint8Array) => {
-  const contentType = contentKey[0]
-  let content = {}
-  switch (contentType) {
-    case 0: {
-      const blockHeaderWithProof = BlockHeaderWithProof.deserialize(res)
-      const header = BlockHeader.fromRLPSerializedHeader(blockHeaderWithProof.header, {
-        setHardfork: true,
-      }).toJSON()
-      const proof =
-        blockHeaderWithProof.proof.selector === 0
-          ? []
-          : (blockHeaderWithProof.proof.value as Uint8Array[])?.map((p) => bytesToHex(p))
-      content = { header, proof }
-      break
-    }
-    case 1: {
-      const blockBody = BlockBodyContentType.deserialize(res)
-      const transactions = blockBody.allTransactions.map((tx) =>
-        TransactionFactory.fromSerializedData(tx).toJSON(),
-      )
-      const unclesRlp = bytesToHex(sszUnclesType.deserialize(blockBody.sszUncles))
-      content = {
-        transactions,
-        uncles: {
-          rlp: unclesRlp,
-          count: RLP.decode(unclesRlp).length.toString(),
-        },
-      }
-      break
-    }
-    case 2: {
-      const receipt = sszReceiptType.deserialize(res)
-      content = receipt
-      break
-    }
-    case 3: {
-      const epochAccumulator = EpochAccumulator.deserialize(res)
-      content = epochAccumulator
-      break
-    }
-    default: {
-      content = {}
-    }
-  }
-  return JSON.stringify(content)
-}
+}

packages/portalnetwork/src/networks/history/history.ts

@@ -2,9 +2,11 @@ import { Block, BlockHeader } from '@ethereumjs/block'
 import { bytesToHex, bytesToInt, equalsBytes, hexToBytes } from '@ethereumjs/util'
 import debug from 'debug'
 
+import type { BaseNetworkConfig, ContentLookupResponse, FindContentMessage } from '../../index.js'
 import {
   ContentMessageType,
   FoundContent,
+  HistoricalSummariesBlockProof,
   MessageCodes,
   PingPongPayloadExtensions,
   PortalWireMessageType,
@@ -19,19 +21,26 @@ import { BaseNetwork } from '../network.js'
 import { NetworkId } from '../types.js'
 
 import {
+  AccumulatorProofType,
   BlockHeaderWithProof,
   BlockNumberKey,
+  HistoricalRootsBlockProof,
   HistoryNetworkContentType,
   MERGE_BLOCK,
   SHANGHAI_BLOCK,
   sszReceiptsListType,
 } from './types.js'
-import { getContentKey, verifyPostCapellaHeaderProof, verifyPreCapellaHeaderProof, verifyPreMergeHeaderProof } from './util.js'
+import {
+  getContentKey,
+  verifyPostCapellaHeaderProof,
+  verifyPreCapellaHeaderProof,
+  verifyPreMergeHeaderProof,
+} from './util.js'
 
 import type { ENR } from '@chainsafe/enr'
-import type { Debugger } from 'debug'
-import type { BaseNetworkConfig, ContentLookupResponse, FindContentMessage } from '../../index.js'
+
 import { RunStatusCode } from '@lodestar/light-client'
+import type { Debugger } from 'debug'
 
 export class HistoryNetwork extends BaseNetwork {
   networkId: NetworkId.HistoryNetwork
@@ -95,7 +104,7 @@ export class HistoryNetwork extends BaseNetwork {
   /**
    * Retrieve a blockheader from the DB by hash
    * @param blockHash the hash of the blockheader sought
-   * @param asBytes return the header as RLP encoded bytes or as an @ethereumjs/block BlockHeader
+   * @param asBytes return the header as RLP encoded bytes or as an `@ethereumjs/block` BlockHeader
    * @returns the bytes or Blockheader if found or else undefined
    */
   public getBlockHeaderFromDB = async (
@@ -109,6 +118,7 @@ export class HistoryNetwork extends BaseNetwork {
     const value = await this.findContentLocally(contentKey)
     if (value === undefined) return undefined
     const header = BlockHeaderWithProof.deserialize(value).header
+
     return asBytes === true
       ? header
       : BlockHeader.fromRLPSerializedHeader(header, { setHardfork: true })
@@ -162,57 +172,76 @@ export class HistoryNetwork extends BaseNetwork {
     const proof = headerProof.proof
 
     if (header.number < MERGE_BLOCK) {
-      if (proof.value === null) {
-        throw new Error('Received pre-merge block header without proof')
+      let deserializedProof: Uint8Array[]
+      try {
+        deserializedProof = AccumulatorProofType.deserialize(proof)
+      } catch (err: any) {
+        this.logger(`invalid proof for block ${bytesToHex(header.hash())}`)
+        throw new Error(`invalid proof for block ${bytesToHex(header.hash())}`)
       }
-      if (Array.isArray(proof.value)) {
-        let validated = false
-        if ('blockHash' in validation) {
-          validated = verifyPreMergeHeaderProof(proof.value, validation.blockHash, header.number)
-        } else {
-          validated = verifyPreMergeHeaderProof(
-            proof.value,
-            bytesToHex(header.hash()),
-            validation.blockNumber,
-          )
-        }
-        if (!validated) {
-          throw new Error('Unable to validate proof for pre-merge header')
-        }
+      let validated = false
+      if ('blockHash' in validation) {
+        validated = verifyPreMergeHeaderProof(
+          deserializedProof,
+          validation.blockHash,
+          header.number,
+        )
+      } else {
+        validated = verifyPreMergeHeaderProof(
+          deserializedProof,
+          bytesToHex(header.hash()),
+          validation.blockNumber,
+        )
+      }
+      if (!validated) {
+        throw new Error('Unable to validate proof for pre-merge header')
       }
     } else if (header.number < SHANGHAI_BLOCK) {
-      if (proof.value === null) {
-        this.logger('Received post-merge block without proof')
-        throw new Error('Received post-merge block header without proof')
+      let deserializedProof: ReturnType<typeof HistoricalRootsBlockProof.deserialize>
+      try {
+        deserializedProof = HistoricalRootsBlockProof.deserialize(proof)
+      } catch (err: any) {
+        this.logger(`invalid proof for block ${bytesToHex(header.hash())}`)
+        throw new Error(`invalid proof for block ${bytesToHex(header.hash())}`)
       }
       let validated = false
       try {
-        validated = verifyPreCapellaHeaderProof(proof.value as any, header.hash())
+        validated = verifyPreCapellaHeaderProof(deserializedProof, header.hash())
       } catch (err: any) {
         this.logger(`Unable to validate proof for post-merge header: ${err.message}`)
       }
       if (!validated) {
         throw new Error('Unable to validate proof for post-merge header')
       }
-    }
-    else {
+    } else {
       // TODO: Check proof slot to ensure header is from previous sync period and handle ephemeral headers separately
-      if (proof.value === null) {
-        this.logger('Received post-merge block without proof')
+
+      let deserializedProof: ReturnType<typeof HistoricalSummariesBlockProof.deserialize>
+      try {
+        deserializedProof = HistoricalSummariesBlockProof.deserialize(proof)
+      } catch (err: any) {
+        this.logger(`invalid proof for block ${bytesToHex(header.hash())}`)
+        throw new Error(`invalid proof for block ${bytesToHex(header.hash())}`)
       }
       const beacon = this.portal.network()['0x500c']
       if (beacon !== undefined && beacon.lightClient?.status === RunStatusCode.started) {
         try {
-          verifyPostCapellaHeaderProof(proof.value as any, header.hash(), beacon.historicalSummaries, beacon.beaconConfig)
+          verifyPostCapellaHeaderProof(
+            deserializedProof,
+            header.hash(),
+            beacon.historicalSummaries,
+            beacon.beaconConfig,
+          )
           this.logger(`Successfully verified proof for block header ${header.number}`)
         } catch {
           this.logger('Received post-capella block header with invalid proof')
           // TODO: throw new Error('Received post-merge block header with invalid proof')
         }
       } else {
-        this.logger('Received post-capella block but Beacon light client is not running so cannot verify proof')
+        this.logger(
+          'Received post-capella block but Beacon light client is not running so cannot verify proof',
+        )
       }
-
     }
     await this.indexBlockHash(header.number, bytesToHex(header.hash()))
     return header.hash()
@@ -348,7 +377,8 @@ export class HistoryNetwork extends BaseNetwork {
       this.gossipManager.add(contentKey)
     }
     this.logger(
-      `${HistoryNetworkContentType[contentType]} added for ${keyOpt instanceof Uint8Array ? bytesToHex(keyOpt) : keyOpt
+      `${HistoryNetworkContentType[contentType]} added for ${
+        keyOpt instanceof Uint8Array ? bytesToHex(keyOpt) : keyOpt
       }`,
     )
   }

packages/portalnetwork/src/networks/history/types.ts

@@ -3,9 +3,7 @@ import {
   ByteVectorType,
   ContainerType,
   ListCompositeType,
-  NoneType,
   UintBigintType,
-  UnionType,
   VectorCompositeType,
 } from '@chainsafe/ssz'
 import { MAX_WITHDRAWALS_PER_PAYLOAD } from '@lodestar/params'
@@ -25,7 +23,7 @@ export const MAX_TRANSACTION_COUNT = 16384 // 2 ** 14
 export const MAX_RECEIPT_LENGTH = 134217728 // 2 ** 27
 export const MAX_HEADER_LENGTH = 8192 // 2 ** 13
 export const MAX_ENCODED_UNCLES_LENGTH = 131072 // MAX_HEADER_LENGTH * 2 ** 4
-
+export const MAX_HEADER_PROOF_LENGTH = 1024
 export const MERGE_BLOCK = 15537393n
 export const SHANGHAI_BLOCK = 17034871n
 
@@ -207,14 +205,7 @@ export const HistoricalSummariesBlockProof = new ContainerType({
   slot: SlotType,
 })
 
-export const BlockHeaderProofType = new UnionType([
-  new NoneType(),
-  AccumulatorProofType,
-  HistoricalRootsBlockProof,
-  HistoricalSummariesBlockProof,
-])
-
 export const BlockHeaderWithProof = new ContainerType({
   header: new ByteListType(MAX_HEADER_LENGTH),
-  proof: BlockHeaderProofType,
+  proof: new ByteListType(MAX_HEADER_PROOF_LENGTH),
 })

packages/portalnetwork/src/networks/history/util.ts

@@ -8,6 +8,7 @@ import { ssz } from '@lodestar/types'
 import { historicalEpochs } from './data/epochHashes.js'
 import { historicalRoots } from './data/historicalRoots.js'
 import {
+  AccumulatorProofType,
   BlockBodyContentType,
   BlockHeaderWithProof,
   BlockNumberKey,
@@ -95,17 +96,17 @@ export const decodeHistoryNetworkContentKey = (
   contentKey: Uint8Array,
 ):
   | {
-    contentType:
-    | HistoryNetworkContentType.BlockHeader
-    | HistoryNetworkContentType.BlockBody
-    | HistoryNetworkContentType.Receipt
-    | HistoryNetworkContentType.HeaderProof
-    keyOpt: Uint8Array
-  }
+      contentType:
+        | HistoryNetworkContentType.BlockHeader
+        | HistoryNetworkContentType.BlockBody
+        | HistoryNetworkContentType.Receipt
+        | HistoryNetworkContentType.HeaderProof
+      keyOpt: Uint8Array
+    }
   | {
-    contentType: HistoryNetworkContentType.BlockHeaderByNumber
-    keyOpt: bigint
-  } => {
+      contentType: HistoryNetworkContentType.BlockHeaderByNumber
+      keyOpt: bigint
+    } => {
   const contentType: HistoryNetworkContentType = contentKey[0]
   if (contentType === HistoryNetworkContentType.BlockHeaderByNumber) {
     const blockNumber = BlockNumberKey.deserialize(contentKey.slice(1)).blockNumber
@@ -208,39 +209,27 @@ export const reassembleBlock = (rawHeader: Uint8Array, rawBody?: Uint8Array) =>
  * @param rlpHex RLP encoded block as hex string
  * @param blockHash block hash as 0x prefixed hex string
  * @param network a running `PortalNetwork` client
+ * @param proof the header proof anchoring the block to an accumulator
+ * (i.e. pre-merge historical accumulator, historical_roots, or historical summaries)
  */
 export const addRLPSerializedBlock = async (
   rlpHex: string,
   blockHash: string,
   network: HistoryNetwork,
-  witnesses: Witnesses,
+  proof: Uint8Array,
 ) => {
   const block = Block.fromRLPSerializedBlock(hexToBytes(rlpHex), {
     setHardfork: true,
   })
   const header = block.header
   const headerKey = getContentKey(HistoryNetworkContentType.BlockHeader, hexToBytes(blockHash))
-  if (header.number < MERGE_BLOCK) {
-    const proof: Witnesses = witnesses
-    const headerProof = BlockHeaderWithProof.serialize({
-      header: header.serialize(),
-      proof: { selector: 1, value: proof },
-    })
-    try {
-      await network.validateHeader(headerProof, { blockHash })
-    } catch {
-      network.logger('Header proof failed validation while loading block from RLP')
-    }
-    await network.store(headerKey, headerProof)
-  } else {
-    const headerProof = BlockHeaderWithProof.serialize({
-      header: header.serialize(),
-      proof: { selector: 0, value: null },
-    })
-    await network.indexBlockHash(header.number, bytesToHex(header.hash()))
+  const headerProof = BlockHeaderWithProof.serialize({
+    header: header.serialize(),
+    proof,
+  })
 
-    await network.store(headerKey, headerProof)
-  }
+  await network.store(headerKey, headerProof)
+  await network.indexBlockHash(header.number, bytesToHex(header.hash()))
   const sszBlock = sszEncodeBlockBody(block)
   await network.addBlockBody(sszBlock, header.hash(), header.serialize())
 }