Fix packet size calculation for normal packets

Author: acolytec3

packages/portalnetwork/src/networks/history/history.ts

@@ -17,14 +17,15 @@ import {
   FoundContent,
   HistoricalSummariesBlockProof,
   HistoryRadius,
-  MAX_PACKET_SIZE,
+  MAX_UDP_PACKET_SIZE,
   MessageCodes,
   PingPongPayloadExtensions,
   PortalWireMessageType,
   RequestCode,
   decodeHistoryNetworkContentKey,
   decodeReceipts,
   encodeClientInfo,
+  getTalkReqOverhead,
   randUint16,
   reassembleBlock,
   saveReceipts,
@@ -439,7 +440,7 @@ export class HistoryNetwork extends BaseNetwork {
         const firstHeaderNumber = this.ephemeralHeaderIndex.getByValue(
           bytesToHex(contentKey.keyOpt.blockHash),
         )
-        for (let x = 1; x < contentKey.keyOpt.ancestorCount; x++) {
+        for (let x = 1; x <= contentKey.keyOpt.ancestorCount; x++) {
           // Determine if we have the ancestor header at block number `firstHeaderNumber - x`
           const ancestorNumber = firstHeaderNumber! - BigInt(x)
           const ancestorHash = this.ephemeralHeaderIndex.getByKey(ancestorNumber)
@@ -459,7 +460,7 @@ export class HistoryNetwork extends BaseNetwork {
           }
         }
         this.logger.extend('FOUNDCONTENT')(
-          `found ${headersList.length} ancestor headers for ${bytesToHex(contentKey.keyOpt.blockHash)}`,
+          `found ${headersList.length - 1} ancestor headers for ${bytesToHex(contentKey.keyOpt.blockHash)}`,
         )
         value = EphemeralHeaderPayload.serialize(headersList)
       }
@@ -468,7 +469,10 @@ export class HistoryNetwork extends BaseNetwork {
     }
     if (!value) {
       await this.enrResponse(decodedContentMessage.contentKey, src, requestId)
-    } else if (value instanceof Uint8Array && value.length < MAX_PACKET_SIZE) {
+    } else if (
+      value instanceof Uint8Array &&
+      value.length < MAX_UDP_PACKET_SIZE - getTalkReqOverhead(hexToBytes(this.networkId).byteLength)
+    ) {
       this.logger.extend('FOUNDCONTENT')(
         'Found value for requested content ' +
           bytesToHex(decodedContentMessage.contentKey) +

packages/portalnetwork/src/networks/network.ts

@@ -35,7 +35,7 @@ import {
   ContentMessageType,
   ErrorPayload,
   HistoryRadius,
-  MAX_PACKET_SIZE,
+  MAX_UDP_PACKET_SIZE,
   MessageCodes,
   NodeLookup,
   PingPongErrorCodes,
@@ -48,6 +48,7 @@ import {
   encodeClientInfo,
   encodeWithVariantPrefix,
   generateRandomNodeIdAtDistance,
+  getTalkReqOverhead,
   randUint16,
   shortId,
 } from '../index.js'
@@ -762,7 +763,10 @@ export abstract class BaseNetwork extends EventEmitter {
     const value = await this.findContentLocally(decodedContentMessage.contentKey)
     if (!value) {
       await this.enrResponse(decodedContentMessage.contentKey, src, requestId)
-    } else if (value instanceof Uint8Array && value.length < MAX_PACKET_SIZE) {
+    } else if (
+      value instanceof Uint8Array &&
+      value.length < MAX_UDP_PACKET_SIZE - getTalkReqOverhead(hexToBytes(this.networkId).byteLength)
+    ) {
       this.logger(
         'Found value for requested content ' +
           bytesToHex(decodedContentMessage.contentKey) +
@@ -818,7 +822,11 @@ export abstract class BaseNetwork extends EventEmitter {
     if (encodedEnrs.length > 0) {
       this.logger.extend('FINDCONTENT')(`Found ${encodedEnrs.length} closer to content`)
       // TODO: Add capability to send multiple TALKRESP messages if # ENRs exceeds packet size
-      while (encodedEnrs.length > 0 && arrayByteLength(encodedEnrs) > MAX_PACKET_SIZE) {
+      while (
+        encodedEnrs.length > 0 &&
+        arrayByteLength(encodedEnrs) >
+          MAX_UDP_PACKET_SIZE - getTalkReqOverhead(hexToBytes(this.networkId).byteLength)
+      ) {
         // Remove ENRs until total ENRs less than 1200 bytes
         encodedEnrs.pop()
       }

packages/portalnetwork/src/wire/utp/Utils/constants.ts

@@ -24,7 +24,7 @@ export const AUTO_ACK_SMALLER_THAN_ACK_NUMBER: boolean = true
 export const MINIMUM_DIFFERENCE_TIMESTAMP_MICROSEC: number = 120000000
 
 export const DEFAULT_PACKET_SIZE = 512
-export const MAX_PACKET_SIZE: number = 1225
+export const MAX_UDP_PACKET_SIZE: number = 1280
 export const MIN_PACKET_SIZE: number = 150
 export const MINIMUM_MTU: number = 576
 export const SEND_IN_BURST: boolean = true
@@ -34,7 +34,7 @@ export const MICROSECOND_WAIT_BETWEEN_BURSTS: number = 28000
 export const TIME_WAIT_AFTER_LAST_PACKET: number = 3000000
 export const ONLY_POSITIVE_GAIN: boolean = false
 export const DEBUG: boolean = false
-export const MAX_UTP_PACKET_LENGTH = MAX_PACKET_SIZE
+export const MAX_UTP_PACKET_LENGTH = MAX_UDP_PACKET_SIZE
 export const MAX_UDP_HEADER_LENGTH = 48
 export const DEF_HEADER_LENGTH = 20
 

packages/portalnetwork/src/wire/utp/Utils/index.ts

@@ -1,3 +1,4 @@
 export * from './constants.js'
 export * from './math.js'
 export * from './variantPrefix.js'
+export * from './packet.js'

packages/portalnetwork/src/wire/utp/Utils/packet.ts

@@ -0,0 +1,21 @@
+/**
+ * Compute the number of bytes required for a TALKREQ message header
+ * @param protocolIdLen is the length of the protocol ID
+ * @returns the number of bytes required for a TALKREQ message header
+ *
+ * @note Shamelessly copied from [Fluffy](https://github.com/status-im/nimbus-eth1/blob/45767278174a48521de46f029f6e66dc526880f6/fluffy/network/wire/messages.nim#L179)
+ */
+
+export const getTalkReqOverhead = (protocolIdLen: number): number => {
+  return (
+    16 + // IV size
+    55 + // header size
+    1 + // talkReq msg id
+    3 + // rlp encoding outer list, max length will be encoded in 2 bytes
+    9 + // request id (max = 8) + 1 byte from rlp encoding byte string
+    protocolIdLen + // bytes length of protocolid (e.g. 0x500b for History Network)
+    1 + // + 1 is necessary due to rlp encoding of byte string
+    3 + // rlp encoding response byte string, max length in 2 bytes
+    16 // HMAC
+  )
+}

packages/portalnetwork/test/integration/ephemeralHeaders.spec.ts

@@ -97,7 +97,7 @@ describe('should be able to retrieve ephemeral headers from a peer', () => {
     assert.exists(res2)
     if ('content' in res2!) {
       const payload = EphemeralHeaderPayload.deserialize(res2.content)
-      assert.equal(payload.length, 1, 'should only get a single ancestor')
+      assert.equal(payload.length, 2, 'should only get a single ancestor')
     } else {
       assert.fail('Expected content in response')
     }