From 315a2096762fb414cb85220d669c63b3158ae1fa Mon Sep 17 00:00:00 2001 From: Scotty <66335769+ScottyPoi@users.noreply.github.com> Date: Wed, 29 Jan 2025 17:51:02 -0700 Subject: [PATCH] Update Discv5 and Remove Buffer (#724) * update discv5 and enr packages * Remove remaining Buffer usage * commit lock --- package-lock.json | 214 ++++++++++-------- packages/cli/package.json | 4 +- packages/portalnetwork/package.json | 4 +- packages/portalnetwork/src/client/client.ts | 44 ++-- .../src/transports/capacitorUdp.ts | 22 +- .../src/transports/websockets.ts | 2 +- .../src/wire/utp/Packets/Packet.ts | 39 ++-- .../src/wire/utp/Packets/PacketHeader.ts | 66 +++--- .../src/wire/utp/PortalNetworkUtp/index.ts | 4 +- .../utp/PortalNetworkUtp/requestManager.ts | 2 +- .../src/wire/utp/Socket/UtpSocket.ts | 2 +- .../src/wire/utp/Utils/variantPrefix.ts | 5 +- .../test/client/provider.spec.ts | 88 +++---- .../postCapellaHeaderProof.spec.ts | 2 +- .../test/wire/utp/packet.spec.ts | 6 +- .../portalnetwork/test/wire/utp/utp.spec.ts | 9 +- 16 files changed, 296 insertions(+), 217 deletions(-) diff --git a/package-lock.json b/package-lock.json index 82bc65ecd..1cd8efd27 100644 --- a/package-lock.json +++ b/package-lock.json @@ -547,18 +547,20 @@ } }, "node_modules/@chainsafe/discv5": { - "version": "10.0.1", + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/@chainsafe/discv5/-/discv5-11.0.0.tgz", + "integrity": "sha512-g/x79Y5b7g6CrNuWazSzXTtnNguC0Sl8xkf9CPMlZ0BrsXmXPzwTp3zgtAiuJcG9x7zDUSDFOejrb4iFcJ0/FQ==", "license": "Apache-2.0", "dependencies": { - "@chainsafe/enr": "^4.0.1", + "@chainsafe/enr": "^5.0.0", + "@ethereumjs/rlp": "^5.0.2", "@libp2p/crypto": "^5.0.1", "@libp2p/interface": "^2.0.1", "@multiformats/multiaddr": "^12.1.10", - "bcrypto": "^5.4.0", - "bigint-buffer": "^1.1.5", + "@noble/hashes": "^1.7.0", + "@noble/secp256k1": "^2.2.2", "debug": "^4.3.1", "lru-cache": "^10.1.0", - "rlp": "^2.2.6", "strict-event-emitter-types": "^2.0.0" } }, @@ -574,33 +576,57 @@ "uint8arraylist": "^2.4.8" } }, + "node_modules/@chainsafe/discv5/node_modules/@noble/hashes": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.7.1.tgz", + "integrity": "sha512-B8XBPsn4vT/KJAGqDzbwztd+6Yte3P4V7iafm24bxgDe/mlRuK6xmWPuCNrKt2vDafZ8MfJLlchDG/vYafQEjQ==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@chainsafe/enr": { - "version": "4.0.1", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@chainsafe/enr/-/enr-5.0.0.tgz", + "integrity": "sha512-xqWBsdFFHy/sLqddsyCT4BH/5PFheyK8Grho27tmiwg/a95nANYANsqXXUP8/HbgCx3F5VDl0/xFacV3Ik7kDA==", "license": "Apache-2.0", "dependencies": { + "@ethereumjs/rlp": "^5.0.2", "@libp2p/crypto": "^5.0.1", "@libp2p/interface": "^2.0.1", "@libp2p/peer-id": "^5.0.1", "@multiformats/multiaddr": "^12.1.10", - "bigint-buffer": "^1.1.5", + "@scure/base": "^1.2.1", "ethereum-cryptography": "^2.2.0", - "rlp": "^2.2.6", - "uint8-varint": "^2.0.2", - "uint8arrays": "^5.0.1" + "uint8-varint": "^2.0.2" } }, "node_modules/@chainsafe/enr/node_modules/@libp2p/interface": { - "version": "2.1.0", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@libp2p/interface/-/interface-2.4.0.tgz", + "integrity": "sha512-PfzxOaz7dU4sdnUNByGLoEk9iqhD0IS+LQMQB12CXh6VyYLA7J8oaoHk3yRBZze3Y4FPa5DHMm5Oi9O/IhreaQ==", "license": "Apache-2.0 OR MIT", "dependencies": { - "@multiformats/multiaddr": "^12.2.3", + "@multiformats/multiaddr": "^12.3.3", "it-pushable": "^3.2.3", - "it-stream-types": "^2.0.1", - "multiformats": "^13.1.0", - "progress-events": "^1.0.0", + "it-stream-types": "^2.0.2", + "multiformats": "^13.3.1", + "progress-events": "^1.0.1", "uint8arraylist": "^2.4.8" } }, + "node_modules/@chainsafe/enr/node_modules/@scure/base": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.2.4.tgz", + "integrity": "sha512-5Yy9czTO47mqz+/J8GM6GIId4umdCk1wc1q8rKERQulIoc8VP9pzDcghv10Tl2E7R96ZUx/PhND3ESYUQX8NuQ==", + "license": "MIT", + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@chainsafe/hashtree": { "version": "1.0.1", "license": "MIT", @@ -1797,40 +1823,75 @@ "uint8arraylist": "^2.4.8" } }, - "node_modules/@libp2p/interface": { - "version": "1.7.0", + "node_modules/@libp2p/peer-id": { + "version": "5.0.10", + "resolved": "https://registry.npmjs.org/@libp2p/peer-id/-/peer-id-5.0.10.tgz", + "integrity": "sha512-+rj61RN3VnmnVoO64LaIZAdLYW2VLsBeSuWIIjeYUXy1U2CpPAzrxmHBQw3YmM2Ozis3FbLgol4pM/9mXsbn2g==", "license": "Apache-2.0 OR MIT", "dependencies": { - "@multiformats/multiaddr": "^12.2.3", - "it-pushable": "^3.2.3", - "it-stream-types": "^2.0.1", - "multiformats": "^13.1.0", - "progress-events": "^1.0.0", - "uint8arraylist": "^2.4.8" + "@libp2p/crypto": "^5.0.9", + "@libp2p/interface": "^2.4.0", + "multiformats": "^13.3.1", + "uint8arrays": "^5.1.0" } }, - "node_modules/@libp2p/peer-id": { - "version": "5.0.2", + "node_modules/@libp2p/peer-id/node_modules/@libp2p/crypto": { + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/@libp2p/crypto/-/crypto-5.0.9.tgz", + "integrity": "sha512-KR+KK1d7BfwUIC/zKN1PhS4elY/6TNWMl//34O2xA/YzSJl6vW/62oXG/XD5ieqjq7qbJZWsgbSRry8w/vDHBg==", "license": "Apache-2.0 OR MIT", "dependencies": { - "@libp2p/crypto": "^5.0.2", - "@libp2p/interface": "^2.1.0", - "multiformats": "^13.1.0", + "@libp2p/interface": "^2.4.0", + "@noble/curves": "^1.7.0", + "@noble/hashes": "^1.6.1", + "asn1js": "^3.0.5", + "multiformats": "^13.3.1", + "protons-runtime": "^5.5.0", + "uint8arraylist": "^2.4.8", "uint8arrays": "^5.1.0" } }, "node_modules/@libp2p/peer-id/node_modules/@libp2p/interface": { - "version": "2.1.0", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@libp2p/interface/-/interface-2.4.0.tgz", + "integrity": "sha512-PfzxOaz7dU4sdnUNByGLoEk9iqhD0IS+LQMQB12CXh6VyYLA7J8oaoHk3yRBZze3Y4FPa5DHMm5Oi9O/IhreaQ==", "license": "Apache-2.0 OR MIT", "dependencies": { - "@multiformats/multiaddr": "^12.2.3", + "@multiformats/multiaddr": "^12.3.3", "it-pushable": "^3.2.3", - "it-stream-types": "^2.0.1", - "multiformats": "^13.1.0", - "progress-events": "^1.0.0", + "it-stream-types": "^2.0.2", + "multiformats": "^13.3.1", + "progress-events": "^1.0.1", "uint8arraylist": "^2.4.8" } }, + "node_modules/@libp2p/peer-id/node_modules/@noble/curves": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.8.1.tgz", + "integrity": "sha512-warwspo+UYUPep0Q+vtdVB4Ugn8GGQj8iyB3gnRWsztmUHTI3S1nhdiWNsPUGL0vud7JlRRk1XEu7Lq1KGTnMQ==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "1.7.1" + }, + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@libp2p/peer-id/node_modules/@noble/hashes": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.7.1.tgz", + "integrity": "sha512-B8XBPsn4vT/KJAGqDzbwztd+6Yte3P4V7iafm24bxgDe/mlRuK6xmWPuCNrKt2vDafZ8MfJLlchDG/vYafQEjQ==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@lodestar/api": { "version": "1.21.0", "license": "Apache-2.0", @@ -1908,12 +1969,13 @@ } }, "node_modules/@multiformats/multiaddr": { - "version": "12.2.3", + "version": "12.3.4", + "resolved": "https://registry.npmjs.org/@multiformats/multiaddr/-/multiaddr-12.3.4.tgz", + "integrity": "sha512-R4pEEUyWGrRo16TSflz80Yr6XNbPirix1pfPqDLXsDZ4aaIrhZ7cez9jnyRQgci6DuuqSyZAdJKV6SdxpZ7Oiw==", "license": "Apache-2.0 OR MIT", "dependencies": { "@chainsafe/is-ip": "^2.0.1", "@chainsafe/netmask": "^2.0.0", - "@libp2p/interface": "^1.0.0", "@multiformats/dns": "^1.0.3", "multiformats": "^13.0.0", "uint8-varint": "^2.0.1", @@ -1978,6 +2040,15 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@noble/secp256k1": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/@noble/secp256k1/-/secp256k1-2.2.3.tgz", + "integrity": "sha512-l7r5oEQym9Us7EAigzg30/PQAvynhMt2uoYtT3t26eGDVm9Yii5mZ5jWSWmZ/oSIR2Et0xfc6DXrG0bZ787V3w==", + "license": "MIT", + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@node-rs/crc32": { "version": "1.10.3", "license": "MIT", @@ -3291,18 +3362,6 @@ ], "license": "MIT" }, - "node_modules/bcrypto": { - "version": "5.5.2", - "hasInstallScript": true, - "license": "MIT", - "dependencies": { - "bufio": "~1.0.7", - "loady": "~0.0.5" - }, - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/bech32": { "version": "1.1.4", "dev": true, @@ -3354,6 +3413,7 @@ }, "node_modules/bn.js": { "version": "5.2.1", + "dev": true, "license": "MIT" }, "node_modules/brace-expansion": { @@ -3424,13 +3484,6 @@ "node": ">=6.14.2" } }, - "node_modules/bufio": { - "version": "1.0.7", - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/builtin-modules": { "version": "1.1.1", "dev": true, @@ -5755,12 +5808,10 @@ } }, "node_modules/it-stream-types": { - "version": "2.0.1", - "license": "Apache-2.0 OR MIT", - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - } + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/it-stream-types/-/it-stream-types-2.0.2.tgz", + "integrity": "sha512-Rz/DEZ6Byn/r9+/SBCuJhpPATDF9D+dz5pbgSUyBsCDtza6wtNATrz/jz1gDyNanC3XdLboriHnOC925bZRBww==", + "license": "Apache-2.0 OR MIT" }, "node_modules/jackspeak": { "version": "3.1.2", @@ -5978,13 +6029,6 @@ "node": ">= 0.8.0" } }, - "node_modules/loady": { - "version": "0.0.5", - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/locate-path": { "version": "6.0.0", "dev": true, @@ -6346,7 +6390,9 @@ "license": "MIT" }, "node_modules/multiformats": { - "version": "13.1.0", + "version": "13.3.1", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.3.1.tgz", + "integrity": "sha512-QxowxTNwJ3r5RMctoGA5p13w5RbRT2QDkoM+yFlqfLiioBp78nhDjnRLvmSBI9+KAqN4VdgOVWM9c0CHd86m3g==", "license": "Apache-2.0 OR MIT" }, "node_modules/nanoid": { @@ -6977,12 +7023,10 @@ } }, "node_modules/progress-events": { - "version": "1.0.0", - "license": "Apache-2.0 OR MIT", - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" - } + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/progress-events/-/progress-events-1.0.1.tgz", + "integrity": "sha512-MOzLIwhpt64KIVN64h1MwdKWiyKFNc/S6BoYKPIVUHFg0/eIEyBulhWCgn678v/4c0ri3FdGuzXymNCv02MUIw==", + "license": "Apache-2.0 OR MIT" }, "node_modules/prom-client": { "version": "14.2.0", @@ -7041,7 +7085,9 @@ } }, "node_modules/protons-runtime": { - "version": "5.4.0", + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/protons-runtime/-/protons-runtime-5.5.0.tgz", + "integrity": "sha512-EsALjF9QsrEk6gbCx3lmfHxVN0ah7nG3cY7GySD4xf4g8cr7g543zB88Foh897Sr1RQJ9yDCUsoT1i1H/cVUFA==", "license": "Apache-2.0 OR MIT", "dependencies": { "uint8-varint": "^2.0.2", @@ -7230,16 +7276,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/rlp": { - "version": "2.2.7", - "license": "MPL-2.0", - "dependencies": { - "bn.js": "^5.2.0" - }, - "bin": { - "rlp": "bin/rlp" - } - }, "node_modules/rollup": { "version": "4.30.1", "dev": true, @@ -8812,8 +8848,8 @@ "version": "0.0.1", "license": "MIT", "dependencies": { - "@chainsafe/discv5": "^10.0.1", - "@chainsafe/enr": "^4.0.1", + "@chainsafe/discv5": "^11.0.0", + "@chainsafe/enr": "^5.0.0", "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/ssz": "^0.17.1", "@ethereumjs/block": "^5.2.0", @@ -8947,8 +8983,8 @@ "dependencies": { "@chainsafe/as-sha256": "^0.5.0", "@chainsafe/blst": "^2.0.3", - "@chainsafe/discv5": "^10.0.1", - "@chainsafe/enr": "^4.0.1", + "@chainsafe/discv5": "^11.0.0", + "@chainsafe/enr": "^5.0.0", "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/ssz": "^0.17.1", "@ethereumjs/block": "^5.3.0", diff --git a/packages/cli/package.json b/packages/cli/package.json index 357d1aa6c..ae30860c6 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -24,8 +24,8 @@ "vitest": "^3.0.2" }, "dependencies": { - "@chainsafe/discv5": "^10.0.1", - "@chainsafe/enr": "^4.0.1", + "@chainsafe/discv5": "^11.0.0", + "@chainsafe/enr": "^5.0.0", "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/ssz": "^0.17.1", "@ethereumjs/block": "^5.2.0", diff --git a/packages/portalnetwork/package.json b/packages/portalnetwork/package.json index f79b156af..e5738cb65 100644 --- a/packages/portalnetwork/package.json +++ b/packages/portalnetwork/package.json @@ -25,8 +25,8 @@ "dependencies": { "@chainsafe/as-sha256": "^0.5.0", "@chainsafe/blst": "^2.0.3", - "@chainsafe/discv5": "^10.0.1", - "@chainsafe/enr": "^4.0.1", + "@chainsafe/discv5": "^11.0.0", + "@chainsafe/enr": "^5.0.0", "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/ssz": "^0.17.1", "@ethereumjs/block": "^5.3.0", diff --git a/packages/portalnetwork/src/client/client.ts b/packages/portalnetwork/src/client/client.ts index 84aff650a..79a4814c1 100644 --- a/packages/portalnetwork/src/client/client.ts +++ b/packages/portalnetwork/src/client/client.ts @@ -34,9 +34,7 @@ import type { PortalNetworkOpts, } from './types.js' import { MessageCodes, PortalWireMessageType } from '../wire/types.js' -import { - type IClientInfo, -} from '../wire/payloadExtensions.js' +import { type IClientInfo } from '../wire/payloadExtensions.js' import { RateLimiter } from '../transports/rateLimiter.js' export class PortalNetwork extends EventEmitter { @@ -132,7 +130,12 @@ export class PortalNetwork extends EventEmitter { switch (opts.transport) { case TransportLayer.WEB: { opts.proxyAddress = opts.proxyAddress ?? 'ws://127.0.0.1:5050' - config.transport = new WebSocketTransportService(ma, config.enr.nodeId, opts.proxyAddress, new RateLimiter()) + config.transport = new WebSocketTransportService( + ma, + config.enr.nodeId, + opts.proxyAddress, + new RateLimiter(), + ) break } case TransportLayer.MOBILE: @@ -428,9 +431,11 @@ export class PortalNetwork extends EventEmitter { * @param msgId uTP message ID * @param packetBuffer uTP packet encoded to Buffer */ - private handleUTP = async (src: INodeAddress, msg: ITalkReqMessage, packetBuffer: Buffer) => { + private handleUTP = async (src: INodeAddress, msg: ITalkReqMessage, packetBuffer: Uint8Array) => { if (this.uTP.requestManagers[src.nodeId] === undefined) { - this.logger.extend('handleUTP').extend('error')(`Received uTP packet from peer with no uTP stream history: ${src.nodeId}. Blacklisting peer.`) + this.logger.extend('handleUTP').extend('error')( + `Received uTP packet from peer with no uTP stream history: ${src.nodeId}. Blacklisting peer.`, + ) this.addToBlackList(src.socketAddr) return } @@ -439,7 +444,8 @@ export class PortalNetwork extends EventEmitter { await this.uTP.handleUtpPacket(packetBuffer, src) } catch (err: any) { this.logger.extend('error')( - `handleUTP error: ${err.message}. SrcId: ${src.nodeId + `handleUTP error: ${err.message}. SrcId: ${ + src.nodeId } MultiAddr: ${src.socketAddr.toString()}`, ) } @@ -465,11 +471,7 @@ export class PortalNetwork extends EventEmitter { : (this.discv5.findEnr(enr.nodeId) ?? fromNodeAddress(enr.socketAddr.nodeAddress(), 'udp')) try { this.metrics?.totalBytesSent.inc(payload.length) - const res = await this.discv5.sendTalkReq( - remote, - Buffer.from(payload), - hexToBytes(messageNetwork), - ) + const res = await this.discv5.sendTalkReq(remote, payload, hexToBytes(messageNetwork)) this.eventLog && this.emit('SendTalkReq', enr.nodeId, bytesToHex(res), bytesToHex(payload)) return res } catch (err: any) { @@ -503,20 +505,20 @@ export class PortalNetwork extends EventEmitter { } public addToBlackList = (ma: Multiaddr) => { - (( - (this.discv5.sessionService.transport)['rateLimiter'] - )).addToBlackList(ma.nodeAddress().address) + ((this.discv5.sessionService.transport)['rateLimiter']).addToBlackList( + ma.nodeAddress().address, + ) } public isBlackListed = (ma: Multiaddr) => { - return (( - (this.discv5.sessionService.transport)['rateLimiter'] - )).isBlackListed(ma.nodeAddress().address) + return ((this.discv5.sessionService.transport)['rateLimiter']).isBlackListed( + ma.nodeAddress().address, + ) } public removeFromBlackList = (ma: Multiaddr) => { - (( - (this.discv5.sessionService.transport)['rateLimiter'] - )).removeFromBlackList(ma.nodeAddress().address) + ((this.discv5.sessionService.transport)['rateLimiter']).removeFromBlackList( + ma.nodeAddress().address, + ) } } diff --git a/packages/portalnetwork/src/transports/capacitorUdp.ts b/packages/portalnetwork/src/transports/capacitorUdp.ts index 7d886fc66..19c2c9d19 100644 --- a/packages/portalnetwork/src/transports/capacitorUdp.ts +++ b/packages/portalnetwork/src/transports/capacitorUdp.ts @@ -16,6 +16,22 @@ import type { import type { ENR } from '@chainsafe/enr' import type { Multiaddr } from '@multiformats/multiaddr' +function uint8ArrayToBase64(array: Uint8Array): string { + return globalThis.btoa( + Array.from(array) + .map(val => String.fromCharCode(val)) + .join('') + ); +} + +function base64ToUint8Array(base64: string): Uint8Array { + const binaryString = globalThis.atob(base64); + return new Uint8Array( + Array.from(binaryString) + .map(char => char.charCodeAt(0)) + ); +} + /** * This class is responsible for encoding outgoing Packets and decoding incoming Packets over UDP */ @@ -53,7 +69,7 @@ export class CapacitorUDPTransportService port, }) UDP.addListener('receive', (ret: any) => { - this.handleIncoming(new Uint8Array(Buffer.from(ret.buffer, 'base64')), { + this.handleIncoming(base64ToUint8Array(ret.buffer), { family: 'IPv4', address: ret.remoteAddress, port: ret.remotePort, @@ -72,7 +88,7 @@ export class CapacitorUDPTransportService socketId: this.socket.socketId, address: nodeAddr.host, port: nodeAddr.port, - buffer: encodePacket(toId, packet).toString('base64'), + buffer: uint8ArrayToBase64(encodePacket(toId, packet)), }) } @@ -84,7 +100,7 @@ export class CapacitorUDPTransportService `/${rinfo.family === 'IPv4' ? 'ip4' : 'ip6'}/${rinfo.address}/udp/${rinfo.port}`, ) try { - const packet = decodePacket(this.srcId, Buffer.from(data)) + const packet = decodePacket(this.srcId, (data)) this.emit('packet', multiaddr, packet) } catch (e) { this.emit('decodeError', e as any, multiaddr) diff --git a/packages/portalnetwork/src/transports/websockets.ts b/packages/portalnetwork/src/transports/websockets.ts index 6dc58d964..4d1e8d42c 100644 --- a/packages/portalnetwork/src/transports/websockets.ts +++ b/packages/portalnetwork/src/transports/websockets.ts @@ -53,7 +53,7 @@ export class WebSocketTransportService this.multiaddr = multiaddr this.srcId = srcId this.socket = new WebSocketAsPromised(proxyAddress, { - packMessage: (data: Buffer) => data.buffer, + packMessage: (data: Uint8Array) => data.buffer, unpackMessage: (data) => data, //@ts-ignore node websocket types don't match browser websocket types - so tell Typescript not to worry about it createWebSocket: (url) => new WebSocket(url), diff --git a/packages/portalnetwork/src/wire/utp/Packets/Packet.ts b/packages/portalnetwork/src/wire/utp/Packets/Packet.ts index 854424397..8c5be7a24 100644 --- a/packages/portalnetwork/src/wire/utp/Packets/Packet.ts +++ b/packages/portalnetwork/src/wire/utp/Packets/Packet.ts @@ -1,5 +1,5 @@ import { createPacketHeader } from './index.js' - +import { concatBytes } from '@ethereumjs/util' import type { PacketHeader, PacketInput, @@ -13,27 +13,28 @@ export class Packet { public readonly payload?: Uint8Array public readonly _size: number - public static fromBuffer(buffer: Buffer): Packet { - const extension = buffer.readUInt8(1) + public static fromBuffer(buffer: Uint8Array): Packet { + const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength) + const metaData = { pType: buffer[0] >> 4, - extension: buffer.readUInt8(1), - connectionId: buffer.readUInt16BE(2), - timestampMicroseconds: buffer.readUint32BE(4), - timestampDifferenceMicroseconds: buffer.readUint32BE(8), - wndSize: buffer.readUInt32BE(12), - seqNr: buffer.readUInt16BE(16), - ackNr: buffer.readUInt16BE(18), + extension: view.getUint8(1), + connectionId: view.getUint16(2, false), // false = big-endian + timestampMicroseconds: view.getUint32(4, false), + timestampDifferenceMicroseconds: view.getUint32(8, false), + wndSize: view.getUint32(12, false), + seqNr: view.getUint16(16, false), + ackNr: view.getUint16(18, false), } - let packet: Packet - if (extension === 1) { - const size = buffer.readUInt8(21) - const bitmask = buffer.subarray(22, 22 + size) - packet = Packet.fromOpts({ + + if (metaData.extension === 1) { + const size = view.getUint8(21) + const bitmask = buffer.slice(22, 22 + size) + const packet = Packet.fromOpts({ header: { version: 1, ...metaData, - bitmask: Uint8Array.from(bitmask), + bitmask, }, }) return packet @@ -43,7 +44,7 @@ export class Packet { version: 1, ...metaData, }, - payload: buffer.subarray(20), + payload: buffer.slice(20), }) return packet } @@ -60,10 +61,10 @@ export class Packet { public get size() { return this._size } - public encode(): Buffer { + public encode(): Uint8Array { const buffer = this.header.encode() if (this.payload) { - return Buffer.concat([buffer, Buffer.from(this.payload)]) + return concatBytes(buffer, (this.payload)) } return buffer } diff --git a/packages/portalnetwork/src/wire/utp/Packets/PacketHeader.ts b/packages/portalnetwork/src/wire/utp/Packets/PacketHeader.ts index 12ba81e8f..5acd0fa9a 100644 --- a/packages/portalnetwork/src/wire/utp/Packets/PacketHeader.ts +++ b/packages/portalnetwork/src/wire/utp/Packets/PacketHeader.ts @@ -1,5 +1,4 @@ import { VERSION } from '../Utils/constants.js' - import { SelectiveAckHeaderExtension } from './Extensions.js' import type { Uint16, Uint32, Uint8 } from '../index.js' @@ -35,27 +34,32 @@ abstract class Header { this.length = 20 } - abstract encode(): Buffer + abstract encode(): Uint8Array } export class BasicPacketHeader extends Header { constructor(options: HeaderInput) { super(options) } - encode(): Buffer { - const buffer = Buffer.alloc(this.length) + encode(): Uint8Array { + const array = new Uint8Array(this.length) + const view = new DataView(array.buffer) + + // Combine packet type and version into a single byte const p = Number(this.pType).toString(16) const v = this.version.toString(16) const pv = p + v const typeAndVer = parseInt(pv, 16) - buffer.writeUInt8(typeAndVer, 0) - buffer.writeUInt8(this.extension, 1) - buffer.writeUInt16BE(this.connectionId, 2) - buffer.writeUint32BE(this.timestampMicroseconds, 4) - buffer.writeUint32BE(this.timestampDifferenceMicroseconds, 8) - buffer.writeUInt32BE(this.wndSize, 12) - buffer.writeUInt16BE(this.seqNr, 16) - buffer.writeUInt16BE(this.ackNr, 18) - return buffer + + view.setUint8(0, typeAndVer) + view.setUint8(1, this.extension) + view.setUint16(2, this.connectionId, false) // false = big-endian + view.setUint32(4, this.timestampMicroseconds, false) + view.setUint32(8, this.timestampDifferenceMicroseconds, false) + view.setUint32(12, this.wndSize, false) + view.setUint16(16, this.seqNr, false) + view.setUint16(18, this.ackNr, false) + + return array } } @@ -70,27 +74,33 @@ export class SelectiveAckHeader extends Header { this.length = this.encode().length } - encode(): Buffer { - const buffer = Buffer.alloc(20 + this.bitmask.length + 2) + encode(): Uint8Array { + const array = new Uint8Array(20 + this.selectiveAckExtension.bitmask.length + 2) + const view = new DataView(array.buffer) + + // Combine packet type and version into a single byte const p = this.pType.toString(16) const v = this.version.toString(16) const pv = p + v const typeAndVer = parseInt(pv, 16) - buffer.writeUInt8(typeAndVer, 0) - buffer.writeUInt8(this.extension, 1) - buffer.writeUInt16BE(this.connectionId, 2) - buffer.writeUInt32BE(this.timestampMicroseconds, 4) - buffer.writeUInt32BE(this.timestampDifferenceMicroseconds, 8) - buffer.writeUInt32BE(this.wndSize, 12) - buffer.writeUInt16BE(this.seqNr, 16) - buffer.writeUInt16BE(this.ackNr, 18) - buffer.writeUInt8(this.selectiveAckExtension.type, 20) - buffer.writeUInt8(this.selectiveAckExtension.len, 21) - //eslint-disable-next-line + + view.setUint8(0, typeAndVer) + view.setUint8(1, this.extension) + view.setUint16(2, this.connectionId, false) // false = big-endian + view.setUint32(4, this.timestampMicroseconds, false) + view.setUint32(8, this.timestampDifferenceMicroseconds, false) + view.setUint32(12, this.wndSize, false) + view.setUint16(16, this.seqNr, false) + view.setUint16(18, this.ackNr, false) + view.setUint8(20, this.selectiveAckExtension.type) + view.setUint8(21, this.selectiveAckExtension.len) + + // Write bitmask values directly to the Uint8Array this.selectiveAckExtension.bitmask.forEach((value, idx) => { - buffer.writeUInt8(value, 22 + idx) + array[22 + idx] = value }) - return buffer + + return array } } diff --git a/packages/portalnetwork/src/wire/utp/PortalNetworkUtp/index.ts b/packages/portalnetwork/src/wire/utp/PortalNetworkUtp/index.ts index 5f2fa2ea0..b9a56ba60 100644 --- a/packages/portalnetwork/src/wire/utp/PortalNetworkUtp/index.ts +++ b/packages/portalnetwork/src/wire/utp/PortalNetworkUtp/index.ts @@ -120,7 +120,7 @@ export class PortalNetworkUTP { return newRequest } - async handleUtpPacket(packetBuffer: Buffer, srcId: INodeAddress): Promise { + async handleUtpPacket(packetBuffer: Uint8Array, srcId: INodeAddress): Promise { if (this.requestManagers[srcId.nodeId] === undefined) { throw new Error(`No request manager for ${srcId.nodeId}`) } @@ -159,7 +159,7 @@ export class PortalNetworkUTP { } } - async send(enr: ENR | INodeAddress, msg: Buffer, networkId: NetworkId) { + async send(enr: ENR | INodeAddress, msg: Uint8Array, networkId: NetworkId) { try { await this.client.sendPortalNetworkMessage(enr, msg, networkId, true) } catch (err) { diff --git a/packages/portalnetwork/src/wire/utp/PortalNetworkUtp/requestManager.ts b/packages/portalnetwork/src/wire/utp/PortalNetworkUtp/requestManager.ts index 323324033..8005dd05a 100644 --- a/packages/portalnetwork/src/wire/utp/PortalNetworkUtp/requestManager.ts +++ b/packages/portalnetwork/src/wire/utp/PortalNetworkUtp/requestManager.ts @@ -78,7 +78,7 @@ export class RequestManager { * Handles an incoming uTP packet. * @param packetBuffer buffer containing the incoming packet */ - async handlePacket(packetBuffer: Buffer) { + async handlePacket(packetBuffer: Uint8Array) { const packet = Packet.fromBuffer(packetBuffer) const request = this.lookupRequest(packet.header.connectionId) switch (request) { diff --git a/packages/portalnetwork/src/wire/utp/Socket/UtpSocket.ts b/packages/portalnetwork/src/wire/utp/Socket/UtpSocket.ts index 709f380f8..d63994483 100644 --- a/packages/portalnetwork/src/wire/utp/Socket/UtpSocket.ts +++ b/packages/portalnetwork/src/wire/utp/Socket/UtpSocket.ts @@ -81,7 +81,7 @@ export abstract class UtpSocket { clearTimeout(this.packetManager.congestionControl.timeoutCounter) } - async sendPacket(packet: Packet): Promise { + async sendPacket(packet: Packet): Promise { const msg = packet.encode() this.logger.extend('SEND').extend(PacketType[packet.header.pType])( `pid: ${packet.header.connectionId} sNr: ${packet.header.seqNr} aNr: ${packet.header.ackNr}`, diff --git a/packages/portalnetwork/src/wire/utp/Utils/variantPrefix.ts b/packages/portalnetwork/src/wire/utp/Utils/variantPrefix.ts index da62fdd0d..227adfd1a 100644 --- a/packages/portalnetwork/src/wire/utp/Utils/variantPrefix.ts +++ b/packages/portalnetwork/src/wire/utp/Utils/variantPrefix.ts @@ -1,3 +1,4 @@ +import { concatBytes } from '@ethereumjs/util' import * as leb from '@thi.ng/leb128' /** @@ -7,7 +8,7 @@ import * as leb from '@thi.ng/leb128' */ export function attatchPrefix(content: Uint8Array): Uint8Array { const prefix = leb.encodeULEB128(content.length) - return Uint8Array.from(Buffer.concat([prefix, content])) + return concatBytes(prefix, content) } /** @@ -19,7 +20,7 @@ export function encodeWithVariantPrefix(contents: Uint8Array[]): Uint8Array { const packed: Uint8Array[] = contents.map((content) => { return attatchPrefix(content) }) - return Uint8Array.from(Buffer.concat(packed)) + return concatBytes(...packed) } type length = number diff --git a/packages/portalnetwork/test/client/provider.spec.ts b/packages/portalnetwork/test/client/provider.spec.ts index d3c1adc5a..8ae02cd0d 100644 --- a/packages/portalnetwork/test/client/provider.spec.ts +++ b/packages/portalnetwork/test/client/provider.spec.ts @@ -7,6 +7,8 @@ import { assert, describe, expect, it } from 'vitest' import { UltralightProvider } from '../../src/client/provider.js' import { TransportLayer } from '../../src/index.js' import { NetworkId } from '../../src/networks/types.js' +import { hexToBytes } from 'ethereum-cryptography/utils' +import { bytesToHex, bytesToUnprefixedHex } from '@ethereumjs/util' describe('Test provider functionality', () => { it('should test provider API', async () => { @@ -24,21 +26,25 @@ describe('Test provider functionality', () => { enr, privateKey, }, - supportedNetworks: [{ networkId: NetworkId.HistoryNetwork }, { networkId: NetworkId.StateNetwork }], bootnodes: [], + supportedNetworks: [ + { networkId: NetworkId.HistoryNetwork }, + { networkId: NetworkId.StateNetwork }, + ], + bootnodes: [], }) - // Stub getBlockByHash for unit testing provider.portal.ETH.getBlockByHash = async (_hash: Uint8Array) => { return Block.fromBlockData({ header: BlockHeader.fromHeaderData({ number: 2n }) }) } - provider.portal.ETH.getBlockByNumber = async (blockNumber: number | bigint | "latest" | "finalized") => { + provider.portal.ETH.getBlockByNumber = async ( + blockNumber: number | bigint | 'latest' | 'finalized', + ) => { return Block.fromBlockData({ header: BlockHeader.fromHeaderData({ number: typeof blockNumber === 'string' ? 0n : blockNumber, - - }) + }), }) } @@ -47,7 +53,7 @@ describe('Test provider functionality', () => { } provider.portal.ETH.getCode = async (_address: Uint8Array) => { - return new Uint8Array(Buffer.from('60806040', 'hex')) + return hexToBytes('0x60806040') } provider.portal.ETH.getBalance = async (_address: Uint8Array) => { @@ -58,63 +64,69 @@ describe('Test provider functionality', () => { const result = new Uint8Array(32) result[30] = 0x00 result[31] = 0x01 - return Buffer.from(result).toString('hex') + return bytesToUnprefixedHex(result) } provider.portal.ETH.call = async (_txObject: any) => { - return Buffer.from([0x00, 0x01]).toString('hex') + return bytesToUnprefixedHex(new Uint8Array([0x00, 0x01])) } - const blockByHash = await provider.request({ + const blockByHash = (await provider.request({ method: 'eth_getBlockByHash', - params: ['0x123', false] - }) as { result: { number: string } } + params: ['0x123', false], + })) as { result: { number: string } } expect(blockByHash.result.number).toBe('0x2') - const blockByNumber = await provider.request({ + const blockByNumber = (await provider.request({ method: 'eth_getBlockByNumber', - params: [100, false] - }) as { result: { number: string } } + params: [100, false], + })) as { result: { number: string } } expect(blockByNumber.result.number).toBe('0x64') - const balance = await provider.request({ + const balance = (await provider.request({ method: 'eth_getBalance', - params: ['0x3DC00AaD844393c110b61aED5849b7c82104e748', '0x0'] - }) as { result: string } + params: ['0x3DC00AaD844393c110b61aED5849b7c82104e748', '0x0'], + })) as { result: string } expect(balance.result).toBe('0xde0b6b3a7640000') - - const storage = await provider.request({ + const storage = (await provider.request({ method: 'eth_getStorageAt', params: [ '0x1234567890123456789012345678901234567890', '0x0000000000000000000000000000000000000000000000000000000000000000', - '0x64' - ] - }) as { result: string } + '0x64', + ], + })) as { result: string } expect(storage.result).toBe('0x' + '00'.repeat(30) + '0001') - const call = await provider.request({ + const call = (await provider.request({ method: 'eth_call', - params: [{ - to: '0x1234567890123456789012345678901234567890', - data: '0x70a08231000000000000000000000000' - }, '0x64'] - }) as { result: string } + params: [ + { + to: '0x1234567890123456789012345678901234567890', + data: '0x70a08231000000000000000000000000', + }, + '0x64', + ], + })) as { result: string } expect(call.result).toBe('0x0001') - await expect(provider.request({ - method: 'eth_unsupportedMethod', - params: [] - })).rejects.toThrow() - - await expect(provider.request({ - method: 'eth_getBlockByHash', - params: ['0x123'] - })).resolves.toEqual({ + await expect( + provider.request({ + method: 'eth_unsupportedMethod', + params: [], + }), + ).rejects.toThrow() + + await expect( + provider.request({ + method: 'eth_getBlockByHash', + params: ['0x123'], + }), + ).resolves.toEqual({ error: { code: -32602, - message: 'Invalid params for eth_getBlockByHash' + message: 'Invalid params for eth_getBlockByHash', }, id: null, jsonrpc: '2.0', diff --git a/packages/portalnetwork/test/integration/postCapellaHeaderProof.spec.ts b/packages/portalnetwork/test/integration/postCapellaHeaderProof.spec.ts index 622963af6..48345cbd5 100644 --- a/packages/portalnetwork/test/integration/postCapellaHeaderProof.spec.ts +++ b/packages/portalnetwork/test/integration/postCapellaHeaderProof.spec.ts @@ -32,7 +32,7 @@ describe('Block Bridge Data Test', () => { ] const pk1 = keys.privateKeyFromProtobuf(hexToBytes(privateKeys[0]).slice(-36)) const enr1 = SignableENR.createFromPrivateKey(pk1) - const initMa: any = multiaddr(`/ip4/127.0.0.1/udp/5034`) + const initMa: any = multiaddr(`/ip4/127.0.0.1/udp/5033`) const client = await PortalNetwork.create({ supportedNetworks: [{ networkId: NetworkId.HistoryNetwork }, { networkId: NetworkId.BeaconChainNetwork }], config: { enr: enr1, bindAddrs: { ip4: initMa }, privateKey: pk1 } diff --git a/packages/portalnetwork/test/wire/utp/packet.spec.ts b/packages/portalnetwork/test/wire/utp/packet.spec.ts index 5336161ef..55cfc25cd 100644 --- a/packages/portalnetwork/test/wire/utp/packet.spec.ts +++ b/packages/portalnetwork/test/wire/utp/packet.spec.ts @@ -164,7 +164,7 @@ export function encodingTest( const encodedPacket = testPacket.encode() assert.equal(bytesToHex(encodedPacket), expectedResult, 'Packet encoding test passed') const testHeader = testPacket.header - const decodedPacket = Packet.fromBuffer(Buffer.from(expectedResult.slice(2), 'hex')) + const decodedPacket = Packet.fromBuffer(hexToBytes(expectedResult)) const decodedHeader = decodedPacket.header assert.equal( Object.entries(decodedHeader).toString(), @@ -190,8 +190,8 @@ export function encodingTest( // } if (packetType === PacketType.ST_DATA) { assert.equal( - bytesToHex(Buffer.from(testData.payload!)), - bytesToHex(Buffer.from(Uint8Array.from(decodedPacket.payload!))), + bytesToHex(testData.payload!), + bytesToHex(Uint8Array.from(decodedPacket.payload!)), `Successfully encoded and decoded DATA Packet payload.`, ) } diff --git a/packages/portalnetwork/test/wire/utp/utp.spec.ts b/packages/portalnetwork/test/wire/utp/utp.spec.ts index 74573270b..9f27ef9af 100644 --- a/packages/portalnetwork/test/wire/utp/utp.spec.ts +++ b/packages/portalnetwork/test/wire/utp/utp.spec.ts @@ -22,6 +22,7 @@ import { WriteSocket } from '../../../src/wire/utp/Socket/WriteSocket.js' import { ENR } from '@chainsafe/enr' import { RequestManager } from '../../../src/wire/utp/PortalNetworkUtp/requestManager.js' +import { utf8ToBytes } from '@ethereumjs/util' const sampleSize = 50000 const enr = ENR.decodeTxt( @@ -142,14 +143,14 @@ describe('PortalNetworkUTP test', async () => { connectionId, socketIds[RequestCode.FOUNDCONTENT_WRITE].sndId, socketIds[RequestCode.FOUNDCONTENT_WRITE].rcvId, - Buffer.from('test'), + utf8ToBytes('test'), ) assert.ok(socket, 'UTPSocket created by PortalNetworkUTP') assert.equal(socket.sndConnectionId, connectionId + 1, 'UTPSocket has correct sndConnectionId') assert.equal(socket.rcvConnectionId, connectionId, 'UTPSocket has correct rcvConnectionId') assert.equal(socket.remoteAddress, enr, 'UTPSocket has correct peerId') assert.equal(socket.type, UtpSocketType.WRITE, 'UTPSocket has correct requestCode') - assert.deepEqual(socket.content, Buffer.from('test'), 'UTPSocket has correct content') + assert.deepEqual(socket.content, utf8ToBytes('test'), 'UTPSocket has correct content') assert.equal( socket.ackNr, startingNrs[RequestCode.FOUNDCONTENT_WRITE].ackNr, @@ -185,7 +186,7 @@ describe('PortalNetworkUTP test', async () => { connectionId, socketIds[RequestCode.OFFER_WRITE].sndId, socketIds[RequestCode.OFFER_WRITE].rcvId, - Buffer.from('test'), + utf8ToBytes('test'), ) assert.equal(socket.type, UtpSocketType.WRITE, 'UTPSocket has correct requestCode') assert.equal(socket.sndConnectionId, connectionId, 'UTPSocket has correct sndConnectionId') @@ -238,7 +239,7 @@ describe('RequestManager', () => { requestManager: mgr, requestCode: RequestCode.FINDCONTENT_READ, contentKeys: [], - content: Buffer.from('test'), + content: utf8ToBytes('test'), }) const packet1 = Packet.fromOpts({ header: {