Bump to new gossipsub (#3858)

* Bump to new gossipsub

* Fix test type errors

* Add getMeshPeers method

* Cleanup test nodes on success and error

* Add allowPublishToZeroPeers option

* Add allowPublishToZeroPeers to dev options

* Add scripts to run dev servers

* Bump gossipsub

* Bump gossipsub

* Expose gossip peer score stats

* bump gossipsub

* Dump gossipsub score params

* Bump lodestar

* Add behaviourPenaltyThreshold to gossip scoring params

* Switch to latest gossipsub update

* Fix cli unit test

* Fix unknownBlockSync e2e test

* Update latest gossipsub

* Fix e2e - network addr in use

* Use latest gossipsub 3efae56c9d39ff3a047a7e6c65fa32e5756bebde

* Use latest gossipsub 039af54c7bf47396c1be414d9fa33aa2e8cc2697

* Increase meshMessageDeliveriesWindow and gossipsubIWantFollowupMs

* Set gossipsubIWantFollowupMs and meshMessageDeliveriesWindow 12s

* Reset gossipsub metric behaviourPenalty per scrape

* Compute score using lodestar score and gossipsub score (#3875)

* Refactor PeerRpcScoreStore: add PeerScore class

* Aggregate lodestarScore, gossipsubScore to compute final score

* updateGossipsubScores util and unit test

* Populate PeerScore on updateGossipsubScore

* Fix peerManager e2e test

* Fix test/sim/multiNodeSingleThread.test.ts

* Update latest gossipsub

* lodestar_gossip_mesh_peers_by_client_count metric

* Log sent peers in gossipsub.publishObject()

* Fix lodestar_gossip_mesh_peers_by_client_count metric

* Fix lodestar_gossip_mesh_peers_by_client_count metric

* Add asyncValidation=true option to gossipsub

* Fix sha256 usage

* Go with libp2p-gossipsub 0.14.0

Co-authored-by: Tuyen Nguyen <[email protected]>
Co-authored-by: Cayman <[email protected]>

Author: 3 people

packages/api/src/routes/lodestar.ts

@@ -17,9 +17,10 @@ export type SyncChainDebugState = {
 
 export type GossipQueueItem = {
   topic: unknown;
-  receivedFrom: string;
+  propagationSource: string;
   data: Uint8Array;
   addedTimeMs: number;
+  seenTimestampSec: number;
 };
 
 export type RegenQueueItem = {
@@ -66,6 +67,8 @@ export type Api = {
   getStateCacheItems(): Promise<StateCacheItem[]>;
   /** Dump a summary of the states in the CheckpointStateCache */
   getCheckpointStateCacheItems(): Promise<StateCacheItem[]>;
+  /** Dump peer gossip stats by peer */
+  getGossipPeerScoreStats(): Promise<Record<string, unknown>>;
   /** Run GC with `global.gc()` */
   runGC(): Promise<void>;
   /** Drop all states in the state cache */
@@ -95,6 +98,7 @@ export const routesData: RoutesData<Api> = {
   getBlockProcessorQueueItems: {url: "/eth/v1/lodestar/block-processor-queue-items", method: "GET"},
   getStateCacheItems: {url: "/eth/v1/lodestar/state-cache-items", method: "GET"},
   getCheckpointStateCacheItems: {url: "/eth/v1/lodestar/checkpoint-state-cache-items", method: "GET"},
+  getGossipPeerScoreStats: {url: "/eth/v1/lodestar/gossip-peer-score-stats", method: "GET"},
   runGC: {url: "/eth/v1/lodestar/gc", method: "POST"},
   dropStateCache: {url: "/eth/v1/lodestar/drop-state-cache", method: "POST"},
   connectPeer: {url: "/eth/v1/lodestar/connect_peer", method: "POST"},
@@ -113,6 +117,7 @@ export type ReqTypes = {
   getBlockProcessorQueueItems: ReqEmpty;
   getStateCacheItems: ReqEmpty;
   getCheckpointStateCacheItems: ReqEmpty;
+  getGossipPeerScoreStats: ReqEmpty;
   runGC: ReqEmpty;
   dropStateCache: ReqEmpty;
   connectPeer: {query: {peerId: string; multiaddr: string[]}};
@@ -140,6 +145,7 @@ export function getReqSerializers(): ReqSerializers<Api, ReqTypes> {
     getBlockProcessorQueueItems: reqEmpty,
     getStateCacheItems: reqEmpty,
     getCheckpointStateCacheItems: reqEmpty,
+    getGossipPeerScoreStats: reqEmpty,
     runGC: reqEmpty,
     dropStateCache: reqEmpty,
     connectPeer: {
@@ -173,6 +179,7 @@ export function getReturnTypes(): ReturnTypes<Api> {
     getBlockProcessorQueueItems: jsonType("camel"),
     getStateCacheItems: jsonType("camel"),
     getCheckpointStateCacheItems: jsonType("camel"),
+    getGossipPeerScoreStats: jsonType("camel"),
     getPeers: jsonType("camel"),
     discv5GetKadValues: jsonType("camel"),
   };

packages/cli/src/cmds/dev/options.ts

@@ -35,8 +35,7 @@ const devOwnOptions: ICliCommandOptions<IDevOwnArgs> = {
   },
 
   startValidators: {
-    description: "Start interop validators in given range",
-    default: "0:7",
+    description: "Start interop validators in inclusive range with notation '0:7'",
     type: "string",
     group: "dev",
   },
@@ -74,6 +73,11 @@ const externalOptionsOverrides: {[k: string]: Options} = {
     defaultDescription: undefined,
     default: true,
   },
+  "network.allowPublishToZeroPeers": {
+    ...beaconNodeOptions["network.allowPublishToZeroPeers"],
+    defaultDescription: undefined,
+    default: true,
+  },
   "network.maxPeers": {
     ...beaconNodeOptions["network.maxPeers"],
     defaultDescription: undefined,

packages/cli/src/options/beaconNodeOptions/network.ts

@@ -17,6 +17,7 @@ export interface INetworkArgs {
   "network.blockCountPeerLimit": number;
   "network.rateTrackerTimeoutMs": number;
   "network.dontSendGossipAttestationsToForkchoice": boolean;
+  "network.allowPublishToZeroPeers": boolean;
 }
 
 export function parseArgs(args: INetworkArgs): IBeaconNodeOptions["network"] {
@@ -40,6 +41,7 @@ export function parseArgs(args: INetworkArgs): IBeaconNodeOptions["network"] {
     blockCountPeerLimit: args["network.blockCountPeerLimit"],
     rateTrackerTimeoutMs: args["network.rateTrackerTimeoutMs"],
     dontSendGossipAttestationsToForkchoice: args["network.dontSendGossipAttestationsToForkchoice"],
+    allowPublishToZeroPeers: args["network.allowPublishToZeroPeers"],
   };
 }
 
@@ -153,4 +155,11 @@ export const options: ICliCommandOptions<INetworkArgs> = {
     description: "Pass gossip attestations to forkchoice or not",
     group: "network",
   },
+
+  "network.allowPublishToZeroPeers": {
+    hidden: true,
+    type: "boolean",
+    description: "Don't error when publishing to zero peers",
+    group: "network",
+  },
 };

packages/cli/test/unit/options/beaconNodeOptions.test.ts

@@ -55,6 +55,7 @@ describe("options / beaconNodeOptions", () => {
       "network.blockCountPeerLimit": 500,
       "network.rateTrackerTimeoutMs": 60000,
       "network.dontSendGossipAttestationsToForkchoice": true,
+      "network.allowPublishToZeroPeers": true,
       "sync.isSingleNode": true,
       "sync.disableProcessAsChainSegment": true,
       "sync.backfillBatchSize": 64,
@@ -120,6 +121,7 @@ describe("options / beaconNodeOptions", () => {
         blockCountPeerLimit: 500,
         rateTrackerTimeoutMs: 60000,
         dontSendGossipAttestationsToForkchoice: true,
+        allowPublishToZeroPeers: true,
       },
       sync: {
         isSingleNode: true,

packages/lodestar/package.json

@@ -80,10 +80,10 @@
     "@chainsafe/ssz": "^0.9.0",
     "@ethersproject/abi": "^5.0.0",
     "@types/datastore-level": "^3.0.0",
-    "datastore-core": "^7.0.1",
     "bl": "^5.0.0",
     "buffer-xor": "^2.0.2",
     "cross-fetch": "^3.1.4",
+    "datastore-core": "^7.0.1",
     "datastore-level": "^6.0.2",
     "deepmerge": "^3.2.0",
     "fastify": "3.15.1",
@@ -96,8 +96,7 @@
     "jwt-simple": "0.5.6",
     "libp2p": "^0.36.2",
     "libp2p-bootstrap": "^0.14.0",
-    "libp2p-gossipsub": "^0.13.2",
-    "libp2p-interfaces": "^4.0.4",
+    "libp2p-gossipsub": "^0.14.0",
     "libp2p-mdns": "^0.18.0",
     "libp2p-mplex": "^0.10.5",
     "libp2p-tcp": "^0.17.2",

packages/lodestar/src/api/impl/lodestar/index.ts

@@ -88,12 +88,13 @@ export function getLodestarApi({
       }
 
       return jobQueue.getItems().map((item) => {
-        const [topic, message] = item.args;
+        const [topic, message, propagationSource, seenTimestampSec] = item.args;
         return {
           topic: topic,
-          receivedFrom: message.receivedFrom,
+          propagationSource,
           data: message.data,
           addedTimeMs: item.addedTimeMs,
+          seenTimestampSec,
         };
       });
     },
@@ -130,6 +131,10 @@ export function getLodestarApi({
       return (chain as BeaconChain)["checkpointStateCache"].dumpSummary();
     },
 
+    async getGossipPeerScoreStats() {
+      return network.gossip.dumpPeerScoreStats();
+    },
+
     async runGC() {
       // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
       if (!global.gc) throw Error("You must expose GC running the Node.js process with 'node --expose_gc'");

packages/lodestar/src/metrics/metrics.ts

@@ -3,7 +3,7 @@
  */
 import {BeaconStateAllForks, getCurrentSlot} from "@chainsafe/lodestar-beacon-state-transition";
 import {IChainForkConfig} from "@chainsafe/lodestar-config";
-import {collectDefaultMetrics, Counter, Registry} from "prom-client";
+import {collectDefaultMetrics, Counter, Metric, Registry} from "prom-client";
 import gcStats from "prometheus-gc-stats";
 import {DbMetricLabels, IDbMetrics} from "@chainsafe/lodestar-db";
 import {createBeaconMetrics, IBeaconMetrics} from "./metrics/beacon";
@@ -12,13 +12,13 @@ import {IMetricsOptions} from "./options";
 import {RegistryMetricCreator} from "./utils/registryMetricCreator";
 import {createValidatorMonitor, IValidatorMonitor} from "./validatorMonitor";
 
-export type IMetrics = IBeaconMetrics & ILodestarMetrics & IValidatorMonitor & {register: Registry};
+export type IMetrics = IBeaconMetrics & ILodestarMetrics & IValidatorMonitor & {register: RegistryMetricCreator};
 
 export function createMetrics(
   opts: IMetricsOptions,
   config: IChainForkConfig,
   anchorState: BeaconStateAllForks,
-  registries: Registry[] = []
+  externalRegistries: Registry[] = []
 ): IMetrics {
   const register = new RegistryMetricCreator();
   const beacon = createBeaconMetrics(register);
@@ -47,7 +47,24 @@ export function createMetrics(
   // - nodejs_gc_reclaimed_bytes_total: The number of bytes GC has freed
   gcStats(register)();
 
-  return {...beacon, ...lodestar, ...validatorMonitor, register: Registry.merge([register, ...registries])};
+  // Merge external registries
+  register;
+  for (const externalRegister of externalRegistries) {
+    // Wrong types, does not return a promise
+    const metrics = (externalRegister.getMetricsAsArray() as unknown) as Resolves<
+      typeof externalRegister.getMetricsAsArray
+    >;
+    for (const metric of metrics) {
+      register.registerMetric((metric as unknown) as Metric<string>);
+    }
+  }
+
+  return {
+    ...beacon,
+    ...lodestar,
+    ...validatorMonitor,
+    register,
+  };
 }
 
 export function createDbMetrics(): {metrics: IDbMetrics; registry: Registry} {
@@ -68,3 +85,6 @@ export function createDbMetrics(): {metrics: IDbMetrics; registry: Registry} {
   registry.registerMetric(metrics.dbWrites);
   return {metrics, registry};
 }
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type Resolves<F extends (...args: any[]) => Promise<any>> = F extends (...args: any[]) => Promise<infer T> ? T : never;

packages/lodestar/src/metrics/metrics/lodestar.ts

@@ -189,6 +189,11 @@ export function createLodestarMetrics(
         help: "Gossip peer score by threashold",
         labelNames: ["threshold"],
       }),
+      meshPeersByClient: register.gauge<"client">({
+        name: "lodestar_gossip_mesh_peers_by_client_count",
+        help: "number of mesh peers, labeled by client",
+        labelNames: ["client"],
+      }),
       score: register.avgMinMax({
         name: "lodestar_gossip_score_avg_min_max",
         help: "Avg min max of all gossip peer scores",