Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update libp2p to ESM version #4114

Merged
merged 51 commits into from
Sep 27, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
377a0e8
WIP
wemeetagain Jun 3, 2022
984584a
Merge branch 'unstable' into cayman/update-libp2p
wemeetagain Jun 29, 2022
1253a9e
Revert eslint changes
wemeetagain Jun 29, 2022
764ecad
Revert dependency update
wemeetagain Jun 29, 2022
e0018e1
Revert dependencies
wemeetagain Jun 29, 2022
449d134
Revert yarn.lock
wemeetagain Jun 29, 2022
03a8554
Update yarn.lock
wemeetagain Jun 29, 2022
e56001d
Update noise
wemeetagain Jun 29, 2022
2b1bd00
Fix lint errors
wemeetagain Jun 29, 2022
b4ee2b3
Update discv5
wemeetagain Jun 29, 2022
ec3afb1
Fix linter errors
wemeetagain Jun 29, 2022
611309c
Fix check-types
wemeetagain Jun 29, 2022
6c6b9cb
Fix up libp2p connection manager handling
wemeetagain Jun 29, 2022
bd898be
Update gossipsub
wemeetagain Jun 30, 2022
65f703f
Add import statement
wemeetagain Jun 30, 2022
dd404cf
Merge branch 'unstable' into cayman/update-libp2p
wemeetagain Jul 6, 2022
d7b0620
Merge branch 'unstable' into cayman/update-libp2p
wemeetagain Jul 6, 2022
56fcb0d
Use upstream GossipsubEvents type
wemeetagain Jul 6, 2022
b28aca6
Remove stray lint disable comment
wemeetagain Jul 6, 2022
1f2c75b
Only ping a single peer when it connects
wemeetagain Jul 14, 2022
7189aa4
Update gossipsub
wemeetagain Jul 14, 2022
824ce9c
Merge branch 'unstable' into cayman/update-libp2p
wemeetagain Jul 14, 2022
fbf39e1
libp2p-gossipsub 3.5.0
twoeths Aug 3, 2022
f84b569
Use libp2p-noise 7.0.3
twoeths Aug 3, 2022
f513f09
Use @libp2p/mplex 4.0.3
twoeths Aug 3, 2022
b534502
Fix write heap dump in libp2p esm branch (#4387)
twoeths Aug 8, 2022
3e27e94
Update libp2p dependencies
wemeetagain Aug 11, 2022
8ac985d
Bump libp2p
wemeetagain Aug 15, 2022
252a148
Fix heap memory issue
twoeths Aug 16, 2022
982711f
Fix some tests
wemeetagain Aug 16, 2022
497a4f9
Merge branch 'unstable' into cayman/update-libp2p
wemeetagain Aug 16, 2022
a0eef4c
Fix type error
wemeetagain Aug 16, 2022
19bd339
Fix req/resp 'index out of bounds' issue
twoeths Aug 18, 2022
b95b702
Fix linter errors
wemeetagain Aug 18, 2022
67da674
Merge branch 'unstable' into cayman/update-libp2p
wemeetagain Aug 23, 2022
e219ad5
Add libp2p metrics and dashboard
wemeetagain Aug 23, 2022
71538ba
Update gossipsub and set max outbound buffer size
wemeetagain Aug 24, 2022
679e998
Update @libp2p/tcp
wemeetagain Aug 31, 2022
07fe735
Update uint8arrays
wemeetagain Sep 5, 2022
dec8c4c
v0.0.0
wemeetagain Sep 8, 2022
f8e4320
Merge branch 'unstable' into cayman/update-libp2p
wemeetagain Sep 8, 2022
d5d93e2
Lint libp2p grafana dashboard
wemeetagain Sep 8, 2022
8aceeb0
Add leveldown dev dependency
wemeetagain Sep 8, 2022
fcce170
Set msgIdToStrFn to Buffer.toString
wemeetagain Sep 9, 2022
7308ba9
Do not set udp to enr (#4591)
twoeths Sep 23, 2022
093e759
Fix e2e reqresp test
twoeths Sep 26, 2022
0a85eb6
Fix lint
twoeths Sep 26, 2022
7531aae
Fix unit tests
twoeths Sep 26, 2022
a346ac3
Merge unstable
twoeths Sep 26, 2022
03d2429
Update yarn.lock after merge
twoeths Sep 26, 2022
afd17d9
Update libp2p
wemeetagain Sep 27, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
657 changes: 657 additions & 0 deletions dashboards/libp2p.json

Large diffs are not rendered by default.

39 changes: 22 additions & 17 deletions packages/beacon-node/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -99,13 +99,24 @@
"dependencies": {
"@chainsafe/as-sha256": "^0.3.1",
"@chainsafe/bls": "7.1.1",
"@chainsafe/discv5": "^0.7.1",
"@chainsafe/libp2p-noise": "5.0.3",
"@chainsafe/discv5": "^1.1.2",
"@chainsafe/libp2p-gossipsub": "^4.1.1",
"@chainsafe/libp2p-noise": "^8.0.0",
"@chainsafe/persistent-merkle-tree": "^0.4.2",
"@chainsafe/snappy-stream": "5.1.1",
"@chainsafe/ssz": "^0.9.2",
"@chainsafe/threads": "^1.10.0",
"@ethersproject/abi": "^5.0.0",
"@libp2p/bootstrap": "^2.0.0",
"@libp2p/interface-connection": "^3.0.2",
"@libp2p/interface-connection-manager": "^1.1.1",
"@libp2p/interface-peer-id": "^1.0.4",
"@libp2p/interface-pubsub": "^2.1.0",
"@libp2p/mdns": "^3.0.1",
"@libp2p/mplex": "^5.2.4",
"@libp2p/peer-id-factory": "^1.0.18",
"@libp2p/tcp": "^3.1.2",
"@multiformats/multiaddr": "^11.0.0",
"@lodestar/api": "^1.1.0",
"@lodestar/config": "^1.1.0",
"@lodestar/db": "^1.1.0",
Expand All @@ -117,33 +128,26 @@
"@lodestar/utils": "^1.1.0",
"@lodestar/validator": "^1.1.0",
"@types/datastore-level": "^3.0.0",
"bl": "^5.0.0",
"buffer-xor": "^2.0.2",
"cross-fetch": "^3.1.4",
"datastore-core": "^7.0.1",
"datastore-level": "^6.0.2",
"datastore-core": "^8.0.1",
"datastore-level": "^9.0.1",
"deepmerge": "^3.2.0",
"fastify": "3.15.1",
"fastify-bearer-auth": "6.1.0",
"fastify-cors": "^6.0.1",
"gc-stats": "^1.4.0",
"interface-datastore": "^5.1.2",
"it-all": "^1.0.4",
"it-pipe": "^1.1.0",
"interface-datastore": "^7.0.0",
"it-all": "^1.0.6",
"it-pipe": "^2.0.4",
"jwt-simple": "0.5.6",
"libp2p": "^0.36.2",
"libp2p-bootstrap": "^0.14.0",
"libp2p-gossipsub": "^0.14.1",
"libp2p-mdns": "^0.18.0",
"libp2p-mplex": "^0.10.5",
"libp2p-tcp": "^0.17.2",
"multiaddr": "^10.0.1",
"peer-id": "^0.16.0",
"libp2p": "0.39.2",
"prom-client": "^13.2.0",
"prometheus-gc-stats": "^0.6.3",
"snappyjs": "^0.7.0",
"stream-to-it": "^0.2.0",
"strict-event-emitter-types": "^2.0.0",
"uint8arraylist": "^2.3.2",
"varint": "^6.0.0"
},
"devDependencies": {
Expand All @@ -153,10 +157,11 @@
"@types/jwt-simple": "0.5.33",
"@types/leveldown": "^4.0.2",
"@types/prometheus-gc-stats": "^0.6.1",
"@types/supertest": "^2.0.8",
"@types/supertest": "^2.0.12",
"@types/tmp": "^0.2.0",
"@types/varint": "^6.0.0",
"eventsource": "^2.0.2",
"leveldown": "^6.1.1",
Copy link
Contributor

@dapplion dapplion Sep 29, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh, why is leveldown required? For libp2p datastore? If so, @wemeetagain can you motivate upstream dep to move to level >= 8.0.0?

"rewiremock": "^3.14.3",
"rimraf": "^3.0.2",
"tmp": "^0.2.1"
Expand Down
2 changes: 1 addition & 1 deletion packages/beacon-node/src/api/impl/beacon/blocks/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ export function getBeaconBlockApi({

chain.processBlock(signedBlock).catch((e) => {
if (e instanceof BlockError && e.type.code === BlockErrorCode.PARENT_UNKNOWN) {
network.events.emit(NetworkEvent.unknownBlockParent, signedBlock, network.peerId.toB58String());
network.events.emit(NetworkEvent.unknownBlockParent, signedBlock, network.peerId.toString());
}
throw e;
}),
Expand Down
13 changes: 5 additions & 8 deletions packages/beacon-node/src/api/impl/lodestar/index.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import {Multiaddr} from "multiaddr";
import {peerIdFromString} from "@libp2p/peer-id";
import {multiaddr} from "@multiformats/multiaddr";
import {routes} from "@lodestar/api";
import {Bucket, Repository} from "@lodestar/db";
import {toHex} from "@lodestar/utils";
Expand All @@ -8,7 +9,6 @@ import {IChainForkConfig} from "@lodestar/config";
import {ssz} from "@lodestar/types";
import {BeaconChain} from "../../../chain/index.js";
import {QueuedStateRegenerator, RegenRequest} from "../../../chain/regen/index.js";
import {createFromB58String} from "../../../util/peerId.js";
import {GossipType} from "../../../network/index.js";
import {IBeaconDb} from "../../../db/interface.js";
import {ApiModules} from "../types.js";
Expand All @@ -25,9 +25,6 @@ export function getLodestarApi({

return {
async writeHeapdump(dirpath = ".") {
// Browser interop
if (typeof require !== "function") throw Error("NodeJS only");

if (writingHeapdump) {
throw Error("Already writing heapdump");
}
Expand Down Expand Up @@ -123,13 +120,13 @@ export function getLodestarApi({
},

async connectPeer(peerIdStr, multiaddrStrs) {
const peerId = createFromB58String(peerIdStr);
const multiaddrs = multiaddrStrs.map((multiaddrStr) => new Multiaddr(multiaddrStr));
const peerId = peerIdFromString(peerIdStr);
const multiaddrs = multiaddrStrs.map((multiaddrStr) => multiaddr(multiaddrStr));
await network.connectToPeer(peerId, multiaddrs);
},

async disconnectPeer(peerIdStr) {
const peerId = createFromB58String(peerIdStr);
const peerId = peerIdFromString(peerIdStr);
await network.disconnectPeer(peerId);
},

Expand Down
8 changes: 4 additions & 4 deletions packages/beacon-node/src/api/impl/node/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ export function getNodeApi(opts: IApiOptions, {network, sync}: Pick<ApiModules,

return {
data: {
peerId: network.peerId.toB58String(),
peerId: network.peerId.toString(),
enr: enr?.encodeTxt(keypair.privateKey) || "",
discoveryAddresses,
p2pAddresses: network.localMultiaddrs.map((m) => m.toString()),
Expand Down Expand Up @@ -60,13 +60,13 @@ export function getNodeApi(opts: IApiOptions, {network, sync}: Pick<ApiModules,
for (const connections of network.getConnectionsByPeer().values()) {
const relevantConnection = getRevelantConnection(connections);
switch (relevantConnection?.stat.status) {
case "open":
case "OPEN":
connected++;
break;
case "closing":
case "CLOSING":
disconnecting++;
break;
case "closed":
case "CLOSED":
disconnected++;
break;
default:
Expand Down
14 changes: 7 additions & 7 deletions packages/beacon-node/src/api/impl/node/utils.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import {Connection} from "libp2p";
import {Connection} from "@libp2p/interface-connection";
import {routes} from "@lodestar/api";
import {PeerStatus} from "../../../network/index.js";

Expand All @@ -9,7 +9,7 @@ export function formatNodePeer(peerIdStr: string, connections: Connection[]): ro
const conn = getRevelantConnection(connections);

return {
peerId: conn ? conn.remotePeer.toB58String() : peerIdStr,
peerId: conn ? conn.remotePeer.toString() : peerIdStr,
// TODO: figure out how to get enr of peer
enr: "",
lastSeenP2pAddress: conn ? conn.remoteAddr.toString() : "",
Expand All @@ -27,11 +27,11 @@ export function formatNodePeer(peerIdStr: string, connections: Connection[]): ro
export function getRevelantConnection(connections: Connection[]): Connection | null {
const byStatus = new Map<PeerStatus, Connection>();
for (const conn of connections) {
if (conn.stat.status === "open") return conn;
if (conn.stat.status === "OPEN") return conn;
if (!byStatus.has(conn.stat.status)) byStatus.set(conn.stat.status, conn);
}

return byStatus.get("open") || byStatus.get("closing") || byStatus.get("closed") || null;
return byStatus.get("OPEN") || byStatus.get("CLOSING") || byStatus.get("CLOSED") || null;
}

/**
Expand All @@ -40,11 +40,11 @@ export function getRevelantConnection(connections: Connection[]): Connection | n
*/
function getPeerState(status: PeerStatus): routes.node.PeerState {
switch (status) {
case "open":
case "OPEN":
return "connected";
case "closing":
case "CLOSING":
return "disconnecting";
case "closed":
case "CLOSED":
default:
return "disconnected";
}
Expand Down
84 changes: 84 additions & 0 deletions packages/beacon-node/src/metrics/metrics/libp2p.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
import {Libp2p} from "libp2p";
import {GaugeExtra} from "../utils/gauge.js";
import {RegistryMetricCreator} from "../utils/registryMetricCreator.js";

export type ILibp2pMetrics = ReturnType<typeof createLibp2pMetrics>;

// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/explicit-function-return-type
export function createLibp2pMetrics(libp2p: Libp2p, register: RegistryMetricCreator) {
const libp2pMetrics = libp2p.metrics;
if (libp2pMetrics === undefined) {
return;
}

const metrics: Record<string, GaugeExtra<string>> = {};

const ensureLibp2pMetrics = async (): Promise<void> => {
// protocol metrics
const protocols = libp2pMetrics.getProtocols();
protocols.forEach((protocol) => {
const protocolStat = libp2pMetrics.forProtocol(protocol);
if (protocolStat === undefined) {
return;
}

// create metric if undefined
if (metrics[protocol] === undefined) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

shouldn't we just add protocol in labelNames and simplify this

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think not, bc they split out the direction. eg: would be helpful to see if we're spammed with identify requests, but not responding.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, just add protocol as an additional label. Yes good idea. Lets tackle metrics cleanup in a separate PR.

const name = `libp2p_protocol_${protocol}_bytes`.replace(/(\/|-|\.)/g, "_");
metrics[protocol] = register.gauge<"direction">({
name,
help: name,
labelNames: ["direction"],
});
}

// set metric
const protocolSnapshot = protocolStat.getSnapshot();
metrics[protocol].set({direction: "received"}, Number(protocolSnapshot.dataReceived));
metrics[protocol].set({direction: "sent"}, Number(protocolSnapshot.dataSent));
});

// component metrics
for (const [systemName, systemMetrics] of libp2pMetrics.getComponentMetrics().entries()) {
for (const [componentName, componentMetrics] of systemMetrics.entries()) {
for (const [metricName, trackedMetric] of componentMetrics.entries()) {
// In practice `systemName` is always libp2p
const name = `${systemName}_${componentName}_${metricName}`.replace(/-/g, "_");

// create metric if undefined
if (metrics[name] === undefined) {
metrics[name] = register.gauge({
name,
help: trackedMetric.help ?? name,
labelNames: trackedMetric.label !== undefined ? [trackedMetric.label] : [],
});
}

// set metric
const m = await trackedMetric.calculate();
if (typeof m === "number") {
metrics[name].set(m);
} else {
const labelName = trackedMetric.label ?? name;
Object.entries(m).forEach(([label, value]) => {
metrics[name].set({[labelName]: label}, value);
});
}
}
}
}
};

metrics.global = register.gauge<"direction">({
name: "libp2p_global_stats",
help: "libp2p global stats",
labelNames: ["direction"],
collect: async () => {
const globalSnapshot = libp2pMetrics.getGlobal().getSnapshot();
metrics.global.set({direction: "received"}, Number(globalSnapshot.dataReceived));
metrics.global.set({direction: "sent"}, Number(globalSnapshot.dataSent));

await ensureLibp2pMetrics();
},
});
}
2 changes: 1 addition & 1 deletion packages/beacon-node/src/network/events.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import {EventEmitter} from "events";
import PeerId from "peer-id";
import {PeerId} from "@libp2p/interface-peer-id";
import StrictEventEmitter from "strict-event-emitter-types";
import {allForks, phase0} from "@lodestar/types";
import {RequestTypedContainer} from "./reqresp/index.js";
Expand Down
11 changes: 8 additions & 3 deletions packages/beacon-node/src/network/gossip/encoding.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import {compress, uncompress} from "snappyjs";
import {RPC} from "libp2p-gossipsub/src/message/rpc";
import {GossipsubMessage} from "libp2p-gossipsub/src/types";
import {Message} from "@libp2p/interface-pubsub";
import {digest} from "@chainsafe/as-sha256";
import {intToBytes} from "@lodestar/utils";
import {ForkName} from "@lodestar/params";
import {RPC} from "@chainsafe/libp2p-gossipsub/message";
import {MESSAGE_DOMAIN_VALID_SNAPPY} from "./constants.js";
import {GossipTopicCache} from "./topic.js";

Expand All @@ -19,10 +19,15 @@ export function fastMsgIdFn(rpcMsg: RPC.IMessage): string {
}
}

export function msgIdToStrFn(msgId: Uint8Array): string {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
return Buffer.prototype.toString.call(msgId, "base64");
}

/**
* Only valid msgId. Messages that fail to snappy_decompress() are not tracked
*/
export function msgIdFn(gossipTopicCache: GossipTopicCache, msg: GossipsubMessage): Uint8Array {
export function msgIdFn(gossipTopicCache: GossipTopicCache, msg: Message): Uint8Array {
const topic = gossipTopicCache.getTopic(msg.topic);

let vec: Uint8Array[];
Expand Down
Loading