chore: Devnet setup for p2p bootstrap node (#6660)

Fixes #6513

iac/main.tf

@@ -26,6 +26,11 @@ provider "aws" {
   region  = "eu-west-2"
 }
 
+# Allocate Elastic IPs for each subnet
+resource "aws_eip" "aztec_network_p2p_eip" {
+  vpc = true
+}
+
 # Create our load balancer.
 resource "aws_lb" "aztec-network" {
   name               = "aztec-network"
@@ -34,10 +39,16 @@ resource "aws_lb" "aztec-network" {
   security_groups = [
     data.terraform_remote_state.setup_iac.outputs.security_group_public_id, aws_security_group.security-group-p2p.id
   ]
-  subnets = [
-    data.terraform_remote_state.setup_iac.outputs.subnet_az1_id,
-    data.terraform_remote_state.setup_iac.outputs.subnet_az2_id
-  ]
+
+  subnet_mapping {
+    subnet_id     = data.terraform_remote_state.setup_iac.outputs.subnet_az1_id
+    allocation_id = aws_eip.aztec_network_p2p_eip.id
+  }
+
+  # No EIP for the second subnet, so it will use a dynamic IP.
+  subnet_mapping {
+    subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_id
+  }
 
   access_logs {
     bucket  = "aztec-logs"

iac/output.tf

@@ -9,3 +9,7 @@ output "nlb_dns" {
 output "p2p_security_group_id" {
   value = aws_security_group.security-group-p2p.id
 }
+
+output "p2p_eip" {
+  value = aws_eip.aztec_network_p2p_eip.public_ip
+}

yarn-project/aztec-faucet/terraform/servicediscovery-drain.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+[ $# -ne 1 ] && echo "Usage: $0 <service-id>" && exit 1
+
+serviceId="--service-id=$1"
+
+echo "Draining servicediscovery instances from $1 ..."
+ids="$(aws servicediscovery list-instances $serviceId --query 'Instances[].Id' --output text | tr '\t' ' ')"
+
+found=
+for id in $ids; do
+  if [ -n "$id" ]; then
+    echo "Deregistering $1 / $id ..."
+    aws servicediscovery deregister-instance $serviceId --instance-id "$id"
+    found=1
+  fi
+done
+
+# Yes, I'm being lazy here...
+[ -n "$found" ] && sleep 5 || true

yarn-project/aztec/package.json

@@ -41,6 +41,7 @@
     "@aztec/l1-artifacts": "workspace:^",
     "@aztec/noir-contracts.js": "workspace:^",
     "@aztec/p2p": "workspace:^",
+    "@aztec/p2p-bootstrap": "workspace:^",
     "@aztec/protocol-contracts": "workspace:^",
     "@aztec/prover-client": "workspace:^",
     "@aztec/pxe": "workspace:^",

yarn-project/aztec/src/cli/cli.ts

@@ -55,7 +55,7 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command {
         services = await startArchiver(options, signalHandlers);
       } else if (options.p2pBootstrap) {
         const { startP2PBootstrap } = await import('./cmds/start_p2p_bootstrap.js');
-        await startP2PBootstrap(options, signalHandlers, userLog, debugLogger);
+        await startP2PBootstrap(options, userLog, debugLogger);
       } else if (options.prover) {
         const { startProver } = await import('./cmds/start_prover.js');
         services = await startProver(options, signalHandlers, userLog);

yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts

@@ -1,21 +1,15 @@
 import { type DebugLogger } from '@aztec/aztec.js';
 import { type LogFn } from '@aztec/foundation/log';
-import { BootstrapNode, type P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p';
+import { type P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p';
+import runBootstrapNode from '@aztec/p2p-bootstrap';
 
 import { mergeEnvVarsAndCliOptions, parseModuleOptions } from '../util.js';
 
-export const startP2PBootstrap = async (
-  options: any,
-  signalHandlers: (() => Promise<void>)[],
-  userLog: LogFn,
-  debugLogger: DebugLogger,
-) => {
+export const startP2PBootstrap = async (options: any, userLog: LogFn, debugLogger: DebugLogger) => {
   // Start a P2P bootstrap node.
   const envVars = getP2PConfigEnvVars();
   const cliOptions = parseModuleOptions(options.p2pBootstrap);
-  const bootstrapNode = new BootstrapNode(debugLogger);
   const config = mergeEnvVarsAndCliOptions<P2PConfig>(envVars, cliOptions);
-  await bootstrapNode.start(config);
-  userLog(`P2P bootstrap node started on ${config.tcpListenIp}:${config.tcpListenPort}`);
-  signalHandlers.push(bootstrapNode.stop);
+  await runBootstrapNode(config, debugLogger);
+  userLog(`P2P bootstrap node started on ${config.udpListenIp}:${config.udpListenPort}`);
 };

yarn-project/aztec/tsconfig.json

@@ -48,6 +48,9 @@
     {
       "path": "../p2p"
     },
+    {
+      "path": "../p2p-bootstrap"
+    },
     {
       "path": "../protocol-contracts"
     },

yarn-project/p2p-bootstrap/package.json

@@ -27,11 +27,15 @@
     "@aztec/foundation": "workspace:^",
     "@aztec/p2p": "workspace:^",
     "dotenv": "^16.0.3",
+    "koa": "^2.15.3",
+    "koa-router": "^12.0.1",
     "tslib": "^2.4.0"
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
     "@types/jest": "^29.5.0",
+    "@types/koa": "^2.15.0",
+    "@types/koa-router": "^7.4.8",
     "@types/node": "^18.14.6",
     "jest": "^29.5.0",
     "ts-node": "^10.9.1",

yarn-project/p2p-bootstrap/src/index.ts

@@ -1,16 +1,31 @@
 import { createDebugLogger } from '@aztec/foundation/log';
-import { BootstrapNode, getP2PConfigEnvVars } from '@aztec/p2p';
+import { BootstrapNode, type P2PConfig } from '@aztec/p2p';
 
-const logger = createDebugLogger('aztec:bootstrap_node');
+import Koa from 'koa';
+import Router from 'koa-router';
+
+const debugLogger = createDebugLogger('aztec:bootstrap_node');
+
+const { HTTP_PORT } = process.env;
 
 /**
  * The application entry point.
  */
-async function main() {
-  const config = getP2PConfigEnvVars();
+async function main(config: P2PConfig, logger = debugLogger) {
   const bootstrapNode = new BootstrapNode(logger);
   await bootstrapNode.start(config);
-  logger.info('Node started');
+  logger.info('DiscV5 Bootnode started');
+
+  const httpApp = new Koa();
+  const router = new Router();
+  router.get('/health', (ctx: Koa.Context) => {
+    ctx.status = 200;
+  });
+
+  httpApp.use(router.routes()).use(router.allowedMethods());
+  httpApp.listen(HTTP_PORT, () => {
+    logger.info(`HTTP server listening on port ${HTTP_PORT}`);
+  });
 
   const stop = async () => {
     logger.debug('Stopping bootstrap node...');
@@ -22,6 +37,4 @@ async function main() {
   process.on('SIGINT', stop);
 }
 
-main().catch(err => {
-  logger.error(err);
-});
+export default main;

yarn-project/p2p-bootstrap/terraform/main.tf

@@ -49,7 +49,7 @@ data "terraform_remote_state" "aztec-network_iac" {
 }
 
 locals {
-  bootnode_keys  = [var.BOOTNODE_1_PRIVATE_KEY, var.BOOTNODE_2_PRIVATE_KEY]
+  bootnode_keys  = [var.BOOTNODE_1_PRIVATE_KEY]
   bootnode_count = length(local.bootnode_keys)
 }
 
@@ -105,13 +105,14 @@ resource "aws_ecs_task_definition" "p2p-bootstrap" {
 [
   {
     "name": "${var.DEPLOY_TAG}-p2p-bootstrap-${count.index + 1}",
-    "image": "${var.DOCKERHUB_ACCOUNT}/aztec:${var.DEPLOY_TAG}",
+    "image": "${var.DOCKERHUB_ACCOUNT}/aztec:devnet",
     "command": ["start", "--p2p-bootstrap"],
     "essential": true,
     "memoryReservation": 3776,
     "portMappings": [
       {
-        "containerPort": ${var.BOOTNODE_LISTEN_PORT + count.index}
+        "containerPort": ${var.BOOTNODE_LISTEN_PORT + count.index},
+        "protocol": "udp"
       },
       {
         "containerPort": 80
@@ -123,17 +124,25 @@ resource "aws_ecs_task_definition" "p2p-bootstrap" {
         "value": "production"
       },
       {
-        "name": "P2P_TCP_LISTEN_PORT",
+        "name": "P2P_UDP_LISTEN_PORT",
         "value": "${var.BOOTNODE_LISTEN_PORT + count.index}"
       },
       {
-        "name": "P2P_TCP_LISTEN_IP",
+        "name": "P2P_UDP_LISTEN_IP",
         "value": "0.0.0.0"
       },
       {
         "name": "PEER_ID_PRIVATE_KEY",
         "value": "${local.bootnode_keys[count.index]}"
       },
+      {
+        "name": "P2P_ANNOUNCE_HOSTNAME",
+        "value": "/ip4/${data.terraform_remote_state.aztec-network_iac.outputs.p2p_eip}"
+      },
+      {
+        "name": "P2P_ANNOUNCE_PORT",
+        "value": "${var.BOOTNODE_LISTEN_PORT + count.index}"
+      },
       {
         "name": "DEBUG",
         "value": "aztec:*"
@@ -145,6 +154,10 @@ resource "aws_ecs_task_definition" "p2p-bootstrap" {
       {
         "name": "P2P_MAX_PEERS",
         "value": "${var.P2P_MAX_PEERS}"
+      },
+      {
+        "name": "HTTP_PORT",
+        "value": "80"
       }
     ],
     "logConfiguration": {
@@ -172,8 +185,7 @@ resource "aws_ecs_service" "p2p-bootstrap" {
 
   network_configuration {
     subnets = [
-      data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id,
-      data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id
+      data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id
     ]
     security_groups = [data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id, data.terraform_remote_state.setup_iac.outputs.security_group_private_id]
   }
@@ -185,53 +197,64 @@ resource "aws_ecs_service" "p2p-bootstrap" {
   }
 
   load_balancer {
-    target_group_arn = aws_lb_target_group.p2p-bootstrap-target-group[count.index].id
+    target_group_arn = aws_lb_target_group.p2p-bootstrap-target-group-udp[count.index].id
     container_name   = "${var.DEPLOY_TAG}-p2p-bootstrap-${count.index + 1}"
     container_port   = var.BOOTNODE_LISTEN_PORT + count.index
   }
 
   task_definition = aws_ecs_task_definition.p2p-bootstrap[count.index].family
 }
 
-resource "aws_lb_target_group" "p2p-bootstrap-target-group" {
+# Create a target group for UDP traffic.
+resource "aws_lb_target_group" "p2p-bootstrap-target-group-udp" {
   count       = local.bootnode_count
-  name        = "p2p-bootstrap-${count.index + 1}-target-group"
+  name        = "p2p-bootstrap-${count.index + 1}-target-group-udp"
   port        = var.BOOTNODE_LISTEN_PORT + count.index
-  protocol    = "TCP"
+  protocol    = "UDP"
   target_type = "ip"
   vpc_id      = data.terraform_remote_state.setup_iac.outputs.vpc_id
 
   health_check {
-    protocol            = "TCP"
-    interval            = 10
+    enabled             = true
+    path                = "/health"
+    port                = "80"
+    interval            = 30
+    timeout             = 10
     healthy_threshold   = 2
     unhealthy_threshold = 2
-    port                = var.BOOTNODE_LISTEN_PORT + count.index
+    matcher             = "200"
+    protocol            = "HTTP"
   }
 }
 
-resource "aws_security_group_rule" "allow-bootstrap-tcp" {
-  count             = local.bootnode_count
-  type              = "ingress"
-  from_port         = var.BOOTNODE_LISTEN_PORT + count.index
-  to_port           = var.BOOTNODE_LISTEN_PORT + count.index
-  protocol          = "tcp"
-  cidr_blocks       = ["0.0.0.0/0"]
-  security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
-}
-
-resource "aws_lb_listener" "p2p-bootstrap-tcp-listener" {
+resource "aws_lb_listener" "p2p-bootstrap-udp-listener" {
   count             = local.bootnode_count
   load_balancer_arn = data.terraform_remote_state.aztec-network_iac.outputs.nlb_arn
   port              = var.BOOTNODE_LISTEN_PORT + count.index
-  protocol          = "TCP"
-
-  tags = {
-    name = "p2p-bootstrap-${count.index}-target-group"
-  }
+  protocol          = "UDP"
 
   default_action {
     type             = "forward"
-    target_group_arn = aws_lb_target_group.p2p-bootstrap-target-group[count.index].arn
+    target_group_arn = aws_lb_target_group.p2p-bootstrap-target-group-udp[count.index].arn
   }
 }
+
+resource "aws_security_group_rule" "allow-bootstrap-udp" {
+  count             = local.bootnode_count
+  type              = "ingress"
+  from_port         = var.BOOTNODE_LISTEN_PORT + count.index
+  to_port           = var.BOOTNODE_LISTEN_PORT + count.index
+  protocol          = "udp"
+  security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
+  cidr_blocks       = ["0.0.0.0/0"]
+}
+
+# Egress Rule for UDP Traffic
+resource "aws_security_group_rule" "allow-bootstrap-udp-egress" {
+  type              = "egress"
+  from_port         = var.BOOTNODE_LISTEN_PORT
+  to_port           = var.BOOTNODE_LISTEN_PORT
+  protocol          = "udp"
+  security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id
+  cidr_blocks       = ["0.0.0.0/0"]
+}

yarn-project/p2p/src/config.ts

@@ -118,6 +118,7 @@ export function getP2PConfigEnvVars(): P2PConfig {
     P2P_MAX_PEERS,
     DATA_DIRECTORY,
     TX_GOSSIP_VERSION,
+    P2P_TX_PROTOCOL,
   } = process.env;
   const envVars: P2PConfig = {
     p2pEnabled: P2P_ENABLED === 'true',
@@ -130,7 +131,7 @@ export function getP2PConfigEnvVars(): P2PConfig {
     udpListenIp: P2P_UDP_LISTEN_IP ? P2P_UDP_LISTEN_IP : '0.0.0.0',
     peerIdPrivateKey: PEER_ID_PRIVATE_KEY,
     bootstrapNodes: BOOTSTRAP_NODES ? BOOTSTRAP_NODES.split(',') : [],
-    transactionProtocol: '',
+    transactionProtocol: P2P_TX_PROTOCOL ? P2P_TX_PROTOCOL : '/aztec/0.1.0',
     announceHostname: P2P_ANNOUNCE_HOSTNAME,
     announcePort: P2P_ANNOUNCE_PORT ? +P2P_ANNOUNCE_PORT : undefined,
     enableNat: P2P_NAT_ENABLED === 'true',