mardizzone/pos-1040: dev: add: stop/start EC2 VMs using terraform (#133)

* dev: add: pos-1040 feature to remotely start/stop aws ec2 instances using express-cli

* dev: add: pos-1040 only restart services in non-dockerized envs

* dev: add: pos-1040 README

* dev: chg: pos-1040 formatting

* dev: add: support for archive nodes on new changes

* dev: chg: prettier fix

README.md

@@ -132,8 +132,8 @@ First off, you need to `--init` terraform on your local machine, by executing th
 - `../../bin/express-cli --destroy`
 
   - Destroys the remote setup and delete the dedicated VMs
-    The `express-cli` also comes with additional utility commands, listed below. Some of them are only available for
-    non-dockerized devnets.
+
+The `express-cli` also comes with additional utility commands, listed below. Some of them are only available for non-dockerized devnets.
 
 - `../../bin/express-cli --update-all [index]`
 
@@ -182,6 +182,14 @@ First off, you need to `--init` terraform on your local machine, by executing th
     If `--send-state-sync` hasn't been used before, only checkpoints will be detected. The execution stops when
     a `state-sync` is found
 
+- ` ../../bin/express-cli --instances-stop`
+
+  - Stop the AWS EC2 VM instances associated with the deployed devnet.
+
+- ` ../../bin/express-cli --instances-start`
+
+  - Start the (previously stopped) AWS EC2 VM instances associated with the deployed devnet. Also, it starts all services, such as ganache, heimdall, and bor
+
 - `../../bin/express-cli --stress [fund]`
 
   - Runs the stress tests on remote nodes. The string `fund` is needed when stress tests are ran for the first time,

main.tf

@@ -1,5 +1,3 @@
-# main.tf
-
 terraform {
   required_providers {
     aws = {
@@ -35,6 +33,18 @@ resource "aws_instance" "app_server" {
   }
 }
 
+resource "aws_eip" "eip" {
+  vpc = true
+  count = (var.DOCKERIZED == "yes") ? 1 : (var.VALIDATOR_COUNT + var.SENTRY_COUNT + var.ARCHIVE_COUNT)
+  instance                  = aws_instance.app_server[count.index].id
+  depends_on                = [aws_internet_gateway.gw]
+}
+
+resource "aws_eip_association" "eip_assoc" {
+  count = (var.DOCKERIZED == "yes") ? 1 : (var.VALIDATOR_COUNT + var.SENTRY_COUNT + var.ARCHIVE_COUNT)
+  instance_id   = aws_instance.app_server[count.index].id
+  allocation_id = aws_eip.eip[count.index].id
+}
 
 resource "aws_security_group" "internet_facing_alb" {
   name        = "internetfacing-loadbalancer-sg"
@@ -114,5 +124,9 @@ variable "Public_Subnet_1" {
 }
 
 output "instance_ips" {
-  value = aws_instance.app_server.*.public_ip
+  value = aws_eip.eip.*.public_ip
+}
+
+output "instance_ids" {
+  value = aws_instance.app_server.*.id
 }

src/express-cli.js

@@ -17,6 +17,8 @@ import { checkDir } from './express/common/files-utils'
 import { program } from 'commander'
 import pkg from '../package.json'
 import { testEip1559 } from '../tests/test-eip-1559'
+import { stopInstances } from './express/commands/instances-stop'
+import { startInstances } from './express/commands/instances-start'
 
 const timer = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
 
@@ -61,6 +63,8 @@ program
   )
   .option('-dd, --setup-datadog', 'Setup DataDog')
   .option('-xxx, --chaos [intensity]', 'Start Chaos')
+  .option('-istop, --instances-stop', 'Stop aws ec2 instances')
+  .option('-istart, --instances-start', 'Start aws ec2 instances')
   .version(pkg.version)
 
 export async function cli() {
@@ -280,5 +284,25 @@ export async function cli() {
 
     await timer(3000)
     await chaos(options.chaos)
+  } else if (options.instancesStop) {
+    console.log('📍Command --instances-stop')
+    if (!checkDir(false)) {
+      console.log(
+        '❌ The command is not called from the appropriate devnet directory!'
+      )
+      process.exit(1)
+    }
+    await timer(3000)
+    await stopInstances()
+  } else if (options.instancesStart) {
+    console.log('📍Command --instances-start')
+    if (!checkDir(false)) {
+      console.log(
+        '❌ The command is not called from the appropriate devnet directory!'
+      )
+      process.exit(1)
+    }
+    await timer(3000)
+    await startInstances()
   }
 }

src/express/commands/instances-start.js

@@ -0,0 +1,47 @@
+// noinspection JSUnresolvedFunction
+
+import { loadDevnetConfig } from '../common/config-utils'
+import { restartAll } from './restart'
+import { maxRetries, runSshCommand } from '../common/remote-worker'
+
+const shell = require('shelljs')
+
+const timer = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+
+async function startGanache(doc) {
+  const ip = `${doc.ethHostUser}@${doc.devnetBorHosts[0]}`
+  console.log('📍Running ganache in machine ' + ip + ' ...')
+  const command = 'sudo systemctl start ganache.service'
+  await runSshCommand(ip, command, maxRetries)
+}
+
+export async function startInstances() {
+  console.log('📍Starting instances...')
+  require('dotenv').config({ path: `${process.cwd()}/.env` })
+  const devnetType =
+    process.env.TF_VAR_DOCKERIZED === 'yes' ? 'docker' : 'remote'
+  const doc = await loadDevnetConfig(devnetType)
+  const instances = doc.instancesIds.toString().replace(/,/g, ' ')
+
+  shell.exec(`aws ec2 start-instances --instance-ids ${instances}`)
+  if (shell.error() !== null) {
+    console.log(
+      `📍Starting instances ${doc.instancesIds.toString()} didn't work. Please check AWS manually`
+    )
+  } else {
+    console.log(`📍Instances ${doc.instancesIds.toString()} are starting...`)
+  }
+
+  if (devnetType === 'remote') {
+    console.log('📍Waiting 30s before restarting all services...')
+    await timer(30000)
+    await startGanache(doc)
+    await restartAll(true)
+  } else {
+    console.log('📍Waiting 20s to ensure instances are started...')
+    await timer(20000)
+    console.log(
+      '📍You can now ssh into the machine and restart the dockerized services...'
+    )
+  }
+}

src/express/commands/instances-stop.js

@@ -0,0 +1,27 @@
+// noinspection JSUnresolvedFunction
+
+import { loadDevnetConfig } from '../common/config-utils'
+
+const shell = require('shelljs')
+
+const timer = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+
+export async function stopInstances() {
+  console.log('📍Stopping instances...')
+  require('dotenv').config({ path: `${process.cwd()}/.env` })
+  const devnetType =
+    process.env.TF_VAR_DOCKERIZED === 'yes' ? 'docker' : 'remote'
+  const doc = await loadDevnetConfig(devnetType)
+  const instances = doc.instancesIds.toString().replace(/,/g, ' ')
+
+  shell.exec(`aws ec2 stop-instances --instance-ids ${instances}`)
+  if (shell.error() !== null) {
+    console.log(
+      `📍Stopping instances ${doc.instancesIds.toString()} didn't work. Please check AWS manually`
+    )
+  } else {
+    console.log('📍Waiting 20s to ensure instances are stopped...')
+    await timer(20000)
+    console.log(`📍Instances ${doc.instancesIds.toString()} are stopping...`)
+  }
+}

src/express/commands/start.js

@@ -396,7 +396,9 @@ export async function start() {
   await terraformApply(devnetId)
   const tfOutput = await terraformOutput()
   const ips = JSON.parse(tfOutput).instance_ips.value.toString()
+  const ids = JSON.parse(tfOutput).instance_ids.value.toString()
   process.env.DEVNET_BOR_HOSTS = ips
+  process.env.INSTANCES_IDS = ids
 
   await validateConfigs()
 

src/express/common/config-utils.js

@@ -106,6 +106,9 @@ function validateEnvVars() {
     }),
     MATIC_CLI_BRANCH: validStr({ default: 'master' }),
     DEVNET_BOR_USERS: validStr({ default: 'ubuntu,ubuntu' }),
+    INSTANCES_IDS: validStr({
+      default: 'i-02a1f3a2884c9edbc,i-03b2d4b3014a4becd'
+    }),
     BOR_DOCKER_BUILD_CONTEXT: url({
       default: 'https://github.com/maticnetwork/bor.git#develop'
     }),
@@ -299,6 +302,7 @@ function setCommonConfigs(doc) {
     process.env.HEIMDALL_DOCKER_BUILD_CONTEXT,
     doc
   )
+  setConfigList('instancesIds', process.env.INSTANCES_IDS, doc)
 }
 
 function setConfigValue(key, value, doc) {