Merge branch 'main' into jdjaustin/issue-137-error-handling-improvements

.github/workflows/deploy-benchmark.yml

@@ -34,5 +34,7 @@ jobs:
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}
           slack_token: ${{ secrets.SLACK_TOKEN }}
+          new_relic_account_id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
+          new_relic_api_key: ${{ secrets.NEW_RELIC_API_KEY }}
           working_directory: terraform/benchmark
           environment: Benchmark Service

archival-snapshots-generate/README.md

@@ -2,7 +2,7 @@
 This tool will generate lite and diff snapshots provided a full snapshot.
 
 # Installation
-Install it with `cargo install --path .`. This tool also requires `forest-cli` in PATH. See more [here](https://github.com/ChainSafe/forest#installation).
+Install it with `cargo install --path .`. This tool also requires `forest-tool` in PATH. See more [here](https://github.com/ChainSafe/forest#installation).
 
 ```
 Usage: archival-snapshots-generate [OPTIONS] <SNAPSHOT_FILE>

archival-snapshots-generate/src/main.rs

@@ -35,9 +35,6 @@ struct Args {
 }
 
 fn main() -> anyhow::Result<()> {
-    if which::which("forest-cli").is_err() {
-        bail!("forest-cli is not installed");
-    }
     if which::which("forest-tool").is_err() {
         bail!("forest-tool is not installed");
     }
@@ -129,7 +126,7 @@ fn generate_lite_snapshot(
     snapshot_file: &Path,
 ) -> anyhow::Result<()> {
     debug!("Generating lite snapshot for epoch {epoch}");
-    std::process::Command::new("forest-cli")
+    std::process::Command::new("forest-tool")
         .args([
             "archive",
             "export",
@@ -168,7 +165,7 @@ fn generate_diff_snapshot(
         return Ok(());
     }
 
-    std::process::Command::new("forest-cli")
+    std::process::Command::new("forest-tool")
         .args([
             "archive",
             "export",

cf/latest-snapshot/.editorconfig

@@ -0,0 +1,13 @@
+# http://editorconfig.org
+root = true
+
+[*]
+indent_style = tab
+tab_width = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.yml]
+indent_style = space

cf/latest-snapshot/.prettierrc

@@ -0,0 +1,19 @@
+{
+	"semi": true,
+	"printWidth": 100,
+	"singleQuote": true,
+	"bracketSpacing": true,
+	"insertPragma": false,
+	"requirePragma": false,
+	"jsxSingleQuote": false,
+	"bracketSameLine": false,
+	"embeddedLanguageFormatting": "auto",
+	"htmlWhitespaceSensitivity": "css",
+	"vueIndentScriptAndStyle": true,
+	"quoteProps": "consistent",
+	"proseWrap": "preserve",
+	"trailingComma": "es5",
+	"arrowParens": "avoid",
+	"useTabs": true,
+	"tabWidth": 2
+}

cf/latest-snapshot/README.md

@@ -0,0 +1,12 @@
+# Snapshot redirect worker
+
+This worker acts on two endpoints:
+
+- `https://forest-archive.chainsafe.dev/latest/calibnet/`
+- `https://forest-archive.chainsafe.dev/latest/mainnet/`
+
+These links will download the latest available snapshot for calibnet and mainnet, respectively.
+
+# Local deployment
+
+Use `wrangler dev` to deploy a local version of this worker which will use the `forest-archive-dev` bucket rather than the production `forest-archive` bucket. Merging changes to this worker will automatically deploy them.

cf/latest-snapshot/package.json

@@ -0,0 +1,14 @@
+{
+	"name": "forest-latest-snapshot",
+	"version": "0.0.0",
+	"private": true,
+	"scripts": {
+		"deploy": "wrangler deploy src/index.ts",
+		"dev": "wrangler dev src/index.ts --local",
+		"start-stackblitz": "WRANGLER_SEND_METRICS=false wrangler dev src/index.ts --local"
+	},
+	"devDependencies": {
+		"@cloudflare/workers-types": "^4.20230904.0",
+		"wrangler": "^3.0.0"
+	}
+}

cf/latest-snapshot/src/index.ts

@@ -0,0 +1,45 @@
+interface Env {
+	FOREST_ARCHIVE: R2Bucket,
+}
+
+function basename(path: String) {
+	return path.split('/').reverse()[0];
+}
+
+// Directly fetch the data for the latest snapshot of a given chain (eg. calibnet or mainnet)
+async function get_latest(env: Env, chain: string): Promise<Response> {
+	const listed = await env.FOREST_ARCHIVE.list({ prefix: chain + "/latest/" });
+	let latest = listed.objects.at(-1);
+	if (latest == null) {
+		return new Response(`No latest snapshot found ${chain}`, {
+			status: 404,
+		});
+	} else {
+		// Should we support range queries?
+		const object = await env.FOREST_ARCHIVE.get(latest.key);
+		if (object === null) {
+			return new Response('No latest snapshot found', {
+				status: 404,
+			});
+		}
+		const headers = new Headers();
+		object.writeHttpMetadata(headers);
+		headers.set('etag', object.httpEtag);
+		let encoded_name = encodeURIComponent(basename(object.key));
+		// Tell browsers and aria2c which filename to use. For 'wget', you have to use `--trust-server-names`.
+		headers.set('content-disposition', `attachment; filename*=UTF-8''${encoded_name}; filename="${encoded_name}"`);
+
+		return new Response(object.body, {
+			headers,
+		});
+	}
+}
+
+export default {
+	async fetch(request: Request, env: Env): Promise<Response> {
+		const url = new URL(request.url);
+		const chain = (url.pathname.match(/\/latest\/(\w*)/) || ["undefined"])[1];
+
+		return await get_latest(env, chain);
+	},
+};

cf/latest-snapshot/tsconfig.json

@@ -0,0 +1,12 @@
+{
+	"compilerOptions": {
+		"noEmit": true,
+		"module": "esnext",
+		"target": "esnext",
+		"lib": ["esnext"],
+		"strict": true,
+		"moduleResolution": "node",
+		"types": ["@cloudflare/workers-types", "@types/jest"]
+	},
+	"exclude": ["node_modules", "dist"]
+}

cf/latest-snapshot/wrangler.toml

@@ -0,0 +1,13 @@
+name = "forest-latest-snapshot"
+main = "./src/index.ts"
+
+compatibility_date = "2022-06-30"
+
+routes = [
+    { pattern = "forest-archive.chainsafe.dev/latest/calibnet/", zone_name = "chainsafe.dev" },
+    { pattern = "forest-archive.chainsafe.dev/latest/mainnet/", zone_name = "chainsafe.dev" },
+]
+
+[[r2_buckets]]
+binding = 'FOREST_ARCHIVE'     # can be any valid JavaScript variable name
+bucket_name = 'forest-archive'

cf/prune-latest/README.md

@@ -0,0 +1,7 @@
+# Deployment
+
+This worker is automatically deployed when modified. To test locally, run `wrangler dev`. This will run the worker against the development bucket `forest-archive-dev`. Once the worker is deployed to production, it'll use the `forest-archive` bucket.
+
+# Pruning
+
+We upload new Filecoin snapshots to CloudFlare every hour and keep only the 10 most recent. The CloudFlare worker script is triggered automatically every hour.

cf/prune-latest/package.json

@@ -0,0 +1,14 @@
+{
+	"name": "template-worker-r2",
+	"version": "0.0.0",
+	"private": true,
+	"scripts": {
+		"deploy": "wrangler deploy src/index.ts",
+		"dev": "wrangler dev src/index.ts --local",
+		"start-stackblitz": "WRANGLER_SEND_METRICS=false wrangler dev src/index.ts --local"
+	},
+	"devDependencies": {
+		"@cloudflare/workers-types": "^4.20230904.0",
+		"wrangler": "^3.0.0"
+	}
+}

cf/prune-latest/src/index.ts

@@ -0,0 +1,26 @@
+export interface Env {
+	FOREST_ARCHIVE: R2Bucket
+}
+
+// Number of recent snapshots to keep. Roughly 1 new snapshot is uploaded every hour.
+const KEEP_COUNT: number = 10;
+
+async function prune(env: Env, chain: string): Promise<String> {
+	const listed = await env.FOREST_ARCHIVE.list({ prefix: chain + "/latest/" });
+	// objects are listed chronologically. Reverse to keep the newest snapshots.
+	listed.objects.reverse();
+	for (let i: number = KEEP_COUNT; i < listed.objects.length; i++) {
+		await env.FOREST_ARCHIVE.delete(listed.objects[i].key);
+	}
+	const pruned = Math.max(listed.objects.length - KEEP_COUNT, 0);
+	const kept = listed.objects.length - pruned;
+	return `Pruned: ${pruned}. Kept: ${kept}`;
+}
+
+export default {
+	async fetch(request: Request, env: Env): Promise<Response> {
+		const calibnet = await prune(env, 'calibnet');
+		const mainnet = await prune(env, 'mainnet');
+		return new Response(`Calibnet: ${calibnet}\nMainnet: ${mainnet}\n`);
+	}
+};

cf/prune-latest/tsconfig.json

@@ -0,0 +1,12 @@
+{
+	"compilerOptions": {
+		"noEmit": true,
+		"module": "esnext",
+		"target": "esnext",
+		"lib": ["esnext"],
+		"strict": true,
+		"moduleResolution": "node",
+		"types": ["@cloudflare/workers-types", "@types/jest"]
+	},
+	"exclude": ["node_modules", "dist"]
+}

cf/prune-latest/wrangler.toml

@@ -0,0 +1,16 @@
+name = "forest-prune-latest"
+main = "./src/index.ts"
+
+compatibility_date = "2022-06-30"
+
+routes = [
+    { pattern = "forest-archive.chainsafe.dev/prune/", zone_name = "chainsafe.dev" },
+]
+
+[[r2_buckets]]
+binding = 'FOREST_ARCHIVE'                 # can be any valid JavaScript variable name
+bucket_name = 'forest-archive'
+preview_bucket_name = 'forest-archive-dev'
+
+[triggers]
+crons = ["0 * * * *"]

do_function/monitor_snapshots/packages/monitor-snapshot/main/__main__.py

@@ -25,9 +25,6 @@ def slack_alert(message, thread_ts=None):
     client = WebClient(token=os.environ['SLACK_TOKEN'])
     CHANNEL_NAME = '#forest-notifications'
 
-    # Format message as a JSON-like string for better readability.
-    message = f'```{json.dumps(message, indent=4, ensure_ascii=False)}```'
-
     # Try sending message, catch and print any errors.
     try:
         response = client.chat_postMessage(channel=CHANNEL_NAME, text=message, thread_ts=thread_ts)
@@ -141,7 +138,7 @@ def main():
     if not checks_passed:
         thread = slack_alert("⛔ Snapshot check failed. 🔥🌲🔥")
         for error_message in error_messages:
-            slack_alert({"error": error_message}, thread_ts=thread)
+            slack_alert(error_message, thread_ts=thread)
         return {
             "result": "⛔ failure",
             "message": "Some checks did not pass. Please review the issues reported. Let's fix them and keep the forest green!. 🔥🌲🔥"

terraform/benchmark/main.tf

@@ -39,6 +39,8 @@ module "benchmark" {
   AWS_ACCESS_KEY_ID     = var.AWS_ACCESS_KEY_ID
   AWS_SECRET_ACCESS_KEY = var.AWS_SECRET_ACCESS_KEY
   digitalocean_token    = var.do_token
+  NEW_RELIC_API_KEY     = var.NEW_RELIC_API_KEY
+  NEW_RELIC_ACCOUNT_ID  = var.NEW_RELIC_ACCOUNT_ID
 }
 
 # This ip address may be used in the future by monitoring software

terraform/benchmark/variable.tf

@@ -17,3 +17,15 @@ variable "slack_token" {
   description = "slack access token"
   type        = string
 }
+
+variable "NEW_RELIC_API_KEY" {
+  description = "New Relic API KEY"
+  default     = ""
+  type        = string
+}
+
+variable "NEW_RELIC_ACCOUNT_ID" {
+  description = "New Relic Account ID"
+  default     = ""
+  type        = string
+}

terraform/forest-calibnet/main.tf

@@ -24,6 +24,7 @@ module "calibnet" {
   attach_volume         = false
   destination_addresses = var.destination_addresses
   chain                 = var.chain
+  rpc_port              = "2345"
   project               = var.project
   fw_name               = var.fw_name
   NR_LICENSE_KEY        = var.NR_LICENSE_KEY

terraform/forest-mainnet/main.tf

@@ -24,6 +24,7 @@ module "mainnet" {
   initial_filesystem_type = var.initial_filesystem_type
   volume_size             = var.volume_size
   attach_volume           = false
+  rpc_port                = "2345"
   destination_addresses   = var.destination_addresses
   chain                   = var.chain
   volume_name             = var.volume_name

terraform/lotus-mainnet/main.tf

@@ -30,6 +30,7 @@ module "lotus-mainnet" {
   fw_name                 = "lotus-mainnet-fw"
   script                  = "lotus.sh"
   forest_user             = "forest"
+  rpc_port                = "1234"
   NR_LICENSE_KEY          = var.NR_LICENSE_KEY
   NEW_RELIC_API_KEY       = var.NEW_RELIC_API_KEY
   NEW_RELIC_ACCOUNT_ID    = var.NEW_RELIC_ACCOUNT_ID

terraform/modules/benchmark/main.tf

@@ -65,6 +65,9 @@ locals {
     "echo 'export SLACK_NOTIF_CHANNEL=\"${var.slack_channel}\"' >> .forest_env",
     "echo 'export BENCHMARK_BUCKET=\"${var.benchmark_bucket}\"' >> .forest_env",
     "echo 'export BENCHMARK_ENDPOINT=\"${var.benchmark_endpoint}\"' >> .forest_env",
+    "echo 'export NEW_RELIC_API_KEY=\"${var.NEW_RELIC_API_KEY}\"' >> .forest_env",
+    "echo 'export NEW_RELIC_ACCOUNT_ID=\"${var.NEW_RELIC_ACCOUNT_ID}\"' >> .forest_env",
+    "echo 'export NEW_RELIC_REGION=\"${var.NEW_RELIC_REGION}\"' >> .forest_env",
     "echo 'export BASE_FOLDER=\"/chainsafe\"' >> .forest_env",
     "echo '. ~/.forest_env' >> .bashrc",
     ". ~/.forest_env",

terraform/modules/benchmark/service/init.sh

@@ -18,3 +18,37 @@ sudo docker run --detach \
   --restart unless-stopped \
   benchmark \
   /bin/bash -c "ruby run_benchmark.rb"
+
+# If New Relic license key and API key are provided,
+# install the new relic agent and New relic agent and OpenMetrics Prometheus integration.
+if [ -n "${NEW_RELIC_API_KEY}" ]; then
+  curl -Ls https://download.newrelic.com/install/newrelic-cli/scripts/install.sh | bash && \
+  sudo  NEW_RELIC_API_KEY="${NEW_RELIC_API_KEY}" \
+  NEW_RELIC_ACCOUNT_ID="${NEW_RELIC_ACCOUNT_ID}" \
+  NEW_RELIC_REGION="${NEW_RELIC_REGION}" \
+  /usr/local/bin/newrelic install -y
+
+# The provided configurations are specific to New Relic. To gain a deeper understanding of these configuration details, you can visit:
+# https://docs.newrelic.com/docs/infrastructure/install-infrastructure-agent/configuration/infrastructure-agent-configuration-settings/#offline-time-to-reset
+cat >> /etc/newrelic-infra.yml <<EOF
+include_matching_metrics:
+  process.name:
+    - regex "forest.*"
+    - regex "^fail2ban.*"
+    - regex "^rsyslog.*"
+    - regex "^syslog.*"
+    - regex "^gpg-agent.*"
+metrics_network_sample_rate: -1
+metrics_process_sample_rate: 600
+metrics_system_sample_rate: 600
+metrics_storage_sample_rate: 600
+metrics_nfs_sample_rate: 600
+container_cache_metadata_limit: 600
+disable_zero_mem_process_filter: true
+disable_all_plugins: true
+disable_cloud_metadata: true
+ignore_system_proxy: true
+EOF
+
+  sudo systemctl restart newrelic-infra
+fi