chore: install new relic on the benchmark service (#258)

Signed-off-by: samuelarogbonlo <[email protected]>

.github/workflows/deploy-benchmark.yml

@@ -34,5 +34,7 @@ jobs:
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}
           slack_token: ${{ secrets.SLACK_TOKEN }}
+          new_relic_account_id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
+          new_relic_api_key: ${{ secrets.NEW_RELIC_API_KEY }}
           working_directory: terraform/benchmark
           environment: Benchmark Service

terraform/benchmark/main.tf

@@ -39,6 +39,8 @@ module "benchmark" {
   AWS_ACCESS_KEY_ID     = var.AWS_ACCESS_KEY_ID
   AWS_SECRET_ACCESS_KEY = var.AWS_SECRET_ACCESS_KEY
   digitalocean_token    = var.do_token
+  NEW_RELIC_API_KEY     = var.NEW_RELIC_API_KEY
+  NEW_RELIC_ACCOUNT_ID  = var.NEW_RELIC_ACCOUNT_ID
 }
 
 # This ip address may be used in the future by monitoring software

terraform/benchmark/variable.tf

@@ -17,3 +17,15 @@ variable "slack_token" {
   description = "slack access token"
   type        = string
 }
+
+variable "NEW_RELIC_API_KEY" {
+  description = "New Relic API KEY"
+  default     = ""
+  type        = string
+}
+
+variable "NEW_RELIC_ACCOUNT_ID" {
+  description = "New Relic Account ID"
+  default     = ""
+  type        = string
+}

terraform/modules/benchmark/main.tf

@@ -65,6 +65,9 @@ locals {
     "echo 'export SLACK_NOTIF_CHANNEL=\"${var.slack_channel}\"' >> .forest_env",
     "echo 'export BENCHMARK_BUCKET=\"${var.benchmark_bucket}\"' >> .forest_env",
     "echo 'export BENCHMARK_ENDPOINT=\"${var.benchmark_endpoint}\"' >> .forest_env",
+    "echo 'export NEW_RELIC_API_KEY=\"${var.NEW_RELIC_API_KEY}\"' >> .forest_env",
+    "echo 'export NEW_RELIC_ACCOUNT_ID=\"${var.NEW_RELIC_ACCOUNT_ID}\"' >> .forest_env",
+    "echo 'export NEW_RELIC_REGION=\"${var.NEW_RELIC_REGION}\"' >> .forest_env",
     "echo 'export BASE_FOLDER=\"/chainsafe\"' >> .forest_env",
     "echo '. ~/.forest_env' >> .bashrc",
     ". ~/.forest_env",

terraform/modules/benchmark/service/init.sh

@@ -18,3 +18,37 @@ sudo docker run --detach \
   --restart unless-stopped \
   benchmark \
   /bin/bash -c "ruby run_benchmark.rb"
+
+# If New Relic license key and API key are provided,
+# install the new relic agent and New relic agent and OpenMetrics Prometheus integration.
+if [ -n "${NEW_RELIC_API_KEY}" ]; then
+  curl -Ls https://download.newrelic.com/install/newrelic-cli/scripts/install.sh | bash && \
+  sudo  NEW_RELIC_API_KEY="${NEW_RELIC_API_KEY}" \
+  NEW_RELIC_ACCOUNT_ID="${NEW_RELIC_ACCOUNT_ID}" \
+  NEW_RELIC_REGION="${NEW_RELIC_REGION}" \
+  /usr/local/bin/newrelic install -y
+
+# The provided configurations are specific to New Relic. To gain a deeper understanding of these configuration details, you can visit:
+# https://docs.newrelic.com/docs/infrastructure/install-infrastructure-agent/configuration/infrastructure-agent-configuration-settings/#offline-time-to-reset
+cat >> /etc/newrelic-infra.yml <<EOF
+include_matching_metrics:
+  process.name:
+    - regex "forest.*"
+    - regex "^fail2ban.*"
+    - regex "^rsyslog.*"
+    - regex "^syslog.*"
+    - regex "^gpg-agent.*"
+metrics_network_sample_rate: -1
+metrics_process_sample_rate: 600
+metrics_system_sample_rate: 600
+metrics_storage_sample_rate: 600
+metrics_nfs_sample_rate: 600
+container_cache_metadata_limit: 600
+disable_zero_mem_process_filter: true
+disable_all_plugins: true
+disable_cloud_metadata: true
+ignore_system_proxy: true
+EOF
+
+  sudo systemctl restart newrelic-infra
+fi

terraform/modules/benchmark/variable.tf

@@ -62,3 +62,21 @@ variable "project" {
   type        = string
   default     = "Forest-DEV" # Alternative: "Default"
 }
+
+variable "NEW_RELIC_API_KEY" {
+  description = "New Relic API KEY"
+  default     = ""
+  type        = string
+}
+
+variable "NEW_RELIC_ACCOUNT_ID" {
+  description = "New Relic Account ID"
+  default     = ""
+  type        = string
+}
+
+variable "NEW_RELIC_REGION" {
+  description = "The New Relic Platform Region"
+  type        = string
+  default     = "EU"
+}

terraform/modules/daily_snapshot/variable.tf

@@ -97,7 +97,7 @@ variable "destination_addresses" {
 }
 
 variable "NEW_RELIC_REGION" {
-  description = "The New Relic Plathom Region"
+  description = "The New Relic Platform Region"
   type        = string
   default     = "EU"
 }

terraform/modules/filecoin_node/variable.tf

@@ -100,7 +100,7 @@ variable "NEW_RELIC_ACCOUNT_ID" {
 }
 
 variable "NEW_RELIC_REGION" {
-  description = "The New Relic Plathom Region"
+  description = "The New Relic Platform Region"
   type        = string
   default     = "EU"
 }

terraform/modules/sync_check/variable.tf

@@ -42,7 +42,7 @@ variable "project" {
 }
 
 variable "NEW_RELIC_REGION" {
-  description = "The New Relic Plathom Region"
+  description = "The New Relic Platform Region"
   type        = string
   default     = "EU"
 }