From fc4709d9de823588966a95cfe598cec52f3acf60 Mon Sep 17 00:00:00 2001
From: "mana\"/scriptalert('xss')/script" <thisismana@users.noreply.github.com>
Date: Tue, 14 Mar 2023 17:14:54 +0100
Subject: [PATCH] feat: use new OpenSearch logging extension (#220)

* feat: use new OpenSearch logging extension

- move lambda into VPC, so it can communicate with the OpenSearch server
- added Logging Lambda Extension

* feat: use new OpenSearch logging extension

- move lambda into VPC, so it can communicate with the OpenSearch server
- added Logging Lambda Extension

* directed to new logging cluster
---
 .../terraform/.terraform.lock.hcl             |  8 +++++
 source/image-handler/terraform/backend.tf     |  4 +++
 source/image-handler/terraform/data.tf        | 32 ++++++++++++++++++-
 source/image-handler/terraform/main.tf        | 30 ++++++++++++++---
 source/image-handler/terraform/provider.tf    |  6 ++++
 5 files changed, 74 insertions(+), 6 deletions(-)

diff --git a/source/image-handler/terraform/.terraform.lock.hcl b/source/image-handler/terraform/.terraform.lock.hcl
index 2881b2a77..2214bfdac 100644
--- a/source/image-handler/terraform/.terraform.lock.hcl
+++ b/source/image-handler/terraform/.terraform.lock.hcl
@@ -24,3 +24,11 @@ provider "registry.terraform.io/hashicorp/aws" {
     "zh:ffc20b7d9f7bd331fb6451d0fc92c68196383d7115e69380de6566cc268cb9b9",
   ]
 }
+
+provider "registry.terraform.io/opensearch-project/opensearch" {
+  version     = "1.0.0-beta.2"
+  constraints = "1.0.0-beta.2"
+  hashes = [
+    "h1:nFbygatyqqJX2pRvNO6GzNq7Z5brOXgzjA69sUOMTC0=",
+  ]
+}
diff --git a/source/image-handler/terraform/backend.tf b/source/image-handler/terraform/backend.tf
index 2d8e1bbd4..5f7388835 100644
--- a/source/image-handler/terraform/backend.tf
+++ b/source/image-handler/terraform/backend.tf
@@ -9,6 +9,10 @@ terraform {
       source  = "hashicorp/aws"
       version = ">= 4.5"
     }
+    opensearch = {
+      source  = "opensearch-project/opensearch"
+      version = "1.0.0-beta.2"
+    }
   }
 
   required_version = "~> 1.0"
diff --git a/source/image-handler/terraform/data.tf b/source/image-handler/terraform/data.tf
index 24c8ac222..289a500ab 100644
--- a/source/image-handler/terraform/data.tf
+++ b/source/image-handler/terraform/data.tf
@@ -15,4 +15,34 @@ data "aws_s3_bucket" "pipeline_artifacts" {
 
 data "aws_s3_bucket" "ci" {
   bucket = "ci-${data.aws_caller_identity.current.account_id}-${data.aws_region.current.name}"
-}
\ No newline at end of file
+}
+
+data "aws_vpc" "selected" {
+  tags = {
+    Name = "main"
+  }
+}
+
+data "aws_subnets" "selected" {
+  filter {
+    name   = "vpc-id"
+    values = [data.aws_vpc.selected.id]
+  }
+
+  tags = {
+    Tier = "private"
+  }
+}
+
+
+data "aws_security_group" "vpc_endpoints" {
+  name = "vpc-endpoint-access"
+}
+
+data "aws_security_group" "all_outbound" {
+  name = "allow-outbound-tcp"
+}
+
+data "aws_security_group" "lambda" {
+  name = "lambda-default"
+}
diff --git a/source/image-handler/terraform/main.tf b/source/image-handler/terraform/main.tf
index 00598283d..d80e05631 100644
--- a/source/image-handler/terraform/main.tf
+++ b/source/image-handler/terraform/main.tf
@@ -10,7 +10,8 @@ module "lambda" {
   version = "6.10.0"
 
   architectures                     = ["x86_64"]
-  cloudwatch_logs_retention_in_days = 1
+  layers                            = ["arn:aws:lambda:eu-west-1:053041861227:layer:CustomLoggingExtensionOpenSearch-Amd64:9"]
+  cloudwatch_logs_enabled           = false
   description                       = "provider of cute kitty pics."
   function_name                     = local.function_name
   ignore_external_function_updates  = true
@@ -29,13 +30,17 @@ module "lambda" {
       CORS_ENABLED   = "Yes"
       CORS_ORIGIN    = "*"
       SOURCE_BUCKETS = aws_s3_bucket.images.bucket
+
+      LOG_EXT_OPEN_SEARCH_URL     = "https://logs.stroeer.engineering"
+      LOG_EXT_BUFFERING_TIMEOUT   = "30000"
+      LOG_EXT_BUFFERING_MAX_BYTES = "1048576"
+      LOG_EXT_BUFFERING_MAX_ITEMS = "10000"
     }
   }
 
-  cloudwatch_log_subscription_filters = {
-    opensearch = {
-      destination_arn = data.aws_lambda_function.log_streaming.arn
-    }
+  vpc_config = {
+    security_group_ids = [data.aws_security_group.vpc_endpoints.id, data.aws_security_group.all_outbound.id, data.aws_security_group.lambda.id]
+    subnet_ids         = data.aws_subnets.selected.ids
   }
 }
 
@@ -85,3 +90,18 @@ module "deployment" {
   s3_key                                      = local.s3_key
   function_name                               = local.function_name
 }
+
+resource "opensearch_role" "logs_write_access" {
+  role_name   = local.function_name
+  description = "Write access for ${local.function_name} lambda"
+
+  index_permissions {
+    index_patterns  = ["${local.function_name}-lambda-*"]
+    allowed_actions = ["write", "create_index"]
+  }
+}
+
+resource "opensearch_roles_mapping" "logs_write_access" {
+  role_name     = opensearch_role.logs_write_access.role_name
+  backend_roles = [module.lambda.role_name]
+}
\ No newline at end of file
diff --git a/source/image-handler/terraform/provider.tf b/source/image-handler/terraform/provider.tf
index d560b442a..516bb925c 100644
--- a/source/image-handler/terraform/provider.tf
+++ b/source/image-handler/terraform/provider.tf
@@ -12,3 +12,9 @@ provider "aws" {
   }
 
 }
+
+provider "opensearch" {
+  aws_region  = data.aws_region.current.name
+  healthcheck = true
+  url         = "https://logs.stroeer.engineering"
+}