From de5b5b2df090481d92ab981f95ab70be2a2364ca Mon Sep 17 00:00:00 2001
From: Frederik Ring <frederik.ring@wikimedia.de>
Date: Mon, 10 Jul 2023 15:12:31 +0200
Subject: [PATCH] feat(tf): add log metric and alert for detecting signup
 throttling

---
 .../monitoring/signup-throttling-alert.tf     | 39 +++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 tf/modules/monitoring/signup-throttling-alert.tf

diff --git a/tf/modules/monitoring/signup-throttling-alert.tf b/tf/modules/monitoring/signup-throttling-alert.tf
new file mode 100644
index 000000000..693783081
--- /dev/null
+++ b/tf/modules/monitoring/signup-throttling-alert.tf
@@ -0,0 +1,39 @@
+resource "google_logging_metric" "api_signup_throttling_applied" {
+  name        = "${var.cluster_name}-api_signup_throttling_applied"
+  description = "A log based metric that detects when api was throttling sign up attempts"
+
+  # Filter for the secondary pod and look for readiness probe failures
+  filter = "resource.labels.cluster_name=\"${var.cluster_name}\" AND resource.labels.pod_name:\"api-app-\" AND resource.type=\"k8s_pod\" AND severity=WARNING AND jsonPayload.message:\"WARN_SIGNUP_THROTTLED\""
+
+  metric_descriptor {
+    metric_kind = "DELTA"
+    value_type  = "INT64"
+  }
+}
+
+resource "google_monitoring_alert_policy" "alert_api_signup_throttling_applied" {
+  display_name = "(${var.cluster_name}): Signup was throttled"
+  combiner     = "OR"
+  notification_channels = [
+    "${var.monitoring_email_group_name}"
+  ]
+  documentation {
+    content = "Alert triggers when a user tried to create a wikibase.cloud account, but the configured limits were already exceeded."
+  }
+  conditions {
+    display_name = "(${var.cluster_name}): Signup was throttled"
+    condition_threshold {
+      filter     = "metric.type=\"logging.googleapis.com/user/${google_logging_metric.api_signup_throttling_applied}\" AND resource.type=\"k8s_pod\""
+      duration   = "0s"
+      comparison = "COMPARISON_GT"
+      aggregations {
+        alignment_period     = "300s"
+        per_series_aligner   = "ALIGN_SUM"
+        cross_series_reducer = "REDUCE_SUM"
+      }
+      trigger {
+        count = 1
+      }
+    }
+  }
+}