fix: deleting pod error (#511)

GoogleCloudPlatform · Oct 18, 2022 · 77fe479 · 77fe479
1 parent 014465b
commit 77fe479
Show file tree

Hide file tree

Showing 6 changed files with 15 additions and 33 deletions.
diff --git a/datasets/cfpb_complaints/infra/cfpb_complaints_dataset.tf b/datasets/cfpb_complaints/infra/cfpb_complaints_dataset.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2021 Google LLC
+ * Copyright 2022 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,19 +24,3 @@ resource "google_bigquery_dataset" "cfpb_complaints" {
 output "bigquery_dataset-cfpb_complaints-dataset_id" {
   value = google_bigquery_dataset.cfpb_complaints.dataset_id
 }
-
-resource "google_storage_bucket" "cfpb-complaints" {
-  name                        = "${var.bucket_name_prefix}-cfpb-complaints"
-  force_destroy               = true
-  location                    = "US"
-  uniform_bucket_level_access = true
-  lifecycle {
-    ignore_changes = [
-      logging,
-    ]
-  }
-}
-
-output "storage_bucket-cfpb-complaints-name" {
-  value = google_storage_bucket.cfpb-complaints.name
-}
diff --git a/datasets/cfpb_complaints/infra/complaint_database_pipeline.tf b/datasets/cfpb_complaints/infra/complaint_database_pipeline.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2021 Google LLC
+ * Copyright 2022 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,15 +16,10 @@
 
 
 resource "google_bigquery_table" "cfpb_complaints_complaint_database" {
-  project    = var.project_id
-  dataset_id = "cfpb_complaints"
-  table_id   = "complaint_database"
-
+  project     = var.project_id
+  dataset_id  = "cfpb_complaints"
+  table_id    = "complaint_database"
   description = "CFPB Complain table"
-
-
-
-
   depends_on = [
     google_bigquery_dataset.cfpb_complaints
   ]

diff --git a/datasets/cfpb_complaints/infra/provider.tf b/datasets/cfpb_complaints/infra/provider.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2021 Google LLC
+ * Copyright 2022 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

diff --git a/datasets/cfpb_complaints/infra/variables.tf b/datasets/cfpb_complaints/infra/variables.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2021 Google LLC
+ * Copyright 2022 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,4 +20,7 @@ variable "bucket_name_prefix" {}
 variable "impersonating_acct" {}
 variable "region" {}
 variable "env" {}
+variable "iam_policies" {
+  default = {}
+}
 
diff --git a/datasets/cfpb_complaints/pipelines/complaint_database/complaint_database_dag.py b/datasets/cfpb_complaints/pipelines/complaint_database/complaint_database_dag.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Google LLC
+# Copyright 2022 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -53,9 +53,9 @@
             "RENAME_MAPPINGS": '{"Complaint ID":"complaint_id","Date received":"date_received","Product":"product","Sub-product":"subproduct","Issue":"issue","Sub-issue":"subissue","Consumer complaint narrative":"consumer_complaint_narrative","Company response to consumer":"company_response_to_consumer","Company public response":"company_public_response","Company":"company_name","State":"state","ZIP code":"zip_code","Tags":"tags","Consumer consent provided?":"consumer_consent_provided","Submitted via":"submitted_via","Date sent to company":"date_sent_to_company","Timely response?":"timely_response","Consumer disputed?":"consumer_disputed"}',
         },
         resources={
-            "request_memory": "3G",
+            "request_memory": "4G",
             "request_cpu": "1",
-            "request_ephemeral_storage": "10G",
+            "request_ephemeral_storage": "8G",
         },
     )
 

diff --git a/datasets/cfpb_complaints/pipelines/complaint_database/pipeline.yaml b/datasets/cfpb_complaints/pipelines/complaint_database/pipeline.yaml
@@ -54,9 +54,9 @@ dag:
           RENAME_MAPPINGS: >-
             {"Complaint ID":"complaint_id","Date received":"date_received","Product":"product","Sub-product":"subproduct","Issue":"issue","Sub-issue":"subissue","Consumer complaint narrative":"consumer_complaint_narrative","Company response to consumer":"company_response_to_consumer","Company public response":"company_public_response","Company":"company_name","State":"state","ZIP code":"zip_code","Tags":"tags","Consumer consent provided?":"consumer_consent_provided","Submitted via":"submitted_via","Date sent to company":"date_sent_to_company","Timely response?":"timely_response","Consumer disputed?":"consumer_disputed"}
         resources:
-          request_memory: "3G"
+          request_memory: "4G"
           request_cpu: "1"
-          request_ephemeral_storage: "10G"
+          request_ephemeral_storage: "8G"
 
     - operator: "GoogleCloudStorageToBigQueryOperator"
       description: "Task to load CSV data to a BigQuery table"