Fix: For COVID-19 Italy, resolve bucket variable in pipeline.yaml (#509)

GoogleCloudPlatform · Oct 14, 2022 · 1f913ac · 1f913ac
1 parent 69128bc
commit 1f913ac
Show file tree

Hide file tree

Showing 6 changed files with 6 additions and 6 deletions.
diff --git a/datasets/covid19_italy/pipelines/data_by_province/data_by_province_dag.py b/datasets/covid19_italy/pipelines/data_by_province/data_by_province_dag.py
@@ -90,7 +90,7 @@
     # Task to load CSV data to a BigQuery table
     load_data_by_province_to_bq_eu = gcs_to_bigquery.GCSToBigQueryOperator(
         task_id="load_data_by_province_to_bq_eu",
-        bucket="public-datasets-dev-covid19-italy-eu",
+        bucket="{{ var.json.covid19_italy.destination_gcs_bucket }}",
         source_objects="province/data_output.csv",
         source_format="CSV",
         destination_project_dataset_table="covid19_italy_eu.data_by_province",

diff --git a/datasets/covid19_italy/pipelines/data_by_province/pipeline.yaml b/datasets/covid19_italy/pipelines/data_by_province/pipeline.yaml
@@ -121,7 +121,7 @@ dag:
       description: "Task to load CSV data to a BigQuery table"
       args:
         task_id: "load_data_by_province_to_bq_eu"
-        bucket: "public-datasets-dev-covid19-italy-eu"
+        bucket: "{{ var.json.covid19_italy.destination_gcs_bucket }}"
         source_objects: "province/data_output.csv"
         source_format: "CSV"
         destination_project_dataset_table: "covid19_italy_eu.data_by_province"

diff --git a/datasets/covid19_italy/pipelines/data_by_region/data_by_region_dag.py b/datasets/covid19_italy/pipelines/data_by_region/data_by_region_dag.py
@@ -121,7 +121,7 @@
     # Task to load CSV data to a BigQuery table
     load_data_by_region_to_bq_eu = gcs_to_bigquery.GCSToBigQueryOperator(
         task_id="load_data_by_region_to_bq_eu",
-        bucket="public-datasets-dev-covid19-italy-eu",
+        bucket="{{ var.json.covid19_italy.destination_gcs_bucket }}",
         source_objects="region/data_output.csv",
         source_format="CSV",
         destination_project_dataset_table="covid19_italy_eu.data_by_region",

diff --git a/datasets/covid19_italy/pipelines/data_by_region/pipeline.yaml b/datasets/covid19_italy/pipelines/data_by_region/pipeline.yaml
@@ -141,7 +141,7 @@ dag:
       description: "Task to load CSV data to a BigQuery table"
       args:
         task_id: "load_data_by_region_to_bq_eu"
-        bucket: "public-datasets-dev-covid19-italy-eu"
+        bucket: "{{ var.json.covid19_italy.destination_gcs_bucket }}"
         source_objects: region/data_output.csv
         source_format: "CSV"
         destination_project_dataset_table: "covid19_italy_eu.data_by_region"

diff --git a/datasets/covid19_italy/pipelines/national_trends/national_trends_dag.py b/datasets/covid19_italy/pipelines/national_trends/national_trends_dag.py
@@ -116,7 +116,7 @@
     # Task to load CSV data to a BigQuery table
     load_national_trends_to_bq_eu = gcs_to_bigquery.GCSToBigQueryOperator(
         task_id="load_national_trends_to_bq_eu",
-        bucket="public-datasets-dev-covid19-italy-eu",
+        bucket="{{ var.json.covid19_italy.destination_gcs_bucket }}",
         source_objects="national/data_output.csv",
         source_format="CSV",
         destination_project_dataset_table="covid19_italy_eu.national_trends",

diff --git a/datasets/covid19_italy/pipelines/national_trends/pipeline.yaml b/datasets/covid19_italy/pipelines/national_trends/pipeline.yaml
@@ -127,7 +127,7 @@ dag:
       description: "Task to load CSV data to a BigQuery table"
       args:
         task_id: "load_national_trends_to_bq_eu"
-        bucket: "public-datasets-dev-covid19-italy-eu"
+        bucket: "{{ var.json.covid19_italy.destination_gcs_bucket }}"
         source_objects: national/data_output.csv
         source_format: "CSV"
         destination_project_dataset_table: "covid19_italy_eu.national_trends"