Add support for Dataproc Metastore CMEK config (#5881) (#4204)

Signed-off-by: Modular Magician <[email protected]>

.changelog/5881.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+metastore: Added support for encryption_config during service creation.
+```

google-beta/resource_dataproc_metastore_service.go

@@ -50,6 +50,24 @@ func resourceDataprocMetastoreService() *schema.Resource {
 and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between
 3 and 63 characters.`,
 			},
+			"encryption_config": {
+				Type:     schema.TypeList,
+				Optional: true,
+				Description: `Information used to configure the Dataproc Metastore service to encrypt
+customer data at rest.`,
+				MaxItems: 1,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"kms_key": {
+							Type:     schema.TypeString,
+							Required: true,
+							ForceNew: true,
+							Description: `The fully qualified customer provided Cloud KMS key name to use for customer data encryption.
+Use the following format: 'projects/([^/]+)/locations/([^/]+)/keyRings/([^/]+)/cryptoKeys/([^/]+)'`,
+						},
+					},
+				},
+			},
 			"hive_metastore_config": {
 				Type:        schema.TypeList,
 				Optional:    true,
@@ -244,6 +262,12 @@ func resourceDataprocMetastoreServiceCreate(d *schema.ResourceData, meta interfa
 	} else if v, ok := d.GetOkExists("maintenance_window"); !isEmptyValue(reflect.ValueOf(maintenanceWindowProp)) && (ok || !reflect.DeepEqual(v, maintenanceWindowProp)) {
 		obj["maintenanceWindow"] = maintenanceWindowProp
 	}
+	encryptionConfigProp, err := expandDataprocMetastoreServiceEncryptionConfig(d.Get("encryption_config"), d, config)
+	if err != nil {
+		return err
+	} else if v, ok := d.GetOkExists("encryption_config"); !isEmptyValue(reflect.ValueOf(encryptionConfigProp)) && (ok || !reflect.DeepEqual(v, encryptionConfigProp)) {
+		obj["encryptionConfig"] = encryptionConfigProp
+	}
 	hiveMetastoreConfigProp, err := expandDataprocMetastoreServiceHiveMetastoreConfig(d.Get("hive_metastore_config"), d, config)
 	if err != nil {
 		return err
@@ -361,6 +385,9 @@ func resourceDataprocMetastoreServiceRead(d *schema.ResourceData, meta interface
 	if err := d.Set("maintenance_window", flattenDataprocMetastoreServiceMaintenanceWindow(res["maintenanceWindow"], d, config)); err != nil {
 		return fmt.Errorf("Error reading Service: %s", err)
 	}
+	if err := d.Set("encryption_config", flattenDataprocMetastoreServiceEncryptionConfig(res["encryptionConfig"], d, config)); err != nil {
+		return fmt.Errorf("Error reading Service: %s", err)
+	}
 	if err := d.Set("hive_metastore_config", flattenDataprocMetastoreServiceHiveMetastoreConfig(res["hiveMetastoreConfig"], d, config)); err != nil {
 		return fmt.Errorf("Error reading Service: %s", err)
 	}
@@ -408,6 +435,12 @@ func resourceDataprocMetastoreServiceUpdate(d *schema.ResourceData, meta interfa
 	} else if v, ok := d.GetOkExists("maintenance_window"); !isEmptyValue(reflect.ValueOf(v)) && (ok || !reflect.DeepEqual(v, maintenanceWindowProp)) {
 		obj["maintenanceWindow"] = maintenanceWindowProp
 	}
+	encryptionConfigProp, err := expandDataprocMetastoreServiceEncryptionConfig(d.Get("encryption_config"), d, config)
+	if err != nil {
+		return err
+	} else if v, ok := d.GetOkExists("encryption_config"); !isEmptyValue(reflect.ValueOf(v)) && (ok || !reflect.DeepEqual(v, encryptionConfigProp)) {
+		obj["encryptionConfig"] = encryptionConfigProp
+	}
 	hiveMetastoreConfigProp, err := expandDataprocMetastoreServiceHiveMetastoreConfig(d.Get("hive_metastore_config"), d, config)
 	if err != nil {
 		return err
@@ -439,6 +472,10 @@ func resourceDataprocMetastoreServiceUpdate(d *schema.ResourceData, meta interfa
 		updateMask = append(updateMask, "maintenanceWindow")
 	}
 
+	if d.HasChange("encryption_config") {
+		updateMask = append(updateMask, "encryptionConfig")
+	}
+
 	if d.HasChange("hive_metastore_config") {
 		updateMask = append(updateMask, "hiveMetastoreConfig")
 	}
@@ -623,6 +660,23 @@ func flattenDataprocMetastoreServiceMaintenanceWindowDayOfWeek(v interface{}, d
 	return v
 }
 
+func flattenDataprocMetastoreServiceEncryptionConfig(v interface{}, d *schema.ResourceData, config *Config) interface{} {
+	if v == nil {
+		return nil
+	}
+	original := v.(map[string]interface{})
+	if len(original) == 0 {
+		return nil
+	}
+	transformed := make(map[string]interface{})
+	transformed["kms_key"] =
+		flattenDataprocMetastoreServiceEncryptionConfigKmsKey(original["kmsKey"], d, config)
+	return []interface{}{transformed}
+}
+func flattenDataprocMetastoreServiceEncryptionConfigKmsKey(v interface{}, d *schema.ResourceData, config *Config) interface{} {
+	return v
+}
+
 func flattenDataprocMetastoreServiceHiveMetastoreConfig(v interface{}, d *schema.ResourceData, config *Config) interface{} {
 	if v == nil {
 		return nil
@@ -747,6 +801,29 @@ func expandDataprocMetastoreServiceMaintenanceWindowDayOfWeek(v interface{}, d T
 	return v, nil
 }
 
+func expandDataprocMetastoreServiceEncryptionConfig(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
+	l := v.([]interface{})
+	if len(l) == 0 || l[0] == nil {
+		return nil, nil
+	}
+	raw := l[0]
+	original := raw.(map[string]interface{})
+	transformed := make(map[string]interface{})
+
+	transformedKmsKey, err := expandDataprocMetastoreServiceEncryptionConfigKmsKey(original["kms_key"], d, config)
+	if err != nil {
+		return nil, err
+	} else if val := reflect.ValueOf(transformedKmsKey); val.IsValid() && !isEmptyValue(val) {
+		transformed["kmsKey"] = transformedKmsKey
+	}
+
+	return transformed, nil
+}
+
+func expandDataprocMetastoreServiceEncryptionConfigKmsKey(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
+	return v, nil
+}
+
 func expandDataprocMetastoreServiceHiveMetastoreConfig(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
 	l := v.([]interface{})
 	if len(l) == 0 || l[0] == nil {

google-beta/resource_dataproc_metastore_service_generated_test.go

@@ -69,6 +69,85 @@ resource "google_dataproc_metastore_service" "default" {
 `, context)
 }
 
+func TestAccDataprocMetastoreService_dataprocMetastoreServiceCmekTestExample(t *testing.T) {
+	t.Parallel()
+
+	context := map[string]interface{}{
+		"random_suffix": randString(t, 10),
+	}
+
+	vcrTest(t, resource.TestCase{
+		PreCheck:     func() { testAccPreCheck(t) },
+		Providers:    testAccProvidersOiCS,
+		CheckDestroy: testAccCheckDataprocMetastoreServiceDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccDataprocMetastoreService_dataprocMetastoreServiceCmekTestExample(context),
+			},
+			{
+				ResourceName:            "google_dataproc_metastore_service.default",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"service_id", "location"},
+			},
+		},
+	})
+}
+
+func testAccDataprocMetastoreService_dataprocMetastoreServiceCmekTestExample(context map[string]interface{}) string {
+	return Nprintf(`
+data "google_project" "project" {
+  provider = google-beta
+}
+
+data "google_storage_project_service_account" "gcs_account" {
+  provider = google-beta
+}
+
+
+resource "google_dataproc_metastore_service" "default" {
+  provider   = google-beta
+  service_id = "tf-test-example-service%{random_suffix}"
+  location   = "us-central1"
+
+  encryption_config {
+    kms_key = google_kms_crypto_key.crypto_key.id
+  }
+
+  hive_metastore_config {
+    version = "3.1.2"
+  }
+
+  depends_on = [google_kms_crypto_key_iam_binding.crypto_key_binding]
+}
+
+resource "google_kms_crypto_key" "crypto_key" {
+  provider = google-beta
+  name     = "tf-test-example-key%{random_suffix}"
+  key_ring = google_kms_key_ring.key_ring.id
+
+  purpose  = "ENCRYPT_DECRYPT"
+}
+
+resource "google_kms_key_ring" "key_ring" {
+  provider = google-beta
+  name     = "tf-test-example-keyring%{random_suffix}"
+  location = "us-central1"
+}
+
+resource "google_kms_crypto_key_iam_binding" "crypto_key_binding" {
+  provider      = google-beta
+  crypto_key_id = google_kms_crypto_key.crypto_key.id
+  role          = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
+
+  members = [
+    "serviceAccount:service-${data.google_project.project.number}@gcp-sa-metastore.iam.gserviceaccount.com",
+    "serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"
+  ]
+}
+`, context)
+}
+
 func testAccCheckDataprocMetastoreServiceDestroyProducer(t *testing.T) func(s *terraform.State) error {
 	return func(s *terraform.State) error {
 		for name, rs := range s.RootModule().Resources {

website/docs/r/dataproc_metastore_service.html.markdown

@@ -54,6 +54,38 @@ resource "google_dataproc_metastore_service" "default" {
   }
 }
 ```
+## Example Usage - Dataproc Metastore Service Cmek Example
+
+
+```hcl
+resource "google_dataproc_metastore_service" "default" {
+  provider   = google-beta
+  service_id = "example-service"
+  location   = "us-central1"
+
+  encryption_config {
+    kms_key = google_kms_crypto_key.crypto_key.id
+  }
+
+  hive_metastore_config {
+    version = "3.1.2"
+  }
+}
+
+resource "google_kms_crypto_key" "crypto_key" {
+  provider = google-beta
+  name     = "example-key"
+  key_ring = google_kms_key_ring.key_ring.id
+
+  purpose  = "ENCRYPT_DECRYPT"
+}
+
+resource "google_kms_key_ring" "key_ring" {
+  provider = google-beta
+  name     = "example-keyring"
+  location = "us-central1"
+}
+```
 
 ## Argument Reference
 
@@ -94,6 +126,12 @@ The following arguments are supported:
   This specifies when the service can be restarted for maintenance purposes in UTC time.
   Structure is [documented below](#nested_maintenance_window).
 
+* `encryption_config` -
+  (Optional)
+  Information used to configure the Dataproc Metastore service to encrypt
+  customer data at rest.
+  Structure is [documented below](#nested_encryption_config).
+
 * `hive_metastore_config` -
   (Optional)
   Configuration information specific to running Hive metastore software as the metastore service.
@@ -119,6 +157,13 @@ The following arguments are supported:
   The day of week, when the window starts.
   Possible values are `MONDAY`, `TUESDAY`, `WEDNESDAY`, `THURSDAY`, `FRIDAY`, `SATURDAY`, and `SUNDAY`.
 
+<a name="nested_encryption_config"></a>The `encryption_config` block supports:
+
+* `kms_key` -
+  (Required)
+  The fully qualified customer provided Cloud KMS key name to use for customer data encryption.
+  Use the following format: `projects/([^/]+)/locations/([^/]+)/keyRings/([^/]+)/cryptoKeys/([^/]+)`
+
 <a name="nested_hive_metastore_config"></a>The `hive_metastore_config` block supports:
 
 * `version` -