From d390b754e03f0b8db64a7867c019add31023e828 Mon Sep 17 00:00:00 2001 From: Julius Kelly Date: Thu, 25 Aug 2022 03:40:45 +0000 Subject: [PATCH 1/3] update for support of CMEK for vertex ai resource --- mmv1/products/vertexai/api.yaml | 21 ++++++++++++++----- mmv1/products/vertexai/terraform.yaml | 6 ++++++ .../examples/vertex_ai_featurestore.tf.erb | 3 +++ .../vertex_ai_featurestore_entitytype.tf.erb | 3 +++ 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/mmv1/products/vertexai/api.yaml b/mmv1/products/vertexai/api.yaml index ac6e40c8b170..a68d7b29b864 100644 --- a/mmv1/products/vertexai/api.yaml +++ b/mmv1/products/vertexai/api.yaml @@ -94,13 +94,13 @@ objects: - !ruby/object:Api::Type::String name: 'kmsKeyName' description: | - Required. The Cloud KMS resource identifier of the customer managed encryption key used to protect a resource. + Required. The Cloud KMS resource identifier of the customer managed encryption key used to protect a resource. Has the form: projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key. The key needs to be in the same region as where the resource is created. input: true - !ruby/object:Api::Type::String name: 'metadataSchemaUri' required: true - input: true + input: true description: | Points to a YAML file stored on Google Cloud Storage describing additional information about the Dataset. The schema is defined as an OpenAPI 3.0.2 Schema Object. The schema files that can be used here are found in gs://google-cloud-aiplatform/schema/dataset/metadata/. # Vertex AI Featurestores @@ -151,7 +151,7 @@ objects: pattern: projects/{{project}}/locations/{{region}}/featurestores/{{name}} - !ruby/object:Api::Type::String name: 'etag' - description: Used to perform consistent read-modify-write updates. + description: Used to perform consistent read-modify-write updates. output: true - !ruby/object:Api::Type::String name: 'createTime' @@ -177,6 +177,17 @@ objects: required: true description: | The number of nodes for each cluster. The number of nodes will not scale automatically but can be scaled manually by providing different values when updating. + - !ruby/object:Api::Type::NestedObject + name: 'encryptionSpec' + description: | + If set, both of the online and offline data storage will be secured by this key. + properties: + - !ruby/object:Api::Type::String + name: 'kmsKeyName' + required: true + description: | + The Cloud KMS resource identifier of the customer managed encryption key used to protect a resource. Has the form: projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key. The key needs to be in the same region as where the compute resource is created. + # Vertex AI Featurestore Entity Type - !ruby/object:Api::Resource name: FeaturestoreEntitytype @@ -227,7 +238,7 @@ objects: pattern: '{featurestore}}/entityTypes/{{name}}' - !ruby/object:Api::Type::String name: 'etag' - description: Used to perform consistent read-modify-write updates. + description: Used to perform consistent read-modify-write updates. output: true - !ruby/object:Api::Type::String name: 'createTime' @@ -334,7 +345,7 @@ objects: - !ruby/object:Api::Type::String name: 'kmsKeyName' description: | - Required. The Cloud KMS resource identifier of the customer managed encryption key used to protect a resource. + Required. The Cloud KMS resource identifier of the customer managed encryption key used to protect a resource. Has the form: projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key. The key needs to be in the same region as where the resource is created. input: true - !ruby/object:Api::Type::NestedObject diff --git a/mmv1/products/vertexai/terraform.yaml b/mmv1/products/vertexai/terraform.yaml index ef41c230667d..b1779359d10f 100644 --- a/mmv1/products/vertexai/terraform.yaml +++ b/mmv1/products/vertexai/terraform.yaml @@ -39,6 +39,9 @@ overrides: !ruby/object:Overrides::ResourceOverrides primary_resource_id: "featurestore" vars: name: "terraform" + kms_key_name: "kms-key" + test_env_vars: + kms_key_name: 'BootstrapKMSKeyInLocation(t, "us-central1").CryptoKey.Name' ignore_read_extra: - "force_destroy" properties: @@ -65,6 +68,9 @@ overrides: !ruby/object:Overrides::ResourceOverrides primary_resource_id: "entity" vars: name: "terraform" + kms_key_name: "kms-key" + test_env_vars: + kms_key_name: 'BootstrapKMSKeyInLocation(t, "us-central1").CryptoKey.Name' properties: etag: !ruby/object:Overrides::Terraform::PropertyOverride ignore_read: true diff --git a/mmv1/templates/terraform/examples/vertex_ai_featurestore.tf.erb b/mmv1/templates/terraform/examples/vertex_ai_featurestore.tf.erb index 816cbc2db021..0eaefed1e142 100644 --- a/mmv1/templates/terraform/examples/vertex_ai_featurestore.tf.erb +++ b/mmv1/templates/terraform/examples/vertex_ai_featurestore.tf.erb @@ -8,5 +8,8 @@ resource "google_vertex_ai_featurestore" "featurestore" { online_serving_config { fixed_node_count = 2 } + encryption_spec { + kms_key_name = "<%= ctx[:vars]['kms_key_name'] %>" + } force_destroy = true } diff --git a/mmv1/templates/terraform/examples/vertex_ai_featurestore_entitytype.tf.erb b/mmv1/templates/terraform/examples/vertex_ai_featurestore_entitytype.tf.erb index 72096c92cdc3..c676aab0201c 100644 --- a/mmv1/templates/terraform/examples/vertex_ai_featurestore_entitytype.tf.erb +++ b/mmv1/templates/terraform/examples/vertex_ai_featurestore_entitytype.tf.erb @@ -8,6 +8,9 @@ resource "google_vertex_ai_featurestore" "featurestore" { online_serving_config { fixed_node_count = 2 } + encryption_spec { + kms_key_name = "<%= ctx[:vars]['kms_key_name'] %>" + } } resource "google_vertex_ai_featurestore_entitytype" "entity" { From c51bad0fdc7dd6f3b7a985f15f21e701356d8e56 Mon Sep 17 00:00:00 2001 From: Julius Kelly Date: Fri, 26 Aug 2022 07:03:55 +0000 Subject: [PATCH 2/3] kms key as a resource added --- mmv1/products/vertexai/terraform.yaml | 10 ++++++---- .../examples/vertex_ai_featurestore.tf.erb | 19 ++++++++++++++++++- .../vertex_ai_featurestore_entitytype.tf.erb | 19 ++++++++++++++++++- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/mmv1/products/vertexai/terraform.yaml b/mmv1/products/vertexai/terraform.yaml index b1779359d10f..402cdcdb8162 100644 --- a/mmv1/products/vertexai/terraform.yaml +++ b/mmv1/products/vertexai/terraform.yaml @@ -39,9 +39,10 @@ overrides: !ruby/object:Overrides::ResourceOverrides primary_resource_id: "featurestore" vars: name: "terraform" - kms_key_name: "kms-key" + project: "appeng-flex" test_env_vars: - kms_key_name: 'BootstrapKMSKeyInLocation(t, "us-central1").CryptoKey.Name' + org_id: :ORG_ID + billing_account: :BILLING_ACCT ignore_read_extra: - "force_destroy" properties: @@ -68,9 +69,10 @@ overrides: !ruby/object:Overrides::ResourceOverrides primary_resource_id: "entity" vars: name: "terraform" - kms_key_name: "kms-key" + project: "vertex-ai" test_env_vars: - kms_key_name: 'BootstrapKMSKeyInLocation(t, "us-central1").CryptoKey.Name' + org_id: :ORG_ID + billing_account: :BILLING_ACCT properties: etag: !ruby/object:Overrides::Terraform::PropertyOverride ignore_read: true diff --git a/mmv1/templates/terraform/examples/vertex_ai_featurestore.tf.erb b/mmv1/templates/terraform/examples/vertex_ai_featurestore.tf.erb index 0eaefed1e142..7a70ae2ccb6a 100644 --- a/mmv1/templates/terraform/examples/vertex_ai_featurestore.tf.erb +++ b/mmv1/templates/terraform/examples/vertex_ai_featurestore.tf.erb @@ -1,3 +1,20 @@ +resource "google_kms_key_ring" "key_ring" { + provider = google-beta + name = "key-ring" + location = "us-central1" +} + +resource "google_kms_crypto_key" "crypto_key" { + provider = google-beta + name = "crypto-key" + key_ring = google_kms_key_ring.key_ring.id + purpose = "ENCRYPT_DECRYPT" + + version_template { + algorithm = "GOOGLE_SYMMETRIC_ENCRYPTION" + } +} + resource "google_vertex_ai_featurestore" "featurestore" { provider = google-beta name = "<%= ctx[:vars]['name'] %>" @@ -9,7 +26,7 @@ resource "google_vertex_ai_featurestore" "featurestore" { fixed_node_count = 2 } encryption_spec { - kms_key_name = "<%= ctx[:vars]['kms_key_name'] %>" + kms_key_name = google_kms_crypto_key.crypto_key.id } force_destroy = true } diff --git a/mmv1/templates/terraform/examples/vertex_ai_featurestore_entitytype.tf.erb b/mmv1/templates/terraform/examples/vertex_ai_featurestore_entitytype.tf.erb index c676aab0201c..db367613ac22 100644 --- a/mmv1/templates/terraform/examples/vertex_ai_featurestore_entitytype.tf.erb +++ b/mmv1/templates/terraform/examples/vertex_ai_featurestore_entitytype.tf.erb @@ -1,3 +1,20 @@ +resource "google_kms_key_ring" "key_ring" { + provider = google-beta + name = "key-ring" + location = "us-central1" +} + +resource "google_kms_crypto_key" "crypto_key" { + provider = google-beta + name = "crypto-key" + key_ring = google_kms_key_ring.key_ring.id + purpose = "ENCRYPT_DECRYPT" + + version_template { + algorithm = "GOOGLE_SYMMETRIC_ENCRYPTION" + } +} + resource "google_vertex_ai_featurestore" "featurestore" { provider = google-beta name = "<%= ctx[:vars]['name'] %>" @@ -9,7 +26,7 @@ resource "google_vertex_ai_featurestore" "featurestore" { fixed_node_count = 2 } encryption_spec { - kms_key_name = "<%= ctx[:vars]['kms_key_name'] %>" + kms_key_name = google_kms_crypto_key.crypto_key.id } } From 87ad2c94a93ecce6093c9aed1f6656c11e0286b0 Mon Sep 17 00:00:00 2001 From: Julius Kelly Date: Tue, 30 Aug 2022 01:01:48 +0000 Subject: [PATCH 3/3] Bootstrapped kms-key-name --- mmv1/products/vertexai/terraform.yaml | 7 +++++++ .../examples/vertex_ai_featurestore.tf.erb | 19 +------------------ .../vertex_ai_featurestore_entitytype.tf.erb | 19 +------------------ 3 files changed, 9 insertions(+), 36 deletions(-) diff --git a/mmv1/products/vertexai/terraform.yaml b/mmv1/products/vertexai/terraform.yaml index 402cdcdb8162..c0b08b0dd35e 100644 --- a/mmv1/products/vertexai/terraform.yaml +++ b/mmv1/products/vertexai/terraform.yaml @@ -40,9 +40,12 @@ overrides: !ruby/object:Overrides::ResourceOverrides vars: name: "terraform" project: "appeng-flex" + kms_key_name: "kms-name" test_env_vars: org_id: :ORG_ID billing_account: :BILLING_ACCT + test_vars_overrides: + kms_key_name: 'BootstrapKMSKeyInLocation(t, "us-central1").CryptoKey.Name' ignore_read_extra: - "force_destroy" properties: @@ -70,9 +73,13 @@ overrides: !ruby/object:Overrides::ResourceOverrides vars: name: "terraform" project: "vertex-ai" + kms_key_name: "kms-name" test_env_vars: org_id: :ORG_ID billing_account: :BILLING_ACCT + test_vars_overrides: + kms_key_name: 'BootstrapKMSKeyInLocation(t, "us-central1").CryptoKey.Name' + properties: etag: !ruby/object:Overrides::Terraform::PropertyOverride ignore_read: true diff --git a/mmv1/templates/terraform/examples/vertex_ai_featurestore.tf.erb b/mmv1/templates/terraform/examples/vertex_ai_featurestore.tf.erb index 7a70ae2ccb6a..0eaefed1e142 100644 --- a/mmv1/templates/terraform/examples/vertex_ai_featurestore.tf.erb +++ b/mmv1/templates/terraform/examples/vertex_ai_featurestore.tf.erb @@ -1,20 +1,3 @@ -resource "google_kms_key_ring" "key_ring" { - provider = google-beta - name = "key-ring" - location = "us-central1" -} - -resource "google_kms_crypto_key" "crypto_key" { - provider = google-beta - name = "crypto-key" - key_ring = google_kms_key_ring.key_ring.id - purpose = "ENCRYPT_DECRYPT" - - version_template { - algorithm = "GOOGLE_SYMMETRIC_ENCRYPTION" - } -} - resource "google_vertex_ai_featurestore" "featurestore" { provider = google-beta name = "<%= ctx[:vars]['name'] %>" @@ -26,7 +9,7 @@ resource "google_vertex_ai_featurestore" "featurestore" { fixed_node_count = 2 } encryption_spec { - kms_key_name = google_kms_crypto_key.crypto_key.id + kms_key_name = "<%= ctx[:vars]['kms_key_name'] %>" } force_destroy = true } diff --git a/mmv1/templates/terraform/examples/vertex_ai_featurestore_entitytype.tf.erb b/mmv1/templates/terraform/examples/vertex_ai_featurestore_entitytype.tf.erb index db367613ac22..c676aab0201c 100644 --- a/mmv1/templates/terraform/examples/vertex_ai_featurestore_entitytype.tf.erb +++ b/mmv1/templates/terraform/examples/vertex_ai_featurestore_entitytype.tf.erb @@ -1,20 +1,3 @@ -resource "google_kms_key_ring" "key_ring" { - provider = google-beta - name = "key-ring" - location = "us-central1" -} - -resource "google_kms_crypto_key" "crypto_key" { - provider = google-beta - name = "crypto-key" - key_ring = google_kms_key_ring.key_ring.id - purpose = "ENCRYPT_DECRYPT" - - version_template { - algorithm = "GOOGLE_SYMMETRIC_ENCRYPTION" - } -} - resource "google_vertex_ai_featurestore" "featurestore" { provider = google-beta name = "<%= ctx[:vars]['name'] %>" @@ -26,7 +9,7 @@ resource "google_vertex_ai_featurestore" "featurestore" { fixed_node_count = 2 } encryption_spec { - kms_key_name = google_kms_crypto_key.crypto_key.id + kms_key_name = "<%= ctx[:vars]['kms_key_name'] %>" } }