Metastore: add auxiliary_versions and metadata_integration args (G…

…oogleCloudPlatform#6280)

Co-authored-by: Shuya Ma <[email protected]>

mmv1/products/metastore/api.yaml

@@ -219,6 +219,30 @@ objects:
                 required: true
                 description: |
                   A Cloud Storage URI that specifies the path to a krb5.conf file. It is of the form gs://{bucket_name}/path/to/krb5.conf, although the file does not need to be named krb5.conf explicitly.
+          - !ruby/object:Api::Type::Map
+            name: 'auxiliaryVersions'
+            min_version: beta
+            description: |
+                  A mapping of Hive metastore version to the auxiliary version configuration.
+                  When specified, a secondary Hive metastore service is created along with the primary service.
+                  All auxiliary versions must be less than the service's primary version.
+                  The key is the auxiliary service name and it must match the regular expression a-z?.
+                  This means that the first character must be a lowercase letter, and all the following characters must be hyphens, lowercase letters, or digits, except the last character, which cannot be a hyphen.
+            key_name: 'key'
+            key_description: |
+              The auxiliary service name
+            value_type: !ruby/object:Api::Type::NestedObject
+              properties:
+                - !ruby/object:Api::Type::String
+                  name: 'version'
+                  required: true
+                  description: |
+                    The Hive metastore version of the auxiliary service. It must be less than the primary Hive metastore service's version.
+                - !ruby/object:Api::Type::KeyValuePairs
+                  name: 'configOverrides'
+                  description: |
+                    A mapping of Hive metastore configuration key-value pairs to apply to the auxiliary Hive metastore (configured in hive-site.xml) in addition to the primary version's overrides.
+                    If keys are present in both the auxiliary version's overrides and the primary version's overrides, the value from the auxiliary version's overrides takes precedence.
       - !ruby/object:Api::Type::Enum
         name: 'databaseType'
         input: true
@@ -241,7 +265,24 @@ objects:
         name: 'uid'
         output: true
         description: |
-          The globally unique resource identifier of the metastore service.          
+          The globally unique resource identifier of the metastore service.
+      - !ruby/object:Api::Type::NestedObject
+        name: 'metadataIntegration'
+        min_version: beta
+        description: |
+          The setting that defines how metastore metadata should be integrated with external services and systems.
+        properties:
+          - !ruby/object:Api::Type::NestedObject
+            name: 'dataCatalogConfig'
+            required: true
+            description: |
+              The integration config for the Data Catalog service.
+            properties:
+              - !ruby/object:Api::Type::Boolean
+                name: 'enabled'
+                required: true
+                description: |
+                  Defines whether the metastore metadata should be synced to Data Catalog. The default value is to disable syncing metastore metadata to Data Catalog.      
   - !ruby/object:Api::Resource
     name: 'Federation'
     min_version: beta

mmv1/products/metastore/terraform.yaml

@@ -19,7 +19,7 @@ overrides: !ruby/object:Overrides::ResourceOverrides
     examples:
       - !ruby/object:Provider::Terraform::Examples
         name: "dataproc_metastore_service_basic"
-        primary_resource_name: "fmt.Sprintf(\"tf-test-metastore-srv%s\", context[\"random_suffix\"])"
+        primary_resource_name: 'fmt.Sprintf("tf-test-metastore-srv%s", context["random_suffix"])'
         primary_resource_id: "default"
         vars:
           metastore_service_name: "metastore-srv"
@@ -45,7 +45,21 @@ overrides: !ruby/object:Overrides::ResourceOverrides
         skip_docs: true
         primary_resource_id: "endpoint"
         vars:
-          metastore_service_name: "metastore-endpoint"          
+          metastore_service_name: "metastore-endpoint"
+      - !ruby/object:Provider::Terraform::Examples
+        name: "dataproc_metastore_service_aux"
+        min_version: beta
+        skip_docs: true
+        primary_resource_id: "aux"
+        vars:
+          metastore_service_name: "metastore-aux"
+      - !ruby/object:Provider::Terraform::Examples
+        name: "dataproc_metastore_service_metadata"
+        min_version: beta
+        skip_docs: true
+        primary_resource_id: "metadata"
+        vars:
+          metastore_service_name: "metastore-metadata"          
     properties:
       network: !ruby/object:Overrides::Terraform::PropertyOverride
         default_from_api: true
@@ -55,7 +69,7 @@ overrides: !ruby/object:Overrides::ResourceOverrides
         default_from_api: true
       hiveMetastoreConfig.configOverrides: !ruby/object:Overrides::Terraform::PropertyOverride
         default_from_api: true
-        diff_suppress_func: 'dataprocMetastoreServiceOverrideSuppress'
+        diff_suppress_func: "dataprocMetastoreServiceOverrideSuppress"
   Federation: !ruby/object:Overrides::Terraform::ResourceOverride
     autogen_async: true
     import_format: ["projects/{{project}}/locations/{{location}}/federations/{{federation_id}}"]
@@ -64,6 +78,6 @@ overrides: !ruby/object:Overrides::ResourceOverrides
         name: "dataproc_metastore_federation_basic"
         primary_resource_id: "default"
         min_version: beta
-        primary_resource_name: "fmt.Sprintf(\"fed%s\", context[\"random_suffix\"])"
+        primary_resource_name: 'fmt.Sprintf("tf-test-metastore-fed%s", context["random_suffix"])'
         vars:
-          metastore_federation_name: "fed"
+          metastore_federation_name: "fed-1"

mmv1/templates/terraform/examples/dataproc_metastore_service_aux.tf.erb

@@ -0,0 +1,14 @@
+resource "google_dataproc_metastore_service" "<%= ctx[:primary_resource_id] %>" {
+  provider   = google-beta
+  service_id = "<%= ctx[:vars]['metastore_service_name'] %>"
+  location   = "us-central1"
+  tier       = "DEVELOPER"
+
+  hive_metastore_config {
+    version = "3.1.2"
+    auxiliary_versions {
+      key     = "aux-test"
+      version = "2.3.6"
+    }
+  }
+}

mmv1/templates/terraform/examples/dataproc_metastore_service_metadata.tf.erb

@@ -0,0 +1,16 @@
+resource "google_dataproc_metastore_service" "<%= ctx[:primary_resource_id] %>" {
+  provider   = google-beta
+  service_id = "<%= ctx[:vars]['metastore_service_name'] %>"
+  location   = "us-central1"
+  tier       = "DEVELOPER"
+
+  metadata_integration {
+    data_catalog_config {
+      enabled = true
+    }
+  }
+
+  hive_metastore_config {
+    version = "3.1.2"
+  }
+}