Add data source for retrieving multiple GCS buckets from a project (#…

…10444) (#7291)

[upstream:0afb6b573d7554b6b232f7738de08945d5fc1f9b]

Signed-off-by: Modular Magician <[email protected]>

google-beta/provider/provider_mmv1_resources.go

@@ -301,6 +301,7 @@ var handwrittenDatasources = map[string]*schema.Resource{
 	"google_sql_database_instances":                       sql.DataSourceSqlDatabaseInstances(),
 	"google_service_networking_peered_dns_domain":         servicenetworking.DataSourceGoogleServiceNetworkingPeeredDNSDomain(),
 	"google_storage_bucket":                               storage.DataSourceGoogleStorageBucket(),
+	"google_storage_buckets":                              storage.DataSourceGoogleStorageBuckets(),
 	"google_storage_bucket_object":                        storage.DataSourceGoogleStorageBucketObject(),
 	"google_storage_bucket_objects":                       storage.DataSourceGoogleStorageBucketObjects(),
 	"google_storage_bucket_object_content":                storage.DataSourceGoogleStorageBucketObjectContent(),

google-beta/services/storage/data_source_google_storage_buckets.go

@@ -0,0 +1,153 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+package storage
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+	"github.com/hashicorp/terraform-provider-google-beta/google-beta/tpgresource"
+	transport_tpg "github.com/hashicorp/terraform-provider-google-beta/google-beta/transport"
+)
+
+func DataSourceGoogleStorageBuckets() *schema.Resource {
+	return &schema.Resource{
+		Read: datasourceGoogleStorageBucketsRead,
+		Schema: map[string]*schema.Schema{
+			"prefix": {
+				Type:     schema.TypeString,
+				Optional: true,
+			},
+			"project": {
+				Type:     schema.TypeString,
+				Optional: true,
+			},
+			"buckets": {
+				Type:     schema.TypeList,
+				Computed: true,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"labels": {
+							Type:     schema.TypeMap,
+							Computed: true,
+							Elem:     &schema.Schema{Type: schema.TypeString},
+						},
+						"location": {
+							Type:     schema.TypeString,
+							Computed: true,
+						},
+						"name": {
+							Type:     schema.TypeString,
+							Computed: true,
+						},
+						"self_link": {
+							Type:     schema.TypeString,
+							Computed: true,
+						},
+						"storage_class": {
+							Type:     schema.TypeString,
+							Computed: true,
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func datasourceGoogleStorageBucketsRead(d *schema.ResourceData, meta interface{}) error {
+	config := meta.(*transport_tpg.Config)
+	userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent)
+	if err != nil {
+		return err
+	}
+
+	params := make(map[string]string)
+	buckets := make([]map[string]interface{}, 0)
+
+	for {
+		url := "https://storage.googleapis.com/storage/v1/b"
+
+		params["project"], err = tpgresource.GetProject(d, config)
+		if err != nil {
+			return fmt.Errorf("Error fetching project for bucket: %s", err)
+		}
+
+		if v, ok := d.GetOk("prefix"); ok {
+			params["prefix"] = v.(string)
+		}
+
+		url, err = transport_tpg.AddQueryParams(url, params)
+		if err != nil {
+			return err
+		}
+
+		res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{
+			Config:    config,
+			Method:    "GET",
+			RawURL:    url,
+			UserAgent: userAgent,
+		})
+		if err != nil {
+			return fmt.Errorf("Error retrieving buckets: %s", err)
+		}
+
+		pageBuckets := flattenDatasourceGoogleBucketsList(res["items"])
+		buckets = append(buckets, pageBuckets...)
+
+		pToken, ok := res["nextPageToken"]
+		if ok && pToken != nil && pToken.(string) != "" {
+			params["pageToken"] = pToken.(string)
+		} else {
+			break
+		}
+	}
+
+	if err := d.Set("buckets", buckets); err != nil {
+		return fmt.Errorf("Error retrieving buckets: %s", err)
+	}
+
+	d.SetId(params["project"])
+
+	return nil
+}
+
+func flattenDatasourceGoogleBucketsList(v interface{}) []map[string]interface{} {
+	if v == nil {
+		return make([]map[string]interface{}, 0)
+	}
+
+	ls := v.([]interface{})
+	buckets := make([]map[string]interface{}, 0, len(ls))
+	for _, raw := range ls {
+		o := raw.(map[string]interface{})
+
+		var mLabels, mLocation, mName, mSelfLink, mStorageClass interface{}
+		if oLabels, ok := o["labels"]; ok {
+			mLabels = oLabels
+		}
+		if oLocation, ok := o["location"]; ok {
+			mLocation = oLocation
+		}
+		if oName, ok := o["name"]; ok {
+			mName = oName
+		}
+		if oSelfLink, ok := o["selfLink"]; ok {
+			mSelfLink = oSelfLink
+		}
+		if oStorageClass, ok := o["storageClass"]; ok {
+			mStorageClass = oStorageClass
+		}
+		buckets = append(buckets, map[string]interface{}{
+			"labels":        mLabels,
+			"location":      mLocation,
+			"name":          mName,
+			"self_link":     mSelfLink,
+			"storage_class": mStorageClass,
+		})
+	}
+
+	return buckets
+}

google-beta/services/storage/data_source_google_storage_buckets_test.go

@@ -0,0 +1,125 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+package storage_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
+	"github.com/hashicorp/terraform-provider-google-beta/google-beta/acctest"
+	"github.com/hashicorp/terraform-provider-google-beta/google-beta/envvar"
+)
+
+func TestAccDataSourceGoogleStorageBuckets_basic(t *testing.T) {
+	t.Parallel()
+
+	static_prefix := "tf-bucket-test"
+	random_suffix := acctest.RandString(t, 10)
+
+	context := map[string]interface{}{
+		"billing_account": envvar.GetTestBillingAccountFromEnv(t),
+		"bucket1":         static_prefix + "-1-" + random_suffix,
+		"bucket2":         static_prefix + "-2-" + random_suffix,
+		"project_id":      static_prefix + "-" + random_suffix,
+		"organization":    envvar.GetTestOrgFromEnv(t),
+	}
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccCheckGoogleStorageBucketsConfig(context),
+				Check: resource.ComposeTestCheckFunc(
+					// Test schema
+					resource.TestCheckResourceAttrSet("data.google_storage_buckets.all", "buckets.0.location"),
+					resource.TestCheckResourceAttrSet("data.google_storage_buckets.all", "buckets.0.name"),
+					resource.TestCheckResourceAttrSet("data.google_storage_buckets.all", "buckets.0.self_link"),
+					resource.TestCheckResourceAttrSet("data.google_storage_buckets.all", "buckets.0.storage_class"),
+					resource.TestCheckResourceAttrSet("data.google_storage_buckets.all", "buckets.1.location"),
+					resource.TestCheckResourceAttrSet("data.google_storage_buckets.all", "buckets.1.name"),
+					resource.TestCheckResourceAttrSet("data.google_storage_buckets.all", "buckets.1.self_link"),
+					resource.TestCheckResourceAttrSet("data.google_storage_buckets.all", "buckets.1.storage_class"),
+					// Test content
+					resource.TestCheckResourceAttr("data.google_storage_buckets.all", "project", context["project_id"].(string)),
+					resource.TestCheckResourceAttr("data.google_storage_buckets.all", "buckets.0.name", context["bucket1"].(string)),
+					resource.TestCheckResourceAttr("data.google_storage_buckets.all", "buckets.1.name", context["bucket2"].(string)),
+					// Test with project
+					resource.TestCheckResourceAttr("data.google_storage_buckets.one", "buckets.0.name", context["bucket1"].(string)),
+					// Test prefix
+					resource.TestCheckResourceAttr("data.google_storage_buckets.two", "buckets.0.name", context["bucket2"].(string)),
+				),
+			},
+		},
+	})
+}
+
+func testAccCheckGoogleStorageBucketsConfig(context map[string]interface{}) string {
+	return fmt.Sprintf(`
+locals {
+  billing_account = "%s"
+  bucket_one      = "%s"
+  bucket_two      = "%s"
+  organization    = "%s"
+  project_id      = "%s"
+}
+
+resource "google_project" "acceptance" {
+  name            = local.project_id
+  project_id      = local.project_id
+  org_id          = local.organization
+  billing_account = local.billing_account
+}
+
+resource "google_storage_bucket" "one" {
+  force_destroy               = true
+  location                    = "EU"
+  name                        = local.bucket_one
+  project                     = google_project.acceptance.project_id
+  uniform_bucket_level_access = true
+}
+
+resource "google_storage_bucket" "two" {
+  force_destroy               = true
+  location                    = "EU"
+  name                        = local.bucket_two
+  project                     = google_project.acceptance.project_id
+  uniform_bucket_level_access = true
+}
+
+data "google_storage_buckets" "all" {
+  project = google_project.acceptance.project_id
+  
+  depends_on = [
+    google_storage_bucket.one,
+    google_storage_bucket.two,
+  ]
+}
+
+data "google_storage_buckets" "one" {
+  prefix  = "tf-bucket-test-1"
+  project = google_project.acceptance.project_id
+
+  depends_on = [
+    google_storage_bucket.one,
+  ]
+}
+
+data "google_storage_buckets" "two" {
+  prefix  = "tf-bucket-test-2"
+  project = google_project.acceptance.project_id
+
+  depends_on = [
+    google_storage_bucket.two,
+  ]
+}`,
+		context["billing_account"].(string),
+		context["bucket1"].(string),
+		context["bucket2"].(string),
+		context["organization"].(string),
+		context["project_id"].(string),
+	)
+}

website/docs/d/storage_buckets.html.markdown

@@ -0,0 +1,44 @@
+---
+subcategory: "Cloud Storage"
+description: |-
+  Retrieve information about a set of GCS buckets in a project.
+---
+
+
+# google\_storage\_buckets
+
+Gets a list of existing GCS buckets.
+See [the official documentation](https://cloud.google.com/storage/docs/introduction)
+and [API](https://cloud.google.com/storage/docs/json_api/v1/buckets/list).
+
+## Example Usage
+
+Example GCS buckets.
+
+```hcl
+data "google_storage_buckets" "example" {
+  project = "example-project"
+}
+```
+
+## Argument Reference
+
+The following arguments are supported:
+
+* `prefix` - (Optional) Filter results to buckets whose names begin with this prefix.
+* `project` - (Optional) The ID of the project. If it is not provided, the provider project is used.
+
+
+## Attributes Reference
+
+The following attributes are exported:
+
+* `buckets` - A list of all retrieved GCS buckets. Structure is [defined below](#nested_buckets).
+
+<a name="nested_buckets"></a>The `buckets` block supports:
+
+* `labels` - User-provided bucket labels, in key/value pairs.
+* `location` - The location of the bucket. 
+* `name` - The name of the bucket.
+* `self_link` - A url reference to the bucket.
+* `storage_class` - The [StorageClass](https://cloud.google.com/storage/docs/storage-classes) of the bucket.