Skip to content

Commit

Permalink
Adds support for lastModifiedSince and lastModifiedBefore fields in s…
Browse files Browse the repository at this point in the history 
…torage_transfer_job (#7496) (#5398)

Signed-off-by: Modular Magician <[email protected]>
  • Loading branch information
@modular-magician
modular-magician authored Mar 30, 2023
1 parent 5c8e7ad commit 0443a0b
Show file tree
Hide file tree
Showing 4 changed files with 180 additions and 0 deletions.
4 changes: 4 additions & 0 deletions .changelog/7496.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
```release-note:enhancement

storagetransfer: add support for 'last_modified_since' and 'last_modified_before' fields to 'google_storage_transfer_job' resource
```
20 changes: 20 additions & 0 deletions google-beta/resource_storage_transfer_job.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ var (
"transfer_spec.0.object_conditions.0.max_time_elapsed_since_last_modification",
"transfer_spec.0.object_conditions.0.include_prefixes",
"transfer_spec.0.object_conditions.0.exclude_prefixes",
"transfer_spec.0.object_conditions.0.last_modified_since",
"transfer_spec.0.object_conditions.0.last_modified_before",
}

transferOptionsKeys = []string{
Expand Down Expand Up @@ -293,6 +295,20 @@ func objectConditionsSchema() *schema.Schema {
},
Description: `exclude_prefixes must follow the requirements described for include_prefixes.`,
},
"last_modified_since": {
Type: schema.TypeString,
ValidateFunc: validateRFC3339Date,
Optional: true,
AtLeastOneOf: objectConditionsKeys,
Description: `If specified, only objects with a "last modification time" on or after this timestamp and objects that don't have a "last modification time" are transferred. A timestamp in RFC3339 UTC "Zulu" format, with nanosecond resolution and up to nine fractional digits. Examples: "2014-10-02T15:01:23Z" and "2014-10-02T15:01:23.045123456Z".`,
},
"last_modified_before": {
Type: schema.TypeString,
ValidateFunc: validateRFC3339Date,
Optional: true,
AtLeastOneOf: objectConditionsKeys,
Description: `If specified, only objects with a "last modification time" before this timestamp and objects that don't have a "last modification time" are transferred. A timestamp in RFC3339 UTC "Zulu" format, with nanosecond resolution and up to nine fractional digits. Examples: "2014-10-02T15:01:23Z" and "2014-10-02T15:01:23.045123456Z".`,
},
},
},
Description: `Only objects that satisfy these object conditions are included in the set of data source and data sink objects. Object conditions based on objects' last_modification_time do not exclude objects in a data sink.`,
Expand Down Expand Up @@ -1033,6 +1049,8 @@ func expandObjectConditions(conditions []interface{}) *storagetransfer.ObjectCon
IncludePrefixes: convertStringArr(condition["include_prefixes"].([]interface{})),
MaxTimeElapsedSinceLastModification: condition["max_time_elapsed_since_last_modification"].(string),
MinTimeElapsedSinceLastModification: condition["min_time_elapsed_since_last_modification"].(string),
LastModifiedSince: condition["last_modified_since"].(string),
LastModifiedBefore: condition["last_modified_before"].(string),
}
}

Expand All @@ -1042,6 +1060,8 @@ func flattenObjectCondition(condition *storagetransfer.ObjectConditions) []map[s
"include_prefixes": condition.IncludePrefixes,
"max_time_elapsed_since_last_modification": condition.MaxTimeElapsedSinceLastModification,
"min_time_elapsed_since_last_modification": condition.MinTimeElapsedSinceLastModification,
"last_modified_since": condition.LastModifiedSince,
"last_modified_before": condition.LastModifiedBefore,
}
return []map[string]interface{}{data}
}
Expand Down
152 changes: 152 additions & 0 deletions google-beta/resource_storage_transfer_job_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,39 @@ func TestAccStorageTransferJob_transferOptions(t *testing.T) {
})
}

func TestAccStorageTransferJob_objectConditions(t *testing.T) {
t.Parallel()

testDataSourceBucketName := RandString(t, 10)
testDataSinkName := RandString(t, 10)
testTransferJobDescription := RandString(t, 10)
testPubSubTopicName := fmt.Sprintf("tf-test-topic-%s", RandString(t, 10))

VcrTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
ProtoV5ProviderFactories: ProtoV5ProviderFactories(t),
CheckDestroy: testAccStorageTransferJobDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccStorageTransferJob_basic(GetTestProjectFromEnv(), testDataSourceBucketName, testDataSinkName, testTransferJobDescription, testPubSubTopicName),
},
{
ResourceName: "google_storage_transfer_job.transfer_job",
ImportState: true,
ImportStateVerify: true,
},
{
Config: testAccStorageTransferJob_objectConditions(GetTestProjectFromEnv(), testDataSourceBucketName, testDataSinkName, testTransferJobDescription, testPubSubTopicName),
},
{
ResourceName: "google_storage_transfer_job.transfer_job",
ImportState: true,
ImportStateVerify: true,
},
},
})
}

func TestAccStorageTransferJob_notificationConfig(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -300,6 +333,7 @@ resource "google_storage_bucket" "data_source" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_source" {
Expand All @@ -313,6 +347,7 @@ resource "google_storage_bucket" "data_sink" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_sink" {
Expand Down Expand Up @@ -355,6 +390,7 @@ resource "google_storage_bucket" "data_source" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_source" {
Expand All @@ -368,6 +404,7 @@ resource "google_storage_bucket" "data_sink" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_sink" {
Expand Down Expand Up @@ -430,6 +467,7 @@ resource "google_storage_bucket" "data_source" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_source" {
Expand All @@ -443,6 +481,7 @@ resource "google_storage_bucket" "data_sink" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_sink" {
Expand Down Expand Up @@ -525,6 +564,7 @@ resource "google_storage_bucket" "data_source" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_source" {
Expand All @@ -538,6 +578,7 @@ resource "google_storage_bucket" "data_sink" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_sink" {
Expand Down Expand Up @@ -592,6 +633,7 @@ resource "google_storage_bucket" "data_sink" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_sink" {
Expand Down Expand Up @@ -668,6 +710,7 @@ resource "google_storage_bucket" "data_source" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_source" {
Expand Down Expand Up @@ -743,6 +786,7 @@ resource "google_storage_bucket" "data_source" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_source" {
Expand All @@ -756,6 +800,7 @@ resource "google_storage_bucket" "data_sink" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_sink" {
Expand Down Expand Up @@ -833,6 +878,107 @@ resource "google_storage_transfer_job" "transfer_job" {
`, project, dataSourceBucketName, project, dataSinkBucketName, project, pubSubTopicName, transferJobDescription, project, overwriteObjectsAlreadyExistingInSink, deleteObjectsUniqueInSink, deleteObjectsFromSourceAfterTransfer, overwriteWhenVal)
}

func testAccStorageTransferJob_objectConditions(project string, dataSourceBucketName string, dataSinkBucketName string, transferJobDescription string, pubSubTopicName string) string {
return fmt.Sprintf(`
data "google_storage_transfer_project_service_account" "default" {
project = "%s"
}
resource "google_storage_bucket" "data_source" {
name = "%s"
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_source" {
bucket = google_storage_bucket.data_source.name
role = "roles/storage.admin"
member = "serviceAccount:${data.google_storage_transfer_project_service_account.default.email}"
}
resource "google_storage_bucket" "data_sink" {
name = "%s"
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_sink" {
bucket = google_storage_bucket.data_sink.name
role = "roles/storage.admin"
member = "serviceAccount:${data.google_storage_transfer_project_service_account.default.email}"
}
resource "google_pubsub_topic" "topic" {
name = "%s"
}
resource "google_pubsub_topic_iam_member" "notification_config" {
topic = google_pubsub_topic.topic.id
role = "roles/pubsub.publisher"
member = "serviceAccount:${data.google_storage_transfer_project_service_account.default.email}"
}
resource "google_storage_transfer_job" "transfer_job" {
description = "%s"
project = "%s"
transfer_spec {
gcs_data_source {
bucket_name = google_storage_bucket.data_source.name
path = "foo/bar/"
}
gcs_data_sink {
bucket_name = google_storage_bucket.data_sink.name
path = "foo/bar/"
}
object_conditions {
last_modified_since = "2020-01-01T00:00:00Z"
last_modified_before = "2020-01-01T00:00:00Z"
}
}
schedule {
schedule_start_date {
year = 2018
month = 10
day = 1
}
schedule_end_date {
year = 2019
month = 10
day = 1
}
start_time_of_day {
hours = 0
minutes = 30
seconds = 0
nanos = 0
}
repeat_interval = "604800s"
}
notification_config {
pubsub_topic = google_pubsub_topic.topic.id
event_types = [
"TRANSFER_OPERATION_SUCCESS",
"TRANSFER_OPERATION_FAILED"
]
payload_format = "JSON"
}
depends_on = [
google_storage_bucket_iam_member.data_source,
google_storage_bucket_iam_member.data_sink,
google_pubsub_topic_iam_member.notification_config,
]
}
`, project, dataSourceBucketName, project, dataSinkBucketName, project, pubSubTopicName, transferJobDescription, project)
}

func testAccStorageTransferJob_notificationPayloadFormat(project string, dataSourceBucketName string, dataSinkBucketName string, transferJobDescription string, pubsubTopicName string, notificationPayloadFormat string) string {
return fmt.Sprintf(`
data "google_storage_transfer_project_service_account" "default" {
Expand All @@ -844,6 +990,7 @@ resource "google_storage_bucket" "data_source" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_source" {
Expand All @@ -857,6 +1004,7 @@ resource "google_storage_bucket" "data_sink" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_sink" {
Expand Down Expand Up @@ -939,6 +1087,7 @@ resource "google_storage_bucket" "data_source" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_source" {
Expand All @@ -952,6 +1101,7 @@ resource "google_storage_bucket" "data_sink" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_sink" {
Expand Down Expand Up @@ -1033,6 +1183,7 @@ resource "google_storage_bucket" "data_source" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_source" {
Expand All @@ -1046,6 +1197,7 @@ resource "google_storage_bucket" "data_sink" {
project = "%s"
location = "US"
force_destroy = true
uniform_bucket_level_access = true
}
resource "google_storage_bucket_iam_member" "data_sink" {
Expand Down
4 changes: 4 additions & 0 deletions website/docs/r/storage_transfer_job.html.markdown
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,10 @@ A duration in seconds with up to nine fractional digits, terminated by 's'. Exam

* `exclude_prefixes` - (Optional) `exclude_prefixes` must follow the requirements described for `include_prefixes`. See [Requirements](https://cloud.google.com/storage-transfer/docs/reference/rest/v1/TransferSpec#ObjectConditions).

* `last_modified_since` - (Optional) If specified, only objects with a "last modification time" on or after this timestamp and objects that don't have a "last modification time" are transferred. A timestamp in RFC3339 UTC "Zulu" format, with nanosecond resolution and up to nine fractional digits. Examples: "2014-10-02T15:01:23Z" and "2014-10-02T15:01:23.045123456Z".

* `last_modified_before` - (Optional) If specified, only objects with a "last modification time" before this timestamp and objects that don't have a "last modification time" are transferred. A timestamp in RFC3339 UTC "Zulu" format, with nanosecond resolution and up to nine fractional digits. Examples: "2014-10-02T15:01:23Z" and "2014-10-02T15:01:23.045123456Z".

<a name="nested_transfer_options"></a>The `transfer_options` block supports:

* `overwrite_objects_already_existing_in_sink` - (Optional) Whether overwriting objects that already exist in the sink is allowed.
Expand Down

0 comments on commit 0443a0b

Please sign in to comment.