Add retries on unready subnetworks (#5418) (#3827)

Signed-off-by: Modular Magician <[email protected]>

.changelog/5418.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+provider: added retries for the `resourceNotReady` error returned when attempting to add resources to a recently-modified subnetwork
+```

google-beta/error_retry_predicates.go

@@ -31,6 +31,12 @@ var defaultErrorRetryPredicates = []RetryErrorPredicateFunc{
 	// we had this in our global default error retries.
 	// Keeping it as a default for now.
 	is409OperationInProgressError,
+
+	// GCE Subnetworks are considered unready for a brief period when certain
+	// operations are performed on them, and the scope is likely too broad to
+	// apply a mutex. If we attempt an operation w/ an unready subnetwork, retry
+	// it.
+	isSubnetworkUnreadyError,
 }
 
 /** END GLOBAL ERROR RETRY PREDICATES HERE **/
@@ -97,6 +103,19 @@ func is409OperationInProgressError(err error) (bool, string) {
 	return false, ""
 }
 
+func isSubnetworkUnreadyError(err error) (bool, string) {
+	gerr, ok := err.(*googleapi.Error)
+	if !ok {
+		return false, ""
+	}
+
+	if gerr.Code == 400 && strings.Contains(gerr.Body, "resourceNotReady") && strings.Contains(gerr.Body, "subnetworks") {
+		log.Printf("[DEBUG] Dismissed an error as retryable based on error code 400 and error reason 'resourceNotReady' w/ `subnetwork`: %s", err)
+		return true, "Subnetwork not ready"
+	}
+	return false, ""
+}
+
 // Retry on comon googleapi error codes for retryable errors.
 // TODO(#5609): This may not need to be applied globally - figure out
 // what retryable error codes apply to which API.