fix: use exponential backoff to attempt reconnect to tsdb (#1267)

* fix: use exponential backoff to attempt reconnect to tsdb

go.mod

@@ -63,6 +63,7 @@ require k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
 
 require (
 	github.com/DataDog/zstd v1.4.5
+	github.com/cenkalti/backoff/v4 v4.2.1
 	github.com/filecoin-project/go-amt-ipld/v4 v4.1.1-0.20230511024927-ecc426107fac
 	github.com/hibiken/asynq v0.23.0
 	github.com/hibiken/asynq/x v0.0.0-20220413130846-5c723f597e01

go.sum

@@ -136,6 +136,8 @@ github.com/buger/goterm v1.0.3/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQt
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=

storage/sql.go

@@ -8,6 +8,7 @@ import (
 	"sort"
 	"strings"
 
+	backoff "github.com/cenkalti/backoff/v4"
 	"github.com/go-pg/pg/v10"
 	"github.com/go-pg/pg/v10/orm"
 	"github.com/go-pg/pg/v10/types"
@@ -228,8 +229,16 @@ func connect(ctx context.Context, opt *pg.Options) (*pg.DB, error) {
 	// db.AddQueryHook(&pgext.OpenTelemetryHook{})
 
 	// Check if connection credentials are valid and PostgreSQL is up and running.
-	if err := db.Ping(ctx); err != nil {
-		return nil, fmt.Errorf("ping database: %w", err)
+	operation := func() error {
+		if err := db.Ping(ctx); err != nil {
+			return fmt.Errorf("ping database: %w", err)
+		}
+		return nil
+	}
+
+	retryErr := backoff.Retry(operation, backoff.NewExponentialBackOff())
+	if retryErr != nil {
+		return nil, fmt.Errorf("ping database retry attempt: %w", retryErr)
 	}
 
 	// Acquire a shared lock on the schema to notify other instances that we are running