diff --git a/go.mod b/go.mod index c387ac14..54d5bdf0 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,7 @@ require k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 require ( github.com/DataDog/zstd v1.4.5 + github.com/cenkalti/backoff/v4 v4.2.1 github.com/filecoin-project/go-amt-ipld/v4 v4.1.1-0.20230511024927-ecc426107fac github.com/hibiken/asynq v0.23.0 github.com/hibiken/asynq/x v0.0.0-20220413130846-5c723f597e01 diff --git a/go.sum b/go.sum index d5e8441c..82b4c604 100644 --- a/go.sum +++ b/go.sum @@ -136,6 +136,8 @@ github.com/buger/goterm v1.0.3/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQt github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= diff --git a/storage/sql.go b/storage/sql.go index 9c765280..252042dc 100644 --- a/storage/sql.go +++ b/storage/sql.go @@ -8,6 +8,7 @@ import ( "sort" "strings" + backoff "github.com/cenkalti/backoff/v4" "github.com/go-pg/pg/v10" "github.com/go-pg/pg/v10/orm" "github.com/go-pg/pg/v10/types" @@ -228,8 +229,16 @@ func connect(ctx context.Context, opt *pg.Options) (*pg.DB, error) { // db.AddQueryHook(&pgext.OpenTelemetryHook{}) // Check if connection credentials are valid and PostgreSQL is up and running. - if err := db.Ping(ctx); err != nil { - return nil, fmt.Errorf("ping database: %w", err) + operation := func() error { + if err := db.Ping(ctx); err != nil { + return fmt.Errorf("ping database: %w", err) + } + return nil + } + + retryErr := backoff.Retry(operation, backoff.NewExponentialBackOff()) + if retryErr != nil { + return nil, fmt.Errorf("ping database retry attempt: %w", retryErr) } // Acquire a shared lock on the schema to notify other instances that we are running