From 2eb6b3bf80b89653c79e01766972c791a003ce14 Mon Sep 17 00:00:00 2001 From: Alexander Matyushentsev Date: Fri, 4 Jun 2021 11:57:04 -0700 Subject: [PATCH] fix: add client side retry to prevent 'transport is closing' errors (#6402) Signed-off-by: Alexander Matyushentsev --- pkg/apiclient/apiclient.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/apiclient/apiclient.go b/pkg/apiclient/apiclient.go index 656c322745a79..112170ff7d888 100644 --- a/pkg/apiclient/apiclient.go +++ b/pkg/apiclient/apiclient.go @@ -19,6 +19,8 @@ import ( "github.com/coreos/go-oidc" "github.com/dgrijalva/jwt-go/v4" "github.com/golang/protobuf/ptypes/empty" + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry" log "github.com/sirupsen/logrus" "golang.org/x/oauth2" "google.golang.org/grpc" @@ -471,9 +473,15 @@ func (c *client) newConn() (*grpc.ClientConn, io.Closer, error) { endpointCredentials := jwtCredentials{ Token: c.AuthToken, } + retryOpts := []grpc_retry.CallOption{ + grpc_retry.WithMax(3), + grpc_retry.WithBackoff(grpc_retry.BackoffLinear(1000 * time.Millisecond)), + } var dialOpts []grpc.DialOption dialOpts = append(dialOpts, grpc.WithPerRPCCredentials(endpointCredentials)) dialOpts = append(dialOpts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(MaxGRPCMessageSize), grpc.MaxCallSendMsgSize(MaxGRPCMessageSize))) + dialOpts = append(dialOpts, grpc.WithStreamInterceptor(grpc_retry.StreamClientInterceptor(retryOpts...))) + dialOpts = append(dialOpts, grpc.WithUnaryInterceptor(grpc_middleware.ChainUnaryClient(grpc_retry.UnaryClientInterceptor(retryOpts...)))) ctx := context.Background()