fix(api): call services retries (#5232)

Signed-off-by: francois samin <[email protected]>
ovh · Jun 5, 2020 · 4678ade · 4678ade
1 parent 4db00e9
commit 4678ade
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 26 deletions.
diff --git a/engine/api/services/http.go b/engine/api/services/http.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -128,25 +129,23 @@ func (s *defaultServiceClient) DoJSONRequest(ctx context.Context, method, path s
 
 // doJSONRequest performs an http request on a service
 func doJSONRequest(ctx context.Context, db gorp.SqlExecutor, srvs []sdk.Service, method, path string, in interface{}, out interface{}, mods ...cdsclient.RequestModifier) (http.Header, int, error) {
-	var lastErr error
+	var lastErr = sdk.WithStack(errors.New("unable to call service: service not found"))
 	var lastCode int
-	var attempt int
-	for {
-		attempt++
+	for attempt := 0; attempt < 5; attempt++ {
 		for i := range srvs {
 			srv := &srvs[i]
 			headers, code, err := _doJSONRequest(ctx, db, srv, method, path, in, out, mods...)
-			if err == nil {
-				return headers, code, nil
+			if err != nil {
+				lastErr = err
+				lastCode = code
+				continue
 			}
-			lastErr = err
-			lastCode = code
-		}
-		if lastErr != nil || attempt > 5 {
-			break
+			return headers, code, nil
 		}
 	}
-	return nil, lastCode, lastErr
+
+	log.Error(ctx, "unable to call service: maximum attempt exceed : %+v", lastErr)
+	return nil, lastCode, sdk.WithStack(lastErr)
 }
 
 // _doJSONRequest is a low level function that performs an http request on service
@@ -182,11 +181,13 @@ func PostMultipart(ctx context.Context, db gorp.SqlExecutor, srvs []sdk.Service,
 	writer := multipart.NewWriter(body)
 	part, err := writer.CreateFormFile("file", filename)
 	if err != nil {
-		return 0, err
+		return 0, sdk.WithStack(err)
+	}
+	if _, err := part.Write(fileContents); err != nil {
+		return 0, sdk.WithStack(err)
 	}
-	part.Write(fileContents)
 	if err := writer.Close(); err != nil {
-		return 0, err
+		return 0, sdk.WithStack(err)
 	}
 
 	mods = append(mods, cdsclient.SetHeader("Content-Type", "multipart/form-data"))

diff --git a/engine/repositories/repositories.go b/engine/repositories/repositories.go
@@ -81,16 +81,19 @@ func (s *Service) Serve(c context.Context) error {
 	defer cancel()
 
 	//Init the cache
+	log.Info(ctx, "Initializing Redis connection (%s)...", s.Cfg.Cache.Redis.Host)
 	var errCache error
 	s.Cache, errCache = cache.New(s.Cfg.Cache.Redis.Host, s.Cfg.Cache.Redis.Password, s.Cfg.Cache.TTL)
 	if errCache != nil {
-		return fmt.Errorf("Cannot connect to redis instance : %v", errCache)
+		return fmt.Errorf("cannot connect to redis instance : %v", errCache)
 	}
 
+	var address = fmt.Sprintf("%s:%d", s.Cfg.HTTP.Addr, s.Cfg.HTTP.Port)
+	log.Info(ctx, "Initializing HTTP router (%s)...", address)
 	//Init the http server
 	s.initRouter(ctx)
 	server := &http.Server{
-		Addr:           fmt.Sprintf("%s:%d", s.Cfg.HTTP.Addr, s.Cfg.HTTP.Port),
+		Addr:           address,
 		Handler:        s.Router.Mux,
 		ReadTimeout:    10 * time.Minute,
 		WriteTimeout:   10 * time.Minute,
@@ -102,31 +105,31 @@ func (s *Service) Serve(c context.Context) error {
 		store: s.Cache,
 	}
 
+	log.Info(ctx, "Initializing processor...")
 	go func() {
 		if err := s.processor(ctx); err != nil {
-			log.Info(ctx, "Repositories> Shutdown processor")
+			log.Info(ctx, "Shutdown processor")
 		}
 	}()
 
+	log.Info(ctx, "Initializing vacuumCleaner...")
 	go func() {
 		if err := s.vacuumCleaner(ctx); err != nil {
-			log.Info(ctx, "Repositories> Shutdown vacuumCleaner")
+			log.Info(ctx, "Shutdown vacuumCleaner")
 		}
 	}()
 
 	//Gracefully shutdown the http server
 	go func() {
-		select {
-		case <-ctx.Done():
-			log.Info(ctx, "Repositories> Shutdown HTTP Server")
-			server.Shutdown(ctx)
-		}
+		<-ctx.Done()
+		log.Info(ctx, "Shutdown HTTP Server")
+		_ = server.Shutdown(ctx)
 	}()
 
 	//Start the http server
-	log.Info(ctx, "Repositories> Starting HTTP Server on port %d", s.Cfg.HTTP.Port)
+	log.Info(ctx, "Starting HTTP Server on port %d", s.Cfg.HTTP.Port)
 	if err := server.ListenAndServe(); err != nil {
-		log.Error(ctx, "Repositories> Listen and serve failed: %s", err)
+		log.Error(ctx, "Listen and serve failed: %s", err)
 	}
 
 	return ctx.Err()