Generate call stack for sdk error and preserve stack trace

cli/cdsctl/admin.go

@@ -22,6 +22,7 @@ func admin() *cobra.Command {
 				adminPlatformModels,
 				adminPlugins,
 				adminBroadcasts,
+				adminErrors,
 				usr,
 				group,
 				worker,
@@ -36,5 +37,6 @@ func admin() *cobra.Command {
 			adminPlugins,
 			adminPluginsAction,
 			adminBroadcasts,
+			adminErrors,
 		})
 }

cli/cdsctl/admin_errors.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/ovh/cds/cli"
+	"github.com/spf13/cobra"
+)
+
+var (
+	adminErrorsCmd = cli.Command{
+		Name:  "errors",
+		Short: "Manage CDS errors",
+	}
+
+	adminErrors = cli.NewCommand(adminErrorsCmd, nil,
+		[]*cobra.Command{
+			cli.NewCommand(adminErrorsGetCmd, adminErrorsGetFunc, nil),
+		})
+)
+
+var adminErrorsGetCmd = cli.Command{
+	Name:  "get",
+	Short: "Get CDS error",
+	Args: []cli.Arg{
+		{Name: "uuid"},
+	},
+}
+
+func adminErrorsGetFunc(v cli.Values) error {
+	res, err := client.MonErrorsGet(v.GetString("uuid"))
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("Message: %s\n", res.Message)
+	if res.StackTrace != "" {
+		fmt.Printf("Stack trace:\n%s", res.StackTrace)
+	}
+
+	return nil
+}

cli/cobra.go

@@ -16,6 +16,8 @@ import (
 	"github.com/olekukonko/tablewriter"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
+
+	"github.com/ovh/cds/sdk"
 )
 
 // ShellMode will os.Exit if false, display only exit code if true
@@ -27,18 +29,23 @@ func ExitOnError(err error, helpFunc ...func() error) {
 		return
 	}
 
-	if e, ok := err.(*Error); ok {
+	code := 50 // default error code
+
+	switch e := err.(type) {
+	case sdk.Error:
+		fmt.Printf("Error(%s): %s\n", e.UUID, e.Message)
+	case *Error:
+		code = e.Code
 		fmt.Println("Error:", e.Error())
-		for _, f := range helpFunc {
-			f()
-		}
-		OSExit(e.Code)
+	default:
+		fmt.Println("Error:", err.Error())
 	}
-	fmt.Println("Error:", err.Error())
+
 	for _, f := range helpFunc {
 		f()
 	}
-	OSExit(50)
+
+	OSExit(code)
 }
 
 // OSExit will os.Exit if ShellMode is false, display only exit code if true

engine/api/api.go

@@ -170,6 +170,10 @@ type Configuration struct {
 	} `toml:"status" comment:"###########################\n CDS Status Settings \n Documentation: https://ovh.github.io/cds/hosting/monitoring/ \n##########################" json:"status"`
 	DefaultOS   string `toml:"defaultOS" default:"linux" comment:"if no model and os/arch is specified in your job's requirements then spawn worker on this operating system (example: freebsd, linux, windows)" json:"defaultOS"`
 	DefaultArch string `toml:"defaultArch" default:"amd64" comment:"if no model and no os/arch is specified in your job's requirements then spawn worker on this architecture (example: amd64, arm, 386)" json:"defaultArch"`
+	Graylog     struct {
+		AccessToken string `toml:"accessToken" json:"-"`
+		URL         string `toml:"url" comment:"Example: http://localhost:9000" json:"url"`
+	} `toml:"graylog"  json:"graylog"`
 }
 
 // ProviderConfiguration is the piece of configuration for each provider authentication

engine/api/api_routes.go

@@ -101,6 +101,7 @@ func (api *API) InitRouter() {
 	r.Handle("/mon/building/{hash}", r.GET(api.getPipelineBuildingCommitHandler))
 	r.Handle("/mon/metrics", r.GET(api.getMetricsHandler, Auth(false)))
 	r.Handle("/mon/stats", r.GET(observability.StatsHandler, Auth(false)))
+	r.Handle("/mon/errors/{uuid}", r.GET(api.getErrorHandler, NeedAdmin(true)))
 
 	r.Handle("/ui/navbar", r.GET(api.getNavbarHandler))
 	r.Handle("/ui/project/{key}/application/{permApplicationName}/overview", r.GET(api.getApplicationOverviewHandler))

engine/api/application_import.go

@@ -71,9 +71,9 @@ func (api *API) postApplicationImportHandler() service.Handler {
 			myError, ok := globalError.(sdk.Error)
 			if ok {
 				log.Warning("postApplicationImportHandler> Unable to import application %s : %v", eapp.Name, myError)
-				msgTranslated, _ := sdk.ProcessError(myError, r.Header.Get("Accept-Language"))
-				msgListString = append(msgListString, msgTranslated)
-				return service.WriteJSON(w, msgListString, myError.Status)
+				sdkErr := sdk.ExtractHTTPError(myError, r.Header.Get("Accept-Language"))
+				msgListString = append(msgListString, sdkErr.Message)
+				return service.WriteJSON(w, msgListString, sdkErr.Status)
 			}
 			return sdk.WrapError(globalError, "postApplicationImportHandler> Unable import application %s", eapp.Name)
 		}

engine/api/ascode_test.go

@@ -37,10 +37,9 @@ func writeError(w *http.Response, err error) (*http.Response, error) {
 	body := new(bytes.Buffer)
 	enc := json.NewEncoder(body)
 	w.Body = ioutil.NopCloser(body)
-	msg, errProcessed := sdk.ProcessError(err, "")
-	sdkErr := sdk.Error{Message: msg}
+	sdkErr := sdk.ExtractHTTPError(err, "")
 	enc.Encode(sdkErr)
-	w.StatusCode = errProcessed.Status
+	w.StatusCode = sdkErr.Status
 	return w, sdkErr
 }
 

engine/api/error.go

@@ -1 +1,76 @@
 package api
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"time"
+
+	"github.com/gorilla/mux"
+
+	"github.com/ovh/cds/sdk"
+	"github.com/ovh/cds/engine/service"
+)
+
+type graylogResponse struct {
+	TotalResult int `json:"total_results"`
+	Messages    []struct {
+		Message map[string]interface{} `json:"message"`
+	} `json:"messages"`
+}
+
+func (api *API) getErrorHandler() service.Handler {
+	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+		vars := mux.Vars(r)
+		uuid := vars["uuid"]
+
+		if api.Config.Graylog.URL == "" || api.Config.Graylog.AccessToken == "" {
+			return sdk.ErrNotImplemented
+		}
+
+		req, err := http.NewRequest("GET", fmt.Sprintf("%s/api/search/universal/absolute", api.Config.Graylog.URL), nil)
+		if err != nil {
+			return sdk.WrapError(err, "invalid given Graylog url")
+		}
+
+		q := req.URL.Query()
+		q.Add("query", fmt.Sprintf("error_uuid:%s", uuid))
+		q.Add("from", "1970-01-01 00:00:00.000")
+		q.Add("to", time.Now().Format("2006-01-02 15:04:05"))
+		q.Add("limit", "1")
+		req.URL.RawQuery = q.Encode()
+
+		req.SetBasicAuth(api.Config.Graylog.AccessToken, "token")
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return sdk.WrapError(err, "cannot send query to Graylog")
+		}
+
+		defer resp.Body.Close()
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return sdk.WrapError(err, "cannot read response from Graylog")
+		}
+
+		var res graylogResponse
+		if err := json.Unmarshal(body, &res); err != nil {
+			return sdk.WrapError(err, "cannot unmarshal response from Graylog")
+		}
+
+		if res.TotalResult < 1 {
+			return sdk.ErrNotFound
+		}
+
+		e := sdk.Error{
+			UUID:    res.Messages[0].Message["error_uuid"].(string),
+			Message: res.Messages[0].Message["message"].(string),
+		}
+		if st, ok := res.Messages[0].Message["stack_trace"]; ok {
+			e.StackTrace = st.(string)
+		}
+
+		return service.WriteJSON(w, e, http.StatusOK)
+	}
+}

engine/api/services/dao.go

@@ -69,7 +69,7 @@ func All(db gorp.SqlExecutor) ([]sdk.Service, error) {
 		if err == sdk.ErrNotFound {
 			return nil, nil
 		}
-		return nil, sdk.WrapError(err, "All> Unable to find all services")
+		return nil, sdk.WrapError(err, "Unable to find all services")
 	}
 	return services, nil
 }
@@ -95,13 +95,13 @@ func findAll(db gorp.SqlExecutor, query string, args ...interface{}) ([]sdk.Serv
 		if err == sql.ErrNoRows {
 			return nil, sdk.ErrNotFound
 		}
-		return nil, sdk.WrapError(err, "findAll> no service found")
+		return nil, sdk.WithStack(err)
 	}
 	ss := make([]sdk.Service, len(sdbs))
 	for i := 0; i < len(sdbs); i++ {
 		s := &sdbs[i]
 		if err := s.PostGet(db); err != nil {
-			return nil, sdk.WrapError(err, "findAll> postGet on srv id:%d name:%s type:%s lastHeartbeat:%v", s.ID, s.Name, s.Type, s.LastHeartbeat)
+			return nil, sdk.WrapError(err, "postGet on srv id:%d name:%s type:%s lastHeartbeat:%v", s.ID, s.Name, s.Type, s.LastHeartbeat)
 		}
 		ss[i] = sdk.Service(sdbs[i])
 	}

engine/api/status.go

@@ -61,7 +61,7 @@ func (api *API) statusHandler() service.Handler {
 
 		srvs, err := services.All(api.mustDB())
 		if err != nil {
-			return sdk.WrapError(err, "statusHandler> error on q.All()")
+			return err
 		}
 
 		mStatus := api.computeGlobalStatus(srvs)

engine/api/workflow/dao.go

@@ -1043,8 +1043,7 @@ func Push(ctx context.Context, db *gorp.DbMap, store cache.Store, proj *sdk.Proj
 	wf, msgList, err := ParseAndImport(ctx, tx, store, proj, &wrkflw, u, ImportOptions{DryRun: dryRun, Force: true})
 	if err != nil {
 		log.Error("Push> Unable to import workflow: %v", err)
-		err = sdk.SetError(err, "unable to import workflow %s", wrkflw.Name)
-		return nil, nil, sdk.WrapError(err, "Push> %v ", err)
+		return nil, nil, sdk.WrapError(err, "Push> unable to import workflow %s", wrkflw.Name)
 	}
 
 	// TODO workflow as code, manage derivation workflow

engine/service/http.go

@@ -8,6 +8,9 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/pborman/uuid"
+	"github.com/sirupsen/logrus"
+
 	"github.com/ovh/cds/sdk"
 	"github.com/ovh/cds/sdk/cdsclient"
 	"github.com/ovh/cds/sdk/log"
@@ -88,17 +91,27 @@ func WriteProcessTime(w http.ResponseWriter) {
 // WriteError is a helper function to return error in a language the called understand
 func WriteError(w http.ResponseWriter, r *http.Request, err error) {
 	al := r.Header.Get("Accept-Language")
-	msg, errProcessed := sdk.ProcessError(err, al)
-	sdkErr := sdk.Error{Message: msg}
+	httpErr := sdk.ExtractHTTPError(err, al)
+	httpErr.UUID = uuid.NewUUID().String()
+	isErrWithStack := sdk.IsErrorWithStack(err)
+
+	entry := logrus.WithField("method", r.Method).
+		WithField("request_uri", r.RequestURI).
+		WithField("status", httpErr.Status).
+		WithField("error_uuid", httpErr.UUID)
+	if isErrWithStack {
+		entry = entry.WithField("stack_trace", fmt.Sprintf("%+v", err))
+	}
 
 	// ErrAlreadyTaken and ErrWorkerModelAlreadyBooked are not useful to log in warning
-	if sdk.ErrorIs(errProcessed, sdk.ErrAlreadyTaken) ||
-		sdk.ErrorIs(errProcessed, sdk.ErrWorkerModelAlreadyBooked) ||
-		sdk.ErrorIs(errProcessed, sdk.ErrJobAlreadyBooked) || r.URL.Path == "/user/me" {
-		log.Debug("%-7s | %-4d | %s \t %s", r.Method, errProcessed.Status, r.RequestURI, err)
+	if sdk.ErrorIs(httpErr, sdk.ErrAlreadyTaken) ||
+		sdk.ErrorIs(httpErr, sdk.ErrWorkerModelAlreadyBooked) ||
+		sdk.ErrorIs(httpErr, sdk.ErrJobAlreadyBooked) || r.URL.Path == "/user/me" {
+		entry.Debugf("%s", err)
 	} else {
-		log.Warning("%-7s | %-4d | %s \t %s", r.Method, errProcessed.Status, r.RequestURI, err)
+		entry.Warningf("%s", err)
 	}
 
-	_ = WriteJSON(w, sdkErr, errProcessed.Status)
+	// safely ignore error returned by WriteJSON
+	_ = WriteJSON(w, httpErr, httpErr.Status)
 }

engine/worker/cmd_cache.go

@@ -378,22 +378,22 @@ func (wk *currentWorker) cachePullHandler(w http.ResponseWriter, r *http.Request
 		case tar.TypeReg, tar.TypeLink:
 			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
 			if err != nil {
-				err = sdk.Error{
+				sdkErr := sdk.Error{
 					Message: "worker cache pull > Unable to open file : " + err.Error(),
 					Status:  http.StatusInternalServerError,
 				}
-				writeJSON(w, err, http.StatusInternalServerError)
+				writeJSON(w, sdkErr, sdkErr.Status)
 				return
 			}
 
 			// copy over contents
 			if _, err := io.Copy(f, tr); err != nil {
 				_ = f.Close()
-				err = sdk.Error{
+				sdkErr := sdk.Error{
 					Message: "worker cache pull > Cannot copy content file : " + err.Error(),
 					Status:  http.StatusInternalServerError,
 				}
-				writeJSON(w, err, http.StatusInternalServerError)
+				writeJSON(w, sdkErr, sdkErr.Status)
 				return
 			}
 

engine/worker/cmd_server.go

@@ -86,7 +86,6 @@ func writeJSON(w http.ResponseWriter, data interface{}, status int) {
 
 func writeError(w http.ResponseWriter, r *http.Request, err error) {
 	al := r.Header.Get("Accept-Language")
-	msg, sdkError := sdk.ProcessError(err, al)
-	sdkErr := sdk.Error{Message: msg}
-	writeJSON(w, sdkErr, sdkError.Status)
+	sdkErr := sdk.ExtractHTTPError(err, al)
+	writeJSON(w, sdkErr, sdkErr.Status)
 }

sdk/cdsclient/client_mon.go

@@ -1,6 +1,9 @@
 package cdsclient
 
 import (
+	"encoding/json"
+	"fmt"
+
 	"github.com/ovh/cds/sdk"
 )
 
@@ -27,3 +30,17 @@ func (c *client) MonDBMigrate() ([]sdk.MonDBMigrate, error) {
 	}
 	return monDBMigrate, nil
 }
+
+func (c *client) MonErrorsGet(uuid string) (*sdk.Error, error) {
+	res, _, _, err := c.Request("GET", fmt.Sprintf("/mon/errors/%s", uuid), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	var sdkError sdk.Error
+	if err := json.Unmarshal(res, &sdkError); err != nil {
+		return nil, err
+	}
+
+	return &sdkError, nil
+}