feat(service): add ip address in router logs (#5789)

engine/api/api.go

@@ -64,10 +64,7 @@ type Configuration struct {
 		API string `toml:"api" default:"http://localhost:8081" json:"api"`
 		UI  string `toml:"ui" default:"http://localhost:8080" json:"ui"`
 	} `toml:"url" comment:"#####################\n CDS URLs Settings \n####################" json:"url"`
-	HTTP struct {
-		Addr string `toml:"addr" default:"" commented:"true" comment:"Listen HTTP address without port, example: 127.0.0.1" json:"addr"`
-		Port int    `toml:"port" default:"8081" json:"port"`
-	} `toml:"http" json:"http"`
+	HTTP    service.HTTPRouterConfiguration `toml:"http" json:"http"`
 	Secrets struct {
 		Key string `toml:"key" json:"-"`
 	} `toml:"secrets" json:"secrets"`
@@ -552,6 +549,7 @@ func (a *API) Serve(ctx context.Context) error {
 	a.Router = &Router{
 		Mux:        mux.NewRouter(),
 		Background: ctx,
+		Config:     a.Config.HTTP,
 	}
 	a.InitRouter()
 	if err := a.initWebsocket(event.DefaultPubSubKey); err != nil {

engine/api/api_routes.go

@@ -443,7 +443,7 @@ func (api *API) InitRouter() {
 	r.Handle("/template/{groupName}/{templateSlug}/usage", Scope(sdk.AuthConsumerScopeTemplate), r.GET(api.getTemplateUsageHandler))
 
 	//Not Found handler
-	r.Mux.NotFoundHandler = http.HandlerFunc(NotFoundHandler)
+	r.Mux.NotFoundHandler = http.HandlerFunc(r.NotFoundHandler)
 
 	r.computeScopeDetails()
 }

engine/api/config.go

@@ -33,14 +33,18 @@ func (api *API) ConfigVCShandler() service.Handler {
 
 func (api *API) ConfigCDNHandler() service.Handler {
 	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-		tcpURL, err := services.GetCDNPublicTCPAdress(ctx, api.mustDB())
+		tcpURL, tcpURLEnableTLS, err := services.GetCDNPublicTCPAdress(ctx, api.mustDB())
 		if err != nil {
 			return err
 		}
 		httpURL, err := services.GetCDNPublicHTTPAdress(ctx, api.mustDB())
 		if err != nil {
 			return err
 		}
-		return service.WriteJSON(w, sdk.CDNConfig{TCPURL: tcpURL, HTTPURL: httpURL}, http.StatusOK)
+		return service.WriteJSON(w,
+			sdk.CDNConfig{TCPURL: tcpURL,
+				TCPURLEnableTLS: tcpURLEnableTLS,
+				HTTPURL:         httpURL},
+			http.StatusOK)
 	}
 }

engine/api/router.go

@@ -59,6 +59,7 @@ type Router struct {
 	nbPanic               int
 	lastPanic             *time.Time
 	scopeDetails          []sdk.AuthConsumerScopeDetail
+	Config                service.HTTPRouterConfiguration
 }
 
 // HandlerConfigFunc is a type used in the router configuration fonction "Handle"
@@ -224,7 +225,6 @@ func (r *Router) computeScopeDetails() {
 				Methods: methods,
 			})
 		}
-
 		details[i].Scope = scope
 		details[i].Endpoints = endpoints
 	}
@@ -340,13 +340,21 @@ func (r *Router) handle(uri string, scope HandlerScope, handlers ...*service.Han
 			telemetry.Path, req.URL.Path,
 			telemetry.Method, req.Method)
 
+		// Retrieve the client ip address from the header (X-Forwarded-For by default)
+		clientIP := req.Header.Get(r.Config.HeaderXForwardedFor)
+		if clientIP == "" {
+			// If the header has not been found, fallback on the remote adress from the http request
+			clientIP = req.RemoteAddr
+		}
+
 		// Prepare logging fields
 		ctx = context.WithValue(ctx, cdslog.Method, req.Method)
 		ctx = context.WithValue(ctx, cdslog.Route, cleanURL)
 		ctx = context.WithValue(ctx, cdslog.RequestURI, req.RequestURI)
 		ctx = context.WithValue(ctx, cdslog.Deprecated, rc.IsDeprecated)
 		ctx = context.WithValue(ctx, cdslog.Handler, rc.Name)
 		ctx = context.WithValue(ctx, cdslog.Action, rc.Name)
+		ctx = context.WithValue(ctx, cdslog.IPAddress, clientIP)
 
 		var fields = mux.Vars(req)
 		for k, v := range fields {
@@ -534,8 +542,22 @@ func MaintenanceAware() service.HandlerConfigParam {
 }
 
 // NotFoundHandler is called by default by Mux is any matching handler has been found
-func NotFoundHandler(w http.ResponseWriter, req *http.Request) {
-	service.WriteError(context.Background(), w, req, sdk.NewError(sdk.ErrNotFound, fmt.Errorf("%s not found", req.URL.Path)))
+func (r *Router) NotFoundHandler(w http.ResponseWriter, req *http.Request) {
+	ctx := req.Context()
+
+	// Retrieve the client ip address from the header (X-Forwarded-For by default)
+	clientIP := req.Header.Get(r.Config.HeaderXForwardedFor)
+	if clientIP == "" {
+		// If the header has not been found, fallback on the remote adress from the http request
+		clientIP = req.RemoteAddr
+	}
+
+	// Prepare logging fields
+	ctx = context.WithValue(ctx, cdslog.Method, req.Method)
+	ctx = context.WithValue(ctx, cdslog.RequestURI, req.RequestURI)
+	ctx = context.WithValue(ctx, cdslog.IPAddress, clientIP)
+
+	service.WriteError(ctx, w, req, sdk.NewError(sdk.ErrNotFound, fmt.Errorf("%s not found", req.URL.Path)))
 }
 
 // StatusPanic returns router status. If nbPanic > 30 -> Alert, if nbPanic > 0 -> Warn

engine/api/services/cdn.go

@@ -8,17 +8,29 @@ import (
 	"github.com/ovh/cds/sdk"
 )
 
-func GetCDNPublicTCPAdress(ctx context.Context, db gorp.SqlExecutor) (string, error) {
+func GetCDNPublicTCPAdress(ctx context.Context, db gorp.SqlExecutor) (string, bool, error) {
 	srvs, err := LoadAllByType(ctx, db, sdk.TypeCDN)
 	if err != nil {
-		return "", err
+		return "", false, err
 	}
+
+	var tcp_addr string
+	var tcp_tls bool
+
+findAddr:
 	for _, svr := range srvs {
 		if addr, ok := svr.Config["public_tcp"]; ok {
-			return addr.(string), nil
+			tcp_addr = addr.(string)
+			if tls, ok := svr.Config["public_tcp_enable_tls"]; ok {
+				tcp_tls = tls.(bool)
+			}
+			break findAddr
 		}
 	}
-	return "", sdk.NewErrorFrom(sdk.ErrNotFound, "unable to find any tcp configuration in CDN Uservice")
+	if tcp_addr != "" {
+		return tcp_addr, tcp_tls, nil
+	}
+	return "", false, sdk.NewErrorFrom(sdk.ErrNotFound, "unable to find any tcp configuration in CDN Uservice")
 }
 
 func GetCDNPublicHTTPAdress(ctx context.Context, db gorp.SqlExecutor) (string, error) {

engine/api/test/assets/assets.go

@@ -545,8 +545,9 @@ func InitCDNService(t *testing.T, db gorpmapper.SqlExecutorWithTx, scopes ...sdk
 			PublicKey:  publicKey,
 			ConsumerID: &hConsumer.ID,
 			Config: map[string]interface{}{
-				"public_tcp":  "cdn.net:4545",
-				"public_http": "http://cdn.net:8080",
+				"public_tcp":            "cdn.net:4545",
+				"public_http":           "http://cdn.net:8080",
+				"public_tcp_enable_tls": true,
 			},
 		},
 	}

engine/api/workflow_queue.go

@@ -111,7 +111,7 @@ func (api *API) postTakeWorkflowJobHandler() service.Handler {
 
 		// Get CDN TCP Addr
 		// Get CDN TCP Addr
-		pbji.GelfServiceAddr, err = services.GetCDNPublicTCPAdress(ctx, api.mustDB())
+		pbji.GelfServiceAddr, pbji.GelfServiceAddrEnableTLS, err = services.GetCDNPublicTCPAdress(ctx, api.mustDB())
 		if err != nil {
 			return err
 		}

engine/api/workflow_queue_test.go

@@ -6,9 +6,6 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"github.com/ovh/cds/engine/api/bootstrap"
-	"github.com/ovh/cds/engine/api/database/gorpmapping"
-	"github.com/ovh/cds/engine/featureflipping"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -18,6 +15,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ovh/cds/engine/api/bootstrap"
+	"github.com/ovh/cds/engine/api/database/gorpmapping"
+	"github.com/ovh/cds/engine/featureflipping"
+
 	"github.com/ovh/venom"
 
 	"github.com/rockbears/log"
@@ -484,6 +485,7 @@ func Test_postTakeWorkflowJobHandler(t *testing.T) {
 	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), pbji))
 
 	assert.Equal(t, "cdn.net:4545", pbji.GelfServiceAddr)
+	assert.Equal(t, true, pbji.GelfServiceAddrEnableTLS)
 
 	run, err := workflow.LoadNodeJobRun(context.TODO(), api.mustDB(), api.Cache, ctx.job.ID)
 	require.NoError(t, err)

engine/cdn/cdn.go

@@ -37,9 +37,6 @@ const (
 func New() *Service {
 	s := new(Service)
 	s.GoRoutines = sdk.NewGoRoutines()
-	s.Router = &api.Router{
-		Mux: mux.NewRouter(),
-	}
 
 	return s
 }
@@ -50,7 +47,10 @@ func (s *Service) Init(config interface{}) (cdsclient.ServiceConfig, error) {
 	if !ok {
 		return cfg, sdk.WithStack(fmt.Errorf("invalid CDN service configuration"))
 	}
-
+	s.Router = &api.Router{
+		Mux:    mux.NewRouter(),
+		Config: sConfig.HTTP,
+	}
 	cfg.Host = sConfig.API.HTTP.URL
 	cfg.Token = sConfig.API.Token
 	cfg.InsecureSkipVerifyTLS = sConfig.API.HTTP.Insecure

engine/cdn/types.go

@@ -64,17 +64,15 @@ type Service struct {
 
 // Configuration is the hooks configuration structure
 type Configuration struct {
-	Name string        `toml:"name" default:"cds-cdn" comment:"Name of this CDS CDN Service\n Enter a name to enable this service" json:"name"`
-	TCP  sdk.TCPServer `toml:"tcp" comment:"######################\n CDS CDN TCP Configuration \n######################" json:"tcp"`
-	HTTP struct {
-		Addr string `toml:"addr" default:"" commented:"true" comment:"Listen address without port, example: 127.0.0.1" json:"addr"`
-		Port int    `toml:"port" default:"8089" json:"port"`
-	} `toml:"http" comment:"######################\n CDS CDN HTTP Configuration \n######################" json:"http"`
-	URL        string                                 `default:"http://localhost:8089" json:"url" comment:"Private URL for communication with API"`
-	PublicTCP  string                                 `toml:"publicTCP" default:"localhost:8090" comment:"Public address to access to CDN TCP server" json:"public_tcp"`
-	PublicHTTP string                                 `toml:"publicHTTP" default:"http://localhost:8089" comment:"Public address to access to CDN HTTP server" json:"public_http"`
-	Database   database.DBConfigurationWithEncryption `toml:"database" comment:"################################\n Postgresql Database settings \n###############################" json:"database"`
-	Cache      struct {
+	Name               string                                 `toml:"name" default:"cds-cdn" comment:"Name of this CDS CDN Service\n Enter a name to enable this service" json:"name"`
+	TCP                sdk.TCPServer                          `toml:"tcp" comment:"######################\n CDS CDN TCP Configuration \n######################" json:"tcp"`
+	HTTP               service.HTTPRouterConfiguration        `toml:"http" comment:"######################\n CDS CDN HTTP Configuration \n######################" json:"http"`
+	URL                string                                 `default:"http://localhost:8089" json:"url" comment:"Private URL for communication with API"`
+	PublicTCP          string                                 `toml:"publicTCP" default:"localhost:8090" comment:"Public address to access to CDN TCP server" json:"public_tcp"`
+	PublicTCPEnableTLS bool                                   `toml:"publicTCPEnableTLS" comment:"Enable TLS on public address to access to CDN TCP server" json:"public_tcp_enable_tls"`
+	PublicHTTP         string                                 `toml:"publicHTTP" default:"http://localhost:8089" comment:"Public address to access to CDN HTTP server" json:"public_http"`
+	Database           database.DBConfigurationWithEncryption `toml:"database" comment:"################################\n Postgresql Database settings \n###############################" json:"database"`
+	Cache              struct {
 		TTL     int   `toml:"ttl" default:"60" json:"ttl"`
 		LruSize int64 `toml:"lruSize" default:"134217728" json:"lruSize" comment:"Redis LRU cache for logs items in bytes (default: 128MB)"`
 		Redis   struct {

engine/config.go

@@ -80,32 +80,39 @@ func configBootstrap(args []string) Configuration {
 				HealthURL:  "https://ovh.github.io",
 				Type:       "doc",
 			})
+			conf.API.HTTP.Port = 8081
 		case sdk.TypeUI:
 			conf.UI = &ui.Configuration{}
 			conf.UI.Name = "cds-ui-" + namesgenerator.GetRandomNameCDS(0)
 			defaults.SetDefaults(conf.UI)
+			conf.UI.HTTP.Port = 8080
 		case "migrate":
 			conf.DatabaseMigrate = &migrateservice.Configuration{}
 			defaults.SetDefaults(conf.DatabaseMigrate)
 			conf.DatabaseMigrate.Name = "cds-migrate-" + namesgenerator.GetRandomNameCDS(0)
 			conf.DatabaseMigrate.ServiceAPI.DB.Schema = "public"
 			conf.DatabaseMigrate.ServiceCDN.DB.Schema = "cdn"
+			conf.DatabaseMigrate.HTTP.Port = 8087
 		case sdk.TypeHatchery + ":local":
 			conf.Hatchery.Local = &local.HatcheryConfiguration{}
 			defaults.SetDefaults(conf.Hatchery.Local)
 			conf.Hatchery.Local.Name = "cds-hatchery-local-" + namesgenerator.GetRandomNameCDS(0)
+			conf.Hatchery.Local.HTTP.Port = 8086
 		case sdk.TypeHatchery + ":kubernetes":
 			conf.Hatchery.Kubernetes = &kubernetes.HatcheryConfiguration{}
 			defaults.SetDefaults(conf.Hatchery.Kubernetes)
 			conf.Hatchery.Kubernetes.Name = "cds-hatchery-kubernetes-" + namesgenerator.GetRandomNameCDS(0)
+			conf.Hatchery.Kubernetes.HTTP.Port = 8086
 		case sdk.TypeHatchery + ":marathon":
 			conf.Hatchery.Marathon = &marathon.HatcheryConfiguration{}
 			defaults.SetDefaults(conf.Hatchery.Marathon)
 			conf.Hatchery.Marathon.Name = "cds-hatchery-marathon-" + namesgenerator.GetRandomNameCDS(0)
+			conf.Hatchery.Marathon.HTTP.Port = 8086
 		case sdk.TypeHatchery + ":openstack":
 			conf.Hatchery.Openstack = &openstack.HatcheryConfiguration{}
 			defaults.SetDefaults(conf.Hatchery.Openstack)
 			conf.Hatchery.Openstack.Name = "cds-hatchery-openstack-" + namesgenerator.GetRandomNameCDS(0)
+			conf.Hatchery.Openstack.HTTP.Port = 8086
 		case sdk.TypeHatchery + ":swarm":
 			conf.Hatchery.Swarm = &swarm.HatcheryConfiguration{}
 			defaults.SetDefaults(conf.Hatchery.Swarm)
@@ -115,14 +122,17 @@ func configBootstrap(args []string) Configuration {
 				},
 			}
 			conf.Hatchery.Swarm.Name = "cds-hatchery-swarm-" + namesgenerator.GetRandomNameCDS(0)
+			conf.Hatchery.Swarm.HTTP.Port = 8086
 		case sdk.TypeHatchery + ":vsphere":
 			conf.Hatchery.VSphere = &vsphere.HatcheryConfiguration{}
 			defaults.SetDefaults(conf.Hatchery.VSphere)
 			conf.Hatchery.VSphere.Name = "cds-hatchery-vsphere-" + namesgenerator.GetRandomNameCDS(0)
+			conf.Hatchery.VSphere.HTTP.Port = 8086
 		case sdk.TypeHooks:
 			conf.Hooks = &hooks.Configuration{}
 			defaults.SetDefaults(conf.Hooks)
 			conf.Hooks.Name = "cds-hooks-" + namesgenerator.GetRandomNameCDS(0)
+			conf.Hooks.HTTP.Port = 8083
 		case sdk.TypeVCS:
 			conf.VCS = &vcs.Configuration{}
 			defaults.SetDefaults(conf.VCS)
@@ -144,13 +154,16 @@ func configBootstrap(args []string) Configuration {
 				"gerrit":         {URL: "http://localhost:8080", Gerrit: &gerrit},
 			}
 			conf.VCS.Name = "cds-vcs-" + namesgenerator.GetRandomNameCDS(0)
+			conf.VCS.HTTP.Port = 8084
 		case sdk.TypeRepositories:
 			conf.Repositories = &repositories.Configuration{}
 			defaults.SetDefaults(conf.Repositories)
 			conf.Repositories.Name = "cds-repositories-" + namesgenerator.GetRandomNameCDS(0)
+			conf.Repositories.HTTP.Port = 8085
 		case sdk.TypeCDN:
 			conf.CDN = &cdn.Configuration{}
 			defaults.SetDefaults(conf.CDN)
+			conf.CDN.HTTP.Port = 8089
 			conf.CDN.Database.Schema = "cdn"
 			conf.CDN.Units.HashLocatorSalt = sdk.RandomString(8)
 			conf.CDN.Units.Buffers = map[string]storage.BufferConfiguration{
@@ -193,6 +206,7 @@ func configBootstrap(args []string) Configuration {
 		case sdk.TypeElasticsearch:
 			conf.ElasticSearch = &elasticsearch.Configuration{}
 			defaults.SetDefaults(conf.ElasticSearch)
+			conf.ElasticSearch.HTTP.Port = 8088
 		default:
 			sdk.Exit("Error service '%s' is unknown", a)
 		}

engine/elasticsearch/elasticsearch.go

@@ -21,9 +21,6 @@ var esClient *elastic.Client
 func New() *Service {
 	s := new(Service)
 	s.GoRoutines = sdk.NewGoRoutines()
-	s.Router = &api.Router{
-		Mux: mux.NewRouter(),
-	}
 	return s
 }
 
@@ -37,7 +34,10 @@ func (s *Service) ApplyConfiguration(config interface{}) error {
 	if !ok {
 		return fmt.Errorf("ApplyConfiguration> Invalid Elasticsearch configuration")
 	}
-
+	s.Router = &api.Router{
+		Mux:    mux.NewRouter(),
+		Config: s.Cfg.HTTP,
+	}
 	s.HTTPURL = s.Cfg.URL
 	s.ServiceName = s.Cfg.Name
 	s.ServiceType = sdk.TypeElasticsearch

engine/elasticsearch/types.go

@@ -16,12 +16,9 @@ type Service struct {
 
 // Configuration is the vcs configuration structure
 type Configuration struct {
-	Name string `toml:"name" comment:"Name of this CDS elasticsearch Service\n Enter a name to enable this service" json:"name"`
-	HTTP struct {
-		Addr string `toml:"addr" default:"" commented:"true" comment:"Listen address without port, example: 127.0.0.1" json:"addr"`
-		Port int    `toml:"port" default:"8088" json:"port"`
-	} `toml:"http" comment:"######################\n CDS Elasticsearch HTTP Configuration \n######################" json:"http"`
-	URL           string `default:"http://localhost:8088" json:"url"`
+	Name          string                          `toml:"name" comment:"Name of this CDS elasticsearch Service\n Enter a name to enable this service" json:"name"`
+	HTTP          service.HTTPRouterConfiguration `toml:"http" comment:"######################\n CDS Elasticsearch HTTP Configuration \n######################" json:"http"`
+	URL           string                          `default:"http://localhost:8088" json:"url"`
 	ElasticSearch struct {
 		URL          string `toml:"url" json:"url"`
 		Username     string `toml:"username" json:"username"`

engine/hatchery/kubernetes/kubernetes.go

@@ -30,9 +30,6 @@ import (
 func New() *HatcheryKubernetes {
 	s := new(HatcheryKubernetes)
 	s.GoRoutines = sdk.NewGoRoutines()
-	s.Router = &api.Router{
-		Mux: mux.NewRouter(),
-	}
 	return s
 }
 
@@ -57,6 +54,11 @@ func (h *HatcheryKubernetes) Init(config interface{}) (cdsclient.ServiceConfig,
 		return cfg, sdk.WithStack(fmt.Errorf("invalid kubernetes hatchery configuration"))
 	}
 
+	h.Router = &api.Router{
+		Mux:    mux.NewRouter(),
+		Config: sConfig.HTTP,
+	}
+
 	cfg.Host = sConfig.API.HTTP.URL
 	cfg.Token = sConfig.API.Token
 	cfg.InsecureSkipVerifyTLS = sConfig.API.HTTP.Insecure