feat(api): add some metrics to /mon/metrics (#3500)

engine/api/api.go

@@ -11,7 +11,6 @@ import (
 	"github.com/go-gorp/gorp"
 	"github.com/gorilla/mux"
 	"go.opencensus.io/stats"
-	"go.opencensus.io/stats/view"
 
 	"github.com/ovh/cds/engine/api/action"
 	"github.com/ovh/cds/engine/api/auth"
@@ -28,7 +27,6 @@ import (
 	"github.com/ovh/cds/engine/api/migrate"
 	"github.com/ovh/cds/engine/api/notification"
 	"github.com/ovh/cds/engine/api/objectstore"
-	"github.com/ovh/cds/engine/api/observability"
 	"github.com/ovh/cds/engine/api/pipeline"
 	"github.com/ovh/cds/engine/api/platform"
 	"github.com/ovh/cds/engine/api/purge"
@@ -48,8 +46,7 @@ import (
 
 // Configuration is the configuraton structure for CDS API
 type Configuration struct {
-	Name string `toml:"name" default:"cdsinstance" comment:"Name of this CDS Instance" json:"name"`
-	URL  struct {
+	URL struct {
 		API string `toml:"api" default:"http://localhost:8081" json:"api"`
 		UI  string `toml:"ui" default:"http://localhost:2015" json:"ui"`
 	} `toml:"url" comment:"#####################\n CDS URLs Settings \n####################" json:"url"`
@@ -232,10 +229,22 @@ type API struct {
 	eventsBroker        *eventsBroker
 	warnChan            chan sdk.Event
 	Cache               cache.Store
-	Stats               struct {
-		WorkflowRunFailed  *stats.Int64Measure
-		WorkflowRunStarted *stats.Int64Measure
-		Sessions           *stats.Int64Measure
+	Metrics             struct {
+		WorkflowRunFailed    *stats.Int64Measure
+		WorkflowRunStarted   *stats.Int64Measure
+		Sessions             *stats.Int64Measure
+		nbUsers              *stats.Int64Measure
+		nbApplications       *stats.Int64Measure
+		nbProjects           *stats.Int64Measure
+		nbGroups             *stats.Int64Measure
+		nbPipelines          *stats.Int64Measure
+		nbWorkflows          *stats.Int64Measure
+		nbArtifacts          *stats.Int64Measure
+		nbWorkerModels       *stats.Int64Measure
+		nbWorkflowRuns       *stats.Int64Measure
+		nbWorkflowNodeRuns   *stats.Int64Measure
+		nbMaxWorkersBuilding *stats.Int64Measure
+		queue                *stats.Int64Measure
 	}
 }
 
@@ -569,13 +578,13 @@ func (a *API) Serve(ctx context.Context) error {
 		Background: ctx,
 	}
 	a.InitRouter()
-	if err := a.Router.InitStats("cds-api", a.Name); err != nil {
-		log.Error("unable to init router stats: %v", err)
+	if err := a.Router.InitMetrics("cds-api", a.Name); err != nil {
+		log.Error("unable to init router metrics: %v", err)
 	}
 
-	log.Info("Initializing Stats")
-	if err := a.initStats(); err != nil {
-		log.Error("unable to init api stats: %v", err)
+	log.Info("Initializing Metrics")
+	if err := a.initMetrics(ctx); err != nil {
+		log.Error("unable to init api metrics: %v", err)
 	}
 
 	//Initiliaze hook package
@@ -659,9 +668,6 @@ func (a *API) Serve(ctx context.Context) error {
 	sdk.GoRoutine(ctx, "auditCleanerRoutine(ctx", func(ctx context.Context) {
 		auditCleanerRoutine(ctx, a.DBConnectionFactory.GetDBMap)
 	})
-	sdk.GoRoutine(ctx, "metrics.Initialize", func(ctx context.Context) {
-		metrics.Initialize(ctx, a.DBConnectionFactory.GetDBMap, a.Config.Name)
-	}, a.PanicDump())
 	sdk.GoRoutine(ctx, "repositoriesmanager.ReceiveEvents", func(ctx context.Context) {
 		repositoriesmanager.ReceiveEvents(ctx, a.DBConnectionFactory.GetDBMap, a.Cache)
 	}, a.PanicDump())
@@ -788,31 +794,6 @@ func (a *API) Serve(ctx context.Context) error {
 	return nil
 }
 
-func (a *API) initStats() error {
-	label := fmt.Sprintf("cds/cds-api/%s/workflow_runs_started", a.Name)
-	a.Stats.WorkflowRunStarted = stats.Int64(label, "number of started workflow runs", stats.UnitDimensionless)
-
-	label = fmt.Sprintf("cds/cds-api/%s/workflow_runs_failed", a.Name)
-	a.Stats.WorkflowRunFailed = stats.Int64(label, "number of failed workflow runs", stats.UnitDimensionless)
-
-	log.Info("api> Stats initialized")
-
-	return observability.RegisterView(
-		&view.View{
-			Name:        "workflow_runs_started",
-			Description: a.Stats.WorkflowRunStarted.Description(),
-			Measure:     a.Stats.WorkflowRunStarted,
-			Aggregation: view.Count(),
-		},
-		&view.View{
-			Name:        "workflow_runs_failed",
-			Description: a.Stats.WorkflowRunFailed.Description(),
-			Measure:     a.Stats.WorkflowRunFailed,
-			Aggregation: view.Count(),
-		},
-	)
-}
-
 const panicDumpTTL = 60 * 60 * 24 // 24 hours
 
 func (a *API) PanicDump() func(s string) (io.WriteCloser, error) {

engine/api/api_routes.go

@@ -94,10 +94,7 @@ func (api *API) InitRouter() {
 	r.Handle("/mon/smtp/ping", r.GET(api.smtpPingHandler, Auth(true)))
 	r.Handle("/mon/version", r.GET(VersionHandler, Auth(false)))
 	r.Handle("/mon/db/migrate", r.GET(api.getMonDBStatusMigrateHandler, NeedAdmin(true)))
-	r.Handle("/mon/building", r.GET(api.getBuildingPipelinesHandler))
-	r.Handle("/mon/building/{hash}", r.GET(api.getPipelineBuildingCommitHandler))
-	r.Handle("/mon/metrics", r.GET(api.getMetricsHandler, Auth(false)))
-	r.Handle("/mon/stats", r.GET(observability.StatsHandler, Auth(false)))
+	r.Handle("/mon/metrics", r.GET(observability.StatsHandler, Auth(false)))
 	r.Handle("/mon/errors/{uuid}", r.GET(api.getErrorHandler, NeedAdmin(true)))
 	r.Handle("/mon/panic/{uuid}", r.GET(api.getPanicDumpHandler, Auth(false)))
 

engine/api/cache/redis.go

@@ -359,9 +359,9 @@ func (s *RedisStore) GetMessageFromSubscription(c context.Context, pb PubSub) (s
 // Status returns the status of the local cache
 func (s *RedisStore) Status() sdk.MonitoringStatusLine {
 	if s.Client.Ping().Err() == nil {
-		return sdk.MonitoringStatusLine{Component: "Cache", Value: "Ping OK", Status: sdk.MonitoringStatusOK}
+		return sdk.MonitoringStatusLine{Component: "Cache Ping", Value: "OK", Status: sdk.MonitoringStatusOK}
 	}
-	return sdk.MonitoringStatusLine{Component: "Cache", Value: "No Ping", Status: sdk.MonitoringStatusAlert}
+	return sdk.MonitoringStatusLine{Component: "Cache Ping", Value: "KO", Status: sdk.MonitoringStatusAlert}
 }
 
 // RemoveFromQueue removes a member from a list

engine/api/database/database.go

@@ -143,14 +143,14 @@ func (f *DBConnectionFactory) dsn() string {
 // Status returns database driver and status in a printable string
 func (f *DBConnectionFactory) Status() sdk.MonitoringStatusLine {
 	if f.db == nil {
-		return sdk.MonitoringStatusLine{Component: "Database", Value: "No Connection", Status: sdk.MonitoringStatusAlert}
+		return sdk.MonitoringStatusLine{Component: "Database Conns", Value: "No Connection", Status: sdk.MonitoringStatusAlert}
 	}
 
 	if err := f.db.Ping(); err != nil {
-		return sdk.MonitoringStatusLine{Component: "Database", Value: "No Ping", Status: sdk.MonitoringStatusAlert}
+		return sdk.MonitoringStatusLine{Component: "Database Conns", Value: "No Ping", Status: sdk.MonitoringStatusAlert}
 	}
 
-	return sdk.MonitoringStatusLine{Component: "Database", Value: fmt.Sprintf("%d conns", f.db.Stats().OpenConnections), Status: sdk.MonitoringStatusOK}
+	return sdk.MonitoringStatusLine{Component: "Database Conns", Value: fmt.Sprintf("%d", f.db.Stats().OpenConnections), Status: sdk.MonitoringStatusOK}
 }
 
 // Close closes the database, releasing any open resources.

engine/api/event/event.go

@@ -133,5 +133,5 @@ func Status() sdk.MonitoringStatusLine {
 		status = sdk.MonitoringStatusAlert
 	}
 
-	return sdk.MonitoringStatusLine{Component: "Event", Value: o, Status: status}
+	return sdk.MonitoringStatusLine{Component: "Event Broker", Value: o, Status: status}
 }

engine/api/mail/mail.go

@@ -46,9 +46,9 @@ func Init(user, password, from, host, port string, tls, disable bool) {
 // Status verification of smtp configuration, returns OK or KO
 func Status() sdk.MonitoringStatusLine {
 	if _, err := smtpClient(); err != nil {
-		return sdk.MonitoringStatusLine{Component: "SMTP", Value: "KO: " + err.Error(), Status: sdk.MonitoringStatusAlert}
+		return sdk.MonitoringStatusLine{Component: "SMTP Ping", Value: "KO: " + err.Error(), Status: sdk.MonitoringStatusAlert}
 	}
-	return sdk.MonitoringStatusLine{Component: "SMTP", Value: "Connect OK", Status: sdk.MonitoringStatusOK}
+	return sdk.MonitoringStatusLine{Component: "SMTP Ping", Value: "Connect OK", Status: sdk.MonitoringStatusOK}
 }
 
 func smtpClient() (*smtp.Client, error) {

engine/api/observability/helper.go

@@ -8,6 +8,9 @@ import (
 
 	"github.com/go-gorp/gorp"
 	"github.com/gorilla/mux"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
 	"go.opencensus.io/trace"
 
 	"github.com/ovh/cds/engine/api/cache"
@@ -106,3 +109,36 @@ func findPrimaryKeyFromRequest(req *http.Request, db gorp.SqlExecutor, store cac
 
 	return pkey, pkey != ""
 }
+
+// NewViewLast creates a new view via aggregation LastValue()
+func NewViewLast(name string, s *stats.Int64Measure, tags []tag.Key) *view.View {
+	return &view.View{
+		Name:        name,
+		Description: s.Description(),
+		Measure:     s,
+		Aggregation: view.LastValue(),
+		TagKeys:     tags,
+	}
+}
+
+// NewViewLastFloat64 creates a new view via aggregation LastValue()
+func NewViewLastFloat64(name string, s *stats.Float64Measure, tags []tag.Key) *view.View {
+	return &view.View{
+		Name:        name,
+		Description: s.Description(),
+		Measure:     s,
+		Aggregation: view.LastValue(),
+		TagKeys:     tags,
+	}
+}
+
+// NewViewCount creates a new view via aggregation LastValue()
+func NewViewCount(name string, s *stats.Int64Measure, tags []tag.Key) *view.View {
+	return &view.View{
+		Name:        name,
+		Description: s.Description(),
+		Measure:     s,
+		Aggregation: view.Count(),
+		TagKeys:     tags,
+	}
+}