fix(hatchery): don't send status alert for only 1 dockerengine failure (

#5111)
ovh · Apr 7, 2020 · 46f44ca · 46f44ca
1 parent 76a9939
commit 46f44ca
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 3 deletions.
diff --git a/engine/hatchery/swarm/swarm_conf.go b/engine/hatchery/swarm/swarm_conf.go
@@ -57,6 +57,7 @@ func (h *HatcherySwarm) ApplyConfiguration(cfg interface{}) error {
 func (h *HatcherySwarm) Status(ctx context.Context) sdk.MonitoringStatus {
 	m := h.CommonMonitoring()
 	m.Lines = append(m.Lines, sdk.MonitoringStatusLine{Component: "Workers", Value: fmt.Sprintf("%d/%d", len(h.WorkersStarted(ctx)), h.Config.Provision.MaxWorker), Status: sdk.MonitoringStatusOK})
+	var nbErrorImageList, nbErrorGetContainers int
 	for dockerName, dockerClient := range h.dockerClients {
 		//Check images
 		status := sdk.MonitoringStatusOK
@@ -65,19 +66,33 @@ func (h *HatcherySwarm) Status(ctx context.Context) sdk.MonitoringStatus {
 		images, err := dockerClient.ImageList(ctxList, types.ImageListOptions{All: true})
 		if err != nil {
 			log.Warning(ctx, "hatchery> swarm> %s> Status> Unable to list images on %s: %s", h.Name(), dockerName, err)
-			status = sdk.MonitoringStatusAlert
+			status = sdk.MonitoringStatusWarn
+			nbErrorImageList++
 		}
 		m.Lines = append(m.Lines, sdk.MonitoringStatusLine{Component: "Images-" + dockerName, Value: fmt.Sprintf("%d", len(images)), Status: status})
 		//Check containers
 		status = sdk.MonitoringStatusOK
 		cs, err := h.getContainers(dockerClient, types.ContainerListOptions{All: true})
 		if err != nil {
 			log.Warning(ctx, "hatchery> swarm> %s> Status> Unable to list containers on %s: %s", h.Name(), dockerName, err)
-			status = sdk.MonitoringStatusAlert
+			status = sdk.MonitoringStatusWarn
+			nbErrorGetContainers++
 		}
 		m.Lines = append(m.Lines, sdk.MonitoringStatusLine{Component: "Containers-" + dockerName, Value: fmt.Sprintf("%d", len(cs)), Status: status})
 	}
 
+	var status = sdk.MonitoringStatusOK
+	if nbErrorImageList > len(h.dockerClients)/2 {
+		status = sdk.MonitoringStatusAlert
+	}
+	m.Lines = append(m.Lines, sdk.MonitoringStatusLine{Component: "DockerEngines.ListImages", Value: fmt.Sprintf("%d/%d", nbErrorImageList, len(h.dockerClients)), Status: status})
+
+	status = sdk.MonitoringStatusOK
+	if nbErrorGetContainers > len(h.dockerClients)/2 {
+		status = sdk.MonitoringStatusAlert
+	}
+	m.Lines = append(m.Lines, sdk.MonitoringStatusLine{Component: "DockerEngines.GetContainers", Value: fmt.Sprintf("%d/%d", nbErrorGetContainers, len(h.dockerClients)), Status: status})
+
 	return m
 }
 

diff --git a/sdk/status.go b/sdk/status.go
@@ -50,7 +50,7 @@ type MonitoringStatusLine struct {
 // HTTPStatusCode return the http status code
 func (m MonitoringStatus) HTTPStatusCode() int {
 	for _, l := range m.Lines {
-		if l.Status != MonitoringStatusOK {
+		if l.Status == MonitoringStatusAlert {
 			return http.StatusServiceUnavailable
 		}
 	}