Making possible to add rules via docker secret

.gitignore

@@ -4,4 +4,5 @@
 /alertmanager.yml
 /tmp
 /*.pem
-/secret
+/secret
+*.out

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.9.2 AS build
+FROM golang:1.12.6 AS build
 ADD . /src
 WORKDIR /src
 RUN go get -t github.com/stretchr/testify/suite
@@ -8,7 +8,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build -v -o docker-flow-monitor
 
 
 
-FROM prom/prometheus:v2.2.0
+FROM prom/prometheus:v2.10.0
 
 ENV GLOBAL_SCRAPE_INTERVAL=10s \
     ARG_CONFIG_FILE=/etc/prometheus/prometheus.yml \

Dockerfile.test

@@ -0,0 +1,6 @@
+FROM golang:1.12.6 AS build
+ADD . /src
+WORKDIR /src
+RUN go get -t github.com/stretchr/testify/suite
+RUN go get -d -v -t
+RUN go test --cover  -coverprofile=coverage.out -v ./... --run UnitTest -p 1

Jenkinsfile

@@ -43,8 +43,7 @@ pipeline {
         label "prod"
       }
       steps {
-        sh "docker service update --image dockerflow/docker-flow-monitor:2-${currentBuild.displayName} monitor_monitor"
-        sh "docker service update --image dockerflow/docker-flow-monitor-docs:2-${currentBuild.displayName} monitor_docs"
+        sh "helm upgrade -i docker-flow-monitor helm/docker-flow-monitor --namespace df --set image.tag=2-${currentBuild.displayName}"
       }
     }
   }

docs/auto-scaling.md

@@ -148,7 +148,7 @@ The full definition of the stack that we just deployed can be found in [docker-f
     command: --config.file=/run/secrets/alert_manager_config --storage.path=/alertmanager
 
   swarm-listener:
-    image: vfarcic/docker-flow-swarm-listener
+    image: dockerflow/docker-flow-swarm-listener
     ...
     environment:
       - DF_NOTIFY_CREATE_SERVICE_URL=http://monitor:8080/v1/docker-flow-monitor/reconfigure

docs/config.md

@@ -130,7 +130,7 @@ The simplest way to add scrape configs is to use Docker [secrets](https://docs.d
 
 ## Scrape Label Configuration With Service and Node Labels
 
-When using a version of [Docker Flow Swarm Listener](https://github.com/vfarcic/docker-flow-swarm-listener), DFSL, newer than `18.03.20-39`, you can configure DFSL to send node information to `Docker Flow Monitor`, DFM. This can be done by setting `DF_INCLUDE_NODE_IP_INFO` to `true` in the DFSL environment. DFM will automatically display the node hostnames as a label for each prometheus target. The `DF_SCRAPE_TARGET_LABELS` env variable allows for additional labels to be displayed.
+When using a version of [Docker Flow Swarm Listener](https://github.com/docker-flow/docker-flow-swarm-listener), DFSL, newer than `18.03.20-39`, you can configure DFSL to send node information to `Docker Flow Monitor`, DFM. This can be done by setting `DF_INCLUDE_NODE_IP_INFO` to `true` in the DFSL environment. DFM will automatically display the node hostnames as a label for each prometheus target. The `DF_SCRAPE_TARGET_LABELS` env variable allows for additional labels to be displayed.
 
 In addition to service labels, DFM can be configured to import node and engine labels prefixed with `com.df.` as prometheus labels for our targets. First, configure DFSL to push node events to DFM by setting the following environment variables in DFSL:
 
@@ -151,7 +151,7 @@ For example, the stack file, [stacks/docker-flow-monitor-flexible-labels.yml](ht
 services:
 ...
   swarm-listener:
-    image: vfarcic/docker-flow-swarm-listener
+    image: dockerflow/docker-flow-swarm-listener
     environment:
       - DF_NOTIFY_CREATE_SERVICE_URL=http://monitor:8080/v1/docker-flow-monitor/reconfigure
       - DF_NOTIFY_REMOVE_SERVICE_URL=http://monitor:8080/v1/docker-flow-monitor/remove

docs/tutorial-flexible-labeling.md

@@ -52,7 +52,7 @@ To get the nodes information, DFSL is configured to send node events to DFM by s
       - DF_GET_NODES_URL=http://swarm-listener:8080/v1/docker-flow-swarm-listener/get-nodes
   ...
   swarm-listener:
-    image: vfarcic/docker-flow-swarm-listener
+    image: dockerflow/docker-flow-swarm-listener
     environment:
       ...
       - DF_NOTIFY_CREATE_NODE_URL=http://monitor:8080/v1/docker-flow-monitor/node/reconfigure

docs/tutorial.md

@@ -67,7 +67,7 @@ Finally, the last service in the stack is `swarm-listener`. The definition is as
 
 ```
   swarm-listener:
-    image: vfarcic/docker-flow-swarm-listener
+    image: dockerflow/docker-flow-swarm-listener
     networks:
       - monitor
     volumes:

helm/docker-flow-monitor/Chart.yaml

@@ -0,0 +1,14 @@
+name: docker-flow-monitor
+version: 0.0.1
+apiVersion: v1
+description: Docker Flow Monitor
+keywords:
+- docker
+- swarm
+- docker-flow
+home: https://monitor.dockerflow.com
+sources:
+- https://github.com/docker-flow/docker-flow-monitor
+maintainers:
+- name: Viktor Farcic
+  email: [email protected]

helm/docker-flow-monitor/README.md

@@ -0,0 +1 @@
+# Docker Flow Monitor

helm/docker-flow-monitor/templates/_helpers.tpl

@@ -0,0 +1,21 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "helm.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "helm.fullname" -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}

helm/docker-flow-monitor/templates/deployment.yaml

@@ -0,0 +1,34 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "helm.fullname" . }}
+  labels:
+    app: {{ template "helm.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ template "helm.name" . }}
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ template "helm.name" . }}
+        release: {{ .Release.Name }}
+    spec:
+      containers:
+      - name: ui
+        image: dockerflow/docker-flow-monitor-docs:{{ .Values.image.tag }}
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 80
+          periodSeconds: 1
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 80
+        resources:
+{{ toYaml .Values.resources | indent 10 }}

helm/docker-flow-monitor/templates/ing.yaml

@@ -0,0 +1,33 @@
+{{- $serviceName := include "helm.fullname" . -}}
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: {{ template "helm.fullname" . }}
+  labels:
+    app: {{ template "helm.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    certmanager.k8s.io/issuer: "letsencrypt-prod"
+    certmanager.k8s.io/acme-challenge-type: http01
+spec:
+  rules:
+  {{- range .Values.ingress.host }}
+  {{- $url := splitList "/" . }}
+  - host: {{ first $url }}
+    http:
+      paths:
+        - path: /{{ rest $url | join "/" }}
+          backend:
+            serviceName: {{ $serviceName }}
+            servicePort: 80
+  {{- end -}}
+  {{- range .Values.ingress.host }}
+  {{- $url := splitList "/" . }}
+  tls:
+  - hosts:
+    - {{ first $url }}
+    secretName: le-{{ $serviceName }}
+  {{- end -}}

helm/docker-flow-monitor/templates/issuer.yaml

@@ -0,0 +1,11 @@
+apiVersion: certmanager.k8s.io/v1alpha1
+kind: Issuer
+metadata:
+  name: letsencrypt-prod
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: [email protected]
+    privateKeySecretRef:
+      name: letsencrypt-prod
+    http01: {}

helm/docker-flow-monitor/templates/svc.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "helm.fullname" . }}
+  labels:
+    app: {{ template "helm.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  ports:
+  - port: 80
+  type: NodePort
+  selector:
+    app: {{ template "helm.name" . }}
+    release: {{ .Release.Name }}

helm/docker-flow-monitor/values.yaml

@@ -0,0 +1,12 @@
+image:
+  tag: latest
+ingress:
+  host:
+  - monitor.dockerflow.com
+resources:
+  limits:
+   cpu: 15m
+   memory: 15Mi
+  requests:
+   cpu: 5m
+   memory: 5Mi

k8s/docker-flow-monitor/templates/deployment.yaml

@@ -0,0 +1,42 @@
+---
+# Source: docker-flow-monitor/templates/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: moonitor-docker-flow-monitor
+  labels:
+    app: docker-flow-monitor
+    chart: docker-flow-monitor-0.0.1
+    release: moonitor
+    heritage: Tiller
+spec:
+  selector:
+    matchLabels:
+      app: docker-flow-monitor
+      release: moonitor
+  template:
+    metadata:
+      labels:
+        app: docker-flow-monitor
+        release: moonitor
+    spec:
+      containers:
+      - name: ui
+        image: dockerflow/docker-flow-monitor-docs:latest
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 80
+          periodSeconds: 1
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 80
+        resources:
+          limits:
+            cpu: 15m
+            memory: 15Mi
+          requests:
+            cpu: 5m
+            memory: 5Mi
+

k8s/docker-flow-monitor/templates/ing.yaml

@@ -0,0 +1,28 @@
+---
+# Source: docker-flow-monitor/templates/ing.yaml
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: moonitor-docker-flow-monitor
+  labels:
+    app: docker-flow-monitor
+    chart: docker-flow-monitor-0.0.1
+    release: moonitor
+    heritage: Tiller
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    certmanager.k8s.io/issuer: "letsencrypt-prod"
+    certmanager.k8s.io/acme-challenge-type: http01
+spec:
+  rules:
+  - host: monitor.dockerflow.com
+    http:
+      paths:
+        - path: /
+          backend:
+            serviceName: moonitor-docker-flow-monitor
+            servicePort: 80
+  tls:
+  - hosts:
+    - monitor.dockerflow.com
+    secretName: le-moonitor-docker-flow-monitor

k8s/docker-flow-monitor/templates/issuer.yaml

@@ -0,0 +1,13 @@
+---
+# Source: docker-flow-monitor/templates/issuer.yaml
+apiVersion: certmanager.k8s.io/v1alpha1
+kind: Issuer
+metadata:
+  name: letsencrypt-prod
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: [email protected]
+    privateKeySecretRef:
+      name: letsencrypt-prod
+    http01: {}

k8s/docker-flow-monitor/templates/svc.yaml

@@ -0,0 +1,18 @@
+---
+# Source: docker-flow-monitor/templates/svc.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: moonitor-docker-flow-monitor
+  labels:
+    app: docker-flow-monitor
+    chart: docker-flow-monitor-0.0.1
+    release: moonitor
+    heritage: Tiller
+spec:
+  ports:
+  - port: 80
+  type: NodePort
+  selector:
+    app: docker-flow-monitor
+    release: moonitor

prometheus/config.go

@@ -33,10 +33,12 @@ func WriteConfig(configPath string, scrapes map[string]Scrape,
 		c.InsertScrapesFromDir(configsDir)
 	}
 
+	c.RuleFiles = []string{"/run/secrets/*.rules"}
 	if len(alerts) > 0 {
 		logPrintf("Writing to alert.rules")
 		afero.WriteFile(FS, alertRulesPath, []byte(GetAlertConfig(alerts)), 0644)
-		c.RuleFiles = []string{"alert.rules"}
+		// c.RuleFiles = []string{"alert.rules"}
+		c.RuleFiles = append(c.RuleFiles, "alert.rules")
 	}
 
 	alertmanagerURLs := os.Getenv("ARG_ALERTMANAGER_URL")
@@ -66,7 +68,6 @@ func WriteConfig(configPath string, scrapes map[string]Scrape,
 	logPrintf("Writing to prometheus.yml")
 	configYAML, _ := yaml.Marshal(c)
 	afero.WriteFile(FS, configPath, configYAML, 0644)
-
 }
 
 // InsertEnv inserts envKey/envValue into config

prometheus/config_test.go

@@ -848,7 +848,7 @@ func (s *ConfigTestSuite) Test_WriteConfig_WriteAlerts() {
 	}
 
 	c := &Config{}
-	c.RuleFiles = []string{"alert.rules"}
+	c.RuleFiles = []string{"/run/secrets/*.rules", "alert.rules"}
 	cYAML, _ := yaml.Marshal(c)
 	expectedAlerts := GetAlertConfig(alerts)