PMM-3387 PMM-5464 Custom base Prometheus file, update code copied from Prometheus to version 2.16.0

.golangci.yml

@@ -40,7 +40,8 @@ issues:
     # gas: duplicated errcheck checks
     - 'G104: Errors unhandled'
   exclude-rules:
-    # gomnd: tests are full of magic numbers
+    # gomnd: tests are full of magic numbers and long functions
     - path: _test\.go
       linters:
         - gomnd
+        - funlen

main.go

@@ -83,7 +83,8 @@ const (
 	http1Addr = "127.0.0.1:7772"
 	debugAddr = "127.0.0.1:7773"
 
-	defaultAlertManagerFile = "/srv/prometheus/rules/pmm.rules.yml"
+	defaultAlertManagerFile  = "/srv/prometheus/rules/pmm.rules.yml"
+	prometheusBaseConfigFile = "/srv/prometheus/prometheus.base.yml"
 )
 
 func addLogsHandler(mux *http.ServeMux, logs *supervisord.Logs) {
@@ -484,7 +485,7 @@ func main() {
 	prom.MustRegister(reformL)
 	db := reform.NewDB(sqlDB, postgresql.Dialect, reformL)
 
-	prometheus, err := prometheus.NewService(*prometheusConfigF, *promtoolF, db, *prometheusURLF)
+	prometheus, err := prometheus.NewService(*prometheusConfigF, prometheusBaseConfigFile, *promtoolF, db, *prometheusURLF)
 	if err != nil {
 		l.Panicf("Prometheus service problem: %+v", err)
 	}

services/prometheus/prometheus.go

@@ -57,30 +57,32 @@ var checkFailedRE = regexp.MustCompile(`FAILED: parsing YAML file \S+: (.+)\n`)
 //   * Prometheus configuration and rule files are accessible;
 //   * promtool is available.
 type Service struct {
-	configPath   string
-	promtoolPath string
-	db           *reform.DB
-	baseURL      *url.URL
-	client       *http.Client
+	configPath     string
+	baseConfigPath string
+	promtoolPath   string
+	db             *reform.DB
+	baseURL        *url.URL
+	client         *http.Client
 
 	l    *logrus.Entry
 	sema chan struct{}
 }
 
 // NewService creates new service.
-func NewService(configPath string, promtoolPath string, db *reform.DB, baseURL string) (*Service, error) {
+func NewService(configPath, baseConfigPath string, promtoolPath string, db *reform.DB, baseURL string) (*Service, error) {
 	u, err := url.Parse(baseURL)
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
 	return &Service{
-		configPath:   configPath,
-		promtoolPath: promtoolPath,
-		db:           db,
-		baseURL:      u,
-		client:       new(http.Client),
-		l:            logrus.WithField("component", "prometheus"),
-		sema:         make(chan struct{}, 1),
+		configPath:     configPath,
+		baseConfigPath: baseConfigPath,
+		promtoolPath:   promtoolPath,
+		db:             db,
+		baseURL:        u,
+		client:         new(http.Client),
+		l:              logrus.WithField("component", "prometheus"),
+		sema:           make(chan struct{}, 1),
 	}, nil
 }
 
@@ -132,6 +134,24 @@ func (svc *Service) reload() error {
 	return errors.Errorf("%d: %s", resp.StatusCode, b)
 }
 
+func (svc *Service) loadBaseConfig() *config.Config {
+	buf, err := ioutil.ReadFile(svc.baseConfigPath)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			svc.l.Errorf("Failed to load base prometheus config %s: %s", svc.baseConfigPath, err)
+		}
+		return new(config.Config)
+	}
+
+	cfg := &config.Config{}
+	if err := yaml.Unmarshal(buf, cfg); err != nil {
+		svc.l.Errorf("Failed to parse base prometheus config %s: %s.", svc.baseConfigPath, err)
+		return new(config.Config)
+	}
+
+	return cfg
+}
+
 // addScrapeConfigs adds Prometheus scrape configs to cfg for all Agents.
 func (svc *Service) addScrapeConfigs(cfg *config.Config, q *reform.Querier, s *models.MetricsResolutions) error {
 	agents, err := q.SelectAllFrom(models.AgentTable, "WHERE NOT disabled AND listen_port IS NOT NULL ORDER BY agent_type, agent_id")
@@ -266,30 +286,33 @@ func (svc *Service) addScrapeConfigs(cfg *config.Config, q *reform.Querier, s *m
 
 // marshalConfig marshals Prometheus configuration.
 func (svc *Service) marshalConfig() ([]byte, error) {
-	var cfg *config.Config
+	cfg := svc.loadBaseConfig()
+
 	e := svc.db.InTransaction(func(tx *reform.TX) error {
 		settings, err := models.GetSettings(tx)
 		if err != nil {
 			return err
 		}
 		s := settings.MetricsResolutions
 
-		cfg = &config.Config{
-			GlobalConfig: config.GlobalConfig{
-				ScrapeInterval:     model.Duration(s.LR),
-				ScrapeTimeout:      scrapeTimeout(s.LR),
-				EvaluationInterval: model.Duration(s.LR),
-			},
-			RuleFiles: []string{
-				"/srv/prometheus/rules/*.rules.yml",
-			},
-			ScrapeConfigs: []*config.ScrapeConfig{
-				scrapeConfigForPrometheus(s.HR),
-				scrapeConfigForGrafana(s.MR),
-				scrapeConfigForPMMManaged(s.MR),
-				scrapeConfigForQANAPI2(s.MR),
-			},
+		if cfg.GlobalConfig.ScrapeInterval == 0 {
+			cfg.GlobalConfig.ScrapeInterval = model.Duration(s.LR)
+		}
+		if cfg.GlobalConfig.ScrapeTimeout == 0 {
+			cfg.GlobalConfig.ScrapeTimeout = scrapeTimeout(s.LR)
 		}
+		if cfg.GlobalConfig.EvaluationInterval == 0 {
+			cfg.GlobalConfig.EvaluationInterval = model.Duration(s.LR)
+		}
+
+		cfg.RuleFiles = append(cfg.RuleFiles, "/srv/prometheus/rules/*.rules.yml")
+
+		cfg.ScrapeConfigs = append(cfg.ScrapeConfigs,
+			scrapeConfigForPrometheus(s.HR),
+			scrapeConfigForGrafana(s.MR),
+			scrapeConfigForPMMManaged(s.MR),
+			scrapeConfigForQANAPI2(s.MR),
+		)
 
 		if settings.AlertManagerURL != "" {
 			u, err := url.Parse(settings.AlertManagerURL)
@@ -309,7 +332,7 @@ func (svc *Service) marshalConfig() ([]byte, error) {
 					}
 				}
 
-				cfg.AlertingConfig.AlertmanagerConfigs = []*config.AlertmanagerConfig{{
+				cfg.AlertingConfig.AlertmanagerConfigs = append(cfg.AlertingConfig.AlertmanagerConfigs, &config.AlertmanagerConfig{
 					ServiceDiscoveryConfig: sd_config.ServiceDiscoveryConfig{
 						StaticConfigs: []*targetgroup.Group{{
 							Targets: []model.LabelSet{{addressLabel: model.LabelValue(u.Host)}},
@@ -318,7 +341,7 @@ func (svc *Service) marshalConfig() ([]byte, error) {
 					HTTPClientConfig: httpClientConfig,
 					Scheme:           u.Scheme,
 					PathPrefix:       u.Path,
-				}}
+				})
 			} else {
 				svc.l.Errorf("Failed to parse Alert Manager URL %q: %s.", settings.AlertManagerURL, err)
 			}

services/prometheus/prometheus_test.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"database/sql"
 	"io/ioutil"
-	"path/filepath"
 	"testing"
 
 	"github.com/AlekSi/pointer"
@@ -33,15 +32,16 @@ import (
 	"github.com/percona/pmm-managed/utils/testdb"
 )
 
-var configPath = filepath.Join("..", "..", "testdata", "prometheus", "prometheus.yml")
+const configPath = "../../testdata/prometheus/prometheus.yml"
+const baseConfigPath = "../../testdata/prometheus/prometheus.base.yml"
 
 func setup(t *testing.T) (*reform.DB, *Service, []byte) {
 	t.Helper()
 
 	sqlDB := testdb.Open(t, models.SkipFixtures)
 	db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf))
 
-	svc, err := NewService(configPath, "promtool", db, "http://127.0.0.1:9090/prometheus/")
+	svc, err := NewService(configPath, "", "promtool", db, "http://127.0.0.1:9090/prometheus/")
 	require.NoError(t, err)
 
 	original, err := ioutil.ReadFile(configPath) //nolint:gosec
@@ -518,3 +518,94 @@ scrape_configs:
 		assert.Equal(t, expected, string(actual), "actual:\n%s", actual)
 	})
 }
+
+func TestBasePrometheusConfig(t *testing.T) {
+	t.Run("Default", func(t *testing.T) {
+		db, svc, original := setup(t)
+		defer teardown(t, db, svc, original)
+
+		svc.baseConfigPath = baseConfigPath
+
+		expected := `# Managed by pmm-managed. DO NOT EDIT.
+---
+global:
+  scrape_interval: 9m
+  scrape_timeout: 19s
+  evaluation_interval: 9m
+rule_files:
+- /srv/prometheus/rules/test.rules.yml
+- /srv/prometheus/rules/*.rules.yml
+scrape_configs:
+- job_name: mysqld_exporter_agent_id_75bb30d3-ef4a-4147-97a8-621a996611dd_hr-5s
+  params:
+    collect[]:
+    - custom_query.hr
+    - global_status
+    - info_schema.innodb_metrics
+    - standard.go
+    - standard.process
+  scrape_interval: 5s
+  scrape_timeout: 4s
+  metrics_path: /metrics
+  scheme: http
+  static_configs:
+  - targets:
+    - 1.2.3.4:12345
+    labels:
+      _agent_label: baz
+      _node_label: foo
+      _service_label: bar
+      agent_id: /agent_id/75bb30d3-ef4a-4147-97a8-621a996611dd
+      agent_type: mysqld_exporter
+      instance: /agent_id/75bb30d3-ef4a-4147-97a8-621a996611dd
+      node_id: /node_id/cc663f36-18ca-40a1-aea9-c6310bb4738d
+      node_name: test-generic-node
+      node_type: generic
+      service_id: /service_id/014647c3-b2f5-44eb-94f4-d943260a968c
+      service_name: test-mysql
+      service_type: mysql
+  basic_auth:
+    username: pmm
+    password: /agent_id/75bb30d3-ef4a-4147-97a8-621a996611dd
+- job_name: prometheus
+  scrape_interval: 5s
+  scrape_timeout: 4s
+  metrics_path: /prometheus/metrics
+  static_configs:
+  - targets:
+    - 127.0.0.1:9090
+    labels:
+      instance: pmm-server
+- job_name: grafana
+  scrape_interval: 5s
+  scrape_timeout: 4s
+  metrics_path: /metrics
+  static_configs:
+  - targets:
+    - 127.0.0.1:3000
+    labels:
+      instance: pmm-server
+- job_name: pmm-managed
+  scrape_interval: 5s
+  scrape_timeout: 4s
+  metrics_path: /debug/metrics
+  static_configs:
+  - targets:
+    - 127.0.0.1:7773
+    labels:
+      instance: pmm-server
+- job_name: qan-api2
+  scrape_interval: 5s
+  scrape_timeout: 4s
+  metrics_path: /debug/metrics
+  static_configs:
+  - targets:
+    - 127.0.0.1:9933
+    labels:
+      instance: pmm-server
+`
+		newcfg, err := svc.marshalConfig()
+		assert.NoError(t, err)
+		assert.Equal(t, expected, string(newcfg), "actual:\n%s", newcfg)
+	})
+}

testdata/prometheus/prometheus.base.yml

@@ -0,0 +1,37 @@
+global:
+  scrape_interval: 9m
+  scrape_timeout: 19s
+  evaluation_interval: 9m
+rule_files:
+  - /srv/prometheus/rules/test.rules.yml
+scrape_configs:
+  - job_name: mysqld_exporter_agent_id_75bb30d3-ef4a-4147-97a8-621a996611dd_hr-5s
+    params:
+      collect[]:
+        - custom_query.hr
+        - global_status
+        - info_schema.innodb_metrics
+        - standard.go
+        - standard.process
+    scrape_interval: 5s
+    scrape_timeout: 4s
+    metrics_path: /metrics
+    static_configs:
+      - targets:
+          - 1.2.3.4:12345
+        labels:
+          _agent_label: baz
+          _node_label: foo
+          _service_label: bar
+          agent_id: /agent_id/75bb30d3-ef4a-4147-97a8-621a996611dd
+          agent_type: mysqld_exporter
+          instance: /agent_id/75bb30d3-ef4a-4147-97a8-621a996611dd
+          node_id: /node_id/cc663f36-18ca-40a1-aea9-c6310bb4738d
+          node_name: test-generic-node
+          node_type: generic
+          service_id: /service_id/014647c3-b2f5-44eb-94f4-d943260a968c
+          service_name: test-mysql
+          service_type: mysql
+    basic_auth:
+      username: pmm
+      password: /agent_id/75bb30d3-ef4a-4147-97a8-621a996611dd