From 745a06bf508ad15ee2ed08cbd0aa77dece3a56ca Mon Sep 17 00:00:00 2001 From: yeya24 Date: Thu, 11 Jul 2019 21:45:39 +0800 Subject: [PATCH] add support for prometheus metrics Signed-off-by: yeya24 --- dfdaemon/constant/constant.go | 7 ++++ dfdaemon/handler/root_handler.go | 3 ++ go.mod | 1 + go.sum | 7 ++++ supernode/config/constants.go | 7 ++++ supernode/server/metrics.go | 68 ++++++++++++++++++++++++++++++++ supernode/server/router.go | 22 ++++++++--- supernode/server/router_test.go | 26 +++++++++++- supernode/server/server.go | 2 +- 9 files changed, 134 insertions(+), 9 deletions(-) create mode 100644 supernode/server/metrics.go diff --git a/dfdaemon/constant/constant.go b/dfdaemon/constant/constant.go index ebd292da9..c03aa0bf2 100644 --- a/dfdaemon/constant/constant.go +++ b/dfdaemon/constant/constant.go @@ -44,3 +44,10 @@ const ( // DefaultConfigPath the default path of dfdaemon configuration file. DefaultConfigPath = "/etc/dragonfly/dfdaemon.yml" ) + +const ( + // Namespace is the prefix of the metrics' name of dragonfly + Namespace = "dragonfly" + // Subsystem represents metrics for dfdaemon + Subsystem = "dfdaemon" +) diff --git a/dfdaemon/handler/root_handler.go b/dfdaemon/handler/root_handler.go index 5c725eea1..5473851b7 100644 --- a/dfdaemon/handler/root_handler.go +++ b/dfdaemon/handler/root_handler.go @@ -22,6 +22,8 @@ import ( _ "net/http/pprof" "github.com/dragonflyoss/Dragonfly/version" + + "github.com/prometheus/client_golang/prometheus/promhttp" ) // New returns a new http mux for dfdaemon @@ -30,5 +32,6 @@ func New() *http.ServeMux { s.HandleFunc("/args", getArgs) s.HandleFunc("/env", getEnv) s.HandleFunc("/debug/version", version.Handler) + s.HandleFunc("/metrics", promhttp.Handler().ServeHTTP) return s } diff --git a/go.mod b/go.mod index 78ab9f795..7f5675924 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/pborman/uuid v0.0.0-20180122190007-c65b2f87fee3 github.com/pkg/errors v0.8.0 github.com/prashantv/gostub v1.0.0 + github.com/prometheus/client_golang v0.9.3 github.com/russross/blackfriday v0.0.0-20171011182219-6d1ef893fcb0 // indirect github.com/sirupsen/logrus v1.2.0 github.com/spf13/afero v1.2.2 diff --git a/go.sum b/go.sum index fff4b7184..405f174ed 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,7 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/asaskevich/govalidator v0.0.0-20170903095215-73945b6115bf h1:wXq5VXJjLole37O6oWZwqBRbKZw6VmC+wuAe8j/w2ZA= github.com/asaskevich/govalidator v0.0.0-20170903095215-73945b6115bf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -64,6 +65,7 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -100,6 +102,7 @@ github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzR github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20170902151237-2a92e673c9a6 h1:xhfqLjTK1g6iq92WjkfuaN6bC7Aoxb5//G8IfwyMyYA= github.com/mailru/easyjson v0.0.0-20170902151237-2a92e673c9a6/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -116,12 +119,16 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prashantv/gostub v1.0.0 h1:wTzvgO04xSS3gHuz6Vhuo0/kvWelyJxwNS0IRBPAwGY= github.com/prashantv/gostub v1.0.0/go.mod h1:dP1v6T1QzyGJJKFocwAU0lSZKpfjstjH8TlhkEU0on0= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= diff --git a/supernode/config/constants.go b/supernode/config/constants.go index 22cc70297..781ca6a56 100644 --- a/supernode/config/constants.go +++ b/supernode/config/constants.go @@ -68,3 +68,10 @@ const ( // CDNWriterRoutineLimit 4 CDNWriterRoutineLimit = 4 ) + +const ( + // Namespace is the prefix of the metrics' name of dragonfly + Namespace = "dragonfly" + // Subsystem represents metrics for supernode + Subsystem = "supernode" +) diff --git a/supernode/server/metrics.go b/supernode/server/metrics.go new file mode 100644 index 000000000..304d69909 --- /dev/null +++ b/supernode/server/metrics.go @@ -0,0 +1,68 @@ +package server + +import ( + "net/http" + + "github.com/dragonflyoss/Dragonfly/supernode/config" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// metrics defines three prometheus metrics for monitoring http handler status +type metrics struct { + requestCounter *prometheus.CounterVec + requestDuration *prometheus.HistogramVec + responseSize *prometheus.HistogramVec +} + +func newMetrics() *metrics { + m := &metrics{ + requestCounter: promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: config.Namespace, + Subsystem: config.Subsystem, + Name: "http_requests_total", + Help: "Counter of HTTP requests.", + }, + []string{"handler", "code"}, + ), + requestDuration: promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: config.Namespace, + Subsystem: config.Subsystem, + Name: "http_request_duration_seconds", + Help: "Histogram of latencies for HTTP requests.", + Buckets: []float64{.1, .2, .4, 1, 3, 8, 20, 60, 120}, + }, + []string{"handler"}, + ), + responseSize: promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: config.Namespace, + Subsystem: config.Subsystem, + Name: "http_response_size_bytes", + Help: "Histogram of response size for HTTP requests.", + Buckets: prometheus.ExponentialBuckets(100, 10, 8), + }, + []string{"handler"}, + ), + } + + return m +} + +// instrumentHandler will update metrics for every http request +func (m *metrics) instrumentHandler(handlerName string, handler http.HandlerFunc) http.HandlerFunc { + return promhttp.InstrumentHandlerCounter( + m.requestCounter.MustCurryWith(prometheus.Labels{"handler": handlerName}), + promhttp.InstrumentHandlerDuration( + m.requestDuration.MustCurryWith(prometheus.Labels{"handler": handlerName}), + promhttp.InstrumentHandlerResponseSize( + m.responseSize.MustCurryWith(prometheus.Labels{"handler": handlerName}), + handler, + ), + ), + ) +} diff --git a/supernode/server/router.go b/supernode/server/router.go index 2f65e3267..7a31e6687 100644 --- a/supernode/server/router.go +++ b/supernode/server/router.go @@ -10,14 +10,21 @@ import ( "github.com/dragonflyoss/Dragonfly/version" "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus/promhttp" ) // versionMatcher defines to parse version url path. const versionMatcher = "/v{version:[0-9.]+}" -func initRoute(s *Server) *mux.Router { - r := mux.NewRouter() +// metricsRouter is a wrapper for mux.Router and metrics. +type metricsRouter struct { + router *mux.Router + metrics *metrics +} +func initRoute(s *Server) *metricsRouter { + r := mux.NewRouter() + router := &metricsRouter{r, newMetrics()} handlers := []*HandlerSpec{ // system {Method: http.MethodGet, Path: "/_ping", HandlerFunc: s.ping}, @@ -40,18 +47,21 @@ func initRoute(s *Server) *mux.Router { // register API for _, h := range handlers { if h != nil { - r.Path(versionMatcher + h.Path).Methods(h.Method).Handler(filter(h.HandlerFunc, s)) - r.Path(h.Path).Methods(h.Method).Handler(filter(h.HandlerFunc, s)) + r.Path(versionMatcher + h.Path).Methods(h.Method).Handler(router.metrics.instrumentHandler(versionMatcher+h.Path, filter(h.HandlerFunc))) + r.Path(h.Path).Methods(h.Method).Handler(router.metrics.instrumentHandler(h.Path, filter(h.HandlerFunc))) } } + // metrics + r.Handle("/metrics", router.metrics.instrumentHandler("/metrics", promhttp.Handler().ServeHTTP)) + if s.Config.Debug || s.Config.EnableProfiler { r.PathPrefix("/debug/pprof/").HandlerFunc(pprof.Index) } - return r + return router } -func filter(handler Handler, s *Server) http.HandlerFunc { +func filter(handler Handler) http.HandlerFunc { pctx := context.Background() return func(w http.ResponseWriter, req *http.Request) { diff --git a/supernode/server/router_test.go b/supernode/server/router_test.go index a364ae452..b44896e47 100644 --- a/supernode/server/router_test.go +++ b/supernode/server/router_test.go @@ -17,6 +17,7 @@ import ( "github.com/dragonflyoss/Dragonfly/version" "github.com/go-check/check" + prom_testutil "github.com/prometheus/client_golang/prometheus/testutil" ) func Test(t *testing.T) { @@ -31,6 +32,7 @@ func init() { type RouterTestSuite struct { addr string listener net.Listener + router *metricsRouter } func (rs *RouterTestSuite) SetUpSuite(c *check.C) { @@ -57,10 +59,11 @@ func (rs *RouterTestSuite) SetUpSuite(c *check.C) { OS: runtime.GOOS, GoVersion: runtime.Version(), } - router := initRoute(s) + + rs.router = initRoute(s) rs.listener, err = net.Listen("tcp", rs.addr) c.Check(err, check.IsNil) - go http.Serve(rs.listener, router) + go http.Serve(rs.listener, rs.router.router) } func (rs *RouterTestSuite) TearDownSuite(c *check.C) { @@ -109,3 +112,22 @@ func (rs *RouterTestSuite) TestVersionHandler(c *check.C) { c.Check(err, check.IsNil) c.Check(string(expectDFVersion), check.Equals, string(res)) } + +func (rs *RouterTestSuite) TestHTTPMetrics(c *check.C) { + // ensure /metrics is accessible + code, _, err := cutil.Get("http://"+rs.addr+"/metrics", 0) + c.Check(err, check.IsNil) + c.Assert(code, check.Equals, 200) + + counter := rs.router.metrics.requestCounter + c.Assert(1, check.Equals, + int(prom_testutil.ToFloat64(counter.WithLabelValues("/metrics", strconv.Itoa(http.StatusOK))))) + + for i := 0; i < 5; i++ { + code, _, err := cutil.Get("http://"+rs.addr+"/_ping", 0) + c.Check(err, check.IsNil) + c.Assert(code, check.Equals, 200) + c.Assert(i+1, check.Equals, + int(prom_testutil.ToFloat64(counter.WithLabelValues("/_ping", strconv.Itoa(http.StatusOK))))) + } +} diff --git a/supernode/server/server.go b/supernode/server/server.go index 93be4ccd4..b92ddaf7c 100644 --- a/supernode/server/server.go +++ b/supernode/server/server.go @@ -91,7 +91,7 @@ func (s *Server) Start() error { } server := &http.Server{ - Handler: router, + Handler: router.router, ReadTimeout: time.Minute * 10, ReadHeaderTimeout: time.Minute * 10, IdleTimeout: time.Minute * 10,