From c6a29e0baa8b1716c49c02028dbbfccdd771c932 Mon Sep 17 00:00:00 2001 From: yeya24 Date: Thu, 5 Sep 2019 13:30:52 -0400 Subject: [PATCH] add some cdn metrics Signed-off-by: yeya24 --- docs/user_guide/metrics.md | 7 +++-- supernode/daemon/mgr/cdn/downloader_test.go | 3 +- supernode/daemon/mgr/cdn/manager.go | 31 +++++++++++++++++++-- supernode/daemon/mgr/task/manager.go | 4 +-- supernode/server/server.go | 2 +- 5 files changed, 39 insertions(+), 8 deletions(-) diff --git a/docs/user_guide/metrics.md b/docs/user_guide/metrics.md index 8c8fddbb6..692c5f13d 100644 --- a/docs/user_guide/metrics.md +++ b/docs/user_guide/metrics.md @@ -16,8 +16,11 @@ This doc contains all the metrics that Dragonfly components currently support. N - dragonfly_supernode_dfgettasks_registered_total{callsystem} - total times of registering new dfgettasks. counter type. - dragonfly_supernode_dfgettasks_failed_total{callsystem} - total times of failed dfgettasks. counter type. - dragonfly_supernode_schedule_duration_milliseconds{peer} - duration for task scheduling in milliseconds -- dragonfly_supernode_trigger_cdn_total{} - total times of triggering cdn. -- dragonfly_supernode_trigger_cdn_failed_total{} - total failed times of triggering cdn. +- dragonfly_supernode_cdn_trigger_total{} - total times of triggering cdn. counter type. +- dragonfly_supernode_cdn_trigger_total{} - total failed times of triggering cdn. counter type. +- dragonfly_supernode_cdn_cache_hit_total{} - total times of hitting cdn cache. counter type. +- dragonfly_supernode_cdn_download_total{} - total times of cdn downloading. counter type. +- dragonfly_supernode_cdn_download_failed_total{} - total failure times of cdn downloading. counter type. ## Dfdaemon diff --git a/supernode/daemon/mgr/cdn/downloader_test.go b/supernode/daemon/mgr/cdn/downloader_test.go index 4ad381d8b..7ba4c0e74 100644 --- a/supernode/daemon/mgr/cdn/downloader_test.go +++ b/supernode/daemon/mgr/cdn/downloader_test.go @@ -31,6 +31,7 @@ import ( "github.com/dragonflyoss/Dragonfly/supernode/httpclient" "github.com/go-check/check" + "github.com/prometheus/client_golang/prometheus" ) func Test(t *testing.T) { @@ -45,7 +46,7 @@ func init() { } func (s *CDNDownloadTestSuite) TestDownload(c *check.C) { - cm, _ := NewManager(config.NewConfig(), nil, nil, httpclient.NewOriginClient()) + cm, _ := NewManager(config.NewConfig(), nil, nil, httpclient.NewOriginClient(), prometheus.DefaultRegisterer) bytes := []byte("hello world") bytesLength := int64(len(bytes)) diff --git a/supernode/daemon/mgr/cdn/manager.go b/supernode/daemon/mgr/cdn/manager.go index 91cbd68f5..0304d5637 100644 --- a/supernode/daemon/mgr/cdn/manager.go +++ b/supernode/daemon/mgr/cdn/manager.go @@ -23,20 +23,41 @@ import ( "github.com/dragonflyoss/Dragonfly/apis/types" "github.com/dragonflyoss/Dragonfly/pkg/limitreader" + "github.com/dragonflyoss/Dragonfly/pkg/metricsutils" "github.com/dragonflyoss/Dragonfly/pkg/netutils" "github.com/dragonflyoss/Dragonfly/pkg/ratelimiter" "github.com/dragonflyoss/Dragonfly/pkg/stringutils" "github.com/dragonflyoss/Dragonfly/supernode/config" "github.com/dragonflyoss/Dragonfly/supernode/daemon/mgr" + "github.com/dragonflyoss/Dragonfly/supernode/httpclient" "github.com/dragonflyoss/Dragonfly/supernode/store" "github.com/dragonflyoss/Dragonfly/supernode/util" - "github.com/dragonflyoss/Dragonfly/supernode/httpclient" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" ) var _ mgr.CDNMgr = &Manager{} +type metrics struct { + cdnCacheHitCount *prometheus.CounterVec + cdnDownloadCount *prometheus.CounterVec + cdnDownloadFailCount *prometheus.CounterVec +} + +func newMetrics(register prometheus.Registerer) *metrics { + return &metrics{ + cdnCacheHitCount: metricsutils.NewCounter(config.SubsystemSupernode, "cdn_cache_hit_total", + "Total times of hitting cdn cache", []string{}, register), + + cdnDownloadCount: metricsutils.NewCounter(config.SubsystemSupernode, "cdn_download_total", + "Total times of cdn download", []string{}, register), + + cdnDownloadFailCount: metricsutils.NewCounter(config.SubsystemSupernode, "cdn_download_failed_total", + "Total failure times of cdn download", []string{}, register), + } +} + // Manager is an implementation of the interface of CDNMgr. type Manager struct { cfg *config.Config @@ -51,10 +72,12 @@ type Manager struct { originClient httpclient.OriginHTTPClient pieceMD5Manager *pieceMD5Mgr writer *superWriter + metrics *metrics } // NewManager returns a new Manager. -func NewManager(cfg *config.Config, cacheStore *store.Store, progressManager mgr.ProgressMgr, originClient httpclient.OriginHTTPClient) (*Manager, error) { +func NewManager(cfg *config.Config, cacheStore *store.Store, progressManager mgr.ProgressMgr, + originClient httpclient.OriginHTTPClient, register prometheus.Registerer) (*Manager, error) { rateLimiter := ratelimiter.NewRateLimiter(ratelimiter.TransRate(config.TransLimit(cfg.MaxBandwidth-cfg.SystemReservedBandwidth)), 2) metaDataManager := newFileMetaDataManager(cacheStore) pieceMD5Manager := newpieceMD5Mgr() @@ -71,6 +94,7 @@ func NewManager(cfg *config.Config, cacheStore *store.Store, progressManager mgr detector: newCacheDetector(cacheStore, metaDataManager, originClient), originClient: originClient, writer: newSuperWriter(cacheStore, cdnReporter), + metrics: newMetrics(register), }, nil } @@ -95,6 +119,7 @@ func (cm *Manager) TriggerCDN(ctx context.Context, task *types.TaskInfo) (*types if startPieceNum == -1 { logrus.Infof("cache full hit for taskId:%s on local", task.ID) + cm.metrics.cdnCacheHitCount.WithLabelValues().Inc() return updateTaskInfo, nil } @@ -107,7 +132,9 @@ func (cm *Manager) TriggerCDN(ctx context.Context, task *types.TaskInfo) (*types // start to download the source file resp, err := cm.download(ctx, task.ID, task.RawURL, task.Headers, startPieceNum, httpFileLength, pieceContSize) + cm.metrics.cdnDownloadCount.WithLabelValues().Inc() if err != nil { + cm.metrics.cdnDownloadFailCount.WithLabelValues().Inc() return getUpdateTaskInfoWithStatusOnly(types.TaskInfoCdnStatusFAILED), err } defer resp.Body.Close() diff --git a/supernode/daemon/mgr/task/manager.go b/supernode/daemon/mgr/task/manager.go index aa4e8da62..6b3f0d02b 100644 --- a/supernode/daemon/mgr/task/manager.go +++ b/supernode/daemon/mgr/task/manager.go @@ -58,10 +58,10 @@ func newMetrics(register prometheus.Registerer) *metrics { tasksRegisterCount: metricsutils.NewCounter(config.SubsystemSupernode, "tasks_registered_total", "Total times of registering tasks", []string{}, register), - triggerCdnCount: metricsutils.NewCounter(config.SubsystemSupernode, "trigger_cdn_total", + triggerCdnCount: metricsutils.NewCounter(config.SubsystemSupernode, "cdn_trigger_total", "Total times of triggering cdn", []string{}, register), - triggerCdnFailCount: metricsutils.NewCounter(config.SubsystemSupernode, "trigger_cdn_failed_total", + triggerCdnFailCount: metricsutils.NewCounter(config.SubsystemSupernode, "cdn_trigger_failed_total", "Total failure times of triggering cdn", []string{}, register), scheduleDurationMilliSeconds: metricsutils.NewHistogram(config.SubsystemSupernode, "schedule_duration_milliseconds", diff --git a/supernode/server/server.go b/supernode/server/server.go index 7a313f5a0..8b812b286 100644 --- a/supernode/server/server.go +++ b/supernode/server/server.go @@ -87,7 +87,7 @@ func New(cfg *config.Config, register prometheus.Registerer) (*Server, error) { return nil, err } - cdnMgr, err := cdn.NewManager(cfg, storeLocal, progressMgr, originClient) + cdnMgr, err := cdn.NewManager(cfg, storeLocal, progressMgr, originClient, register) if err != nil { return nil, err }