From bb26e6c5b6b7c42ffc8c48db7b68f81fef5aec37 Mon Sep 17 00:00:00 2001 From: jing Date: Tue, 4 Aug 2020 17:46:42 +0800 Subject: [PATCH] Feature/integrate influxdata (#29) * add more tests for cache * fix cache's exipration mechanism broken and add purge key test --- backends/redis_test.go | 12 +- cache.go | 4 +- cache_test.go | 42 ++++- caddyfile.go | 29 --- purge.go => endpoint.go | 45 ++++- endpoint_test.go | 185 ++++++++++++++++++++ example/influxdb_monitor/Caddyfile | 1 + example/influxdb_monitor/Caddyfile.json | 120 +++++++++++++ example/influxdb_monitor/docker-compose.yml | 67 ++++--- example/influxdb_monitor/readme.org | 36 +++- extends/influxlog/influx.go | 42 ++++- go.mod | 1 + go.sum | 7 + pkg/helper/helper.go | 28 ++- readme.org | 2 - response.go | 1 - 16 files changed, 539 insertions(+), 83 deletions(-) rename purge.go => endpoint.go (80%) create mode 100644 endpoint_test.go create mode 100644 example/influxdb_monitor/Caddyfile.json diff --git a/backends/redis_test.go b/backends/redis_test.go index 7812e9f..9216668 100644 --- a/backends/redis_test.go +++ b/backends/redis_test.go @@ -24,16 +24,22 @@ func (suite *RedisBackendTestSuite) SetupSuite() { log.Fatal(err) } - resource, err := pool.Run("redis", "5.0.9", []string{}) + redisVersion := "5.0.9" + resource, err := pool.Run("redis", redisVersion, []string{}) if err != nil { log.Fatal(err) } suite.pool = pool suite.resource = resource - port := resource.GetPort("6379/tcp") - InitRedisClient(fmt.Sprintf("localhost:%s", port), "", 0) + + err = suite.pool.Retry(func() error { + return InitRedisClient(fmt.Sprintf("localhost:%s", port), "", 0) + }) + if err != nil { + log.Fatal(err) + } } func (suite *RedisBackendTestSuite) TestParseRedisConfig() { diff --git a/cache.go b/cache.go index f8080b8..32e4a62 100644 --- a/cache.go +++ b/cache.go @@ -31,7 +31,7 @@ var ( ) // intend to mock for test -var now = time.Now().UTC +var now = func() time.Time { return time.Now().UTC() } // RuleMatcherType specifies the type of matching rule to cache. type RuleMatcherType string @@ -431,7 +431,7 @@ func (h *HTTPCache) Get(key string, request *http.Request) (*Entry, bool) { return nil, false } -// Keys list the keys holded by this cache +// Keys list the keys holden by this cache func (h *HTTPCache) Keys() []string { keys := []string{} for index, l := range h.entriesLock { diff --git a/cache_test.go b/cache_test.go index cf1c65c..b6a8de7 100644 --- a/cache_test.go +++ b/cache_test.go @@ -202,15 +202,54 @@ func (suite *EntryTestSuite) TearDownSuite() { type HTTPCacheTestSuite struct { suite.Suite config *Config + cache *HTTPCache } func (suite *HTTPCacheTestSuite) SetupSuite() { err := backends.InitGroupCacheRes(50 * 1024 * 1024) suite.Nil(err) suite.config = getDefaultConfig() + suite.cache = NewHTTPCache(suite.config) } -// TODO: add http cache test +func (suite *HTTPCacheTestSuite) TestGetNonExistEntry() { + req := makeRequest("/", http.Header{}) + entry, exists := suite.cache.Get("abc", req) + suite.Nil(entry) + suite.False(exists) +} + +func (suite *HTTPCacheTestSuite) TestGetExistEntry() { + req := makeRequest("/", http.Header{}) + res := makeResponse(200, http.Header{}) + entry := NewEntry("hello", req, res, suite.config) + suite.cache.Put(req, entry) + + prevEntry, exists := suite.cache.Get("hello", req) + suite.Equal(prevEntry, entry) + suite.True(exists) +} + +func (suite *HTTPCacheTestSuite) TestCleanEntry() { + req := makeRequest("/", http.Header{}) + res := makeResponse(200, http.Header{}) + key := "friday" + + entry := NewEntry(key, req, res, suite.config) + suite.cache.Put(req, entry) + + keyInKeys := false + keys := suite.cache.Keys() + for _, k := range keys { + if k == key { + keyInKeys = true + } + } + suite.True(keyInKeys) + + err := suite.cache.Del(key) + suite.Nil(err) +} func (suite *HTTPCacheTestSuite) TearDownSuite() { err := backends.ReleaseGroupCacheRes() @@ -219,6 +258,7 @@ func (suite *HTTPCacheTestSuite) TearDownSuite() { func TestCacheStatusTestSuite(t *testing.T) { suite.Run(t, new(CacheStatusTestSuite)) + suite.Run(t, new(HTTPCacheTestSuite)) suite.Run(t, new(RuleMatcherTestSuite)) suite.Run(t, new(EntryTestSuite)) } diff --git a/caddyfile.go b/caddyfile.go index dd8f47d..346b74b 100644 --- a/caddyfile.go +++ b/caddyfile.go @@ -250,13 +250,6 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { h.DistributedRaw = caddyconfig.JSONModuleObject(unm, "distributed", "consul", nil) - // case keyInfluxLog: - // raw, err := setupInfluxLog(keyInfluxLog, d, args) - // if err != nil { - // return err - // } - // h.InfluxLogRaw = raw - default: return d.Err("Unknown cache parameter: " + parameter) } @@ -268,28 +261,6 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { return nil } -func setupInfluxLog(key string, d *caddyfile.Dispenser, args []string) (json.RawMessage, error) { - - mod, err := caddy.GetModule("caddy.logging.writers." + key) - if err != nil { - return nil, d.Errf("getting influxlog module '%s': '%v'", mod, err) - } - - unm, ok := mod.New().(caddyfile.Unmarshaler) - if !ok { - return nil, d.Errf("influxlog module '%s' is not a Caddyfile unmarshaler", mod) - } - - err = unm.UnmarshalCaddyfile(d.NewFromNextSegment()) - if err != nil { - return nil, err - } - - raw := caddyconfig.JSONModuleObject(unm, "mylog", "influxlog", nil) - - return raw, nil -} - // Interface guards var ( _ caddyfile.Unmarshaler = (*Handler)(nil) diff --git a/purge.go b/endpoint.go similarity index 80% rename from purge.go rename to endpoint.go index f42fd1c..7c62217 100644 --- a/purge.go +++ b/endpoint.go @@ -11,6 +11,7 @@ import ( "sync" "github.com/caddyserver/caddy/v2" + "github.com/sillygod/cdp-cache/pkg/helper" ) var ( @@ -65,8 +66,8 @@ func (p *PurgePayload) parseURI() { func (p *PurgePayload) pruneHost() { - if strings.HasPrefix(p.Host, "http") { - p.Host = strings.Split(p.Host, ":")[1] + if strings.HasPrefix(p.Host, "http") || strings.HasPrefix(p.Host, "https") { + p.Host = strings.Split(p.Host, "//")[1] } if !strings.HasSuffix(p.Host, "/") { @@ -120,8 +121,8 @@ func (c cachePurge) Routes() []caddy.AdminRoute { Handler: caddy.AdminHandlerFunc(c.handlePurge), }, { - Pattern: "/caches", - Handler: caddy.AdminHandlerFunc(c.handleListCacheKeys), + Pattern: "/caches/", + Handler: caddy.AdminHandlerFunc(c.handleCacheEndpoints), }, } } @@ -132,7 +133,43 @@ func health(w http.ResponseWriter, r *http.Request) error { return nil } +func (c cachePurge) handleShowCache(w http.ResponseWriter, r *http.Request) error { + var err error + + if r.Method != http.MethodGet { + return caddy.APIError{ + Code: http.StatusMethodNotAllowed, + Err: fmt.Errorf("method not allowed"), + } + } + + key := helper.TrimBy(r.URL.Path, "/", 2) + cache := getHandlerCache() + + entry, exists := cache.Get(key, r) + if exists { + err = entry.WriteBodyTo(w) + } + + return err +} + +func (c cachePurge) handleCacheEndpoints(w http.ResponseWriter, r *http.Request) error { + // a workaround for handling the wildcard endpoint. Caddy uses the standard library's mux + // so it doesn't support this natively. + + path := r.URL.Path + + switch path { + case "/caches/": + return c.handleListCacheKeys(w, r) + default: + return c.handleShowCache(w, r) + } +} + func (c cachePurge) handleListCacheKeys(w http.ResponseWriter, r *http.Request) error { + if r.Method != http.MethodGet { return caddy.APIError{ Code: http.StatusMethodNotAllowed, diff --git a/endpoint_test.go b/endpoint_test.go new file mode 100644 index 0000000..63aa91f --- /dev/null +++ b/endpoint_test.go @@ -0,0 +1,185 @@ +package httpcache + +import ( + "bytes" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strings" + "testing" + + "github.com/caddyserver/caddy/v2/caddytest" + "github.com/stretchr/testify/suite" +) + +type CacheEndpointTestSuite struct { + suite.Suite + caddyTester *caddytest.Tester + url string +} + +func (suite *CacheEndpointTestSuite) assertKeyNotIn(key string, keys []string, msgAndArgs ...interface{}) { + exists := false + + for _, k := range keys { + if k == key { + exists = true + } + } + + suite.False(exists, msgAndArgs) +} + +func (suite *CacheEndpointTestSuite) assertKeyIn(key string, keys []string, msgAndArgs ...interface{}) { + exists := false + + for _, k := range keys { + if k == key { + exists = true + } + } + + suite.True(exists, msgAndArgs) +} + +func (suite *CacheEndpointTestSuite) SetupSuite() { + suite.caddyTester = caddytest.NewTester(suite.T()) + suite.url = "http://localhost:9898/hello" + suite.caddyTester.InitServer(` + { + order http_cache before reverse_proxy + admin 0.0.0.0:7777 + } + + :9898 { + + reverse_proxy { + to localhost:9988 + } + + http_cache { + cache_type in_memory + } + + } + + :9988 { + respond /hello 200 { + body "hope anything will be good" + } + } + + `, "caddyfile") +} + +func (suite *CacheEndpointTestSuite) TestListCacheKeys() { + r, err := http.NewRequest("GET", suite.url, nil) + suite.Assert().NoError(err) + + res, err := suite.caddyTester.Client.Do(r) + suite.Assert().NoError(err) + // create the cache first + + r, err = http.NewRequest("GET", "http://localhost:7777/caches", nil) + suite.Assert().NoError(err) + + res, err = suite.caddyTester.Client.Do(r) + suite.Assert().NoError(err) + + result, err := ioutil.ReadAll(res.Body) + res.Body.Close() + suite.Assert().NoError(err) + suite.True(strings.Contains(string(result), "GET localhost/hello?")) + +} + +func (suite *CacheEndpointTestSuite) TestHealthCheck() { + r, err := http.NewRequest("GET", "http://localhost:7777/health", nil) + suite.Assert().NoError(err) + _, err = suite.caddyTester.Client.Do(r) + suite.Assert().NoError(err) +} + +func (suite *CacheEndpointTestSuite) TestShowCache() { + r, err := http.NewRequest("GET", suite.url, nil) + suite.Assert().NoError(err) + + _, err = suite.caddyTester.Client.Do(r) + suite.Assert().NoError(err) + // create the cache first + url := fmt.Sprintf("http://localhost:7777/caches/%s", url.PathEscape("GET localhost/hello?")) + + r, err = http.NewRequest("GET", url, nil) + suite.Assert().NoError(err) + + res, err := suite.caddyTester.Client.Do(r) + suite.Assert().NoError(err) + + result, err := ioutil.ReadAll(res.Body) + res.Body.Close() + suite.Assert().NoError(err) + suite.True(strings.Contains(string(result), "hope anything will be good"), fmt.Sprintf("result: %s", string(result))) +} + +func (suite *CacheEndpointTestSuite) TestPurgeCache() { + + testdata := []struct { + uri string + host string + body []byte + cacheKey string + }{ + { + host: "http://localhost:9898/", + uri: "hello", + body: []byte(`{"method": "GET", "host": "http://localhost", "uri": "hello"}`), + cacheKey: "GET localhost/hello?", + }, + { + host: "http://localhost:9898/", + uri: "hello?abc.txt", + body: []byte(`{"host": "http://localhost", "uri": "hello?abc.txt"}`), // default method is GET + cacheKey: "GET localhost/hello?abc.txt", + }, + { + host: "http://localhost:9898/", + uri: "hello", + body: []byte(`{"host": "http://localhost/", "uri": "hello"}`), // host with trailing forward slash is also ok + cacheKey: "GET localhost/hello?", + }, + } + + for _, data := range testdata { + + r, err := http.NewRequest("GET", data.host+data.uri, nil) + suite.Assert().NoError(err) + + _, err = suite.caddyTester.Client.Do(r) + suite.Assert().NoError(err) + // create the cache first + + r, err = http.NewRequest("DELETE", "http://localhost:7777/caches/purge", bytes.NewBuffer(data.body)) + suite.Assert().NoError(err) + r.Header.Set("Content-Type", "application/json") + + cache = getHandlerCache() + keys := cache.Keys() + + suite.assertKeyIn(data.cacheKey, keys, fmt.Sprintf("%s should be in keys: %s", data.cacheKey, keys)) + + res, err := suite.caddyTester.Client.Do(r) + suite.Assert().NoError(err) + suite.Equal(200, res.StatusCode) + + // now, the cache is deleted so the key is not existed. + keys = cache.Keys() + suite.assertKeyNotIn(data.cacheKey, keys, fmt.Sprintf("%s should not be in keys: %s", data.cacheKey, keys)) + + } + +} + +func TestCacheEndpoingTestSuite(t *testing.T) { + suite.Run(t, new(CacheEndpointTestSuite)) +} diff --git a/example/influxdb_monitor/Caddyfile b/example/influxdb_monitor/Caddyfile index bdb3def..9c758ed 100644 --- a/example/influxdb_monitor/Caddyfile +++ b/example/influxdb_monitor/Caddyfile @@ -13,6 +13,7 @@ cache_type in_memory match_path / + default_max_age 1m } log { diff --git a/example/influxdb_monitor/Caddyfile.json b/example/influxdb_monitor/Caddyfile.json new file mode 100644 index 0000000..c4d580a --- /dev/null +++ b/example/influxdb_monitor/Caddyfile.json @@ -0,0 +1,120 @@ +{ + "admin": { + "listen": "0.0.0.0:7777" + }, + "apps": { + "http": { + "servers": { + "srv0": { + "listen": [ + ":9991" + ], + "logs": { + "default_logger_name": "log0" + }, + "routes": [ + { + "handle": [ + { + "config": { + "cache_buckets_num": 256, + "cache_key_template": "{http.request.method} {http.request.host}{http.request.uri.path}?{http.request.uri.query}", + "cache_max_memory_size": 1073741824, + "default_max_age": 300000000000, + "lock_timeout": 300000000000, + "path": "/tmp/caddy_cache", + "redis_connection_setting": "localhost:6379 0", + "rule_matcher_raws": [ + { + "Data": { + "path": "/" + }, + "Type": "path" + } + ], + "status_header": "X-Cache-Status", + "type": "in_memory" + }, + "handler": "http_cache" + }, + { + "handler": "reverse_proxy", + "upstreams": [ + { + "dial": "localhost:9995" + } + ] + } + ] + } + ] + }, + "srv1": { + "listen": [ + ":9995" + ], + "logs": { + "default_logger_name": "log1" + }, + "routes": [ + { + "handle": [ + { + "handler": "vars", + "root": "/tmp/caddy-benchmark" + }, + { + "handler": "headers", + "response": { + "set": { + "Cache-Control": [ + "public" + ] + } + } + }, + { + "handler": "file_server", + "hide": [ + "/Users/jing/Desktop/cdp-cache/example/influxdb_monitor/Caddyfile" + ] + } + ] + } + ] + } + } + } + }, + "logging": { + "logs": { + "default": { + "level": "info" + }, + "log0": { + "include": [ + "http.log.access.log0", + "http.handlers.http_cache" + ], + "level": "debug", + "writer": { + "Client": null, + "Config": { + "addr": "http://localhost:8086", + "bucket": "telegraf", + "organization": "", + "token": "user:pass1234" + }, + "output": "influxlog" + } + }, + + "log1": { + "include": [ + "http.log.access.log1" + ], + "level": "info" + } + } + } +} \ No newline at end of file diff --git a/example/influxdb_monitor/docker-compose.yml b/example/influxdb_monitor/docker-compose.yml index 7552de3..c2b5c37 100644 --- a/example/influxdb_monitor/docker-compose.yml +++ b/example/influxdb_monitor/docker-compose.yml @@ -16,27 +16,26 @@ services: networks: - influxdata - ports: - "8086:8086" - "8082:8082" # telegraf to collects the syslog - telegraf: - image: telegraf:1.15.1 - environment: - - HOSTNAME=songa-telegraf - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ./example/influxdb_monitor/telegraf.conf:/etc/telegraf/telegraf.conf + # telegraf: + # image: telegraf:1.15.1 + # environment: + # - HOSTNAME=songa-telegraf + # volumes: + # - /var/run/docker.sock:/var/run/docker.sock + # - ./example/influxdb_monitor/telegraf.conf:/etc/telegraf/telegraf.conf - restart: always + # restart: always - networks: - - influxdata + # networks: + # - influxdata - depends_on: - - influxdb + # depends_on: + # - influxdb chronograf: image: chronograf:1.8.4 @@ -54,28 +53,37 @@ services: depends_on: - influxdb - - telegraf # in the future add the grafana log panel + grafana: + image: grafana/grafana:7.1.1 + volumes: + - grafana:/var/lib/grafana + ports: + - "3000:3000" + networks: + - influxdata + depends_on: + - influxdb # cdp to write the log as syslog - cdp: &cdp-source - image: golang:1.14.2 - container_name: cdp - volumes: - - cdp:/go/ - - go-build-cache:/root/.cache/go-build - - .:${PROJECT_PATH} + # cdp: &cdp-source + # image: golang:1.14.2 + # container_name: cdp + # volumes: + # - cdp:/go/ + # - go-build-cache:/root/.cache/go-build + # - .:${PROJECT_PATH} - command: > - sh -c "echo 'spin up caddy proxy cache...' && - cd ${PROJECT_PATH} && - go run $PROJECT_PATH/cmd/main.go run --config example/influxdb_monitor/Caddyfile" + # command: > + # sh -c "echo 'spin up caddy proxy cache...' && + # cd ${PROJECT_PATH} && + # go run $PROJECT_PATH/cmd/main.go run --config example/influxdb_monitor/Caddyfile" - ports: - - "9991:9991" - environment: - - PROJECT_PATH=${PROJECT_PATH} + # ports: + # - "9991:9991" + # environment: + # - PROJECT_PATH=${PROJECT_PATH} volumes: @@ -83,6 +91,7 @@ volumes: go-build-cache: influxdb: chronograf: + grafana: networks: influxdata: diff --git a/example/influxdb_monitor/readme.org b/example/influxdb_monitor/readme.org index e4d6571..0d0e58a 100644 --- a/example/influxdb_monitor/readme.org +++ b/example/influxdb_monitor/readme.org @@ -1,7 +1,35 @@ * Influxdb Monitor Example -Let's try to monitor our cache server with influxdb + keep notes for manually set the influxLog -#+begin_src sh -PROJECT_PATH=/app docker-compose --project-directory=./ -f example/influxdb_monitor/docker-compose.yml up -#+end_src + #+begin_src go + func setupInfluxLog(key string, d *caddyfile.Dispenser, args []string) (json.RawMessage, error) { + + mod, err := caddy.GetModule("caddy.logging.writers." + key) + if err != nil { + return nil, d.Errf("getting influxlog module '%s': '%v'", mod, err) + } + + unm, ok := mod.New().(caddyfile.Unmarshaler) + if !ok { + return nil, d.Errf("influxlog module '%s' is not a Caddyfile unmarshaler", mod) + } + + err = unm.UnmarshalCaddyfile(d.NewFromNextSegment()) + if err != nil { + return nil, err + } + + raw := caddyconfig.JSONModuleObject(unm, "mylog", "influxlog", nil) + + return raw, nil + } + #+end_src + + Let's try to monitor our cache server with influxdb + + #+begin_src sh + PROJECT_PATH=/app docker-compose --project-directory=./ -f example/influxdb_monitor/docker-compose.yml up + #+end_src + + Note: I can't find any way to customize the log configuration for the ~include~, ~exclude~ or other detailed settings in the ~Caddyfile~. Therefore, I decide to use json config which is more configurable. diff --git a/extends/influxlog/influx.go b/extends/influxlog/influx.go index 15da14d..e7a7d01 100644 --- a/extends/influxlog/influx.go +++ b/extends/influxlog/influx.go @@ -1,10 +1,14 @@ package influxlog import ( + "encoding/json" "fmt" "io" + "strings" "time" + "github.com/sillygod/cdp-cache/pkg/helper" + "github.com/caddyserver/caddy/v2" influxdb2 "github.com/influxdata/influxdb-client-go" influxdb2api "github.com/influxdata/influxdb-client-go/api" @@ -18,19 +22,43 @@ type influxWriteCloser struct { api influxdb2api.WriteAPI } -func (i *influxWriteCloser) Write(b []byte) (int, error) { +func (i *influxWriteCloser) write(b []byte) (int, error) { + // TODO: consider to do this in background worker n := len(b) content := string(b) - // TODO: parse the content {time} {level} {module} {messages} - // Remember to rewrite the function below in next PR. - fmt.Println(content) - tags := map[string]string{"ip": "135.22.5.3"} - fields := map[string]interface{}{"Country": "HI"} - p := influxdb2.NewPoint("syslog", tags, fields, time.Now()) + // parse the content {time} {level} {module} {action} {messages} + caddy.Log().Named("influxlog will write").Debug(content) + + var ts time.Time + var err error + + tags := map[string]string{} + fields := map[string]interface{}{} + tokens := strings.Split(content, "\t") + + if len(tokens) == 5 { + ts, err = time.Parse(helper.LogUTCTimeFormat, tokens[0]) + if err != nil { + ts = time.Now() + } + err := json.Unmarshal([]byte(tokens[4]), &fields) + fmt.Println(tokens[4]) + if err != nil { + fmt.Println(err) + return 0, err + } + fields["message"] = fmt.Sprintf("%s %s", tokens[3], tokens[4]) + } + + p := influxdb2.NewPoint("syslog", tags, fields, ts) i.api.WritePoint(p) return n, nil } +func (i *influxWriteCloser) Write(b []byte) (int, error) { + return i.write(b) +} + func (i *influxWriteCloser) Close() error { i.api.Flush() i.api.Close() diff --git a/go.mod b/go.mod index cfbc3bf..892a2b8 100644 --- a/go.mod +++ b/go.mod @@ -28,4 +28,5 @@ require ( go.uber.org/zap v1.15.0 golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect + golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d // indirect ) diff --git a/go.sum b/go.sum index 1b7985f..fa926f9 100644 --- a/go.sum +++ b/go.sum @@ -128,6 +128,7 @@ github.com/armon/go-metrics v0.3.3/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a h1:pv34s756C4pEXnjgPfGYgdhg/ZdajGhyOvzx8k+23nw= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -1016,6 +1017,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180611182652-db08ff08e862/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1074,6 +1077,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180622082034-63fc586f45fe/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1198,6 +1203,8 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb h1:iKlO7ROJc6SttHKlxzwGytRtBUqX4VARrNTgP2YLX5M= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d h1:szSOL78iTCl0LF1AMjhSWJj8tIM0KixlUUnBtYXsmd8= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/pkg/helper/helper.go b/pkg/helper/helper.go index 519cdd0..6b8d6e5 100644 --- a/pkg/helper/helper.go +++ b/pkg/helper/helper.go @@ -1,6 +1,11 @@ package helper -import "net" +import ( + "net" + "strings" +) + +var LogUTCTimeFormat = "2006/01/02 15:04:05" // IPAddr get the local node's ip address func IPAddr() (net.IP, error) { @@ -18,3 +23,24 @@ func IPAddr() (net.IP, error) { } return nil, nil } + +// TrimBy trims the string with provided substring and repeat +// it multiple times decided by the count. +// ex. +// TrimBy("/localhost/caches/abc.txt", "/", 2) +// will get "caches/abc.txt" +func TrimBy(str, substr string, count int) string { + tmp := 0 + + for { + if tmp == count { + break + } + + index := strings.Index(str, substr) + str = str[index+1:] + tmp++ + } + + return str +} diff --git a/readme.org b/readme.org index 7638e15..311aedc 100644 --- a/readme.org +++ b/readme.org @@ -186,7 +186,5 @@ In the same time, write some fragement benchmark to find out which part can be optimized go test -bench=. -benchmem -cpuprofile profile.out go tool pprof profile.out (in the interactive mod, you can type web to open a web interface to see the graph) - - [ ] custom log format (currently only add zap logger to print info) - Idealy, We can implement a custom log module. - [ ] distributed cache (in progress) - [ ] more optimization.. diff --git a/response.go b/response.go index 868c0ab..ab6c569 100644 --- a/response.go +++ b/response.go @@ -146,7 +146,6 @@ func (r *Response) WriteHeader(code int) { } func shouldUseCache(req *http.Request) bool { - // TODO Add more logic like get params, ?nocache=true if req.Method != "GET" && req.Method != "HEAD" { // Only cache Get and head request