Feature/integrate consul sdk&write tests (#22)

* add token for consul's acl

* add tests for cache

.codeclimate.yml

@@ -0,0 +1,23 @@
+version: "2"
+checks:
+  method-lines:
+    config:
+      threshold: 100
+plugins:
+  fixme:
+    enabled: true
+  gofmt:
+    enabled: true
+  golint:
+    enabled: true
+  govet:
+    enabled: true
+exclude_patterns:
+  - "**/*_test.go"
+  - "*_test.go"
+  - "**_test.go"
+  - "build/*"
+  - "benchmark/*"
+  - "example/*"
+  - "test/*"
+

.github/stale.yml

@@ -1,5 +1,5 @@
 # Number of days of inactivity before an issue becomes stale
-daysUntilStale: 60
+daysUntilStale: 20
 # Number of days of inactivity before a stale issue is closed
 daysUntilClose: 7
 # Issues with these labels will never be considered stale

build/ci.dockerfile

@@ -1,5 +1,5 @@
 FROM alpine:3.11.6
-COPY caddy /app
+COPY caddy /app/caddy
 CMD ["./app/caddy", "run", "--config", "/app/Caddyfile"]
 
 HEALTHCHECK --interval=5s --timeout=10s --start-period=5s \

cache.go

@@ -30,6 +30,9 @@ var (
 	l           sync.RWMutex
 )
 
+// intend to mock for test
+var now = time.Now().UTC
+
 // RuleMatcherType specifies the type of matching rule to cache.
 type RuleMatcherType string
 
@@ -186,7 +189,7 @@ func judgeResponseShouldCacheOrNot(req *http.Request,
 		ReqHeaders:    reqHeaders,
 		ReqMethod:     reqMethod,
 
-		NowUTC: time.Now().UTC(),
+		NowUTC: now(),
 	}
 	rv := cacheobject.ObjectResults{}
 
@@ -206,11 +209,11 @@ func judgeResponseShouldCacheOrNot(req *http.Request,
 func getCacheStatus(req *http.Request, response *Response, config *Config) (bool, time.Time) {
 	// TODO: what does the lock time do, add more rule
 	if response.Code == http.StatusPartialContent || response.snapHeader.Get("Content-Range") != "" {
-		return false, time.Now().Add(config.LockTimeout)
+		return false, now().Add(config.LockTimeout)
 	}
 
 	if response.Code == http.StatusNotModified {
-		return false, time.Now()
+		return false, now()
 	}
 
 	reasonsNotToCache, expiration, _, _, err := judgeResponseShouldCacheOrNot(req, response.Code, response.snapHeader, false)
@@ -220,22 +223,22 @@ func getCacheStatus(req *http.Request, response *Response, config *Config) (bool
 
 	isPublic := len(reasonsNotToCache) == 0
 	if !isPublic {
-		return false, time.Now().Add(config.LockTimeout)
+		return false, now().Add(config.LockTimeout)
 	}
 
-	varyHeader := response.HeaderMap.Get("Vary")
+	varyHeader := response.snapHeader.Get("Vary")
 	if varyHeader == "*" {
-		return false, time.Now().Add(config.LockTimeout)
+		return false, now().Add(config.LockTimeout)
 	}
 
 	for _, rule := range config.RuleMatchers {
 		if !rule.matches(req, response.Code, response.snapHeader) {
-			return false, time.Now()
+			return false, now()
 		}
 	}
 
-	if expiration.Before(time.Now()) {
-		expiration = time.Now().Add(config.DefaultMaxAge)
+	if now().After(expiration.Add(-1 * time.Second)) {
+		expiration = now().Add(config.DefaultMaxAge)
 	}
 
 	return true, expiration

cache_test.go

@@ -0,0 +1,148 @@
+package httpcache
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/suite"
+)
+
+func makeRequest(url string, headers http.Header) *http.Request {
+	r := httptest.NewRequest("GET", url, nil)
+	copyHeaders(headers, r.Header)
+	return r
+}
+
+func makeResponse(code int, headers http.Header) *Response {
+	return &Response{
+		Code:       code,
+		snapHeader: headers,
+	}
+}
+
+func makeHeader(key string, value string) http.Header {
+	h := http.Header{}
+	h.Add(key, value)
+	return h
+}
+
+type CacheStatusTestSuite struct {
+	suite.Suite
+	c *Config
+}
+
+func (suite *CacheStatusTestSuite) SetupSuite() {
+	suite.c = &Config{}
+	suite.c.DefaultMaxAge = 1 * time.Second
+	suite.c.LockTimeout = 5 * time.Hour
+	suite.c.RuleMatchers = []RuleMatcher{
+		&PathRuleMatcher{Path: "/public"},
+	}
+
+	testTime := time.Now().UTC()
+	// monkey patch the origin definition of now
+	now = func() time.Time {
+		return testTime
+	}
+}
+
+func (suite *CacheStatusTestSuite) TearDownSuite() {
+	now = time.Now().UTC
+}
+
+func (suite *CacheStatusTestSuite) TestCacheControlParseError() {
+	// cache-control: https://www.imperva.com/learn/performance/cache-control/#:~:text=Cache%2DControl%3A%20Max%2DAge,another%20request%20to%20a%20server.
+	req := makeRequest("/", http.Header{})
+	res := makeResponse(200, makeHeader("Cache-Control", "max-age=song"))
+	isPublic, expiration := getCacheStatus(req, res, suite.c)
+	suite.False(isPublic)
+	suite.Equal(time.Time{}, expiration)
+}
+
+func (suite *CacheStatusTestSuite) TestCacheControlIsPrivate() {
+	req := makeRequest("/", http.Header{})
+	res := makeResponse(200, makeHeader("Cache-Control", "private"))
+	isPublic, expiration := getCacheStatus(req, res, suite.c)
+	suite.False(isPublic)
+	suite.Equal(now().Add(suite.c.LockTimeout), expiration, "lockTimeout should be returned")
+}
+
+func (suite *CacheStatusTestSuite) TestVaryWildCardInResponseHeader() {
+	// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Vary
+	req := makeRequest("/", http.Header{})
+	res := makeResponse(200, makeHeader("Vary", "*"))
+	isPublic, expiration := getCacheStatus(req, res, suite.c)
+	suite.False(isPublic)
+	suite.Equal(now().Add(suite.c.LockTimeout), expiration)
+}
+
+func (suite *CacheStatusTestSuite) TestUpstreamReturned502() {
+	req := makeRequest("/", http.Header{})
+	res := makeResponse(502, http.Header{})
+	isPublic, _ := getCacheStatus(req, res, suite.c)
+	suite.False(isPublic)
+}
+
+func (suite *CacheStatusTestSuite) TestUpstreamReturned304() {
+	req := makeRequest("/", http.Header{})
+	res := makeResponse(304, http.Header{})
+	isPublic, _ := getCacheStatus(req, res, suite.c)
+	suite.False(isPublic)
+}
+
+func (suite *CacheStatusTestSuite) TestPathMatchedWithExpirationSpecified() {
+	req := makeRequest("/public", http.Header{})
+	res := makeResponse(200, makeHeader("Cache-control", "max-age=5"))
+	isPublic, expiration := getCacheStatus(req, res, suite.c)
+	suite.True(isPublic)
+	suite.Equal(now().Add(time.Duration(5)*time.Second).Round(time.Second), expiration.Round(time.Second))
+}
+
+func (suite *CacheStatusTestSuite) TestPathMatchedWithoutExpirationSpecified() {
+	req := makeRequest("/public", http.Header{})
+	res := makeResponse(200, http.Header{})
+	isPublic, expiration := getCacheStatus(req, res, suite.c)
+	suite.True(isPublic)
+	suite.Equal(now().Add(suite.c.DefaultMaxAge).Round(time.Second), expiration.Round(time.Second))
+}
+
+type RuleMatcherTestSuite struct {
+	suite.Suite
+}
+
+func (suite *RuleMatcherTestSuite) TestPathMatched() {
+	m := PathRuleMatcher{Path: "/"}
+	match := m.matches(makeRequest("/", http.Header{}), 200, http.Header{})
+	suite.True(match)
+}
+
+func (suite *RuleMatcherTestSuite) TestPathNotMatched() {
+	m := PathRuleMatcher{Path: "/is"}
+	match := m.matches(makeRequest("/", http.Header{}), 200, http.Header{})
+	suite.False(match)
+}
+
+func (suite *RuleMatcherTestSuite) TestHeaderMatched() {
+	m := HeaderRuleMatcher{
+		Header: "Content-Type",
+		Value:  []string{"image/png", "image/jpg"}}
+
+	match := m.matches(makeRequest("/", http.Header{}), 200, makeHeader("Content-Type", "image/jpg"))
+	suite.True(match)
+}
+
+func (suite *RuleMatcherTestSuite) TestHeaderNotMatched() {
+	m := HeaderRuleMatcher{
+		Header: "Content-Type",
+		Value:  []string{"image/png", "image/jpg"}}
+
+	match := m.matches(makeRequest("/", http.Header{}), 200, makeHeader("Content-Type", "application/json"))
+	suite.False(match)
+}
+
+func TestCacheStatusTestSuite(t *testing.T) {
+	suite.Run(t, new(CacheStatusTestSuite))
+	suite.Run(t, new(RuleMatcherTestSuite))
+}

extends/distributed/caddyfile.go

@@ -8,13 +8,15 @@ const (
 	keyServiceName = "service_name"
 	keyAddr        = "addr"
 	keyHealthCheck = "health_check"
+	keyToken       = "token"
 )
 
 // Config is the configuration for the consul
 type Config struct {
 	ServiceName string `json:"service_name,omitempty"`
 	Addr        string `json:"addr,omitempty"`
 	HealthURL   string `json:"health_url,omitempty"`
+	Token       string `json:"token,omitempty"`
 }
 
 func getDefaultConfig() *Config {
@@ -48,6 +50,9 @@ func (c *ConsulService) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 			case keyHealthCheck:
 				config.HealthURL = args[0]
 
+			case keyToken:
+				config.Token = args[0]
+
 			default:
 				return d.Errf("unrecognized subdirective %s", parameter)
 

extends/distributed/register.go

@@ -65,6 +65,7 @@ func (c *ConsulService) Provision(ctx caddy.Context) error {
 
 	config := api.DefaultConfig()
 	config.Address = c.Config.Addr
+	config.Token = c.Config.Token
 
 	consulClient, err := api.NewClient(config)
 	if err != nil {
@@ -73,6 +74,7 @@ func (c *ConsulService) Provision(ctx caddy.Context) error {
 
 	c.Client = consulClient
 	c.Catalog = c.Client.Catalog()
+
 	c.KV = c.Client.KV()
 
 	ip, err := helper.IPAddr()

go.mod

@@ -3,6 +3,7 @@ module github.com/sillygod/cdp-cache
 go 1.14
 
 require (
+	github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38
 	github.com/armon/go-metrics v0.3.3 // indirect
 	github.com/caddyserver/caddy/v2 v2.0.0
 	github.com/go-redis/redis v6.15.8+incompatible

readme.org

@@ -49,9 +49,9 @@
       Content-Type: application/json
 
       {
-        "method": "GET" ,
-        "host": "localhost",
-        "uri": ".*\\.txt"
+      "method": "GET" ,
+      "host": "localhost",
+      "uri": ".*\\.txt"
       }
     #+end_src
 ** Support cluster with consul
@@ -123,7 +123,7 @@
     ex.
 
     #+begin_quote
-      match_header Content-Type image/jpg image/png "text/plain; charset=utf-8"
+    match_header Content-Type image/jpg image/png "text/plain; charset=utf-8"
     #+end_quote
 
 *** path
@@ -141,7 +141,23 @@
 *** cache_max_memory_size
 
     The max memory usage for in_memory backend.
+
+*** distributed
+
+    Working in process. Currently, only support =consul= to establish the cluster of cache server node.
+
+    To see a example config, please refer [[file:example/distributed_cache/Caddyfile::health_check ":7777/health"][this]] 
+
+**** service_name
+     specify your service to be registered in the consul agent.
 
+**** addr
+     the address of the consul agent.
+
+**** health_check
+     indicate the health_check endpoint which consul agent will use this endpoint to check the cache server is healthy
+
+
 ** Example configs
    You can go to the directory [[file:example/][example]]. It shows you each type of cache's configuration.
 
@@ -162,8 +178,8 @@
 
 * Todo list
 
+  - [ ] add more tests (first priority)
   - [ ] custom log format (currently only add zap logger to print info)
     Idealy, We can implement a custom log module.
   - [ ] distributed cache (in progress)
-  - [ ] add more tests
   - [ ] more optimization..