Merge branch 'master' into error-log-enhance

# Conflicts:
#	api/internal/core/storage/etcd.go
#	api/internal/core/store/validate.go
#	api/main.go
#	api/test/shell/cli_test.sh

api/conf/conf.go

@@ -45,7 +45,7 @@ var (
 	WorkDir          = "."
 	ServerHost       = "127.0.0.1"
 	ServerPort       = 80
-	ETCDEndpoints    = []string{"127.0.0.1:2379"}
+	ETCDConfig       *Etcd
 	ErrorLogLevel    = "warn"
 	ErrorLogPath     = "logs/error.log"
 	UserList         = make(map[string]User, 2)
@@ -55,6 +55,8 @@ var (
 
 type Etcd struct {
 	Endpoints []string
+	Username  string
+	Password  string
 }
 
 type Listen struct {
@@ -128,9 +130,9 @@ func setConf() {
 			ServerHost = config.Conf.Listen.Host
 		}
 
-		//etcd
+		// for etcd
 		if len(config.Conf.Etcd.Endpoints) > 0 {
-			ETCDEndpoints = config.Conf.Etcd.Endpoints
+			initEtcdConfig(config.Conf.Etcd)
 		}
 
 		//error log
@@ -180,3 +182,17 @@ func initSchema() {
 		Schema = gjson.ParseBytes(schemaContent)
 	}
 }
+
+// initialize etcd config
+func initEtcdConfig(conf Etcd) {
+	var endpoints = []string{"127.0.0.1:2379"}
+	if len(conf.Endpoints) > 0 {
+		endpoints = conf.Endpoints
+	}
+
+	ETCDConfig = &Etcd{
+		Endpoints: endpoints,
+		Username:  conf.Username,
+		Password:  conf.Password,
+	}
+}

api/conf/conf.yaml

@@ -22,6 +22,10 @@ conf:
   etcd:
     endpoints:          # supports defining multiple etcd host addresses for an etcd cluster
       - 127.0.0.1:2379
+
+                        # etcd basic auth info
+    # username: "root"    # ignore etcd username if not enable etcd auth
+    # password: "123456"  # ignore etcd password if not enable etcd auth
   log:
     error_log:
       level: warn       # supports levels, lower to higher: debug, info, warn, error, panic, fatal

api/go.sum

@@ -118,6 +118,7 @@ github.com/shiningrush/droplet v0.2.1 h1:p2utttTbCfgiL+x0Zrb2jFeWspB7/o+v3e+R94G
 github.com/shiningrush/droplet v0.2.1/go.mod h1:akW2vIeamvMD6zj6wIBfzYn6StGXBxwlW3gA+hcHu5M=
 github.com/shiningrush/droplet v0.2.2 h1:jEqSGoJXlybt1yQdauu+waE+l7KYlguNs8VayMfQ96Q=
 github.com/shiningrush/droplet v0.2.2/go.mod h1:akW2vIeamvMD6zj6wIBfzYn6StGXBxwlW3gA+hcHu5M=
+github.com/shiningrush/droplet v0.2.3 h1:bzPDzkE0F54r94XsultGS8uAPeL3pZIRmjqM0zIlpeI=
 github.com/shiningrush/droplet v0.2.3/go.mod h1:akW2vIeamvMD6zj6wIBfzYn6StGXBxwlW3gA+hcHu5M=
 github.com/shiningrush/droplet/wrapper/gin v0.2.0 h1:LHkU+TbSkpePgXrTg3hJoSZlCMS03GeWMl0t+oLkd44=
 github.com/shiningrush/droplet/wrapper/gin v0.2.0/go.mod h1:ZJu+sCRrVXn5Pg618c1KK3Ob2UiXGuPM1ROx5uMM9YQ=

api/internal/core/storage/etcd.go

@@ -23,6 +23,7 @@ import (
 
 	"go.etcd.io/etcd/clientv3"
 
+	"github.com/apisix/manager-api/conf"
 	"github.com/apisix/manager-api/internal/utils"
 	"github.com/apisix/manager-api/log"
 )
@@ -34,10 +35,12 @@ var (
 type EtcdV3Storage struct {
 }
 
-func InitETCDClient(endpoints []string) error {
+func InitETCDClient(etcdConf *conf.Etcd) error {
 	cli, err := clientv3.New(clientv3.Config{
-		Endpoints:   endpoints,
+		Endpoints:   etcdConf.Endpoints,
 		DialTimeout: 5 * time.Second,
+		Username:    etcdConf.Username,
+		Password:    etcdConf.Password,
 	})
 	if err != nil {
 		log.Errorf("init etcd failed: %s", err)

api/internal/core/store/validate.go

@@ -17,10 +17,10 @@
 package store
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"regexp"
 
 	"github.com/xeipuuv/gojsonschema"
 	"go.uber.org/zap/buffer"
@@ -252,32 +252,52 @@ func (v *APISIXJsonSchemaValidator) Validate(obj interface{}) error {
 	}
 
 	plugins, schemaType := getPlugins(obj)
-	//fix lua json.encode transform lua{properties={}} to json{"properties":[]}
-	reg := regexp.MustCompile(`\"properties\":\[\]`)
 	for pluginName, pluginConf := range plugins {
-		var schemaDef string
-		schemaDef = conf.Schema.Get("plugins." + pluginName + "." + schemaType).String()
-		if schemaDef == "" && schemaType == "consumer_schema" {
-			schemaDef = conf.Schema.Get("plugins." + pluginName + ".schema").String()
+		schemaValue := conf.Schema.Get("plugins." + pluginName + "." + schemaType).Value()
+		if schemaValue == nil && schemaType == "consumer_schema" {
+			schemaValue = conf.Schema.Get("plugins." + pluginName + ".schema").Value()
 		}
-		if schemaDef == "" {
-			log.Errorf("schema validate failed: schema not found, path: %s", "plugins."+pluginName)
+
+		if schemaValue == nil {
+			log.Errorf("schema validate failed: schema not found,  %s, %s", "plugins."+pluginName, schemaType)
 			return fmt.Errorf("schema validate failed: schema not found, path: %s", "plugins."+pluginName)
 		}
+		schemaMap := schemaValue.(map[string]interface{})
+		schemaByte, err := json.Marshal(schemaMap)
+		if err != nil {
+			log.Warnf("schema validate failed: schema json encode failed, path: %s, %w", "plugins."+pluginName, err)
+			return fmt.Errorf("schema validate failed: schema json encode failed, path: %s, %w", "plugins."+pluginName, err)
+		}
 
-		schemaDef = reg.ReplaceAllString(schemaDef, `"properties":{}`)
-		s, err := gojsonschema.NewSchema(gojsonschema.NewStringLoader(schemaDef))
+		s, err := gojsonschema.NewSchema(gojsonschema.NewBytesLoader(schemaByte))
 		if err != nil {
 			log.Errorf("init schema validate failed: %s", err)
 			return fmt.Errorf("schema validate failed: %s", err)
 		}
 
-		ret, err := s.Validate(gojsonschema.NewGoLoader(pluginConf))
+		// check property disable, if is bool, remove from json schema checking
+		conf := pluginConf.(map[string]interface{})
+		var exchange bool
+		disable, ok := conf["disable"]
+		if ok {
+			if fmt.Sprintf("%T", disable) == "bool" {
+				delete(conf, "disable")
+				exchange = true
+			}
+		}
+
+		// check schema
+		ret, err := s.Validate(gojsonschema.NewGoLoader(conf))
 		if err != nil {
 			log.Errorf("schema validate failed: %s", err)
 			return fmt.Errorf("schema validate failed: %s", err)
 		}
 
+		// put the value back to the property disable
+		if exchange {
+			conf["disable"] = disable
+		}
+
 		if !ret.Valid() {
 			errString := buffer.Buffer{}
 			for i, vErr := range ret.Errors() {

api/internal/core/store/validate_test.go

@@ -137,7 +137,6 @@ func TestAPISIXJsonSchemaValidator_Validate(t *testing.T) {
 	err = validator.Validate(consumer3)
 	assert.NotNil(t, err)
 	assert.EqualError(t, err, "schema validate failed: (root): count is required")
-
 }
 
 func TestAPISIXJsonSchemaValidator_checkUpstream(t *testing.T) {
@@ -249,6 +248,66 @@ func TestAPISIXJsonSchemaValidator_checkUpstream(t *testing.T) {
 	assert.EqualError(t, err, "schema validate failed: (root): Does not match pattern '^((uri|server_name|server_addr|request_uri|remote_port|remote_addr|query_string|host|hostname)|arg_[0-9a-zA-z_-]+)$'")
 }
 
+func TestAPISIXJsonSchemaValidator_Plugin(t *testing.T) {
+	validator, err := NewAPISIXJsonSchemaValidator("main.route")
+	assert.Nil(t, err)
+
+	// validate plugin's schema which has no `properties` or empty `properties`
+	route := &entity.Route{}
+	reqBody := `{
+		"id": "1",
+		"uri": "/hello",
+		"plugins": {
+			"prometheus": {
+				"disable": false
+			},
+			"key-auth": {
+				"disable": true
+			}
+		}
+	}`
+	err = json.Unmarshal([]byte(reqBody), route)
+	assert.Nil(t, err)
+	err = validator.Validate(route)
+	assert.Nil(t, err)
+
+	// validate plugin's schema which use `oneOf`
+	reqBody = `{
+		"id": "1",
+		"uri": "/hello",
+		"plugins": {
+                        "ip-restriction": {
+                            "blacklist": [
+                                "127.0.0.0/24"
+                            ],
+                            "disable": true
+                        }
+		}
+	}`
+	err = json.Unmarshal([]byte(reqBody), route)
+	assert.Nil(t, err)
+	err = validator.Validate(route)
+	assert.Nil(t, err)
+
+	// validate plugin's schema with invalid type for `disable`
+	reqBody = `{
+		"id": "1",
+		"uri": "/hello",
+		"plugins": {
+                        "ip-restriction": {
+                            "blacklist": [
+                                "127.0.0.0/24"
+                            ],
+                            "disable": 1
+                        }
+		}
+	}`
+	err = json.Unmarshal([]byte(reqBody), route)
+	assert.Nil(t, err)
+	err = validator.Validate(route)
+	assert.Equal(t, fmt.Errorf("schema validate failed: (root): Must validate one and only one schema (oneOf)\n(root): Additional property disable is not allowed"), err)
+}
+
 func TestAPISIXJsonSchemaValidator_Route_checkRemoteAddr(t *testing.T) {
 	tests := []struct {
 		caseDesc        string

api/internal/handler/consumer/consumer_test.go

@@ -19,20 +19,22 @@ package consumer
 
 import (
 	"encoding/json"
+
 	"testing"
 	"time"
 
 	"github.com/shiningrush/droplet"
 	"github.com/stretchr/testify/assert"
 
+	"github.com/apisix/manager-api/conf"
 	"github.com/apisix/manager-api/internal/core/entity"
 	"github.com/apisix/manager-api/internal/core/storage"
 	"github.com/apisix/manager-api/internal/core/store"
 )
 
 func TestConsumer(t *testing.T) {
 	// init
-	err := storage.InitETCDClient([]string{"127.0.0.1:2379"})
+	err := storage.InitETCDClient(conf.ETCDConfig)
 	assert.Nil(t, err)
 	err = store.InitStores()
 	assert.Nil(t, err)

api/internal/handler/route/route_test.go

@@ -27,14 +27,15 @@ import (
 	"github.com/shiningrush/droplet/data"
 	"github.com/stretchr/testify/assert"
 
+	"github.com/apisix/manager-api/conf"
 	"github.com/apisix/manager-api/internal/core/entity"
 	"github.com/apisix/manager-api/internal/core/storage"
 	"github.com/apisix/manager-api/internal/core/store"
 )
 
 func TestRoute(t *testing.T) {
 	// init
-	err := storage.InitETCDClient([]string{"127.0.0.1:2379"})
+	err := storage.InitETCDClient(conf.ETCDConfig)
 	assert.Nil(t, err)
 	err = store.InitStores()
 	assert.Nil(t, err)
@@ -989,7 +990,7 @@ func TestRoute(t *testing.T) {
 
 func Test_Route_With_Script(t *testing.T) {
 	// init
-	err := storage.InitETCDClient([]string{"127.0.0.1:2379"})
+	err := storage.InitETCDClient(conf.ETCDConfig)
 	assert.Nil(t, err)
 	err = store.InitStores()
 	assert.Nil(t, err)

api/internal/handler/service/service_test.go

@@ -25,14 +25,15 @@ import (
 	"github.com/shiningrush/droplet"
 	"github.com/stretchr/testify/assert"
 
+	"github.com/apisix/manager-api/conf"
 	"github.com/apisix/manager-api/internal/core/entity"
 	"github.com/apisix/manager-api/internal/core/storage"
 	"github.com/apisix/manager-api/internal/core/store"
 )
 
 func TestService(t *testing.T) {
 	// init
-	err := storage.InitETCDClient([]string{"127.0.0.1:2379"})
+	err := storage.InitETCDClient(conf.ETCDConfig)
 	assert.Nil(t, err)
 	err = store.InitStores()
 	assert.Nil(t, err)

api/internal/handler/ssl/ssl_test.go

@@ -25,14 +25,15 @@ import (
 	"github.com/shiningrush/droplet"
 	"github.com/stretchr/testify/assert"
 
+	"github.com/apisix/manager-api/conf"
 	"github.com/apisix/manager-api/internal/core/entity"
 	"github.com/apisix/manager-api/internal/core/storage"
 	"github.com/apisix/manager-api/internal/core/store"
 )
 
 func TestSSL(t *testing.T) {
 	// init
-	err := storage.InitETCDClient([]string{"127.0.0.1:2379"})
+	err := storage.InitETCDClient(conf.ETCDConfig)
 	assert.Nil(t, err)
 	err = store.InitStores()
 	assert.Nil(t, err)

api/internal/handler/upstream/upstream_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/shiningrush/droplet"
 	"github.com/stretchr/testify/assert"
 
+	"github.com/apisix/manager-api/conf"
 	"github.com/apisix/manager-api/internal/core/entity"
 	"github.com/apisix/manager-api/internal/core/storage"
 	"github.com/apisix/manager-api/internal/core/store"
@@ -34,7 +35,7 @@ var upstreamHandler *Handler
 
 func TestUpstream(t *testing.T) {
 	// init
-	err := storage.InitETCDClient([]string{"127.0.0.1:2379"})
+	err := storage.InitETCDClient(conf.ETCDConfig)
 	assert.Nil(t, err)
 	err = store.InitStores()
 	assert.Nil(t, err)

api/main.go

@@ -19,14 +19,15 @@ package main
 import (
 	"context"
 	"fmt"
-	"github.com/apisix/manager-api/internal/handler"
-	"github.com/shiningrush/droplet"
 	"net/http"
 	"os"
 	"os/signal"
 	"syscall"
 	"time"
 
+	"github.com/apisix/manager-api/internal/handler"
+	"github.com/shiningrush/droplet"
+
 	"github.com/apisix/manager-api/conf"
 	"github.com/apisix/manager-api/internal"
 	"github.com/apisix/manager-api/internal/core/storage"
@@ -44,8 +45,9 @@ func main() {
 		newMws = append(newMws, mws[1:]...)
 		return newMws
 	}
-	if err := storage.InitETCDClient(conf.ETCDEndpoints); err != nil {
-		log.Errorf("init etcd client fail: %s", err)
+
+	if err := storage.InitETCDClient(conf.ETCDConfig); err != nil {
+		log.Errorf("init etcd client fail: %w", err)
 		panic(err)
 	}
 	if err := store.InitStores(); err != nil {