make the struct Properties embedded into config Ref#933

Signed-off-by: chentanjun <[email protected]>

cmd/dfget/app/root.go

@@ -116,8 +116,8 @@ func initProperties() {
 		}
 	}
 
-	if cfg.Node == nil {
-		cfg.Node = properties.Nodes
+	if cfg.Nodes == nil {
+		cfg.Nodes = properties.Nodes
 	}
 
 	if cfg.LocalLimit == 0 {
@@ -196,7 +196,7 @@ func initFlags() {
 			"\nin this way, different but actually the same URLs can reuse the same downloading task")
 	flagSet.StringSliceVar(&cfg.Header, "header", nil,
 		"http header, eg: --header='Accept: *' --header='Host: abc'")
-	flagSet.StringSliceVarP(&cfg.Node, "node", "n", nil,
+	flagSet.StringSliceVarP(&cfg.Nodes, "node", "n", nil,
 		"specify the addresses(host:port) of supernodes")
 	flagSet.BoolVar(&cfg.Notbs, "notbs", false,
 		"disable back source downloading for requested file when p2p fails to download it")
@@ -238,15 +238,15 @@ func transFilter(filter string) []string {
 func handleNodes() error {
 	nodes := make([]string, 0)
 
-	for _, v := range cfg.Node {
+	for _, v := range cfg.Nodes {
 		// TODO: check the validity of v.
 		if strings.IndexByte(v, ':') > 0 {
 			nodes = append(nodes, v)
 			continue
 		}
 		nodes = append(nodes, fmt.Sprintf("%s:%d", v, config.DefaultSupernodePort))
 	}
-	cfg.Node = nodes
+	cfg.Nodes = nodes
 	return nil
 }
 

cmd/dfget/app/root_test.go

@@ -27,7 +27,6 @@ import (
 	"github.com/dragonflyoss/Dragonfly/dfget/config"
 	"github.com/dragonflyoss/Dragonfly/pkg/errortypes"
 	"github.com/dragonflyoss/Dragonfly/pkg/rate"
-
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
 )
@@ -37,9 +36,10 @@ type dfgetSuit struct {
 }
 
 func (suit *dfgetSuit) Test_initFlagsNoArguments() {
-	suit.Nil(cfg.Node)
+	initProperties()
+	suit.Equal(cfg.Nodes, []string{"127.0.0.1"})
 	suit.Equal(cfg.LocalLimit, 20*rate.MB)
-	suit.Equal(cfg.TotalLimit, rate.Rate(0))
+	suit.Equal(cfg.TotalLimit, 20*rate.MB)
 	suit.Equal(cfg.Notbs, false)
 	suit.Equal(cfg.DFDaemon, false)
 	suit.Equal(cfg.Console, false)
@@ -55,7 +55,7 @@ func (suit *dfgetSuit) Test_initProperties() {
 	iniFile := filepath.Join(dirName, "dragonfly.ini")
 	yamlFile := filepath.Join(dirName, "dragonfly.yaml")
 	iniContent := []byte("[node]\naddress=1.1.1.1")
-	yamlContent := []byte("nodes:\n  - 1.1.1.2\nlocalLimit: 1000K")
+	yamlContent := []byte("nodes:\n  - 1.1.1.2\nlocalLimit: 1000K\ntotalLimit: 1000k")
 	ioutil.WriteFile(iniFile, iniContent, os.ModePerm)
 	ioutil.WriteFile(yamlFile, yamlContent, os.ModePerm)
 
@@ -71,9 +71,9 @@ func (suit *dfgetSuit) Test_initProperties() {
 		{configs: []string{iniFile, yamlFile},
 			expected: newProp(0, 0, 0, "1.1.1.1")},
 		{configs: []string{yamlFile, iniFile},
-			expected: newProp(int(rate.MB*20), 0, 0, "1.1.1.2")},
+			expected: newProp(int(rate.KB*1000), int(rate.KB*1000), 0, "1.1.1.2")},
 		{configs: []string{filepath.Join(dirName, "x"), yamlFile},
-			expected: newProp(int(rate.MB*20), 0, 0, "1.1.1.2")},
+			expected: newProp(int(rate.KB*1000), int(rate.KB*1000), 0, "1.1.1.2")},
 	}
 
 	for _, v := range cases {
@@ -84,7 +84,7 @@ func (suit *dfgetSuit) Test_initProperties() {
 			"--locallimit", v.expected.LocalLimit.String(),
 			"--totallimit", v.expected.TotalLimit.String()})
 		initProperties()
-		suit.EqualValues(cfg.Node, v.expected.Nodes)
+		suit.EqualValues(cfg.Nodes, v.expected.Nodes)
 		suit.Equal(cfg.LocalLimit, v.expected.LocalLimit)
 		suit.Equal(cfg.TotalLimit, v.expected.TotalLimit)
 		suit.Equal(cfg.ClientQueueSize, v.expected.ClientQueueSize)

dfget/config/config.go

@@ -86,6 +86,7 @@ func NewProperties() *Properties {
 		LocalLimit:      DefaultLocalLimit,
 		MinRate:         DefaultMinRate,
 		ClientQueueSize: DefaultClientQueueSize,
+		TotalLimit:      DefaultTotalLimit,
 	}
 }
 
@@ -146,18 +147,6 @@ type Config struct {
 	// Output full output path.
 	Output string `json:"output"`
 
-	// LocalLimit rate limit about a single download task, format: G(B)/g/M(B)/m/K(B)/k/B
-	// pure number will also be parsed as Byte.
-	LocalLimit rate.Rate `json:"localLimit,omitempty"`
-
-	// Minimal rate about a single download task, format: G(B)/g/M(B)/m/K(B)/k/B
-	// pure number will also be parsed as Byte.
-	MinRate rate.Rate `json:"minRate,omitempty"`
-
-	// TotalLimit rate limit about the whole host, format: G(B)/g/M(B)/m/K(B)/k/B
-	// pure number will also be parsed as Byte.
-	TotalLimit rate.Rate `json:"totalLimit,omitempty"`
-
 	// Timeout download timeout(second).
 	Timeout time.Duration `json:"timeout,omitempty"`
 
@@ -186,9 +175,6 @@ type Config struct {
 	// eg: --header='Accept: *' --header='Host: abc'.
 	Header []string `json:"header,omitempty"`
 
-	// Node specify supernodes.
-	Node []string `json:"node,omitempty"`
-
 	// Notbs indicates whether to not back source to download when p2p fails.
 	Notbs bool `json:"notbs,omitempty"`
 
@@ -208,12 +194,6 @@ type Config struct {
 	// If set true, log level will be 'debug'.
 	Verbose bool `json:"verbose,omitempty"`
 
-	// ClientQueueSize is the size of client queue
-	// which controls the number of pieces that can be processed simultaneously.
-	// It is only useful when the pattern not equals "source".
-	// The default value is 6.
-	ClientQueueSize int `json:"clientQueueSize,omitempty"`
-
 	// Start time.
 	StartTime time.Time `json:"-"`
 
@@ -240,6 +220,9 @@ type Config struct {
 
 	// The reason of backing to source.
 	BackSourceReason int `json:"-"`
+
+	// Embedded Properties holds all configurable properties.
+	Properties
 }
 
 func (cfg *Config) String() string {
@@ -267,8 +250,6 @@ func NewConfig() *Config {
 	cfg.RV.SystemDataDir = path.Join(cfg.WorkHome, "data")
 	cfg.RV.FileLength = -1
 	cfg.ConfigFiles = []string{DefaultYamlConfigFile, DefaultIniConfigFile}
-	cfg.LocalLimit = DefaultLocalLimit
-	cfg.MinRate = DefaultMinRate
 	return cfg
 }
 

dfget/config/config_test.go

@@ -58,9 +58,10 @@ func (suite *ConfigSuite) TestConfig_String(c *check.C) {
 	expected := "{\"url\":\"\",\"output\":\"\""
 	c.Assert(strings.Contains(cfg.String(), expected), check.Equals, true)
 	cfg.LocalLimit = 20 * rate.MB
+	cfg.MinRate = 64 * rate.KB
 	cfg.Pattern = "p2p"
-	expected = "\"url\":\"\",\"output\":\"\",\"localLimit\":\"20MB\"," +
-		"\"minRate\":\"64KB\",\"pattern\":\"p2p\""
+	expected = "\"url\":\"\",\"output\":\"\",\"pattern\":\"p2p\"," +
+		"\"localLimit\":\"20MB\",\"minRate\":\"64KB\""
 	c.Assert(strings.Contains(cfg.String(), expected), check.Equals, true)
 }
 

dfget/config/constants.go

@@ -52,6 +52,7 @@ const (
 	DefaultNode            = "127.0.0.1"
 	DefaultLocalLimit      = 20 * rate.MB
 	DefaultMinRate         = 64 * rate.KB
+	DefaultTotalLimit      = 20 * rate.MB
 	DefaultClientQueueSize = 6
 )
 

dfget/core/core.go

@@ -109,9 +109,9 @@ func prepare(cfg *config.Config) (err error) {
 	}
 	rv.DataDir = cfg.RV.SystemDataDir
 
-	cfg.Node = adjustSupernodeList(cfg.Node)
+	cfg.Nodes = adjustSupernodeList(cfg.Nodes)
 	if stringutils.IsEmptyStr(rv.LocalIP) {
-		rv.LocalIP = checkConnectSupernode(cfg.Node)
+		rv.LocalIP = checkConnectSupernode(cfg.Nodes)
 	}
 	rv.Cid = getCid(rv.LocalIP, cfg.Sign)
 	rv.TaskFileName = getTaskFileName(rv.RealTarget, cfg.Sign)
@@ -142,7 +142,7 @@ func registerToSuperNode(cfg *config.Config, register regist.SupernodeRegister)
 		panic("user specified")
 	}
 
-	if len(cfg.Node) == 0 {
+	if len(cfg.Nodes) == 0 {
 		cfg.BackSourceReason = config.BackSourceReasonNodeEmpty
 		panic("supernode empty")
 	}
@@ -202,10 +202,10 @@ func downloadFile(cfg *config.Config, supernodeAPI api.SupernodeAPI,
 
 	if success {
 		logrus.Infof("download SUCCESS from supernode %s cost:%.3fs length:%d",
-			cfg.Node, time.Since(cfg.StartTime).Seconds(), cfg.RV.FileLength)
+			cfg.Nodes, time.Since(cfg.StartTime).Seconds(), cfg.RV.FileLength)
 	} else {
 		logrus.Infof("download FAIL from supernode %s cost:%.3fs length:%d reason:%d",
-			cfg.Node, time.Since(cfg.StartTime).Seconds(), cfg.RV.FileLength, cfg.BackSourceReason)
+			cfg.Nodes, time.Since(cfg.StartTime).Seconds(), cfg.RV.FileLength, cfg.BackSourceReason)
 	}
 	return err
 }
@@ -294,7 +294,7 @@ func reportMetrics(cfg *config.Config, supernodeAPI api.SupernodeAPI, downloadTi
 		Success:          success,
 		TaskID:           taskID,
 	}
-	for _, node := range cfg.Node {
+	for _, node := range cfg.Nodes {
 		resp, err := supernodeAPI.ReportMetrics(node, req)
 		if err != nil {
 			logrus.Errorf("failed to report metrics to supernode %s: %v", node, err)

dfget/core/core_test.go

@@ -94,11 +94,11 @@ func (s *CoreTestSuite) TestRegisterToSupernode(c *check.C) {
 
 	uploader.SetupPeerServerExecutor(nil)
 	cfg.Pattern = config.PatternP2P
-	cfg.Node = []string{"x"}
+	cfg.Nodes = []string{"x"}
 	cfg.URL = "http://x.com"
 	f(config.BackSourceReasonRegisterFail, true, nil)
 
-	cfg.Node = []string{"x"}
+	cfg.Nodes = []string{"x"}
 	cfg.URL = "http://taobao.com"
 	cfg.BackSourceReason = config.BackSourceReasonNone
 }

dfget/core/regist/register.go

@@ -64,8 +64,8 @@ func (s *supernodeRegister) Register(peerPort int) (*RegisterResult, *errortypes
 		start      = time.Now()
 	)
 
-	logrus.Infof("do register to one of %v", s.cfg.Node)
-	nodes, nLen := s.cfg.Node, len(s.cfg.Node)
+	logrus.Infof("do register to one of %v", s.cfg.Nodes)
+	nodes, nLen := s.cfg.Nodes, len(s.cfg.Nodes)
 	req := s.constructRegisterRequest(peerPort)
 	for i = 0; i < nLen; i++ {
 		req.SupernodeIP = netutils.ExtractHost(nodes[i])
@@ -92,7 +92,7 @@ func (s *supernodeRegister) Register(peerPort int) (*RegisterResult, *errortypes
 		return nil, err
 	}
 
-	result := NewRegisterResult(nodes[i], s.cfg.Node, s.cfg.URL,
+	result := NewRegisterResult(nodes[i], s.cfg.Nodes, s.cfg.URL,
 		resp.Data.TaskID, resp.Data.FileLength, resp.Data.PieceSize)
 
 	logrus.Infof("do register result:%s and cost:%.3fs", resp,
@@ -114,14 +114,14 @@ func (s *supernodeRegister) checkResponse(resp *types.RegisterResponse, e error)
 }
 
 func (s *supernodeRegister) setRemainderNodes(idx int) {
-	nLen := len(s.cfg.Node)
+	nLen := len(s.cfg.Nodes)
 	if nLen <= 0 {
 		return
 	}
 	if idx < nLen {
-		s.cfg.Node = s.cfg.Node[idx+1:]
+		s.cfg.Nodes = s.cfg.Nodes[idx+1:]
 	} else {
-		s.cfg.Node = []string{}
+		s.cfg.Nodes = []string{}
 	}
 }
 

dfget/core/regist/register_test.go

@@ -91,21 +91,21 @@ func (s *RegistTestSuite) TestSupernodeRegister_Register(c *check.C) {
 		}
 	}
 
-	cfg.Node = []string{""}
+	cfg.Nodes = []string{""}
 	f(constants.HTTPError, "connection refused", nil)
 
-	cfg.Node = []string{"x"}
+	cfg.Nodes = []string{"x"}
 	f(501, "invalid source url", nil)
 
-	cfg.Node = []string{"x"}
+	cfg.Nodes = []string{"x"}
 	cfg.URL = "http://taobao.com"
 	f(constants.CodeNeedAuth, "need auth", nil)
 
-	cfg.Node = []string{"x"}
+	cfg.Nodes = []string{"x"}
 	cfg.URL = "http://github.com"
 	f(constants.CodeWaitAuth, "wait auth", nil)
 
-	cfg.Node = []string{"x"}
+	cfg.Nodes = []string{"x"}
 	cfg.URL = "http://lowzj.com"
 	f(constants.Success, "", &RegisterResult{
 		Node: "x", RemainderNodes: []string{}, URL: cfg.URL, TaskID: "a",