Improve Survey and Log Collection

.gitignore

@@ -34,6 +34,7 @@ pkg/visor/foo/
 /*-server
 /*.json
 !/dmsghttp-config.json
+!/services-config.json
 /*.sh
 /*.log
 

.goreleaser-archlinux.yml

@@ -290,6 +290,7 @@ archives:
     name_template: 'skywire-v{{ .Version }}-{{ .Os }}-{{ .Arch }}'
     files:
       - dmsghttp-config.json
+      - services-config.json
     builds:
       - skywire-visor-amd64
       - skywire-cli-amd64
@@ -305,6 +306,7 @@ archives:
     name_template: 'skywire-v{{ .Version }}-{{ .Os }}-{{ .Arch }}'
     files:
       - dmsghttp-config.json
+      - services-config.json
     builds:
       - skywire-visor-arm64
       - skywire-cli-arm64
@@ -320,6 +322,7 @@ archives:
     name_template: 'skywire-v{{ .Version }}-{{ .Os }}-{{ .Arch }}'
     files:
       - dmsghttp-config.json
+      - services-config.json
     builds:
       - skywire-visor-arm
       - skywire-cli-arm

.goreleaser-darwin.yml

@@ -86,6 +86,7 @@ archives:
     name_template: 'skywire-v{{ .Version }}-{{ .Os }}-{{ .Arch }}'
     files:
       - dmsghttp-config.json
+      - services-config.json
     builds:
       - skywire-visor
       - skywire-cli

.goreleaser-linux.yml

@@ -472,6 +472,7 @@ archives:
     name_template: 'skywire-v{{ .Version }}-{{ .Os }}-{{ .Arch }}'
     files:
       - dmsghttp-config.json
+      - services-config.json
     builds:
       - skywire-visor-amd64
       - skywire-cli-amd64
@@ -487,6 +488,7 @@ archives:
     name_template: 'skywire-v{{ .Version }}-{{ .Os }}-{{ .Arch }}'
     files:
       - dmsghttp-config.json
+      - services-config.json
     builds:
       - skywire-visor-arm64
       - skywire-cli-arm64
@@ -502,6 +504,7 @@ archives:
     name_template: 'skywire-v{{ .Version }}-{{ .Os }}-{{ .Arch }}'
     files:
       - dmsghttp-config.json
+      - services-config.json
     builds:
       - skywire-visor-arm
       - skywire-cli-arm
@@ -517,6 +520,7 @@ archives:
     name_template: 'skywire-v{{ .Version }}-{{ .Os }}-{{ .Arch }}hf'
     files:
       - dmsghttp-config.json
+      - services-config.json
     builds:
       - skywire-visor-armhf
       - skywire-cli-armhf
@@ -532,6 +536,7 @@ archives:
     name_template: 'skywire-v{{ .Version }}-{{ .Os }}-{{ .Arch }}'
     files:
       - dmsghttp-config.json
+      - services-config.json
     builds:
       - skywire-visor-riscv64
       - skywire-cli-riscv64

.goreleaser-windows.yml

@@ -70,6 +70,7 @@ archives:
     name_template: 'skywire-v{{ .Version }}-{{ .Os }}-{{ .Arch }}'
     files:
       - dmsghttp-config.json
+      - services-config.json
     builds:
       - skywire-visor
       - skywire-cli

cmd/skywire-cli/README.md

@@ -124,6 +124,7 @@ Available Commands:
   tree                    subcommand tree
   doc                     generate markdown docs
   dmsghttp                update dmsghttp-config.json file from config bootstrap service
+  services                update services-config.json file from config service
 
 
 ```
@@ -2190,3 +2191,17 @@ Flags:
 
 
 ```
+
+### services
+
+```
+update services-config.json file from config service
+
+Usage:
+  cli services update [flags]
+
+Flags:
+  -p, --path string   path of services-config file, default is for pkg installation (default "/opt/skywire/services-config.json")
+
+
+```

cmd/skywire-cli/commands/config/gen.go

@@ -498,14 +498,17 @@ var genConfigCmd = &cobra.Command{
 					}
 				} else {
 					//fill in services struct with the response
-					err = json.Unmarshal(body, &services)
+					err = json.Unmarshal(body, &servicesConfig)
 					if err != nil {
 						log.WithError(err).Fatal("Failed to unmarshal json response\n")
 					}
 					if !isStdout {
 						log.Infof("Fetched service endpoints from '%s'", serviceConfURL)
 					}
-
+					services = servicesConfig.Prod
+					if isTestEnv {
+						services = servicesConfig.Test
+					}
 					// reset the state of isStdout
 					isStdout = wasStdout
 				}

cmd/skywire-cli/commands/config/root.go

@@ -52,6 +52,7 @@ var (
 	isBestProtocol              bool
 	serviceConfURL              string
 	services                    visorconfig.Services
+	servicesConfig              servicesConf
 	isForce                     bool
 	isHide                      bool
 	isAll                       bool
@@ -102,3 +103,8 @@ var RootCmd = &cobra.Command{
 	Short: "Generate or update a skywire config",
 	Long:  "Generate or update the config file used by skywire-visor.",
 }
+
+type servicesConf struct { //nolint
+	Test visorconfig.Services `json:"test"`
+	Prod visorconfig.Services `json:"prod"`
+}

cmd/skywire-cli/commands/log/root.go

@@ -61,7 +61,7 @@ func init() {
 	logCmd.Flags().IntVarP(&duration, "duration", "n", 0, "number of days before today to fetch transport logs for")
 	logCmd.Flags().BoolVar(&allVisors, "all", false, "consider all visors ; no version filtering")
 	logCmd.Flags().IntVar(&batchSize, "batchSize", 50, "number of visor in each batch")
-	logCmd.Flags().Int64Var(&maxFileSize, "maxfilesize", 30, "maximum file size allowed to download during collecting logs, in KB")
+	logCmd.Flags().Int64Var(&maxFileSize, "maxfilesize", 1024, "maximum file size allowed to download during collecting logs, in KB")
 	logCmd.Flags().StringVarP(&dmsgDisc, "dmsg-disc", "D", skyenv.DmsgDiscAddr, "dmsg discovery url\n")
 	logCmd.Flags().StringVarP(&utAddr, "ut", "u", "", "custom uptime tracker url")
 	if os.Getenv("DMSGCURL_SK") != "" {

cmd/skywire-cli/commands/root.go

@@ -22,6 +22,7 @@ import (
 	cliskysocksc "github.com/skycoin/skywire/cmd/skywire-cli/commands/proxy"
 	clireward "github.com/skycoin/skywire/cmd/skywire-cli/commands/reward"
 	clirtfind "github.com/skycoin/skywire/cmd/skywire-cli/commands/rtfind"
+	cliservices "github.com/skycoin/skywire/cmd/skywire-cli/commands/services"
 	cliskyfwd "github.com/skycoin/skywire/cmd/skywire-cli/commands/skyfwd"
 	cliskyrev "github.com/skycoin/skywire/cmd/skywire-cli/commands/skyrev"
 	clisurvey "github.com/skycoin/skywire/cmd/skywire-cli/commands/survey"
@@ -197,6 +198,7 @@ func init() {
 		treeCmd,
 		docCmd,
 		clidmsghttp.RootCmd,
+		cliservices.RootCmd,
 	)
 	var jsonOutput bool
 	RootCmd.PersistentFlags().BoolVar(&jsonOutput, internal.JSONString, false, "print output in json")

cmd/skywire-cli/commands/services/root.go

@@ -0,0 +1,101 @@
+// Package cliservices cmd/skywire-cli/commands/services/root.go
+package cliservices
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/skycoin/skywire-utilities/pkg/cmdutil"
+	"github.com/skycoin/skywire-utilities/pkg/logging"
+	"github.com/skycoin/skywire-utilities/pkg/skyenv"
+	"github.com/skycoin/skywire/pkg/visor/visorconfig"
+)
+
+var (
+	path string
+)
+
+func init() {
+	servicesCmd.Flags().SortFlags = false
+	servicesCmd.Flags().StringVarP(&path, "path", "p", "/opt/skywire/services-config.json", "path of services-config file, default is for pkg installation")
+}
+
+// RootCmd is servicesCmd
+var RootCmd = servicesCmd
+
+var servicesCmd = &cobra.Command{
+	Use:   "services update",
+	Short: "update services-config.json file from config bootstrap service",
+	Run: func(cmd *cobra.Command, args []string) {
+		log := logging.MustGetLogger("services_updater")
+
+		ctx, cancel := cmdutil.SignalContext(context.Background(), log)
+		defer cancel()
+		go func() {
+			<-ctx.Done()
+			cancel()
+			os.Exit(1)
+		}()
+
+		servicesConf, err := fetchServicesConf()
+		if err != nil {
+			log.WithError(err).Error("Cannot fetching updated services-config data")
+		}
+
+		file, err := json.MarshalIndent(servicesConf, "", " ")
+		if err != nil {
+			log.WithError(err).Error("Error accurs during marshal content to json file")
+		}
+
+		err = os.WriteFile(path, file, 0600)
+		if err != nil {
+			log.WithError(err).Errorf("Cannot save new services-config.json file at %s", path)
+		}
+	},
+}
+
+type servicesConf struct { //nolint
+	Test visorconfig.Services `json:"test"`
+	Prod visorconfig.Services `json:"prod"`
+}
+
+func fetchServicesConf() (servicesConf, error) {
+	var newConf servicesConf
+	var prodConf visorconfig.Services
+	prodResp, err := http.Get(skyenv.ServiceConfAddr)
+	if err != nil {
+		return newConf, err
+	}
+	defer prodResp.Body.Close() //nolint
+	body, err := io.ReadAll(prodResp.Body)
+	if err != nil {
+		return newConf, err
+	}
+	err = json.Unmarshal(body, &prodConf)
+	if err != nil {
+		return newConf, err
+	}
+	newConf.Prod = prodConf
+
+	var testConf visorconfig.Services
+	testResp, err := http.Get(skyenv.TestServiceConfAddr)
+	if err != nil {
+		return newConf, err
+	}
+	defer testResp.Body.Close() //nolint
+	body, err = io.ReadAll(testResp.Body)
+	if err != nil {
+		return newConf, err
+	}
+	err = json.Unmarshal(body, &testConf)
+	if err != nil {
+		return newConf, err
+	}
+	newConf.Test = testConf
+	return newConf, nil
+}

go.sum

@@ -546,8 +546,6 @@ golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWP
 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
-golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
 golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -634,8 +632,6 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
-golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
@@ -646,8 +642,6 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX
 golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
-golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
 golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
 golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -658,8 +652,6 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
-golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

pkg/skyenv/skyenv.go

@@ -10,6 +10,8 @@ const (
 	ConfigName = "skywire-config.json"
 	// DMSGHTTPName is the default dmsghttp config name
 	DMSGHTTPName = "dmsghttp-config.json"
+	// ServicesName is the default services config name
+	ServicesName = "services-config.json"
 
 	// Dmsg port constants.
 	// TODO(evanlinjin): Define these properly. These are currently random.

pkg/visor/api.go

@@ -367,7 +367,7 @@ func (v *Visor) SetRewardAddress(p string) (string, error) {
 		return p, fmt.Errorf("failed to write config to file. err=%v", err)
 	}
 	// generate survey after set/update reward address
-	visorconfig.GenerateSurvey(v.conf, v.log, false)
+	GenerateSurvey(v, v.log, false)
 	return p, nil
 }
 

pkg/visor/init.go

@@ -403,7 +403,7 @@ func initDmsgHTTPLogServer(ctx context.Context, v *Visor, log *logging.Logger) e
 		}
 	}
 
-	lsAPI := logserver.New(logger, v.conf.Transport.LogStore.Location, v.conf.LocalPath, v.conf.DmsgHTTPServerPath, whitelistedPKs, v.conf.PK, printLog)
+	lsAPI := logserver.New(logger, v.conf.Transport.LogStore.Location, v.conf.LocalPath, v.conf.DmsgHTTPServerPath, whitelistedPKs, &v.survey, printLog)
 
 	lis, err := dmsgC.Listen(visorconfig.DmsgHTTPPort)
 	if err != nil {
@@ -449,7 +449,7 @@ func initDmsgHTTPLogServer(ctx context.Context, v *Visor, log *logging.Logger) e
 }
 
 func initSystemSurvey(ctx context.Context, v *Visor, log *logging.Logger) error { //nolint:all
-	go visorconfig.GenerateSurvey(v.conf, log, true)
+	go GenerateSurvey(v, log, true)
 	return nil
 }