merge with develop, fixes merge conflict

.gitignore

@@ -53,3 +53,4 @@ Char
 /SkywireInstaller*.pkg
 *.dmg
 /scripts/mac_installer/icon.iconset/
+releaseChangelog.md

CHANGELOG.md

@@ -16,6 +16,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - added `update` and `summary` as subcommand to `skywire-cli visor`
 - added multiple new flag to update configuration in `skywire-cli config update`
 - added shell autocompletion command to `skywire-cli` and `skywire-visor`
+- added `dsmgHTTPStruct` in visorconfig pkg to usable other repos, such as `skybian`
+- added `dmsghttp-config.json` which contains the `dmsg-urls` of services and info of `dmsg-servers` for both prod and test
+- added `servers` filed to `dmsg` in config
+- added `-d,--dmsghttp` flag to `skywire-cli config gen`
+- added `dmsgdirect` client to connect to services over dmsg
 ## 0.5.0
 
 ### Changed

Makefile

@@ -149,7 +149,10 @@ build-deploy: ## Build for deployment Docker images
 	${OPTS} go build ${BUILD_OPTS_DEPLOY} -o /release/apps/skysocks-client ./cmd/apps/skysocks-client
 
 github-release:
-	goreleaser --rm-dist
+	$(eval GITHUB_TAG=$(shell git describe --abbrev=0 --tags | cut -c 2-))
+	sed '/^## ${GITHUB_TAG}$$/,/^## .*/!d;//d;/^$$/d' ./CHANGELOG.md > releaseChangelog.md
+	goreleaser --rm-dist --release-notes releaseChangelog.md 
+
 
 build-docker: ## Build docker image
 	./ci_scripts/docker-push.sh -t ${BRANCH} -b

cmd/skywire-cli/commands/config/gen.go

@@ -104,10 +104,10 @@ var genConfigCmd = &cobra.Command{
 			logger.WithError(err).Fatal("Failed to create config.")
 		}
 
-		// Use local servers
+		// Use dmsg urls for services and add dmsg-servers
 		if dmsgHTTP {
-			var dmsgHTTPServersList dmsgHTTPServers
-			serversListJSON, err := ioutil.ReadFile("localServers.json")
+			var dmsgHTTPServersList visorconfig.DmsgHTTPServers
+			serversListJSON, err := ioutil.ReadFile("dmsghttp-config.json")
 			if err != nil {
 				logger.WithError(err).Fatal("Failed to read servers.json file.")
 			}
@@ -186,17 +186,3 @@ func readOldConfig(log *logging.MasterLogger, confPath string, replace bool) (*v
 
 	return conf, true
 }
-
-type dmsgHTTPServers struct {
-	Test dmsgHTTPServersData `json:"test"`
-	Prod dmsgHTTPServersData `json:"prod"`
-}
-type dmsgHTTPServersData struct {
-	DMSGServers        []string `json:"dmsg_servers"`
-	DMSGDiscovery      string   `json:"dmsg_discovery"`
-	TransportDiscovery string   `json:"transport_discovery"`
-	AddressResolver    string   `json:"address_resolver"`
-	RouteFinder        string   `json:"route_finder"`
-	UptimeTracker      string   `json:"uptime_tracker"`
-	ServiceDiscovery   string   `json:"service_discovery"`
-}

cmd/skywire-cli/commands/mdisc/root.go

@@ -3,6 +3,7 @@ package mdisc
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"os"
 	"text/tabwriter"
 	"time"
@@ -41,7 +42,7 @@ var entryCmd = &cobra.Command{
 		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
 		defer cancel()
 		pk := internal.ParsePK("visor-public-key", args[0])
-		entry, err := disc.NewHTTP(mdAddr).Entry(ctx, pk)
+		entry, err := disc.NewHTTP(mdAddr, http.Client{}).Entry(ctx, pk)
 		internal.Catch(err)
 		fmt.Println(entry)
 	},
@@ -53,7 +54,7 @@ var availableServersCmd = &cobra.Command{
 	Run: func(_ *cobra.Command, _ []string) {
 		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
 		defer cancel()
-		entries, err := disc.NewHTTP(mdAddr).AvailableServers(ctx)
+		entries, err := disc.NewHTTP(mdAddr, http.Client{}).AvailableServers(ctx)
 		internal.Catch(err)
 		printAvailableServers(entries)
 	},

cmd/skywire-cli/commands/rtfind/root.go

@@ -2,6 +2,7 @@ package rtfind
 
 import (
 	"fmt"
+	"net/http"
 	"time"
 
 	"github.com/skycoin/dmsg/cipher"
@@ -31,7 +32,7 @@ var RootCmd = &cobra.Command{
 	Short: "Queries the Route Finder for available routes between two visors",
 	Args:  cobra.MinimumNArgs(2),
 	Run: func(_ *cobra.Command, args []string) {
-		rfc := rfclient.NewHTTP(frAddr, timeout)
+		rfc := rfclient.NewHTTP(frAddr, timeout, http.Client{})
 
 		var srcPK, dstPK cipher.PubKey
 		internal.Catch(srcPK.Set(args[0]))

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/schollz/progressbar/v2 v2.15.0
 	github.com/shirou/gopsutil/v3 v3.21.4
 	github.com/sirupsen/logrus v1.8.1
-	github.com/skycoin/dmsg v0.0.0-20211007145032-962409e5845f
+	github.com/skycoin/dmsg v0.0.0-20211125122021-388f2fc645c9
 	github.com/skycoin/skycoin v0.27.1
 	github.com/skycoin/yamux v0.0.0-20200803175205-571ceb89da9f
 	github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8

go.sum

@@ -306,8 +306,8 @@ github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5k
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/skycoin/dmsg v0.0.0-20211007145032-962409e5845f h1:0/PJaqsGvkfuf4I/Jg2ayQq0AJg5qhIhK4YzevX72Bg=
-github.com/skycoin/dmsg v0.0.0-20211007145032-962409e5845f/go.mod h1:p9RQVoY18Rpi5mrbIfE55z7XcKTscIUiNh8nv1ou1/8=
+github.com/skycoin/dmsg v0.0.0-20211125122021-388f2fc645c9 h1:Cy62MJBJkPWc/zoNBnwJC7+ak+Hj20TgwxOMKkhi8mI=
+github.com/skycoin/dmsg v0.0.0-20211125122021-388f2fc645c9/go.mod h1:EgRg8fy5RjF67OJlh9w+vhq3+Phyn6AXKSedkzhf1ww=
 github.com/skycoin/noise v0.0.0-20180327030543-2492fe189ae6 h1:1Nc5EBY6pjfw1kwW0duwyG+7WliWz5u9kgk1h5MnLuA=
 github.com/skycoin/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:UXghlricA7J3aRD/k7p/zBObQfmBawwCxIVPVjz2Q3o=
 github.com/skycoin/skycoin v0.26.0/go.mod h1:78nHjQzd8KG0jJJVL/j0xMmrihXi70ti63fh8vXScJw=
@@ -479,7 +479,6 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210406210042-72f3dc4e9b72/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210503060354-a79de5458b56 h1:b8jxX3zqjpqb2LklXPzKSGJhzyxCOZSz8ncv8Nv+y7w=
 golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

internal/httpauth/client.go

@@ -63,10 +63,10 @@ type Client struct {
 // * SW-Public: The specified public key
 // * SW-Nonce:  The nonce for that public key
 // * SW-Sig:    The signature of the payload + the nonce
-func NewClient(ctx context.Context, addr string, key cipher.PubKey, sec cipher.SecKey) (*Client, error) {
+func NewClient(ctx context.Context, addr string, key cipher.PubKey, sec cipher.SecKey, client *http.Client) (*Client, error) {
 	c := &Client{
-		client:      &http.Client{},
-		reuseClient: &http.Client{},
+		client:      client,
+		reuseClient: client,
 		key:         key,
 		sec:         sec,
 		addr:        sanitizedAddr(addr),

internal/httpauth/client_test.go

@@ -45,7 +45,7 @@ func TestClient(t *testing.T) {
 	ts := newTestServer(t, pk, headerCh)
 	defer ts.Close()
 
-	c, err := NewClient(context.TODO(), ts.URL, pk, sk)
+	c, err := NewClient(context.TODO(), ts.URL, pk, sk, &http.Client{})
 	require.NoError(t, err)
 
 	req, err := http.NewRequest(http.MethodGet, ts.URL+"/foo", bytes.NewBufferString(payload))
@@ -71,7 +71,7 @@ func TestClient_BadNonce(t *testing.T) {
 	ts := newTestServer(t, pk, headerCh)
 	defer ts.Close()
 
-	c, err := NewClient(context.TODO(), ts.URL, pk, sk)
+	c, err := NewClient(context.TODO(), ts.URL, pk, sk, &http.Client{})
 	require.NoError(t, err)
 
 	c.nonce = 999

internal/utclient/client.go

@@ -50,13 +50,13 @@ const (
 // * SW-Public: The specified public key
 // * SW-Nonce:  The nonce for that public key
 // * SW-Sig:    The signature of the payload + the nonce
-func NewHTTP(addr string, pk cipher.PubKey, sk cipher.SecKey) (APIClient, error) {
+func NewHTTP(addr string, pk cipher.PubKey, sk cipher.SecKey, httpC http.Client) (APIClient, error) {
 	var client *httpauth.Client
 	var err error
 
 	retrier := netutil.NewRetrier(createRetryDelay, 10, 2, log)
 	retrierFunc := func() error {
-		client, err = httpauth.NewClient(context.Background(), addr, pk, sk)
+		client, err = httpauth.NewClient(context.Background(), addr, pk, sk, &httpC)
 		if err != nil {
 			return fmt.Errorf("uptime tracker httpauth: %w", err)
 		}

internal/utclient/client_test.go

@@ -42,7 +42,7 @@ func TestClientAuth(t *testing.T) {
 	))
 	defer srv.Close()
 
-	client, err := NewHTTP(srv.URL, testPubKey, testSecKey)
+	client, err := NewHTTP(srv.URL, testPubKey, testSecKey, http.Client{})
 	require.NoError(t, err)
 	c := client.(*httpClient)
 
@@ -67,7 +67,7 @@ func TestUpdateVisorUptime(t *testing.T) {
 
 	defer srv.Close()
 
-	c, err := NewHTTP(srv.URL, testPubKey, testSecKey)
+	c, err := NewHTTP(srv.URL, testPubKey, testSecKey, http.Client{})
 	require.NoError(t, err)
 
 	err = c.UpdateVisorUptime(context.TODO())

pkg/app/appdisc/factory.go

@@ -1,6 +1,8 @@
 package appdisc
 
 import (
+	"net/http"
+
 	"github.com/sirupsen/logrus"
 	"github.com/skycoin/dmsg/cipher"
 	"github.com/skycoin/skycoin/src/util/logging"
@@ -16,6 +18,7 @@ type Factory struct {
 	PK          cipher.PubKey
 	SK          cipher.SecKey
 	ServiceDisc string // Address of service-discovery
+	Client      http.Client
 }
 
 func (f *Factory) setDefaults() {
@@ -43,7 +46,7 @@ func (f *Factory) VisorUpdater(port uint16) Updater {
 	}
 
 	return &serviceUpdater{
-		client: servicedisc.NewClient(f.Log, conf),
+		client: servicedisc.NewClient(f.Log, conf, f.Client),
 	}
 }
 
@@ -74,11 +77,11 @@ func (f *Factory) AppUpdater(conf appcommon.ProcConfig) (Updater, bool) {
 	switch conf.AppName {
 	case skyenv.VPNServerName:
 		return &serviceUpdater{
-			client: servicedisc.NewClient(log, getServiceDiscConf(conf, servicedisc.ServiceTypeVPN)),
+			client: servicedisc.NewClient(log, getServiceDiscConf(conf, servicedisc.ServiceTypeVPN), f.Client),
 		}, true
 	case skyenv.SkysocksName:
 		return &serviceUpdater{
-			client: servicedisc.NewClient(log, getServiceDiscConf(conf, servicedisc.ServiceTypeSkysocks)),
+			client: servicedisc.NewClient(log, getServiceDiscConf(conf, servicedisc.ServiceTypeSkysocks), f.Client),
 		}, true
 	default:
 		return &emptyUpdater{}, false

pkg/dmsgc/dmsgc.go

@@ -2,6 +2,7 @@ package dmsgc
 
 import (
 	"context"
+	"net/http"
 
 	"github.com/skycoin/dmsg"
 	"github.com/skycoin/dmsg/cipher"
@@ -13,13 +14,13 @@ import (
 
 // DmsgConfig defines config for Dmsg network.
 type DmsgConfig struct {
-	Discovery     string   `json:"discovery"`
-	SessionsCount int      `json:"sessions_count"`
-	Servers       []string `json:"servers"`
+	Discovery     string        `json:"discovery"`
+	SessionsCount int           `json:"sessions_count"`
+	Servers       []*disc.Entry `json:"servers"`
 }
 
 // New makes new dmsg client from configuration
-func New(pk cipher.PubKey, sk cipher.SecKey, eb *appevent.Broadcaster, conf *DmsgConfig) *dmsg.Client {
+func New(pk cipher.PubKey, sk cipher.SecKey, eb *appevent.Broadcaster, conf *DmsgConfig, httpC http.Client) *dmsg.Client {
 	dmsgConf := &dmsg.Config{
 		MinSessions: conf.SessionsCount,
 		Callbacks: &dmsg.ClientCallbacks{
@@ -37,7 +38,8 @@ func New(pk cipher.PubKey, sk cipher.SecKey, eb *appevent.Broadcaster, conf *Dms
 			},
 		},
 	}
-	dmsgC := dmsg.NewClient(pk, sk, disc.NewHTTP(conf.Discovery), dmsgConf)
+
+	dmsgC := dmsg.NewClient(pk, sk, disc.NewHTTP(conf.Discovery, httpC), dmsgConf)
 	dmsgC.SetLogger(logging.MustGetLogger("dmsgC"))
 	return dmsgC
 }

pkg/routefinder/rfclient/client.go

@@ -62,14 +62,14 @@ type apiClient struct {
 }
 
 // NewHTTP constructs new Client that communicates over http.
-func NewHTTP(addr string, apiTimeout time.Duration) Client {
+func NewHTTP(addr string, apiTimeout time.Duration, client http.Client) Client {
 	if apiTimeout == 0 {
 		apiTimeout = defaultContextTimeout
 	}
 
 	return &apiClient{
 		addr:       sanitizedAddr(addr),
-		client:     http.Client{},
+		client:     client,
 		apiTimeout: apiTimeout,
 	}
 }

pkg/servicedisc/autoconnect.go

@@ -2,6 +2,7 @@ package servicedisc
 
 import (
 	"context"
+	"net/http"
 	"time"
 
 	"github.com/sirupsen/logrus"
@@ -38,9 +39,9 @@ type autoconnector struct {
 
 // MakeConnector returns a new connector that will try to connect to at most maxConns
 // services
-func MakeConnector(conf Config, maxConns int, tm *transport.Manager, log *logging.Logger) Autoconnector {
+func MakeConnector(conf Config, maxConns int, tm *transport.Manager, httpC http.Client, log *logging.Logger) Autoconnector {
 	connector := &autoconnector{}
-	connector.client = NewClient(log, conf)
+	connector.client = NewClient(log, conf, httpC)
 	connector.maxConns = maxConns
 	connector.log = log
 	connector.tm = tm

pkg/servicedisc/client.go

@@ -50,7 +50,7 @@ type HTTPClient struct {
 }
 
 // NewClient creates a new HTTPClient.
-func NewClient(log logrus.FieldLogger, conf Config) *HTTPClient {
+func NewClient(log logrus.FieldLogger, conf Config, client http.Client) *HTTPClient {
 	return &HTTPClient{
 		log:  log,
 		conf: conf,
@@ -59,7 +59,7 @@ func NewClient(log logrus.FieldLogger, conf Config) *HTTPClient {
 			Type:    conf.Type,
 			Version: buildinfo.Version(),
 		},
-		client: http.Client{},
+		client: client,
 	}
 }
 
@@ -96,7 +96,7 @@ func (c *HTTPClient) Auth(ctx context.Context) (*httpauth.Client, error) {
 		return auth, nil
 	}
 
-	auth, err := httpauth.NewClient(ctx, c.conf.DiscAddr, c.conf.PK, c.conf.SK)
+	auth, err := httpauth.NewClient(ctx, c.conf.DiscAddr, c.conf.PK, c.conf.SK, &c.client)
 	if err != nil {
 		return nil, err
 	}

pkg/setup/node.go

@@ -3,6 +3,7 @@ package setup
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"net/rpc"
 	"time"
 
@@ -33,7 +34,7 @@ func NewNode(conf *Config) (*Node, error) {
 	}
 
 	// Connect to dmsg network.
-	dmsgDisc := disc.NewHTTP(conf.Dmsg.Discovery)
+	dmsgDisc := disc.NewHTTP(conf.Dmsg.Discovery, http.Client{})
 	dmsgConf := &dmsg.Config{MinSessions: conf.Dmsg.SessionsCount}
 	dmsgC := dmsg.NewClient(conf.PK, conf.SK, dmsgDisc, dmsgConf)
 	go dmsgC.Serve(context.Background())