Appserver and visor improvements. (#357)

Appserver improvements:
* Combined the appserver.Server and appserver.ProcManager concepts.
* Have one rpc.Server per app proc.
* Simplified app.Client initialization.
* Visor to App initialization is done via a single env defined for the app PROC_CONFIG which contains a JSON object.
* App to Visor initialization is done by App connecting the Visor via app server address. The first 16 bytes that the App sends to Visor is the appcommon.ProcKey.
* Changed appcommon.ProcKey format so that it is a deterministic length.
* Improved app logging and removed any possibilities of log database race conditions with bbolt.
* Various logging improvements.

App management, visor config, startup and shutdown improvements:
* Separated app-associated functionality within /pkg/visor into /pkg/app/launcher.
* Added the concept of visor.BaseConfig() to simplify obtaining default values.
* Made startup/shutdown logic modular and introduced init and shutdown stacks.
* Shutdown now uses module-based timeouts and wait groups.

Changes as suggested by @Darkren and @nkryuchkov

Makefile

@@ -56,6 +56,15 @@ stop: ## Stop running skywire-visor on host
 config: ## Generate skywire.json
 	-./skywire-cli visor gen-config -o  ./skywire.json -r
 
+install-generate: ## Installs required execs for go generate.
+	${OPTS} go install github.com/mjibson/esc
+	${OPTS} go install github.com/vektra/mockery/cmd/mockery
+	# If the following does not work, you may need to run:
+	# 	git config --global [email protected]:.insteadOf https://github.com/
+	# Source: https://stackoverflow.com/questions/27500861/whats-the-proper-way-to-go-get-a-private-repository
+	# We are using 'go get' instead of 'go install' here, because we don't have a git tag in which 'readmegen' is already implemented.
+	${OPTS} go get -u github.com/SkycoinPro/skywire-services/cmd/readmegen@master
+
 generate: ## Generate mocks and config README's
 	go generate ./...
 

cmd/apps/helloworld/helloworld.go

@@ -4,11 +4,10 @@ simple client server app for skywire visor testing
 package main
 
 import (
-	"log"
 	"os"
 
 	"github.com/SkycoinProject/dmsg/cipher"
-	"github.com/SkycoinProject/skycoin/src/util/logging"
+	"github.com/sirupsen/logrus"
 
 	"github.com/SkycoinProject/skywire-mainnet/pkg/app"
 	"github.com/SkycoinProject/skywire-mainnet/pkg/app/appnet"
@@ -20,25 +19,19 @@ const (
 	netType = appnet.TypeSkynet
 )
 
+var log = logrus.New()
+
 func main() {
+	appC := app.NewClient()
+	defer appC.Close()
+
 	if _, err := buildinfo.Get().WriteTo(log.Writer()); err != nil {
 		log.Printf("Failed to output build info: %v", err)
 	}
 
-	clientConfig, err := app.ClientConfigFromEnv()
-	if err != nil {
-		log.Fatalf("Error getting client config: %v\n", err)
-	}
-
-	app, err := app.NewClient(logging.MustGetLogger("helloworld"), clientConfig)
-	if err != nil {
-		log.Fatalf("Error creating app client: %v\n", err)
-	}
-	defer app.Close()
-
 	if len(os.Args) == 1 {
 		port := routing.Port(1024)
-		l, err := app.Listen(netType, port)
+		l, err := appC.Listen(netType, port)
 		if err != nil {
 			log.Fatalf("Error listening network %v on port %d: %v\n", netType, port, err)
 		}
@@ -72,7 +65,7 @@ func main() {
 		log.Fatal("Failed to construct PubKey: ", err, os.Args[1])
 	}
 
-	conn, err := app.Dial(appnet.Addr{
+	conn, err := appC.Dial(appnet.Addr{
 		Net:    netType,
 		PubKey: remotePK,
 		Port:   10,

cmd/apps/skychat/chat.go

@@ -11,11 +11,12 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"os"
 	"sync"
 	"time"
 
 	"github.com/SkycoinProject/dmsg/cipher"
-	"github.com/SkycoinProject/skycoin/src/util/logging"
+	"github.com/sirupsen/logrus"
 
 	"github.com/SkycoinProject/skywire-mainnet/internal/netutil"
 	"github.com/SkycoinProject/skywire-mainnet/pkg/app"
@@ -25,7 +26,6 @@ import (
 )
 
 const (
-	appName = "skychat"
 	netType = appnet.TypeSkynet
 	port    = routing.Port(1)
 )
@@ -34,40 +34,28 @@ var addr = flag.String("addr", ":8001", "address to bind")
 var r = netutil.NewRetrier(50*time.Millisecond, 5, 2)
 
 var (
-	chatApp   *app.Client
-	clientCh  chan string
-	chatConns map[cipher.PubKey]net.Conn
-	connsMu   sync.Mutex
-	log       *logging.MasterLogger
+	log      = logrus.New()
+	appC     *app.Client
+	clientCh chan string
+	conns    map[cipher.PubKey]net.Conn // Chat connections
+	connsMu  sync.Mutex
 )
 
 func main() {
-	log = app.NewLogger(appName)
-	flag.Parse()
+	appC = app.NewClient()
+	defer appC.Close()
 
-	if _, err := buildinfo.Get().WriteTo(log.Writer()); err != nil {
+	if _, err := buildinfo.Get().WriteTo(os.Stdout); err != nil {
 		log.Printf("Failed to output build info: %v", err)
 	}
 
-	clientConfig, err := app.ClientConfigFromEnv()
-	if err != nil {
-		log.Fatalf("Error getting client config: %v\n", err)
-	}
-
-	// TODO: pass `log`?
-	a, err := app.NewClient(logging.MustGetLogger(fmt.Sprintf("app_%s", appName)), clientConfig)
-	if err != nil {
-		log.Fatal("Setup failure: ", err)
-	}
-	defer a.Close()
-	log.Println("Successfully created skychat app")
-
-	chatApp = a
+	flag.Parse()
+	log.Println("Successfully started skychat.")
 
 	clientCh = make(chan string)
 	defer close(clientCh)
 
-	chatConns = make(map[cipher.PubKey]net.Conn)
+	conns = make(map[cipher.PubKey]net.Conn)
 	go listenLoop()
 
 	http.Handle("/", http.FileServer(FS(false)))
@@ -79,7 +67,7 @@ func main() {
 }
 
 func listenLoop() {
-	l, err := chatApp.Listen(netType, port)
+	l, err := appC.Listen(netType, port)
 	if err != nil {
 		log.Printf("Error listening network %v on port %d: %v\n", netType, port, err)
 		return
@@ -96,7 +84,7 @@ func listenLoop() {
 
 		raddr := conn.RemoteAddr().(appnet.Addr)
 		connsMu.Lock()
-		chatConns[raddr.PubKey] = conn
+		conns[raddr.PubKey] = conn
 		connsMu.Unlock()
 		log.Printf("Accepted skychat conn on %s from %s\n", conn.LocalAddr(), raddr.PubKey)
 
@@ -113,7 +101,7 @@ func handleConn(conn net.Conn) {
 			log.Println("Failed to read packet:", err)
 			raddr := conn.RemoteAddr().(appnet.Addr)
 			connsMu.Lock()
-			delete(chatConns, raddr.PubKey)
+			delete(conns, raddr.PubKey)
 			connsMu.Unlock()
 			return
 		}
@@ -150,13 +138,13 @@ func messageHandler(w http.ResponseWriter, req *http.Request) {
 		Port:   1,
 	}
 	connsMu.Lock()
-	conn, ok := chatConns[pk]
+	conn, ok := conns[pk]
 	connsMu.Unlock()
 
 	if !ok {
 		var err error
 		err = r.Do(func() error {
-			conn, err = chatApp.Dial(addr)
+			conn, err = appC.Dial(addr)
 			return err
 		})
 		if err != nil {
@@ -165,7 +153,7 @@ func messageHandler(w http.ResponseWriter, req *http.Request) {
 		}
 
 		connsMu.Lock()
-		chatConns[pk] = conn
+		conns[pk] = conn
 		connsMu.Unlock()
 
 		go handleConn(conn)
@@ -176,7 +164,7 @@ func messageHandler(w http.ResponseWriter, req *http.Request) {
 		http.Error(w, err.Error(), http.StatusBadRequest)
 
 		connsMu.Lock()
-		delete(chatConns, pk)
+		delete(conns, pk)
 		connsMu.Unlock()
 
 		return

cmd/apps/skysocks-client/skysocks-client.go

@@ -5,13 +5,12 @@ package main
 
 import (
 	"flag"
-	"fmt"
 	"io"
 	"net"
 	"time"
 
 	"github.com/SkycoinProject/dmsg/cipher"
-	"github.com/SkycoinProject/skycoin/src/util/logging"
+	"github.com/sirupsen/logrus"
 
 	"github.com/SkycoinProject/skywire-mainnet/internal/netutil"
 	"github.com/SkycoinProject/skywire-mainnet/internal/skysocks"
@@ -23,11 +22,12 @@ import (
 )
 
 const (
-	appName   = "skysocks-client"
 	netType   = appnet.TypeSkynet
 	socksPort = routing.Port(3)
 )
 
+var log = logrus.New()
+
 var r = netutil.NewRetrier(time.Second, 0, 1)
 
 func dialServer(appCl *app.Client, pk cipher.PubKey) (net.Conn, error) {
@@ -49,8 +49,10 @@ func dialServer(appCl *app.Client, pk cipher.PubKey) (net.Conn, error) {
 }
 
 func main() {
-	log := app.NewLogger(appName)
-	skysocks.Log = log.PackageLogger("skysocks")
+	appC := app.NewClient()
+	defer appC.Close()
+
+	skysocks.Log = log
 
 	if _, err := buildinfo.Get().WriteTo(log.Writer()); err != nil {
 		log.Printf("Failed to output build info: %v", err)
@@ -60,20 +62,6 @@ func main() {
 	var serverPK = flag.String("srv", "", "PubKey of the server to connect to")
 	flag.Parse()
 
-	config, err := app.ClientConfigFromEnv()
-	if err != nil {
-		log.Fatalf("Error getting client config: %v\n", err)
-	}
-
-	socksApp, err := app.NewClient(logging.MustGetLogger(fmt.Sprintf("app_%s", appName)), config)
-
-	if err != nil {
-		log.Fatal("Setup failure: ", err)
-	}
-	defer func() {
-		socksApp.Close()
-	}()
-
 	if *serverPK == "" {
 		log.Warn("Empty server PubKey. Exiting")
 		return
@@ -85,7 +73,7 @@ func main() {
 	}
 
 	for {
-		conn, err := dialServer(socksApp, pk)
+		conn, err := dialServer(appC, pk)
 		if err != nil {
 			log.Fatalf("Failed to dial to a server: %v", err)
 		}