Skip to content

Commit

Permalink
Fix health endpoint timeout
Browse files Browse the repository at this point in the history
  • Loading branch information
nkryuchkov committed Jul 23, 2020
1 parent 60c3fbd commit 3ea795a
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 35 deletions.
5 changes: 2 additions & 3 deletions pkg/hypervisor/hypervisor.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,7 @@ import (
)

const (
healthTimeout = 5 * time.Second
httpTimeout = 30 * time.Second
httpTimeout = 30 * time.Second
)

const (
Expand Down Expand Up @@ -366,7 +365,7 @@ func (hv *Hypervisor) getHealth() http.HandlerFunc {
}

resCh := make(chan healthRes)
tCh := time.After(healthTimeout)
tCh := time.After(visor.HealthTimeout)

go func() {
hi, err := ctx.RPC.Health()
Expand Down
9 changes: 7 additions & 2 deletions pkg/routefinder/rfclient/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,13 @@ func (c *apiClient) FindRoutes(ctx context.Context, rts []routing.PathEdges, opt
}

// Health checks route finder health.
func (c *apiClient) Health(_ context.Context) (int, error) {
res, err := http.Get(c.addr + "/health")
func (c *apiClient) Health(ctx context.Context) (int, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.addr+"/health", nil)
if err != nil {
return 0, err
}

res, err := http.DefaultClient.Do(req)
if err != nil {
return 0, err
}
Expand Down
83 changes: 53 additions & 30 deletions pkg/visor/rpc.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"net/http"
"net/rpc"
"sync"
"time"

"github.com/SkycoinProject/dmsg/buildinfo"
Expand All @@ -23,7 +24,8 @@ import (

const (
// RPCPrefix is the prefix used with all RPC calls.
RPCPrefix = "app-visor"
RPCPrefix = "app-visor"
HealthTimeout = 5 * time.Second
)

var (
Expand Down Expand Up @@ -74,34 +76,45 @@ type HealthInfo struct {
func (r *RPC) Health(_ *struct{}, out *HealthInfo) (err error) {
defer rpcutil.LogCall(r.log, "Health", nil)(out, &err)

ctx := context.Background()
ctx, cancel := context.WithTimeout(context.Background(), HealthTimeout/2)
defer cancel()

out.TransportDiscovery = http.StatusNotFound
out.RouteFinder = http.StatusNotFound
out.SetupNode = http.StatusNotFound
out.UptimeTracker = http.StatusNotFound
out.AddressResolver = http.StatusNotFound

if tdClient := r.visor.TpDiscClient(); tdClient != nil {
tdStatus, err := tdClient.Health(ctx)
if err != nil {
r.log.WithError(err).Warnf("Failed to check transport discovery health")
var wg sync.WaitGroup

out.TransportDiscovery = http.StatusInternalServerError
}
if tdClient := r.visor.TpDiscClient(); tdClient != nil {
wg.Add(1)
go func() {
defer wg.Done()
tdStatus, err := tdClient.Health(ctx)
if err != nil {
r.log.WithError(err).Warnf("Failed to check transport discovery health")

out.TransportDiscovery = http.StatusInternalServerError
}

out.TransportDiscovery = tdStatus
out.TransportDiscovery = tdStatus
}()
}

if rfClient := r.visor.RouteFinderClient(); rfClient != nil {
rfStatus, err := rfClient.Health(ctx)
if err != nil {
r.log.WithError(err).Warnf("Failed to check route finder health")

out.RouteFinder = http.StatusInternalServerError
}
wg.Add(1)
go func() {
defer wg.Done()
rfStatus, err := rfClient.Health(ctx)
if err != nil {
r.log.WithError(err).Warnf("Failed to check route finder health")

out.RouteFinder = http.StatusInternalServerError
}

out.RouteFinder = rfStatus
out.RouteFinder = rfStatus
}()
}

// TODO(evanlinjin): This should actually poll the setup nodes services.
Expand All @@ -112,27 +125,37 @@ func (r *RPC) Health(_ *struct{}, out *HealthInfo) (err error) {
}

if utClient := r.visor.UptimeTrackerClient(); utClient != nil {
utStatus, err := utClient.Health(ctx)
if err != nil {
r.log.WithError(err).Warnf("Failed to check uptime tracker health")

out.UptimeTracker = http.StatusInternalServerError
}
wg.Add(1)
go func() {
defer wg.Done()
utStatus, err := utClient.Health(ctx)
if err != nil {
r.log.WithError(err).Warnf("Failed to check uptime tracker health")

out.UptimeTracker = http.StatusInternalServerError
}

out.UptimeTracker = utStatus
out.UptimeTracker = utStatus
}()
}

if arClient := r.visor.AddressResolverClient(); arClient != nil {
arStatus, err := arClient.Health(ctx)
if err != nil {
r.log.WithError(err).Warnf("Failed to check address resolver health")

out.AddressResolver = http.StatusInternalServerError
}
wg.Add(1)
go func() {
defer wg.Done()
arStatus, err := arClient.Health(ctx)
if err != nil {
r.log.WithError(err).Warnf("Failed to check address resolver health")

out.AddressResolver = http.StatusInternalServerError
}

out.AddressResolver = arStatus
out.AddressResolver = arStatus
}()
}

wg.Wait()

return nil
}

Expand Down

0 comments on commit 3ea795a

Please sign in to comment.