From 88b45fde111c190ec09949adf1280224ae7b8867 Mon Sep 17 00:00:00 2001 From: Antonio Navarro Perez Date: Tue, 31 Jan 2023 12:28:57 +0100 Subject: [PATCH] WIP: Implement new ResourceManager partial limits config object Signed-off-by: Antonio Navarro Perez --- config/swarm.go | 4 +- core/commands/swarm.go | 2 +- core/node/libp2p/rcmgr.go | 221 +++++++++++-------------- core/node/libp2p/rcmgr_defaults.go | 165 ++++++++---------- core/node/libp2p/rcmgr_logging_test.go | 16 +- go.mod | 8 +- go.sum | 15 +- 7 files changed, 190 insertions(+), 241 deletions(-) diff --git a/config/swarm.go b/config/swarm.go index d8fd17e946d8..abff91422076 100644 --- a/config/swarm.go +++ b/config/swarm.go @@ -141,8 +141,8 @@ type ConnMgr struct { // type ResourceMgr struct { // Enables the Network Resource Manager feature, default to on. - Enabled Flag `json:",omitempty"` - Limits *rcmgr.LimitConfig `json:",omitempty"` + Enabled Flag `json:",omitempty"` + Limits *rcmgr.PartialLimitConfig `json:",omitempty"` MaxMemory *OptionalString `json:",omitempty"` MaxFileDescriptors *OptionalInteger `json:",omitempty"` diff --git a/core/commands/swarm.go b/core/commands/swarm.go index d00291f78f5f..0ef9a2ac19c6 100644 --- a/core/commands/swarm.go +++ b/core/commands/swarm.go @@ -430,7 +430,7 @@ Changes made via command line are persisted in the Swarm.ResourceMgr.Limits fiel // set scope limit to new values (when limit.json is passed as a second arg) if req.Files != nil { - var newLimit rcmgr.BaseLimit + var newLimit *rcmgr.ResourceLimits it := req.Files.Entries() if it.Next() { file := files.FileFromEntry(it) diff --git a/core/node/libp2p/rcmgr.go b/core/node/libp2p/rcmgr.go index e0ce4693be19..bca680e11497 100644 --- a/core/node/libp2p/rcmgr.go +++ b/core/node/libp2p/rcmgr.go @@ -3,6 +3,7 @@ package libp2p import ( "context" "fmt" + "math" "os" "path/filepath" "strings" @@ -52,33 +53,15 @@ func ResourceManager(cfg config.SwarmConfig) interface{} { return nil, opts, fmt.Errorf("opening IPFS_PATH: %w", err) } - var limitConfig rcmgr.LimitConfig - defaultComputedLimitConfig, err := createDefaultLimitConfig(cfg) + concreteLimitConfig, err := createDefaultLimitConfig(cfg, true) if err != nil { return nil, opts, err } - // The logic for defaults and overriding with specified SwarmConfig.ResourceMgr.Limits - // is documented in docs/config.md. - // Any changes here should be reflected there. - if cfg.ResourceMgr.Limits != nil { - userSuppliedOverrideLimitConfig := *cfg.ResourceMgr.Limits - // This effectively overrides the computed default LimitConfig with any non-zero values from cfg.ResourceMgr.Limits. - // Because of how how Apply works, any 0 value for a user supplied override - // will be overriden with a computed default value. - // There currently isn't a way for a user to supply a 0-value override. - userSuppliedOverrideLimitConfig.Apply(defaultComputedLimitConfig) - limitConfig = userSuppliedOverrideLimitConfig - } else { - limitConfig = defaultComputedLimitConfig - } - - if err := ensureConnMgrMakeSenseVsResourceMgr(limitConfig, cfg.ConnMgr); err != nil { + if err := ensureConnMgrMakeSenseVsResourceMgr(concreteLimitConfig, cfg.ConnMgr); err != nil { return nil, opts, err } - limiter := rcmgr.NewFixedLimiter(limitConfig) - str, err := rcmgrObs.NewStatsTraceReporter() if err != nil { return nil, opts, err @@ -110,6 +93,8 @@ func ResourceManager(cfg config.SwarmConfig) interface{} { ropts = append(ropts, rcmgr.WithTrace(traceFilePath)) } + limiter := rcmgr.NewFixedLimiter(concreteLimitConfig) + manager, err = rcmgr.NewResourceManager(limiter, ropts...) if err != nil { return nil, opts, fmt.Errorf("creating libp2p resource manager: %w", err) @@ -138,11 +123,11 @@ func ResourceManager(cfg config.SwarmConfig) interface{} { } type NetStatOut struct { - System *rcmgr.BaseLimit `json:",omitempty"` - Transient *rcmgr.BaseLimit `json:",omitempty"` - Services map[string]rcmgr.BaseLimit `json:",omitempty"` - Protocols map[string]rcmgr.BaseLimit `json:",omitempty"` - Peers map[string]rcmgr.BaseLimit `json:",omitempty"` + System *rcmgr.ResourceLimits `json:",omitempty"` + Transient *rcmgr.ResourceLimits `json:",omitempty"` + Services map[string]rcmgr.ResourceLimits `json:",omitempty"` + Protocols map[string]rcmgr.ResourceLimits `json:",omitempty"` + Peers map[string]rcmgr.ResourceLimits `json:",omitempty"` } func NetStat(mgr network.ResourceManager, scope string, percentage int) (NetStatOut, error) { @@ -164,7 +149,7 @@ func NetStat(mgr network.ResourceManager, scope string, percentage int) (NetStat result.System = compareLimits(scopeToLimit(&stat.System), limits.System, percentage) result.Transient = compareLimits(scopeToLimit(&stat.Transient), limits.Transient, percentage) if len(stat.Services) > 0 { - result.Services = make(map[string]rcmgr.BaseLimit, len(stat.Services)) + result.Services = make(map[string]rcmgr.ResourceLimits, len(stat.Services)) for srv, stat := range stat.Services { ls := limits.Services[srv] fstat := compareLimits(scopeToLimit(&stat), &ls, percentage) @@ -174,7 +159,7 @@ func NetStat(mgr network.ResourceManager, scope string, percentage int) (NetStat } } if len(stat.Protocols) > 0 { - result.Protocols = make(map[string]rcmgr.BaseLimit, len(stat.Protocols)) + result.Protocols = make(map[string]rcmgr.ResourceLimits, len(stat.Protocols)) for proto, stat := range stat.Protocols { ls := limits.Protocols[string(proto)] fstat := compareLimits(scopeToLimit(&stat), &ls, percentage) @@ -184,7 +169,7 @@ func NetStat(mgr network.ResourceManager, scope string, percentage int) (NetStat } } if len(stat.Peers) > 0 { - result.Peers = make(map[string]rcmgr.BaseLimit, len(stat.Peers)) + result.Peers = make(map[string]rcmgr.ResourceLimits, len(stat.Peers)) for p, stat := range stat.Peers { ls := limits.Peers[p.Pretty()] fstat := compareLimits(scopeToLimit(&stat), &ls, percentage) @@ -216,7 +201,7 @@ func NetStat(mgr network.ResourceManager, scope string, percentage int) (NetStat svc := strings.TrimPrefix(scope, config.ResourceMgrServiceScopePrefix) err = mgr.ViewService(svc, func(s network.ServiceScope) error { stat := s.Stat() - result.Services = map[string]rcmgr.BaseLimit{ + result.Services = map[string]rcmgr.ResourceLimits{ svc: *scopeToLimit(&stat), } return nil @@ -227,7 +212,7 @@ func NetStat(mgr network.ResourceManager, scope string, percentage int) (NetStat proto := strings.TrimPrefix(scope, config.ResourceMgrProtocolScopePrefix) err = mgr.ViewProtocol(protocol.ID(proto), func(s network.ProtocolScope) error { stat := s.Stat() - result.Protocols = map[string]rcmgr.BaseLimit{ + result.Protocols = map[string]rcmgr.ResourceLimits{ proto: *scopeToLimit(&stat), } return nil @@ -242,7 +227,7 @@ func NetStat(mgr network.ResourceManager, scope string, percentage int) (NetStat } err = mgr.ViewPeer(pid, func(s network.PeerScope) error { stat := s.Stat() - result.Peers = map[string]rcmgr.BaseLimit{ + result.Peers = map[string]rcmgr.ResourceLimits{ p: *scopeToLimit(&stat), } return nil @@ -262,48 +247,49 @@ var scopes = []string{ config.ResourceMgrPeerScopePrefix, } -func scopeToLimit(s *network.ScopeStat) *rcmgr.BaseLimit { - return &rcmgr.BaseLimit{ - Streams: s.NumStreamsInbound + s.NumStreamsOutbound, - StreamsInbound: s.NumStreamsInbound, - StreamsOutbound: s.NumStreamsOutbound, - Conns: s.NumConnsInbound + s.NumConnsOutbound, - ConnsInbound: s.NumConnsInbound, - ConnsOutbound: s.NumConnsOutbound, - FD: s.NumFD, - Memory: s.Memory, +func scopeToLimit(s *network.ScopeStat) *rcmgr.ResourceLimits { + + return &rcmgr.ResourceLimits{ + Streams: rcmgr.LimitVal(s.NumStreamsInbound + s.NumStreamsOutbound), + StreamsInbound: rcmgr.LimitVal(s.NumStreamsInbound), + StreamsOutbound: rcmgr.LimitVal(s.NumStreamsOutbound), + Conns: rcmgr.LimitVal(s.NumConnsInbound + s.NumConnsOutbound), + ConnsInbound: rcmgr.LimitVal(s.NumConnsInbound), + ConnsOutbound: rcmgr.LimitVal(s.NumConnsOutbound), + FD: rcmgr.LimitVal(s.NumFD), + Memory: rcmgr.LimitVal64(s.Memory), } } // compareLimits compares stat and limit. // If any of the stats value are equals or above the specified percentage, // stat object is returned. -func compareLimits(stat, limit *rcmgr.BaseLimit, percentage int) *rcmgr.BaseLimit { +func compareLimits(stat, limit *rcmgr.ResourceLimits, percentage int) *rcmgr.ResourceLimits { if stat == nil || limit == nil { return nil } if abovePercentage(int(stat.Memory), int(limit.Memory), percentage) { return stat } - if abovePercentage(stat.ConnsInbound, limit.ConnsInbound, percentage) { + if abovePercentage(int(stat.ConnsInbound), int(limit.ConnsInbound), percentage) { return stat } - if abovePercentage(stat.ConnsOutbound, limit.ConnsOutbound, percentage) { + if abovePercentage(int(stat.ConnsOutbound), int(limit.ConnsOutbound), percentage) { return stat } - if abovePercentage(stat.Conns, limit.Conns, percentage) { + if abovePercentage(int(stat.Conns), int(limit.Conns), percentage) { return stat } - if abovePercentage(stat.FD, limit.FD, percentage) { + if abovePercentage(int(stat.FD), int(limit.FD), percentage) { return stat } - if abovePercentage(stat.StreamsInbound, limit.StreamsInbound, percentage) { + if abovePercentage(int(stat.StreamsInbound), int(limit.StreamsInbound), percentage) { return stat } - if abovePercentage(stat.StreamsOutbound, limit.StreamsOutbound, percentage) { + if abovePercentage(int(stat.StreamsOutbound), int(limit.StreamsOutbound), percentage) { return stat } - if abovePercentage(stat.Streams, limit.Streams, percentage) { + if abovePercentage(int(stat.Streams), int(limit.Streams), percentage) { return stat } @@ -322,6 +308,17 @@ func abovePercentage(v1, v2, percentage int) bool { return int((float64(v1)/float64(v2))*100) >= percentage } +var infiniteBaseLimit = rcmgr.BaseLimit{ + Streams: math.MaxInt, + StreamsInbound: math.MaxInt, + StreamsOutbound: math.MaxInt, + Conns: math.MaxInt, + ConnsInbound: math.MaxInt, + ConnsOutbound: math.MaxInt, + FD: math.MaxInt, + Memory: math.MaxInt64, +} + func NetLimitAll(mgr network.ResourceManager) (*NetStatOut, error) { var result = &NetStatOut{} lister, ok := mgr.(rcmgr.ResourceManagerState) @@ -336,41 +333,41 @@ func NetLimitAll(mgr network.ResourceManager) (*NetStatOut, error) { if err != nil { return nil, err } - result.System = &s + result.System = s case config.ResourceMgrTransientScope: s, err := NetLimit(mgr, config.ResourceMgrSystemScope) if err != nil { return nil, err } - result.Transient = &s + result.Transient = s case config.ResourceMgrServiceScopePrefix: - result.Services = make(map[string]rcmgr.BaseLimit) + result.Services = make(map[string]rcmgr.ResourceLimits) for _, serv := range lister.ListServices() { s, err := NetLimit(mgr, config.ResourceMgrServiceScopePrefix+serv) if err != nil { return nil, err } - result.Services[serv] = s + result.Services[serv] = *s } case config.ResourceMgrProtocolScopePrefix: - result.Protocols = make(map[string]rcmgr.BaseLimit) + result.Protocols = make(map[string]rcmgr.ResourceLimits) for _, prot := range lister.ListProtocols() { ps := string(prot) s, err := NetLimit(mgr, config.ResourceMgrProtocolScopePrefix+ps) if err != nil { return nil, err } - result.Protocols[ps] = s + result.Protocols[ps] = *s } case config.ResourceMgrPeerScopePrefix: - result.Peers = make(map[string]rcmgr.BaseLimit) + result.Peers = make(map[string]rcmgr.ResourceLimits) for _, peer := range lister.ListPeers() { ps := peer.Pretty() s, err := NetLimit(mgr, config.ResourceMgrPeerScopePrefix+ps) if err != nil { return nil, err } - result.Peers[ps] = s + result.Peers[ps] = *s } } } @@ -378,8 +375,8 @@ func NetLimitAll(mgr network.ResourceManager) (*NetStatOut, error) { return result, nil } -func NetLimit(mgr network.ResourceManager, scope string) (rcmgr.BaseLimit, error) { - var result rcmgr.BaseLimit +func NetLimit(mgr network.ResourceManager, scope string) (*rcmgr.ResourceLimits, error) { + var result *rcmgr.ResourceLimits getLimit := func(s network.ResourceScope) error { limiter, ok := s.(rcmgr.ResourceScopeLimiter) if !ok { // NullResourceManager @@ -388,14 +385,7 @@ func NetLimit(mgr network.ResourceManager, scope string) (rcmgr.BaseLimit, error limit := limiter.Limit() switch l := limit.(type) { case *rcmgr.BaseLimit: - result.Memory = l.Memory - result.Streams = l.Streams - result.StreamsInbound = l.StreamsInbound - result.StreamsOutbound = l.StreamsOutbound - result.Conns = l.Conns - result.ConnsInbound = l.ConnsInbound - result.ConnsOutbound = l.ConnsOutbound - result.FD = l.FD + result = l.ToResourceLimits() default: return fmt.Errorf("unknown limit type %T", limit) } @@ -427,14 +417,15 @@ func NetLimit(mgr network.ResourceManager, scope string) (rcmgr.BaseLimit, error } // NetSetLimit sets new ResourceManager limits for the given scope. The limits take effect immediately, and are also persisted to the repo config. -func NetSetLimit(mgr network.ResourceManager, repo repo.Repo, scope string, limit rcmgr.BaseLimit) error { +func NetSetLimit(mgr network.ResourceManager, repo repo.Repo, scope string, limit *rcmgr.ResourceLimits) error { setLimit := func(s network.ResourceScope) error { limiter, ok := s.(rcmgr.ResourceScopeLimiter) if !ok { // NullResourceManager return ErrNoResourceMgr } - limiter.SetLimit(&limit) + baseLimit := limit.Build(infiniteBaseLimit) + limiter.SetLimit(&baseLimit) return nil } @@ -444,7 +435,7 @@ func NetSetLimit(mgr network.ResourceManager, repo repo.Repo, scope string, limi } if cfg.Swarm.ResourceMgr.Limits == nil { - cfg.Swarm.ResourceMgr.Limits = &rcmgr.LimitConfig{} + cfg.Swarm.ResourceMgr.Limits = &rcmgr.PartialLimitConfig{} } configLimits := cfg.Swarm.ResourceMgr.Limits @@ -461,18 +452,18 @@ func NetSetLimit(mgr network.ResourceManager, repo repo.Repo, scope string, limi err = mgr.ViewService(svc, func(s network.ServiceScope) error { return setLimit(s) }) setConfigFunc = func() { if configLimits.Service == nil { - configLimits.Service = map[string]rcmgr.BaseLimit{} + configLimits.Service = map[string]rcmgr.ResourceLimits{} } - configLimits.Service[svc] = limit + configLimits.Service[svc] = *limit } case strings.HasPrefix(scope, config.ResourceMgrProtocolScopePrefix): proto := strings.TrimPrefix(scope, config.ResourceMgrProtocolScopePrefix) err = mgr.ViewProtocol(protocol.ID(proto), func(s network.ProtocolScope) error { return setLimit(s) }) setConfigFunc = func() { if configLimits.Protocol == nil { - configLimits.Protocol = map[protocol.ID]rcmgr.BaseLimit{} + configLimits.Protocol = map[protocol.ID]rcmgr.ResourceLimits{} } - configLimits.Protocol[protocol.ID(proto)] = limit + configLimits.Protocol[protocol.ID(proto)] = *limit } case strings.HasPrefix(scope, config.ResourceMgrPeerScopePrefix): p := strings.TrimPrefix(scope, config.ResourceMgrPeerScopePrefix) @@ -484,9 +475,9 @@ func NetSetLimit(mgr network.ResourceManager, repo repo.Repo, scope string, limi err = mgr.ViewPeer(pid, func(s network.PeerScope) error { return setLimit(s) }) setConfigFunc = func() { if configLimits.Peer == nil { - configLimits.Peer = map[peer.ID]rcmgr.BaseLimit{} + configLimits.Peer = map[peer.ID]rcmgr.ResourceLimits{} } - configLimits.Peer[pid] = limit + configLimits.Peer[pid] = *limit } default: return fmt.Errorf("invalid scope %q", scope) @@ -497,7 +488,7 @@ func NetSetLimit(mgr network.ResourceManager, repo repo.Repo, scope string, limi } if cfg.Swarm.ResourceMgr.Limits == nil { - cfg.Swarm.ResourceMgr.Limits = &rcmgr.LimitConfig{} + cfg.Swarm.ResourceMgr.Limits = &rcmgr.PartialLimitConfig{} } setConfigFunc() @@ -509,16 +500,18 @@ func NetSetLimit(mgr network.ResourceManager, repo repo.Repo, scope string, limi } // NetResetLimit resets ResourceManager limits to defaults. The limits take effect immediately, and are also persisted to the repo config. -func NetResetLimit(mgr network.ResourceManager, repo repo.Repo, scope string) (rcmgr.BaseLimit, error) { - var result rcmgr.BaseLimit +func NetResetLimit(mgr network.ResourceManager, repo repo.Repo, scope string) (*rcmgr.ResourceLimits, error) { + var result *rcmgr.ResourceLimits - setLimit := func(s network.ResourceScope, l rcmgr.Limit) error { + setLimit := func(s network.ResourceScope, limit *rcmgr.ResourceLimits) error { limiter, ok := s.(rcmgr.ResourceScopeLimiter) if !ok { return ErrNoResourceMgr } - limiter.SetLimit(l) + baseLimit := limit.Build(infiniteBaseLimit) + result = limit + limiter.SetLimit(&baseLimit) return nil } @@ -527,53 +520,43 @@ func NetResetLimit(mgr network.ResourceManager, repo repo.Repo, scope string) (r return result, fmt.Errorf("reading config to reset limit: %w", err) } - defaults, err := createDefaultLimitConfig(cfg.Swarm) + clc, err := createDefaultLimitConfig(cfg.Swarm, false) if err != nil { return result, fmt.Errorf("creating default limit config: %w", err) } + defaults := clc.ToLimitConfig() + if cfg.Swarm.ResourceMgr.Limits == nil { - cfg.Swarm.ResourceMgr.Limits = &rcmgr.LimitConfig{} + cfg.Swarm.ResourceMgr.Limits = &rcmgr.PartialLimitConfig{} } configLimits := cfg.Swarm.ResourceMgr.Limits - var setConfigFunc func() rcmgr.BaseLimit switch { case scope == config.ResourceMgrSystemScope: - err = mgr.ViewSystem(func(s network.ResourceScope) error { return setLimit(s, &defaults.System) }) - setConfigFunc = func() rcmgr.BaseLimit { - configLimits.System = defaults.System - return defaults.System - } + err = mgr.ViewSystem(func(s network.ResourceScope) error { return setLimit(s, defaults.System) }) + configLimits.System = nil case scope == config.ResourceMgrTransientScope: - err = mgr.ViewTransient(func(s network.ResourceScope) error { return setLimit(s, &defaults.Transient) }) - setConfigFunc = func() rcmgr.BaseLimit { - configLimits.Transient = defaults.Transient - return defaults.Transient - } + err = mgr.ViewTransient(func(s network.ResourceScope) error { return setLimit(s, defaults.Transient) }) + configLimits.Transient = nil case strings.HasPrefix(scope, config.ResourceMgrServiceScopePrefix): svc := strings.TrimPrefix(scope, config.ResourceMgrServiceScopePrefix) - err = mgr.ViewService(svc, func(s network.ServiceScope) error { return setLimit(s, &defaults.ServiceDefault) }) - setConfigFunc = func() rcmgr.BaseLimit { - if configLimits.Service == nil { - configLimits.Service = map[string]rcmgr.BaseLimit{} - } - configLimits.Service[svc] = defaults.ServiceDefault - return defaults.ServiceDefault + err = mgr.ViewService(svc, func(s network.ServiceScope) error { return setLimit(s, defaults.ServiceDefault) }) + if configLimits.Service == nil { + configLimits.Service = map[string]rcmgr.ResourceLimits{} } + + delete(configLimits.Service, svc) case strings.HasPrefix(scope, config.ResourceMgrProtocolScopePrefix): proto := strings.TrimPrefix(scope, config.ResourceMgrProtocolScopePrefix) - err = mgr.ViewProtocol(protocol.ID(proto), func(s network.ProtocolScope) error { return setLimit(s, &defaults.ProtocolDefault) }) - setConfigFunc = func() rcmgr.BaseLimit { - if configLimits.Protocol == nil { - configLimits.Protocol = map[protocol.ID]rcmgr.BaseLimit{} - } - configLimits.Protocol[protocol.ID(proto)] = defaults.ProtocolDefault - - return defaults.ProtocolDefault + err = mgr.ViewProtocol(protocol.ID(proto), func(s network.ProtocolScope) error { return setLimit(s, defaults.ProtocolDefault) }) + if configLimits.Protocol == nil { + configLimits.Protocol = map[protocol.ID]rcmgr.ResourceLimits{} } + + delete(configLimits.Protocol, protocol.ID(proto)) case strings.HasPrefix(scope, config.ResourceMgrPeerScopePrefix): p := strings.TrimPrefix(scope, config.ResourceMgrPeerScopePrefix) @@ -583,15 +566,12 @@ func NetResetLimit(mgr network.ResourceManager, repo repo.Repo, scope string) (r return result, fmt.Errorf("invalid peer ID: %q: %w", p, err) } - err = mgr.ViewPeer(pid, func(s network.PeerScope) error { return setLimit(s, &defaults.PeerDefault) }) - setConfigFunc = func() rcmgr.BaseLimit { - if configLimits.Peer == nil { - configLimits.Peer = map[peer.ID]rcmgr.BaseLimit{} - } - configLimits.Peer[pid] = defaults.PeerDefault - - return defaults.PeerDefault + err = mgr.ViewPeer(pid, func(s network.PeerScope) error { return setLimit(s, defaults.PeerDefault) }) + if configLimits.Peer == nil { + configLimits.Peer = map[peer.ID]rcmgr.ResourceLimits{} } + + delete(configLimits.Peer, pid) default: return result, fmt.Errorf("invalid scope %q", scope) } @@ -600,8 +580,6 @@ func NetResetLimit(mgr network.ResourceManager, repo repo.Repo, scope string) (r return result, fmt.Errorf("resetting new limits on resource manager: %w", err) } - result = setConfigFunc() - if err := repo.SetConfig(cfg); err != nil { return result, fmt.Errorf("writing new limits to repo config: %w", err) } @@ -609,10 +587,13 @@ func NetResetLimit(mgr network.ResourceManager, repo repo.Repo, scope string) (r return result, nil } -func ensureConnMgrMakeSenseVsResourceMgr(rcm rcmgr.LimitConfig, cmgr config.ConnMgr) error { +func ensureConnMgrMakeSenseVsResourceMgr(concreteLimits rcmgr.ConcreteLimitConfig, cmgr config.ConnMgr) error { if cmgr.Type.WithDefault(config.DefaultConnMgrType) == "none" { return nil // none connmgr, no checks to do } + + rcm := concreteLimits.ToLimitConfig() + highWater := cmgr.HighWater.WithDefault(config.DefaultConnMgrHighWater) if rcm.System.ConnsInbound <= rcm.System.Conns { if int64(rcm.System.ConnsInbound) <= highWater { diff --git a/core/node/libp2p/rcmgr_defaults.go b/core/node/libp2p/rcmgr_defaults.go index 9a3825108f39..a64409414a8e 100644 --- a/core/node/libp2p/rcmgr_defaults.go +++ b/core/node/libp2p/rcmgr_defaults.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/dustin/go-humanize" - "github.com/libp2p/go-libp2p" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" "github.com/pbnjay/memory" @@ -12,33 +11,15 @@ import ( "github.com/ipfs/kubo/core/node/libp2p/fd" ) -// We are doing some magic when parsing config files (we are using a map[string]interface{} to compare config files). -// When you don't have a type the JSON Parse function cast numbers to float64 by default, -// losing precision when writing the final number. So if we use math.MaxInt as our infinite number, -// after writing the config file we will have 9223372036854776000 instead of 9223372036854775807, -// making the parsing process fail. Setting 1e9 (1000000000) as "no limit" value. It also avoids to overflow on 32 bit architectures. -const bigEnough = 1e9 - -var infiniteBaseLimit = rcmgr.BaseLimit{ - Streams: bigEnough, - StreamsInbound: bigEnough, - StreamsOutbound: bigEnough, - Conns: bigEnough, - ConnsInbound: bigEnough, - ConnsOutbound: bigEnough, - FD: bigEnough, - Memory: bigEnough, -} - -var noLimitIncrease = rcmgr.BaseLimitIncrease{ - ConnsInbound: 0, - ConnsOutbound: 0, - Conns: 0, - StreamsInbound: 0, - StreamsOutbound: 0, - Streams: 0, - Memory: 0, - FDFraction: 0, +var infiniteResourceLimit = &rcmgr.ResourceLimits{ + Streams: rcmgr.Unlimited, + StreamsInbound: rcmgr.Unlimited, + StreamsOutbound: rcmgr.Unlimited, + Conns: rcmgr.Unlimited, + ConnsInbound: rcmgr.Unlimited, + ConnsOutbound: rcmgr.Unlimited, + FD: rcmgr.Unlimited, + Memory: rcmgr.Unlimited64, } // This file defines implicit limit defaults used when Swarm.ResourceMgr.Enabled @@ -46,12 +27,12 @@ var noLimitIncrease = rcmgr.BaseLimitIncrease{ // createDefaultLimitConfig creates LimitConfig to pass to libp2p's resource manager. // The defaults follow the documentation in docs/libp2p-resource-management.md. // Any changes in the logic here should be reflected there. -func createDefaultLimitConfig(cfg config.SwarmConfig) (rcmgr.LimitConfig, error) { +func createDefaultLimitConfig(cfg config.SwarmConfig, applyConfigLimits bool) (rcmgr.ConcreteLimitConfig, error) { maxMemoryDefaultString := humanize.Bytes(uint64(memory.TotalMemory()) / 2) maxMemoryString := cfg.ResourceMgr.MaxMemory.WithDefault(maxMemoryDefaultString) maxMemory, err := humanize.ParseBytes(maxMemoryString) if err != nil { - return rcmgr.LimitConfig{}, err + return rcmgr.ConcreteLimitConfig{}, err } maxMemoryMB := maxMemory / (1024 * 1024) @@ -79,108 +60,79 @@ Run 'ipfs swarm limit all' to see the resulting limits. // (see https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/limit_defaults.go#L357 ). systemConnsInbound := int(1 * maxMemoryMB) - scalingLimitConfig := rcmgr.ScalingLimitConfig{ - SystemBaseLimit: rcmgr.BaseLimit{ - Memory: int64(maxMemory), - FD: maxFD, + partialLimits := rcmgr.PartialLimitConfig{ + System: &rcmgr.ResourceLimits{ + Memory: rcmgr.LimitVal64(maxMemory), + FD: rcmgr.LimitVal(maxFD), - // By default, we just limit connections on the inbound side. - Conns: bigEnough, - ConnsInbound: systemConnsInbound, - ConnsOutbound: bigEnough, + Conns: rcmgr.Unlimited, + ConnsInbound: rcmgr.LimitVal(systemConnsInbound), + ConnsOutbound: rcmgr.Unlimited, - Streams: bigEnough, - StreamsInbound: bigEnough, - StreamsOutbound: bigEnough, + Streams: rcmgr.Unlimited, + StreamsOutbound: rcmgr.Unlimited, + StreamsInbound: rcmgr.Unlimited, }, - SystemLimitIncrease: noLimitIncrease, // Transient connections won't cause any memory to accounted for by the resource manager. // Only established connections do. // As a result, we can't rely on System.Memory to protect us from a bunch of transient connection being opened. // We limit the same values as the System scope, but only allow the Transient scope to take 25% of what is allowed for the System scope. - TransientBaseLimit: rcmgr.BaseLimit{ - Memory: int64(maxMemory / 4), - FD: maxFD / 4, + Transient: &rcmgr.ResourceLimits{ + Memory: rcmgr.LimitVal64(maxMemory / 4), + FD: rcmgr.LimitVal(maxFD / 4), - Conns: bigEnough, - ConnsInbound: systemConnsInbound / 4, - ConnsOutbound: bigEnough, + Conns: rcmgr.Unlimited, + ConnsInbound: rcmgr.LimitVal(systemConnsInbound / 4), + ConnsOutbound: rcmgr.Unlimited, - Streams: bigEnough, - StreamsInbound: bigEnough, - StreamsOutbound: bigEnough, + Streams: rcmgr.Unlimited, + StreamsInbound: rcmgr.Unlimited, + StreamsOutbound: rcmgr.Unlimited, }, - TransientLimitIncrease: noLimitIncrease, - // Lets get out of the way of the allow list functionality. // If someone specified "Swarm.ResourceMgr.Allowlist" we should let it go through. - AllowlistedSystemBaseLimit: infiniteBaseLimit, - AllowlistedSystemLimitIncrease: noLimitIncrease, + AllowlistedSystem: infiniteResourceLimit, - AllowlistedTransientBaseLimit: infiniteBaseLimit, - AllowlistedTransientLimitIncrease: noLimitIncrease, + AllowlistedTransient: infiniteResourceLimit, // Keep it simple by not having Service, ServicePeer, Protocol, ProtocolPeer, Conn, or Stream limits. - ServiceBaseLimit: infiniteBaseLimit, - ServiceLimitIncrease: noLimitIncrease, + ServiceDefault: infiniteResourceLimit, - ServicePeerBaseLimit: infiniteBaseLimit, - ServicePeerLimitIncrease: noLimitIncrease, + ServicePeerDefault: infiniteResourceLimit, - ProtocolBaseLimit: infiniteBaseLimit, - ProtocolLimitIncrease: noLimitIncrease, + ProtocolDefault: infiniteResourceLimit, - ProtocolPeerBaseLimit: infiniteBaseLimit, - ProtocolPeerLimitIncrease: noLimitIncrease, + ProtocolPeerDefault: infiniteResourceLimit, - ConnBaseLimit: infiniteBaseLimit, - ConnLimitIncrease: noLimitIncrease, + Conn: infiniteResourceLimit, - StreamBaseLimit: infiniteBaseLimit, - StreamLimitIncrease: noLimitIncrease, + Stream: infiniteResourceLimit, // Limit the resources consumed by a peer. // This doesn't protect us against intentional DoS attacks since an attacker can easily spin up multiple peers. // We specify this limit against unintentional DoS attacks (e.g., a peer has a bug and is sending too much traffic intentionally). // In that case we want to keep that peer's resource consumption contained. // To keep this simple, we only constrain inbound connections and streams. - PeerBaseLimit: rcmgr.BaseLimit{ - Memory: bigEnough, - FD: bigEnough, - Conns: bigEnough, - ConnsInbound: rcmgr.DefaultLimits.PeerBaseLimit.ConnsInbound, - ConnsOutbound: bigEnough, - Streams: bigEnough, - StreamsInbound: rcmgr.DefaultLimits.PeerBaseLimit.StreamsInbound, - StreamsOutbound: bigEnough, - }, - // Most limits don't see an increase because they're already infinite/bigEnough. - // The values that should scale based on the amount of memory allocated to libp2p need to increase accordingly. - PeerLimitIncrease: rcmgr.BaseLimitIncrease{ - Memory: 0, - FDFraction: 0, - Conns: 0, - ConnsInbound: rcmgr.DefaultLimits.PeerLimitIncrease.ConnsInbound, - ConnsOutbound: 0, - Streams: 0, - StreamsInbound: rcmgr.DefaultLimits.PeerLimitIncrease.StreamsInbound, - StreamsOutbound: 0, + PeerDefault: &rcmgr.ResourceLimits{ + Memory: rcmgr.Unlimited64, + FD: rcmgr.Unlimited, + Conns: rcmgr.Unlimited, + ConnsInbound: rcmgr.DefaultLimit, + ConnsOutbound: rcmgr.Unlimited, + Streams: rcmgr.Unlimited, + StreamsInbound: rcmgr.DefaultLimit, + StreamsOutbound: rcmgr.Unlimited, }, } - // Whatever limits libp2p has specifically tuned for its protocols/services we'll apply. - libp2p.SetDefaultServiceLimits(&scalingLimitConfig) - - defaultLimitConfig := scalingLimitConfig.Scale(int64(maxMemory), maxFD) - // Simple checks to overide autoscaling ensuring limits make sense versus the connmgr values. // There are ways to break this, but this should catch most problems already. // We might improve this in the future. // See: https://github.com/ipfs/kubo/issues/9545 if cfg.ConnMgr.Type.WithDefault(config.DefaultConnMgrType) != "none" { - maxInboundConns := int64(defaultLimitConfig.System.ConnsInbound) + maxInboundConns := int64(partialLimits.System.ConnsInbound) if connmgrHighWaterTimesTwo := cfg.ConnMgr.HighWater.WithDefault(config.DefaultConnMgrHighWater) * 2; maxInboundConns < connmgrHighWaterTimesTwo { maxInboundConns = connmgrHighWaterTimesTwo } @@ -190,9 +142,24 @@ Run 'ipfs swarm limit all' to see the resulting limits. } // Scale System.StreamsInbound as well, but use the existing ratio of StreamsInbound to ConnsInbound - defaultLimitConfig.System.StreamsInbound = int(maxInboundConns * int64(defaultLimitConfig.System.StreamsInbound) / int64(defaultLimitConfig.System.ConnsInbound)) - defaultLimitConfig.System.ConnsInbound = int(maxInboundConns) + partialLimits.System.StreamsInbound = rcmgr.LimitVal(maxInboundConns * int64(partialLimits.System.StreamsInbound) / int64(partialLimits.System.ConnsInbound)) + partialLimits.System.ConnsInbound = rcmgr.LimitVal(maxInboundConns) + } + + limitConfig := partialLimits + + // The logic for defaults and overriding with specified SwarmConfig.ResourceMgr.Limits + // is documented in docs/config.md. + // Any changes here should be reflected there. + if cfg.ResourceMgr.Limits != nil && applyConfigLimits { + userSuppliedOverrideLimitConfig := *cfg.ResourceMgr.Limits + // This effectively overrides the computed default LimitConfig with any non-zero values from cfg.ResourceMgr.Limits. + // Because of how how Apply works, any 0 value for a user supplied override + // will be overriden with a computed default value. + // There currently isn't a way for a user to supply a 0-value override. + userSuppliedOverrideLimitConfig.Apply(partialLimits) + limitConfig = userSuppliedOverrideLimitConfig } - return defaultLimitConfig, nil + return limitConfig.Build(rcmgr.DefaultLimits.Scale(int64(maxMemory), maxFD)), nil } diff --git a/core/node/libp2p/rcmgr_logging_test.go b/core/node/libp2p/rcmgr_logging_test.go index 512168d4a52f..477e1c44214a 100644 --- a/core/node/libp2p/rcmgr_logging_test.go +++ b/core/node/libp2p/rcmgr_logging_test.go @@ -16,11 +16,17 @@ import ( func TestLoggingResourceManager(t *testing.T) { clock := clock.NewMock() - limits := rcmgr.DefaultLimits.AutoScale() - limits.System.Conns = 1 - limits.System.ConnsInbound = 1 - limits.System.ConnsOutbound = 1 - limiter := rcmgr.NewFixedLimiter(limits) + partialLimits := rcmgr.PartialLimitConfig{ + System: &rcmgr.ResourceLimits{ + Conns: 1, + ConnsInbound: 1, + ConnsOutbound: 1, + }, + } + + concreteLimits := partialLimits.Build(rcmgr.DefaultLimits.AutoScale()) + + limiter := rcmgr.NewFixedLimiter(concreteLimits) rm, err := rcmgr.NewResourceManager(limiter) if err != nil { t.Fatal(err) diff --git a/go.mod b/go.mod index 3d91f40e77f0..3ca431fed92e 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,7 @@ require ( github.com/jbenet/go-temp-err-catcher v0.1.0 github.com/jbenet/goprocess v0.1.4 github.com/libp2p/go-doh-resolver v0.4.0 - github.com/libp2p/go-libp2p v0.24.2 + github.com/libp2p/go-libp2p v0.24.3-0.20230120204155-c68e24ccd286 github.com/libp2p/go-libp2p-http v0.4.0 github.com/libp2p/go-libp2p-kad-dht v0.20.0 github.com/libp2p/go-libp2p-kbucket v0.5.0 @@ -154,6 +154,7 @@ require ( github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect github.com/huin/goupnp v1.0.3 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.0.0 // indirect @@ -175,10 +176,9 @@ require ( github.com/libp2p/go-libp2p-gostream v0.5.0 // indirect github.com/libp2p/go-libp2p-xor v0.1.0 // indirect github.com/libp2p/go-mplex v0.7.0 // indirect - github.com/libp2p/go-msgio v0.2.0 // indirect + github.com/libp2p/go-msgio v0.3.0 // indirect github.com/libp2p/go-nat v0.1.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect - github.com/libp2p/go-openssl v0.1.0 // indirect github.com/libp2p/go-reuseport v0.2.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.0 // indirect github.com/libp2p/zeroconf/v2 v2.2.0 // indirect @@ -190,7 +190,6 @@ require ( github.com/marten-seemann/webtransport-go v0.4.3 // indirect github.com/mattn/go-colorable v0.1.4 // indirect github.com/mattn/go-isatty v0.0.17 // indirect - github.com/mattn/go-pointer v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.4 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect @@ -215,7 +214,6 @@ require ( github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rs/cors v1.7.0 // indirect github.com/samber/lo v1.36.0 // indirect - github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect diff --git a/go.sum b/go.sum index 471acc8080b7..4b4f04728956 100644 --- a/go.sum +++ b/go.sum @@ -416,6 +416,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= +github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -734,8 +736,8 @@ github.com/libp2p/go-libp2p v0.7.0/go.mod h1:hZJf8txWeCduQRDC/WSqBGMxaTHCOYHt2xS github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniVO7zIHGMw= github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0= -github.com/libp2p/go-libp2p v0.24.2 h1:iMViPIcLY0D6zr/f+1Yq9EavCZu2i7eDstsr1nEwSAk= -github.com/libp2p/go-libp2p v0.24.2/go.mod h1:WuxtL2V8yGjam03D93ZBC19tvOUiPpewYv1xdFGWu1k= +github.com/libp2p/go-libp2p v0.24.3-0.20230120204155-c68e24ccd286 h1:wgKtjGrXdLm1OqPAwgZzt6JpfCs383lF8Qa5brUUezo= +github.com/libp2p/go-libp2p v0.24.3-0.20230120204155-c68e24ccd286/go.mod h1:ZqQXmVB7kTkKtm+LoPODTqgKE/j/nMPpRWDgw5Lcvog= github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= @@ -870,8 +872,8 @@ github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+ github.com/libp2p/go-msgio v0.0.3/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.6/go.mod h1:4ecVB6d9f4BDSL5fqvPiC4A3KivjWn+Venn/1ALLMWA= -github.com/libp2p/go-msgio v0.2.0 h1:W6shmB+FeynDrUVl2dgFQvzfBZcXiyqY4VmpQLu9FqU= -github.com/libp2p/go-msgio v0.2.0/go.mod h1:dBVM1gW3Jk9XqHkU4eKdGvVHdLa51hoGfll6jMJMSlY= +github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= +github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36GchpcVc1yI= github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/+KSDo= github.com/libp2p/go-nat v0.0.5/go.mod h1:B7NxsVNPZmRLvMOwiEO1scOSyjA56zxYAGv1yQgRkEU= @@ -888,8 +890,6 @@ github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.5/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= -github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo= -github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc= github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= github.com/libp2p/go-reuseport v0.2.0 h1:18PRvIMlpY6ZK85nIAicSBuXXvrYoSw3dsBAR7zc560= @@ -964,8 +964,6 @@ github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= -github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -1265,7 +1263,6 @@ github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJ github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= -github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=