From 6990b70d951999f52c14450202d6c0be8b31537a Mon Sep 17 00:00:00 2001 From: Merlin Ran Date: Thu, 14 Oct 2021 17:46:26 -0400 Subject: [PATCH] respect the rate limit from cid gravity Signed-off-by: Merlin Ran --- main.go | 8 ++++++- service/pricing/cid_gravity.go | 35 +++++++++++++++++++---------- service/pricing/cid_gravity_test.go | 16 +++++++++---- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/main.go b/main.go index b6bae4f..e0233b8 100644 --- a/main.go +++ b/main.go @@ -171,6 +171,12 @@ Zero means no limits`, Description: `The timeout to fetch deal data. Be conservative to leave enough room for network instability.`, }, + { + Name: "cid-gravity-api-url", + DefValue: "https://api.cidgravity.com/api/integrations/bidbot", + Description: "The CID gravity system API endpoint to use. DO NOT CHANGE. Only for test purposes.", + }, + { Name: "cid-gravity-key", DefValue: "", @@ -409,7 +415,7 @@ var daemonCmd = &cobra.Command{ PricingRulesStrict: v.GetBool("cid-gravity-strict"), } if cidGravityKey := v.GetString("cid-gravity-key"); cidGravityKey != "" { - config.PricingRules = pricing.NewCIDGravityRules(cidGravityKey) + config.PricingRules = pricing.NewCIDGravityRules(v.GetString("cid-gravity-api-url"), cidGravityKey) } serv, err := service.New(config, store, lc, fc) diff --git a/service/pricing/cid_gravity.go b/service/pricing/cid_gravity.go index 11e7be8..acbdc4a 100644 --- a/service/pricing/cid_gravity.go +++ b/service/pricing/cid_gravity.go @@ -20,8 +20,7 @@ const ( ) var ( - log = golog.Logger("bidbot/pricing") - cidGravityAPIUrl = "https://api.cidgravity.com/api/integrations/bidbot" + log = golog.Logger("bidbot/pricing") // Use cached rules if they are loaded no earlier than this period. cidGravityCachePeriod = time.Minute ) @@ -29,6 +28,8 @@ var ( type rawRules struct { Blocked bool MaintenanceMode bool + DealRateLimit int + CurrentDealRate int PricingRules []struct { Verified bool MinSize uint64 // bytes @@ -40,14 +41,15 @@ type rawRules struct { } type cidGravityRules struct { + apiURL string apiKey string perClientRules map[string]*clientRules lkRules sync.Mutex } // NewCIDGravityRules returns PricingRules based on CID gravity configuration for the storage provider. -func NewCIDGravityRules(apiKey string) PricingRules { - return &cidGravityRules{apiKey: apiKey, perClientRules: make(map[string]*clientRules)} +func NewCIDGravityRules(apiURL, apiKey string) PricingRules { + return &cidGravityRules{apiURL: apiURL, apiKey: apiKey, perClientRules: make(map[string]*clientRules)} } // PricesFor looks up prices for the auction based on its client address. @@ -55,7 +57,7 @@ func (cg *cidGravityRules) PricesFor(auction *pb.Auction) (prices ResolvedPrices cg.lkRules.Lock() rules, exists := cg.perClientRules[auction.ClientAddress] if !exists { - rules = newClientRulesFor(cg.apiKey, auction.ClientAddress) + rules = newClientRulesFor(cg.apiURL, cg.apiKey, auction.ClientAddress) } cg.perClientRules[auction.ClientAddress] = rules cg.lkRules.Unlock() @@ -63,6 +65,7 @@ func (cg *cidGravityRules) PricesFor(auction *pb.Auction) (prices ResolvedPrices } type clientRules struct { + apiURL string apiKey string clientAddress string rules atomic.Value // *CIDGravityRules @@ -71,8 +74,9 @@ type clientRules struct { lkLoadRules sync.Mutex } -func newClientRulesFor(apiKey, clientAddress string) *clientRules { +func newClientRulesFor(apiURL, apiKey, clientAddress string) *clientRules { rules := &clientRules{ + apiURL: apiURL, apiKey: apiKey, clientAddress: clientAddress, } @@ -84,24 +88,31 @@ func newClientRulesFor(apiKey, clientAddress string) *clientRules { // auction. The rules are cached locally for some time. It returns valid = false if the cached rules were expired but // couldn't be reloaded from the CID gravity API in time. func (cg *clientRules) PricesFor(auction *pb.Auction) (prices ResolvedPrices, valid bool) { - valid = cg.maybeReloadRules(cidGravityAPIUrl, cidGravityLoadRulesTimeout, cidGravityCachePeriod) + valid = cg.maybeReloadRules(cg.apiURL, cidGravityLoadRulesTimeout, cidGravityCachePeriod) if !valid { return } - rules := cg.rules.Load() + var rules *rawRules + // preventive but should not happen - if rules == nil { + r := cg.rules.Load() + if r == nil { valid = false return } - if rules.(*rawRules).Blocked { + rules = r.(*rawRules) + + if rules.CurrentDealRate >= rules.DealRateLimit { + return + } + if rules.Blocked { return } - if rules.(*rawRules).MaintenanceMode { + if rules.MaintenanceMode { return } // rules are checked in sequence and the first match wins. - for _, r := range rules.(*rawRules).PricingRules { + for _, r := range rules.PricingRules { if auction.DealSize >= r.MinSize && auction.DealSize <= r.MaxSize && auction.DealDuration >= r.MinDuration && auction.DealDuration <= r.MaxDuration { if r.Verified && !prices.VerifiedPriceValid { diff --git a/service/pricing/cid_gravity_test.go b/service/pricing/cid_gravity_test.go index 794583e..e29fae3 100644 --- a/service/pricing/cid_gravity_test.go +++ b/service/pricing/cid_gravity_test.go @@ -14,7 +14,6 @@ import ( ) func TestPriceFor(t *testing.T) { - cidGravityAPIUrl = "http://localhost:invalid" // do not care about rules loading cidGravityCachePeriod = time.Second rules := &rawRules{ PricingRules: []struct { @@ -62,7 +61,7 @@ func TestPriceFor(t *testing.T) { DealSize: 1, DealDuration: 1, } - cg := newClientRulesFor("key", auction.ClientAddress) + cg := newClientRulesFor("http://localhost:invalid", "key", auction.ClientAddress) rp, valid := cg.PricesFor(auction) assert.False(t, valid, "prices should be invalid before the rules are loaded") @@ -93,6 +92,13 @@ func TestPriceFor(t *testing.T) { assert.True(t, rp.VerifiedPriceValid) assert.Equal(t, int64(100), rp.VerifiedPrice) + rules.CurrentDealRate = rules.DealRateLimit + rp, valid = cg.PricesFor(auction) + assert.True(t, valid) + assert.False(t, rp.UnverifiedPriceValid) + assert.False(t, rp.VerifiedPriceValid) + + rules.CurrentDealRate = 0 rules.MaintenanceMode = true rp, valid = cg.PricesFor(auction) assert.True(t, valid) @@ -112,7 +118,7 @@ func TestPriceFor(t *testing.T) { } func TestMaybeReloadRules(t *testing.T) { - cg := newClientRulesFor("key", "pk") + cg := newClientRulesFor("http://localhost:invalid", "key", "pk") apiResponse := []byte(`{ "pricingRules": [ { @@ -125,7 +131,9 @@ func TestMaybeReloadRules(t *testing.T) { } ], "blocked": false, - "maintenanceMode": false + "maintenanceMode": false, + "dealRateLimit": 50, + "currentDealRate": 0 }`) for _, testCase := range []struct { name string