respect the rate limit from cid gravity

Signed-off-by: Merlin Ran <[email protected]>

main.go

@@ -171,6 +171,12 @@ Zero means no limits`,
 			Description: `The timeout to fetch deal data. Be conservative to leave enough room for network instability.`,
 		},
 
+		{
+			Name:        "cid-gravity-api-url",
+			DefValue:    "https://api.cidgravity.com/api/integrations/bidbot",
+			Description: "The CID gravity system API endpoint to use. DO NOT CHANGE. Only for test purposes.",
+		},
+
 		{
 			Name:        "cid-gravity-key",
 			DefValue:    "",
@@ -409,7 +415,7 @@ var daemonCmd = &cobra.Command{
 			PricingRulesStrict:  v.GetBool("cid-gravity-strict"),
 		}
 		if cidGravityKey := v.GetString("cid-gravity-key"); cidGravityKey != "" {
-			config.PricingRules = pricing.NewCIDGravityRules(cidGravityKey)
+			config.PricingRules = pricing.NewCIDGravityRules(v.GetString("cid-gravity-api-url"), cidGravityKey)
 		}
 
 		serv, err := service.New(config, store, lc, fc)

service/pricing/cid_gravity.go

@@ -20,15 +20,16 @@ const (
 )
 
 var (
-	log              = golog.Logger("bidbot/pricing")
-	cidGravityAPIUrl = "https://api.cidgravity.com/api/integrations/bidbot"
+	log = golog.Logger("bidbot/pricing")
 	// Use cached rules if they are loaded no earlier than this period.
 	cidGravityCachePeriod = time.Minute
 )
 
 type rawRules struct {
 	Blocked         bool
 	MaintenanceMode bool
+	DealRateLimit   int
+	CurrentDealRate int
 	PricingRules    []struct {
 		Verified    bool
 		MinSize     uint64 // bytes
@@ -40,29 +41,31 @@ type rawRules struct {
 }
 
 type cidGravityRules struct {
+	apiURL         string
 	apiKey         string
 	perClientRules map[string]*clientRules
 	lkRules        sync.Mutex
 }
 
 // NewCIDGravityRules returns PricingRules based on CID gravity configuration for the storage provider.
-func NewCIDGravityRules(apiKey string) PricingRules {
-	return &cidGravityRules{apiKey: apiKey, perClientRules: make(map[string]*clientRules)}
+func NewCIDGravityRules(apiURL, apiKey string) PricingRules {
+	return &cidGravityRules{apiURL: apiURL, apiKey: apiKey, perClientRules: make(map[string]*clientRules)}
 }
 
 // PricesFor looks up prices for the auction based on its client address.
 func (cg *cidGravityRules) PricesFor(auction *pb.Auction) (prices ResolvedPrices, valid bool) {
 	cg.lkRules.Lock()
 	rules, exists := cg.perClientRules[auction.ClientAddress]
 	if !exists {
-		rules = newClientRulesFor(cg.apiKey, auction.ClientAddress)
+		rules = newClientRulesFor(cg.apiURL, cg.apiKey, auction.ClientAddress)
 	}
 	cg.perClientRules[auction.ClientAddress] = rules
 	cg.lkRules.Unlock()
 	return rules.PricesFor(auction)
 }
 
 type clientRules struct {
+	apiURL           string
 	apiKey           string
 	clientAddress    string
 	rules            atomic.Value // *CIDGravityRules
@@ -71,8 +74,9 @@ type clientRules struct {
 	lkLoadRules      sync.Mutex
 }
 
-func newClientRulesFor(apiKey, clientAddress string) *clientRules {
+func newClientRulesFor(apiURL, apiKey, clientAddress string) *clientRules {
 	rules := &clientRules{
+		apiURL:        apiURL,
 		apiKey:        apiKey,
 		clientAddress: clientAddress,
 	}
@@ -84,24 +88,31 @@ func newClientRulesFor(apiKey, clientAddress string) *clientRules {
 // auction. The rules are cached locally for some time. It returns valid = false if the cached rules were expired but
 // couldn't be reloaded from the CID gravity API in time.
 func (cg *clientRules) PricesFor(auction *pb.Auction) (prices ResolvedPrices, valid bool) {
-	valid = cg.maybeReloadRules(cidGravityAPIUrl, cidGravityLoadRulesTimeout, cidGravityCachePeriod)
+	valid = cg.maybeReloadRules(cg.apiURL, cidGravityLoadRulesTimeout, cidGravityCachePeriod)
 	if !valid {
 		return
 	}
-	rules := cg.rules.Load()
+	var rules *rawRules
+
 	// preventive but should not happen
-	if rules == nil {
+	r := cg.rules.Load()
+	if r == nil {
 		valid = false
 		return
 	}
-	if rules.(*rawRules).Blocked {
+	rules = r.(*rawRules)
+
+	if rules.CurrentDealRate >= rules.DealRateLimit {
+		return
+	}
+	if rules.Blocked {
 		return
 	}
-	if rules.(*rawRules).MaintenanceMode {
+	if rules.MaintenanceMode {
 		return
 	}
 	// rules are checked in sequence and the first match wins.
-	for _, r := range rules.(*rawRules).PricingRules {
+	for _, r := range rules.PricingRules {
 		if auction.DealSize >= r.MinSize && auction.DealSize <= r.MaxSize &&
 			auction.DealDuration >= r.MinDuration && auction.DealDuration <= r.MaxDuration {
 			if r.Verified && !prices.VerifiedPriceValid {

service/pricing/cid_gravity_test.go

@@ -14,7 +14,6 @@ import (
 )
 
 func TestPriceFor(t *testing.T) {
-	cidGravityAPIUrl = "http://localhost:invalid" // do not care about rules loading
 	cidGravityCachePeriod = time.Second
 	rules := &rawRules{
 		PricingRules: []struct {
@@ -62,7 +61,7 @@ func TestPriceFor(t *testing.T) {
 		DealSize:      1,
 		DealDuration:  1,
 	}
-	cg := newClientRulesFor("key", auction.ClientAddress)
+	cg := newClientRulesFor("http://localhost:invalid", "key", auction.ClientAddress)
 
 	rp, valid := cg.PricesFor(auction)
 	assert.False(t, valid, "prices should be invalid before the rules are loaded")
@@ -93,6 +92,13 @@ func TestPriceFor(t *testing.T) {
 	assert.True(t, rp.VerifiedPriceValid)
 	assert.Equal(t, int64(100), rp.VerifiedPrice)
 
+	rules.CurrentDealRate = rules.DealRateLimit
+	rp, valid = cg.PricesFor(auction)
+	assert.True(t, valid)
+	assert.False(t, rp.UnverifiedPriceValid)
+	assert.False(t, rp.VerifiedPriceValid)
+
+	rules.CurrentDealRate = 0
 	rules.MaintenanceMode = true
 	rp, valid = cg.PricesFor(auction)
 	assert.True(t, valid)
@@ -112,7 +118,7 @@ func TestPriceFor(t *testing.T) {
 }
 
 func TestMaybeReloadRules(t *testing.T) {
-	cg := newClientRulesFor("key", "pk")
+	cg := newClientRulesFor("http://localhost:invalid", "key", "pk")
 	apiResponse := []byte(`{
 	     "pricingRules": [
 		     {
@@ -125,7 +131,9 @@ func TestMaybeReloadRules(t *testing.T) {
 		     }
 		 ],
 		"blocked": false,
-		"maintenanceMode": false
+		"maintenanceMode": false,
+		"dealRateLimit": 50,
+		"currentDealRate": 0
     }`)
 	for _, testCase := range []struct {
 		name         string