Improve logging for dynamic config

cadence-workflow · Nov 17, 2020 · f9bb732 · f9bb732
1 parent 11fde99
commit f9bb732
Show file tree

Hide file tree

Showing 6 changed files with 31 additions and 21 deletions.
diff --git a/common/persistence/visibilitySamplingClient.go b/common/persistence/visibilitySamplingClient.go
@@ -40,7 +40,7 @@ const (
 	numOfPriorityForList   = 1
 )
 
-// ErrPersistenceLimitExceededForList is the error indicating QPS limit reached for list visibility. 
+// ErrPersistenceLimitExceededForList is the error indicating QPS limit reached for list visibility.
 var ErrPersistenceLimitExceededForList = &workflow.ServiceBusyError{Message: "Persistence Max QPS Reached for List Operations."}
 
 type visibilitySamplingClient struct {

diff --git a/common/service/dynamicconfig/clientInterface.go b/common/service/dynamicconfig/clientInterface.go
@@ -24,6 +24,8 @@ package dynamicconfig
 
 import (
 	"time"
+
+	"github.com/uber/cadence/common/types"
 )
 
 // Client allows fetching values from a dynamic configuration system NOTE: This does not have async
@@ -45,3 +47,7 @@ type Client interface {
 	// UpdateValue takes value as map and updates by overriding. It doesn't support update with filters.
 	UpdateValue(name Key, value interface{}) error
 }
+
+var notFoundError = &types.EntityNotExistsError{
+	Message: "unable to find key",
+}
diff --git a/common/service/dynamicconfig/config.go b/common/service/dynamicconfig/config.go
@@ -28,6 +28,7 @@ import (
 
 	"github.com/uber/cadence/common/log"
 	"github.com/uber/cadence/common/log/tag"
+	"github.com/uber/cadence/common/types"
 )
 
 const (
@@ -65,7 +66,11 @@ func (c *Collection) logError(key Key, err error) {
 	errCount := atomic.AddInt64(&c.errCount, 1)
 	if errCount%errCountLogThreshold == 0 {
 		// log only every 'x' errors to reduce mem allocs and to avoid log noise
-		c.logger.Warn("Failed to fetch key from dynamic config", tag.Key(key.String()), tag.Error(err))
+		if _, ok := err.(*types.EntityNotExistsError); ok {
+			c.logger.Info("dynamic config not set, use default value", tag.Key(key.String()))
+		} else {
+			c.logger.Warn("Failed to fetch key from dynamic config", tag.Key(key.String()), tag.Error(err))
+		}
 	}
 }
 
@@ -84,7 +89,7 @@ func (c *Collection) logValue(
 			c.logger.Info("Dynamic config has changed",
 				tag.Key(key.String()), tag.Value(value), tag.DefaultValue(loadedValue))
 			// update the logKeys so that we can capture the changes again
-			// (ignore the racing condition here because it's just for logging, we need a lock if really need to solve it) 
+			// (ignore the racing condition here because it's just for logging, we need a lock if really need to solve it)
 			c.logKeys.Store(key, value)
 		}
 	}

diff --git a/common/service/dynamicconfig/file_based_client.go b/common/service/dynamicconfig/file_based_client.go
@@ -275,7 +275,7 @@ func (fc *fileBasedClient) getValueWithFilters(key Key, filters map[Filter]inter
 		}
 	}
 	if !found {
-		return defaultValue, errors.New("unable to find key")
+		return defaultValue, notFoundError
 	}
 	return defaultValue, nil
 }

diff --git a/common/service/dynamicconfig/inMemoryClient.go b/common/service/dynamicconfig/inMemoryClient.go
@@ -21,7 +21,6 @@
 package dynamicconfig
 
 import (
-	"errors"
 	"sync"
 	"time"
 )
@@ -53,7 +52,7 @@ func (mc *inMemoryClient) GetValue(key Key, defaultValue interface{}) (interface
 	if val, ok := mc.globalValues[key]; ok {
 		return val, nil
 	}
-	return defaultValue, errors.New("unable to find key")
+	return defaultValue, notFoundError
 }
 
 func (mc *inMemoryClient) GetValueWithFilters(
@@ -72,7 +71,7 @@ func (mc *inMemoryClient) GetIntValue(name Key, filters map[Filter]interface{},
 	if val, ok := mc.globalValues[name]; ok {
 		return val.(int), nil
 	}
-	return defaultValue, errors.New("unable to find key")
+	return defaultValue, notFoundError
 }
 
 func (mc *inMemoryClient) GetFloatValue(name Key, filters map[Filter]interface{}, defaultValue float64) (float64, error) {
@@ -82,7 +81,7 @@ func (mc *inMemoryClient) GetFloatValue(name Key, filters map[Filter]interface{}
 	if val, ok := mc.globalValues[name]; ok {
 		return val.(float64), nil
 	}
-	return defaultValue, errors.New("unable to find key")
+	return defaultValue, notFoundError
 }
 
 func (mc *inMemoryClient) GetBoolValue(name Key, filters map[Filter]interface{}, defaultValue bool) (bool, error) {
@@ -92,7 +91,7 @@ func (mc *inMemoryClient) GetBoolValue(name Key, filters map[Filter]interface{},
 	if val, ok := mc.globalValues[name]; ok {
 		return val.(bool), nil
 	}
-	return defaultValue, errors.New("unable to find key")
+	return defaultValue, notFoundError
 }
 
 func (mc *inMemoryClient) GetStringValue(name Key, filters map[Filter]interface{}, defaultValue string) (string, error) {
@@ -102,7 +101,7 @@ func (mc *inMemoryClient) GetStringValue(name Key, filters map[Filter]interface{
 	if val, ok := mc.globalValues[name]; ok {
 		return val.(string), nil
 	}
-	return defaultValue, errors.New("unable to find key")
+	return defaultValue, notFoundError
 }
 
 func (mc *inMemoryClient) GetMapValue(
@@ -114,7 +113,7 @@ func (mc *inMemoryClient) GetMapValue(
 	if val, ok := mc.globalValues[name]; ok {
 		return val.(map[string]interface{}), nil
 	}
-	return defaultValue, errors.New("unable to find key")
+	return defaultValue, notFoundError
 }
 
 func (mc *inMemoryClient) GetDurationValue(
@@ -126,7 +125,7 @@ func (mc *inMemoryClient) GetDurationValue(
 	if val, ok := mc.globalValues[name]; ok {
 		return val.(time.Duration), nil
 	}
-	return defaultValue, errors.New("unable to find key")
+	return defaultValue, notFoundError
 }
 
 func (mc *inMemoryClient) UpdateValue(key Key, value interface{}) error {

diff --git a/common/service/dynamicconfig/nopClient.go b/common/service/dynamicconfig/nopClient.go
@@ -31,44 +31,44 @@ import (
 type nopClient struct{}
 
 func (mc *nopClient) GetValue(name Key, defaultValue interface{}) (interface{}, error) {
-	return nil, errors.New("unable to find key")
+	return nil, notFoundError
 }
 
 func (mc *nopClient) GetValueWithFilters(
 	name Key, filters map[Filter]interface{}, defaultValue interface{},
 ) (interface{}, error) {
-	return nil, errors.New("unable to find key")
+	return nil, notFoundError
 }
 
 func (mc *nopClient) GetIntValue(name Key, filters map[Filter]interface{}, defaultValue int) (int, error) {
-	return defaultValue, errors.New("unable to find key")
+	return defaultValue, notFoundError
 }
 
 func (mc *nopClient) GetFloatValue(name Key, filters map[Filter]interface{}, defaultValue float64) (float64, error) {
-	return defaultValue, errors.New("unable to find key")
+	return defaultValue, notFoundError
 }
 
 func (mc *nopClient) GetBoolValue(name Key, filters map[Filter]interface{}, defaultValue bool) (bool, error) {
 	if filters[DomainName] == "TestRawHistoryDomain" {
-		return true, errors.New("unable to find key")
+		return true, notFoundError
 	}
-	return defaultValue, errors.New("unable to find key")
+	return defaultValue, notFoundError
 }
 
 func (mc *nopClient) GetStringValue(name Key, filters map[Filter]interface{}, defaultValue string) (string, error) {
-	return defaultValue, errors.New("unable to find key")
+	return defaultValue, notFoundError
 }
 
 func (mc *nopClient) GetMapValue(
 	name Key, filters map[Filter]interface{}, defaultValue map[string]interface{},
 ) (map[string]interface{}, error) {
-	return defaultValue, errors.New("unable to find key")
+	return defaultValue, notFoundError
 }
 
 func (mc *nopClient) GetDurationValue(
 	name Key, filters map[Filter]interface{}, defaultValue time.Duration,
 ) (time.Duration, error) {
-	return defaultValue, errors.New("unable to find key")
+	return defaultValue, notFoundError
 }
 
 func (mc *nopClient) UpdateValue(name Key, value interface{}) error {