update docs and config

launchdarkly · Feb 13, 2024 · a01fb46 · a01fb46
1 parent e9063a9
commit a01fb46
Show file tree

Hide file tree

Showing 8 changed files with 106 additions and 46 deletions.
diff --git a/config/config.go b/config/config.go
@@ -158,11 +158,12 @@ type AutoConfigConfig struct {
 
 // OfflineModeConfig contains configuration parameters for the offline/file data source feature.
 type OfflineModeConfig struct {
-	FileDataSource        string           `conf:"FILE_DATA_SOURCE"`
-	EnvDatastorePrefix    string           `conf:"ENV_DATASTORE_PREFIX"`
-	EnvDatastoreTableName string           `conf:"ENV_DATASTORE_TABLE_NAME"`
-	EnvAllowedOrigin      ct.OptStringList `conf:"ENV_ALLOWED_ORIGIN"`
-	EnvAllowedHeader      ct.OptStringList `conf:"ENV_ALLOWED_HEADER"`
+	FileDataSource                   string           `conf:"FILE_DATA_SOURCE"`
+	FileDataSourceMonitoringInterval ct.OptDuration   `conf:"FILE_DATA_SOURCE_MONITORING_INTERVAL"`
+	EnvDatastorePrefix               string           `conf:"ENV_DATASTORE_PREFIX"`
+	EnvDatastoreTableName            string           `conf:"ENV_DATASTORE_TABLE_NAME"`
+	EnvAllowedOrigin                 ct.OptStringList `conf:"ENV_ALLOWED_ORIGIN"`
+	EnvAllowedHeader                 ct.OptStringList `conf:"ENV_ALLOWED_HEADER"`
 }
 
 // EventsConfig contains configuration parameters for proxying events.

diff --git a/config/config_validation.go b/config/config_validation.go
@@ -3,10 +3,10 @@ package config
 import (
 	"errors"
 	"fmt"
-	"strings"
-
 	ct "github.com/launchdarkly/go-configtypes"
 	"github.com/launchdarkly/go-sdk-common/v3/ldlog"
+	"strings"
+	"time"
 )
 
 var (
@@ -18,11 +18,12 @@ var (
 	errOfflineModeWithEnvironments     = errors.New("cannot configure specific environments if offline mode is enabled")
 	errAutoConfWithoutDBDisambig       = errors.New(`when using auto-configuration with database storage, database prefix (or,` +
 		` if using DynamoDB, table name) must be specified and must contain "` + AutoConfigEnvironmentIDPlaceholder + `"`)
-	errRedisURLWithHostAndPort = errors.New("please specify Redis URL or host/port, but not both")
-	errRedisBadHostname        = errors.New("invalid Redis hostname")
-	errConsulTokenAndTokenFile = errors.New("Consul token must be specified as either an inline value or a file, but not both") //nolint:stylecheck
-	errAutoConfWithFilters     = errors.New("cannot configure filters if auto-configuration is enabled")
-	errMissingProjKey          = errors.New("when filters are configured, all environments must specify a 'projKey'")
+	errRedisURLWithHostAndPort           = errors.New("please specify Redis URL or host/port, but not both")
+	errRedisBadHostname                  = errors.New("invalid Redis hostname")
+	errConsulTokenAndTokenFile           = errors.New("Consul token must be specified as either an inline value or a file, but not both") //nolint:stylecheck
+	errAutoConfWithFilters               = errors.New("cannot configure filters if auto-configuration is enabled")
+	errMissingProjKey                    = errors.New("when filters are configured, all environments must specify a 'projKey'")
+	errInvalidFileDataSourcePollInterval = errors.New("file data source poll interval must be at least 100ms")
 )
 
 func errEnvironmentWithNoSDKKey(envName string) error {
@@ -76,6 +77,7 @@ func ValidateConfig(c *Config, loggers ldlog.Loggers) error {
 	validateConfigEnvironments(&result, c)
 	validateConfigDatabases(&result, c, loggers)
 	validateConfigFilters(&result, c)
+	validateOfflineMode(&result, c)
 
 	return result.GetError()
 }
@@ -122,7 +124,7 @@ func validateConfigEnvironments(result *ct.ValidationResult, c *Config) {
 	}
 	if c.OfflineMode.FileDataSource == "" {
 		if c.OfflineMode.EnvDatastorePrefix != "" || c.OfflineMode.EnvDatastoreTableName != "" ||
-			len(c.OfflineMode.EnvAllowedOrigin.Values()) != 0 || len(c.OfflineMode.EnvAllowedHeader.Values()) != 0 {
+			len(c.OfflineMode.EnvAllowedOrigin.Values()) != 0 || len(c.OfflineMode.EnvAllowedHeader.Values()) != 0 || c.OfflineMode.FileDataSourceMonitoringInterval.IsDefined() {
 			result.AddError(nil, errOfflineModePropertiesWithNoFile)
 		}
 	} else {
@@ -184,6 +186,16 @@ func validateConfigFilters(result *ct.ValidationResult, c *Config) {
 		}
 	}
 }
+
+func validateOfflineMode(result *ct.ValidationResult, c *Config) {
+	if c.OfflineMode.FileDataSourceMonitoringInterval.IsDefined() {
+		interval := c.OfflineMode.FileDataSourceMonitoringInterval.GetOrElse(0)
+		if interval < 100*time.Millisecond {
+			result.AddError(nil, errInvalidFileDataSourcePollInterval)
+		}
+	}
+}
+
 func validateConfigDatabases(result *ct.ValidationResult, c *Config, loggers ldlog.Loggers) {
 	normalizeRedisConfig(result, c)
 

diff --git a/config/test_data_configs_invalid_test.go b/config/test_data_configs_invalid_test.go
@@ -26,6 +26,10 @@ func makeInvalidConfigs() []testDataInvalidConfig {
 		makeInvalidConfigOfflineModeAllowedHeaderWithNoFile(),
 		makeInvalidConfigOfflineModePrefixWithNoFile(),
 		makeInvalidConfigOfflineModeTableNameWithNoFile(),
+		makeInvalidConfigOfflineModeWithMonitoringInterval("0s"),
+		makeInvalidConfigOfflineModeWithMonitoringInterval("not a duration"),
+		makeInvalidConfigOfflineModeWithMonitoringInterval("-1s"),
+		makeInvalidConfigOfflineModeWithMonitoringInterval("99ms"),
 		makeInvalidConfigRedisInvalidHostname(),
 		makeInvalidConfigRedisInvalidDockerPort(),
 		makeInvalidConfigRedisConflictingParams(),
@@ -240,6 +244,18 @@ EnvDatastoreTableName = table
 	return c
 }
 
+func makeInvalidConfigOfflineModeWithMonitoringInterval(interval string) testDataInvalidConfig {
+	c := testDataInvalidConfig{name: "offline mode table name with no file"}
+	c.fileError = errOfflineModePropertiesWithNoFile.Error()
+	c.fileContent = `
+[OfflineMode]
+fileDataSource = foo.tar.gz
+fileDataSourceMonitoringInterval = ` + interval + `
+
+`
+	return c
+}
+
 func makeInvalidConfigRedisInvalidHostname() testDataInvalidConfig {
 	c := testDataInvalidConfig{name: "Redis - invalid hostname"}
 	c.envVarsError = "invalid Redis hostname"

diff --git a/config/test_data_configs_valid_test.go b/config/test_data_configs_valid_test.go
@@ -65,7 +65,10 @@ func makeValidConfigs() []testDataValidConfig {
 		makeValidConfigExplicitOldDefaultBaseURI(),
 		makeValidConfigAutoConfig(),
 		makeValidConfigAutoConfigWithDatabase(),
-		makeValidConfigFileData(),
+		makeValidConfigOfflineModeMinimal(),
+		makeValidConfigOfflineModeWithMonitoringInterval("100ms"),
+		makeValidConfigOfflineModeWithMonitoringInterval("1s"),
+		makeValidConfigOfflineModeWithMonitoringInterval("5m"),
 		makeValidConfigRedisMinimal(),
 		makeValidConfigRedisAll(),
 		makeValidConfigRedisURL(),
@@ -335,7 +338,7 @@ Enabled = true
 	return c
 }
 
-func makeValidConfigFileData() testDataValidConfig {
+func makeValidConfigOfflineModeMinimal() testDataValidConfig {
 	c := testDataValidConfig{name: "file data properties"}
 	c.makeConfig = func(c *Config) {
 		c.OfflineMode.FileDataSource = "my-file-path"
@@ -350,6 +353,32 @@ FileDataSource = my-file-path
 	return c
 }
 
+func MustOptDurationFromString(duration string) ct.OptDuration {
+	opt, err := ct.NewOptDurationFromString(duration)
+	if err != nil {
+		panic(err)
+	}
+	return opt
+}
+
+func makeValidConfigOfflineModeWithMonitoringInterval(interval string) testDataValidConfig {
+	c := testDataValidConfig{name: "file data properties"}
+	c.makeConfig = func(c *Config) {
+		c.OfflineMode.FileDataSource = "my-file-path"
+		c.OfflineMode.FileDataSourceMonitoringInterval = MustOptDurationFromString(interval)
+	}
+	c.envVars = map[string]string{
+		"FILE_DATA_SOURCE":                     "my-file-path",
+		"FILE_DATA_SOURCE_MONITORING_INTERVAL": interval,
+	}
+	c.fileContent = `
+[OfflineMode]
+FileDataSource = my-file-path
+FileDataSourceMonitoringInterval = ` + interval + `
+`
+	return c
+}
+
 func makeValidConfigRedisMinimal() testDataValidConfig {
 	c := testDataValidConfig{name: "Redis - minimal parameters"}
 	c.makeConfig = func(c *Config) {

diff --git a/docs/configuration.md b/docs/configuration.md
@@ -93,13 +93,14 @@ _(6)_ When using a database store, if there are multiple environments, it is nec
 
 This section is only applicable if [offline mode](https://docs.launchdarkly.com/home/advanced/relay-proxy-enterprise/offline) is enabled for your account.
 
-| Property in file         | Environment var            |  Type  | Default | Description                                                                                                                                                                                                                         |
-|--------------------------|----------------------------|:------:|:--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `fileDataSource`         | `FILE_DATA_SOURCE`         | String |         | Path to the offline mode data file that you have downloaded from LaunchDarkly.                                                                                                                                                      |
-| `envDatastorePrefix`     | `ENV_DATASTORE_PREFIX`     | String |         | If using a Redis, Consul, or DynamoDB store, this string will be added to all database keys to distinguish them from any other environments that are using the database. _(6)_                                                      |
-| `envDatastoreTableName ` | `ENV_DATASTORE_TABLE_NAME` | String |         | If using a DynamoDB store, this specifies the table name. _(6)_                                                                                                                                                                     |
-| `envAllowedOrigin`       | `ENV_ALLOWED_ORIGIN`       |  URI   |         | If provided, adds CORS headers to prevent access from other domains. This variable can be provided multiple times per environment (if using the `ENV_ALLOWED_ORIGIN` variable, specify a comma-delimited list).                     |
-| `envAllowedHeader`       | `ENV_ALLOWED_HEADER`       | String |         | If provided, adds the specify headers to the list of accepted headers for CORS requests. This variable can be provided multiple times per environment (if using the `ENV_ALLOWED_HEADER` variable, specify a comma-delimited list). |
+| Property in file                   | Environment var                        |   Type   | Default | Description                                                                                                                                                                                                                         |
+|------------------------------------|----------------------------------------|:--------:|:--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `fileDataSource`                   | `FILE_DATA_SOURCE`                     |  String  |         | Path to the offline mode data file that you have downloaded from LaunchDarkly.                                                                                                                                                      |
+| `fileDataSourceMonitoringInterval` | `FILE_DATA_SOURCE_MONITORING_INTERVAL` | Duration | `1s`    | How often the file data source is checked for changes. Minimum is 100ms. To reduce computation and syscalls, raise the interval (for example, `5m` for every 5 minutes.)                                                            |
+| `envDatastorePrefix`               | `ENV_DATASTORE_PREFIX`                 |  String  |         | If using a Redis, Consul, or DynamoDB store, this string will be added to all database keys to distinguish them from any other environments that are using the database. _(6)_                                                      |
+| `envDatastoreTableName `           | `ENV_DATASTORE_TABLE_NAME`             |  String  |         | If using a DynamoDB store, this specifies the table name. _(6)_                                                                                                                                                                     |
+| `envAllowedOrigin`                 | `ENV_ALLOWED_ORIGIN`                   |   URI    |         | If provided, adds CORS headers to prevent access from other domains. This variable can be provided multiple times per environment (if using the `ENV_ALLOWED_ORIGIN` variable, specify a comma-delimited list).                     |
+| `envAllowedHeader`                 | `ENV_ALLOWED_HEADER`                   |  String  |         | If provided, adds the specify headers to the list of accepted headers for CORS requests. This variable can be provided multiple times per environment (if using the `ENV_ALLOWED_HEADER` variable, specify a comma-delimited list). |
 
 Note that the last three properties have the same meanings and the same environment variables names as the corresponding properties in the `[AutoConfig]` section described above. It is not possible to use `[OfflineMode]` and `[AutoConfig]` at the same time.
 

diff --git a/internal/filedata/archive_manager.go b/internal/filedata/archive_manager.go
@@ -11,8 +11,8 @@ import (
 )
 
 const (
-	defaultPollInterval = 1 * time.Second
-	maxRetryDuration    = time.Second * 2
+	defaultMonitoringInterval = 1 * time.Second
+	maxRetryDuration          = time.Second * 2
 )
 
 // ArchiveManager manages the file data source.
@@ -23,13 +23,13 @@ const (
 // Relay provides an implementation of the UpdateHandler interface which will be called for all changes that
 // it needs to know about.
 type ArchiveManager struct {
-	filePath      string
-	pollInterval  time.Duration
-	handler       UpdateHandler
-	lastKnownEnvs map[config.EnvironmentID]environmentMetadata
-	loggers       ldlog.Loggers
-	closeCh       chan struct{}
-	closeOnce     sync.Once
+	filePath           string
+	monitoringInterval time.Duration
+	handler            UpdateHandler
+	lastKnownEnvs      map[config.EnvironmentID]environmentMetadata
+	loggers            ldlog.Loggers
+	closeCh            chan struct{}
+	closeOnce          sync.Once
 }
 
 // ArchiveManagerInterface is an interface containing the public methods of ArchiveManager. This is used
@@ -45,7 +45,7 @@ type ArchiveManagerInterface interface {
 func NewArchiveManager(
 	filePath string,
 	handler UpdateHandler,
-	pollInterval time.Duration, // zero = use the default; we set a nonzero brief interval in unit tests
+	monitoringInterval time.Duration, // zero = use the default; we set a nonzero brief interval in unit tests
 	loggers ldlog.Loggers,
 ) (*ArchiveManager, error) {
 	fileInfo, err := os.Stat(filePath)
@@ -54,15 +54,15 @@ func NewArchiveManager(
 	}
 
 	am := &ArchiveManager{
-		filePath:      filePath,
-		handler:       handler,
-		pollInterval:  pollInterval,
-		lastKnownEnvs: make(map[config.EnvironmentID]environmentMetadata),
-		loggers:       loggers,
-		closeCh:       make(chan struct{}),
+		filePath:           filePath,
+		handler:            handler,
+		monitoringInterval: monitoringInterval,
+		lastKnownEnvs:      make(map[config.EnvironmentID]environmentMetadata),
+		loggers:            loggers,
+		closeCh:            make(chan struct{}),
 	}
-	if am.pollInterval == 0 {
-		am.pollInterval = defaultPollInterval
+	if am.monitoringInterval == 0 {
+		am.monitoringInterval = defaultMonitoringInterval
 	}
 	am.loggers.SetPrefix("[FileDataSource]")
 
@@ -87,12 +87,12 @@ func (am *ArchiveManager) Close() error {
 }
 
 func (am *ArchiveManager) monitorForChangesByPolling(original os.FileInfo) {
-	ticker := time.NewTicker(am.pollInterval)
+	ticker := time.NewTicker(am.monitoringInterval)
 	defer ticker.Stop()
 
 	prevInfo := original
 
-	am.loggers.Infof("Monitoring %s (size=%d, mtime=%s)", am.filePath, original.Size(), original.ModTime())
+	am.loggers.Infof("Monitoring %s for changes (every %s) (size=%d, mtime=%s)", am.filePath, am.monitoringInterval, original.Size(), original.ModTime())
 
 	for {
 		select {

diff --git a/internal/filedata/archive_manager_test.go b/internal/filedata/archive_manager_test.go
@@ -90,7 +90,7 @@ func TestDefaultRetryInterval(t *testing.T) {
 		require.NoError(t, err)
 		defer archiveManager.Close()
 
-		assert.Equal(t, defaultPollInterval, archiveManager.pollInterval)
+		assert.Equal(t, defaultMonitoringInterval, archiveManager.monitoringInterval)
 	})
 }
 

diff --git a/relay/relay.go b/relay/relay.go
@@ -81,7 +81,7 @@ type ClientFactoryFunc func(sdkKey config.SDKKey, config ld.Config) (*ld.LDClien
 type relayInternalOptions struct {
 	loggers               ldlog.Loggers
 	clientFactory         sdks.ClientFactoryFunc
-	archiveManagerFactory func(string, filedata.UpdateHandler, ldlog.Loggers) (filedata.ArchiveManagerInterface, error)
+	archiveManagerFactory func(path string, monitoringInterval time.Duration, environmentUpdates filedata.UpdateHandler, loggers ldlog.Loggers) (filedata.ArchiveManagerInterface, error)
 }
 
 // NewRelay creates a new Relay given a configuration and a method to create a client.
@@ -226,6 +226,7 @@ func newRelayInternal(c config.Config, options relayInternalOptions) (*Relay, er
 		}
 		archiveManager, err := factory(
 			c.OfflineMode.FileDataSource,
+			c.OfflineMode.FileDataSourceMonitoringInterval.GetOrElse(0),
 			&relayFileDataActions{r: r},
 			loggers,
 		)
@@ -290,9 +291,9 @@ func makeFilteredEnvironments(c *config.Config) map[string]*config.EnvConfig {
 	return out
 }
 
-func defaultArchiveManagerFactory(filePath string, handler filedata.UpdateHandler, loggers ldlog.Loggers) (
+func defaultArchiveManagerFactory(filePath string, monitoringInterval time.Duration, handler filedata.UpdateHandler, loggers ldlog.Loggers) (
 	filedata.ArchiveManagerInterface, error) {
-	am, err := filedata.NewArchiveManager(filePath, handler, 0, loggers)
+	am, err := filedata.NewArchiveManager(filePath, handler, monitoringInterval, loggers)
 	return am, err
 }