feat: offline mode key rotation (#408)

This PR adds support for SDK key rotation (both with and without
deprecation periods) to offline mode.

What this means is that whenever an archive is loaded, if there is a
deprecated key, it will be accepted for new SDK connections until it
reaches its expiration date.

config/config.go

@@ -263,6 +263,7 @@ type EnvConfig struct {
 	TTL           ct.OptDuration   `conf:"LD_TTL_"`
 	ProjKey       string           `conf:"LD_PROJ_KEY_"`
 	FilterKey     FilterKey        // injected based on [filters] section
+	Offline       bool             // set to true if this environment was created in offline mode
 }
 
 type FiltersConfig struct {

integrationtests/autoconfig_test.go

@@ -77,7 +77,7 @@ func testPolicyUpdate(t *testing.T, manager *integrationTestManager) {
 		manager.awaitRelayStatus(t, func(status api.StatusRep) bool {
 			if len(status.Environments) == 1 {
 				if envStatus, ok := status.Environments[string(remainingEnv.id)]; ok {
-					verifyEnvProperties(t, testData.project, remainingEnv, envStatus, true)
+					verifyEnvProperties(t, testData.project, remainingEnv, envStatus, &envPropertyExpectations{nameAndKey: true})
 					return true
 				}
 			}
@@ -96,7 +96,7 @@ func testAddEnvironment(t *testing.T, manager *integrationTestManager) {
 		manager.awaitRelayStatus(t, func(status api.StatusRep) bool {
 			if len(status.Environments) == len(testData.environments)+1 {
 				if envStatus, ok := status.Environments[string(newEnv.id)]; ok {
-					verifyEnvProperties(t, testData.project, newEnv, envStatus, true)
+					verifyEnvProperties(t, testData.project, newEnv, envStatus, &envPropertyExpectations{nameAndKey: true})
 					return true
 				}
 			}
@@ -136,7 +136,7 @@ func testUpdatedSDKKeyWithoutExpiry(t *testing.T, manager *integrationTestManage
 
 		manager.awaitRelayStatus(t, func(status api.StatusRep) bool {
 			if envStatus, ok := status.Environments[string(envToUpdate.id)]; ok {
-				verifyEnvProperties(t, testData.project, updatedEnv, envStatus, true)
+				verifyEnvProperties(t, testData.project, updatedEnv, envStatus, &envPropertyExpectations{nameAndKey: true})
 				return last5(envStatus.SDKKey) == last5(string(newKey)) && envStatus.ExpiringSDKKey == ""
 			}
 			return false
@@ -166,7 +166,7 @@ func testUpdatedSDKKeyWithExpiry(t *testing.T, manager *integrationTestManager)
 				return false
 			}
 			if envStatus, ok := status.Environments[string(envToUpdate.id)]; ok {
-				verifyEnvProperties(t, testData.project, updatedEnv, envStatus, true)
+				verifyEnvProperties(t, testData.project, updatedEnv, envStatus, &envPropertyExpectations{nameAndKey: true})
 				return last5(envStatus.SDKKey) == last5(string(newKey)) &&
 					last5(envStatus.ExpiringSDKKey) == last5(string(oldKey))
 			}
@@ -204,13 +204,13 @@ func testUpdatedSDKKeyWithExpiryBeforeStartingRelay(t *testing.T, manager *integ
 	})
 	defer manager.stopRelay(t)
 
-	manager.awaitEnvironments(t, projAndEnvs, false, func(proj projectInfo, env environmentInfo) string {
+	manager.awaitEnvironments(t, projAndEnvs, nil, func(proj projectInfo, env environmentInfo) string {
 		return string(env.id)
 	})
 
 	manager.awaitRelayStatus(t, func(status api.StatusRep) bool {
 		if envStatus, ok := status.Environments[string(envToUpdate.id)]; ok {
-			verifyEnvProperties(t, testData.project, updatedEnv, envStatus, true)
+			verifyEnvProperties(t, testData.project, updatedEnv, envStatus, &envPropertyExpectations{nameAndKey: true})
 			return last5(envStatus.SDKKey) == last5(string(newKey)) &&
 				last5(envStatus.ExpiringSDKKey) == last5(string(oldKey))
 		}
@@ -238,7 +238,7 @@ func testUpdatedMobileKey(t *testing.T, manager *integrationTestManager) {
 
 		manager.awaitRelayStatus(t, func(status api.StatusRep) bool {
 			if envStatus, ok := status.Environments[string(envToUpdate.id)]; ok {
-				verifyEnvProperties(t, testData.project, updatedEnv, envStatus, true)
+				verifyEnvProperties(t, testData.project, updatedEnv, envStatus, &envPropertyExpectations{nameAndKey: true})
 				return last5(envStatus.MobileKey) == last5(string(newKey))
 			}
 			return false
@@ -286,7 +286,7 @@ func withRelayAndTestData(t *testing.T, manager *integrationTestManager, action
 
 func awaitInitialState(t *testing.T, manager *integrationTestManager, testData autoConfigTestData) {
 	projsAndEnvs := projsAndEnvs{testData.project: testData.environments}
-	manager.awaitEnvironments(t, projsAndEnvs, true, func(proj projectInfo, env environmentInfo) string {
+	manager.awaitEnvironments(t, projsAndEnvs, &envPropertyExpectations{nameAndKey: true}, func(proj projectInfo, env environmentInfo) string {
 		return string(env.id)
 	})
 }

integrationtests/offline_mode_test.go

@@ -7,10 +7,12 @@ package integrationtests
 import (
 	"fmt"
 	"io"
+	"maps"
 	"net/http"
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/launchdarkly/ld-relay/v8/config"
 
@@ -25,8 +27,35 @@ type offlineModeTestData struct {
 	autoConfigID  autoConfigID
 }
 
+// Used to configure the environment/project setup for an offline mode test.
+type apiParams struct {
+	// How many projects to create.
+	numProjects int
+	// Within each project, how many environments to create. Note: this must be >= 2 due to the way test flag
+	// variations are setup.
+	numEnvironments int
+}
+
 func testOfflineMode(t *testing.T, manager *integrationTestManager) {
-	withOfflineModeTestData(t, manager, func(testData offlineModeTestData) {
+	t.Run("expected environments and flag values", func(t *testing.T) {
+		testExpectedEnvironmentsAndFlagValues(t, manager)
+	})
+	t.Run("sdk key is rotated with deprecation after relay has started", func(t *testing.T) {
+		testSDKKeyRotatedAfterRelayStarted(t, manager)
+	})
+	t.Run("sdk key is rotated with deprecation before relay has started", func(t *testing.T) {
+		testSDKKeyRotatedBeforeRelayStarted(t, manager)
+	})
+	t.Run("sdk key is rotated and then expires", func(t *testing.T) {
+		testSDKKeyExpires(t, manager)
+	})
+	t.Run("sdk key is rotated multiple times without deprecation after relay started", func(t *testing.T) {
+		testKeyIsRotatedWithoutGracePeriod(t, manager)
+	})
+}
+
+func testExpectedEnvironmentsAndFlagValues(t *testing.T, manager *integrationTestManager) {
+	withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 2}, func(testData offlineModeTestData) {
 		helpers.WithTempDir(func(dirPath string) {
 			fileName := "archive.tar.gz"
 			filePath := filepath.Join(manager.relaySharedDir, fileName)
@@ -40,16 +69,194 @@ func testOfflineMode(t *testing.T, manager *integrationTestManager) {
 			})
 			defer manager.stopRelay(t)
 
-			manager.awaitEnvironments(t, testData.projsAndEnvs, true, func(proj projectInfo, env environmentInfo) string {
+			manager.awaitEnvironments(t, testData.projsAndEnvs, &envPropertyExpectations{nameAndKey: true}, func(proj projectInfo, env environmentInfo) string {
+				return string(env.id)
+			})
+			manager.verifyFlagValues(t, testData.projsAndEnvs)
+		})
+	})
+}
+
+func testSDKKeyRotatedAfterRelayStarted(t *testing.T, manager *integrationTestManager) {
+	// If we download an archive with a primary SDK key, and then it is subsequently updated
+	// with a deprecated key, we become initialized with both keys present.
+	withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 1}, func(testData offlineModeTestData) {
+		helpers.WithTempDir(func(dirPath string) {
+			fileName := "archive.tar.gz"
+			filePath := filepath.Join(manager.relaySharedDir, fileName)
+
+			err := downloadRelayArchive(manager, testData.autoConfigKey, filePath)
+			manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err)
+			require.NoError(t, err)
+
+			// Ensure the key won't expire during this test.
+			const keyGracePeriod = 1 * time.Hour
+			// Relay will check for expired keys at this interval.
+			const cleanupInterval = 100 * time.Millisecond
+			// We'll sleep longer than the interval after rotating keys, to try and reduce test flakiness.
+			const cleanupIntervalBuffer = 1 * time.Second
+
+			manager.startRelay(t, map[string]string{
+				"FILE_DATA_SOURCE":                    filepath.Join(relayContainerSharedDir, fileName),
+				"EXPIRED_CREDENTIAL_CLEANUP_INTERVAL": cleanupInterval.String(),
+			})
+			defer manager.stopRelay(t)
+
+			manager.awaitEnvironments(t, testData.projsAndEnvs, &envPropertyExpectations{nameAndKey: true}, func(proj projectInfo, env environmentInfo) string {
 				return string(env.id)
 			})
 			manager.verifyFlagValues(t, testData.projsAndEnvs)
+
+			// The updated map will is modified to contain expiringSdkKey field (with the old SDK key) and
+			// the new key set to whatever the API call returned.
+			updated := manager.rotateSDKKeys(t, testData.projsAndEnvs, time.Now().Add(keyGracePeriod))
+
+			err = downloadRelayArchive(manager, testData.autoConfigKey, filePath)
+			manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err)
+			require.NoError(t, err)
+
+			time.Sleep(cleanupIntervalBuffer)
+
+			// We are now asserting that the environment credentials returned by the status endpoint contains not just
+			// the new SDK key, but the expiring one as well.
+			manager.awaitEnvironments(t, updated, &envPropertyExpectations{nameAndKey: true, sdkKeys: true}, func(proj projectInfo, env environmentInfo) string {
+				return string(env.id)
+			})
+		})
+	})
+}
+
+func testSDKKeyRotatedBeforeRelayStarted(t *testing.T, manager *integrationTestManager) {
+	// Upon startup if an archive contains a primary and deprecated key, we become initialized with both keys.
+	withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 1}, func(testData offlineModeTestData) {
+		helpers.WithTempDir(func(dirPath string) {
+			fileName := "archive.tar.gz"
+			filePath := filepath.Join(manager.relaySharedDir, fileName)
+
+			// Relay will check for expired keys at this interval.
+			const cleanupInterval = 100 * time.Millisecond
+			// We'll sleep longer than the interval after rotating keys, to try and reduce test flakiness.
+			const cleanupIntervalBuffer = 1 * time.Second
+
+			// Rotation happens before starting up the relay.
+			updated := manager.rotateSDKKeys(t, testData.projsAndEnvs, time.Now().Add(1*time.Hour))
+
+			err := downloadRelayArchive(manager, testData.autoConfigKey, filePath)
+			manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err)
+			require.NoError(t, err)
+
+			manager.startRelay(t, map[string]string{
+				"FILE_DATA_SOURCE":                    filepath.Join(relayContainerSharedDir, fileName),
+				"EXPIRED_CREDENTIAL_CLEANUP_INTERVAL": cleanupInterval.String(),
+			})
+			defer manager.stopRelay(t)
+
+			time.Sleep(cleanupIntervalBuffer)
+
+			manager.awaitEnvironments(t, updated, &envPropertyExpectations{nameAndKey: true, sdkKeys: true}, func(proj projectInfo, env environmentInfo) string {
+				return string(env.id)
+			})
+			manager.verifyFlagValues(t, testData.projsAndEnvs)
+		})
+	})
+}
+
+func testSDKKeyExpires(t *testing.T, manager *integrationTestManager) {
+	// If a key is deprecated and then expires, it should be removed from the environment credentials.
+	withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 1}, func(testData offlineModeTestData) {
+		helpers.WithTempDir(func(dirPath string) {
+			fileName := "archive.tar.gz"
+			filePath := filepath.Join(manager.relaySharedDir, fileName)
+
+			const keyGracePeriod = 5 * time.Second
+			// Relay will check for expired keys at this interval.
+			const cleanupInterval = 100 * time.Millisecond
+
+			// Rotation happens before starting up the relay.
+			updated := manager.rotateSDKKeys(t, testData.projsAndEnvs, time.Now().Add(keyGracePeriod))
+			then := time.Now()
+
+			err := downloadRelayArchive(manager, testData.autoConfigKey, filePath)
+			manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err)
+			require.NoError(t, err)
+
+			manager.startRelay(t, map[string]string{
+				"FILE_DATA_SOURCE":                    filepath.Join(relayContainerSharedDir, fileName),
+				"EXPIRED_CREDENTIAL_CLEANUP_INTERVAL": cleanupInterval.String(),
+			})
+			defer manager.stopRelay(t)
+
+			manager.awaitEnvironments(t, updated, &envPropertyExpectations{nameAndKey: true, sdkKeys: true}, func(proj projectInfo, env environmentInfo) string {
+				return string(env.id)
+			})
+
+			// This test is timing-dependant on Relay removing the expired keys before we check the /status endpoint.
+			// To keep the test fast, only sleep as long as necessary to ensure the keys have expired.
+			toSleep := keyGracePeriod - time.Since(then)
+			if toSleep > 0 {
+				time.Sleep(toSleep)
+			}
+			manager.awaitEnvironments(t, updated.withoutExpiringKeys(), &envPropertyExpectations{nameAndKey: true, sdkKeys: true}, func(proj projectInfo, env environmentInfo) string {
+				return string(env.id)
+			})
+		})
+	})
+}
+
+func testKeyIsRotatedWithoutGracePeriod(t *testing.T, manager *integrationTestManager) {
+	// If a key is rotated without a grace period, then the old one should be revoked immediately.
+	// If a key is deprecated and then expires, it should be removed from the environment credentials.
+	withOfflineModeTestData(t, manager, apiParams{numEnvironments: 2, numProjects: 1}, func(testData offlineModeTestData) {
+		helpers.WithTempDir(func(dirPath string) {
+			fileName := "archive.tar.gz"
+			filePath := filepath.Join(manager.relaySharedDir, fileName)
+
+			err := downloadRelayArchive(manager, testData.autoConfigKey, filePath)
+			manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err)
+			require.NoError(t, err)
+
+			// Relay will check for expired keys at this interval.
+			const cleanupInterval = 100 * time.Millisecond
+			// We'll sleep longer than the interval after rotating keys, to try and reduce test flakiness.
+			const cleanupIntervalBuffer = 1 * time.Second
+
+			fmt.Println(cleanupInterval)
+			manager.startRelay(t, map[string]string{
+				"FILE_DATA_SOURCE":                    filepath.Join(relayContainerSharedDir, fileName),
+				"EXPIRED_CREDENTIAL_CLEANUP_INTERVAL": cleanupInterval.String(),
+			})
+			defer manager.stopRelay(t)
+
+			manager.awaitEnvironments(t, testData.projsAndEnvs, &envPropertyExpectations{nameAndKey: true}, func(proj projectInfo, env environmentInfo) string {
+				return string(env.id)
+			})
+			manager.verifyFlagValues(t, testData.projsAndEnvs)
+
+			updated := maps.Clone(testData.projsAndEnvs)
+
+			// Check that the rotation logic holds for more than one rotation.
+			const numRotations = 3
+			for i := 0; i < numRotations; i++ {
+				// time.Time{} to signify that there's no deprecation period.
+				updated = manager.rotateSDKKeys(t, updated, time.Time{})
+
+				err = downloadRelayArchive(manager, testData.autoConfigKey, filePath)
+				manager.apiHelper.logResult("Download data archive from /relay/latest-all to "+filePath, err)
+				require.NoError(t, err)
+
+				time.Sleep(cleanupIntervalBuffer)
+
+				// We are now asserting that the SDK key was rotated (and that there's no expiringSDKKey).
+				manager.awaitEnvironments(t, updated, &envPropertyExpectations{nameAndKey: true, sdkKeys: true}, func(proj projectInfo, env environmentInfo) string {
+					return string(env.id)
+				})
+			}
 		})
 	})
 }
 
-func withOfflineModeTestData(t *testing.T, manager *integrationTestManager, fn func(offlineModeTestData)) {
-	projsAndEnvs, err := manager.apiHelper.createProjectsAndEnvironments(2, 2)
+func withOfflineModeTestData(t *testing.T, manager *integrationTestManager, cfg apiParams, fn func(offlineModeTestData)) {
+	projsAndEnvs, err := manager.apiHelper.createProjectsAndEnvironments(cfg.numProjects, cfg.numEnvironments)
 	require.NoError(t, err)
 	defer manager.apiHelper.deleteProjects(projsAndEnvs)
 

integrationtests/projects_and_environments_test.go

@@ -15,20 +15,30 @@ type projectInfo struct {
 }
 
 type environmentInfo struct {
-	id        config.EnvironmentID
-	key       string
-	name      string
-	sdkKey    config.SDKKey
-	mobileKey config.MobileKey
-	prefix    string
-	projKey   string
+	id             config.EnvironmentID
+	key            string
+	name           string
+	sdkKey         config.SDKKey
+	expiringSdkKey config.SDKKey
+	mobileKey      config.MobileKey
+	prefix         string
+	projKey        string
 
 	// this is a synthetic field, set only when this environment is a filtered environment.
 	filterKey config.FilterKey
 }
 
 type projsAndEnvs map[projectInfo][]environmentInfo
 
+func (pe projsAndEnvs) withoutExpiringKeys() projsAndEnvs {
+	for _, envs := range pe {
+		for i := range envs {
+			envs[i].expiringSdkKey = ""
+		}
+	}
+	return pe
+}
+
 func (pe projsAndEnvs) enumerateEnvs(fn func(projectInfo, environmentInfo)) {
 	for proj, envs := range pe {
 		for _, env := range envs {

integrationtests/standard_mode_payload_filters_test.go

@@ -42,7 +42,7 @@ func testStandardModeWithDefaultFilters(t *testing.T, manager *integrationTestMa
 		manager.startRelay(t, envVars)
 		defer manager.stopRelay(t)
 
-		manager.awaitEnvironments(t, testData.projsAndEnvs, false, func(proj projectInfo, env environmentInfo) string {
+		manager.awaitEnvironments(t, testData.projsAndEnvs, nil, func(proj projectInfo, env environmentInfo) string {
 			if env.filterKey == "" {
 				return env.key
 			}
@@ -105,7 +105,7 @@ func testStandardModeWithSpecificFilters(t *testing.T, manager *integrationTestM
 		manager.startRelay(t, envVars)
 		defer manager.stopRelay(t)
 
-		manager.awaitEnvironments(t, testData.projsAndEnvs, false, func(proj projectInfo, env environmentInfo) string {
+		manager.awaitEnvironments(t, testData.projsAndEnvs, nil, func(proj projectInfo, env environmentInfo) string {
 			if env.filterKey == "" {
 				return env.key
 			}

integrationtests/standard_mode_test.go

@@ -25,7 +25,7 @@ func testStandardMode(t *testing.T, manager *integrationTestManager) {
 		manager.startRelay(t, envVars)
 		defer manager.stopRelay(t)
 
-		manager.awaitEnvironments(t, testData.projsAndEnvs, false, func(proj projectInfo, env environmentInfo) string {
+		manager.awaitEnvironments(t, testData.projsAndEnvs, nil, func(proj projectInfo, env environmentInfo) string {
 			return string(env.name)
 		})
 		manager.verifyFlagValues(t, testData.projsAndEnvs)