This repository has been archived by the owner on Oct 9, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 59
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add support for GCP secret manager (#547)
Signed-off-by: Jeev B <[email protected]>
- Loading branch information
Showing
7 changed files
with
279 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,156 @@ | ||
| package webhook | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
|
|
||
| "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" | ||
| "github.com/flyteorg/flytepropeller/pkg/webhook/config" | ||
| "github.com/flyteorg/flytestdlib/logger" | ||
| corev1 "k8s.io/api/core/v1" | ||
| ) | ||
|
|
||
| const ( | ||
| // GCPSecretsVolumeName defines the static name of the volume used for mounting/sharing secrets between init-container | ||
| // sidecar and the rest of the containers in the pod. | ||
| GCPSecretsVolumeName = "gcp-secret-vol" // #nosec | ||
| ) | ||
|
|
||
| var ( | ||
| // GCPSecretMountPath defines the default mount path for secrets | ||
| GCPSecretMountPath = filepath.Join(string(os.PathSeparator), "etc", "flyte", "secrets") | ||
| ) | ||
|
|
||
| // GCPSecretManagerInjector allows injecting of secrets from GCP Secret Manager as files. It uses a Google Cloud | ||
| // SDK SideCar as an init-container to download the secret and save it to a local volume shared with all other | ||
| // containers in the pod. It supports multiple secrets to be mounted but that will result into adding an init | ||
| // container for each secret. The Google serviceaccount (GSA) associated with the Pod, either via Workload | ||
| // Identity (recommended) or the underlying node's serviceacccount, must have permissions to pull the secret | ||
| // from GCP Secret Manager. Currently, the secret must also exist in the same project. Otherwise, the Pod will | ||
| // fail with an init-error. | ||
| // Files will be mounted on /etc/flyte/secrets/<SecretGroup>/<SecretGroupVersion> | ||
| type GCPSecretManagerInjector struct { | ||
| cfg config.GCPSecretManagerConfig | ||
| } | ||
|
|
||
| func formatGCPSecretAccessCommand(secret *core.Secret) []string { | ||
| // `gcloud` writes this file with permission 0600. | ||
| // This will cause permission issues in the main container when using non-root | ||
| // users, so we fix the file permissions with `chmod`. | ||
| secretDir := strings.ToLower(filepath.Join(GCPSecretMountPath, secret.Group)) | ||
| secretPath := strings.ToLower(filepath.Join(secretDir, secret.GroupVersion)) | ||
| args := []string{ | ||
| "gcloud", | ||
| "secrets", | ||
| "versions", | ||
| "access", | ||
| secret.GroupVersion, | ||
| fmt.Sprintf("--secret=%s", secret.Group), | ||
| fmt.Sprintf( | ||
| "--out-file=%s", | ||
| secretPath, | ||
| ), | ||
| "&&", | ||
| "chmod", | ||
| "+rX", | ||
| secretDir, | ||
| secretPath, | ||
| } | ||
| return []string{"sh", "-c", strings.Join(args, " ")} | ||
| } | ||
|
|
||
| func formatGCPInitContainerName(index int) string { | ||
| return fmt.Sprintf("gcp-pull-secret-%v", index) | ||
| } | ||
|
|
||
| func (i GCPSecretManagerInjector) Type() config.SecretManagerType { | ||
| return config.SecretManagerTypeGCP | ||
| } | ||
|
|
||
| func (i GCPSecretManagerInjector) Inject(ctx context.Context, secret *core.Secret, p *corev1.Pod) (newP *corev1.Pod, injected bool, err error) { | ||
| if len(secret.Group) == 0 || len(secret.GroupVersion) == 0 { | ||
| return nil, false, fmt.Errorf("GCP Secrets Webhook require both group and group version to be set. "+ | ||
| "Secret: [%v]", secret) | ||
| } | ||
|
|
||
| switch secret.MountRequirement { | ||
| case core.Secret_ANY: | ||
| fallthrough | ||
| case core.Secret_FILE: | ||
| // A Volume with a static name so that if we try to inject multiple secrets, we won't mount multiple volumes. | ||
| // We use Memory as the storage medium for volume source to avoid | ||
| vol := corev1.Volume{ | ||
| Name: GCPSecretsVolumeName, | ||
| VolumeSource: corev1.VolumeSource{ | ||
| EmptyDir: &corev1.EmptyDirVolumeSource{ | ||
| Medium: corev1.StorageMediumMemory, | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| p.Spec.Volumes = appendVolumeIfNotExists(p.Spec.Volumes, vol) | ||
| p.Spec.InitContainers = append(p.Spec.InitContainers, createGCPSidecarContainer(i.cfg, p, secret)) | ||
|
|
||
| secretVolumeMount := corev1.VolumeMount{ | ||
| Name: GCPSecretsVolumeName, | ||
| ReadOnly: true, | ||
| MountPath: GCPSecretMountPath, | ||
| } | ||
|
|
||
| p.Spec.Containers = AppendVolumeMounts(p.Spec.Containers, secretVolumeMount) | ||
| p.Spec.InitContainers = AppendVolumeMounts(p.Spec.InitContainers, secretVolumeMount) | ||
|
|
||
| // Inject GCP secret-inject webhook annotations to mount the secret in a predictable location. | ||
| envVars := []corev1.EnvVar{ | ||
| // Set environment variable to let the container know where to find the mounted files. | ||
| { | ||
| Name: SecretPathDefaultDirEnvVar, | ||
| Value: GCPSecretMountPath, | ||
| }, | ||
| // Sets an empty prefix to let the containers know the file names will match the secret keys as-is. | ||
| { | ||
| Name: SecretPathFilePrefixEnvVar, | ||
| Value: "", | ||
| }, | ||
| } | ||
|
|
||
| for _, envVar := range envVars { | ||
| p.Spec.InitContainers = AppendEnvVars(p.Spec.InitContainers, envVar) | ||
| p.Spec.Containers = AppendEnvVars(p.Spec.Containers, envVar) | ||
| } | ||
| case core.Secret_ENV_VAR: | ||
| fallthrough | ||
| default: | ||
| err := fmt.Errorf("unrecognized mount requirement [%v] for secret [%v]", secret.MountRequirement.String(), secret.Key) | ||
| logger.Error(ctx, err) | ||
| return p, false, err | ||
| } | ||
|
|
||
| return p, true, nil | ||
| } | ||
|
|
||
| func createGCPSidecarContainer(cfg config.GCPSecretManagerConfig, p *corev1.Pod, secret *core.Secret) corev1.Container { | ||
| return corev1.Container{ | ||
| Image: cfg.SidecarImage, | ||
| // Create a unique name to allow multiple secrets to be mounted. | ||
| Name: formatGCPInitContainerName(len(p.Spec.InitContainers)), | ||
| Command: formatGCPSecretAccessCommand(secret), | ||
| VolumeMounts: []corev1.VolumeMount{ | ||
| { | ||
| Name: GCPSecretsVolumeName, | ||
| MountPath: GCPSecretMountPath, | ||
| }, | ||
| }, | ||
| Resources: cfg.Resources, | ||
| } | ||
| } | ||
|
|
||
| // NewGCPSecretManagerInjector creates a SecretInjector that's able to mount secrets from GCP Secret Manager. | ||
| func NewGCPSecretManagerInjector(cfg config.GCPSecretManagerConfig) GCPSecretManagerInjector { | ||
| return GCPSecretManagerInjector{ | ||
| cfg: cfg, | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| package webhook | ||
|
|
||
| import ( | ||
| "context" | ||
| "testing" | ||
|
|
||
| "github.com/flyteorg/flytepropeller/pkg/webhook/config" | ||
|
|
||
| "github.com/go-test/deep" | ||
|
|
||
| "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" | ||
| "github.com/stretchr/testify/assert" | ||
| corev1 "k8s.io/api/core/v1" | ||
| ) | ||
|
|
||
| func TestGCPSecretManagerInjector_Inject(t *testing.T) { | ||
| injector := NewGCPSecretManagerInjector(config.DefaultConfig.GCPSecretManagerConfig) | ||
| inputSecret := &core.Secret{ | ||
| Group: "TestSecret", | ||
| GroupVersion: "2", | ||
| } | ||
|
|
||
| expected := &corev1.Pod{ | ||
| Spec: corev1.PodSpec{ | ||
| Volumes: []corev1.Volume{ | ||
| { | ||
| Name: "gcp-secret-vol", | ||
| VolumeSource: corev1.VolumeSource{ | ||
| EmptyDir: &corev1.EmptyDirVolumeSource{ | ||
| Medium: corev1.StorageMediumMemory, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
|
|
||
| InitContainers: []corev1.Container{ | ||
| { | ||
| Name: "gcp-pull-secret-0", | ||
| Image: "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine", | ||
| Command: []string{ | ||
| "sh", | ||
| "-c", | ||
| "gcloud secrets versions access 2 --secret=TestSecret --out-file=/etc/flyte/secrets/testsecret/2 && chmod +rX /etc/flyte/secrets/testsecret /etc/flyte/secrets/testsecret/2", | ||
| }, | ||
| Env: []corev1.EnvVar{ | ||
| { | ||
| Name: "FLYTE_SECRETS_DEFAULT_DIR", | ||
| Value: "/etc/flyte/secrets", | ||
| }, | ||
| { | ||
| Name: "FLYTE_SECRETS_FILE_PREFIX", | ||
| Value: "", | ||
| }, | ||
| }, | ||
| VolumeMounts: []corev1.VolumeMount{ | ||
| { | ||
| Name: "gcp-secret-vol", | ||
| MountPath: "/etc/flyte/secrets", | ||
| }, | ||
| }, | ||
| Resources: config.DefaultConfig.GCPSecretManagerConfig.Resources, | ||
| }, | ||
| }, | ||
| Containers: []corev1.Container{}, | ||
| }, | ||
| } | ||
|
|
||
| p := &corev1.Pod{ | ||
| Spec: corev1.PodSpec{ | ||
| Containers: []corev1.Container{}, | ||
| }, | ||
| } | ||
| actualP, injected, err := injector.Inject(context.Background(), inputSecret, p) | ||
| assert.NoError(t, err) | ||
| assert.True(t, injected) | ||
| if diff := deep.Equal(actualP, expected); diff != nil { | ||
| assert.Fail(t, "actual != expected", "Diff: %v", diff) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters