Skip to content
This repository has been archived by the owner on Oct 9, 2023. It is now read-only.

Commit

Permalink
Merge branch 'master' into bug/delete-image-pull-backoff-ng
Browse files Browse the repository at this point in the history 
Signed-off-by: Daniel Rammer <[email protected]>
  • Loading branch information
@hamersaw
hamersaw committed Apr 21, 2023
2 parents 6ed1729 + dd49c9d commit b2a3e46
Show file tree
Hide file tree
Showing 6 changed files with 127 additions and 18 deletions.
2 changes: 2 additions & 0 deletions pkg/controller/nodes/subworkflow/launchplan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@ func (l *launchPlanHandler) StartLaunchPlan(ctx context.Context, nCtx handler.No
RawOutputDataConfig: nCtx.ExecutionContext().GetRawOutputDataConfig().RawOutputDataConfig,
Labels: nCtx.ExecutionContext().GetLabels(),
Annotations: nCtx.ExecutionContext().GetAnnotations(),
Interruptible: nCtx.ExecutionContext().GetExecutionConfig().Interruptible,
OverwriteCache: nCtx.ExecutionContext().GetExecutionConfig().OverwriteCache,
}

if nCtx.ExecutionContext().GetExecutionConfig().RecoveryExecution.WorkflowExecutionIdentifier != nil {
Expand Down
29 changes: 20 additions & 9 deletions pkg/controller/nodes/subworkflow/launchplan/admin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,23 +6,25 @@ import (
"fmt"
"time"

evtErr "github.com/flyteorg/flytepropeller/events/errors"
"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin"
"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service"

"github.com/flyteorg/flytestdlib/cache"
"golang.org/x/time/rate"
"k8s.io/client-go/util/workqueue"

stdErr "github.com/flyteorg/flytestdlib/errors"

"github.com/flyteorg/flytestdlib/logger"

"github.com/flyteorg/flytestdlib/promutils"

"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin"
"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service"
evtErr "github.com/flyteorg/flytepropeller/events/errors"

"github.com/golang/protobuf/ptypes/wrappers"

"golang.org/x/time/rate"

"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"

"k8s.io/client-go/util/workqueue"
)

var isRecovery = true
Expand Down Expand Up @@ -93,6 +95,13 @@ func (a *adminLaunchPlanExecutor) Launch(ctx context.Context, launchCtx LaunchCo
}
}

var interruptible *wrappers.BoolValue
if launchCtx.Interruptible != nil {
interruptible = &wrappers.BoolValue{
Value: *launchCtx.Interruptible,
}
}

req := &admin.ExecutionCreateRequest{
Project: executionID.Project,
Domain: executionID.Domain,
Expand All @@ -111,6 +120,8 @@ func (a *adminLaunchPlanExecutor) Launch(ctx context.Context, launchCtx LaunchCo
SecurityContext: &launchCtx.SecurityContext,
MaxParallelism: int32(launchCtx.MaxParallelism),
RawOutputDataConfig: launchCtx.RawOutputDataConfig,
Interruptible: interruptible,
OverwriteCache: launchCtx.OverwriteCache,
},
}

Expand Down
2 changes: 2 additions & 0 deletions pkg/controller/nodes/subworkflow/launchplan/launchplan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ type LaunchContext struct {
RawOutputDataConfig *admin.RawOutputDataConfig
Annotations map[string]string
Labels map[string]string
Interruptible *bool
OverwriteCache bool
}

// Executor interface to be implemented by the remote system that can allow workflow launching capabilities
Expand Down
5 changes: 3 additions & 2 deletions pkg/webhook/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,9 @@ type GCPSecretManagerConfig struct {
}

type VaultSecretManagerConfig struct {
Role string `json:"role" pflag:",Specifies the vault role to use"`
KVVersion KVVersion `json:"kvVersion" pflag:"-,The KV Engine Version. Defaults to 2. Use 1 for unversioned secrets. Refer to - https://www.vaultproject.io/docs/secrets/kv#kv-secrets-engine."`
Role string `json:"role" pflag:",Specifies the vault role to use"`
KVVersion KVVersion `json:"kvVersion" pflag:"-,The KV Engine Version. Defaults to 2. Use 1 for unversioned secrets. Refer to - https://www.vaultproject.io/docs/secrets/kv#kv-secrets-engine."`
Annotations map[string]string `json:"annotations" pflag:"-,Annotation to be added to user task pod. The annotation can also be used to override default annotations added by Flyte. Useful to customize Vault integration (https://developer.hashicorp.com/vault/docs/platform/k8s/injector/annotations)"`
}

func GetConfig() *Config {
Expand Down
3 changes: 1 addition & 2 deletions pkg/webhook/vault_secret_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,7 @@ func (i VaultSecretManagerInjector) Inject(ctx context.Context, secret *coreIdl.
return p, false, err
}

p.ObjectMeta.Annotations = utils.UnionMaps(p.ObjectMeta.Annotations, commonVaultAnnotations)
p.ObjectMeta.Annotations = utils.UnionMaps(p.ObjectMeta.Annotations, secretVaultAnnotations)
p.ObjectMeta.Annotations = utils.UnionMaps(secretVaultAnnotations, commonVaultAnnotations, i.cfg.Annotations, p.ObjectMeta.Annotations)

case coreIdl.Secret_ENV_VAR:
return p, false, fmt.Errorf("Env_Var is not a supported mount requirement for Vault Secret Manager")
Expand Down
104 changes: 99 additions & 5 deletions pkg/webhook/vault_secret_manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,11 +82,69 @@ func ExpectedKVv2(uuid string) *corev1.Pod {
return expected
}

func NewInputPod() *corev1.Pod {
func ExpectedKVv3(uuid string) *corev1.Pod {
// Injects uuid into expected output for KV v2 secrets
expected := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
"vault.hashicorp.com/agent-inject": "true",
"vault.hashicorp.com/secret-volume-path": "/etc/flyte/secrets",
"vault.hashicorp.com/role": "flyte",
"vault.hashicorp.com/agent-pre-populate-only": "true",
fmt.Sprintf("vault.hashicorp.com/agent-inject-secret-%s", uuid): "foo",
fmt.Sprintf("vault.hashicorp.com/agent-inject-file-%s", uuid): "foo/bar",
fmt.Sprintf("vault.hashicorp.com/agent-inject-template-%s", uuid): `{{- with secret "foo" -}}{{ .Data.data.bar }}{{- end -}}`,
"vault.hashicorp.com/auth-config-type": "gce",
},
},
Spec: PodSpec,
}
return expected
}

func ExpectedKVv4(uuid string) *corev1.Pod {
// Injects uuid into expected output for KV v2 secrets
expected := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
"vault.hashicorp.com/agent-inject": "true",
"vault.hashicorp.com/secret-volume-path": "/etc/flyte/secrets",
"vault.hashicorp.com/role": "my-role",
"vault.hashicorp.com/agent-pre-populate-only": "true",
fmt.Sprintf("vault.hashicorp.com/agent-inject-secret-%s", uuid): "foo",
fmt.Sprintf("vault.hashicorp.com/agent-inject-file-%s", uuid): "foo/bar",
fmt.Sprintf("vault.hashicorp.com/agent-inject-template-%s", uuid): `{{- with secret "foo" -}}{{ .Data.data.bar }}{{- end -}}`,
},
},
Spec: PodSpec,
}
return expected
}

func ExpectedKVv5(uuid string) *corev1.Pod {
// Injects uuid into expected output for KV v2 secrets
expected := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
"vault.hashicorp.com/agent-inject": "true",
"vault.hashicorp.com/secret-volume-path": "/etc/flyte/secrets",
"vault.hashicorp.com/role": "flyte",
"vault.hashicorp.com/agent-pre-populate-only": "false",
fmt.Sprintf("vault.hashicorp.com/agent-inject-secret-%s", uuid): "foo",
fmt.Sprintf("vault.hashicorp.com/agent-inject-file-%s", uuid): "foo/bar",
fmt.Sprintf("vault.hashicorp.com/agent-inject-template-%s", uuid): `{{- with secret "foo" -}}{{ .Data.data.bar }}{{- end -}}`,
},
},
Spec: PodSpec,
}
return expected
}

func NewInputPod(annotations map[string]string) *corev1.Pod {
// Need to create a new Pod for every test since annotations are otherwise appended to original reference object
p := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{},
Annotations: annotations,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
Expand Down Expand Up @@ -122,7 +180,7 @@ func TestVaultSecretManagerInjector_Inject(t *testing.T) {
args: args{
cfg: config.VaultSecretManagerConfig{Role: "flyte", KVVersion: config.KVVersion1},
secret: inputSecret,
p: NewInputPod(),
p: NewInputPod(map[string]string{}),
},
want: ExpectedKVv1,
wantErr: false,
Expand All @@ -132,17 +190,53 @@ func TestVaultSecretManagerInjector_Inject(t *testing.T) {
args: args{
cfg: config.VaultSecretManagerConfig{Role: "flyte", KVVersion: config.KVVersion2},
secret: inputSecret,
p: NewInputPod(),
p: NewInputPod(map[string]string{}),
},
want: ExpectedKVv2,
wantErr: false,
},
{
name: "KVv3 Secret - extra annotations",
args: args{
cfg: config.VaultSecretManagerConfig{Role: "flyte", KVVersion: config.KVVersion2, Annotations: map[string]string{
"vault.hashicorp.com/auth-config-type": "gce",
}},
secret: inputSecret,
p: NewInputPod(map[string]string{}),
},
want: ExpectedKVv3,
wantErr: false,
},
{
name: "KVv4 Secret - user override annotation",
args: args{
cfg: config.VaultSecretManagerConfig{Role: "flyte", KVVersion: config.KVVersion2, Annotations: map[string]string{}},
secret: inputSecret,
p: NewInputPod(map[string]string{
"vault.hashicorp.com/role": "my-role",
}),
},
want: ExpectedKVv4,
wantErr: false,
},
{
name: "KVv5 Secret - system override annotation",
args: args{
cfg: config.VaultSecretManagerConfig{Role: "flyte", KVVersion: config.KVVersion2, Annotations: map[string]string{
"vault.hashicorp.com/agent-pre-populate-only": "false", // override vault.hashicorp.com/agent-pre-populate-only
}},
secret: inputSecret,
p: NewInputPod(map[string]string{}),
},
want: ExpectedKVv5,
wantErr: false,
},
{
name: "Unsupported KV version",
args: args{
cfg: config.VaultSecretManagerConfig{Role: "flyte", KVVersion: 3},
secret: inputSecret,
p: NewInputPod(),
p: NewInputPod(map[string]string{}),
},
want: nil,
wantErr: true,
Expand Down

0 comments on commit b2a3e46

Please sign in to comment.