Skip to content

Commit

Permalink
Update more e2e flags to use transforms and remove special logic
Browse files Browse the repository at this point in the history 
Updates various flags that result in choosing the appropriate e2e
conformance image. These help simplify logic in the rest of the
application and move the complexity just into the flags themselves.

Signed-off-by: John Schnake <[email protected]>
  • Loading branch information
@johnSchnake
johnSchnake committed Jul 7, 2021
1 parent 0ae0a6f commit 1b626d7
Show file tree
Hide file tree
Showing 29 changed files with 299 additions and 94 deletions.
134 changes: 119 additions & 15 deletions cmd/sonobuoy/app/args.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ const (
e2eSkipFlag = "e2e-skip"
e2eParallelFlag = "e2e-parallel"
e2eRegistryConfigFlag = "e2e-repo-config"
pluginImageFlag = "plugin-image"

// Quick runs a single E2E test and the systemd log tests.
Quick string = "quick"
Expand Down Expand Up @@ -103,30 +104,57 @@ func AddSonobuoyImage(image *string, flags *pflag.FlagSet) {
)
}

// AddKubeConformanceImage initialises an image url flag.
func AddKubeConformanceImage(image *string, flags *pflag.FlagSet) {
flags.StringVar(
image, "kube-conformance-image", "",
"Container image override for the kube conformance image. Overrides --kube-conformance-image-version.",
func AddPluginImage(pluginTransforms *map[string][]func(*manifest.Manifest) error, fs *pflag.FlagSet) {
fs.Var(
&pluginImageFlagType{
transforms: *pluginTransforms,
},
pluginImageFlag,
"Override a plugins image from what is in its definition (expects format plugin:image)",
)
}

// AddSystemdLogsImage initialises the systemd-logs-image flag.
func AddSystemdLogsImage(image *string, flags *pflag.FlagSet) {
flags.StringVar(
image, "systemd-logs-image", config.DefaultSystemdLogsImage,
// AddKubeConformanceImage initialises the kube-conformance-image flag. Really just a legacy wrapper for --plugin-image=e2e:imageName
func AddKubeConformanceImage(pluginTransforms *map[string][]func(*manifest.Manifest) error, fs *pflag.FlagSet) {
fs.Var(
&hardcodedPluginImageFlagType{
plugin: e2ePlugin,
underlyingFlag: &pluginImageFlagType{
transforms: *pluginTransforms,
},
},
"kube-conformance-image",
"Container image override for the kube conformance image.",
)
}

// AddSystemdLogsImage initialises the systemd-logs-image flag. Really just a legacy wrapper for --plugin-image=systemd-logs:imageName
func AddSystemdLogsImage(pluginTransforms *map[string][]func(*manifest.Manifest) error, fs *pflag.FlagSet) {
fs.Var(
&hardcodedPluginImageFlagType{
plugin: systemdLogsPlugin,
underlyingFlag: &pluginImageFlagType{
transforms: *pluginTransforms,
},
},
"systemd-logs-image",
"Container image override for the systemd-logs plugin image.",
)
}

// AddKubeConformanceImageVersion initialises an image version flag.
func AddKubeConformanceImageVersion(imageVersion *image.ConformanceImageVersion, flags *pflag.FlagSet) {
func AddKubeConformanceImageVersion(imageVersion *image.ConformanceImageVersion, pluginTransforms *map[string][]func(*manifest.Manifest) error, flags *pflag.FlagSet) {
help := "Use default Conformance image, but override the version. "
help += "Default is 'auto', which will be set to your cluster's version if detected, erroring otherwise."
help += "You can also choose 'latest' which will find the latest dev image upstream."

*imageVersion = image.ConformanceImageVersionAuto
flags.Var(imageVersion, "kube-conformance-image-version", help)
flags.Var(
&kubernetesVersionLogicFlag{
raw: imageVersion,
underlyingFlag: &pluginImageFlagType{
transforms: *pluginTransforms,
},
}, "kube-conformance-image-version", help)
if err := flags.MarkDeprecated("kube-conformance-image-version", "Use --kubernetes-version instead."); err != nil {
panic("Failed to setup flags properly")
}
Expand Down Expand Up @@ -330,14 +358,19 @@ func AddPluginListFlag(p *[]string, flags *pflag.FlagSet) {
}

// AddKubernetesVersionFlag initialises an image version flag.
func AddKubernetesVersionFlag(imageVersion *image.ConformanceImageVersion, flags *pflag.FlagSet) {
func AddKubernetesVersionFlag(imageVersion *image.ConformanceImageVersion, pluginTransforms *map[string][]func(*manifest.Manifest) error, flags *pflag.FlagSet) {
help := "Use default Conformance image, but override the version. "
help += "Default is 'auto', which will be set to your cluster's version if detected, erroring otherwise. "
help += "'ignore' will try version resolution but ignore errors. "
help += "'latest' will find the latest dev image/version upstream."

*imageVersion = image.ConformanceImageVersionAuto
flags.Var(imageVersion, "kubernetes-version", help)
flags.Var(
&kubernetesVersionLogicFlag{
raw: imageVersion,
underlyingFlag: &pluginImageFlagType{
transforms: *pluginTransforms,
},
}, "kubernetes-version", help)
}

// AddShortFlag adds a boolean flag to just print the Sonobuoy version and
Expand Down Expand Up @@ -513,3 +546,74 @@ func (f *sshPathFlag) Set(str string) error {
})
return nil
}

type pluginImageFlagType struct {
overrides map[string]string
transforms map[string][]func(*manifest.Manifest) error
}

func (f *pluginImageFlagType) String() string { return fmt.Sprint(f.overrides) }
func (f *pluginImageFlagType) Type() string { return "plugin:image" }
func (f *pluginImageFlagType) Set(str string) error {
parts := strings.SplitN(str, ":", 2)
if len(parts) != 2 {
return fmt.Errorf("failed to parse plugin image flag, expected format plugin:image and got %v", str)
}
if f.overrides == nil {
f.overrides = map[string]string{}
}
f.overrides[parts[0]] = parts[1]

f.transforms[parts[0]] = append(f.transforms[parts[0]], func(m *manifest.Manifest) error {
m.Spec.Image = parts[1]
return nil
})
return nil
}

type hardcodedPluginImageFlagType struct {
underlyingFlag *pluginImageFlagType
plugin string
}

func (f *hardcodedPluginImageFlagType) String() string {
return f.underlyingFlag.String()
}
func (f *hardcodedPluginImageFlagType) Type() string { return "image" }
func (f *hardcodedPluginImageFlagType) Set(str string) error {
return f.underlyingFlag.Set(fmt.Sprintf("%v:%v", f.plugin, str))
}

type kubernetesVersionLogicFlag struct {
underlyingFlag *pluginImageFlagType
raw *image.ConformanceImageVersion
}

func (f *kubernetesVersionLogicFlag) String() string {
return f.raw.String()
}
func (f *kubernetesVersionLogicFlag) Type() string { return "string" }
func (f *kubernetesVersionLogicFlag) Set(str string) error {
if err := f.raw.Set(str); err != nil {
return err
}

var img, ver string
switch *f.raw {
case "", image.ConformanceImageVersionIgnore, image.ConformanceImageVersionAuto:
img = config.UpstreamKubeConformanceImageURL
ver = "$SONOBUOY_K8S_VERSION"
case image.ConformanceImageVersionLatest:
version, err := image.GetLatestDevVersion(image.DevVersionURL)
if err != nil {
return errors.Wrap(err, "couldn't identify latest dev image")
}

img = image.DevVersionImageURL
ver = version
default:
img = config.UpstreamKubeConformanceImageURL
ver = f.raw.String()
}
return f.underlyingFlag.Set(fmt.Sprintf("%v:%v:%v", e2ePlugin, img, ver))
}
71 changes: 29 additions & 42 deletions cmd/sonobuoy/app/gen.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,14 @@ import (
)

type genFlags struct {
sonobuoyConfig SonobuoyConfig
rbacMode RBACMode
kubecfg Kubeconfig
dnsNamespace string
dnsPodLabels []string
kubeConformanceImage string
systemdLogsImage string
sshKeyPath string
k8sVersion imagepkg.ConformanceImageVersion
showDefaultPodSpec bool
sonobuoyConfig SonobuoyConfig
rbacMode RBACMode
kubecfg Kubeconfig
dnsNamespace string
dnsPodLabels []string
sshKeyPath string
k8sVersion imagepkg.ConformanceImageVersion
showDefaultPodSpec bool

// plugins will keep a list of the plugins we want. Custom type for
// flag support.
Expand Down Expand Up @@ -76,8 +74,6 @@ func GenFlagSet(cfg *genFlags, rbac RBACMode) *pflag.FlagSet {
AddDNSNamespaceFlag(&cfg.dnsNamespace, genset)
AddDNSPodLabelsFlag(&cfg.dnsPodLabels, genset)
AddSonobuoyImage(&cfg.sonobuoyConfig.WorkerImage, genset)
AddKubeConformanceImage(&cfg.kubeConformanceImage, genset)
AddSystemdLogsImage(&cfg.systemdLogsImage, genset)
AddSSHKeyPathFlag(&cfg.sshKeyPath, &cfg.pluginTransforms, genset)

AddPluginSetFlag(&cfg.plugins, genset)
Expand All @@ -86,8 +82,13 @@ func GenFlagSet(cfg *genFlags, rbac RBACMode) *pflag.FlagSet {

AddNodeSelectorsFlag(&cfg.nodeSelectors, genset)

AddKubeConformanceImageVersion(&cfg.k8sVersion, genset)
AddKubernetesVersionFlag(&cfg.k8sVersion, genset)
AddKubeConformanceImageVersion(&cfg.k8sVersion, &cfg.pluginTransforms, genset)
AddKubernetesVersionFlag(&cfg.k8sVersion, &cfg.pluginTransforms, genset)

AddPluginImage(&cfg.pluginTransforms, genset)
AddKubeConformanceImage(&cfg.pluginTransforms, genset)
AddSystemdLogsImage(&cfg.pluginTransforms, genset)

return genset
}

Expand All @@ -96,8 +97,6 @@ func (g *genFlags) Config() (*client.GenConfig, error) {
g.plugins.DynamicPlugins = []string{e2ePlugin, systemdLogsPlugin}
}

// TODO: Refactor this logic to be less convuled: https://github.com/vmware-tanzu/sonobuoy/issues/481

// In some configurations, the kube client isn't actually needed for correct executation
// Therefore, delay reporting the error until we're sure we need the client
kubeclient, kubeError := getClient(&g.kubecfg)
Expand All @@ -112,8 +111,7 @@ func (g *genFlags) Config() (*client.GenConfig, error) {
return nil, err
}

var e2eImage, e2eRegistry, imageVersion string

var k8sVersion string
switch g.k8sVersion {
case "", imagepkg.ConformanceImageVersionAuto, imagepkg.ConformanceImageVersionLatest, imagepkg.ConformanceImageVersionIgnore:
var discoveryClient discovery.ServerVersionInterface
Expand All @@ -124,7 +122,7 @@ func (g *genFlags) Config() (*client.GenConfig, error) {
// `auto` k8s version needs resolution as well as any static plugins which use the
// variable SONOBUOY_K8S_VERSION. Just check for it all by default but allow skipping
// errors/resolution via flag.
e2eRegistry, imageVersion, err = g.k8sVersion.Get(discoveryClient, imagepkg.DevVersionURL)
_, k8sVersion, err = g.k8sVersion.Get(discoveryClient, imagepkg.DevVersionURL)
if err != nil {
if errors.Cause(err) == imagepkg.ErrImageVersionNoClient &&
g.k8sVersion != imagepkg.ConformanceImageVersionIgnore {
Expand All @@ -133,32 +131,21 @@ func (g *genFlags) Config() (*client.GenConfig, error) {
return nil, err
}
default:
// Sane default if specifying version by itself.
e2eRegistry = config.UpstreamKubeConformanceImageURL
imageVersion = g.k8sVersion.String()
}

// --kube-conformance-image overrides --kube-conformance-image-version
if g.kubeConformanceImage != "" {
e2eImage = g.kubeConformanceImage
} else {
e2eImage = fmt.Sprintf("%v:%v", e2eRegistry, imageVersion)
k8sVersion = g.k8sVersion.String()
}

return &client.GenConfig{
Config: &g.sonobuoyConfig.Config,
EnableRBAC: rbacEnabled,
KubeConformanceImage: e2eImage,
SystemdLogsImage: g.systemdLogsImage,
ImagePullPolicy: g.sonobuoyConfig.ImagePullPolicy,
SSHKeyPath: g.sshKeyPath,
DynamicPlugins: g.plugins.DynamicPlugins,
StaticPlugins: g.plugins.StaticPlugins,
PluginEnvOverrides: g.pluginEnvs,
ShowDefaultPodSpec: g.showDefaultPodSpec,
NodeSelectors: g.nodeSelectors,
KubeVersion: imageVersion,
PluginTransforms: g.pluginTransforms,
Config: &g.sonobuoyConfig.Config,
EnableRBAC: rbacEnabled,
ImagePullPolicy: g.sonobuoyConfig.ImagePullPolicy,
SSHKeyPath: g.sshKeyPath,
DynamicPlugins: g.plugins.DynamicPlugins,
StaticPlugins: g.plugins.StaticPlugins,
PluginEnvOverrides: g.pluginEnvs,
ShowDefaultPodSpec: g.showDefaultPodSpec,
NodeSelectors: g.nodeSelectors,
KubeVersion: k8sVersion,
PluginTransforms: g.pluginTransforms,
}, nil
}

Expand Down
16 changes: 11 additions & 5 deletions cmd/sonobuoy/app/images.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/vmware-tanzu/sonobuoy/pkg/config"
"github.com/vmware-tanzu/sonobuoy/pkg/errlog"
"github.com/vmware-tanzu/sonobuoy/pkg/image"
"github.com/vmware-tanzu/sonobuoy/pkg/plugin/manifest"
)

// Number times to retry docker commands before giving up
Expand All @@ -46,6 +47,11 @@ type imagesFlags struct {
k8sVersion image.ConformanceImageVersion
}

var (
// transformSink avoids nil issues despite not really needing the transforms for these commands.
transformSink = map[string][]func(*manifest.Manifest) error{}
)

func NewCmdImages() *cobra.Command {
var flags imagesFlags
// Main command
Expand Down Expand Up @@ -74,7 +80,7 @@ func NewCmdImages() *cobra.Command {

AddKubeconfigFlag(&flags.kubeconfig, cmd.Flags())
AddPluginListFlag(&flags.plugins, cmd.Flags())
AddKubernetesVersionFlag(&flags.k8sVersion, cmd.Flags())
AddKubernetesVersionFlag(&flags.k8sVersion, &transformSink, cmd.Flags())

cmd.AddCommand(pullCmd())
cmd.AddCommand(pushCmd())
Expand Down Expand Up @@ -115,7 +121,7 @@ func pullCmd() *cobra.Command {
AddKubeconfigFlag(&flags.kubeconfig, pullCmd.Flags())
AddPluginListFlag(&flags.plugins, pullCmd.Flags())
AddDryRunFlag(&flags.dryRun, pullCmd.Flags())
AddKubernetesVersionFlag(&flags.k8sVersion, pullCmd.Flags())
AddKubernetesVersionFlag(&flags.k8sVersion, &transformSink, pullCmd.Flags())

return pullCmd
}
Expand Down Expand Up @@ -158,7 +164,7 @@ func pushCmd() *cobra.Command {
AddCustomRegistryFlag(&flags.customRegistry, pushCmd.Flags())
AddDryRunFlag(&flags.dryRun, pushCmd.Flags())
pushCmd.MarkFlagRequired(customRegistryFlag)
AddKubernetesVersionFlag(&flags.k8sVersion, pushCmd.Flags())
AddKubernetesVersionFlag(&flags.k8sVersion, &transformSink, pushCmd.Flags())

return pushCmd
}
Expand Down Expand Up @@ -191,7 +197,7 @@ func downloadCmd() *cobra.Command {
AddKubeconfigFlag(&flags.kubeconfig, downloadCmd.Flags())
AddPluginListFlag(&flags.plugins, downloadCmd.Flags())
AddDryRunFlag(&flags.dryRun, downloadCmd.Flags())
AddKubernetesVersionFlag(&flags.k8sVersion, downloadCmd.Flags())
AddKubernetesVersionFlag(&flags.k8sVersion, &transformSink, downloadCmd.Flags())

return downloadCmd
}
Expand Down Expand Up @@ -222,7 +228,7 @@ func deleteCmd() *cobra.Command {
AddKubeconfigFlag(&flags.kubeconfig, deleteCmd.Flags())
AddPluginListFlag(&flags.plugins, deleteCmd.Flags())
AddDryRunFlag(&flags.dryRun, deleteCmd.Flags())
AddKubernetesVersionFlag(&flags.k8sVersion, deleteCmd.Flags())
AddKubernetesVersionFlag(&flags.k8sVersion, &transformSink, deleteCmd.Flags())

return deleteCmd
}
Expand Down
7 changes: 4 additions & 3 deletions pkg/client/gen.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ func (*SonobuoyClient) GenerateManifestAndPlugins(cfg *GenConfig) ([]byte, []*ma
return strings.ToLower(plugins[i].SonobuoyConfig.PluginName) < strings.ToLower(plugins[j].SonobuoyConfig.PluginName)
})

// Apply transforms. Ensure this is before handling configmaps.
// Apply transforms. Ensure this is before handling configmaps and applying the k8s_version.
for pluginName, transforms := range cfg.PluginTransforms {
for _, p := range plugins {
if p.SonobuoyConfig.PluginName == pluginName {
Expand Down Expand Up @@ -201,6 +201,7 @@ func (*SonobuoyClient) GenerateManifestAndPlugins(cfg *GenConfig) ([]byte, []*ma
for _, p := range plugins {
pluginNames = append(pluginNames, p.SonobuoyConfig.PluginName)
}

return nil, nil, fmt.Errorf("failed to override env vars for plugin %v, no plugin with that name found; have plugins: %v", pluginName, pluginNames)
}
}
Expand Down Expand Up @@ -304,7 +305,7 @@ func SystemdLogsManifest(cfg *GenConfig) *manifest.Manifest {
Spec: manifest.Container{
Container: corev1.Container{
Name: "systemd-logs",
Image: cfg.SystemdLogsImage,
Image: config.DefaultSystemdLogsImage,
Command: []string{"/bin/sh", "-c", `/get_systemd_logs.sh; while true; do echo "Plugin is complete. Sleeping indefinitely to avoid container exit and automatic restarts from Kubernetes"; sleep 3600; done`},
ImagePullPolicy: corev1.PullPolicy(cfg.ImagePullPolicy),
Env: []corev1.EnvVar{
Expand Down Expand Up @@ -349,7 +350,7 @@ func E2EManifest(cfg *GenConfig) *manifest.Manifest {
Spec: manifest.Container{
Container: corev1.Container{
Name: "e2e",
Image: cfg.KubeConformanceImage,
Image: fmt.Sprintf("%v:%v", config.UpstreamKubeConformanceImageURL, "$SONOBUOY_K8S_VERSION"),
Command: []string{"/run_e2e.sh"},
ImagePullPolicy: corev1.PullPolicy(cfg.ImagePullPolicy),
Env: []corev1.EnvVar{
Expand Down
Loading

0 comments on commit 1b626d7

Please sign in to comment.