Implements e2e subcommand

This subcommand allows users to more easily inspect their
results archive. They can see which tests passed, failed or
were skipped. This also allows users to rerun the tests that
failed.

Closes #248

Signed-off-by: Chuck Ha <[email protected]>

cmd/sonobuoy/app/e2e.go

@@ -0,0 +1,118 @@
+/*
+Copyright 2018 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"compress/gzip"
+	"fmt"
+	"os"
+
+	"github.com/heptio/sonobuoy/pkg/client"
+	"github.com/heptio/sonobuoy/pkg/errlog"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"k8s.io/client-go/kubernetes"
+)
+
+var e2eFlags struct {
+	show  string
+	rerun bool
+}
+
+func init() {
+	cmd := &cobra.Command{
+		Use:   "e2e archive.tar.gz",
+		Short: "Inspect e2e test results. Optionally rerun failed tests",
+		Run:   e2es,
+		Args:  cobra.ExactArgs(1),
+	}
+	AddGenFlags(&runopts.GenConfig, cmd)
+	AddModeFlag(&runFlags.mode, cmd)
+	AddSonobuoyConfigFlag(&runFlags.sonobuoyConfig, cmd)
+	AddKubeconfigFlag(&runFlags.kubecfg, cmd)
+	// Default to detect since we need a kubeconfig regardless
+	AddRBACModeFlags(&runFlags.rbacMode, cmd, DetectRBACMode)
+	AddSkipPreflightFlag(&runopts.SkipPreflight, cmd)
+
+	cmd.PersistentFlags().StringVar(&e2eFlags.show, "show", "failed", "Defines which tests to show, options are [passed, failed (default) or all]. Cannot be combined with --rerun-failed.")
+	cmd.PersistentFlags().BoolVar(&e2eFlags.rerun, "rerun-failed", false, "Rerun the failed tests reported by the archive. The --show flag will be ignored.")
+
+	RootCmd.AddCommand(cmd)
+}
+
+func e2es(cmd *cobra.Command, args []string) {
+	f, err := os.Open(args[0])
+	if err != nil {
+		errlog.LogError(errors.Wrapf(err, "could not open sonobuoy archive: %v", args[0]))
+		os.Exit(1)
+	}
+	defer f.Close()
+	// As documented, ignore show if we are doing a rerun of failed tests.
+	if e2eFlags.rerun {
+		e2eFlags.show = "failed"
+	}
+	gzr, err := gzip.NewReader(f)
+	if err != nil {
+		errlog.LogError(errors.Wrap(err, "could not make a gzip reader"))
+		os.Exit(1)
+	}
+	defer gzr.Close()
+	sonobuoy := client.NewSonobuoyClient()
+	testCases, err := sonobuoy.GetTests(gzr, e2eFlags.show)
+	if err != nil {
+		errlog.LogError(errors.Wrap(err, "could not get tests from archive"))
+		os.Exit(1)
+	}
+
+	// If we are not doing a rerun, print and exit.
+	if !e2eFlags.rerun {
+		fmt.Printf("%v tests\n", e2eFlags.show)
+		fmt.Println(client.PrintableTestCases(testCases))
+		return
+	}
+
+	restConfig, err := runFlags.kubecfg.Get()
+	if err != nil {
+		errlog.LogError(errors.Wrap(err, "couldn't get REST client"))
+		os.Exit(1)
+	}
+
+	clientset, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		errlog.LogError(errors.Wrap(err, "couldn't create Kubernetes clientset"))
+		os.Exit(1)
+	}
+
+	rbacEnabled, err := runFlags.rbacMode.Enabled(clientset)
+	if err != nil {
+		errlog.LogError(errors.Wrap(err, "couldn't detect RBAC mode"))
+		os.Exit(1)
+	}
+	runopts.GenConfig.EnableRBAC = rbacEnabled
+
+	runopts.Config = GetConfigWithMode(&runFlags.sonobuoyConfig, runFlags.mode)
+	runopts.E2EConfig = &client.E2EConfig{
+		Focus: client.Focus(testCases),
+		Skip:  "",
+	}
+
+	fmt.Printf("Rerunning %d tests:\n", len(testCases))
+	if err := sonobuoy.Run(&runopts, restConfig); err != nil {
+		errlog.LogError(errors.Wrap(err, "error attempting to rerun failed tests"))
+		os.Exit(1)
+	}
+}

cmd/sonobuoy/app/sonobuoyconfig.go

@@ -51,7 +51,7 @@ func (c *SonobuoyConfig) Get() *config.Config {
 	return &c.Config
 }
 
-// GetConfigWithMode creates a config with the following algorithim:
+// GetConfigWithMode creates a config with the following algorithm:
 // If the SonobuoyConfig isn't nil, use that
 // If not, use the supplied Mode to modify a default config
 func GetConfigWithMode(sonobuoyCfg *SonobuoyConfig, mode client.Mode) *config.Config {

pkg/client/e2e.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2018 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"io"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/heptio/sonobuoy/pkg/results"
+	"github.com/heptio/sonobuoy/pkg/results/e2e"
+	"github.com/onsi/ginkgo/reporters"
+	"github.com/pkg/errors"
+)
+
+// GetTests extracts the junit results from a sonobuoy archive and returns the requested tests.
+func (c *SonobuoyClient) GetTests(reader io.Reader, show string) ([]reporters.JUnitTestCase, error) {
+	read := results.NewReaderWithVersion(reader, "irrelevant")
+	junitResults := reporters.JUnitTestSuite{}
+	err := read.WalkFiles(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		// TODO(chuckha) consider reusing this function for any generic e2e-esque plugin results.
+		// TODO(chuckha) consider using path.Join()
+		return results.ExtractFileIntoStruct(results.PluginsDir+e2e.ResultsSubdirectory+e2e.JUnitResultsFile, path, info, &junitResults)
+	})
+	out := make([]reporters.JUnitTestCase, 0)
+	if err != nil {
+		return out, errors.Wrap(err, "failed to walk results archive")
+	}
+	if show == "passed" || show == "all" {
+		out = append(out, results.Filter(results.Passed, junitResults)...)
+	}
+	if show == "failed" || show == "all" {
+		out = append(out, results.Filter(results.Failed, junitResults)...)
+	}
+	if show == "skipped" || show == "all" {
+		out = append(out, results.Filter(results.Skipped, junitResults)...)
+	}
+	sort.Sort(results.AlphabetizedTestCases(out))
+	return out, nil
+}
+
+// Focus returns a value to be used in the E2E_FOCUS variable that is
+// representative of the test cases in the struct.
+func Focus(testCases []reporters.JUnitTestCase) string {
+	// YAML doesn't like escaped characters and regex needs escaped characters. Therefore a double escape is necessary.
+	r := strings.NewReplacer("[", `\\[`, "]", `\\]`)
+	testNames := make([]string, len(testCases))
+	for i, tc := range testCases {
+		testNames[i] = r.Replace(tc.Name)
+	}
+	return strings.Join(testNames, "|")
+}
+
+// PrintableTestCases nicely strings a []reporters.JunitTestCase
+type PrintableTestCases []reporters.JUnitTestCase
+
+func (p PrintableTestCases) String() string {
+	out := make([]string, len(p))
+	for i, tc := range p {
+		out[i] = tc.Name
+	}
+	return strings.Join(out, "\n")
+}

pkg/results/e2e/consts.go

@@ -0,0 +1,24 @@
+/*
+Copyright 2018 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// package e2e defines files and directories found in the e2e plugin results.
+package e2e
+
+const (
+	// ResultsDirectory is the directory where the results will be found
+	ResultsSubdirectory = "e2e/results/"
+	JUnitResultsFile    = "junit_01.xml"
+)

pkg/results/junit_utils.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2018 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package results
+
+import (
+	"github.com/onsi/ginkgo/reporters"
+)
+
+// Filter keeps only the tests that match the predicate function.
+func Filter(predicate func(testCase reporters.JUnitTestCase) bool, testSuite reporters.JUnitTestSuite) []reporters.JUnitTestCase {
+	out := make([]reporters.JUnitTestCase, 0)
+	for _, tc := range testSuite.TestCases {
+		if predicate(tc) {
+			out = append(out, tc)
+		}
+	}
+	return out
+}
+
+// AlphabetizedTestCases implements Sort over the list of testCases we have.s
+type AlphabetizedTestCases []reporters.JUnitTestCase
+
+func (a AlphabetizedTestCases) Len() int           { return len(a) }
+func (a AlphabetizedTestCases) Less(i, j int) bool { return a[i].Name < a[j].Name }
+func (a AlphabetizedTestCases) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+
+// predicate functions
+
+// Skipped returns true if the test was skipped.
+func Skipped(testCase reporters.JUnitTestCase) bool { return testCase.Skipped != nil }
+
+// Passed returns true if the test passed.
+func Passed(testCase reporters.JUnitTestCase) bool {
+	return testCase.Skipped == nil && testCase.FailureMessage == nil
+}
+
+// Failed returns true if the test failed.
+func Failed(testCase reporters.JUnitTestCase) bool {
+	return testCase.Skipped == nil && testCase.FailureMessage != nil
+}

pkg/results/reader.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2018 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package results
 
 import (
@@ -16,17 +32,18 @@ import (
 )
 
 const (
+	// PluginsDir defines where in the archive the plugin results are.
+	PluginsDir = "plugins/"
+
 	hostsDir                  = "hosts/"
 	namespacedResourcesDir    = "resources/ns/"
 	nonNamespacedResourcesDir = "resources/cluster/"
-	pluginsDir                = "plugins/"
 	podLogs                   = "podlogs/"
 	metadataDir               = "meta/"
-
-	defaultServicesFile      = "Services.json"
-	defaultNodesFile         = "Nodes.json"
-	defaultServerVersionFile = "serverversion.json"
-	defaultServerGroupsFile  = "servergroups.json"
+	defaultServicesFile       = "Services.json"
+	defaultNodesFile          = "Nodes.json"
+	defaultServerVersionFile  = "serverversion.json"
+	defaultServerGroupsFile   = "servergroups.json"
 )
 
 const (