Added ability for calicoctl to read multiple yaml documents in the sa…

…me file

calicoctl/resourcemgr/resourcemgr.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2016 Tigera, Inc. All rights reserved.
+// Copyright (c) 2017 Tigera, Inc. All rights reserved.
 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,21 +15,18 @@
 package resourcemgr
 
 import (
+	"bytes"
 	"errors"
-
 	"fmt"
+	"io"
+	"os"
 	"reflect"
 	"strings"
 
-	"io/ioutil"
-	"os"
-
-	"github.com/projectcalico/libcalico-go/lib/api/unversioned"
-
-	"bytes"
-
 	log "github.com/Sirupsen/logrus"
+	yamlsep "github.com/projectcalico/calicoctl/calicoctl/util/yaml"
 	"github.com/projectcalico/go-yaml-wrapper"
+	"github.com/projectcalico/libcalico-go/lib/api/unversioned"
 	"github.com/projectcalico/libcalico-go/lib/client"
 	"github.com/projectcalico/libcalico-go/lib/validator"
 )
@@ -57,7 +54,7 @@ type ResourceActionCommand func(*client.Client, unversioned.Resource) (unversion
 //	-  Template strings used to format output for each resource type.
 //	-  Functions to handle resource management actions (apply, create, update, delete, list).
 //         These functions are an untyped interface (generic Resource interfaces) that map through
-//         to the Calicc clients typed interface.
+//         to the Calico clients typed interface.
 type resourceHelper struct {
 	typeMetadata      unversioned.TypeMetadata
 	resourceType      reflect.Type
@@ -228,19 +225,37 @@ func unmarshalSliceOfResources(tml []unversioned.TypeMetadata, b []byte) ([]unve
 // Resources.  If the file does not contain any valid Resources this function returns an error.
 func CreateResourcesFromFile(f string) ([]unversioned.Resource, error) {
 	// Load the bytes from file or from stdin.
-	var b []byte
+	var reader io.Reader
 	var err error
-
 	if f == "-" {
-		b, err = ioutil.ReadAll(os.Stdin)
+		reader = os.Stdin
 	} else {
-		b, err = ioutil.ReadFile(f)
+		reader, err = os.Open(f)
 	}
 	if err != nil {
 		return nil, err
 	}
 
-	return createResourcesFromBytes(b)
+	var resources []unversioned.Resource
+	separator := yamlsep.NewYAMLDocumentSeparator(reader)
+	for {
+		b, err := separator.Next()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+
+		r, err := createResourcesFromBytes(b)
+		if err != nil {
+			return nil, err
+		}
+
+		resources = append(resources, r...)
+	}
+
+	return resources, nil
 }
 
 // Implement the ResourceManager interface on the resourceHelper struct.

calicoctl/util/yaml/separator.go

@@ -0,0 +1,83 @@
+// Copyright (c) 2017 Tigera, Inc. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package yaml
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+)
+
+const yamlSeparator = "\n---"
+
+type Separator interface {
+	Next() ([]byte, error)
+}
+
+type YAMLSeparator struct {
+	scanner *bufio.Scanner
+}
+
+func NewYAMLDocumentSeparator(reader io.Reader) Separator {
+	scanner := bufio.NewScanner(reader)
+	scanner.Split(splitYAMLDocument)
+	return &YAMLSeparator{
+		scanner: scanner,
+	}
+}
+
+// Copyright 2014 The Kubernetes Authors.
+// Function originally from:
+// https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder.go .
+func splitYAMLDocument(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	if atEOF && len(data) == 0 {
+		return 0, nil, nil
+	}
+	sep := len([]byte(yamlSeparator))
+	if i := bytes.Index(data, []byte(yamlSeparator)); i >= 0 {
+		// We have a potential document terminator
+		i += sep
+		after := data[i:]
+		if len(after) == 0 {
+			// we can't read any more characters
+			if atEOF {
+				return len(data), data[:len(data)-sep], nil
+			}
+			return 0, nil, nil
+		}
+		if j := bytes.IndexByte(after, '\n'); j >= 0 {
+			return i + j + 1, data[0 : i-sep], nil
+		}
+		return 0, nil, nil
+	}
+	// If we're at EOF, we have a final, non-terminated line. Return it.
+	if atEOF {
+		return len(data), data, nil
+	}
+	// Request more data.
+	return 0, nil, nil
+}
+
+func (t *YAMLSeparator) Next() ([]byte, error) {
+	if !t.scanner.Scan() {
+		err := t.scanner.Err()
+		if err == nil {
+			err = io.EOF
+		}
+		return []byte{}, err
+	}
+
+	return t.scanner.Bytes(), nil
+}

calicoctl/util/yaml/separator_test.go

@@ -0,0 +1,121 @@
+// Copyright (c) 2017 Tigera, Inc. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package yaml
+
+import (
+	"bytes"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// Copyright 2014 The Kubernetes Authors.
+// The test cases were originally written in:
+// https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder_test.go
+// and modified to use the Ginkgo testing framework
+var _ = Describe("Test splitYAMLDocument", func() {
+	It("should not split up full documents without separators", func() {
+		ValidateSplitYAMLDocument("foo", true, "foo", 3)
+	})
+
+	It("should not return anything without a separator and not at the EOF", func() {
+		ValidateSplitYAMLDocument("fo", false, "", 0)
+	})
+
+	It("should return part of YAML separator at EOF", func() {
+		ValidateSplitYAMLDocument("---", true, "---", 3)
+	})
+
+	It("should return something similar to YAML separator with newline at EOF", func() {
+		ValidateSplitYAMLDocument("---\n", true, "---\n", 4)
+	})
+
+	It("should not return something similar to YAML separator if not at EOF", func() {
+		ValidateSplitYAMLDocument("---\n", false, "", 0)
+	})
+
+	It("should not return yaml separator before EOF but advance the bytes read", func() {
+		ValidateSplitYAMLDocument("\n---\n", false, "", 5)
+	})
+
+	It("should not return yaml separator at EOF but advance the bytes read", func() {
+		ValidateSplitYAMLDocument("\n---\n", true, "", 5)
+	})
+
+	It("should split out and read the first document in multiple documents", func() {
+		ValidateSplitYAMLDocument("abc\n---\ndef", true, "abc", 8)
+	})
+
+	It("should read the rest of the multiple documents at the EOF", func() {
+		ValidateSplitYAMLDocument("def", true, "def", 3)
+	})
+
+	It("should read nothing from an empty file", func() {
+		ValidateSplitYAMLDocument("", true, "", 0)
+	})
+})
+
+func ValidateSplitYAMLDocument(input string, atEOF bool, expect string, advAmt int) {
+	adv, token, err := splitYAMLDocument([]byte(input), atEOF)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(expect).To(Equal(string(token)))
+	Expect(advAmt).To(Equal(adv))
+}
+
+var _ = Describe("Test YAML Separator Next", func() {
+	Context("with 2 YAML documents in one file", func() {
+		reader := bytes.NewReader([]byte(testYAMLDocFull))
+		separator := NewYAMLDocumentSeparator(reader)
+
+		var doc []byte
+		var err error
+		BeforeEach(func() {
+			doc, err = separator.Next()
+		})
+
+		It("should correctly separate the first document and return it", func() {
+			Expect(err).NotTo(HaveOccurred())
+			Expect(bytes.Equal([]byte(testYAMLDoc1), doc)).To(Equal(true))
+		})
+
+		It("should correctly separate the second document and return it", func() {
+			Expect(err).NotTo(HaveOccurred())
+			Expect(bytes.Equal([]byte(testYAMLDoc2), doc)).To(Equal(true))
+		})
+
+		It("should return an EOF error when there is nothing left", func() {
+			Expect(err).To(HaveOccurred())
+			Expect(len(doc)).To(Equal(0))
+		})
+	})
+})
+
+const testYAMLDoc1 = `
+a: TestFieldA
+b: TestFieldB
+c:
+  c1: TestFieldC1
+  c2: TestFieldC2`
+
+const testYAMLDoc2 = `
+d: TestFieldD
+e: TestFieldE
+f:
+  - name: TestFieldFListItem1
+  - name: TestFieldFListItem2`
+
+const testYAMLDocFull = testYAMLDoc1 + `
+---
+` + testYAMLDoc2

calicoctl/util/yaml/yaml_suite_test.go

@@ -0,0 +1,13 @@
+package yaml_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"testing"
+)
+
+func TestYaml(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Yaml Suite")
+}