Merge pull request #1782 from gunjan5/offline-migration-tool

Add calicoctl convert command for manifest offline conversions

calicoctl/calicoctl.go

@@ -36,6 +36,7 @@ func main() {
               name.
     get       Get a resource identified by file, stdin or resource type and
               name.
+    convert   Convert config files between different API versions.
     ipam      IP address management.
     node      Calico node management.
     version   Display the version of calicoctl.
@@ -81,6 +82,8 @@ Description:
 			commands.Delete(args)
 		case "get":
 			commands.Get(args)
+		case "convert":
+			commands.Convert(args)
 		case "version":
 			commands.Version(args)
 		case "node":

calicoctl/commands/convert.go

@@ -0,0 +1,176 @@
+// Copyright (c) 2017 Tigera, Inc. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package commands
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/docopt/docopt-go"
+	log "github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+
+	"github.com/projectcalico/calicoctl/calicoctl/commands/argutils"
+	"github.com/projectcalico/calicoctl/calicoctl/commands/constants"
+	"github.com/projectcalico/calicoctl/calicoctl/commands/v1resourceloader"
+	"github.com/projectcalico/libcalico-go/lib/apis/v1/unversioned"
+	conversion "github.com/projectcalico/libcalico-go/lib/upgrade/etcd/conversionv1v3"
+	validator "github.com/projectcalico/libcalico-go/lib/validator/v3"
+)
+
+func Convert(args []string) {
+	doc := constants.DatastoreIntro + `Usage:
+  calicoctl convert --filename=<FILENAME>
+                [--output=<OUTPUT>] [--ignore-validation]
+
+Examples:
+  # Convert the contents of policy.yaml to v3 policy.
+  calicoctl convert -f ./policy.yaml -o yaml
+
+  # Convert a policy based on the JSON passed into stdin.
+  cat policy.json | calicoctl convert -f -
+
+Options:
+  -h --help                     Show this screen.
+  -f --filename=<FILENAME>      Filename to use to create the resource. If set to
+                                "-" loads from stdin.
+  -o --output=<OUTPUT FORMAT>   Output format. One of: yaml or json.
+                                [Default: yaml]
+  --ignore-validation           Skip validation on the converted manifest.
+
+
+Description:
+  Convert config files from Calico v1 to v3 API versions. Both YAML and JSON formats are accepted.
+
+  The default output will be printed to stdout in YAML format.
+`
+	parsedArgs, err := docopt.Parse(doc, args, true, "", false, false)
+	if err != nil {
+		fmt.Printf("Invalid option: 'calicoctl %s'. Use flag '--help' to read about a specific subcommand.\n", strings.Join(args, " "))
+		os.Exit(1)
+	}
+	if len(parsedArgs) == 0 {
+		return
+	}
+
+	var rp resourcePrinter
+	output := parsedArgs["--output"].(string)
+	// Only supported output formats are yaml (default) and json.
+	switch output {
+	case "yaml", "yml":
+		rp = resourcePrinterYAML{}
+	case "json":
+		rp = resourcePrinterJSON{}
+	default:
+		fmt.Printf("unrecognized output format '%s'\n", output)
+		os.Exit(1)
+	}
+
+	filename := argutils.ArgStringOrBlank(parsedArgs, "--filename")
+
+	// Load the V1 resource from file and convert to a slice
+	// of resources for easier handling.
+	resV1, err := v1resourceloader.CreateResourcesFromFile(filename)
+	if err != nil {
+		fmt.Printf("Failed to execute command: %v\n", err)
+		os.Exit(1)
+	}
+
+	var results []runtime.Object
+	for _, v1Resource := range resV1 {
+		v3Resource, err := convertResource(v1Resource)
+		if err != nil {
+			fmt.Printf("Failed to execute command: %v\n", err)
+			os.Exit(1)
+		}
+
+		// Remove any extra metadata the object might have.
+		rom := v3Resource.(v1.ObjectMetaAccessor).GetObjectMeta()
+		rom.SetNamespace("")
+		rom.SetUID("")
+		rom.SetResourceVersion("")
+		rom.SetCreationTimestamp(v1.Time{})
+		rom.SetDeletionTimestamp(nil)
+		rom.SetDeletionGracePeriodSeconds(nil)
+		rom.SetClusterName("")
+
+		ignoreValidation := argutils.ArgBoolOrFalse(parsedArgs, "--ignore-validation")
+		if !ignoreValidation {
+			if err := validator.Validate(v3Resource); err != nil {
+				fmt.Printf("Converted manifest resource(s) failed validation: %s\n", err)
+				fmt.Printf("Re-run the command with '--ignore-validation' flag to see the converted output.\n")
+				os.Exit(1)
+			}
+		}
+
+		results = append(results, v3Resource)
+	}
+
+	log.Infof("results: %+v", results)
+
+	err = rp.print(nil, results)
+	if err != nil {
+		fmt.Printf("Failed to execute command: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+// convertResource converts v1 resource into a v3 resource.
+func convertResource(v1resource unversioned.Resource) (conversion.Resource, error) {
+	// Get the type converter for the v1 resource.
+	convRes, err := getTypeConverter(v1resource.GetTypeMetadata().Kind)
+	if err != nil {
+		return nil, err
+	}
+
+	// Convert v1 API resource to v1 backend KVPair.
+	kvp, err := convRes.APIV1ToBackendV1(v1resource)
+	if err != nil {
+		return nil, err
+	}
+
+	// Convert v1 backend KVPair to v3 API resource.
+	res, err := convRes.BackendV1ToAPIV3(kvp)
+	if err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}
+
+// getTypeConverter returns a type specific converter for a given v1 resource.
+func getTypeConverter(resKind string) (conversion.Converter, error) {
+	switch strings.ToLower(resKind) {
+	case "node":
+		return conversion.Node{}, nil
+	case "hostendpoint":
+		return conversion.HostEndpoint{}, nil
+	case "workloadendpoint":
+		return conversion.WorkloadEndpoint{}, nil
+	case "profile":
+		return conversion.Profile{}, nil
+	case "policy":
+		return conversion.Policy{}, nil
+	case "ippool":
+		return conversion.IPPool{}, nil
+	case "bgppeer":
+		return conversion.BGPPeer{}, nil
+
+	default:
+		return nil, fmt.Errorf("conversion for the resource type '%s' is not supported", resKind)
+	}
+}

calicoctl/commands/v1resourceloader/v1resourceloader.go

@@ -0,0 +1,202 @@
+// Copyright (c) 2017 Tigera, Inc. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1resourceloader
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"reflect"
+
+	log "github.com/sirupsen/logrus"
+
+	yamlsep "github.com/projectcalico/calicoctl/calicoctl/util/yaml"
+	"github.com/projectcalico/go-yaml-wrapper"
+	apiv1 "github.com/projectcalico/libcalico-go/lib/apis/v1"
+	"github.com/projectcalico/libcalico-go/lib/apis/v1/unversioned"
+	v1validator "github.com/projectcalico/libcalico-go/lib/validator/v1"
+)
+
+// Store a resourceHelper for each resource unversioned.TypeMetadata.
+var resourceToType map[unversioned.TypeMetadata]reflect.Type
+
+func init() {
+	resourceToType = make(map[unversioned.TypeMetadata]reflect.Type)
+	populateResourceTypes()
+}
+
+// populateResourceTypes register all the V1 resource types in the resourceToType map.
+func populateResourceTypes() {
+	resTypes := []unversioned.Resource{
+		apiv1.NewBGPPeer(),
+		apiv1.NewIPPool(),
+		apiv1.NewHostEndpoint(),
+		apiv1.NewNode(),
+		apiv1.NewPolicy(),
+		apiv1.NewProfile(),
+		apiv1.NewWorkloadEndpoint(),
+	}
+
+	for _, rt := range resTypes {
+		resourceToType[rt.GetTypeMetadata()] = reflect.ValueOf(rt).Elem().Type()
+	}
+}
+
+// Create a new concrete resource structure based on the type.  If the type is
+// a list, this creates a concrete Resource-List of the required type.
+func newResource(tm unversioned.TypeMetadata) (unversioned.Resource, error) {
+	rh, ok := resourceToType[tm]
+	if !ok {
+		return nil, errors.New(fmt.Sprintf("Unknown resource type (%s) and/or version (%s)", tm.Kind, tm.APIVersion))
+	}
+	log.Debugf("Found resource helper: %s", rh)
+
+	// Create new resource and fill in the type metadata.
+	new := reflect.New(rh)
+	elem := new.Elem()
+	elem.FieldByName("Kind").SetString(tm.GetTypeMetadata().Kind)
+	elem.FieldByName("APIVersion").SetString(tm.GetTypeMetadata().APIVersion)
+
+	return new.Interface().(unversioned.Resource), nil
+}
+
+// Create the resource from the specified byte array encapsulating the resource.
+// -  The byte array may be JSON or YAML encoding of either a single resource or list of
+//    resources as defined by the API objects in /api.
+//
+// The returned Resource will either be a single resource document or a List of documents.
+// If the file does not contain any valid Resources this function returns an error.
+func createResourcesFromBytes(b []byte) ([]unversioned.Resource, error) {
+	// Start by unmarshalling the bytes into a TypeMetadata structure - this will ignore
+	// other fields.
+	var err error
+	tm := unversioned.TypeMetadata{}
+	tms := []unversioned.TypeMetadata{}
+	if err = yaml.Unmarshal(b, &tm); err == nil {
+		// We processed a metadata, so create a concrete resource struct to unpack
+		// into.
+		return unmarshalResource(tm, b)
+	} else if err = yaml.Unmarshal(b, &tms); err == nil {
+		// We processed a slice of metadata's, create a list of concrete resource
+		// structs to unpack into.
+		return unmarshalSliceOfResources(tms, b)
+	} else {
+		// Failed to parse a single resource or list of resources.
+		return nil, err
+	}
+}
+
+// Unmarshal a bytearray containing a single resource of the specified type into
+// a concrete structure for that resource type.
+//
+// Return as a slice of Resource interfaces, containing a single element that is
+// the unmarshalled resource.
+func unmarshalResource(tm unversioned.TypeMetadata, b []byte) ([]unversioned.Resource, error) {
+	log.Infof("Processing type %s", tm.Kind)
+	unpacked, err := newResource(tm)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = yaml.UnmarshalStrict(b, unpacked); err != nil {
+		return nil, err
+	}
+
+	log.Infof("Type of unpacked data: %v", reflect.TypeOf(unpacked))
+	if err = v1validator.Validate(unpacked); err != nil {
+		return nil, err
+	}
+
+	log.Infof("Unpacked: %+v", unpacked)
+
+	return []unversioned.Resource{unpacked}, nil
+}
+
+// Unmarshal a bytearray containing a list of resources of the specified types into
+// a slice of concrete structures for those resource types.
+//
+// Return as a slice of Resource interfaces, containing an element that is each of
+// the unmarshalled resources.
+func unmarshalSliceOfResources(tml []unversioned.TypeMetadata, b []byte) ([]unversioned.Resource, error) {
+	log.Infof("Processing list of resources")
+	unpacked := make([]unversioned.Resource, len(tml))
+	for i, tm := range tml {
+		log.Infof("  - processing type %s", tm.Kind)
+		r, err := newResource(tm)
+		if err != nil {
+			return nil, err
+		}
+		unpacked[i] = r
+	}
+
+	if err := yaml.UnmarshalStrict(b, &unpacked); err != nil {
+		return nil, err
+	}
+
+	// Validate the data in the structures.  The v1validator does not handle slices, so
+	// validate each resource separately.
+	for _, r := range unpacked {
+		if err := v1validator.Validate(r); err != nil {
+			return nil, err
+		}
+	}
+
+	log.Infof("Unpacked: %+v", unpacked)
+
+	return unpacked, nil
+}
+
+// Create the Resource from the specified file f.
+// 	-  The file format may be JSON or YAML encoding of either a single resource or list of
+// 	   resources as defined by the API objects in /api.
+// 	-  A filename of "-" means "Read from stdin".
+//
+// The returned Resource will either be a single Resource or a List containing zero or more
+// Resources.  If the file does not contain any valid Resources this function returns an error.
+func CreateResourcesFromFile(f string) ([]unversioned.Resource, error) {
+	// Load the bytes from file or from stdin.
+	var reader io.Reader
+	var err error
+	if f == "-" {
+		reader = os.Stdin
+	} else {
+		reader, err = os.Open(f)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	var resources []unversioned.Resource
+	separator := yamlsep.NewYAMLDocumentSeparator(reader)
+	for {
+		b, err := separator.Next()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+
+		r, err := createResourcesFromBytes(b)
+		if err != nil {
+			return nil, err
+		}
+
+		resources = append(resources, r...)
+	}
+
+	return resources, nil
+}