duffle export optionally to produce OCI image layout

Provides some basic digest verification. See
#408.

Fixes #715

cmd/duffle/export.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/pkg/errors"
+
 	"github.com/deislabs/duffle/pkg/duffle/home"
 	"github.com/deislabs/duffle/pkg/loader"
 	"github.com/deislabs/duffle/pkg/packager"
@@ -12,20 +14,26 @@ import (
 )
 
 const exportDesc = `
-Packages a bundle, invocation images, and all referenced images within
-a single gzipped tarfile. All images specified in the bundle metadata
-are saved as tar files in the artifacts/ directory along with an 
-artifacts.json file which describes the contents of artifacts/ directory.
+Packages a bundle, and by default any images referenced by the bundle, within
+a single gzipped tarfile.
+
+If neither --oci-layout nor --thin is specified, all images (incuding invocation
+images) referenced by the bundle metadata are saved as tar files in the
+artifacts/ directory along with an artifacts.json file which describes the
+contents of the artifacts/ directory.
+
+If --oci-layout is specified, all images (incuding invocation images) referenced
+by the bundle metadata are saved as an OCI image layout in the artifacts/layout/
+directory.
+
+If --thin specified, only the bundle manifest is exported.
 
 By default, this command will use the name and version information of
 the bundle to create a compressed archive file called
 <name>-<version>.tgz in the current directory. This destination can be
 updated by specifying a file path to save the compressed bundle to using
 the --output-file flag.
 
-Use the --thin flag to export the bundle manifest without the invocation
-images and referenced images.
-
 Pass in a path to a bundle file instead of a bundle in local storage by
 using the --bundle-is-file flag like below:
 $ duffle export [PATH] --bundle-is-file
@@ -39,6 +47,7 @@ type exportCmd struct {
 	thin         bool
 	verbose      bool
 	bundleIsFile bool
+	ociLayout    bool
 }
 
 func newExportCmd(w io.Writer) *cobra.Command {
@@ -52,6 +61,9 @@ func newExportCmd(w io.Writer) *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			export.home = home.Home(homePath())
 			export.bundle = args[0]
+			if export.thin && export.ociLayout {
+				return errors.New("--thin and --oci-layout must not both be specified")
+			}
 
 			return export.run()
 		},
@@ -60,6 +72,7 @@ func newExportCmd(w io.Writer) *cobra.Command {
 	f := cmd.Flags()
 	f.StringVarP(&export.dest, "output-file", "o", "", "Save exported bundle to file path")
 	f.BoolVarP(&export.bundleIsFile, "bundle-is-file", "f", false, "Indicates that the bundle source is a file path")
+	f.BoolVarP(&export.ociLayout, "oci-layout", "l", false, "Export images as an OCI image layout")
 	f.BoolVarP(&export.thin, "thin", "t", false, "Export only the bundle manifest")
 	f.BoolVarP(&export.verbose, "verbose", "v", false, "Verbose output")
 
@@ -79,7 +92,12 @@ func (ex *exportCmd) run() error {
 }
 
 func (ex *exportCmd) Export(bundlefile string, l loader.BundleLoader) error {
-	exp, err := packager.NewExporter(bundlefile, ex.dest, ex.home.Logs(), l, ex.thin)
+	is, err := packager.NewImageStore(ex.thin, ex.ociLayout)
+	if err != nil {
+		return err
+	}
+
+	exp, err := packager.NewExporter(bundlefile, ex.dest, ex.home.Logs(), l, is)
 	if err != nil {
 		return fmt.Errorf("Unable to set up exporter: %s", err)
 	}

pkg/packager/export.go

@@ -1,17 +1,13 @@
 package packager
 
 import (
-	"context"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"github.com/deislabs/cnab-go/bundle"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/archive"
 
 	"github.com/deislabs/duffle/pkg/loader"
@@ -20,9 +16,7 @@ import (
 type Exporter struct {
 	Source      string
 	Destination string
-	Thin        bool
-	Client      *client.Client
-	Context     context.Context
+	ImageStore  ImageStore
 	Logs        string
 	Loader      loader.BundleLoader
 }
@@ -31,34 +25,40 @@ type Exporter struct {
 //  lives, where the compressed bundle should be exported to,
 //  and what form a bundle should be exported in (thin or thick/full). It also
 //  sets up a docker client to work with images.
-func NewExporter(source, dest, logsDir string, l loader.BundleLoader, thin bool) (*Exporter, error) {
-	cli, err := client.NewClientWithOpts(client.FromEnv)
-	if err != nil {
-		return nil, err
-	}
-	ctx := context.Background()
-	cli.NegotiateAPIVersion(ctx)
-
+func NewExporter(source, dest, logsDir string, l loader.BundleLoader, is ImageStore) (*Exporter, error) {
 	logs := filepath.Join(logsDir, "export-"+time.Now().Format("20060102150405"))
 
 	return &Exporter{
 		Source:      source,
 		Destination: dest,
-		Thin:        thin,
-		Client:      cli,
-		Context:     ctx,
+		ImageStore:  is,
 		Logs:        logs,
 		Loader:      l,
 	}, nil
 }
 
+type ImageStore interface {
+	configure(archiveDir string, logs io.Writer) error
+	add(img string) (contentDigest string, err error)
+}
+
+func NewImageStore(thin bool, ociLayout bool) (ImageStore, error) {
+	if thin {
+		return newNop(), nil
+	}
+	if ociLayout {
+		return newOciLayout(), nil
+	}
+	return newTarFiles()
+
+}
+
 // Export prepares an artifacts directory containing all of the necessary
 //  images, packages the bundle along with the artifacts in a gzipped tar
 //  file, and saves that file to the file path specified as destination.
 //  If the any part of the destination path doesn't, it will be created.
 //  exist
 func (ex *Exporter) Export() error {
-
 	//prepare log file for this export
 	logsf, err := os.Create(ex.Logs)
 	if err != nil {
@@ -106,10 +106,11 @@ func (ex *Exporter) Export() error {
 		return err
 	}
 
-	if !ex.Thin {
-		if err := ex.prepareArtifacts(bun, archiveDir, logsf); err != nil {
-			return fmt.Errorf("Error preparing artifacts: %s", err)
-		}
+	if err := ex.ImageStore.configure(archiveDir, logsf); err != nil {
+		return fmt.Errorf("Error creating artifacts: %s", err)
+	}
+	if err := ex.prepareArtifacts(bun); err != nil {
+		return fmt.Errorf("Error preparing artifacts: %s", err)
 	}
 
 	dest := name + ".tgz"
@@ -139,64 +140,42 @@ func (ex *Exporter) Export() error {
 	return err
 }
 
-// prepareArtifacts pulls all images, verifies their digests (TODO: verify digest) and
-//  saves them to a directory called artifacts/ in the bundle directory
-func (ex *Exporter) prepareArtifacts(bun *bundle.Bundle, archiveDir string, logs io.Writer) error {
-	artifactsDir := filepath.Join(archiveDir, "artifacts")
-	if err := os.MkdirAll(artifactsDir, 0755); err != nil {
-		return err
-	}
-
+// prepareArtifacts pulls all images, verifies their digests and
+// saves them to a directory called artifacts/ in the bundle directory
+func (ex *Exporter) prepareArtifacts(bun *bundle.Bundle) error {
 	for _, image := range bun.Images {
-		_, err := ex.archiveImage(image.Image, artifactsDir, logs)
-		if err != nil {
+		if err := ex.addImage(image.BaseImage); err != nil {
 			return err
 		}
 	}
 
 	for _, in := range bun.InvocationImages {
-		_, err := ex.archiveImage(in.Image, artifactsDir, logs)
-		if err != nil {
+		if err := ex.addImage(in.BaseImage); err != nil {
 			return err
 		}
-
 	}
 
 	return nil
 }
 
-func (ex *Exporter) archiveImage(image, artifactsDir string, logs io.Writer) (string, error) {
-	ctx := ex.Context
-
-	imagePullOptions := types.ImagePullOptions{} //TODO: add platform info
-	pullLogs, err := ex.Client.ImagePull(ctx, image, imagePullOptions)
+// addImage pulls an image, adds it to the artifacts/ directory, and verifies its digest
+func (ex *Exporter) addImage(image bundle.BaseImage) error {
+	dig, err := ex.ImageStore.add(image.Image)
 	if err != nil {
-		return "", fmt.Errorf("Error pulling image %s: %s", image, err)
-	}
-	defer pullLogs.Close()
-	io.Copy(logs, pullLogs)
-
-	reader, err := ex.Client.ImageSave(ctx, []string{image})
-	if err != nil {
-		return "", err
+		return err
 	}
-	defer reader.Close()
+	return checkDigest(image, dig)
+}
 
-	name := buildFileName(image) + ".tar"
-	out, err := os.Create(filepath.Join(artifactsDir, name))
-	if err != nil {
-		return name, err
+// checkDigest compares the content digest of the given image to the given content digest and returns an error if they
+// are both non-empty and do not match
+func checkDigest(image bundle.BaseImage, dig string) error {
+	digestFromManifest := image.Digest
+	if dig == "" || digestFromManifest == "" {
+		return nil
 	}
-	defer out.Close()
-	if _, err := io.Copy(out, reader); err != nil {
-		return name, err
+	if digestFromManifest != dig {
+		return fmt.Errorf("content digest mismatch: image %s has digest %s but the digest should be %s according to the bundle manifest", image.Image, dig, digestFromManifest)
 	}
-
-	return name, nil
-}
-
-func buildFileName(uri string) string {
-	filename := strings.Replace(uri, "/", "-", -1)
-	return strings.Replace(filename, ":", "-", -1)
-
+	return nil
 }