Merge pull request moby#3968 from sipsma/docker-zstd-apply

compression: register docker zstd stream processor
cpuguy83 · Jul 10, 2023 · 31a9120 · 31a9120
2 parents f21a96c + 6ce499c
commit 31a9120
Show file tree

Hide file tree

Showing 4 changed files with 86 additions and 52 deletions.
diff --git a/client/client_test.go b/client/client_test.go
@@ -3765,66 +3765,95 @@ func testBuildExportZstd(t *testing.T, sb integration.Sandbox) {
 
 func testPullZstdImage(t *testing.T, sb integration.Sandbox) {
 	integration.CheckFeatureCompat(t, sb, integration.FeatureDirectPush)
-	c, err := New(sb.Context(), sb.Address())
-	require.NoError(t, err)
-	defer c.Close()
+	for _, ociMediaTypes := range []bool{true, false} {
+		ociMediaTypes := ociMediaTypes
+		t.Run(t.Name()+fmt.Sprintf("/ociMediaTypes=%t", ociMediaTypes), func(t *testing.T) {
+			c, err := New(sb.Context(), sb.Address())
+			require.NoError(t, err)
+			defer c.Close()
 
-	busybox := llb.Image("busybox:latest")
-	cmd := `sh -e -c "echo -n zstd > data"`
+			busybox := llb.Image("busybox:latest")
+			cmd := `sh -e -c "echo -n zstd > data"`
 
-	st := llb.Scratch()
-	st = busybox.Run(llb.Shlex(cmd), llb.Dir("/wd")).AddMount("/wd", st)
+			st := llb.Scratch()
+			st = busybox.Run(llb.Shlex(cmd), llb.Dir("/wd")).AddMount("/wd", st)
 
-	def, err := st.Marshal(sb.Context())
-	require.NoError(t, err)
+			def, err := st.Marshal(sb.Context())
+			require.NoError(t, err)
 
-	registry, err := sb.NewRegistry()
-	if errors.Is(err, integration.ErrRequirements) {
-		t.Skip(err.Error())
-	}
-	require.NoError(t, err)
+			registry, err := sb.NewRegistry()
+			if errors.Is(err, integration.ErrRequirements) {
+				t.Skip(err.Error())
+			}
+			require.NoError(t, err)
 
-	target := registry + "/buildkit/build/exporter:zstd"
+			target := registry + "/buildkit/build/exporter:zstd"
 
-	_, err = c.Solve(sb.Context(), def, SolveOpt{
-		Exports: []ExportEntry{
-			{
-				Type: ExporterImage,
-				Attrs: map[string]string{
-					"name":        target,
-					"push":        "true",
-					"compression": "zstd",
+			_, err = c.Solve(sb.Context(), def, SolveOpt{
+				Exports: []ExportEntry{
+					{
+						Type: ExporterImage,
+						Attrs: map[string]string{
+							"name":           target,
+							"push":           "true",
+							"compression":    "zstd",
+							"oci-mediatypes": strconv.FormatBool(ociMediaTypes),
+						},
+					},
+				},
+			}, nil)
+			require.NoError(t, err)
 
-					// containerd applier supports only zstd with oci-mediatype.
-					"oci-mediatypes": "true",
+			ensurePruneAll(t, c, sb)
+
+			st = llb.Image(target).File(llb.Copy(llb.Image(target), "/data", "/zdata"))
+			def, err = st.Marshal(sb.Context())
+			require.NoError(t, err)
+
+			destDir := t.TempDir()
+
+			out := filepath.Join(destDir, "out.tar")
+			outW, err := os.Create(out)
+			require.NoError(t, err)
+
+			_, err = c.Solve(sb.Context(), def, SolveOpt{
+				Exports: []ExportEntry{
+					{
+						Type:   ExporterOCI,
+						Output: fixedWriteCloser(outW),
+						Attrs: map[string]string{
+							"oci-mediatypes": strconv.FormatBool(ociMediaTypes),
+						},
+					},
 				},
-			},
-		},
-	}, nil)
-	require.NoError(t, err)
+			}, nil)
+			require.NoError(t, err)
 
-	ensurePruneAll(t, c, sb)
+			dt, err := os.ReadFile(out)
+			require.NoError(t, err)
 
-	st = llb.Scratch().File(llb.Copy(llb.Image(target), "/data", "/zdata"))
+			m, err := testutil.ReadTarToMap(dt, false)
+			require.NoError(t, err)
 
-	def, err = st.Marshal(sb.Context())
-	require.NoError(t, err)
+			var index ocispecs.Index
+			err = json.Unmarshal(m["index.json"].Data, &index)
+			require.NoError(t, err)
 
-	destDir := t.TempDir()
+			var mfst ocispecs.Manifest
+			err = json.Unmarshal(m["blobs/sha256/"+index.Manifests[0].Digest.Hex()].Data, &mfst)
+			require.NoError(t, err)
 
-	_, err = c.Solve(sb.Context(), def, SolveOpt{
-		Exports: []ExportEntry{
-			{
-				Type:      ExporterLocal,
-				OutputDir: destDir,
-			},
-		},
-	}, nil)
-	require.NoError(t, err)
+			firstLayer := mfst.Layers[0]
+			if ociMediaTypes {
+				require.Equal(t, ocispecs.MediaTypeImageLayer+"+zstd", firstLayer.MediaType)
+			} else {
+				require.Equal(t, images.MediaTypeDockerSchema2Layer+".zstd", firstLayer.MediaType)
+			}
 
-	dt, err := os.ReadFile(filepath.Join(destDir, "zdata"))
-	require.NoError(t, err)
-	require.Equal(t, dt, []byte("zstd"))
+			zstdLayerDigest := firstLayer.Digest.Hex()
+			require.Equal(t, m["blobs/sha256/"+zstdLayerDigest].Data[:4], []byte{0x28, 0xb5, 0x2f, 0xfd})
+		})
+	}
 }
 func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
 	integration.CheckFeatureCompat(t, sb, integration.FeatureDirectPush)

diff --git a/util/compression/compression.go b/util/compression/compression.go
@@ -78,7 +78,6 @@ func (c Config) SetLevel(l int) Config {
 
 const (
 	mediaTypeDockerSchema2LayerZstd = images.MediaTypeDockerSchema2Layer + ".zstd"
-	mediaTypeImageLayerZstd         = ocispecs.MediaTypeImageLayer + "+zstd" // unreleased image-spec#790
 )
 
 var Default = Gzip
@@ -104,7 +103,7 @@ func fromMediaType(mediaType string) (Type, error) {
 		return Uncompressed, nil
 	case ocispecs.MediaTypeImageLayerGzip, ocispecs.MediaTypeImageLayerNonDistributableGzip: //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
 		return Gzip, nil
-	case mediaTypeImageLayerZstd, ocispecs.MediaTypeImageLayerNonDistributableZstd: //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
+	case ocispecs.MediaTypeImageLayerZstd, ocispecs.MediaTypeImageLayerNonDistributableZstd: //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
 		return Zstd, nil
 	default:
 		return nil, errors.Errorf("unsupported media type %s", mediaType)
@@ -193,7 +192,7 @@ var toDockerLayerType = map[string]string{
 	images.MediaTypeDockerSchema2LayerForeignGzip:    images.MediaTypeDockerSchema2LayerForeignGzip,
 	ocispecs.MediaTypeImageLayerNonDistributable:     images.MediaTypeDockerSchema2LayerForeign,     //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
 	ocispecs.MediaTypeImageLayerNonDistributableGzip: images.MediaTypeDockerSchema2LayerForeignGzip, //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
-	mediaTypeImageLayerZstd:                          mediaTypeDockerSchema2LayerZstd,
+	ocispecs.MediaTypeImageLayerZstd:                 mediaTypeDockerSchema2LayerZstd,
 	mediaTypeDockerSchema2LayerZstd:                  mediaTypeDockerSchema2LayerZstd,
 }
 
@@ -207,8 +206,8 @@ var toOCILayerType = map[string]string{
 	images.MediaTypeDockerSchema2LayerGzip:           ocispecs.MediaTypeImageLayerGzip,
 	images.MediaTypeDockerSchema2LayerForeign:        ocispecs.MediaTypeImageLayerNonDistributable,     //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
 	images.MediaTypeDockerSchema2LayerForeignGzip:    ocispecs.MediaTypeImageLayerNonDistributableGzip, //nolint:staticcheck // ignore SA1019: Non-distributable layers are deprecated, and not recommended for future use.
-	mediaTypeImageLayerZstd:                          mediaTypeImageLayerZstd,
-	mediaTypeDockerSchema2LayerZstd:                  mediaTypeImageLayerZstd,
+	ocispecs.MediaTypeImageLayerZstd:                 ocispecs.MediaTypeImageLayerZstd,
+	mediaTypeDockerSchema2LayerZstd:                  ocispecs.MediaTypeImageLayerZstd,
 }
 
 func convertLayerMediaType(ctx context.Context, mediaType string, oci bool) string {

diff --git a/util/compression/zstd.go b/util/compression/zstd.go
@@ -47,7 +47,7 @@ func (c zstdType) NeedsForceCompression() bool {
 }
 
 func (c zstdType) MediaType() string {
-	return mediaTypeImageLayerZstd
+	return ocispecs.MediaTypeImageLayerZstd
 }
 
 func (c zstdType) String() string {

diff --git a/util/winlayers/applier.go b/util/winlayers/applier.go
@@ -37,6 +37,12 @@ type winApplier struct {
 }
 
 func (s *winApplier) Apply(ctx context.Context, desc ocispecs.Descriptor, mounts []mount.Mount, opts ...diff.ApplyOpt) (d ocispecs.Descriptor, err error) {
+	// HACK:, containerd doesn't know about vnd.docker.image.rootfs.diff.tar.zstd, but that
+	// media type is compatible w/ the oci type, so just lie and say it's the oci type
+	if desc.MediaType == images.MediaTypeDockerSchema2Layer+".zstd" {
+		desc.MediaType = ocispecs.MediaTypeImageLayerZstd
+	}
+
 	if !hasWindowsLayerMode(ctx) {
 		return s.apply(ctx, desc, mounts, opts...)
 	}