Load pipeline on yaml tree vs go structs

cli/options.go

@@ -52,7 +52,7 @@ type ProjectOptions struct {
 
 	// ConfigPaths are file paths to one or more Compose files.
 	//
-	// These are applied in order by the loader following the merge logic
+	// These are applied in order by the loader following the override logic
 	// as described in the spec.
 	//
 	// The first entry is required and is the primary Compose file.

consts/consts.go

@@ -22,3 +22,7 @@ const (
 	ComposeFilePath      = "COMPOSE_FILE"
 	ComposeProfiles      = "COMPOSE_PROFILES"
 )
+
+const Extensions = "#extensions" // Using # prefix, we prevent risk to conflict with an actual yaml key
+
+type ComposeFileKey struct{}

loader/volume.go → format/volume.go

@@ -14,7 +14,7 @@
    limitations under the License.
 */
 
-package loader
+package format
 
 import (
 	"strings"

loader/volume_test.go → format/volume_test.go

@@ -14,7 +14,7 @@
    limitations under the License.
 */
 
-package loader
+package format
 
 import (
 	"fmt"

loader/extends.go

@@ -0,0 +1,129 @@
+/*
+   Copyright 2020 The Compose Specification Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package loader
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+
+	"github.com/compose-spec/compose-go/consts"
+	"github.com/compose-spec/compose-go/override"
+	"github.com/compose-spec/compose-go/types"
+)
+
+func ApplyExtends(ctx context.Context, dict map[string]any, workingdir string, opts *Options, ct *cycleTracker, post ...PostProcessor) error {
+	a, ok := dict["services"]
+	if !ok {
+		return nil
+	}
+	services := a.(map[string]any)
+	for name, s := range services {
+		service := s.(map[string]any)
+		x, ok := service["extends"]
+		if !ok {
+			continue
+		}
+		if err := ct.Add(ctx.Value(consts.ComposeFileKey{}).(string), name); err != nil {
+			return err
+		}
+		extends := x.(map[string]any)
+		var base any
+		ref := extends["service"].(string)
+		if file, ok := extends["file"]; ok {
+			path := file.(string)
+			for _, loader := range opts.ResourceLoaders {
+				if !loader.Accept(path) {
+					continue
+				}
+				local, err := loader.Load(ctx, path)
+				if err != nil {
+					return err
+				}
+				relworkingdir := filepath.Dir(local)
+				if !filepath.IsAbs(local) {
+					relworkingdir, err = filepath.Rel(workingdir, relworkingdir)
+					if err != nil {
+						return err
+					}
+				}
+				extendsOpts := opts.clone()
+				extendsOpts.ResolvePaths = true
+				extendsOpts.SkipNormalization = true
+				extendsOpts.SkipConsistencyCheck = true
+				extendsOpts.SkipInclude = true
+				source, err := loadYamlModel(ctx, types.ConfigDetails{
+					WorkingDir: relworkingdir,
+					ConfigFiles: []types.ConfigFile{
+						{Filename: local},
+					},
+				}, extendsOpts, ct, nil)
+				if err != nil {
+					return err
+				}
+				services := source["services"].(map[string]any)
+				base, ok = services[ref]
+				if !ok {
+					return fmt.Errorf("cannot extend service %q in %s: service not found", name, path)
+				}
+			}
+			if base == nil {
+				return fmt.Errorf("cannot read %s", path)
+			}
+		} else {
+			base, ok = services[ref]
+			if !ok {
+				return fmt.Errorf("cannot extend service %q in %s: service not found", name, "filename") // TODO track filename
+			}
+		}
+		source := deepClone(base).(map[string]any)
+		for _, processor := range post {
+			processor.Apply(map[string]any{
+				"services": map[string]any{
+					name: source,
+				},
+			})
+		}
+		merged, err := override.ExtendService(source, service)
+		if err != nil {
+			return err
+		}
+		delete(merged, "extends")
+		services[name] = merged
+	}
+	dict["services"] = services
+	return nil
+}
+
+func deepClone(value any) any {
+	switch v := value.(type) {
+	case []any:
+		cp := make([]any, len(v))
+		for i, e := range v {
+			cp[i] = deepClone(e)
+		}
+		return cp
+	case map[string]any:
+		cp := make(map[string]any, len(v))
+		for k, e := range v {
+			cp[k] = deepClone(e)
+		}
+		return cp
+	default:
+		return value
+	}
+}

loader/fix.go

@@ -0,0 +1,36 @@
+/*
+   Copyright 2020 The Compose Specification Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package loader
+
+// fixEmptyNotNull is a workaround for https://github.com/xeipuuv/gojsonschema/issues/141
+// as go-yaml `[]` will load as a `[]any(nil)`, which is not the same as an empty array
+func fixEmptyNotNull(value any) interface{} {
+	switch v := value.(type) {
+	case []any:
+		if v == nil {
+			return []any{}
+		}
+		for i, e := range v {
+			v[i] = fixEmptyNotNull(e)
+		}
+	case map[string]any:
+		for k, e := range v {
+			v[k] = fixEmptyNotNull(e)
+		}
+	}
+	return value
+}

loader/full-example.yml

@@ -295,22 +295,22 @@ services:
 
     volumes:
       # Just specify a path and let the Engine create a volume
-      - /var/lib/mysql
+      - /var/lib/anonymous
       # Specify an absolute path mapping
-      - /opt/data:/var/lib/mysql
+      - /opt/data:/var/lib/data
       # Path on the host, relative to the Compose file
       - .:/code
       - ./static:/var/www/html
       # User-relative path
       - ~/configs:/etc/configs:ro
       # Named volume
-      - datavolume:/var/lib/mysql
+      - datavolume:/var/lib/volume
       - type: bind
         source: ./opt
-        target: /opt
+        target: /opt/cached
         consistency: cached
       - type: tmpfs
-        target: /opt
+        target: /opt/tmpfs
         tmpfs:
           size: 10000
 

loader/full-struct_test.go

@@ -423,14 +423,14 @@ func services(workingDir, homeDir string) []types.ServiceConfig {
 			},
 			User: "someone",
 			Volumes: []types.ServiceVolumeConfig{
-				{Target: "/var/lib/mysql", Type: "volume", Volume: &types.ServiceVolumeVolume{}},
-				{Source: "/opt/data", Target: "/var/lib/mysql", Type: "bind", Bind: &types.ServiceVolumeBind{CreateHostPath: true}},
+				{Target: "/var/lib/anonymous", Type: "volume", Volume: &types.ServiceVolumeVolume{}},
+				{Source: "/opt/data", Target: "/var/lib/data", Type: "bind", Bind: &types.ServiceVolumeBind{CreateHostPath: true}},
 				{Source: workingDir, Target: "/code", Type: "bind", Bind: &types.ServiceVolumeBind{CreateHostPath: true}},
 				{Source: filepath.Join(workingDir, "static"), Target: "/var/www/html", Type: "bind", Bind: &types.ServiceVolumeBind{CreateHostPath: true}},
 				{Source: filepath.Join(homeDir, "configs"), Target: "/etc/configs", Type: "bind", ReadOnly: true, Bind: &types.ServiceVolumeBind{CreateHostPath: true}},
-				{Source: "datavolume", Target: "/var/lib/mysql", Type: "volume", Volume: &types.ServiceVolumeVolume{}},
-				{Source: filepath.Join(workingDir, "opt"), Target: "/opt", Consistency: "cached", Type: "bind"},
-				{Target: "/opt", Type: "tmpfs", Tmpfs: &types.ServiceVolumeTmpfs{
+				{Source: "datavolume", Target: "/var/lib/volume", Type: "volume", Volume: &types.ServiceVolumeVolume{}},
+				{Source: filepath.Join(workingDir, "opt"), Target: "/opt/cached", Consistency: "cached", Type: "bind"},
+				{Target: "/opt/tmpfs", Type: "tmpfs", Tmpfs: &types.ServiceVolumeTmpfs{
 					Size: types.UnitBytes(10000),
 				}},
 			},
@@ -908,11 +908,11 @@ services:
     uts: host
     volumes:
       - type: volume
-        target: /var/lib/mysql
+        target: /var/lib/anonymous
         volume: {}
       - type: bind
         source: /opt/data
-        target: /var/lib/mysql
+        target: /var/lib/data
         bind:
           create_host_path: true
       - type: bind
@@ -933,14 +933,14 @@ services:
           create_host_path: true
       - type: volume
         source: datavolume
-        target: /var/lib/mysql
+        target: /var/lib/volume
         volume: {}
       - type: bind
         source: %s
-        target: /opt
+        target: /opt/cached
         consistency: cached
       - type: tmpfs
-        target: /opt
+        target: /opt/tmpfs
         tmpfs:
           size: "10000"
     working_dir: /code
@@ -1586,13 +1586,13 @@ func fullExampleJSON(workingDir, homeDir string) string {
       "volumes": [
         {
           "type": "volume",
-          "target": "/var/lib/mysql",
+          "target": "/var/lib/anonymous",
           "volume": {}
         },
         {
           "type": "bind",
           "source": "/opt/data",
-          "target": "/var/lib/mysql",
+          "target": "/var/lib/data",
           "bind": {
             "create_host_path": true
           }
@@ -1625,18 +1625,18 @@ func fullExampleJSON(workingDir, homeDir string) string {
         {
           "type": "volume",
           "source": "datavolume",
-          "target": "/var/lib/mysql",
+          "target": "/var/lib/volume",
           "volume": {}
         },
         {
           "type": "bind",
           "source": "%s",
-          "target": "/opt",
+          "target": "/opt/cached",
           "consistency": "cached"
         },
         {
           "type": "tmpfs",
-          "target": "/opt",
+          "target": "/opt/tmpfs",
           "tmpfs": {
             "size": "10000"
           }