Auto sync with Skaffold

Signed-off-by: David Gageot <[email protected]>

pkg/skaffold/build/buildpacks/build_test.go

@@ -135,7 +135,7 @@ func TestBuild(t *testing.T) {
 				Add("img:latest", "builtImageID")
 			localDocker := docker.NewLocalDaemon(test.api, nil, false, nil)
 
-			builder := NewArtifactBuilder(localDocker, test.pushImages)
+			builder := NewArtifactBuilder(localDocker, test.pushImages, false)
 			_, err := builder.Build(context.Background(), ioutil.Discard, &latest.Artifact{
 				Workspace: ".",
 				ArtifactType: latest.ArtifactType{

pkg/skaffold/build/buildpacks/lifecycle.go

@@ -82,6 +82,10 @@ func (b *Builder) build(ctx context.Context, out io.Writer, a *latest.Artifact,
 		return "", errors.Wrap(err, "unable to evaluate env variables")
 	}
 
+	if b.devMode && a.Sync != nil && len(a.Sync.Infer) > 0 {
+		env = append(env, "EXPERIMENTAL_DEV=1")
+	}
+
 	if err := runPackBuildFunc(ctx, out, pack.BuildOptions{
 		AppPath:  workspace,
 		Builder:  builderImage,

pkg/skaffold/build/buildpacks/metadata.go

@@ -54,3 +54,48 @@ func (b *Builder) findRunImage(ctx context.Context, a *latest.BuildpackArtifact,
 
 	return m.Stack.RunImage.Image, nil
 }
+
+type buildMetadata struct {
+	Bom []bom `json:"bom"`
+}
+
+type bom struct {
+	Metadata bomMetadata `json:"metadata"`
+}
+
+type bomMetadata struct {
+	Sync []sync `json:"skaffold.sync"`
+}
+
+type sync struct {
+	Src   string `json:"src"`
+	Dest  string `json:"dest"`
+	Strip string `json:"strip"`
+}
+
+// $ docker inspect demo/buildpacks | jq -r '.[].Config.Labels["io.buildpacks.build.metadata"] | fromjson.bom[].metadata["skaffold.sync"]'
+func SyncRules(labels map[string]string) ([]*latest.SyncRule, error) {
+	metadataJSON, present := labels["io.buildpacks.build.metadata"]
+	if !present {
+		return nil, nil
+	}
+
+	m := buildMetadata{}
+	if err := json.Unmarshal([]byte(metadataJSON), &m); err != nil {
+		return nil, err
+	}
+
+	var rules []*latest.SyncRule
+
+	for _, b := range m.Bom {
+		for _, sync := range b.Metadata.Sync {
+			rules = append(rules, &latest.SyncRule{
+				Src:   sync.Src,
+				Dest:  sync.Dest,
+				Strip: sync.Strip,
+			})
+		}
+	}
+
+	return rules, nil
+}

pkg/skaffold/build/buildpacks/types.go

@@ -22,12 +22,14 @@ import "github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
 type Builder struct {
 	localDocker docker.LocalDaemon
 	pushImages  bool
+	devMode     bool
 }
 
 // NewArtifactBuilder returns a new buildpack artifact builder
-func NewArtifactBuilder(localDocker docker.LocalDaemon, pushImages bool) *Builder {
+func NewArtifactBuilder(localDocker docker.LocalDaemon, pushImages, devMode bool) *Builder {
 	return &Builder{
 		localDocker: localDocker,
 		pushImages:  pushImages,
+		devMode:     devMode,
 	}
 }

pkg/skaffold/build/cache/cache.go

@@ -50,6 +50,7 @@ type cache struct {
 	insecureRegistries map[string]bool
 	cacheFile          string
 	imagesAreLocal     bool
+	devMode            bool
 }
 
 // DependencyLister fetches a list of dependencies for an artifact
@@ -85,6 +86,7 @@ func NewCache(runCtx *runcontext.RunContext, imagesAreLocal bool, dependencies D
 		insecureRegistries: runCtx.InsecureRegistries,
 		cacheFile:          cacheFile,
 		imagesAreLocal:     imagesAreLocal,
+		devMode:            runCtx.DevMode,
 	}, nil
 }
 

pkg/skaffold/build/cache/hash.go

@@ -41,11 +41,11 @@ var (
 	artifactConfigFunction = artifactConfig
 )
 
-func getHashForArtifact(ctx context.Context, depLister DependencyLister, a *latest.Artifact) (string, error) {
+func getHashForArtifact(ctx context.Context, depLister DependencyLister, a *latest.Artifact, devMode bool) (string, error) {
 	var inputs []string
 
 	// Append the artifact's configuration
-	config, err := artifactConfigFunction(a)
+	config, err := artifactConfigFunction(a, devMode)
 	if err != nil {
 		return "", errors.Wrapf(err, "getting artifact's configuration for %s", a.ImageName)
 	}
@@ -100,12 +100,17 @@ func getHashForArtifact(ctx context.Context, depLister DependencyLister, a *late
 	return hex.EncodeToString(hasher.Sum(nil)), nil
 }
 
-func artifactConfig(a *latest.Artifact) (string, error) {
+// TODO(dgageot): when the buildpacks builder image digest changes, we need to change the hash
+func artifactConfig(a *latest.Artifact, devMode bool) (string, error) {
 	buf, err := json.Marshal(a.ArtifactType)
 	if err != nil {
 		return "", errors.Wrapf(err, "marshalling the artifact's configuration for %s", a.ImageName)
 	}
 
+	if devMode && a.BuildpackArtifact != nil && a.Sync != nil && len(a.Sync.Infer) > 0 {
+		return string(buf) + ".DEV", nil
+	}
+
 	return string(buf), nil
 }
 

pkg/skaffold/build/cache/hash_test.go

@@ -39,7 +39,7 @@ var mockCacheHasher = func(s string) (string, error) {
 	return s, nil
 }
 
-var fakeArtifactConfig = func(a *latest.Artifact) (string, error) {
+var fakeArtifactConfig = func(a *latest.Artifact, _ bool) (string, error) {
 	if a.ArtifactType.DockerArtifact != nil {
 		return "docker/target=" + a.ArtifactType.DockerArtifact.Target, nil
 	}
@@ -51,6 +51,7 @@ func TestGetHashForArtifact(t *testing.T) {
 		description  string
 		dependencies []string
 		artifact     *latest.Artifact
+		devMode      bool
 		expected     string
 	}{
 		{
@@ -131,7 +132,7 @@ func TestGetHashForArtifact(t *testing.T) {
 			t.Override(&artifactConfigFunction, fakeArtifactConfig)
 
 			depLister := stubDependencyLister(test.dependencies)
-			actual, err := getHashForArtifact(context.Background(), depLister, test.artifact)
+			actual, err := getHashForArtifact(context.Background(), depLister, test.artifact, test.devMode)
 
 			t.CheckNoError(err)
 			t.CheckDeepEqual(test.expected, actual)
@@ -147,7 +148,7 @@ func TestArtifactConfig(t *testing.T) {
 					Target: "target",
 				},
 			},
-		})
+		}, false)
 		t.CheckNoError(err)
 
 		config2, err := artifactConfig(&latest.Artifact{
@@ -156,7 +157,39 @@ func TestArtifactConfig(t *testing.T) {
 					Target: "other",
 				},
 			},
-		})
+		}, false)
+		t.CheckNoError(err)
+
+		if config1 == config2 {
+			t.Errorf("configs should be different: [%s] [%s]", config1, config2)
+		}
+	})
+}
+
+func TestArtifactConfigDevMode(t *testing.T) {
+	testutil.Run(t, "", func(t *testutil.T) {
+		config1, err := artifactConfig(&latest.Artifact{
+			ArtifactType: latest.ArtifactType{
+				BuildpackArtifact: &latest.BuildpackArtifact{
+					Builder: "any/builder",
+				},
+			},
+			Sync: &latest.Sync{
+				Infer: []string{"**/*"},
+			},
+		}, false)
+		t.CheckNoError(err)
+
+		config2, err := artifactConfig(&latest.Artifact{
+			ArtifactType: latest.ArtifactType{
+				BuildpackArtifact: &latest.BuildpackArtifact{
+					Builder: "any/builder",
+				},
+			},
+			Sync: &latest.Sync{
+				Infer: []string{"**/*"},
+			},
+		}, true)
 		t.CheckNoError(err)
 
 		if config1 == config2 {
@@ -180,21 +213,21 @@ func TestBuildArgs(t *testing.T) {
 		t.Override(&hashFunction, mockCacheHasher)
 		t.Override(&artifactConfigFunction, fakeArtifactConfig)
 
-		actual, err := getHashForArtifact(context.Background(), stubDependencyLister(nil), artifact)
+		actual, err := getHashForArtifact(context.Background(), stubDependencyLister(nil), artifact, false)
 
 		t.CheckNoError(err)
 		t.CheckDeepEqual(expected, actual)
 
 		// Change order of buildargs
 		artifact.ArtifactType.DockerArtifact.BuildArgs = map[string]*string{"two": stringPointer("2"), "one": stringPointer("1")}
-		actual, err = getHashForArtifact(context.Background(), stubDependencyLister(nil), artifact)
+		actual, err = getHashForArtifact(context.Background(), stubDependencyLister(nil), artifact, false)
 
 		t.CheckNoError(err)
 		t.CheckDeepEqual(expected, actual)
 
 		// Change build args, get different hash
 		artifact.ArtifactType.DockerArtifact.BuildArgs = map[string]*string{"one": stringPointer("1")}
-		actual, err = getHashForArtifact(context.Background(), stubDependencyLister(nil), artifact)
+		actual, err = getHashForArtifact(context.Background(), stubDependencyLister(nil), artifact, false)
 
 		t.CheckNoError(err)
 		if actual == expected {
@@ -223,7 +256,7 @@ func TestBuildArgsEnvSubstitution(t *testing.T) {
 		t.Override(&artifactConfigFunction, fakeArtifactConfig)
 
 		depLister := stubDependencyLister([]string{"dep"})
-		hash1, err := getHashForArtifact(context.Background(), depLister, artifact)
+		hash1, err := getHashForArtifact(context.Background(), depLister, artifact, false)
 
 		t.CheckNoError(err)
 
@@ -233,7 +266,7 @@ func TestBuildArgsEnvSubstitution(t *testing.T) {
 			return []string{"FOO=baz"}
 		}
 
-		hash2, err := getHashForArtifact(context.Background(), depLister, artifact)
+		hash2, err := getHashForArtifact(context.Background(), depLister, artifact, false)
 
 		t.CheckNoError(err)
 		if hash1 == hash2 {
@@ -290,7 +323,7 @@ func TestCacheHasher(t *testing.T) {
 			path := originalFile
 			depLister := stubDependencyLister([]string{tmpDir.Path(originalFile)})
 
-			oldHash, err := getHashForArtifact(context.Background(), depLister, &latest.Artifact{})
+			oldHash, err := getHashForArtifact(context.Background(), depLister, &latest.Artifact{}, false)
 			t.CheckNoError(err)
 
 			test.update(originalFile, tmpDir)
@@ -299,7 +332,7 @@ func TestCacheHasher(t *testing.T) {
 			}
 
 			depLister = stubDependencyLister([]string{tmpDir.Path(path)})
-			newHash, err := getHashForArtifact(context.Background(), depLister, &latest.Artifact{})
+			newHash, err := getHashForArtifact(context.Background(), depLister, &latest.Artifact{}, false)
 
 			t.CheckNoError(err)
 			t.CheckFalse(test.differentHash && oldHash == newHash)

pkg/skaffold/build/cache/lookup.go

@@ -53,7 +53,7 @@ func (c *cache) lookupArtifacts(ctx context.Context, tags tag.ImageTags, artifac
 }
 
 func (c *cache) lookup(ctx context.Context, a *latest.Artifact, tag string) cacheDetails {
-	hash, err := hashForArtifact(ctx, c.dependencies, a)
+	hash, err := hashForArtifact(ctx, c.dependencies, a, c.devMode)
 	if err != nil {
 		return failed{err: fmt.Errorf("getting hash for artifact %s: %v", a.ImageName, err)}
 	}

pkg/skaffold/build/cache/lookup_test.go

@@ -30,7 +30,7 @@ import (
 func TestLookupLocal(t *testing.T) {
 	tests := []struct {
 		description string
-		hasher      func(context.Context, DependencyLister, *latest.Artifact) (string, error)
+		hasher      func(context.Context, DependencyLister, *latest.Artifact, bool) (string, error)
 		cache       map[string]ImageDetails
 		api         *testutil.FakeAPIClient
 		expected    cacheDetails
@@ -126,7 +126,7 @@ func TestLookupLocal(t *testing.T) {
 func TestLookupRemote(t *testing.T) {
 	tests := []struct {
 		description string
-		hasher      func(context.Context, DependencyLister, *latest.Artifact) (string, error)
+		hasher      func(context.Context, DependencyLister, *latest.Artifact, bool) (string, error)
 		cache       map[string]ImageDetails
 		api         *testutil.FakeAPIClient
 		expected    cacheDetails
@@ -208,14 +208,14 @@ func TestLookupRemote(t *testing.T) {
 	}
 }
 
-func mockHasher(value string) func(context.Context, DependencyLister, *latest.Artifact) (string, error) {
-	return func(context.Context, DependencyLister, *latest.Artifact) (string, error) {
+func mockHasher(value string) func(context.Context, DependencyLister, *latest.Artifact, bool) (string, error) {
+	return func(context.Context, DependencyLister, *latest.Artifact, bool) (string, error) {
 		return value, nil
 	}
 }
 
-func failingHasher(errMessage string) func(context.Context, DependencyLister, *latest.Artifact) (string, error) {
-	return func(context.Context, DependencyLister, *latest.Artifact) (string, error) {
+func failingHasher(errMessage string) func(context.Context, DependencyLister, *latest.Artifact, bool) (string, error) {
+	return func(context.Context, DependencyLister, *latest.Artifact, bool) (string, error) {
 		return "", errors.New(errMessage)
 	}
 }

pkg/skaffold/build/local/local.go

@@ -89,7 +89,7 @@ func (b *Builder) runBuildForArtifact(ctx context.Context, out io.Writer, artifa
 		return custom.NewArtifactBuilder(b.localDocker, b.insecureRegistries, b.pushImages, b.retrieveExtraEnv()).Build(ctx, out, artifact, tag)
 
 	case artifact.BuildpackArtifact != nil:
-		return buildpacks.NewArtifactBuilder(b.localDocker, b.pushImages).Build(ctx, out, artifact, tag)
+		return buildpacks.NewArtifactBuilder(b.localDocker, b.pushImages, b.devMode).Build(ctx, out, artifact, tag)
 
 	default:
 		return "", fmt.Errorf("undefined artifact type: %+v", artifact.ArtifactType)

pkg/skaffold/build/local/types.go

@@ -41,6 +41,7 @@ type Builder struct {
 	prune              bool
 	pruneChildren      bool
 	skipTests          bool
+	devMode            bool
 	kubeContext        string
 	builtImages        []string
 	insecureRegistries map[string]bool
@@ -78,6 +79,7 @@ func NewBuilder(runCtx *runcontext.RunContext) (*Builder, error) {
 		localCluster:       localCluster,
 		pushImages:         pushImages,
 		skipTests:          runCtx.Opts.SkipTests,
+		devMode:            runCtx.DevMode,
 		prune:              runCtx.Opts.Prune(),
 		pruneChildren:      !runCtx.Opts.NoPruneChildren,
 		insecureRegistries: runCtx.InsecureRegistries,

pkg/skaffold/docker/image_util.go

@@ -65,3 +65,15 @@ func RetrieveWorkingDir(tagged string, insecureRegistries map[string]bool) (stri
 		return cf.Config.WorkingDir, nil
 	}
 }
+
+func RetrieveLabels(tagged string, insecureRegistries map[string]bool) (map[string]string, error) {
+	cf, err := RetrieveConfigFile(tagged, insecureRegistries)
+	switch {
+	case err != nil:
+		return nil, err
+	case cf == nil:
+		return nil, nil
+	default:
+		return cf.Config.Labels, nil
+	}
+}

pkg/skaffold/runner/runcontext/context.go

@@ -37,6 +37,7 @@ type RunContext struct {
 	WorkingDir         string
 	Namespaces         []string
 	InsecureRegistries map[string]bool
+	DevMode            bool
 }
 
 func GetRunContext(opts config.SkaffoldOptions, cfg latest.Pipeline) (*RunContext, error) {
@@ -70,13 +71,17 @@ func GetRunContext(opts config.SkaffoldOptions, cfg latest.Pipeline) (*RunContex
 		insecureRegistries[r] = true
 	}
 
+	// TODO(dgageot): what about debug?
+	devMode := opts.Command == "dev"
+
 	return &RunContext{
 		Opts:               opts,
 		Cfg:                cfg,
 		WorkingDir:         cwd,
 		KubeContext:        kubeContext,
 		Namespaces:         namespaces,
 		InsecureRegistries: insecureRegistries,
+		DevMode:            devMode,
 	}, nil
 }