fix(sync): allow local paths in vendir config (#191)

examples/default/envs/mykso/dev/_apps/argocd/.myks/vendir.lock.yaml

@@ -7,5 +7,5 @@ directories:
       tags:
       - v2.7.3
     path: .
-  path: ytt/argocd
+  path: envs/mykso/dev/_apps/argocd/vendor/ytt/argocd
 kind: LockConfig

examples/default/envs/mykso/dev/_apps/argocd/.myks/vendir.yaml

@@ -1,7 +1,7 @@
 apiVersion: vendir.k14s.io/v1alpha1
 kind: Config
 directories:
-- path: ytt/argocd
+- path: envs/mykso/dev/_apps/argocd/vendor/ytt/argocd
   contents:
   - path: .
     git:

examples/default/envs/mykso/dev/_apps/httpbingo/.myks/vendir.lock.yaml

@@ -5,5 +5,5 @@ directories:
       appVersion: v2.2.2
       version: 0.1.1
     path: .
-  path: charts/httpbingo
+  path: envs/mykso/dev/_apps/httpbingo/vendor/charts/httpbingo
 kind: LockConfig

examples/default/envs/mykso/dev/_apps/httpbingo/.myks/vendir.yaml

@@ -1,7 +1,7 @@
 apiVersion: vendir.k14s.io/v1alpha1
 kind: Config
 directories:
-- path: charts/httpbingo
+- path: envs/mykso/dev/_apps/httpbingo/vendor/charts/httpbingo
   contents:
   - path: .
     helmChart:

examples/integration-tests/envs/dev/_apps/helm-installation/.myks/vendir.lock.yaml

@@ -3,5 +3,5 @@ directories:
 - contents:
   - directory: {}
     path: .
-  path: charts/render-test
+  path: envs/dev/_apps/helm-installation/vendor/charts/render-test
 kind: LockConfig

examples/integration-tests/envs/dev/_apps/helm-installation/.myks/vendir.yaml

@@ -1,9 +1,9 @@
 apiVersion: vendir.k14s.io/v1alpha1
 kind: Config
 directories:
-- path: charts/render-test
+- path: envs/dev/_apps/helm-installation/vendor/charts/render-test
   contents:
   - path: .
     directory:
       name: render-test
-      path: ../../../../../../charts/render-test-chart
+      path: ../charts/render-test-chart

examples/integration-tests/prototypes/helm-render-test/vendir/base.ytt.yaml

@@ -9,5 +9,4 @@ directories:
     contents:
       - path: .
         directory:
-          name: #@ app.name
-          path: ../../../../../../charts/render-test-chart
+          path: ../charts/render-test-chart

examples/simple/envs/dev/_apps/httpbingo/.myks/vendir.lock.yaml

@@ -5,5 +5,5 @@ directories:
       appVersion: v2.2.2
       version: 0.1.1
     path: .
-  path: charts/httpbingo
+  path: envs/dev/_apps/httpbingo/vendor/charts/httpbingo
 kind: LockConfig

examples/simple/envs/dev/_apps/httpbingo/.myks/vendir.yaml

@@ -1,7 +1,7 @@
 apiVersion: vendir.k14s.io/v1alpha1
 kind: Config
 directories:
-- path: charts/httpbingo
+- path: envs/dev/_apps/httpbingo/vendor/charts/httpbingo
   contents:
   - path: .
     helmChart:

examples/simple/envs/prod/_apps/httpbingo/.myks/vendir.lock.yaml

@@ -5,5 +5,5 @@ directories:
       appVersion: v2.2.2
       version: 0.1.0
     path: .
-  path: charts/httpbingo
+  path: envs/prod/_apps/httpbingo/vendor/charts/httpbingo
 kind: LockConfig

examples/simple/envs/prod/_apps/httpbingo/.myks/vendir.yaml

@@ -1,7 +1,7 @@
 apiVersion: vendir.k14s.io/v1alpha1
 kind: Config
 directories:
-- path: charts/httpbingo
+- path: envs/prod/_apps/httpbingo/vendor/charts/httpbingo
   contents:
   - path: .
     helmChart:

go.mod

@@ -109,4 +109,4 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
-replace carvel.dev/vendir => github.com/mykso/vendir v0.0.0-20231226140829-0b5478366ac6 // unique-tmp-dir
+replace carvel.dev/vendir => github.com/mykso/vendir v0.38.0-unique-tmp-dir

go.sum

@@ -84,8 +84,6 @@ github.com/cppforlife/cobrautil v0.0.0-20221021151949-d60711905d65 h1:+3J1K6yQFR
 github.com/cppforlife/cobrautil v0.0.0-20221021151949-d60711905d65/go.mod h1:2w+qxVu2KSGW78Ex/XaIqfh/OvBgjEsmN53S4T8vEyA=
 github.com/cppforlife/color v1.9.1-0.20200716202919-6706ac40b835 h1:mYQweUIBD+TBRjIeQnJmXr0GSVMpI6O0takyb/aaOgo=
 github.com/cppforlife/color v1.9.1-0.20200716202919-6706ac40b835/go.mod h1:dYeVsKp1vvK8XjdTPR1gF+uk+9doxKeO3hqQTOCr7T4=
-github.com/cppforlife/go-cli-ui v0.0.0-20220425131040-94f26b16bc14 h1:MjRdR01xh0sfkeS3OOBv+MYkYsrbHuTDc4rfBnVdFaI=
-github.com/cppforlife/go-cli-ui v0.0.0-20220425131040-94f26b16bc14/go.mod h1:AlgTssDlstr4mf92TR4DPITLfl5+7wEY4cKStCmeeto=
 github.com/cppforlife/go-cli-ui v0.0.0-20220622150351-995494831c6c h1:Au0iPWQ6E8TMil9HiGW/DI4CBttUpBOZWkzgqwq+PLg=
 github.com/cppforlife/go-cli-ui v0.0.0-20220622150351-995494831c6c/go.mod h1:ci7nWkU0g40w486NlpUpXXpTD3pkOBjH090Uc0wpER4=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
@@ -241,10 +239,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/mykso/vendir v0.0.0-20231226140829-0b5478366ac6 h1:P44L6H/jK3ju8K6QJpFHE/5/FIfX6KfZxAk64BOfNkY=
-github.com/mykso/vendir v0.0.0-20231226140829-0b5478366ac6/go.mod h1:7cRMieK3dDvzokbq1LmKIOWzZ+hw9Rm+BAUOLoIaBG8=
-github.com/mykso/vendir v0.0.0-20231230003508-84b2be48e389 h1:i4g6aYWEDCpqO2xJ6vi5IfM/cApV9D/A1SJwSbDQ9BY=
-github.com/mykso/vendir v0.0.0-20231230003508-84b2be48e389/go.mod h1:7cRMieK3dDvzokbq1LmKIOWzZ+hw9Rm+BAUOLoIaBG8=
+github.com/mykso/vendir v0.38.0-unique-tmp-dir h1:xaqkHV4OEOnKgnt22WN6gOnSSENhFdz1BsEQF82M/QI=
+github.com/mykso/vendir v0.38.0-unique-tmp-dir/go.mod h1:7cRMieK3dDvzokbq1LmKIOWzZ+hw9Rm+BAUOLoIaBG8=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=

internal/myks/sync.go

@@ -3,6 +3,7 @@ package myks
 import (
 	_ "embed"
 	"errors"
+	"fmt"
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -11,6 +12,15 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+const vendorDirOverlayTemplate = `
+#@ load("@ytt:overlay", "overlay")
+#@overlay/match by=overlay.subset({"kind": "Config", "apiVersion": "vendir.k14s.io/v1alpha1"})
+---
+directories:
+  - #@overlay/match by=overlay.all, expects="1+"
+    #@overlay/replace via=lambda l, r: (r + "/" + l).replace("//", "/")
+    path: %s`
+
 func (a *Application) Sync(vendirSecrets string) error {
 	log.Debug().Msg(a.Msg(syncStepName, "Starting"))
 	if err := a.prepareSync(); err != nil {
@@ -49,7 +59,9 @@ func (a *Application) prepareSync() error {
 		return ErrNoVendirConfig
 	}
 
-	vendirConfig, err := a.ytt(syncStepName, "creating vendir config", yttFiles)
+	vendorDir := a.expandPath(a.e.g.VendorDirName)
+	overlayReader := strings.NewReader(fmt.Sprintf(vendorDirOverlayTemplate, vendorDir))
+	vendirConfig, err := a.yttS(syncStepName, "creating vendir config", yttFiles, overlayReader)
 	if err != nil {
 		return err
 	}
@@ -76,30 +88,23 @@ func (a *Application) prepareSync() error {
 }
 
 func (a *Application) doSync(vendirSecrets string) error {
-	// Paths are relative to the vendor directory (BUG: this will brake with multi-level vendor directory, e.g. `vendor/shmendor`)
-	vendirConfigFileRelativePath := filepath.Join("..", a.e.g.ServiceDirName, a.e.g.VendirConfigFileName)
-	vendirLockFileRelativePath := filepath.Join("..", a.e.g.ServiceDirName, a.e.g.VendirLockFileName)
-	vendorDir := a.expandPath(a.e.g.VendorDirName)
-
-	if err := createDirectory(vendorDir); err != nil {
-		return err
-	}
+	vendirConfigPath := a.expandServicePath(a.e.g.VendirConfigFileName)
+	vendirLockFilePath := a.expandServicePath(a.e.g.VendirLockFileName)
 
 	// TODO sync retry
-	if err := a.runVendirSync(vendorDir, vendirConfigFileRelativePath, vendirLockFileRelativePath, vendirSecrets); err != nil {
+	if err := a.runVendirSync(vendirConfigPath, vendirLockFilePath, vendirSecrets); err != nil {
 		log.Error().Err(err).Msg(a.Msg(syncStepName, "Vendir sync failed"))
 		return err
 	}
 
-	vendirConfigFile := a.expandServicePath(a.e.g.VendirConfigFileName)
-	return a.cleanupVendorDir(vendorDir, vendirConfigFile)
+	vendorDir := a.expandPath(a.e.g.VendorDirName)
+	return a.cleanupVendorDir(vendorDir, vendirConfigPath)
 }
 
-func (a *Application) runVendirSync(targetDir string, vendirConfig string, vendirLock string, vendirSecrets string) error {
+func (a *Application) runVendirSync(vendirConfig, vendirLock, vendirSecrets string) error {
 	args := []string{
 		"vendir",
 		"sync",
-		"--chdir=" + targetDir,
 		"--file=" + vendirConfig,
 		"--lock-file=" + vendirLock,
 		"--file=-",
@@ -129,7 +134,7 @@ func (a Application) cleanupVendorDir(vendorDir, vendirConfigFile string) error
 		dirs = append(dirs, filepath.Clean(path)+string(filepath.Separator))
 	}
 
-	log.Debug().Strs("managed dirs", dirs).Msg("")
+	log.Debug().Strs("vendir-managed dirs", dirs).Msg("")
 
 	return filepath.WalkDir(vendorDir, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
@@ -139,32 +144,20 @@ func (a Application) cleanupVendorDir(vendorDir, vendirConfigFile string) error
 		if !d.IsDir() {
 			return nil
 		}
-		log.Debug().Msg(a.Msg(syncStepName, "Checking directory "+path))
-
-		relPath, err := filepath.Rel(vendorDir, path)
-		if err != nil {
-			return err
-		}
-		if relPath == "." {
-			return nil
-		}
 
-		relPath = relPath + string(filepath.Separator)
+		path = path + string(filepath.Separator)
 		for _, dir := range dirs {
-			log.Debug().Str("dir", dir).Str("relPath", relPath).Msg("Checking dir")
-			if dir == relPath {
-				log.Debug().Msgf("%s == %s", dir, relPath)
+			if dir == path {
 				return fs.SkipDir
 			}
 
-			if strings.HasPrefix(dir, relPath) {
-				log.Debug().Msgf("%s has prefix %s", dir, relPath)
+			if strings.HasPrefix(dir, path) {
 				return nil
 			}
 
 			// This should never happen
-			if strings.HasPrefix(relPath, dir) {
-				log.Debug().Msgf("%s has prefix %s", relPath, dir)
+			if strings.HasPrefix(path, dir) {
+				log.Debug().Msgf("%s has prefix %s", path, dir)
 				return fs.SkipDir
 			}
 		}

internal/myks/sync_test.go

@@ -3,6 +3,7 @@ package myks
 import (
 	"os"
 	"path/filepath"
+	"regexp"
 	"testing"
 
 	"github.com/creasty/defaults"
@@ -131,6 +132,7 @@ func Test_cleanupVendorDir(t *testing.T) {
 		},
 	}
 
+	vendorPathRegex := regexp.MustCompile(`path: (\S*)`)
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			const keeperFile = "keeper"
@@ -143,6 +145,9 @@ func Test_cleanupVendorDir(t *testing.T) {
 				t.Errorf("creating directory %s error = %v", vendorDir, err)
 			}
 
+			// Prepend vendor directory to paths in vendir config
+			tt.vendirYaml = vendorPathRegex.ReplaceAllString(tt.vendirYaml, "path: "+vendorDir+"/$1")
+
 			// Create vendir config file
 			if err := os.MkdirAll(filepath.Dir(vendirConfigFile), 0o755); err != nil {
 				t.Errorf("creating directory %s error = %v", filepath.Dir(vendirConfigFile), err)