#1665. Improve Package Selection (#1687)

## Is your feature request related to a problem? Please describe.
Our package selection workflow in the UI is a bit slow due to searching
the entire home directory at one time.

## Describe the solution you'd like
Recommend the following changes:

### For 'init' package deployment
- [x] in the selection window, only show init packages that exist in the
three places that Zarf searches for them:
  - Current Working Directory
  -  Zarf Binary Directory
  - Home `.zarf-cache` folder
  - Only show init packages that match the current build version. 

### Regular Package Deployment
For 'regular' packages -  the UX flow proceeds in two parts:
- [x] Search # 1. 
- Search and show packages located in the current working directory w/o
subdirectories (matching the tab behavior on `zarf package deploy`)
- If packages are found show in the table and add a expand search button
to table that executes search # 2
- If no packages are found - show no package found error message (gives
the user option to cancel deployment or proceed to part # 2
- [x] search part # 2
    - Search and show packages located in the entire home directory
    - Exclude trash 
    - Exclude hidden folders

- [x] For both workflows, increase the height of the package selection
table to show more packages at once

## Design File

https://www.figma.com/file/MUItIzpzLBLuIyt225Bwgl/Zarf-Web-Ui?type=design&node-id=1544%3A56348&t=JeB5xVvOb7lcP6ZB-1


## Type of change

- [x] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)

---------

Co-authored-by: Wayne Starr <[email protected]>

src/cmd/destroy.go

@@ -52,7 +52,7 @@ var destroyCmd = &cobra.Command{
 
 			// Run all the scripts!
 			pattern := regexp.MustCompile(`(?mi)zarf-clean-.+\.sh$`)
-			scripts, _ := utils.RecursiveFileList(config.ZarfCleanupScriptsPath, pattern, false, true)
+			scripts, _ := utils.RecursiveFileList(config.ZarfCleanupScriptsPath, pattern, true)
 			// Iterate over all matching zarf-clean scripts and exec them
 			for _, script := range scripts {
 				// Run the matched script

src/internal/api/packages/find.go

@@ -5,52 +5,180 @@
 package packages
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/fs"
 	"net/http"
 	"os"
+	"path/filepath"
 	"regexp"
 
-	"github.com/defenseunicorns/zarf/src/internal/api/common"
-	"github.com/defenseunicorns/zarf/src/pkg/message"
+	"github.com/defenseunicorns/zarf/src/config"
+	"github.com/defenseunicorns/zarf/src/pkg/packager"
 	"github.com/defenseunicorns/zarf/src/pkg/utils"
 )
 
 // Find zarf-packages on the local system (https://regex101.com/r/TUUftK/1)
 var packagePattern = regexp.MustCompile(`zarf-package[^\s\\\/]*\.tar(\.zst)?$`)
-// Find zarf-init packages on the local system (https://regex101.com/r/6aTl3O/2)
-var initPattern = regexp.MustCompile(`zarf-init[^\s\\\/]*\.tar(\.zst)?$`)
 
-// Find returns all packages anywhere down the directory tree of the working directory.
-func Find(w http.ResponseWriter, _ *http.Request) {
-	message.Debug("packages.Find()")
-	findPackage(packagePattern, w, os.Getwd)
+// Find zarf-init packages on the local system
+var currentInitPattern = regexp.MustCompile(packager.GetInitPackageName("") + "$")
+
+// FindInHomeStream returns all packages in the user's home directory.
+// If the init query parameter is true, only init packages will be returned.
+func FindInHomeStream(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/event-stream; charset=utf-8")
+	w.Header().Set("Cache-Control", "no-cache")
+	w.Header().Set("Connection", "keep-alive")
+
+	init := r.URL.Query().Get("init")
+	regexp := packagePattern
+	if init == "true" {
+		regexp = currentInitPattern
+	}
+
+	done := make(chan bool)
+	go func() {
+		// User home directory
+		homePath, err := os.UserHomeDir()
+		if err != nil {
+			streamError(err, w)
+		} else {
+			// Recursively search for and stream packages in the home directory
+			recursivePackageStream(homePath, regexp, w)
+		}
+		close(done)
+	}()
+
+	<-done
 }
 
-// FindInHome returns all packages in the user's home directory.
-func FindInHome(w http.ResponseWriter, _ *http.Request) {
-	message.Debug("packages.FindInHome()")
-	findPackage(packagePattern, w, os.UserHomeDir)
+// FindInitStream finds and streams all init packages in the current working directory, the cache directory, and execution directory
+func FindInitStream(w http.ResponseWriter, _ *http.Request) {
+	w.Header().Set("Content-Type", "text/event-stream; charset=utf-8")
+	w.Header().Set("Cache-Control", "no-cache")
+	w.Header().Set("Connection", "keep-alive")
+
+	done := make(chan bool)
+	go func() {
+		// stream init packages in the execution directory
+		if execDir, err := utils.GetFinalExecutablePath(); err == nil {
+			streamDirPackages(execDir, currentInitPattern, w)
+		} else {
+			streamError(err, w)
+		}
+
+		// Cache directory
+		cachePath := config.GetAbsCachePath()
+		// Create the cache directory if it doesn't exist
+		if utils.InvalidPath(cachePath) {
+			if err := os.MkdirAll(cachePath, 0755); err != nil {
+				streamError(err, w)
+			}
+		}
+		streamDirPackages(cachePath, currentInitPattern, w)
+
+		// Find init packages in the current working directory
+		if cwd, err := os.Getwd(); err == nil {
+			streamDirPackages(cwd, currentInitPattern, w)
+		} else {
+			streamError(err, w)
+		}
+		close(done)
+	}()
+	<-done
 }
 
-// FindInitPackage returns all init packages anywhere down the directory tree of the users home directory.
-func FindInitPackage(w http.ResponseWriter, _ *http.Request) {
-	message.Debug("packages.FindInitPackage()")
-	findPackage(initPattern, w, os.UserHomeDir)
+// FindPackageStream finds and streams all packages in the current working directory
+func FindPackageStream(w http.ResponseWriter, _ *http.Request) {
+	w.Header().Set("Content-Type", "text/event-stream; charset=utf-8")
+	w.Header().Set("Cache-Control", "no-cache")
+	w.Header().Set("Connection", "keep-alive")
+	done := make(chan bool)
+
+	go func() {
+		if cwd, err := os.Getwd(); err == nil {
+			streamDirPackages(cwd, packagePattern, w)
+		} else {
+			streamError(err, w)
+		}
+		close(done)
+	}()
+
+	<-done
+	// Find init packages in the current working directory
 }
 
-func findPackage(pattern *regexp.Regexp, w http.ResponseWriter, setDir func() (string, error)) {
-	targetDir, err := setDir()
+// recursivePackageStream recursively searches for and streams packages in the given directory
+func recursivePackageStream(dir string, pattern *regexp.Regexp, w http.ResponseWriter) {
+	err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+		// ignore files/dirs that it does not have permission to read
+		if err != nil && os.IsPermission(err) {
+			return nil
+		}
+
+		// Return error if the pattern is invalid
+		if pattern == nil {
+			return filepath.ErrBadPattern
+		}
+
+		// Return errors
+		if err != nil {
+			return err
+		}
+
+		if !d.IsDir() {
+			if len(pattern.FindStringIndex(path)) > 0 {
+				streamPackage(path, w)
+			}
+			// Skip the trash bin and hidden directories
+		} else if utils.IsTrashBin(path) {
+			return filepath.SkipDir
+		}
+
+		return nil
+	})
 	if err != nil {
-		message.ErrorWebf(err, w, "Error getting directory")
-		return
+		streamError(err, w)
 	}
+}
 
-	// Skip permission errors, search dot-prefixed directories.
-	files, err := utils.RecursiveFileList(targetDir, pattern, true, false)
-	if err != nil || len(files) == 0 {
-		pkgNotFoundMsg := fmt.Sprintf("Unable to locate the package: %s", pattern.String())
-		message.ErrorWebf(err, w, pkgNotFoundMsg)
-		return
+// streamDirPackages streams all packages in the given directory
+func streamDirPackages(dir string, pattern *regexp.Regexp, w http.ResponseWriter) {
+	files, err := os.ReadDir(dir)
+	if err != nil {
+		streamError(err, w)
+	}
+	for _, file := range files {
+		if !file.IsDir() {
+			path := fmt.Sprintf("%s/%s", dir, file.Name())
+			if pattern != nil {
+				if len(pattern.FindStringIndex(path)) > 0 {
+					streamPackage(path, w)
+				}
+			}
+		}
 	}
-	common.WriteJSONResponse(w, files, http.StatusOK)
+}
+
+// streamPackage streams the package at the given path
+func streamPackage(path string, w http.ResponseWriter) {
+	pkg, err := ReadPackage(path)
+	if err != nil {
+		streamError(err, w)
+	} else {
+		jsonData, err := json.Marshal(pkg)
+		if err != nil {
+			streamError(err, w)
+		} else {
+			fmt.Fprintf(w, "data: %s\n\n", jsonData)
+			w.(http.Flusher).Flush()
+		}
+	}
+}
+
+// streamError streams the given error to the client
+func streamError(err error, w http.ResponseWriter) {
+	fmt.Fprintf(w, "data: %s\n\n", err.Error())
+	w.(http.Flusher).Flush()
 }

src/internal/api/packages/read.go

@@ -24,15 +24,15 @@ func Read(w http.ResponseWriter, r *http.Request) {
 
 	path := chi.URLParam(r, "path")
 
-	if pkg, err := readPackage(path); err != nil {
+	if pkg, err := ReadPackage(path); err != nil {
 		message.ErrorWebf(err, w, "Unable to read the package at: `%s`", path)
 	} else {
 		common.WriteJSONResponse(w, pkg, http.StatusOK)
 	}
 }
 
-// internal function to read a package from the local filesystem.
-func readPackage(path string) (pkg types.APIZarfPackage, err error) {
+// ReadPackage reads a packages yaml from the local filesystem and returns an APIZarfPackage.
+func ReadPackage(path string) (pkg types.APIZarfPackage, err error) {
 	var file []byte
 
 	pkg.Path, err = url.QueryUnescape(path)

src/internal/api/start.go

@@ -86,9 +86,6 @@ func LaunchAPIServer() {
 		})
 
 		r.Route("/packages", func(r chi.Router) {
-			r.Get("/find", packages.Find)
-			r.Get("/find-in-home", packages.FindInHome)
-			r.Get("/find-init", packages.FindInitPackage)
 			r.Get("/read/{path}", packages.Read)
 			r.Get("/list", packages.ListDeployedPackages)
 			r.Put("/deploy", packages.DeployPackage)
@@ -100,6 +97,13 @@ func LaunchAPIServer() {
 			r.Get("/connections", packages.ListConnections)
 			r.Get("/sbom/{path}", packages.ExtractSBOM)
 			r.Delete("/sbom", packages.DeleteSBOM)
+			r.Route("/find", func(r chi.Router) {
+				r.Route("/stream", func(r chi.Router) {
+					r.Get("/", packages.FindPackageStream)
+					r.Get("/init", packages.FindInitStream)
+					r.Get("/home", packages.FindInHomeStream)
+				})
+			})
 		})
 
 		r.Route("/components", func(r chi.Router) {

src/internal/packager/template/yaml.go

@@ -15,7 +15,7 @@ import (
 func ProcessYamlFilesInPath(path string, component types.ZarfComponent, values Values) ([]string, error) {
 	// Only pull in yml and yaml files
 	pattern := regexp.MustCompile(`(?mi)\.ya?ml$`)
-	manifests, _ := utils.RecursiveFileList(path, pattern, false, true)
+	manifests, _ := utils.RecursiveFileList(path, pattern, false)
 
 	for _, manifest := range manifests {
 		if err := values.Apply(component, manifest, false); err != nil {

src/pkg/packager/create.go

@@ -302,7 +302,7 @@ func (p *Packager) getFilesToSBOM(component types.ZarfComponent) (*types.Compone
 
 	appendSBOMFiles := func(path string) {
 		if utils.IsDir(path) {
-			files, _ := utils.RecursiveFileList(path, nil, false, true)
+			files, _ := utils.RecursiveFileList(path, nil, false)
 			componentSBOM.Files = append(componentSBOM.Files, files...)
 		} else {
 			componentSBOM.Files = append(componentSBOM.Files, path)

src/pkg/packager/deploy.go

@@ -317,7 +317,7 @@ func (p *Packager) processComponentFiles(component types.ZarfComponent, pkgLocat
 
 		fileList := []string{}
 		if utils.IsDir(fileLocation) {
-			files, _ := utils.RecursiveFileList(fileLocation, nil, false, true)
+			files, _ := utils.RecursiveFileList(fileLocation, nil, false)
 			fileList = append(fileList, files...)
 		} else {
 			fileList = append(fileList, fileLocation)

src/pkg/utils/io.go

@@ -168,20 +168,9 @@ func ReplaceTextTemplate(path string, mappings map[string]*TextTemplate, depreca
 }
 
 // RecursiveFileList walks a path with an optional regex pattern and returns a slice of file paths.
-// the skipPermission flag can be provided to ignore unauthorized files/dirs when true
-// the skipHidden flag can be provided to ignore dot prefixed files/dirs when true
-func RecursiveFileList(dir string, pattern *regexp.Regexp, skipPermission bool, skipHidden bool) (files []string, err error) {
+// If skipHidden is true, hidden directories will be skipped.
+func RecursiveFileList(dir string, pattern *regexp.Regexp, skipHidden bool) (files []string, err error) {
 	err = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
-		// ignore files/dirs that it does not have permission to read
-		if err != nil && os.IsPermission(err) && skipPermission {
-			return nil
-		}
-		// Skip hidden directories
-		if skipHidden {
-			if d.IsDir() && d.Name()[0] == dotCharacter {
-				return filepath.SkipDir
-			}
-		}
 
 		// Return errors
 		if err != nil {
@@ -196,6 +185,9 @@ func RecursiveFileList(dir string, pattern *regexp.Regexp, skipPermission bool,
 			} else {
 				files = append(files, path)
 			}
+			// Skip hidden directories
+		} else if skipHidden && IsHidden(d.Name()) {
+			return filepath.SkipDir
 		}
 
 		return nil
@@ -289,6 +281,33 @@ func IsTextFile(path string) (bool, error) {
 	return hasText || hasJSON || hasXML, nil
 }
 
+// IsTrashBin checks if the given directory path corresponds to an operating system's trash bin.
+func IsTrashBin(dirPath string) bool {
+	dirPath = filepath.Clean(dirPath)
+
+	// Check if the directory path matches a Linux trash bin
+	if strings.HasSuffix(dirPath, "/Trash") || strings.HasSuffix(dirPath, "/.Trash-1000") {
+		return true
+	}
+
+	// Check if the directory path matches a macOS trash bin
+	if strings.HasSuffix(dirPath, "./Trash") || strings.HasSuffix(dirPath, "/.Trashes") {
+		return true
+	}
+
+	// Check if the directory path matches a Windows trash bin
+	if strings.HasSuffix(dirPath, "\\$RECYCLE.BIN") {
+		return true
+	}
+
+	return false
+}
+
+// IsHidden returns true if the given file name starts with a dot.
+func IsHidden(name string) bool {
+	return name[0] == dotCharacter
+}
+
 // GetDirSize walks through all files and directories in the provided path and returns the total size in bytes.
 func GetDirSize(path string) (int64, error) {
 	dirSize := int64(0)

src/test/ui/02_initialize_cluster.spec.ts

@@ -7,12 +7,20 @@ test.beforeEach(async ({ page }) => {
 	page.on('pageerror', (err) => console.log(err.message));
 });
 
+const expandedSearch = async (page) => {
+	const expanded = (await page.locator('.button-label:has-text("Search Directory")')).first();
+	if (expanded.isVisible()) {
+		await expanded.click();
+	}
+};
+
 const getToSelectPage = async (page) => {
 	await page.goto('/auth?token=insecure&next=/packages?init=true', { waitUntil: 'networkidle' });
 };
 
 const getToConfigurePage = async (page) => {
 	await getToSelectPage(page);
+	await expandedSearch(page);
 	// Find first init package deploy button.
 	const deployInit = page.getByTitle('init').first();
 	// click the init package deploy button.
@@ -42,8 +50,10 @@ test.describe('initialize a zarf cluster', () => {
 			'4 Deploy',
 		]);
 
+		await expandedSearch(page);
+
 		// Find first init package deploy button.
-		const deployInit = page.getByTitle('init').first();
+		let deployInit = page.getByTitle('init').first();
 		// click the init package deploy button.
 		deployInit.click();
 

src/test/ui/03_deploy_non_init.spec.ts

@@ -13,6 +13,10 @@ const getToSelectPage = async (page) => {
 
 const getToReview = async (page) => {
 	await getToSelectPage(page);
+	const expanded = (await page.locator('.button-label:has-text("Search Directory")')).first();
+	if (expanded.isVisible()) {
+		await expanded.click();
+	}
 	// Find first dos-games package deploy button.
 	const dosGames = page.getByTitle('dos-games').first();
 	// click the dos-games package deploy button.