From 77141b788cf547ab557f80f6802c1491865df10f Mon Sep 17 00:00:00 2001 From: Cole Winberry <86802655+mike-winberry@users.noreply.github.com> Date: Mon, 12 Jun 2023 16:45:52 -0700 Subject: [PATCH] #1665. Improve Package Selection (#1687) ## Is your feature request related to a problem? Please describe. Our package selection workflow in the UI is a bit slow due to searching the entire home directory at one time. ## Describe the solution you'd like Recommend the following changes: ### For 'init' package deployment - [x] in the selection window, only show init packages that exist in the three places that Zarf searches for them: - Current Working Directory - Zarf Binary Directory - Home `.zarf-cache` folder - Only show init packages that match the current build version. ### Regular Package Deployment For 'regular' packages - the UX flow proceeds in two parts: - [x] Search # 1. - Search and show packages located in the current working directory w/o subdirectories (matching the tab behavior on `zarf package deploy`) - If packages are found show in the table and add a expand search button to table that executes search # 2 - If no packages are found - show no package found error message (gives the user option to cancel deployment or proceed to part # 2 - [x] search part # 2 - Search and show packages located in the entire home directory - Exclude trash - Exclude hidden folders - [x] For both workflows, increase the height of the package selection table to show more packages at once ## Design File https://www.figma.com/file/MUItIzpzLBLuIyt225Bwgl/Zarf-Web-Ui?type=design&node-id=1544%3A56348&t=JeB5xVvOb7lcP6ZB-1 ## Type of change - [x] Bug fix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) --------- Co-authored-by: Wayne Starr --- src/cmd/destroy.go | 2 +- src/internal/api/packages/find.go | 182 ++++++++++-- src/internal/api/packages/read.go | 6 +- src/internal/api/start.go | 10 +- src/internal/packager/template/yaml.go | 2 +- src/pkg/packager/create.go | 2 +- src/pkg/packager/deploy.go | 2 +- src/pkg/utils/io.go | 45 ++- src/test/ui/02_initialize_cluster.spec.ts | 12 +- src/test/ui/03_deploy_non_init.spec.ts | 4 + src/ui/lib/api.ts | 8 +- .../lib/components/local-package-table.svelte | 266 +++++++++++------- 12 files changed, 392 insertions(+), 149 deletions(-) diff --git a/src/cmd/destroy.go b/src/cmd/destroy.go index 6c6e44548f..af96944023 100644 --- a/src/cmd/destroy.go +++ b/src/cmd/destroy.go @@ -52,7 +52,7 @@ var destroyCmd = &cobra.Command{ // Run all the scripts! pattern := regexp.MustCompile(`(?mi)zarf-clean-.+\.sh$`) - scripts, _ := utils.RecursiveFileList(config.ZarfCleanupScriptsPath, pattern, false, true) + scripts, _ := utils.RecursiveFileList(config.ZarfCleanupScriptsPath, pattern, true) // Iterate over all matching zarf-clean scripts and exec them for _, script := range scripts { // Run the matched script diff --git a/src/internal/api/packages/find.go b/src/internal/api/packages/find.go index 50fd9c39a4..226a0f6682 100644 --- a/src/internal/api/packages/find.go +++ b/src/internal/api/packages/find.go @@ -5,52 +5,180 @@ package packages import ( + "encoding/json" "fmt" + "io/fs" "net/http" "os" + "path/filepath" "regexp" - "github.com/defenseunicorns/zarf/src/internal/api/common" - "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/utils" ) // Find zarf-packages on the local system (https://regex101.com/r/TUUftK/1) var packagePattern = regexp.MustCompile(`zarf-package[^\s\\\/]*\.tar(\.zst)?$`) -// Find zarf-init packages on the local system (https://regex101.com/r/6aTl3O/2) -var initPattern = regexp.MustCompile(`zarf-init[^\s\\\/]*\.tar(\.zst)?$`) -// Find returns all packages anywhere down the directory tree of the working directory. -func Find(w http.ResponseWriter, _ *http.Request) { - message.Debug("packages.Find()") - findPackage(packagePattern, w, os.Getwd) +// Find zarf-init packages on the local system +var currentInitPattern = regexp.MustCompile(packager.GetInitPackageName("") + "$") + +// FindInHomeStream returns all packages in the user's home directory. +// If the init query parameter is true, only init packages will be returned. +func FindInHomeStream(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream; charset=utf-8") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + + init := r.URL.Query().Get("init") + regexp := packagePattern + if init == "true" { + regexp = currentInitPattern + } + + done := make(chan bool) + go func() { + // User home directory + homePath, err := os.UserHomeDir() + if err != nil { + streamError(err, w) + } else { + // Recursively search for and stream packages in the home directory + recursivePackageStream(homePath, regexp, w) + } + close(done) + }() + + <-done } -// FindInHome returns all packages in the user's home directory. -func FindInHome(w http.ResponseWriter, _ *http.Request) { - message.Debug("packages.FindInHome()") - findPackage(packagePattern, w, os.UserHomeDir) +// FindInitStream finds and streams all init packages in the current working directory, the cache directory, and execution directory +func FindInitStream(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/event-stream; charset=utf-8") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + + done := make(chan bool) + go func() { + // stream init packages in the execution directory + if execDir, err := utils.GetFinalExecutablePath(); err == nil { + streamDirPackages(execDir, currentInitPattern, w) + } else { + streamError(err, w) + } + + // Cache directory + cachePath := config.GetAbsCachePath() + // Create the cache directory if it doesn't exist + if utils.InvalidPath(cachePath) { + if err := os.MkdirAll(cachePath, 0755); err != nil { + streamError(err, w) + } + } + streamDirPackages(cachePath, currentInitPattern, w) + + // Find init packages in the current working directory + if cwd, err := os.Getwd(); err == nil { + streamDirPackages(cwd, currentInitPattern, w) + } else { + streamError(err, w) + } + close(done) + }() + <-done } -// FindInitPackage returns all init packages anywhere down the directory tree of the users home directory. -func FindInitPackage(w http.ResponseWriter, _ *http.Request) { - message.Debug("packages.FindInitPackage()") - findPackage(initPattern, w, os.UserHomeDir) +// FindPackageStream finds and streams all packages in the current working directory +func FindPackageStream(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/event-stream; charset=utf-8") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + done := make(chan bool) + + go func() { + if cwd, err := os.Getwd(); err == nil { + streamDirPackages(cwd, packagePattern, w) + } else { + streamError(err, w) + } + close(done) + }() + + <-done + // Find init packages in the current working directory } -func findPackage(pattern *regexp.Regexp, w http.ResponseWriter, setDir func() (string, error)) { - targetDir, err := setDir() +// recursivePackageStream recursively searches for and streams packages in the given directory +func recursivePackageStream(dir string, pattern *regexp.Regexp, w http.ResponseWriter) { + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + // ignore files/dirs that it does not have permission to read + if err != nil && os.IsPermission(err) { + return nil + } + + // Return error if the pattern is invalid + if pattern == nil { + return filepath.ErrBadPattern + } + + // Return errors + if err != nil { + return err + } + + if !d.IsDir() { + if len(pattern.FindStringIndex(path)) > 0 { + streamPackage(path, w) + } + // Skip the trash bin and hidden directories + } else if utils.IsTrashBin(path) { + return filepath.SkipDir + } + + return nil + }) if err != nil { - message.ErrorWebf(err, w, "Error getting directory") - return + streamError(err, w) } +} - // Skip permission errors, search dot-prefixed directories. - files, err := utils.RecursiveFileList(targetDir, pattern, true, false) - if err != nil || len(files) == 0 { - pkgNotFoundMsg := fmt.Sprintf("Unable to locate the package: %s", pattern.String()) - message.ErrorWebf(err, w, pkgNotFoundMsg) - return +// streamDirPackages streams all packages in the given directory +func streamDirPackages(dir string, pattern *regexp.Regexp, w http.ResponseWriter) { + files, err := os.ReadDir(dir) + if err != nil { + streamError(err, w) + } + for _, file := range files { + if !file.IsDir() { + path := fmt.Sprintf("%s/%s", dir, file.Name()) + if pattern != nil { + if len(pattern.FindStringIndex(path)) > 0 { + streamPackage(path, w) + } + } + } } - common.WriteJSONResponse(w, files, http.StatusOK) +} + +// streamPackage streams the package at the given path +func streamPackage(path string, w http.ResponseWriter) { + pkg, err := ReadPackage(path) + if err != nil { + streamError(err, w) + } else { + jsonData, err := json.Marshal(pkg) + if err != nil { + streamError(err, w) + } else { + fmt.Fprintf(w, "data: %s\n\n", jsonData) + w.(http.Flusher).Flush() + } + } +} + +// streamError streams the given error to the client +func streamError(err error, w http.ResponseWriter) { + fmt.Fprintf(w, "data: %s\n\n", err.Error()) + w.(http.Flusher).Flush() } diff --git a/src/internal/api/packages/read.go b/src/internal/api/packages/read.go index c73a2f1b73..ed3a306724 100644 --- a/src/internal/api/packages/read.go +++ b/src/internal/api/packages/read.go @@ -24,15 +24,15 @@ func Read(w http.ResponseWriter, r *http.Request) { path := chi.URLParam(r, "path") - if pkg, err := readPackage(path); err != nil { + if pkg, err := ReadPackage(path); err != nil { message.ErrorWebf(err, w, "Unable to read the package at: `%s`", path) } else { common.WriteJSONResponse(w, pkg, http.StatusOK) } } -// internal function to read a package from the local filesystem. -func readPackage(path string) (pkg types.APIZarfPackage, err error) { +// ReadPackage reads a packages yaml from the local filesystem and returns an APIZarfPackage. +func ReadPackage(path string) (pkg types.APIZarfPackage, err error) { var file []byte pkg.Path, err = url.QueryUnescape(path) diff --git a/src/internal/api/start.go b/src/internal/api/start.go index 436f3acde6..cf31c90eef 100644 --- a/src/internal/api/start.go +++ b/src/internal/api/start.go @@ -86,9 +86,6 @@ func LaunchAPIServer() { }) r.Route("/packages", func(r chi.Router) { - r.Get("/find", packages.Find) - r.Get("/find-in-home", packages.FindInHome) - r.Get("/find-init", packages.FindInitPackage) r.Get("/read/{path}", packages.Read) r.Get("/list", packages.ListDeployedPackages) r.Put("/deploy", packages.DeployPackage) @@ -100,6 +97,13 @@ func LaunchAPIServer() { r.Get("/connections", packages.ListConnections) r.Get("/sbom/{path}", packages.ExtractSBOM) r.Delete("/sbom", packages.DeleteSBOM) + r.Route("/find", func(r chi.Router) { + r.Route("/stream", func(r chi.Router) { + r.Get("/", packages.FindPackageStream) + r.Get("/init", packages.FindInitStream) + r.Get("/home", packages.FindInHomeStream) + }) + }) }) r.Route("/components", func(r chi.Router) { diff --git a/src/internal/packager/template/yaml.go b/src/internal/packager/template/yaml.go index a2fdd81574..46321d7f4d 100644 --- a/src/internal/packager/template/yaml.go +++ b/src/internal/packager/template/yaml.go @@ -15,7 +15,7 @@ import ( func ProcessYamlFilesInPath(path string, component types.ZarfComponent, values Values) ([]string, error) { // Only pull in yml and yaml files pattern := regexp.MustCompile(`(?mi)\.ya?ml$`) - manifests, _ := utils.RecursiveFileList(path, pattern, false, true) + manifests, _ := utils.RecursiveFileList(path, pattern, false) for _, manifest := range manifests { if err := values.Apply(component, manifest, false); err != nil { diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 8a4f36dd88..62be1ddd68 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -302,7 +302,7 @@ func (p *Packager) getFilesToSBOM(component types.ZarfComponent) (*types.Compone appendSBOMFiles := func(path string) { if utils.IsDir(path) { - files, _ := utils.RecursiveFileList(path, nil, false, true) + files, _ := utils.RecursiveFileList(path, nil, false) componentSBOM.Files = append(componentSBOM.Files, files...) } else { componentSBOM.Files = append(componentSBOM.Files, path) diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index fd4d56669f..46809709e1 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -317,7 +317,7 @@ func (p *Packager) processComponentFiles(component types.ZarfComponent, pkgLocat fileList := []string{} if utils.IsDir(fileLocation) { - files, _ := utils.RecursiveFileList(fileLocation, nil, false, true) + files, _ := utils.RecursiveFileList(fileLocation, nil, false) fileList = append(fileList, files...) } else { fileList = append(fileList, fileLocation) diff --git a/src/pkg/utils/io.go b/src/pkg/utils/io.go index cb2d4999ad..d4b2fcb2f2 100755 --- a/src/pkg/utils/io.go +++ b/src/pkg/utils/io.go @@ -168,20 +168,9 @@ func ReplaceTextTemplate(path string, mappings map[string]*TextTemplate, depreca } // RecursiveFileList walks a path with an optional regex pattern and returns a slice of file paths. -// the skipPermission flag can be provided to ignore unauthorized files/dirs when true -// the skipHidden flag can be provided to ignore dot prefixed files/dirs when true -func RecursiveFileList(dir string, pattern *regexp.Regexp, skipPermission bool, skipHidden bool) (files []string, err error) { +// If skipHidden is true, hidden directories will be skipped. +func RecursiveFileList(dir string, pattern *regexp.Regexp, skipHidden bool) (files []string, err error) { err = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { - // ignore files/dirs that it does not have permission to read - if err != nil && os.IsPermission(err) && skipPermission { - return nil - } - // Skip hidden directories - if skipHidden { - if d.IsDir() && d.Name()[0] == dotCharacter { - return filepath.SkipDir - } - } // Return errors if err != nil { @@ -196,6 +185,9 @@ func RecursiveFileList(dir string, pattern *regexp.Regexp, skipPermission bool, } else { files = append(files, path) } + // Skip hidden directories + } else if skipHidden && IsHidden(d.Name()) { + return filepath.SkipDir } return nil @@ -289,6 +281,33 @@ func IsTextFile(path string) (bool, error) { return hasText || hasJSON || hasXML, nil } +// IsTrashBin checks if the given directory path corresponds to an operating system's trash bin. +func IsTrashBin(dirPath string) bool { + dirPath = filepath.Clean(dirPath) + + // Check if the directory path matches a Linux trash bin + if strings.HasSuffix(dirPath, "/Trash") || strings.HasSuffix(dirPath, "/.Trash-1000") { + return true + } + + // Check if the directory path matches a macOS trash bin + if strings.HasSuffix(dirPath, "./Trash") || strings.HasSuffix(dirPath, "/.Trashes") { + return true + } + + // Check if the directory path matches a Windows trash bin + if strings.HasSuffix(dirPath, "\\$RECYCLE.BIN") { + return true + } + + return false +} + +// IsHidden returns true if the given file name starts with a dot. +func IsHidden(name string) bool { + return name[0] == dotCharacter +} + // GetDirSize walks through all files and directories in the provided path and returns the total size in bytes. func GetDirSize(path string) (int64, error) { dirSize := int64(0) diff --git a/src/test/ui/02_initialize_cluster.spec.ts b/src/test/ui/02_initialize_cluster.spec.ts index 479e9785dc..a6e4755dc5 100644 --- a/src/test/ui/02_initialize_cluster.spec.ts +++ b/src/test/ui/02_initialize_cluster.spec.ts @@ -7,12 +7,20 @@ test.beforeEach(async ({ page }) => { page.on('pageerror', (err) => console.log(err.message)); }); +const expandedSearch = async (page) => { + const expanded = (await page.locator('.button-label:has-text("Search Directory")')).first(); + if (expanded.isVisible()) { + await expanded.click(); + } +}; + const getToSelectPage = async (page) => { await page.goto('/auth?token=insecure&next=/packages?init=true', { waitUntil: 'networkidle' }); }; const getToConfigurePage = async (page) => { await getToSelectPage(page); + await expandedSearch(page); // Find first init package deploy button. const deployInit = page.getByTitle('init').first(); // click the init package deploy button. @@ -42,8 +50,10 @@ test.describe('initialize a zarf cluster', () => { '4 Deploy', ]); + await expandedSearch(page); + // Find first init package deploy button. - const deployInit = page.getByTitle('init').first(); + let deployInit = page.getByTitle('init').first(); // click the init package deploy button. deployInit.click(); diff --git a/src/test/ui/03_deploy_non_init.spec.ts b/src/test/ui/03_deploy_non_init.spec.ts index 396db81380..69165920ca 100644 --- a/src/test/ui/03_deploy_non_init.spec.ts +++ b/src/test/ui/03_deploy_non_init.spec.ts @@ -13,6 +13,10 @@ const getToSelectPage = async (page) => { const getToReview = async (page) => { await getToSelectPage(page); + const expanded = (await page.locator('.button-label:has-text("Search Directory")')).first(); + if (expanded.isVisible()) { + await expanded.click(); + } // Find first dos-games package deploy button. const dosGames = page.getByTitle('dos-games').first(); // click the dos-games package deploy button. diff --git a/src/ui/lib/api.ts b/src/ui/lib/api.ts index 5f0b26b49b..dcf22c7df8 100644 --- a/src/ui/lib/api.ts +++ b/src/ui/lib/api.ts @@ -36,9 +36,6 @@ const Cluster = { }; const Packages = { - find: () => http.get('/packages/find'), - findInHome: () => http.get('/packages/find-in-home'), - findInit: () => http.get('/packages/find-init'), read: (name: string) => http.get(`/packages/read/${encodeURIComponent(name)}`), getDeployedPackages: () => http.get('/packages/list'), deploy: (options: APIZarfDeployPayload) => http.put(`/packages/deploy`, options), @@ -59,6 +56,11 @@ const Packages = { ), sbom: (path: string) => http.get(`/packages/sbom/${encodeURIComponent(path)}`), cleanSBOM: () => http.del('/packages/sbom'), + find: (eventParams: EventParams) => http.eventStream('/packages/find/stream', eventParams), + findInit: (eventParams: EventParams) => + http.eventStream('/packages/find/stream/init', eventParams), + findHome: (eventParams: EventParams, init: boolean) => + http.eventStream(`/packages/find/stream/home?init=${init}`, eventParams), }; const DeployingComponents = { diff --git a/src/ui/lib/components/local-package-table.svelte b/src/ui/lib/components/local-package-table.svelte index 7079959395..e13ae666b2 100644 --- a/src/ui/lib/components/local-package-table.svelte +++ b/src/ui/lib/components/local-package-table.svelte @@ -3,37 +3,76 @@ // SPDX-FileCopyrightText: 2021-Present The Zarf Authors --> Local Directory - This table shows all of the Zarf{initString} packages that exist on your local machine. + This table shows all of the Zarf{initString} packages that exist {tooltip}. + {#if doneStreaming || expandedSearch} + + {expandedSearchButtonText} + + {/if} {#each tableLabels as l} @@ -141,75 +204,88 @@ {/each} - {#await readPackages()} -
- - - Searching your local machine for Zarf{initString} Packages. This may take a minute. + {#each packages as pkg} + + + check_circle + {pkg.zarfPackage.metadata?.name} -
- {:then packages} - {#if !packages.length} -
- - No Zarf{initString} Packages found on local system - -
- {:else} - {#each packages as pkg} - - - - check_circle - - {pkg.zarfPackage.metadata?.name} - - - {#if !initPkg && pkg.zarfPackage?.metadata?.version} - - {pkg.zarfPackage.metadata.version} - - {:else if initPkg && pkg.zarfPackage?.build?.version} - - {pkg.zarfPackage.build.version} - - {/if} - + + {#if !initPkg && pkg.zarfPackage?.metadata?.version} + + {pkg.zarfPackage.metadata.version} + + {:else if initPkg && pkg.zarfPackage?.build?.version} + + {pkg.zarfPackage.build.version} + + {/if} + - - {#if pkg.zarfPackage?.build?.architecture} - - {pkg.zarfPackage.build.architecture} - - {/if} - - {pkg.zarfPackage.kind} - - - - {pkg.zarfPackage.metadata?.description} - - - - - - {/each} - {/if} - {:catch err} -
- {err.message} -
- {/await} + + {#if pkg.zarfPackage?.build?.architecture} + + {pkg.zarfPackage.build.architecture} + + {/if} + + {pkg.zarfPackage.kind} + + + + {pkg.zarfPackage.metadata?.description} + + + + + + {/each} + {#if !doneStreaming} + +
+ + {searchText} +
+ + {/if} + {#if doneStreaming && !packages.length && expandedSearch} + +
+ warning + No packages found. +
+ + {/if} + {#if !packages.length && doneStreaming && !expandedSearch} + + + No Zarf packages were found in the current working directory. Would you like to search the + home directory? + + + goto('/')} variant="outlined" backgroundColor="white"> + cancel deployment + + + Search Directory + + + + {/if}