From 112cd1959982a71f91680326f74078857857f8c5 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Mon, 4 Nov 2024 12:05:44 +0000 Subject: [PATCH 1/3] Go: Move `IsGolangVendorDirectory` to `util` package --- .../configurebaseline/configurebaseline.go | 14 +------------- go/extractor/util/util.go | 12 ++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/go/extractor/configurebaseline/configurebaseline.go b/go/extractor/configurebaseline/configurebaseline.go index 04037d61425e..062289f7baa4 100644 --- a/go/extractor/configurebaseline/configurebaseline.go +++ b/go/extractor/configurebaseline/configurebaseline.go @@ -3,24 +3,12 @@ package configurebaseline import ( "encoding/json" "io/fs" - "os" "path" "path/filepath" "github.com/github/codeql-go/extractor/util" ) -func fileExists(path string) bool { - stat, err := os.Stat(path) - return err == nil && stat.Mode().IsRegular() -} - -// Decides if `dirPath` is a vendor directory by testing whether it is called `vendor` -// and contains a `modules.txt` file. -func isGolangVendorDirectory(dirPath string) bool { - return filepath.Base(dirPath) == "vendor" && fileExists(filepath.Join(dirPath, "modules.txt")) -} - type BaselineConfig struct { PathsIgnore []string `json:"paths-ignore"` } @@ -38,7 +26,7 @@ func GetConfigBaselineAsJSON(rootDir string) ([]byte, error) { // it will not be extracted either. return nil } - if isGolangVendorDirectory(dirPath) { + if util.IsGolangVendorDirectory(dirPath) { // Note that CodeQL expects a forward-slash-separated path, even on Windows. vendorDirs = append(vendorDirs, path.Join(filepath.ToSlash(dirPath), "**")) return filepath.SkipDir diff --git a/go/extractor/util/util.go b/go/extractor/util/util.go index 595326496d8a..9a2d12988150 100644 --- a/go/extractor/util/util.go +++ b/go/extractor/util/util.go @@ -287,3 +287,15 @@ func getImportPathFromRepoURL(repourl string) string { path = regexp.MustCompile(`^/+|\.git$`).ReplaceAllString(path, "") return host + "/" + path } + +// Decides if `path` refers to a file that exists. +func fileExists(path string) bool { + stat, err := os.Stat(path) + return err == nil && stat.Mode().IsRegular() +} + +// Decides if `dirPath` is a vendor directory by testing whether it is called `vendor` +// and contains a `modules.txt` file. +func IsGolangVendorDirectory(dirPath string) bool { + return filepath.Base(dirPath) == "vendor" && fileExists(filepath.Join(dirPath, "modules.txt")) +} From 40e5010239cd515033fa172512de1cd7a5156d44 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Mon, 4 Nov 2024 12:16:41 +0000 Subject: [PATCH 2/3] Go: Allow `FindAllFilesWithName` to use predicate functions for `dirsToSkip` --- go/extractor/project/project.go | 8 ++++---- go/extractor/util/util.go | 13 +++++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/go/extractor/project/project.go b/go/extractor/project/project.go index c870e6a5458a..a82443329fdf 100644 --- a/go/extractor/project/project.go +++ b/go/extractor/project/project.go @@ -184,12 +184,12 @@ func RemoveTemporaryExtractorFiles() { // Find all go.work files in the working directory and its subdirectories func findGoWorkFiles() []string { - return util.FindAllFilesWithName(".", "go.work", "vendor") + return util.FindAllFilesWithName(".", "go.work", util.SkipVendorChecks...) } // Find all go.mod files in the specified directory and its subdirectories func findGoModFiles(root string) []string { - return util.FindAllFilesWithName(root, "go.mod", "vendor") + return util.FindAllFilesWithName(root, "go.mod", util.SkipVendorChecks...) } // A regular expression for the Go toolchain version syntax. @@ -547,8 +547,8 @@ func startsWithAnyOf(str string, prefixes []string) bool { // Finds Go workspaces in the current working directory. func GetWorkspaceInfo(emitDiagnostics bool) []GoWorkspace { bazelPaths := slices.Concat( - util.FindAllFilesWithName(".", "BUILD", "vendor"), - util.FindAllFilesWithName(".", "BUILD.bazel", "vendor"), + util.FindAllFilesWithName(".", "BUILD", util.SkipVendorChecks...), + util.FindAllFilesWithName(".", "BUILD.bazel", util.SkipVendorChecks...), ) if len(bazelPaths) > 0 { // currently not supported diff --git a/go/extractor/util/util.go b/go/extractor/util/util.go index 9a2d12988150..e98e1a3d539a 100644 --- a/go/extractor/util/util.go +++ b/go/extractor/util/util.go @@ -152,7 +152,16 @@ func FindGoFiles(root string) bool { return found } -func FindAllFilesWithName(root string, name string, dirsToSkip ...string) []string { +// The type of check function used by `FindAllFilesWithName` to decide whether to skip the directory named by `path`. +type FindAllFilesWithNameSkipCheck func(path string) bool + +// Commonly we only want to skip `vendor` directories in `FindAllFilesWithName`. This array is a suitable +// argument for `dirsToSkip` which skips `vendor` directories. +var SkipVendorChecks = []FindAllFilesWithNameSkipCheck{IsGolangVendorDirectory} + +// Returns an array of all files matching `name` within the path at `root`. +// The `dirsToSkip` array contains check functions used to decide which directories to skip. +func FindAllFilesWithName(root string, name string, dirsToSkip ...FindAllFilesWithNameSkipCheck) []string { paths := make([]string, 0, 1) filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { @@ -160,7 +169,7 @@ func FindAllFilesWithName(root string, name string, dirsToSkip ...string) []stri } if d.IsDir() { for _, dirToSkip := range dirsToSkip { - if path == dirToSkip { + if dirToSkip(path) { return filepath.SkipDir } } From 74cea2fedfe8b64ddca5ecb672a2f3f51a7a0d55 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Mon, 4 Nov 2024 12:44:42 +0000 Subject: [PATCH 3/3] Go: Add `vendor` directory to `mixed-layout` test with `go.work` file The `go.work` file here should not get discovered by the autobuilder --- .../mixed-layout/src/module/vendor/example.com/test/add.go | 5 +++++ .../mixed-layout/src/module/vendor/example.com/test/go.work | 3 +++ .../mixed-layout/src/module/vendor/modules.txt | 3 +++ .../integration-tests/mixed-layout/src/workspace/go.work.sum | 3 +++ 4 files changed, 14 insertions(+) create mode 100644 go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/add.go create mode 100644 go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/go.work create mode 100644 go/ql/integration-tests/mixed-layout/src/module/vendor/modules.txt create mode 100644 go/ql/integration-tests/mixed-layout/src/workspace/go.work.sum diff --git a/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/add.go b/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/add.go new file mode 100644 index 000000000000..b1ce6a2a3a39 --- /dev/null +++ b/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/add.go @@ -0,0 +1,5 @@ +package test + +func Add(a, b int) int { + return a + b +} diff --git a/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/go.work b/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/go.work new file mode 100644 index 000000000000..96b89a39cb9a --- /dev/null +++ b/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/go.work @@ -0,0 +1,3 @@ +go 1.22.0 + +use . diff --git a/go/ql/integration-tests/mixed-layout/src/module/vendor/modules.txt b/go/ql/integration-tests/mixed-layout/src/module/vendor/modules.txt new file mode 100644 index 000000000000..023bcb386e2d --- /dev/null +++ b/go/ql/integration-tests/mixed-layout/src/module/vendor/modules.txt @@ -0,0 +1,3 @@ +# example.com/test v0.1.0 +## explicit; go 1.14 +example.com/test diff --git a/go/ql/integration-tests/mixed-layout/src/workspace/go.work.sum b/go/ql/integration-tests/mixed-layout/src/workspace/go.work.sum new file mode 100644 index 000000000000..d89ea795aab4 --- /dev/null +++ b/go/ql/integration-tests/mixed-layout/src/workspace/go.work.sum @@ -0,0 +1,3 @@ +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=