From bc61a58000976fb6f551bafccbad632ffa1127c3 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Mon, 8 Jul 2024 14:05:06 +0100 Subject: [PATCH 1/2] Go: Add integration test for extracting vendored dependencies --- .../go/extract-vendor/build-environment.expected | 5 +++++ .../go/extract-vendor/diagnostics.expected | 14 ++++++++++++++ .../extract-vendor/force_sequential_test_execution | 2 ++ .../all-platforms/go/extract-vendor/src/go.mod | 5 +++++ .../all-platforms/go/extract-vendor/src/go.sum | 1 + .../all-platforms/go/extract-vendor/src/test.go | 11 +++++++++++ .../src/vendor/example.com/test/add.go | 5 +++++ .../go/extract-vendor/src/vendor/modules.txt | 3 +++ .../all-platforms/go/extract-vendor/test.expected | 5 +++++ .../all-platforms/go/extract-vendor/test.py | 4 ++++ .../all-platforms/go/extract-vendor/test.ql | 8 ++++++++ 11 files changed, 63 insertions(+) create mode 100644 go/ql/integration-tests/all-platforms/go/extract-vendor/build-environment.expected create mode 100644 go/ql/integration-tests/all-platforms/go/extract-vendor/diagnostics.expected create mode 100644 go/ql/integration-tests/all-platforms/go/extract-vendor/force_sequential_test_execution create mode 100644 go/ql/integration-tests/all-platforms/go/extract-vendor/src/go.mod create mode 100644 go/ql/integration-tests/all-platforms/go/extract-vendor/src/go.sum create mode 100644 go/ql/integration-tests/all-platforms/go/extract-vendor/src/test.go create mode 100644 go/ql/integration-tests/all-platforms/go/extract-vendor/src/vendor/example.com/test/add.go create mode 100644 go/ql/integration-tests/all-platforms/go/extract-vendor/src/vendor/modules.txt create mode 100644 go/ql/integration-tests/all-platforms/go/extract-vendor/test.expected create mode 100644 go/ql/integration-tests/all-platforms/go/extract-vendor/test.py create mode 100644 go/ql/integration-tests/all-platforms/go/extract-vendor/test.ql diff --git a/go/ql/integration-tests/all-platforms/go/extract-vendor/build-environment.expected b/go/ql/integration-tests/all-platforms/go/extract-vendor/build-environment.expected new file mode 100644 index 000000000000..0b225ce00857 --- /dev/null +++ b/go/ql/integration-tests/all-platforms/go/extract-vendor/build-environment.expected @@ -0,0 +1,5 @@ +{ + "configuration" : { + "go" : { } + } +} diff --git a/go/ql/integration-tests/all-platforms/go/extract-vendor/diagnostics.expected b/go/ql/integration-tests/all-platforms/go/extract-vendor/diagnostics.expected new file mode 100644 index 000000000000..56d774b7037c --- /dev/null +++ b/go/ql/integration-tests/all-platforms/go/extract-vendor/diagnostics.expected @@ -0,0 +1,14 @@ +{ + "markdownMessage": "A single `go.mod` file was found.\n\n`go.mod`", + "severity": "note", + "source": { + "extractorName": "go", + "id": "go/autobuilder/single-root-go-mod-found", + "name": "A single `go.mod` file was found in the root" + }, + "visibility": { + "cliSummaryTable": false, + "statusPage": false, + "telemetry": true + } +} diff --git a/go/ql/integration-tests/all-platforms/go/extract-vendor/force_sequential_test_execution b/go/ql/integration-tests/all-platforms/go/extract-vendor/force_sequential_test_execution new file mode 100644 index 000000000000..47ca99290999 --- /dev/null +++ b/go/ql/integration-tests/all-platforms/go/extract-vendor/force_sequential_test_execution @@ -0,0 +1,2 @@ +# go get has been observed to sometimes fail when multiple tests try to simultaneously fetch the same package. +goget diff --git a/go/ql/integration-tests/all-platforms/go/extract-vendor/src/go.mod b/go/ql/integration-tests/all-platforms/go/extract-vendor/src/go.mod new file mode 100644 index 000000000000..bfb907e7b813 --- /dev/null +++ b/go/ql/integration-tests/all-platforms/go/extract-vendor/src/go.mod @@ -0,0 +1,5 @@ +go 1.14 + +require example.com/test v0.1.0 + +module test diff --git a/go/ql/integration-tests/all-platforms/go/extract-vendor/src/go.sum b/go/ql/integration-tests/all-platforms/go/extract-vendor/src/go.sum new file mode 100644 index 000000000000..77b7c845ca63 --- /dev/null +++ b/go/ql/integration-tests/all-platforms/go/extract-vendor/src/go.sum @@ -0,0 +1 @@ +example.com/test v0.1.0/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= diff --git a/go/ql/integration-tests/all-platforms/go/extract-vendor/src/test.go b/go/ql/integration-tests/all-platforms/go/extract-vendor/src/test.go new file mode 100644 index 000000000000..1939e3478d48 --- /dev/null +++ b/go/ql/integration-tests/all-platforms/go/extract-vendor/src/test.go @@ -0,0 +1,11 @@ +package test + +import ( + subdir "example.com/test" +) + +func Test() { + + foo := subdir.Add(2, 2) + println(foo) +} diff --git a/go/ql/integration-tests/all-platforms/go/extract-vendor/src/vendor/example.com/test/add.go b/go/ql/integration-tests/all-platforms/go/extract-vendor/src/vendor/example.com/test/add.go new file mode 100644 index 000000000000..b1ce6a2a3a39 --- /dev/null +++ b/go/ql/integration-tests/all-platforms/go/extract-vendor/src/vendor/example.com/test/add.go @@ -0,0 +1,5 @@ +package test + +func Add(a, b int) int { + return a + b +} diff --git a/go/ql/integration-tests/all-platforms/go/extract-vendor/src/vendor/modules.txt b/go/ql/integration-tests/all-platforms/go/extract-vendor/src/vendor/modules.txt new file mode 100644 index 000000000000..023bcb386e2d --- /dev/null +++ b/go/ql/integration-tests/all-platforms/go/extract-vendor/src/vendor/modules.txt @@ -0,0 +1,3 @@ +# example.com/test v0.1.0 +## explicit; go 1.14 +example.com/test diff --git a/go/ql/integration-tests/all-platforms/go/extract-vendor/test.expected b/go/ql/integration-tests/all-platforms/go/extract-vendor/test.expected new file mode 100644 index 000000000000..d03518bd540f --- /dev/null +++ b/go/ql/integration-tests/all-platforms/go/extract-vendor/test.expected @@ -0,0 +1,5 @@ +extractedFiles +| src/go.mod:0:0:0:0 | src/go.mod | +| src/test.go:0:0:0:0 | src/test.go | +| src/vendor/example.com/test/add.go:0:0:0:0 | src/vendor/example.com/test/add.go | +#select diff --git a/go/ql/integration-tests/all-platforms/go/extract-vendor/test.py b/go/ql/integration-tests/all-platforms/go/extract-vendor/test.py new file mode 100644 index 000000000000..2bd482201b8b --- /dev/null +++ b/go/ql/integration-tests/all-platforms/go/extract-vendor/test.py @@ -0,0 +1,4 @@ +from go_integration_test import * + +os.environ['CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS'] = "true" +go_integration_test() diff --git a/go/ql/integration-tests/all-platforms/go/extract-vendor/test.ql b/go/ql/integration-tests/all-platforms/go/extract-vendor/test.ql new file mode 100644 index 000000000000..459a43015602 --- /dev/null +++ b/go/ql/integration-tests/all-platforms/go/extract-vendor/test.ql @@ -0,0 +1,8 @@ +import go +import semmle.go.DiagnosticsReporting + +query predicate extractedFiles(File f) { any() } + +from string msg, int sev +where reportableDiagnostics(_, msg, sev) +select msg, sev From 7ca57e114f0d132c08ffe7369486d094ddf22454 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Mon, 8 Jul 2024 14:08:19 +0100 Subject: [PATCH 2/2] Go: Add `CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS` env var If set to `true`, this allows `vendor` directories to be extracted --- go/extractor/extractor.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index 090bd486c3a7..df3a43f80cfa 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -193,10 +193,20 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error { log.Println("Starting to extract packages.") sep := regexp.QuoteMeta(string(filepath.Separator)) - // if a path matches this regexp, we don't want to extract this package. Currently, it checks - // - that the path does not contain a `..` segment, and - // - the path does not contain a `vendor` directory. - noExtractRe := regexp.MustCompile(`.*(^|` + sep + `)(\.\.|vendor)($|` + sep + `).*`) + + // Construct a list of directory segments to exclude from extraction, starting with ".." + excludedDirs := []string{`\.\.`} + + // If CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS is "true", we extract `vendor` directories; + // otherwise (the default) is to exclude them from extraction + includeVendor := os.Getenv("CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS") == "true" + if !includeVendor { + excludedDirs = append(excludedDirs, "vendor") + } + + // If a path matches this regexp, we don't extract this package. It checks whether the path + // contains one of the `excludedDirs`. + noExtractRe := regexp.MustCompile(`.*(^|` + sep + `)(` + strings.Join(excludedDirs, "|") + `)($|` + sep + `).*`) // extract AST information for all packages packages.Visit(pkgs, nil, func(pkg *packages.Package) {