Merge pull request #16925 from github/mbg/go/add-vendor-env-var

Go: Add environment variable to include `vendor` directories in extraction

go/extractor/extractor.go

@@ -193,10 +193,20 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 	log.Println("Starting to extract packages.")
 
 	sep := regexp.QuoteMeta(string(filepath.Separator))
-	// if a path matches this regexp, we don't want to extract this package. Currently, it checks
-	//   - that the path does not contain a `..` segment, and
-	//   - the path does not contain a `vendor` directory.
-	noExtractRe := regexp.MustCompile(`.*(^|` + sep + `)(\.\.|vendor)($|` + sep + `).*`)
+
+	// Construct a list of directory segments to exclude from extraction, starting with ".."
+	excludedDirs := []string{`\.\.`}
+
+	// If CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS is "true", we extract `vendor` directories;
+	// otherwise (the default) is to exclude them from extraction
+	includeVendor := os.Getenv("CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS") == "true"
+	if !includeVendor {
+		excludedDirs = append(excludedDirs, "vendor")
+	}
+
+	// If a path matches this regexp, we don't extract this package. It checks whether the path
+	// contains one of the `excludedDirs`.
+	noExtractRe := regexp.MustCompile(`.*(^|` + sep + `)(` + strings.Join(excludedDirs, "|") + `)($|` + sep + `).*`)
 
 	// extract AST information for all packages
 	packages.Visit(pkgs, nil, func(pkg *packages.Package) {

go/ql/integration-tests/all-platforms/go/extract-vendor/build-environment.expected

@@ -0,0 +1,5 @@
+{
+  "configuration" : {
+    "go" : { }
+  }
+}

go/ql/integration-tests/all-platforms/go/extract-vendor/diagnostics.expected

@@ -0,0 +1,14 @@
+{
+  "markdownMessage": "A single `go.mod` file was found.\n\n`go.mod`",
+  "severity": "note",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/single-root-go-mod-found",
+    "name": "A single `go.mod` file was found in the root"
+  },
+  "visibility": {
+    "cliSummaryTable": false,
+    "statusPage": false,
+    "telemetry": true
+  }
+}

go/ql/integration-tests/all-platforms/go/extract-vendor/force_sequential_test_execution

@@ -0,0 +1,2 @@
+# go get has been observed to sometimes fail when multiple tests try to simultaneously fetch the same package.
+goget

go/ql/integration-tests/all-platforms/go/extract-vendor/src/go.mod

@@ -0,0 +1,5 @@
+go 1.14
+
+require example.com/test v0.1.0
+
+module test

go/ql/integration-tests/all-platforms/go/extract-vendor/src/go.sum

@@ -0,0 +1 @@
+example.com/test v0.1.0/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=

go/ql/integration-tests/all-platforms/go/extract-vendor/src/test.go

@@ -0,0 +1,11 @@
+package test
+
+import (
+	subdir "example.com/test"
+)
+
+func Test() {
+
+	foo := subdir.Add(2, 2)
+	println(foo)
+}

go/ql/integration-tests/all-platforms/go/extract-vendor/src/vendor/modules.txt

@@ -0,0 +1,3 @@
+# example.com/test v0.1.0
+## explicit; go 1.14
+example.com/test

go/ql/integration-tests/all-platforms/go/extract-vendor/test.expected

@@ -0,0 +1,5 @@
+extractedFiles
+| src/go.mod:0:0:0:0 | src/go.mod |
+| src/test.go:0:0:0:0 | src/test.go |
+| src/vendor/example.com/test/add.go:0:0:0:0 | src/vendor/example.com/test/add.go |
+#select

go/ql/integration-tests/all-platforms/go/extract-vendor/test.py

@@ -0,0 +1,4 @@
+from go_integration_test import *
+
+os.environ['CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS'] = "true"
+go_integration_test()

go/ql/integration-tests/all-platforms/go/extract-vendor/test.ql

@@ -0,0 +1,8 @@
+import go
+import semmle.go.DiagnosticsReporting
+
+query predicate extractedFiles(File f) { any() }
+
+from string msg, int sev
+where reportableDiagnostics(_, msg, sev)
+select msg, sev