[do not merge] Dummy PR for Go test extraction testing

go/codeql-extractor.yml

@@ -19,3 +19,11 @@ file_types:
     extensions:
       - .go
 legacy_qltest_extraction: true
+options:
+  extract_tests:
+    title: Whether to include Go test files and functions in the CodeQL database.
+    description: >
+      A value indicating whether Go test files and functions should be included in the CodeQL database.
+      The default is 'false'.
+    type: string
+    pattern: "^(false|true)$"

go/codeql-tools/autobuild.sh

@@ -11,6 +11,9 @@ fi
 LGTM_SRC="$(pwd)"
 export LGTM_SRC
 
+CODEQL_EXTRACTOR_GO_OPTION_EXTRACT_TESTS=true
+export CODEQL_EXTRACTOR_GO_OPTION_EXTRACT_TESTS
+
 if [ "${CODEQL_EXTRACTOR_GO_BUILD_TRACING:-}" = "on" ]; then
   echo "Tracing enabled"
   "$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-build-runner"

go/extractor/cli/go-extractor/go-extractor.go

@@ -21,7 +21,7 @@ func usage() {
 	fmt.Fprintf(os.Stderr, "--help                Print this help.\n")
 }
 
-func parseFlags(args []string, mimic bool) ([]string, []string) {
+func parseFlags(args []string, mimic bool, extractTests bool) ([]string, []string, bool) {
 	i := 0
 	buildFlags := []string{}
 	for ; i < len(args) && strings.HasPrefix(args[i], "-"); i++ {
@@ -44,9 +44,9 @@ func parseFlags(args []string, mimic bool) ([]string, []string) {
 					if i+1 < len(args) {
 						i++
 						command := args[i]
-						if command == "build" || command == "install" || command == "run" {
-							log.Printf("Intercepting build")
-							return parseFlags(args[i+1:], true)
+						if command == "build" || command == "install" || command == "run" || command == "test" {
+							log.Printf("Intercepting build for %s command", command)
+							return parseFlags(args[i+1:], true, command == "test")
 						} else {
 							log.Printf("Non-build command '%s'; skipping", strings.Join(args[1:], " "))
 							os.Exit(0)
@@ -63,12 +63,12 @@ func parseFlags(args []string, mimic bool) ([]string, []string) {
 
 		// parse go build flags
 		switch args[i] {
-		// skip `-o output` and `-i`, if applicable
+		// skip `-o output`, `-i` and `-c`, if applicable
 		case "-o":
 			if i+1 < len(args) {
 				i++
 			}
-		case "-i":
+		case "-i", "-c":
 		case "-p", "-asmflags", "-buildmode", "-compiler", "-gccgoflags", "-gcflags", "-installsuffix",
 			"-ldflags", "-mod", "-modfile", "-pkgdir", "-tags", "-toolexec", "-overlay":
 			if i+1 < len(args) {
@@ -90,11 +90,12 @@ func parseFlags(args []string, mimic bool) ([]string, []string) {
 	cpuprofile = os.Getenv("CODEQL_EXTRACTOR_GO_CPU_PROFILE")
 	memprofile = os.Getenv("CODEQL_EXTRACTOR_GO_MEM_PROFILE")
 
-	return buildFlags, args[i:]
+	return buildFlags, args[i:], extractTests
 }
 
 func main() {
-	buildFlags, patterns := parseFlags(os.Args[1:], false)
+	extractTestsDefault := os.Getenv("CODEQL_EXTRACTOR_GO_OPTION_EXTRACT_TESTS") == "true"
+	buildFlags, patterns, extractTests := parseFlags(os.Args[1:], false, extractTestsDefault)
 
 	if cpuprofile != "" {
 		f, err := os.Create(cpuprofile)
@@ -114,7 +115,7 @@ func main() {
 	}
 
 	log.Printf("Build flags: '%s'; patterns: '%s'\n", strings.Join(buildFlags, " "), strings.Join(patterns, " "))
-	err := extractor.ExtractWithFlags(buildFlags, patterns)
+	err := extractor.ExtractWithFlags(buildFlags, patterns, extractTests)
 	if err != nil {
 		errString := err.Error()
 		if strings.Contains(errString, "unexpected directory layout:") {

go/extractor/extractor.go

@@ -59,11 +59,11 @@ func init() {
 
 // Extract extracts the packages specified by the given patterns
 func Extract(patterns []string) error {
-	return ExtractWithFlags(nil, patterns)
+	return ExtractWithFlags(nil, patterns, false)
 }
 
 // ExtractWithFlags extracts the packages specified by the given patterns and build flags
-func ExtractWithFlags(buildFlags []string, patterns []string) error {
+func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool) error {
 	startTime := time.Now()
 
 	extraction := NewExtraction(buildFlags, patterns)
@@ -81,14 +81,22 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 		}
 	}
 
-	log.Println("Running packages.Load.")
+	testMessage := ""
+	if extractTests {
+		testMessage = " (test extraction enabled)"
+	}
+	log.Printf("Running packages.Load%s.", testMessage)
+
+	// This includes test packages if either we're tracing a `go test` command,
+	// or if CODEQL_EXTRACTOR_GO_OPTION_EXTRACT_TESTS is set to "true".
 	cfg := &packages.Config{
 		Mode: packages.NeedName | packages.NeedFiles |
 			packages.NeedCompiledGoFiles |
 			packages.NeedImports | packages.NeedDeps |
 			packages.NeedTypes | packages.NeedTypesSizes |
 			packages.NeedTypesInfo | packages.NeedSyntax,
 		BuildFlags: buildFlags,
+		Tests:      extractTests,
 	}
 	pkgs, err := packages.Load(cfg, patterns...)
 	if err != nil {
@@ -123,7 +131,7 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 	if os.Getenv("CODEQL_EXTRACTOR_GO_FAST_PACKAGE_INFO") != "false" {
 		log.Printf("Running go list to resolve package and module directories.")
 		// get all packages information
-		pkgInfos, err = toolchain.GetPkgsInfo(patterns, true, modFlags...)
+		pkgInfos, err = toolchain.GetPkgsInfo(patterns, true, extractTests, modFlags...)
 		if err != nil {
 			log.Fatalf("Error getting dependency package or module directories: %v.", err)
 		}
@@ -132,8 +140,36 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 
 	pkgsNotFound := make([]string, 0, len(pkgs))
 
+	// Build a map from package paths to their longest IDs--
+	// in the context of a `go test -c` compilation, we will see the same package more than
+	// once, with IDs like "abc.com/pkgname [abc.com/pkgname.test]" to distinguish the version
+	// that contains and is used by test code.
+	// For our purposes it is simplest to just ignore the non-test version, since the test
+	// version seems to be a superset of it.
+	longestPackageIds := make(map[string]string)
+	packages.Visit(pkgs, nil, func(pkg *packages.Package) {
+		if longestIDSoFar, present := longestPackageIds[pkg.PkgPath]; present {
+			if len(pkg.ID) > len(longestIDSoFar) {
+				longestPackageIds[pkg.PkgPath] = pkg.ID
+			}
+		} else {
+			longestPackageIds[pkg.PkgPath] = pkg.ID
+		}
+	})
+
 	// Do a post-order traversal and extract the package scope of each package
 	packages.Visit(pkgs, nil, func(pkg *packages.Package) {
+		// Note that if test extraction is enabled, we will encounter a package twice here:
+		// once as the main package, and once as the test package (with a package ID like
+		// "abc.com/pkgname [abc.com/pkgname.test]").
+		//
+		// We will extract it both times however, because we need to visit the packages
+		// in the right order in order to visit used types before their users, and the
+		// ordering determined by packages.Visit for the main and the test package may differ.
+		//
+		// This should only cause some wasted time and not inconsistency because the names for
+		// objects seen in this process should be the same each time.
+
 		log.Printf("Processing package %s.", pkg.PkgPath)
 
 		if _, ok := pkgInfos[pkg.PkgPath]; !ok {
@@ -210,6 +246,19 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
 
 	// extract AST information for all packages
 	packages.Visit(pkgs, nil, func(pkg *packages.Package) {
+
+		// If this is a variant of a package that also occurs with a longer ID, skip it;
+		// otherwise we would extract the same file more than once including extracting the
+		// body of methods twice, causing database inconsistencies.
+		//
+		// We prefer the version with the longest ID because that is (so far as I know) always
+		// the version that defines more entities -- the only case I'm aware of being a test
+		// variant of a package, which includes test-only functions in addition to the complete
+		// contents of the main variant.
+		if pkg.ID != longestPackageIds[pkg.PkgPath] {
+			return
+		}
+
 		for root := range wantedRoots {
 			pkgInfo := pkgInfos[pkg.PkgPath]
 			relDir, err := filepath.Rel(root, pkgInfo.PkgDir)

go/extractor/toolchain/toolchain.go

@@ -223,14 +223,18 @@ type PkgInfo struct {
 // GetPkgsInfo gets the absolute module and package root directories for the packages matched by the
 // patterns `patterns`. It passes to `go list` the flags specified by `flags`.  If `includingDeps`
 // is true, all dependencies will also be included.
-func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[string]PkgInfo, error) {
+func GetPkgsInfo(patterns []string, includingDeps bool, extractTests bool, flags ...string) (map[string]PkgInfo, error) {
 	// enable module mode so that we can find a module root if it exists, even if go module support is
 	// disabled by a build
 	if includingDeps {
 		// the flag `-deps` causes all dependencies to be retrieved
 		flags = append(flags, "-deps")
 	}
 
+	if extractTests {
+		flags = append(flags, "-test")
+	}
+
 	// using -json overrides -f format
 	output, err := RunList("", patterns, append(flags, "-json")...)
 	if err != nil {
@@ -272,6 +276,12 @@ func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[st
 			PkgDir: pkgAbsDir,
 			ModDir: modAbsDir,
 		}
+
+		if extractTests && strings.Contains(pkgInfo.ImportPath, " [") {
+			// Assume " [" is the start of a qualifier, and index the package by its base name
+			baseImportPath := strings.Split(pkgInfo.ImportPath, " [")[0]
+			pkgInfoMapping[baseImportPath] = pkgInfoMapping[pkgInfo.ImportPath]
+		}
 	}
 	return pkgInfoMapping, nil
 }

go/ql/integration-tests/go-mod-sample/src/blackbox_test.go

@@ -0,0 +1,16 @@
+package makesample_test
+
+import (
+	"makesample"
+	"testing"
+)
+
+// Note because this is a test we do NOT expect this to be extracted.
+func TestTestMe(t *testing.T) {
+
+	publicResult := makesample.PublicFunction()
+	if publicResult != 1 {
+		t.Errorf("Expected 1, got %d", publicResult)
+	}
+
+}

go/ql/integration-tests/go-mod-sample/src/test.go

@@ -10,3 +10,7 @@ func test() {
 	header.Version = 4
 
 }
+
+func PublicFunction() int {
+	return 1
+}

go/ql/integration-tests/go-mod-sample/src/test_test.go

@@ -0,0 +1,15 @@
+package makesample
+
+import (
+	"testing"
+)
+
+func TestTestMe(t *testing.T) {
+
+	// Note because this is a test we do NOT expect this to be extracted.
+	publicResult := PublicFunction()
+	if publicResult != 1 {
+		t.Errorf("Expected 1, got %d", publicResult)
+	}
+
+}

go/ql/integration-tests/test-extraction-autobuild/src/Makefile

@@ -0,0 +1,2 @@
+all:
+	go get

go/ql/integration-tests/test-extraction-autobuild/src/go.mod

@@ -0,0 +1,3 @@
+go 1.14
+
+module testsample

go/ql/integration-tests/test-extraction-autobuild/src/go.sum

@@ -0,0 +1,45 @@
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

go/ql/integration-tests/test-extraction-autobuild/src/testme.go

@@ -0,0 +1,5 @@
+package testsample
+
+func PublicFunction() int { return 1 }
+
+func privateFunction() int { return 2 }

go/ql/integration-tests/test-extraction-autobuild/src/testme_blackbox_test.go

@@ -0,0 +1,15 @@
+package testsample_test
+
+import (
+	"testing"
+	"testsample"
+)
+
+func TestTestMe(t *testing.T) {
+
+	publicResult := testsample.PublicFunction()
+	if publicResult != 1 {
+		t.Errorf("Expected 1, got %d", publicResult)
+	}
+
+}

go/ql/integration-tests/test-extraction-autobuild/src/testme_test.go

@@ -0,0 +1,19 @@
+package testsample
+
+import (
+	"testing"
+)
+
+func TestTestMe(t *testing.T) {
+
+	publicResult := PublicFunction()
+	if publicResult != 1 {
+		t.Errorf("Expected 1, got %d", publicResult)
+	}
+
+	privateResult := privateFunction()
+	if privateResult != 2 {
+		t.Errorf("Expected 2, got %d", privateResult)
+	}
+
+}

go/ql/integration-tests/test-extraction-autobuild/test.expected

@@ -0,0 +1,9 @@
+#select
+| src/testme.go:0:0:0:0 | src/testme.go |
+| src/testme_blackbox_test.go:0:0:0:0 | src/testme_blackbox_test.go |
+| src/testme_test.go:0:0:0:0 | src/testme_test.go |
+calls
+| src/testme_blackbox_test.go:10:18:10:44 | call to PublicFunction | src/testme.go:3:1:3:38 | function declaration |
+| src/testme_test.go:9:18:9:33 | call to PublicFunction | src/testme.go:3:1:3:38 | function declaration |
+| src/testme_test.go:14:19:14:35 | call to privateFunction | src/testme.go:5:1:5:39 | function declaration |
+extractionErrors

go/ql/integration-tests/test-extraction-autobuild/test.py

@@ -0,0 +1,4 @@
+import os
+
+def test(codeql, go):
+    codeql.database.create(source_root="src", extractor_option = ["extract_tests=true"])

go/ql/integration-tests/test-extraction-autobuild/test.ql

@@ -0,0 +1,9 @@
+import go
+import semmle.go.DiagnosticsReporting
+
+from GoFile f
+select f
+
+query predicate calls(CallExpr ce, FuncDecl f) { f = ce.getTarget().getFuncDecl() }
+
+query predicate extractionErrors(string msg, int sev) { reportableDiagnostics(_, msg, sev) }

go/ql/integration-tests/test-extraction-traced/src/Makefile

@@ -0,0 +1,2 @@
+all:
+	go get

go/ql/integration-tests/test-extraction-traced/src/go.mod

@@ -0,0 +1,3 @@
+go 1.14
+
+module testsample