From 76e694259427dfa87792b5871da495906bdaa0df Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Tue, 13 Aug 2024 19:24:28 +0100 Subject: [PATCH 01/11] Go: support extracting test code This implements support for test extraction by two mechanisms: * In autobuild mode, setting `CODEQL_EXTRACTOR_GO_EXTRACT_TESTS` to `true`. * In manual build mode, tracing a `go test` command (`go test -c` is to be recommended for efficiency). Go deals with test compilation by creating several extra packages on top of those expected from inspection of the source code (see docs of `packages.Load` for more detail): packages whose IDs include a suffix like `mydomain.com/mypackage [mydomain.com/mypackage.test]`, and packages containing generated test driver code like `mydomain.com/mypackage.test`. There are also additional packages like `mydomain.com/mypackage_tests` which are explicitly present in source code, but not compiled by a normal `go build`. So far as I can tell, the purpose of the two variants of the package is to resolve dependency cycles (because the tests variant of the package can have more dependencies than the non-tests variant, and non-test code can compile against non-test package variants). Since the test package variants seems to be a superset of the non-tests variant, I employ the simple heuristic of ignoring the variant of each package with the shortest ID. I haven't seen a case where there are three or more variants of a package, so I expect this to always identify the tests variant as the preferred one. If several variants were extracted, and we were to attempt to match Golang's linkage strategy among the different variants, we would need to extend trap-file name and most top-level symbol trap IDs with the package variant they come from; I hope this won't prove necessary. "Real" `_tests` packages, and wholly synthetic driver code packages, are extracted just like normal. --- go/extractor/cli/go-extractor/go-extractor.go | 19 +++++----- go/extractor/extractor.go | 35 +++++++++++++++++-- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/go/extractor/cli/go-extractor/go-extractor.go b/go/extractor/cli/go-extractor/go-extractor.go index 00d3c8e7de13..2f40d2980cfa 100644 --- a/go/extractor/cli/go-extractor/go-extractor.go +++ b/go/extractor/cli/go-extractor/go-extractor.go @@ -21,7 +21,7 @@ func usage() { fmt.Fprintf(os.Stderr, "--help Print this help.\n") } -func parseFlags(args []string, mimic bool) ([]string, []string) { +func parseFlags(args []string, mimic bool, extractTests bool) ([]string, []string, bool) { i := 0 buildFlags := []string{} for ; i < len(args) && strings.HasPrefix(args[i], "-"); i++ { @@ -44,9 +44,9 @@ func parseFlags(args []string, mimic bool) ([]string, []string) { if i+1 < len(args) { i++ command := args[i] - if command == "build" || command == "install" || command == "run" { - log.Printf("Intercepting build") - return parseFlags(args[i+1:], true) + if command == "build" || command == "install" || command == "run" || command == "test" { + log.Printf("Intercepting build for %s command", command) + return parseFlags(args[i+1:], true, command == "test") } else { log.Printf("Non-build command '%s'; skipping", strings.Join(args[1:], " ")) os.Exit(0) @@ -63,12 +63,12 @@ func parseFlags(args []string, mimic bool) ([]string, []string) { // parse go build flags switch args[i] { - // skip `-o output` and `-i`, if applicable + // skip `-o output`, `-i` and `-c`, if applicable case "-o": if i+1 < len(args) { i++ } - case "-i": + case "-i", "-c": case "-p", "-asmflags", "-buildmode", "-compiler", "-gccgoflags", "-gcflags", "-installsuffix", "-ldflags", "-mod", "-modfile", "-pkgdir", "-tags", "-toolexec", "-overlay": if i+1 < len(args) { @@ -90,11 +90,12 @@ func parseFlags(args []string, mimic bool) ([]string, []string) { cpuprofile = os.Getenv("CODEQL_EXTRACTOR_GO_CPU_PROFILE") memprofile = os.Getenv("CODEQL_EXTRACTOR_GO_MEM_PROFILE") - return buildFlags, args[i:] + return buildFlags, args[i:], extractTests } func main() { - buildFlags, patterns := parseFlags(os.Args[1:], false) + extractTestsDefault := os.Getenv("CODEQL_EXTRACTOR_GO_EXTRACT_TESTS") == "true" + buildFlags, patterns, extractTests := parseFlags(os.Args[1:], false, extractTestsDefault) if cpuprofile != "" { f, err := os.Create(cpuprofile) @@ -114,7 +115,7 @@ func main() { } log.Printf("Build flags: '%s'; patterns: '%s'\n", strings.Join(buildFlags, " "), strings.Join(patterns, " ")) - err := extractor.ExtractWithFlags(buildFlags, patterns) + err := extractor.ExtractWithFlags(buildFlags, patterns, extractTests) if err != nil { errString := err.Error() if strings.Contains(errString, "unexpected directory layout:") { diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index 31fa2ceb4131..d10e0d38d8f5 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -59,11 +59,11 @@ func init() { // Extract extracts the packages specified by the given patterns func Extract(patterns []string) error { - return ExtractWithFlags(nil, patterns) + return ExtractWithFlags(nil, patterns, false) } // ExtractWithFlags extracts the packages specified by the given patterns and build flags -func ExtractWithFlags(buildFlags []string, patterns []string) error { +func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool) error { startTime := time.Now() extraction := NewExtraction(buildFlags, patterns) @@ -89,6 +89,7 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error { packages.NeedTypes | packages.NeedTypesSizes | packages.NeedTypesInfo | packages.NeedSyntax, BuildFlags: buildFlags, + Tests: extractTests, } pkgs, err := packages.Load(cfg, patterns...) if err != nil { @@ -132,10 +133,33 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error { pkgsNotFound := make([]string, 0, len(pkgs)) + // Build a map from package paths to their longest IDs-- + // in the context of a `go test -c` compilation, we will see the same package more than + // once, with IDs like "abc.com/pkgname [abc.com/pkgname.test]" to distinguish the version + // that contains and is used by test code. + // For our purposes it is simplest to just ignore the non-test version, since the test + // version seems to be a superset of it. + longestPackageIds := make(map[string]string) + packages.Visit(pkgs, nil, func(pkg *packages.Package) { + if shortestID, present := longestPackageIds[pkg.PkgPath]; present { + if len(pkg.ID) > len(shortestID) { + longestPackageIds[pkg.PkgPath] = pkg.ID + } + } else { + longestPackageIds[pkg.PkgPath] = pkg.ID + } + }) + // Do a post-order traversal and extract the package scope of each package packages.Visit(pkgs, nil, func(pkg *packages.Package) { log.Printf("Processing package %s.", pkg.PkgPath) + // If this is a variant of a package that also occurs with a longer ID, skip it. + if pkg.ID != longestPackageIds[pkg.PkgPath] { + log.Printf("Skipping variant of package %s with ID %s.", pkg.PkgPath, pkg.ID) + return + } + if _, ok := pkgInfos[pkg.PkgPath]; !ok { pkgInfos[pkg.PkgPath] = toolchain.GetPkgInfo(pkg.PkgPath, modFlags...) } @@ -210,6 +234,13 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error { // extract AST information for all packages packages.Visit(pkgs, nil, func(pkg *packages.Package) { + + // If this is a variant of a package that also occurs with a longer ID, skip it. + if pkg.ID != longestPackageIds[pkg.PkgPath] { + // Don't log here; we already mentioned this above. + return + } + for root := range wantedRoots { pkgInfo := pkgInfos[pkg.PkgPath] relDir, err := filepath.Rel(root, pkgInfo.PkgDir) From c3dffc955bfd97507b2bafe8de90754f19c669f7 Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Wed, 21 Aug 2024 11:42:07 +0100 Subject: [PATCH 02/11] Apply review comments --- go/extractor/extractor.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index d10e0d38d8f5..68e64d46f905 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -81,7 +81,14 @@ func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool) } } - log.Println("Running packages.Load.") + testMessage := "" + if extractTests { + testMessage = " (test extraction enabled)" + } + log.Printf("Running packages.Load%s.", testMessage) + + // This includes test packages if either we're tracing a `go test` command, + // or if CODEQL_EXTRACTOR_GO_EXTRACT_TESTS is set to "true". cfg := &packages.Config{ Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | @@ -141,8 +148,8 @@ func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool) // version seems to be a superset of it. longestPackageIds := make(map[string]string) packages.Visit(pkgs, nil, func(pkg *packages.Package) { - if shortestID, present := longestPackageIds[pkg.PkgPath]; present { - if len(pkg.ID) > len(shortestID) { + if longestIDSoFar, present := longestPackageIds[pkg.PkgPath]; present { + if len(pkg.ID) > len(longestIDSoFar) { longestPackageIds[pkg.PkgPath] = pkg.ID } } else { @@ -154,7 +161,7 @@ func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool) packages.Visit(pkgs, nil, func(pkg *packages.Package) { log.Printf("Processing package %s.", pkg.PkgPath) - // If this is a variant of a package that also occurs with a longer ID, skip it. + // If this is a variant of a package that also occurs with a shorter ID, skip it. if pkg.ID != longestPackageIds[pkg.PkgPath] { log.Printf("Skipping variant of package %s with ID %s.", pkg.PkgPath, pkg.ID) return From fd592fa18f325111711c590c05bfd8eccf6b939e Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Wed, 21 Aug 2024 12:27:34 +0100 Subject: [PATCH 03/11] Add tests --- .../go-mod-sample/src/blackbox_test.go | 16 +++++++ .../go-mod-sample/src/test.go | 4 ++ .../go-mod-sample/src/test_test.go | 15 +++++++ .../test-extraction-autobuild/src/Makefile | 2 + .../test-extraction-autobuild/src/go.mod | 3 ++ .../test-extraction-autobuild/src/go.sum | 45 +++++++++++++++++++ .../test-extraction-autobuild/src/testme.go | 5 +++ .../src/testme_blackbox_test.go | 15 +++++++ .../src/testme_test.go | 19 ++++++++ .../test-extraction-autobuild/test.expected | 9 ++++ .../test-extraction-autobuild/test.py | 5 +++ .../test-extraction-autobuild/test.ql | 11 +++++ .../test-extraction-traced/src/Makefile | 2 + .../test-extraction-traced/src/go.mod | 3 ++ .../test-extraction-traced/src/go.sum | 45 +++++++++++++++++++ .../test-extraction-traced/src/testme.go | 5 +++ .../src/testme_blackbox_test.go | 15 +++++++ .../test-extraction-traced/src/testme_test.go | 19 ++++++++ .../test-extraction-traced/test.expected | 9 ++++ .../test-extraction-traced/test.py | 4 ++ .../test-extraction-traced/test.ql | 11 +++++ .../traced-extraction/src/Makefile | 2 + .../traced-extraction/src/go.mod | 3 ++ .../traced-extraction/src/go.sum | 45 +++++++++++++++++++ .../traced-extraction/src/testme.go | 5 +++ .../src/testme_blackbox_test.go | 16 +++++++ .../traced-extraction/src/testme_test.go | 20 +++++++++ .../traced-extraction/test.expected | 4 ++ .../traced-extraction/test.py | 4 ++ .../traced-extraction/test.ql | 11 +++++ 30 files changed, 372 insertions(+) create mode 100644 go/ql/integration-tests/go-mod-sample/src/blackbox_test.go create mode 100644 go/ql/integration-tests/go-mod-sample/src/test_test.go create mode 100644 go/ql/integration-tests/test-extraction-autobuild/src/Makefile create mode 100644 go/ql/integration-tests/test-extraction-autobuild/src/go.mod create mode 100644 go/ql/integration-tests/test-extraction-autobuild/src/go.sum create mode 100644 go/ql/integration-tests/test-extraction-autobuild/src/testme.go create mode 100644 go/ql/integration-tests/test-extraction-autobuild/src/testme_blackbox_test.go create mode 100644 go/ql/integration-tests/test-extraction-autobuild/src/testme_test.go create mode 100644 go/ql/integration-tests/test-extraction-autobuild/test.expected create mode 100644 go/ql/integration-tests/test-extraction-autobuild/test.py create mode 100644 go/ql/integration-tests/test-extraction-autobuild/test.ql create mode 100644 go/ql/integration-tests/test-extraction-traced/src/Makefile create mode 100644 go/ql/integration-tests/test-extraction-traced/src/go.mod create mode 100644 go/ql/integration-tests/test-extraction-traced/src/go.sum create mode 100644 go/ql/integration-tests/test-extraction-traced/src/testme.go create mode 100644 go/ql/integration-tests/test-extraction-traced/src/testme_blackbox_test.go create mode 100644 go/ql/integration-tests/test-extraction-traced/src/testme_test.go create mode 100644 go/ql/integration-tests/test-extraction-traced/test.expected create mode 100644 go/ql/integration-tests/test-extraction-traced/test.py create mode 100644 go/ql/integration-tests/test-extraction-traced/test.ql create mode 100644 go/ql/integration-tests/traced-extraction/src/Makefile create mode 100644 go/ql/integration-tests/traced-extraction/src/go.mod create mode 100644 go/ql/integration-tests/traced-extraction/src/go.sum create mode 100644 go/ql/integration-tests/traced-extraction/src/testme.go create mode 100644 go/ql/integration-tests/traced-extraction/src/testme_blackbox_test.go create mode 100644 go/ql/integration-tests/traced-extraction/src/testme_test.go create mode 100644 go/ql/integration-tests/traced-extraction/test.expected create mode 100644 go/ql/integration-tests/traced-extraction/test.py create mode 100644 go/ql/integration-tests/traced-extraction/test.ql diff --git a/go/ql/integration-tests/go-mod-sample/src/blackbox_test.go b/go/ql/integration-tests/go-mod-sample/src/blackbox_test.go new file mode 100644 index 000000000000..739d60e070ca --- /dev/null +++ b/go/ql/integration-tests/go-mod-sample/src/blackbox_test.go @@ -0,0 +1,16 @@ +package makesample_test + +import ( + "testing" + "makesample" +) + +// Note because this is a test we do NOT expect this to be extracted. +func TestTestMe(t *testing.T) { + + publicResult := makesample.PublicFunction() + if publicResult != 1 { + t.Errorf("Expected 1, got %d", publicResult) + } + +} diff --git a/go/ql/integration-tests/go-mod-sample/src/test.go b/go/ql/integration-tests/go-mod-sample/src/test.go index 824c0b04c726..71f2a463011a 100644 --- a/go/ql/integration-tests/go-mod-sample/src/test.go +++ b/go/ql/integration-tests/go-mod-sample/src/test.go @@ -10,3 +10,7 @@ func test() { header.Version = 4 } + +func PublicFunction() int { + return 1 +} diff --git a/go/ql/integration-tests/go-mod-sample/src/test_test.go b/go/ql/integration-tests/go-mod-sample/src/test_test.go new file mode 100644 index 000000000000..0b11ee38283b --- /dev/null +++ b/go/ql/integration-tests/go-mod-sample/src/test_test.go @@ -0,0 +1,15 @@ +package makesample + +import ( + "testing" +) + +func TestTestMe(t *testing.T) { + + // Note because this is a test we do NOT expect this to be extracted. + publicResult := PublicFunction() + if publicResult != 1 { + t.Errorf("Expected 1, got %d", publicResult) + } + +} diff --git a/go/ql/integration-tests/test-extraction-autobuild/src/Makefile b/go/ql/integration-tests/test-extraction-autobuild/src/Makefile new file mode 100644 index 000000000000..266e02877884 --- /dev/null +++ b/go/ql/integration-tests/test-extraction-autobuild/src/Makefile @@ -0,0 +1,2 @@ +all: + go get diff --git a/go/ql/integration-tests/test-extraction-autobuild/src/go.mod b/go/ql/integration-tests/test-extraction-autobuild/src/go.mod new file mode 100644 index 000000000000..c4a9f55df6c1 --- /dev/null +++ b/go/ql/integration-tests/test-extraction-autobuild/src/go.mod @@ -0,0 +1,3 @@ +go 1.14 + +module testsample diff --git a/go/ql/integration-tests/test-extraction-autobuild/src/go.sum b/go/ql/integration-tests/test-extraction-autobuild/src/go.sum new file mode 100644 index 000000000000..a8e1b59ae4b1 --- /dev/null +++ b/go/ql/integration-tests/test-extraction-autobuild/src/go.sum @@ -0,0 +1,45 @@ +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/go/ql/integration-tests/test-extraction-autobuild/src/testme.go b/go/ql/integration-tests/test-extraction-autobuild/src/testme.go new file mode 100644 index 000000000000..e24138553ce9 --- /dev/null +++ b/go/ql/integration-tests/test-extraction-autobuild/src/testme.go @@ -0,0 +1,5 @@ +package testsample + +func PublicFunction() int { return 1 } + +func privateFunction() int { return 2 } diff --git a/go/ql/integration-tests/test-extraction-autobuild/src/testme_blackbox_test.go b/go/ql/integration-tests/test-extraction-autobuild/src/testme_blackbox_test.go new file mode 100644 index 000000000000..84a3c30deb42 --- /dev/null +++ b/go/ql/integration-tests/test-extraction-autobuild/src/testme_blackbox_test.go @@ -0,0 +1,15 @@ +package testsample_test + +import ( + "testing" + "testsample" +) + +func TestTestMe(t *testing.T) { + + publicResult := testsample.PublicFunction() + if publicResult != 1 { + t.Errorf("Expected 1, got %d", publicResult) + } + +} diff --git a/go/ql/integration-tests/test-extraction-autobuild/src/testme_test.go b/go/ql/integration-tests/test-extraction-autobuild/src/testme_test.go new file mode 100644 index 000000000000..bac05af5f7f1 --- /dev/null +++ b/go/ql/integration-tests/test-extraction-autobuild/src/testme_test.go @@ -0,0 +1,19 @@ +package testsample + +import ( + "testing" +) + +func TestTestMe(t *testing.T) { + + publicResult := PublicFunction() + if publicResult != 1 { + t.Errorf("Expected 1, got %d", publicResult) + } + + privateResult := privateFunction() + if privateResult != 2 { + t.Errorf("Expected 2, got %d", privateResult) + } + +} diff --git a/go/ql/integration-tests/test-extraction-autobuild/test.expected b/go/ql/integration-tests/test-extraction-autobuild/test.expected new file mode 100644 index 000000000000..d39d8143fa6c --- /dev/null +++ b/go/ql/integration-tests/test-extraction-autobuild/test.expected @@ -0,0 +1,9 @@ +#select +| src/testme.go:0:0:0:0 | src/testme.go | +| src/testme_blackbox_test.go:0:0:0:0 | src/testme_blackbox_test.go | +| src/testme_test.go:0:0:0:0 | src/testme_test.go | +calls +| src/testme_blackbox_test.go:10:19:10:45 | call to PublicFunction | src/testme.go:3:1:3:38 | function declaration | +| src/testme_test.go:9:19:9:34 | call to PublicFunction | src/testme.go:3:1:3:38 | function declaration | +| src/testme_test.go:14:20:14:36 | call to privateFunction | src/testme.go:5:1:5:39 | function declaration | +extractionErrors diff --git a/go/ql/integration-tests/test-extraction-autobuild/test.py b/go/ql/integration-tests/test-extraction-autobuild/test.py new file mode 100644 index 000000000000..417b1e8bb04e --- /dev/null +++ b/go/ql/integration-tests/test-extraction-autobuild/test.py @@ -0,0 +1,5 @@ +import os + +def test(codeql, go): + os.environ["CODEQL_EXTRACTOR_GO_EXTRACT_TESTS"] = "true" + codeql.database.create(source_root="src") diff --git a/go/ql/integration-tests/test-extraction-autobuild/test.ql b/go/ql/integration-tests/test-extraction-autobuild/test.ql new file mode 100644 index 000000000000..cb06aeadf9c3 --- /dev/null +++ b/go/ql/integration-tests/test-extraction-autobuild/test.ql @@ -0,0 +1,11 @@ +import go +import semmle.go.DiagnosticsReporting + +from GoFile f +select f + +query predicate calls(CallExpr ce, FuncDecl f) { + f = ce.getTarget().getFuncDecl() +} + +query predicate extractionErrors(string msg, int sev) { reportableDiagnostics(_, msg, sev) } diff --git a/go/ql/integration-tests/test-extraction-traced/src/Makefile b/go/ql/integration-tests/test-extraction-traced/src/Makefile new file mode 100644 index 000000000000..266e02877884 --- /dev/null +++ b/go/ql/integration-tests/test-extraction-traced/src/Makefile @@ -0,0 +1,2 @@ +all: + go get diff --git a/go/ql/integration-tests/test-extraction-traced/src/go.mod b/go/ql/integration-tests/test-extraction-traced/src/go.mod new file mode 100644 index 000000000000..c4a9f55df6c1 --- /dev/null +++ b/go/ql/integration-tests/test-extraction-traced/src/go.mod @@ -0,0 +1,3 @@ +go 1.14 + +module testsample diff --git a/go/ql/integration-tests/test-extraction-traced/src/go.sum b/go/ql/integration-tests/test-extraction-traced/src/go.sum new file mode 100644 index 000000000000..a8e1b59ae4b1 --- /dev/null +++ b/go/ql/integration-tests/test-extraction-traced/src/go.sum @@ -0,0 +1,45 @@ +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/go/ql/integration-tests/test-extraction-traced/src/testme.go b/go/ql/integration-tests/test-extraction-traced/src/testme.go new file mode 100644 index 000000000000..e24138553ce9 --- /dev/null +++ b/go/ql/integration-tests/test-extraction-traced/src/testme.go @@ -0,0 +1,5 @@ +package testsample + +func PublicFunction() int { return 1 } + +func privateFunction() int { return 2 } diff --git a/go/ql/integration-tests/test-extraction-traced/src/testme_blackbox_test.go b/go/ql/integration-tests/test-extraction-traced/src/testme_blackbox_test.go new file mode 100644 index 000000000000..84a3c30deb42 --- /dev/null +++ b/go/ql/integration-tests/test-extraction-traced/src/testme_blackbox_test.go @@ -0,0 +1,15 @@ +package testsample_test + +import ( + "testing" + "testsample" +) + +func TestTestMe(t *testing.T) { + + publicResult := testsample.PublicFunction() + if publicResult != 1 { + t.Errorf("Expected 1, got %d", publicResult) + } + +} diff --git a/go/ql/integration-tests/test-extraction-traced/src/testme_test.go b/go/ql/integration-tests/test-extraction-traced/src/testme_test.go new file mode 100644 index 000000000000..bac05af5f7f1 --- /dev/null +++ b/go/ql/integration-tests/test-extraction-traced/src/testme_test.go @@ -0,0 +1,19 @@ +package testsample + +import ( + "testing" +) + +func TestTestMe(t *testing.T) { + + publicResult := PublicFunction() + if publicResult != 1 { + t.Errorf("Expected 1, got %d", publicResult) + } + + privateResult := privateFunction() + if privateResult != 2 { + t.Errorf("Expected 2, got %d", privateResult) + } + +} diff --git a/go/ql/integration-tests/test-extraction-traced/test.expected b/go/ql/integration-tests/test-extraction-traced/test.expected new file mode 100644 index 000000000000..d39d8143fa6c --- /dev/null +++ b/go/ql/integration-tests/test-extraction-traced/test.expected @@ -0,0 +1,9 @@ +#select +| src/testme.go:0:0:0:0 | src/testme.go | +| src/testme_blackbox_test.go:0:0:0:0 | src/testme_blackbox_test.go | +| src/testme_test.go:0:0:0:0 | src/testme_test.go | +calls +| src/testme_blackbox_test.go:10:19:10:45 | call to PublicFunction | src/testme.go:3:1:3:38 | function declaration | +| src/testme_test.go:9:19:9:34 | call to PublicFunction | src/testme.go:3:1:3:38 | function declaration | +| src/testme_test.go:14:20:14:36 | call to privateFunction | src/testme.go:5:1:5:39 | function declaration | +extractionErrors diff --git a/go/ql/integration-tests/test-extraction-traced/test.py b/go/ql/integration-tests/test-extraction-traced/test.py new file mode 100644 index 000000000000..2887d3fa0d09 --- /dev/null +++ b/go/ql/integration-tests/test-extraction-traced/test.py @@ -0,0 +1,4 @@ +import os + +def test(codeql, go): + codeql.database.create(source_root="src", command="go test -c") diff --git a/go/ql/integration-tests/test-extraction-traced/test.ql b/go/ql/integration-tests/test-extraction-traced/test.ql new file mode 100644 index 000000000000..cb06aeadf9c3 --- /dev/null +++ b/go/ql/integration-tests/test-extraction-traced/test.ql @@ -0,0 +1,11 @@ +import go +import semmle.go.DiagnosticsReporting + +from GoFile f +select f + +query predicate calls(CallExpr ce, FuncDecl f) { + f = ce.getTarget().getFuncDecl() +} + +query predicate extractionErrors(string msg, int sev) { reportableDiagnostics(_, msg, sev) } diff --git a/go/ql/integration-tests/traced-extraction/src/Makefile b/go/ql/integration-tests/traced-extraction/src/Makefile new file mode 100644 index 000000000000..266e02877884 --- /dev/null +++ b/go/ql/integration-tests/traced-extraction/src/Makefile @@ -0,0 +1,2 @@ +all: + go get diff --git a/go/ql/integration-tests/traced-extraction/src/go.mod b/go/ql/integration-tests/traced-extraction/src/go.mod new file mode 100644 index 000000000000..c4a9f55df6c1 --- /dev/null +++ b/go/ql/integration-tests/traced-extraction/src/go.mod @@ -0,0 +1,3 @@ +go 1.14 + +module testsample diff --git a/go/ql/integration-tests/traced-extraction/src/go.sum b/go/ql/integration-tests/traced-extraction/src/go.sum new file mode 100644 index 000000000000..a8e1b59ae4b1 --- /dev/null +++ b/go/ql/integration-tests/traced-extraction/src/go.sum @@ -0,0 +1,45 @@ +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/go/ql/integration-tests/traced-extraction/src/testme.go b/go/ql/integration-tests/traced-extraction/src/testme.go new file mode 100644 index 000000000000..e24138553ce9 --- /dev/null +++ b/go/ql/integration-tests/traced-extraction/src/testme.go @@ -0,0 +1,5 @@ +package testsample + +func PublicFunction() int { return 1 } + +func privateFunction() int { return 2 } diff --git a/go/ql/integration-tests/traced-extraction/src/testme_blackbox_test.go b/go/ql/integration-tests/traced-extraction/src/testme_blackbox_test.go new file mode 100644 index 000000000000..497b5e81410a --- /dev/null +++ b/go/ql/integration-tests/traced-extraction/src/testme_blackbox_test.go @@ -0,0 +1,16 @@ +package testsample_test + +import ( + "testing" + "testsample" +) + +func TestTestMe(t *testing.T) { + + // Note because this is a test we do NOT expect it to be extracted + publicResult := testsample.PublicFunction() + if publicResult != 1 { + t.Errorf("Expected 1, got %d", publicResult) + } + +} diff --git a/go/ql/integration-tests/traced-extraction/src/testme_test.go b/go/ql/integration-tests/traced-extraction/src/testme_test.go new file mode 100644 index 000000000000..c5f28c409de2 --- /dev/null +++ b/go/ql/integration-tests/traced-extraction/src/testme_test.go @@ -0,0 +1,20 @@ +package testsample + +import ( + "testing" +) + +func TestTestMe(t *testing.T) { + + // Note because this is a test we do NOT expect it to be extracted + publicResult := PublicFunction() + if publicResult != 1 { + t.Errorf("Expected 1, got %d", publicResult) + } + + privateResult := privateFunction() + if privateResult != 2 { + t.Errorf("Expected 2, got %d", privateResult) + } + +} diff --git a/go/ql/integration-tests/traced-extraction/test.expected b/go/ql/integration-tests/traced-extraction/test.expected new file mode 100644 index 000000000000..7b62d6d92983 --- /dev/null +++ b/go/ql/integration-tests/traced-extraction/test.expected @@ -0,0 +1,4 @@ +#select +| src/testme.go:0:0:0:0 | src/testme.go | +calls +extractionErrors diff --git a/go/ql/integration-tests/traced-extraction/test.py b/go/ql/integration-tests/traced-extraction/test.py new file mode 100644 index 000000000000..fbb3f6a6d6c5 --- /dev/null +++ b/go/ql/integration-tests/traced-extraction/test.py @@ -0,0 +1,4 @@ +import os + +def test(codeql, go): + codeql.database.create(source_root="src", command="go build") diff --git a/go/ql/integration-tests/traced-extraction/test.ql b/go/ql/integration-tests/traced-extraction/test.ql new file mode 100644 index 000000000000..cb06aeadf9c3 --- /dev/null +++ b/go/ql/integration-tests/traced-extraction/test.ql @@ -0,0 +1,11 @@ +import go +import semmle.go.DiagnosticsReporting + +from GoFile f +select f + +query predicate calls(CallExpr ce, FuncDecl f) { + f = ce.getTarget().getFuncDecl() +} + +query predicate extractionErrors(string msg, int sev) { reportableDiagnostics(_, msg, sev) } From eb6918f88f16590f80dad07f9082498ce95ab4dd Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Wed, 21 Aug 2024 12:32:27 +0100 Subject: [PATCH 04/11] Autoformat --- go/ql/integration-tests/test-extraction-autobuild/test.ql | 4 +--- go/ql/integration-tests/test-extraction-traced/test.ql | 4 +--- go/ql/integration-tests/traced-extraction/test.ql | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/go/ql/integration-tests/test-extraction-autobuild/test.ql b/go/ql/integration-tests/test-extraction-autobuild/test.ql index cb06aeadf9c3..15eae8959862 100644 --- a/go/ql/integration-tests/test-extraction-autobuild/test.ql +++ b/go/ql/integration-tests/test-extraction-autobuild/test.ql @@ -4,8 +4,6 @@ import semmle.go.DiagnosticsReporting from GoFile f select f -query predicate calls(CallExpr ce, FuncDecl f) { - f = ce.getTarget().getFuncDecl() -} +query predicate calls(CallExpr ce, FuncDecl f) { f = ce.getTarget().getFuncDecl() } query predicate extractionErrors(string msg, int sev) { reportableDiagnostics(_, msg, sev) } diff --git a/go/ql/integration-tests/test-extraction-traced/test.ql b/go/ql/integration-tests/test-extraction-traced/test.ql index cb06aeadf9c3..15eae8959862 100644 --- a/go/ql/integration-tests/test-extraction-traced/test.ql +++ b/go/ql/integration-tests/test-extraction-traced/test.ql @@ -4,8 +4,6 @@ import semmle.go.DiagnosticsReporting from GoFile f select f -query predicate calls(CallExpr ce, FuncDecl f) { - f = ce.getTarget().getFuncDecl() -} +query predicate calls(CallExpr ce, FuncDecl f) { f = ce.getTarget().getFuncDecl() } query predicate extractionErrors(string msg, int sev) { reportableDiagnostics(_, msg, sev) } diff --git a/go/ql/integration-tests/traced-extraction/test.ql b/go/ql/integration-tests/traced-extraction/test.ql index cb06aeadf9c3..15eae8959862 100644 --- a/go/ql/integration-tests/traced-extraction/test.ql +++ b/go/ql/integration-tests/traced-extraction/test.ql @@ -4,8 +4,6 @@ import semmle.go.DiagnosticsReporting from GoFile f select f -query predicate calls(CallExpr ce, FuncDecl f) { - f = ce.getTarget().getFuncDecl() -} +query predicate calls(CallExpr ce, FuncDecl f) { f = ce.getTarget().getFuncDecl() } query predicate extractionErrors(string msg, int sev) { reportableDiagnostics(_, msg, sev) } From 9d79feb4d3882c043ff9a5119b08215780492794 Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Wed, 21 Aug 2024 12:41:27 +0100 Subject: [PATCH 05/11] Autoformat go --- .../go-mod-sample/src/blackbox_test.go | 12 +++++------ .../go-mod-sample/src/test_test.go | 12 +++++------ .../src/testme_blackbox_test.go | 12 +++++------ .../src/testme_test.go | 18 ++++++++--------- .../src/testme_blackbox_test.go | 12 +++++------ .../test-extraction-traced/src/testme_test.go | 18 ++++++++--------- .../src/testme_blackbox_test.go | 14 ++++++------- .../traced-extraction/src/testme_test.go | 20 +++++++++---------- 8 files changed, 59 insertions(+), 59 deletions(-) diff --git a/go/ql/integration-tests/go-mod-sample/src/blackbox_test.go b/go/ql/integration-tests/go-mod-sample/src/blackbox_test.go index 739d60e070ca..c3f434a7aa12 100644 --- a/go/ql/integration-tests/go-mod-sample/src/blackbox_test.go +++ b/go/ql/integration-tests/go-mod-sample/src/blackbox_test.go @@ -1,16 +1,16 @@ package makesample_test import ( - "testing" - "makesample" + "makesample" + "testing" ) // Note because this is a test we do NOT expect this to be extracted. func TestTestMe(t *testing.T) { - publicResult := makesample.PublicFunction() - if publicResult != 1 { - t.Errorf("Expected 1, got %d", publicResult) - } + publicResult := makesample.PublicFunction() + if publicResult != 1 { + t.Errorf("Expected 1, got %d", publicResult) + } } diff --git a/go/ql/integration-tests/go-mod-sample/src/test_test.go b/go/ql/integration-tests/go-mod-sample/src/test_test.go index 0b11ee38283b..6693a016350c 100644 --- a/go/ql/integration-tests/go-mod-sample/src/test_test.go +++ b/go/ql/integration-tests/go-mod-sample/src/test_test.go @@ -1,15 +1,15 @@ package makesample import ( - "testing" + "testing" ) func TestTestMe(t *testing.T) { - // Note because this is a test we do NOT expect this to be extracted. - publicResult := PublicFunction() - if publicResult != 1 { - t.Errorf("Expected 1, got %d", publicResult) - } + // Note because this is a test we do NOT expect this to be extracted. + publicResult := PublicFunction() + if publicResult != 1 { + t.Errorf("Expected 1, got %d", publicResult) + } } diff --git a/go/ql/integration-tests/test-extraction-autobuild/src/testme_blackbox_test.go b/go/ql/integration-tests/test-extraction-autobuild/src/testme_blackbox_test.go index 84a3c30deb42..18a507b5aa38 100644 --- a/go/ql/integration-tests/test-extraction-autobuild/src/testme_blackbox_test.go +++ b/go/ql/integration-tests/test-extraction-autobuild/src/testme_blackbox_test.go @@ -1,15 +1,15 @@ package testsample_test import ( - "testing" - "testsample" + "testing" + "testsample" ) func TestTestMe(t *testing.T) { - publicResult := testsample.PublicFunction() - if publicResult != 1 { - t.Errorf("Expected 1, got %d", publicResult) - } + publicResult := testsample.PublicFunction() + if publicResult != 1 { + t.Errorf("Expected 1, got %d", publicResult) + } } diff --git a/go/ql/integration-tests/test-extraction-autobuild/src/testme_test.go b/go/ql/integration-tests/test-extraction-autobuild/src/testme_test.go index bac05af5f7f1..183d7cd3dffe 100644 --- a/go/ql/integration-tests/test-extraction-autobuild/src/testme_test.go +++ b/go/ql/integration-tests/test-extraction-autobuild/src/testme_test.go @@ -1,19 +1,19 @@ package testsample import ( - "testing" + "testing" ) func TestTestMe(t *testing.T) { - publicResult := PublicFunction() - if publicResult != 1 { - t.Errorf("Expected 1, got %d", publicResult) - } + publicResult := PublicFunction() + if publicResult != 1 { + t.Errorf("Expected 1, got %d", publicResult) + } - privateResult := privateFunction() - if privateResult != 2 { - t.Errorf("Expected 2, got %d", privateResult) - } + privateResult := privateFunction() + if privateResult != 2 { + t.Errorf("Expected 2, got %d", privateResult) + } } diff --git a/go/ql/integration-tests/test-extraction-traced/src/testme_blackbox_test.go b/go/ql/integration-tests/test-extraction-traced/src/testme_blackbox_test.go index 84a3c30deb42..18a507b5aa38 100644 --- a/go/ql/integration-tests/test-extraction-traced/src/testme_blackbox_test.go +++ b/go/ql/integration-tests/test-extraction-traced/src/testme_blackbox_test.go @@ -1,15 +1,15 @@ package testsample_test import ( - "testing" - "testsample" + "testing" + "testsample" ) func TestTestMe(t *testing.T) { - publicResult := testsample.PublicFunction() - if publicResult != 1 { - t.Errorf("Expected 1, got %d", publicResult) - } + publicResult := testsample.PublicFunction() + if publicResult != 1 { + t.Errorf("Expected 1, got %d", publicResult) + } } diff --git a/go/ql/integration-tests/test-extraction-traced/src/testme_test.go b/go/ql/integration-tests/test-extraction-traced/src/testme_test.go index bac05af5f7f1..183d7cd3dffe 100644 --- a/go/ql/integration-tests/test-extraction-traced/src/testme_test.go +++ b/go/ql/integration-tests/test-extraction-traced/src/testme_test.go @@ -1,19 +1,19 @@ package testsample import ( - "testing" + "testing" ) func TestTestMe(t *testing.T) { - publicResult := PublicFunction() - if publicResult != 1 { - t.Errorf("Expected 1, got %d", publicResult) - } + publicResult := PublicFunction() + if publicResult != 1 { + t.Errorf("Expected 1, got %d", publicResult) + } - privateResult := privateFunction() - if privateResult != 2 { - t.Errorf("Expected 2, got %d", privateResult) - } + privateResult := privateFunction() + if privateResult != 2 { + t.Errorf("Expected 2, got %d", privateResult) + } } diff --git a/go/ql/integration-tests/traced-extraction/src/testme_blackbox_test.go b/go/ql/integration-tests/traced-extraction/src/testme_blackbox_test.go index 497b5e81410a..728cc651d7a3 100644 --- a/go/ql/integration-tests/traced-extraction/src/testme_blackbox_test.go +++ b/go/ql/integration-tests/traced-extraction/src/testme_blackbox_test.go @@ -1,16 +1,16 @@ package testsample_test import ( - "testing" - "testsample" + "testing" + "testsample" ) func TestTestMe(t *testing.T) { - // Note because this is a test we do NOT expect it to be extracted - publicResult := testsample.PublicFunction() - if publicResult != 1 { - t.Errorf("Expected 1, got %d", publicResult) - } + // Note because this is a test we do NOT expect it to be extracted + publicResult := testsample.PublicFunction() + if publicResult != 1 { + t.Errorf("Expected 1, got %d", publicResult) + } } diff --git a/go/ql/integration-tests/traced-extraction/src/testme_test.go b/go/ql/integration-tests/traced-extraction/src/testme_test.go index c5f28c409de2..cc77c9dd37ff 100644 --- a/go/ql/integration-tests/traced-extraction/src/testme_test.go +++ b/go/ql/integration-tests/traced-extraction/src/testme_test.go @@ -1,20 +1,20 @@ package testsample import ( - "testing" + "testing" ) func TestTestMe(t *testing.T) { - // Note because this is a test we do NOT expect it to be extracted - publicResult := PublicFunction() - if publicResult != 1 { - t.Errorf("Expected 1, got %d", publicResult) - } + // Note because this is a test we do NOT expect it to be extracted + publicResult := PublicFunction() + if publicResult != 1 { + t.Errorf("Expected 1, got %d", publicResult) + } - privateResult := privateFunction() - if privateResult != 2 { - t.Errorf("Expected 2, got %d", privateResult) - } + privateResult := privateFunction() + if privateResult != 2 { + t.Errorf("Expected 2, got %d", privateResult) + } } From 94cb99e51de887c8e8d76f9c29c47c4d98cd25d5 Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Wed, 21 Aug 2024 12:44:04 +0100 Subject: [PATCH 06/11] Adjust test expectations --- .../test-extraction-autobuild/test.expected | 6 +++--- .../integration-tests/test-extraction-traced/test.expected | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go/ql/integration-tests/test-extraction-autobuild/test.expected b/go/ql/integration-tests/test-extraction-autobuild/test.expected index d39d8143fa6c..9e1585fc5ef8 100644 --- a/go/ql/integration-tests/test-extraction-autobuild/test.expected +++ b/go/ql/integration-tests/test-extraction-autobuild/test.expected @@ -3,7 +3,7 @@ | src/testme_blackbox_test.go:0:0:0:0 | src/testme_blackbox_test.go | | src/testme_test.go:0:0:0:0 | src/testme_test.go | calls -| src/testme_blackbox_test.go:10:19:10:45 | call to PublicFunction | src/testme.go:3:1:3:38 | function declaration | -| src/testme_test.go:9:19:9:34 | call to PublicFunction | src/testme.go:3:1:3:38 | function declaration | -| src/testme_test.go:14:20:14:36 | call to privateFunction | src/testme.go:5:1:5:39 | function declaration | +| src/testme_blackbox_test.go:10:18:10:44 | call to PublicFunction | src/testme.go:3:1:3:38 | function declaration | +| src/testme_test.go:9:18:9:33 | call to PublicFunction | src/testme.go:3:1:3:38 | function declaration | +| src/testme_test.go:14:19:14:35 | call to privateFunction | src/testme.go:5:1:5:39 | function declaration | extractionErrors diff --git a/go/ql/integration-tests/test-extraction-traced/test.expected b/go/ql/integration-tests/test-extraction-traced/test.expected index d39d8143fa6c..9e1585fc5ef8 100644 --- a/go/ql/integration-tests/test-extraction-traced/test.expected +++ b/go/ql/integration-tests/test-extraction-traced/test.expected @@ -3,7 +3,7 @@ | src/testme_blackbox_test.go:0:0:0:0 | src/testme_blackbox_test.go | | src/testme_test.go:0:0:0:0 | src/testme_test.go | calls -| src/testme_blackbox_test.go:10:19:10:45 | call to PublicFunction | src/testme.go:3:1:3:38 | function declaration | -| src/testme_test.go:9:19:9:34 | call to PublicFunction | src/testme.go:3:1:3:38 | function declaration | -| src/testme_test.go:14:20:14:36 | call to privateFunction | src/testme.go:5:1:5:39 | function declaration | +| src/testme_blackbox_test.go:10:18:10:44 | call to PublicFunction | src/testme.go:3:1:3:38 | function declaration | +| src/testme_test.go:9:18:9:33 | call to PublicFunction | src/testme.go:3:1:3:38 | function declaration | +| src/testme_test.go:14:19:14:35 | call to privateFunction | src/testme.go:5:1:5:39 | function declaration | extractionErrors From f5ff82268148e0672344298582d5fa544ded11ad Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Wed, 21 Aug 2024 16:04:39 +0100 Subject: [PATCH 07/11] Convert extract-tests option to an official extractor option --- go/codeql-extractor.yml | 8 ++++++++ go/extractor/cli/go-extractor/go-extractor.go | 2 +- go/extractor/extractor.go | 2 +- go/ql/integration-tests/test-extraction-autobuild/test.py | 3 +-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/go/codeql-extractor.yml b/go/codeql-extractor.yml index 20cfe987ef32..e3333ec7c06d 100644 --- a/go/codeql-extractor.yml +++ b/go/codeql-extractor.yml @@ -19,3 +19,11 @@ file_types: extensions: - .go legacy_qltest_extraction: true +options: + extract_tests: + title: Whether to include Go test files and functions in the CodeQL database. + description: > + A value indicating whether Go test files and functions should be included in the CodeQL database. + The default is 'false'. + type: string + pattern: "^(false|true)$" diff --git a/go/extractor/cli/go-extractor/go-extractor.go b/go/extractor/cli/go-extractor/go-extractor.go index 2f40d2980cfa..142a950d9f12 100644 --- a/go/extractor/cli/go-extractor/go-extractor.go +++ b/go/extractor/cli/go-extractor/go-extractor.go @@ -94,7 +94,7 @@ func parseFlags(args []string, mimic bool, extractTests bool) ([]string, []strin } func main() { - extractTestsDefault := os.Getenv("CODEQL_EXTRACTOR_GO_EXTRACT_TESTS") == "true" + extractTestsDefault := os.Getenv("CODEQL_EXTRACTOR_GO_OPTION_EXTRACT_TESTS") == "true" buildFlags, patterns, extractTests := parseFlags(os.Args[1:], false, extractTestsDefault) if cpuprofile != "" { diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index 68e64d46f905..448bb83deee5 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -88,7 +88,7 @@ func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool) log.Printf("Running packages.Load%s.", testMessage) // This includes test packages if either we're tracing a `go test` command, - // or if CODEQL_EXTRACTOR_GO_EXTRACT_TESTS is set to "true". + // or if CODEQL_EXTRACTOR_GO_OPTION_EXTRACT_TESTS is set to "true". cfg := &packages.Config{ Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | diff --git a/go/ql/integration-tests/test-extraction-autobuild/test.py b/go/ql/integration-tests/test-extraction-autobuild/test.py index 417b1e8bb04e..0dc91b5212c5 100644 --- a/go/ql/integration-tests/test-extraction-autobuild/test.py +++ b/go/ql/integration-tests/test-extraction-autobuild/test.py @@ -1,5 +1,4 @@ import os def test(codeql, go): - os.environ["CODEQL_EXTRACTOR_GO_EXTRACT_TESTS"] = "true" - codeql.database.create(source_root="src") + codeql.database.create(source_root="src", extractor_option = ["extract_tests=true"]) From bcb84a84e1184b08efccd9d9edec39c41d9d4fc6 Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Fri, 20 Sep 2024 12:48:08 +0100 Subject: [PATCH 08/11] Only skip test packages at the file-extraction phase --- go/extractor/extractor.go | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index 448bb83deee5..a42e56866baa 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -159,13 +159,18 @@ func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool) // Do a post-order traversal and extract the package scope of each package packages.Visit(pkgs, nil, func(pkg *packages.Package) { - log.Printf("Processing package %s.", pkg.PkgPath) + // Note that if test extraction is enabled, we will encounter a package twice here: + // once as the main package, and once as the test package (with a package ID like + // "abc.com/pkgname [abc.com/pkgname.test]"). + // + // We will extract it both times however, because we need to visit the packages + // in the right order in order to visit used types before their users, and the + // ordering determined by packages.Visit for the main and the test package may differ. + // + // This should only cause some wasted time and not inconsistency because the names for + // objects seen in this process should be the same each time. - // If this is a variant of a package that also occurs with a shorter ID, skip it. - if pkg.ID != longestPackageIds[pkg.PkgPath] { - log.Printf("Skipping variant of package %s with ID %s.", pkg.PkgPath, pkg.ID) - return - } + log.Printf("Processing package %s.", pkg.PkgPath) if _, ok := pkgInfos[pkg.PkgPath]; !ok { pkgInfos[pkg.PkgPath] = toolchain.GetPkgInfo(pkg.PkgPath, modFlags...) @@ -242,9 +247,15 @@ func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool) // extract AST information for all packages packages.Visit(pkgs, nil, func(pkg *packages.Package) { - // If this is a variant of a package that also occurs with a longer ID, skip it. + // If this is a variant of a package that also occurs with a longer ID, skip it; + // otherwise we would extract the same file more than once including extracting the + // body of methods twice, causing database inconsistencies. + // + // We prefer the version with the longest ID because that is (so far as I know) always + // the version that defines more entities -- the only case I'm aware of being a test + // variant of a package, which includes test-only functions in addition to the complete + // contents of the main variant. if pkg.ID != longestPackageIds[pkg.PkgPath] { - // Don't log here; we already mentioned this above. return } From bb44a2fc8ca094560d98d1503bdd75360b7d45ed Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Sat, 21 Sep 2024 13:38:41 +0100 Subject: [PATCH 09/11] Populate pkgInfoMapping for test packages if relevant --- go/extractor/extractor.go | 2 +- go/extractor/toolchain/toolchain.go | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index a42e56866baa..81e7e99dee54 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -131,7 +131,7 @@ func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool) if os.Getenv("CODEQL_EXTRACTOR_GO_FAST_PACKAGE_INFO") != "false" { log.Printf("Running go list to resolve package and module directories.") // get all packages information - pkgInfos, err = toolchain.GetPkgsInfo(patterns, true, modFlags...) + pkgInfos, err = toolchain.GetPkgsInfo(patterns, true, extractTests, modFlags...) if err != nil { log.Fatalf("Error getting dependency package or module directories: %v.", err) } diff --git a/go/extractor/toolchain/toolchain.go b/go/extractor/toolchain/toolchain.go index 029390432376..046f9af65b2f 100644 --- a/go/extractor/toolchain/toolchain.go +++ b/go/extractor/toolchain/toolchain.go @@ -223,7 +223,7 @@ type PkgInfo struct { // GetPkgsInfo gets the absolute module and package root directories for the packages matched by the // patterns `patterns`. It passes to `go list` the flags specified by `flags`. If `includingDeps` // is true, all dependencies will also be included. -func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[string]PkgInfo, error) { +func GetPkgsInfo(patterns []string, includingDeps bool, extractTests bool, flags ...string) (map[string]PkgInfo, error) { // enable module mode so that we can find a module root if it exists, even if go module support is // disabled by a build if includingDeps { @@ -231,6 +231,10 @@ func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[st flags = append(flags, "-deps") } + if extractTests { + flags = append(flags, "-test") + } + // using -json overrides -f format output, err := RunList("", patterns, append(flags, "-json")...) if err != nil { @@ -272,6 +276,12 @@ func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[st PkgDir: pkgAbsDir, ModDir: modAbsDir, } + + if extractTests && strings.Contains(pkgInfo.ImportPath, " [") { + // Assume " [" is the start of a qualifier, and index the package by its base name + baseImportPath := strings.Split(pkgInfo.ImportPath, " [")[0] + pkgInfoMapping[baseImportPath] = pkgInfoMapping[pkgInfo.ImportPath] + } } return pkgInfoMapping, nil } From fe273e9084a301e64c4878f71d95e0b6895b1602 Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Wed, 21 Aug 2024 12:34:50 +0100 Subject: [PATCH 10/11] Force test extraction on for evaluation purposes --- go/codeql-tools/autobuild.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/codeql-tools/autobuild.sh b/go/codeql-tools/autobuild.sh index 585152e676e2..9d89a651a54d 100755 --- a/go/codeql-tools/autobuild.sh +++ b/go/codeql-tools/autobuild.sh @@ -11,6 +11,9 @@ fi LGTM_SRC="$(pwd)" export LGTM_SRC +CODEQL_EXTRACTOR_GO_OPTION_EXTRACT_TESTS=true +export CODEQL_EXTRACTOR_GO_OPTION_EXTRACT_TESTS + if [ "${CODEQL_EXTRACTOR_GO_BUILD_TRACING:-}" = "on" ]; then echo "Tracing enabled" "$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-build-runner" From 8ceacdfa0dac8d700bf90a31ec38693407082414 Mon Sep 17 00:00:00 2001 From: Chris Smowton Date: Sat, 21 Sep 2024 22:11:13 +0100 Subject: [PATCH 11/11] Autoformat --- go/ql/integration-tests/go-mod-sample/src/test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/ql/integration-tests/go-mod-sample/src/test.go b/go/ql/integration-tests/go-mod-sample/src/test.go index 71f2a463011a..c3f0f284d4a8 100644 --- a/go/ql/integration-tests/go-mod-sample/src/test.go +++ b/go/ql/integration-tests/go-mod-sample/src/test.go @@ -12,5 +12,5 @@ func test() { } func PublicFunction() int { - return 1 + return 1 }