From c62b2221046000d91b0a3fc63ea6a18d9e7c0539 Mon Sep 17 00:00:00 2001 From: Xiaoyang Tan Date: Tue, 17 May 2022 23:13:27 -0700 Subject: [PATCH] finally owrks --- go/tools/builders/BUILD.bazel | 5 +- go/tools/builders/env.go | 2 +- go/tools/builders/stdliblist.go | 369 +++++++++++++-------------- go/tools/builders/stdliblist_test.go | 21 +- 4 files changed, 191 insertions(+), 206 deletions(-) diff --git a/go/tools/builders/BUILD.bazel b/go/tools/builders/BUILD.bazel index 192c67993c..762983af13 100644 --- a/go/tools/builders/BUILD.bazel +++ b/go/tools/builders/BUILD.bazel @@ -34,7 +34,10 @@ go_test( "stdliblist_test.go", ], data = ["@go_sdk//:files"], - rundir = ".", + # -sdk in stdliblist needs to be a relative path, otherwise it breaks + # assumptions of cloning go_sdk, thus we need to set up the test in a + # way that go_sdk is under the directory where test is run. + rundir = "..", deps = ["//go/tools/bazel"], ) diff --git a/go/tools/builders/env.go b/go/tools/builders/env.go index 5e3afd5b06..3ed12e9745 100644 --- a/go/tools/builders/env.go +++ b/go/tools/builders/env.go @@ -66,7 +66,7 @@ type env struct { // configured with those flags. func envFlags(flags *flag.FlagSet) *env { env := &env{} - flags.StringVar(&env.sdk, "sdk", "", "Path to the Go SDK.") + flags.StringVar(&env.sdk, "sdk", "", "Relative path to the Go SDK.") flags.Var(&tagFlag{}, "tags", "List of build tags considered true.") flags.StringVar(&env.installSuffix, "installsuffix", "", "Standard library under GOROOT/pkg") flags.BoolVar(&env.verbose, "v", false, "Whether subprocess command lines should be printed") diff --git a/go/tools/builders/stdliblist.go b/go/tools/builders/stdliblist.go index eedf74d143..34243c6b81 100644 --- a/go/tools/builders/stdliblist.go +++ b/go/tools/builders/stdliblist.go @@ -15,142 +15,138 @@ package main import ( - "bytes" - "encoding/json" - "flag" - "fmt" - "go/build" - "os" - "path/filepath" - "strings" + "bytes" + "encoding/json" + "flag" + "fmt" + "go/build" + "os" + "path/filepath" + "strings" ) // Copy and pasted from golang.org/x/tools/go/packages type flatPackagesError struct { - Pos string // "file:line:col" or "file:line" or "" or "-" - Msg string - Kind flatPackagesErrorKind + Pos string // "file:line:col" or "file:line" or "" or "-" + Msg string + Kind flatPackagesErrorKind } type flatPackagesErrorKind int const ( - UnknownError flatPackagesErrorKind = iota - ListError - ParseError - TypeError + UnknownError flatPackagesErrorKind = iota + ListError + ParseError + TypeError ) func (err flatPackagesError) Error() string { - pos := err.Pos - if pos == "" { - pos = "-" // like token.Position{}.String() - } - return pos + ": " + err.Msg + pos := err.Pos + if pos == "" { + pos = "-" // like token.Position{}.String() + } + return pos + ": " + err.Msg } // flatPackage is the JSON form of Package // It drops all the type and syntax fields, and transforms the Imports type flatPackage struct { - ID string - Name string `json:",omitempty"` - PkgPath string `json:",omitempty"` - Standard bool `json:",omitempty"` - Errors []flatPackagesError `json:",omitempty"` - GoFiles []string `json:",omitempty"` - CompiledGoFiles []string `json:",omitempty"` - OtherFiles []string `json:",omitempty"` - ExportFile string `json:",omitempty"` - Imports map[string]string `json:",omitempty"` + ID string + Name string `json:",omitempty"` + PkgPath string `json:",omitempty"` + Standard bool `json:",omitempty"` + Errors []flatPackagesError `json:",omitempty"` + GoFiles []string `json:",omitempty"` + CompiledGoFiles []string `json:",omitempty"` + OtherFiles []string `json:",omitempty"` + ExportFile string `json:",omitempty"` + Imports map[string]string `json:",omitempty"` } type goListPackage struct { - Dir string // directory containing package sources - ImportPath string // import path of package in dir - Name string // package name - Target string // install path - Goroot bool // is this package in the Go root? - Standard bool // is this package part of the standard Go library? - Root string // Go root or Go path dir containing this package - Export string // file containing export data (when using -export) - // Source files - GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) - CgoFiles []string // .go source files that import "C" - CompiledGoFiles []string // .go files presented to compiler (when using -compiled) - IgnoredGoFiles []string // .go source files ignored due to build constraints - IgnoredOtherFiles []string // non-.go source files ignored due to build constraints - CFiles []string // .c source files - CXXFiles []string // .cc, .cxx and .cpp source files - MFiles []string // .m source files - HFiles []string // .h, .hh, .hpp and .hxx source files - FFiles []string // .f, .F, .for and .f90 Fortran source files - SFiles []string // .s source files - SwigFiles []string // .swig files - SwigCXXFiles []string // .swigcxx files - SysoFiles []string // .syso object files to add to archive - TestGoFiles []string // _test.go files in package - XTestGoFiles []string // _test.go files outside package - // Embedded files - EmbedPatterns []string // //go:embed patterns - EmbedFiles []string // files matched by EmbedPatterns - TestEmbedPatterns []string // //go:embed patterns in TestGoFiles - TestEmbedFiles []string // files matched by TestEmbedPatterns - XTestEmbedPatterns []string // //go:embed patterns in XTestGoFiles - XTestEmbedFiles []string // files matched by XTestEmbedPatterns - // Dependency information - Imports []string // import paths used by this package - ImportMap map[string]string // map from source import to ImportPath (identity entries omitted) - // Error information - Incomplete bool // this package or a dependency has an error - Error *flatPackagesError // error loading package - DepsErrors []*flatPackagesError // errors loading dependencies + Dir string // directory containing package sources + ImportPath string // import path of package in dir + Name string // package name + Target string // install path + Goroot bool // is this package in the Go root? + Standard bool // is this package part of the standard Go library? + Root string // Go root or Go path dir containing this package + Export string // file containing export data (when using -export) + // Source files + GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) + CgoFiles []string // .go source files that import "C" + CompiledGoFiles []string // .go files presented to compiler (when using -compiled) + IgnoredGoFiles []string // .go source files ignored due to build constraints + IgnoredOtherFiles []string // non-.go source files ignored due to build constraints + CFiles []string // .c source files + CXXFiles []string // .cc, .cxx and .cpp source files + MFiles []string // .m source files + HFiles []string // .h, .hh, .hpp and .hxx source files + FFiles []string // .f, .F, .for and .f90 Fortran source files + SFiles []string // .s source files + SwigFiles []string // .swig files + SwigCXXFiles []string // .swigcxx files + SysoFiles []string // .syso object files to add to archive + TestGoFiles []string // _test.go files in package + XTestGoFiles []string // _test.go files outside package + // Embedded files + EmbedPatterns []string // //go:embed patterns + EmbedFiles []string // files matched by EmbedPatterns + TestEmbedPatterns []string // //go:embed patterns in TestGoFiles + TestEmbedFiles []string // files matched by TestEmbedPatterns + XTestEmbedPatterns []string // //go:embed patterns in XTestGoFiles + XTestEmbedFiles []string // files matched by XTestEmbedPatterns + // Dependency information + Imports []string // import paths used by this package + ImportMap map[string]string // map from source import to ImportPath (identity entries omitted) + // Error information + Incomplete bool // this package or a dependency has an error + Error *flatPackagesError // error loading package + DepsErrors []*flatPackagesError // errors loading dependencies } func stdlibPackageID(importPath string) string { - return "@io_bazel_rules_go//stdlib:" + importPath + return "@io_bazel_rules_go//stdlib:" + importPath } // labelledPath replace the cloneBase with output base label func labelledPath(cloneBase, p string) string { - dir, _ := filepath.Rel(cloneBase, p) - println("rel:" + dir) - return filepath.Join("__BAZEL_OUTPUT_BASE__", dir) + dir, _ := filepath.Rel(cloneBase, p) + return filepath.Join("__BAZEL_OUTPUT_BASE__", dir) } // absoluteSourcesPaths replace cloneBase of the absolution // paths with the label for all source files in a package func absoluteSourcesPaths(cloneBase, pkgDir string, srcs []string) []string { - ret := make([]string, 0, len(srcs)) - pkgDir = labelledPath(cloneBase, pkgDir) - println("labelled:" + pkgDir) - for _, src := range srcs { - ret = append(ret, filepath.Join(pkgDir, src)) - } - return ret + ret := make([]string, 0, len(srcs)) + pkgDir = labelledPath(cloneBase, pkgDir) + for _, src := range srcs { + ret = append(ret, filepath.Join(pkgDir, src)) + } + return ret } func flatPackageForStd(cloneBase string, pkg *goListPackage) *flatPackage { - // Don't use generated files from the stdlib - println("pkg.Dir:" + pkg.Dir) - println("cloneBase:" + cloneBase) - goFiles := absoluteSourcesPaths(cloneBase, pkg.Dir, pkg.GoFiles) - - newPkg := &flatPackage{ - ID: stdlibPackageID(pkg.ImportPath), - Name: pkg.Name, - PkgPath: pkg.ImportPath, - ExportFile: labelledPath(cloneBase, pkg.Target), - Imports: map[string]string{}, - Standard: pkg.Standard, - GoFiles: goFiles, - CompiledGoFiles: goFiles, - } - for _, imp := range pkg.Imports { - newPkg.Imports[imp] = stdlibPackageID(imp) - } - // We don't support CGo for now - delete(newPkg.Imports, "C") - return newPkg + // Don't use generated files from the stdlib + goFiles := absoluteSourcesPaths(cloneBase, pkg.Dir, pkg.GoFiles) + + newPkg := &flatPackage{ + ID: stdlibPackageID(pkg.ImportPath), + Name: pkg.Name, + PkgPath: pkg.ImportPath, + ExportFile: labelledPath(cloneBase, pkg.Target), + Imports: map[string]string{}, + Standard: pkg.Standard, + GoFiles: goFiles, + CompiledGoFiles: goFiles, + } + for _, imp := range pkg.Imports { + newPkg.Imports[imp] = stdlibPackageID(imp) + } + // We don't support CGo for now + delete(newPkg.Imports, "C") + return newPkg } // In Go 1.18, the standard library started using go:embed directives. @@ -161,100 +157,95 @@ func flatPackageForStd(cloneBase string, pkg *goListPackage) *flatPackage { // go:embed directives will refuse to include the symlinks in the sandbox. // // To work around this, cloneGoRoot creates a copy of a subset of external/go_sdk -// that is sufficient to call "go list" into a new cloneBase directory while -// retaining its path relative to the root directory, e.g. "go list" needs to -// call "compile", which needs "pkg/tool". -// So "$OUTPUT_BASE/external/go_sdk" becomes +// that is sufficient to call "go list" into a new cloneBase directory, e.g. +// "go list" needs to call "compile", which needs "pkg/tool". +// We also need to retain the same relative path to the root directory, e.g. +// "$OUTPUT_BASE/external/go_sdk" becomes // {cloneBase}/external/go_sdk", which will be set at GOROOT later. This ensures // that file paths in the generated JSON are still valid. // // cloneGoRoot returns the new GOROOT we should run under. -func cloneGoRoot(goRoot, relativeGoroot, cloneBase string) (newGoRoot string, err error) { - newGoRoot = filepath.Join(cloneBase, relativeGoroot) - if err := os.MkdirAll(newGoRoot, 01755); err != nil { - return "", err - } - - if err := replicate(goRoot, newGoRoot, replicatePaths("src", "pkg/tool", "pkg/include")); err != nil { - return "", err - } - - return newGoRoot, nil +func cloneGoRoot(relativeGoroot, cloneBase string) (newGoRoot string, err error) { + goRoot := abs(relativeGoroot) + newGoRoot = filepath.Join(cloneBase, relativeGoroot) + if err := os.MkdirAll(newGoRoot, 01755); err != nil { + return "", err + } + + if err := replicate(goRoot, newGoRoot, replicatePaths("src", "pkg/tool", "pkg/include")); err != nil { + return "", err + } + + return newGoRoot, nil } // stdliblist runs `go list -json` on the standard library and saves it to a file. func stdliblist(args []string) error { - // process the args - flags := flag.NewFlagSet("stdliblist", flag.ExitOnError) - goenv := envFlags(flags) - out := flags.String("out", "", "Path to output go list json") - if err := flags.Parse(args); err != nil { - return err - } - if err := goenv.checkFlags(); err != nil { - return err - } - - goRoot, err := filepath.Abs(goenv.sdk) - if err != nil { - return err - } - cloneBase, cleanup, err := goenv.workDir() - if err != nil { - return err - } - defer func() { cleanup() }() - - cloneGoRoot, err := cloneGoRoot(goRoot, goenv.sdk, cloneBase) - if err != nil { - return fmt.Errorf("failed to clone new go root %v", err) - } - - // Ensure paths are absolute. - absPaths := []string{} - for _, path := range filepath.SplitList(os.Getenv("PATH")) { - absPaths = append(absPaths, abs(path)) - } - os.Setenv("PATH", strings.Join(absPaths, string(os.PathListSeparator))) - os.Setenv("GOROOT", cloneGoRoot) - // Make sure we have an absolute path to the C compiler. - // TODO(#1357): also take absolute paths of includes and other paths in flags. - os.Setenv("CC", abs(os.Getenv("CC"))) - - cachePath := abs(*out + ".gocache") - defer os.RemoveAll(cachePath) - os.Setenv("GOCACHE", cachePath) - os.Setenv("GOMODCACHE", cachePath) - os.Setenv("GOPATH", cachePath) - - listArgs := goenv.goCmd("list") - println("listargs:" + strings.Join(listArgs, ",")) - if len(build.Default.BuildTags) > 0 { - listArgs = append(listArgs, "-tags", strings.Join(build.Default.BuildTags, ",")) - } - listArgs = append(listArgs, "-json", "builtin", "std", "runtime/cgo") - - jsonFile, err := os.Create(*out) - if err != nil { - return err - } - defer jsonFile.Close() - - jsonData := &bytes.Buffer{} - if err := goenv.runCommandToFile(jsonData, listArgs); err != nil { - return err - } - encoder := json.NewEncoder(jsonFile) - decoder := json.NewDecoder(jsonData) - for decoder.More() { - var pkg *goListPackage - if err := decoder.Decode(&pkg); err != nil { - return err - } - if err := encoder.Encode(flatPackageForStd(cloneBase, pkg)); err != nil { - return err - } - } - - return nil + // process the args + flags := flag.NewFlagSet("stdliblist", flag.ExitOnError) + goenv := envFlags(flags) + out := flags.String("out", "", "Path to output go list json") + if err := flags.Parse(args); err != nil { + return err + } + if err := goenv.checkFlags(); err != nil { + return err + } + + cloneBase, cleanup, err := goenv.workDir() + if err != nil { + return err + } + defer func() { cleanup() }() + + cloneGoRoot, err := cloneGoRoot(goenv.sdk, cloneBase) + if err != nil { + return fmt.Errorf("failed to clone new go root %v", err) + } + // Ensure paths are absolute. + absPaths := []string{} + for _, path := range filepath.SplitList(os.Getenv("PATH")) { + absPaths = append(absPaths, abs(path)) + } + os.Setenv("PATH", strings.Join(absPaths, string(os.PathListSeparator))) + os.Setenv("GOROOT", cloneGoRoot) + // Make sure we have an absolute path to the C compiler. + // TODO(#1357): also take absolute paths of includes and other paths in flags. + os.Setenv("CC", abs(os.Getenv("CC"))) + + cachePath := abs(*out + ".gocache") + defer os.RemoveAll(cachePath) + os.Setenv("GOCACHE", cachePath) + os.Setenv("GOMODCACHE", cachePath) + os.Setenv("GOPATH", cachePath) + + listArgs := goenv.goCmd("list") + if len(build.Default.BuildTags) > 0 { + listArgs = append(listArgs, "-tags", strings.Join(build.Default.BuildTags, ",")) + } + listArgs = append(listArgs, "-json", "builtin", "std", "runtime/cgo") + + jsonFile, err := os.Create(*out) + if err != nil { + return err + } + defer jsonFile.Close() + + jsonData := &bytes.Buffer{} + if err := goenv.runCommandToFile(jsonData, listArgs); err != nil { + return err + } + encoder := json.NewEncoder(jsonFile) + decoder := json.NewDecoder(jsonData) + for decoder.More() { + var pkg *goListPackage + if err := decoder.Decode(&pkg); err != nil { + return err + } + if err := encoder.Encode(flatPackageForStd(cloneBase, pkg)); err != nil { + return err + } + } + + return nil } diff --git a/go/tools/builders/stdliblist_test.go b/go/tools/builders/stdliblist_test.go index ec86f68164..0cc5b7e488 100644 --- a/go/tools/builders/stdliblist_test.go +++ b/go/tools/builders/stdliblist_test.go @@ -8,31 +8,22 @@ import ( "path/filepath" "strings" "testing" - - "github.com/bazelbuild/rules_go/go/tools/bazel" ) func Test_stdliblist(t *testing.T) { testDir := t.TempDir() outJSON := filepath.Join(testDir, "out.json") - // test files are at run file directory, but this test is run at - // {runfile directory}/bazel.TestWorkspace() - // since -sdk is assumed to be a relative path to execRoot - // (go.sdk.root_file.dirname), thus setting wd to - // {runfile directory} so that go_sdk is discoverable - // {runfile directory} is the parent directory of bazel.RunfilesPath() - runFilesPath, err := bazel.RunfilesPath() - if err != nil { - t.Error("failed to find runfiles path") - } - runFilesPath = filepath.Dir(filepath.Clean(runFilesPath)) + // test files are at {runfile directory}/go_sdk, + // this test is run at {runfile directory}/{workspace}/../ + // thus go_sdk is the relative path to current working + // directory test_args := []string{ fmt.Sprintf("-out=%s", outJSON), - fmt.Sprintf("-sdk=%s", filepath.Join(runFilesPath, "go_sdk")), + fmt.Sprintf("-sdk=%s", "go_sdk"), } - err = stdliblist(test_args) + err := stdliblist(test_args) if err != nil { t.Errorf("calling stdliblist got err: %v", err) }