Skip to content
This repository has been archived by the owner on Nov 5, 2024. It is now read-only.

Commit

Permalink
go/packages: allow types loading without NeedDeps
Browse files Browse the repository at this point in the history 
Before separating Load* into Need* we could use LoadSyntax
to get types information by loading inital packages
from source code and loading it's direct dependencies from export data.
It was broken when separation was introduced and before this commit
everything was loading from source code what resulted into slow
loading times. This commit fixes it.

Also, do backwards-incompatible fix of definition of deprecated
LoadImports and LoadAllSyntax.

Improve an internal error message
"internal error: nil Pkg importing x from y": replace it with
"internal error: package x without types was imported from y".

Remove packages.NeedDeps request for loading in tests
TestLoadTypesBits and TestContainsOverlayXTest.

Fixes golang/go#31752, fixes golang/go#33077, fixes golang/go#32814,
      fixes golang/go#31699, fixes golang/go#31930
  • Loading branch information
@jirfag
jirfag committed Sep 10, 2019
1 parent 5b82db0 commit 271b1bd
Show file tree
Hide file tree
Showing 3 changed files with 132 additions and 49 deletions.
3 changes: 1 addition & 2 deletions go/packages/golist.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -831,8 +831,7 @@ func golistargs(cfg *Config, words []string) []string {
fmt.Sprintf("-compiled=%t", cfg.Mode&(NeedCompiledGoFiles|NeedSyntax|NeedTypesInfo|NeedTypesSizes) != 0),
fmt.Sprintf("-test=%t", cfg.Tests),
fmt.Sprintf("-export=%t", usesExportData(cfg)),
fmt.Sprintf("-deps=%t", cfg.Mode&NeedDeps != 0 ||
cfg.Mode&NeedTypesInfo != 0), // Dependencies are required to do typechecking on sources, which is required for the TypesInfo.
fmt.Sprintf("-deps=%t", cfg.Mode&NeedImports != 0),
// go list doesn't let you pass -test and -find together,
// probably because you'd just get the TestMain.
fmt.Sprintf("-find=%t", !cfg.Tests && cfg.Mode&findFlags == 0),
Expand Down
71 changes: 36 additions & 35 deletions go/packages/packages.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ const (
// "placeholder" Packages with only the ID set.
NeedImports

// NeedDeps adds the fields requested by the LoadMode in the packages in Imports. If NeedImports
// is not set NeedDeps has no effect.
// NeedDeps adds the fields requested by the LoadMode in the packages in Imports.
// If NeedImports is not set, it will be added automatically.
NeedDeps

// NeedExportsFile adds ExportsFile.
Expand All @@ -61,7 +61,7 @@ const (
// NeedSyntax adds Syntax.
NeedSyntax

// NeedTypesInfo adds TypesInfo.
// NeedTypesInfo adds TypesInfo. If NeedImports is not set, it will be added automatically.
NeedTypesInfo

// NeedTypesSizes adds TypesSizes.
Expand All @@ -75,7 +75,7 @@ const (

// Deprecated: LoadImports exists for historical compatibility
// and should not be used. Please directly specify the needed fields using the Need values.
LoadImports = LoadFiles | NeedImports | NeedDeps
LoadImports = LoadFiles | NeedImports

// Deprecated: LoadTypes exists for historical compatibility
// and should not be used. Please directly specify the needed fields using the Need values.
Expand All @@ -87,7 +87,7 @@ const (

// Deprecated: LoadAllSyntax exists for historical compatibility
// and should not be used. Please directly specify the needed fields using the Need values.
LoadAllSyntax = LoadSyntax
LoadAllSyntax = LoadSyntax | NeedDeps
)

// A Config specifies details about how packages should be loaded.
Expand Down Expand Up @@ -476,6 +476,8 @@ func newLoader(cfg *Config) *loader {
}
}
}

ld.addDependingLoadModes()
return ld
}

Expand All @@ -496,7 +498,7 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
}
lpkg := &loaderPackage{
Package: pkg,
needtypes: (ld.Mode&(NeedTypes|NeedTypesInfo) != 0 && rootIndex < 0) || rootIndex >= 0,
needtypes: (ld.Mode&(NeedTypes|NeedTypesInfo) != 0 && ld.Mode&NeedDeps != 0 && rootIndex < 0) || rootIndex >= 0,
needsrc: (ld.Mode&(NeedSyntax|NeedTypesInfo) != 0 && ld.Mode&NeedDeps != 0 && rootIndex < 0) || rootIndex >= 0 ||
len(ld.Overlay) > 0 || // Overlays can invalidate export data. TODO(matloob): make this check fine-grained based on dependencies on overlaid files
pkg.ExportFile == "" && pkg.PkgPath != "unsafe",
Expand Down Expand Up @@ -544,10 +546,8 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
lpkg.color = grey
stack = append(stack, lpkg) // push
stubs := lpkg.Imports // the structure form has only stubs with the ID in the Imports
// If NeedTypesInfo we need dependencies (at least for the roots) to typecheck the package.
// If NeedImports isn't set, the imports fields will all be zeroed out.
// If NeedDeps isn't also set we want to keep the stubs.
if ld.Mode&NeedTypesInfo != 0 || (ld.Mode&NeedImports != 0 && ld.Mode&NeedDeps != 0) {
if ld.Mode&NeedImports != 0 {
lpkg.Imports = make(map[string]*Package, len(stubs))
for importPath, ipkg := range stubs {
var importErr error
Expand All @@ -566,11 +566,8 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
continue
}

// If !NeedDeps, just fill Imports for the root. No need to recurse further.
if ld.Mode&NeedDeps != 0 {
if visit(imp) {
lpkg.needsrc = true
}
if visit(imp) {
lpkg.needsrc = true
}
lpkg.Imports[importPath] = imp.Package
}
Expand All @@ -587,7 +584,7 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
return lpkg.needsrc
}

if ld.Mode&(NeedImports|NeedDeps|NeedTypesInfo) == 0 {
if ld.Mode&NeedImports == 0 {
// We do this to drop the stub import packages that we are not even going to try to resolve.
for _, lpkg := range initial {
lpkg.Imports = nil
Expand All @@ -598,7 +595,7 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
visit(lpkg)
}
}
if ld.Mode&NeedDeps != 0 { // TODO(matloob): This is only the case if NeedTypes is also set, right?
if ld.Mode&NeedImports != 0 && ld.Mode&NeedTypes != 0 {
for _, lpkg := range srcPkgs {
// Complete type information is required for the
// immediate dependencies of each source package.
Expand All @@ -623,7 +620,6 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
}

result := make([]*Package, len(initial))
importPlaceholders := make(map[string]*Package)
for i, lpkg := range initial {
result[i] = lpkg.Package
}
Expand Down Expand Up @@ -660,17 +656,8 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
if ld.Mode&NeedTypesSizes == 0 {
ld.pkgs[i].TypesSizes = nil
}
if ld.Mode&NeedDeps == 0 {
for j, pkg := range ld.pkgs[i].Imports {
ph, ok := importPlaceholders[pkg.ID]
if !ok {
ph = &Package{ID: pkg.ID}
importPlaceholders[pkg.ID] = ph
}
ld.pkgs[i].Imports[j] = ph
}
}
}

return result, nil
}

Expand All @@ -691,7 +678,6 @@ func (ld *loader) loadRecursive(lpkg *loaderPackage) {
}(imp)
}
wg.Wait()

ld.loadPackage(lpkg)
})
}
Expand Down Expand Up @@ -818,7 +804,7 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) {
if ipkg.Types != nil && ipkg.Types.Complete() {
return ipkg.Types, nil
}
log.Fatalf("internal error: nil Pkg importing %q from %q", path, lpkg)
log.Fatalf("internal error: package %q without types was imported from %q", path, lpkg)
panic("unreachable")
})

Expand All @@ -829,7 +815,7 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) {
// Type-check bodies of functions only in non-initial packages.
// Example: for import graph A->B->C and initial packages {A,C},
// we can ignore function bodies in B.
IgnoreFuncBodies: (ld.Mode&(NeedDeps|NeedTypesInfo) == 0) && !lpkg.initial,
IgnoreFuncBodies: ld.Mode&NeedDeps == 0 && !lpkg.initial,

Error: appendError,
Sizes: ld.sizes,
Expand Down Expand Up @@ -1091,10 +1077,25 @@ func (ld *loader) loadFromExportData(lpkg *loaderPackage) (*types.Package, error
return tpkg, nil
}

// addDependingLoadModes adds dependencies for choosed LoadMode in ld.Mode
func (ld *loader) addDependingLoadModes() {
if ld.Mode&NeedTypesInfo != 0 && ld.Mode&NeedImports == 0 {
// If NeedTypesInfo, go/packages needs to do typechecking itself so it can
// associate type info with the AST. To do so, we need the export data
// for dependencies, which means we need to ask for the direct dependencies.
// NeedImports is used to ask for the direct dependencies.
ld.Mode |= NeedImports
ld.Logf("Added load mode dependency of NeedTypesInfo: NeedImports")
}

if ld.Mode&NeedDeps != 0 && ld.Mode&NeedImports == 0 {
// With NeedDeps we need to load at least direct dependencies.
// NeedImports is used to ask for the direct dependencies.
ld.Mode |= NeedImports
ld.Logf("Added load mode dependency of NeedDeps: NeedImports")
}
}

func usesExportData(cfg *Config) bool {
return cfg.Mode&NeedExportsFile != 0 ||
// If NeedTypes but not NeedTypesInfo we won't typecheck using sources, so we need export data.
(cfg.Mode&NeedTypes != 0 && cfg.Mode&NeedTypesInfo == 0) ||
// If NeedTypesInfo but not NeedDeps, we're typechecking a package using its sources plus its dependencies' export data
(cfg.Mode&NeedTypesInfo != 0 && cfg.Mode&NeedDeps == 0)
return cfg.Mode&NeedExportsFile != 0 || cfg.Mode&NeedTypes != 0 && cfg.Mode&NeedDeps == 0
}
107 changes: 95 additions & 12 deletions go/packages/packages_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -577,7 +577,7 @@ func testLoadTypesBits(t *testing.T, exporter packagestest.Exporter) {
}}})
defer exported.Cleanup()

exported.Config.Mode = packages.NeedTypes | packages.NeedDeps | packages.NeedImports
exported.Config.Mode = packages.NeedTypes | packages.NeedImports
initial, err := packages.Load(exported.Config, "golang.org/fake/a", "golang.org/fake/c")
if err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -678,14 +678,16 @@ func testLoadSyntaxOK(t *testing.T, exporter packagestest.Exporter) {
}

for _, test := range []struct {
id string
id string
wantSyntax bool
wantComplete bool
}{
{"golang.org/fake/a"}, // source package
{"golang.org/fake/b"}, // source package
{"golang.org/fake/c"}, // source package
{"golang.org/fake/d"}, // export data package
{"golang.org/fake/e"}, // export data package
{"golang.org/fake/f"}, // export data package
{"golang.org/fake/a", true, true}, // source package
{"golang.org/fake/b", true, true}, // source package because depends on initial package
{"golang.org/fake/c", true, true}, // source package
{"golang.org/fake/d", false, true}, // export data package
{"golang.org/fake/e", false, false}, // export data package
{"golang.org/fake/f", false, false}, // export data package
} {
// TODO(matloob): LoadSyntax and LoadAllSyntax are now equivalent, wantSyntax and wantComplete
// are true for all packages in the transitive dependency set. Add test cases on the individual
Expand All @@ -698,9 +700,19 @@ func testLoadSyntaxOK(t *testing.T, exporter packagestest.Exporter) {
if p.Types == nil {
t.Errorf("missing types.Package for %s", p)
continue
} else if p.Types.Complete() != test.wantComplete {
if test.wantComplete {
t.Errorf("incomplete types.Package for %s", p)
} else {
t.Errorf("unexpected complete types.Package for %s", p)
}
}
if p.Syntax == nil {
t.Errorf("missing ast.Files for %s", p)
if (p.Syntax != nil) != test.wantSyntax {
if test.wantSyntax {
t.Errorf("missing ast.Files for %s", p)
} else {
t.Errorf("unexpected ast.Files for for %s", p)
}
}
if p.Errors != nil {
t.Errorf("errors in package: %s: %s", p, p.Errors)
Expand Down Expand Up @@ -796,7 +808,7 @@ func testLoadSyntaxError(t *testing.T, exporter packagestest.Exporter) {
{"golang.org/fake/c", true, true},
{"golang.org/fake/d", true, true},
{"golang.org/fake/e", true, true},
{"golang.org/fake/f", true, false},
{"golang.org/fake/f", false, false},
} {
p := all[test.id]
if p == nil {
Expand Down Expand Up @@ -1364,7 +1376,7 @@ func testContainsOverlayXTest(t *testing.T, exporter packagestest.Exporter) {
}}})
defer exported.Cleanup()
bOverlayXTestFile := filepath.Join(filepath.Dir(exported.File("golang.org/fake", "b/b.go")), "b_overlay_x_test.go")
exported.Config.Mode = packages.NeedName | packages.NeedFiles | packages.NeedImports | packages.NeedDeps
exported.Config.Mode = packages.NeedName | packages.NeedFiles | packages.NeedImports
exported.Config.Overlay = map[string][]byte{bOverlayXTestFile: []byte(`package b_test; import "golang.org/fake/b"`)}
initial, err := packages.Load(exported.Config, "file="+bOverlayXTestFile)
if err != nil {
Expand Down Expand Up @@ -2178,6 +2190,77 @@ func testIssue32814(t *testing.T, exporter packagestest.Exporter) {
}
}

func TestLoadTypesInfoWithoutNeedDeps(t *testing.T) {
packagestest.TestAll(t, testLoadTypesInfoWithoutNeedDeps)
}
func testLoadTypesInfoWithoutNeedDeps(t *testing.T, exporter packagestest.Exporter) {
exported := packagestest.Export(t, exporter, []packagestest.Module{{
Name: "golang.org/fake",
Files: map[string]interface{}{
"a/a.go": `package a; import _ "golang.org/fake/b"`,
"b/b.go": `package b`,
}}})
defer exported.Cleanup()

exported.Config.Mode = packages.NeedTypes | packages.NeedTypesInfo | packages.NeedImports
pkgs, err := packages.Load(exported.Config, "golang.org/fake/a")
if err != nil {
t.Fatal(err)
}
pkg := pkgs[0]
if pkg.IllTyped {
t.Fatal("Loaded package is ill typed")
}
const expectedImport = "golang.org/fake/b"
if _, ok := pkg.Imports[expectedImport]; !ok || len(pkg.Imports) != 1 {
t.Fatalf("Imports of loaded package: want [%s], got %v", expectedImport, pkg.Imports)
}
}

func TestLoadWithNeedDeps(t *testing.T) {
packagestest.TestAll(t, testLoadWithNeedDeps)
}
func testLoadWithNeedDeps(t *testing.T, exporter packagestest.Exporter) {
exported := packagestest.Export(t, exporter, []packagestest.Module{{
Name: "golang.org/fake",
Files: map[string]interface{}{
"a/a.go": `package a; import _ "golang.org/fake/b"`,
"b/b.go": `package b; import _ "golang.org/fake/c"`,
"c/c.go": `package c`,
}}})
defer exported.Cleanup()

exported.Config.Mode = packages.NeedTypes | packages.NeedTypesInfo | packages.NeedImports | packages.NeedDeps
pkgs, err := packages.Load(exported.Config, "golang.org/fake/a")
if err != nil {
t.Fatal(err)
}
if len(pkgs) != 1 {
t.Fatalf("Expected 1 package, got %d", len(pkgs))
}

pkgA := pkgs[0]
if pkgA.IllTyped {
t.Fatal("Loaded package is ill typed")
}

pkgB := pkgA.Imports["golang.org/fake/b"]
if pkgB == nil || len(pkgA.Imports) != 1 {
t.Fatalf("Imports of loaded package 'a' are invalid: %v", pkgA.Imports)
}
if pkgB.Types == nil || !pkgB.Types.Complete() || pkgB.TypesInfo == nil {
t.Fatalf("Types of package 'b' are nil or incomplete: %v, %v", pkgB.Types, pkgB.TypesInfo)
}

pkgC := pkgB.Imports["golang.org/fake/c"]
if pkgC == nil || len(pkgB.Imports) != 1 {
t.Fatalf("Imports of loaded package 'c' are invalid: %v", pkgB.Imports)
}
if pkgC.Types == nil || !pkgC.Types.Complete() || pkgC.TypesInfo == nil {
t.Fatalf("Types of package 'b' are nil or incomplete: %v, %v", pkgC.Types, pkgC.TypesInfo)
}
}

func errorMessages(errors []packages.Error) []string {
var msgs []string
for _, err := range errors {
Expand Down

0 comments on commit 271b1bd

Please sign in to comment.