go_repository: read configuration from main WORKSPACE (#550)

'gazelle fix' and 'gazelle update' now accept -repo_config, the path
to a file where information about repositories can be loaded. By
default, this is WORKSPACE in the repository root directory.
'gazelle fix' and 'gazelle update-repos' still update the WORKSPACE
file in the repository root directory when this flag is set.

go_repository passes the path to @//:WORKSPACE to -repo_config.

go_repository resolves @//:WORKSPACE and any files mentioned in
'# gazelle:repository_macro' directives. When these files, all
go_repository rules will be invalidated. It should not be necessary to
download cached repositories (except vcs repositories; see #549).
On a Macbook Pro, it takes about 22.5s to re-evaluate 70 cached,
invalidated go_repository rules for github.com/gohugoio/hugo. If this
becomes a project for large projects, we can provide a way to disable
or limit this behavior in the future.

go_repository_tools and go_repository_cache are moved to their own
.bzl files. Changes in go_repository.bzl should not invalidate these
in the future.

Fixes #529

cmd/gazelle/fix-update.go

@@ -63,8 +63,10 @@ func getUpdateConfig(c *config.Config) *updateConfig {
 }
 
 type updateConfigurer struct {
-	mode      string
-	recursive bool
+	mode           string
+	recursive      bool
+	knownImports   []string
+	repoConfigPath string
 }
 
 func (ucr *updateConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) {
@@ -76,6 +78,8 @@ func (ucr *updateConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *conf
 	fs.StringVar(&ucr.mode, "mode", "fix", "print: prints all of the updated BUILD files\n\tfix: rewrites all of the BUILD files in place\n\tdiff: computes the rewrite but then just does a diff")
 	fs.BoolVar(&ucr.recursive, "r", true, "when true, gazelle will update subdirectories recursively")
 	fs.StringVar(&uc.patchPath, "patch", "", "when set with -mode=diff, gazelle will write to a file instead of stdout")
+	fs.Var(&gzflag.MultiFlag{Values: &ucr.knownImports}, "known_import", "import path for which external resolution is skipped (can specify multiple times)")
+	fs.StringVar(&ucr.repoConfigPath, "repo_config", "", "file where Gazelle should load repository configuration. Defaults to WORKSPACE.")
 }
 
 func (ucr *updateConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) error {
@@ -118,6 +122,34 @@ func (ucr *updateConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) erro
 		uc.walkMode = walk.UpdateDirsMode
 	}
 
+	if ucr.repoConfigPath == "" {
+		ucr.repoConfigPath = filepath.Join(c.RepoRoot, "WORKSPACE")
+	}
+	if repoConfigFile, err := rule.LoadWorkspaceFile(ucr.repoConfigPath, ""); err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+	} else {
+		uc.repos, _, err = repo.ListRepositories(repoConfigFile)
+		if err != nil {
+			return err
+		}
+	}
+	repoPrefixes := make(map[string]bool)
+	for _, r := range uc.repos {
+		repoPrefixes[r.GoPrefix] = true
+	}
+	for _, imp := range ucr.knownImports {
+		if repoPrefixes[imp] {
+			continue
+		}
+		repo := repo.Repo{
+			Name:     label.ImportPathToBazelRepoName(imp),
+			GoPrefix: imp,
+		}
+		uc.repos = append(uc.repos, repo)
+	}
+
 	return nil
 }
 
@@ -331,9 +363,6 @@ func newFixUpdateConfiguration(cmd command, args []string, cexts []config.Config
 	// -h or -help were passed explicitly.
 	fs.Usage = func() {}
 
-	var knownImports []string
-	fs.Var(&gzflag.MultiFlag{Values: &knownImports}, "known_import", "import path for which external resolution is skipped (can specify multiple times)")
-
 	for _, cext := range cexts {
 		cext.RegisterFlags(fs, cmd.String(), c)
 	}
@@ -353,7 +382,6 @@ func newFixUpdateConfiguration(cmd command, args []string, cexts []config.Config
 		}
 	}
 
-	uc := getUpdateConfig(c)
 	workspacePath := filepath.Join(c.RepoRoot, "WORKSPACE")
 	if workspace, err := rule.LoadWorkspaceFile(workspacePath, ""); err != nil {
 		if !os.IsNotExist(err) {
@@ -362,7 +390,7 @@ func newFixUpdateConfiguration(cmd command, args []string, cexts []config.Config
 	} else {
 		c.RepoName = findWorkspaceName(workspace)
 		var reposFiles map[*rule.File][]string
-		uc.repos, reposFiles, err = repo.ListRepositories(workspace)
+		_, reposFiles, err = repo.ListRepositories(workspace)
 		if err != nil {
 			return nil, err
 		}
@@ -378,20 +406,6 @@ func newFixUpdateConfiguration(cmd command, args []string, cexts []config.Config
 			return nil, err
 		}
 	}
-	repoPrefixes := make(map[string]bool)
-	for _, r := range uc.repos {
-		repoPrefixes[r.GoPrefix] = true
-	}
-	for _, imp := range knownImports {
-		if repoPrefixes[imp] {
-			continue
-		}
-		repo := repo.Repo{
-			Name:     label.ImportPathToBazelRepoName(imp),
-			GoPrefix: imp,
-		}
-		uc.repos = append(uc.repos, repo)
-	}
 
 	return c, nil
 }

cmd/gazelle/integration_test.go

@@ -1064,7 +1064,7 @@ go_library(
 	})
 }
 
-func TestCustomRepoNames(t *testing.T) {
+func TestCustomRepoNamesMain(t *testing.T) {
 	files := []testtools.FileSpec{
 		{
 			Path: "WORKSPACE",
@@ -1110,6 +1110,60 @@ go_library(
 	})
 }
 
+func TestCustomRepoNamesExternal(t *testing.T) {
+	files := []testtools.FileSpec{
+		{
+			Path: "main/WORKSPACE",
+			Content: `go_repository(
+    name = "custom_repo",
+    importpath = "example.com/bar",
+    commit = "123456",
+)
+`,
+		}, {
+			Path:    "ext/WORKSPACE",
+			Content: "",
+		}, {
+			Path: "ext/foo.go",
+			Content: `
+package foo
+
+import _ "example.com/bar"
+`,
+		},
+	}
+	dir, cleanup := testtools.CreateFiles(t, files)
+	defer cleanup()
+
+	extDir := filepath.Join(dir, "ext")
+	args := []string{
+		"-go_prefix=example.com/foo",
+		"-mode=fix",
+		"-repo_root=" + extDir,
+		"-repo_config=" + filepath.Join(dir, "main", "WORKSPACE"),
+	}
+	if err := runGazelle(extDir, args); err != nil {
+		t.Fatal(err)
+	}
+
+	testtools.CheckFiles(t, dir, []testtools.FileSpec{
+		{
+			Path: "ext/BUILD.bazel",
+			Content: `
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["foo.go"],
+    importpath = "example.com/foo",
+    visibility = ["//visibility:public"],
+    deps = ["@custom_repo//:go_default_library"],
+)
+`,
+		},
+	})
+}
+
 func TestImportReposFromDepToWorkspace(t *testing.T) {
 	files := []testtools.FileSpec{
 		{

deps.bzl

@@ -15,8 +15,14 @@
 load(
     "@bazel_gazelle//internal:go_repository.bzl",
     _go_repository = "go_repository",
-    _go_repository_cache = "go_repository_cache",
-    _go_repository_tools = "go_repository_tools",
+)
+load(
+    "@bazel_gazelle//internal:go_repository_cache.bzl",
+    "go_repository_cache",
+)
+load(
+    "@bazel_gazelle//internal:go_repository_tools.bzl",
+    "go_repository_tools",
 )
 load(
     "@bazel_gazelle//internal:overlay_repository.bzl",
@@ -35,7 +41,7 @@ go_repository = _go_repository
 
 def gazelle_dependencies(go_sdk = ""):
     if go_sdk:
-        _go_repository_cache(
+        go_repository_cache(
             name = "bazel_gazelle_go_repository_cache",
             go_sdk_name = go_sdk,
         )
@@ -52,12 +58,12 @@ def gazelle_dependencies(go_sdk = ""):
             else:
                 platform = "host"
             go_sdk_info[name] = platform
-        _go_repository_cache(
+        go_repository_cache(
             name = "bazel_gazelle_go_repository_cache",
             go_sdk_info = go_sdk_info,
         )
 
-    _go_repository_tools(
+    go_repository_tools(
         name = "bazel_gazelle_go_repository_tools",
         go_cache = "@bazel_gazelle_go_repository_cache//:go.env",
     )
@@ -74,15 +80,15 @@ def gazelle_dependencies(go_sdk = ""):
         name = "com_github_bazelbuild_buildtools",
         importpath = "github.com/bazelbuild/buildtools",
         sum = "h1:KLCFP96KTydy03EMRwdGnngaD76gt34BBWK4g7TChgk=",
-        version = "v0.0.0-20190329162354-3f7be923c4b0",        
+        version = "v0.0.0-20190329162354-3f7be923c4b0",
     )
 
     _maybe(
         go_repository,
         name = "com_github_fsnotify_fsnotify",
         importpath = "github.com/fsnotify/fsnotify",
         sum = "h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=",
-        version = "v1.4.7",      
+        version = "v1.4.7",
     )
 
     _maybe(

go.mod

@@ -1,5 +1,7 @@
 module github.com/bazelbuild/bazel-gazelle
 
+go 1.12
+
 require (
 	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/bazelbuild/buildtools v0.0.0-20190329162354-3f7be923c4b0