repo: repository directive (#651)

A repository directive specifies a repository rule that Gazelle should know about. The directive can be repeated multiple times and can be declared from within a macro definition that Gazelle knows about. At the very least the directive must define a rule kind and a name 
attribute, but it can define extra attributes after that

Fixes #132

README.rst

@@ -707,15 +707,31 @@ discover custom repository names and known prefixes. The ``fix`` and ``update``
 commands use these directives for dependency resolution. ``update-repos`` uses
 them to learn about repository rules defined in alternate locations.
 
-+-------------------------------------------------------+----------------------------------------+
-| **WORKSPACE Directive**                               | **Default value**                      |
-+=======================================================+========================================+
-| :direc:`# gazelle:repository_macro macroFile%defName` | n/a                                    |
-+-------------------------------------------------------+----------------------------------------+
-| Tells Gazelle to look for repository rules in a macro in a .bzl file. The directive can be     |
-| repeated multiple times.                                                                       |
-| The macro can be generated by calling ``update-repos`` with the ``to_macro`` flag.             |
-+-------------------------------------------------------+----------------------------------------+
++--------------------------------------------------------------------+----------------------------------------+
+| **WORKSPACE Directive**                                            | **Default value**                      |
++====================================================================+========================================+
+| :direc:`# gazelle:repository_macro macroFile%defName`              | n/a                                    |
++--------------------------------------------------------------------+----------------------------------------+
+| Tells Gazelle to look for repository rules in a macro in a .bzl file. The directive can be                  |
+| repeated multiple times.                                                                                    |
+| The macro can be generated by calling ``update-repos`` with the ``to_macro`` flag.                          |
++--------------------------------------------------------------------+----------------------------------------+
+| :direc:`# gazelle:repository rule_kind attr1_name=attr1_value ...` | n/a                                    |
++--------------------------------------------------------------------+----------------------------------------+
+| Specifies a repository rule that Gazelle should know about. The directive can be repeated multiple times,   |
+| and can be declared from within a macro definition that Gazelle knows about. At the very least the          |
+| directive must define a rule kind and a name attribute, but it can define extra attributes after that.      |
+|                                                                                                             |
+| This is useful for teaching Gazelle about repos declared in external macros. The directive can also be used |
+| to override an actual repository rule. For example, a ``git_repository`` rule for ``org_golang_x_tools``    |
+| could be overriden with the directive:                                                                      |
+|                                                                                                             |
+| .. code:: bzl                                                                                               |
+|                                                                                                             |
+|   # gazelle:repository go_repository name=org_golang_x_tools importpath=golang.org/x/tools                  |
+|                                                                                                             |
+| Gazelle would then proceed as if ``org_golang_x_tools`` was declared as a ``go_repository`` rule.           |
++--------------------------------------------------------------------+----------------------------------------+
 
 Keep comments
 ~~~~~~~~~~~~~

cmd/generate_repo_config/generate_repo_config_test.go

@@ -42,8 +42,8 @@ go_repository(
 			Path: "repositories.bzl",
 			Content: `
 load("@bazel_gazelle//:deps.bzl", "go_repository")
-# gazelle:repo test2
 def go_repositories():
+    # gazelle:repo test2
     go_repository(
         name = "org_golang_x_net",
         importpath = "golang.org/x/net",

repo/repo.go

@@ -69,12 +69,26 @@ func FindExternalRepo(repoRoot, name string) (string, error) {
 // ListRepositories extracts metadata about repositories declared in a
 // file.
 func ListRepositories(workspace *rule.File) (repos []*rule.Rule, repoFileMap map[string]*rule.File, err error) {
+	repoIndexMap := make(map[string]int)
 	repoFileMap = make(map[string]*rule.File)
 	for _, repo := range workspace.Rules {
 		if name := repo.Name(); name != "" {
+				repos = append(repos, repo)
+				repoFileMap[name] = workspace
+				repoIndexMap[name] = len(repos) - 1
+		}
+	}
+	extraRepos, err := parseRepositoryDirectives(workspace.Directives)
+	if err != nil {
+		return nil, nil, err
+	}
+	for _, repo := range extraRepos {
+		if i, ok := repoIndexMap[repo.Name()]; ok {
+			repos[i] = repo
+		} else {
 			repos = append(repos, repo)
-			repoFileMap[name] = workspace
 		}
+		repoFileMap[repo.Name()] = workspace
 	}
 
 	for _, d := range workspace.Directives {
@@ -93,13 +107,52 @@ func ListRepositories(workspace *rule.File) (repos []*rule.Rule, repoFileMap map
 				if name := repo.Name(); name != "" {
 					repos = append(repos, repo)
 					repoFileMap[name] = macroFile
+					repoIndexMap[name] = len(repos) - 1
+				}
+			}
+			extraRepos, err = parseRepositoryDirectives(macroFile.Directives)
+			if err != nil {
+				return nil, nil, err
+			}
+			for _, repo := range extraRepos {
+				if i, ok := repoIndexMap[repo.Name()]; ok {
+					repos[i] = repo
+				} else {
+					repos = append(repos, repo)
 				}
+				repoFileMap[repo.Name()] = macroFile
 			}
 		}
 	}
 	return repos, repoFileMap, nil
 }
 
+func parseRepositoryDirectives(directives []rule.Directive) (repos []*rule.Rule, err error) {
+	for _, d := range directives {
+		switch d.Key {
+		case "repository":
+			vals := strings.Fields(d.Value)
+			if len(vals) < 2 {
+				return nil, fmt.Errorf("failure parsing repository: %s, expected repository kind and attributes", d.Value)
+			}
+			kind := vals[0]
+			r := rule.NewRule(kind, "")
+			for _, val := range vals[1:] {
+				kv := strings.SplitN(val, "=", 2)
+				if len(kv) != 2 {
+					return nil, fmt.Errorf("failure parsing repository: %s, expected format for attributes is attr1_name=attr1_value", d.Value)
+				}
+				r.SetAttr(kv[0], kv[1])
+			}
+			if r.Name() == "" {
+				return nil, fmt.Errorf("failure parsing repository: %s, expected a name attribute for the given repository", d.Value)
+			}
+			repos = append(repos, r)
+		}
+	}
+	return repos, nil
+}
+
 func parseRepositoryMacroDirective(directive string) (string, string, error) {
 	vals := strings.Split(directive, "%")
 	if len(vals) != 2 {

repo/repo_test.go

@@ -107,6 +107,46 @@ go_repository(
 	}
 }
 
+func TestListRepositoriesWithRepositoryDirective(t *testing.T) {
+	for _, tc := range []struct {
+		desc, workspace, want string
+	}{
+		{
+			desc: "empty",
+			want: "",
+		}, {
+			desc: "git_repository",
+			workspace: `
+git_repository(
+    name = "custom_repo",
+    commit = "123456",
+    remote = "https://example.com/repo",
+    importpath = "example.com/repo",
+)
+# gazelle:repository go_repository name=custom_repo importpath=example.com/repo1
+# gazelle:repository go_repository name=custom_repo_2 importpath=example.com/repo2
+`,
+			want: `custom_repo example.com/repo1
+custom_repo_2 example.com/repo2`,
+		},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			workspace, err := rule.LoadData("WORKSPACE", "", []byte(tc.workspace))
+			if err != nil {
+				t.Fatal(err)
+			}
+			repos, _, err := repo.ListRepositories(workspace)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got := reposToString(repos)
+			if got != tc.want {
+				t.Errorf("got\n%s\n\nwant:\n%s", got, tc.want)
+			}
+		})
+	}
+}
+
 func TestListRepositoriesWithRepositoryMacroDirective(t *testing.T) {
 	files := []testtools.FileSpec{{
 		Path: "repos1.bzl",
@@ -130,6 +170,7 @@ def foo_repositories():
 		Path: "repos2.bzl",
 		Content: `
 def bar_repositories():
+    # gazelle:repository go_repository name=extra_repo importpath=example.com/extra
     go_repository(
         name = "bar_repo",
         commit = "123456",
@@ -138,6 +179,7 @@ def bar_repositories():
     )
 
 def baz_repositories():
+    # gazelle:repository go_repository name=ignored_repo importpath=example.com/ignored
     go_repository(
         name = "ignored_repo",
         commit = "123456",
@@ -163,7 +205,8 @@ def baz_repositories():
 	got := reposToString(repos)
 	want := `go_repo example.com/go
 foo_repo example.com/foo
-bar_repo example.com/bar`
+bar_repo example.com/bar
+extra_repo example.com/extra`
 	if got != want {
 		t.Errorf("got\n%s\n\nwant:\n%s", got, want)
 	}

rule/directives.go

@@ -39,6 +39,17 @@ type Directive struct {
 // is returned. Errors are reported for unrecognized directives and directives
 // out of place (after the first statement).
 func ParseDirectives(f *bzl.File) []Directive {
+	return parseDirectives(f.Stmt)
+}
+
+// ParseDirectivesFromMacro scans a macro body for Gazelle directives. The
+// full list of directives is returned. Errors are reported for unrecognized
+// directives and directives out of place (after the first statement).
+func ParseDirectivesFromMacro(f *bzl.DefStmt) []Directive {
+	return parseDirectives(f.Body)
+}
+
+func parseDirectives(stmt []bzl.Expr) []Directive {
 	var directives []Directive
 	parseComment := func(com bzl.Comment) {
 		match := directiveRe.FindStringSubmatch(com.Token)
@@ -49,7 +60,7 @@ func ParseDirectives(f *bzl.File) []Directive {
 		directives = append(directives, Directive{key, value})
 	}
 
-	for _, s := range f.Stmt {
+	for _, s := range stmt {
 		coms := s.Comment()
 		for _, com := range coms.Before {
 			parseComment(com)

rule/directives_test.go

@@ -54,7 +54,7 @@ foo(
 				t.Fatal(err)
 			}
 
-			got := ParseDirectives(f)
+			got := parseDirectives(f.Stmt)
 			if !reflect.DeepEqual(got, tc.want) {
 				t.Errorf("got %#v ; want %#v", got, tc.want)
 			}

rule/rule.go

@@ -205,7 +205,11 @@ func ScanASTBody(pkg, defName string, bzlFile *bzl.File) *File {
 			inserted: false,
 		}
 	}
-	f.Directives = ParseDirectives(bzlFile)
+	if f.function != nil {
+		f.Directives = ParseDirectivesFromMacro(f.function.stmt)
+	} else {
+		f.Directives = ParseDirectives(bzlFile)
+	}
 	return f
 }