From c79e9424fca4994f3d7e968a783bea60e32b55ff Mon Sep 17 00:00:00 2001 From: Noam Gal Date: Thu, 17 Jun 2021 18:23:16 +0300 Subject: [PATCH] fix-ref/sha/tag (#107) * revert #98 * checkout after clone Co-authored-by: roi.kramer --- Makefile | 4 +- cmd/commands/app.go | 4 +- cmd/commands/repo.go | 5 +- cmd/commands/repo_test.go | 1 + docs/App-Specifier.md | 41 +++ .../argocd-autopilot_repo_bootstrap.md | 2 +- .../namespace-install/kustomization.yaml | 2 +- mkdocs.yml | 5 +- pkg/application/application.go | 15 +- pkg/git/repository.go | 208 ++++--------- pkg/git/repository_test.go | 183 +++++++++-- pkg/git/repospec_test.go | 215 ------------- pkg/util/repospec.go | 165 ++++++++++ pkg/util/repospec_test.go | 284 ++++++++++++++++++ 14 files changed, 732 insertions(+), 402 deletions(-) create mode 100644 docs/App-Specifier.md delete mode 100644 pkg/git/repospec_test.go create mode 100644 pkg/util/repospec.go create mode 100644 pkg/util/repospec_test.go diff --git a/Makefile b/Makefile index d7d599e9..b32eaebf 100644 --- a/Makefile +++ b/Makefile @@ -5,8 +5,8 @@ CLI_NAME?=argocd-autopilot IMAGE_REPOSITORY?=quay.io IMAGE_NAMESPACE?=argoprojlabs -INSTALLATION_MANIFESTS_URL="github.com/argoproj-labs/argocd-autopilot/manifests?tag=$(VERSION)" -INSTALLATION_MANIFESTS_NAMESPACED_URL="github.com/argoproj-labs/argocd-autopilot/manifests/namespace-install?tag=$(VERSION)" +INSTALLATION_MANIFESTS_URL="github.com/argoproj-labs/argocd-autopilot/manifests?ref=$(VERSION)" +INSTALLATION_MANIFESTS_NAMESPACED_URL="github.com/argoproj-labs/argocd-autopilot/manifests/namespace-install?ref=$(VERSION)" DEV_INSTALLATION_MANIFESTS_URL="manifests/" DEV_INSTALLATION_MANIFESTS_NAMESPACED_URL="manifests/namespace-install" diff --git a/cmd/commands/app.go b/cmd/commands/app.go index 20cd28ef..7a4d3b4b 100644 --- a/cmd/commands/app.go +++ b/cmd/commands/app.go @@ -215,7 +215,9 @@ var setAppOptsDefaults = func(ctx context.Context, repofs fs.FS, opts *AppCreate // local directory fsys = fs.Create(osfs.New(opts.AppOpts.AppSpecifier)) } else { - log.G().Infof("trying to infer application type from '%s'", opts.AppOpts.AppSpecifier) + host, orgRepo, p, _, _, suffix, _ := util.ParseGitUrl(opts.AppOpts.AppSpecifier) + url := host + orgRepo + suffix + log.G().Infof("cloning repo: '%s', to infer app type from path '%s'", url, p) cloneOpts := &git.CloneOptions{ Repo: opts.AppOpts.AppSpecifier, Auth: opts.CloneOpts.Auth, diff --git a/cmd/commands/repo.go b/cmd/commands/repo.go index 386a1640..fa0b3f9c 100644 --- a/cmd/commands/repo.go +++ b/cmd/commands/repo.go @@ -182,7 +182,7 @@ func NewRepoBootstrapCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&appSpecifier, "app", "", "The application specifier (e.g. argocd@v1.0.2)") + cmd.Flags().StringVar(&appSpecifier, "app", "", "The application specifier (e.g. github.com/argoproj-labs/argocd-autopilot/manifests?ref=v0.2.5), overrides the default installation argo-cd manifests") cmd.Flags().BoolVar(&namespaced, "namespaced", false, "If true, install a namespaced version of argo-cd (no need for cluster-role)") cmd.Flags().BoolVar(&dryRun, "dry-run", false, "If true, print manifests instead of applying them to the cluster (nothing will be commited to git)") cmd.Flags().BoolVar(&hidePassword, "hide-password", false, "If true, will not print initial argo cd password") @@ -264,7 +264,6 @@ func RunRepoCreate(ctx context.Context, opts *RepoCreateOptions) (*git.CloneOpti Password: opts.Token, }, } - co.Parse() return co, nil } @@ -664,7 +663,7 @@ func createBootstrapKustomization(namespace, repoURL, appSpecifier string) (*kus } func createCreds(repoUrl string) ([]byte, error) { - host, _, _, _, _ := git.ParseGitUrl(repoUrl) + host, _, _, _, _, _, _ := util.ParseGitUrl(repoUrl) creds := []argocdsettings.RepositoryCredentials{ { URL: host, diff --git a/cmd/commands/repo_test.go b/cmd/commands/repo_test.go index cce3c224..8fb1bbb7 100644 --- a/cmd/commands/repo_test.go +++ b/cmd/commands/repo_test.go @@ -296,6 +296,7 @@ func Test_buildBootstrapManifests(t *testing.T) { for tname, tt := range tests { t.Run(tname, func(t *testing.T) { tt.args.cloneOpts.Parse() + b, ret := buildBootstrapManifests( tt.args.namespace, tt.args.appSpecifier, diff --git a/docs/App-Specifier.md b/docs/App-Specifier.md new file mode 100644 index 00000000..050c76d6 --- /dev/null +++ b/docs/App-Specifier.md @@ -0,0 +1,41 @@ +# The Application Specifier + +The application specifier is a string denoting the entrypoint to the application that you want to create. You specify it when using the `--app` +flag in the `app create` and `repo bootstrap` commands. + +## Structure +Lets look at the following example of adding argo workflows v3.0.7 to project `prod` to better understand the structure of the application specifier: +```bash +argocd-autopilot app create workflows --app "github.com/argoproj/argo-workflows/manifests/cluster-install?ref=v3.0.7" --project prod +``` +In this example the app specifier is: `github.com/argoproj/argo-workflows/manifests/cluster-install?ref=v3.0.7`, which is composed of three parts: + +1. `github.com/argoproj/argo-workflows`: The repository +2. `manifests/cluster-install`: The path inside the repository to the directory containing the base `kustomization.yaml` +3. `?ref=v3.0.7`: The git ref to use, in this case, the tag `v3.0.7` + +!!! note + The `ref` that will be used to get the application manifests is calculated using the following logic: + + 1. If not specified - uses the HEAD of the main branch of the repository + 2. If there is a commit with the same SHA use this commit + 3. Looks for a tag with the same name + 4. Looks for a branch with the same name + +## Application Type Inference +By default, `argocd-autopilot` will try to automatically infer the correct application type from the supported [application types](https://argoproj.github.io/argo-cd/user-guide/application_sources/#tools) (currently only kustomize and directory types are supported). To do that it would try to clone the repository, checkout the correct ref, and look at the specified path for the following: + +1. If there is a `kustomization.yaml` - the infered application type is `kustomize` +2. Else - the infered application type is `directory` + +!!! tip + If you don't want `argocd-autopilot` to infer the type automatically, you can specify the application type yourself using the `--type` flag. + +## Local Application Path +If the application specifier is a path to a local directory on your machine, `argocd autopilot` will automatically detect that and use `flat` installation mode, meaning it would build all of the manifests and write them into one `install.yaml` file, which would be required by a base `kustomization.yaml`. + +For example: +``` +argocd-autopilot app create someapp --app ./path/to/kustomization/dir --project dev +``` +Assuming the file `./path/to/kustomization/dir/kustomization.yaml` exists, `argocd-autopilot` will run `kustomize build`, then commit the resulting manifests to the gitops repository under: `apps/someapp/base/install.yaml`, with the base kustomization, located at `apps/someapp/base/kustomization.yaml`, requiring it. \ No newline at end of file diff --git a/docs/commands/argocd-autopilot_repo_bootstrap.md b/docs/commands/argocd-autopilot_repo_bootstrap.md index 17907dbe..874b8d0b 100644 --- a/docs/commands/argocd-autopilot_repo_bootstrap.md +++ b/docs/commands/argocd-autopilot_repo_bootstrap.md @@ -34,7 +34,7 @@ argocd-autopilot repo bootstrap [flags] ### Options ``` - --app string The application specifier (e.g. argocd@v1.0.2) + --app string The application specifier (e.g. github.com/argoproj-labs/argocd-autopilot/manifests?ref=v0.2.5), overrides the default installation argo-cd manifests --as string Username to impersonate for the operation --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. --cache-dir string Default cache directory (default "/home/user/.kube/cache") diff --git a/manifests/namespace-install/kustomization.yaml b/manifests/namespace-install/kustomization.yaml index 201caff3..453e7c5f 100644 --- a/manifests/namespace-install/kustomization.yaml +++ b/manifests/namespace-install/kustomization.yaml @@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - https://raw.githubusercontent.com/argoproj/argo-cd/v2.0.0/manifests/namespace-install.yaml - - https://raw.githubusercontent.com/argoproj-labs/applicationset/v0.1.0/manifests/install.yaml + - https://raw.githubusercontent.com/argoproj-labs/applicationset/master/manifests/install.yaml # TODO: switch to the next release when available diff --git a/mkdocs.yml b/mkdocs.yml index 1b4f3f85..865a16e9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,8 +22,11 @@ markdown_extensions: permalink: true nav: - Introduction: index.md - - Getting Started: Getting-Started.md - Installation: Installation-Guide.md + - User Guide: + - Getting Started: Getting-Started.md + - Application Specifier: App-Specifier.md + - Command Reference: commands/argocd-autopilot.md - Development: Development.md - Roadmap: Roadmap.md - Blogs: Blogs.md diff --git a/pkg/application/application.go b/pkg/application/application.go index 0db354e9..0db184fd 100644 --- a/pkg/application/application.go +++ b/pkg/application/application.go @@ -10,10 +10,10 @@ import ( "reflect" "github.com/argoproj-labs/argocd-autopilot/pkg/fs" - "github.com/argoproj-labs/argocd-autopilot/pkg/git" "github.com/argoproj-labs/argocd-autopilot/pkg/kube" "github.com/argoproj-labs/argocd-autopilot/pkg/log" "github.com/argoproj-labs/argocd-autopilot/pkg/store" + "github.com/argoproj-labs/argocd-autopilot/pkg/util" "github.com/ghodss/yaml" billyUtils "github.com/go-git/go-billy/v5/util" @@ -399,18 +399,17 @@ func newDirApp(opts *CreateOptions) *dirApp { app := &dirApp{ baseApp: baseApp{opts}, } - cloneOpts := &git.CloneOptions{ - Repo: opts.AppSpecifier, - } - cloneOpts.Parse() + + host, orgRepo, path, gitRef, _, suffix, _ := util.ParseGitUrl(opts.AppSpecifier) + url := host + orgRepo + suffix app.config = &Config{ AppName: opts.AppName, UserGivenName: opts.AppName, DestNamespace: opts.DestNamespace, DestServer: opts.DestServer, - SrcRepoURL: cloneOpts.URL(), - SrcPath: cloneOpts.Path(), - SrcTargetRevision: cloneOpts.Revision(), + SrcRepoURL: url, + SrcPath: path, + SrcTargetRevision: gitRef, } return app diff --git a/pkg/git/repository.go b/pkg/git/repository.go index ccbd6122..950d43b6 100644 --- a/pkg/git/repository.go +++ b/pkg/git/repository.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "net/url" "os" "strings" @@ -58,20 +57,20 @@ type ( } ) -const ( - gitSuffix = ".git" - gitDelimiter = "_git/" -) - // Errors var ( ErrNilOpts = errors.New("options cannot be nil") ErrNoParse = errors.New("must call Parse before using CloneOptions") ErrRepoNotFound = errors.New("git repository not found") + ErrNoRemotes = errors.New("no remotes in repository") ) // go-git functions (we mock those in tests) var ( + checkoutRef = func(r *repo, ref string) error { + return r.checkoutRef(ref) + } + ggClone = func(ctx context.Context, s storage.Storer, worktree billy.Filesystem, o *gg.CloneOptions) (gogit.Repository, error) { return gg.CloneContext(ctx, s, worktree, o) } @@ -122,7 +121,7 @@ func (o *CloneOptions) Parse() { suffix string ) - host, orgRepo, o.path, o.revision, suffix = ParseGitUrl(o.Repo) + host, orgRepo, o.path, o.revision, _, suffix, _ = util.ParseGitUrl(o.Repo) o.url = host + orgRepo + suffix } @@ -160,7 +159,7 @@ func (o *CloneOptions) URL() string { } func (o *CloneOptions) Revision() string { - return plumbing.ReferenceName(o.revision).Short() + return o.revision } func (o *CloneOptions) Path() string { @@ -211,23 +210,23 @@ var clone = func(ctx context.Context, opts *CloneOptions) (*repo, error) { Auth: getAuth(opts.Auth), Depth: 1, Progress: opts.Progress, - Tags: gg.NoTags, - } - - if opts.revision != "" { - cloneOpts.ReferenceName = plumbing.ReferenceName(opts.revision) } - log.G(ctx).WithFields(log.Fields{ - "url": opts.url, - "rev": opts.revision, - }).Debug("cloning git repo") + log.G(ctx).WithField("url", opts.url).Debug("cloning git repo") r, err := ggClone(ctx, memory.NewStorage(), opts.FS, cloneOpts) if err != nil { return nil, err } - return &repo{Repository: r, auth: opts.Auth}, nil + repo := &repo{Repository: r, auth: opts.Auth} + + if opts.revision != "" { + if err := checkoutRef(repo, opts.revision); err != nil { + return nil, err + } + } + + return repo, nil } var initRepo = func(ctx context.Context, opts *CloneOptions) (*repo, error) { @@ -244,6 +243,44 @@ var initRepo = func(ctx context.Context, opts *CloneOptions) (*repo, error) { return r, r.initBranch(ctx, opts.revision) } +func (r *repo) checkoutRef(ref string) error { + hash, err := r.ResolveRevision(plumbing.Revision(ref)) + if err != nil { + if err != plumbing.ErrReferenceNotFound { + return err + } + + log.G().WithField("ref", ref).Debug("failed resolving ref, trying to resolve from remote branch") + remotes, err := r.Remotes() + if err != nil { + return err + } + + if len(remotes) == 0 { + return ErrNoRemotes + } + + remoteref := fmt.Sprintf("%s/%s", remotes[0].Config().Name, ref) + hash, err = r.ResolveRevision(plumbing.Revision(remoteref)) + if err != nil { + return err + } + } + + wt, err := worktree(r) + if err != nil { + return err + } + + log.G().WithFields(log.Fields{ + "ref": ref, + "hash": hash.String(), + }).Debug("checking out commit") + return wt.Checkout(&gg.CheckoutOptions{ + Hash: *hash, + }) +} + func (r *repo) addRemote(name, url string) error { _, err := r.CreateRemote(&config.RemoteConfig{Name: name, URLs: []string{url}}) return err @@ -287,138 +324,3 @@ func getAuth(auth Auth) transport.AuthMethod { Password: auth.Password, } } - -// ParseGitUrl returns the different parts of the repo url -// example: "https://github.com/owner/name/repo/path?ref=branch" -// host: "https://github.com" -// orgRepo: "owner/name" -// path: "path" -// ref: "refs/heads/branch" -// gitSuff: ".git" -// For tags use "?tag=" -// For specific git commit sha use "?sha=" -func ParseGitUrl(n string) (host, orgRepo, path, ref, gitSuff string) { - if strings.Contains(n, gitDelimiter) { - index := strings.Index(n, gitDelimiter) - // Adding _git/ to host - host = normalizeGitHostSpec(n[:index+len(gitDelimiter)]) - orgRepo = strings.Split(strings.Split(n[index+len(gitDelimiter):], "/")[0], "?")[0] - path, ref = peelQuery(n[index+len(gitDelimiter)+len(orgRepo):]) - return - } - - host, n = parseHostSpec(n) - gitSuff = gitSuffix - if strings.Contains(n, gitSuffix) { - index := strings.Index(n, gitSuffix) - orgRepo = n[0:index] - n = n[index+len(gitSuffix):] - path, ref = peelQuery(n) - return - } - - i := strings.Index(n, "/") - if i < 1 { - path, ref = peelQuery(n) - return - } - - j := strings.Index(n[i+1:], "/") - if j >= 0 { - j += i + 1 - orgRepo = n[:j] - path, ref = peelQuery(n[j+1:]) - return - } - - path = "" - orgRepo, ref = peelQuery(n) - return -} - -func peelQuery(arg string) (path, ref string) { - parsed, err := url.Parse(arg) - if err != nil { - return path, "" - } - - path = parsed.Path - values := parsed.Query() - branch := values.Get("ref") - tag := values.Get("tag") - sha := values.Get("sha") - if sha != "" { - ref = sha - return - } - - if tag != "" { - ref = "refs/tags/" + tag - return - } - - if branch != "" { - ref = "refs/heads/" + branch - return - } - - return -} - -func parseHostSpec(n string) (string, string) { - var host string - // Start accumulating the host part. - for _, p := range [...]string{ - // Order matters here. - "git::", "gh:", "ssh://", "https://", "http://", - "git@", "github.com:", "github.com/"} { - if len(p) < len(n) && strings.ToLower(n[:len(p)]) == p { - n = n[len(p):] - host += p - } - } - if host == "git@" { - i := strings.Index(n, "/") - if i > -1 { - host += n[:i+1] - n = n[i+1:] - } else { - i = strings.Index(n, ":") - if i > -1 { - host += n[:i+1] - n = n[i+1:] - } - } - return host, n - } - - // If host is a http(s) or ssh URL, grab the domain part. - for _, p := range [...]string{ - "ssh://", "https://", "http://"} { - if strings.HasSuffix(host, p) { - i := strings.Index(n, "/") - if i > -1 { - host = host + n[0:i+1] - n = n[i+1:] - } - break - } - } - - return normalizeGitHostSpec(host), n -} - -func normalizeGitHostSpec(host string) string { - s := strings.ToLower(host) - if strings.Contains(s, "github.com") { - if strings.Contains(s, "git@") || strings.Contains(s, "ssh:") { - host = "git@github.com:" - } else { - host = "https://github.com/" - } - } - if strings.HasPrefix(s, "git::") { - host = strings.TrimPrefix(s, "git::") - } - return host -} diff --git a/pkg/git/repository_test.go b/pkg/git/repository_test.go index c7a718c6..2f475126 100644 --- a/pkg/git/repository_test.go +++ b/pkg/git/repository_test.go @@ -2,6 +2,7 @@ package git import ( "context" + "errors" "fmt" "os" "reflect" @@ -233,16 +234,17 @@ func Test_clone(t *testing.T) { opts *CloneOptions retErr error wantErr bool - assertFn func(*testing.T, *repo) expectedOpts *gg.CloneOptions + checkoutRef func(t *testing.T, r *repo, ref string) error + assertFn func(t *testing.T, r *repo) }{ - "NilOpts": { + "Should fail when there are no CloneOptions": { wantErr: true, assertFn: func(t *testing.T, r *repo) { assert.Nil(t, r) }, }, - "No Auth": { + "Should clone without Auth data": { opts: &CloneOptions{ Repo: "https://github.com/owner/name", }, @@ -251,15 +253,14 @@ func Test_clone(t *testing.T) { Auth: nil, Depth: 1, Progress: os.Stderr, - Tags: gg.NoTags, }, assertFn: func(t *testing.T, r *repo) { assert.NotNil(t, r) }, }, - "With Auth": { + "Should clone with Auth data": { opts: &CloneOptions{ - Repo: "https://github.com/owner/name", + Repo: "https://github.com/owner/name.git", Auth: Auth{ Username: "asd", Password: "123", @@ -273,13 +274,12 @@ func Test_clone(t *testing.T) { }, Depth: 1, Progress: os.Stderr, - Tags: gg.NoTags, }, assertFn: func(t *testing.T, r *repo) { assert.NotNil(t, r) }, }, - "Error": { + "Should fail if go-git.Clone fails": { opts: &CloneOptions{ Repo: "https://github.com/owner/name", }, @@ -287,7 +287,6 @@ func Test_clone(t *testing.T) { URL: "https://github.com/owner/name.git", Depth: 1, Progress: os.Stderr, - Tags: gg.NoTags, }, retErr: fmt.Errorf("error"), wantErr: true, @@ -295,25 +294,48 @@ func Test_clone(t *testing.T) { assert.Nil(t, r) }, }, - "With Revision": { + "Should checkout revision after clone": { opts: &CloneOptions{ Repo: "https://github.com/owner/name?ref=test", }, expectedOpts: &gg.CloneOptions{ - URL: "https://github.com/owner/name.git", - Depth: 1, - Progress: os.Stderr, - Tags: gg.NoTags, - ReferenceName: plumbing.NewBranchReferenceName("test"), + URL: "https://github.com/owner/name.git", + Depth: 1, + Progress: os.Stderr, + }, + checkoutRef: func(t *testing.T, _ *repo, ref string) error { + assert.Equal(t, "test", ref) + return nil }, assertFn: func(t *testing.T, r *repo) { assert.NotNil(t, r) }, }, + "Should fail if checkout fails": { + opts: &CloneOptions{ + Repo: "https://github.com/owner/name?ref=test", + }, + expectedOpts: &gg.CloneOptions{ + URL: "https://github.com/owner/name.git", + Depth: 1, + Progress: os.Stderr, + }, + wantErr: true, + checkoutRef: func(t *testing.T, _ *repo, ref string) error { + assert.Equal(t, "test", ref) + return errors.New("some error") + }, + assertFn: func(t *testing.T, r *repo) { + assert.Nil(t, r) + }, + }, } - orgClone := ggClone - defer func() { ggClone = orgClone }() + origCheckoutRef, origClone := checkoutRef, ggClone + defer func() { + checkoutRef = origCheckoutRef + ggClone = origClone + }() for tname, tt := range tests { t.Run(tname, func(t *testing.T) { @@ -334,6 +356,12 @@ func Test_clone(t *testing.T) { tt.opts.Parse() } + if tt.checkoutRef != nil { + checkoutRef = func(r *repo, ref string) error { + return tt.checkoutRef(t, r, ref) + } + } + got, err := clone(context.Background(), tt.opts) if (err != nil) != tt.wantErr { t.Errorf("clone() error = %v, wantErr %v", err, tt.wantErr) @@ -519,3 +547,124 @@ func Test_repo_Persist(t *testing.T) { }) } } + +func Test_repo_checkoutRef(t *testing.T) { + tests := map[string]struct { + ref string + hash string + wantErr string + beforeFn func() *mocks.Repository + }{ + "Should checkout a specific hash": { + ref: "3992c4", + hash: "3992c4", + beforeFn: func() *mocks.Repository { + r := &mocks.Repository{} + hash := plumbing.NewHash("3992c4") + r.On("ResolveRevision", plumbing.Revision("3992c4")).Return(&hash, nil) + return r + }, + }, + "Should checkout a tag": { + ref: "v1.0.0", + hash: "3992c4", + beforeFn: func() *mocks.Repository { + r := &mocks.Repository{} + hash := plumbing.NewHash("3992c4") + r.On("ResolveRevision", plumbing.Revision("v1.0.0")).Return(&hash, nil) + return r + }, + }, + "Should checkout a branch": { + ref: "CR-1234", + hash: "3992c4", + beforeFn: func() *mocks.Repository { + r := &mocks.Repository{} + r.On("ResolveRevision", plumbing.Revision("CR-1234")).Return(nil, plumbing.ErrReferenceNotFound) + r.On("Remotes").Return([]*gg.Remote{ + gg.NewRemote(nil, &config.RemoteConfig{ + Name: "origin", + }), + }, nil) + hash := plumbing.NewHash("3992c4") + r.On("ResolveRevision", plumbing.Revision("origin/CR-1234")).Return(&hash, nil) + return r + }, + }, + "Should fail if ResolveRevision fails": { + ref: "CR-1234", + hash: "3992c4", + wantErr: "some error", + beforeFn: func() *mocks.Repository { + r := &mocks.Repository{} + r.On("ResolveRevision", plumbing.Revision("CR-1234")).Return(nil, errors.New("some error")) + return r + }, + }, + "Should fail if Remotes fails": { + ref: "CR-1234", + hash: "3992c4", + wantErr: "some error", + beforeFn: func() *mocks.Repository { + r := &mocks.Repository{} + r.On("ResolveRevision", plumbing.Revision("CR-1234")).Return(nil, plumbing.ErrReferenceNotFound) + r.On("Remotes").Return(nil, errors.New("some error")) + return r + }, + }, + "Should fail if repo has no remotes": { + ref: "CR-1234", + hash: "3992c4", + wantErr: ErrNoRemotes.Error(), + beforeFn: func() *mocks.Repository { + r := &mocks.Repository{} + r.On("ResolveRevision", plumbing.Revision("CR-1234")).Return(nil, plumbing.ErrReferenceNotFound) + r.On("Remotes").Return([]*gg.Remote{}, nil) + return r + }, + }, + "Should fail if branch not found": { + ref: "CR-1234", + hash: "3992c4", + wantErr: plumbing.ErrReferenceNotFound.Error(), + beforeFn: func() *mocks.Repository { + r := &mocks.Repository{} + r.On("ResolveRevision", plumbing.Revision("CR-1234")).Return(nil, plumbing.ErrReferenceNotFound) + r.On("Remotes").Return([]*gg.Remote{ + gg.NewRemote(nil, &config.RemoteConfig{ + Name: "origin", + }), + }, nil) + r.On("ResolveRevision", plumbing.Revision("origin/CR-1234")).Return(nil, plumbing.ErrReferenceNotFound) + return r + }, + }, + } + origWorktree := worktree + defer func() { worktree = origWorktree }() + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + mockwt := &mocks.Worktree{} + worktree = func(r gogit.Repository) (gogit.Worktree, error) { + return mockwt, nil + } + mockwt.On("Checkout", &gg.CheckoutOptions{ + Hash: plumbing.NewHash(tt.hash), + }).Return(nil) + mockrepo := tt.beforeFn() + r := &repo{Repository: mockrepo} + if err := r.checkoutRef(tt.ref); err != nil { + if tt.wantErr != "" { + assert.EqualError(t, err, tt.wantErr) + } else { + t.Errorf("repo.checkoutRef() error = %v, wantErr %v", err, tt.wantErr) + } + + return + } + + mockrepo.AssertExpectations(t) + mockwt.AssertExpectations(t) + }) + } +} diff --git a/pkg/git/repospec_test.go b/pkg/git/repospec_test.go deleted file mode 100644 index 3ad4c518..00000000 --- a/pkg/git/repospec_test.go +++ /dev/null @@ -1,215 +0,0 @@ -// The following code was copied from https://github.com/kubernetes-sigs/kustomize/blob/master/api/internal/git/repospec_test.go -// and modified to test the copied repospec.go - -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package git - -import ( - "fmt" - "path/filepath" - "testing" -) - -const refQuery = "?ref=" - -var orgRepos = []string{"someOrg/someRepo", "kubernetes/website"} - -var pathNames = []string{"README.md", "foo/krusty.txt", ""} - -var hrefArgs = []string{"someBranch", "master", "v0.1.0", ""} - -var hostNamesRawAndNormalized = [][]string{ - {"gh:", "gh:"}, - {"GH:", "gh:"}, - {"gitHub.com/", "https://github.com/"}, - {"github.com:", "https://github.com/"}, - {"http://github.com/", "https://github.com/"}, - {"https://github.com/", "https://github.com/"}, - {"hTTps://github.com/", "https://github.com/"}, - {"ssh://git.example.com:7999/", "ssh://git.example.com:7999/"}, - {"git::https://gitlab.com/", "https://gitlab.com/"}, - {"git::http://git.example.com/", "http://git.example.com/"}, - {"git::https://git.example.com/", "https://git.example.com/"}, - {"git@github.com:", "git@github.com:"}, - {"git@github.com/", "git@github.com:"}, - {"git@gitlab2.sqtools.ru:10022/", "git@gitlab2.sqtools.ru:10022/"}, -} - -func makeUrl(hostFmt, orgRepo, path, href string) string { - if len(path) > 0 { - orgRepo = filepath.Join(orgRepo, path) - } - url := hostFmt + orgRepo - if href != "" { - url += refQuery + href - } - return url -} - -func TestNewRepoSpecFromUrl(t *testing.T) { - var bad [][]string - for _, tuple := range hostNamesRawAndNormalized { - hostRaw := tuple[0] - hostSpec := tuple[1] - for _, orgRepo := range orgRepos { - for _, pathName := range pathNames { - for _, hrefArg := range hrefArgs { - uri := makeUrl(hostRaw, orgRepo, pathName, hrefArg) - host, org, path, ref, _ := ParseGitUrl(uri) - if host != hostSpec { - bad = append(bad, []string{"host", uri, host, hostSpec}) - } - - if org != orgRepo { - bad = append(bad, []string{"orgRepo", uri, orgRepo, orgRepo}) - } - - if path != pathName { - bad = append(bad, []string{"path", uri, path, pathName}) - } - - if hrefArg != "" && ref != "refs/heads/"+hrefArg { - bad = append(bad, []string{"ref", uri, ref, hrefArg}) - } - } - } - } - } - - if len(bad) > 0 { - for _, tuple := range bad { - fmt.Printf("\n"+ - " from uri: %s\n"+ - " actual %4s: %s\n"+ - "expected %4s: %s\n", - tuple[1], tuple[0], tuple[2], tuple[0], tuple[3]) - } - t.Fail() - } -} - -func TestNewRepoSpecFromUrl_CloneSpecs(t *testing.T) { - testcases := []struct { - input string - cloneSpec string - absPath string - ref string - tag string - sha string - }{ - { - input: "http://github.com/someorg/somerepo/somedir", - cloneSpec: "https://github.com/someorg/somerepo.git", - absPath: "somedir", - ref: "", - }, - { - input: "git@github.com:someorg/somerepo/somedir", - cloneSpec: "git@github.com:someorg/somerepo.git", - absPath: "somedir", - ref: "", - }, - { - input: "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git?ref=branch", - cloneSpec: "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git", - absPath: "", - ref: "refs/heads/branch", - }, - { - input: "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git?tag=v0.1.0", - cloneSpec: "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git", - absPath: "", - ref: "refs/tags/v0.1.0", - }, - { - input: "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git?sha=some_sha", - cloneSpec: "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git", - absPath: "", - ref: "some_sha", - }, - { - input: "https://itfs.mycompany.com/collection/project/_git/somerepos", - cloneSpec: "https://itfs.mycompany.com/collection/project/_git/somerepos", - absPath: "", - ref: "", - }, - { - input: "git::https://itfs.mycompany.com/collection/project/_git/somerepos", - cloneSpec: "https://itfs.mycompany.com/collection/project/_git/somerepos", - absPath: "", - ref: "", - }, - } - for _, testcase := range testcases { - host, orgRepo, path, ref, suffix := ParseGitUrl(testcase.input) - cloneSpec := host + orgRepo + suffix - if cloneSpec != testcase.cloneSpec { - t.Errorf("CloneSpec expected to be %v, but got %v on %s", testcase.cloneSpec, cloneSpec, testcase.input) - } - - if path != testcase.absPath { - t.Errorf("AbsPath expected to be %v, but got %v on %s", testcase.absPath, path, testcase.input) - } - - if ref != testcase.ref { - t.Errorf("ref expected to be %v, but got %v on %s", testcase.ref, ref, testcase.input) - } - } -} - -func TestPeelQuery(t *testing.T) { - testcases := []struct { - input string - - path string - ref string - }{ - { - // All empty. - input: "somerepos", - path: "somerepos", - }, - { - input: "somerepos?ref=branch", - path: "somerepos", - ref: "refs/heads/branch", - }, - { - input: "somerepos?tag=v1.0.0", - path: "somerepos", - ref: "refs/tags/v1.0.0", - }, - { - input: "somerepos?sha=some_sha", - path: "somerepos", - ref: "some_sha", - }, - { - input: "somerepos?ref=branch&tag=v1.0.0", - path: "somerepos", - ref: "refs/tags/v1.0.0", - }, - { - input: "somerepos?ref=branch&sha=some_sha", - path: "somerepos", - ref: "some_sha", - }, - { - input: "somerepos?sha=some_sha&tag=v1.0.0", - path: "somerepos", - ref: "some_sha", - }, - } - - for _, testcase := range testcases { - path, ref := peelQuery(testcase.input) - if path != testcase.path || ref != testcase.ref { - t.Errorf("peelQuery: expected (%s, %s) got (%s, %s) on %s", - testcase.path, testcase.ref, - path, ref, - testcase.input) - } - } -} diff --git a/pkg/util/repospec.go b/pkg/util/repospec.go new file mode 100644 index 00000000..b17f87c5 --- /dev/null +++ b/pkg/util/repospec.go @@ -0,0 +1,165 @@ +// The following file was copied from https://github.com/kubernetes-sigs/kustomize/blob/master/api/internal/git/repospec.go +// and modified to expose the ParseGitUrl function + +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package util + +import ( + "net/url" + "strconv" + "strings" + "time" +) + +const ( + gitSuffix = ".git" + gitDelimiter = "_git/" +) + +// From strings like git@github.com:someOrg/someRepo.git or +// https://github.com/someOrg/someRepo?ref=someHash, extract +// the parts. +func ParseGitUrl(n string) ( + host string, orgRepo string, path string, gitRef string, gitSubmodules bool, gitSuff string, gitTimeout time.Duration) { + + if strings.Contains(n, gitDelimiter) { + index := strings.Index(n, gitDelimiter) + // Adding _git/ to host + host = normalizeGitHostSpec(n[:index+len(gitDelimiter)]) + orgRepo = strings.Split(strings.Split(n[index+len(gitDelimiter):], "/")[0], "?")[0] + path, gitRef, gitTimeout, gitSubmodules = peelQuery(n[index+len(gitDelimiter)+len(orgRepo):]) + return + } + host, n = parseHostSpec(n) + gitSuff = gitSuffix + if strings.Contains(n, gitSuffix) { + index := strings.Index(n, gitSuffix) + orgRepo = n[0:index] + n = n[index+len(gitSuffix):] + path, gitRef, gitTimeout, gitSubmodules = peelQuery(n) + return + } + + i := strings.Index(n, "/") + if i < 1 { + path, gitRef, gitTimeout, gitSubmodules = peelQuery(n) + return + } + j := strings.Index(n[i+1:], "/") + if j >= 0 { + j += i + 1 + orgRepo = n[:j] + path, gitRef, gitTimeout, gitSubmodules = peelQuery(n[j+1:]) + return + } + path = "" + orgRepo, gitRef, gitTimeout, gitSubmodules = peelQuery(n) + return host, orgRepo, path, gitRef, gitSubmodules, gitSuff, gitTimeout +} + +// Clone git submodules by default. +const defaultSubmodules = true + +// Arbitrary, but non-infinite, timeout for running commands. +const defaultTimeout = 27 * time.Second + +func peelQuery(arg string) (string, string, time.Duration, bool) { + // Parse the given arg into a URL. In the event of a parse failure, return + // our defaults. + parsed, err := url.Parse(arg) + if err != nil { + return arg, "", defaultTimeout, defaultSubmodules + } + values := parsed.Query() + + // ref is the desired git ref to target. Can be specified by in a git URL + // with ?ref= or ?version=, although ref takes precedence. + ref := values.Get("version") + if queryValue := values.Get("ref"); queryValue != "" { + ref = queryValue + } + + // depth is the desired git exec timeout. Can be specified by in a git URL + // with ?timeout=. + duration := defaultTimeout + if queryValue := values.Get("timeout"); queryValue != "" { + // Attempt to first parse as a number of integer seconds (like "61"), + // and then attempt to parse as a suffixed duration (like "61s"). + if intValue, err := strconv.Atoi(queryValue); err == nil && intValue > 0 { + duration = time.Duration(intValue) * time.Second + } else if durationValue, err := time.ParseDuration(queryValue); err == nil && durationValue > 0 { + duration = durationValue + } + } + + // submodules indicates if git submodule cloning is desired. Can be + // specified by in a git URL with ?submodules=. + submodules := defaultSubmodules + if queryValue := values.Get("submodules"); queryValue != "" { + if boolValue, err := strconv.ParseBool(queryValue); err == nil { + submodules = boolValue + } + } + + return parsed.Path, ref, duration, submodules +} + +func parseHostSpec(n string) (string, string) { + var host string + // Start accumulating the host part. + for _, p := range [...]string{ + // Order matters here. + "git::", "gh:", "ssh://", "https://", "http://", + "git@", "github.com:", "github.com/"} { + if len(p) < len(n) && strings.ToLower(n[:len(p)]) == p { + n = n[len(p):] + host += p + } + } + if host == "git@" { + i := strings.Index(n, "/") + if i > -1 { + host += n[:i+1] + n = n[i+1:] + } else { + i = strings.Index(n, ":") + if i > -1 { + host += n[:i+1] + n = n[i+1:] + } + } + return host, n + } + + // If host is a http(s) or ssh URL, grab the domain part. + for _, p := range [...]string{ + "ssh://", "https://", "http://"} { + if strings.HasSuffix(host, p) { + i := strings.Index(n, "/") + if i > -1 { + host = host + n[0:i+1] + n = n[i+1:] + } + break + } + } + + return normalizeGitHostSpec(host), n +} + +func normalizeGitHostSpec(host string) string { + s := strings.ToLower(host) + if strings.Contains(s, "github.com") { + if strings.Contains(s, "git@") || strings.Contains(s, "ssh:") { + host = "git@github.com:" + } else { + host = "https://github.com/" + } + } + if strings.HasPrefix(s, "git::") { + host = strings.TrimPrefix(s, "git::") + } + return host +} diff --git a/pkg/util/repospec_test.go b/pkg/util/repospec_test.go new file mode 100644 index 00000000..a6735a7f --- /dev/null +++ b/pkg/util/repospec_test.go @@ -0,0 +1,284 @@ +// The following code was copied from https://github.com/kubernetes-sigs/kustomize/blob/master/api/internal/git/repospec_test.go +// and modified to test the copied repospec.go + +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package util + +import ( + "fmt" + "path/filepath" + "testing" + "time" +) + +const refQuery = "?ref=" + +var orgRepos = []string{"someOrg/someRepo", "kubernetes/website"} + +var pathNames = []string{"README.md", "foo/krusty.txt", ""} + +var hrefArgs = []string{"someBranch", "master", "v0.1.0", ""} + +var hostNamesRawAndNormalized = [][]string{ + {"gh:", "gh:"}, + {"GH:", "gh:"}, + {"gitHub.com/", "https://github.com/"}, + {"github.com:", "https://github.com/"}, + {"http://github.com/", "https://github.com/"}, + {"https://github.com/", "https://github.com/"}, + {"hTTps://github.com/", "https://github.com/"}, + {"ssh://git.example.com:7999/", "ssh://git.example.com:7999/"}, + {"git::https://gitlab.com/", "https://gitlab.com/"}, + {"git::http://git.example.com/", "http://git.example.com/"}, + {"git::https://git.example.com/", "https://git.example.com/"}, + {"git@github.com:", "git@github.com:"}, + {"git@github.com/", "git@github.com:"}, + {"git@gitlab2.sqtools.ru:10022/", "git@gitlab2.sqtools.ru:10022/"}, +} + +func makeUrl(hostFmt, orgRepo, path, href string) string { + if len(path) > 0 { + orgRepo = filepath.Join(orgRepo, path) + } + url := hostFmt + orgRepo + if href != "" { + url += refQuery + href + } + return url +} + +func TestNewRepoSpecFromUrl(t *testing.T) { + var bad [][]string + for _, tuple := range hostNamesRawAndNormalized { + hostRaw := tuple[0] + hostSpec := tuple[1] + for _, orgRepo := range orgRepos { + for _, pathName := range pathNames { + for _, hrefArg := range hrefArgs { + uri := makeUrl(hostRaw, orgRepo, pathName, hrefArg) + host, org, path, ref, _, _, _ := ParseGitUrl(uri) + if host != hostSpec { + bad = append(bad, []string{"host", uri, host, hostSpec}) + } + if org != orgRepo { + bad = append(bad, []string{"orgRepo", uri, orgRepo, orgRepo}) + } + if path != pathName { + bad = append(bad, []string{"path", uri, path, pathName}) + } + if ref != hrefArg { + bad = append(bad, []string{"ref", uri, ref, hrefArg}) + } + } + } + } + } + if len(bad) > 0 { + for _, tuple := range bad { + fmt.Printf("\n"+ + " from uri: %s\n"+ + " actual %4s: %s\n"+ + "expected %4s: %s\n", + tuple[1], tuple[0], tuple[2], tuple[0], tuple[3]) + } + t.Fail() + } +} + +func TestNewRepoSpecFromUrl_CloneSpecs(t *testing.T) { + testcases := []struct { + input string + cloneSpec string + absPath string + ref string + }{ + { + input: "http://github.com/someorg/somerepo/somedir", + cloneSpec: "https://github.com/someorg/somerepo.git", + absPath: "somedir", + ref: "", + }, + { + input: "git@github.com:someorg/somerepo/somedir", + cloneSpec: "git@github.com:someorg/somerepo.git", + absPath: "somedir", + ref: "", + }, + { + input: "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git?ref=v0.1.0", + cloneSpec: "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git", + absPath: "", + ref: "v0.1.0", + }, + { + input: "https://itfs.mycompany.com/collection/project/_git/somerepos", + cloneSpec: "https://itfs.mycompany.com/collection/project/_git/somerepos", + absPath: "", + ref: "", + }, + { + input: "https://itfs.mycompany.com/collection/project/_git/somerepos?version=v1.0.0", + cloneSpec: "https://itfs.mycompany.com/collection/project/_git/somerepos", + absPath: "", + ref: "v1.0.0", + }, + { + input: "git::https://itfs.mycompany.com/collection/project/_git/somerepos", + cloneSpec: "https://itfs.mycompany.com/collection/project/_git/somerepos", + absPath: "", + ref: "", + }, + } + for _, testcase := range testcases { + host, orgRepo, path, ref, _, suffix, _ := ParseGitUrl(testcase.input) + cloneSpec := host + orgRepo + suffix + if cloneSpec != testcase.cloneSpec { + t.Errorf("CloneSpec expected to be %v, but got %v on %s", + testcase.cloneSpec, cloneSpec, testcase.input) + } + if path != testcase.absPath { + t.Errorf("AbsPath expected to be %v, but got %v on %s", + testcase.absPath, path, testcase.input) + } + if ref != testcase.ref { + t.Errorf("ref expected to be %v, but got %v on %s", + testcase.ref, ref, testcase.input) + } + } +} + +func TestPeelQuery(t *testing.T) { + testcases := []struct { + input string + + path string + ref string + submodules bool + timeout time.Duration + }{ + { + // All empty. + input: "somerepos", + path: "somerepos", + ref: "", + submodules: defaultSubmodules, + timeout: defaultTimeout, + }, + { + input: "somerepos?ref=v1.0.0", + path: "somerepos", + ref: "v1.0.0", + submodules: defaultSubmodules, + timeout: defaultTimeout, + }, + { + input: "somerepos?version=master", + path: "somerepos", + ref: "master", + submodules: defaultSubmodules, + timeout: defaultTimeout, + }, + { + // A ref value takes precedence over a version value. + input: "somerepos?version=master&ref=v1.0.0", + path: "somerepos", + ref: "v1.0.0", + submodules: defaultSubmodules, + timeout: defaultTimeout, + }, + { + // Empty submodules value uses default. + input: "somerepos?version=master&submodules=", + path: "somerepos", + ref: "master", + submodules: defaultSubmodules, + timeout: defaultTimeout, + }, + { + // Malformed submodules value uses default. + input: "somerepos?version=master&submodules=maybe", + path: "somerepos", + ref: "master", + submodules: defaultSubmodules, + timeout: defaultTimeout, + }, + { + input: "somerepos?version=master&submodules=true", + path: "somerepos", + ref: "master", + submodules: true, + timeout: defaultTimeout, + }, + { + input: "somerepos?version=master&submodules=false", + path: "somerepos", + ref: "master", + submodules: false, + timeout: defaultTimeout, + }, + { + // Empty timeout value uses default. + input: "somerepos?version=master&timeout=", + path: "somerepos", + ref: "master", + submodules: defaultSubmodules, + timeout: defaultTimeout, + }, + { + // Malformed timeout value uses default. + input: "somerepos?version=master&timeout=jiffy", + path: "somerepos", + ref: "master", + submodules: defaultSubmodules, + timeout: defaultTimeout, + }, + { + // Zero timeout value uses default. + input: "somerepos?version=master&timeout=0", + path: "somerepos", + ref: "master", + submodules: defaultSubmodules, + timeout: defaultTimeout, + }, + { + input: "somerepos?version=master&timeout=0s", + path: "somerepos", + ref: "master", + submodules: defaultSubmodules, + timeout: defaultTimeout, + }, + { + input: "somerepos?version=master&timeout=61", + path: "somerepos", + ref: "master", + submodules: defaultSubmodules, + timeout: 61 * time.Second, + }, + { + input: "somerepos?version=master&timeout=1m1s", + path: "somerepos", + ref: "master", + submodules: defaultSubmodules, + timeout: 61 * time.Second, + }, + { + input: "somerepos?version=master&submodules=false&timeout=1m1s", + path: "somerepos", + ref: "master", + submodules: false, + timeout: 61 * time.Second, + }, + } + + for _, testcase := range testcases { + path, ref, timeout, submodules := peelQuery(testcase.input) + if path != testcase.path || ref != testcase.ref || timeout != testcase.timeout || submodules != testcase.submodules { + t.Errorf("peelQuery: expected (%s, %s, %v, %v) got (%s, %s, %v, %v) on %s", + testcase.path, testcase.ref, testcase.timeout, testcase.submodules, + path, ref, timeout, submodules, + testcase.input) + } + } +}