From 40b3019a787271fdf173a4950a503878d0be1d03 Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Wed, 2 Jun 2021 15:11:20 +0300 Subject: [PATCH 01/20] merge --- cmd/commands/assets/kustomization_readme.md | 27 ------- cmd/commands/common.go | 62 +++++++++++++++ cmd/commands/project.go | 86 ++++++++++++++++----- cmd/commands/project_test.go | 8 +- cmd/commands/repo.go | 80 ++++--------------- pkg/store/store.go | 2 + 6 files changed, 155 insertions(+), 110 deletions(-) delete mode 100644 cmd/commands/assets/kustomization_readme.md diff --git a/cmd/commands/assets/kustomization_readme.md b/cmd/commands/assets/kustomization_readme.md deleted file mode 100644 index 9750d7c5..00000000 --- a/cmd/commands/assets/kustomization_readme.md +++ /dev/null @@ -1,27 +0,0 @@ -# Apps -This directory contains all of the applications you installed by using: -```bash -argocd-autopilot app create --app -p -``` - -## Application Types -> If you don't specify the application `--type` argocd-autopilot will try to clone the source repository and infer the application type [automatically](https://argoproj.github.io/argo-cd/user-guide/tool_detection/#tool-detection) - -* ### Directory application - Such an application references a specific directory at a given repo URL, path and revision. It will be persisted in the GitOps Repository as a single file at `apps///config.json`. - #### Example: - ```bash - argocd-autopilot app create dir-example --app github.com/argoproj-labs/argocd-autopilot/examples/demo-dir/ -p --type dir - ``` - -* ### Kustomize application - A Kustomize application will have exactly one: `apps//base/kustomization.yaml` file, and one or more `apps//overlays//` folders. - - The `apps//base/kustomization.yaml` file is created the first time you create the application. The `apps//overlays//` folder is created for each project you install this application on. So all overlays of the same application are using the same base `kustomization.yaml`. - #### Example: - Try running the following command: - ```bash - argocd-autopilot app create hello-world --app github.com/argoproj-labs/argocd-autopilot/examples/demo-app/ -p --type kustomize - ``` - -###### * If you did not create a project yet take a look at: [creating a project](https://argocd-autopilot.readthedocs.io/en/stable/Getting-Started/#add-a-project-and-an-application). \ No newline at end of file diff --git a/cmd/commands/common.go b/cmd/commands/common.go index 02e95f5a..dc7e029a 100644 --- a/cmd/commands/common.go +++ b/cmd/commands/common.go @@ -10,6 +10,9 @@ import ( "github.com/argoproj-labs/argocd-autopilot/pkg/log" "github.com/argoproj-labs/argocd-autopilot/pkg/store" "github.com/argoproj-labs/argocd-autopilot/pkg/util" + argocdv1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + "github.com/ghodss/yaml" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/go-git/go-billy/v5/memfs" "github.com/spf13/cobra" @@ -86,3 +89,62 @@ func addFlags(cmd *cobra.Command) (*BaseOptions, error) { return o, nil } + +func createApp(opts *createAppOptions) ([]byte, error) { + if opts.destServer == "" { + opts.destServer = store.Default.DestServer + } + + app := &argocdv1alpha1.Application{ + TypeMeta: metav1.TypeMeta{ + Kind: argocdv1alpha1.ApplicationSchemaGroupVersionKind.Kind, + APIVersion: argocdv1alpha1.ApplicationSchemaGroupVersionKind.GroupVersion().String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: opts.namespace, + Name: opts.name, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": store.Default.ManagedBy, + "app.kubernetes.io/name": opts.name, + }, + Finalizers: []string{ + "resources-finalizer.argocd.argoproj.io", + }, + }, + Spec: argocdv1alpha1.ApplicationSpec{ + Project: "default", + Source: argocdv1alpha1.ApplicationSource{ + RepoURL: opts.repoURL, + Path: opts.srcPath, + TargetRevision: opts.revision, + }, + Destination: argocdv1alpha1.ApplicationDestination{ + Server: opts.destServer, + Namespace: opts.namespace, + }, + SyncPolicy: &argocdv1alpha1.SyncPolicy{ + Automated: &argocdv1alpha1.SyncPolicyAutomated{ + SelfHeal: true, + Prune: true, + }, + SyncOptions: []string{ + "allowEmpty=true", + }, + }, + IgnoreDifferences: []argocdv1alpha1.ResourceIgnoreDifferences{ + { + Group: "argoproj.io", + Kind: "Application", + JSONPointers: []string{ + "/status", + }, + }, + }, + }, + } + if opts.noFinalizer { + app.ObjectMeta.Finalizers = []string{} + } + + return yaml.Marshal(app) +} diff --git a/cmd/commands/project.go b/cmd/commands/project.go index ebfc637e..f44e9404 100644 --- a/cmd/commands/project.go +++ b/cmd/commands/project.go @@ -2,10 +2,12 @@ package commands import ( "context" + _ "embed" "fmt" "io" "os" "path/filepath" + "strings" "text/tabwriter" "github.com/argoproj-labs/argocd-autopilot/pkg/application" @@ -27,6 +29,9 @@ import ( var DefaultApplicationSetGeneratorInterval int64 = 20 +//go:embed assets/cluster_res_readme.md +var clusterResReadmeTpl []byte + type ( ProjectCreateOptions struct { BaseOptions @@ -98,7 +103,7 @@ func NewProjectCreateCommand(opts *BaseOptions) *cobra.Command { # Create a new project in a specific path inside the GitOps repo - project create --installation-path path/to/installation_root + project create --installation-path path/to/installation_root `), RunE: func(cmd *cobra.Command, args []string) error { if len(args) < 1 { @@ -155,8 +160,9 @@ func RunProjectCreate(ctx context.Context, opts *ProjectCreateOptions) error { return err } } + cleanDestServer := cleanServerAddr(destServer) - project, appSet := generateProject(&GenerateProjectOptions{ + projectYAML, appsetYAML, clusterResAppYAML, clusterResReadme, err := generateProjectManifests(&GenerateProjectOptions{ Name: opts.Name, Namespace: installationNamespace, RepoURL: opts.CloneOptions.URL, @@ -164,21 +170,12 @@ func RunProjectCreate(ctx context.Context, opts *ProjectCreateOptions) error { InstallationPath: opts.CloneOptions.RepoRoot, DefaultDestServer: destServer, }) - - projectYAML, err := yaml.Marshal(project) - if err != nil { - return fmt.Errorf("failed to marshal project: %w", err) - } - - appsetYAML, err := yaml.Marshal(appSet) if err != nil { - return fmt.Errorf("failed to marshal appSet: %w", err) + return fmt.Errorf("failed to generate project resources: %w", err) } - joinedYAML := util.JoinManifests(projectYAML, appsetYAML) - if opts.DryRun { - log.G().Printf("%s", joinedYAML) + log.G().Printf("%s", util.JoinManifests(projectYAML, appsetYAML, clusterResAppYAML)) return nil } @@ -189,12 +186,35 @@ func RunProjectCreate(ctx context.Context, opts *ProjectCreateOptions) error { } } - if err = billyUtils.WriteFile(repofs, repofs.Join(store.Default.ProjectsDir, opts.Name+".yaml"), joinedYAML, 0666); err != nil { + if err = billyUtils.WriteFile( + repofs, + repofs.Join(store.Default.ProjectsDir, opts.Name+".yaml"), + util.JoinManifests(projectYAML, appsetYAML), + 0666, + ); err != nil { return fmt.Errorf("failed to create project file: %w", err) } + if err = billyUtils.WriteFile( + repofs, + repofs.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, cleanDestServer+".yaml"), + clusterResAppYAML, + 0666, + ); err != nil { + return fmt.Errorf("failed to create cluster resources application file: %w", err) + } + + if err = billyUtils.WriteFile( + repofs, + repofs.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, cleanDestServer, "README.md"), + clusterResReadme, + 0666, + ); err != nil { + return fmt.Errorf("failed to create cluster resources README.md file: %w", err) + } + log.G().Infof("pushing new project manifest to repo") - if err = r.Persist(ctx, &git.PushOptions{CommitMsg: "Added project " + opts.Name}); err != nil { + if err = r.Persist(ctx, &git.PushOptions{CommitMsg: fmt.Sprintf("Added project '%s'", opts.Name)}); err != nil { return err } @@ -203,8 +223,8 @@ func RunProjectCreate(ctx context.Context, opts *ProjectCreateOptions) error { return nil } -var generateProject = func(o *GenerateProjectOptions) (*argocdv1alpha1.AppProject, *appset.ApplicationSet) { - appProject := &argocdv1alpha1.AppProject{ +func generateProjectManifests(o *GenerateProjectOptions) (projectYAML, appSetYAML, clusterResAppYAML, clusterResReadme []byte, err error) { + project := &argocdv1alpha1.AppProject{ TypeMeta: metav1.TypeMeta{ Kind: argocdv1alpha1.AppProjectSchemaGroupVersionKind.Kind, APIVersion: argocdv1alpha1.AppProjectSchemaGroupVersionKind.GroupVersion().String(), @@ -240,6 +260,10 @@ var generateProject = func(o *GenerateProjectOptions) (*argocdv1alpha1.AppProjec }, }, } + if projectYAML, err = yaml.Marshal(project); err != nil { + err = fmt.Errorf("failed to marshal AppProject: %w", err) + return + } appSet := &appset.ApplicationSet{ TypeMeta: metav1.TypeMeta{ @@ -295,8 +319,26 @@ var generateProject = func(o *GenerateProjectOptions) (*argocdv1alpha1.AppProjec }, }, } + if appSetYAML, err = yaml.Marshal(appSet); err != nil { + err = fmt.Errorf("failed to marshal ApplicationSet: %w", err) + return + } + + if clusterResAppYAML, err = createApp(&createAppOptions{ + repoURL: o.RepoURL, + revision: o.Revision, + name: "cluster-resources-" + cleanServerAddr(o.DefaultDestServer), + srcPath: filepath.Join(o.InstallationPath, store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, cleanServerAddr(o.DefaultDestServer)), + destServer: o.DefaultDestServer, + noFinalizer: true, + }); err != nil { + err = fmt.Errorf("failed to marshal cluster-resources application: %w", err) + return + } + + clusterResReadme = []byte(strings.ReplaceAll(string(clusterResReadmeTpl), "{CLUSTER}", o.DefaultDestServer)) - return appProject, appSet + return } var getInstallationNamespace = func(repofs fs.FS) (string, error) { @@ -441,3 +483,11 @@ func RunProjectDelete(ctx context.Context, opts *BaseOptions) error { return nil } + +func cleanServerAddr(addr string) string { + return strings.NewReplacer( + "https://", "", + "http://", "", + "/", ".", + ).Replace(addr) +} diff --git a/cmd/commands/project_test.go b/cmd/commands/project_test.go index baf8545e..0aebb8e6 100644 --- a/cmd/commands/project_test.go +++ b/cmd/commands/project_test.go @@ -17,6 +17,7 @@ import ( gitmocks "github.com/argoproj-labs/argocd-autopilot/pkg/git/mocks" "github.com/argoproj-labs/argocd-autopilot/pkg/store" "github.com/argoproj-labs/argocd-autopilot/pkg/util" + "github.com/ghodss/yaml" appset "github.com/argoproj-labs/applicationset/api/v1alpha1" argocdv1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" @@ -184,7 +185,12 @@ func Test_generateProject(t *testing.T) { for ttname, tt := range tests { t.Run(ttname, func(t *testing.T) { assert := assert.New(t) - gotProject, gotAppSet := generateProject(tt.o) + gotProject := &argocdv1alpha1.AppProject{} + gotAppSet := &appset.ApplicationSet{} + gotProjectYAML, gotAppSetYAML, _, _, _ := generateProjectManifests(tt.o) + assert.NoError(yaml.Unmarshal(gotProjectYAML, gotProject)) + assert.NoError(yaml.Unmarshal(gotAppSetYAML, gotAppSet)) + assert.Equal(tt.wantName, gotProject.Name, "Project Name") assert.Equal(tt.wantNamespace, gotProject.Namespace, "Project Namespace") assert.Equal(tt.wantProjectDescription, gotProject.Spec.Description, "Project Description") diff --git a/cmd/commands/repo.go b/cmd/commands/repo.go index 0810db0c..beaa3102 100644 --- a/cmd/commands/repo.go +++ b/cmd/commands/repo.go @@ -18,7 +18,6 @@ import ( "github.com/argoproj-labs/argocd-autopilot/pkg/store" "github.com/argoproj-labs/argocd-autopilot/pkg/util" - argocdv1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" argocdsettings "github.com/argoproj/argo-cd/v2/util/settings" "github.com/ghodss/yaml" "github.com/go-git/go-billy/v5/memfs" @@ -38,8 +37,8 @@ const ( //go:embed assets/projects_readme.md var projectReadme []byte -//go:embed assets/kustomization_readme.md -var kustomizationReadme []byte +//go:embed assets/apps_readme.md +var appsReadme []byte // used for mocking var ( @@ -76,6 +75,7 @@ type ( bootstrapManifests struct { bootstrapApp []byte rootApp []byte + clusterResAppSet []byte argocdApp []byte repoCreds []byte applyManifests []byte @@ -332,7 +332,11 @@ func RunRepoBootstrap(ctx context.Context, opts *RepoBootstrapOptions) error { // push results to repo log.G().Infof("pushing bootstrap manifests to repo") - if err = r.Persist(ctx, &git.PushOptions{CommitMsg: "Autopilot Bootstrap at " + opts.CloneOptions.RepoRoot}); err != nil { + commitMsg := "Autopilot Bootstrap" + if opts.CloneOptions.RepoRoot != "" { + commitMsg = "Autopilot Bootstrap at " + opts.CloneOptions.RepoRoot + } + if err = r.Persist(ctx, &git.PushOptions{CommitMsg: commitMsg}); err != nil { return err } @@ -409,70 +413,16 @@ func validateRepo(repofs fs.FS) error { return nil } -type createBootstrapAppOptions struct { +type createAppOptions struct { name string namespace string repoURL string revision string srcPath string + destServer string noFinalizer bool } -func createApp(opts *createBootstrapAppOptions) ([]byte, error) { - app := &argocdv1alpha1.Application{ - TypeMeta: metav1.TypeMeta{ - Kind: argocdv1alpha1.ApplicationSchemaGroupVersionKind.Kind, - APIVersion: argocdv1alpha1.ApplicationSchemaGroupVersionKind.GroupVersion().String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: opts.namespace, - Name: opts.name, - Labels: map[string]string{ - "app.kubernetes.io/managed-by": store.Default.ManagedBy, - "app.kubernetes.io/name": opts.name, - }, - Finalizers: []string{ - "resources-finalizer.argocd.argoproj.io", - }, - }, - Spec: argocdv1alpha1.ApplicationSpec{ - Project: "default", - Source: argocdv1alpha1.ApplicationSource{ - RepoURL: opts.repoURL, - Path: opts.srcPath, - TargetRevision: opts.revision, - }, - Destination: argocdv1alpha1.ApplicationDestination{ - Server: store.Default.DestServer, - Namespace: opts.namespace, - }, - SyncPolicy: &argocdv1alpha1.SyncPolicy{ - Automated: &argocdv1alpha1.SyncPolicyAutomated{ - SelfHeal: true, - Prune: true, - }, - SyncOptions: []string{ - "allowEmpty=true", - }, - }, - IgnoreDifferences: []argocdv1alpha1.ResourceIgnoreDifferences{ - { - Group: "argoproj.io", - Kind: "Application", - JSONPointers: []string{ - "/status", - }, - }, - }, - }, - } - if opts.noFinalizer { - app.ObjectMeta.Finalizers = []string{} - } - - return yaml.Marshal(app) -} - func waitClusterReady(ctx context.Context, f kube.Factory, timeout time.Duration, namespace string) error { return f.Wait(ctx, &kube.WaitOptions{ Interval: store.Default.WaitInterval, @@ -531,7 +481,7 @@ func buildBootstrapManifests(namespace, appSpecifier string, cloneOpts *git.Clon var err error manifests := &bootstrapManifests{} - manifests.bootstrapApp, err = createApp(&createBootstrapAppOptions{ + manifests.bootstrapApp, err = createApp(&createAppOptions{ name: store.Default.BootsrtrapAppName, namespace: namespace, repoURL: cloneOpts.URL, @@ -542,7 +492,7 @@ func buildBootstrapManifests(namespace, appSpecifier string, cloneOpts *git.Clon return nil, err } - manifests.rootApp, err = createApp(&createBootstrapAppOptions{ + manifests.rootApp, err = createApp(&createAppOptions{ name: store.Default.RootAppName, namespace: namespace, repoURL: cloneOpts.URL, @@ -553,7 +503,7 @@ func buildBootstrapManifests(namespace, appSpecifier string, cloneOpts *git.Clon return nil, err } - manifests.argocdApp, err = createApp(&createBootstrapAppOptions{ + manifests.argocdApp, err = createApp(&createAppOptions{ name: store.Default.ArgoCDName, namespace: namespace, repoURL: cloneOpts.URL, @@ -621,13 +571,15 @@ func writeManifestsToRepo(repoFS fs.FS, manifests *bootstrapManifests, installat return err } + // write cluster-resources appset + // write ./projects/README.md if err = billyUtils.WriteFile(repoFS, repoFS.Join(store.Default.ProjectsDir, "README.md"), projectReadme, 0666); err != nil { return err } // write ./apps/README.md - if err = billyUtils.WriteFile(repoFS, repoFS.Join(store.Default.AppsDir, "README.md"), kustomizationReadme, 0666); err != nil { + if err = billyUtils.WriteFile(repoFS, repoFS.Join(store.Default.AppsDir, "README.md"), appsReadme, 0666); err != nil { return err } diff --git a/pkg/store/store.go b/pkg/store/store.go index 9891797d..984ada8e 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -38,6 +38,7 @@ var Default = struct { AppsDir string OverlaysDir string BaseDir string + ClusterResourcesDir string ArgoCDName string ArgoCDNamespace string BootsrtrapAppName string @@ -55,6 +56,7 @@ var Default = struct { BootsrtrapDir: "bootstrap", OverlaysDir: "overlays", BaseDir: "base", + ClusterResourcesDir: "cluster-resources", ArgoCDName: "argo-cd", ArgoCDNamespace: "argocd", BootsrtrapAppName: "autopilot-bootstrap", From 316fafb372777eb760455eebd631ba9986ab9313 Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Tue, 1 Jun 2021 17:19:08 +0300 Subject: [PATCH 02/20] added readme --- cmd/commands/assets/apps_readme.md | 27 +++++++++++++++++++++++ cmd/commands/assets/cluster_res_readme.md | 3 +++ 2 files changed, 30 insertions(+) create mode 100644 cmd/commands/assets/apps_readme.md create mode 100644 cmd/commands/assets/cluster_res_readme.md diff --git a/cmd/commands/assets/apps_readme.md b/cmd/commands/assets/apps_readme.md new file mode 100644 index 00000000..9750d7c5 --- /dev/null +++ b/cmd/commands/assets/apps_readme.md @@ -0,0 +1,27 @@ +# Apps +This directory contains all of the applications you installed by using: +```bash +argocd-autopilot app create --app -p +``` + +## Application Types +> If you don't specify the application `--type` argocd-autopilot will try to clone the source repository and infer the application type [automatically](https://argoproj.github.io/argo-cd/user-guide/tool_detection/#tool-detection) + +* ### Directory application + Such an application references a specific directory at a given repo URL, path and revision. It will be persisted in the GitOps Repository as a single file at `apps///config.json`. + #### Example: + ```bash + argocd-autopilot app create dir-example --app github.com/argoproj-labs/argocd-autopilot/examples/demo-dir/ -p --type dir + ``` + +* ### Kustomize application + A Kustomize application will have exactly one: `apps//base/kustomization.yaml` file, and one or more `apps//overlays//` folders. + + The `apps//base/kustomization.yaml` file is created the first time you create the application. The `apps//overlays//` folder is created for each project you install this application on. So all overlays of the same application are using the same base `kustomization.yaml`. + #### Example: + Try running the following command: + ```bash + argocd-autopilot app create hello-world --app github.com/argoproj-labs/argocd-autopilot/examples/demo-app/ -p --type kustomize + ``` + +###### * If you did not create a project yet take a look at: [creating a project](https://argocd-autopilot.readthedocs.io/en/stable/Getting-Started/#add-a-project-and-an-application). \ No newline at end of file diff --git a/cmd/commands/assets/cluster_res_readme.md b/cmd/commands/assets/cluster_res_readme.md new file mode 100644 index 00000000..0d20c022 --- /dev/null +++ b/cmd/commands/assets/cluster_res_readme.md @@ -0,0 +1,3 @@ +# Cluster Resources +This directory contains all cluster resources that should be applied to cluster: {CLUSTER}. +For example `Namespace` resources that are shared by multiple applications on the same namespace. From 2defa806f388334e9cfc3df262f368963ec13fb2 Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Wed, 2 Jun 2021 19:52:28 +0300 Subject: [PATCH 03/20] wip --- cmd/commands/assets/cluster_res_readme.md | 2 +- cmd/commands/common.go | 98 ++++++++++++ cmd/commands/project.go | 185 ++++++++-------------- cmd/commands/repo.go | 109 +++++++------ pkg/application/application.go | 17 +- pkg/application/util.go | 27 ++++ pkg/fs/fs.go | 30 ++++ pkg/fs/mocks/fs.go | 25 ++- pkg/store/store.go | 2 + 9 files changed, 326 insertions(+), 169 deletions(-) create mode 100644 pkg/application/util.go diff --git a/cmd/commands/assets/cluster_res_readme.md b/cmd/commands/assets/cluster_res_readme.md index 0d20c022..f39aece5 100644 --- a/cmd/commands/assets/cluster_res_readme.md +++ b/cmd/commands/assets/cluster_res_readme.md @@ -1,3 +1,3 @@ # Cluster Resources -This directory contains all cluster resources that should be applied to cluster: {CLUSTER}. +This directory contains all cluster resources that should be applied to cluster: `{CLUSTER}`. For example `Namespace` resources that are shared by multiple applications on the same namespace. diff --git a/cmd/commands/common.go b/cmd/commands/common.go index dc7e029a..2aeecfb0 100644 --- a/cmd/commands/common.go +++ b/cmd/commands/common.go @@ -2,14 +2,17 @@ package commands import ( "context" + _ "embed" "fmt" "os" + appset "github.com/argoproj-labs/applicationset/api/v1alpha1" "github.com/argoproj-labs/argocd-autopilot/pkg/fs" "github.com/argoproj-labs/argocd-autopilot/pkg/git" "github.com/argoproj-labs/argocd-autopilot/pkg/log" "github.com/argoproj-labs/argocd-autopilot/pkg/store" "github.com/argoproj-labs/argocd-autopilot/pkg/util" + appsetv1alpha1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1" argocdv1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/ghodss/yaml" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,6 +34,17 @@ var ( die = util.Die exit = os.Exit + DefaultApplicationSetGeneratorInterval int64 = 20 + + //go:embed assets/cluster_res_readme.md + clusterResReadmeTpl []byte + + //go:embed assets/projects_readme.md + projectReadme []byte + + //go:embed assets/apps_readme.md + appsReadme []byte + clone = func(ctx context.Context, cloneOpts *git.CloneOptions, filesystem fs.FS) (git.Repository, fs.FS, error) { return cloneOpts.Clone(ctx, filesystem) } @@ -90,6 +104,16 @@ func addFlags(cmd *cobra.Command) (*BaseOptions, error) { return o, nil } +type createAppOptions struct { + name string + namespace string + repoURL string + revision string + srcPath string + destServer string + noFinalizer bool +} + func createApp(opts *createAppOptions) ([]byte, error) { if opts.destServer == "" { opts.destServer = store.Default.DestServer @@ -148,3 +172,77 @@ func createApp(opts *createAppOptions) ([]byte, error) { return yaml.Marshal(app) } + +type createAppSetOptions struct { + name string + namespace string + appName string + appNamespace string + repoURL string + revision string + srcPath string + destServer string + destNamespace string + noFinalizer bool + prune bool + appLabels map[string]string + generators []appset.ApplicationSetGenerator +} + +func createAppSet(o *createAppSetOptions) ([]byte, error) { + if o.destServer == "" { + o.destServer = store.Default.DestServer + } + + appSet := &appset.ApplicationSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "ApplicationSet", + APIVersion: appset.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: o.name, + Namespace: o.namespace, + }, + Spec: appset.ApplicationSetSpec{ + Generators: o.generators, + Template: appset.ApplicationSetTemplate{ + ApplicationSetTemplateMeta: appset.ApplicationSetTemplateMeta{ + Namespace: o.appNamespace, + Name: o.appName, + Labels: o.appLabels, + }, + Spec: appsetv1alpha1.ApplicationSpec{ + Source: appsetv1alpha1.ApplicationSource{ + RepoURL: o.repoURL, + Path: o.srcPath, + TargetRevision: o.revision, + }, + Destination: appsetv1alpha1.ApplicationDestination{ + Server: o.destServer, + Namespace: o.destNamespace, + }, + SyncPolicy: &appsetv1alpha1.SyncPolicy{ + Automated: &appsetv1alpha1.SyncPolicyAutomated{ + SelfHeal: true, + Prune: o.prune, + }, + }, + }, + }, + }, + } + + if o.appLabels == nil { + // default labels + appSet.Spec.Template.ApplicationSetTemplateMeta.Labels = map[string]string{ + "app.kubernetes.io/managed-by": store.Default.ManagedBy, + "app.kubernetes.io/name": o.appName, + } + } + + if o.noFinalizer { + appSet.ObjectMeta.Finalizers = []string{} + } + + return yaml.Marshal(appSet) +} diff --git a/cmd/commands/project.go b/cmd/commands/project.go index f44e9404..4c302f28 100644 --- a/cmd/commands/project.go +++ b/cmd/commands/project.go @@ -2,7 +2,6 @@ package commands import ( "context" - _ "embed" "fmt" "io" "os" @@ -19,7 +18,6 @@ import ( "github.com/argoproj-labs/argocd-autopilot/pkg/util" appset "github.com/argoproj-labs/applicationset/api/v1alpha1" - appsetv1alpha1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1" argocdv1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/ghodss/yaml" billyUtils "github.com/go-git/go-billy/v5/util" @@ -27,11 +25,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var DefaultApplicationSetGeneratorInterval int64 = 20 - -//go:embed assets/cluster_res_readme.md -var clusterResReadmeTpl []byte - type ( ProjectCreateOptions struct { BaseOptions @@ -47,12 +40,13 @@ type ( } GenerateProjectOptions struct { - Name string - Namespace string - DefaultDestServer string - RepoURL string - Revision string - InstallationPath string + Name string + Namespace string + DefaultDestServer string + DefaultDestContext string + RepoURL string + Revision string + InstallationPath string } ) @@ -160,57 +154,56 @@ func RunProjectCreate(ctx context.Context, opts *ProjectCreateOptions) error { return err } } - cleanDestServer := cleanServerAddr(destServer) - - projectYAML, appsetYAML, clusterResAppYAML, clusterResReadme, err := generateProjectManifests(&GenerateProjectOptions{ - Name: opts.Name, - Namespace: installationNamespace, - RepoURL: opts.CloneOptions.URL, - Revision: opts.CloneOptions.Revision, - InstallationPath: opts.CloneOptions.RepoRoot, - DefaultDestServer: destServer, + + projectYAML, appsetYAML, clusterResReadme, clusterResConf, err := generateProjectManifests(&GenerateProjectOptions{ + Name: opts.Name, + Namespace: installationNamespace, + RepoURL: opts.CloneOptions.URL, + Revision: opts.CloneOptions.Revision, + InstallationPath: opts.CloneOptions.RepoRoot, + DefaultDestServer: destServer, + DefaultDestContext: opts.DestKubeContext, }) if err != nil { return fmt.Errorf("failed to generate project resources: %w", err) } if opts.DryRun { - log.G().Printf("%s", util.JoinManifests(projectYAML, appsetYAML, clusterResAppYAML)) + log.G().Printf("%s", util.JoinManifests(projectYAML, appsetYAML)) return nil } + bulkWrites := []fs.BulkWriteRequest{} + if opts.DestKubeContext != "" { log.G().Infof("adding cluster: %s", opts.DestKubeContext) if err = opts.AddCmd.Execute(ctx, opts.DestKubeContext); err != nil { return fmt.Errorf("failed to add new cluster credentials: %w", err) } - } - if err = billyUtils.WriteFile( - repofs, - repofs.Join(store.Default.ProjectsDir, opts.Name+".yaml"), - util.JoinManifests(projectYAML, appsetYAML), - 0666, - ); err != nil { - return fmt.Errorf("failed to create project file: %w", err) - } + if !repofs.ExistsOrDie(repofs.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, opts.DestKubeContext)) { + bulkWrites = append(bulkWrites, fs.BulkWriteRequest{ + Filename: repofs.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, opts.DestKubeContext+".json"), + Data: clusterResConf, + ErrMsg: "failed to write cluster config", + }) - if err = billyUtils.WriteFile( - repofs, - repofs.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, cleanDestServer+".yaml"), - clusterResAppYAML, - 0666, - ); err != nil { - return fmt.Errorf("failed to create cluster resources application file: %w", err) + bulkWrites = append(bulkWrites, fs.BulkWriteRequest{ + Filename: repofs.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, opts.DestKubeContext, "README.md"), + Data: clusterResReadme, + ErrMsg: "failed to write cluster resources readme", + }) + } } - if err = billyUtils.WriteFile( - repofs, - repofs.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, cleanDestServer, "README.md"), - clusterResReadme, - 0666, - ); err != nil { - return fmt.Errorf("failed to create cluster resources README.md file: %w", err) + bulkWrites = append(bulkWrites, fs.BulkWriteRequest{ + Filename: repofs.Join(store.Default.ProjectsDir, opts.Name+".yaml"), + Data: util.JoinManifests(projectYAML, appsetYAML), + ErrMsg: "failed to create project file", + }) + + if err = repofs.BulkWrite(bulkWrites...); err != nil { + return err } log.G().Infof("pushing new project manifest to repo") @@ -223,7 +216,7 @@ func RunProjectCreate(ctx context.Context, opts *ProjectCreateOptions) error { return nil } -func generateProjectManifests(o *GenerateProjectOptions) (projectYAML, appSetYAML, clusterResAppYAML, clusterResReadme []byte, err error) { +func generateProjectManifests(o *GenerateProjectOptions) (projectYAML, appSetYAML, clusterResReadme, clusterResConfig []byte, err error) { project := &argocdv1alpha1.AppProject{ TypeMeta: metav1.TypeMeta{ Kind: argocdv1alpha1.AppProjectSchemaGroupVersionKind.Kind, @@ -265,79 +258,49 @@ func generateProjectManifests(o *GenerateProjectOptions) (projectYAML, appSetYAM return } - appSet := &appset.ApplicationSet{ - TypeMeta: metav1.TypeMeta{ - Kind: "ApplicationSet", - APIVersion: appset.GroupVersion.String(), + appSetYAML, err = createAppSet(&createAppSetOptions{ + name: o.Name, + namespace: o.Namespace, + appName: fmt.Sprintf("%s-{{ userGivenName }}", o.Name), + appNamespace: o.Namespace, + repoURL: "{{ srcRepoURL }}", + srcPath: "{{ srcPath }}", + revision: "{{ srcTargetRevision }}", + destServer: "{{ destServer }}", + destNamespace: "{{ destNamespace }}", + prune: true, + appLabels: map[string]string{ + "app.kubernetes.io/managed-by": store.Default.ManagedBy, + "app.kubernetes.io/name": "{{ appName }}", }, - ObjectMeta: metav1.ObjectMeta{ - Name: o.Name, - Namespace: o.Namespace, - }, - Spec: appset.ApplicationSetSpec{ - Generators: []appset.ApplicationSetGenerator{ - { - Git: &appset.GitGenerator{ - RepoURL: o.RepoURL, - Revision: o.Revision, - Files: []appset.GitFileGeneratorItem{ - { - Path: filepath.Join(o.InstallationPath, store.Default.AppsDir, "**", o.Name, "config.json"), - }, - }, - RequeueAfterSeconds: &DefaultApplicationSetGeneratorInterval, - }, - }, - }, - Template: appset.ApplicationSetTemplate{ - ApplicationSetTemplateMeta: appset.ApplicationSetTemplateMeta{ - Namespace: o.Namespace, - Name: fmt.Sprintf("%s-{{ userGivenName }}", o.Name), - Labels: map[string]string{ - "app.kubernetes.io/managed-by": store.Default.ManagedBy, - "app.kubernetes.io/name": "{{ appName }}", - }, - }, - Spec: appsetv1alpha1.ApplicationSpec{ - Project: o.Name, - Source: appsetv1alpha1.ApplicationSource{ - RepoURL: "{{ srcRepoURL }}", - Path: "{{ srcPath }}", - TargetRevision: "{{ srcTargetRevision }}", - }, - Destination: appsetv1alpha1.ApplicationDestination{ - Server: "{{ destServer }}", - Namespace: "{{ destNamespace }}", - }, - SyncPolicy: &appsetv1alpha1.SyncPolicy{ - Automated: &appsetv1alpha1.SyncPolicyAutomated{ - SelfHeal: true, - Prune: true, + generators: []appset.ApplicationSetGenerator{ + { + Git: &appset.GitGenerator{ + RepoURL: o.RepoURL, + Revision: o.Revision, + Files: []appset.GitFileGeneratorItem{ + { + Path: filepath.Join(o.InstallationPath, store.Default.AppsDir, "**", o.Name, "config.json"), }, }, + RequeueAfterSeconds: &DefaultApplicationSetGeneratorInterval, }, }, }, - } - if appSetYAML, err = yaml.Marshal(appSet); err != nil { + }) + if err != nil { err = fmt.Errorf("failed to marshal ApplicationSet: %w", err) return } - if clusterResAppYAML, err = createApp(&createAppOptions{ - repoURL: o.RepoURL, - revision: o.Revision, - name: "cluster-resources-" + cleanServerAddr(o.DefaultDestServer), - srcPath: filepath.Join(o.InstallationPath, store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, cleanServerAddr(o.DefaultDestServer)), - destServer: o.DefaultDestServer, - noFinalizer: true, - }); err != nil { - err = fmt.Errorf("failed to marshal cluster-resources application: %w", err) + clusterResReadme = []byte(strings.ReplaceAll(string(clusterResReadmeTpl), "{CLUSTER}", o.DefaultDestServer)) + + clusterResConfig, err = yaml.Marshal(&application.ClusterResConfig{Name: o.DefaultDestContext, Server: o.DefaultDestServer}) + if err != nil { + err = fmt.Errorf("failed to create cluster resources config: %w", err) return } - clusterResReadme = []byte(strings.ReplaceAll(string(clusterResReadmeTpl), "{CLUSTER}", o.DefaultDestServer)) - return } @@ -483,11 +446,3 @@ func RunProjectDelete(ctx context.Context, opts *BaseOptions) error { return nil } - -func cleanServerAddr(addr string) string { - return strings.NewReplacer( - "https://", "", - "http://", "", - "/", ".", - ).Replace(addr) -} diff --git a/cmd/commands/repo.go b/cmd/commands/repo.go index beaa3102..e5e528bc 100644 --- a/cmd/commands/repo.go +++ b/cmd/commands/repo.go @@ -2,7 +2,6 @@ package commands import ( "context" - _ "embed" "fmt" "os" "path/filepath" @@ -18,10 +17,10 @@ import ( "github.com/argoproj-labs/argocd-autopilot/pkg/store" "github.com/argoproj-labs/argocd-autopilot/pkg/util" + appset "github.com/argoproj-labs/applicationset/api/v1alpha1" argocdsettings "github.com/argoproj/argo-cd/v2/util/settings" "github.com/ghodss/yaml" "github.com/go-git/go-billy/v5/memfs" - billyUtils "github.com/go-git/go-billy/v5/util" "github.com/spf13/cobra" "github.com/spf13/viper" v1 "k8s.io/api/core/v1" @@ -34,12 +33,6 @@ const ( installationModeNormal = "normal" ) -//go:embed assets/projects_readme.md -var projectReadme []byte - -//go:embed assets/apps_readme.md -var appsReadme []byte - // used for mocking var ( argocdLogin = argocd.Login @@ -76,6 +69,7 @@ type ( bootstrapApp []byte rootApp []byte clusterResAppSet []byte + clusterResConfig []byte argocdApp []byte repoCreds []byte applyManifests []byte @@ -413,16 +407,6 @@ func validateRepo(repofs fs.FS) error { return nil } -type createAppOptions struct { - name string - namespace string - repoURL string - revision string - srcPath string - destServer string - noFinalizer bool -} - func waitClusterReady(ctx context.Context, f kube.Factory, timeout time.Duration, namespace string) error { return f.Wait(ctx, &kube.WaitOptions{ Interval: store.Default.WaitInterval, @@ -515,6 +499,46 @@ func buildBootstrapManifests(namespace, appSpecifier string, cloneOpts *git.Clon return nil, err } + manifests.clusterResAppSet, err = createAppSet(&createAppSetOptions{ + name: store.Default.ClusterResourcesDir, + namespace: namespace, + repoURL: cloneOpts.URL, + revision: cloneOpts.Revision, + appName: store.Default.ClusterResourcesDir + "-{{name}}", + appNamespace: namespace, + destServer: "{{server}}", + noFinalizer: true, + prune: false, + srcPath: filepath.Join(cloneOpts.RepoRoot, store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, "{{name}}"), + generators: []appset.ApplicationSetGenerator{ + { + Git: &appset.GitGenerator{ + RepoURL: cloneOpts.URL, + Revision: cloneOpts.Revision, + Files: []appset.GitFileGeneratorItem{ + { + Path: filepath.Join( + cloneOpts.RepoRoot, + store.Default.BootsrtrapDir, + store.Default.ClusterResourcesDir, + "*.json", + ), + }, + }, + RequeueAfterSeconds: &DefaultApplicationSetGeneratorInterval, + }, + }, + }, + }) + if err != nil { + return nil, err + } + + manifests.clusterResConfig, err = yaml.Marshal(&application.ClusterResConfig{Name: store.Default.ClusterContextName, Server: store.Default.DestServer}) + if err != nil { + return nil, err + } + k, err := createBootstrapKustomization(namespace, cloneOpts.URL, appSpecifier) if err != nil { return nil, err @@ -545,45 +569,32 @@ func buildBootstrapManifests(namespace, appSpecifier string, cloneOpts *git.Clon } func writeManifestsToRepo(repoFS fs.FS, manifests *bootstrapManifests, installationMode string) error { + var bulkWrites []fs.BulkWriteRequest argocdPath := repoFS.Join(store.Default.BootsrtrapDir, store.Default.ArgoCDName) - var err error - if installationMode == installationModeNormal { - if err = billyUtils.WriteFile(repoFS, repoFS.Join(argocdPath, "kustomization.yaml"), manifests.bootstrapKustomization, 0666); err != nil { - return err - } + clusterResReadme := []byte(strings.ReplaceAll(string(clusterResReadmeTpl), "{CLUSTER}", store.Default.ClusterContextName)) - if err = billyUtils.WriteFile(repoFS, repoFS.Join(argocdPath, "namespace.yaml"), manifests.namespace, 0666); err != nil { - return err + if installationMode == installationModeNormal { + bulkWrites = []fs.BulkWriteRequest{ + {Filename: repoFS.Join(argocdPath, "kustomization.yaml"), Data: manifests.bootstrapKustomization}, + {Filename: repoFS.Join(argocdPath, "namespace.yaml"), Data: manifests.namespace}, } } else { - if err = billyUtils.WriteFile(repoFS, repoFS.Join(argocdPath, "install.yaml"), manifests.applyManifests, 0666); err != nil { - return err + bulkWrites = []fs.BulkWriteRequest{ + {Filename: repoFS.Join(argocdPath, "install.yaml"), Data: manifests.applyManifests}, } } - // write projects root app - if err = billyUtils.WriteFile(repoFS, repoFS.Join(store.Default.BootsrtrapDir, store.Default.RootAppName+".yaml"), manifests.rootApp, 0666); err != nil { - return err - } - - // write argocd app - if err = billyUtils.WriteFile(repoFS, repoFS.Join(store.Default.BootsrtrapDir, store.Default.ArgoCDName+".yaml"), manifests.argocdApp, 0666); err != nil { - return err - } - - // write cluster-resources appset - - // write ./projects/README.md - if err = billyUtils.WriteFile(repoFS, repoFS.Join(store.Default.ProjectsDir, "README.md"), projectReadme, 0666); err != nil { - return err - } + bulkWrites = append(bulkWrites, []fs.BulkWriteRequest{ + {Filename: repoFS.Join(store.Default.BootsrtrapDir, store.Default.RootAppName+".yaml"), Data: manifests.rootApp}, // write projects root app + {Filename: repoFS.Join(store.Default.BootsrtrapDir, store.Default.ArgoCDName+".yaml"), Data: manifests.argocdApp}, // write argocd app + {Filename: repoFS.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir+".yaml"), Data: manifests.clusterResAppSet}, // write cluster-resources appset + {Filename: repoFS.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, store.Default.ClusterContextName, "README.md"), Data: clusterResReadme}, // write ./bootstrap/cluster-resources/in-cluster/README.md + {Filename: repoFS.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, store.Default.ClusterContextName+".json"), Data: manifests.clusterResConfig}, // write ./bootstrap/cluster-resources/in-cluster.json + {Filename: repoFS.Join(store.Default.ProjectsDir, "README.md"), Data: projectReadme}, // write ./projects/README.md + {Filename: repoFS.Join(store.Default.AppsDir, "README.md"), Data: appsReadme}, // write ./apps/README.md + }...) - // write ./apps/README.md - if err = billyUtils.WriteFile(repoFS, repoFS.Join(store.Default.AppsDir, "README.md"), appsReadme, 0666); err != nil { - return err - } - - return nil + return repoFS.BulkWrite(bulkWrites...) } func createBootstrapKustomization(namespace, repoURL, appSpecifier string) (*kusttypes.Kustomization, error) { diff --git a/pkg/application/application.go b/pkg/application/application.go index 090b3567..f74f1ca9 100644 --- a/pkg/application/application.go +++ b/pkg/application/application.go @@ -63,6 +63,11 @@ type ( SrcTargetRevision string `json:"srcTargetRevision"` } + ClusterResConfig struct { + Name string `json:"name"` + Server string `json:"server"` + } + CreateOptions struct { AppName string AppType string @@ -265,7 +270,6 @@ func newKustApp(o *CreateOptions, projectName, repoURL, targetRevision string) ( } if o.DestNamespace != "" && o.DestNamespace != "default" { - app.overlay.Resources = append(app.overlay.Resources, "namespace.yaml") app.overlay.Namespace = o.DestNamespace app.namespace = kube.GenerateNamespace(o.DestNamespace) } @@ -330,14 +334,23 @@ func kustCreateFiles(app *kustApp, repofs fs.FS, projectName string) error { } } + // verify that the dest server already exists + clusterName, err := serverToClusterName(repofs, app.opts.DestServer) + if err != nil { + return fmt.Errorf("failed to get cluster name for the specified dest-server: %w", err) + } + if clusterName == "" { + return fmt.Errorf("cluster '%s' is yet configured, you need to create a project that uses this cluster first", app.opts.DestServer) + } + // if we override the namespace we also need to write the namespace manifests next to the overlay if app.namespace != nil { - nsPath := repofs.Join(overlayPath, "namespace.yaml") nsYAML, err := yaml.Marshal(app.namespace) if err != nil { return fmt.Errorf("failed to marshal app overlay namespace: %w", err) } + nsPath := repofs.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, clusterName, app.namespace.Name+"-ns.yaml") if _, err = writeFile(repofs, nsPath, "application namespace", nsYAML); err != nil { return err } diff --git a/pkg/application/util.go b/pkg/application/util.go new file mode 100644 index 00000000..4146b9c4 --- /dev/null +++ b/pkg/application/util.go @@ -0,0 +1,27 @@ +package application + +import ( + "github.com/argoproj-labs/argocd-autopilot/pkg/fs" + "github.com/argoproj-labs/argocd-autopilot/pkg/store" + + billyUtils "github.com/go-git/go-billy/v5/util" +) + +func serverToClusterName(repofs fs.FS, server string) (string, error) { + confs, err := billyUtils.Glob(repofs, repofs.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, "*.json")) + if err != nil { + return "", err + } + + for _, confFile := range confs { + conf := &ClusterResConfig{} + if err = repofs.ReadYamls(confFile, conf); err != nil { + return "", err + } + if conf.Server == server { + return conf.Name, nil + } + } + + return "", nil +} diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 1d1251a3..974490a8 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -33,6 +33,12 @@ type FS interface { // WriteYamls write the data as yaml into the file WriteYamls(filename string, o ...interface{}) error + + // BulkWrite accepts multiple write requests and perform them sequentially. If it encounters + // an error it stops it's operation and returns this error. + // + // default file permission is 0644. + BulkWrite(writes ...BulkWriteRequest) error } type fsimpl struct { @@ -43,6 +49,13 @@ type File interface { billy.File } +type BulkWriteRequest struct { + Filename string + Data []byte + Perm os.FileMode + ErrMsg string +} + func Create(bfs billy.Filesystem) FS { return &fsimpl{bfs} } @@ -130,3 +143,20 @@ func (fs *fsimpl) WriteYamls(filename string, o ...interface{}) error { data := util.JoinManifests(yamls...) return billyUtils.WriteFile(fs, filename, data, 0666) } + +func (fs *fsimpl) BulkWrite(writes ...BulkWriteRequest) error { + for _, req := range writes { + var perm os.FileMode = 0644 + if req.Perm != 0 { + perm = req.Perm + } + + if err := billyUtils.WriteFile(fs, req.Filename, req.Data, perm); err != nil { + if req.ErrMsg != "" { + return fmt.Errorf("%s: %w", req.ErrMsg, err) + } + return err + } + } + return nil +} diff --git a/pkg/fs/mocks/fs.go b/pkg/fs/mocks/fs.go index 8e0f1c34..8dc35933 100644 --- a/pkg/fs/mocks/fs.go +++ b/pkg/fs/mocks/fs.go @@ -3,10 +3,11 @@ package mocks import ( - billy "github.com/go-git/go-billy/v5" - iofs "io/fs" + fs "github.com/argoproj-labs/argocd-autopilot/pkg/fs" + billy "github.com/go-git/go-billy/v5" + mock "github.com/stretchr/testify/mock" ) @@ -15,6 +16,26 @@ type FS struct { mock.Mock } +// BulkWrite provides a mock function with given fields: writes +func (_m *FS) BulkWrite(writes ...fs.BulkWriteRequest) error { + _va := make([]interface{}, len(writes)) + for _i := range writes { + _va[_i] = writes[_i] + } + var _ca []interface{} + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 error + if rf, ok := ret.Get(0).(func(...fs.BulkWriteRequest) error); ok { + r0 = rf(writes...) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // CheckExistsOrWrite provides a mock function with given fields: filename, data func (_m *FS) CheckExistsOrWrite(filename string, data []byte) (bool, error) { ret := _m.Called(filename, data) diff --git a/pkg/store/store.go b/pkg/store/store.go index 984ada8e..1cf54c96 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -51,6 +51,7 @@ var Default = struct { WaitInterval time.Duration DestServer string DestServerAnnotation string + ClusterContextName string }{ AppsDir: "apps", BootsrtrapDir: "bootstrap", @@ -68,6 +69,7 @@ var Default = struct { WaitInterval: time.Second * 3, DestServer: "https://kubernetes.default.svc", DestServerAnnotation: "argocd-autopilot.argoproj-labs.io/default-dest-server", + ClusterContextName: "in-cluster", } // Get returns the global store From fc4842a4a7c472c5b08349ae4077d21d2edcbeef Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Thu, 3 Jun 2021 11:12:43 +0300 Subject: [PATCH 04/20] fsutils --- Makefile | 4 +- cmd/commands/project.go | 11 +++--- cmd/commands/repo.go | 11 +++--- .../argocd-autopilot_project_create.md | 2 +- pkg/application/application.go | 2 +- pkg/application/mocks/application.go | 2 +- pkg/fs/fs.go | 38 +------------------ pkg/fs/fs_test.go | 1 - pkg/fs/mocks/file.go | 2 +- pkg/fs/mocks/fs.go | 27 ++----------- pkg/fs/utils/utils.go | 38 +++++++++++++++++++ pkg/git/github/mocks/repositories.go | 2 +- pkg/git/github/mocks/users.go | 2 +- pkg/git/gogit/mocks/repository.go | 2 +- pkg/git/gogit/mocks/worktree.go | 2 +- pkg/git/mocks/provider.go | 2 +- pkg/git/mocks/repository.go | 2 +- pkg/git/provider.go | 2 +- pkg/git/provider_github.go | 2 +- pkg/git/repository.go | 4 +- pkg/kube/kube.go | 2 +- pkg/kube/mocks/kube.go | 2 +- 22 files changed, 73 insertions(+), 89 deletions(-) create mode 100644 pkg/fs/utils/utils.go diff --git a/Makefile b/Makefile index 94cea794..d75691c9 100644 --- a/Makefile +++ b/Makefile @@ -151,13 +151,13 @@ check-worktree: $(GOBIN)/mockery: @mkdir dist || true @echo installing: mockery - @curl -L -o dist/mockery.tar.gz -- https://github.com/vektra/mockery/releases/download/v1.1.1/mockery_1.1.1_$(shell uname -s)_$(shell uname -m).tar.gz + @curl -L -o dist/mockery.tar.gz -- https://github.com/vektra/mockery/releases/download/v2.8.0/mockery_2.8.0_$(shell uname -s)_$(shell uname -m).tar.gz @tar zxvf dist/mockery.tar.gz mockery @rm dist/mockery.tar.gz @chmod +x mockery @mkdir -p $(GOBIN) @mv mockery $(GOBIN)/mockery - @mockery -version + @mockery --version $(GOBIN)/golangci-lint: @mkdir dist || true diff --git a/cmd/commands/project.go b/cmd/commands/project.go index 4c302f28..d093a11f 100644 --- a/cmd/commands/project.go +++ b/cmd/commands/project.go @@ -12,6 +12,7 @@ import ( "github.com/argoproj-labs/argocd-autopilot/pkg/application" "github.com/argoproj-labs/argocd-autopilot/pkg/argocd" "github.com/argoproj-labs/argocd-autopilot/pkg/fs" + fsutils "github.com/argoproj-labs/argocd-autopilot/pkg/fs/utils" "github.com/argoproj-labs/argocd-autopilot/pkg/git" "github.com/argoproj-labs/argocd-autopilot/pkg/log" "github.com/argoproj-labs/argocd-autopilot/pkg/store" @@ -173,7 +174,7 @@ func RunProjectCreate(ctx context.Context, opts *ProjectCreateOptions) error { return nil } - bulkWrites := []fs.BulkWriteRequest{} + bulkWrites := []fsutils.BulkWriteRequest{} if opts.DestKubeContext != "" { log.G().Infof("adding cluster: %s", opts.DestKubeContext) @@ -182,13 +183,13 @@ func RunProjectCreate(ctx context.Context, opts *ProjectCreateOptions) error { } if !repofs.ExistsOrDie(repofs.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, opts.DestKubeContext)) { - bulkWrites = append(bulkWrites, fs.BulkWriteRequest{ + bulkWrites = append(bulkWrites, fsutils.BulkWriteRequest{ Filename: repofs.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, opts.DestKubeContext+".json"), Data: clusterResConf, ErrMsg: "failed to write cluster config", }) - bulkWrites = append(bulkWrites, fs.BulkWriteRequest{ + bulkWrites = append(bulkWrites, fsutils.BulkWriteRequest{ Filename: repofs.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, opts.DestKubeContext, "README.md"), Data: clusterResReadme, ErrMsg: "failed to write cluster resources readme", @@ -196,13 +197,13 @@ func RunProjectCreate(ctx context.Context, opts *ProjectCreateOptions) error { } } - bulkWrites = append(bulkWrites, fs.BulkWriteRequest{ + bulkWrites = append(bulkWrites, fsutils.BulkWriteRequest{ Filename: repofs.Join(store.Default.ProjectsDir, opts.Name+".yaml"), Data: util.JoinManifests(projectYAML, appsetYAML), ErrMsg: "failed to create project file", }) - if err = repofs.BulkWrite(bulkWrites...); err != nil { + if err = fsutils.BulkWrite(repofs, bulkWrites...); err != nil { return err } diff --git a/cmd/commands/repo.go b/cmd/commands/repo.go index e5e528bc..8a0b71c9 100644 --- a/cmd/commands/repo.go +++ b/cmd/commands/repo.go @@ -11,6 +11,7 @@ import ( "github.com/argoproj-labs/argocd-autopilot/pkg/application" "github.com/argoproj-labs/argocd-autopilot/pkg/argocd" "github.com/argoproj-labs/argocd-autopilot/pkg/fs" + fsutils "github.com/argoproj-labs/argocd-autopilot/pkg/fs/utils" "github.com/argoproj-labs/argocd-autopilot/pkg/git" "github.com/argoproj-labs/argocd-autopilot/pkg/kube" "github.com/argoproj-labs/argocd-autopilot/pkg/log" @@ -569,22 +570,22 @@ func buildBootstrapManifests(namespace, appSpecifier string, cloneOpts *git.Clon } func writeManifestsToRepo(repoFS fs.FS, manifests *bootstrapManifests, installationMode string) error { - var bulkWrites []fs.BulkWriteRequest + var bulkWrites []fsutils.BulkWriteRequest argocdPath := repoFS.Join(store.Default.BootsrtrapDir, store.Default.ArgoCDName) clusterResReadme := []byte(strings.ReplaceAll(string(clusterResReadmeTpl), "{CLUSTER}", store.Default.ClusterContextName)) if installationMode == installationModeNormal { - bulkWrites = []fs.BulkWriteRequest{ + bulkWrites = []fsutils.BulkWriteRequest{ {Filename: repoFS.Join(argocdPath, "kustomization.yaml"), Data: manifests.bootstrapKustomization}, {Filename: repoFS.Join(argocdPath, "namespace.yaml"), Data: manifests.namespace}, } } else { - bulkWrites = []fs.BulkWriteRequest{ + bulkWrites = []fsutils.BulkWriteRequest{ {Filename: repoFS.Join(argocdPath, "install.yaml"), Data: manifests.applyManifests}, } } - bulkWrites = append(bulkWrites, []fs.BulkWriteRequest{ + bulkWrites = append(bulkWrites, []fsutils.BulkWriteRequest{ {Filename: repoFS.Join(store.Default.BootsrtrapDir, store.Default.RootAppName+".yaml"), Data: manifests.rootApp}, // write projects root app {Filename: repoFS.Join(store.Default.BootsrtrapDir, store.Default.ArgoCDName+".yaml"), Data: manifests.argocdApp}, // write argocd app {Filename: repoFS.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir+".yaml"), Data: manifests.clusterResAppSet}, // write cluster-resources appset @@ -594,7 +595,7 @@ func writeManifestsToRepo(repoFS fs.FS, manifests *bootstrapManifests, installat {Filename: repoFS.Join(store.Default.AppsDir, "README.md"), Data: appsReadme}, // write ./apps/README.md }...) - return repoFS.BulkWrite(bulkWrites...) + return fsutils.BulkWrite(repoFS, bulkWrites...) } func createBootstrapKustomization(namespace, repoURL, appSpecifier string) (*kusttypes.Kustomization, error) { diff --git a/docs/commands/argocd-autopilot_project_create.md b/docs/commands/argocd-autopilot_project_create.md index 860cfb8c..3b080f65 100644 --- a/docs/commands/argocd-autopilot_project_create.md +++ b/docs/commands/argocd-autopilot_project_create.md @@ -26,7 +26,7 @@ argocd-autopilot project create [PROJECT] [flags] # Create a new project in a specific path inside the GitOps repo - argocd-autopilot project create --installation-path path/to/installation_root + argocd-autopilot project create --installation-path path/to/installation_root ``` diff --git a/pkg/application/application.go b/pkg/application/application.go index f74f1ca9..8f880843 100644 --- a/pkg/application/application.go +++ b/pkg/application/application.go @@ -24,7 +24,7 @@ import ( kusttypes "sigs.k8s.io/kustomize/api/types" ) -//go:generate mockery -name Application -filename application.go +//go:generate mockery --name Application const ( InstallationModeFlat = "flat" diff --git a/pkg/application/mocks/application.go b/pkg/application/mocks/application.go index f3f91785..0289e719 100644 --- a/pkg/application/mocks/application.go +++ b/pkg/application/mocks/application.go @@ -1,4 +1,4 @@ -// Code generated by mockery v1.1.1. DO NOT EDIT. +// Code generated by mockery (devel). DO NOT EDIT. package mocks diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 974490a8..46ab08ff 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -12,8 +12,8 @@ import ( billyUtils "github.com/go-git/go-billy/v5/util" ) -//go:generate mockery -name FS -filename fs.go -//go:generate mockery -name File -filename file.go +//go:generate mockery --name FS + type FS interface { billy.Filesystem @@ -33,29 +33,12 @@ type FS interface { // WriteYamls write the data as yaml into the file WriteYamls(filename string, o ...interface{}) error - - // BulkWrite accepts multiple write requests and perform them sequentially. If it encounters - // an error it stops it's operation and returns this error. - // - // default file permission is 0644. - BulkWrite(writes ...BulkWriteRequest) error } type fsimpl struct { billy.Filesystem } -type File interface { - billy.File -} - -type BulkWriteRequest struct { - Filename string - Data []byte - Perm os.FileMode - ErrMsg string -} - func Create(bfs billy.Filesystem) FS { return &fsimpl{bfs} } @@ -143,20 +126,3 @@ func (fs *fsimpl) WriteYamls(filename string, o ...interface{}) error { data := util.JoinManifests(yamls...) return billyUtils.WriteFile(fs, filename, data, 0666) } - -func (fs *fsimpl) BulkWrite(writes ...BulkWriteRequest) error { - for _, req := range writes { - var perm os.FileMode = 0644 - if req.Perm != 0 { - perm = req.Perm - } - - if err := billyUtils.WriteFile(fs, req.Filename, req.Data, perm); err != nil { - if req.ErrMsg != "" { - return fmt.Errorf("%s: %w", req.ErrMsg, err) - } - return err - } - } - return nil -} diff --git a/pkg/fs/fs_test.go b/pkg/fs/fs_test.go index 0f22cefb..b97fc019 100644 --- a/pkg/fs/fs_test.go +++ b/pkg/fs/fs_test.go @@ -105,7 +105,6 @@ func Test_fsimpl_CheckExistsOrWrite(t *testing.T) { want bool wantErr string beforeFn func(m *mocks.FS) - assertFn func(t *testing.T, mockedFile *mocks.File) }{ "should exists": { args: args{path: "/usr/bar", data: []byte{}}, diff --git a/pkg/fs/mocks/file.go b/pkg/fs/mocks/file.go index 1f38ca80..6ddc8bd5 100644 --- a/pkg/fs/mocks/file.go +++ b/pkg/fs/mocks/file.go @@ -1,4 +1,4 @@ -// Code generated by mockery v1.1.1. DO NOT EDIT. +// Code generated by mockery (devel). DO NOT EDIT. package mocks diff --git a/pkg/fs/mocks/fs.go b/pkg/fs/mocks/fs.go index 8dc35933..bd75842d 100644 --- a/pkg/fs/mocks/fs.go +++ b/pkg/fs/mocks/fs.go @@ -1,13 +1,12 @@ -// Code generated by mockery v1.1.1. DO NOT EDIT. +// Code generated by mockery (devel). DO NOT EDIT. package mocks import ( - iofs "io/fs" - - fs "github.com/argoproj-labs/argocd-autopilot/pkg/fs" billy "github.com/go-git/go-billy/v5" + iofs "io/fs" + mock "github.com/stretchr/testify/mock" ) @@ -16,26 +15,6 @@ type FS struct { mock.Mock } -// BulkWrite provides a mock function with given fields: writes -func (_m *FS) BulkWrite(writes ...fs.BulkWriteRequest) error { - _va := make([]interface{}, len(writes)) - for _i := range writes { - _va[_i] = writes[_i] - } - var _ca []interface{} - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) - - var r0 error - if rf, ok := ret.Get(0).(func(...fs.BulkWriteRequest) error); ok { - r0 = rf(writes...) - } else { - r0 = ret.Error(0) - } - - return r0 -} - // CheckExistsOrWrite provides a mock function with given fields: filename, data func (_m *FS) CheckExistsOrWrite(filename string, data []byte) (bool, error) { ret := _m.Called(filename, data) diff --git a/pkg/fs/utils/utils.go b/pkg/fs/utils/utils.go new file mode 100644 index 00000000..a1c7b3af --- /dev/null +++ b/pkg/fs/utils/utils.go @@ -0,0 +1,38 @@ +package utils + +import ( + "fmt" + "os" + + "github.com/argoproj-labs/argocd-autopilot/pkg/fs" + + billyUtils "github.com/go-git/go-billy/v5/util" +) + +type BulkWriteRequest struct { + Filename string + Data []byte + Perm os.FileMode + ErrMsg string +} + +// BulkWrite accepts multiple write requests and perform them sequentially. If it encounters +// an error it stops it's operation and returns this error. +// +// default file permission is 0644. +func BulkWrite(fsys fs.FS, writes ...BulkWriteRequest) error { + for _, req := range writes { + var perm os.FileMode = 0644 + if req.Perm != 0 { + perm = req.Perm + } + + if err := billyUtils.WriteFile(fsys, req.Filename, req.Data, perm); err != nil { + if req.ErrMsg != "" { + return fmt.Errorf("%s: %w", req.ErrMsg, err) + } + return err + } + } + return nil +} diff --git a/pkg/git/github/mocks/repositories.go b/pkg/git/github/mocks/repositories.go index f0c51439..a4de23ea 100644 --- a/pkg/git/github/mocks/repositories.go +++ b/pkg/git/github/mocks/repositories.go @@ -1,4 +1,4 @@ -// Code generated by mockery v1.1.1. DO NOT EDIT. +// Code generated by mockery (devel). DO NOT EDIT. package mocks diff --git a/pkg/git/github/mocks/users.go b/pkg/git/github/mocks/users.go index c3dcd601..557461e4 100644 --- a/pkg/git/github/mocks/users.go +++ b/pkg/git/github/mocks/users.go @@ -1,4 +1,4 @@ -// Code generated by mockery v1.1.1. DO NOT EDIT. +// Code generated by mockery (devel). DO NOT EDIT. package mocks diff --git a/pkg/git/gogit/mocks/repository.go b/pkg/git/gogit/mocks/repository.go index 9d4cbc97..c043b83e 100644 --- a/pkg/git/gogit/mocks/repository.go +++ b/pkg/git/gogit/mocks/repository.go @@ -1,4 +1,4 @@ -// Code generated by mockery v1.1.1. DO NOT EDIT. +// Code generated by mockery (devel). DO NOT EDIT. package mocks diff --git a/pkg/git/gogit/mocks/worktree.go b/pkg/git/gogit/mocks/worktree.go index a41057d7..287dfd07 100644 --- a/pkg/git/gogit/mocks/worktree.go +++ b/pkg/git/gogit/mocks/worktree.go @@ -1,4 +1,4 @@ -// Code generated by mockery v1.1.1. DO NOT EDIT. +// Code generated by mockery (devel). DO NOT EDIT. package mocks diff --git a/pkg/git/mocks/provider.go b/pkg/git/mocks/provider.go index 231a4133..c7709e5b 100644 --- a/pkg/git/mocks/provider.go +++ b/pkg/git/mocks/provider.go @@ -1,4 +1,4 @@ -// Code generated by mockery v1.1.1. DO NOT EDIT. +// Code generated by mockery (devel). DO NOT EDIT. package mocks diff --git a/pkg/git/mocks/repository.go b/pkg/git/mocks/repository.go index 41382c7d..29a621fa 100644 --- a/pkg/git/mocks/repository.go +++ b/pkg/git/mocks/repository.go @@ -1,4 +1,4 @@ -// Code generated by mockery v1.1.1. DO NOT EDIT. +// Code generated by mockery (devel). DO NOT EDIT. package mocks diff --git a/pkg/git/provider.go b/pkg/git/provider.go index da942579..eaec9595 100644 --- a/pkg/git/provider.go +++ b/pkg/git/provider.go @@ -6,7 +6,7 @@ import ( "fmt" ) -//go:generate mockery -name Provider -filename provider.go +//go:generate mockery --name Provider type ( // Provider represents a git provider diff --git a/pkg/git/provider_github.go b/pkg/git/provider_github.go index 0c738f2b..3617926c 100644 --- a/pkg/git/provider_github.go +++ b/pkg/git/provider_github.go @@ -10,7 +10,7 @@ import ( gh "github.com/google/go-github/v34/github" ) -//go:generate mockery -dir github -all -output github/mocks -case snake +//go:generate mockery --dir github --all --output github/mocks --case snake type github struct { opts *ProviderOptions Repositories g.Repositories diff --git a/pkg/git/repository.go b/pkg/git/repository.go index ffba959a..a6eb49fe 100644 --- a/pkg/git/repository.go +++ b/pkg/git/repository.go @@ -21,8 +21,8 @@ import ( "github.com/spf13/viper" ) -//go:generate mockery -dir gogit -all -output gogit/mocks -case snake -//go:generate mockery -name Repository -filename repository.go +//go:generate mockery --dir gogit --all --output gogit/mocks --case snake +//go:generate mockery --name Repository type ( // Repository represents a git repository diff --git a/pkg/kube/kube.go b/pkg/kube/kube.go index aca4080e..45864d6f 100644 --- a/pkg/kube/kube.go +++ b/pkg/kube/kube.go @@ -20,7 +20,7 @@ import ( cmdutil "k8s.io/kubectl/pkg/cmd/util" ) -//go:generate mockery -name Factory -filename kube.go +//go:generate mockery --name Factory --filename kube.go const ( defaultPollInterval = time.Second * 2 diff --git a/pkg/kube/mocks/kube.go b/pkg/kube/mocks/kube.go index 511008bf..f1189e05 100644 --- a/pkg/kube/mocks/kube.go +++ b/pkg/kube/mocks/kube.go @@ -1,4 +1,4 @@ -// Code generated by mockery v1.1.1. DO NOT EDIT. +// Code generated by mockery (devel). DO NOT EDIT. package mocks From 9dfdcb43b69a06d9b74953da39e83f03dff4637a Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Thu, 3 Jun 2021 12:41:15 +0300 Subject: [PATCH 05/20] wip --- hack/check_worktree.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/hack/check_worktree.sh b/hack/check_worktree.sh index 451f489c..5577c843 100755 --- a/hack/check_worktree.sh +++ b/hack/check_worktree.sh @@ -6,6 +6,7 @@ if [[ -z "$res" ]]; then echo worktree is clean! else echo error: working tree is not clean! make sure you run 'make pre-push' and commit the changes. + echo "$res" GIT_PAGER=cat git diff --minimal exit 1 fi From cb54d82c2e94e40061e2879f6c13be6a5b87502e Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Thu, 3 Jun 2021 12:44:09 +0300 Subject: [PATCH 06/20] fix tests --- cmd/commands/project_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/commands/project_test.go b/cmd/commands/project_test.go index 0aebb8e6..bb9419e9 100644 --- a/cmd/commands/project_test.go +++ b/cmd/commands/project_test.go @@ -86,7 +86,7 @@ func TestRunProjectCreate(t *testing.T) { prepareRepo: func() (git.Repository, fs.FS, error) { memfs := memfs.New() mockedRepo := &gitmocks.Repository{} - mockedRepo.On("Persist", mock.AnythingOfType("*context.emptyCtx"), &git.PushOptions{CommitMsg: "Added project project"}).Return(fmt.Errorf("failed to persist")) + mockedRepo.On("Persist", mock.AnythingOfType("*context.emptyCtx"), &git.PushOptions{CommitMsg: "Added project 'project'"}).Return(fmt.Errorf("failed to persist")) return mockedRepo, fs.Create(memfs), nil }, getInstallationNamespace: func(_ fs.FS) (string, error) { @@ -99,7 +99,7 @@ func TestRunProjectCreate(t *testing.T) { prepareRepo: func() (git.Repository, fs.FS, error) { memfs := memfs.New() mockedRepo := &gitmocks.Repository{} - mockedRepo.On("Persist", mock.AnythingOfType("*context.emptyCtx"), &git.PushOptions{CommitMsg: "Added project project"}).Return(nil) + mockedRepo.On("Persist", mock.AnythingOfType("*context.emptyCtx"), &git.PushOptions{CommitMsg: "Added project 'project'"}).Return(nil) return mockedRepo, fs.Create(memfs), nil }, getInstallationNamespace: func(_ fs.FS) (string, error) { From 569d637e23cd88641f36ee172c1f99256268ac8d Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Thu, 3 Jun 2021 13:29:00 +0300 Subject: [PATCH 07/20] fix codegen --- pkg/application/application.go | 2 +- pkg/fs/fs.go | 2 +- pkg/git/provider.go | 2 +- pkg/git/repository.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/application/application.go b/pkg/application/application.go index 8f880843..13af31e5 100644 --- a/pkg/application/application.go +++ b/pkg/application/application.go @@ -24,7 +24,7 @@ import ( kusttypes "sigs.k8s.io/kustomize/api/types" ) -//go:generate mockery --name Application +//go:generate mockery --name Application --filename application.go const ( InstallationModeFlat = "flat" diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 46ab08ff..90b4d964 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -12,7 +12,7 @@ import ( billyUtils "github.com/go-git/go-billy/v5/util" ) -//go:generate mockery --name FS +//go:generate mockery --name FS --filename fs.go type FS interface { billy.Filesystem diff --git a/pkg/git/provider.go b/pkg/git/provider.go index eaec9595..117a02a5 100644 --- a/pkg/git/provider.go +++ b/pkg/git/provider.go @@ -6,7 +6,7 @@ import ( "fmt" ) -//go:generate mockery --name Provider +//go:generate mockery --name Provider --filename provider.go type ( // Provider represents a git provider diff --git a/pkg/git/repository.go b/pkg/git/repository.go index a6eb49fe..f2724b46 100644 --- a/pkg/git/repository.go +++ b/pkg/git/repository.go @@ -22,7 +22,7 @@ import ( ) //go:generate mockery --dir gogit --all --output gogit/mocks --case snake -//go:generate mockery --name Repository +//go:generate mockery --name Repository --filename repository.go type ( // Repository represents a git repository From 1153d8be09d98d50678a4b5f54bed95e798760f5 Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Thu, 3 Jun 2021 14:37:23 +0300 Subject: [PATCH 08/20] fixed tests --- cmd/commands/common.go | 5 +++++ cmd/commands/project.go | 1 + 2 files changed, 6 insertions(+) diff --git a/cmd/commands/common.go b/cmd/commands/common.go index 2aeecfb0..e8ecdd2c 100644 --- a/cmd/commands/common.go +++ b/cmd/commands/common.go @@ -178,6 +178,7 @@ type createAppSetOptions struct { namespace string appName string appNamespace string + appProject string repoURL string revision string srcPath string @@ -240,6 +241,10 @@ func createAppSet(o *createAppSetOptions) ([]byte, error) { } } + if o.appProject != "" { + appSet.Spec.Template.Spec.Project = o.appProject + } + if o.noFinalizer { appSet.ObjectMeta.Finalizers = []string{} } diff --git a/cmd/commands/project.go b/cmd/commands/project.go index d093a11f..0bc9197e 100644 --- a/cmd/commands/project.go +++ b/cmd/commands/project.go @@ -264,6 +264,7 @@ func generateProjectManifests(o *GenerateProjectOptions) (projectYAML, appSetYAM namespace: o.Namespace, appName: fmt.Sprintf("%s-{{ userGivenName }}", o.Name), appNamespace: o.Namespace, + appProject: o.Name, repoURL: "{{ srcRepoURL }}", srcPath: "{{ srcPath }}", revision: "{{ srcTargetRevision }}", From 9f241d9a901c740868926dd8ec91194842e4fd7a Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Thu, 3 Jun 2021 15:11:36 +0300 Subject: [PATCH 09/20] fixed tests --- pkg/application/application_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/application/application_test.go b/pkg/application/application_test.go index b0afc657..716a1723 100644 --- a/pkg/application/application_test.go +++ b/pkg/application/application_test.go @@ -99,8 +99,8 @@ func Test_newKustApp(t *testing.T) { assertFn: func(t *testing.T, a *kustApp) { assert.Equal(t, "install.yaml", a.base.Resources[0]) assert.Equal(t, []byte("foo"), a.manifests) + assert.Equal(t, 1, len(a.overlay.Resources)) assert.Equal(t, "../../base", a.overlay.Resources[0]) - assert.Equal(t, "namespace.yaml", a.overlay.Resources[1]) assert.Equal(t, "namespace", a.namespace.ObjectMeta.Name) assert.True(t, reflect.DeepEqual(&Config{ AppName: "name", From ac53d9a22c846dd6d3e8a8cdda17dfca56ced7c6 Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Thu, 3 Jun 2021 16:11:44 +0300 Subject: [PATCH 10/20] fixed tests --- pkg/application/application.go | 2 +- pkg/application/application_test.go | 43 +++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/pkg/application/application.go b/pkg/application/application.go index 13af31e5..bb0b2613 100644 --- a/pkg/application/application.go +++ b/pkg/application/application.go @@ -340,7 +340,7 @@ func kustCreateFiles(app *kustApp, repofs fs.FS, projectName string) error { return fmt.Errorf("failed to get cluster name for the specified dest-server: %w", err) } if clusterName == "" { - return fmt.Errorf("cluster '%s' is yet configured, you need to create a project that uses this cluster first", app.opts.DestServer) + return fmt.Errorf("cluster '%s' is not configured yet, you need to create a project that uses this cluster first", app.opts.DestServer) } // if we override the namespace we also need to write the namespace manifests next to the overlay diff --git a/pkg/application/application_test.go b/pkg/application/application_test.go index 716a1723..7ef920a8 100644 --- a/pkg/application/application_test.go +++ b/pkg/application/application_test.go @@ -11,6 +11,8 @@ import ( fsmocks "github.com/argoproj-labs/argocd-autopilot/pkg/fs/mocks" "github.com/argoproj-labs/argocd-autopilot/pkg/kube" "github.com/argoproj-labs/argocd-autopilot/pkg/store" + "github.com/ghodss/yaml" + v1 "k8s.io/api/core/v1" "github.com/go-git/go-billy/v5/memfs" billyUtils "github.com/go-git/go-billy/v5/util" @@ -19,6 +21,19 @@ import ( kusttypes "sigs.k8s.io/kustomize/api/types" ) +func bootstrapMockFS(t *testing.T, repofs fs.FS) { + clusterResConf := &ClusterResConfig{Name: store.Default.ClusterContextName, Server: store.Default.DestServer} + clusterResConfJSON, err := yaml.Marshal(clusterResConf) + assert.NoError(t, err) + err = billyUtils.WriteFile( + repofs, + repofs.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, store.Default.ClusterContextName+".json"), + clusterResConfJSON, + 0644, + ) + assert.NoError(t, err) +} + func Test_newKustApp(t *testing.T) { orgGenerateManifests := generateManifests defer func() { generateManifests = orgGenerateManifests }() @@ -243,7 +258,8 @@ func Test_kustCreateFiles(t *testing.T) { app := &kustApp{ baseApp: baseApp{ opts: &CreateOptions{ - AppName: "app", + AppName: "app", + DestServer: store.Default.DestServer, }, }, } @@ -262,9 +278,11 @@ func Test_kustCreateFiles(t *testing.T) { "Should create install.yaml when manifests exist": { beforeFn: func() (*kustApp, fs.FS, string) { app := &kustApp{ + baseApp: baseApp{ opts: &CreateOptions{ - AppName: "app", + AppName: "app", + DestServer: store.Default.DestServer, }, }, manifests: []byte("some manifests"), @@ -279,21 +297,27 @@ func Test_kustCreateFiles(t *testing.T) { assert.Equal(t, "some manifests", string(data)) }, }, - "Should create namespace.yaml when needed": { + "Should create namespace.yaml on the correct cluster resources directory when needed": { beforeFn: func() (*kustApp, fs.FS, string) { app := &kustApp{ baseApp: baseApp{ opts: &CreateOptions{ - AppName: "app", + AppName: "app", + DestServer: store.Default.DestServer, }, }, - namespace: kube.GenerateNamespace("namespace"), + namespace: kube.GenerateNamespace("foo"), } return app, fs.Create(memfs.New()), "project" }, assertFn: func(t *testing.T, repofs fs.FS, err error) { assert.NoError(t, err) - assert.True(t, repofs.ExistsOrDie(repofs.Join(store.Default.AppsDir, "app", store.Default.OverlaysDir, "project", "namespace.yaml")), "overlay namespace should exist") + path := repofs.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, store.Default.ClusterContextName, "foo-ns.yaml") + ns, err := repofs.ReadFile(path) + assert.NoError(t, err, "namespace file should exist in cluster-resources dir") + namespace := &v1.Namespace{} + assert.NoError(t, yaml.Unmarshal(ns, namespace)) + assert.Equal(t, "foo", namespace.Name) }, }, "Should fail when base kustomization is different from kustRes": { @@ -301,7 +325,8 @@ func Test_kustCreateFiles(t *testing.T) { app := &kustApp{ baseApp: baseApp{ opts: &CreateOptions{ - AppName: "app", + AppName: "app", + DestServer: store.Default.DestServer, }, }, base: &kusttypes.Kustomization{ @@ -332,7 +357,8 @@ func Test_kustCreateFiles(t *testing.T) { app := &kustApp{ baseApp: baseApp{ opts: &CreateOptions{ - AppName: "app", + AppName: "app", + DestServer: store.Default.DestServer, }, }, } @@ -355,6 +381,7 @@ func Test_kustCreateFiles(t *testing.T) { for tname, tt := range tests { t.Run(tname, func(t *testing.T) { app, repofs, projectName := tt.beforeFn() + bootstrapMockFS(t, repofs) err := app.CreateFiles(repofs, projectName) tt.assertFn(t, repofs, err) }) From b92159ffa805064cf5c4e6b0c7e772dbe8d86e1f Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Thu, 3 Jun 2021 17:37:29 +0300 Subject: [PATCH 11/20] fixed tests --- cmd/commands/project_test.go | 23 +++-- pkg/application/application.go | 64 ++++++++------ pkg/application/application_test.go | 128 +++++++++++++++++++++++++++- pkg/util/repospec.go | 4 +- 4 files changed, 183 insertions(+), 36 deletions(-) diff --git a/cmd/commands/project_test.go b/cmd/commands/project_test.go index bb9419e9..1e71624e 100644 --- a/cmd/commands/project_test.go +++ b/cmd/commands/project_test.go @@ -11,6 +11,7 @@ import ( "strings" "testing" + "github.com/argoproj-labs/argocd-autopilot/pkg/application" "github.com/argoproj-labs/argocd-autopilot/pkg/fs" fsmocks "github.com/argoproj-labs/argocd-autopilot/pkg/fs/mocks" "github.com/argoproj-labs/argocd-autopilot/pkg/git" @@ -164,15 +165,17 @@ func Test_generateProject(t *testing.T) { wantRevision string wantDefaultDestServer string wantProject string + wantContextName string }{ "should generate project and appset with correct values": { o: &GenerateProjectOptions{ - Name: "name", - Namespace: "namespace", - DefaultDestServer: "defaultDestServer", - RepoURL: "repoUrl", - Revision: "revision", - InstallationPath: "some/path", + Name: "name", + Namespace: "namespace", + DefaultDestServer: "defaultDestServer", + DefaultDestContext: "some-context-name", + RepoURL: "repoUrl", + Revision: "revision", + InstallationPath: "some/path", }, wantName: "name", wantNamespace: "namespace", @@ -180,6 +183,7 @@ func Test_generateProject(t *testing.T) { wantRepoURL: "repoUrl", wantRevision: "revision", wantDefaultDestServer: "defaultDestServer", + wantContextName: "some-context-name", }, } for ttname, tt := range tests { @@ -187,9 +191,14 @@ func Test_generateProject(t *testing.T) { assert := assert.New(t) gotProject := &argocdv1alpha1.AppProject{} gotAppSet := &appset.ApplicationSet{} - gotProjectYAML, gotAppSetYAML, _, _, _ := generateProjectManifests(tt.o) + gotClusterResConf := &application.ClusterResConfig{} + gotProjectYAML, gotAppSetYAML, _, gotClusterResConfigYAML, _ := generateProjectManifests(tt.o) assert.NoError(yaml.Unmarshal(gotProjectYAML, gotProject)) assert.NoError(yaml.Unmarshal(gotAppSetYAML, gotAppSet)) + assert.NoError(yaml.Unmarshal(gotClusterResConfigYAML, gotClusterResConf)) + + assert.Equal(tt.wantContextName, gotClusterResConf.Name) + assert.Equal(tt.wantDefaultDestServer, gotClusterResConf.Server) assert.Equal(tt.wantName, gotProject.Name, "Project Name") assert.Equal(tt.wantNamespace, gotProject.Namespace, "Project Namespace") diff --git a/pkg/application/application.go b/pkg/application/application.go index bb0b2613..c0c9ee68 100644 --- a/pkg/application/application.go +++ b/pkg/application/application.go @@ -334,24 +334,13 @@ func kustCreateFiles(app *kustApp, repofs fs.FS, projectName string) error { } } - // verify that the dest server already exists - clusterName, err := serverToClusterName(repofs, app.opts.DestServer) + clusterName, err := getClusterName(repofs, app.opts.DestServer) if err != nil { - return fmt.Errorf("failed to get cluster name for the specified dest-server: %w", err) - } - if clusterName == "" { - return fmt.Errorf("cluster '%s' is not configured yet, you need to create a project that uses this cluster first", app.opts.DestServer) + return err } - // if we override the namespace we also need to write the namespace manifests next to the overlay if app.namespace != nil { - nsYAML, err := yaml.Marshal(app.namespace) - if err != nil { - return fmt.Errorf("failed to marshal app overlay namespace: %w", err) - } - - nsPath := repofs.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, clusterName, app.namespace.Name+"-ns.yaml") - if _, err = writeFile(repofs, nsPath, "application namespace", nsYAML); err != nil { + if err = createNamespaceManifest(repofs, clusterName, app.namespace); err != nil { return err } } @@ -369,6 +358,31 @@ func kustCreateFiles(app *kustApp, repofs fs.FS, projectName string) error { return nil } +func getClusterName(repofs fs.FS, destServer string) (string, error) { + // verify that the dest server already exists + clusterName, err := serverToClusterName(repofs, destServer) + if err != nil { + return "", fmt.Errorf("failed to get cluster name for the specified dest-server: %w", err) + } + if clusterName == "" { + return "", fmt.Errorf("cluster '%s' is not configured yet, you need to create a project that uses this cluster first", destServer) + } + return clusterName, nil +} + +func createNamespaceManifest(repofs fs.FS, clusterName string, namespace *v1.Namespace) error { + nsYAML, err := yaml.Marshal(namespace) + if err != nil { + return fmt.Errorf("failed to marshal app overlay namespace: %w", err) + } + + nsPath := repofs.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, clusterName, namespace.Name+"-ns.yaml") + if _, err = writeFile(repofs, nsPath, "application namespace", nsYAML); err != nil { + return err + } + return nil +} + /* dirApp Application impl */ func newDirApp(opts *CreateOptions) *dirApp { app := &dirApp{ @@ -401,6 +415,17 @@ func (app *dirApp) CreateFiles(repofs fs.FS, projectName string) error { return err } + clusterName, err := getClusterName(repofs, app.opts.DestServer) + if err != nil { + return err + } + + if app.opts.DestNamespace != "" && app.opts.DestNamespace != "default" { + if err = createNamespaceManifest(repofs, clusterName, kube.GenerateNamespace(app.opts.DestNamespace)); err != nil { + return err + } + } + return nil } @@ -471,17 +496,6 @@ var generateManifests = func(k *kusttypes.Kustomization) ([]byte, error) { return nil, err } - if k.Namespace != "" { - log.G().Debug("detected namespace on kustomization, generating namespace.yaml file") - ns, err := yaml.Marshal(kube.GenerateNamespace(k.Namespace)) - if err != nil { - return nil, err - } - if err = ioutil.WriteFile(filepath.Join(td, "namespace.yaml"), ns, 0400); err != nil { - return nil, err - } - } - log.G().WithFields(log.Fields{ "bootstrapKustPath": kustomizationPath, "resourcePath": k.Resources[0], diff --git a/pkg/application/application_test.go b/pkg/application/application_test.go index 7ef920a8..c6fbb552 100644 --- a/pkg/application/application_test.go +++ b/pkg/application/application_test.go @@ -12,12 +12,11 @@ import ( "github.com/argoproj-labs/argocd-autopilot/pkg/kube" "github.com/argoproj-labs/argocd-autopilot/pkg/store" "github.com/ghodss/yaml" - v1 "k8s.io/api/core/v1" - "github.com/go-git/go-billy/v5/memfs" billyUtils "github.com/go-git/go-billy/v5/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + v1 "k8s.io/api/core/v1" kusttypes "sigs.k8s.io/kustomize/api/types" ) @@ -320,6 +319,22 @@ func Test_kustCreateFiles(t *testing.T) { assert.Equal(t, "foo", namespace.Name) }, }, + "Should fail if trying to install an application with destServer that is not configured yet": { + beforeFn: func() (*kustApp, fs.FS, string) { + app := &kustApp{ + baseApp: baseApp{ + opts: &CreateOptions{ + AppName: "app", + DestServer: "foo", + }, + }, + } + return app, fs.Create(memfs.New()), "project" + }, + assertFn: func(t *testing.T, repofs fs.FS, err error) { + assert.Error(t, err, "cluster 'foo' is not configured yet, you need to create a project that uses this cluster first") + }, + }, "Should fail when base kustomization is different from kustRes": { beforeFn: func() (*kustApp, fs.FS, string) { app := &kustApp{ @@ -566,3 +581,112 @@ func TestDeleteFromProject(t *testing.T) { }) } } + +func Test_newDirApp(t *testing.T) { + tests := map[string]struct { + opts *CreateOptions + want *dirApp + }{ + "Basic": { + opts: &CreateOptions{ + AppName: "fooapp", + AppSpecifier: "github.com/foo/bar/somepath/in/repo?ref=v0.1.2", + DestNamespace: "fizz", + DestServer: "buzz", + }, + want: &dirApp{ + config: &Config{ + AppName: "fooapp", + UserGivenName: "fooapp", + DestNamespace: "fizz", + DestServer: "buzz", + SrcRepoURL: "github.com/foo/bar", + SrcTargetRevision: "v0.1.2", + SrcPath: "somepath/in/repo", + }, + }, + }, + } + for tname, tt := range tests { + t.Run(tname, func(t *testing.T) { + if got := newDirApp(tt.opts); !reflect.DeepEqual(got.config, tt.want.config) { + t.Errorf("newDirApp() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_dirApp_CreateFiles(t *testing.T) { + tests := map[string]struct { + projectName string + app *dirApp + assertFn func(*testing.T, fs.FS, error) + }{ + "Should not create namespace if app namespace is 'default'": { + app: &dirApp{ + baseApp: baseApp{ + opts: &CreateOptions{ + AppName: "foo", + AppSpecifier: "github.com/foo/bar/path", + DestNamespace: "default", + DestServer: store.Default.DestServer, + }, + }, + }, + assertFn: func(t *testing.T, repofs fs.FS, e error) { + exists, err := repofs.Exists(repofs.Join( + store.Default.BootsrtrapDir, + store.Default.ClusterResourcesDir, + store.Default.ClusterContextName, + "default-ns.yaml", + )) + assert.NoError(t, err) + assert.False(t, exists) + }, + }, + "Should fail with destServer that is not configured yet": { + app: &dirApp{ + baseApp: baseApp{ + opts: &CreateOptions{ + AppName: "foo", + AppSpecifier: "github.com/foo/bar/path", + DestNamespace: "default", + DestServer: "some.new.server", + }, + }, + }, + assertFn: func(t *testing.T, repofs fs.FS, err error) { + assert.Error(t, err, "cluster 'some.new.server' is not configured yet, you need to create a project that uses this cluster first") + }, + }, + "Should create namespace in correct cluster resources dir": { + app: &dirApp{ + baseApp: baseApp{ + opts: &CreateOptions{ + AppName: "foo", + AppSpecifier: "github.com/foo/bar/path", + DestNamespace: "buzz", + DestServer: store.Default.DestServer, + }, + }, + }, + assertFn: func(t *testing.T, repofs fs.FS, e error) { + exists, err := repofs.Exists(repofs.Join( + store.Default.BootsrtrapDir, + store.Default.ClusterResourcesDir, + store.Default.ClusterContextName, + "buzz-ns.yaml", + )) + assert.NoError(t, err) + assert.True(t, exists) + }, + }, + } + for tname, tt := range tests { + t.Run(tname, func(t *testing.T) { + repofs := fs.Create(memfs.New()) + bootstrapMockFS(t, repofs) + tt.assertFn(t, repofs, tt.app.CreateFiles(repofs, tt.projectName)) + }) + } +} diff --git a/pkg/util/repospec.go b/pkg/util/repospec.go index 95ff205e..b17f87c5 100644 --- a/pkg/util/repospec.go +++ b/pkg/util/repospec.go @@ -109,7 +109,7 @@ func peelQuery(arg string) (string, string, time.Duration, bool) { func parseHostSpec(n string) (string, string) { var host string // Start accumulating the host part. - for _, p := range []string{ + for _, p := range [...]string{ // Order matters here. "git::", "gh:", "ssh://", "https://", "http://", "git@", "github.com:", "github.com/"} { @@ -134,7 +134,7 @@ func parseHostSpec(n string) (string, string) { } // If host is a http(s) or ssh URL, grab the domain part. - for _, p := range []string{ + for _, p := range [...]string{ "ssh://", "https://", "http://"} { if strings.HasSuffix(host, p) { i := strings.Index(n, "/") From 6177b3040da0bf117038b54a1e5e1febd22ac84b Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Thu, 3 Jun 2021 17:59:04 +0300 Subject: [PATCH 12/20] fixed tests --- pkg/application/application_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/application/application_test.go b/pkg/application/application_test.go index c6fbb552..b7ed2deb 100644 --- a/pkg/application/application_test.go +++ b/pkg/application/application_test.go @@ -600,7 +600,7 @@ func Test_newDirApp(t *testing.T) { UserGivenName: "fooapp", DestNamespace: "fizz", DestServer: "buzz", - SrcRepoURL: "github.com/foo/bar", + SrcRepoURL: "https://github.com/foo/bar", SrcTargetRevision: "v0.1.2", SrcPath: "somepath/in/repo", }, @@ -610,7 +610,7 @@ func Test_newDirApp(t *testing.T) { for tname, tt := range tests { t.Run(tname, func(t *testing.T) { if got := newDirApp(tt.opts); !reflect.DeepEqual(got.config, tt.want.config) { - t.Errorf("newDirApp() = %v, want %v", got, tt.want) + t.Errorf("newDirApp() = %+v, want %+v", got.config, tt.want.config) } }) } From 4d707e29fd30f17c4812eb46726c59f984428232 Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Thu, 3 Jun 2021 18:32:18 +0300 Subject: [PATCH 13/20] fixed json marshal --- cmd/commands/repo.go | 3 ++- pkg/application/application_test.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/commands/repo.go b/cmd/commands/repo.go index 8a0b71c9..b9545cdc 100644 --- a/cmd/commands/repo.go +++ b/cmd/commands/repo.go @@ -2,6 +2,7 @@ package commands import ( "context" + "encoding/json" "fmt" "os" "path/filepath" @@ -535,7 +536,7 @@ func buildBootstrapManifests(namespace, appSpecifier string, cloneOpts *git.Clon return nil, err } - manifests.clusterResConfig, err = yaml.Marshal(&application.ClusterResConfig{Name: store.Default.ClusterContextName, Server: store.Default.DestServer}) + manifests.clusterResConfig, err = json.Marshal(&application.ClusterResConfig{Name: store.Default.ClusterContextName, Server: store.Default.DestServer}) if err != nil { return nil, err } diff --git a/pkg/application/application_test.go b/pkg/application/application_test.go index b7ed2deb..cdd4f84c 100644 --- a/pkg/application/application_test.go +++ b/pkg/application/application_test.go @@ -1,6 +1,7 @@ package application import ( + "encoding/json" "fmt" "io/ioutil" "path/filepath" @@ -22,7 +23,7 @@ import ( func bootstrapMockFS(t *testing.T, repofs fs.FS) { clusterResConf := &ClusterResConfig{Name: store.Default.ClusterContextName, Server: store.Default.DestServer} - clusterResConfJSON, err := yaml.Marshal(clusterResConf) + clusterResConfJSON, err := json.Marshal(clusterResConf) assert.NoError(t, err) err = billyUtils.WriteFile( repofs, From 1caf6f95359691f46c9a3e0a2ec9f567fa946222 Mon Sep 17 00:00:00 2001 From: Noam Gal Date: Sun, 6 Jun 2021 14:40:11 +0300 Subject: [PATCH 14/20] fixed ns during bootstrap --- cmd/commands/repo.go | 3 +-- pkg/application/application.go | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/cmd/commands/repo.go b/cmd/commands/repo.go index b9545cdc..f2f16490 100644 --- a/cmd/commands/repo.go +++ b/cmd/commands/repo.go @@ -308,7 +308,7 @@ func RunRepoBootstrap(ctx context.Context, opts *RepoBootstrapOptions) error { // apply built manifest to k8s cluster log.G().Infof("using context: \"%s\", namespace: \"%s\"", opts.KubeContext, opts.Namespace) - log.G().Infof("applying bootstrap manifests to cluster...") + log.G().Infof("applying bootstrap manifests to cluster...\n%s", manifests.applyManifests) if err = opts.KubeFactory.Apply(ctx, opts.Namespace, util.JoinManifests(manifests.applyManifests, manifests.repoCreds)); err != nil { return fmt.Errorf("failed to apply bootstrap manifests to cluster: %w", err) } @@ -608,7 +608,6 @@ func createBootstrapKustomization(namespace, repoURL, appSpecifier string) (*kus k := &kusttypes.Kustomization{ Resources: []string{ appSpecifier, - "namespace.yaml", }, TypeMeta: kusttypes.TypeMeta{ APIVersion: kusttypes.KustomizationVersion, diff --git a/pkg/application/application.go b/pkg/application/application.go index c0c9ee68..1841ff9f 100644 --- a/pkg/application/application.go +++ b/pkg/application/application.go @@ -482,6 +482,23 @@ var generateManifests = func(k *kusttypes.Kustomization) ([]byte, error) { } defer os.RemoveAll(td) + if k.Namespace != "" { + log.G().Debug("detected namespace on kustomization, generating namespace.yaml file") + ns, err := yaml.Marshal(kube.GenerateNamespace(k.Namespace)) + if err != nil { + return nil, err + } + + if err = ioutil.WriteFile(filepath.Join(td, "namespace.yaml"), ns, 0400); err != nil { + return nil, err + } + + k.Resources = append(k.Resources, "namespace.yaml") + defer func() { + k.Resources = k.Resources[:1] + }() + } + kustomizationPath := filepath.Join(td, "kustomization.yaml") if err = fixResourcesPaths(k, kustomizationPath); err != nil { return nil, err From f93c0192168f38db9a6af22ef38df29336cca342 Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Sun, 6 Jun 2021 15:26:01 +0300 Subject: [PATCH 15/20] added brew bump step to release pipeline --- build/release.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/build/release.yml b/build/release.yml index 5bc83d49..0e225239 100644 --- a/build/release.yml +++ b/build/release.yml @@ -175,6 +175,18 @@ steps: on: - success + bump_brew_formula: + stage: Release + title: bump brew formula version + image: linuxbrew/linuxbrew:latest + commands: + - brew bump-formula-pr --strict --no-browser --tag ${{CF_BRANCH_TAG_NORMALIZED}} --revision ${{CF_REVISION}} + when: + steps: + - name: create_release + on: + - success + push_prod: stage: Release title: promote images From 8e548cd3e0bf287de63e952600b20667f75ca4d2 Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Sun, 6 Jun 2021 15:31:35 +0300 Subject: [PATCH 16/20] added brew bump step to release pipeline --- build/release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/release.yml b/build/release.yml index 0e225239..34532669 100644 --- a/build/release.yml +++ b/build/release.yml @@ -179,8 +179,10 @@ steps: stage: Release title: bump brew formula version image: linuxbrew/linuxbrew:latest + environment: + - HOMEBREW_GITHUB_API_TOKEN=${{GITHUB_TOKEN}} commands: - - brew bump-formula-pr --strict --no-browser --tag ${{CF_BRANCH_TAG_NORMALIZED}} --revision ${{CF_REVISION}} + - brew bump-formula-pr --strict --no-browser --tag ${{CF_BRANCH_TAG_NORMALIZED}} --revision ${{CF_REVISION}} --fork-org codefresh-io when: steps: - name: create_release From 7ff4af980052434bbb3b32e6eb46572bc2b7f07d Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Sun, 6 Jun 2021 15:35:08 +0300 Subject: [PATCH 17/20] fix log --- cmd/commands/repo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/commands/repo.go b/cmd/commands/repo.go index f2f16490..a3893784 100644 --- a/cmd/commands/repo.go +++ b/cmd/commands/repo.go @@ -308,7 +308,7 @@ func RunRepoBootstrap(ctx context.Context, opts *RepoBootstrapOptions) error { // apply built manifest to k8s cluster log.G().Infof("using context: \"%s\", namespace: \"%s\"", opts.KubeContext, opts.Namespace) - log.G().Infof("applying bootstrap manifests to cluster...\n%s", manifests.applyManifests) + log.G().Infof("applying bootstrap manifests to cluster...") if err = opts.KubeFactory.Apply(ctx, opts.Namespace, util.JoinManifests(manifests.applyManifests, manifests.repoCreds)); err != nil { return fmt.Errorf("failed to apply bootstrap manifests to cluster: %w", err) } From 0b443fa2f89c07f65da1b3a2cd1099064562f40d Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Sun, 6 Jun 2021 16:01:31 +0300 Subject: [PATCH 18/20] fix namespace in bootstrap --- cmd/commands/repo.go | 26 ++++++++++++++++++-------- pkg/application/application.go | 17 ----------------- pkg/util/util.go | 3 +++ 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/cmd/commands/repo.go b/cmd/commands/repo.go index a3893784..cacaf0fd 100644 --- a/cmd/commands/repo.go +++ b/cmd/commands/repo.go @@ -281,6 +281,7 @@ func RunRepoBootstrap(ctx context.Context, opts *RepoBootstrapOptions) error { // Dry Run check if opts.DryRun { fmt.Printf("%s", util.JoinManifests( + manifests.namespace, manifests.applyManifests, manifests.repoCreds, manifests.bootstrapApp, @@ -309,12 +310,12 @@ func RunRepoBootstrap(ctx context.Context, opts *RepoBootstrapOptions) error { // apply built manifest to k8s cluster log.G().Infof("using context: \"%s\", namespace: \"%s\"", opts.KubeContext, opts.Namespace) log.G().Infof("applying bootstrap manifests to cluster...") - if err = opts.KubeFactory.Apply(ctx, opts.Namespace, util.JoinManifests(manifests.applyManifests, manifests.repoCreds)); err != nil { + if err = opts.KubeFactory.Apply(ctx, opts.Namespace, util.JoinManifests(manifests.namespace, manifests.applyManifests, manifests.repoCreds)); err != nil { return fmt.Errorf("failed to apply bootstrap manifests to cluster: %w", err) } // write argocd manifests - if err = writeManifestsToRepo(opts.FS, manifests, opts.InstallationMode); err != nil { + if err = writeManifestsToRepo(opts.FS, manifests, opts.InstallationMode, opts.Namespace); err != nil { return fmt.Errorf("failed to write manifests to repo: %w", err) } @@ -546,10 +547,12 @@ func buildBootstrapManifests(namespace, appSpecifier string, cloneOpts *git.Clon return nil, err } - ns := kube.GenerateNamespace(namespace) - manifests.namespace, err = yaml.Marshal(ns) - if err != nil { - return nil, err + if namespace != "" && namespace != "default" { + ns := kube.GenerateNamespace(namespace) + manifests.namespace, err = yaml.Marshal(ns) + if err != nil { + return nil, err + } } manifests.applyManifests, err = runKustomizeBuild(k) @@ -570,7 +573,7 @@ func buildBootstrapManifests(namespace, appSpecifier string, cloneOpts *git.Clon return manifests, nil } -func writeManifestsToRepo(repoFS fs.FS, manifests *bootstrapManifests, installationMode string) error { +func writeManifestsToRepo(repoFS fs.FS, manifests *bootstrapManifests, installationMode, namespace string) error { var bulkWrites []fsutils.BulkWriteRequest argocdPath := repoFS.Join(store.Default.BootsrtrapDir, store.Default.ArgoCDName) clusterResReadme := []byte(strings.ReplaceAll(string(clusterResReadmeTpl), "{CLUSTER}", store.Default.ClusterContextName)) @@ -578,7 +581,6 @@ func writeManifestsToRepo(repoFS fs.FS, manifests *bootstrapManifests, installat if installationMode == installationModeNormal { bulkWrites = []fsutils.BulkWriteRequest{ {Filename: repoFS.Join(argocdPath, "kustomization.yaml"), Data: manifests.bootstrapKustomization}, - {Filename: repoFS.Join(argocdPath, "namespace.yaml"), Data: manifests.namespace}, } } else { bulkWrites = []fsutils.BulkWriteRequest{ @@ -596,6 +598,14 @@ func writeManifestsToRepo(repoFS fs.FS, manifests *bootstrapManifests, installat {Filename: repoFS.Join(store.Default.AppsDir, "README.md"), Data: appsReadme}, // write ./apps/README.md }...) + if manifests.namespace != nil { + // write ./bootstrap/cluster-resources/in-cluster/...-ns.yaml + bulkWrites = append( + bulkWrites, + fsutils.BulkWriteRequest{Filename: repoFS.Join(store.Default.BootsrtrapDir, store.Default.ClusterResourcesDir, store.Default.ClusterContextName, namespace+"-ns.yaml"), Data: manifests.namespace}, + ) + } + return fsutils.BulkWrite(repoFS, bulkWrites...) } diff --git a/pkg/application/application.go b/pkg/application/application.go index 1841ff9f..c0c9ee68 100644 --- a/pkg/application/application.go +++ b/pkg/application/application.go @@ -482,23 +482,6 @@ var generateManifests = func(k *kusttypes.Kustomization) ([]byte, error) { } defer os.RemoveAll(td) - if k.Namespace != "" { - log.G().Debug("detected namespace on kustomization, generating namespace.yaml file") - ns, err := yaml.Marshal(kube.GenerateNamespace(k.Namespace)) - if err != nil { - return nil, err - } - - if err = ioutil.WriteFile(filepath.Join(td, "namespace.yaml"), ns, 0400); err != nil { - return nil, err - } - - k.Resources = append(k.Resources, "namespace.yaml") - defer func() { - k.Resources = k.Resources[:1] - }() - } - kustomizationPath := filepath.Join(td, "kustomization.yaml") if err = fixResourcesPaths(k, kustomizationPath); err != nil { return nil, err diff --git a/pkg/util/util.go b/pkg/util/util.go index 4f074a96..259f7cf5 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -131,6 +131,9 @@ func MustParseDuration(dur string) time.Duration { func JoinManifests(manifests ...[]byte) []byte { res := make([]string, 0, len(manifests)) for _, m := range manifests { + if m == nil { + continue + } res = append(res, string(m)) } return []byte(strings.Join(res, yamlSeperator)) From aeeb4a810425897c761b03801c3f6d35dc1597b0 Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Sun, 6 Jun 2021 16:19:57 +0300 Subject: [PATCH 19/20] fix kust app --- cmd/commands/app.go | 2 +- pkg/application/application.go | 8 ++++---- pkg/application/application_test.go | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cmd/commands/app.go b/cmd/commands/app.go index d74aa8ee..2c5959b6 100644 --- a/cmd/commands/app.go +++ b/cmd/commands/app.go @@ -114,7 +114,7 @@ func RunAppCreate(ctx context.Context, opts *AppCreateOptions) error { return err } - app, err := opts.AppOpts.Parse(opts.ProjectName, opts.CloneOptions.URL, opts.CloneOptions.Revision) + app, err := opts.AppOpts.Parse(opts.ProjectName, opts.CloneOptions.URL, opts.CloneOptions.Revision, opts.CloneOptions.RepoRoot) if err != nil { return fmt.Errorf("failed to parse application from flags: %v", err) } diff --git a/pkg/application/application.go b/pkg/application/application.go index c0c9ee68..3efeec0e 100644 --- a/pkg/application/application.go +++ b/pkg/application/application.go @@ -188,10 +188,10 @@ func GenerateManifests(k *kusttypes.Kustomization) ([]byte, error) { /* CreateOptions impl */ // Parse tries to parse `CreateOptions` into an `Application`. -func (o *CreateOptions) Parse(projectName, repoURL, targetRevision string) (Application, error) { +func (o *CreateOptions) Parse(projectName, repoURL, targetRevision, repoRoot string) (Application, error) { switch o.AppType { case AppTypeKustomize: - return newKustApp(o, projectName, repoURL, targetRevision) + return newKustApp(o, projectName, repoURL, targetRevision, repoRoot) case AppTypeDirectory: return newDirApp(o), nil default: @@ -205,7 +205,7 @@ func (app *baseApp) Name() string { } /* kustApp Application impl */ -func newKustApp(o *CreateOptions, projectName, repoURL, targetRevision string) (*kustApp, error) { +func newKustApp(o *CreateOptions, projectName, repoURL, targetRevision, repoRoot string) (*kustApp, error) { var err error app := &kustApp{ baseApp: baseApp{o}, @@ -280,7 +280,7 @@ func newKustApp(o *CreateOptions, projectName, repoURL, targetRevision string) ( DestNamespace: o.DestNamespace, DestServer: o.DestServer, SrcRepoURL: repoURL, - SrcPath: filepath.Join(store.Default.AppsDir, o.AppName, store.Default.OverlaysDir, projectName), + SrcPath: filepath.Join(repoRoot, store.Default.AppsDir, o.AppName, store.Default.OverlaysDir, projectName), SrcTargetRevision: targetRevision, } diff --git a/pkg/application/application_test.go b/pkg/application/application_test.go index cdd4f84c..dab6c621 100644 --- a/pkg/application/application_test.go +++ b/pkg/application/application_test.go @@ -46,6 +46,7 @@ func Test_newKustApp(t *testing.T) { opts *CreateOptions srcRepoURL string srcTargetRevision string + srcRepoRoot string projectName string wantErr string assertFn func(*testing.T, *kustApp) @@ -130,7 +131,7 @@ func Test_newKustApp(t *testing.T) { } for tname, tt := range tests { t.Run(tname, func(t *testing.T) { - app, err := newKustApp(tt.opts, tt.projectName, tt.srcRepoURL, tt.srcTargetRevision) + app, err := newKustApp(tt.opts, tt.projectName, tt.srcRepoURL, tt.srcTargetRevision, tt.srcRepoRoot) if err != nil { if tt.wantErr != "" { assert.EqualError(t, err, tt.wantErr) From da0c53a668ca55bd0e975b10f13092d896c446eb Mon Sep 17 00:00:00 2001 From: "roi.kramer" Date: Sun, 6 Jun 2021 16:27:12 +0300 Subject: [PATCH 20/20] fix yaml instead of json --- cmd/commands/project.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/commands/project.go b/cmd/commands/project.go index 0bc9197e..5abad758 100644 --- a/cmd/commands/project.go +++ b/cmd/commands/project.go @@ -2,6 +2,7 @@ package commands import ( "context" + "encoding/json" "fmt" "io" "os" @@ -297,7 +298,7 @@ func generateProjectManifests(o *GenerateProjectOptions) (projectYAML, appSetYAM clusterResReadme = []byte(strings.ReplaceAll(string(clusterResReadmeTpl), "{CLUSTER}", o.DefaultDestServer)) - clusterResConfig, err = yaml.Marshal(&application.ClusterResConfig{Name: o.DefaultDestContext, Server: o.DefaultDestServer}) + clusterResConfig, err = json.Marshal(&application.ClusterResConfig{Name: o.DefaultDestContext, Server: o.DefaultDestServer}) if err != nil { err = fmt.Errorf("failed to create cluster resources config: %w", err) return