diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..0385e3430 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true diff --git a/README.md b/README.md index 3f9fdb9e4..f694825a7 100644 --- a/README.md +++ b/README.md @@ -176,3 +176,10 @@ Development instructions for the CircleCI CLI can be found in [HACKING.md](HACKI ## More Please see the [documentation](https://circleci-public.github.io/circleci-cli) or `circleci help` for more. + + +## Version Compatibility + +As of version `0.1.24705` - we no longer support Server 3.x instances. In order to upgrade the CLI to the latest version, you'll need to update your instance of server to 4.x. + +`0.1.23845` is the last version to support Server 3.x and 2.x. diff --git a/api/rest/client.go b/api/rest/client.go index 57c7bfcc5..219363301 100644 --- a/api/rest/client.go +++ b/api/rest/client.go @@ -17,14 +17,14 @@ import ( ) type Client struct { - baseURL *url.URL + BaseURL *url.URL circleToken string client *http.Client } func New(baseURL *url.URL, token string, httpClient *http.Client) *Client { return &Client{ - baseURL: baseURL, + BaseURL: baseURL, circleToken: token, client: httpClient, } @@ -60,7 +60,7 @@ func (c *Client) NewRequest(method string, u *url.URL, payload interface{}) (req } } - req, err = http.NewRequest(method, c.baseURL.ResolveReference(u).String(), r) + req, err = http.NewRequest(method, c.BaseURL.ResolveReference(u).String(), r) if err != nil { return nil, err } diff --git a/cmd/config.go b/cmd/config.go index 9c46e444b..d2afbef52 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -2,33 +2,16 @@ package cmd import ( "fmt" - "net/url" - "os" - "strings" - "github.com/CircleCI-Public/circleci-cli/api/rest" "github.com/CircleCI-Public/circleci-cli/config" "github.com/CircleCI-Public/circleci-cli/filetree" - "github.com/CircleCI-Public/circleci-cli/local" - "github.com/CircleCI-Public/circleci-cli/pipeline" "github.com/CircleCI-Public/circleci-cli/proxy" "github.com/CircleCI-Public/circleci-cli/settings" "github.com/pkg/errors" "github.com/spf13/cobra" - "github.com/spf13/pflag" "gopkg.in/yaml.v3" ) -var ( - CollaborationsPath = "me/collaborations" -) - -type configOptions struct { - cfg *settings.Config - rest *rest.Client - args []string -} - // Path to the config.yml file to operate on. // Used to for compatibility with `circleci config validate --path` var configPath string @@ -39,19 +22,7 @@ var configAnnotations = map[string]string{ "": "The path to your config (use \"-\" for STDIN)", } -func GetConfigAPIHost(cfg *settings.Config) string { - if cfg.Host != defaultHost { - return cfg.Host - } else { - return cfg.ConfigAPIHost - } -} - -func newConfigCommand(config *settings.Config) *cobra.Command { - opts := configOptions{ - cfg: config, - } - +func newConfigCommand(globalConfig *settings.Config) *cobra.Command { configCmd := &cobra.Command{ Use: "config", Short: "Operate on build config files", @@ -60,11 +31,8 @@ func newConfigCommand(config *settings.Config) *cobra.Command { packCommand := &cobra.Command{ Use: "pack ", Short: "Pack up your CircleCI configuration into a single file.", - PreRun: func(cmd *cobra.Command, args []string) { - opts.args = args - }, - RunE: func(_ *cobra.Command, _ []string) error { - return packConfig(opts) + RunE: func(_ *cobra.Command, args []string) error { + return packConfig(args) }, Args: cobra.ExactArgs(1), Annotations: make(map[string]string), @@ -75,12 +43,24 @@ func newConfigCommand(config *settings.Config) *cobra.Command { Use: "validate ", Aliases: []string{"check"}, Short: "Check that the config file is well formed.", - PreRun: func(cmd *cobra.Command, args []string) { - opts.args = args - opts.rest = rest.NewFromConfig(GetConfigAPIHost(opts.cfg), config) - }, - RunE: func(cmd *cobra.Command, _ []string) error { - return validateConfig(opts, cmd.Flags()) + RunE: func(cmd *cobra.Command, args []string) error { + compiler := config.New(globalConfig) + orgID, _ := cmd.Flags().GetString("org-id") + orgSlug, _ := cmd.Flags().GetString("org-slug") + path := config.DefaultConfigPath + if configPath != "" { + path = configPath + } + if len(args) == 1 { + path = args[0] + } + return compiler.ValidateConfig(config.ValidateConfigOpts{ + ConfigPath: path, + OrgID: orgID, + OrgSlug: orgSlug, + IgnoreDeprecatedImages: ignoreDeprecatedImages, + VerboseOutput: verboseOutput, + }) }, Args: cobra.MaximumNArgs(1), Annotations: make(map[string]string), @@ -89,6 +69,7 @@ func newConfigCommand(config *settings.Config) *cobra.Command { validateCommand.PersistentFlags().StringVarP(&configPath, "config", "c", ".circleci/config.yml", "path to config file") validateCommand.PersistentFlags().BoolVarP(&verboseOutput, "verbose", "v", false, "Enable verbose output") validateCommand.PersistentFlags().BoolVar(&ignoreDeprecatedImages, "ignore-deprecated-images", false, "ignores the deprecated images error") + if err := validateCommand.PersistentFlags().MarkHidden("config"); err != nil { panic(err) } @@ -98,12 +79,25 @@ func newConfigCommand(config *settings.Config) *cobra.Command { processCommand := &cobra.Command{ Use: "process ", Short: "Validate config and display expanded configuration.", - PreRun: func(cmd *cobra.Command, args []string) { - opts.args = args - opts.rest = rest.NewFromConfig(GetConfigAPIHost(opts.cfg), config) - }, - RunE: func(cmd *cobra.Command, _ []string) error { - return processConfig(opts, cmd.Flags()) + RunE: func(cmd *cobra.Command, args []string) error { + compiler := config.New(globalConfig) + pipelineParamsFilePath, _ := cmd.Flags().GetString("pipeline-parameters") + orgID, _ := cmd.Flags().GetString("org-id") + orgSlug, _ := cmd.Flags().GetString("org-slug") + path := config.DefaultConfigPath + if configPath != "" { + path = configPath + } + if len(args) == 1 { + path = args[0] + } + return compiler.ProcessConfig(config.ProcessConfigOpts{ + ConfigPath: path, + OrgID: orgID, + OrgSlug: orgSlug, + PipelineParamsFilePath: pipelineParamsFilePath, + VerboseOutput: verboseOutput, + }) }, Args: cobra.ExactArgs(1), Annotations: make(map[string]string), @@ -112,16 +106,13 @@ func newConfigCommand(config *settings.Config) *cobra.Command { processCommand.Flags().StringP("org-slug", "o", "", "organization slug (for example: github/example-org), used when a config depends on private orbs belonging to that org") processCommand.Flags().String("org-id", "", "organization id used when a config depends on private orbs belonging to that org") processCommand.Flags().StringP("pipeline-parameters", "", "", "YAML/JSON map of pipeline parameters, accepts either YAML/JSON directly or file path (for example: my-params.yml)") - processCommand.Flags().StringP("circleci-api-host", "", "", "the api-host you want to use for config processing and validation") + processCommand.PersistentFlags().BoolVar(&verboseOutput, "verbose", false, "adds verbose output to the command") migrateCommand := &cobra.Command{ Use: "migrate", Short: "Migrate a pre-release 2.0 config to the official release version", - PreRun: func(cmd *cobra.Command, args []string) { - opts.args = args - }, - RunE: func(_ *cobra.Command, _ []string) error { - return migrateConfig(opts) + RunE: func(_ *cobra.Command, args []string) error { + return migrateConfig(args) }, Hidden: true, DisableFlagParsing: true, @@ -138,115 +129,8 @@ func newConfigCommand(config *settings.Config) *cobra.Command { return configCmd } -// The arg is actually optional, in order to support compatibility with the --path flag. -func validateConfig(opts configOptions, flags *pflag.FlagSet) error { - var err error - var response *config.ConfigResponse - path := local.DefaultConfigPath - // First, set the path to configPath set by --path flag for compatibility - if configPath != "" { - path = configPath - } - - // Then, if an arg is passed in, choose that instead - if len(opts.args) == 1 { - path = opts.args[0] - } - - //if no orgId provided use org slug - values := pipeline.LocalPipelineValues() - if verboseOutput { - printValues(values) - } - - var orgID string - orgID, _ = flags.GetString("org-id") - if strings.TrimSpace(orgID) != "" { - response, err = config.ConfigQuery(opts.rest, path, orgID, nil, pipeline.LocalPipelineValues()) - if err != nil { - return err - } - } else { - orgSlug, _ := flags.GetString("org-slug") - orgs, err := GetOrgCollaborations(opts.cfg) - if err != nil { - fmt.Println(err.Error()) - } - orgID = GetOrgIdFromSlug(orgSlug, orgs) - response, err = config.ConfigQuery(opts.rest, path, orgID, nil, pipeline.LocalPipelineValues()) - if err != nil { - return err - } - } - - // check if a deprecated Linux VM image is being used - // link here to blog post when available - // returns an error if a deprecated image is used - if !ignoreDeprecatedImages { - err := config.DeprecatedImageCheck(response) - if err != nil { - return err - } - } - - if path == "-" { - fmt.Printf("\nConfig input is valid.\n") - } else { - fmt.Printf("\nConfig file at %s is valid.\n", path) - } - - return nil -} - -func processConfig(opts configOptions, flags *pflag.FlagSet) error { - paramsYaml, _ := flags.GetString("pipeline-parameters") - var response *config.ConfigResponse - var params pipeline.Parameters - var err error - - if len(paramsYaml) > 0 { - // The 'src' value can be a filepath, or a yaml string. If the file cannot be read successfully, - // proceed with the assumption that the value is already valid yaml. - raw, err := os.ReadFile(paramsYaml) - if err != nil { - raw = []byte(paramsYaml) - } - - err = yaml.Unmarshal(raw, ¶ms) - if err != nil { - return fmt.Errorf("invalid 'pipeline-parameters' provided: %s", err.Error()) - } - } - - //if no orgId provided use org slug - values := pipeline.LocalPipelineValues() - printValues(values) - - orgID, _ := flags.GetString("org-id") - if strings.TrimSpace(orgID) != "" { - response, err = config.ConfigQuery(opts.rest, opts.args[0], orgID, params, values) - if err != nil { - return err - } - } else { - orgSlug, _ := flags.GetString("org-slug") - orgs, err := GetOrgCollaborations(opts.cfg) - if err != nil { - fmt.Println(err.Error()) - } - orgID = GetOrgIdFromSlug(orgSlug, orgs) - response, err = config.ConfigQuery(opts.rest, opts.args[0], orgID, params, values) - if err != nil { - return err - } - } - - fmt.Print(response.OutputYaml) - return nil -} - -func packConfig(opts configOptions) error { - tree, err := filetree.NewTree(opts.args[0]) +func packConfig(args []string) error { + tree, err := filetree.NewTree(args[0]) if err != nil { return errors.Wrap(err, "An error occurred trying to build the tree") } @@ -259,44 +143,6 @@ func packConfig(opts configOptions) error { return nil } -func migrateConfig(opts configOptions) error { - return proxy.Exec([]string{"config", "migrate"}, opts.args) -} - -func printValues(values pipeline.Values) { - fmt.Fprintln(os.Stderr, "Processing config with following values:") - for key, value := range values { - fmt.Fprintf(os.Stderr, "%-18s %s\n", key+":", value) - } -} - -type CollaborationResult struct { - VcsTye string `json:"vcs_type"` - OrgSlug string `json:"slug"` - OrgName string `json:"name"` - OrgId string `json:"id"` - AvatarUrl string `json:"avatar_url"` -} - -// GetOrgCollaborations - fetches all the collaborations for a given user. -func GetOrgCollaborations(cfg *settings.Config) ([]CollaborationResult, error) { - baseClient := rest.NewFromConfig(cfg.Host, cfg) - req, err := baseClient.NewRequest("GET", &url.URL{Path: CollaborationsPath}, nil) - if err != nil { - return nil, err - } - - var resp []CollaborationResult - _, err = baseClient.DoRequest(req, &resp) - return resp, err -} - -// GetOrgIdFromSlug - converts a slug into an orgID. -func GetOrgIdFromSlug(slug string, collaborations []CollaborationResult) string { - for _, v := range collaborations { - if v.OrgSlug == slug { - return v.OrgId - } - } - return "" +func migrateConfig(args []string) error { + return proxy.Exec([]string{"config", "migrate"}, args) } diff --git a/cmd/config_test.go b/cmd/config_test.go index 1d0967e79..d0ed9016d 100644 --- a/cmd/config_test.go +++ b/cmd/config_test.go @@ -4,15 +4,11 @@ import ( "fmt" "os/exec" "path/filepath" - "testing" "github.com/CircleCI-Public/circleci-cli/clitest" - "github.com/CircleCI-Public/circleci-cli/cmd" - "github.com/CircleCI-Public/circleci-cli/settings" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/onsi/gomega/gexec" - "github.com/stretchr/testify/assert" "gotest.tools/v3/golden" ) @@ -250,17 +246,3 @@ var _ = Describe("Config", func() { }) }) }) - -func TestGetConfigAPIHost(t *testing.T) { - t.Run("tests that we correctly get the config api host when the host is not the default one", func(t *testing.T) { - // if the host isn't equal to `https://circleci.com` then this is likely a server instance and - // wont have the api.X.com subdomain so we should instead just respect the host for config commands - host := cmd.GetConfigAPIHost(&settings.Config{Host: "test"}) - assert.Equal(t, host, "test") - - // If the host passed in is the same as the defaultHost 'https://circleci.com' - then we know this is cloud - // and as such should use the `api.circleci.com` subdomain - host = cmd.GetConfigAPIHost(&settings.Config{Host: "https://circleci.com", ConfigAPIHost: "https://api.circleci.com"}) - assert.Equal(t, host, "https://api.circleci.com") - }) -} diff --git a/cmd/root.go b/cmd/root.go index 0d7695e8a..a6257771f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -22,7 +22,6 @@ import ( var defaultEndpoint = "graphql-unstable" var defaultHost = "https://circleci.com" -var defaultAPIHost = "https://api.circleci.com" var defaultRestEndpoint = "api/v2" var trueString = "true" @@ -105,11 +104,6 @@ func MakeCommands() *cobra.Command { RestEndpoint: defaultRestEndpoint, Endpoint: defaultEndpoint, GitHubAPI: "https://api.github.com/", - // The config api host differs for both cloud and server setups. - // For cloud, the base domain will be https://api.circleci.com - // for server, this should match the host as we don't have the same - // api subdomain setup - ConfigAPIHost: defaultAPIHost, } if err := rootOptions.Load(); err != nil { @@ -188,7 +182,6 @@ func MakeCommands() *cobra.Command { flags.StringVar(&rootOptions.Host, "host", rootOptions.Host, "URL to your CircleCI host, also CIRCLECI_CLI_HOST") flags.StringVar(&rootOptions.Endpoint, "endpoint", rootOptions.Endpoint, "URI to your CircleCI GraphQL API endpoint") flags.StringVar(&rootOptions.GitHubAPI, "github-api", "https://api.github.com/", "Change the default endpoint to GitHub API for retrieving updates") - flags.StringVar(&rootOptions.ConfigAPIHost, "config-api-host", "https://api.circleci.com", "Change the default endpoint for the config api host") flags.BoolVar(&rootOptions.SkipUpdateCheck, "skip-update-check", skipUpdateByDefault(), "Skip the check for updates check run before every command.") hidden := []string{"github-api", "debug", "endpoint"} diff --git a/config/collaborators.go b/config/collaborators.go new file mode 100644 index 000000000..30da51d17 --- /dev/null +++ b/config/collaborators.go @@ -0,0 +1,39 @@ +package config + +import ( + "net/url" +) + +var ( + CollaborationsPath = "me/collaborations" +) + +type CollaborationResult struct { + VcsTye string `json:"vcs_type"` + OrgSlug string `json:"slug"` + OrgName string `json:"name"` + OrgId string `json:"id"` + AvatarUrl string `json:"avatar_url"` +} + +// GetOrgCollaborations - fetches all the collaborations for a given user. +func (c *ConfigCompiler) GetOrgCollaborations() ([]CollaborationResult, error) { + req, err := c.collaboratorRestClient.NewRequest("GET", &url.URL{Path: CollaborationsPath}, nil) + if err != nil { + return nil, err + } + + var resp []CollaborationResult + _, err = c.collaboratorRestClient.DoRequest(req, &resp) + return resp, err +} + +// GetOrgIdFromSlug - converts a slug into an orgID. +func GetOrgIdFromSlug(slug string, collaborations []CollaborationResult) string { + for _, v := range collaborations { + if v.OrgSlug == slug { + return v.OrgId + } + } + return "" +} diff --git a/config/collaborators_test.go b/config/collaborators_test.go new file mode 100644 index 000000000..115c4f8c9 --- /dev/null +++ b/config/collaborators_test.go @@ -0,0 +1,43 @@ +package config + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/CircleCI-Public/circleci-cli/settings" + "github.com/stretchr/testify/assert" +) + +func TestGetOrgCollaborations(t *testing.T) { + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `[{"vcs_type":"circleci","slug":"gh/test","id":"2345"}]`) + })) + defer svr.Close() + compiler := New(&settings.Config{Host: svr.URL, HTTPClient: http.DefaultClient}) + + t.Run("assert compiler has correct host", func(t *testing.T) { + assert.Equal(t, "http://"+compiler.collaboratorRestClient.BaseURL.Host, svr.URL) + }) + + t.Run("getOrgCollaborations can parse response correctly", func(t *testing.T) { + collabs, err := compiler.GetOrgCollaborations() + assert.NoError(t, err) + assert.Equal(t, 1, len(collabs)) + assert.Equal(t, "circleci", collabs[0].VcsTye) + }) + + t.Run("can fetch orgID from a slug", func(t *testing.T) { + expected := "1234" + actual := GetOrgIdFromSlug("gh/test", []CollaborationResult{{OrgSlug: "gh/test", OrgId: "1234"}}) + assert.Equal(t, expected, actual) + }) + + t.Run("returns empty if no slug match", func(t *testing.T) { + expected := "" + actual := GetOrgIdFromSlug("gh/doesntexist", []CollaborationResult{{OrgSlug: "gh/test", OrgId: "1234"}}) + assert.Equal(t, expected, actual) + }) +} diff --git a/config/commands.go b/config/commands.go new file mode 100644 index 000000000..2167e6859 --- /dev/null +++ b/config/commands.go @@ -0,0 +1,158 @@ +package config + +import ( + "fmt" + "os" + "strings" + + "github.com/pkg/errors" + "gopkg.in/yaml.v3" +) + +func printValues(values Values) { + for key, value := range values { + fmt.Printf("\t%s:\t%s", key, value) + } +} + +type ProcessConfigOpts struct { + ConfigPath string + OrgID string + OrgSlug string + PipelineParamsFilePath string + + VerboseOutput bool +} + +func (c *ConfigCompiler) getOrgID( + optsOrgID string, + optsOrgSlug string, +) (string, error) { + if optsOrgID == "" && optsOrgSlug == "" { + fmt.Println("No org id or slug has been provided") + return "", nil + } + + var orgID string + if strings.TrimSpace(optsOrgID) != "" { + orgID = optsOrgID + } else { + orgs, err := c.GetOrgCollaborations() + if err != nil { + return "", err + } + orgID = GetOrgIdFromSlug(optsOrgSlug, orgs) + if orgID == "" { + fmt.Println("Could not fetch a valid org-id from collaborators endpoint.") + fmt.Println("Check if you have access to this org by hitting https://circleci.com/api/v2/me/collaborations") + fmt.Println("Continuing on - private orb resolution will not work as intended") + } + } + + return orgID, nil +} + +func (c *ConfigCompiler) ProcessConfig(opts ProcessConfigOpts) error { + var response *ConfigResponse + var params Parameters + var err error + + if len(opts.PipelineParamsFilePath) > 0 { + // The 'src' value can be a filepath, or a yaml string. If the file cannot be read successfully, + // proceed with the assumption that the value is already valid yaml. + raw, err := os.ReadFile(opts.PipelineParamsFilePath) + if err != nil { + raw = []byte(opts.PipelineParamsFilePath) + } + + err = yaml.Unmarshal(raw, ¶ms) + if err != nil { + return fmt.Errorf("invalid 'pipeline-parameters' provided: %s", err.Error()) + } + } + + //if no orgId provided use org slug + values := LocalPipelineValues() + if opts.VerboseOutput { + fmt.Println("Processing config with following values") + printValues(values) + } + + orgID, err := c.getOrgID(opts.OrgID, opts.OrgSlug) + if err != nil { + return fmt.Errorf("failed to get the appropriate org-id: %s", err.Error()) + } + + response, err = c.ConfigQuery( + opts.ConfigPath, + orgID, + params, + values, + ) + if err != nil { + return err + } + + if !response.Valid { + fmt.Println(response.Errors) + return errors.New("config is invalid") + } + + fmt.Print(response.OutputYaml) + return nil +} + +type ValidateConfigOpts struct { + ConfigPath string + OrgID string + OrgSlug string + + IgnoreDeprecatedImages bool + VerboseOutput bool +} + +// The arg is actually optional, in order to support compatibility with the --path flag. +func (c *ConfigCompiler) ValidateConfig(opts ValidateConfigOpts) error { + var err error + var response *ConfigResponse + + //if no orgId provided use org slug + values := LocalPipelineValues() + if opts.VerboseOutput { + fmt.Println("Validating config with following values") + printValues(values) + } + + orgID, err := c.getOrgID(opts.OrgID, opts.OrgSlug) + if err != nil { + return fmt.Errorf("failed to get the appropriate org-id: %s", err.Error()) + } + + response, err = c.ConfigQuery( + opts.ConfigPath, + orgID, + nil, + LocalPipelineValues(), + ) + if err != nil { + return err + } + + if !response.Valid { + fmt.Println(response.Errors) + return errors.New("config is invalid") + } + + // check if a deprecated Linux VM image is being used + // link here to blog post when available + // returns an error if a deprecated image is used + if !opts.IgnoreDeprecatedImages { + err := deprecatedImageCheck(response) + if err != nil { + return err + } + } + + fmt.Printf("\nConfig file at %s is valid.\n", opts.ConfigPath) + return nil +} diff --git a/config/commands_test.go b/config/commands_test.go new file mode 100644 index 000000000..dc6e7a16a --- /dev/null +++ b/config/commands_test.go @@ -0,0 +1,163 @@ +package config + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/CircleCI-Public/circleci-cli/settings" + "github.com/stretchr/testify/assert" +) + +func TestGetOrgID(t *testing.T) { + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `[{"vcs_type":"circleci","slug":"gh/test","id":"2345"}]`) + })) + defer svr.Close() + compiler := New(&settings.Config{Host: svr.URL, HTTPClient: http.DefaultClient}) + + t.Run("returns the original org-id passed if it is set", func(t *testing.T) { + expected := "1234" + actual, err := compiler.getOrgID("1234", "") + assert.NoError(t, err) + assert.Equal(t, expected, actual) + }) + + t.Run("returns the correct org id from org-slug", func(t *testing.T) { + expected := "2345" + actual, err := compiler.getOrgID("", "gh/test") + assert.NoError(t, err) + assert.Equal(t, expected, actual) + }) + + t.Run("returns the correct org id with org-id and org-slug both set", func(t *testing.T) { + expected := "1234" + actual, err := compiler.getOrgID("1234", "gh/test") + assert.NoError(t, err) + assert.Equal(t, expected, actual) + }) + + t.Run("does not return an error if org-id cannot be found", func(t *testing.T) { + expected := "" + actual, err := compiler.getOrgID("", "gh/doesntexist") + assert.NoError(t, err) + assert.Equal(t, expected, actual) + }) + +} + +var testYaml = `version: 2.1\n\norbs:\n node: circleci/node@5.0.3\n\njobs:\n datadog-hello-world:\n docker:\n - image: cimg/base:stable\n steps:\n - run: |\n echo \"doing something really cool\"\nworkflows:\n datadog-hello-world:\n jobs:\n - datadog-hello-world\n` + +func TestValidateConfig(t *testing.T) { + t.Run("validate config works as expected", func(t *testing.T) { + t.Run("validate config is able to send a request with no owner-id", func(t *testing.T) { + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + reqBody, err := io.ReadAll(r.Body) + assert.NoError(t, err) + + var req CompileConfigRequest + err = json.Unmarshal(reqBody, &req) + assert.NoError(t, err) + fmt.Fprintf(w, `{"valid":true,"source-yaml":"%s","output-yaml":"%s","errors":[]}`, testYaml, testYaml) + })) + defer svr.Close() + compiler := New(&settings.Config{Host: svr.URL, HTTPClient: http.DefaultClient}) + + err := compiler.ValidateConfig(ValidateConfigOpts{ + ConfigPath: "testdata/config.yml", + }) + assert.NoError(t, err) + }) + + t.Run("validate config is able to send a request with owner-id", func(t *testing.T) { + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + reqBody, err := io.ReadAll(r.Body) + assert.NoError(t, err) + + var req CompileConfigRequest + err = json.Unmarshal(reqBody, &req) + assert.NoError(t, err) + assert.Equal(t, "1234", req.Options.OwnerID) + fmt.Fprintf(w, `{"valid":true,"source-yaml":"%s","output-yaml":"%s","errors":[]}`, testYaml, testYaml) + })) + defer svr.Close() + compiler := New(&settings.Config{Host: svr.URL, HTTPClient: http.DefaultClient}) + + err := compiler.ValidateConfig(ValidateConfigOpts{ + ConfigPath: "testdata/config.yml", + OrgID: "1234", + }) + assert.NoError(t, err) + }) + + t.Run("validate config is able to send a request with owner-id from slug", func(t *testing.T) { + mux := http.NewServeMux() + + mux.HandleFunc("/compile-config-with-defaults", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + reqBody, err := io.ReadAll(r.Body) + assert.NoError(t, err) + + var req CompileConfigRequest + err = json.Unmarshal(reqBody, &req) + assert.NoError(t, err) + assert.Equal(t, "2345", req.Options.OwnerID) + fmt.Fprintf(w, `{"valid":true,"source-yaml":"%s","output-yaml":"%s","errors":[]}`, testYaml, testYaml) + }) + + mux.HandleFunc("/me/collaborations", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `[{"vcs_type":"circleci","slug":"gh/test","id":"2345"}]`) + }) + + svr := httptest.NewServer(mux) + defer svr.Close() + + compiler := New(&settings.Config{Host: svr.URL, HTTPClient: http.DefaultClient}) + + err := compiler.ValidateConfig(ValidateConfigOpts{ + ConfigPath: "testdata/config.yml", + OrgSlug: "gh/test", + }) + assert.NoError(t, err) + }) + + t.Run("validate config is able to send a request with no owner-id after failed collaborations lookup", func(t *testing.T) { + mux := http.NewServeMux() + + mux.HandleFunc("/compile-config-with-defaults", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + reqBody, err := io.ReadAll(r.Body) + assert.NoError(t, err) + + var req CompileConfigRequest + err = json.Unmarshal(reqBody, &req) + assert.NoError(t, err) + assert.Equal(t, "", req.Options.OwnerID) + fmt.Fprintf(w, `{"valid":true,"source-yaml":"%s","output-yaml":"%s","errors":[]}`, testYaml, testYaml) + }) + + mux.HandleFunc("/me/collaborations", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `[{"vcs_type":"circleci","slug":"gh/test","id":"2345"}]`) + }) + + svr := httptest.NewServer(mux) + defer svr.Close() + + compiler := New(&settings.Config{Host: svr.URL, HTTPClient: http.DefaultClient}) + + err := compiler.ValidateConfig(ValidateConfigOpts{ + ConfigPath: "testdata/config.yml", + OrgSlug: "gh/nonexistent", + }) + assert.NoError(t, err) + }) + }) +} diff --git a/config/config.go b/config/config.go index 0abbaa84d..7d17b75e5 100644 --- a/config/config.go +++ b/config/config.go @@ -7,14 +7,47 @@ import ( "os" "github.com/CircleCI-Public/circleci-cli/api/rest" - "github.com/CircleCI-Public/circleci-cli/pipeline" + "github.com/CircleCI-Public/circleci-cli/settings" "github.com/pkg/errors" ) +var ( + defaultHost = "https://circleci.com" + defaultAPIHost = "https://api.circleci.com" + + // Making this the one true source for default config path + DefaultConfigPath = ".circleci/config.yml" +) + +type ConfigCompiler struct { + host string + compileRestClient *rest.Client + collaboratorRestClient *rest.Client +} + +func New(cfg *settings.Config) *ConfigCompiler { + hostValue := getCompileHost(cfg.Host) + configCompiler := &ConfigCompiler{ + host: hostValue, + compileRestClient: rest.NewFromConfig(hostValue, cfg), + collaboratorRestClient: rest.NewFromConfig(cfg.Host, cfg), + } + return configCompiler +} + +func getCompileHost(cfgHost string) string { + if cfgHost != defaultHost { + return cfgHost + } else { + return defaultAPIHost + } +} + type ConfigError struct { Message string `json:"message"` } +// ConfigResponse - the structure of what is returned from the downstream compilation endpoint type ConfigResponse struct { Valid bool `json:"valid"` SourceYaml string `json:"source-yaml"` @@ -22,6 +55,7 @@ type ConfigResponse struct { Errors []ConfigError `json:"errors"` } +// CompileConfigRequest - the structure of the data we send to the downstream compilation service. type CompileConfigRequest struct { ConfigYaml string `json:"config_yaml"` Options Options `json:"options"` @@ -33,55 +67,30 @@ type Options struct { PipelineValues map[string]string `json:"pipeline_values,omitempty"` } -// #nosec -func loadYaml(path string) (string, error) { - var err error - var config []byte - if path == "-" { - config, err = io.ReadAll(os.Stdin) - } else { - config, err = os.ReadFile(path) - } - - if err != nil { - return "", errors.Wrapf(err, "Could not load config file at %s", path) - } - - return string(config), nil -} - // ConfigQuery - attempts to compile or validate a given config file with the // passed in params/values. // If the orgID is passed in, the config-compilation with private orbs should work. -func ConfigQuery( - rest *rest.Client, +func (c *ConfigCompiler) ConfigQuery( configPath string, orgID string, - params pipeline.Parameters, - values pipeline.Values, + params Parameters, + values Values, ) (*ConfigResponse, error) { - configString, err := loadYaml(configPath) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to load yaml config from config path provider: %w", err) } compileRequest := CompileConfigRequest{ ConfigYaml: configString, Options: Options{ - PipelineValues: values, + OwnerID: orgID, + PipelineValues: values, + PipelineParameters: params, }, } - if orgID != "" { - compileRequest.Options.OwnerID = orgID - } - - if len(params) >= 1 { - compileRequest.Options.PipelineParameters = params - } - - req, err := rest.NewRequest( + req, err := c.compileRestClient.NewRequest( "POST", &url.URL{ Path: "compile-config-with-defaults", @@ -89,21 +98,41 @@ func ConfigQuery( compileRequest, ) if err != nil { - return nil, err + return nil, fmt.Errorf("an error occurred creating the request: %w", err) } configCompilationResp := &ConfigResponse{} - statusCode, err := rest.DoRequest(req, configCompilationResp) + statusCode, err := c.compileRestClient.DoRequest(req, configCompilationResp) if err != nil { - return nil, err + if statusCode == 404 { + return nil, errors.New("this version of the CLI does not support your instance of server, please refer to https://github.com/CircleCI-Public/circleci-cli for version compatibility") + } + return nil, fmt.Errorf("config compilation request returned an error: %w", err) } + if statusCode != 200 { return nil, errors.New("unable to validate or compile config") } if len(configCompilationResp.Errors) > 0 { - return nil, errors.New(fmt.Sprintf("config compilation contains errors: %s", configCompilationResp.Errors)) + return nil, fmt.Errorf("config compilation contains errors: %s", configCompilationResp.Errors) } return configCompilationResp, nil } + +func loadYaml(path string) (string, error) { + var err error + var config []byte + if path == "-" { + config, err = io.ReadAll(os.Stdin) + } else { + config, err = os.ReadFile(path) + } + + if err != nil { + return "", errors.Wrapf(err, "Could not load config file at %s", path) + } + + return string(config), nil +} diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 000000000..deeba3c00 --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,140 @@ +package config + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/CircleCI-Public/circleci-cli/settings" + "github.com/stretchr/testify/assert" +) + +func TestCompiler(t *testing.T) { + t.Run("test compiler setup", func(t *testing.T) { + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `[{"vcs_type":"circleci","slug":"gh/test","id":"2345"}]`) + })) + defer svr.Close() + compiler := New(&settings.Config{Host: svr.URL, HTTPClient: http.DefaultClient}) + + t.Run("assert compiler has correct host", func(t *testing.T) { + assert.Equal(t, "http://"+compiler.compileRestClient.BaseURL.Host, svr.URL) + }) + + t.Run("assert compiler has default api host", func(t *testing.T) { + newCompiler := New(&settings.Config{Host: defaultHost, HTTPClient: http.DefaultClient}) + assert.Equal(t, "https://"+newCompiler.compileRestClient.BaseURL.Host, defaultAPIHost) + }) + + t.Run("tests that we correctly get the config api host when the host is not the default one", func(t *testing.T) { + // if the host isn't equal to `https://circleci.com` then this is likely a server instance and + // wont have the api.X.com subdomain so we should instead just respect the host for config commands + host := getCompileHost("test") + assert.Equal(t, host, "test") + + // If the host passed in is the same as the defaultHost 'https://circleci.com' - then we know this is cloud + // and as such should use the `api.circleci.com` subdomain + host = getCompileHost("https://circleci.com") + assert.Equal(t, host, "https://api.circleci.com") + }) + }) + + t.Run("test ConfigQuery", func(t *testing.T) { + t.Run("returns the correct configCompilation response", func(t *testing.T) { + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"valid":true,"source-yaml":"source","output-yaml":"output","errors":[]}`) + })) + defer svr.Close() + compiler := New(&settings.Config{Host: svr.URL, HTTPClient: http.DefaultClient}) + + result, err := compiler.ConfigQuery("testdata/config.yml", "1234", Parameters{}, Values{}) + assert.NoError(t, err) + assert.Equal(t, true, result.Valid) + assert.Equal(t, "output", result.OutputYaml) + assert.Equal(t, "source", result.SourceYaml) + }) + + t.Run("returns error when config file could not be found", func(t *testing.T) { + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"valid":true,"source-yaml":"source","output-yaml":"output","errors":[]}`) + })) + defer svr.Close() + compiler := New(&settings.Config{Host: svr.URL, HTTPClient: http.DefaultClient}) + + _, err := compiler.ConfigQuery("testdata/nonexistent.yml", "1234", Parameters{}, Values{}) + assert.Error(t, err) + assert.Contains(t, err.Error(), "Could not load config file at testdata/nonexistent.yml") + }) + + t.Run("handles 404 status correctly", func(t *testing.T) { + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer svr.Close() + compiler := New(&settings.Config{Host: svr.URL, HTTPClient: http.DefaultClient}) + + _, err := compiler.ConfigQuery("testdata/config.yml", "1234", Parameters{}, Values{}) + assert.Error(t, err) + assert.Contains(t, err.Error(), "this version of the CLI does not support your instance of server") + }) + + t.Run("handles non-200 status correctly", func(t *testing.T) { + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer svr.Close() + compiler := New(&settings.Config{Host: svr.URL, HTTPClient: http.DefaultClient}) + + _, err := compiler.ConfigQuery("testdata/config.yml", "1234", Parameters{}, Values{}) + assert.Error(t, err) + assert.Contains(t, err.Error(), "config compilation request returned an error") + }) + + t.Run("server gets correct information owner ID", func(t *testing.T) { + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + reqBody, err := io.ReadAll(r.Body) + assert.NoError(t, err) + + var req CompileConfigRequest + err = json.Unmarshal(reqBody, &req) + assert.NoError(t, err) + assert.Equal(t, "1234", req.Options.OwnerID) + assert.Equal(t, "test: test\n", req.ConfigYaml) + fmt.Fprintf(w, `{"valid":true,"source-yaml":"source","output-yaml":"output","errors":[]}`) + })) + defer svr.Close() + compiler := New(&settings.Config{Host: svr.URL, HTTPClient: http.DefaultClient}) + + resp, err := compiler.ConfigQuery("testdata/test.yml", "1234", Parameters{}, Values{}) + assert.NoError(t, err) + assert.Equal(t, true, resp.Valid) + assert.Equal(t, "output", resp.OutputYaml) + assert.Equal(t, "source", resp.SourceYaml) + }) + + }) + +} + +func TestLoadYaml(t *testing.T) { + t.Run("tests load yaml", func(t *testing.T) { + expected := `test: test +` + actual, err := loadYaml("testdata/test.yml") + assert.NoError(t, err) + assert.Equal(t, expected, actual) + }) + + t.Run("returns error for non-existent yml file", func(t *testing.T) { + actual, err := loadYaml("testdata/non-existent.yml") + assert.Error(t, err) + assert.Equal(t, "", actual) + }) +} diff --git a/config/deprecated-images.go b/config/deprecated.go similarity index 96% rename from config/deprecated-images.go rename to config/deprecated.go index fcc998fc9..fc8501ae0 100644 --- a/config/deprecated-images.go +++ b/config/deprecated.go @@ -38,8 +38,7 @@ type processedConfig struct { } // Processes the config down to v2.0, then checks image used against the block list -func DeprecatedImageCheck(response *ConfigResponse) error { - +func deprecatedImageCheck(response *ConfigResponse) error { aConfig := processedConfig{} err := yaml.Unmarshal([]byte(response.OutputYaml), &aConfig) if err != nil { diff --git a/config/deprecated_test.go b/config/deprecated_test.go new file mode 100644 index 000000000..8f7c53c00 --- /dev/null +++ b/config/deprecated_test.go @@ -0,0 +1,35 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var TestErrorCase = ` +jobs: + job: + machine: circleci/classic:201710-01 +` + +var TestHappyCase = ` +jobs: + job: + image: non/deprecated +` + +func TestDeprecatedImageCheck(t *testing.T) { + t.Run("happy path - tests deprecated image check works", func(t *testing.T) { + err := deprecatedImageCheck(&ConfigResponse{ + OutputYaml: TestErrorCase, + }) + assert.Error(t, err) + }) + + t.Run("happy path - no error if image used", func(t *testing.T) { + err := deprecatedImageCheck(&ConfigResponse{ + OutputYaml: TestHappyCase, + }) + assert.Nil(t, err) + }) +} diff --git a/pipeline/pipeline.go b/config/pipeline.go similarity index 64% rename from pipeline/pipeline.go rename to config/pipeline.go index 7daefa96b..09375d918 100644 --- a/pipeline/pipeline.go +++ b/config/pipeline.go @@ -1,8 +1,7 @@ -package pipeline +package config import ( "fmt" - "sort" "github.com/CircleCI-Public/circleci-cli/git" ) @@ -45,27 +44,3 @@ func LocalPipelineValues() Values { return vals } - -// TODO: type Parameters map[string]string - -// KeyVal is a data structure specifically for passing pipeline data to GraphQL which doesn't support free-form maps. -type KeyVal struct { - Key string `json:"key"` - Val string `json:"val"` -} - -// PrepareForGraphQL takes a golang homogenous map, and transforms it into a list of keyval pairs, since GraphQL does not support homogenous maps. -func PrepareForGraphQL(kvMap Values) []KeyVal { - // we need to create the slice of KeyVals in a deterministic order for testing purposes - keys := make([]string, 0, len(kvMap)) - for k := range kvMap { - keys = append(keys, k) - } - sort.Strings(keys) - - kvs := make([]KeyVal, 0, len(kvMap)) - for _, k := range keys { - kvs = append(kvs, KeyVal{Key: k, Val: kvMap[k]}) - } - return kvs -} diff --git a/config/testdata/config-no-orb.yml b/config/testdata/config-no-orb.yml new file mode 100644 index 000000000..35f2573f2 --- /dev/null +++ b/config/testdata/config-no-orb.yml @@ -0,0 +1,13 @@ +version: 2.1 + +jobs: + datadog-hello-world: + docker: + - image: cimg/base:stable + steps: + - run: | + echo "doing something really cool" +workflows: + datadog-hello-world: + jobs: + - datadog-hello-world diff --git a/config/testdata/config.yml b/config/testdata/config.yml new file mode 100644 index 000000000..d5f89b865 --- /dev/null +++ b/config/testdata/config.yml @@ -0,0 +1,16 @@ +version: 2.1 + +orbs: + node: circleci/node@5.0.3 + +jobs: + datadog-hello-world: + docker: + - image: cimg/base:stable + steps: + - run: | + echo "doing something really cool" +workflows: + datadog-hello-world: + jobs: + - datadog-hello-world diff --git a/config/testdata/test.yml b/config/testdata/test.yml new file mode 100644 index 000000000..e5239010e --- /dev/null +++ b/config/testdata/test.yml @@ -0,0 +1 @@ +test: test diff --git a/integration_tests/features/circleci_config.feature b/integration_tests/features/circleci_config.feature index 439103b78..3f645f936 100644 --- a/integration_tests/features/circleci_config.feature +++ b/integration_tests/features/circleci_config.feature @@ -56,7 +56,7 @@ Feature: Config checking jobs: - datadog-hello-world """ - When I run `circleci --config-api-host https://k9s.sphereci.com config validate --skip-update-check -c config.yml` + When I run `circleci --host https://k9s.sphereci.com config validate --skip-update-check -c config.yml` Then the exit status should be 0 And the output should contain "Config file at config.yml is valid" diff --git a/local/local.go b/local/local.go index 066eafee7..320789c66 100644 --- a/local/local.go +++ b/local/local.go @@ -11,9 +11,7 @@ import ( "strings" "syscall" - "github.com/CircleCI-Public/circleci-cli/api/rest" "github.com/CircleCI-Public/circleci-cli/config" - "github.com/CircleCI-Public/circleci-cli/pipeline" "github.com/CircleCI-Public/circleci-cli/settings" "github.com/pkg/errors" "github.com/spf13/pflag" @@ -26,20 +24,20 @@ const DefaultConfigPath = ".circleci/config.yml" func Execute(flags *pflag.FlagSet, cfg *settings.Config, args []string) error { var err error var configResponse *config.ConfigResponse - restClient := rest.NewFromConfig(cfg.Host, cfg) - processedArgs, configPath := buildAgentArguments(flags) + compiler := config.New(cfg) + //if no orgId provided use org slug orgID, _ := flags.GetString("org-id") if strings.TrimSpace(orgID) != "" { - configResponse, err = config.ConfigQuery(restClient, configPath, orgID, nil, pipeline.LocalPipelineValues()) + configResponse, err = compiler.ConfigQuery(configPath, orgID, nil, config.LocalPipelineValues()) if err != nil { return err } } else { orgSlug, _ := flags.GetString("org-slug") - configResponse, err = config.ConfigQuery(restClient, configPath, orgSlug, nil, pipeline.LocalPipelineValues()) + configResponse, err = compiler.ConfigQuery(configPath, orgSlug, nil, config.LocalPipelineValues()) if err != nil { return err } diff --git a/settings/settings.go b/settings/settings.go index a612cd26d..36928fac5 100644 --- a/settings/settings.go +++ b/settings/settings.go @@ -36,10 +36,6 @@ type Config struct { GitHubAPI string `yaml:"-"` SkipUpdateCheck bool `yaml:"-"` OrbPublishing OrbPublishingInfo `yaml:"orb_publishing"` - // Represents the API host we want to use for config compilation and validation - // requests - this is typically on the api.circleci.com subdomain for cloud, or the - // same domain for server instances. - ConfigAPIHost string `yaml:"-"` } type OrbPublishingInfo struct { @@ -133,10 +129,6 @@ func (cfg *Config) WriteToDisk() error { func (cfg *Config) LoadFromEnv(prefix string) { if host := ReadFromEnv(prefix, "host"); host != "" { cfg.Host = host - // If the user is a server customer and overwrites the default - // https://circleci.com host - we then have to use this as the host for - // any config compilation or validation requests as opposed to https://api.circleci.com - cfg.ConfigAPIHost = host } if restEndpoint := ReadFromEnv(prefix, "rest_endpoint"); restEndpoint != "" {