From 5469dabd86fcfcf042175aa6e12a1a2f87d61c81 Mon Sep 17 00:00:00 2001 From: Elliot Forbes Date: Thu, 9 Mar 2023 14:16:27 +0000 Subject: [PATCH] First pass at removing the api-service dependency for config compilation and validation --- api/api.go | 119 ------ api/rest/client.go | 44 +- api/rest/client_test.go | 37 ++ cmd/config.go | 73 +++- cmd/config_test.go | 398 +++++------------- cmd/root.go | 7 + config/config.go | 109 +++++ {cmd => config}/deprecated-images.go | 6 +- .../features/circleci_config.feature | 203 +++++++++ local/local.go | 12 +- settings/settings.go | 8 + 11 files changed, 578 insertions(+), 438 deletions(-) create mode 100644 config/config.go rename {cmd => config}/deprecated-images.go (93%) diff --git a/api/api.go b/api/api.go index 75fd208a9..ff6ca667d 100644 --- a/api/api.go +++ b/api/api.go @@ -11,7 +11,6 @@ import ( "strings" "github.com/CircleCI-Public/circleci-cli/api/graphql" - "github.com/CircleCI-Public/circleci-cli/pipeline" "github.com/CircleCI-Public/circleci-cli/references" "github.com/CircleCI-Public/circleci-cli/settings" "github.com/Masterminds/semver" @@ -513,124 +512,6 @@ func WhoamiQuery(cl *graphql.Client) (*WhoamiResponse, error) { return &response, nil } -// ConfigQueryLegacy calls the GQL API to validate and process config with the legacy orgSlug -func ConfigQueryLegacy(cl *graphql.Client, configPath string, orgSlug string, params pipeline.Parameters, values pipeline.Values) (*ConfigResponse, error) { - var response BuildConfigResponse - var query string - config, err := loadYaml(configPath) - if err != nil { - return nil, err - } - // GraphQL isn't forwards-compatible, so we are unusually selective here about - // passing only non-empty fields on to the API, to minimize user impact if the - // backend is out of date. - var fieldAddendums string - if orgSlug != "" { - fieldAddendums += ", orgSlug: $orgSlug" - } - if len(params) > 0 { - fieldAddendums += ", pipelineParametersJson: $pipelineParametersJson" - } - query = fmt.Sprintf( - `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgSlug: String) { - buildConfig(configYaml: $config, pipelineValues: $pipelineValues%s) { - valid, - errors { message }, - sourceYaml, - outputYaml - } - }`, - fieldAddendums) - - request := graphql.NewRequest(query) - request.SetToken(cl.Token) - request.Var("config", config) - - if values != nil { - request.Var("pipelineValues", pipeline.PrepareForGraphQL(values)) - } - if params != nil { - pipelineParameters, err := json.Marshal(params) - if err != nil { - return nil, fmt.Errorf("unable to serialize pipeline values: %s", err.Error()) - } - request.Var("pipelineParametersJson", string(pipelineParameters)) - } - - if orgSlug != "" { - request.Var("orgSlug", orgSlug) - } - - err = cl.Run(request, &response) - if err != nil { - return nil, errors.Wrap(err, "Unable to validate config") - } - if len(response.BuildConfig.ConfigResponse.Errors) > 0 { - return nil, &response.BuildConfig.ConfigResponse.Errors - } - - return &response.BuildConfig.ConfigResponse, nil -} - -// ConfigQuery calls the GQL API to validate and process config with the org id -func ConfigQuery(cl *graphql.Client, configPath string, orgId string, params pipeline.Parameters, values pipeline.Values) (*ConfigResponse, error) { - var response BuildConfigResponse - var query string - config, err := loadYaml(configPath) - if err != nil { - return nil, err - } - // GraphQL isn't forwards-compatible, so we are unusually selective here about - // passing only non-empty fields on to the API, to minimize user impact if the - // backend is out of date. - var fieldAddendums string - if orgId != "" { - fieldAddendums += ", orgId: $orgId" - } - if len(params) > 0 { - fieldAddendums += ", pipelineParametersJson: $pipelineParametersJson" - } - query = fmt.Sprintf( - `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgId: UUID!) { - buildConfig(configYaml: $config, pipelineValues: $pipelineValues%s) { - valid, - errors { message }, - sourceYaml, - outputYaml - } - }`, - fieldAddendums) - - request := graphql.NewRequest(query) - request.SetToken(cl.Token) - request.Var("config", config) - - if values != nil { - request.Var("pipelineValues", pipeline.PrepareForGraphQL(values)) - } - if params != nil { - pipelineParameters, err := json.Marshal(params) - if err != nil { - return nil, fmt.Errorf("unable to serialize pipeline values: %s", err.Error()) - } - request.Var("pipelineParametersJson", string(pipelineParameters)) - } - - if orgId != "" { - request.Var("orgId", orgId) - } - - err = cl.Run(request, &response) - if err != nil { - return nil, errors.Wrap(err, "Unable to validate config") - } - if len(response.BuildConfig.ConfigResponse.Errors) > 0 { - return nil, &response.BuildConfig.ConfigResponse.Errors - } - - return &response.BuildConfig.ConfigResponse, nil -} - // OrbQuery validated and processes an orb. func OrbQuery(cl *graphql.Client, configPath string) (*ConfigResponse, error) { var response OrbConfigResponse diff --git a/api/rest/client.go b/api/rest/client.go index d745b887e..918f64c2e 100644 --- a/api/rest/client.go +++ b/api/rest/client.go @@ -17,7 +17,12 @@ import ( ) type Client struct { - baseURL *url.URL + baseURL *url.URL + // The config api host differs for both cloud and server setups. + // For cloud, the base domain will be https://api.circleci.com + // for server, this should match the host as we don't have the same + // api subdomain setup + apiURL *url.URL circleToken string client *http.Client } @@ -29,13 +34,15 @@ func New(host string, config *settings.Config) *Client { endpoint += "/" } - u, _ := url.Parse(host) + baseURL, _ := url.Parse(host) + apiURL, _ := url.Parse(config.ConfigAPIHost) client := config.HTTPClient client.Timeout = 10 * time.Second return &Client{ - baseURL: u.ResolveReference(&url.URL{Path: endpoint}), + apiURL: apiURL.ResolveReference(&url.URL{Path: endpoint}), + baseURL: baseURL.ResolveReference(&url.URL{Path: endpoint}), circleToken: config.Token, client: client, } @@ -57,6 +64,34 @@ func (c *Client) NewRequest(method string, u *url.URL, payload interface{}) (req return nil, err } + c.enrichRequestHeaders(req, payload) + return req, nil +} + +// NewAPIRequest - similar to NewRequest except it uses the apiURL as the base URL. +func (c *Client) NewAPIRequest(method string, u *url.URL, payload interface{}) (req *http.Request, err error) { + var r io.Reader + if payload != nil { + buf := &bytes.Buffer{} + r = buf + err = json.NewEncoder(buf).Encode(payload) + if err != nil { + fmt.Printf("failed to encode payload as json: %s\n", err.Error()) + return nil, err + } + } + + req, err = http.NewRequest(method, c.apiURL.ResolveReference(u).String(), r) + if err != nil { + fmt.Printf("failed to create new http request: %s\n", err.Error()) + return nil, err + } + + c.enrichRequestHeaders(req, payload) + return req, nil +} + +func (c *Client) enrichRequestHeaders(req *http.Request, payload interface{}) { req.Header.Set("Circle-Token", c.circleToken) req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", version.UserAgent()) @@ -67,12 +102,12 @@ func (c *Client) NewRequest(method string, u *url.URL, payload interface{}) (req if payload != nil { req.Header.Set("Content-Type", "application/json") } - return req, nil } func (c *Client) DoRequest(req *http.Request, resp interface{}) (statusCode int, err error) { httpResp, err := c.client.Do(req) if err != nil { + fmt.Printf("failed to make http request: %s\n", err.Error()) return 0, err } defer httpResp.Body.Close() @@ -83,6 +118,7 @@ func (c *Client) DoRequest(req *http.Request, resp interface{}) (statusCode int, }{} err = json.NewDecoder(httpResp.Body).Decode(&httpError) if err != nil { + fmt.Printf("failed to decode body: %s", err.Error()) return httpResp.StatusCode, err } return httpResp.StatusCode, &HTTPError{Code: httpResp.StatusCode, Message: httpError.Message} diff --git a/api/rest/client_test.go b/api/rest/client_test.go index a45d4bc13..f66281f7b 100644 --- a/api/rest/client_test.go +++ b/api/rest/client_test.go @@ -118,6 +118,43 @@ func TestClient_DoRequest(t *testing.T) { }) } +func TestAPIRequest(t *testing.T) { + fix := &fixture{} + c, cleanup := fix.Run(http.StatusCreated, `{"key": "value"}`) + defer cleanup() + + t.Run("test new api request sets the default headers", func(t *testing.T) { + req, err := c.NewAPIRequest("GET", &url.URL{}, struct{}{}) + assert.NilError(t, err) + + assert.Equal(t, req.Header.Get("User-Agent"), "circleci-cli/0.0.0-dev+dirty-local-tree (source)") + assert.Equal(t, req.Header.Get("Circle-Token"), c.circleToken) + assert.Equal(t, req.Header.Get("Accept"), "application/json") + }) + + type testPayload struct { + Message string + } + + t.Run("test new api request sets the default headers", func(t *testing.T) { + req, err := c.NewAPIRequest("GET", &url.URL{}, testPayload{Message: "hello"}) + assert.NilError(t, err) + + assert.Equal(t, req.Header.Get("Circleci-Cli-Command"), "") + assert.Equal(t, req.Header.Get("Content-Type"), "application/json") + }) + + t.Run("test new api request doesn't set content-type with empty payload", func(t *testing.T) { + req, err := c.NewAPIRequest("GET", &url.URL{}, nil) + assert.NilError(t, err) + + assert.Equal(t, req.Header.Get("Circleci-Cli-Command"), "") + if req.Header.Get("Content-Type") != "" { + t.Fail() + } + }) +} + type fixture struct { mu sync.Mutex url url.URL diff --git a/cmd/config.go b/cmd/config.go index 5ae76fcde..7d05200be 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -3,10 +3,11 @@ package cmd import ( "fmt" "io/ioutil" + "net/url" "strings" - "github.com/CircleCI-Public/circleci-cli/api" - "github.com/CircleCI-Public/circleci-cli/api/graphql" + "github.com/CircleCI-Public/circleci-cli/api/rest" + "github.com/CircleCI-Public/circleci-cli/config" "github.com/CircleCI-Public/circleci-cli/filetree" "github.com/CircleCI-Public/circleci-cli/local" "github.com/CircleCI-Public/circleci-cli/pipeline" @@ -18,9 +19,13 @@ import ( "gopkg.in/yaml.v3" ) +var ( + CollaborationsPath = "me/collaborations" +) + type configOptions struct { cfg *settings.Config - cl *graphql.Client + rest *rest.Client args []string } @@ -63,7 +68,7 @@ func newConfigCommand(config *settings.Config) *cobra.Command { Short: "Check that the config file is well formed.", PreRun: func(cmd *cobra.Command, args []string) { opts.args = args - opts.cl = graphql.NewClient(config.HTTPClient, config.Host, config.Endpoint, config.Token, config.Debug) + opts.rest = rest.New(config.Host, config) }, RunE: func(cmd *cobra.Command, _ []string) error { return validateConfig(opts, cmd.Flags()) @@ -85,7 +90,7 @@ func newConfigCommand(config *settings.Config) *cobra.Command { Short: "Validate config and display expanded configuration.", PreRun: func(cmd *cobra.Command, args []string) { opts.args = args - opts.cl = graphql.NewClient(config.HTTPClient, config.Host, config.Endpoint, config.Token, config.Debug) + opts.rest = rest.New(config.Host, config) }, RunE: func(cmd *cobra.Command, _ []string) error { return processConfig(opts, cmd.Flags()) @@ -97,6 +102,7 @@ func newConfigCommand(config *settings.Config) *cobra.Command { processCommand.Flags().StringP("org-slug", "o", "", "organization slug (for example: github/example-org), used when a config depends on private orbs belonging to that org") processCommand.Flags().String("org-id", "", "organization id used when a config depends on private orbs belonging to that org") processCommand.Flags().StringP("pipeline-parameters", "", "", "YAML/JSON map of pipeline parameters, accepts either YAML/JSON directly or file path (for example: my-params.yml)") + processCommand.Flags().StringP("circleci-api-host", "", "", "the api-host you want to use for config processing and validation") migrateCommand := &cobra.Command{ Use: "migrate", @@ -125,7 +131,7 @@ func newConfigCommand(config *settings.Config) *cobra.Command { // The arg is actually optional, in order to support compatibility with the --path flag. func validateConfig(opts configOptions, flags *pflag.FlagSet) error { var err error - var response *api.ConfigResponse + var response *config.ConfigResponse path := local.DefaultConfigPath // First, set the path to configPath set by --path flag for compatibility if configPath != "" { @@ -142,15 +148,21 @@ func validateConfig(opts configOptions, flags *pflag.FlagSet) error { fmt.Println("Validating config with following values") printValues(values) - orgID, _ := flags.GetString("org-id") + var orgID string + orgID, _ = flags.GetString("org-id") if strings.TrimSpace(orgID) != "" { - response, err = api.ConfigQuery(opts.cl, path, orgID, nil, values) + response, err = config.ConfigQuery(opts.rest, path, orgID, nil, pipeline.LocalPipelineValues()) if err != nil { return err } } else { orgSlug, _ := flags.GetString("org-slug") - response, err = api.ConfigQueryLegacy(opts.cl, path, orgSlug, nil, values) + orgs, err := GetOrgCollaborations(opts.rest) + if err != nil { + fmt.Println(err.Error()) + } + orgID = GetOrgIdFromSlug(orgSlug, orgs) + response, err = config.ConfigQuery(opts.rest, path, orgID, nil, pipeline.LocalPipelineValues()) if err != nil { return err } @@ -160,7 +172,7 @@ func validateConfig(opts configOptions, flags *pflag.FlagSet) error { // link here to blog post when available // returns an error if a deprecated image is used if !ignoreDeprecatedImages { - err := deprecatedImageCheck(response) + err := config.DeprecatedImageCheck(response) if err != nil { return err } @@ -177,7 +189,7 @@ func validateConfig(opts configOptions, flags *pflag.FlagSet) error { func processConfig(opts configOptions, flags *pflag.FlagSet) error { paramsYaml, _ := flags.GetString("pipeline-parameters") - var response *api.ConfigResponse + var response *config.ConfigResponse var params pipeline.Parameters var err error @@ -202,13 +214,18 @@ func processConfig(opts configOptions, flags *pflag.FlagSet) error { orgID, _ := flags.GetString("org-id") if strings.TrimSpace(orgID) != "" { - response, err = api.ConfigQuery(opts.cl, opts.args[0], orgID, params, values) + response, err = config.ConfigQuery(opts.rest, opts.args[0], orgID, params, values) if err != nil { return err } } else { orgSlug, _ := flags.GetString("org-slug") - response, err = api.ConfigQueryLegacy(opts.cl, opts.args[0], orgSlug, params, values) + orgs, err := GetOrgCollaborations(opts.rest) + if err != nil { + fmt.Println(err.Error()) + } + orgID = GetOrgIdFromSlug(orgSlug, orgs) + response, err = config.ConfigQuery(opts.rest, opts.args[0], orgID, params, values) if err != nil { return err } @@ -241,3 +258,33 @@ func printValues(values pipeline.Values) { fmt.Printf("\t%s:\t%s", key, value) } } + +type CollaborationResult struct { + VcsTye string `json:"vcs_type"` + OrgSlug string `json:"slug"` + OrgName string `json:"name"` + OrgId string `json:"id"` + AvatarUrl string `json:"avatar_url"` +} + +// GetOrgCollaborations - fetches all the collaborations for a given user. +func GetOrgCollaborations(client *rest.Client) ([]CollaborationResult, error) { + req, err := client.NewRequest("GET", &url.URL{Path: CollaborationsPath}, nil) + if err != nil { + return nil, err + } + + var resp []CollaborationResult + _, err = client.DoRequest(req, &resp) + return resp, err +} + +// GetOrgIdFromSlug - converts a slug into an orgID. +func GetOrgIdFromSlug(slug string, collaborations []CollaborationResult) string { + for _, v := range collaborations { + if v.OrgSlug == slug { + return v.OrgId + } + } + return "" +} diff --git a/cmd/config_test.go b/cmd/config_test.go index 85b6e8632..d0ed9016d 100644 --- a/cmd/config_test.go +++ b/cmd/config_test.go @@ -1,20 +1,13 @@ package cmd_test import ( - "encoding/json" "fmt" - "io" - "net/http" "os/exec" "path/filepath" - "time" - "github.com/CircleCI-Public/circleci-cli/api/graphql" "github.com/CircleCI-Public/circleci-cli/clitest" - "github.com/CircleCI-Public/circleci-cli/pipeline" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/onsi/gomega/gbytes" "github.com/onsi/gomega/gexec" "gotest.tools/v3/golden" ) @@ -25,7 +18,6 @@ var _ = Describe("Config", func() { command *exec.Cmd results []byte tempSettings *clitest.TempSettings - token string = "testtoken" ) BeforeEach(func() { @@ -95,6 +87,112 @@ var _ = Describe("Config", func() { }) }) + It("packs all YAML contents as expected", func() { + command = exec.Command(pathCLI, + "config", "pack", + "--skip-update-check", + "testdata/hugo-pack/.circleci") + results = golden.Get(GinkgoT(), filepath.FromSlash("hugo-pack/result.yml")) + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + session.Wait() + Expect(err).ShouldNot(HaveOccurred()) + Eventually(session.Err.Contents()).Should(BeEmpty()) + Eventually(session.Out.Contents()).Should(MatchYAML(results)) + Eventually(session).Should(gexec.Exit(0)) + }) + + It("given a .circleci folder with config.yml and local orb, packs all YAML contents as expected", func() { + command = exec.Command(pathCLI, + "config", "pack", + "--skip-update-check", + "testdata/hugo-pack/.circleci") + results = golden.Get(GinkgoT(), filepath.FromSlash("hugo-pack/result.yml")) + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + session.Wait() + Expect(err).ShouldNot(HaveOccurred()) + Eventually(session.Err.Contents()).Should(BeEmpty()) + Eventually(session.Out.Contents()).Should(MatchYAML(results)) + Eventually(session).Should(gexec.Exit(0)) + }) + + It("given a local orbs folder with mixed inline and local commands, jobs, etc, packs all YAML contents as expected", func() { + var path string = "nested-orbs-and-local-commands-etc" + command = exec.Command(pathCLI, + "config", "pack", + "--skip-update-check", + filepath.Join("testdata", path, "test")) + results = golden.Get(GinkgoT(), filepath.FromSlash(fmt.Sprintf("%s/result.yml", path))) + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + session.Wait() + Expect(err).ShouldNot(HaveOccurred()) + Eventually(session.Err.Contents()).Should(BeEmpty()) + Eventually(session.Out.Contents()).Should(MatchYAML(results)) + Eventually(session).Should(gexec.Exit(0)) + }) + + It("returns an error when validating a config", func() { + var path string = "nested-orbs-and-local-commands-etc" + command = exec.Command(pathCLI, + "config", "pack", + "--skip-update-check", + filepath.Join("testdata", path, "test")) + results = golden.Get(GinkgoT(), filepath.FromSlash(fmt.Sprintf("%s/result.yml", path))) + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + session.Wait() + Expect(err).ShouldNot(HaveOccurred()) + Eventually(session.Err.Contents()).Should(BeEmpty()) + Eventually(session.Out.Contents()).Should(MatchYAML(results)) + Eventually(session).Should(gexec.Exit(0)) + }) + + It("packs successfully given an orb containing local executors and commands in folder", func() { + command = exec.Command(pathCLI, + "config", "pack", + "--skip-update-check", + "testdata/myorb/test") + + results = golden.Get(GinkgoT(), filepath.FromSlash("myorb/result.yml")) + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + session.Wait() + Expect(err).ShouldNot(HaveOccurred()) + Eventually(session.Err.Contents()).Should(BeEmpty()) + Eventually(session.Out.Contents()).Should(MatchYAML(results)) + Eventually(session).Should(gexec.Exit(0)) + }) + + It("packs as expected given a large nested config including rails orbs", func() { + var path string = "test-with-large-nested-rails-orb" + command = exec.Command(pathCLI, + "config", "pack", + "--skip-update-check", + filepath.Join("testdata", path, "test")) + results = golden.Get(GinkgoT(), filepath.FromSlash(fmt.Sprintf("%s/result.yml", path))) + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + session.Wait() + Expect(err).ShouldNot(HaveOccurred()) + Eventually(session.Err.Contents()).Should(BeEmpty()) + Eventually(session.Out.Contents()).Should(MatchYAML(results)) + Eventually(session).Should(gexec.Exit(0)) + }) + + It("prints an error given a config which is a list and not a map", func() { + config := clitest.OpenTmpFile(filepath.Join(tempSettings.Home, "myorb"), "config.yaml") + command = exec.Command(pathCLI, + "config", "pack", + "--skip-update-check", + config.RootDir, + ) + config.Write([]byte(`[]`)) + + expected := fmt.Sprintf("Error: Failed trying to marshal the tree to YAML : expected a map, got a `[]interface {}` which is not supported at this time for \"%s\"\n", config.Path) + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).ShouldNot(HaveOccurred()) + stderr := session.Wait().Err.Contents() + Expect(string(stderr)).To(Equal(expected)) + Eventually(session).Should(clitest.ShouldFail()) + config.Close() + }) + Describe("with a large nested config including rails orb", func() { BeforeEach(func() { var path string = "test-with-large-nested-rails-orb" @@ -146,289 +244,5 @@ var _ = Describe("Config", func() { Eventually(session).Should(clitest.ShouldFail()) }) }) - - Describe("validating configs", func() { - config := "version: 2.1" - var expReq string - - BeforeEach(func() { - command = exec.Command(pathCLI, - "config", "validate", - "--skip-update-check", - "--token", token, - "--host", tempSettings.TestServer.URL(), - "-", - ) - - stdin, err := command.StdinPipe() - Expect(err).ToNot(HaveOccurred()) - _, err = io.WriteString(stdin, config) - Expect(err).ToNot(HaveOccurred()) - stdin.Close() - - query := `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgSlug: String) { - buildConfig(configYaml: $config, pipelineValues: $pipelineValues) { - valid, - errors { message }, - sourceYaml, - outputYaml - } - }` - - r := graphql.NewRequest(query) - r.SetToken(token) - r.Variables["config"] = config - r.Variables["pipelineValues"] = pipeline.PrepareForGraphQL(pipeline.LocalPipelineValues()) - - req, err := r.Encode() - Expect(err).ShouldNot(HaveOccurred()) - expReq = req.String() - }) - - It("returns an error when validating a config", func() { - expResp := `{ - "buildConfig": { - "errors": [ - {"message": "error1"} - ] - } - }` - - tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ - Status: http.StatusOK, - Request: expReq, - Response: expResp, - }) - - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Err, time.Second*3).Should(gbytes.Say("Error: error1")) - Eventually(session).Should(clitest.ShouldFail()) - }) - - It("returns successfully when validating a config", func() { - expResp := `{ - "buildConfig": {} - }` - - tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ - Status: http.StatusOK, - Request: expReq, - Response: expResp, - }) - - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Out, time.Second*3).Should(gbytes.Say("Config input is valid.")) - Eventually(session).Should(gexec.Exit(0)) - }) - }) - - Describe("validating configs with pipeline parameters", func() { - config := "version: 2.1" - var expReq string - - BeforeEach(func() { - command = exec.Command(pathCLI, - "config", "process", - "--skip-update-check", - "--token", token, - "--host", tempSettings.TestServer.URL(), - "--pipeline-parameters", `{"foo": "test", "bar": true, "baz": 10}`, - "-", - ) - - stdin, err := command.StdinPipe() - Expect(err).ToNot(HaveOccurred()) - _, err = io.WriteString(stdin, config) - Expect(err).ToNot(HaveOccurred()) - stdin.Close() - - query := `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgSlug: String) { - buildConfig(configYaml: $config, pipelineValues: $pipelineValues, pipelineParametersJson: $pipelineParametersJson) { - valid, - errors { message }, - sourceYaml, - outputYaml - } - }` - - r := graphql.NewRequest(query) - r.SetToken(token) - r.Variables["config"] = config - r.Variables["pipelineValues"] = pipeline.PrepareForGraphQL(pipeline.LocalPipelineValues()) - - pipelineParams, err := json.Marshal(pipeline.Parameters{ - "foo": "test", - "bar": true, - "baz": 10, - }) - Expect(err).ToNot(HaveOccurred()) - r.Variables["pipelineParametersJson"] = string(pipelineParams) - - req, err := r.Encode() - Expect(err).ShouldNot(HaveOccurred()) - expReq = req.String() - }) - - It("returns successfully when validating a config", func() { - expResp := `{ - "buildConfig": {} - }` - - tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ - Status: http.StatusOK, - Request: expReq, - Response: expResp, - }) - - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session).Should(gexec.Exit(0)) - }) - }) - - Describe("validating configs with private orbs", func() { - config := "version: 2.1" - orgId := "bb604b45-b6b0-4b81-ad80-796f15eddf87" - var expReq string - - BeforeEach(func() { - command = exec.Command(pathCLI, - "config", "validate", - "--skip-update-check", - "--token", token, - "--host", tempSettings.TestServer.URL(), - "--org-id", orgId, - "-", - ) - - stdin, err := command.StdinPipe() - Expect(err).ToNot(HaveOccurred()) - _, err = io.WriteString(stdin, config) - Expect(err).ToNot(HaveOccurred()) - stdin.Close() - - query := `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgId: UUID!) { - buildConfig(configYaml: $config, pipelineValues: $pipelineValues, orgId: $orgId) { - valid, - errors { message }, - sourceYaml, - outputYaml - } - }` - - r := graphql.NewRequest(query) - r.SetToken(token) - r.Variables["config"] = config - r.Variables["orgId"] = orgId - r.Variables["pipelineValues"] = pipeline.PrepareForGraphQL(pipeline.LocalPipelineValues()) - - req, err := r.Encode() - Expect(err).ShouldNot(HaveOccurred()) - expReq = req.String() - }) - - It("returns an error when validating a config with a private orb", func() { - expResp := `{ - "buildConfig": { - "errors": [ - {"message": "permission denied"} - ] - } - }` - - tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ - Status: http.StatusOK, - Request: expReq, - Response: expResp, - }) - - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Err, time.Second*3).Should(gbytes.Say("Error: permission denied")) - Eventually(session).Should(clitest.ShouldFail()) - }) - }) - - Describe("validating configs with private orbs Legacy", func() { - config := "version: 2.1" - orgSlug := "circleci" - var expReq string - - BeforeEach(func() { - command = exec.Command(pathCLI, - "config", "validate", - "--skip-update-check", - "--token", token, - "--host", tempSettings.TestServer.URL(), - "--org-slug", orgSlug, - "-", - ) - - stdin, err := command.StdinPipe() - Expect(err).ToNot(HaveOccurred()) - _, err = io.WriteString(stdin, config) - Expect(err).ToNot(HaveOccurred()) - stdin.Close() - - query := `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgSlug: String) { - buildConfig(configYaml: $config, pipelineValues: $pipelineValues, orgSlug: $orgSlug) { - valid, - errors { message }, - sourceYaml, - outputYaml - } - }` - - r := graphql.NewRequest(query) - r.SetToken(token) - r.Variables["config"] = config - r.Variables["orgSlug"] = orgSlug - r.Variables["pipelineValues"] = pipeline.PrepareForGraphQL(pipeline.LocalPipelineValues()) - - req, err := r.Encode() - Expect(err).ShouldNot(HaveOccurred()) - expReq = req.String() - }) - - It("returns an error when validating a config with a private orb", func() { - expResp := `{ - "buildConfig": { - "errors": [ - {"message": "permission denied"} - ] - } - }` - - tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ - Status: http.StatusOK, - Request: expReq, - Response: expResp, - }) - - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Err, time.Second*3).Should(gbytes.Say("Error: permission denied")) - Eventually(session).Should(clitest.ShouldFail()) - }) - - It("returns successfully when validating a config with private orbs", func() { - expResp := `{ - "buildConfig": {} - }` - - tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ - Status: http.StatusOK, - Request: expReq, - Response: expResp, - }) - - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Out, time.Second*3).Should(gbytes.Say("Config input is valid.")) - Eventually(session).Should(gexec.Exit(0)) - }) - }) }) }) diff --git a/cmd/root.go b/cmd/root.go index 6a636cd0c..7f7ed4ec1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -21,6 +21,7 @@ import ( var defaultEndpoint = "graphql-unstable" var defaultHost = "https://circleci.com" +var defaultAPIHost = "https://api.circleci.com" var defaultRestEndpoint = "api/v2" var trueString = "true" @@ -103,6 +104,11 @@ func MakeCommands() *cobra.Command { RestEndpoint: defaultRestEndpoint, Endpoint: defaultEndpoint, GitHubAPI: "https://api.github.com/", + // The config api host differs for both cloud and server setups. + // For cloud, the base domain will be https://api.circleci.com + // for server, this should match the host as we don't have the same + // api subdomain setup + ConfigAPIHost: defaultAPIHost, } if err := rootOptions.Load(); err != nil { @@ -174,6 +180,7 @@ func MakeCommands() *cobra.Command { flags.StringVar(&rootOptions.Host, "host", rootOptions.Host, "URL to your CircleCI host, also CIRCLECI_CLI_HOST") flags.StringVar(&rootOptions.Endpoint, "endpoint", rootOptions.Endpoint, "URI to your CircleCI GraphQL API endpoint") flags.StringVar(&rootOptions.GitHubAPI, "github-api", "https://api.github.com/", "Change the default endpoint to GitHub API for retrieving updates") + flags.StringVar(&rootOptions.ConfigAPIHost, "config-api-host", "https://api.circleci.com", "Change the default endpoint for the config api host") flags.BoolVar(&rootOptions.SkipUpdateCheck, "skip-update-check", skipUpdateByDefault(), "Skip the check for updates check run before every command.") hidden := []string{"github-api", "debug", "endpoint"} diff --git a/config/config.go b/config/config.go new file mode 100644 index 000000000..8605690a6 --- /dev/null +++ b/config/config.go @@ -0,0 +1,109 @@ +package config + +import ( + "fmt" + "io/ioutil" + "net/url" + "os" + + "github.com/CircleCI-Public/circleci-cli/api/rest" + "github.com/CircleCI-Public/circleci-cli/pipeline" + "github.com/pkg/errors" +) + +type ConfigError struct { + Message string `json:"message"` +} + +type ConfigResponse struct { + Valid bool `json:"valid"` + SourceYaml string `json:"source-yaml"` + OutputYaml string `json:"output-yaml"` + Errors []ConfigError `json:"errors"` +} + +type CompileConfigRequest struct { + ConfigYaml string `json:"config_yaml"` + Options Options `json:"options"` +} + +type Options struct { + OwnerID string `json:"owner-id,omitempty"` + PipelineParameters map[string]interface{} `json:"pipeline_parameters,omitempty"` + PipelineValues map[string]string `json:"pipeline_values,omitempty"` +} + +// #nosec +func loadYaml(path string) (string, error) { + var err error + var config []byte + if path == "-" { + config, err = ioutil.ReadAll(os.Stdin) + } else { + config, err = ioutil.ReadFile(path) + } + + if err != nil { + return "", errors.Wrapf(err, "Could not load config file at %s", path) + } + + return string(config), nil +} + +// ConfigQuery - attempts to compile or validate a given config file with the +// passed in params/values. +// If the orgID is passed in, the config-compilation with private orbs should work. +func ConfigQuery( + rest *rest.Client, + configPath string, + orgID string, + params pipeline.Parameters, + values pipeline.Values, +) (*ConfigResponse, error) { + + configString, err := loadYaml(configPath) + if err != nil { + return nil, err + } + + compileRequest := CompileConfigRequest{ + ConfigYaml: configString, + Options: Options{ + PipelineValues: values, + }, + } + + if orgID != "" { + compileRequest.Options.OwnerID = orgID + } + + if len(params) >= 1 { + compileRequest.Options.PipelineParameters = params + } + + req, err := rest.NewAPIRequest( + "POST", + &url.URL{ + Path: "compile-config-with-defaults", + }, + compileRequest, + ) + if err != nil { + return nil, err + } + + configCompilationResp := &ConfigResponse{} + statusCode, err := rest.DoRequest(req, configCompilationResp) + if err != nil { + return nil, err + } + if statusCode != 200 { + return nil, errors.New("non 200 status code") + } + + if len(configCompilationResp.Errors) > 0 { + return nil, errors.New(fmt.Sprintf("config compilation contains errors: %s", configCompilationResp.Errors)) + } + + return configCompilationResp, nil +} diff --git a/cmd/deprecated-images.go b/config/deprecated-images.go similarity index 93% rename from cmd/deprecated-images.go rename to config/deprecated-images.go index 607edfc88..fcc998fc9 100644 --- a/cmd/deprecated-images.go +++ b/config/deprecated-images.go @@ -1,8 +1,6 @@ -package cmd +package config import ( - "github.com/CircleCI-Public/circleci-cli/api" - "github.com/pkg/errors" "gopkg.in/yaml.v3" ) @@ -40,7 +38,7 @@ type processedConfig struct { } // Processes the config down to v2.0, then checks image used against the block list -func deprecatedImageCheck(response *api.ConfigResponse) error { +func DeprecatedImageCheck(response *ConfigResponse) error { aConfig := processedConfig{} err := yaml.Unmarshal([]byte(response.OutputYaml), &aConfig) diff --git a/integration_tests/features/circleci_config.feature b/integration_tests/features/circleci_config.feature index d9085cc2b..439103b78 100644 --- a/integration_tests/features/circleci_config.feature +++ b/integration_tests/features/circleci_config.feature @@ -15,6 +15,209 @@ Feature: Config checking Then the exit status should be 0 And the output should contain "Config file at config.yml is valid." + Scenario: Checking a valid config file with an orb + Given a file named "config.yml" with: + """ + version: 2.1 + + orbs: + node: circleci/node@5.0.3 + + jobs: + datadog-hello-world: + docker: + - image: cimg/base:stable + steps: + - run: | + echo "doing something really cool" + workflows: + datadog-hello-world: + jobs: + - datadog-hello-world + """ + When I run `circleci config validate --skip-update-check -c config.yml` + Then the exit status should be 0 + And the output should contain "Config file at config.yml is valid" + + Scenario: Checking a valid config against the k9s server + Given a file named "config.yml" with: + """ + version: 2.1 + + jobs: + datadog-hello-world: + docker: + - image: cimg/base:stable + steps: + - run: | + echo "doing something really cool" + workflows: + datadog-hello-world: + jobs: + - datadog-hello-world + """ + When I run `circleci --config-api-host https://k9s.sphereci.com config validate --skip-update-check -c config.yml` + Then the exit status should be 0 + And the output should contain "Config file at config.yml is valid" + + Scenario: Checking a valid config file with an orb + Given a file named "config.yml" with: + """ + version: 2.1 + + orbs: + node: circleci/node@5.0.3 + + jobs: + datadog-hello-world: + docker: + - image: cimg/base:stable + steps: + - run: | + echo "doing something really cool" + workflows: + datadog-hello-world: + jobs: + - datadog-hello-world + """ + When I run `circleci config validate --skip-update-check -c config.yml` + Then the exit status should be 0 + And the output should contain "Config file at config.yml is valid" + + Scenario: Checking a valid config file with a private org + Given a file named "config.yml" with: + """ + version: 2.1 + + orbs: + node: circleci/node@5.0.3 + + jobs: + datadog-hello-world: + docker: + - image: cimg/base:stable + steps: + - run: | + echo "doing something really cool" + workflows: + datadog-hello-world: + jobs: + - datadog-hello-world + """ + When I run `circleci config validate --skip-update-check --org-id bb604b45-b6b0-4b81-ad80-796f15eddf87 -c config.yml` + Then the output should contain "Config file at config.yml is valid" + And the exit status should be 0 + + Scenario: Checking a valid config file with a non-existant orb + Given a file named "config.yml" with: + """ + version: 2.1 + + orbs: + node: circleci/doesnt-exist@5.0.3 + + jobs: + datadog-hello-world: + docker: + - image: cimg/base:stable + steps: + - run: | + echo "doing something really cool" + workflows: + datadog-hello-world: + jobs: + - datadog-hello-world + """ + When I run `circleci config validate --skip-update-check -c config.yml` + Then the exit status should be 255 + And the output should contain "config compilation contains errors" + + Scenario: Checking a valid config file with pipeline-parameters + Given a file named "config.yml" with: + """ + version: 2.1 + + parameters: + foo: + type: string + default: "bar" + + jobs: + datadog-hello-world: + docker: + - image: cimg/base:stable + steps: + - run: | + echo "doing something really cool" + echo << pipeline.parameters.foo >> + workflows: + datadog-hello-world: + jobs: + - datadog-hello-world + """ + When I run `circleci config process config.yml --pipeline-parameters "foo: fighters"` + Then the output should contain "fighters" + And the exit status should be 0 + + Scenario: Checking a valid config file with default pipeline params + Given a file named "config.yml" with: + """ + version: 2.1 + + parameters: + foo: + type: string + default: "bar" + + jobs: + datadog-hello-world: + docker: + - image: cimg/base:stable + steps: + - run: | + echo "doing something really cool" + echo << pipeline.parameters.foo >> + workflows: + datadog-hello-world: + jobs: + - datadog-hello-world + """ + When I run `circleci config process config.yml` + Then the output should contain "bar" + And the exit status should be 0 + + Scenario: Checking a valid config file with file pipeline-parameters + Given a file named "config.yml" with: + """ + version: 2.1 + + parameters: + foo: + type: string + default: "bar" + + jobs: + datadog-hello-world: + docker: + - image: cimg/base:stable + steps: + - run: | + echo "doing something really cool" + echo << pipeline.parameters.foo >> + workflows: + datadog-hello-world: + jobs: + - datadog-hello-world + """ + And I write to "params.yml" with: + """ + foo: "totallyawesome" + """ + When I run `circleci config process config.yml --pipeline-parameters params.yml` + Then the output should contain "totallyawesome" + And the exit status should be 0 + + Scenario: Checking an invalid config file Given a file named "config.yml" with: """ diff --git a/local/local.go b/local/local.go index 373a88dd2..2137cf355 100644 --- a/local/local.go +++ b/local/local.go @@ -12,8 +12,8 @@ import ( "strings" "syscall" - "github.com/CircleCI-Public/circleci-cli/api" - "github.com/CircleCI-Public/circleci-cli/api/graphql" + "github.com/CircleCI-Public/circleci-cli/api/rest" + "github.com/CircleCI-Public/circleci-cli/config" "github.com/CircleCI-Public/circleci-cli/pipeline" "github.com/CircleCI-Public/circleci-cli/settings" "github.com/pkg/errors" @@ -26,21 +26,21 @@ const DefaultConfigPath = ".circleci/config.yml" func Execute(flags *pflag.FlagSet, cfg *settings.Config, args []string) error { var err error - var configResponse *api.ConfigResponse - cl := graphql.NewClient(cfg.HTTPClient, cfg.Host, cfg.Endpoint, cfg.Token, cfg.Debug) + var configResponse *config.ConfigResponse + restClient := rest.New(cfg.Host, cfg) processedArgs, configPath := buildAgentArguments(flags) //if no orgId provided use org slug orgID, _ := flags.GetString("org-id") if strings.TrimSpace(orgID) != "" { - configResponse, err = api.ConfigQuery(cl, configPath, orgID, nil, pipeline.LocalPipelineValues()) + configResponse, err = config.ConfigQuery(restClient, configPath, orgID, nil, pipeline.LocalPipelineValues()) if err != nil { return err } } else { orgSlug, _ := flags.GetString("org-slug") - configResponse, err = api.ConfigQueryLegacy(cl, configPath, orgSlug, nil, pipeline.LocalPipelineValues()) + configResponse, err = config.ConfigQuery(restClient, configPath, orgSlug, nil, pipeline.LocalPipelineValues()) if err != nil { return err } diff --git a/settings/settings.go b/settings/settings.go index 24bc07f33..241538018 100644 --- a/settings/settings.go +++ b/settings/settings.go @@ -35,6 +35,10 @@ type Config struct { GitHubAPI string `yaml:"-"` SkipUpdateCheck bool `yaml:"-"` OrbPublishing OrbPublishingInfo `yaml:"orb_publishing"` + // Represents the API host we want to use for config compilation and validation + // requests - this is typically on the api.circleci.com subdomain for cloud, or the + // same domain for server instances. + ConfigAPIHost string `yaml:"-"` } type OrbPublishingInfo struct { @@ -128,6 +132,10 @@ func (cfg *Config) WriteToDisk() error { func (cfg *Config) LoadFromEnv(prefix string) { if host := ReadFromEnv(prefix, "host"); host != "" { cfg.Host = host + // If the user is a server customer and overwrites the default + // https://circleci.com host - we then have to use this as the host for + // any config compilation or validation requests as opposed to https://api.circleci.com + cfg.ConfigAPIHost = host } if restEndpoint := ReadFromEnv(prefix, "rest_endpoint"); restEndpoint != "" {