From 092ba1a5e54e21953eb8cc79140d459572dc9d15 Mon Sep 17 00:00:00 2001 From: Elliot Forbes Date: Mon, 20 Mar 2023 16:46:41 +0000 Subject: [PATCH] Revert "PIPE-2315 - Removes API-Service from config compilation" --- api/api.go | 119 ++++++ api/rest/client.go | 44 +- api/rest/client_test.go | 37 -- cmd/config.go | 73 +--- cmd/config_test.go | 398 +++++++++++++----- {config => cmd}/deprecated-images.go | 6 +- cmd/root.go | 7 - config/config.go | 109 ----- .../features/circleci_config.feature | 203 --------- local/local.go | 12 +- settings/settings.go | 8 - 11 files changed, 438 insertions(+), 578 deletions(-) rename {config => cmd}/deprecated-images.go (93%) delete mode 100644 config/config.go diff --git a/api/api.go b/api/api.go index ff6ca667d..75fd208a9 100644 --- a/api/api.go +++ b/api/api.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/CircleCI-Public/circleci-cli/api/graphql" + "github.com/CircleCI-Public/circleci-cli/pipeline" "github.com/CircleCI-Public/circleci-cli/references" "github.com/CircleCI-Public/circleci-cli/settings" "github.com/Masterminds/semver" @@ -512,6 +513,124 @@ func WhoamiQuery(cl *graphql.Client) (*WhoamiResponse, error) { return &response, nil } +// ConfigQueryLegacy calls the GQL API to validate and process config with the legacy orgSlug +func ConfigQueryLegacy(cl *graphql.Client, configPath string, orgSlug string, params pipeline.Parameters, values pipeline.Values) (*ConfigResponse, error) { + var response BuildConfigResponse + var query string + config, err := loadYaml(configPath) + if err != nil { + return nil, err + } + // GraphQL isn't forwards-compatible, so we are unusually selective here about + // passing only non-empty fields on to the API, to minimize user impact if the + // backend is out of date. + var fieldAddendums string + if orgSlug != "" { + fieldAddendums += ", orgSlug: $orgSlug" + } + if len(params) > 0 { + fieldAddendums += ", pipelineParametersJson: $pipelineParametersJson" + } + query = fmt.Sprintf( + `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgSlug: String) { + buildConfig(configYaml: $config, pipelineValues: $pipelineValues%s) { + valid, + errors { message }, + sourceYaml, + outputYaml + } + }`, + fieldAddendums) + + request := graphql.NewRequest(query) + request.SetToken(cl.Token) + request.Var("config", config) + + if values != nil { + request.Var("pipelineValues", pipeline.PrepareForGraphQL(values)) + } + if params != nil { + pipelineParameters, err := json.Marshal(params) + if err != nil { + return nil, fmt.Errorf("unable to serialize pipeline values: %s", err.Error()) + } + request.Var("pipelineParametersJson", string(pipelineParameters)) + } + + if orgSlug != "" { + request.Var("orgSlug", orgSlug) + } + + err = cl.Run(request, &response) + if err != nil { + return nil, errors.Wrap(err, "Unable to validate config") + } + if len(response.BuildConfig.ConfigResponse.Errors) > 0 { + return nil, &response.BuildConfig.ConfigResponse.Errors + } + + return &response.BuildConfig.ConfigResponse, nil +} + +// ConfigQuery calls the GQL API to validate and process config with the org id +func ConfigQuery(cl *graphql.Client, configPath string, orgId string, params pipeline.Parameters, values pipeline.Values) (*ConfigResponse, error) { + var response BuildConfigResponse + var query string + config, err := loadYaml(configPath) + if err != nil { + return nil, err + } + // GraphQL isn't forwards-compatible, so we are unusually selective here about + // passing only non-empty fields on to the API, to minimize user impact if the + // backend is out of date. + var fieldAddendums string + if orgId != "" { + fieldAddendums += ", orgId: $orgId" + } + if len(params) > 0 { + fieldAddendums += ", pipelineParametersJson: $pipelineParametersJson" + } + query = fmt.Sprintf( + `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgId: UUID!) { + buildConfig(configYaml: $config, pipelineValues: $pipelineValues%s) { + valid, + errors { message }, + sourceYaml, + outputYaml + } + }`, + fieldAddendums) + + request := graphql.NewRequest(query) + request.SetToken(cl.Token) + request.Var("config", config) + + if values != nil { + request.Var("pipelineValues", pipeline.PrepareForGraphQL(values)) + } + if params != nil { + pipelineParameters, err := json.Marshal(params) + if err != nil { + return nil, fmt.Errorf("unable to serialize pipeline values: %s", err.Error()) + } + request.Var("pipelineParametersJson", string(pipelineParameters)) + } + + if orgId != "" { + request.Var("orgId", orgId) + } + + err = cl.Run(request, &response) + if err != nil { + return nil, errors.Wrap(err, "Unable to validate config") + } + if len(response.BuildConfig.ConfigResponse.Errors) > 0 { + return nil, &response.BuildConfig.ConfigResponse.Errors + } + + return &response.BuildConfig.ConfigResponse, nil +} + // OrbQuery validated and processes an orb. func OrbQuery(cl *graphql.Client, configPath string) (*ConfigResponse, error) { var response OrbConfigResponse diff --git a/api/rest/client.go b/api/rest/client.go index 918f64c2e..d745b887e 100644 --- a/api/rest/client.go +++ b/api/rest/client.go @@ -17,12 +17,7 @@ import ( ) type Client struct { - baseURL *url.URL - // The config api host differs for both cloud and server setups. - // For cloud, the base domain will be https://api.circleci.com - // for server, this should match the host as we don't have the same - // api subdomain setup - apiURL *url.URL + baseURL *url.URL circleToken string client *http.Client } @@ -34,15 +29,13 @@ func New(host string, config *settings.Config) *Client { endpoint += "/" } - baseURL, _ := url.Parse(host) - apiURL, _ := url.Parse(config.ConfigAPIHost) + u, _ := url.Parse(host) client := config.HTTPClient client.Timeout = 10 * time.Second return &Client{ - apiURL: apiURL.ResolveReference(&url.URL{Path: endpoint}), - baseURL: baseURL.ResolveReference(&url.URL{Path: endpoint}), + baseURL: u.ResolveReference(&url.URL{Path: endpoint}), circleToken: config.Token, client: client, } @@ -64,34 +57,6 @@ func (c *Client) NewRequest(method string, u *url.URL, payload interface{}) (req return nil, err } - c.enrichRequestHeaders(req, payload) - return req, nil -} - -// NewAPIRequest - similar to NewRequest except it uses the apiURL as the base URL. -func (c *Client) NewAPIRequest(method string, u *url.URL, payload interface{}) (req *http.Request, err error) { - var r io.Reader - if payload != nil { - buf := &bytes.Buffer{} - r = buf - err = json.NewEncoder(buf).Encode(payload) - if err != nil { - fmt.Printf("failed to encode payload as json: %s\n", err.Error()) - return nil, err - } - } - - req, err = http.NewRequest(method, c.apiURL.ResolveReference(u).String(), r) - if err != nil { - fmt.Printf("failed to create new http request: %s\n", err.Error()) - return nil, err - } - - c.enrichRequestHeaders(req, payload) - return req, nil -} - -func (c *Client) enrichRequestHeaders(req *http.Request, payload interface{}) { req.Header.Set("Circle-Token", c.circleToken) req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", version.UserAgent()) @@ -102,12 +67,12 @@ func (c *Client) enrichRequestHeaders(req *http.Request, payload interface{}) { if payload != nil { req.Header.Set("Content-Type", "application/json") } + return req, nil } func (c *Client) DoRequest(req *http.Request, resp interface{}) (statusCode int, err error) { httpResp, err := c.client.Do(req) if err != nil { - fmt.Printf("failed to make http request: %s\n", err.Error()) return 0, err } defer httpResp.Body.Close() @@ -118,7 +83,6 @@ func (c *Client) DoRequest(req *http.Request, resp interface{}) (statusCode int, }{} err = json.NewDecoder(httpResp.Body).Decode(&httpError) if err != nil { - fmt.Printf("failed to decode body: %s", err.Error()) return httpResp.StatusCode, err } return httpResp.StatusCode, &HTTPError{Code: httpResp.StatusCode, Message: httpError.Message} diff --git a/api/rest/client_test.go b/api/rest/client_test.go index f66281f7b..a45d4bc13 100644 --- a/api/rest/client_test.go +++ b/api/rest/client_test.go @@ -118,43 +118,6 @@ func TestClient_DoRequest(t *testing.T) { }) } -func TestAPIRequest(t *testing.T) { - fix := &fixture{} - c, cleanup := fix.Run(http.StatusCreated, `{"key": "value"}`) - defer cleanup() - - t.Run("test new api request sets the default headers", func(t *testing.T) { - req, err := c.NewAPIRequest("GET", &url.URL{}, struct{}{}) - assert.NilError(t, err) - - assert.Equal(t, req.Header.Get("User-Agent"), "circleci-cli/0.0.0-dev+dirty-local-tree (source)") - assert.Equal(t, req.Header.Get("Circle-Token"), c.circleToken) - assert.Equal(t, req.Header.Get("Accept"), "application/json") - }) - - type testPayload struct { - Message string - } - - t.Run("test new api request sets the default headers", func(t *testing.T) { - req, err := c.NewAPIRequest("GET", &url.URL{}, testPayload{Message: "hello"}) - assert.NilError(t, err) - - assert.Equal(t, req.Header.Get("Circleci-Cli-Command"), "") - assert.Equal(t, req.Header.Get("Content-Type"), "application/json") - }) - - t.Run("test new api request doesn't set content-type with empty payload", func(t *testing.T) { - req, err := c.NewAPIRequest("GET", &url.URL{}, nil) - assert.NilError(t, err) - - assert.Equal(t, req.Header.Get("Circleci-Cli-Command"), "") - if req.Header.Get("Content-Type") != "" { - t.Fail() - } - }) -} - type fixture struct { mu sync.Mutex url url.URL diff --git a/cmd/config.go b/cmd/config.go index 7d05200be..5ae76fcde 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -3,11 +3,10 @@ package cmd import ( "fmt" "io/ioutil" - "net/url" "strings" - "github.com/CircleCI-Public/circleci-cli/api/rest" - "github.com/CircleCI-Public/circleci-cli/config" + "github.com/CircleCI-Public/circleci-cli/api" + "github.com/CircleCI-Public/circleci-cli/api/graphql" "github.com/CircleCI-Public/circleci-cli/filetree" "github.com/CircleCI-Public/circleci-cli/local" "github.com/CircleCI-Public/circleci-cli/pipeline" @@ -19,13 +18,9 @@ import ( "gopkg.in/yaml.v3" ) -var ( - CollaborationsPath = "me/collaborations" -) - type configOptions struct { cfg *settings.Config - rest *rest.Client + cl *graphql.Client args []string } @@ -68,7 +63,7 @@ func newConfigCommand(config *settings.Config) *cobra.Command { Short: "Check that the config file is well formed.", PreRun: func(cmd *cobra.Command, args []string) { opts.args = args - opts.rest = rest.New(config.Host, config) + opts.cl = graphql.NewClient(config.HTTPClient, config.Host, config.Endpoint, config.Token, config.Debug) }, RunE: func(cmd *cobra.Command, _ []string) error { return validateConfig(opts, cmd.Flags()) @@ -90,7 +85,7 @@ func newConfigCommand(config *settings.Config) *cobra.Command { Short: "Validate config and display expanded configuration.", PreRun: func(cmd *cobra.Command, args []string) { opts.args = args - opts.rest = rest.New(config.Host, config) + opts.cl = graphql.NewClient(config.HTTPClient, config.Host, config.Endpoint, config.Token, config.Debug) }, RunE: func(cmd *cobra.Command, _ []string) error { return processConfig(opts, cmd.Flags()) @@ -102,7 +97,6 @@ func newConfigCommand(config *settings.Config) *cobra.Command { processCommand.Flags().StringP("org-slug", "o", "", "organization slug (for example: github/example-org), used when a config depends on private orbs belonging to that org") processCommand.Flags().String("org-id", "", "organization id used when a config depends on private orbs belonging to that org") processCommand.Flags().StringP("pipeline-parameters", "", "", "YAML/JSON map of pipeline parameters, accepts either YAML/JSON directly or file path (for example: my-params.yml)") - processCommand.Flags().StringP("circleci-api-host", "", "", "the api-host you want to use for config processing and validation") migrateCommand := &cobra.Command{ Use: "migrate", @@ -131,7 +125,7 @@ func newConfigCommand(config *settings.Config) *cobra.Command { // The arg is actually optional, in order to support compatibility with the --path flag. func validateConfig(opts configOptions, flags *pflag.FlagSet) error { var err error - var response *config.ConfigResponse + var response *api.ConfigResponse path := local.DefaultConfigPath // First, set the path to configPath set by --path flag for compatibility if configPath != "" { @@ -148,21 +142,15 @@ func validateConfig(opts configOptions, flags *pflag.FlagSet) error { fmt.Println("Validating config with following values") printValues(values) - var orgID string - orgID, _ = flags.GetString("org-id") + orgID, _ := flags.GetString("org-id") if strings.TrimSpace(orgID) != "" { - response, err = config.ConfigQuery(opts.rest, path, orgID, nil, pipeline.LocalPipelineValues()) + response, err = api.ConfigQuery(opts.cl, path, orgID, nil, values) if err != nil { return err } } else { orgSlug, _ := flags.GetString("org-slug") - orgs, err := GetOrgCollaborations(opts.rest) - if err != nil { - fmt.Println(err.Error()) - } - orgID = GetOrgIdFromSlug(orgSlug, orgs) - response, err = config.ConfigQuery(opts.rest, path, orgID, nil, pipeline.LocalPipelineValues()) + response, err = api.ConfigQueryLegacy(opts.cl, path, orgSlug, nil, values) if err != nil { return err } @@ -172,7 +160,7 @@ func validateConfig(opts configOptions, flags *pflag.FlagSet) error { // link here to blog post when available // returns an error if a deprecated image is used if !ignoreDeprecatedImages { - err := config.DeprecatedImageCheck(response) + err := deprecatedImageCheck(response) if err != nil { return err } @@ -189,7 +177,7 @@ func validateConfig(opts configOptions, flags *pflag.FlagSet) error { func processConfig(opts configOptions, flags *pflag.FlagSet) error { paramsYaml, _ := flags.GetString("pipeline-parameters") - var response *config.ConfigResponse + var response *api.ConfigResponse var params pipeline.Parameters var err error @@ -214,18 +202,13 @@ func processConfig(opts configOptions, flags *pflag.FlagSet) error { orgID, _ := flags.GetString("org-id") if strings.TrimSpace(orgID) != "" { - response, err = config.ConfigQuery(opts.rest, opts.args[0], orgID, params, values) + response, err = api.ConfigQuery(opts.cl, opts.args[0], orgID, params, values) if err != nil { return err } } else { orgSlug, _ := flags.GetString("org-slug") - orgs, err := GetOrgCollaborations(opts.rest) - if err != nil { - fmt.Println(err.Error()) - } - orgID = GetOrgIdFromSlug(orgSlug, orgs) - response, err = config.ConfigQuery(opts.rest, opts.args[0], orgID, params, values) + response, err = api.ConfigQueryLegacy(opts.cl, opts.args[0], orgSlug, params, values) if err != nil { return err } @@ -258,33 +241,3 @@ func printValues(values pipeline.Values) { fmt.Printf("\t%s:\t%s", key, value) } } - -type CollaborationResult struct { - VcsTye string `json:"vcs_type"` - OrgSlug string `json:"slug"` - OrgName string `json:"name"` - OrgId string `json:"id"` - AvatarUrl string `json:"avatar_url"` -} - -// GetOrgCollaborations - fetches all the collaborations for a given user. -func GetOrgCollaborations(client *rest.Client) ([]CollaborationResult, error) { - req, err := client.NewRequest("GET", &url.URL{Path: CollaborationsPath}, nil) - if err != nil { - return nil, err - } - - var resp []CollaborationResult - _, err = client.DoRequest(req, &resp) - return resp, err -} - -// GetOrgIdFromSlug - converts a slug into an orgID. -func GetOrgIdFromSlug(slug string, collaborations []CollaborationResult) string { - for _, v := range collaborations { - if v.OrgSlug == slug { - return v.OrgId - } - } - return "" -} diff --git a/cmd/config_test.go b/cmd/config_test.go index d0ed9016d..85b6e8632 100644 --- a/cmd/config_test.go +++ b/cmd/config_test.go @@ -1,13 +1,20 @@ package cmd_test import ( + "encoding/json" "fmt" + "io" + "net/http" "os/exec" "path/filepath" + "time" + "github.com/CircleCI-Public/circleci-cli/api/graphql" "github.com/CircleCI-Public/circleci-cli/clitest" + "github.com/CircleCI-Public/circleci-cli/pipeline" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/onsi/gomega/gbytes" "github.com/onsi/gomega/gexec" "gotest.tools/v3/golden" ) @@ -18,6 +25,7 @@ var _ = Describe("Config", func() { command *exec.Cmd results []byte tempSettings *clitest.TempSettings + token string = "testtoken" ) BeforeEach(func() { @@ -87,112 +95,6 @@ var _ = Describe("Config", func() { }) }) - It("packs all YAML contents as expected", func() { - command = exec.Command(pathCLI, - "config", "pack", - "--skip-update-check", - "testdata/hugo-pack/.circleci") - results = golden.Get(GinkgoT(), filepath.FromSlash("hugo-pack/result.yml")) - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - session.Wait() - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Err.Contents()).Should(BeEmpty()) - Eventually(session.Out.Contents()).Should(MatchYAML(results)) - Eventually(session).Should(gexec.Exit(0)) - }) - - It("given a .circleci folder with config.yml and local orb, packs all YAML contents as expected", func() { - command = exec.Command(pathCLI, - "config", "pack", - "--skip-update-check", - "testdata/hugo-pack/.circleci") - results = golden.Get(GinkgoT(), filepath.FromSlash("hugo-pack/result.yml")) - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - session.Wait() - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Err.Contents()).Should(BeEmpty()) - Eventually(session.Out.Contents()).Should(MatchYAML(results)) - Eventually(session).Should(gexec.Exit(0)) - }) - - It("given a local orbs folder with mixed inline and local commands, jobs, etc, packs all YAML contents as expected", func() { - var path string = "nested-orbs-and-local-commands-etc" - command = exec.Command(pathCLI, - "config", "pack", - "--skip-update-check", - filepath.Join("testdata", path, "test")) - results = golden.Get(GinkgoT(), filepath.FromSlash(fmt.Sprintf("%s/result.yml", path))) - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - session.Wait() - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Err.Contents()).Should(BeEmpty()) - Eventually(session.Out.Contents()).Should(MatchYAML(results)) - Eventually(session).Should(gexec.Exit(0)) - }) - - It("returns an error when validating a config", func() { - var path string = "nested-orbs-and-local-commands-etc" - command = exec.Command(pathCLI, - "config", "pack", - "--skip-update-check", - filepath.Join("testdata", path, "test")) - results = golden.Get(GinkgoT(), filepath.FromSlash(fmt.Sprintf("%s/result.yml", path))) - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - session.Wait() - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Err.Contents()).Should(BeEmpty()) - Eventually(session.Out.Contents()).Should(MatchYAML(results)) - Eventually(session).Should(gexec.Exit(0)) - }) - - It("packs successfully given an orb containing local executors and commands in folder", func() { - command = exec.Command(pathCLI, - "config", "pack", - "--skip-update-check", - "testdata/myorb/test") - - results = golden.Get(GinkgoT(), filepath.FromSlash("myorb/result.yml")) - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - session.Wait() - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Err.Contents()).Should(BeEmpty()) - Eventually(session.Out.Contents()).Should(MatchYAML(results)) - Eventually(session).Should(gexec.Exit(0)) - }) - - It("packs as expected given a large nested config including rails orbs", func() { - var path string = "test-with-large-nested-rails-orb" - command = exec.Command(pathCLI, - "config", "pack", - "--skip-update-check", - filepath.Join("testdata", path, "test")) - results = golden.Get(GinkgoT(), filepath.FromSlash(fmt.Sprintf("%s/result.yml", path))) - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - session.Wait() - Expect(err).ShouldNot(HaveOccurred()) - Eventually(session.Err.Contents()).Should(BeEmpty()) - Eventually(session.Out.Contents()).Should(MatchYAML(results)) - Eventually(session).Should(gexec.Exit(0)) - }) - - It("prints an error given a config which is a list and not a map", func() { - config := clitest.OpenTmpFile(filepath.Join(tempSettings.Home, "myorb"), "config.yaml") - command = exec.Command(pathCLI, - "config", "pack", - "--skip-update-check", - config.RootDir, - ) - config.Write([]byte(`[]`)) - - expected := fmt.Sprintf("Error: Failed trying to marshal the tree to YAML : expected a map, got a `[]interface {}` which is not supported at this time for \"%s\"\n", config.Path) - session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - Expect(err).ShouldNot(HaveOccurred()) - stderr := session.Wait().Err.Contents() - Expect(string(stderr)).To(Equal(expected)) - Eventually(session).Should(clitest.ShouldFail()) - config.Close() - }) - Describe("with a large nested config including rails orb", func() { BeforeEach(func() { var path string = "test-with-large-nested-rails-orb" @@ -244,5 +146,289 @@ var _ = Describe("Config", func() { Eventually(session).Should(clitest.ShouldFail()) }) }) + + Describe("validating configs", func() { + config := "version: 2.1" + var expReq string + + BeforeEach(func() { + command = exec.Command(pathCLI, + "config", "validate", + "--skip-update-check", + "--token", token, + "--host", tempSettings.TestServer.URL(), + "-", + ) + + stdin, err := command.StdinPipe() + Expect(err).ToNot(HaveOccurred()) + _, err = io.WriteString(stdin, config) + Expect(err).ToNot(HaveOccurred()) + stdin.Close() + + query := `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgSlug: String) { + buildConfig(configYaml: $config, pipelineValues: $pipelineValues) { + valid, + errors { message }, + sourceYaml, + outputYaml + } + }` + + r := graphql.NewRequest(query) + r.SetToken(token) + r.Variables["config"] = config + r.Variables["pipelineValues"] = pipeline.PrepareForGraphQL(pipeline.LocalPipelineValues()) + + req, err := r.Encode() + Expect(err).ShouldNot(HaveOccurred()) + expReq = req.String() + }) + + It("returns an error when validating a config", func() { + expResp := `{ + "buildConfig": { + "errors": [ + {"message": "error1"} + ] + } + }` + + tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ + Status: http.StatusOK, + Request: expReq, + Response: expResp, + }) + + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).ShouldNot(HaveOccurred()) + Eventually(session.Err, time.Second*3).Should(gbytes.Say("Error: error1")) + Eventually(session).Should(clitest.ShouldFail()) + }) + + It("returns successfully when validating a config", func() { + expResp := `{ + "buildConfig": {} + }` + + tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ + Status: http.StatusOK, + Request: expReq, + Response: expResp, + }) + + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).ShouldNot(HaveOccurred()) + Eventually(session.Out, time.Second*3).Should(gbytes.Say("Config input is valid.")) + Eventually(session).Should(gexec.Exit(0)) + }) + }) + + Describe("validating configs with pipeline parameters", func() { + config := "version: 2.1" + var expReq string + + BeforeEach(func() { + command = exec.Command(pathCLI, + "config", "process", + "--skip-update-check", + "--token", token, + "--host", tempSettings.TestServer.URL(), + "--pipeline-parameters", `{"foo": "test", "bar": true, "baz": 10}`, + "-", + ) + + stdin, err := command.StdinPipe() + Expect(err).ToNot(HaveOccurred()) + _, err = io.WriteString(stdin, config) + Expect(err).ToNot(HaveOccurred()) + stdin.Close() + + query := `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgSlug: String) { + buildConfig(configYaml: $config, pipelineValues: $pipelineValues, pipelineParametersJson: $pipelineParametersJson) { + valid, + errors { message }, + sourceYaml, + outputYaml + } + }` + + r := graphql.NewRequest(query) + r.SetToken(token) + r.Variables["config"] = config + r.Variables["pipelineValues"] = pipeline.PrepareForGraphQL(pipeline.LocalPipelineValues()) + + pipelineParams, err := json.Marshal(pipeline.Parameters{ + "foo": "test", + "bar": true, + "baz": 10, + }) + Expect(err).ToNot(HaveOccurred()) + r.Variables["pipelineParametersJson"] = string(pipelineParams) + + req, err := r.Encode() + Expect(err).ShouldNot(HaveOccurred()) + expReq = req.String() + }) + + It("returns successfully when validating a config", func() { + expResp := `{ + "buildConfig": {} + }` + + tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ + Status: http.StatusOK, + Request: expReq, + Response: expResp, + }) + + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).ShouldNot(HaveOccurred()) + Eventually(session).Should(gexec.Exit(0)) + }) + }) + + Describe("validating configs with private orbs", func() { + config := "version: 2.1" + orgId := "bb604b45-b6b0-4b81-ad80-796f15eddf87" + var expReq string + + BeforeEach(func() { + command = exec.Command(pathCLI, + "config", "validate", + "--skip-update-check", + "--token", token, + "--host", tempSettings.TestServer.URL(), + "--org-id", orgId, + "-", + ) + + stdin, err := command.StdinPipe() + Expect(err).ToNot(HaveOccurred()) + _, err = io.WriteString(stdin, config) + Expect(err).ToNot(HaveOccurred()) + stdin.Close() + + query := `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgId: UUID!) { + buildConfig(configYaml: $config, pipelineValues: $pipelineValues, orgId: $orgId) { + valid, + errors { message }, + sourceYaml, + outputYaml + } + }` + + r := graphql.NewRequest(query) + r.SetToken(token) + r.Variables["config"] = config + r.Variables["orgId"] = orgId + r.Variables["pipelineValues"] = pipeline.PrepareForGraphQL(pipeline.LocalPipelineValues()) + + req, err := r.Encode() + Expect(err).ShouldNot(HaveOccurred()) + expReq = req.String() + }) + + It("returns an error when validating a config with a private orb", func() { + expResp := `{ + "buildConfig": { + "errors": [ + {"message": "permission denied"} + ] + } + }` + + tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ + Status: http.StatusOK, + Request: expReq, + Response: expResp, + }) + + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).ShouldNot(HaveOccurred()) + Eventually(session.Err, time.Second*3).Should(gbytes.Say("Error: permission denied")) + Eventually(session).Should(clitest.ShouldFail()) + }) + }) + + Describe("validating configs with private orbs Legacy", func() { + config := "version: 2.1" + orgSlug := "circleci" + var expReq string + + BeforeEach(func() { + command = exec.Command(pathCLI, + "config", "validate", + "--skip-update-check", + "--token", token, + "--host", tempSettings.TestServer.URL(), + "--org-slug", orgSlug, + "-", + ) + + stdin, err := command.StdinPipe() + Expect(err).ToNot(HaveOccurred()) + _, err = io.WriteString(stdin, config) + Expect(err).ToNot(HaveOccurred()) + stdin.Close() + + query := `query ValidateConfig ($config: String!, $pipelineParametersJson: String, $pipelineValues: [StringKeyVal!], $orgSlug: String) { + buildConfig(configYaml: $config, pipelineValues: $pipelineValues, orgSlug: $orgSlug) { + valid, + errors { message }, + sourceYaml, + outputYaml + } + }` + + r := graphql.NewRequest(query) + r.SetToken(token) + r.Variables["config"] = config + r.Variables["orgSlug"] = orgSlug + r.Variables["pipelineValues"] = pipeline.PrepareForGraphQL(pipeline.LocalPipelineValues()) + + req, err := r.Encode() + Expect(err).ShouldNot(HaveOccurred()) + expReq = req.String() + }) + + It("returns an error when validating a config with a private orb", func() { + expResp := `{ + "buildConfig": { + "errors": [ + {"message": "permission denied"} + ] + } + }` + + tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ + Status: http.StatusOK, + Request: expReq, + Response: expResp, + }) + + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).ShouldNot(HaveOccurred()) + Eventually(session.Err, time.Second*3).Should(gbytes.Say("Error: permission denied")) + Eventually(session).Should(clitest.ShouldFail()) + }) + + It("returns successfully when validating a config with private orbs", func() { + expResp := `{ + "buildConfig": {} + }` + + tempSettings.AppendPostHandler(token, clitest.MockRequestResponse{ + Status: http.StatusOK, + Request: expReq, + Response: expResp, + }) + + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).ShouldNot(HaveOccurred()) + Eventually(session.Out, time.Second*3).Should(gbytes.Say("Config input is valid.")) + Eventually(session).Should(gexec.Exit(0)) + }) + }) }) }) diff --git a/config/deprecated-images.go b/cmd/deprecated-images.go similarity index 93% rename from config/deprecated-images.go rename to cmd/deprecated-images.go index fcc998fc9..607edfc88 100644 --- a/config/deprecated-images.go +++ b/cmd/deprecated-images.go @@ -1,6 +1,8 @@ -package config +package cmd import ( + "github.com/CircleCI-Public/circleci-cli/api" + "github.com/pkg/errors" "gopkg.in/yaml.v3" ) @@ -38,7 +40,7 @@ type processedConfig struct { } // Processes the config down to v2.0, then checks image used against the block list -func DeprecatedImageCheck(response *ConfigResponse) error { +func deprecatedImageCheck(response *api.ConfigResponse) error { aConfig := processedConfig{} err := yaml.Unmarshal([]byte(response.OutputYaml), &aConfig) diff --git a/cmd/root.go b/cmd/root.go index 7f7ed4ec1..6a636cd0c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -21,7 +21,6 @@ import ( var defaultEndpoint = "graphql-unstable" var defaultHost = "https://circleci.com" -var defaultAPIHost = "https://api.circleci.com" var defaultRestEndpoint = "api/v2" var trueString = "true" @@ -104,11 +103,6 @@ func MakeCommands() *cobra.Command { RestEndpoint: defaultRestEndpoint, Endpoint: defaultEndpoint, GitHubAPI: "https://api.github.com/", - // The config api host differs for both cloud and server setups. - // For cloud, the base domain will be https://api.circleci.com - // for server, this should match the host as we don't have the same - // api subdomain setup - ConfigAPIHost: defaultAPIHost, } if err := rootOptions.Load(); err != nil { @@ -180,7 +174,6 @@ func MakeCommands() *cobra.Command { flags.StringVar(&rootOptions.Host, "host", rootOptions.Host, "URL to your CircleCI host, also CIRCLECI_CLI_HOST") flags.StringVar(&rootOptions.Endpoint, "endpoint", rootOptions.Endpoint, "URI to your CircleCI GraphQL API endpoint") flags.StringVar(&rootOptions.GitHubAPI, "github-api", "https://api.github.com/", "Change the default endpoint to GitHub API for retrieving updates") - flags.StringVar(&rootOptions.ConfigAPIHost, "config-api-host", "https://api.circleci.com", "Change the default endpoint for the config api host") flags.BoolVar(&rootOptions.SkipUpdateCheck, "skip-update-check", skipUpdateByDefault(), "Skip the check for updates check run before every command.") hidden := []string{"github-api", "debug", "endpoint"} diff --git a/config/config.go b/config/config.go deleted file mode 100644 index 8605690a6..000000000 --- a/config/config.go +++ /dev/null @@ -1,109 +0,0 @@ -package config - -import ( - "fmt" - "io/ioutil" - "net/url" - "os" - - "github.com/CircleCI-Public/circleci-cli/api/rest" - "github.com/CircleCI-Public/circleci-cli/pipeline" - "github.com/pkg/errors" -) - -type ConfigError struct { - Message string `json:"message"` -} - -type ConfigResponse struct { - Valid bool `json:"valid"` - SourceYaml string `json:"source-yaml"` - OutputYaml string `json:"output-yaml"` - Errors []ConfigError `json:"errors"` -} - -type CompileConfigRequest struct { - ConfigYaml string `json:"config_yaml"` - Options Options `json:"options"` -} - -type Options struct { - OwnerID string `json:"owner-id,omitempty"` - PipelineParameters map[string]interface{} `json:"pipeline_parameters,omitempty"` - PipelineValues map[string]string `json:"pipeline_values,omitempty"` -} - -// #nosec -func loadYaml(path string) (string, error) { - var err error - var config []byte - if path == "-" { - config, err = ioutil.ReadAll(os.Stdin) - } else { - config, err = ioutil.ReadFile(path) - } - - if err != nil { - return "", errors.Wrapf(err, "Could not load config file at %s", path) - } - - return string(config), nil -} - -// ConfigQuery - attempts to compile or validate a given config file with the -// passed in params/values. -// If the orgID is passed in, the config-compilation with private orbs should work. -func ConfigQuery( - rest *rest.Client, - configPath string, - orgID string, - params pipeline.Parameters, - values pipeline.Values, -) (*ConfigResponse, error) { - - configString, err := loadYaml(configPath) - if err != nil { - return nil, err - } - - compileRequest := CompileConfigRequest{ - ConfigYaml: configString, - Options: Options{ - PipelineValues: values, - }, - } - - if orgID != "" { - compileRequest.Options.OwnerID = orgID - } - - if len(params) >= 1 { - compileRequest.Options.PipelineParameters = params - } - - req, err := rest.NewAPIRequest( - "POST", - &url.URL{ - Path: "compile-config-with-defaults", - }, - compileRequest, - ) - if err != nil { - return nil, err - } - - configCompilationResp := &ConfigResponse{} - statusCode, err := rest.DoRequest(req, configCompilationResp) - if err != nil { - return nil, err - } - if statusCode != 200 { - return nil, errors.New("non 200 status code") - } - - if len(configCompilationResp.Errors) > 0 { - return nil, errors.New(fmt.Sprintf("config compilation contains errors: %s", configCompilationResp.Errors)) - } - - return configCompilationResp, nil -} diff --git a/integration_tests/features/circleci_config.feature b/integration_tests/features/circleci_config.feature index 439103b78..d9085cc2b 100644 --- a/integration_tests/features/circleci_config.feature +++ b/integration_tests/features/circleci_config.feature @@ -15,209 +15,6 @@ Feature: Config checking Then the exit status should be 0 And the output should contain "Config file at config.yml is valid." - Scenario: Checking a valid config file with an orb - Given a file named "config.yml" with: - """ - version: 2.1 - - orbs: - node: circleci/node@5.0.3 - - jobs: - datadog-hello-world: - docker: - - image: cimg/base:stable - steps: - - run: | - echo "doing something really cool" - workflows: - datadog-hello-world: - jobs: - - datadog-hello-world - """ - When I run `circleci config validate --skip-update-check -c config.yml` - Then the exit status should be 0 - And the output should contain "Config file at config.yml is valid" - - Scenario: Checking a valid config against the k9s server - Given a file named "config.yml" with: - """ - version: 2.1 - - jobs: - datadog-hello-world: - docker: - - image: cimg/base:stable - steps: - - run: | - echo "doing something really cool" - workflows: - datadog-hello-world: - jobs: - - datadog-hello-world - """ - When I run `circleci --config-api-host https://k9s.sphereci.com config validate --skip-update-check -c config.yml` - Then the exit status should be 0 - And the output should contain "Config file at config.yml is valid" - - Scenario: Checking a valid config file with an orb - Given a file named "config.yml" with: - """ - version: 2.1 - - orbs: - node: circleci/node@5.0.3 - - jobs: - datadog-hello-world: - docker: - - image: cimg/base:stable - steps: - - run: | - echo "doing something really cool" - workflows: - datadog-hello-world: - jobs: - - datadog-hello-world - """ - When I run `circleci config validate --skip-update-check -c config.yml` - Then the exit status should be 0 - And the output should contain "Config file at config.yml is valid" - - Scenario: Checking a valid config file with a private org - Given a file named "config.yml" with: - """ - version: 2.1 - - orbs: - node: circleci/node@5.0.3 - - jobs: - datadog-hello-world: - docker: - - image: cimg/base:stable - steps: - - run: | - echo "doing something really cool" - workflows: - datadog-hello-world: - jobs: - - datadog-hello-world - """ - When I run `circleci config validate --skip-update-check --org-id bb604b45-b6b0-4b81-ad80-796f15eddf87 -c config.yml` - Then the output should contain "Config file at config.yml is valid" - And the exit status should be 0 - - Scenario: Checking a valid config file with a non-existant orb - Given a file named "config.yml" with: - """ - version: 2.1 - - orbs: - node: circleci/doesnt-exist@5.0.3 - - jobs: - datadog-hello-world: - docker: - - image: cimg/base:stable - steps: - - run: | - echo "doing something really cool" - workflows: - datadog-hello-world: - jobs: - - datadog-hello-world - """ - When I run `circleci config validate --skip-update-check -c config.yml` - Then the exit status should be 255 - And the output should contain "config compilation contains errors" - - Scenario: Checking a valid config file with pipeline-parameters - Given a file named "config.yml" with: - """ - version: 2.1 - - parameters: - foo: - type: string - default: "bar" - - jobs: - datadog-hello-world: - docker: - - image: cimg/base:stable - steps: - - run: | - echo "doing something really cool" - echo << pipeline.parameters.foo >> - workflows: - datadog-hello-world: - jobs: - - datadog-hello-world - """ - When I run `circleci config process config.yml --pipeline-parameters "foo: fighters"` - Then the output should contain "fighters" - And the exit status should be 0 - - Scenario: Checking a valid config file with default pipeline params - Given a file named "config.yml" with: - """ - version: 2.1 - - parameters: - foo: - type: string - default: "bar" - - jobs: - datadog-hello-world: - docker: - - image: cimg/base:stable - steps: - - run: | - echo "doing something really cool" - echo << pipeline.parameters.foo >> - workflows: - datadog-hello-world: - jobs: - - datadog-hello-world - """ - When I run `circleci config process config.yml` - Then the output should contain "bar" - And the exit status should be 0 - - Scenario: Checking a valid config file with file pipeline-parameters - Given a file named "config.yml" with: - """ - version: 2.1 - - parameters: - foo: - type: string - default: "bar" - - jobs: - datadog-hello-world: - docker: - - image: cimg/base:stable - steps: - - run: | - echo "doing something really cool" - echo << pipeline.parameters.foo >> - workflows: - datadog-hello-world: - jobs: - - datadog-hello-world - """ - And I write to "params.yml" with: - """ - foo: "totallyawesome" - """ - When I run `circleci config process config.yml --pipeline-parameters params.yml` - Then the output should contain "totallyawesome" - And the exit status should be 0 - - Scenario: Checking an invalid config file Given a file named "config.yml" with: """ diff --git a/local/local.go b/local/local.go index 2137cf355..373a88dd2 100644 --- a/local/local.go +++ b/local/local.go @@ -12,8 +12,8 @@ import ( "strings" "syscall" - "github.com/CircleCI-Public/circleci-cli/api/rest" - "github.com/CircleCI-Public/circleci-cli/config" + "github.com/CircleCI-Public/circleci-cli/api" + "github.com/CircleCI-Public/circleci-cli/api/graphql" "github.com/CircleCI-Public/circleci-cli/pipeline" "github.com/CircleCI-Public/circleci-cli/settings" "github.com/pkg/errors" @@ -26,21 +26,21 @@ const DefaultConfigPath = ".circleci/config.yml" func Execute(flags *pflag.FlagSet, cfg *settings.Config, args []string) error { var err error - var configResponse *config.ConfigResponse - restClient := rest.New(cfg.Host, cfg) + var configResponse *api.ConfigResponse + cl := graphql.NewClient(cfg.HTTPClient, cfg.Host, cfg.Endpoint, cfg.Token, cfg.Debug) processedArgs, configPath := buildAgentArguments(flags) //if no orgId provided use org slug orgID, _ := flags.GetString("org-id") if strings.TrimSpace(orgID) != "" { - configResponse, err = config.ConfigQuery(restClient, configPath, orgID, nil, pipeline.LocalPipelineValues()) + configResponse, err = api.ConfigQuery(cl, configPath, orgID, nil, pipeline.LocalPipelineValues()) if err != nil { return err } } else { orgSlug, _ := flags.GetString("org-slug") - configResponse, err = config.ConfigQuery(restClient, configPath, orgSlug, nil, pipeline.LocalPipelineValues()) + configResponse, err = api.ConfigQueryLegacy(cl, configPath, orgSlug, nil, pipeline.LocalPipelineValues()) if err != nil { return err } diff --git a/settings/settings.go b/settings/settings.go index 241538018..24bc07f33 100644 --- a/settings/settings.go +++ b/settings/settings.go @@ -35,10 +35,6 @@ type Config struct { GitHubAPI string `yaml:"-"` SkipUpdateCheck bool `yaml:"-"` OrbPublishing OrbPublishingInfo `yaml:"orb_publishing"` - // Represents the API host we want to use for config compilation and validation - // requests - this is typically on the api.circleci.com subdomain for cloud, or the - // same domain for server instances. - ConfigAPIHost string `yaml:"-"` } type OrbPublishingInfo struct { @@ -132,10 +128,6 @@ func (cfg *Config) WriteToDisk() error { func (cfg *Config) LoadFromEnv(prefix string) { if host := ReadFromEnv(prefix, "host"); host != "" { cfg.Host = host - // If the user is a server customer and overwrites the default - // https://circleci.com host - we then have to use this as the host for - // any config compilation or validation requests as opposed to https://api.circleci.com - cfg.ConfigAPIHost = host } if restEndpoint := ReadFromEnv(prefix, "rest_endpoint"); restEndpoint != "" {