From 1bb9c113e25916cdb4d5e5ac7077a77ccd6f697b Mon Sep 17 00:00:00 2001 From: Charles Francoise Date: Tue, 13 Jun 2023 09:33:50 +0200 Subject: [PATCH 1/5] create node project testdata this is used for `circleci config generate` tests --- cmd/testdata/node/package.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 cmd/testdata/node/package.json diff --git a/cmd/testdata/node/package.json b/cmd/testdata/node/package.json new file mode 100644 index 000000000..00e66575a --- /dev/null +++ b/cmd/testdata/node/package.json @@ -0,0 +1,10 @@ +{ + "name": "circleci-cli-node-test", + "version": "1.0.0", + "private": true, + "scripts": { + "build": "webpack", + "lint": "eslint .", + "test": "jest ." + } +} From b7555a903ee1157e27e774844548c3356c2d9386 Mon Sep 17 00:00:00 2001 From: Charles Francoise Date: Tue, 13 Jun 2023 09:34:08 +0200 Subject: [PATCH 2/5] use os.Getwd instead of `"."` --- cmd/config.go | 11 +++++++++-- cmd/config_test.go | 24 +++++++++++++++++++----- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/cmd/config.go b/cmd/config.go index 44e18b1a4..f20b9add1 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -162,8 +162,15 @@ func migrateConfig(args []string) error { } func generateConfig(args []string) error { - path := "." - if len(args) == 1 { + var err error + var path string + if len(args) == 0 { + // use working directory as default + path, err = os.Getwd() + if err != nil { + return fmt.Errorf("couldn't get working directory") + } + } else { path = args[0] } diff --git a/cmd/config_test.go b/cmd/config_test.go index 6bd041cff..3b1fb3089 100644 --- a/cmd/config_test.go +++ b/cmd/config_test.go @@ -2,6 +2,7 @@ package cmd_test import ( "fmt" + "os" "os/exec" "path/filepath" @@ -245,22 +246,35 @@ var _ = Describe("Config", func() { }) }) }) + Describe("generate", func() { It("works without a path", func() { command := exec.Command(pathCLI, "config", "generate") + command.Dir = "testdata/node" session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - session.Wait() Expect(err).ShouldNot(HaveOccurred()) + + session.Wait() + Eventually(session.Err.Contents()).Should(BeEmpty()) - Eventually(session.Out.Contents()).Should(MatchRegexp("#.*")) + Eventually(session.Out.Contents()).Should(MatchRegexp("npm run test")) + Eventually(session).Should(gexec.Exit(0)) }) + It("works with a path", func() { - command := exec.Command(pathCLI, "config", "generate", "..") + wd, err := os.Getwd() + Expect(err).ShouldNot(HaveOccurred()) + + command := exec.Command(pathCLI, "config", "generate", "node") + command.Dir = filepath.Join(wd, "testdata") session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) - session.Wait() Expect(err).ShouldNot(HaveOccurred()) + + session.Wait() + Eventually(session.Err.Contents()).Should(BeEmpty()) - Eventually(session.Out.Contents()).Should(MatchRegexp("#.*")) + Eventually(session.Out.Contents()).Should(MatchRegexp("npm run test")) + Eventually(session).Should(gexec.Exit(0)) }) }) }) From 6490c1831163975b2d5b4a3d1e73f517cdfff48d Mon Sep 17 00:00:00 2001 From: Brent Marks Date: Wed, 14 Jun 2023 18:28:55 -0400 Subject: [PATCH 3/5] pin frozen version of build-agent cli --- cmd/build.go | 2 -- local/local.go | 74 ++++++-------------------------------------------- 2 files changed, 8 insertions(+), 68 deletions(-) diff --git a/cmd/build.go b/cmd/build.go index 9d1d773f2..aec7b43db 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -22,8 +22,6 @@ func newLocalExecuteCommand(config *settings.Config) *cobra.Command { } local.AddFlagsForDocumentation(buildCommand.Flags()) - buildAgentVersionUsage := `The version of the build agent image you want to use. This can be configured by writing in $HOME/.circleci/build_agent_settings.json: '{"LatestSha256":""}'` - buildCommand.Flags().String("build-agent-version", "", buildAgentVersionUsage) buildCommand.Flags().StringP("org-slug", "o", "", "organization slug (for example: github/example-org), used when a config depends on private orbs belonging to that org") buildCommand.Flags().String("org-id", "", "organization id, used when a config depends on private orbs belonging to that org") diff --git a/local/local.go b/local/local.go index 32e55320c..c9fe792f5 100644 --- a/local/local.go +++ b/local/local.go @@ -1,12 +1,10 @@ package local import ( - "encoding/json" "fmt" "io" "os" "os/exec" - "path" "regexp" "strings" "syscall" @@ -73,9 +71,7 @@ func Execute(flags *pflag.FlagSet, cfg *settings.Config, args []string) error { return err } - picardVersion, _ := flags.GetString("build-agent-version") - image, err := picardImage(os.Stdout, picardVersion) - + image, err := picardImage(os.Stdout) if err != nil { return errors.Wrap(err, "Could not find picard image") } @@ -134,7 +130,7 @@ func buildAgentArguments(flags *pflag.FlagSet) ([]string, string) { // build a list of all supplied flags, that we will pass on to build-agent flags.Visit(func(flag *pflag.Flag) { - if flag.Name != "build-agent-version" && flag.Name != "org-slug" && flag.Name != "config" && flag.Name != "debug" && flag.Name != "org-id" && flag.Name != "docker-socket-path" { + if flag.Name != "org-slug" && flag.Name != "config" && flag.Name != "debug" && flag.Name != "org-id" && flag.Name != "docker-socket-path" { result = append(result, unparseFlag(flags, flag)...) } }) @@ -145,11 +141,10 @@ func buildAgentArguments(flags *pflag.FlagSet) ([]string, string) { return result, configPath } -func picardImage(output io.Writer, picardVersion string) (string, error) { - +func picardImage(output io.Writer) (string, error) { fmt.Fprintf(output, "Fetching latest build environment...\n") - sha, err := getPicardSha(output, picardVersion) + sha, err := findLatestPicardSha() if err != nil { return "", err } @@ -158,31 +153,6 @@ func picardImage(output io.Writer, picardVersion string) (string, error) { return fmt.Sprintf("%s@%s", picardRepo, sha), nil } -func getPicardSha(output io.Writer, picardVersion string) (string, error) { - // If the version was passed as argument, we take it - if picardVersion != "" { - return picardVersion, nil - } - - var sha string - var err error - - sha, err = loadBuildAgentShaFromConfig() - if sha != "" && err == nil { - return sha, nil - } - if err != nil && !os.IsNotExist(err) { - fmt.Fprintf(output, "Unable to parse JSON file %s because: %s\n", buildAgentSettingsPath(), err) - fmt.Fprintf(output, "Falling back to latest build-agent version\n") - } - - sha, err = findLatestPicardSha() - if err != nil { - return "", err - } - return sha, nil -} - func ensureDockerIsAvailable() (string, error) { dockerPath, err := exec.LookPath("docker") @@ -202,12 +172,14 @@ func ensureDockerIsAvailable() (string, error) { // Still depends on a function in cmd/build.go func findLatestPicardSha() (string, error) { - if _, err := ensureDockerIsAvailable(); err != nil { return "", err } - outputBytes, err := exec.Command("docker", "pull", picardRepo).CombinedOutput() // #nosec + // We are freezing build-agent cli as we would like to deprecate this path + // this is frozen to build-agent branch https://github.com/circleci/build-agent/tree/frozen-circleci-cli + imageName := fmt.Sprintf("%s:1.0.183556-5a2aabb6", picardRepo) + outputBytes, err := exec.Command("docker", "pull", imageName).CombinedOutput() // #nosec if err != nil { return "", errors.Wrap(err, "failed to pull latest docker image") @@ -224,36 +196,6 @@ func findLatestPicardSha() (string, error) { return latest, nil } -type buildAgentSettings struct { - LatestSha256 string -} - -func loadBuildAgentShaFromConfig() (string, error) { - if _, err := os.Stat(buildAgentSettingsPath()); os.IsNotExist(err) { - // Settings file does not exist. - return "", nil - } - - file, err := os.Open(buildAgentSettingsPath()) - if err != nil { - return "", errors.Wrap(err, "Could not open build settings config") - } - defer file.Close() - - var settings buildAgentSettings - - if err := json.NewDecoder(file).Decode(&settings); err != nil { - - return "", errors.Wrap(err, "Could not parse build settings config") - } - - return settings.LatestSha256, nil -} - -func buildAgentSettingsPath() string { - return path.Join(settings.SettingsPath(), "build_agent_settings.json") -} - // Write data to a temp file, and return the path to that file. func writeStringToTempFile(data string) (string, error) { // It's important to specify `/tmp` here as the location of the temp file. From 0d9fa65c3f0e8dbf33c3ee99b74486bb142e8efd Mon Sep 17 00:00:00 2001 From: JulesFaucherre Date: Thu, 15 Jun 2023 09:45:44 +0200 Subject: [PATCH 4/5] fix: Put back build-agent customisation --- cmd/build.go | 1 + local/local.go | 70 +++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/cmd/build.go b/cmd/build.go index aec7b43db..c32ed969c 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -22,6 +22,7 @@ func newLocalExecuteCommand(config *settings.Config) *cobra.Command { } local.AddFlagsForDocumentation(buildCommand.Flags()) + buildCommand.Flags().String("build-agent-version", "", `The version of the build agent image you want to use. This can be configured by writing in $HOME/.circleci/build_agent_settings.json: '{"LatestSha256":""}'`) buildCommand.Flags().StringP("org-slug", "o", "", "organization slug (for example: github/example-org), used when a config depends on private orbs belonging to that org") buildCommand.Flags().String("org-id", "", "organization id, used when a config depends on private orbs belonging to that org") diff --git a/local/local.go b/local/local.go index c9fe792f5..57af6b32b 100644 --- a/local/local.go +++ b/local/local.go @@ -1,10 +1,12 @@ package local import ( + "encoding/json" "fmt" "io" "os" "os/exec" + "path" "regexp" "strings" "syscall" @@ -71,7 +73,8 @@ func Execute(flags *pflag.FlagSet, cfg *settings.Config, args []string) error { return err } - image, err := picardImage(os.Stdout) + picardVersion, _ := flags.GetString("build-agent-version") + image, err := picardImage(os.Stdout, picardVersion) if err != nil { return errors.Wrap(err, "Could not find picard image") } @@ -130,7 +133,7 @@ func buildAgentArguments(flags *pflag.FlagSet) ([]string, string) { // build a list of all supplied flags, that we will pass on to build-agent flags.Visit(func(flag *pflag.Flag) { - if flag.Name != "org-slug" && flag.Name != "config" && flag.Name != "debug" && flag.Name != "org-id" && flag.Name != "docker-socket-path" { + if flag.Name != "build-agent-version" && flag.Name != "org-slug" && flag.Name != "config" && flag.Name != "debug" && flag.Name != "org-id" && flag.Name != "docker-socket-path" { result = append(result, unparseFlag(flags, flag)...) } }) @@ -141,10 +144,10 @@ func buildAgentArguments(flags *pflag.FlagSet) ([]string, string) { return result, configPath } -func picardImage(output io.Writer) (string, error) { +func picardImage(output io.Writer, picardVersion string) (string, error) { fmt.Fprintf(output, "Fetching latest build environment...\n") - sha, err := findLatestPicardSha() + sha, err := getPicardSha(output, picardVersion) if err != nil { return "", err } @@ -153,6 +156,31 @@ func picardImage(output io.Writer) (string, error) { return fmt.Sprintf("%s@%s", picardRepo, sha), nil } +func getPicardSha(output io.Writer, picardVersion string) (string, error) { + // If the version was passed as argument, we take it + if picardVersion != "" { + return picardVersion, nil + } + + var sha string + var err error + + sha, err = loadBuildAgentShaFromConfig() + if sha != "" && err == nil { + return sha, nil + } + if err != nil && !os.IsNotExist(err) { + fmt.Fprintf(output, "Unable to parse JSON file %s because: %s\n", buildAgentSettingsPath(), err) + fmt.Fprintf(output, "Falling back to latest build-agent version\n") + } + + sha, err = findLatestPicardSha() + if err != nil { + return "", err + } + return sha, nil +} + func ensureDockerIsAvailable() (string, error) { dockerPath, err := exec.LookPath("docker") @@ -247,3 +275,37 @@ func unparseFlag(flags *pflag.FlagSet, flag *pflag.Flag) []string { } return result } + +type buildAgentSettings struct { + LatestSha256 string +} + +func loadBuildAgentShaFromConfig() (string, error) { + if _, err := os.Stat(buildAgentSettingsPath()); os.IsNotExist(err) { + // Settings file does not exist. + return "", nil + } + + file, err := os.Open(buildAgentSettingsPath()) + if err != nil { + return "", errors.Wrap(err, "Could not open build settings config") + } + defer file.Close() + + var settings buildAgentSettings + + buf, err := io.ReadAll(file) + if err != nil { + return "", errors.Wrap(err, "Couldn't read from build settings file") + } + + if err = json.Unmarshal(buf, &settings); err != nil { + return "", errors.Wrap(err, "Could not parse build settings config") + } + + return settings.LatestSha256, nil +} + +func buildAgentSettingsPath() string { + return path.Join(settings.SettingsPath(), "build_agent_settings.json") +} From b18c5407e3a8e523c039e5a4f9dbfdea165fc530 Mon Sep 17 00:00:00 2001 From: JulesFaucherre Date: Thu, 15 Jun 2023 11:59:26 +0200 Subject: [PATCH 5/5] fix: Changed the fixed version --- local/local.go | 35 +++-------------------------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/local/local.go b/local/local.go index 57af6b32b..39272c91a 100644 --- a/local/local.go +++ b/local/local.go @@ -7,7 +7,6 @@ import ( "os" "os/exec" "path" - "regexp" "strings" "syscall" @@ -174,11 +173,9 @@ func getPicardSha(output io.Writer, picardVersion string) (string, error) { fmt.Fprintf(output, "Falling back to latest build-agent version\n") } - sha, err = findLatestPicardSha() - if err != nil { - return "", err - } - return sha, nil + // We are freezing build-agent cli as we would like to deprecate this path + fixedSha := "sha256:008ba7f4223f1e26c11df9575283491b620074fa96da6961e0dcde47fb757014" + return fixedSha, nil } func ensureDockerIsAvailable() (string, error) { @@ -198,32 +195,6 @@ func ensureDockerIsAvailable() (string, error) { return dockerPath, nil } -// Still depends on a function in cmd/build.go -func findLatestPicardSha() (string, error) { - if _, err := ensureDockerIsAvailable(); err != nil { - return "", err - } - - // We are freezing build-agent cli as we would like to deprecate this path - // this is frozen to build-agent branch https://github.com/circleci/build-agent/tree/frozen-circleci-cli - imageName := fmt.Sprintf("%s:1.0.183556-5a2aabb6", picardRepo) - outputBytes, err := exec.Command("docker", "pull", imageName).CombinedOutput() // #nosec - - if err != nil { - return "", errors.Wrap(err, "failed to pull latest docker image") - } - - output := string(outputBytes) - sha256 := regexp.MustCompile("(?m)sha256:[0-9a-f]+") - latest := sha256.FindString(output) - - if latest == "" { - return "", fmt.Errorf("failed to parse sha256 from docker pull output") - } - - return latest, nil -} - // Write data to a temp file, and return the path to that file. func writeStringToTempFile(data string) (string, error) { // It's important to specify `/tmp` here as the location of the temp file.