From 15af7f2f5e799c6f53d44690886f8bdf8658a393 Mon Sep 17 00:00:00 2001 From: Marc O'Morain Date: Wed, 4 Jul 2018 15:46:01 +0100 Subject: [PATCH] Add build command Port the majority of the current bash script that runs the existing CLI tool. The allows people to run `circleci build` locally using the new CLI. --- cmd/build.go | 203 +++++++++++++++++++++++++++++++++++++++++++++ cmd/config.go | 4 - cmd/config_test.go | 4 +- cmd/configure.go | 11 ++- cmd/orb_test.go | 4 +- cmd/root.go | 6 +- logger/logger.go | 9 -- 7 files changed, 217 insertions(+), 24 deletions(-) create mode 100644 cmd/build.go diff --git a/cmd/build.go b/cmd/build.go new file mode 100644 index 000000000..0e0afc580 --- /dev/null +++ b/cmd/build.go @@ -0,0 +1,203 @@ +package cmd + +import ( + "bufio" + "context" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path" + "regexp" + "strconv" + "strings" + + "github.com/CircleCI-Public/circleci-cli/settings" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +var dockerAPIVersion = "1.23" + +var job = "build" +var nodeTotal = 1 +var checkoutKey = "~/.ssh/id_rsa" +var skipCheckout = true + +func newBuildCommand() *cobra.Command { + + command := &cobra.Command{ + Use: "build", + Short: "Run a build", + RunE: runBuild, + } + + /* + TODO: Support these flags: + --branch string Git branch + -e, --env -e VAR=VAL Set environment variables, e.g. -e VAR=VAL + --index int node index of parallelism + --repo-url string Git Url + --revision string Git Revision + -v, --volume stringSlice Volume bind-mounting + */ + + flags := command.Flags() + flags.StringVarP(&dockerAPIVersion, "docker-api-version", "d", dockerAPIVersion, "The Docker API version to use") + flags.StringVar(&checkoutKey, "checkout-key", checkoutKey, "Git Checkout key") + flags.StringVar(&job, "job", job, "job to be executed") + flags.BoolVar(&skipCheckout, "skip-checkout", skipCheckout, "use local path as-is") + flags.IntVar(&nodeTotal, "node-total", nodeTotal, "total number of parallel nodes") + + return command +} + +var picardRepo = "circleci/picard" +var circleCiDir = path.Join(settings.UserHomeDir(), ".circleci") +var currentDigestFile = path.Join(circleCiDir, "current_picard_digest") +var latestDigestFile = path.Join(circleCiDir, "latest_picard_digest") + +func getDigest(file string) string { + + if _, err := os.Stat(file); !os.IsNotExist(err) { + digest, err := ioutil.ReadFile(file) + + if err != nil { + Logger.Error("Could not load digest file", err) + } else { + return strings.TrimSpace(string(digest)) + } + } + + return "" // Unknown digest +} + +func newPullLatestImageCommand() *exec.Cmd { + return exec.Command("docker", "pull", picardRepo) +} + +func getLatestImageSha() (string, error) { + + cmd := newPullLatestImageCommand() + Logger.Info("Pulling latest build image") + bytes, err := cmd.CombinedOutput() + + if err != nil { + return "", errors.Wrap(err, "failed to pull latest image") + } + + output := string(bytes) + + sha256 := regexp.MustCompile("(?m)sha256.*$") + + //latest_image=$(docker pull picardRepo | grep -o "sha256.*$") + //echo $latest_image > $LATEST_DIGEST_FILE + latest := sha256.FindString(output) + + if latest == "" { + return latest, errors.New("Failed to find latest image") + } + + return latest, nil +} + +func picardImage() (string, error) { + currentDigest := getDigest(currentDigestFile) + + if currentDigest == "" { + // Receiving latest image of picard in case of there's no current digest stored + Logger.Info("Downloading latest CircleCI build agent...") + + var err error + + currentDigest, err = getLatestImageSha() + + // TODO - write the digest to a file so that we can + // use it again. + + if err != nil { + return "", err + } + + } else { + // Otherwise pulling latest image in background + // TODO - this should write to a file to record + // the fact that we have the latest image. + newPullLatestImageCommand().Start() + + } + + Logger.Infof("Docker image digest: %s", currentDigest) + + return fmt.Sprintf("%s@%s", picardRepo, currentDigest), nil +} + +func streamOutout(stream io.Reader) { + scanner := bufio.NewScanner(stream) + go func() { + for scanner.Scan() { + Logger.Info(scanner.Text()) + } + }() +} + +func runBuild(cmd *cobra.Command, args []string) error { + + ctx := context.Background() + + pwd, err := os.Getwd() + + if err != nil { + return errors.Wrap(err, "Could not find pwd") + } + + image, err := picardImage() + + if err != nil { + return errors.Wrap(err, "Could not find picard image") + } + + // TODO: marc: + // I can't find a way to pass `-it` and have carriage return + // characters work correctly. + arguments := []string{"run", "--rm", + "-e", fmt.Sprintf("DOCKER_API_VERSION=%s", dockerAPIVersion), + "-v", "/var/run/docker.sock:/var/run/docker.sock", + "-v", fmt.Sprintf("%s:%s", pwd, pwd), + "-v", fmt.Sprintf("%s:/root/.circleci", circleCiDir), + "--workdir", pwd, + image, "circleci", "build", + + // Proxied arguments + "--config", configPath, + "--skip-checkout", strconv.FormatBool(skipCheckout), + "--node-total", strconv.Itoa(nodeTotal), + "--checkout-key", checkoutKey, + "--job", job} + + arguments = append(arguments, args...) + + Logger.Debug(fmt.Sprintf("Starting docker with args: %s", arguments)) + + build := exec.CommandContext(ctx, "docker", arguments...) + + build.Stdin = os.Stdin + + stdout, err := build.StdoutPipe() + + if err != nil { + return errors.Wrap(err, "Failed to connect to stdout") + } + + stderr, err := build.StderrPipe() + + if err != nil { + return errors.Wrap(err, "Failed to connect to stderr") + } + + streamOutout(stdout) + streamOutout(stderr) + + return build.Run() +} diff --git a/cmd/config.go b/cmd/config.go index e28062bac..f686076f0 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -7,9 +7,6 @@ import ( "github.com/spf13/cobra" ) -// Path to the config.yml file to operate on. -var configPath string - func newConfigCommand() *cobra.Command { configCmd := &cobra.Command{ Use: "config", @@ -29,7 +26,6 @@ func newConfigCommand() *cobra.Command { RunE: expandConfig, } - configCmd.PersistentFlags().StringVarP(&configPath, "path", "p", ".circleci/config.yml", "path to build config") configCmd.AddCommand(validateCommand) configCmd.AddCommand(expandCommand) diff --git a/cmd/config_test.go b/cmd/config_test.go index be179adf4..19edfde45 100644 --- a/cmd/config_test.go +++ b/cmd/config_test.go @@ -44,7 +44,7 @@ var _ = Describe("Config", func() { "config", "validate", "-t", token, "-e", testServer.URL(), - "-p", config.Path, + "-c", config.Path, ) }) @@ -120,7 +120,7 @@ var _ = Describe("Config", func() { "config", "expand", "-t", token, "-e", testServer.URL(), - "-p", config.Path, + "-c", config.Path, ) }) diff --git a/cmd/configure.go b/cmd/configure.go index 33761ba32..8d21665b5 100644 --- a/cmd/configure.go +++ b/cmd/configure.go @@ -1,7 +1,6 @@ package cmd import ( - "fmt" "strings" "github.com/pkg/errors" @@ -76,12 +75,12 @@ type testingUI struct { } func (ui testingUI) readStringFromUser(message string, defaultValue string) string { - fmt.Println(message) + Logger.Info(message) return ui.input } func (ui testingUI) askUserToConfirm(message string) bool { - fmt.Println(message) + Logger.Info(message) return ui.confirm } @@ -108,10 +107,10 @@ func configure(cmd *cobra.Command, args []string) error { if shouldAskForToken(token, ui) { viper.Set("token", ui.readStringFromUser("CircleCI API Token", "")) - fmt.Println("API token has been set.") + Logger.Info("API token has been set.") } viper.Set("endpoint", ui.readStringFromUser("CircleCI API End Point", viper.GetString("endpoint"))) - fmt.Println("API endpoint has been set.") + Logger.Info("API endpoint has been set.") // Marc: I can't find a way to prevent the verbose flag from // being written to the config file, so set it to false in @@ -122,6 +121,6 @@ func configure(cmd *cobra.Command, args []string) error { return errors.Wrap(err, "Failed to save config file") } - fmt.Println("Configuration has been saved.") + Logger.Info("Configuration has been saved.") return nil } diff --git a/cmd/orb_test.go b/cmd/orb_test.go index 13bda98d6..9f6421073 100644 --- a/cmd/orb_test.go +++ b/cmd/orb_test.go @@ -44,7 +44,7 @@ var _ = Describe("Orb", func() { "orb", "validate", "-t", token, "-e", testServer.URL(), - "-p", orb.Path, + "-c", orb.Path, ) }) @@ -123,7 +123,7 @@ var _ = Describe("Orb", func() { "orb", "expand", "-t", token, "-e", testServer.URL(), - "-p", orb.Path, + "-c", orb.Path, ) }) diff --git a/cmd/root.go b/cmd/root.go index bbb9360dd..f8f62d033 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,6 +13,9 @@ import ( var defaultEndpoint = "https://circleci.com/graphql-unstable" +// Path to the config.yml file to operate on. +var configPath = ".circleci/config.yml" + // Execute adds all child commands to rootCmd and // sets flags appropriately. This function is called // by main.main(). It only needs to happen once to @@ -42,10 +45,11 @@ func makeCommands() *cobra.Command { rootCmd.AddCommand(newConfigureCommand()) rootCmd.AddCommand(newConfigCommand()) rootCmd.AddCommand(newOrbCommand()) - + rootCmd.AddCommand(newBuildCommand()) rootCmd.PersistentFlags().BoolP("verbose", "v", false, "Enable verbose logging.") rootCmd.PersistentFlags().StringP("endpoint", "e", defaultEndpoint, "the endpoint of your CircleCI GraphQL API") rootCmd.PersistentFlags().StringP("token", "t", "", "your token for using CircleCI") + rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", configPath, "path to build config") for _, flag := range []string{"endpoint", "token", "verbose"} { bindCobraFlagToViper(rootCmd, flag) diff --git a/logger/logger.go b/logger/logger.go index 8361f9abf..51b2acef1 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -66,15 +66,6 @@ func (l *Logger) Error(msg string, err error) { } } -// FatalOnError prints a message and error's message to os.Stderr then QUITS! -// Please be aware this method will exit the program via os.Exit(1). -// This method wraps log.Logger.Fatalln -func (l *Logger) FatalOnError(msg string, err error) { - if err != nil { - l.error.Fatalln(msg, err.Error()) - } -} - // Prettyify accepts a map of data and pretty prints it. // It's using json.MarshalIndent and printing with log.Logger.Infoln func (l *Logger) Prettyify(data interface{}) {