diff --git a/cli/cdsctl/workflow_artifact.go b/cli/cdsctl/workflow_artifact.go index 323152656d..a3da655b28 100644 --- a/cli/cdsctl/workflow_artifact.go +++ b/cli/cdsctl/workflow_artifact.go @@ -97,6 +97,11 @@ var workflowArtifactDownloadCmd = cli.Command{ Usage: "exclude files from download - could be a regex: *.log", Default: "", }, + { + Name: "cdn-url", + Usage: "overwrite cdn url", + Default: "", + }, }, } @@ -106,10 +111,15 @@ func workflowArtifactDownloadRun(v cli.Values) error { return cli.NewError("number parameter have to be an integer") } - confCDN, err := client.ConfigCDN() - if err != nil { - return err + cdnURL := v.GetString("cdn-url") + if cdnURL == "" { + confCDN, err := client.ConfigCDN() + if err != nil { + return err + } + cdnURL = confCDN.HTTPURL } + ok, err := downloadFromCDSAPI(v, number) if err != nil { return err @@ -176,7 +186,7 @@ func workflowArtifactDownloadRun(v cli.Values) error { return err } fmt.Printf("Downloading %s...\n", artifactData.Name) - r, err := client.CDNItemDownload(context.Background(), confCDN.HTTPURL, artifactData.CDNRefHash, sdk.CDNTypeItemRunResult) + r, err := client.CDNItemDownload(context.Background(), cdnURL, artifactData.CDNRefHash, sdk.CDNTypeItemRunResult) if err != nil { return err } diff --git a/contrib/integrations/artifactory/artifactory.go b/contrib/integrations/artifactory/artifactory.go index e9d46a0db1..895d044998 100644 --- a/contrib/integrations/artifactory/artifactory.go +++ b/contrib/integrations/artifactory/artifactory.go @@ -4,12 +4,16 @@ import ( "encoding/json" "fmt" "net/url" + "strings" "time" "github.com/jfrog/jfrog-client-go/artifactory" "github.com/jfrog/jfrog-client-go/artifactory/auth" "github.com/jfrog/jfrog-client-go/artifactory/services" "github.com/jfrog/jfrog-client-go/config" + "github.com/jfrog/jfrog-client-go/distribution" + authdistrib "github.com/jfrog/jfrog-client-go/distribution/auth" + "github.com/ovh/cds/sdk" ) @@ -43,6 +47,21 @@ type FileChildren struct { Folder bool `json:"folder"` } +func CreateDistributionClient(url, token string) (*distribution.DistributionServicesManager, error) { + dtb := authdistrib.NewDistributionDetails() + dtb.SetUrl(strings.Replace(url, "/artifactory/", "/distribution/", -1)) + dtb.SetAccessToken(token) + serviceConfig, err := config.NewConfigBuilder(). + SetServiceDetails(dtb). + SetThreads(1). + SetDryRun(false). + Build() + if err != nil { + return nil, fmt.Errorf("unable to create service config: %v", err) + } + return distribution.New(serviceConfig) +} + func CreateArtifactoryClient(url, token string) (artifactory.ArtifactoryServicesManager, error) { rtDetails := auth.NewArtifactoryDetails() rtDetails.SetUrl(url) diff --git a/contrib/integrations/artifactory/plugin-artifactory-build-info/main.go b/contrib/integrations/artifactory/plugin-artifactory-build-info/main.go index 733604f362..2fabd6b9e6 100644 --- a/contrib/integrations/artifactory/plugin-artifactory-build-info/main.go +++ b/contrib/integrations/artifactory/plugin-artifactory-build-info/main.go @@ -50,7 +50,6 @@ type executionContext struct { workflowName string version string lowMaturitySuffix string - cdsRepo string } func (e *artifactoryBuildInfoPlugin) Manifest(_ context.Context, _ *empty.Empty) (*integrationplugin.IntegrationPluginManifest, error) { @@ -67,7 +66,6 @@ func (e *artifactoryBuildInfoPlugin) Run(_ context.Context, opts *integrationplu token := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigToken)] tokenName := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigTokenName)] lowMaturitySuffix := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigPromotionLowMaturity)] - cdsRepo := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigCdsRepository)] buildInfo := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigBuildInfoPath)] version := opts.GetOptions()["cds.version"] @@ -117,7 +115,6 @@ func (e *artifactoryBuildInfoPlugin) Run(_ context.Context, opts *integrationplu workflowName: workflowName, version: version, projectKey: projectKey, - cdsRepo: cdsRepo, } modules, err := e.computeBuildInfoModules(artiClient, execContext) if err != nil { @@ -195,9 +192,7 @@ func (e *artifactoryBuildInfoPlugin) retrieveModulesArtifacts(client artifactory props["build.number"] = execContext.version props["build.timestamp"] = strconv.FormatInt(time.Now().Unix(), 10) repoSrc := repoName - if repoName != execContext.cdsRepo { - repoSrc += "-" + execContext.lowMaturitySuffix - } + repoSrc += "-" + execContext.lowMaturitySuffix if err := art.SetProperties(client, repoSrc, path, props); err != nil { return nil, err } diff --git a/contrib/integrations/artifactory/plugin-artifactory-release/Makefile b/contrib/integrations/artifactory/plugin-artifactory-release/Makefile new file mode 100644 index 0000000000..2069cd5a74 --- /dev/null +++ b/contrib/integrations/artifactory/plugin-artifactory-release/Makefile @@ -0,0 +1,59 @@ +.PHONY: clean + +VERSION := $(if ${CDS_SEMVER},${CDS_SEMVER},snapshot) +GITHASH := $(if ${GIT_HASH},${GIT_HASH},`git log -1 --format="%H"`) +BUILDTIME := `date "+%m/%d/%y-%H:%M:%S"` +CDSCTL := $(if ${CDSCTL},${CDSCTL},cdsctl) + +TARGET_DIR = ./dist +TARGET_NAME = plugin-artifactory-release + +define PLUGIN_MANIFEST_BINARY +os: %os% +arch: %arch% +cmd: ./%filename% +endef +export PLUGIN_MANIFEST_BINARY + +TARGET_LDFLAGS = -ldflags "-X github.com/ovh/cds/sdk.VERSION=$(VERSION) -X github.com/ovh/cds/sdk.GOOS=$$GOOS -X github.com/ovh/cds/sdk.GOARCH=$$GOARCH -X github.com/ovh/cds/sdk.GITHASH=$(GITHASH) -X github.com/ovh/cds/sdk.BUILDTIME=$(BUILDTIME) -X github.com/ovh/cds/sdk.BINARY=$(TARGET_NAME)" +TARGET_OS = $(if ${OS},${OS},windows darwin linux freebsd) +TARGET_ARCH = $(if ${ARCH},${ARCH},amd64 arm 386 arm64) + +GO_BUILD = go build + +$(TARGET_DIR): + $(info create $(TARGET_DIR) directory) + @mkdir -p $(TARGET_DIR) + +default: build + +clean: + @rm -rf $(TARGET_DIR) + +build: $(TARGET_DIR) + @cp $(TARGET_NAME).yml $(TARGET_DIR)/$(TARGET_NAME).yml + @for GOOS in $(TARGET_OS); do \ + for GOARCH in $(TARGET_ARCH); do \ + EXTENSION=""; \ + if test "$$GOOS" = "windows" ; then EXTENSION=".exe"; fi; \ + echo Compiling $(TARGET_DIR)/$(TARGET_NAME)-$$GOOS-$$GOARCH$$EXTENSION $(VERSION); \ + FILENAME=$(TARGET_NAME)-$$GOOS-$$GOARCH$$EXTENSION; \ + GOOS=$$GOOS GOARCH=$$GOARCH CGO_ENABLED=0 $(GO_BUILD) $(TARGET_LDFLAGS) -o $(TARGET_DIR)/$$FILENAME; \ + echo "$$PLUGIN_MANIFEST_BINARY" > $(TARGET_DIR)/plugin-artifactory-release-$$GOOS-$$GOARCH.yml; \ + perl -pi -e s,%os%,$$GOOS,g $(TARGET_DIR)/plugin-artifactory-release-$$GOOS-$$GOARCH.yml; \ + perl -pi -e s,%arch%,$$GOARCH,g $(TARGET_DIR)/plugin-artifactory-release-$$GOOS-$$GOARCH.yml; \ + perl -pi -e s,%filename%,$$FILENAME,g $(TARGET_DIR)/plugin-artifactory-release-$$GOOS-$$GOARCH.yml; \ + done; \ + done + +publish: + @echo "Updating plugin..." + $(CDSCTL) admin plugins import $(TARGET_DIR)/$(TARGET_NAME).yml + @for GOOS in $(TARGET_OS); do \ + for GOARCH in $(TARGET_ARCH); do \ + EXTENSION=""; \ + if test "$$GOOS" = "windows" ; then EXTENSION=".exe"; fi; \ + echo "Updating plugin binary $(TARGET_NAME)-$$GOOS-$$GOARCH$$EXTENSION"; \ + $(CDSCTL) admin plugins binary-add artifactory-release-plugin $(TARGET_DIR)/$(TARGET_NAME)-$$GOOS-$$GOARCH.yml $(TARGET_DIR)/$(TARGET_NAME)-$$GOOS-$$GOARCH$$EXTENSION; \ + done; \ + done diff --git a/contrib/integrations/artifactory/plugin-artifactory-release/main.go b/contrib/integrations/artifactory/plugin-artifactory-release/main.go new file mode 100644 index 0000000000..537601378b --- /dev/null +++ b/contrib/integrations/artifactory/plugin-artifactory-release/main.go @@ -0,0 +1,303 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "path/filepath" + + "os" + "regexp" + "strings" + + "github.com/golang/protobuf/ptypes/empty" + "github.com/jfrog/jfrog-client-go/artifactory" + artService "github.com/jfrog/jfrog-client-go/artifactory/services" + "github.com/jfrog/jfrog-client-go/artifactory/services/utils" + "github.com/jfrog/jfrog-client-go/distribution" + authdistrib "github.com/jfrog/jfrog-client-go/distribution/auth" + "github.com/jfrog/jfrog-client-go/distribution/services" + distriUtils "github.com/jfrog/jfrog-client-go/distribution/services/utils" + "github.com/jfrog/jfrog-client-go/utils/log" + + "github.com/ovh/cds/contrib/grpcplugins" + art "github.com/ovh/cds/contrib/integrations/artifactory" + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/grpcplugin/integrationplugin" +) + +/* +This plugin have to be used as a releaseplugin + +Artifactory release plugin must configured as following: + name: artifactory-release-plugin + type: integration + author: "Steven Guiheux" + description: "OVH Artifactory Release Plugin" + +$ cdsctl admin plugins import artifactory-release-plugin.yml + +Build the present binaries and import in CDS: + os: linux + arch: amd64 + cmd: + +$ cdsctl admin plugins binary-add artifactory-release-plugin artifactory-release-plugin-bin.yml +*/ + +type artifactoryReleasePlugin struct { + integrationplugin.Common +} + +type EdgeNode struct { + Name string `json:"name"` + SiteName string `json:"site_name"` + City struct { + Name string `json:"name"` + CountryCode string `json:"country_code"` + } `json:"city"` + LicenseType string `json:"license_type"` + LicenseStatus string `json:"license_status"` +} + +func (e *artifactoryReleasePlugin) Manifest(_ context.Context, _ *empty.Empty) (*integrationplugin.IntegrationPluginManifest, error) { + return &integrationplugin.IntegrationPluginManifest{ + Name: "OVH Artifactory Release Plugin", + Author: "Steven Guiheux", + Description: "OVH Artifactory Release Plugin", + Version: sdk.VERSION, + }, nil +} + +func (e *artifactoryReleasePlugin) Run(_ context.Context, opts *integrationplugin.RunQuery) (*integrationplugin.RunResult, error) { + artifactoryURL := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigURL)] + token := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigToken)] + releaseToken := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigReleaseToken)] + buildInfo := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigBuildInfoPath)] + lowMaturitySuffix := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigPromotionLowMaturity)] + highMaturitySuffix := opts.GetOptions()[fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactManagerConfigPromotionHighMaturity)] + + version := opts.GetOptions()["cds.version"] + projectKey := opts.GetOptions()["cds.project"] + workflowName := opts.GetOptions()["cds.workflow"] + + artifactList := opts.GetOptions()["artifacts"] + releaseNote := opts.GetOptions()["releaseNote"] + + runResult, err := grpcplugins.GetRunResults(e.HTTPPort) + if err != nil { + return fail("unable to list run results: %v", err) + } + + log.SetLogger(log.NewLogger(log.ERROR, os.Stdout)) + distriClient, err := art.CreateDistributionClient(artifactoryURL, releaseToken) + if err != nil { + return fail("unable to create distribution client: %v", err) + } + + // Promotion + artiClient, err := art.CreateArtifactoryClient(artifactoryURL, token) + if err != nil { + return fail("unable to create artifactory client: %v", err) + } + + artSplitted := strings.Split(artifactList, ",") + artRegs := make([]*regexp.Regexp, 0, len(artSplitted)) + for _, art := range artSplitted { + r, err := regexp.Compile(art) + if err != nil { + return fail("unable compile regexp in artifact list: %v", err) + } + artRegs = append(artRegs, r) + } + + type promotedArtifact struct { + Pattern string + Target string + } + + artifactPromoted := make([]promotedArtifact, 0) + for _, r := range runResult { + rData, err := r.GetArtifactManager() + if err != nil { + return fail("unable to read result %s: %v", r.ID, err) + } + skip := true + for _, reg := range artRegs { + if reg.MatchString(rData.Name) { + skip = false + break + } + } + if skip { + continue + } + switch rData.RepoType { + case "docker": + if err := e.promoteDockerImage(artiClient, rData, lowMaturitySuffix, highMaturitySuffix); err != nil { + return fail("unable to promote docker image: %s: %v", rData.Name+"-"+highMaturitySuffix, err) + } + + // Pattern must be like: "//(*)" + // Target must be like: "//$1" + artifactPromoted = append(artifactPromoted, promotedArtifact{ + Pattern: fmt.Sprintf("%s/%s/(*)", rData.RepoName+"-"+highMaturitySuffix, rData.Path), + Target: fmt.Sprintf("%s/%s/{1}", rData.RepoName, rData.Path), + }) + default: + if err := e.promoteFile(artiClient, rData, lowMaturitySuffix, highMaturitySuffix); err != nil { + return fail("unable to promote file: %s: %v", rData.Name, err) + } + dir, _ := filepath.Split(rData.Path) + // Pattern must be like: "//(*)" + // Target must be like: "//$1" + artifactPromoted = append(artifactPromoted, promotedArtifact{ + Pattern: fmt.Sprintf("%s/%s(*)", rData.RepoName+"-"+highMaturitySuffix, dir), + Target: fmt.Sprintf("%s/%s{1}", rData.RepoName, dir), + }) + } + + } + + // Release bundle + buildInfoName := fmt.Sprintf("%s/%s/%s", buildInfo, projectKey, workflowName) + + params := services.NewCreateReleaseBundleParams(strings.Replace(buildInfoName, "/", "-", -1), version) + params.ReleaseNotes = releaseNote + params.ReleaseNotesSyntax = "plain_text" + + paramsBuild := fmt.Sprintf("%s/%s", strings.Replace(buildInfoName, "/", "\\/", -1), version) + if artifactList == "" { + params.SpecFiles = []*utils.ArtifactoryCommonParams{ + { + Recursive: true, + Build: paramsBuild, + }, + } + } else { + params.SpecFiles = make([]*utils.ArtifactoryCommonParams, 0, len(artifactPromoted)) + for _, art := range artifactPromoted { + query := &utils.ArtifactoryCommonParams{ + Recursive: true, + Build: paramsBuild, + Pattern: art.Pattern, + Target: art.Target, + } + params.SpecFiles = append(params.SpecFiles, query) + } + } + params.SignImmediately = true + fmt.Printf("Creating release %s %s\n", params.Name, params.Version) + if _, err := distriClient.CreateReleaseBundle(params); err != nil { + return fail("unable to create release bundle: %v", err) + } + + fmt.Printf("Listing Edge nodes to distribute the release \n") + edges, err := e.listEdgeNodes(distriClient, artifactoryURL, releaseToken) + if err != nil { + return fail("%v", err) + } + edges = e.removeNonEdge(edges) + + fmt.Printf("Distribute Release %s %s\n", params.Name, params.Version) + distributionParams := services.NewDistributeReleaseBundleParams(params.Name, params.Version) + distributionParams.DistributionRules = make([]*distriUtils.DistributionCommonParams, 0, len(edges)) + for _, e := range edges { + distributionParams.DistributionRules = append(distributionParams.DistributionRules, &distriUtils.DistributionCommonParams{ + SiteName: e.SiteName, + CityName: e.City.Name, + CountryCodes: []string{e.City.CountryCode}, + }) + } + if err := distriClient.DistributeReleaseBundle(distributionParams); err != nil { + return fail("unable to distribution version: %v", err) + } + + return &integrationplugin.RunResult{ + Status: sdk.StatusSuccess, + }, nil +} + +func (e *artifactoryReleasePlugin) listEdgeNodes(distriClient *distribution.DistributionServicesManager, url, token string) ([]EdgeNode, error) { + // action=x distribute + listEdgeNodePath := fmt.Sprintf("api/ui/distribution/edge_nodes?action=x") + dtb := authdistrib.NewDistributionDetails() + dtb.SetUrl(strings.Replace(url, "/artifactory/", "/distribution/", -1)) + dtb.SetAccessToken(token) + + fakeService := services.NewCreateReleaseBundleService(distriClient.Client()) + fakeService.DistDetails = dtb + clientDetail := fakeService.DistDetails.CreateHttpClientDetails() + listEdgeURL := fmt.Sprintf("%s%s", fakeService.DistDetails.GetUrl(), listEdgeNodePath) + utils.SetContentType("application/json", &clientDetail.Headers) + + resp, body, _, err := distriClient.Client().SendGet(listEdgeURL, true, &clientDetail) + if err != nil { + return nil, fmt.Errorf("unable to list edge node from distribution: %v", err) + } + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("http error %d: %s", resp.StatusCode, string(body)) + } + + var edges []EdgeNode + if err := json.Unmarshal(body, &edges); err != nil { + return nil, fmt.Errorf("unable to unmarshal response %s: %v", string(body), err) + } + return edges, nil +} + +func (e *artifactoryReleasePlugin) removeNonEdge(edges []EdgeNode) []EdgeNode { + edgeFiltered := make([]EdgeNode, 0, len(edges)) + for _, e := range edges { + if e.LicenseType != "EDGE" { + continue + } + edgeFiltered = append(edgeFiltered, e) + } + return edgeFiltered +} + +func (e *artifactoryReleasePlugin) promoteFile(artiClient artifactory.ArtifactoryServicesManager, data sdk.WorkflowRunResultArtifactManager, lowMaturity, highMaturity string) error { + srcRepo := fmt.Sprintf("%s-%s", data.RepoName, lowMaturity) + targetRepo := fmt.Sprintf("%s-%s", data.RepoName, highMaturity) + params := artService.NewMoveCopyParams() + params.Pattern = fmt.Sprintf("%s/%s", srcRepo, data.Path) + params.Target = fmt.Sprintf("%s/%s", targetRepo, data.Path) + params.Flat = true + fmt.Printf("Promoting file %s from %s to %s\n", data.Name, srcRepo, targetRepo) + nbSuccess, nbFailed, err := artiClient.Move(params) + if err != nil { + return err + } + if nbFailed > 0 || nbSuccess == 0 { + return fmt.Errorf("%s: copy failed with no reason", data.Name) + } + return nil +} + +func (e *artifactoryReleasePlugin) promoteDockerImage(artiClient artifactory.ArtifactoryServicesManager, data sdk.WorkflowRunResultArtifactManager, lowMaturity, highMaturity string) error { + sourceRepo := fmt.Sprintf("%s-%s", data.RepoName, lowMaturity) + targetRepo := fmt.Sprintf("%s-%s", data.RepoName, highMaturity) + params := artService.NewDockerPromoteParams(data.Path, sourceRepo, targetRepo) + params.Copy = false + fmt.Printf("Promoting docker image %s from %s to %s\n", data.Name, params.SourceRepo, params.TargetRepo) + return artiClient.PromoteDocker(params) +} + +func main() { + e := artifactoryReleasePlugin{} + if err := integrationplugin.Start(context.Background(), &e); err != nil { + panic(err) + } + return + +} + +func fail(format string, args ...interface{}) (*integrationplugin.RunResult, error) { + msg := fmt.Sprintf(format, args...) + fmt.Println(msg) + return &integrationplugin.RunResult{ + Details: msg, + Status: sdk.StatusFail, + }, nil +} diff --git a/contrib/integrations/artifactory/plugin-artifactory-release/plugin-artifactory-release.yml b/contrib/integrations/artifactory/plugin-artifactory-release/plugin-artifactory-release.yml new file mode 100644 index 0000000000..2142e57a53 --- /dev/null +++ b/contrib/integrations/artifactory/plugin-artifactory-release/plugin-artifactory-release.yml @@ -0,0 +1,5 @@ +name: artifactory-release-plugin +type: integration-release +integration: ArtifactManager +author: "OVH SAS" +description: "OVH Artifactory Release Plugin" diff --git a/engine/worker/internal/action/builtin_release.go b/engine/worker/internal/action/builtin_release.go index bb8316ab01..85941b45b8 100644 --- a/engine/worker/internal/action/builtin_release.go +++ b/engine/worker/internal/action/builtin_release.go @@ -4,79 +4,97 @@ import ( "context" "errors" "fmt" - "strconv" "strings" - "github.com/rockbears/log" + "github.com/golang/protobuf/ptypes/empty" "github.com/ovh/cds/engine/worker/pkg/workerruntime" "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/grpcplugin/integrationplugin" ) -func RunReleaseVCS(ctx context.Context, wk workerruntime.Runtime, a sdk.Action, secrets []sdk.Variable) (sdk.Result, error) { - var res sdk.Result - res.Status = sdk.StatusFail - jobID, err := workerruntime.JobID(ctx) - if err != nil { - return res, err +func RunRelease(ctx context.Context, wk workerruntime.Runtime, a sdk.Action, _ []sdk.Variable) (sdk.Result, error) { + pfName := sdk.ParameterFind(wk.Parameters(), "cds.integration.artifact_manager") + if pfName == nil { + return sdk.Result{}, errors.New("unable to retrieve artifact manager integration... Aborting") } - artifactList := sdk.ParameterFind(a.Parameters, "artifacts") - tag := sdk.ParameterFind(a.Parameters, "tag") - title := sdk.ParameterFind(a.Parameters, "title") - releaseNote := sdk.ParameterFind(a.Parameters, "releaseNote") - - pkey := sdk.ParameterFind(wk.Parameters(), "cds.project") - wName := sdk.ParameterFind(wk.Parameters(), "cds.workflow") - workflowNum := sdk.ParameterFind(wk.Parameters(), "cds.run.number") + plugin := wk.GetPlugin(sdk.GRPCPluginRelease) + if plugin == nil { + return sdk.Result{}, sdk.NewErrorFrom(sdk.ErrNotFound, "unable to find plugin of type %s", sdk.GRPCPluginRelease) + } - if pkey == nil { - return res, errors.New("cds.project variable not found") + //First check OS and Architecture + binary := plugin.GetBinary(strings.ToLower(sdk.GOOS), strings.ToLower(sdk.GOARCH)) + if binary == nil { + return sdk.Result{}, fmt.Errorf("unable to retrieve the plugin for release on integration %s... Aborting", pfName.Value) } - if wName == nil { - return res, errors.New("cds.workflow variable not found") + pluginSocket, err := startGRPCPlugin(ctx, binary.PluginName, wk, binary, startGRPCPluginOptions{}) + if err != nil { + return sdk.Result{}, fmt.Errorf("unable to start GRPCPlugin: %v", err) } - if workflowNum == nil { - return res, errors.New("cds.run.number variable not found") + c, err := integrationplugin.Client(context.Background(), pluginSocket.Socket) + if err != nil { + return sdk.Result{}, fmt.Errorf("unable to call GRPCPlugin: %v", err) } - if tag == nil || tag.Value == "" { - return res, errors.New("tag name is not set. Nothing to perform") + qPort := integrationplugin.WorkerHTTPPortQuery{Port: wk.HTTPPort()} + if _, err := c.WorkerHTTPPort(ctx, &qPort); err != nil { + return sdk.Result{}, fmt.Errorf("unable to setup plugin with worker port: %v", err) } - if title == nil || title.Value == "" { - return res, errors.New("release title is not set") + pluginSocket.Client = c + if _, err := c.Manifest(context.Background(), new(empty.Empty)); err != nil { + return sdk.Result{}, fmt.Errorf("unable to call GRPCPlugin: %v", err) } - if releaseNote == nil || releaseNote.Value == "" { - return res, errors.New("release note is not set") + pluginClient := pluginSocket.Client + integrationPluginClient, ok := pluginClient.(integrationplugin.IntegrationPluginClient) + if !ok { + return sdk.Result{}, fmt.Errorf("unable to retrieve integration GRPCPlugin: %v", err) } - wRunNumber, errI := strconv.ParseInt(workflowNum.Value, 10, 64) - if errI != nil { - return res, fmt.Errorf("Workflow number is not a number. Got %s: %s", workflowNum.Value, errI) + logCtx, stopLogs := context.WithCancel(ctx) + done := make(chan struct{}) + go enablePluginLogger(logCtx, done, pluginSocket, wk) + + defer integrationPluginClientStop(ctx, integrationPluginClient, done, stopLogs) + + manifest, err := integrationPluginClient.Manifest(ctx, &empty.Empty{}) + if err != nil { + integrationPluginClientStop(ctx, integrationPluginClient, done, stopLogs) + return sdk.Result{}, fmt.Errorf("unable to retrieve retrieve plugin manifest: %v", err) } - artSplitted := strings.Split(artifactList.Value, ",") - req := sdk.WorkflowNodeRunRelease{ - ReleaseContent: releaseNote.Value, - ReleaseTitle: title.Value, - TagName: tag.Value, - Artifacts: artSplitted, + wk.SendLog(ctx, workerruntime.LevelInfo, fmt.Sprintf("# Plugin %s v%s is ready", manifest.Name, manifest.Version)) + + query := integrationplugin.RunQuery{ + Options: sdk.ParametersToMap(wk.Parameters()), + } + for _, v := range a.Parameters { + query.Options[v.Name] = v.Value } - jobrun, err := wk.Client().QueueJobInfo(ctx, jobID) + res, err := integrationPluginClient.Run(ctx, &query) if err != nil { - return res, fmt.Errorf("unable to get job info: %v", err) + integrationPluginClientStop(ctx, integrationPluginClient, done, stopLogs) + return sdk.Result{}, fmt.Errorf("error while running integration plugin: %v", err) } - log.Info(ctx, "RunRelease> jobRunID=%v WorkflowNodeRunID:%v", jobID, jobrun.WorkflowNodeRunID) + wk.SendLog(ctx, workerruntime.LevelInfo, fmt.Sprintf("# Details: %s", res.Details)) + wk.SendLog(ctx, workerruntime.LevelInfo, fmt.Sprintf("# Status: %s", res.Status)) - if err := wk.Client().WorkflowNodeRunRelease(pkey.Value, wName.Value, wRunNumber, jobrun.WorkflowNodeRunID, req); err != nil { - return res, fmt.Errorf("unable to make workflow node run release: %v", err) + if strings.ToUpper(res.Status) == strings.ToUpper(sdk.StatusSuccess) { + integrationPluginClientStop(ctx, integrationPluginClient, done, stopLogs) + return sdk.Result{ + Status: sdk.StatusSuccess, + }, nil } - return sdk.Result{Status: sdk.StatusSuccess}, nil + return sdk.Result{ + Status: sdk.StatusFail, + Reason: res.Details, + }, nil } diff --git a/engine/worker/internal/action/builtin_release_vcs.go b/engine/worker/internal/action/builtin_release_vcs.go new file mode 100644 index 0000000000..bb8316ab01 --- /dev/null +++ b/engine/worker/internal/action/builtin_release_vcs.go @@ -0,0 +1,82 @@ +package action + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/rockbears/log" + + "github.com/ovh/cds/engine/worker/pkg/workerruntime" + "github.com/ovh/cds/sdk" +) + +func RunReleaseVCS(ctx context.Context, wk workerruntime.Runtime, a sdk.Action, secrets []sdk.Variable) (sdk.Result, error) { + var res sdk.Result + res.Status = sdk.StatusFail + jobID, err := workerruntime.JobID(ctx) + if err != nil { + return res, err + } + + artifactList := sdk.ParameterFind(a.Parameters, "artifacts") + tag := sdk.ParameterFind(a.Parameters, "tag") + title := sdk.ParameterFind(a.Parameters, "title") + releaseNote := sdk.ParameterFind(a.Parameters, "releaseNote") + + pkey := sdk.ParameterFind(wk.Parameters(), "cds.project") + wName := sdk.ParameterFind(wk.Parameters(), "cds.workflow") + workflowNum := sdk.ParameterFind(wk.Parameters(), "cds.run.number") + + if pkey == nil { + return res, errors.New("cds.project variable not found") + } + + if wName == nil { + return res, errors.New("cds.workflow variable not found") + } + + if workflowNum == nil { + return res, errors.New("cds.run.number variable not found") + } + + if tag == nil || tag.Value == "" { + return res, errors.New("tag name is not set. Nothing to perform") + } + + if title == nil || title.Value == "" { + return res, errors.New("release title is not set") + } + + if releaseNote == nil || releaseNote.Value == "" { + return res, errors.New("release note is not set") + } + + wRunNumber, errI := strconv.ParseInt(workflowNum.Value, 10, 64) + if errI != nil { + return res, fmt.Errorf("Workflow number is not a number. Got %s: %s", workflowNum.Value, errI) + } + + artSplitted := strings.Split(artifactList.Value, ",") + req := sdk.WorkflowNodeRunRelease{ + ReleaseContent: releaseNote.Value, + ReleaseTitle: title.Value, + TagName: tag.Value, + Artifacts: artSplitted, + } + + jobrun, err := wk.Client().QueueJobInfo(ctx, jobID) + if err != nil { + return res, fmt.Errorf("unable to get job info: %v", err) + } + + log.Info(ctx, "RunRelease> jobRunID=%v WorkflowNodeRunID:%v", jobID, jobrun.WorkflowNodeRunID) + + if err := wk.Client().WorkflowNodeRunRelease(pkey.Value, wName.Value, wRunNumber, jobrun.WorkflowNodeRunID, req); err != nil { + return res, fmt.Errorf("unable to make workflow node run release: %v", err) + } + + return sdk.Result{Status: sdk.StatusSuccess}, nil +} diff --git a/engine/worker/internal/action/builtin_release_test.go b/engine/worker/internal/action/builtin_release_vcs_test.go similarity index 100% rename from engine/worker/internal/action/builtin_release_test.go rename to engine/worker/internal/action/builtin_release_vcs_test.go diff --git a/engine/worker/internal/builtin.go b/engine/worker/internal/builtin.go index 89ace328aa..9f7235dd21 100644 --- a/engine/worker/internal/builtin.go +++ b/engine/worker/internal/builtin.go @@ -21,6 +21,7 @@ func init() { mapBuiltinActions[sdk.GitCloneAction] = action.RunGitClone mapBuiltinActions[sdk.GitTagAction] = action.RunGitTag mapBuiltinActions[sdk.ReleaseVCSAction] = action.RunReleaseVCS + mapBuiltinActions[sdk.ReleaseAction] = action.RunRelease mapBuiltinActions[sdk.CheckoutApplicationAction] = action.RunCheckoutApplication mapBuiltinActions[sdk.DeployApplicationAction] = action.RunDeployApplication mapBuiltinActions[sdk.CoverageAction] = action.RunParseCoverageResultAction diff --git a/go.mod b/go.mod index a416ce3155..ce9ae6a7dd 100644 --- a/go.mod +++ b/go.mod @@ -156,6 +156,8 @@ require ( sigs.k8s.io/yaml v1.1.0 // indirect ) +replace github.com/jfrog/jfrog-client-go => github.com/sguiheux/jfrog-client-go v0.21.1-0.20210526124627-b559bbbcdf25 + replace github.com/vmware/go-nfs-client => github.com/sguiheux/go-nfs-client v0.0.0-20210311091651-4f075a6103cc replace github.com/alecthomas/jsonschema => github.com/sguiheux/jsonschema v0.2.0 diff --git a/go.sum b/go.sum index 864eee6bc9..30adcf04cb 100644 --- a/go.sum +++ b/go.sum @@ -326,8 +326,6 @@ github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/U github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jfrog/gofrog v1.0.6 h1:yUDxSCw8gTK6vC4PvtG0HTnEOQJSZ+O4lWGCgkev1nU= github.com/jfrog/gofrog v1.0.6/go.mod h1:HkDzg+tMNw23UryoOv0+LB94BzYcl6MCIoz8Tmlb+s8= -github.com/jfrog/jfrog-client-go v0.22.3 h1:CqircVnWso+EbpYymsLq1CplHlQ8zxHdy/VOhaAX6Bs= -github.com/jfrog/jfrog-client-go v0.22.3/go.mod h1:1XMLr/yzslzV9uABPrX4gpA0Bvc51ZX6LRvu/L0WQPc= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= @@ -550,6 +548,8 @@ github.com/sguiheux/go-coverage v0.0.0-20190710153556-287b082a7197 h1:qu90yDtRE5 github.com/sguiheux/go-coverage v0.0.0-20190710153556-287b082a7197/go.mod h1:0hhKrsUsoT7yvxwNGKa+TSYNA26DNWMqReeZEQq/9FI= github.com/sguiheux/go-nfs-client v0.0.0-20210311091651-4f075a6103cc h1:2hQK9ZA+R4QK6YSeI6J8h40fv1pQVmsoLwdn0omv4NE= github.com/sguiheux/go-nfs-client v0.0.0-20210311091651-4f075a6103cc/go.mod h1:JWMmlL5pWPL6DVIvix8TwfsDIfw8Cu1uyvid9Js3nyE= +github.com/sguiheux/jfrog-client-go v0.21.1-0.20210526124627-b559bbbcdf25 h1:szpBX0VzkKReVit+MI4Qm9izvO60NSgP9JukQq4OKJU= +github.com/sguiheux/jfrog-client-go v0.21.1-0.20210526124627-b559bbbcdf25/go.mod h1:1XMLr/yzslzV9uABPrX4gpA0Bvc51ZX6LRvu/L0WQPc= github.com/sguiheux/jsonschema v0.2.0 h1:hFHEPxudR6sNcsg50/iuJzHT5d3h3KOvtcg2Hrshs2k= github.com/sguiheux/jsonschema v0.2.0/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= github.com/shirou/gopsutil v0.0.0-20170406131756-e49a95f3d5f8 h1:05R1OwSk31dkzqf2Jf27n2IOoF9zkK9LcPgPsEm8U7U= diff --git a/sdk/action.go b/sdk/action.go index 5c6c4aeb3e..2c51cf841e 100644 --- a/sdk/action.go +++ b/sdk/action.go @@ -26,6 +26,7 @@ const ( DeployApplicationAction = "DeployApplication" PushBuildInfo = "PushBuildInfo" InstallKeyAction = "InstallKey" + ReleaseAction = "Release" DefaultGitCloneParameterTagValue = "{{.git.tag}}" ) @@ -147,7 +148,6 @@ func (a Action) IsValid() error { } } } - return nil } diff --git a/sdk/action/action.go b/sdk/action/action.go index 234bada284..97442a2e6a 100644 --- a/sdk/action/action.go +++ b/sdk/action/action.go @@ -23,6 +23,7 @@ var List = []Manifest{ InstallKey, JUnit, ReleaseVCS, + Release, Script, ServeStaticFiles, } diff --git a/sdk/action/release.go b/sdk/action/release.go new file mode 100644 index 0000000000..d08f05a1e1 --- /dev/null +++ b/sdk/action/release.go @@ -0,0 +1,42 @@ +package action + +import ( + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/exportentities" +) + +// Release action definition. +var Release = Manifest{ + Action: sdk.Action{ + Name: sdk.ReleaseAction, + Description: "This action creates a release on a artifact manager. It promotes artifacts.", + Parameters: []sdk.Parameter{ + { + Name: "releaseNote", + Description: "(optional) Set a release note for the release.", + Type: sdk.TextParameter, + }, + { + Name: "artifacts", + Description: "(optional) Set a list of artifacts, separate by ','. You can also use regexp.", + Type: sdk.StringParameter, + }, + }, + }, + Example: exportentities.PipelineV1{ + Version: exportentities.PipelineVersion1, + Name: "Pipeline1", + Stages: []string{"Stage1"}, + Jobs: []exportentities.Job{{ + Name: "Job1", + Stage: "Stage1", + Steps: []exportentities.Step{ + { + Release: &exportentities.StepRelease{ + Artifacts: "*.zip", + }, + }, + }, + }}, + }, +} diff --git a/sdk/exportentities/step.go b/sdk/exportentities/step.go index 7d35ccbafa..ca2897f492 100644 --- a/sdk/exportentities/step.go +++ b/sdk/exportentities/step.go @@ -167,6 +167,16 @@ func newStep(act sdk.Action) Step { if prefix != nil { s.GitTag.Prefix = prefix.Value } + case sdk.ReleaseAction: + s.Release = &StepRelease{} + artifacts := sdk.ParameterFind(act.Parameters, "artifacts") + if artifacts != nil { + s.Release.Artifacts = artifacts.Value + } + releaseNote := sdk.ParameterFind(act.Parameters, "releaseNote") + if releaseNote != nil { + s.Release.ReleaseNote = releaseNote.Value + } case sdk.ReleaseVCSAction: s.ReleaseVCS = &StepReleaseVCS{} artifacts := sdk.ParameterFind(act.Parameters, "artifacts") @@ -294,6 +304,12 @@ type StepGitClone struct { User string `json:"user,omitempty" yaml:"user,omitempty"` } +// StepRelease represents exported release step. +type StepRelease struct { + Artifacts string `json:"artifacts,omitempty" yaml:"artifacts,omitempty"` + ReleaseNote string `json:"releaseNote,omitempty" yaml:"releaseNote,omitempty"` +} + // StepReleaseVCS represents exported release step. type StepReleaseVCS struct { Artifacts string `json:"artifacts,omitempty" yaml:"artifacts,omitempty"` @@ -342,6 +358,7 @@ type Step struct { GitClone *StepGitClone `json:"gitClone,omitempty" yaml:"gitClone,omitempty" jsonschema:"oneof_required=actionGitClone" jsonschema_description:"Clone a git repository.\nhttps://ovh.github.io/cds/docs/actions/builtin-gitclone"` GitTag *StepGitTag `json:"gitTag,omitempty" yaml:"gitTag,omitempty" jsonschema:"oneof_required=actionGitTag" jsonschema_description:"Create a git tag.\nhttps://ovh.github.io/cds/docs/actions/builtin-gittag"` ReleaseVCS *StepReleaseVCS `json:"releaseVCS,omitempty" yaml:"releaseVCS,omitempty" jsonschema:"oneof_required=actionReleaseVCS" jsonschema_description:"Release an application.\nhttps://ovh.github.io/cds/docs/actions/builtin-releasevcs"` + Release *StepRelease `json:"release,omitempty" yaml:"release,omitempty" jsonschema:"oneof_required=actionRelease" jsonschema_description:"Release an application.\nhttps://ovh.github.io/cds/docs/actions/builtin-release"` JUnitReport *StepJUnitReport `json:"jUnitReport,omitempty" yaml:"jUnitReport,omitempty" jsonschema:"oneof_required=actionJUNit" jsonschema_description:"Parse JUnit report.\nhttps://ovh.github.io/cds/docs/actions/builtin-junit"` Checkout *StepCheckout `json:"checkout,omitempty" yaml:"checkout,omitempty" jsonschema:"oneof_required=actionCheckout" jsonschema_description:"Checkout repository for an application.\nhttps://ovh.github.io/cds/docs/actions/builtin-checkoutapplication"` InstallKey *StepInstallKey `json:"installKey,omitempty" yaml:"installKey,omitempty" jsonschema:"oneof_required=actionInstallKey" jsonschema_description:"Install a key (GPG, SSH) in your current workspace.\nhttps://ovh.github.io/cds/docs/actions/builtin-installkey"` @@ -450,6 +467,9 @@ func (s Step) IsValid() bool { if s.isReleaseVCS() { count++ } + if s.isRelease() { + count++ + } if s.isCheckout() { count++ } @@ -494,6 +514,8 @@ func (s Step) toAction() (*sdk.Action, error) { a, err = s.asGitTag() } else if s.isReleaseVCS() { a, err = s.asReleaseVCS() + } else if s.isRelease() { + a, err = s.asRelease() } else if s.isCheckout() { a = s.asCheckoutApplication() } else if s.isInstallKey() { @@ -731,6 +753,22 @@ func (s Step) asGitTag() (sdk.Action, error) { return a, nil } +func (s Step) isRelease() bool { return s.Release != nil } + +func (s Step) asRelease() (sdk.Action, error) { + var a sdk.Action + m, err := stepToMap(s.Release) + if err != nil { + return a, err + } + a = sdk.Action{ + Name: sdk.ReleaseAction, + Type: sdk.BuiltinAction, + Parameters: sdk.ParametersFromMap(m), + } + return a, nil +} + func (s Step) isReleaseVCS() bool { return s.ReleaseVCS != nil } func (s Step) asReleaseVCS() (sdk.Action, error) { diff --git a/sdk/integration.go b/sdk/integration.go index fb0f5a2eda..f5a28fbd9b 100644 --- a/sdk/integration.go +++ b/sdk/integration.go @@ -19,6 +19,7 @@ const ( ArtifactManagerConfigURL = "url" ArtifactManagerConfigTokenName = "token.name" ArtifactManagerConfigToken = "token" + ArtifactManagerConfigReleaseToken = "release.token" ArtifactManagerConfigCdsRepository = "cds.repository" ArtifactManagerConfigPromotionLowMaturity = "promotion.maturity.low" ArtifactManagerConfigPromotionHighMaturity = "promotion.maturity.high" @@ -135,6 +136,9 @@ var ( ArtifactManagerConfigToken: IntegrationConfigValue{ Type: IntegrationConfigTypePassword, }, + ArtifactManagerConfigReleaseToken: IntegrationConfigValue{ + Type: IntegrationConfigTypePassword, + }, ArtifactManagerConfigCdsRepository: IntegrationConfigValue{ Type: IntegrationConfigTypeString, }, diff --git a/sdk/plugin.go b/sdk/plugin.go index a5bc9514da..d1b2625c45 100644 --- a/sdk/plugin.go +++ b/sdk/plugin.go @@ -6,6 +6,7 @@ const ( GRPCPluginUploadArtifact = "integration-upload_artifact" GRPCPluginDownloadArtifact = "integration-download_artifact" GRPCPluginBuildInfo = "integration-build_info" + GRPCPluginRelease = "integration-release" GRPCPluginAction = "action" )