feat: add artifactory integration (#5802)

cli/cdsctl/workflow_run_result.go

@@ -237,6 +237,12 @@ func toCLIRunResult(results []sdk.WorkflowRunResult) ([]RunResultCli, error) {
 				return nil, err
 			}
 			name = artiResult.Name
+		case sdk.WorkflowRunResultTypeArtifactManager:
+			artiResult, err := r.GetArtifactManager()
+			if err != nil {
+				return nil, err
+			}
+			name = artiResult.Name
 		}
 
 		cliresults = append(cliresults, RunResultCli{

contrib/grpcplugins/action/plugin-archive/go.mod

@@ -5,7 +5,6 @@ replace github.com/ovh/cds => ../../../../
 go 1.16
 
 require (
-	github.com/dsnet/compress v0.0.1 // indirect
 	github.com/golang/protobuf v1.5.0
 	github.com/mholt/archiver v3.1.1+incompatible
 	github.com/ovh/cds v0.0.0-00010101000000-000000000000

contrib/grpcplugins/action/plugin-clair/go.mod

@@ -36,5 +36,5 @@ require (
 	github.com/quay/clair/v2 v2.1.4
 	github.com/spf13/viper v1.7.0
 	github.com/vbatts/tar-split v0.11.1 // indirect
-	golang.org/x/net v0.0.0-20200602114024-627f9648deb9
+	golang.org/x/net v0.0.0-20201209123823-ac852fbbde11
 )

contrib/grpcplugins/action/plugin-ssh-cmd/go.mod

@@ -7,5 +7,5 @@ go 1.16
 require (
 	github.com/golang/protobuf v1.5.0
 	github.com/ovh/cds v0.0.0-00010101000000-000000000000
-	golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899
+	golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9
 )

contrib/integrations/arsenal/arsenal.yml

@@ -3,7 +3,7 @@ default_config:
   host:
     type: string
 deployment: true
-deployment_default_config:
+additional_default_config:
   version:
     type: string
     value: "{{.cds.version}}"

contrib/integrations/artifactory/artifactory.yml

@@ -0,0 +1,19 @@
+name: Artifactory
+default_config:
+  artifactory.url:
+    type: string
+  artifactory.token.name:
+    type: string
+  artifactory.token:
+    type: password
+  artifactory.cds_repository:
+    type: string
+  artifactory.promotion.low.maturity:
+    type: string
+  artifactory.promotion.high.maturity:
+    type: string
+additional_default_config:
+  build.info.path:
+    type: string
+    value: ""
+artifact_manager: true

contrib/integrations/artifactory/plugin-artifactory-download-artifact/Makefile

@@ -0,0 +1,59 @@
+.PHONY: clean
+
+VERSION := $(if ${CDS_SEMVER},${CDS_SEMVER},snapshot)
+GITHASH := $(if ${GIT_HASH},${GIT_HASH},`git log -1 --format="%H"`)
+BUILDTIME := `date "+%m/%d/%y-%H:%M:%S"`
+CDSCTL := $(if ${CDSCTL},${CDSCTL},cdsctl)
+
+TARGET_DIR = ./dist
+TARGET_NAME = plugin-artifactory-download-artifact
+
+define PLUGIN_MANIFEST_BINARY
+os: %os%
+arch: %arch%
+cmd: ./%filename%
+endef
+export PLUGIN_MANIFEST_BINARY
+
+TARGET_LDFLAGS = -ldflags "-X github.com/ovh/cds/sdk.VERSION=$(VERSION) -X github.com/ovh/cds/sdk.GOOS=$$GOOS -X github.com/ovh/cds/sdk.GOARCH=$$GOARCH -X github.com/ovh/cds/sdk.GITHASH=$(GITHASH) -X github.com/ovh/cds/sdk.BUILDTIME=$(BUILDTIME) -X github.com/ovh/cds/sdk.BINARY=$(TARGET_NAME)"
+TARGET_OS = $(if ${OS},${OS},windows darwin linux freebsd)
+TARGET_ARCH = $(if ${ARCH},${ARCH},amd64 arm 386 arm64)
+
+GO_BUILD = go build
+
+$(TARGET_DIR):
+	$(info create $(TARGET_DIR) directory)
+	@mkdir -p $(TARGET_DIR)
+
+default: build
+
+clean:
+	@rm -rf $(TARGET_DIR)
+
+build: $(TARGET_DIR)
+	@cp $(TARGET_NAME).yml $(TARGET_DIR)/$(TARGET_NAME).yml
+	@for GOOS in $(TARGET_OS); do \
+		for GOARCH in $(TARGET_ARCH); do \
+			EXTENSION=""; \
+			if test "$$GOOS" = "windows" ; then EXTENSION=".exe"; fi; \
+			echo Compiling $(TARGET_DIR)/$(TARGET_NAME)-$$GOOS-$$GOARCH$$EXTENSION $(VERSION); \
+			FILENAME=$(TARGET_NAME)-$$GOOS-$$GOARCH$$EXTENSION; \
+			GOOS=$$GOOS GOARCH=$$GOARCH CGO_ENABLED=0 $(GO_BUILD) $(TARGET_LDFLAGS) -o $(TARGET_DIR)/$$FILENAME; \
+			echo "$$PLUGIN_MANIFEST_BINARY" > $(TARGET_DIR)/plugin-artifactory-download-artifact-$$GOOS-$$GOARCH.yml; \
+			perl -pi -e s,%os%,$$GOOS,g $(TARGET_DIR)/plugin-artifactory-download-artifact-$$GOOS-$$GOARCH.yml; \
+			perl -pi -e s,%arch%,$$GOARCH,g $(TARGET_DIR)/plugin-artifactory-download-artifact-$$GOOS-$$GOARCH.yml; \
+			perl -pi -e s,%filename%,$$FILENAME,g $(TARGET_DIR)/plugin-artifactory-download-artifact-$$GOOS-$$GOARCH.yml; \
+		done; \
+	done
+
+publish:
+	@echo "Updating plugin..."
+	$(CDSCTL) admin plugins import $(TARGET_DIR)/$(TARGET_NAME).yml
+	@for GOOS in $(TARGET_OS); do \
+		for GOARCH in $(TARGET_ARCH); do \
+			EXTENSION=""; \
+			if test "$$GOOS" = "windows" ; then EXTENSION=".exe"; fi; \
+			echo "Updating plugin binary $(TARGET_NAME)-$$GOOS-$$GOARCH$$EXTENSION"; \
+			$(CDSCTL) admin plugins binary-add artifactory-download-artifact-plugin $(TARGET_DIR)/$(TARGET_NAME)-$$GOOS-$$GOARCH.yml $(TARGET_DIR)/$(TARGET_NAME)-$$GOOS-$$GOARCH$$EXTENSION; \
+		done; \
+	done

contrib/integrations/artifactory/plugin-artifactory-download-artifact/main.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strconv"
+
+	"github.com/golang/protobuf/ptypes/empty"
+	"github.com/jfrog/jfrog-client-go/artifactory"
+	"github.com/jfrog/jfrog-client-go/artifactory/auth"
+	"github.com/jfrog/jfrog-client-go/artifactory/services"
+	"github.com/jfrog/jfrog-client-go/artifactory/services/utils"
+	"github.com/jfrog/jfrog-client-go/config"
+	"github.com/jfrog/jfrog-client-go/utils/log"
+
+	"github.com/ovh/cds/sdk"
+	"github.com/ovh/cds/sdk/grpcplugin/integrationplugin"
+)
+
+/*
+This plugin have to be used as a download artifact integration plugin
+
+Artifactory download artifact plugin must configured as following:
+	name: artifactory-download-artifact-plugin
+	type: integration
+	author: "Steven Guiheux"
+	description: "OVH Artifactory Upload Artifact Plugin"
+
+$ cdsctl admin plugins import artifactory-download-artifact-plugin.yml
+
+Build the present binaries and import in CDS:
+	os: linux
+	arch: amd64
+	cmd: <path-to-binary-file>
+
+$ cdsctl admin plugins binary-add artifactory-download-artifact-plugin artifactory-download-artifact-plugin-bin.yml <path-to-binary-file>
+
+Artifactory integration must configured as following
+	name: Artifactory
+	default_config:
+	  artifactory.url:
+		type: string
+	  artifactory.token:
+		type: password
+	  artifactory.cds_repository:
+		type: string
+	artifact_manager: true
+*/
+
+type artifactoryDownloadArtifactPlugin struct {
+	integrationplugin.Common
+}
+
+func (e *artifactoryDownloadArtifactPlugin) Manifest(_ context.Context, _ *empty.Empty) (*integrationplugin.IntegrationPluginManifest, error) {
+	return &integrationplugin.IntegrationPluginManifest{
+		Name:        "OVH Artifactory Download Artifact Plugin",
+		Author:      "Steven Guiheux",
+		Description: "OVH Artifactory Download Artifact Plugin",
+		Version:     sdk.VERSION,
+	}, nil
+}
+
+func createArtifactoryClient(url, token string) (artifactory.ArtifactoryServicesManager, error) {
+	rtDetails := auth.NewArtifactoryDetails()
+	rtDetails.SetUrl(url)
+	rtDetails.SetAccessToken(token)
+	serviceConfig, err := config.NewConfigBuilder().
+		SetServiceDetails(rtDetails).
+		SetThreads(1).
+		SetDryRun(false).
+		Build()
+	if err != nil {
+		return nil, fmt.Errorf("unable to create service config: %v", err)
+	}
+	return artifactory.New(serviceConfig)
+}
+
+func (e *artifactoryDownloadArtifactPlugin) Run(_ context.Context, opts *integrationplugin.RunQuery) (*integrationplugin.RunResult, error) {
+	cdsRepo := opts.GetOptions()["cds.integration.artifact_manager.artifactory.cds_repository"]
+	artifactoryURL := opts.GetOptions()["cds.integration.artifact_manager.artifactory.url"]
+	token := opts.GetOptions()["cds.integration.artifact_manager.artifactory.token"]
+
+	filePath := opts.GetOptions()[sdk.ArtifactDownloadPluginInputFilePath]
+	path := opts.GetOptions()[sdk.ArtifactDownloadPluginInputDestinationPath]
+	md5 := opts.GetOptions()[sdk.ArtifactDownloadPluginInputMd5]
+	permS := opts.GetOptions()[sdk.ArtifactDownloadPluginInputPerm]
+
+	perm, err := strconv.ParseUint(permS, 10, 32)
+	if err != nil {
+		return fail("unable to read file permission %s: %v", permS, err)
+	}
+
+	artiClient, err := createArtifactoryClient(artifactoryURL, token)
+	if err != nil {
+		return fail("unable to create artifactory client: %v", err)
+	}
+	log.SetLogger(log.NewLogger(log.ERROR, os.Stdout))
+
+	params := services.NewDownloadParams()
+	params.Pattern = fmt.Sprintf("%s/%s", cdsRepo, filePath)
+	params.Target = path
+	params.Flat = true
+	params.Retries = 5
+
+	summary, err := artiClient.DownloadFilesWithSummary(params)
+	if err != nil || summary.TotalFailed > 0 {
+		return fail("unable to download files %s from artifactory %s: %v", filePath, params.Target, err)
+	}
+	defer summary.Close()
+	for artDetails := new(utils.ArtifactDetails); summary.ArtifactsDetailsReader.NextRecord(artDetails) == nil; artDetails = new(utils.ArtifactDetails) {
+		if md5 != artDetails.Checksums.Md5 {
+			return fail("wrong md5 for file %s. Got %s Want %s", filePath, artDetails.Checksums.Md5, md5)
+		}
+	}
+
+	fileMode, err := os.Stat(path)
+	if err != nil {
+		return fail("unable to get file stat: %v", err)
+	}
+	currentperm := uint32(fileMode.Mode().Perm())
+	if currentperm != uint32(perm) {
+		if err := os.Chmod(path, os.FileMode(uint32(perm))); err != nil {
+			return fail("unable to chmod file %s: %v", path, err)
+		}
+	}
+	return &integrationplugin.RunResult{
+		Status: sdk.StatusSuccess,
+	}, nil
+}
+
+func main() {
+	e := artifactoryDownloadArtifactPlugin{}
+	if err := integrationplugin.Start(context.Background(), &e); err != nil {
+		panic(err)
+	}
+	return
+
+}
+
+func fail(format string, args ...interface{}) (*integrationplugin.RunResult, error) {
+	msg := fmt.Sprintf(format, args...)
+	fmt.Println(msg)
+	return &integrationplugin.RunResult{
+		Details: msg,
+		Status:  sdk.StatusFail,
+	}, nil
+}

contrib/integrations/artifactory/plugin-artifactory-download-artifact/plugin-artifactory-download-artifact.yml

@@ -0,0 +1,5 @@
+name: artifactory-download-artifact-plugin
+type: integration-download_artifact
+integration: Artifactory
+author: "OVH SAS"
+description: "OVH Artifactory Download Artifact Plugin"

contrib/integrations/artifactory/plugin-artifactory-upload-artifact/Makefile

@@ -0,0 +1,59 @@
+.PHONY: clean
+
+VERSION := $(if ${CDS_SEMVER},${CDS_SEMVER},snapshot)
+GITHASH := $(if ${GIT_HASH},${GIT_HASH},`git log -1 --format="%H"`)
+BUILDTIME := `date "+%m/%d/%y-%H:%M:%S"`
+CDSCTL := $(if ${CDSCTL},${CDSCTL},cdsctl)
+
+TARGET_DIR = ./dist
+TARGET_NAME = plugin-artifactory-upload-artifact
+
+define PLUGIN_MANIFEST_BINARY
+os: %os%
+arch: %arch%
+cmd: ./%filename%
+endef
+export PLUGIN_MANIFEST_BINARY
+
+TARGET_LDFLAGS = -ldflags "-X github.com/ovh/cds/sdk.VERSION=$(VERSION) -X github.com/ovh/cds/sdk.GOOS=$$GOOS -X github.com/ovh/cds/sdk.GOARCH=$$GOARCH -X github.com/ovh/cds/sdk.GITHASH=$(GITHASH) -X github.com/ovh/cds/sdk.BUILDTIME=$(BUILDTIME) -X github.com/ovh/cds/sdk.BINARY=$(TARGET_NAME)"
+TARGET_OS = $(if ${OS},${OS},windows darwin linux freebsd)
+TARGET_ARCH = $(if ${ARCH},${ARCH},amd64 arm 386 arm64)
+
+GO_BUILD = go build
+
+$(TARGET_DIR):
+	$(info create $(TARGET_DIR) directory)
+	@mkdir -p $(TARGET_DIR)
+
+default: build
+
+clean:
+	@rm -rf $(TARGET_DIR)
+
+build: $(TARGET_DIR)
+	@cp $(TARGET_NAME).yml $(TARGET_DIR)/$(TARGET_NAME).yml
+	@for GOOS in $(TARGET_OS); do \
+		for GOARCH in $(TARGET_ARCH); do \
+			EXTENSION=""; \
+			if test "$$GOOS" = "windows" ; then EXTENSION=".exe"; fi; \
+			echo Compiling $(TARGET_DIR)/$(TARGET_NAME)-$$GOOS-$$GOARCH$$EXTENSION $(VERSION); \
+			FILENAME=$(TARGET_NAME)-$$GOOS-$$GOARCH$$EXTENSION; \
+			GOOS=$$GOOS GOARCH=$$GOARCH CGO_ENABLED=0 $(GO_BUILD) $(TARGET_LDFLAGS) -o $(TARGET_DIR)/$$FILENAME; \
+			echo "$$PLUGIN_MANIFEST_BINARY" > $(TARGET_DIR)/plugin-artifactory-upload-artifact-$$GOOS-$$GOARCH.yml; \
+			perl -pi -e s,%os%,$$GOOS,g $(TARGET_DIR)/plugin-artifactory-upload-artifact-$$GOOS-$$GOARCH.yml; \
+			perl -pi -e s,%arch%,$$GOARCH,g $(TARGET_DIR)/plugin-artifactory-upload-artifact-$$GOOS-$$GOARCH.yml; \
+			perl -pi -e s,%filename%,$$FILENAME,g $(TARGET_DIR)/plugin-artifactory-upload-artifact-$$GOOS-$$GOARCH.yml; \
+		done; \
+	done
+
+publish:
+	@echo "Updating plugin..."
+	$(CDSCTL) admin plugins import $(TARGET_DIR)/$(TARGET_NAME).yml
+	@for GOOS in $(TARGET_OS); do \
+		for GOARCH in $(TARGET_ARCH); do \
+			EXTENSION=""; \
+			if test "$$GOOS" = "windows" ; then EXTENSION=".exe"; fi; \
+			echo "Updating plugin binary $(TARGET_NAME)-$$GOOS-$$GOARCH$$EXTENSION"; \
+			$(CDSCTL) admin plugins binary-add artifactory-upload-artifact-plugin $(TARGET_DIR)/$(TARGET_NAME)-$$GOOS-$$GOARCH.yml $(TARGET_DIR)/$(TARGET_NAME)-$$GOOS-$$GOARCH$$EXTENSION; \
+		done; \
+	done