Skip to content

Commit

Permalink
fix(api,cdsctl): download artifacts and coverage from artifacts manag…
Browse files Browse the repository at this point in the history
…er (#6041)
  • Loading branch information
fsamin authored Dec 17, 2021
1 parent 74696a1 commit 139ea40
Showing 20 changed files with 137 additions and 43 deletions.
51 changes: 44 additions & 7 deletions cli/cdsctl/workflow_run_result.go
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"os"
"os/exec"
"regexp"

"github.com/spf13/cobra"
@@ -81,6 +82,9 @@ func workflowRunResultGet(v cli.Values) error {
var fileName string
var perm uint32
var md5 string
var artifactManagerPath string
var artifactManagerRepo string

switch r.Type {
case sdk.WorkflowRunResultTypeArtifact:
art, err := r.GetArtifact()
@@ -100,6 +104,16 @@ func workflowRunResultGet(v cli.Values) error {
fileName = cov.Name
perm = cov.Perm
md5 = cov.MD5
case sdk.WorkflowRunResultTypeArtifactManager:
art, err := r.GetArtifactManager()
if err != nil {
return err
}
fileName = art.Name
perm = art.Perm
md5 = art.MD5
artifactManagerPath = art.Path
artifactManagerRepo = art.RepoName
default:
return cli.NewError("cannot get result of type %s", r.Type)
}
@@ -124,7 +138,27 @@ func workflowRunResultGet(v cli.Values) error {
}
}

if toDownload {
if !toDownload {
fmt.Printf("File %s already downloaded, checksum OK\n", f.Name())
return nil
}

if artifactManagerPath != "" && artifactManagerRepo != "" {
_, err := exec.LookPath("jfrog")
if err != nil {
fmt.Printf("# File is available on repository %s: %s\n", artifactManagerRepo, artifactManagerPath)
fmt.Printf("# to download the file use the following command\n")
fmt.Printf("jfrog rt download %s %q", artifactManagerRepo, artifactManagerPath)
return err
}
cmd := exec.Command("jfrog", "rt", "download", "--flat", artifactManagerRepo+"/"+artifactManagerPath)
output, err := cmd.CombinedOutput()
fmt.Println(string(output))
if err != nil {
return err
}
return nil
} else {
var err error
f, err = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.FileMode(perm))
if err != nil {
@@ -139,11 +173,8 @@ func workflowRunResultGet(v cli.Values) error {
return cli.NewError("unable to close file %s: %v", fileName, err)
}
}
if toDownload {
fmt.Printf("File %s created, checksum OK\n", f.Name())
} else {
fmt.Printf("File %s already downloaded, checksum OK\n", f.Name())
}

fmt.Printf("File %s created, checksum OK\n", f.Name())
}
return nil
}
@@ -224,13 +255,19 @@ func toCLIRunResult(results []sdk.WorkflowRunResult) ([]RunResultCli, error) {
return nil, err
}
name = artiResult.Name
artiType = "file"
case sdk.WorkflowRunResultTypeArtifactManager:
artiResult, err := r.GetArtifactManager()
if err != nil {
return nil, err
}
name = artiResult.Name
artiType = artiResult.RepoType
if artiResult.FileType != "" {
artiType = artiResult.FileType
} else {
artiType = artiResult.RepoType
}

}

cliresults = append(cliresults, RunResultCli{
2 changes: 2 additions & 0 deletions engine/api/artifact.go
Original file line number Diff line number Diff line change
@@ -10,6 +10,8 @@ import (
"github.com/ovh/cds/sdk"
)

// DEPRECATED
// TODO: remove this code after CDN would be mandatory
func (api *API) getArtifactsStoreHandler() service.Handler {
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
vars := mux.Vars(r)
2 changes: 1 addition & 1 deletion engine/api/objectstore/objectstore.go
Original file line number Diff line number Diff line change
@@ -21,7 +21,7 @@ type Driver interface {
GetProjectIntegration() sdk.ProjectIntegration
Status(ctx context.Context) sdk.MonitoringStatusLine
Store(o Object, data io.ReadCloser) (string, error)
ServeStaticFiles(o Object, entrypoint string, data io.ReadCloser) (string, error)
ServeStaticFiles(o Object, entrypoint string, data io.ReadCloser) (string, error) // DEPRECATED
Fetch(ctx context.Context, o Object) (io.ReadCloser, error)
Delete(ctx context.Context, o Object) error
DeleteContainer(ctx context.Context, containerPath string) error
1 change: 1 addition & 0 deletions engine/api/workflow/dao_node_run.go
Original file line number Diff line number Diff line change
@@ -92,6 +92,7 @@ func LoadNodeRun(db gorp.SqlExecutor, projectkey, workflowname string, noderunID
}
r.Artifacts = arts
}
// DEPRECATED
if loadOpts.WithStaticFiles {
staticFiles, errS := loadStaticFilesByNodeRunID(db, r.ID)
if errS != nil {
3 changes: 2 additions & 1 deletion engine/api/workflow/dao_run.go
Original file line number Diff line number Diff line change
@@ -48,7 +48,7 @@ workflow_run.header
type LoadRunOptions struct {
WithCoverage bool
WithArtifacts bool
WithStaticFiles bool
WithStaticFiles bool //DEPRECATED
WithTests bool
WithLightTests bool
WithVulnerabilities bool
@@ -981,6 +981,7 @@ func syncNodeRuns(db gorp.SqlExecutor, wr *sdk.WorkflowRun, loadOpts LoadRunOpti
wnr.Artifacts = arts
}

// DEPRECATED
if loadOpts.WithStaticFiles {
staticFiles, errS := loadStaticFilesByNodeRunID(db, wnr.ID)
if errS != nil {
2 changes: 2 additions & 0 deletions engine/api/workflow/dao_staticfiles.go
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ import (
"github.com/ovh/cds/sdk"
)

// DEPRECATED
func loadStaticFilesByNodeRunID(db gorp.SqlExecutor, nodeRunID int64) ([]sdk.StaticFiles, error) {
var dbstaticFiles []dbStaticFiles
if _, err := db.Select(&dbstaticFiles, `SELECT
@@ -32,6 +33,7 @@ func loadStaticFilesByNodeRunID(db gorp.SqlExecutor, nodeRunID int64) ([]sdk.Sta
return staticFiles, nil
}

// DEPRECATED
// InsertStaticFiles insert in table workflow_artifacts
func InsertStaticFiles(db gorp.SqlExecutor, sf *sdk.StaticFiles) error {
sf.Created = time.Now()
3 changes: 3 additions & 0 deletions engine/api/workflow/workflow_run_results.go
Original file line number Diff line number Diff line change
@@ -216,6 +216,9 @@ func verifyAddResultArtifactManager(ctx context.Context, db gorp.SqlExecutor, st
artResult.Size = fileInfo.Size
artResult.MD5 = fileInfo.Md5
artResult.RepoType = fileInfo.Type
if artResult.FileType == "" {
artResult.FileType = artResult.RepoType
}

if err := artResult.IsValid(); err != nil {
return "", err
7 changes: 7 additions & 0 deletions engine/api/workflow_queue_storage.go
Original file line number Diff line number Diff line change
@@ -18,6 +18,7 @@ import (
"github.com/ovh/cds/sdk"
)

// DEPRECATED
func (api *API) postWorkflowJobStaticFilesHandler() service.Handler {
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
if isWorker := isWorker(ctx); !isWorker {
@@ -121,6 +122,8 @@ func (api *API) postWorkflowJobStaticFilesHandler() service.Handler {
}
}

// DEPRECATED
// TODO: remove this code after CDN would be mandatory
func (api *API) postWorkflowJobArtifactHandler() service.Handler {
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
if isWorker := isWorker(ctx); !isWorker {
@@ -251,6 +254,8 @@ func (api *API) postWorkflowJobArtifactHandler() service.Handler {
}
}

// DEPRECATED
// TODO: remove this code after CDN would be mandatory
func (api *API) postWorkflowJobArtifactWithTempURLCallbackHandler() service.Handler {
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
if isWorker := isWorker(ctx); !isWorker {
@@ -307,6 +312,8 @@ func (api *API) postWorkflowJobArtifactWithTempURLCallbackHandler() service.Hand
}
}

// DEPRECATED
// TODO: remove this code after CDN would be mandatory
func (api *API) postWorkflowJobArtifacWithTempURLHandler() service.Handler {
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
if isWorker := isWorker(ctx); !isWorker {
9 changes: 7 additions & 2 deletions engine/worker/cmd_run_result.go
Original file line number Diff line number Diff line change
@@ -96,19 +96,24 @@ Example:

func addArtifactManagerRunResultCmd() func(cmd *cobra.Command, args []string) {
return func(cmd *cobra.Command, args []string) {
if len(args) != 3 {
sdk.Exit("missing arguments. Cmd: worker run-result add artifact-manager <fileName> <repo-name> <file-path>")
if len(args) != 3 && len(args) != 4 {
sdk.Exit("missing arguments. Cmd: worker run-result add artifact-manager <fileName> <repo-name> <file-path> [file-type]")
}

fileName := args[0]
repositoryName := args[1]
filePath := args[2]
var fileType string
if len(args) == 4 {
fileType = args[3]
}

payload := sdk.WorkflowRunResultArtifactManager{
Name: fileName,
Perm: 0,
Path: filePath,
RepoName: repositoryName,
FileType: fileType,
}
data, _ := json.Marshal(payload)

30 changes: 20 additions & 10 deletions engine/worker/internal/action/builtin_artifact_upload.go
Original file line number Diff line number Diff line change
@@ -49,7 +49,17 @@ func RunArtifactUpload(ctx context.Context, wk workerruntime.Runtime, a sdk.Acti
artifactPath = filepath.Join(abs, artifactPath)
}

tag := sdk.ParameterFind(a.Parameters, "tag")
tagParam := sdk.ParameterFind(a.Parameters, "tag")
var tag string
if tagParam != nil {
tag = tagParam.Value
}

fileTypeParam := sdk.ParameterFind(a.Parameters, "type")
var fileType string
if fileTypeParam != nil {
fileType = fileTypeParam.Value
}

// Global all files matching filePath
filesPath, err := afero.Glob(afero.NewOsFs(), artifactPath)
@@ -91,18 +101,18 @@ func RunArtifactUpload(ctx context.Context, wk workerruntime.Runtime, a sdk.Acti
// 2. Integration artifact manager on workflow
// 3. CDN activated or not
if integrationName != sdk.DefaultStorageIntegrationName {
if err := uploadArtifactByApiCall(path, wk, ctx, projectKey, integrationName, jobID, tag); err != nil {
if err := uploadArtifactByApiCall(path, wk, ctx, projectKey, integrationName, jobID, tag, fileType); err != nil {
chanError <- sdk.WrapError(err, "Error while uploading artifact by api call %s", path)
wgErrors.Add(1)
}
return
} else if pluginArtifactManagement != nil {
if err := uploadArtifactByIntegrationPlugin(path, ctx, wk, pluginArtifactManagement); err != nil {
if err := uploadArtifactByIntegrationPlugin(path, ctx, wk, pluginArtifactManagement, fileType); err != nil {
chanError <- sdk.WrapError(err, "Error while uploading artifact by plugin %s", path)
wgErrors.Add(1)
}
} else if !cdnArtifactEnabled {
if err := uploadArtifactByApiCall(path, wk, ctx, projectKey, integrationName, jobID, tag); err != nil {
if err := uploadArtifactByApiCall(path, wk, ctx, projectKey, integrationName, jobID, tag, fileType); err != nil {
chanError <- sdk.WrapError(err, "Error while uploading artifact by api call %s", path)
wgErrors.Add(1)
}
@@ -133,7 +143,7 @@ func RunArtifactUpload(ctx context.Context, wk workerruntime.Runtime, a sdk.Acti
return res, nil
}

func uploadArtifactByIntegrationPlugin(path string, ctx context.Context, wk workerruntime.Runtime, artiManager *sdk.GRPCPlugin) error {
func uploadArtifactByIntegrationPlugin(path string, ctx context.Context, wk workerruntime.Runtime, artiManager *sdk.GRPCPlugin, fileType string) error {
_, fileName := filepath.Split(path)

// Check run result
@@ -205,6 +215,8 @@ func uploadArtifactByIntegrationPlugin(path string, ctx context.Context, wk work
return fmt.Errorf("error uploading artifact: %v", err)
}

res.Outputs[sdk.ArtifactUploadPluginOutputFileType] = fileType

if strings.ToUpper(res.Status) != strings.ToUpper(sdk.StatusSuccess) {
return fmt.Errorf("plugin execution failed %s: %s", res.Status, res.Details)
}
@@ -237,11 +249,8 @@ func uploadArtifactIntoCDN(path string, ctx context.Context, wk workerruntime.Ru
return nil
}

func uploadArtifactByApiCall(path string, wk workerruntime.Runtime, ctx context.Context, projectKey string, integrationName string, jobID int64, tag *sdk.Parameter) error {
if tag == nil {
return sdk.NewError(sdk.ErrWorkerErrorCommand, fmt.Errorf("tag variable is empty. aborting"))
}
throughTempURL, duration, err := wk.Client().QueueArtifactUpload(ctx, projectKey, integrationName, jobID, tag.Value, path)
func uploadArtifactByApiCall(path string, wk workerruntime.Runtime, ctx context.Context, projectKey string, integrationName string, jobID int64, tag, fileType string) error {
throughTempURL, duration, err := wk.Client().QueueArtifactUpload(ctx, projectKey, integrationName, jobID, tag, path, fileType)
if err != nil {
return err
}
@@ -282,6 +291,7 @@ func addWorkflowRunResult(ctx context.Context, wk workerruntime.Runtime, filePat
Perm: uint32(perm),
RepoName: uploadResult.Outputs[sdk.ArtifactUploadPluginOutputPathRepoName],
Path: uploadResult.Outputs[sdk.ArtifactUploadPluginOutputPathFilePath],
FileType: uploadResult.Outputs[sdk.ArtifactUploadPluginOutputFileType],
}

bts, err := json.Marshal(data)
2 changes: 1 addition & 1 deletion engine/worker/internal/action/builtin_coverage.go
Original file line number Diff line number Diff line change
@@ -84,7 +84,7 @@ func RunParseCoverageResultAction(ctx context.Context, wk workerruntime.Runtime,

pluginArtifactManagement := wk.GetPlugin(sdk.GRPCPluginUploadArtifact)
if pluginArtifactManagement != nil {
if err := uploadArtifactByIntegrationPlugin(fpath, ctx, wk, pluginArtifactManagement); err != nil {
if err := uploadArtifactByIntegrationPlugin(fpath, ctx, wk, pluginArtifactManagement, sdk.ArtifactFileTypeCoverage); err != nil {
return res, fmt.Errorf("coverage parser: unable to upload in artifact manager: %v", err)
}
} else {
3 changes: 3 additions & 0 deletions sdk/artifact.go
Original file line number Diff line number Diff line change
@@ -15,11 +15,14 @@ const (
ArtifactUploadPluginOutputPathMD5 = "md5"
ArtifactUploadPluginOutputPerm = "perm"
ArtifactUploadPluginOutputSize = "size"
ArtifactUploadPluginOutputFileType = "file_type"

ArtifactDownloadPluginInputDestinationPath = "cds.integration.artifact_manager.download.destination.path"
ArtifactDownloadPluginInputFilePath = "cds.integration.artifact_manager.download.file.path"
ArtifactDownloadPluginInputMd5 = "cds.integration.artifact_manager.download.file.md5"
ArtifactDownloadPluginInputPerm = "cds.integration.artifact_manager.download.file.perm"

ArtifactFileTypeCoverage = "coverage"
)

// ArtifactsStore represents
Loading

0 comments on commit 139ea40

Please sign in to comment.