Skip to content

Commit

Permalink
feat: upload coverage report as run result (#5749)
Browse files Browse the repository at this point in the history
  • Loading branch information
sguiheux authored Mar 8, 2021
1 parent 75360ed commit c547ca2
Show file tree
Hide file tree
Showing 7 changed files with 323 additions and 16 deletions.
1 change: 1 addition & 0 deletions cli/cdsctl/workflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ func workflow() *cobra.Command {
workflowArtifact(),
workflowLog(),
workflowAdvanced(),
workflowRunResult(),
})
}

Expand Down
249 changes: 249 additions & 0 deletions cli/cdsctl/workflow_run_result.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,249 @@
package main

import (
"context"
"fmt"
"io"
"os"
"regexp"

"github.com/spf13/cobra"

"github.com/ovh/cds/cli"
"github.com/ovh/cds/sdk"
)

var workflowRunResultCmd = cli.Command{
Name: "result",
Aliases: []string{"results"},
Short: "Manage Workflow Run Result",
}

type RunResultCli struct {
ID string `cli:"id"`
Type sdk.WorkflowRunResultType `cli:"type"`
Name string `cli:"name"`
}

func workflowRunResult() *cobra.Command {
return cli.NewCommand(workflowRunResultCmd, nil, []*cobra.Command{
cli.NewListCommand(workflowRunResultListCmd, workflowRunResultList, nil, withAllCommandModifiers()...),
cli.NewCommand(workflowRunResultGetCmd, workflowRunResultGet, nil, withAllCommandModifiers()...),
})
}

var workflowRunResultGetCmd = cli.Command{
Name: "download",
Short: "download workflow run result",
Ctx: []cli.Arg{
{Name: _ProjectKey},
{Name: _WorkflowName},
},
Args: []cli.Arg{
{
Name: "run-number",
IsValid: func(s string) bool {
match, _ := regexp.MatchString(`[0-9]?`, s)
return match
},
},
{
Name: "result-id",
IsValid: func(s string) bool {
match, _ := regexp.MatchString(`[0-9]?`, s)
return match
},
},
},
}

func workflowRunResultGet(v cli.Values) error {
ctx := context.Background()
runNumber, err := v.GetInt64("run-number")
if err != nil {
return err
}
resultID := v.GetString("result-id")

confCDN, err := client.ConfigCDN()
if err != nil {
return err
}

runResults, err := client.WorkflowRunResultsList(ctx, v.GetString(_ProjectKey), v.GetString(_WorkflowName), runNumber)
if err != nil {
return err
}
for _, r := range runResults {
if r.ID != resultID {
continue
}
var cdnHash string
var fileName string
var perm uint32
var md5 string
switch r.Type {
case sdk.WorkflowRunResultTypeArtifact:
art, err := r.GetArtifact()
if err != nil {
return err
}
cdnHash = art.CDNRefHash
fileName = art.Name
perm = art.Perm
md5 = art.MD5
case sdk.WorkflowRunResultTypeCoverage:
cov, err := r.GetCoverage()
if err != nil {
return err
}
cdnHash = cov.CDNRefHash
fileName = cov.Name
perm = cov.Perm
md5 = cov.MD5
default:
return fmt.Errorf("cannot get result of type %s", r.Type)
}

var f *os.File
var toDownload bool
if _, err := os.Stat(fileName); os.IsNotExist(err) {
toDownload = true
} else {
// file exists, check sha512
var errf error
f, errf = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, os.FileMode(perm))
if errf != nil {
return errf
}
md5Sum, err := sdk.FileMd5sum(fileName)
if err != nil {
return err
}
if md5Sum != md5 {
toDownload = true
}
}

if toDownload {
var err error
f, err = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.FileMode(perm))
if err != nil {
return err
}
fmt.Printf("Downloading %s...\n", fileName)
r, err := client.CDNItemDownload(context.Background(), confCDN.HTTPURL, cdnHash, sdk.CDNTypeItemRunResult)
if err != nil {
return err
}
if _, err := io.Copy(f, r); err != nil {
return sdk.WrapError(err, "unable to write file")
}
if err := f.Close(); err != nil {
return err
}
}

md5Sum, err := sdk.FileMd5sum(fileName)
if err != nil {
return err
}

if md5Sum != md5 {
return fmt.Errorf("Invalid md5Sum \ndownloaded file:%s\n%s:%s", md5Sum, f.Name(), md5)
}

if toDownload {
fmt.Printf("File %s created, checksum OK\n", f.Name())
} else {
fmt.Printf("File %s already downloaded, checksum OK\n", f.Name())
}
}
return nil
}

var workflowRunResultListCmd = cli.Command{
Name: "list",
Short: "List workflow run result",
Ctx: []cli.Arg{
{Name: _ProjectKey},
{Name: _WorkflowName},
},
Args: []cli.Arg{
{
Name: "run-number",
IsValid: func(s string) bool {
match, _ := regexp.MatchString(`[0-9]?`, s)
return match
},
},
},
Flags: []cli.Flag{
{
Name: "type",
ShortHand: "t",
Usage: "List only result of one type",
IsValid: func(s string) bool {
match, _ := regexp.MatchString(`[0-9]?`, s)
return match
},
},
},
}

func workflowRunResultList(v cli.Values) (cli.ListResult, error) {
ctx := context.Background()

runNumber, err := v.GetInt64("run-number")
if err != nil {
return nil, err
}

results, err := client.WorkflowRunResultsList(ctx, v.GetString(_ProjectKey), v.GetString(_WorkflowName), runNumber)
if err != nil {
return nil, err
}

if v.GetString("type") == "" {
cliResults, err := toCLIRunResult(results)
return cli.AsListResult(cliResults), err
}

resultsFiltered := make([]sdk.WorkflowRunResult, 0)
for _, r := range results {
if string(r.Type) != v.GetString("type") {
continue
}
resultsFiltered = append(resultsFiltered, r)
}
cliResults, err := toCLIRunResult(resultsFiltered)
return cli.AsListResult(cliResults), err
}

func toCLIRunResult(results []sdk.WorkflowRunResult) ([]RunResultCli, error) {
cliresults := make([]RunResultCli, 0, len(results))
for _, r := range results {
var name string
switch r.Type {
case sdk.WorkflowRunResultTypeCoverage:
covResult, err := r.GetCoverage()
if err != nil {
return nil, err
}
name = covResult.Name
case sdk.WorkflowRunResultTypeArtifact:
artiResult, err := r.GetArtifact()
if err != nil {
return nil, err
}
name = artiResult.Name
}

cliresults = append(cliresults, RunResultCli{
ID: r.ID,
Type: r.Type,
Name: name,
})
}
return cliresults, nil
}
26 changes: 26 additions & 0 deletions engine/api/workflow/workflow_run_results.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,12 @@ func AddResult(db *gorp.DbMap, store cache.Store, runResult *sdk.WorkflowRunResu
if err != nil {
return err
}
case sdk.WorkflowRunResultTypeCoverage:
var err error
cacheKey, err = verifyAddResultCoverage(store, runResult)
if err != nil {
return err
}
default:
return sdk.WrapError(sdk.ErrInvalidData, "unkonwn result type %s", runResult.Type)
}
Expand All @@ -140,6 +146,26 @@ func AddResult(db *gorp.DbMap, store cache.Store, runResult *sdk.WorkflowRunResu
return sdk.WithStack(store.Delete(cacheKey))
}

func verifyAddResultCoverage(store cache.Store, runResult *sdk.WorkflowRunResult) (string, error) {
coverageRunResult, err := runResult.GetCoverage()
if err != nil {
return "", err
}
if err := coverageRunResult.IsValid(); err != nil {
return "", err
}

cacheKey := GetRunResultKey(runResult.WorkflowRunID, runResult.Type, coverageRunResult.Name)
b, err := store.Exist(cacheKey)
if err != nil {
return cacheKey, err
}
if !b {
return cacheKey, sdk.WrapError(sdk.ErrForbidden, "unable to upload an unchecked coverage")
}
return cacheKey, nil
}

func verifyAddResultArtifact(store cache.Store, runResult *sdk.WorkflowRunResult) (string, error) {
artifactRunResult, err := runResult.GetArtifact()
if err != nil {
Expand Down
4 changes: 0 additions & 4 deletions engine/api/workflow_run.go
Original file line number Diff line number Diff line change
Expand Up @@ -1448,10 +1448,6 @@ func (api *API) postWorkflowRunResultsHandler() service.Handler {
return sdk.WithStack(err)
}

if runResult.Type != sdk.WorkflowRunResultTypeArtifact {
return sdk.WrapError(sdk.ErrForbidden, "unable to manage non artifact result")
}

wr, err := workflow.LoadRun(ctx, api.mustDB(), key, name, number, workflow.LoadRunOptions{WithArtifacts: true})
if err != nil {
return err
Expand Down
32 changes: 22 additions & 10 deletions engine/cdn/cdn_file.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,26 +129,38 @@ func (s *Service) storeFile(ctx context.Context, sig cdn.Signature, reader io.Re
return err
}

runResultApiRef, _ := it.GetCDNRunResultApiRef()
switch itemType {
case sdk.CDNTypeItemRunResult:
artiApiRef, _ := it.GetCDNRunResultApiRef()
// Call CDS to insert workflow run result
artiResult := sdk.WorkflowRunResultArtifact{
Name: apiRef.ToFilename(),
Size: it.Size,
MD5: it.MD5,
CDNRefHash: it.APIRefHash,
Perm: artiApiRef.Perm,
var result interface{}
switch runResultApiRef.RunResultType {
case sdk.WorkflowRunResultTypeArtifact:
result = sdk.WorkflowRunResultArtifact{
Name: apiRef.ToFilename(),
Size: it.Size,
MD5: it.MD5,
CDNRefHash: it.APIRefHash,
Perm: runResultApiRef.Perm,
}
case sdk.WorkflowRunResultTypeCoverage:
result = sdk.WorkflowRunResultCoverage{
Name: apiRef.ToFilename(),
Size: it.Size,
MD5: it.MD5,
CDNRefHash: it.APIRefHash,
Perm: runResultApiRef.Perm,
}
}
bts, err := json.Marshal(artiResult)

bts, err := json.Marshal(result)
if err != nil {
return sdk.WithStack(err)
}
wrResult := sdk.WorkflowRunResult{
WorkflowRunID: sig.RunID,
WorkflowNodeRunID: sig.NodeRunID,
WorkflowRunJobID: sig.JobID,
Type: sdk.WorkflowRunResultTypeArtifact,
Type: runResultApiRef.RunResultType,
DataRaw: json.RawMessage(bts),
}
if err := s.Client.WorkflowRunResultsAdd(ctx, sig.ProjectKey, sig.WorkflowName, sig.RunNumber, wrResult); err != nil {
Expand Down
Loading

0 comments on commit c547ca2

Please sign in to comment.