Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix/action param interpolate #5107

Merged
merged 3 commits into from
Apr 8, 2020
Merged

Fix/action param interpolate #5107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
26dad72
fix(api): remove dead code
fsamin Apr 6, 2020
21c1961
fix(worker): variables and params interpolation
fsamin Apr 8, 2020
747d989
fix: cr
fsamin Apr 8, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 0 additions & 26 deletions engine/api/pipeline/pipeline_parameter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,29 +121,3 @@ func DeleteAllParameterFromPipeline(db gorp.SqlExecutor, pipelineID int64) error
_, err := db.Exec(query, pipelineID)
return sdk.WrapError(err, "Unable to delete all parameters")
}

// CountInParamValue counts how many time a pattern is in parameter value for the given project
func CountInParamValue(db gorp.SqlExecutor, key string, value string) ([]string, error) {
query := `
SELECT DISTINCT pipeline.name
FROM pipeline_parameter
JOIN pipeline ON pipeline.id = pipeline_parameter.pipeline_id
JOIN project ON project.id = pipeline.project_id
WHERE value like $2 AND project.projectkey = $1;
`
rows, err := db.Query(query, key, fmt.Sprintf("%%%s%%", value))
if err != nil {
return nil, sdk.WrapError(err, "Unable to count usage")
}
defer rows.Close()

var results []string
for rows.Next() {
var pipName string
if err := rows.Scan(&pipName); err != nil {
return nil, sdk.WrapError(err, "Unable to scan")
}
results = append(results, pipName)
}
return results, nil
}
2 changes: 0 additions & 2 deletions engine/api/workflow/gorp_model.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"time"

"github.com/ovh/cds/engine/api/database/gorpmapping"
"github.com/ovh/cds/sdk/log"

"github.com/ovh/cds/sdk"
)
Expand Down Expand Up @@ -108,7 +107,6 @@ func (j *JobRun) ToJobRun(jr *sdk.WorkflowNodeJobRun) (err error) {
j.ExecGroups, err = gorpmapping.JSONToNullString(jr.ExecGroups)
j.WorkerName = jr.WorkerName
j.HatcheryName = jr.HatcheryName
log.Debug("ToJobRun> %+v", j)
if err != nil {
return sdk.WrapError(err, "column exec_groups")
}
Expand Down
86 changes: 47 additions & 39 deletions engine/worker/internal/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,32 +14,51 @@ import (
"github.com/spf13/afero"

"github.com/ovh/cds/engine/worker/pkg/workerruntime"

"github.com/ovh/cds/sdk"
"github.com/ovh/cds/sdk/interpolate"
"github.com/ovh/cds/sdk/log"
)

func processJobParameter(parameters []sdk.Parameter, secrets []sdk.Variable) {
for i := range parameters {
keepReplacing := true
for keepReplacing {
t := parameters[i].Value
func processVariablesAndParameters(action *sdk.Action, jobParameters []sdk.Parameter, jobSecrets []sdk.Variable) error {
if err := processJobParameter(jobParameters, jobSecrets); err != nil {
return err
}

if err := processActionVariables(action, nil, jobParameters, jobSecrets); err != nil {
return err
}
return nil
}

func processJobParameter(parameters []sdk.Parameter, secrets []sdk.Variable) error {
secretParam := sdk.VariablesToParameters("", secrets)
secretMap := sdk.ParametersToMap(secretParam)

for _, p := range parameters {
parameters[i].Value = strings.Replace(parameters[i].Value, "{{."+p.Name+"}}", p.Value, -1)
for i := range parameters {
var err error
var oldValue = parameters[i].Value
var x int
var keepReplacing = true
for keepReplacing && x < 10 {
var paramMap = sdk.ParametersToMap(parameters)
parameters[i].Value, err = interpolate.Do(parameters[i].Value, paramMap)
if err != nil {
return sdk.WrapError(err, "Unable to interpolate job parameters")
}

for _, p := range secrets {
parameters[i].Value = strings.Replace(parameters[i].Value, "{{."+p.Name+"}}", p.Value, -1)
parameters[i].Value, err = interpolate.Do(parameters[i].Value, secretMap)
if err != nil {
return sdk.WrapError(err, "Unable to interpolate job parameters")
}

// If parameters wasn't updated, consider it done
if parameters[i].Value == t {
if parameters[i].Value == oldValue {
keepReplacing = false
}
x++
}
}
return nil
}

// ProcessActionVariables replaces all placeholders inside action recursively using
Expand All @@ -48,38 +67,29 @@ func processJobParameter(parameters []sdk.Parameter, secrets []sdk.Variable) {
// - Secrets from project, application and environment
//
// This function should be called ONLY from worker
func (w *CurrentWorker) processActionVariables(a *sdk.Action, parent *sdk.Action, jobParameters []sdk.Parameter, secrets []sdk.Variable) error {
func processActionVariables(a *sdk.Action, parent *sdk.Action, jobParameters []sdk.Parameter, secrets []sdk.Variable) error {
// replaces placeholder in parameters with ActionBuild variables
// replaces placeholder in parameters with Parent params
secretParam := sdk.VariablesToParameters("", secrets)
secretMap := sdk.ParametersToMap(secretParam)
var parentParamMap = map[string]string{}
if parent != nil {
parentParamMap = sdk.ParametersToMap(parent.Parameters)
}
jobParamMap := sdk.ParametersToMap(jobParameters)
allParams := sdk.ParametersMapMerge(parentParamMap, jobParamMap)
allParams = sdk.ParametersMapMerge(allParams, secretMap)
for i := range a.Parameters {
keepReplacing := true
for keepReplacing {
t := a.Parameters[i].Value

if parent != nil {
for _, p := range parent.Parameters {
a.Parameters[i].Value = strings.Replace(a.Parameters[i].Value, "{{."+p.Name+"}}", p.Value, -1)
}
}

for _, p := range jobParameters {
a.Parameters[i].Value = strings.Replace(a.Parameters[i].Value, "{{."+p.Name+"}}", p.Value, -1)
}

for _, p := range secrets {
a.Parameters[i].Value = strings.Replace(a.Parameters[i].Value, "{{."+p.Name+"}}", p.Value, -1)
}

// If parameters wasn't updated, consider it done
if a.Parameters[i].Value == t {
keepReplacing = false
}
var err error
a.Parameters[i].Value, err = interpolate.Do(a.Parameters[i].Value, allParams)
if err != nil {
return err
}
}

// replaces placeholder in all children recursively
for i := range a.Actions {
if err := w.processActionVariables(&a.Actions[i], a, jobParameters, secrets); err != nil {
if err := processActionVariables(&a.Actions[i], a, jobParameters, secrets); err != nil {
return nil
}
}
Expand Down Expand Up @@ -514,12 +524,10 @@ func (w *CurrentWorker) ProcessJob(jobInfo sdk.WorkflowNodeJobRunData) (sdk.Resu
})

// REPLACE ALL VARIABLE EVEN SECRETS HERE
processJobParameter(jobParameters, jobInfo.Secrets)
if err := w.processActionVariables(&jobInfo.NodeJobRun.Job.Action, nil, jobParameters, jobInfo.Secrets); err != nil {
log.Warning(ctx, "processJob> Cannot process action %s parameters: %s", jobInfo.NodeJobRun.Job.Action.Name, err)
if err := processVariablesAndParameters(&jobInfo.NodeJobRun.Job.Action, jobParameters, jobInfo.Secrets); err != nil {
return sdk.Result{
Status: sdk.StatusFail,
Reason: fmt.Sprintf("Error: cannot process action %s parameters", jobInfo.NodeJobRun.Job.Action.Name),
Reason: fmt.Sprintf("Error: cannot process job %s parameters", jobInfo.NodeJobRun.Job.Action.Name),
}, err
}

Expand Down
Loading