feat(api): disable project secrets auto injection for given regions

engine/api/api.go

@@ -67,7 +67,7 @@ type Configuration struct {
 	} `toml:"url" comment:"#####################\n CDS URLs Settings \n####################" json:"url"`
 	HTTP    service.HTTPRouterConfiguration `toml:"http" json:"http"`
 	Secrets struct {
-		Key string `toml:"key" json:"-"`
+		SkipProjectSecretsOnRegion []string `toml:"skipProjectSecretsOnRegion" json:"-"`
 	} `toml:"secrets" json:"secrets"`
 	Database database.DBConfigurationWithEncryption `toml:"database" comment:"################################\n Postgresql Database settings \n###############################" json:"database"`
 	Cache    struct {
@@ -356,10 +356,6 @@ func (a *API) CheckConfiguration(config interface{}) error {
 		}
 	}
 
-	if len(aConfig.Secrets.Key) != 32 {
-		return fmt.Errorf("invalid secret key. It should be 32 bits (%d)", len(aConfig.Secrets.Key))
-	}
-
 	if aConfig.DefaultArch == "" {
 		log.Warn(context.Background(), `You should add a default architecture in your configuration (example: defaultArch: "amd64"). It means if there is no model and os/arch requirement on your job then spawn on a worker based on this architecture`)
 	}

engine/api/application_deployment_test.go

@@ -299,6 +299,7 @@ func Test_postApplicationDeploymentStrategyConfigHandlerAsProvider(t *testing.T)
 
 	_, jws, err := builtin.NewConsumer(context.TODO(), db, sdk.RandomString(10), sdk.RandomString(10), 0, localConsumer, u.GetGroupIDs(),
 		sdk.NewAuthConsumerScopeDetails(sdk.AuthConsumerScopeProject))
+	require.NoError(t, err)
 
 	pkey := sdk.RandomString(10)
 	proj := assets.InsertTestProject(t, db, api.Cache, pkey, pkey)

engine/api/application_import_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	yaml "gopkg.in/yaml.v2"
 
 	"github.com/ovh/cds/engine/api/application"
@@ -462,6 +463,7 @@ func Test_postApplicationImportHandler_NewAppFromYAMLWithKeysAndSecretsAndReImpo
 	eapp.Keys[k2.Name] = ek2
 
 	btes, err := yaml.Marshal(eapp)
+	require.NoError(t, err)
 	body = string(btes)
 
 	t.Log(body)

engine/api/application_test.go

@@ -38,6 +38,7 @@ func Test_postApplicationMetadataHandler_AsProvider(t *testing.T) {
 	require.NoError(t, err)
 	_, jws, err := builtin.NewConsumer(context.TODO(), db, sdk.RandomString(10), sdk.RandomString(10), 0, localConsumer, u.GetGroupIDs(),
 		sdk.NewAuthConsumerScopeDetails(sdk.AuthConsumerScopeProject))
+	require.NoError(t, err)
 
 	pkey := sdk.RandomString(10)
 	proj := assets.InsertTestProject(t, db, api.Cache, pkey, pkey)

engine/api/ascode_test.go

@@ -339,6 +339,7 @@ vcs_ssh_key: proj-blabla
 
 	// mock service
 	allSrv, err := services.LoadAll(context.TODO(), db)
+	require.NoError(t, err)
 	for _, s := range allSrv {
 		if err := services.Delete(db, &s); err != nil {
 			t.Fatalf("unable to delete service: %v", err)

engine/api/project/dao_key.go

@@ -57,8 +57,8 @@ func LoadAllKeys(ctx context.Context, db gorp.SqlExecutor, projectID int64) ([]s
 }
 
 // LoadAllKeysWithPrivateContent load all keys for the given project
-func LoadAllKeysWithPrivateContent(ctx context.Context, db gorp.SqlExecutor, appID int64) ([]sdk.ProjectKey, error) {
-	keys, err := LoadAllKeys(ctx, db, appID)
+func LoadAllKeysWithPrivateContent(ctx context.Context, db gorp.SqlExecutor, projID int64) ([]sdk.ProjectKey, error) {
+	keys, err := LoadAllKeys(ctx, db, projID)
 	if err != nil {
 		return nil, err
 	}

engine/api/project_test.go

@@ -38,39 +38,25 @@ func TestVariableInProject(t *testing.T) {
 		Value: "value1",
 		Type:  "PASSWORD",
 	}
-	err := project.InsertVariable(db, project1.ID, var1, &sdk.AuthentifiedUser{Username: "foo"})
-	if err != nil {
-		t.Fatalf("cannot insert var1 in project1: %s", err)
-	}
+	require.NoError(t, project.InsertVariable(db, project1.ID, var1, &sdk.AuthentifiedUser{Username: "foo"}))
 
 	// 3. Test Update variable
 	var2 := var1
 	var2.Value = "value1Updated"
-	err = project.UpdateVariable(db, project1.ID, var2, var1, &sdk.AuthentifiedUser{Username: "foo"})
-	if err != nil {
-		t.Fatalf("cannot update var1 in project1: %s", err)
-	}
+	require.NoError(t, project.UpdateVariable(db, project1.ID, var2, var1, &sdk.AuthentifiedUser{Username: "foo"}))
 
 	// 4. Delete variable
-	err = project.DeleteVariable(api.mustDB(), project1.ID, var1, &sdk.AuthentifiedUser{Username: "foo"})
-	if err != nil {
-		t.Fatalf("cannot delete var1 from project: %s", err)
-	}
-	varTest, err := project.LoadVariable(api.mustDB(), project1.ID, var1.Name)
-	if varTest != nil {
-		t.Fatalf("var1 should be deleted: %+v", varTest)
-	}
+	require.NoError(t, project.DeleteVariable(api.mustDB(), project1.ID, var1, &sdk.AuthentifiedUser{Username: "foo"}))
+	_, err := project.LoadVariable(api.mustDB(), project1.ID, var1.Name)
+	require.Error(t, err)
 
 	// 5. Insert new var
 	var3 := &sdk.ProjectVariable{
 		Name:  "var2",
 		Value: "value2",
 		Type:  "STRING",
 	}
-	err = project.InsertVariable(db, project1.ID, var3, &sdk.AuthentifiedUser{Username: "foo"})
-	if err != nil {
-		t.Fatalf("cannot insert var1 in project1: %s", err)
-	}
+	require.NoError(t, project.InsertVariable(db, project1.ID, var3, &sdk.AuthentifiedUser{Username: "foo"}))
 }
 
 func Test_getProjectsHandler(t *testing.T) {
@@ -324,6 +310,7 @@ func Test_getprojectsHandler_AsProviderWithRequestedUsername(t *testing.T) {
 
 	_, jws, err := builtin.NewConsumer(context.TODO(), db, sdk.RandomString(10), sdk.RandomString(10), 0, localConsumer, admin.GetGroupIDs(),
 		sdk.NewAuthConsumerScopeDetails(sdk.AuthConsumerScopeProject))
+	require.NoError(t, err)
 
 	u, _ := assets.InsertLambdaUser(t, db)
 
@@ -436,6 +423,7 @@ func Test_getProjectsHandler_FilterByRepo(t *testing.T) {
 
 	_, jws, err := builtin.NewConsumer(context.TODO(), db, sdk.RandomString(10), sdk.RandomString(10), 0, localConsumer, admin.GetGroupIDs(),
 		sdk.NewAuthConsumerScopeDetails(sdk.AuthConsumerScopeProject))
+	require.NoError(t, err)
 
 	u, _ := assets.InsertLambdaUser(t, db)
 

engine/api/purge/purge.go

@@ -346,7 +346,7 @@ func DeleteArtifactsFromRepositoryManager(ctx context.Context, db gorp.SqlExecut
 	var rtToken string
 	for _, s := range secrets {
 		if s.Name == fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactoryConfigToken) {
-			rtToken = s.Value
+			rtToken = string(s.Value)
 			break
 		}
 	}

engine/api/websocket_test.go

@@ -34,6 +34,7 @@ func Test_websocketWrongFilters(t *testing.T) {
 
 	_, jws, err := builtin.NewConsumer(context.TODO(), db, sdk.RandomString(10), sdk.RandomString(10), 0, localConsumer, u.GetGroupIDs(),
 		sdk.NewAuthConsumerScopeDetails(sdk.AuthConsumerScopeProject))
+	require.NoError(t, err)
 
 	chanMessageReceived := make(chan sdk.WebsocketEvent)
 	chanMessageToSend := make(chan []sdk.WebsocketFilter)
@@ -72,6 +73,7 @@ func Test_websocketFilterRetroCompatibility(t *testing.T) {
 
 	u, _ := assets.InsertLambdaUser(t, db)
 	localConsumer, err := authentication.LoadConsumerByTypeAndUserID(context.TODO(), db, sdk.ConsumerLocal, u.ID, authentication.LoadConsumerOptions.WithAuthentifiedUser)
+	require.NoError(t, err)
 
 	c := &websocketClientData{
 		AuthConsumer: *localConsumer,
@@ -188,6 +190,7 @@ func Test_websocketDeconnection(t *testing.T) {
 
 	_, jws, err := builtin.NewConsumer(context.TODO(), db, sdk.RandomString(10), sdk.RandomString(10), 0, localConsumer, u.GetGroupIDs(),
 		sdk.NewAuthConsumerScopeDetails(sdk.AuthConsumerScopeProject))
+	require.NoError(t, err)
 
 	// Open websocket
 	client := cdsclient.New(cdsclient.Config{

engine/api/workflow/dao_run_secret.go

@@ -29,27 +29,23 @@ func InsertRunSecret(ctx context.Context, db gorpmapper.SqlExecutorWithTx, wrSec
 	return nil
 }
 
-func loadRunSecretWithDecryption(ctx context.Context, db gorp.SqlExecutor, runID int64, entities []string) ([]sdk.Variable, error) {
+func loadRunSecretWithDecryption(ctx context.Context, db gorp.SqlExecutor, runID int64, entities []string) (sdk.WorkflowRunSecrets, error) {
 	var dbSecrets []dbWorkflowRunSecret
 	query := gorpmapping.NewQuery(`SELECT * FROM workflow_run_secret WHERE workflow_run_id = $1 AND context = ANY(string_to_array($2, ',')::text[])`).Args(runID, gorpmapping.IDStringsToQueryString(entities))
 	if err := gorpmapping.GetAll(ctx, db, query, &dbSecrets, gorpmapping.GetOptions.WithDecryption); err != nil {
 		return nil, err
 	}
-	secrets := make([]sdk.Variable, len(dbSecrets))
+	secrets := make(sdk.WorkflowRunSecrets, 0, len(dbSecrets))
 	for i := range dbSecrets {
 		isValid, err := gorpmapping.CheckSignature(dbSecrets[i], dbSecrets[i].Signature)
 		if err != nil {
 			return nil, err
 		}
 		if !isValid {
-			log.Error(ctx, "workflow.loadRunSecretWithDecryption> secret value corrupted %s", dbSecrets[i].ID)
+			log.Error(ctx, "secret value corrupted %s", dbSecrets[i].ID)
 			continue
 		}
-		secrets[i] = sdk.Variable{
-			Name:  dbSecrets[i].Name,
-			Type:  dbSecrets[i].Type,
-			Value: string(dbSecrets[i].Value),
-		}
+		secrets = append(secrets, dbSecrets[i].WorkflowRunSecret)
 	}
 	return secrets, nil
 }

engine/api/workflow/execute_node_job_run.go

@@ -305,7 +305,7 @@ func checkStatusWaiting(ctx context.Context, store cache.Store, jobID int64, sta
 }
 
 // LoadDecryptSecrets loads all secrets for a job run
-func LoadDecryptSecrets(ctx context.Context, db gorp.SqlExecutor, wr *sdk.WorkflowRun, nodeRun *sdk.WorkflowNodeRun) ([]sdk.Variable, error) {
+func LoadDecryptSecrets(ctx context.Context, db gorp.SqlExecutor, wr *sdk.WorkflowRun, nodeRun *sdk.WorkflowNodeRun) (sdk.WorkflowRunSecrets, error) {
 	entities := []string{SecretProjContext}
 
 	for _, integ := range wr.Workflow.Integrations {

engine/api/workflow/workflow_run_results.go

@@ -198,7 +198,7 @@ func verifyAddResultArtifactManager(ctx context.Context, db gorp.SqlExecutor, st
 	var artifactManagerToken string
 	for _, s := range secrets {
 		if s.Name == fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactoryConfigToken) {
-			artifactManagerToken = s.Value
+			artifactManagerToken = string(s.Value)
 			break
 		}
 	}
@@ -325,7 +325,7 @@ func LoadRunResultsByRunID(ctx context.Context, db gorp.SqlExecutor, runID int64
 		SELECT data->>'name' AS name, sub_num, id
 		FROM workflow_run_result
 		WHERE workflow_run_id = $1
-	), 
+	),
 	deduplication AS (
 		SELECT distinct on (name) *
 		FROM allResults

engine/api/workflow_ascode_with_hooks_test.go

@@ -478,7 +478,7 @@ version: v1.0`),
 
 	require.NoError(t, waitCraftinWorkflow(t, api, api.mustDB(), wrun.ID))
 	wr, err := workflow.LoadRunByID(context.Background(), db, wrun.ID, workflow.LoadRunOptions{})
-	require.NoError(t, nil)
+	require.NoError(t, err)
 	require.NotEqual(t, "Fail", wr.Status)
 
 	wk, err = workflow.Load(context.Background(), db, api.Cache, *proj, "w-go-repo", workflow.LoadOptions{})

engine/api/workflow_ascode_with_secrets_test.go

@@ -353,8 +353,10 @@ version: v1.0`),
 	t.Logf("%d %+v", wrDB.Workflow.WorkflowData.Node.ID, wrDB.WorkflowNodeRuns)
 	require.NotNil(t, wrDB.WorkflowNodeRuns[wrDB.Workflow.WorkflowData.Node.ID])
 	require.Len(t, wrDB.WorkflowNodeRuns[wrDB.Workflow.WorkflowData.Node.ID], 1)
-	secrets, errS := workflow.LoadDecryptSecrets(context.TODO(), db, wrDB, &wrDB.WorkflowNodeRuns[wrDB.Workflow.WorkflowData.Node.ID][0])
-	require.NoError(t, errS)
+	secretsRaw, err := workflow.LoadDecryptSecrets(context.TODO(), db, wrDB, &wrDB.WorkflowNodeRuns[wrDB.Workflow.WorkflowData.Node.ID][0])
+	require.NoError(t, err)
+
+	secrets := secretsRaw.ToVariables()
 
 	t.Logf("%+v", secrets)
 	require.Len(t, secrets, 6)

engine/api/workflow_hook.go

@@ -289,7 +289,7 @@ func (api *API) postWorkflowJobHookCallbackHandler() service.Handler {
 
 		// Hide secrets in payload
 		for _, s := range secrets {
-			callback.Log = strings.Replace(callback.Log, s.Value, "**"+s.Name+"**", -1)
+			callback.Log = strings.Replace(callback.Log, string(s.Value), sdk.PasswordPlaceholder, -1)
 		}
 
 		report, err := workflow.UpdateOutgoingHookRunStatus(ctx, tx, api.Cache, *proj, wr, hookRunID, callback)
@@ -336,11 +336,6 @@ func (api *API) getWorkflowJobHookDetailsHandler() service.Handler {
 			return sdk.WithStack(sdk.ErrNotFound)
 		}
 
-		secrets, errSecret := workflow.LoadDecryptSecrets(ctx, db, wr, nil)
-		if errSecret != nil {
-			return sdk.WrapError(errSecret, "cannot load secrets")
-		}
-		hr.BuildParameters = append(hr.BuildParameters, sdk.VariablesToParameters("", secrets)...)
 		return service.WriteJSON(w, hr, http.StatusOK)
 	}
 }

engine/api/workflow_purge_test.go

@@ -41,6 +41,7 @@ func Test_purgeDryRunHandler(t *testing.T) {
 
 	_, jws, err := builtin.NewConsumer(context.TODO(), db, sdk.RandomString(10), sdk.RandomString(10), 0, localConsumer, u.GetGroupIDs(),
 		sdk.NewAuthConsumerScopeDetails(sdk.AuthConsumerScopeProject))
+	require.NoError(t, err)
 
 	key := sdk.RandomString(10)
 	proj := assets.InsertTestProject(t, db, api.Cache, key, key)

engine/api/workflow_queue.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
+	"regexp"
 	"strings"
 
 	"net/http"
@@ -106,7 +107,7 @@ func (api *API) postTakeWorkflowJobHandler() service.Handler {
 		}
 
 		pbji := &sdk.WorkflowNodeJobRunData{}
-		report, err := takeJob(ctx, api.mustDB, api.Cache, p, id, workerModelName, pbji, wk, hatcheryName)
+		report, err := takeJob(ctx, api.mustDB, api.Cache, p, id, workerModelName, pbji, wk, hatcheryName, api.Config.Secrets.SkipProjectSecretsOnRegion)
 		if err != nil {
 			return sdk.WrapError(err, "cannot takeJob nodeJobRunID:%d", id)
 		}
@@ -136,15 +137,14 @@ func (api *API) postTakeWorkflowJobHandler() service.Handler {
 	}
 }
 
-func takeJob(ctx context.Context, dbFunc func() *gorp.DbMap, store cache.Store, p *sdk.Project, id int64, workerModel string, wnjri *sdk.WorkflowNodeJobRunData, wk *sdk.Worker, hatcheryName string) (*workflow.ProcessorReport, error) {
-	// Start a tx
-	tx, errBegin := dbFunc().Begin()
-	if errBegin != nil {
-		return nil, sdk.WrapError(errBegin, "Cannot start transaction")
+func takeJob(ctx context.Context, dbFunc func() *gorp.DbMap, store cache.Store, p *sdk.Project, id int64, workerModel string, wnjri *sdk.WorkflowNodeJobRunData, wk *sdk.Worker, hatcheryName string, skipProjectSecretsOnRegion []string) (*workflow.ProcessorReport, error) {
+	tx, err := dbFunc().Begin()
+	if err != nil {
+		return nil, sdk.WrapError(err, "cannot start transaction")
 	}
 	defer tx.Rollback() // nolint
 
-	//Prepare spawn infos
+	// Prepare spawn infos
 	m1 := sdk.SpawnMsg{ID: sdk.MsgSpawnInfoJobTaken.ID, Args: []interface{}{fmt.Sprintf("%d", id), wk.Name}}
 	m2 := sdk.SpawnMsg{ID: sdk.MsgSpawnInfoJobTakenWorkerVersion.ID, Args: []interface{}{wk.Name, wk.Version, wk.OS, wk.Arch}}
 	infos := []sdk.SpawnInfo{
@@ -186,15 +186,15 @@ func takeJob(ctx context.Context, dbFunc func() *gorp.DbMap, store cache.Store,
 	if noderun.Status == sdk.StatusWaiting {
 		noderun.Status = sdk.StatusBuilding
 		if err := workflow.UpdateNodeRun(tx, noderun); err != nil {
-			return nil, sdk.WrapError(err, "Cannot update node run")
+			return nil, sdk.WrapError(err, "cannot update node run")
 		}
 		report.Add(ctx, *noderun)
 	}
 
 	// Load workflow run
 	workflowRun, err := workflow.LoadRunByID(ctx, tx, noderun.WorkflowRunID, workflow.LoadRunOptions{})
 	if err != nil {
-		return nil, sdk.WrapError(err, "Unable to load workflow run")
+		return nil, sdk.WrapError(err, "unable to load workflow run")
 	}
 
 	secrets, err := workflow.LoadDecryptSecrets(ctx, tx, workflowRun, noderun)
@@ -207,7 +207,7 @@ func takeJob(ctx context.Context, dbFunc func() *gorp.DbMap, store cache.Store,
 	wnjri.NodeJobRun = *job
 	wnjri.Number = noderun.Number
 	wnjri.SubNumber = noderun.SubNumber
-	wnjri.Secrets = secrets
+	wnjri.Secrets = make([]sdk.Variable, 0, len(secrets))
 	wnjri.RunID = workflowRun.ID
 	wnjri.WorkflowID = workflowRun.WorkflowID
 	wnjri.WorkflowName = workflowRun.Workflow.Name
@@ -217,6 +217,34 @@ func takeJob(ctx context.Context, dbFunc func() *gorp.DbMap, store cache.Store,
 		return nil, sdk.WithStack(err)
 	}
 
+	secretsReqs := job.Job.Action.Requirements.FilterByType(sdk.SecretRequirement).Values()
+	secretsReqsRegs := make([]*regexp.Regexp, 0, len(secretsReqs))
+	for i := range secretsReqs {
+		r, err := regexp.Compile(secretsReqs[i])
+		if err != nil {
+			return nil, sdk.WithStack(err)
+		}
+		secretsReqsRegs = append(secretsReqsRegs, r)
+	}
+
+	// Filter project's secrets depending of the region requirement that was set on job
+	skipProjectSecrets := job.Region != nil && sdk.IsInArray(*job.Region, skipProjectSecretsOnRegion)
+	for i := range secrets {
+		if skipProjectSecrets && secrets[i].Context == workflow.SecretProjContext {
+			var inRequirements bool
+			for _, reg := range secretsReqsRegs {
+				if reg.MatchString(secrets[i].Name) {
+					inRequirements = true
+					break
+				}
+			}
+			if !inRequirements {
+				continue
+			}
+		}
+		wnjri.Secrets = append(wnjri.Secrets, secrets[i].ToVariable())
+	}
+
 	return report, nil
 }