feat(api): disable project secrets auto injection for given regions

docs/content/docs/concepts/requirement/_index.md

@@ -15,6 +15,7 @@ Requirement types:
 - [Memory]({{< relref "/docs/concepts/requirement/requirement_memory.md" >}})
 - [OS & Architecture]({{< relref "/docs/concepts/requirement/requirement_os_arch.md" >}})
 - [Region]({{< relref "/docs/concepts/requirement/requirement_region.md" >}})
+- [Secret]({{< relref "/docs/concepts/requirement/requirement_secret.md" >}})
 
 A [Job]({{< relref "/docs/concepts/job.md" >}}) will be executed by a **worker**.
 

docs/content/docs/concepts/requirement/requirement_secret.md

@@ -0,0 +1,42 @@
+---
+title: "Secret Requirement"
+weight: 4
+---
+
+The `Secret` prerequisite allows you to require a worker to start with some project's secrets when those secrets are not automatically injected.
+
+Secret automatic injection can be disabled if a job requires to run in a specific region (using a "Region" prerequisite) that was added in CDS API configuration (key: skipProjectSecretsOnRegion).
+
+The value for the requirement should be a valid regex. In the following example it is used to match both default SSH and PGP keys for a CDS project.
+
+Example of job configuration:
+```
+- job: build
+  requirements:
+  - region: myregion
+  - secret: ^cds.key.proj-(ssh|pgp)-test.priv$
+  steps:
+  ...
+```
+
+Example of CDS API configuration:
+```
+[api]
+  ...
+  [api.secrets]
+    ...
+    skipProjectSecretsOnRegion = ["myregion"]
+```
+
+Example of CDS Hatchery configuration:
+```
+[hatchery]
+  [hatchery.local]
+    ...
+    [hatchery.local.commonConfiguration]
+      ...
+      [hatchery.local.commonConfiguration.provision]
+        ...
+        region = "myregion"
+        ignoreJobWithNoRegion = true
+```

engine/api/api.go

@@ -67,7 +67,7 @@ type Configuration struct {
 	} `toml:"url" comment:"#####################\n CDS URLs Settings \n####################" json:"url"`
 	HTTP    service.HTTPRouterConfiguration `toml:"http" json:"http"`
 	Secrets struct {
-		Key string `toml:"key" json:"-"`
+		SkipProjectSecretsOnRegion []string `toml:"skipProjectSecretsOnRegion" json:"-"`
 	} `toml:"secrets" json:"secrets"`
 	Database database.DBConfigurationWithEncryption `toml:"database" comment:"################################\n Postgresql Database settings \n###############################" json:"database"`
 	Cache    struct {
@@ -356,10 +356,6 @@ func (a *API) CheckConfiguration(config interface{}) error {
 		}
 	}
 
-	if len(aConfig.Secrets.Key) != 32 {
-		return fmt.Errorf("invalid secret key. It should be 32 bits (%d)", len(aConfig.Secrets.Key))
-	}
-
 	if aConfig.DefaultArch == "" {
 		log.Warn(context.Background(), `You should add a default architecture in your configuration (example: defaultArch: "amd64"). It means if there is no model and os/arch requirement on your job then spawn on a worker based on this architecture`)
 	}

engine/api/application_deployment_test.go

@@ -299,6 +299,7 @@ func Test_postApplicationDeploymentStrategyConfigHandlerAsProvider(t *testing.T)
 
 	_, jws, err := builtin.NewConsumer(context.TODO(), db, sdk.RandomString(10), sdk.RandomString(10), 0, localConsumer, u.GetGroupIDs(),
 		sdk.NewAuthConsumerScopeDetails(sdk.AuthConsumerScopeProject))
+	require.NoError(t, err)
 
 	pkey := sdk.RandomString(10)
 	proj := assets.InsertTestProject(t, db, api.Cache, pkey, pkey)

engine/api/application_import_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	yaml "gopkg.in/yaml.v2"
 
 	"github.com/ovh/cds/engine/api/application"
@@ -462,6 +463,7 @@ func Test_postApplicationImportHandler_NewAppFromYAMLWithKeysAndSecretsAndReImpo
 	eapp.Keys[k2.Name] = ek2
 
 	btes, err := yaml.Marshal(eapp)
+	require.NoError(t, err)
 	body = string(btes)
 
 	t.Log(body)

engine/api/application_test.go

@@ -38,6 +38,7 @@ func Test_postApplicationMetadataHandler_AsProvider(t *testing.T) {
 	require.NoError(t, err)
 	_, jws, err := builtin.NewConsumer(context.TODO(), db, sdk.RandomString(10), sdk.RandomString(10), 0, localConsumer, u.GetGroupIDs(),
 		sdk.NewAuthConsumerScopeDetails(sdk.AuthConsumerScopeProject))
+	require.NoError(t, err)
 
 	pkey := sdk.RandomString(10)
 	proj := assets.InsertTestProject(t, db, api.Cache, pkey, pkey)

engine/api/ascode_test.go

@@ -339,6 +339,7 @@ vcs_ssh_key: proj-blabla
 
 	// mock service
 	allSrv, err := services.LoadAll(context.TODO(), db)
+	require.NoError(t, err)
 	for _, s := range allSrv {
 		if err := services.Delete(db, &s); err != nil {
 			t.Fatalf("unable to delete service: %v", err)

engine/api/pipeline.go

@@ -40,11 +40,8 @@ func (api *API) updateAsCodePipelineHandler() service.Handler {
 		if err := service.UnmarshalBody(r, &p); err != nil {
 			return err
 		}
-
-		// check pipeline name pattern
-		regexp := sdk.NamePatternRegex
-		if !regexp.MatchString(p.Name) {
-			return sdk.WrapError(sdk.ErrInvalidPipelinePattern, "updateAsCodePipelineHandler: pipeline name %s do not respect pattern", p.Name)
+		if err := p.IsValid(); err != nil {
+			return err
 		}
 
 		tx, err := api.mustDB().Begin()
@@ -154,13 +151,10 @@ func (api *API) updatePipelineHandler() service.Handler {
 
 		var p sdk.Pipeline
 		if err := service.UnmarshalBody(r, &p); err != nil {
-			return sdk.WrapError(err, "Cannot read body")
+			return err
 		}
-
-		// check pipeline name pattern
-		regexp := sdk.NamePatternRegex
-		if !regexp.MatchString(p.Name) {
-			return sdk.WrapError(sdk.ErrInvalidPipelinePattern, "updatePipelineHandler: Pipeline name %s do not respect pattern", p.Name)
+		if err := p.IsValid(); err != nil {
+			return err
 		}
 
 		pipelineDB, err := pipeline.LoadPipeline(ctx, api.mustDB(), key, name, true)
@@ -172,9 +166,9 @@ func (api *API) updatePipelineHandler() service.Handler {
 			return sdk.WithStack(sdk.ErrForbidden)
 		}
 
-		tx, errB := api.mustDB().Begin()
-		if errB != nil {
-			sdk.WrapError(errB, "updatePipelineHandler> Cannot start transaction")
+		tx, err := api.mustDB().Begin()
+		if err != nil {
+			return sdk.WrapError(err, "cannot start transaction")
 		}
 		defer tx.Rollback() // nolint
 
@@ -283,10 +277,8 @@ func (api *API) addPipelineHandler() service.Handler {
 		if err := service.UnmarshalBody(r, &p); err != nil {
 			return err
 		}
-
-		// check pipeline name pattern
-		if regexp := sdk.NamePatternRegex; !regexp.MatchString(p.Name) {
-			return sdk.NewErrorFrom(sdk.ErrInvalidPipelinePattern, "pipeline name %s do not respect pattern %s", p.Name, sdk.NamePattern)
+		if err := p.IsValid(); err != nil {
+			return err
 		}
 
 		// Check that pipeline does not already exists

engine/api/pipeline/pipeline_parser.go

@@ -22,9 +22,9 @@ type ImportOptions struct {
 // ParseAndImport parse an exportentities.pipeline and insert or update the pipeline in database
 func ParseAndImport(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj sdk.Project, epip exportentities.Pipeliner, u sdk.Identifiable, opts ImportOptions) (*sdk.Pipeline, []sdk.Message, error) {
 	//Transform payload to a sdk.Pipeline
-	pip, errP := epip.Pipeline()
-	if errP != nil {
-		return pip, nil, sdk.WrapError(sdk.NewError(sdk.ErrWrongRequest, errP), "unable to parse pipeline")
+	pip, err := epip.Pipeline()
+	if err != nil {
+		return nil, nil, err
 	}
 
 	pip.FromRepository = opts.FromRepository
@@ -34,9 +34,9 @@ func ParseAndImport(ctx context.Context, db gorp.SqlExecutor, cache cache.Store,
 	}
 
 	// Check if pipeline exists
-	exist, errE := ExistPipeline(db, proj.ID, pip.Name)
-	if errE != nil {
-		return pip, nil, sdk.WrapError(errE, "unable to check if pipeline %v exists", pip.Name)
+	exist, err := ExistPipeline(db, proj.ID, pip.Name)
+	if err != nil {
+		return pip, nil, sdk.WrapError(err, "unable to check if pipeline %v exists", pip.Name)
 	}
 
 	done := new(sync.WaitGroup)

engine/api/pipeline_import.go

@@ -149,7 +149,7 @@ func (api *API) putImportPipelineHandler() service.Handler {
 		pip, allMsg, err := pipeline.ParseAndImport(ctx, tx, api.Cache, *proj, data, getAPIConsumer(ctx), pipeline.ImportOptions{Force: true, PipelineName: pipelineName})
 		msgListString := translate(allMsg)
 		if err != nil {
-			return sdk.NewErrorWithStack(err, sdk.NewErrorFrom(sdk.ErrInvalidPipeline, "unable to parse and import pipeline"))
+			return sdk.ErrorWithFallback(err, sdk.ErrWrongRequest, "unable to import pipeline")
 		}
 
 		if err := tx.Commit(); err != nil {

engine/api/project/dao_key.go

@@ -57,8 +57,8 @@ func LoadAllKeys(ctx context.Context, db gorp.SqlExecutor, projectID int64) ([]s
 }
 
 // LoadAllKeysWithPrivateContent load all keys for the given project
-func LoadAllKeysWithPrivateContent(ctx context.Context, db gorp.SqlExecutor, appID int64) ([]sdk.ProjectKey, error) {
-	keys, err := LoadAllKeys(ctx, db, appID)
+func LoadAllKeysWithPrivateContent(ctx context.Context, db gorp.SqlExecutor, projID int64) ([]sdk.ProjectKey, error) {
+	keys, err := LoadAllKeys(ctx, db, projID)
 	if err != nil {
 		return nil, err
 	}

engine/api/project_test.go

@@ -38,39 +38,25 @@ func TestVariableInProject(t *testing.T) {
 		Value: "value1",
 		Type:  "PASSWORD",
 	}
-	err := project.InsertVariable(db, project1.ID, var1, &sdk.AuthentifiedUser{Username: "foo"})
-	if err != nil {
-		t.Fatalf("cannot insert var1 in project1: %s", err)
-	}
+	require.NoError(t, project.InsertVariable(db, project1.ID, var1, &sdk.AuthentifiedUser{Username: "foo"}))
 
 	// 3. Test Update variable
 	var2 := var1
 	var2.Value = "value1Updated"
-	err = project.UpdateVariable(db, project1.ID, var2, var1, &sdk.AuthentifiedUser{Username: "foo"})
-	if err != nil {
-		t.Fatalf("cannot update var1 in project1: %s", err)
-	}
+	require.NoError(t, project.UpdateVariable(db, project1.ID, var2, var1, &sdk.AuthentifiedUser{Username: "foo"}))
 
 	// 4. Delete variable
-	err = project.DeleteVariable(api.mustDB(), project1.ID, var1, &sdk.AuthentifiedUser{Username: "foo"})
-	if err != nil {
-		t.Fatalf("cannot delete var1 from project: %s", err)
-	}
-	varTest, err := project.LoadVariable(api.mustDB(), project1.ID, var1.Name)
-	if varTest != nil {
-		t.Fatalf("var1 should be deleted: %+v", varTest)
-	}
+	require.NoError(t, project.DeleteVariable(api.mustDB(), project1.ID, var1, &sdk.AuthentifiedUser{Username: "foo"}))
+	_, err := project.LoadVariable(api.mustDB(), project1.ID, var1.Name)
+	require.Error(t, err)
 
 	// 5. Insert new var
 	var3 := &sdk.ProjectVariable{
 		Name:  "var2",
 		Value: "value2",
 		Type:  "STRING",
 	}
-	err = project.InsertVariable(db, project1.ID, var3, &sdk.AuthentifiedUser{Username: "foo"})
-	if err != nil {
-		t.Fatalf("cannot insert var1 in project1: %s", err)
-	}
+	require.NoError(t, project.InsertVariable(db, project1.ID, var3, &sdk.AuthentifiedUser{Username: "foo"}))
 }
 
 func Test_getProjectsHandler(t *testing.T) {
@@ -324,6 +310,7 @@ func Test_getprojectsHandler_AsProviderWithRequestedUsername(t *testing.T) {
 
 	_, jws, err := builtin.NewConsumer(context.TODO(), db, sdk.RandomString(10), sdk.RandomString(10), 0, localConsumer, admin.GetGroupIDs(),
 		sdk.NewAuthConsumerScopeDetails(sdk.AuthConsumerScopeProject))
+	require.NoError(t, err)
 
 	u, _ := assets.InsertLambdaUser(t, db)
 
@@ -436,6 +423,7 @@ func Test_getProjectsHandler_FilterByRepo(t *testing.T) {
 
 	_, jws, err := builtin.NewConsumer(context.TODO(), db, sdk.RandomString(10), sdk.RandomString(10), 0, localConsumer, admin.GetGroupIDs(),
 		sdk.NewAuthConsumerScopeDetails(sdk.AuthConsumerScopeProject))
+	require.NoError(t, err)
 
 	u, _ := assets.InsertLambdaUser(t, db)
 

engine/api/purge/purge.go

@@ -346,7 +346,7 @@ func DeleteArtifactsFromRepositoryManager(ctx context.Context, db gorp.SqlExecut
 	var rtToken string
 	for _, s := range secrets {
 		if s.Name == fmt.Sprintf("cds.integration.artifact_manager.%s", sdk.ArtifactoryConfigToken) {
-			rtToken = s.Value
+			rtToken = string(s.Value)
 			break
 		}
 	}

engine/api/websocket_test.go

@@ -34,6 +34,7 @@ func Test_websocketWrongFilters(t *testing.T) {
 
 	_, jws, err := builtin.NewConsumer(context.TODO(), db, sdk.RandomString(10), sdk.RandomString(10), 0, localConsumer, u.GetGroupIDs(),
 		sdk.NewAuthConsumerScopeDetails(sdk.AuthConsumerScopeProject))
+	require.NoError(t, err)
 
 	chanMessageReceived := make(chan sdk.WebsocketEvent)
 	chanMessageToSend := make(chan []sdk.WebsocketFilter)
@@ -72,6 +73,7 @@ func Test_websocketFilterRetroCompatibility(t *testing.T) {
 
 	u, _ := assets.InsertLambdaUser(t, db)
 	localConsumer, err := authentication.LoadConsumerByTypeAndUserID(context.TODO(), db, sdk.ConsumerLocal, u.ID, authentication.LoadConsumerOptions.WithAuthentifiedUser)
+	require.NoError(t, err)
 
 	c := &websocketClientData{
 		AuthConsumer: *localConsumer,
@@ -188,6 +190,7 @@ func Test_websocketDeconnection(t *testing.T) {
 
 	_, jws, err := builtin.NewConsumer(context.TODO(), db, sdk.RandomString(10), sdk.RandomString(10), 0, localConsumer, u.GetGroupIDs(),
 		sdk.NewAuthConsumerScopeDetails(sdk.AuthConsumerScopeProject))
+	require.NoError(t, err)
 
 	// Open websocket
 	client := cdsclient.New(cdsclient.Config{

engine/api/workflow/dao_node_job_run_info.go

@@ -49,9 +49,9 @@ func LoadNodeRunJobInfo(ctx context.Context, db gorp.SqlExecutor, nodeRunID int6
 // a temporary data, as workflow_node_job_run table. After the end of the Job,
 // swpawninfos values will be in WorfklowRun table in stages column
 func insertNodeRunJobInfo(db gorp.SqlExecutor, info *sdk.WorkflowNodeJobRunInfo) error {
-	spawnJSON, errJ := json.Marshal(info.SpawnInfos)
-	if errJ != nil {
-		return sdk.WrapError(errJ, "insertNodeRunJobInfo> cannot Marshal")
+	spawnJSON, err := json.Marshal(info.SpawnInfos)
+	if err != nil {
+		return sdk.WithStack(err)
 	}
 
 	query := "insert into workflow_node_run_job_info (workflow_node_run_id, workflow_node_run_job_id, spawninfos, created) values ($1, $2, $3, $4)"

engine/api/workflow/dao_run_secret.go

@@ -29,27 +29,23 @@ func InsertRunSecret(ctx context.Context, db gorpmapper.SqlExecutorWithTx, wrSec
 	return nil
 }
 
-func loadRunSecretWithDecryption(ctx context.Context, db gorp.SqlExecutor, runID int64, entities []string) ([]sdk.Variable, error) {
+func loadRunSecretWithDecryption(ctx context.Context, db gorp.SqlExecutor, runID int64, entities []string) (sdk.WorkflowRunSecrets, error) {
 	var dbSecrets []dbWorkflowRunSecret
 	query := gorpmapping.NewQuery(`SELECT * FROM workflow_run_secret WHERE workflow_run_id = $1 AND context = ANY(string_to_array($2, ',')::text[])`).Args(runID, gorpmapping.IDStringsToQueryString(entities))
 	if err := gorpmapping.GetAll(ctx, db, query, &dbSecrets, gorpmapping.GetOptions.WithDecryption); err != nil {
 		return nil, err
 	}
-	secrets := make([]sdk.Variable, len(dbSecrets))
+	secrets := make(sdk.WorkflowRunSecrets, 0, len(dbSecrets))
 	for i := range dbSecrets {
 		isValid, err := gorpmapping.CheckSignature(dbSecrets[i], dbSecrets[i].Signature)
 		if err != nil {
 			return nil, err
 		}
 		if !isValid {
-			log.Error(ctx, "workflow.loadRunSecretWithDecryption> secret value corrupted %s", dbSecrets[i].ID)
+			log.Error(ctx, "secret value corrupted %s", dbSecrets[i].ID)
 			continue
 		}
-		secrets[i] = sdk.Variable{
-			Name:  dbSecrets[i].Name,
-			Type:  dbSecrets[i].Type,
-			Value: string(dbSecrets[i].Value),
-		}
+		secrets = append(secrets, dbSecrets[i].WorkflowRunSecret)
 	}
 	return secrets, nil
 }

engine/api/workflow/execute_node_job_run.go

@@ -223,12 +223,16 @@ func PrepareSpawnInfos(infos []sdk.SpawnInfo) []sdk.SpawnInfo {
 	now := time.Now()
 	prepared := make([]sdk.SpawnInfo, 0)
 	for _, info := range infos {
-		prepared = append(prepared, sdk.SpawnInfo{
+		preparedInfo := sdk.SpawnInfo{
 			APITime:     now,
 			RemoteTime:  info.RemoteTime,
 			Message:     info.Message,
 			UserMessage: info.Message.DefaultUserMessage(),
-		})
+		}
+		if preparedInfo.RemoteTime.IsZero() {
+			preparedInfo.RemoteTime = now
+		}
+		prepared = append(prepared, preparedInfo)
 	}
 	return prepared
 }
@@ -275,7 +279,7 @@ func TakeNodeJobRun(ctx context.Context, db gorpmapper.SqlExecutorWithTx, store
 		return nil, nil, sdk.WrapError(err, "cannot update worker_id in node job run %d", jobID)
 	}
 
-	if err := AddSpawnInfosNodeJobRun(db, job.WorkflowNodeRunID, jobID, PrepareSpawnInfos(infos)); err != nil {
+	if err := AddSpawnInfosNodeJobRun(db, job.WorkflowNodeRunID, jobID, infos); err != nil {
 		return nil, nil, sdk.WrapError(err, "cannot save spawn info on node job run %d", jobID)
 	}
 
@@ -305,7 +309,7 @@ func checkStatusWaiting(ctx context.Context, store cache.Store, jobID int64, sta
 }
 
 // LoadDecryptSecrets loads all secrets for a job run
-func LoadDecryptSecrets(ctx context.Context, db gorp.SqlExecutor, wr *sdk.WorkflowRun, nodeRun *sdk.WorkflowNodeRun) ([]sdk.Variable, error) {
+func LoadDecryptSecrets(ctx context.Context, db gorp.SqlExecutor, wr *sdk.WorkflowRun, nodeRun *sdk.WorkflowNodeRun) (sdk.WorkflowRunSecrets, error) {
 	entities := []string{SecretProjContext}
 
 	for _, integ := range wr.Workflow.Integrations {