From 44eae68460abaf5e71f5192b4b2fd27ba0d4b1ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Samin?= Date: Mon, 25 Jan 2021 09:41:51 +0100 Subject: [PATCH] feat(api,cli): new feature to list and check encrypted secrets (#5648) Signed-off-by: francois samin --- cli/cdsctl/encrypt.go | 22 +++++++- engine/api/api_routes.go | 1 + .../api/environment/environment_importer.go | 2 +- engine/api/project/key.go | 14 ++++- engine/api/project_variable.go | 34 ++++++++++++ engine/cdn/storage/storageunit_run.go | 4 +- sdk/cdsclient/client_project_variable.go | 8 +++ sdk/cdsclient/interface.go | 1 + .../mock_cdsclient/interface_mock.go | 53 +++++++++++++++++-- sdk/variable.go | 7 +++ .../parameter/value/parameter.value.html | 2 +- 11 files changed, 139 insertions(+), 9 deletions(-) diff --git a/cli/cdsctl/encrypt.go b/cli/cdsctl/encrypt.go index 282aa9a90d..8faa7dbf6f 100644 --- a/cli/cdsctl/encrypt.go +++ b/cli/cdsctl/encrypt.go @@ -59,7 +59,7 @@ my-data: 01234567890987654321`, } func encrypt() *cobra.Command { - return cli.NewCommand(encryptCmd, encryptRun, nil, withAllCommandModifiers()...) + return cli.NewCommand(encryptCmd, encryptRun, cli.SubCommands{encryptList()}, withAllCommandModifiers()...) } func encryptRun(v cli.Values) error { @@ -76,3 +76,23 @@ func encryptRun(v cli.Values) error { fmt.Printf("%s: %s\n", variable.Name, variable.Value) return nil } + +var encryptListCmd = cli.Command{ + Name: "list", + Short: "List all the encrypted variable of your CDS project", + Ctx: []cli.Arg{ + {Name: _ProjectKey}, + }, +} + +func encryptList() *cobra.Command { + return cli.NewListCommand(encryptListCmd, encryptListRun, nil, withAllCommandModifiers()...) +} + +func encryptListRun(v cli.Values) (cli.ListResult, error) { + secrets, err := client.VariableListEncrypt(v.GetString(_ProjectKey)) + if err != nil { + return nil, err + } + return cli.AsListResult(secrets), nil +} diff --git a/engine/api/api_routes.go b/engine/api/api_routes.go index f4128a4d9f..81d45c9d8c 100644 --- a/engine/api/api_routes.go +++ b/engine/api/api_routes.go @@ -158,6 +158,7 @@ func (api *API) InitRouter() { r.Handle("/project/{permProjectKey}/group/{groupName}", Scope(sdk.AuthConsumerScopeProject), r.PUT(api.putGroupRoleOnProjectHandler), r.DELETE(api.deleteGroupFromProjectHandler)) r.Handle("/project/{permProjectKey}/variable", Scope(sdk.AuthConsumerScopeProject), r.GET(api.getVariablesInProjectHandler)) r.Handle("/project/{permProjectKey}/encrypt", Scope(sdk.AuthConsumerScopeProject), r.POST(api.postEncryptVariableHandler)) + r.Handle("/project/{permProjectKey}/encrypt/list", Scope(sdk.AuthConsumerScopeProject), r.GET(api.getListEncryptVariableHandler)) r.Handle("/project/{permProjectKey}/variable/audit", Scope(sdk.AuthConsumerScopeProject), r.GET(api.getVariablesAuditInProjectnHandler)) r.Handle("/project/{permProjectKey}/variable/{name}", Scope(sdk.AuthConsumerScopeProject), r.GET(api.getVariableInProjectHandler), r.POST(api.addVariableInProjectHandler), r.PUT(api.updateVariableInProjectHandler), r.DELETE(api.deleteVariableFromProjectHandler)) r.Handle("/project/{permProjectKey}/variable/{name}/audit", Scope(sdk.AuthConsumerScopeProject), r.GET(api.getVariableAuditInProjectHandler)) diff --git a/engine/api/environment/environment_importer.go b/engine/api/environment/environment_importer.go index e5954feb9c..27e6b1ae3b 100644 --- a/engine/api/environment/environment_importer.go +++ b/engine/api/environment/environment_importer.go @@ -68,7 +68,7 @@ func Import(db gorpmapper.SqlExecutorWithTx, proj sdk.Project, env *sdk.Environm //ImportInto import variables and groups on an existing environment func ImportInto(ctx context.Context, db gorpmapper.SqlExecutorWithTx, env *sdk.Environment, into *sdk.Environment, msgChan chan<- sdk.Message, u sdk.Identifiable) error { var updateVar = func(v *sdk.EnvironmentVariable) { - log.Debug(ctx, "ImportInto> Updating var %s", v.Name) + log.Debug(ctx, "ImportInto> Updating var %q with value %q", v.Name, v.Value) varBefore, errV := LoadVariable(db, into.ID, v.Name) if errV != nil { diff --git a/engine/api/project/key.go b/engine/api/project/key.go index 5399a42a9f..a4c1634227 100644 --- a/engine/api/project/key.go +++ b/engine/api/project/key.go @@ -3,6 +3,7 @@ package project import ( "bytes" "compress/gzip" + "context" "crypto/rand" "database/sql" "encoding/base64" @@ -18,7 +19,9 @@ import ( ) func init() { - gorpmapping.Register(gorpmapping.New(dbEncryptedData{}, "encrypted_data", false, "token")) + gorpmapping.Register(gorpmapping.New(dbEncryptedData{}, "encrypted_data", false, "project_id", "content_name")) + gorpmapping.Register(gorpmapping.New(sdk.Secret{}, "encrypted_data", false, "project_id", "content_name")) + } type dbEncryptedData struct { @@ -28,6 +31,15 @@ type dbEncryptedData struct { EncyptedContent []byte `db:"encrypted_content"` } +func ListEncryptedData(ctx context.Context, db gorp.SqlExecutor, projectID int64) ([]sdk.Secret, error) { + var res []sdk.Secret + query := gorpmapping.NewQuery("select content_name, token from encrypted_data where project_id = $1").Args(projectID) + if err := gorpmapping.GetAll(ctx, db, query, &res); err != nil { + return nil, sdk.WithStack(err) + } + return res, nil +} + // EncryptWithBuiltinKey encrypt a content with the builtin gpg key encode, compress it and encode with base64 func EncryptWithBuiltinKey(db gorp.SqlExecutor, projectID int64, name, content string) (string, error) { existingToken, err := db.SelectStr("select token from encrypted_data where project_id = $1 and content_name = $2", projectID, name) diff --git a/engine/api/project_variable.go b/engine/api/project_variable.go index 9d6f97f2e0..26a6c0f954 100644 --- a/engine/api/project_variable.go +++ b/engine/api/project_variable.go @@ -5,6 +5,7 @@ import ( "net/http" "github.com/gorilla/mux" + "github.com/rockbears/log" "github.com/ovh/cds/engine/api/event" "github.com/ovh/cds/engine/api/project" @@ -12,6 +13,39 @@ import ( "github.com/ovh/cds/sdk" ) +func (api *API) getListEncryptVariableHandler() service.Handler { + return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error { + vars := mux.Vars(r) + key := vars[permProjectKey] + + p, err := project.Load(ctx, api.mustDB(), key) + if err != nil { + return err + } + + res, err := project.ListEncryptedData(ctx, api.mustDB(), p.ID) + if err != nil { + return err + } + + for i := range res { + decryptedData, err := project.DecryptWithBuiltinKey(api.mustDB(), p.ID, res[i].Token) + if err != nil { + log.Error(ctx, "unable to decrypt data %s: %v", res[i].Token, err) + res[i].Status = "decryption failed" + } + + if decryptedData == sdk.PasswordPlaceholder { + res[i].Status = "password placeholder detected" + } + + res[i].Status = "OK" + } + + return service.WriteJSON(w, res, http.StatusOK) + } +} + func (api *API) postEncryptVariableHandler() service.Handler { return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error { vars := mux.Vars(r) diff --git a/engine/cdn/storage/storageunit_run.go b/engine/cdn/storage/storageunit_run.go index be50609dd8..1916eeccf8 100644 --- a/engine/cdn/storage/storageunit_run.go +++ b/engine/cdn/storage/storageunit_run.go @@ -23,7 +23,9 @@ func (x *RunningStorageUnits) FillSyncItemChannel(ctx context.Context, s Storage if err := x.cache.ScoredSetRevRange(ctx, cache.Key(KeyBackendSync, s.Name()), 0, nbItem, &itemIDs); err != nil { return err } - log.Info(ctx, "FillSyncItemChannel> Item to sync for %s: %d", s.Name(), len(itemIDs)) + if len(itemIDs) > 0 { + log.Info(ctx, "FillSyncItemChannel> Item to sync for %s: %d", s.Name(), len(itemIDs)) + } for _, id := range itemIDs { select { case s.SyncItemChannel() <- id: diff --git a/sdk/cdsclient/client_project_variable.go b/sdk/cdsclient/client_project_variable.go index 87e83addef..22d7c8ea5d 100644 --- a/sdk/cdsclient/client_project_variable.go +++ b/sdk/cdsclient/client_project_variable.go @@ -49,3 +49,11 @@ func (c *client) VariableEncrypt(projectKey string, varName string, content stri } return variable, nil } + +func (c *client) VariableListEncrypt(projectKey string) ([]sdk.Secret, error) { + secrets := []sdk.Secret{} + if _, err := c.GetJSON(context.Background(), "/project/"+projectKey+"/encrypt/list", &secrets, nil); err != nil { + return nil, err + } + return secrets, nil +} diff --git a/sdk/cdsclient/interface.go b/sdk/cdsclient/interface.go index e5cf202f59..1b71739bd5 100644 --- a/sdk/cdsclient/interface.go +++ b/sdk/cdsclient/interface.go @@ -241,6 +241,7 @@ type ProjectVariablesClient interface { ProjectVariableGet(projectKey string, varName string) (*sdk.Variable, error) ProjectVariableUpdate(projectKey string, variable *sdk.Variable) error VariableEncrypt(projectKey string, varName string, content string) (*sdk.Variable, error) + VariableListEncrypt(projectKey string) ([]sdk.Secret, error) } // QueueClient exposes queue related functions diff --git a/sdk/cdsclient/mock_cdsclient/interface_mock.go b/sdk/cdsclient/mock_cdsclient/interface_mock.go index a20762e47c..1d54a3e1a4 100644 --- a/sdk/cdsclient/mock_cdsclient/interface_mock.go +++ b/sdk/cdsclient/mock_cdsclient/interface_mock.go @@ -13,7 +13,7 @@ import ( sdk "github.com/ovh/cds/sdk" cdsclient "github.com/ovh/cds/sdk/cdsclient" venom "github.com/ovh/venom" - coverage "github.com/sguiheux/go-coverage" + go_coverage "github.com/sguiheux/go-coverage" io "io" http "net/http" reflect "reflect" @@ -2652,6 +2652,21 @@ func (mr *MockProjectClientMockRecorder) VariableEncrypt(projectKey, varName, co return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VariableEncrypt", reflect.TypeOf((*MockProjectClient)(nil).VariableEncrypt), projectKey, varName, content) } +// VariableListEncrypt mocks base method +func (m *MockProjectClient) VariableListEncrypt(projectKey string) ([]sdk.Secret, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VariableListEncrypt", projectKey) + ret0, _ := ret[0].([]sdk.Secret) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// VariableListEncrypt indicates an expected call of VariableListEncrypt +func (mr *MockProjectClientMockRecorder) VariableListEncrypt(projectKey interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VariableListEncrypt", reflect.TypeOf((*MockProjectClient)(nil).VariableListEncrypt), projectKey) +} + // ProjectGroupsImport mocks base method func (m *MockProjectClient) ProjectGroupsImport(projectKey string, content io.Reader, mods ...cdsclient.RequestModifier) (sdk.Project, error) { m.ctrl.T.Helper() @@ -2941,6 +2956,21 @@ func (mr *MockProjectVariablesClientMockRecorder) VariableEncrypt(projectKey, va return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VariableEncrypt", reflect.TypeOf((*MockProjectVariablesClient)(nil).VariableEncrypt), projectKey, varName, content) } +// VariableListEncrypt mocks base method +func (m *MockProjectVariablesClient) VariableListEncrypt(projectKey string) ([]sdk.Secret, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VariableListEncrypt", projectKey) + ret0, _ := ret[0].([]sdk.Secret) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// VariableListEncrypt indicates an expected call of VariableListEncrypt +func (mr *MockProjectVariablesClientMockRecorder) VariableListEncrypt(projectKey interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VariableListEncrypt", reflect.TypeOf((*MockProjectVariablesClient)(nil).VariableListEncrypt), projectKey) +} + // MockQueueClient is a mock of QueueClient interface type MockQueueClient struct { ctrl *gomock.Controller @@ -3086,7 +3116,7 @@ func (mr *MockQueueClientMockRecorder) QueueJobSendSpawnInfo(ctx, id, in interfa } // QueueSendCoverage mocks base method -func (m *MockQueueClient) QueueSendCoverage(ctx context.Context, id int64, report coverage.Report) error { +func (m *MockQueueClient) QueueSendCoverage(ctx context.Context, id int64, report go_coverage.Report) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "QueueSendCoverage", ctx, id, report) ret0, _ := ret[0].(error) @@ -6907,6 +6937,21 @@ func (mr *MockInterfaceMockRecorder) VariableEncrypt(projectKey, varName, conten return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VariableEncrypt", reflect.TypeOf((*MockInterface)(nil).VariableEncrypt), projectKey, varName, content) } +// VariableListEncrypt mocks base method +func (m *MockInterface) VariableListEncrypt(projectKey string) ([]sdk.Secret, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VariableListEncrypt", projectKey) + ret0, _ := ret[0].([]sdk.Secret) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// VariableListEncrypt indicates an expected call of VariableListEncrypt +func (mr *MockInterfaceMockRecorder) VariableListEncrypt(projectKey interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VariableListEncrypt", reflect.TypeOf((*MockInterface)(nil).VariableListEncrypt), projectKey) +} + // ProjectGroupsImport mocks base method func (m *MockInterface) ProjectGroupsImport(projectKey string, content io.Reader, mods ...cdsclient.RequestModifier) (sdk.Project, error) { m.ctrl.T.Helper() @@ -7142,7 +7187,7 @@ func (mr *MockInterfaceMockRecorder) QueueJobSendSpawnInfo(ctx, id, in interface } // QueueSendCoverage mocks base method -func (m *MockInterface) QueueSendCoverage(ctx context.Context, id int64, report coverage.Report) error { +func (m *MockInterface) QueueSendCoverage(ctx context.Context, id int64, report go_coverage.Report) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "QueueSendCoverage", ctx, id, report) ret0, _ := ret[0].(error) @@ -8808,7 +8853,7 @@ func (mr *MockWorkerInterfaceMockRecorder) QueueJobSendSpawnInfo(ctx, id, in int } // QueueSendCoverage mocks base method -func (m *MockWorkerInterface) QueueSendCoverage(ctx context.Context, id int64, report coverage.Report) error { +func (m *MockWorkerInterface) QueueSendCoverage(ctx context.Context, id int64, report go_coverage.Report) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "QueueSendCoverage", ctx, id, report) ret0, _ := ret[0].(error) diff --git a/sdk/variable.go b/sdk/variable.go index 68412cbed4..bfbd7e6176 100644 --- a/sdk/variable.go +++ b/sdk/variable.go @@ -6,6 +6,13 @@ import ( "time" ) +type Secret struct { + ProjectID int64 `json:"project_id" db:"project_id" cli:"-"` + Name string `json:"name" db:"content_name" cli:"name,key"` + Token string `json:"token" db:"token" cli:"token"` + Status string `json:"status" db:"-" cli:"status"` +} + // Variable represent a variable for a project or pipeline type Variable struct { ID int64 `json:"id,omitempty" cli:"-"` diff --git a/ui/src/app/shared/parameter/value/parameter.value.html b/ui/src/app/shared/parameter/value/parameter.value.html index 21f9ed8781..30fa103672 100644 --- a/ui/src/app/shared/parameter/value/parameter.value.html +++ b/ui/src/app/shared/parameter/value/parameter.value.html @@ -12,7 +12,7 @@
- +