feat(api,vcs): use new vcs

Signed-off-by: Yvonnick Esnault <[email protected]>

engine/api/project/dao.go

@@ -16,6 +16,7 @@ import (
 	"github.com/ovh/cds/engine/api/group"
 	"github.com/ovh/cds/engine/api/keys"
 	"github.com/ovh/cds/engine/api/repositoriesmanager"
+	"github.com/ovh/cds/engine/api/vcs"
 	"github.com/ovh/cds/engine/cache"
 	"github.com/ovh/cds/engine/gorpmapper"
 	"github.com/ovh/cds/sdk"
@@ -314,11 +315,34 @@ func unwrap(ctx context.Context, db gorp.SqlExecutor, p *dbProject, opts []LoadO
 		end()
 	}
 
-	vcsServers, err := repositoriesmanager.LoadAllProjectVCSServerLinksByProjectID(ctx, db, p.ID)
+	vcsProjects, err := vcs.LoadAllVCSByProject(ctx, db, p.Key)
 	if err != nil {
 		return nil, err
 	}
-	proj.VCSServers = vcsServers
+
+	proj.VCSServers = vcsProjects
+
+	// DEPRECATED VCS
+	vcsServersDeprecated, err := repositoriesmanager.LoadAllProjectVCSServerLinksByProjectID(ctx, db, p.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, vcsDeprecated := range vcsServersDeprecated {
+		var found bool
+		for _, v := range vcsProjects {
+			if vcsDeprecated.Name == v.Name {
+				found = true
+				break
+			}
+		}
+		if !found {
+			toadd := sdk.VCSProject{
+				Name: vcsDeprecated.Name,
+			}
+			proj.VCSServers = append(proj.VCSServers, toadd)
+		}
+	}
 
 	return &proj, nil
 }

engine/api/repositoriesmanager/repositories_manager.go

@@ -3,6 +3,7 @@ package repositoriesmanager
 import (
 	"context"
 	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -23,16 +24,22 @@ import (
 	"github.com/ovh/cds/sdk/telemetry"
 )
 
-func LoadByName(ctx context.Context, db gorp.SqlExecutor, vcsName string) (sdk.VCSConfiguration, error) {
+func (c *vcsClient) IsGerrit(ctx context.Context, db gorp.SqlExecutor) (bool, error) {
+	if c.vcsProject != nil {
+		return c.vcsProject.Type == "gerrit", nil
+	}
+
+	// DEPRECATED VCS
 	var vcsServer sdk.VCSConfiguration
 	srvs, err := services.LoadAllByType(ctx, db, sdk.TypeVCS)
 	if err != nil {
-		return vcsServer, sdk.WrapError(err, "Unable to load services")
+		return false, sdk.WrapError(err, "Unable to load services")
 	}
-	if _, _, err := services.NewClient(db, srvs).DoJSONRequest(ctx, "GET", fmt.Sprintf("/vcs/%s", vcsName), nil, &vcsServer); err != nil {
-		return vcsServer, sdk.WithStack(err)
+	if _, _, err := services.NewClient(db, srvs).DoJSONRequest(ctx, "GET", fmt.Sprintf("/vcs/%s", c.name), nil, &vcsServer); err != nil {
+		return false, sdk.WrapError(err, "error on requesting vcs service")
 	}
-	return vcsServer, nil
+
+	return vcsServer.Type == "gerrit", nil
 }
 
 //LoadAll Load all RepositoriesManager from the database
@@ -241,10 +248,20 @@ func deprecatedAuthorizedClient(ctx context.Context, db gorpmapper.SqlExecutorWi
 
 func (c *vcsClient) doJSONRequest(ctx context.Context, method, path string, in interface{}, out interface{}) (int, error) {
 	headers, code, err := services.NewClient(c.db, c.srvs).DoJSONRequest(ctx, method, path, in, out, func(req *http.Request) {
-		req.Header.Set(sdk.HeaderXAccessToken, base64.StdEncoding.EncodeToString([]byte(c.token)))
-		req.Header.Set(sdk.HeaderXAccessTokenSecret, base64.StdEncoding.EncodeToString([]byte(c.secret)))
-		if c.created != 0 {
-			req.Header.Set(sdk.HeaderXAccessTokenCreated, base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%d", c.created))))
+		if c.vcsProject != nil {
+			log.Debug(ctx, "requesting vcs via vcs project")
+			btes, err := json.Marshal(c.vcsProject)
+			if err != nil {
+				log.Error(ctx, "invalid vcs project conf. err: %v", err)
+			}
+			req.Header.Set(sdk.HeaderXVCSProjectConf, base64.StdEncoding.EncodeToString(btes))
+		} else {
+			log.Debug(ctx, "requesting vcs via vcs oauth2")
+			req.Header.Set(sdk.HeaderXAccessToken, base64.StdEncoding.EncodeToString([]byte(c.token)))
+			req.Header.Set(sdk.HeaderXAccessTokenSecret, base64.StdEncoding.EncodeToString([]byte(c.secret)))
+			if c.created != 0 {
+				req.Header.Set(sdk.HeaderXAccessTokenCreated, base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%d", c.created))))
+			}
 		}
 	})
 
@@ -274,8 +291,14 @@ func (c *vcsClient) doJSONRequest(ctx context.Context, method, path string, in i
 
 func (c *vcsClient) postBinary(ctx context.Context, path string, fileLength int, r io.Reader, out interface{}) (int, error) {
 	return services.PostBinary(ctx, c.srvs, path, r, out, func(req *http.Request) {
-		req.Header.Set(sdk.HeaderXAccessToken, base64.StdEncoding.EncodeToString([]byte(c.token)))
-		req.Header.Set(sdk.HeaderXAccessTokenSecret, base64.StdEncoding.EncodeToString([]byte(c.secret)))
+		if c.vcsProject != nil {
+			if token, ok := c.vcsProject.Auth["token"]; ok {
+				req.Header.Set(sdk.HeaderXVCSProjectConf, base64.StdEncoding.EncodeToString([]byte(token)))
+			}
+		} else {
+			req.Header.Set(sdk.HeaderXAccessToken, base64.StdEncoding.EncodeToString([]byte(c.token)))
+			req.Header.Set(sdk.HeaderXAccessTokenSecret, base64.StdEncoding.EncodeToString([]byte(c.secret)))
+		}
 		req.Header.Set("Content-Type", "application/octet-stream")
 		req.Header.Set("Content-Length", strconv.Itoa(fileLength))
 	})

engine/api/v2_project.go

@@ -154,7 +154,7 @@ func (api *API) getVCSProjectHandler() ([]service.RbacChecker, service.Handler)
 			}
 			defer tx.Rollback() // nolint
 
-			vcsProject, err := vcs.LoadVCSByProject(context.Background(), tx, pKey, vcsProjectName, gorpmapping.GetOptions.WithDecryption)
+			vcsProject, err := vcs.LoadVCSByProject(context.Background(), tx, pKey, vcsProjectName)
 			if err != nil {
 				return err
 			}

engine/api/workflow/workflow_run_event.go

@@ -93,14 +93,19 @@ loopNotif:
 	repoFullName := wr.Workflow.Applications[node.Context.ApplicationID].RepositoryFullname
 
 	//Get the RepositoriesManager Client
+	log.Info(ctx, "######## SendVCSEvent: AAAA")
 	if e.vcsClient == nil {
+		log.Info(ctx, "######## SendVCSEvent: BBBBB")
 		var err error
 		e.vcsClient, err = repositoriesmanager.AuthorizedClient(ctx, tx, store, proj.Key, vcsServerName)
 		if err != nil {
-			return err
+			log.Info(ctx, "######## SendVCSEvent: CCCC")
+			return sdk.WrapError(err, "can't get AuthorizedClient for %v/%v", proj.Key, vcsServerName)
 		}
+		log.Info(ctx, "######## SendVCSEvent: DDDDD")
 	}
 
+	log.Info(ctx, "######## SendVCSEvent: EEEEE")
 	ref := nodeRun.VCSHash
 	if nodeRun.VCSTag != "" {
 		ref = nodeRun.VCSTag
@@ -111,7 +116,7 @@ loopNotif:
 		var err error
 		statuses, err = e.vcsClient.ListStatuses(ctx, repoFullName, ref)
 		if err != nil {
-			return err
+			return sdk.WrapError(err, "can't ListStatuses for %v with vcs %v/%v", repoFullName, proj.Key, vcsServerName)
 		}
 		e.commitsStatuses[ref] = statuses
 	}
@@ -129,7 +134,7 @@ loopNotif:
 
 	if statusFound == nil || statusFound.State == "" {
 		if err := e.sendVCSEventStatus(ctx, tx, store, proj.Key, wr, &nodeRun, notif, vcsServerName); err != nil {
-			return err
+			return sdk.WrapError(err, "can't sendVCSEventStatus vcs %v/%v", proj.Key, vcsServerName)
 		}
 	} else {
 		skipStatus := false
@@ -154,7 +159,7 @@ loopNotif:
 
 		if !skipStatus {
 			if err := e.sendVCSEventStatus(ctx, tx, store, proj.Key, wr, &nodeRun, notif, vcsServerName); err != nil {
-				return err
+				return sdk.WrapError(err, "can't sendVCSEventStatus vcs %v/%v", proj.Key, vcsServerName)
 			}
 		}
 	}
@@ -163,7 +168,7 @@ loopNotif:
 		return nil
 	}
 	if err := e.sendVCSPullRequestComment(ctx, tx, wr, &nodeRun, notif, vcsServerName); err != nil {
-		return err
+		return sdk.WrapError(err, "can't sendVCSPullRequestComment vcs %v/%v", proj.Key, vcsServerName)
 	}
 
 	if err := tx.Commit(); err != nil {
@@ -237,12 +242,11 @@ func (e *VCSEventMessenger) sendVCSEventStatus(ctx context.Context, db gorp.SqlE
 	}
 
 	// Check if it's a gerrit or not
-	vcsConf, err := repositoriesmanager.LoadByName(ctx, db, vcsServerName)
+	isGerrit, err := e.vcsClient.IsGerrit(ctx, db)
 	if err != nil {
 		return err
 	}
-
-	if vcsConf.Type == "gerrit" {
+	if isGerrit {
 		// Get gerrit variable
 		var project, changeID, branch, revision, url string
 		projectParam := sdk.ParameterFind(nodeRun.BuildParameters, "git.repository")
@@ -275,7 +279,6 @@ func (e *VCSEventMessenger) sendVCSEventStatus(ctx context.Context, db gorp.SqlE
 				URL:        url,
 			}
 		}
-
 	}
 
 	payload, _ := json.Marshal(eventWNR)
@@ -329,12 +332,6 @@ func (e *VCSEventMessenger) sendVCSPullRequestComment(ctx context.Context, db go
 		return err
 	}
 
-	// Check if it's a gerrit or not
-	vcsConf, err := repositoriesmanager.LoadByName(ctx, db, vcsServerName)
-	if err != nil {
-		return err
-	}
-
 	var changeID string
 	changeIDParam := sdk.ParameterFind(nodeRun.BuildParameters, "gerrit.change.id")
 	if changeIDParam != nil {
@@ -350,13 +347,17 @@ func (e *VCSEventMessenger) sendVCSPullRequestComment(ctx context.Context, db go
 	reqComment := sdk.VCSPullRequestCommentRequest{Message: report}
 	reqComment.Revision = revision
 
-	// If we are on Gerrit
-	if changeID != "" && vcsConf.Type == "gerrit" {
+	isGerrit, err := e.vcsClient.IsGerrit(ctx, db)
+	if err != nil {
+		return err
+	}
+
+	if changeID != "" && isGerrit {
 		reqComment.ChangeID = changeID
 		if err := e.vcsClient.PullRequestComment(ctx, app.RepositoryFullname, reqComment); err != nil {
 			return err
 		}
-	} else if vcsConf.Type != "gerrit" {
+	} else if !isGerrit {
 		//Check if this branch and this commit is a pullrequest
 		prs, err := e.vcsClient.PullRequests(ctx, app.RepositoryFullname)
 		if err != nil {

engine/api/workflow_event.go

@@ -61,7 +61,7 @@ func (api *API) WorkflowSendEvent(ctx context.Context, proj sdk.Project, report
 		event.PublishWorkflowNodeRun(ctx, *nr, wr.Workflow, eventsNotif)
 		e := &workflow.VCSEventMessenger{}
 		if err := e.SendVCSEvent(ctx, db, api.Cache, proj, *wr, wnr); err != nil {
-			log.Warn(ctx, "WorkflowSendEvent> Cannot send vcs notification")
+			log.Warn(ctx, "WorkflowSendEvent> Cannot send vcs notification err:%v", err)
 		}
 	}
 

engine/api/workflow_queue_test.go

@@ -1171,9 +1171,6 @@ func TestInsertNewCodeCoverageReport(t *testing.T) {
 	}))
 	u.Groups = append(u.Groups, proj.ProjectGroups[0].Group)
 
-	// Add repo manager
-	proj.VCSServers = make([]sdk.ProjectVCSServerLink, 0, 1)
-
 	vcsServer := sdk.ProjectVCSServerLink{
 		ProjectID: proj.ID,
 		Name:      "repoManServ",

engine/vcs/bitbucketcloud/bitbucketcloud.go

@@ -36,7 +36,17 @@ type bitbucketcloudConsumer struct {
 }
 
 //New creates a new GithubConsumer
-func New(ClientID, ClientSecret, apiURL, uiURL, proxyURL string, store cache.Store, disableStatus, disableStatusDetail bool) sdk.VCSServer {
+func New(apiURL, uiURL, proxyURL string, store cache.Store) sdk.VCSServer {
+	return &bitbucketcloudConsumer{
+		Cache:    store,
+		apiURL:   apiURL,
+		uiURL:    uiURL,
+		proxyURL: proxyURL,
+	}
+}
+
+// DEPRECATED VCS
+func NewDeprecated(ClientID, ClientSecret, apiURL, uiURL, proxyURL string, store cache.Store, disableStatus, disableStatusDetail bool) sdk.VCSServer {
 	return &bitbucketcloudConsumer{
 		ClientID:            ClientID,
 		ClientSecret:        ClientSecret,

engine/vcs/bitbucketcloud/bitbucketcloud_test.go

@@ -44,7 +44,7 @@ func getNewConsumer(t *testing.T) sdk.VCSServer {
 		t.Fatalf("Unable to init cache (%s): %v", redisHost, err)
 	}
 
-	bbConsumer := New(clientID, clientSecret, "http://localhost", "", "", cache, true, true)
+	bbConsumer := NewDeprecated(clientID, clientSecret, "http://localhost", "", "", cache, true, true)
 	return bbConsumer
 }
 
@@ -66,7 +66,7 @@ func getNewAuthorizedClient(t *testing.T) sdk.VCSAuthorizedClient {
 		t.Fatalf("Unable to init cache (%s): %v", redisHost, err)
 	}
 
-	bbConsumer := New(clientID, clientSecret, "http://localhost", "", "", cache, true, true)
+	bbConsumer := NewDeprecated(clientID, clientSecret, "http://localhost", "", "", cache, true, true)
 	vcsAuth := sdk.VCSAuth{
 		AccessToken:        currentAccessToken,
 		AccessTokenSecret:  currentRefreshToken,

engine/vcs/bitbucketcloud/oauth_consumer.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"net/url"
-	"time"
 
 	"github.com/rockbears/log"
 
@@ -92,62 +91,39 @@ func (consumer *bitbucketcloudConsumer) RefreshToken(ctx context.Context, refres
 	return resp.AccessToken, resp.RefreshToken, nil
 }
 
-//keep client in memory
-var instancesAuthorizedClient = map[string]*bitbucketcloudClient{}
-
 //GetAuthorized returns an authorized client
 func (consumer *bitbucketcloudConsumer) GetAuthorizedClient(ctx context.Context, vcsAuth sdk.VCSAuth) (sdk.VCSAuthorizedClient, error) {
-	token := vcsAuth.PersonalAccessTokens
-	if vcsAuth.PersonalAccessTokens == "" { // DEPRECATED
-		token = vcsAuth.AccessToken
-	}
-
-	createdTime := time.Unix(vcsAuth.AccessTokenCreated, 0)
-
-	c, ok := instancesAuthorizedClient[token]
-	if createdTime.Add(2 * time.Hour).Before(time.Now()) {
-		if ok {
-			delete(instancesAuthorizedClient, token)
-		}
-		var newAccessToken string
-		if vcsAuth.AccessToken != "" { // DEPRECATED
-			var err error
-			newAccessToken, _, err = consumer.RefreshToken(ctx, vcsAuth.AccessTokenSecret)
-			if err != nil {
-				return nil, sdk.WrapError(err, "cannot refresh token")
-			}
-		}
-
-		c = &bitbucketcloudClient{
-			PersonalAccessToken: vcsAuth.PersonalAccessTokens,
-			ClientID:            consumer.ClientID,         // DEPRECATED
-			OAuthToken:          newAccessToken,            // DEPRECATED
-			RefreshToken:        vcsAuth.AccessTokenSecret, // DEPRECATED
+	if vcsAuth.VCSProject != nil {
+		c := &bitbucketcloudClient{
+			PersonalAccessToken: vcsAuth.VCSProject.Auth["token"],
 			Cache:               consumer.Cache,
 			apiURL:              consumer.apiURL,
 			uiURL:               consumer.uiURL,
-			DisableStatus:       consumer.disableStatus,
-			DisableStatusDetail: consumer.disableStatusDetail,
-			proxyURL:            consumer.proxyURL,
 		}
-		instancesAuthorizedClient[newAccessToken] = c
-	} else {
-		if !ok {
-			c = &bitbucketcloudClient{
-				PersonalAccessToken: vcsAuth.PersonalAccessTokens,
-				ClientID:            consumer.ClientID,         // DEPRECATED
-				OAuthToken:          vcsAuth.AccessToken,       // DEPRECATED
-				RefreshToken:        vcsAuth.AccessTokenSecret, // DEPRECATED
-				Cache:               consumer.Cache,
-				apiURL:              consumer.apiURL,
-				uiURL:               consumer.uiURL,
-				DisableStatus:       consumer.disableStatus,
-				DisableStatusDetail: consumer.disableStatusDetail,
-				proxyURL:            consumer.proxyURL,
-			}
-			instancesAuthorizedClient[token] = c
+		return c, nil
+	}
+
+	// DEPRECATED VCS
+	var newAccessToken string
+	if vcsAuth.AccessToken != "" {
+		var err error
+		newAccessToken, _, err = consumer.RefreshToken(ctx, vcsAuth.AccessTokenSecret)
+		if err != nil {
+			return nil, sdk.WrapError(err, "cannot refresh token")
 		}
 	}
 
+	c := &bitbucketcloudClient{
+		ClientID:            consumer.ClientID,
+		OAuthToken:          newAccessToken,
+		RefreshToken:        vcsAuth.AccessTokenSecret,
+		Cache:               consumer.Cache,
+		apiURL:              consumer.apiURL,
+		uiURL:               consumer.uiURL,
+		DisableStatus:       consumer.disableStatus,
+		DisableStatusDetail: consumer.disableStatusDetail,
+		proxyURL:            consumer.proxyURL,
+	}
+
 	return c, nil
 }