feat: migrate organization to new model (#6303)

engine/api/admin.go

@@ -82,7 +82,7 @@ func (api *API) postAdminOrganizationHandler() service.Handler {
 
 func (api *API) getAdminOrganizationsHandler() service.Handler {
 	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-		orgas, err := organization.LoadAllOrganizations(ctx, api.mustDB())
+		orgas, err := organization.LoadOrganizations(ctx, api.mustDB())
 		if err != nil {
 			return err
 		}

engine/api/admin_test.go

@@ -34,9 +34,6 @@ func Test_getAdminOrganizationCRUD(t *testing.T) {
 
 	_, jwt := assets.InsertAdminUser(t, db)
 
-	_, err := db.Exec("DELETE FROM organization")
-	require.NoError(t, err)
-
 	orga := sdk.Organization{
 		Name: sdk.RandomString(10),
 	}
@@ -58,18 +55,17 @@ func Test_getAdminOrganizationCRUD(t *testing.T) {
 	body := wList.Body.Bytes()
 	require.NoError(t, json.Unmarshal(body, &orgs))
 
-	require.Equal(t, 1, len(orgs))
-	require.Equal(t, orga.Name, orgs[0].Name)
+	require.Equal(t, 2, len(orgs))
 
-	uriDelete := api.Router.GetRoute("DELETE", api.deleteAdminOrganizationsHandler, map[string]string{"organizationIdentifier": orgs[0].Name})
+	uriDelete := api.Router.GetRoute("DELETE", api.deleteAdminOrganizationsHandler, map[string]string{"organizationIdentifier": orgs[1].Name})
 	reqDelete := assets.NewJWTAuthentifiedRequest(t, jwt, "DELETE", uriDelete, nil)
 	wDelete := httptest.NewRecorder()
 	api.Router.Mux.ServeHTTP(wDelete, reqDelete)
 	require.Equal(t, 204, wDelete.Code)
 
-	orgsDb, err := organization.LoadAllOrganizations(context.TODO(), db)
+	orgsDb, err := organization.LoadOrganizations(context.TODO(), db)
 	require.NoError(t, err)
-	require.Equal(t, 0, len(orgsDb))
+	require.Equal(t, 1, len(orgsDb))
 
 }
 

engine/api/api.go

@@ -39,6 +39,7 @@ import (
 	"github.com/ovh/cds/engine/api/migrate"
 	"github.com/ovh/cds/engine/api/notification"
 	"github.com/ovh/cds/engine/api/objectstore"
+	"github.com/ovh/cds/engine/api/organization"
 	"github.com/ovh/cds/engine/api/purge"
 	"github.com/ovh/cds/engine/api/repositoriesmanager"
 	"github.com/ovh/cds/engine/api/services"
@@ -102,7 +103,7 @@ type Configuration struct {
 		DisableAddUserInDefaultGroup bool                       `toml:"disableAddUserInDefaultGroup" default:"false" comment:"If false, user are automatically added in the default group" json:"disableAddUserInDefaultGroup"`
 		RSAPrivateKey                string                     `toml:"rsaPrivateKey" default:"" comment:"The RSA Private Key used to sign and verify the JWT Tokens issued by the API \nThis is mandatory." json:"-"`
 		RSAPrivateKeys               []authentication.KeyConfig `toml:"rsaPrivateKeys" default:"" comment:"RSA Private Keys used to sign and verify the JWT Tokens issued by the API \nThis is mandatory." json:"-" mapstructure:"rsaPrivateKeys"`
-		AllowedOrganizations         sdk.StringSlice            `toml:"allowedOrganizations" default:"[default]" comment:"The list of allowed organizations for CDS users, let empty to authorize all organizations." json:"allowedOrganizations"`
+		AllowedOrganizations         sdk.StringSlice            `toml:"allowedOrganizations" comment:"The list of allowed organizations for CDS users, let empty to authorize all organizations." json:"allowedOrganizations"`
 		LDAP                         struct {
 			Enabled         bool   `toml:"enabled" default:"false" json:"enabled"`
 			SignupDisabled  bool   `toml:"signupDisabled" default:"false" json:"signupDisabled"`
@@ -120,7 +121,7 @@ type Configuration struct {
 			Enabled              bool   `toml:"enabled" default:"true" json:"enabled"`
 			SignupDisabled       bool   `toml:"signupDisabled" default:"false" json:"signupDisabled"`
 			SignupAllowedDomains string `toml:"signupAllowedDomains" default:"" comment:"Allow signup from selected domains only - comma separated. Example: your-domain.com,another-domain.com" commented:"true" json:"signupAllowedDomains"`
-			Organization         string `toml:"organization" default:"default" comment:"Organization assigned to user created by local authentication" commented:"true" json:"organization"`
+			Organization         string `toml:"organization" default:"default" comment:"Organization assigned to user created by local authentication" json:"organization"`
 		} `toml:"local" json:"local"`
 		CorporateSSO struct {
 			MFASupportEnabled bool   `json:"mfa_support_enabled" default:"false" toml:"mfaSupportEnabled"`
@@ -144,23 +145,23 @@ type Configuration struct {
 			APIURL         string `toml:"apiUrl" json:"apiUrl" default:"https://api.github.com" comment:"GitHub API URL"`
 			ClientID       string `toml:"clientId" json:"-" comment:"GitHub OAuth Client ID"`
 			ClientSecret   string `toml:"clientSecret" json:"-" comment:"GitHub OAuth Client Secret"`
-			Organization   string `toml:"organization" default:"default" comment:"Organization assigned to user created by github authentication" commented:"true" json:"organization"`
+			Organization   string `toml:"organization" default:"default" comment:"Organization assigned to user created by github authentication" json:"organization"`
 		} `toml:"github" json:"github" comment:"#######\n CDS <-> GitHub Auth. Documentation on https://ovh.github.io/cds/docs/integrations/github/github_authentication/ \n######"`
 		Gitlab struct {
 			Enabled        bool   `toml:"enabled" default:"false" json:"enabled"`
 			SignupDisabled bool   `toml:"signupDisabled" default:"false" json:"signupDisabled"`
 			URL            string `toml:"url" json:"url" default:"https://gitlab.com" comment:"GitLab URL"`
 			ApplicationID  string `toml:"applicationID" json:"-" comment:"GitLab OAuth Application ID"`
 			Secret         string `toml:"secret" json:"-" comment:"GitLab OAuth Application Secret"`
-			Organization   string `toml:"organization" default:"default" comment:"Organization assigned to user created by gitlab authentication" commented:"true" json:"organization"`
+			Organization   string `toml:"organization" default:"default" comment:"Organization assigned to user created by gitlab authentication" json:"organization"`
 		} `toml:"gitlab" json:"gitlab" comment:"#######\n CDS <-> GitLab Auth. Documentation on https://ovh.github.io/cds/docs/integrations/gitlab/gitlab_authentication/ \n######"`
 		OIDC struct {
 			Enabled        bool   `toml:"enabled" default:"false" json:"enabled"`
 			SignupDisabled bool   `toml:"signupDisabled" default:"false" json:"signupDisabled"`
 			URL            string `toml:"url" json:"url" default:"" comment:"Open ID connect config URL"`
 			ClientID       string `toml:"clientId" json:"-" comment:"OIDC Client ID"`
 			ClientSecret   string `toml:"clientSecret" json:"-" comment:"OIDC Client Secret"`
-			Organization   string `toml:"organization" default:"default" comment:"Organization assigned to user created by openid authentication" commented:"true" json:"organization"`
+			Organization   string `toml:"organization" default:"default" comment:"Organization assigned to user created by openid authentication" json:"organization"`
 		} `toml:"oidc" json:"oidc" comment:"#######\n CDS <-> Open ID Connect Auth. Documentation on https://ovh.github.io/cds/docs/integrations/openid-connect/ \n######"`
 	} `toml:"auth" comment:"##############################\n CDS Authentication Settings# \n#############################" json:"auth"`
 	SMTP struct {
@@ -378,6 +379,24 @@ func (a *API) CheckConfiguration(config interface{}) error {
 		return errors.New("invalid given authentication rsa private key")
 	}
 
+	if len(aConfig.Auth.AllowedOrganizations) == 0 {
+		return errors.New("you must allow at least one organization in field 'allowedOrganizations'")
+	}
+
+	// Check authentication driver
+	if aConfig.Auth.Local.Enabled && (aConfig.Auth.Local.Organization == "" || !aConfig.Auth.AllowedOrganizations.Contains(aConfig.Auth.Local.Organization)) {
+		return errors.New("local authentication driver organization empty or not allowed in field 'allowedOrganizations'")
+	}
+	if aConfig.Auth.OIDC.Enabled && (aConfig.Auth.OIDC.Organization == "" || !aConfig.Auth.AllowedOrganizations.Contains(aConfig.Auth.OIDC.Organization)) {
+		return errors.New("oidc authentication driver organization empty or not allowed in field 'allowedOrganizations'")
+	}
+	if aConfig.Auth.Gitlab.Enabled && (aConfig.Auth.Gitlab.Organization == "" || !aConfig.Auth.AllowedOrganizations.Contains(aConfig.Auth.Gitlab.Organization)) {
+		return errors.New("gitlab authentication driver organization empty or not allowed in field 'allowedOrganizations'")
+	}
+	if aConfig.Auth.Github.Enabled && (aConfig.Auth.Github.Organization == "" || !aConfig.Auth.AllowedOrganizations.Contains(aConfig.Auth.Github.Organization)) {
+		return errors.New("github authentication driver organization empty or not allowed in field 'allowedOrganizations'")
+	}
+
 	return nil
 }
 
@@ -564,6 +583,28 @@ func (a *API) Serve(ctx context.Context) error {
 		return migrate.ArtifactoryIntegration(ctx, a.DBConnectionFactory.GetDBMap(gorpmapping.Mapper))
 	}})
 
+	migrate.Add(ctx, sdk.Migration{Name: "OrganizationMigration", Release: "0.51.0", Blocker: true, Automatic: true, ExecFunc: func(ctx context.Context) error {
+		usersToMigrate, err := migrate.GetOrganizationUsersToMigrate(ctx, a.DBConnectionFactory.GetDBMap(gorpmapping.Mapper)())
+		if err != nil {
+			return err
+		}
+		for _, u := range usersToMigrate {
+			tx, err := a.DBConnectionFactory.GetDBMap(gorpmapping.Mapper)().Begin()
+			if err != nil {
+				return sdk.WithStack(err)
+			}
+
+			if err := a.userSetOrganization(ctx, tx, u.User, u.OrganizationName); err != nil {
+				return err
+			}
+
+			if err := tx.Commit(); err != nil {
+				return sdk.WithStack(err)
+			}
+		}
+		return nil
+	}})
+
 	isFreshInstall, errF := version.IsFreshInstall(a.mustDB())
 	if errF != nil {
 		return sdk.WrapError(errF, "Unable to check if it's a fresh installation of CDS")
@@ -787,6 +828,10 @@ func (a *API) Serve(ctx context.Context) error {
 		return fmt.Errorf("cannot setup integrations: %v", err)
 	}
 
+	if err := organization.CreateDefaultOrganization(ctx, a.DBConnectionFactory.GetDBMap(gorpmapping.Mapper)(), a.Config.Auth.AllowedOrganizations); err != nil {
+		return sdk.WrapError(err, "unable to initialize organizations")
+	}
+
 	pubKey, err := jws.ExportPublicKey(authentication.GetSigningKey())
 	if err != nil {
 		return sdk.WrapError(err, "Unable to export public signing key")

engine/api/api_test.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"context"
+	"github.com/ovh/cds/engine/api/organization"
 	"net/http/httptest"
 	"net/url"
 	"testing"
@@ -38,12 +39,16 @@ func newTestAPI(t *testing.T, bootstrapFunc ...test.Bootstrapf) (*API, *test.Fak
 		Cache:               store,
 	}
 	api.AuthenticationDrivers = make(map[sdk.AuthConsumerType]sdk.AuthDriver)
-	api.AuthenticationDrivers[sdk.ConsumerLocal] = local.NewDriver(context.TODO(), false, "http://localhost:8080", "", "")
+	api.AuthenticationDrivers[sdk.ConsumerLocal] = local.NewDriver(context.TODO(), false, "http://localhost:8080", "default", "default")
 	api.AuthenticationDrivers[sdk.ConsumerBuiltin] = builtin.NewDriver()
 	api.AuthenticationDrivers[sdk.ConsumerTest] = authdrivertest.NewDriver(t)
 	api.AuthenticationDrivers[sdk.ConsumerTest2] = authdrivertest.NewDriver(t)
 	api.GoRoutines = sdk.NewGoRoutines(context.TODO())
 
+	// Reset organization
+	_, err := db.Exec("DELETE FROM organization")
+	require.NoError(t, err)
+	require.NoError(t, organization.CreateDefaultOrganization(context.TODO(), db.DbMap, []string{"default"}))
 	api.InitRouter()
 
 	ctx, cancel := context.WithCancel(context.Background())

engine/api/auth_test.go

@@ -3,7 +3,7 @@ package api
 import (
 	"context"
 	"encoding/json"
-	"fmt"
+	"github.com/ovh/cds/engine/api/organization"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -58,6 +58,10 @@ func Test_postAuthSignoutHandler(t *testing.T) {
 
 func Test_postAuthSigninHandler_ShouldSuccessWithANewUser(t *testing.T) {
 	api, db, _ := newTestAPI(t)
+	api.Config.Auth.AllowedOrganizations = append(api.Config.Auth.AllowedOrganizations, "planet-express")
+
+	newOrga := sdk.Organization{Name: "planet-express"}
+	require.NoError(t, organization.Insert(context.TODO(), db, &newOrga))
 
 	uri := api.Router.GetRoute(http.MethodPost, api.postAuthSigninHandler, map[string]string{
 		"consumerType": "futurama",
@@ -312,15 +316,18 @@ dg/94O8U5bC2T8a9CsA/q8eGuucP
 )
 
 func Test_postAuthSigninHandler_WithCorporateSSO(t *testing.T) {
-	api, _, _ := newTestAPI(t)
+	api, db, _ := newTestAPI(t)
+	api.Config.Auth.AllowedOrganizations = []string{"planet-express"}
 
 	var cfg corpsso.Config
 	cfg.Request.Keys.RequestSigningKey = AuthKey
 	cfg.Request.RedirectMethod = "POST"
 	cfg.Request.RedirectURL = "https://lolcat.local/sso/jwt"
 	cfg.Token.KeySigningKey.KeySigningKey = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmDMEXF1XRhYJKwYBBAHaRw8BAQdABEHVkfddwOIEFd7V0hsGrudgRuOlnV4/VSK6\nYJGFag+0HnRlc3QtbG9ja2VyIDx0ZXN0QGxvbGNhdC5ob3N0PoiQBBMWCAA4FiEE\nBN0dlUe5Vi8hx0ZsWXCoyV8Z2eQFAlxdV0YCGwMFCwkIBwIGFQoJCAsCBBYCAwEC\nHgECF4AACgkQWXCoyV8Z2eQt5gEAycwThBk4CzuQ8XtPvLA/kml3Jkclgw6ACGsP\nYOrnz+gA/2XOjnhYOA6S3sn9g4UMVtON8TofBMTTSqCdgrghu3kFuDgEXF1XRhIK\nKwYBBAGXVQEFAQEHQGlq7X9fCeXKxlmcWgT+fFJyS1MlL2uwKQteXl8yIadwAwEI\nB4h4BBgWCAAgFiEEBN0dlUe5Vi8hx0ZsWXCoyV8Z2eQFAlxdV0YCGwwACgkQWXCo\nyV8Z2eR4rgD/cPn9TStAoXc4Pa+sKgAFmG3NVCNln8FtkH5cQ1g0ouUA/AzcLTL4\nVQHT6ArvDWzJKKrh2PepZ5PVMS/Hwh/GDH4J\n=n1Ws\n-----END PGP PUBLIC KEY BLOCK-----"
 	cfg.Token.KeySigningKey.SigningKeyClaim = "key"
-	cfg.AllowedOrganizations = []string{"planet-express"}
+	cfg.AllowedOrganizations = api.Config.Auth.AllowedOrganizations
+
+	require.NoError(t, organization.Insert(context.TODO(), db, &sdk.Organization{Name: "planet-express"}))
 
 	api.AuthenticationDrivers[sdk.ConsumerCorporateSSO] = corpsso.NewDriver(cfg)
 
@@ -450,45 +457,30 @@ func TestUserSetOrganization_EnsureGroup(t *testing.T) {
 	g1 := assets.InsertTestGroup(t, db, sdk.RandomString(10))
 
 	u1, _ := assets.InsertLambdaUser(t, db, g1)
-	u2, _ := assets.InsertLambdaUser(t, db, g1)
-	u3, _ := assets.InsertLambdaUser(t, db, g1)
+
+	require.NoError(t, organization.Insert(context.TODO(), db, &sdk.Organization{Name: "org0"}))
+	require.NoError(t, organization.Insert(context.TODO(), db, &sdk.Organization{Name: "org1"}))
+	require.NoError(t, organization.Insert(context.TODO(), db, &sdk.Organization{Name: "org2"}))
 
 	// Set not allowed org should return an error
 	api.Config.Auth.AllowedOrganizations = []string{"org0"}
 	err := api.userSetOrganization(context.TODO(), db, u1, "org1")
 	require.Error(t, err)
 	require.Equal(t, "forbidden (from: user organization \"org1\" is not allowed)", err.Error())
 
-	// Set org on user should also update its group
+	// update org on user must return error
 	api.Config.Auth.AllowedOrganizations = []string{"org1", "org2"}
-	require.NoError(t, api.userSetOrganization(context.TODO(), db, u1, "org1"))
-	orgU1, err := user.LoadOrganizationByUserID(context.TODO(), db, u1.ID)
-	require.NoError(t, err)
-	require.Equal(t, "org1", orgU1.Organization)
-	orgG1, err := group.LoadOrganizationByGroupID(context.TODO(), db, g1.ID)
-	require.NoError(t, err)
-	require.Equal(t, "org1", orgG1.Organization)
-
-	// Set org should be ok when no conflict on groups
-	require.NoError(t, api.userSetOrganization(context.TODO(), db, u3, "org1"))
-	orgU3, err := user.LoadOrganizationByUserID(context.TODO(), db, u3.ID)
-	require.NoError(t, err)
-	require.Equal(t, "org1", orgU3.Organization)
-
-	// Set org on a user that is part of group with another org should return an error
-	err = api.userSetOrganization(context.TODO(), db, u2, "org2")
-	require.Error(t, err)
-	require.Equal(t, "Cannot validate given data (from: group members organization conflict \"org1\" and \"org2\")", err.Error())
-
-	// Change user org should return an error
-	err = api.userSetOrganization(context.TODO(), db, u1, "org2")
+	err = api.userSetOrganization(context.TODO(), db, u1, "org1")
 	require.Error(t, err)
-	require.Equal(t, "forbidden (from: cannot change user organization to \"org2\", value already set to \"org1\")", err.Error())
+	require.Equal(t, "forbidden (from: cannot change user organization to \"org1\", value already set to \"default\")", err.Error())
 }
 
 func TestUserSetOrganization_EnsureProject(t *testing.T) {
 	api, db, _ := newTestAPI(t)
 
+	require.NoError(t, organization.Insert(context.TODO(), db, &sdk.Organization{Name: "org1"}))
+	require.NoError(t, organization.Insert(context.TODO(), db, &sdk.Organization{Name: "org2"}))
+
 	pKey := sdk.RandomString(10)
 	p1 := assets.InsertTestProject(t, db, api.Cache, pKey, pKey)
 
@@ -501,21 +493,15 @@ func TestUserSetOrganization_EnsureProject(t *testing.T) {
 	}))
 
 	u1, _ := assets.InsertLambdaUser(t, db, g1)
-	u2, _ := assets.InsertLambdaUser(t, db, g2)
 
 	// Assert project info
 	require.NoError(t, project.LoadOptions.WithGroups(context.TODO(), db, p1))
-	require.Equal(t, "", p1.Organization)
+	require.Equal(t, "default", p1.Organization)
 	require.Len(t, p1.ProjectGroups, 2)
 
 	// Set org on u1 should change project organization
 	api.Config.Auth.AllowedOrganizations = []string{"org1", "org2"}
-	require.NoError(t, api.userSetOrganization(context.TODO(), db, u1, "org1"))
-	require.NoError(t, project.LoadOptions.WithGroups(context.TODO(), db, p1))
-	require.Equal(t, "org1", p1.Organization)
-
-	// Set another org on a user that is part of group with permission on the project should return an error
-	err := api.userSetOrganization(context.TODO(), db, u2, "org2")
+	err := api.userSetOrganization(context.TODO(), db, u1, "org1")
 	require.Error(t, err)
-	require.Equal(t, fmt.Sprintf("forbidden (from: changing group organization conflict on project with id: %d) (caused by: group permissions organization conflict)", p1.ID), err.Error())
+	require.Equal(t, "forbidden (from: cannot change user organization to \"org1\", value already set to \"default\")", err.Error())
 }

engine/api/group/crud.go

@@ -2,6 +2,7 @@ package group
 
 import (
 	"context"
+	"github.com/ovh/cds/engine/api/organization"
 
 	"github.com/go-gorp/gorp"
 
@@ -23,13 +24,16 @@ func Create(ctx context.Context, db gorpmapper.SqlExecutorWithTx, grp *sdk.Group
 		return err
 	}
 
-	if user.Organization != "" {
-		if err := InsertOrganization(ctx, db, &Organization{
-			GroupID:      grp.ID,
-			Organization: user.Organization,
-		}); err != nil {
-			return err
-		}
+	org, err := organization.LoadOrganizationByName(ctx, db, user.Organization)
+	if err != nil {
+		return err
+	}
+
+	if err := InsertGroupOrganization(ctx, db, &GroupOrganization{
+		GroupID:        grp.ID,
+		OrganizationID: org.ID,
+	}); err != nil {
+		return err
 	}
 
 	return nil
@@ -82,32 +86,42 @@ func EnsureOrganization(ctx context.Context, db gorpmapper.SqlExecutorWithTx, g
 	if err := LoadOptions.WithMembers(ctx, db, g); err != nil {
 		return err
 	}
-	exitingGroupOrganization, err := LoadOrganizationByGroupID(ctx, db, g.ID)
+	exitingGroupOrganization, err := LoadGroupOrganizationByGroupID(ctx, db, g.ID)
 	if err != nil && !sdk.ErrorIs(err, sdk.ErrNotFound) {
 		return err
 	}
-	newGroupOrganization, err := g.Members.ComputeOrganization()
+
+	newOrganizationName, err := g.Members.ComputeOrganization()
 	if err != nil {
 		return err
 	}
 	if exitingGroupOrganization != nil {
-		g.Organization = exitingGroupOrganization.Organization
+		currentOrganization, err := organization.LoadOrganizationByID(ctx, db, exitingGroupOrganization.OrganizationID)
+		if err != nil {
+			return err
+		}
+		g.Organization = currentOrganization.Name
 	}
-	if g.Organization == newGroupOrganization {
+	if g.Organization == newOrganizationName {
 		return nil
 	}
 
-	g.Organization = newGroupOrganization
+	newGroupOrganization, err := organization.LoadOrganizationByName(ctx, db, newOrganizationName)
+	if err != nil {
+		return err
+	}
+
+	g.Organization = newOrganizationName
 	if exitingGroupOrganization == nil {
-		if err := InsertOrganization(ctx, db, &Organization{
-			GroupID:      g.ID,
-			Organization: newGroupOrganization,
+		if err := InsertGroupOrganization(ctx, db, &GroupOrganization{
+			GroupID:        g.ID,
+			OrganizationID: newGroupOrganization.ID,
 		}); err != nil {
 			return err
 		}
 	} else {
-		exitingGroupOrganization.Organization = newGroupOrganization
-		if err := UpdateOrganization(ctx, db, exitingGroupOrganization); err != nil {
+		exitingGroupOrganization.OrganizationID = newGroupOrganization.ID
+		if err := UpdateGroupOrganization(ctx, db, exitingGroupOrganization); err != nil {
 			return err
 		}
 	}