Skip to content

Commit

Permalink
feat(cdsctl,api): cdsctl admin database list / delete (#3842)
Browse files Browse the repository at this point in the history 
* feat(cdsctl,api): cdsctl admin database list / delete

Signed-off-by: Yvonnick Esnault <[email protected]>
  • Loading branch information
@yesnault
yesnault authored Jan 14, 2019
1 parent f1a668c commit eeb1665
Show file tree
Hide file tree
Showing 17 changed files with 290 additions and 133 deletions.
27 changes: 27 additions & 0 deletions cli/cdsctl/admin_database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ var adminDatabaseCmd = cli.Command{
func adminDatabase() *cobra.Command {
return cli.NewCommand(adminDatabaseCmd, nil, []*cobra.Command{
cli.NewCommand(adminDatabaseUnlockCmd, adminDatabaseUnlockFunc, nil),
cli.NewCommand(adminDatabaseDeleteMigrationCmd, adminDatabaseDeleteFunc, nil),
cli.NewListCommand(adminDatabaseMigrationsList, adminDatabaseMigrationsListFunc, nil),
})
}

Expand All @@ -28,3 +30,28 @@ var adminDatabaseUnlockCmd = cli.Command{
func adminDatabaseUnlockFunc(v cli.Values) error {
return client.AdminDatabaseMigrationUnlock(v.GetString("id"))
}

var adminDatabaseDeleteMigrationCmd = cli.Command{
Name: "delete",
Short: "Delete a database migration from table gorp_migration (use with caution)",
Args: []cli.Arg{
{Name: "id"},
},
}

func adminDatabaseDeleteFunc(v cli.Values) error {
return client.AdminDatabaseMigrationDelete(v.GetString("id"))
}

var adminDatabaseMigrationsList = cli.Command{
Name: "list",
Short: "List all CDS DB migrations",
}

func adminDatabaseMigrationsListFunc(_ cli.Values) (cli.ListResult, error) {
migrations, err := client.AdminDatabaseMigrationsList()
if err != nil {
return nil, err
}
return cli.AsListResult(migrations), nil
}
6 changes: 6 additions & 0 deletions cli/cdsctl/admin_migrations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,12 @@ import (
var adminMigrationsCmd = cli.Command{
Name: "migration",
Short: "Manage CDS Migrations",
Long: `Theses commands manage CDS Migration and DO NOT concern database migrations.
A CDS Migration is an internal routine. This helps manage a complex data migration with code included
in CDS Engine. It's totally transpartent to CDS Users & Administrators - but these commands can help
CDS Administrators and core CDS Developers to debug something if needed.
`,
}

func adminMigrations() *cobra.Command {
Expand Down
4 changes: 3 additions & 1 deletion engine/api/api_routes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@ func (api *API) InitRouter() {
r.Handle("/admin/cds/migration", r.GET(api.getAdminMigrationsHandler, NeedAdmin(true)))
r.Handle("/admin/cds/migration/{id}/cancel", r.POST(api.postAdminMigrationCancelHandler, NeedAdmin(true)))
r.Handle("/admin/cds/migration/{id}/todo", r.POST(api.postAdminMigrationTodoHandler, NeedAdmin(true)))
r.Handle("/admin/database/migration/unlock/{id}", r.POST(api.postMigrationUnlockedHandler, NeedAdmin(true)))
r.Handle("/admin/database/migration/delete/{id}", r.DELETE(api.deleteDatabaseMigrationHandler, NeedAdmin(true)))
r.Handle("/admin/database/migration/unlock/{id}", r.POST(api.postDatabaseMigrationUnlockedHandler, NeedAdmin(true)))
r.Handle("/admin/database/migration", r.GET(api.getDatabaseMigrationHandler, NeedAdmin(true)))
r.Handle("/admin/debug", r.GET(api.getProfileIndexHandler, Auth(false)))
r.Handle("/admin/debug/trace", r.POST(api.getTraceHandler, NeedAdmin(true)), r.GET(api.getTraceHandler, NeedAdmin(true)))
r.Handle("/admin/debug/cpu", r.POST(api.getCPUProfileHandler, NeedAdmin(true)), r.GET(api.getCPUProfileHandler, NeedAdmin(true)))
Expand Down
25 changes: 24 additions & 1 deletion engine/api/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,20 @@ import (
"github.com/ovh/cds/sdk"
)

func (api *API) postMigrationUnlockedHandler() service.Handler {
func (api *API) deleteDatabaseMigrationHandler() service.Handler {
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
vars := mux.Vars(r)
id := vars["id"]

if len(id) == 0 {
return sdk.NewErrorFrom(sdk.ErrWrongRequest, "Id is mandatory. Check id from table gorp_migrations")
}

return dbmigrate.DeleteMigrate(api.mustDB().Db, id)
}
}

func (api *API) postDatabaseMigrationUnlockedHandler() service.Handler {
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
vars := mux.Vars(r)
id := vars["id"]
Expand All @@ -23,3 +36,13 @@ func (api *API) postMigrationUnlockedHandler() service.Handler {
return dbmigrate.UnlockMigrate(api.mustDB().Db, id)
}
}

func (api *API) getDatabaseMigrationHandler() service.Handler {
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
a, err := dbmigrate.List(api.mustDB().Db)
if err != nil {
return sdk.WrapError(err, "Cannot load database migration list %d", err)
}
return service.WriteJSON(w, a, http.StatusOK)
}
}
108 changes: 54 additions & 54 deletions engine/api/database/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,40 +15,40 @@ import (

// DBConnectionFactory is a database connection factory on postgres with gorp
type DBConnectionFactory struct {
dbDriver string
dbRole string
dbUser string
dbPassword string
dbName string
dbHost string
dbPort int
dbSSLMode string
dbTimeout int
dbConnectTimeout int
dbMaxConn int
db *sql.DB
DBDriver string
DBRole string
DBUser string
DBPassword string
DBName string
DBHost string
DBPort int
DBSSLMode string
DBTimeout int
DBConnectTimeout int
DBMaxConn int
Database *sql.DB
mutex *sync.Mutex
}

// DB returns the current sql.DB object
func (f *DBConnectionFactory) DB() *sql.DB {
if f.db == nil {
if f.dbName == "" {
if f.Database == nil {
if f.DBName == "" {
return nil
}
newF, err := Init(f.dbUser, f.dbRole, f.dbPassword, f.dbName, f.dbHost, f.dbPort, f.dbSSLMode, f.dbConnectTimeout, f.dbTimeout, f.dbMaxConn)
newF, err := Init(f.DBUser, f.DBRole, f.DBPassword, f.DBName, f.DBHost, f.DBPort, f.DBSSLMode, f.DBConnectTimeout, f.DBTimeout, f.DBMaxConn)
if err != nil {
log.Error("Database> cannot init db connection : %s", err)
return nil
}
*f = *newF
}
if err := f.db.Ping(); err != nil {
if err := f.Database.Ping(); err != nil {
log.Error("Database> cannot ping db : %s", err)
f.db = nil
f.Database = nil
return nil
}
return f.db
return f.Database
}

// GetDBMap returns a gorp.DbMap pointer
Expand All @@ -58,76 +58,76 @@ func (f *DBConnectionFactory) GetDBMap() *gorp.DbMap {

//Set is for tetsing purpose, we need to set manually the connection
func (f *DBConnectionFactory) Set(d *sql.DB) {
f.db = d
f.Database = d
}

// Init initialize sql.DB object by checking environment variables and connecting to database
func Init(user, role, password, name, host string, port int, sslmode string, connectTimeout, timeout, maxconn int) (*DBConnectionFactory, error) {
f := &DBConnectionFactory{
dbDriver: "postgres",
dbRole: role,
dbUser: user,
dbPassword: password,
dbName: name,
dbHost: host,
dbPort: port,
dbSSLMode: sslmode,
dbTimeout: timeout,
dbConnectTimeout: connectTimeout,
dbMaxConn: maxconn,
DBDriver: "postgres",
DBRole: role,
DBUser: user,
DBPassword: password,
DBName: name,
DBHost: host,
DBPort: port,
DBSSLMode: sslmode,
DBTimeout: timeout,
DBConnectTimeout: connectTimeout,
DBMaxConn: maxconn,
mutex: &sync.Mutex{},
}

f.mutex.Lock()
defer f.mutex.Unlock()

// Try to close before reinit
if f.db != nil {
if err := f.db.Close(); err != nil {
if f.Database != nil {
if err := f.Database.Close(); err != nil {
log.Error("Cannot close connection to DB : %s", err)
}
}

var err error

if f.dbUser == "" ||
f.dbPassword == "" ||
f.dbName == "" ||
f.dbHost == "" ||
f.dbPort == 0 {
if f.DBUser == "" ||
f.DBPassword == "" ||
f.DBName == "" ||
f.DBHost == "" ||
f.DBPort == 0 {
return nil, fmt.Errorf("Missing database infos")
}

if f.dbTimeout < 200 || f.dbTimeout > 30000 {
f.dbTimeout = 3000
if f.DBTimeout < 200 || f.DBTimeout > 30000 {
f.DBTimeout = 3000
}

if f.dbConnectTimeout <= 0 {
f.dbConnectTimeout = 10
if f.DBConnectTimeout <= 0 {
f.DBConnectTimeout = 10
}

// connect_timeout in seconds
// statement_timeout in milliseconds
dsn := f.dsn()
f.db, err = sql.Open(f.dbDriver, dsn)
f.Database, err = sql.Open(f.DBDriver, dsn)
if err != nil {
f.db = nil
f.Database = nil
log.Error("cannot open database: %s", err)
return nil, err
}

if err = f.db.Ping(); err != nil {
f.db = nil
if err = f.Database.Ping(); err != nil {
f.Database = nil
return nil, err
}

f.db.SetMaxOpenConns(f.dbMaxConn)
f.db.SetMaxIdleConns(int(f.dbMaxConn / 2))
f.Database.SetMaxOpenConns(f.DBMaxConn)
f.Database.SetMaxIdleConns(int(f.DBMaxConn / 2))

// Set role if specified
if role != "" {
log.Debug("database> setting role %s on database", role)
if _, err := f.db.Exec("SET ROLE '" + role + "'"); err != nil {
if _, err := f.Database.Exec("SET ROLE '" + role + "'"); err != nil {
log.Error("unable to set role %s on database: %s", role, err)
return nil, sdk.WrapError(err, "unable to set role %s", role)
}
Expand All @@ -137,26 +137,26 @@ func Init(user, role, password, name, host string, port int, sslmode string, con
}

func (f *DBConnectionFactory) dsn() string {
return fmt.Sprintf("user=%s password=%s dbname=%s host=%s port=%d sslmode=%s connect_timeout=%d statement_timeout=%d", f.dbUser, f.dbPassword, f.dbName, f.dbHost, f.dbPort, f.dbSSLMode, f.dbConnectTimeout, f.dbTimeout)
return fmt.Sprintf("user=%s password=%s dbname=%s host=%s port=%d sslmode=%s connect_timeout=%d statement_timeout=%d", f.DBUser, f.DBPassword, f.DBName, f.DBHost, f.DBPort, f.DBSSLMode, f.DBConnectTimeout, f.DBTimeout)
}

// Status returns database driver and status in a printable string
func (f *DBConnectionFactory) Status() sdk.MonitoringStatusLine {
if f.db == nil {
if f.Database == nil {
return sdk.MonitoringStatusLine{Component: "Database Conns", Value: "No Connection", Status: sdk.MonitoringStatusAlert}
}

if err := f.db.Ping(); err != nil {
if err := f.Database.Ping(); err != nil {
return sdk.MonitoringStatusLine{Component: "Database Conns", Value: "No Ping", Status: sdk.MonitoringStatusAlert}
}

return sdk.MonitoringStatusLine{Component: "Database Conns", Value: fmt.Sprintf("%d", f.db.Stats().OpenConnections), Status: sdk.MonitoringStatusOK}
return sdk.MonitoringStatusLine{Component: "Database Conns", Value: fmt.Sprintf("%d", f.Database.Stats().OpenConnections), Status: sdk.MonitoringStatusOK}
}

// Close closes the database, releasing any open resources.
func (f *DBConnectionFactory) Close() error {
if f.db != nil {
return f.db.Close()
if f.Database != nil {
return f.Database.Close()
}
return nil
}
Expand Down
Loading

0 comments on commit eeb1665

Please sign in to comment.