Skip to content

Commit

Permalink
feat(api,cdsctl,sdk): add export worker model (#3774)
Browse files Browse the repository at this point in the history
* feat(api,cdsctl,sdk): refactor and add exportentities worker model

Signed-off-by: Benjamin Coenen <[email protected]>

* feat(api,cdsctl,sdk): add export worker model (#3767)

Signed-off-by: Benjamin Coenen <[email protected]>

* feat(api,cdsctl,sdk): add export worker model (#3767)

Signed-off-by: Benjamin Coenen <[email protected]>

* feat(api,cdsctl,sdk): add export worker model (#3767)

Signed-off-by: Benjamin Coenen <[email protected]>

* Apply suggestions from code review

Co-Authored-By: bnjjj <[email protected]>

* feat(api,cdsctl,sdk): add export worker model (#3767)

Signed-off-by: Benjamin Coenen <[email protected]>
  • Loading branch information
bnjjj authored and richardlt committed Dec 26, 2018
1 parent 50a3fa7 commit 689b82c
Show file tree
Hide file tree
Showing 16 changed files with 601 additions and 146 deletions.
174 changes: 35 additions & 139 deletions cli/cdsctl/worker_model.go
Original file line number Diff line number Diff line change
@@ -1,14 +1,11 @@
package main

import (
"bytes"
"encoding/json"
"fmt"
"reflect"
"strings"

"github.com/spf13/cobra"
yaml "gopkg.in/yaml.v2"

"github.com/ovh/cds/cli"
"github.com/ovh/cds/sdk"
Expand All @@ -26,6 +23,7 @@ func workerModel() *cobra.Command {
cli.NewGetCommand(workerModelShowCmd, workerModelShowRun, nil, withAllCommandModifiers()...),
cli.NewDeleteCommand(workerModelDeleteCmd, workerModelDeleteRun, nil),
cli.NewCommand(workerModelImportCmd, workerModelImportRun, nil),
cli.NewCommand(workerModelExportCmd, workerModelExportRun, nil, withAllCommandModifiers()...),
})
}

Expand Down Expand Up @@ -101,24 +99,6 @@ For admin:
},
}

type workerModelFile struct {
Name string `json:"name" yaml:"name"`
Group string `json:"group" yaml:"group"`
Communication string `json:"communication,omitempty" yaml:"communication,omitempty"`
Provision int `json:"provision,omitempty" yaml:"provision,omitempty"`
Image string `json:"image" yaml:"image"`
Description string `json:"description" yaml:"description"`
Type string `json:"type" yaml:"type"`
Flavor string `json:"flavor,omitempty" yaml:"flavor,omitempty"`
Envs map[string]string `json:"envs,omitempty" yaml:"envs,omitempty"`
PatternName string `json:"pattern_name,omitempty" yaml:"pattern_name,omitempty"`
Shell string `json:"shell,omitempty" yaml:"shell,omitempty"`
PreCmd string `json:"pre_cmd,omitempty" yaml:"pre_cmd,omitempty"`
Cmd string `json:"cmd,omitempty" yaml:"cmd,omitempty"`
PostCmd string `json:"post_cmd,omitempty" yaml:"post_cmd,omitempty"`
Restricted bool `json:"restricted" yaml:"restricted"`
}

func workerModelImportRun(c cli.Values) error {
force := c.GetBool("force")
if c.GetString("filepath") == "" {
Expand All @@ -132,126 +112,12 @@ func workerModelImportRun(c cli.Values) error {
return fmt.Errorf("Error: Cannot read file %s (%v)", filepath, err)
}

buf := new(bytes.Buffer)
if _, errR := buf.ReadFrom(reader); errR != nil {
reader.Close()
return fmt.Errorf("Error: cannot read file content %s : %v", filepath, errR)
}
reader.Close()

var modelInfos workerModelFile
switch format {
case exportentities.FormatJSON:
if err := json.Unmarshal(buf.Bytes(), &modelInfos); err != nil {
return fmt.Errorf("Error: cannot unmarshal json file %s : %v", filepath, err)
}
case exportentities.FormatYAML:
if err := yaml.Unmarshal(buf.Bytes(), &modelInfos); err != nil {
return fmt.Errorf("Error: cannot unmarshal yaml file %s : %v", filepath, err)
}
default:
return fmt.Errorf("Invalid file format")
}

var t string
var modelDocker sdk.ModelDocker
var modelVM sdk.ModelVirtualMachine
switch modelInfos.Type {
case sdk.Docker:
t = sdk.Docker
if modelInfos.Image == "" {
sdk.Exit("Error: Docker image not provided\n")
}
modelDocker.Shell = modelInfos.Shell
modelDocker.Image = modelInfos.Image
modelDocker.Cmd = modelInfos.Cmd
if modelInfos.PatternName == "" {
if modelDocker.Shell == "" {
sdk.Exit("Error: main shell command not provided\n")
}
if modelDocker.Cmd == "" {
sdk.Exit("Error: main worker command not provided\n")
}
}

break
case sdk.Openstack:
t = sdk.Openstack
d := sdk.ModelVirtualMachine{
Image: modelInfos.Image,
Flavor: modelInfos.Flavor,
Cmd: modelInfos.Cmd,
PostCmd: modelInfos.PostCmd,
PreCmd: modelInfos.PreCmd,
}
if d.Image == "" {
return fmt.Errorf("Error: Openstack image not provided")
}
if d.Flavor == "" {
return fmt.Errorf("Error: Openstack flavor not provided")
}
if modelInfos.PatternName == "" {
if d.Cmd == "" {
return fmt.Errorf("Error: Openstack command not provided")
}
}
modelVM = d
break
case sdk.VSphere:
t = sdk.VSphere
d := sdk.ModelVirtualMachine{
Image: modelInfos.Image,
Flavor: modelInfos.Flavor,
Cmd: modelInfos.Cmd,
PostCmd: modelInfos.PostCmd,
PreCmd: modelInfos.PreCmd,
}
if d.Image == "" {
return fmt.Errorf("Error: VSphere image not provided")
}
if modelInfos.PatternName == "" {
if d.Cmd == "" {
return fmt.Errorf("Error: VSphere main worker command empty")
}
}

modelVM = d
break
default:
return fmt.Errorf("Unknown worker type: %s", modelInfos.Type)
}

if modelInfos.Name == "" {
return fmt.Errorf("Error: worker model name is not provided")
}

if modelInfos.Group == "" {
return fmt.Errorf("Error: group is not provided")
}

g, err := client.GroupGet(modelInfos.Group)
formatStr, _ := exportentities.GetFormatStr(format)
wm, err := client.WorkerModelImport(reader, formatStr, force)
if err != nil {
return fmt.Errorf("Error : Unable to get group %s : %s", modelInfos.Group, err)
}

if force {
if existingWm, err := client.WorkerModel(modelInfos.Name); err != nil {
if _, errAdd := client.WorkerModelAdd(modelInfos.Name, t, modelInfos.PatternName, &modelDocker, &modelVM, g.ID); errAdd != nil {
return fmt.Errorf("Error: cannot add worker model %s (%s)", modelInfos.Name, errAdd)
}
fmt.Printf("Worker model %s added with success", modelInfos.Name)
} else {
if _, errU := client.WorkerModelUpdate(existingWm.ID, modelInfos.Name, t, &modelDocker, &modelVM, g.ID); errU != nil {
return fmt.Errorf("Error: cannot update worker model %s (%s)", modelInfos.Name, errU)
}
fmt.Printf("Worker model %s updated with success", modelInfos.Name)
}
} else {
if _, errAdd := client.WorkerModelAdd(modelInfos.Name, t, modelInfos.PatternName, &modelDocker, &modelVM, g.ID); errAdd != nil {
return fmt.Errorf("Error: cannot add worker model %s (%s)", modelInfos.Name, errAdd)
}
fmt.Printf("Worker model %s added with success", modelInfos.Name)
return err
}
fmt.Printf("Worker model %s imported with success\n", wm.Name)
}

return nil
Expand Down Expand Up @@ -291,3 +157,33 @@ func workerModelDeleteRun(v cli.Values) error {
}
return nil
}

var workerModelExportCmd = cli.Command{
Name: "export",
Short: "Export a worker model",
Args: []cli.Arg{
{Name: "name"},
},
Flags: []cli.Flag{
{
Kind: reflect.String,
Name: "format",
Usage: "Specify export format (json or yaml)",
Default: "yaml",
},
},
}

func workerModelExportRun(c cli.Values) error {
wmName := c.GetString("name")
wm, err := client.WorkerModel(wmName)
if err != nil {
return sdk.WrapError(err, "cannot load worker model %s", wmName)
}
btes, err := client.WorkerModelExport(wm.ID, c.GetString("format"))
if err != nil {
return err
}
fmt.Println(string(btes))
return nil
}
2 changes: 2 additions & 0 deletions engine/api/api_routes.go
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,7 @@ func (api *API) InitRouter() {

// Worker models
r.Handle("/worker/model", r.POST(api.addWorkerModelHandler), r.GET(api.getWorkerModelsHandler))
r.Handle("/worker/model/import", r.POST(api.postWorkerModelImportHandler))
r.Handle("/worker/model/pattern", r.POST(api.postAddWorkerModelPatternHandler, NeedAdmin(true)), r.GET(api.getWorkerModelPatternsHandler))
r.Handle("/worker/model/pattern/{type}/{name}", r.GET(api.getWorkerModelPatternHandler), r.PUT(api.putWorkerModelPatternHandler, NeedAdmin(true)), r.DELETE(api.deleteWorkerModelPatternHandler, NeedAdmin(true)))
r.Handle("/worker/model/book/{permModelID}", r.PUT(api.bookWorkerModelHandler, NeedHatchery()))
Expand All @@ -424,6 +425,7 @@ func (api *API) InitRouter() {
r.Handle("/worker/model/type", r.GET(api.getWorkerModelTypesHandler))
r.Handle("/worker/model/communication", r.GET(api.getWorkerModelCommunicationsHandler))
r.Handle("/worker/model/{permModelID}", r.PUT(api.updateWorkerModelHandler), r.DELETE(api.deleteWorkerModelHandler))
r.Handle("/worker/model/{permModelID}/export", r.GET(api.getWorkerModelExportHandler))
r.Handle("/worker/model/{permModelID}/usage", r.GET(api.getWorkerModelUsageHandler))
r.Handle("/worker/model/capability/type", r.GET(api.getRequirementTypesHandler))

Expand Down
16 changes: 16 additions & 0 deletions engine/api/group/group.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,22 @@ func LoadGroupByID(db gorp.SqlExecutor, id int64) (*sdk.Group, error) {
}, nil
}

// LoadGroupByName retrieves group informations from database given his name
func LoadGroupByName(db gorp.SqlExecutor, name string) (*sdk.Group, error) {
query := `SELECT "group".id FROM "group" WHERE "group".name = $1`
var id int64
if err := db.QueryRow(query, name).Scan(&id); err != nil {
if err == sql.ErrNoRows {
err = sdk.ErrGroupNotFound
}
return nil, sdk.WithStack(err)
}
return &sdk.Group{
ID: id,
Name: name,
}, nil
}

// LoadUserGroup retrieves all group users from database
func LoadUserGroup(db gorp.SqlExecutor, group *sdk.Group) error {
query := `SELECT "user".username, "user".data, "group_user".group_admin FROM "user"
Expand Down
21 changes: 21 additions & 0 deletions engine/api/worker/model_export.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package worker

import (
"io"

"github.com/ovh/cds/sdk"
"github.com/ovh/cds/sdk/exportentities"
)

// Export convert sdk.Model to an exportentities.WorkerModel, format and write into a io.Writer
func Export(wm sdk.Model, f exportentities.Format, w io.Writer) (int, error) {
eWm := exportentities.NewWorkerModel(wm)

// Marshal to the desired format
b, err := exportentities.Marshal(eWm, f)
if err != nil {
return 0, sdk.WithStack(err)
}

return w.Write(b)
}
124 changes: 124 additions & 0 deletions engine/api/worker/model_import.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
package worker

import (
"database/sql"

"github.com/go-gorp/gorp"
"github.com/lib/pq"

"github.com/ovh/cds/engine/api/database/gorpmapping"
"github.com/ovh/cds/engine/api/group"
"github.com/ovh/cds/sdk"
"github.com/ovh/cds/sdk/exportentities"
)

// ParseAndImport parse and import an exportentities.WorkerModel
func ParseAndImport(db gorp.SqlExecutor, eWorkerModel *exportentities.WorkerModel, force bool, u *sdk.User) (*sdk.Model, error) {
sdkWm, err := eWorkerModel.GetWorkerModel()
if err != nil {
return nil, err
}

gr, err := group.LoadGroupByName(db, sdkWm.Group.Name)
if err != nil {
return nil, sdk.WrapError(err, "Unable to get group %s", sdkWm.Group.Name)
}
sdkWm.Group = *gr
sdkWm.GroupID = gr.ID

var modelPattern *sdk.ModelPattern
if sdkWm.PatternName != "" {
var errP error
modelPattern, errP = LoadWorkerModelPatternByName(db, sdkWm.Type, sdkWm.PatternName)
if errP != nil {
return nil, sdk.WrapError(sdk.ErrWrongRequest, "Cannot load worker model pattern %s : %v", sdkWm.PatternName, errP)
}
}

//User must be admin of the group set in the model
var isGroupAdmin bool
currentUGroup:
for _, g := range u.Groups {
if g.ID == sdkWm.GroupID {
for _, a := range g.Admins {
if a.ID == u.ID {
isGroupAdmin = true
break currentUGroup
}
}
}
}

//User should have the right permission or be admin
if !u.Admin && !isGroupAdmin {
return nil, sdk.ErrWorkerModelNoAdmin
}

switch sdkWm.Type {
case sdk.Docker:
if sdkWm.ModelDocker.Image == "" {
return nil, sdk.WrapError(sdk.ErrWrongRequest, "Invalid worker image")
}
if !u.Admin && !sdkWm.Restricted {
if modelPattern == nil {
return nil, sdk.ErrWorkerModelNoPattern
}
}
if sdkWm.ModelDocker.Cmd == "" || sdkWm.ModelDocker.Shell == "" {
return nil, sdk.WrapError(sdk.ErrWrongRequest, "Invalid worker command or invalid shell command")
}
default:
if sdkWm.ModelVirtualMachine.Image == "" {
return nil, sdk.WrapError(sdk.ErrWrongRequest, "Invalid worker command or invalid image")
}
if !u.Admin && !sdkWm.Restricted {
if modelPattern == nil {
return nil, sdk.ErrWorkerModelNoPattern
}
sdkWm.ModelVirtualMachine.PreCmd = modelPattern.Model.PreCmd
sdkWm.ModelVirtualMachine.Cmd = modelPattern.Model.Cmd
sdkWm.ModelVirtualMachine.PostCmd = modelPattern.Model.PostCmd
}
}

if sdkWm.GroupID == 0 {
return nil, sdk.WrapError(sdk.ErrWrongRequest, "groupID should be set")
}

if group.IsDefaultGroupID(sdkWm.GroupID) {
return nil, sdk.WrapError(sdk.ErrWrongRequest, "this group can't be owner of a worker model")
}

// provision is allowed only for CDS Admin
// or by currentUser with a restricted model
if !u.Admin && !sdkWm.Restricted {
sdkWm.Provision = 0
}

if force {
if existingWm, err := LoadWorkerModelByName(db, sdkWm.Name); err != nil {
if sdk.Cause(err) == sql.ErrNoRows {
if errAdd := InsertWorkerModel(db, &sdkWm); errAdd != nil {
return nil, sdk.WrapError(errAdd, "cannot add worker model %s", sdkWm.Name)
}
} else {
return nil, sdk.WrapError(err, "cannot find worker model %s", sdkWm.Name)
}
} else {
sdkWm.ID = existingWm.ID
if errU := UpdateWorkerModel(db, &sdkWm); errU != nil {
return nil, sdk.WrapError(errU, "cannot update worker model %s", sdkWm.Name)
}
}
return &sdkWm, nil
}

if errAdd := InsertWorkerModel(db, &sdkWm); errAdd != nil {
if errPG, ok := sdk.Cause(errAdd).(*pq.Error); ok && errPG.Code == gorpmapping.ViolateUniqueKeyPGCode {
errAdd = sdk.ErrConflict
}
return nil, sdk.WrapError(errAdd, "cannot add worker model %s", sdkWm.Name)
}

return &sdkWm, nil
}
Loading

0 comments on commit 689b82c

Please sign in to comment.