Skip to content

Commit

Permalink
fix(api): rename worker model (#5183)
Browse files Browse the repository at this point in the history 
* fix(api): rename worker model

Signed-off-by: Yvonnick Esnault <[email protected]>
  • Loading branch information
@yesnault
yesnault authored May 15, 2020
1 parent 2497dfa commit d21537d
Show file tree
Hide file tree
Showing 4 changed files with 231 additions and 14 deletions.
8 changes: 6 additions & 2 deletions engine/api/action/dao.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package action

import (
"context"
"fmt"
"strings"

"github.com/go-gorp/gorp"
Expand Down Expand Up @@ -197,11 +198,14 @@ func GetRequirementsDistinctBinary(db gorp.SqlExecutor) (sdk.RequirementList, er
func GetRequirementsTypeModelAndValueStartBy(ctx context.Context, db gorp.SqlExecutor, value string) ([]sdk.Requirement, error) {
rs := []sdk.Requirement{}

// if value equals Debian9, the regex should match "Debian9" and "Debian9 --foo", but not "Debian9-Foo"
reg := fmt.Sprintf("^%s(?!\\S)", value)

query := gorpmapping.NewQuery(`
SELECT *
FROM action_requirement
WHERE type = 'model' AND value LIKE $1
`).Args(value + "%")
WHERE type = 'model' AND (value ~ $1)
`).Args(reg)

if err := gorpmapping.GetAll(ctx, db, query, &rs); err != nil {
return nil, sdk.WrapError(err, "cannot get requirements")
Expand Down
30 changes: 18 additions & 12 deletions engine/api/pipeline/pipeline.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,10 @@ func LoadByWorkerModel(ctx context.Context, db gorp.SqlExecutor, model *sdk.Mode
var query gorpmapping.Query

isSharedInfraModel := model.GroupID == group.SharedInfraGroup.ID
modelNamePattern := model.Group.Name + "/" + model.Name + "%"
modelNamePatternWithGroup := model.Group.Name + "/" + model.Name

modelNamePattern1 := fmt.Sprintf("^%s(?!\\S)", model.Name)
modelNamePattern2 := fmt.Sprintf("^%s(?!\\S)", modelNamePatternWithGroup)

if isSharedInfraModel {
query = gorpmapping.NewQuery(`
Expand All @@ -104,8 +107,8 @@ func LoadByWorkerModel(ctx context.Context, db gorp.SqlExecutor, model *sdk.Mode
JOIN pipeline ON pipeline.id = pipeline_stage.pipeline_id
JOIN project ON project.id = pipeline.project_id
WHERE action_requirement.type = 'model'
AND (action_requirement.value LIKE $1 OR action_requirement.value LIKE $2)
`).Args(model.Name+"%", modelNamePattern)
AND (action_requirement.value ~ $1 OR action_requirement.value ~ $2)
`).Args(modelNamePattern1, modelNamePattern2)
} else {
query = gorpmapping.NewQuery(`
SELECT DISTINCT pipeline.*, project.projectkey AS projectKey
Expand All @@ -115,13 +118,13 @@ func LoadByWorkerModel(ctx context.Context, db gorp.SqlExecutor, model *sdk.Mode
JOIN pipeline ON pipeline.id = pipeline_stage.pipeline_id
JOIN project ON project.id = pipeline.project_id
WHERE action_requirement.type = 'model'
AND action_requirement.value LIKE $1
`).Args(modelNamePattern)
AND action_requirement.value ~ $1
`).Args(modelNamePattern2)
}

var dbPips Pipelines
if err := gorpmapping.GetAll(ctx, db, query, &dbPips); err != nil {
return nil, sdk.WrapError(err, "unable to load pipelines linked to worker model pattern %s", modelNamePattern)
return nil, sdk.WrapError(err, "unable to load pipelines linked to worker model pattern %s", modelNamePattern2)
}

return dbPips.Cast(), nil
Expand All @@ -132,7 +135,10 @@ func LoadByWorkerModelAndGroupIDs(ctx context.Context, db gorp.SqlExecutor, mode
var query gorpmapping.Query

isSharedInfraModel := model.GroupID == group.SharedInfraGroup.ID
modelNamePattern := model.Group.Name + "/" + model.Name + "%"
modelNamePatternWithGroup := model.Group.Name + "/" + model.Name

modelNamePattern1 := fmt.Sprintf("^%s(?!\\S)", model.Name)
modelNamePattern2 := fmt.Sprintf("^%s(?!\\S)", modelNamePatternWithGroup)

if isSharedInfraModel {
query = gorpmapping.NewQuery(`
Expand All @@ -143,13 +149,13 @@ func LoadByWorkerModelAndGroupIDs(ctx context.Context, db gorp.SqlExecutor, mode
JOIN pipeline ON pipeline.id = pipeline_stage.pipeline_id
JOIN project ON project.id = pipeline.project_id
WHERE action_requirement.type = 'model'
AND (action_requirement.value LIKE $1 OR action_requirement.value LIKE $2)
AND (action_requirement.value ~ $1 OR action_requirement.value ~ $2)
AND project.id IN (
SELECT project_group.project_id
FROM project_group
WHERE project_group.group_id = ANY(string_to_array($3, ',')::int[])
)
`).Args(model.Name+"%", modelNamePattern, gorpmapping.IDsToQueryString(groupIDs))
`).Args(modelNamePattern1, modelNamePattern2, gorpmapping.IDsToQueryString(groupIDs))
} else {
query = gorpmapping.NewQuery(`
SELECT DISTINCT pipeline.*, project.projectkey AS projectKey
Expand All @@ -159,19 +165,19 @@ func LoadByWorkerModelAndGroupIDs(ctx context.Context, db gorp.SqlExecutor, mode
JOIN pipeline ON pipeline.id = pipeline_stage.pipeline_id
JOIN project ON project.id = pipeline.project_id
WHERE action_requirement.type = 'model'
AND action_requirement.value LIKE $1
AND action_requirement.value ~ $1
AND project.id IN (
SELECT project_group.project_id
FROM project_group
WHERE
project_group.group_id = ANY(string_to_array($2, ',')::int[])
)
`).Args(modelNamePattern, gorpmapping.IDsToQueryString(groupIDs))
`).Args(modelNamePattern2, gorpmapping.IDsToQueryString(groupIDs))
}

var pips Pipelines
if err := gorpmapping.GetAll(ctx, db, query, &pips); err != nil {
return nil, sdk.WrapError(err, "unable to load pipelines linked to worker model pattern %s", modelNamePattern)
return nil, sdk.WrapError(err, "unable to load pipelines linked to worker model pattern %s", modelNamePattern2)
}

return pips.Cast(), nil
Expand Down
12 changes: 12 additions & 0 deletions engine/api/pipeline/pipeline_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ func TestLoadByWorkerModel(t *testing.T) {

model1 := sdk.Model{Name: sdk.RandomString(10), Group: g1, GroupID: g1.ID}
model2 := sdk.Model{Name: sdk.RandomString(10), Group: g2, GroupID: g2.ID}
model3 := sdk.Model{Name: model1.Name[:5], Group: g1, GroupID: g1.ID}

projectKey := sdk.RandomString(10)
proj := assets.InsertTestProject(t, db, cache, projectKey, projectKey)
Expand Down Expand Up @@ -267,6 +268,13 @@ func TestLoadByWorkerModel(t *testing.T) {
assert.Equal(t, pip1.Name, pips[0].Name)
assert.Equal(t, pip2.Name, pips[1].Name)

pips, err = pipeline.LoadByWorkerModel(context.TODO(), db, &model3)
assert.NoError(t, err)

if !assert.Equal(t, 0, len(pips)) {
t.FailNow()
}

pips, err = pipeline.LoadByWorkerModel(context.TODO(), db, &model2)
assert.NoError(t, err)

Expand All @@ -275,6 +283,10 @@ func TestLoadByWorkerModel(t *testing.T) {
}
assert.Equal(t, pip3.Name, pips[0].Name)

pips, err = pipeline.LoadByWorkerModelAndGroupIDs(context.TODO(), db, &model3, []int64{})
assert.NoError(t, err)
assert.Equal(t, 0, len(pips))

pips, err = pipeline.LoadByWorkerModelAndGroupIDs(context.TODO(), db, &model1, []int64{})
assert.NoError(t, err)
assert.Equal(t, 0, len(pips))
Expand Down
195 changes: 195 additions & 0 deletions engine/api/workermodel/crud_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,15 @@ package workermodel_test

import (
"context"
"fmt"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/ovh/cds/engine/api/bootstrap"
"github.com/ovh/cds/engine/api/group"
"github.com/ovh/cds/engine/api/pipeline"
"github.com/ovh/cds/engine/api/test"
"github.com/ovh/cds/engine/api/test/assets"
"github.com/ovh/cds/engine/api/workermodel"
Expand Down Expand Up @@ -122,6 +125,198 @@ func TestUpdateModel(t *testing.T) {
cpy.GroupID = g2.ID
res, err = workermodel.Update(context.TODO(), db, res, cpy)
require.Error(t, err)

}

// create a worker model aaa
// a pipeline use worker model aaa-foo
// rename worker model to aaa-bar
// the pipeline should keep the name aaa-foo
func TestUpdateModelInPipeline(t *testing.T) {
db, cache, end := test.SetupPG(t, bootstrap.InitiliazeDB)
defer end()

g1 := assets.InsertTestGroup(t, db, sdk.RandomString(10))
u, _ := assets.InsertLambdaUser(t, db)

pattern := sdk.ModelPattern{
Name: sdk.RandomString(10),
Type: sdk.Docker,
Model: sdk.ModelCmds{
Cmd: "pattern cmd",
Shell: "pattern shell",
},
}
require.NoError(t, workermodel.InsertPattern(db, &pattern))

model1Name := sdk.RandomString(10)
model1, err := workermodel.Create(context.TODO(), db, sdk.Model{
Type: sdk.Docker,
Name: model1Name,
Group: g1,
GroupID: g1.ID,
ModelDocker: sdk.ModelDocker{},
}, u)
require.NoError(t, err)

model1NameFoo := model1Name + "-foo"
model1Foo, err := workermodel.Create(context.TODO(), db, sdk.Model{
Type: sdk.Docker,
Name: model1NameFoo,
Group: g1,
GroupID: g1.ID,
ModelDocker: sdk.ModelDocker{},
}, u)
require.NoError(t, err)

projectKey := sdk.RandomString(10)
proj := assets.InsertTestProject(t, db, cache, projectKey, projectKey)

require.NoError(t, group.InsertLinkGroupProject(context.TODO(), db, &group.LinkGroupProject{
GroupID: g1.ID,
ProjectID: proj.ID,
Role: sdk.PermissionReadWriteExecute,
}))

pip1 := sdk.Pipeline{ProjectID: proj.ID, ProjectKey: proj.Key, Name: sdk.RandomString(10)}
test.NoError(t, pipeline.InsertPipeline(db, &pip1))
job1 := sdk.Job{
Enabled: true,
Action: sdk.Action{
Enabled: true,
Requirements: []sdk.Requirement{{
Type: sdk.ModelRequirement,
Name: fmt.Sprintf("%s/%s-foo --privileged", g1.Name, model1.Name),
Value: fmt.Sprintf("%s/%s-foo --privileged", g1.Name, model1.Name),
}},
},
}
test.NoError(t, pipeline.InsertJob(db, &job1, 0, &pip1))

model1FooLoad, err := workermodel.LoadByNameAndGroupID(db, model1Foo.Name, g1.ID)
require.NoError(t, err)

pips, err := pipeline.LoadByWorkerModel(context.TODO(), db, model1FooLoad)
assert.NoError(t, err)
require.Equal(t, 1, len(pips))

model1Load, err := workermodel.LoadByIDWithClearPassword(db, model1.ID)
require.NoError(t, err)

pips, err = pipeline.LoadByWorkerModel(context.TODO(), db, model1Load)
assert.NoError(t, err)
require.Equal(t, 0, len(pips))

// Test rename worker model
res, err := workermodel.Update(context.TODO(), db, model1Load, sdk.Model{
Type: sdk.Docker,
Name: model1Name + "-bar",
PatternName: pattern.Name,
GroupID: g1.ID,
ModelDocker: sdk.ModelDocker{
Private: true,
Password: sdk.PasswordPlaceholder,
},
})
require.NoError(t, err)

model2Load, err := workermodel.LoadByIDWithClearPassword(db, res.ID)
require.NoError(t, err)

pips, err = pipeline.LoadByWorkerModel(context.TODO(), db, model1Load)
assert.NoError(t, err)
require.Equal(t, 0, len(pips))

pips, err = pipeline.LoadByWorkerModel(context.TODO(), db, model2Load)
assert.NoError(t, err)
require.Equal(t, 0, len(pips))

pips, err = pipeline.LoadByWorkerModel(context.TODO(), db, model1FooLoad)
assert.NoError(t, err)
require.Equal(t, 1, len(pips))

}

func TestUpdateModelInPipelineSimple(t *testing.T) {
db, cache, end := test.SetupPG(t, bootstrap.InitiliazeDB)
defer end()

g1 := assets.InsertTestGroup(t, db, sdk.RandomString(10))
u, _ := assets.InsertLambdaUser(t, db)

pattern := sdk.ModelPattern{
Name: sdk.RandomString(10),
Type: sdk.Docker,
Model: sdk.ModelCmds{
Cmd: "pattern cmd",
Shell: "pattern shell",
},
}
require.NoError(t, workermodel.InsertPattern(db, &pattern))

model1Name := sdk.RandomString(10)
model1, err := workermodel.Create(context.TODO(), db, sdk.Model{
Type: sdk.Docker,
Name: model1Name,
Group: g1,
GroupID: g1.ID,
ModelDocker: sdk.ModelDocker{},
}, u)
require.NoError(t, err)

projectKey := sdk.RandomString(10)
proj := assets.InsertTestProject(t, db, cache, projectKey, projectKey)

require.NoError(t, group.InsertLinkGroupProject(context.TODO(), db, &group.LinkGroupProject{
GroupID: g1.ID,
ProjectID: proj.ID,
Role: sdk.PermissionReadWriteExecute,
}))

pip1 := sdk.Pipeline{ProjectID: proj.ID, ProjectKey: proj.Key, Name: sdk.RandomString(10)}
test.NoError(t, pipeline.InsertPipeline(db, &pip1))
job1 := sdk.Job{
Enabled: true,
Action: sdk.Action{
Enabled: true,
Requirements: []sdk.Requirement{{
Type: sdk.ModelRequirement,
Name: fmt.Sprintf("%s/%s --privileged", g1.Name, model1.Name),
Value: fmt.Sprintf("%s/%s --privileged", g1.Name, model1.Name),
}},
},
}
test.NoError(t, pipeline.InsertJob(db, &job1, 0, &pip1))

model1Load, err := workermodel.LoadByIDWithClearPassword(db, model1.ID)
require.NoError(t, err)

pips, err := pipeline.LoadByWorkerModel(context.TODO(), db, model1Load)
assert.NoError(t, err)
require.Equal(t, 1, len(pips))

model1NameFoo := model1Name + "-foo"
res, err := workermodel.Update(context.TODO(), db, model1Load, sdk.Model{
Type: sdk.Docker,
Name: model1NameFoo,
PatternName: pattern.Name,
GroupID: g1.ID,
ModelDocker: sdk.ModelDocker{
Private: true,
Password: sdk.PasswordPlaceholder,
},
})
require.NoError(t, err)

model1FooLoad, err := workermodel.LoadByIDWithClearPassword(db, res.ID)
require.NoError(t, err)

pips, err = pipeline.LoadByWorkerModel(context.TODO(), db, model1FooLoad)
assert.NoError(t, err)
require.Equal(t, 1, len(pips))

pip, err := pipeline.LoadPipelineByID(context.TODO(), db, pips[0].ID, true)
require.Equal(t, fmt.Sprintf("%s/%s-foo --privileged", g1.Name, model1.Name), pip.Stages[0].Jobs[0].Action.Requirements[0].Value)
}

func TestCopyModelTypeData(t *testing.T) {
Expand Down

0 comments on commit d21537d

Please sign in to comment.