From 80140d72472aaa1dcd721081c5a01f9eecdaee19 Mon Sep 17 00:00:00 2001 From: Richard LT Date: Wed, 10 Jun 2020 15:14:50 +0200 Subject: [PATCH] feat(hatchery): openstack max CPUs count and flavor weigth (#5190) * feat(hatchery): openstask max CPUs count and flavor weigth * feat: sort and filter models enabled, check max CPUs count * feat: impl CountSmallerFlavorToKeep check * test: unit test func --- engine/hatchery/openstack/get.go | 28 +++- engine/hatchery/openstack/init.go | 29 +++- engine/hatchery/openstack/init_test.go | 41 +++++ engine/hatchery/openstack/openstack.go | 39 +++-- engine/hatchery/openstack/openstack_test.go | 85 ++++++---- engine/hatchery/openstack/spawn.go | 82 ++++++++-- engine/hatchery/openstack/spawn_test.go | 163 ++++++++++++++++++++ engine/hatchery/openstack/types.go | 11 ++ go.mod | 2 +- 9 files changed, 415 insertions(+), 65 deletions(-) create mode 100644 engine/hatchery/openstack/init_test.go create mode 100644 engine/hatchery/openstack/spawn_test.go diff --git a/engine/hatchery/openstack/get.go b/engine/hatchery/openstack/get.go index b778640702..65988dc510 100644 --- a/engine/hatchery/openstack/get.go +++ b/engine/hatchery/openstack/get.go @@ -6,6 +6,7 @@ import ( "sync" "time" + "github.com/gophercloud/gophercloud/openstack/compute/v2/flavors" "github.com/gophercloud/gophercloud/openstack/compute/v2/images" "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" @@ -20,17 +21,32 @@ func (h *HatcheryOpenstack) imageID(ctx context.Context, img string) (string, er return i.ID, nil } } - return "", fmt.Errorf("imageID> image '%s' not found", img) + return "", sdk.WithStack(fmt.Errorf("image '%s' not found", img)) } // Find flavor ID from flavor name -func (h *HatcheryOpenstack) flavorID(flavor string) (string, error) { - for _, f := range h.flavors { - if f.Name == flavor { - return f.ID, nil +func (h *HatcheryOpenstack) flavor(flavor string) (flavors.Flavor, error) { + for i := range h.flavors { + if h.flavors[i].Name == flavor { + return h.flavors[i], nil } } - return "", fmt.Errorf("flavorID> flavor '%s' not found", flavor) + return flavors.Flavor{}, sdk.WithStack(fmt.Errorf("flavor '%s' not found", flavor)) +} + +// Find flavor ID from flavor name +func (h *HatcheryOpenstack) getSmallerFlavorThan(flavor flavors.Flavor) flavors.Flavor { + var smaller *flavors.Flavor + for i := range h.flavors { + // If the flavor is not the given one and need less CPUs its + if h.flavors[i].ID != flavor.ID && h.flavors[i].VCPUs < flavor.VCPUs && (smaller == nil || smaller.VCPUs < h.flavors[i].VCPUs) { + smaller = &h.flavors[i] + } + } + if smaller == nil { + return flavor + } + return *smaller } //This a embedded cache for images list diff --git a/engine/hatchery/openstack/init.go b/engine/hatchery/openstack/init.go index b338525ba0..20b4acebba 100644 --- a/engine/hatchery/openstack/init.go +++ b/engine/hatchery/openstack/init.go @@ -61,14 +61,41 @@ func (h *HatcheryOpenstack) initFlavors() error { if err != nil { return sdk.WithStack(fmt.Errorf("initFlavors> error on flavors.ListDetail: %v", err)) } + lflavors, err := flavors.ExtractFlavors(all) if err != nil { return sdk.WithStack(fmt.Errorf("initFlavors> error on flavors.ExtractFlavors: %v", err)) } - h.flavors = lflavors + + h.flavors = h.filterAllowedFlavors(lflavors) + return nil } +func (h HatcheryOpenstack) filterAllowedFlavors(allFlavors []flavors.Flavor) []flavors.Flavor { + // If allowed flavors are given in configuration we should check that given flavor is part of the list. + if len(h.Config.AllowedFlavors) == 0 { + return allFlavors + } + + filteredFlavors := make([]flavors.Flavor, 0, len(allFlavors)) + for i := range allFlavors { + var allowed bool + for j := range h.Config.AllowedFlavors { + if h.Config.AllowedFlavors[j] == allFlavors[i].Name { + allowed = true + break + } + } + if !allowed { + log.Debug("initFlavors> flavor '%s' is not allowed", allFlavors[i].Name) + continue + } + filteredFlavors = append(filteredFlavors, allFlavors[i]) + } + return filteredFlavors +} + func (h *HatcheryOpenstack) initNetworks() error { all, err := tenantnetworks.List(h.openstackClient).AllPages() if err != nil { diff --git a/engine/hatchery/openstack/init_test.go b/engine/hatchery/openstack/init_test.go new file mode 100644 index 0000000000..578f9cd635 --- /dev/null +++ b/engine/hatchery/openstack/init_test.go @@ -0,0 +1,41 @@ +package openstack + +import ( + "testing" + + "github.com/gophercloud/gophercloud/openstack/compute/v2/flavors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ovh/cds/sdk/log" +) + +func TestHatcheryOpenstack_initFlavors(t *testing.T) { + log.SetLogger(t) + + h := &HatcheryOpenstack{} + + allFlavors := []flavors.Flavor{ + {Name: "b2-7", VCPUs: 2}, + {Name: "b2-15", VCPUs: 4}, + {Name: "b2-30", VCPUs: 8}, + {Name: "b2-60", VCPUs: 16}, + {Name: "b2-120", VCPUs: 32}, + } + + filteredFlavors := h.filterAllowedFlavors(allFlavors) + require.Len(t, filteredFlavors, 5, "no filter as allowed flavor list is empty in config") + + h.Config.AllowedFlavors = []string{"b2-15", "b2-60"} + + filteredFlavors = h.filterAllowedFlavors(allFlavors) + require.Len(t, filteredFlavors, 2) + assert.Equal(t, "b2-15", filteredFlavors[0].Name) + assert.Equal(t, "b2-60", filteredFlavors[1].Name) + + h.Config.AllowedFlavors = []string{"s1-4", "b2-15"} + + filteredFlavors = h.filterAllowedFlavors(allFlavors) + require.Len(t, filteredFlavors, 1) + assert.Equal(t, "b2-15", filteredFlavors[0].Name) +} diff --git a/engine/hatchery/openstack/openstack.go b/engine/hatchery/openstack/openstack.go index 5e4a28f772..c3d39f0569 100644 --- a/engine/hatchery/openstack/openstack.go +++ b/engine/hatchery/openstack/openstack.go @@ -3,6 +3,7 @@ package openstack import ( "context" "fmt" + "sort" "strings" "sync" "time" @@ -154,7 +155,34 @@ func (*HatcheryOpenstack) ModelType() string { // WorkerModelsEnabled returns Worker model enabled. func (h *HatcheryOpenstack) WorkerModelsEnabled() ([]sdk.Model, error) { - return h.CDSClient().WorkerModelEnabledList() + allModels, err := h.CDSClient().WorkerModelEnabledList() + if err != nil { + return nil, err + } + + filteredModels := make([]sdk.Model, 0, len(allModels)) + for i := range allModels { + if allModels[i].Type != sdk.Openstack { + continue + } + + // Required flavor should be available on target OpenStack project + if _, err := h.flavor(allModels[i].ModelVirtualMachine.Flavor); err != nil { + log.Debug("WorkerModelsEnabled> model %s/%s is not usable because flavor '%s' not found", allModels[i].Group.Name, allModels[i].Name, allModels[i].ModelVirtualMachine.Flavor) + continue + } + + filteredModels = append(filteredModels, allModels[i]) + } + + // Sort models by required CPUs, this will allows to starts job without defined model on the smallest flavor. + sort.Slice(filteredModels, func(i, j int) bool { + flavorI, _ := h.flavor(filteredModels[i].ModelVirtualMachine.Flavor) + flavorJ, _ := h.flavor(filteredModels[j].ModelVirtualMachine.Flavor) + return flavorI.VCPUs < flavorJ.VCPUs + }) + + return filteredModels, nil } // WorkerModelSecretList returns secret for given model. @@ -165,15 +193,6 @@ func (h *HatcheryOpenstack) WorkerModelSecretList(m sdk.Model) (sdk.WorkerModelS // CanSpawn return wether or not hatchery can spawn model // requirements are not supported func (h *HatcheryOpenstack) CanSpawn(ctx context.Context, model *sdk.Model, jobID int64, requirements []sdk.Requirement) bool { - // if there is a model, we have to check if the flavor attached to model is knowned by this hatchery - if model != nil { - if _, err := h.flavorID(model.ModelVirtualMachine.Flavor); err != nil { - log.Debug("CanSpawn> h.flavorID on %s err:%v", model.ModelVirtualMachine.Flavor, err) - return false - } - log.Debug("CanSpawn> flavor '%s' found", model.ModelVirtualMachine.Flavor) - } - for _, r := range requirements { if r.Type == sdk.ServiceRequirement || r.Type == sdk.MemoryRequirement || r.Type == sdk.HostnameRequirement { return false diff --git a/engine/hatchery/openstack/openstack_test.go b/engine/hatchery/openstack/openstack_test.go index 0eb2d78676..351c40ca0f 100644 --- a/engine/hatchery/openstack/openstack_test.go +++ b/engine/hatchery/openstack/openstack_test.go @@ -4,10 +4,14 @@ import ( "context" "testing" + "github.com/golang/mock/gomock" "github.com/gophercloud/gophercloud/openstack/compute/v2/flavors" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/cdsclient/mock_cdsclient" + "github.com/ovh/cds/sdk/log" ) func TestHatcheryOpenstack_CanSpawn(t *testing.T) { @@ -28,41 +32,60 @@ func TestHatcheryOpenstack_CanSpawn(t *testing.T) { // no model, hostname prerequisite, canSpawn must be false: hostname can't be managed by openstack hatchery canSpawn = h.CanSpawn(context.TODO(), nil, 1, []sdk.Requirement{{Type: sdk.HostnameRequirement, Value: "localhost"}}) require.False(t, canSpawn) +} - flavors := []flavors.Flavor{ - {Name: "b2-7"}, - } - h.flavors = flavors +func TestHatcheryOpenstack_WorkerModelsEnabled(t *testing.T) { + log.SetLogger(t) - m := &sdk.Model{ - ID: 1, - Name: "my-model", - Group: &sdk.Group{ - ID: 1, - Name: "mygroup", - }, - ModelVirtualMachine: sdk.ModelVirtualMachine{ - Flavor: "vps-ssd-3", - }, - } + h := &HatcheryOpenstack{} - // model with a unknowned flavor - canSpawn = h.CanSpawn(context.TODO(), m, 1, nil) - require.False(t, canSpawn) + ctrl := gomock.NewController(t) + mockClient := mock_cdsclient.NewMockInterface(ctrl) + h.Client = mockClient + t.Cleanup(func() { ctrl.Finish() }) + + mockClient.EXPECT().WorkerModelEnabledList().DoAndReturn(func() ([]sdk.Model, error) { + return []sdk.Model{ + { + ID: 1, + Type: sdk.Docker, + Name: "my-model-1", + Group: &sdk.Group{ID: 1, Name: "mygroup"}, + }, + { + ID: 2, + Type: sdk.Openstack, + Name: "my-model-2", + Group: &sdk.Group{ID: 1, Name: "mygroup"}, + ModelVirtualMachine: sdk.ModelVirtualMachine{Flavor: "b2-120"}, + }, + { + ID: 3, + Type: sdk.Openstack, + Name: "my-model-3", + Group: &sdk.Group{ID: 1, Name: "mygroup"}, + ModelVirtualMachine: sdk.ModelVirtualMachine{Flavor: "b2-7"}, + }, + { + ID: 4, + Type: sdk.Openstack, + Name: "my-model-4", + Group: &sdk.Group{ID: 1, Name: "mygroup"}, + ModelVirtualMachine: sdk.ModelVirtualMachine{Flavor: "unknown"}, + }, + }, nil + }) - m = &sdk.Model{ - ID: 1, - Name: "my-model", - Group: &sdk.Group{ - ID: 1, - Name: "mygroup", - }, - ModelVirtualMachine: sdk.ModelVirtualMachine{ - Flavor: "b2-7", - }, + h.flavors = []flavors.Flavor{ + {Name: "b2-7", VCPUs: 2}, + {Name: "b2-30", VCPUs: 16}, + {Name: "b2-120", VCPUs: 32}, } - // model with a knowned flavor - canSpawn = h.CanSpawn(context.TODO(), m, 1, nil) - require.True(t, canSpawn) + // Only model that match a known flavor should be returned and sorted by CPUs asc + ms, err := h.WorkerModelsEnabled() + require.NoError(t, err) + require.Len(t, ms, 2) + assert.Equal(t, "my-model-3", ms[0].Name) + assert.Equal(t, "my-model-2", ms[1].Name) } diff --git a/engine/hatchery/openstack/spawn.go b/engine/hatchery/openstack/spawn.go index 1b7b379f78..833b1cd1b9 100644 --- a/engine/hatchery/openstack/spawn.go +++ b/engine/hatchery/openstack/spawn.go @@ -5,11 +5,13 @@ import ( "context" "encoding/base64" "fmt" + "math" "strings" "text/template" "time" "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" + "github.com/sirupsen/logrus" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/hatchery" @@ -29,21 +31,26 @@ func (h *HatcheryOpenstack) SpawnWorker(ctx context.Context, spawnArgs hatchery. return sdk.WithStack(fmt.Errorf("no job ID and no register")) } - if len(h.getServers(ctx)) == h.Configuration().Provision.MaxWorker { - log.Debug("MaxWorker limit (%d) reached", h.Configuration().Provision.MaxWorker) + if err := h.checkSpawnLimits(ctx, *spawnArgs.Model); err != nil { + isErrWithStack := sdk.IsErrorWithStack(err) + fields := logrus.Fields{} + if isErrWithStack { + fields["stack_trace"] = fmt.Sprintf("%+v", err) + } + log.InfoWithFields(ctx, fields, "%s", err) return nil } - // Get image ID - imageID, erri := h.imageID(ctx, spawnArgs.Model.ModelVirtualMachine.Image) - if erri != nil { - return erri + // Get flavor for target model + flavor, err := h.flavor(spawnArgs.Model.ModelVirtualMachine.Flavor) + if err != nil { + return err } - // Get flavor ID - flavorID, errf := h.flavorID(spawnArgs.Model.ModelVirtualMachine.Flavor) - if errf != nil { - return errf + // Get image ID + imageID, err := h.imageID(ctx, spawnArgs.Model.ModelVirtualMachine.Image) + if err != nil { + return err } var withExistingImage bool @@ -68,9 +75,9 @@ func (h *HatcheryOpenstack) SpawnWorker(ctx context.Context, spawnArgs hatchery. udata := spawnArgs.Model.ModelVirtualMachine.PreCmd + "\n" + spawnArgs.Model.ModelVirtualMachine.Cmd + "\n" + spawnArgs.Model.ModelVirtualMachine.PostCmd - tmpl, errt := template.New("udata").Parse(udata) - if errt != nil { - return errt + tmpl, err := template.New("udata").Parse(udata) + if err != nil { + return err } udataParam := sdk.WorkerArgs{ API: h.Configuration().API.HTTP.URL, @@ -124,7 +131,7 @@ func (h *HatcheryOpenstack) SpawnWorker(ctx context.Context, spawnArgs hatchery. networks := []servers.Network{{UUID: h.networkID, FixedIP: ip}} r := servers.Create(h.openstackClient, servers.CreateOpts{ Name: spawnArgs.WorkerName, - FlavorRef: flavorID, + FlavorRef: flavor.ID, ImageRef: imageID, Metadata: meta, UserData: []byte(udata64), @@ -134,13 +141,56 @@ func (h *HatcheryOpenstack) SpawnWorker(ctx context.Context, spawnArgs hatchery. server, err := r.Extract() if err != nil { if strings.Contains(err.Error(), "is already in use on instance") && try < maxTries { // Fixed IP address X.X.X.X is already in use on instance - log.Warning(ctx, "SpawnWorker> Unable to create server: name:%s flavor:%s image:%s metadata:%v networks:%s err:%v body:%s - Try %d/%d", spawnArgs.WorkerName, flavorID, imageID, meta, networks, err, r.Body, try, maxTries) + log.Warning(ctx, "SpawnWorker> Unable to create server: name:%s flavor:%s image:%s metadata:%v networks:%s err:%v body:%s - Try %d/%d", spawnArgs.WorkerName, flavor.ID, imageID, meta, networks, err, r.Body, try, maxTries) continue } - return fmt.Errorf("SpawnWorker> Unable to create server: name:%s flavor:%s image:%s metadata:%v networks:%s err:%v body:%s", spawnArgs.WorkerName, flavorID, imageID, meta, networks, err, r.Body) + return fmt.Errorf("SpawnWorker> Unable to create server: name:%s flavor:%s image:%s metadata:%v networks:%s err:%v body:%s", spawnArgs.WorkerName, flavor.ID, imageID, meta, networks, err, r.Body) } log.Debug("SpawnWorker> Created Server ID: %s", server.ID) break } return nil } + +func (h *HatcheryOpenstack) checkSpawnLimits(ctx context.Context, model sdk.Model) error { + existingServers := h.getServers(ctx) + if len(existingServers) >= h.Configuration().Provision.MaxWorker { + return sdk.WithStack(fmt.Errorf("MaxWorker limit (%d) reached", h.Configuration().Provision.MaxWorker)) + } + + // Get flavor for target model + flavor, err := h.flavor(model.ModelVirtualMachine.Flavor) + if err != nil { + return err + } + + // If a max CPUs count is set in configuration we will check that there are enough CPUs available to spawn the model + var totalCPUsUsed int + if h.Config.MaxCPUs > 0 { + for i := range existingServers { + totalCPUsUsed += existingServers[i].Flavor["vcpus"].(int) + } + if totalCPUsUsed+flavor.VCPUs > h.Config.MaxCPUs { + return sdk.WithStack(fmt.Errorf("MaxCPUs limit (%d) reached", h.Config.MaxCPUs)) + } + } + + // If the CountSmallerFlavorToKeep is set in config, we should check that there will be enough CPUs to spawn a smaller flavor after this one + if h.Config.MaxCPUs > 0 && h.Config.CountSmallerFlavorToKeep > 0 { + smallerFlavor := h.getSmallerFlavorThan(flavor) + // If same id, means that the requested flavor is the smallest one so we want to start it. + log.Debug("checkSpawnLimits> smaller flavor found for %s is %s", flavor.Name, smallerFlavor.Name) + if smallerFlavor.ID != flavor.ID { + minCPUsNeededToStart := flavor.VCPUs + h.Config.CountSmallerFlavorToKeep*smallerFlavor.VCPUs + countCPUsLeft := int(math.Max(.0, float64(h.Config.MaxCPUs-totalCPUsUsed))) // Set zero as min value in case that the limit changed and count of used greater than max count + if minCPUsNeededToStart > countCPUsLeft { + return sdk.WithStack(fmt.Errorf("CountSmallerFlavorToKeep limit reached, can't start model %s/%s with flavor %s that requires %d CPUs. Smaller flavor is %s and need %d CPUs. There are currently %d/%d left CPUs", + model.Group.Name, model.Name, flavor.Name, flavor.VCPUs, smallerFlavor.Name, smallerFlavor.VCPUs, countCPUsLeft, h.Config.MaxCPUs)) + } + log.Debug("checkSpawnLimits> %d/%d CPUs left is enougth to start model %s/%s with flavor %s that require %d CPUs", + countCPUsLeft, h.Config.MaxCPUs, model.Group.Name, model.Name, flavor.Name, flavor.VCPUs) + } + } + + return nil +} diff --git a/engine/hatchery/openstack/spawn_test.go b/engine/hatchery/openstack/spawn_test.go new file mode 100644 index 0000000000..41a11d4a27 --- /dev/null +++ b/engine/hatchery/openstack/spawn_test.go @@ -0,0 +1,163 @@ +package openstack + +import ( + "context" + "testing" + + "github.com/gophercloud/gophercloud/openstack/compute/v2/flavors" + "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/log" +) + +func TestHatcheryOpenstack_checkSpawnLimits_MaxWorker(t *testing.T) { + log.SetLogger(t) + + h := &HatcheryOpenstack{} + h.Config.Provision.MaxWorker = 3 + h.flavors = []flavors.Flavor{ + {Name: "my-flavor", VCPUs: 2}, + } + + m := sdk.Model{ + ID: 1, + Name: "my-model", + Group: &sdk.Group{ID: 1, Name: "my-group"}, + ModelVirtualMachine: sdk.ModelVirtualMachine{Flavor: "my-flavor"}, + } + + lservers.list = []servers.Server{ + {Flavor: map[string]interface{}{"vcpus": 8}}, + {Flavor: map[string]interface{}{"vcpus": 16}}, + } + + err := h.checkSpawnLimits(context.TODO(), m) + require.NoError(t, err) + + lservers.list = []servers.Server{ + {Flavor: map[string]interface{}{"vcpus": 8}}, + {Flavor: map[string]interface{}{"vcpus": 16}}, + {Flavor: map[string]interface{}{"vcpus": 32}}, + } + + err = h.checkSpawnLimits(context.TODO(), m) + require.Error(t, err) + assert.Contains(t, err.Error(), "MaxWorker") +} + +func TestHatcheryOpenstack_checkSpawnLimits_MaxCPUs(t *testing.T) { + log.SetLogger(t) + + h := &HatcheryOpenstack{} + h.Config.Provision.MaxWorker = 10 + h.Config.MaxCPUs = 6 + h.flavors = []flavors.Flavor{ + {Name: "my-flavor", VCPUs: 2}, + } + + m := sdk.Model{ + ID: 1, + Name: "my-model", + Group: &sdk.Group{ID: 1, Name: "my-group"}, + ModelVirtualMachine: sdk.ModelVirtualMachine{Flavor: "my-flavor"}, + } + + lservers.list = []servers.Server{ + {Flavor: map[string]interface{}{"vcpus": 2}}, + {Flavor: map[string]interface{}{"vcpus": 2}}, + } + + err := h.checkSpawnLimits(context.TODO(), m) + require.NoError(t, err) + + lservers.list = []servers.Server{ + {Flavor: map[string]interface{}{"vcpus": 2}}, + {Flavor: map[string]interface{}{"vcpus": 2}}, + {Flavor: map[string]interface{}{"vcpus": 2}}, + } + + err = h.checkSpawnLimits(context.TODO(), m) + require.Error(t, err) + assert.Contains(t, err.Error(), "MaxCPUs") +} + +func TestHatcheryOpenstack_checkSpawnLimits_CountSmallerFlavorToKeep(t *testing.T) { + log.SetLogger(t) + + h := &HatcheryOpenstack{} + h.Config.Provision.MaxWorker = 10 + h.Config.MaxCPUs = 30 + h.Config.CountSmallerFlavorToKeep = 2 + h.flavors = []flavors.Flavor{ + {ID: "1", Name: "b2-7", VCPUs: 2}, + {ID: "3", Name: "b2-30", VCPUs: 8}, + {ID: "2", Name: "b2-15", VCPUs: 4}, + } + + m1 := sdk.Model{ + ID: 1, + Name: "my-model-1", + Group: &sdk.Group{ID: 1, Name: "my-group"}, + ModelVirtualMachine: sdk.ModelVirtualMachine{Flavor: "b2-7"}, + } + m2 := sdk.Model{ + ID: 2, + Name: "my-model-2", + Group: &sdk.Group{ID: 1, Name: "my-group"}, + ModelVirtualMachine: sdk.ModelVirtualMachine{Flavor: "b2-15"}, + } + m3 := sdk.Model{ + ID: 3, + Name: "my-model-3", + Group: &sdk.Group{ID: 1, Name: "my-group"}, + ModelVirtualMachine: sdk.ModelVirtualMachine{Flavor: "b2-30"}, + } + + lservers.list = []servers.Server{ + {Flavor: map[string]interface{}{"vcpus": 8}}, + } + + err := h.checkSpawnLimits(context.TODO(), m3) + require.NoError(t, err, "22 CPUs left (30-8) should be enough to start 8 CPUs flavor (8+4*2=16)") + + lservers.list = []servers.Server{ + {Flavor: map[string]interface{}{"vcpus": 8}}, + {Flavor: map[string]interface{}{"vcpus": 8}}, + } + + err = h.checkSpawnLimits(context.TODO(), m3) + require.Error(t, err, "14 CPUs left (30-8*2) should be not be enough to start 8 CPUs flavor (8+4*2=16)") + assert.Contains(t, err.Error(), "CountSmallerFlavorToKeep") + + err = h.checkSpawnLimits(context.TODO(), m2) + require.NoError(t, err, "14 CPUs left (30-8*2) should be enough to start 4 CPUs flavor (4+2*2=8)") + + lservers.list = []servers.Server{ + {Flavor: map[string]interface{}{"vcpus": 8}}, + {Flavor: map[string]interface{}{"vcpus": 8}}, + {Flavor: map[string]interface{}{"vcpus": 4}}, + {Flavor: map[string]interface{}{"vcpus": 4}}, + {Flavor: map[string]interface{}{"vcpus": 2}}, + {Flavor: map[string]interface{}{"vcpus": 2}}, + } + + err = h.checkSpawnLimits(context.TODO(), m1) + require.NoError(t, err, "2 CPUs left (30-8*2-4*2-2*2) should be enough to start the smallest flavor with 2 CPUs") + + lservers.list = []servers.Server{ + {Flavor: map[string]interface{}{"vcpus": 8}}, + {Flavor: map[string]interface{}{"vcpus": 8}}, + {Flavor: map[string]interface{}{"vcpus": 4}}, + {Flavor: map[string]interface{}{"vcpus": 4}}, + {Flavor: map[string]interface{}{"vcpus": 2}}, + {Flavor: map[string]interface{}{"vcpus": 2}}, + {Flavor: map[string]interface{}{"vcpus": 2}}, + } + + err = h.checkSpawnLimits(context.TODO(), m1) + require.Error(t, err, "0 CPUs left to start new flavor") + assert.Contains(t, err.Error(), "CountSmallerFlavorToKeep") +} diff --git a/engine/hatchery/openstack/types.go b/engine/hatchery/openstack/types.go index 18601840c3..c64d3e3d28 100644 --- a/engine/hatchery/openstack/types.go +++ b/engine/hatchery/openstack/types.go @@ -47,6 +47,17 @@ type HatcheryConfiguration struct { // CreateImageTimeout max wait for create an openstack image (in seconds) CreateImageTimeout int `mapstructure:"createImageTimeout" toml:"createImageTimeout" default:"180" commented:"false" comment:"max wait for create an openstack image (in seconds)" json:"createImageTimeout"` + + // AllowedFlavors if not empty the hatchery will be able to start a model only if its flavor is listed in allowed flavors + AllowedFlavors []string `mapstructure:"allowedFlavors" toml:"allowedFlavors" default:"" commented:"true" comment:"List of allowed flavors that can be used by the hatchery." json:"allowedFlavors"` + + // MaxCPUs if set the hatchery will stop starting new models if its flavors requires more CPUs than availables + MaxCPUs int `mapstructure:"maxCpus" toml:"maxCpus" default:"" commented:"true" comment:"Maximum count of CPUs that can be used at a same time by the hatchery." json:"maxCpus"` + + // CountSmallerFlavorToKeep define the count of smaller flavors that the hatchery should be able to boot when booting a larger flavor. + // This count will prevent big flavor to take all the CPUs available for the hatchery and will keep some available for smaller flavors. + // Ex: if two flavors are available with 8 and 4 cpus and count to keep equals 2 the hatchery will need 8+4*2=16cpus available to start a 8cpus flavor. + CountSmallerFlavorToKeep int `mapstructure:"countSmallerFlavorToKeep" toml:"countSmallerFlavorToKeep" default:"" commented:"true" comment:"Count of smaller flavors that the hatchery should be able to boot when booting a larger flavor." json:"countSmallerFlavorToKeep"` } // HatcheryOpenstack spawns instances of worker model with type 'ISO' diff --git a/go.mod b/go.mod index b2ce8943c6..93e337ceff 100644 --- a/go.mod +++ b/go.mod @@ -211,7 +211,7 @@ require ( gopkg.in/stomp.v1 v1.0.1 // indirect gopkg.in/vmihailenco/msgpack.v2 v2.9.1 // indirect gopkg.in/yaml.v2 v2.2.4 - gotest.tools v2.1.0+incompatible // indirect + gotest.tools v2.1.0+incompatible k8s.io/api v0.0.0-20181204000039-89a74a8d264d k8s.io/apimachinery v0.0.0-20190223094358-dcb391cde5ca k8s.io/client-go v10.0.0+incompatible