Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(hatchery:k8s): set memory and CPU limits for workers and services #6187

Merged
merged 5 commits into from
Jun 7, 2022
Merged

feat(hatchery:k8s): set memory and CPU limits for workers and services #6187

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6f05542
feat(hatchery:k8s): set memory and CPU limits for workers and services
richardlt Jun 2, 2022
a0eab2f
fix: apply default values
richardlt Jun 2, 2022
e931424
fix: memory limit in MB
richardlt Jun 7, 2022
38e322a
fix: unit test
richardlt Jun 7, 2022
17c002c
feat: add ephemeral storage limit
richardlt Jun 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 55 additions & 11 deletions engine/hatchery/kubernetes/kubernetes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,15 @@ func (h *HatcheryKubernetes) SpawnWorker(ctx context.Context, spawnArgs hatchery
logJob = fmt.Sprintf("for workflow job %d,", spawnArgs.JobID)
}

cpu := h.Config.DefaultCPU
if cpu == "" {
cpu = "500m"
}

memory := int64(h.Config.DefaultMemory)
if memory == 0 {
memory = 1024
}
for _, r := range spawnArgs.Requirements {
if r.Type == sdk.MemoryRequirement {
var err error
Expand All @@ -192,9 +200,15 @@ func (h *HatcheryKubernetes) SpawnWorker(ctx context.Context, spawnArgs hatchery
log.Warn(ctx, "spawnKubernetesDockerWorker> %s unable to parse memory requirement %d: %v", logJob, memory, err)
return err
}
break
}
}

ephemeralStorage := h.Config.DefaultEphemeralStorage
if ephemeralStorage == "" {
ephemeralStorage = "1Gi"
}

workerConfig := h.GenerateWorkerConfig(ctx, h, spawnArgs)
udataParam := struct {
API string
Expand Down Expand Up @@ -283,7 +297,14 @@ func (h *HatcheryKubernetes) SpawnWorker(ctx context.Context, spawnArgs hatchery
Args: []string{cmd},
Resources: apiv1.ResourceRequirements{
Requests: apiv1.ResourceList{
apiv1.ResourceMemory: resource.MustParse(fmt.Sprintf("%d", memory)),
apiv1.ResourceCPU: resource.MustParse(cpu),
apiv1.ResourceMemory: *resource.NewScaledQuantity(memory, resource.Mega),
apiv1.ResourceEphemeralStorage: resource.MustParse(ephemeralStorage),
},
Limits: apiv1.ResourceList{
apiv1.ResourceCPU: resource.MustParse(cpu),
apiv1.ResourceMemory: *resource.NewScaledQuantity(memory, resource.Mega),
apiv1.ResourceEphemeralStorage: resource.MustParse(ephemeralStorage),
},
},
},
Expand Down Expand Up @@ -313,28 +334,51 @@ func (h *HatcheryKubernetes) SpawnWorker(ctx context.Context, spawnArgs hatchery
podSchema.Spec.HostAliases[0].Hostnames[0] = "worker"
}

serviceCPU := h.Config.DefaultServiceCPU
if serviceCPU == "" {
serviceCPU = "256m"
}

serviceMemory := int64(h.Config.DefaultServiceMemory)
if serviceMemory == 0 {
serviceMemory = 512
}

serviceEphemeralStorage := h.Config.DefaultServiceEphemeralStorage
if serviceEphemeralStorage == "" {
serviceEphemeralStorage = "512Mi"
}

for i, serv := range services {
//name= <alias> => the name of the host put in /etc/hosts of the worker
//value= "postgres:latest env_1=blabla env_2=blabla"" => we can add env variables in requirement name
img, envm := hatchery.ParseRequirementModel(serv.Value)

servContainer := apiv1.Container{
Name: fmt.Sprintf("service-%d-%s", serv.ID, strings.ToLower(serv.Name)),
Image: img,
}

if sm, ok := envm["CDS_SERVICE_MEMORY"]; ok {
mq, err := resource.ParseQuantity(sm)
var err error
serviceMemory, err = strconv.ParseInt(sm, 10, 64)
if err != nil {
log.Warn(ctx, "hatchery> kubernetes> SpawnWorker> Unable to parse CDS_SERVICE_MEMORY value '%s': %s", sm, err)
continue
}
servContainer.Resources = apiv1.ResourceRequirements{
delete(envm, "CDS_SERVICE_MEMORY")
}

servContainer := apiv1.Container{
Name: fmt.Sprintf("service-%d-%s", serv.ID, strings.ToLower(serv.Name)),
Image: img,
Resources: apiv1.ResourceRequirements{
Requests: apiv1.ResourceList{
apiv1.ResourceMemory: mq,
apiv1.ResourceCPU: resource.MustParse(serviceCPU),
apiv1.ResourceMemory: *resource.NewScaledQuantity(serviceMemory, resource.Mega),
apiv1.ResourceEphemeralStorage: resource.MustParse(serviceEphemeralStorage),
},
}
delete(envm, "CDS_SERVICE_MEMORY")
Limits: apiv1.ResourceList{
apiv1.ResourceCPU: resource.MustParse(serviceCPU),
apiv1.ResourceMemory: *resource.NewScaledQuantity(serviceMemory, resource.Mega),
apiv1.ResourceEphemeralStorage: resource.MustParse(serviceEphemeralStorage),
},
},
}

if sa, ok := envm["CDS_SERVICE_ARGS"]; ok {
Expand Down
2 changes: 1 addition & 1 deletion engine/hatchery/kubernetes/kubernetes_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ func TestHatcheryKubernetes_Status(t *testing.T) {

require.Equal(t, 2, len(podRequest.Spec.Containers))
require.Equal(t, "my-worker", podRequest.Spec.Containers[0].Name)
require.Equal(t, int64(4096), podRequest.Spec.Containers[0].Resources.Requests.Memory().Value())
require.Equal(t, int64(4096000000), podRequest.Spec.Containers[0].Resources.Requests.Memory().Value())
var foundEnv, foundSecret bool
for _, env := range podRequest.Spec.Containers[0].Env {
if env.Name == "PROVISION_ENV" && env.Value == "MYVALUE" {
Expand Down
10 changes: 10 additions & 0 deletions engine/hatchery/kubernetes/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,18 @@ var containerServiceNameRegexp = regexp.MustCompile(`service-([0-9]+)-(.*)`)
// HatcheryConfiguration is the configuration for local hatchery
type HatcheryConfiguration struct {
service.HatcheryCommonConfiguration `mapstructure:"commonConfiguration" toml:"commonConfiguration" json:"commonConfiguration"`
// DefaultCPU Worker default CPU
DefaultCPU string `mapstructure:"defaultCPU" toml:"defaultCPU" default:"500m" commented:"false" comment:"Worker default CPU" json:"defaultCPU"`
// DefaultMemory Worker default memory
DefaultMemory int `mapstructure:"defaultMemory" toml:"defaultMemory" default:"1024" commented:"false" comment:"Worker default memory in Mo" json:"defaultMemory"`
// DefaultEphemeralStorage Worker default ephemeral storage size
DefaultEphemeralStorage string `mapstructure:"defaultEphemeralStorage" toml:"defaultEphemeralStorage" default:"1Gi" commented:"false" comment:"Worker default ephemeral storage size" json:"defaultEphemeralStorage"`
// DefaultServiceCPU Service default CPU
DefaultServiceCPU string `mapstructure:"defaultServiceCPU" toml:"defaultServiceCPU" default:"256m" commented:"false" comment:"Service default CPU" json:"defaultServiceCPU"`
// DefaultServiceMemory Service default memory
DefaultServiceMemory int `mapstructure:"defaultServiceMemory" toml:"defaultServiceMemory" default:"512" commented:"false" comment:"Service default memory in Mo" json:"defaultServiceMemory"`
// DefaultServiceEphemeralStorage Service default ephemeral storage size
DefaultServiceEphemeralStorage string `mapstructure:"defaultServiceEphemeralStorage" toml:"defaultServiceEphemeralStorage" default:"512Mi" commented:"false" comment:"Service default ephemeral storage size" json:"defaultServiceEphemeralStorage"`
// Namespace is the kubernetes namespace in which workers are spawned"
Namespace string `mapstructure:"namespace" toml:"namespace" default:"cds" commented:"false" comment:"Kubernetes namespace in which workers are spawned" json:"namespace"`
// KubernetesMasterURL Address of kubernetes master
Expand Down