feat(api): hatchery config property IgnoreJobWithNoRegion on consumer (…

…#6162)

cli/cdsctl/consumer.go

@@ -194,6 +194,11 @@ func authConsumerNewRun(v cli.Values) error {
 		svcRegion = cli.AskValue("Service region")
 	}
 
+	var svcIgnoreJobWithNoRegion = v.GetBool("service-ignore-job-with-no-region")
+	if !svcIgnoreJobWithNoRegion && !v.GetBool("no-interactive") {
+		svcIgnoreJobWithNoRegion = cli.AskConfirm("Service ignore job with no region")
+	}
+
 	var consumer = sdk.AuthConsumer{
 		Name:            name,
 		Description:     description,
@@ -211,6 +216,9 @@ func authConsumerNewRun(v cli.Values) error {
 	if svcRegion != "" {
 		consumer.ServiceRegion = &svcRegion
 	}
+	if svcIgnoreJobWithNoRegion {
+		consumer.ServiceIgnoreJobWithNoRegion = &svcIgnoreJobWithNoRegion
+	}
 
 	res, err := client.AuthConsumerCreateForUser(username, consumer)
 	if err != nil {

engine/api/auth_builtin.go

@@ -77,11 +77,24 @@ func (api *API) postAuthBuiltinSigninHandler() service.Handler {
 			if consumer.ServiceType != nil && *consumer.ServiceType != srv.Type {
 				return sdk.NewErrorFrom(sdk.ErrForbidden, "service type %q doesn't match with consumer %q", srv.Type, *consumer.ServiceType)
 			}
-			if consumer.ServiceRegion != nil && *consumer.ServiceRegion != *srv.Region {
-				return sdk.NewErrorFrom(sdk.ErrForbidden, "service region %q doesn't match with consumer %q", srv.Type, *consumer.ServiceRegion)
+			if consumer.ServiceRegion != nil {
+				if srv.Region == nil {
+					return sdk.NewErrorFrom(sdk.ErrForbidden, "unknown service region doesn't match with consumer %q", *consumer.ServiceRegion)
+				}
+				if *consumer.ServiceRegion != *srv.Region {
+					return sdk.NewErrorFrom(sdk.ErrForbidden, "service region %q doesn't match with consumer %q", *srv.Region, *consumer.ServiceRegion)
+				}
+			}
+			if consumer.ServiceIgnoreJobWithNoRegion != nil {
+				if srv.IgnoreJobWithNoRegion == nil {
+					return sdk.NewErrorFrom(sdk.ErrForbidden, "unknown service ignore job with no region value doesn't match with consumer '%t'", *consumer.ServiceIgnoreJobWithNoRegion)
+				}
+				if *consumer.ServiceIgnoreJobWithNoRegion != *srv.IgnoreJobWithNoRegion {
+					return sdk.NewErrorFrom(sdk.ErrForbidden, "service ignore job with no region flag value '%t' doesn't match with consumer '%t'", *srv.IgnoreJobWithNoRegion, *consumer.ServiceIgnoreJobWithNoRegion)
+				}
 			}
 		} else {
-			if consumer.ServiceName != nil || consumer.ServiceType != nil || consumer.ServiceRegion != nil {
+			if consumer.ServiceName != nil || consumer.ServiceType != nil || consumer.ServiceRegion != nil || consumer.ServiceIgnoreJobWithNoRegion != nil {
 				return sdk.NewErrorFrom(sdk.ErrForbidden, "signing request doesn't match with consumer %q service definition. missing service payload", consumer.Name)
 			}
 		}

engine/api/auth_consumer.go

@@ -85,14 +85,15 @@ func (api *API) postConsumerByUserHandler() service.Handler {
 
 		// Create the new built in consumer from request data
 		consumerOpts := builtin.NewConsumerOptions{
-			Name:          reqData.Name,
-			Description:   reqData.Description,
-			Duration:      reqData.ValidityPeriods.Latest().Duration,
-			GroupIDs:      reqData.GroupIDs,
-			Scopes:        reqData.ScopeDetails,
-			ServiceName:   reqData.ServiceName,
-			ServiceType:   reqData.ServiceType,
-			ServiceRegion: reqData.ServiceRegion,
+			Name:                         reqData.Name,
+			Description:                  reqData.Description,
+			Duration:                     reqData.ValidityPeriods.Latest().Duration,
+			GroupIDs:                     reqData.GroupIDs,
+			Scopes:                       reqData.ScopeDetails,
+			ServiceName:                  reqData.ServiceName,
+			ServiceType:                  reqData.ServiceType,
+			ServiceRegion:                reqData.ServiceRegion,
+			ServiceIgnoreJobWithNoRegion: reqData.ServiceIgnoreJobWithNoRegion,
 		}
 		newConsumer, token, err := builtin.NewConsumer(ctx, tx, consumerOpts, consumer)
 		if err != nil {

engine/api/authentication/builtin/builtin.go

@@ -66,14 +66,15 @@ func (d AuthDriver) CheckSigninRequest(req sdk.AuthConsumerSigninRequest) error
 // NewConsumer returns a new builtin consumer for given data.
 // The parent consumer should be given with all data loaded including the authentified user.
 type NewConsumerOptions struct {
-	Name          string
-	Description   string
-	Duration      time.Duration
-	GroupIDs      []int64
-	Scopes        sdk.AuthConsumerScopeDetails
-	ServiceName   *string
-	ServiceType   *string
-	ServiceRegion *string
+	Name                         string
+	Description                  string
+	Duration                     time.Duration
+	GroupIDs                     []int64
+	Scopes                       sdk.AuthConsumerScopeDetails
+	ServiceName                  *string
+	ServiceType                  *string
+	ServiceRegion                *string
+	ServiceIgnoreJobWithNoRegion *bool
 }
 
 func NewConsumer(ctx context.Context, db gorpmapper.SqlExecutorWithTx, opts NewConsumerOptions, parentConsumer *sdk.AuthConsumer) (*sdk.AuthConsumer, string, error) {
@@ -102,18 +103,19 @@ func NewConsumer(ctx context.Context, db gorpmapper.SqlExecutorWithTx, opts NewC
 	}
 
 	c := sdk.AuthConsumer{
-		Name:               opts.Name,
-		Description:        opts.Description,
-		ParentID:           &parentConsumer.ID,
-		AuthentifiedUserID: parentConsumer.AuthentifiedUserID,
-		Type:               sdk.ConsumerBuiltin,
-		Data:               map[string]string{},
-		GroupIDs:           opts.GroupIDs,
-		ScopeDetails:       opts.Scopes,
-		ValidityPeriods:    sdk.NewAuthConsumerValidityPeriod(time.Now(), opts.Duration),
-		ServiceName:        opts.ServiceName,
-		ServiceType:        opts.ServiceType,
-		ServiceRegion:      opts.ServiceRegion,
+		Name:                         opts.Name,
+		Description:                  opts.Description,
+		ParentID:                     &parentConsumer.ID,
+		AuthentifiedUserID:           parentConsumer.AuthentifiedUserID,
+		Type:                         sdk.ConsumerBuiltin,
+		Data:                         map[string]string{},
+		GroupIDs:                     opts.GroupIDs,
+		ScopeDetails:                 opts.Scopes,
+		ValidityPeriods:              sdk.NewAuthConsumerValidityPeriod(time.Now(), opts.Duration),
+		ServiceName:                  opts.ServiceName,
+		ServiceType:                  opts.ServiceType,
+		ServiceRegion:                opts.ServiceRegion,
+		ServiceIgnoreJobWithNoRegion: opts.ServiceIgnoreJobWithNoRegion,
 	}
 
 	if err := authentication.InsertConsumer(ctx, db, &c); err != nil {

engine/api/authentication/gorp_model.go

@@ -12,9 +12,9 @@ type authConsumer struct {
 }
 
 func (c authConsumer) Canonical() gorpmapper.CanonicalForms {
-	_ = []interface{}{c.ID, c.AuthentifiedUserID, c.Type, c.Data, c.Created, c.GroupIDs, c.ScopeDetails, c.Disabled, c.ServiceName, c.ServiceType, c.ServiceRegion} // Checks that fields exists at compilation
+	_ = []interface{}{c.ID, c.AuthentifiedUserID, c.Type, c.Data, c.Created, c.GroupIDs, c.ScopeDetails, c.Disabled, c.ServiceName, c.ServiceType, c.ServiceRegion, c.ServiceIgnoreJobWithNoRegion} // Checks that fields exists at compilation
 	return []gorpmapper.CanonicalForm{
-		//"{{.ID}}{{.AuthentifiedUserID}}{{print .Type}}{{print .Data}}{{printDate .Created}}{{print .GroupIDs}}{{print .ScopeDetails}}{{print .Disabled}}{{.ServiceName}}{{.ServiceType}}{{.ServiceRegion}}",
+		//"{{.ID}}{{.AuthentifiedUserID}}{{print .Type}}{{print .Data}}{{printDate .Created}}{{print .GroupIDs}}{{print .ScopeDetails}}{{print .Disabled}}{{.ServiceName}}{{.ServiceType}}{{.ServiceRegion}}{{.ServiceIgnoreJobWithNoRegion}}",
 		"{{.ID}}{{.AuthentifiedUserID}}{{print .Type}}{{print .Data}}{{printDate .Created}}{{print .GroupIDs}}{{print .ScopeDetails}}{{print .Disabled}}",
 	}
 }

engine/api/services/gorp_model.go

@@ -12,7 +12,9 @@ type service struct {
 }
 
 func (s service) Canonical() gorpmapper.CanonicalForms {
+	_ = []interface{}{s.ID, s.Name, s.Type, s.Region, s.IgnoreJobWithNoRegion} // Checks that fields exists at compilation
 	return []gorpmapper.CanonicalForm{
+		//"{{.ID}}{{.Name}}{{.Type}}{{.Region}}{{.IgnoreJobWithNoRegion}}",
 		"{{.ID}}{{.Name}}{{.Type}}",
 	}
 }

engine/api/worker/registration.go

@@ -111,12 +111,15 @@ func RegisterWorker(ctx context.Context, db gorpmapper.SqlExecutorWithTx, store
 			}
 		}
 
-		// Check additional information based on the consumer if a region is set, allows to register only job with same region.
-		// TODO add another information on the consumer to allow job with a specific region requirement or no region (runNodeJob.Region == nil).
+		// Check additional information based on the consumer if a region is set.
+		// Allows to register only job with same region or job without region if ServiceIgnoreJobWithNoRegion is not true.
 		if hatcheryConsumer.ServiceRegion != nil && *hatcheryConsumer.ServiceRegion != "" {
-			canTakeJobWithRegion := runNodeJob.Region != nil && *runNodeJob.Region == *hatcheryConsumer.ServiceRegion
-			if !canTakeJobWithRegion {
-				return nil, sdk.WrapError(sdk.ErrForbidden, "hatchery can't register job with id %q for region %s", spawnArgs.JobID, *runNodeJob.Region)
+			if runNodeJob.Region == nil {
+				if hatcheryConsumer.ServiceIgnoreJobWithNoRegion != nil && *hatcheryConsumer.ServiceIgnoreJobWithNoRegion {
+					return nil, sdk.WrapError(sdk.ErrForbidden, "hatchery can't register job with id %d without region requirement", spawnArgs.JobID)
+				}
+			} else if *runNodeJob.Region != *hatcheryConsumer.ServiceRegion {
+				return nil, sdk.WrapError(sdk.ErrForbidden, "hatchery can't register job with id %d for region %s", spawnArgs.JobID, *runNodeJob.Region)
 			}
 		}
 	}

engine/gorpmapper/signature_test.go

@@ -0,0 +1,78 @@
+package gorpmapper
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/ovh/cds/sdk"
+	"github.com/stretchr/testify/require"
+)
+
+type testService struct {
+	sdk.Service
+	SignedEntity
+}
+
+func (s testService) Canonical() CanonicalForms {
+	return []CanonicalForm{
+		"{{.ID}}{{.Name}}{{.Type}}{{.Region}}{{.IgnoreJobWithNoRegion}}",
+	}
+}
+
+func Test_CanonicalFormWithPointer(t *testing.T) {
+	m := New()
+	m.Register(m.NewTableMapping(testService{}, "service", false, "id"))
+
+	region := "test"
+
+	cases := []struct {
+		name string
+		s    testService
+		res  string
+	}{
+		{
+			name: "Service with empty values",
+			s: testService{
+				Service: sdk.Service{
+					CanonicalService: sdk.CanonicalService{
+						ID:                    123,
+						Name:                  "my-service",
+						Type:                  sdk.TypeHatchery,
+						Region:                nil,
+						IgnoreJobWithNoRegion: nil,
+					},
+				},
+			},
+			res: "123my-servicehatchery<nil><nil>",
+		},
+		{
+			name: "Service without empty values",
+			s: testService{
+				Service: sdk.Service{
+					CanonicalService: sdk.CanonicalService{
+						ID:                    123,
+						Name:                  "my-service",
+						Type:                  sdk.TypeHatchery,
+						Region:                &region,
+						IgnoreJobWithNoRegion: &sdk.True,
+					},
+				},
+			},
+			res: "123my-servicehatcherytesttrue",
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			f, _ := c.s.Canonical().Latest()
+			tmpl, err := m.getCanonicalTemplate(f)
+			require.NoError(t, err)
+			require.NotNil(t, tmpl)
+
+			var clearContent = new(bytes.Buffer)
+			require.NoError(t, tmpl.Execute(clearContent, c.s))
+
+			require.Equal(t, c.res, clearContent.String())
+		})
+	}
+}

engine/hatchery/kubernetes/kubernetes.go

@@ -94,6 +94,7 @@ func (h *HatcheryKubernetes) ApplyConfiguration(cfg interface{}) error {
 		return fmt.Errorf("unable to parse RSA private Key: %v", err)
 	}
 	h.Common.Common.Region = h.Config.Provision.Region
+	h.Common.Common.IgnoreJobWithNoRegion = h.Config.Provision.IgnoreJobWithNoRegion
 
 	return nil
 }

engine/hatchery/local/local.go

@@ -77,6 +77,7 @@ func (h *HatcheryLocal) ApplyConfiguration(cfg interface{}) error {
 		return fmt.Errorf("unable to parse RSA private Key: %v", err)
 	}
 	h.Common.Common.Region = h.Config.Provision.Region
+	h.Common.Common.IgnoreJobWithNoRegion = h.Config.Provision.IgnoreJobWithNoRegion
 
 	return nil
 }

engine/hatchery/marathon/marathon.go

@@ -81,6 +81,7 @@ func (h *HatcheryMarathon) ApplyConfiguration(cfg interface{}) error {
 		return fmt.Errorf("unable to parse RSA private Key: %v", err)
 	}
 	h.Common.Common.Region = h.Config.Provision.Region
+	h.Common.Common.IgnoreJobWithNoRegion = h.Config.Provision.IgnoreJobWithNoRegion
 
 	return nil
 }

engine/hatchery/openstack/openstack.go

@@ -85,6 +85,7 @@ func (h *HatcheryOpenstack) ApplyConfiguration(cfg interface{}) error {
 		return fmt.Errorf("unable to parse RSA private Key: %v", err)
 	}
 	h.Common.Common.Region = h.Config.Provision.Region
+	h.Common.Common.IgnoreJobWithNoRegion = h.Config.Provision.IgnoreJobWithNoRegion
 
 	return nil
 }

engine/hatchery/swarm/swarm_conf.go

@@ -55,6 +55,7 @@ func (h *HatcherySwarm) ApplyConfiguration(cfg interface{}) error {
 		return fmt.Errorf("unable to parse RSA private Key: %v", err)
 	}
 	h.Common.Common.Region = h.Config.Provision.Region
+	h.Common.Common.IgnoreJobWithNoRegion = h.Config.Provision.IgnoreJobWithNoRegion
 
 	return nil
 }

engine/hatchery/vsphere/hatchery.go

@@ -74,6 +74,7 @@ func (h *HatcheryVSphere) ApplyConfiguration(cfg interface{}) error {
 		return sdk.WithStack(fmt.Errorf("unable to parse RSA private Key: %v", err))
 	}
 	h.Common.Common.Region = h.Config.Provision.Region
+	h.Common.Common.IgnoreJobWithNoRegion = h.Config.Provision.IgnoreJobWithNoRegion
 
 	if h.Config.WorkerTTL == 0 {
 		h.Config.WorkerTTL = 120

engine/service/service.go

@@ -111,6 +111,9 @@ func (c *Common) Signin(ctx context.Context, cdsclientConfig cdsclient.ServiceCo
 	if c.Region != "" {
 		registerPayload.CanonicalService.Region = &c.Region
 	}
+	if c.IgnoreJobWithNoRegion {
+		registerPayload.CanonicalService.IgnoreJobWithNoRegion = &c.IgnoreJobWithNoRegion
+	}
 
 	initClient := func(ctx context.Context) error {
 		var err error

engine/service/types.go

@@ -106,21 +106,22 @@ func (hcc HatcheryCommonConfiguration) Check() error {
 
 // Common is the struct representing a CDS µService
 type Common struct {
-	Client               cdsclient.Interface
-	APIPublicKey         []byte
-	ParsedAPIPublicKey   *rsa.PublicKey
-	StartupTime          time.Time
-	HTTPURL              string
-	MaxHeartbeatFailures int
-	ServiceName          string
-	ServiceType          string
-	ServiceInstance      *sdk.Service
-	PrivateKey           *rsa.PrivateKey
-	Signer               jose.Signer
-	CDNLogsURL           string
-	ServiceLogger        *logrus.Logger
-	GoRoutines           *sdk.GoRoutines
-	Region               string
+	Client                cdsclient.Interface
+	APIPublicKey          []byte
+	ParsedAPIPublicKey    *rsa.PublicKey
+	StartupTime           time.Time
+	HTTPURL               string
+	MaxHeartbeatFailures  int
+	ServiceName           string
+	ServiceType           string
+	ServiceInstance       *sdk.Service
+	PrivateKey            *rsa.PrivateKey
+	Signer                jose.Signer
+	CDNLogsURL            string
+	ServiceLogger         *logrus.Logger
+	GoRoutines            *sdk.GoRoutines
+	Region                string
+	IgnoreJobWithNoRegion bool
 }
 
 // Service is the interface for a engine service

engine/sql/api/241_add_hatchery_field_on_consumer.sql

@@ -0,0 +1,9 @@
+-- +migrate Up
+ALTER TABLE "auth_consumer" ADD COLUMN IF NOT EXISTS "service_ignore_job_with_no_region" BOOLEAN;
+
+ALTER TABLE "service" ADD COLUMN IF NOT EXISTS "ignore_job_with_no_region" BOOLEAN;
+
+-- +migrate Down
+ALTER TABLE "auth_consumer" DROP COLUMN IF EXISTS "service_ignore_job_with_no_region";
+
+ALTER TABLE "service" DROP COLUMN IF EXISTS "ignore_job_with_no_region";

sdk/services.go

@@ -21,14 +21,15 @@ const (
 )
 
 type CanonicalService struct {
-	ID         int64         `json:"id" db:"id" mapstructure:"id"`
-	Name       string        `json:"name" db:"name" cli:"name,key" mapstructure:"name"`
-	ConsumerID *string       `json:"-" db:"auth_consumer_id" mapstructure:"-"`
-	Type       string        `json:"type" db:"type" cli:"type" mapstructure:"type"`
-	HTTPURL    string        `json:"http_url" db:"http_url" cli:"url" mapstructure:"http_url"`
-	Config     ServiceConfig `json:"config" db:"config" cli:"-" mapstructure:"config"`
-	PublicKey  []byte        `json:"public_key" db:"public_key" mapstructure:"public_key"`
-	Region     *string       `json:"region" db:"region" mapstructure:"region"`
+	ID                    int64         `json:"id" db:"id" mapstructure:"id"`
+	Name                  string        `json:"name" db:"name" cli:"name,key" mapstructure:"name"`
+	ConsumerID            *string       `json:"-" db:"auth_consumer_id" mapstructure:"-"`
+	Type                  string        `json:"type" db:"type" cli:"type" mapstructure:"type"`
+	HTTPURL               string        `json:"http_url" db:"http_url" cli:"url" mapstructure:"http_url"`
+	Config                ServiceConfig `json:"config" db:"config" cli:"-" mapstructure:"config"`
+	PublicKey             []byte        `json:"public_key" db:"public_key" mapstructure:"public_key"`
+	Region                *string       `json:"region" db:"region" mapstructure:"region"`
+	IgnoreJobWithNoRegion *bool         `json:"ignore_job_with_no_region" db:"ignore_job_with_no_region" mapstructure:"ignore_job_with_no_region"`
 }
 
 // Service is a µService registered on CDS API.