Issue 354: Segment store pods are not coming up with external access

go.sum

@@ -194,6 +194,7 @@ github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
 github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
@@ -256,10 +257,12 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q=
 github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/operator-framework/operator-sdk v0.4.0 h1:5LKhvld7AZZaFkbA5Uvt3y/BSjXgtmjNrM1mJHY/+CI=
 github.com/operator-framework/operator-sdk v0.4.0/go.mod h1:iVyukRkam5JZa8AnjYf+/G3rk7JI1+M6GsU0sq0B9NA=
@@ -412,12 +415,14 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/mail.v2 v2.0.0-20180731213649-a0242b2233b4/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

pkg/controller/pravega/pravega_segmentstore.go

@@ -433,7 +433,7 @@ func MakeSegmentStoreExternalServices(pravegaCluster *api.PravegaCluster) []*cor
 	services := make([]*corev1.Service, pravegaCluster.Spec.Pravega.SegmentStoreReplicas)
 
 	for i := int32(0); i < pravegaCluster.Spec.Pravega.SegmentStoreReplicas; i++ {
-		ssPodName := util.ServiceNameForSegmentStore(pravegaCluster.Name, i)
+		ssPodName := util.ServiceNameForSegmentStore(pravegaCluster, i)
 		annotationMap := pravegaCluster.Spec.Pravega.SegmentStoreServiceAnnotations
 		annotationValue := generateDNSAnnotationForSvc(pravegaCluster.Spec.ExternalAccess.DomainName, ssPodName)
 

pkg/controller/pravegacluster/pravegacluster_controller_test.go

@@ -186,7 +186,7 @@ var _ = Describe("PravegaCluster Controller", func() {
 						// should not be any client service
 						foundSvc := &corev1.Service{}
 						nn := types.NamespacedName{
-							Name:      util.ServiceNameForSegmentStore(p.Name, 0),
+							Name:      util.ServiceNameForSegmentStore(p, 0),
 							Namespace: Namespace,
 						}
 						err = client.Get(context.TODO(), nn, foundSvc)
@@ -432,21 +432,21 @@ var _ = Describe("PravegaCluster Controller", func() {
 				BeforeEach(func() {
 					foundSegmentStoreSvc1 = &corev1.Service{}
 					nn1 := types.NamespacedName{
-						Name:      util.ServiceNameForSegmentStore(p.Name, 0),
+						Name:      util.ServiceNameForSegmentStore(p, 0),
 						Namespace: Namespace,
 					}
 					err = client.Get(context.TODO(), nn1, foundSegmentStoreSvc1)
 
 					foundSegmentStoreSvc2 = &corev1.Service{}
 					nn2 := types.NamespacedName{
-						Name:      util.ServiceNameForSegmentStore(p.Name, 1),
+						Name:      util.ServiceNameForSegmentStore(p, 1),
 						Namespace: Namespace,
 					}
 					err = client.Get(context.TODO(), nn2, foundSegmentStoreSvc2)
 
 					foundSegmentStoreSvc3 = &corev1.Service{}
 					nn3 := types.NamespacedName{
-						Name:      util.ServiceNameForSegmentStore(p.Name, 2),
+						Name:      util.ServiceNameForSegmentStore(p, 2),
 						Namespace: Namespace,
 					}
 					err = client.Get(context.TODO(), nn3, foundSegmentStoreSvc3)
@@ -468,15 +468,15 @@ var _ = Describe("PravegaCluster Controller", func() {
 					mapLength := len(foundSegmentStoreSvc1.GetAnnotations())
 					Ω(mapLength).To(Equal(1))
 
-					svcName1 := util.ServiceNameForSegmentStore(p.Name, 0) + "." + domainName
+					svcName1 := util.ServiceNameForSegmentStore(p, 0) + "." + domainName
 					Expect(foundSegmentStoreSvc1.GetAnnotations()).To(HaveKeyWithValue(
 						"external-dns.alpha.kubernetes.io/hostname", svcName1))
 
-					svcName2 := util.ServiceNameForSegmentStore(p.Name, 1) + "." + domainName
+					svcName2 := util.ServiceNameForSegmentStore(p, 1) + "." + domainName
 					Expect(foundSegmentStoreSvc2.GetAnnotations()).To(HaveKeyWithValue(
 						"external-dns.alpha.kubernetes.io/hostname", svcName2))
 
-					svcName3 := util.ServiceNameForSegmentStore(p.Name, 2) + "." + domainName
+					svcName3 := util.ServiceNameForSegmentStore(p, 2) + "." + domainName
 					Expect(foundSegmentStoreSvc3.GetAnnotations()).To(HaveKeyWithValue(
 						"external-dns.alpha.kubernetes.io/hostname", svcName3))
 				})
@@ -556,21 +556,21 @@ var _ = Describe("PravegaCluster Controller", func() {
 				BeforeEach(func() {
 					foundSegmentStoreSvc1 = &corev1.Service{}
 					nn1 := types.NamespacedName{
-						Name:      util.ServiceNameForSegmentStore(p.Name, 0),
+						Name:      util.ServiceNameForSegmentStore(p, 0),
 						Namespace: Namespace,
 					}
 					err = client.Get(context.TODO(), nn1, foundSegmentStoreSvc1)
 
 					foundSegmentStoreSvc2 = &corev1.Service{}
 					nn2 := types.NamespacedName{
-						Name:      util.ServiceNameForSegmentStore(p.Name, 1),
+						Name:      util.ServiceNameForSegmentStore(p, 1),
 						Namespace: Namespace,
 					}
 					err = client.Get(context.TODO(), nn2, foundSegmentStoreSvc2)
 
 					foundSegmentStoreSvc3 = &corev1.Service{}
 					nn3 := types.NamespacedName{
-						Name:      util.ServiceNameForSegmentStore(p.Name, 2),
+						Name:      util.ServiceNameForSegmentStore(p, 2),
 						Namespace: Namespace,
 					}
 					err = client.Get(context.TODO(), nn3, foundSegmentStoreSvc3)
@@ -601,15 +601,15 @@ var _ = Describe("PravegaCluster Controller", func() {
 					Expect(foundSegmentStoreSvc3.GetAnnotations()).To(HaveKeyWithValue(
 						"service.beta.kubernetes.io/aws-load-balancer-type",
 						"nlb"))
-					svcName1 := util.ServiceNameForSegmentStore(p.Name, 0) + "." + domainName
+					svcName1 := util.ServiceNameForSegmentStore(p, 0) + "." + domainName
 					Expect(foundSegmentStoreSvc1.GetAnnotations()).To(HaveKeyWithValue(
 						"external-dns.alpha.kubernetes.io/hostname", svcName1))
 
-					svcName2 := util.ServiceNameForSegmentStore(p.Name, 1) + "." + domainName
+					svcName2 := util.ServiceNameForSegmentStore(p, 1) + "." + domainName
 					Expect(foundSegmentStoreSvc2.GetAnnotations()).To(HaveKeyWithValue(
 						"external-dns.alpha.kubernetes.io/hostname", svcName2))
 
-					svcName3 := util.ServiceNameForSegmentStore(p.Name, 2) + "." + domainName
+					svcName3 := util.ServiceNameForSegmentStore(p, 2) + "." + domainName
 					Expect(foundSegmentStoreSvc3.GetAnnotations()).To(HaveKeyWithValue(
 						"external-dns.alpha.kubernetes.io/hostname", svcName3))
 				})
@@ -655,21 +655,21 @@ var _ = Describe("PravegaCluster Controller", func() {
 				BeforeEach(func() {
 					foundSegmentStoreSvc1 = &corev1.Service{}
 					nn1 := types.NamespacedName{
-						Name:      util.ServiceNameForSegmentStore(p.Name, 0),
+						Name:      util.ServiceNameForSegmentStore(p, 0),
 						Namespace: Namespace,
 					}
 					err = client.Get(context.TODO(), nn1, foundSegmentStoreSvc1)
 
 					foundSegmentStoreSvc2 = &corev1.Service{}
 					nn2 := types.NamespacedName{
-						Name:      util.ServiceNameForSegmentStore(p.Name, 1),
+						Name:      util.ServiceNameForSegmentStore(p, 1),
 						Namespace: Namespace,
 					}
 					err = client.Get(context.TODO(), nn2, foundSegmentStoreSvc2)
 
 					foundSegmentStoreSvc3 = &corev1.Service{}
 					nn3 := types.NamespacedName{
-						Name:      util.ServiceNameForSegmentStore(p.Name, 2),
+						Name:      util.ServiceNameForSegmentStore(p, 2),
 						Namespace: Namespace,
 					}
 					err = client.Get(context.TODO(), nn3, foundSegmentStoreSvc3)
@@ -734,21 +734,21 @@ var _ = Describe("PravegaCluster Controller", func() {
 				BeforeEach(func() {
 					foundSegmentStoreSvc1 = &corev1.Service{}
 					nn1 := types.NamespacedName{
-						Name:      util.ServiceNameForSegmentStore(p.Name, 0),
+						Name:      util.ServiceNameForSegmentStore(p, 0),
 						Namespace: Namespace,
 					}
 					err = client.Get(context.TODO(), nn1, foundSegmentStoreSvc1)
 
 					foundSegmentStoreSvc2 = &corev1.Service{}
 					nn2 := types.NamespacedName{
-						Name:      util.ServiceNameForSegmentStore(p.Name, 1),
+						Name:      util.ServiceNameForSegmentStore(p, 1),
 						Namespace: Namespace,
 					}
 					err = client.Get(context.TODO(), nn2, foundSegmentStoreSvc2)
 
 					foundSegmentStoreSvc3 = &corev1.Service{}
 					nn3 := types.NamespacedName{
-						Name:      util.ServiceNameForSegmentStore(p.Name, 2),
+						Name:      util.ServiceNameForSegmentStore(p, 2),
 						Namespace: Namespace,
 					}
 					err = client.Get(context.TODO(), nn3, foundSegmentStoreSvc3)
@@ -762,15 +762,15 @@ var _ = Describe("PravegaCluster Controller", func() {
 				It("should set provided domain name as annotation", func() {
 					mapLength := len(foundSegmentStoreSvc1.GetAnnotations())
 					Ω(mapLength).To(Equal(1))
-					svcName1 := util.ServiceNameForSegmentStore(p.Name, 0) + "." + domainName
+					svcName1 := util.ServiceNameForSegmentStore(p, 0) + "." + domainName
 					Expect(foundSegmentStoreSvc1.GetAnnotations()).To(HaveKeyWithValue(
 						"external-dns.alpha.kubernetes.io/hostname", svcName1))
 
-					svcName2 := util.ServiceNameForSegmentStore(p.Name, 1) + "." + domainName
+					svcName2 := util.ServiceNameForSegmentStore(p, 1) + "." + domainName
 					Expect(foundSegmentStoreSvc2.GetAnnotations()).To(HaveKeyWithValue(
 						"external-dns.alpha.kubernetes.io/hostname", svcName2))
 
-					svcName3 := util.ServiceNameForSegmentStore(p.Name, 2) + "." + domainName
+					svcName3 := util.ServiceNameForSegmentStore(p, 2) + "." + domainName
 					Expect(foundSegmentStoreSvc3.GetAnnotations()).To(HaveKeyWithValue(
 						"external-dns.alpha.kubernetes.io/hostname", svcName3))
 				})
@@ -817,7 +817,7 @@ var _ = Describe("PravegaCluster Controller", func() {
 					foundSegmentStoreSvc1 = &corev1.Service{}
 
 					nn1 := types.NamespacedName{
-						Name:      util.ServiceNameForSegmentStore(p.Name, 0),
+						Name:      util.ServiceNameForSegmentStore(p, 0),
 						Namespace: Namespace,
 					}
 
@@ -829,7 +829,7 @@ var _ = Describe("PravegaCluster Controller", func() {
 					foundSegmentStoreSvc2 = &corev1.Service{}
 
 					nn2 := types.NamespacedName{
-						Name:      util.ServiceNameForSegmentStore(p.Name, 2),
+						Name:      util.ServiceNameForSegmentStore(p, 2),
 						Namespace: Namespace,
 					}
 					err = client.Get(context.TODO(), nn2, foundSegmentStoreSvc2)
@@ -840,7 +840,7 @@ var _ = Describe("PravegaCluster Controller", func() {
 					foundSegmentStoreSvc3 = &corev1.Service{}
 
 					nn3 := types.NamespacedName{
-						Name:      util.ServiceNameForSegmentStore(p.Name, 3),
+						Name:      util.ServiceNameForSegmentStore(p, 3),
 						Namespace: Namespace,
 					}
 					err = client.Get(context.TODO(), nn3, foundSegmentStoreSvc3)

pkg/controller/pravegacluster/upgrade.go

@@ -320,7 +320,6 @@ func (r *ReconcilePravegaCluster) syncControllerVersion(p *pravegav1alpha1.Prave
 }
 
 func (r *ReconcilePravegaCluster) syncSegmentStoreVersion(p *pravegav1alpha1.PravegaCluster) (synced bool, err error) {
-
 	sts := &appsv1.StatefulSet{}
 	name := util.StatefulSetNameForSegmentstore(p)
 	err = r.client.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: p.Namespace}, sts)
@@ -435,6 +434,39 @@ func (r *ReconcilePravegaCluster) syncStoreVersion(p *pravegav1alpha1.PravegaClu
 	return r.syncSegmentStoreVersion(p)
 }
 
+func (r *ReconcilePravegaCluster) createExternalServices(p *pravegav1alpha1.PravegaCluster) error {
+	services := pravega.MakeSegmentStoreExternalServices(p)
+	for _, service := range services {
+		controllerutil.SetControllerReference(p, service, r.scheme)
+		err := r.client.Create(context.TODO(), service)
+		if err != nil && !errors.IsAlreadyExists(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+func (r *ReconcilePravegaCluster) deleteExternalServices(p *pravegav1alpha1.PravegaCluster) error {
+	var name string = ""
+	for i := int32(0); i < p.Spec.Pravega.SegmentStoreReplicas; i++ {
+		service := &corev1.Service{}
+		name = util.ServiceNameForSegmentStore(p, i)
+		err := r.client.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: p.Namespace}, service)
+		if err != nil {
+			if errors.IsNotFound(err) {
+				continue
+			} else {
+				return err
+			}
+		}
+		err = r.client.Delete(context.TODO(), service)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 //To handle upgrade/rollback from Pravega version < 0.7 to Pravega Version >= 0.7
 func (r *ReconcilePravegaCluster) syncSegmentStoreVersionTo07(p *pravegav1alpha1.PravegaCluster) (synced bool, err error) {
 	p.Status.UpdateProgress(pravegav1alpha1.UpdatingSegmentstoreReason, "0")
@@ -444,6 +476,18 @@ func (r *ReconcilePravegaCluster) syncSegmentStoreVersionTo07(p *pravegav1alpha1
 	//this check is to see if the newsts is present or not if it's not present it will be created here
 	if err != nil {
 		if errors.IsNotFound(err) {
+			if p.Spec.ExternalAccess.Enabled {
+				err = r.createExternalServices(p)
+				if err != nil {
+					*newsts.Spec.Replicas = 0
+					err2 := r.client.Create(context.TODO(), newsts)
+					if err2 != nil {
+						log.Printf("failed to create StatefulSet: %v", err2)
+						return false, err2
+					}
+					return false, err
+				}
+			}
 			*newsts.Spec.Replicas = 0
 			err2 := r.client.Create(context.TODO(), newsts)
 			if err2 != nil {
@@ -564,6 +608,9 @@ func (r *ReconcilePravegaCluster) transitionToNewSTS(p *pravegav1alpha1.PravegaC
 	}
 	//this is to check if all the new ss pods have comeup before deleteing the old sts
 	if newsts.Status.ReadyReplicas == p.Spec.Pravega.SegmentStoreReplicas {
+		if p.Spec.ExternalAccess.Enabled {
+			r.deleteExternalServices(p)
+		}
 		err = r.client.Delete(context.TODO(), oldsts)
 	}
 	if err != nil {

pkg/util/pravegacluster.go

@@ -19,7 +19,7 @@ import (
 	v "github.com/hashicorp/go-version"
 	"github.com/pravega/pravega-operator/pkg/apis/pravega/v1alpha1"
 	api "github.com/pravega/pravega-operator/pkg/apis/pravega/v1alpha1"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 )
 
 var (
@@ -62,8 +62,11 @@ func ServiceNameForController(clusterName string) string {
 	return fmt.Sprintf("%s-pravega-controller", clusterName)
 }
 
-func ServiceNameForSegmentStore(clusterName string, index int32) string {
-	return fmt.Sprintf("%s-pravega-segmentstore-%d", clusterName, index)
+func ServiceNameForSegmentStore(p *api.PravegaCluster, index int32) string {
+	if IsVersionBelow07(p.Spec.Version) {
+		return fmt.Sprintf("%s-pravega-segmentstore-%d", p.Name, index)
+	}
+	return fmt.Sprintf("%s-pravega-segment-store-%d", p.Name, index)
 }
 
 func HeadlessServiceNameForSegmentStore(clusterName string) string {