Issue 439: Post upgrade script for helm upgrades

doc/operator-upgrade.md

@@ -115,6 +115,22 @@ The upgrade to Operator 0.5.0 can be triggered using the following command
 helm upgrade <operator release name> <location of 0.5.0 charts> --set webhookCert.crt=<tls.crt> --set webhookCert.generate=false --set webhookCert.certName=<cert-name> --set webhookCert.secretName=<secret-name>
 ```
 
+Once the upgrade completes and the following command starts returning a response instead of throwing an error message (this might take around 7 to 10 minutes after the operator upgrade has been done)
+```
+kubectl describe PravegaCluster
+```
+Execute the script `post-upgrade.sh` inside the [scripts](https://github.com/pravega/pravega-operator/blob/master/scripts) folder. The format of the command is
+```
+./post-upgrade.sh <PravegaCluster resource name> <PravegaCluster release name> <BookkeeperCluster release name> <version> <namespace> <zookeeper svc name>
+```
+This script patches the `PravegaCluster` and `BookkeeperCluster` resources with the required annotations and labels, and updates their corresponding helm releases. This script needs the following arguments
+1. Name of the PravegaCluster or BookkeeperCluster resource (check the output of `kubectl get PravegaCluster` to obtain this name).
+2. Name of the release that has been created for the v1alpha1 PravegaCluster resource (check the output of `helm ls` to obtain this name).
+3. Name of the release that needs to be created for the BookkeeperCluster resource.
+4. Version of the PravegaCluster or BookkeeperCluster resources (check the output of `kubectl get PravegaCluster` to obtain the version number).
+5. Namespace in which PravegaCluster and BookkeeperCluster resources are deployed (this is an optional parameter and its default value is `default`).
+6. Name of the zookeeper client service (this is an optional parameter and its default value is `zookeeper-client`).
+
 #### Upgrade manually
 
 To manually trigger the upgrade to Operator 0.5.0, run the script `operatorUpgrade.sh` under [tools](https://github.com/pravega/pravega-operator/blob/master/tools) folder. This script patches the Pravega Cluster CRD and creates necessary K8s artifacts, needed by 0.5.0 Operator, prior to triggering the upgrade by updating the image tag in Operator deployment.

doc/rollback-cluster.md

@@ -190,7 +190,4 @@ Status:
     Type:                  RollbackInProgress
 ```
 
-When a rollback failure happens, manual intervention would be required to resolve this.
-After checking and solving the root cause of failure, to bring the cluster back to a stable state, a user can upgrade to:
-1. The version to which a user initially intended to upgrade.(when upgrade failure was noticed)
-2. To any other supported version based versions of all pods in the cluster.
+When a rollback failure happens, the operator cannot recover the cluster from this failed state and manual intervention would be required to resolve this.

pkg/apis/pravega/v1beta1/pravegacluster_types.go

@@ -550,7 +550,7 @@ func (dst *PravegaCluster) updateControllerReferences(ownerRefs []metav1.OwnerRe
 }
 
 func createConfigMap(p *v1alpha1.PravegaCluster) error {
-	cmName := "pravega-config"
+	cmName := fmt.Sprintf("%s-configmap", p.Name)
 	cfgMap := &corev1.ConfigMap{}
 	err := Mgr.GetClient().Get(context.TODO(),
 		types.NamespacedName{Name: cmName, Namespace: p.Namespace}, cfgMap)
@@ -605,12 +605,19 @@ func (p *PravegaCluster) migrateBookkeeper(srcObj *v1alpha1.PravegaCluster) erro
 	}
 
 	log.Printf("Created Bookkeeper CR by name %s", b.Name)
-	err = migrateConfigMap(srcObj, b)
+	configmap := fmt.Sprintf("%s-configmap", srcObj.Name)
+	err = migrateConfigMap(srcObj, b, configmap)
 	if err != nil {
-		log.Fatalf("Error releasing BK CM %v", err)
+		log.Fatalf("Error releasing BK CM %s %v", configmap, err)
 		return err
 	}
-	log.Print("Migrated Bookkeeper ConfigMap.")
+	configmap = nameForBookie(srcObj.Name)
+	err = migrateConfigMap(srcObj, b, configmap)
+	if err != nil {
+		log.Fatalf("Error releasing BK CM %s %v", configmap, err)
+		return err
+	}
+	log.Print("Migrated Bookkeeper ConfigMaps.")
 
 	err = migratePVC(srcObj, b)
 	if err != nil {
@@ -666,9 +673,8 @@ func migrateSTS(p *v1alpha1.PravegaCluster, b *bkapi.BookkeeperCluster) error {
 	return nil
 }
 
-func migrateConfigMap(p *v1alpha1.PravegaCluster, b *bkapi.BookkeeperCluster) error {
+func migrateConfigMap(p *v1alpha1.PravegaCluster, b *bkapi.BookkeeperCluster, name string) error {
 	configmap := &corev1.ConfigMap{}
-	name := nameForBookie(p.Name)
 	err := Mgr.GetClient().Get(context.TODO(),
 		types.NamespacedName{Name: name, Namespace: p.Namespace}, configmap)
 	if err != nil {
@@ -822,7 +828,7 @@ func specCopy(srcObj *v1alpha1.PravegaCluster, b *bkapi.BookkeeperCluster) {
 
 	b.Spec.ZookeeperUri = srcObj.Spec.ZookeeperUri
 	// name of config-map having pravega configuration
-	b.Spec.EnvVars = "pravega-config"
+	b.Spec.EnvVars = fmt.Sprintf("%s-configmap", srcObj.Name)
 	b.Spec.Version = srcObj.Spec.Version
 }
 

scripts/post-upgrade.sh

@@ -0,0 +1,51 @@
+#! /bin/bash
+set -ex
+
+if [[ "$#" -lt 4 || "$#" -gt 6 ]]; then
+	echo "Error : Invalid number of arguments"
+	Usage: "./post-upgrade.sh <pravegacluster name> <pravega-release-name> <bookkeeper-release-name> <version> <namespace> <zk-svc-name>"
+	exit 1
+fi
+
+name=$1
+pname=$2
+bkname=$3
+version=$4
+namespace=${5:-default}
+zksvc=${6:-zookeeper-client}
+
+echo "Checking that the PravegaCluster resource is in ready state"
+kubectl describe PravegaCluster $name -n $namespace
+currentReplicasPc=`kubectl get PravegaCluster $name -n $namespace -o jsonpath='{.status.currentReplicas}'`
+readyReplicasPc=`kubectl get PravegaCluster $name -n $namespace -o jsonpath='{.status.readyReplicas}'`
+if [ $currentReplicasPc != $readyReplicasPc ]; then
+	echo "Error : Pravega Cluster is not in ready state"
+  exit 1
+fi
+echo "Adding required annotations and labels to the PravegaCluster resource to make sure they're owned by the right chart"
+kubectl annotate PravegaCluster $name meta.helm.sh/release-name=$pname -n $namespace --overwrite
+kubectl annotate PravegaCluster $name meta.helm.sh/release-namespace=$namespace -n $namespace --overwrite
+kubectl label PravegaCluster $name app.kubernetes.io/managed-by=Helm -n $namespace --overwrite
+
+echo "Checking that the BookkeeperCluster resource is in ready state"
+kubectl get BookkeeperCluster $name -n $namespace
+currentReplicasBk=`kubectl get BookkeeperCluster $name -n $namespace -o jsonpath='{.status.replicas}'`
+readyReplicasBk=`kubectl get BookkeeperCluster $name -n $namespace -o jsonpath='{.status.readyReplicas}'`
+if [ $currentReplicasBk != $readyReplicasBk ]; then
+	echo "Error : Bookkeeper Cluster is not in ready state"
+  exit 2
+fi
+echo "Adding required annotations and labels to the BookkeeperCluster resource to make sure they're owned by the right chart"
+kubectl annotate BookkeeperCluster $name meta.helm.sh/release-name=$bkname -n $namespace --overwrite
+kubectl annotate BookkeeperCluster $name meta.helm.sh/release-namespace=$namespace -n $namespace --overwrite
+kubectl label BookkeeperCluster $name app.kubernetes.io/managed-by=Helm -n $namespace --overwrite
+kubectl annotate ConfigMap $name-configmap meta.helm.sh/release-name=$bkname -n $namespace --overwrite
+kubectl annotate ConfigMap $name-configmap meta.helm.sh/release-namespace=$namespace -n $namespace --overwrite
+kubectl label ConfigMap $name-configmap app.kubernetes.io/managed-by=Helm -n $namespace --overwrite
+
+helm repo add pravega https://charts.pravega.io
+helm repo update
+echo "Upgrading the pravega charts"
+helm upgrade $pname pravega/pravega --version=$version --set fullnameOverride=$name --set zookeeperUri="$zksvc:2181" --set bookkeeperUri="$name-bookie-headless:3181"
+echo "Installing the bookkeeper charts"
+helm install $bkname pravega/bookkeeper --version=$version --set fullnameOverride=$name --set zookeeperUri="$zksvc:2181" --set pravegaClusterName=$name