Improve Rollout health.lua (#1554)

resource_customizations/argoproj.io/Rollout/health.lua

@@ -1,3 +1,55 @@
+function checkReplicasStatus(obj)
+  hs = {}
+  if obj.spec.replicas ~= nil and obj.status.updatedReplicas < obj.spec.replicas then
+    hs.status = "Progressing"
+    hs.message = "Waiting for roll out to finish: More replicas need to be updated"
+    return hs
+  end
+  if obj.status.replicas > obj.status.updatedReplicas then
+    hs.status = "Progressing"
+    hs.message = "Waiting for roll out to finish: old replicas are pending termination"
+    return hs
+  end
+  if obj.status.availableReplicas < obj.status.updatedReplicas then
+    hs.status = "Progressing"
+    hs.message = "Waiting for roll out to finish: updated replicas are still becoming available"
+    return hs
+  end
+  if obj.spec.replicas ~= nil and obj.status.updatedReplicas < obj.spec.replicas then
+    hs.status = "Progressing"
+    hs.message = "Waiting for roll out to finish: More replicas need to be updated"
+    return hs
+  end
+  if obj.status.replicas > obj.status.updatedReplicas then
+    hs.status = "Progressing"
+    hs.message = "Waiting for roll out to finish: old replicas are pending termination"
+    return hs
+  end
+  if obj.status.availableReplicas < obj.status.updatedReplicas then
+    hs.status = "Progressing"
+    hs.message = "Waiting for roll out to finish: updated replicas are still becoming available"
+    return hs
+  end
+  return nil
+end
+
+function checkPaused(obj)
+  hs = {}
+  local paused = false
+  if obj.status.verifyingPreview ~= nil then
+    paused = obj.status.verifyingPreview
+  elseif obj.spec.paused ~= nil then
+    paused = obj.spec.paused
+  end
+
+  if paused then
+    hs.status = "Suspended"
+    hs.message = "Rollout is paused"
+    return hs
+  end
+  return nil
+end
+
 hs = {}
 if obj.status ~= nil then
   if obj.status.conditions ~= nil then
@@ -7,51 +59,76 @@ if obj.status ~= nil then
         hs.message = condition.message
         return hs
       end
+      if condition.type == "Progressing" and condition.reason == "ProgressDeadlineExceeded" then
+        hs.status = "Degraded"
+        hs.message = condition.message
+        return hs
+      end
     end
   end
   if obj.status.currentPodHash ~= nil then
-    if obj.spec.replicas ~= nil and obj.status.updatedReplicas < obj.spec.replicas then
-      hs.status = "Progressing"
-	  hs.message = "Waiting for roll out to finish: More replicas need to be updated"
-      return hs
-    end
-
-    local paused = false
-    if obj.status.verifyingPreview ~= nil then
-      paused = obj.status.verifyingPreview
-    elseif obj.spec.paused ~= nil then
-      paused = obj.spec.paused
-    end
-
-    if paused then
-      hs.status = "Suspended"
-      if obj.status.blueGreen.previewSelector ~= nil and obj.status.blueGreen.previewSelector == obj.status.currentPodHash then
-        hs.message = "The preview Service is serving traffic to the current pod spec"
-      elseif obj.status.blueGreen.activeSelector ~= nil and obj.status.blueGreen.activeSelector == obj.status.currentPodHash then
+    if obj.spec.strategy.blueGreen ~= nil then
+      isPaused = checkPaused(obj)
+      if isPaused ~= nil then
+        return isPaused
+      end
+      replicasHS = checkReplicasStatus(obj)
+      if replicasHS ~= nil then
+        return replicasHS
+      end
+      if obj.status.blueGreen.activeSelector ~= nil and obj.status.blueGreen.activeSelector == obj.status.currentPodHash then
+        hs.status = "Healthy"
         hs.message = "The active Service is serving traffic to the current pod spec"
+        return hs
       end
-      return hs
-    end
-
-    if obj.status.replicas > obj.status.updatedReplicas then
-      hs.status = "Progressing"
-      hs.message = "Waiting for roll out to finish: old replicas are pending termination"
-      return hs
-    end
-    if obj.status.availableReplicas < obj.status.updatedReplicas then
       hs.status = "Progressing"
-      hs.message = "Waiting for roll out to finish: updated replicas are still becoming available"
+      hs.message = "The current pod spec is not receiving traffic from the active service"
       return hs
     end
+    if obj.spec.strategy.canary ~= nil then
+      currentRSIsStable = obj.status.canary.stableRS == obj.status.currentPodHash
+      if obj.spec.strategy.canary.steps ~= nil then
+        stepCount = table.getn(obj.spec.strategy.canary.steps)
+        if obj.status.currentStepIndex ~= nil then
+          currentStepIndex = obj.status.currentStepIndex
+          isPaused = checkPaused(obj)
+          if isPaused ~= nil then
+            return isPaused
+          end
+
+          if paused then
+            hs.status = "Suspended"
+            hs.message = "Rollout is paused"
+            return hs
+          end
+          if currentRSIsStable and stepCount == currentStepIndex then
+            replicasHS = checkReplicasStatus(obj)
+            if replicasHS ~= nil then
+              return replicasHS
+            end
+            hs.status = "Healthy"
+            hs.message = "The rollout has completed all steps"
+            return hs
+          end
+        end
+        hs.status = "Progressing"
+        hs.message = "Waiting for rollout to finish steps"
+        return hs
+      end
 
-    if obj.status.blueGreen.activeSelector ~= nil and obj.status.blueGreen.activeSelector == obj.status.currentPodHash then
-      hs.status = "Healthy"
-      hs.message = "The active Service is serving traffic to the current pod spec"
-      return hs
+      -- The detecting the health of the Canary deployment when there are no steps
+      replicasHS = checkReplicasStatus(obj)
+      if replicasHS ~= nil then
+        return replicasHS
+      end
+      if currentRSIsStable then
+        hs.status = "Healthy"
+        hs.message = "The rollout has completed canary deployment"
+        return hs
+      end
+      hs.status = "Progressing"
+      hs.message = "Waiting for rollout to finish canary deployment"
     end
-    hs.status = "Progressing"
-    hs.message = "The current pod spec is not receiving traffic from the active service"
-    return hs
   end
 end
 hs.status = "Progressing"

resource_customizations/argoproj.io/Rollout/health_test.yaml

@@ -1,29 +1,55 @@
 tests:
+- healthStatus:
+    status: Degraded
+    message: Rollout has missing field '.Spec.Strategy.Type'
+  inputPath: testdata/degraded_invalidSpec.yaml
+- healthStatus:
+    status: Degraded
+    message: ReplicaSet "guestbook-bluegreen-helm-guestbook-6b8cf6f7db" has timed out progressing.
+  inputPath: testdata/degraded_rolloutTimeout.yaml
+#BlueGreen
 - healthStatus:
     status: Healthy
     message: The active Service is serving traffic to the current pod spec
-  inputPath: testdata/healthy_servingActiveService.yaml
+  inputPath: testdata/bluegreen/healthy_servingActiveService.yaml
 - healthStatus:
     status: Suspended
-    message: The preview Service is serving traffic to the current pod spec
-  inputPath: testdata/suspended_servingPreviewService.yaml
+    message: Rollout is paused
+  inputPath: testdata/bluegreen/suspended_servingPreviewService.yaml
 - healthStatus:
     status: Suspended
-    message: The preview Service is serving traffic to the current pod spec
+    message: Rollout is paused
   inputPath: testdata/v0.2_suspended_servingPreviewService.yaml
 - healthStatus:
     status: Progressing
     message: "Waiting for roll out to finish: More replicas need to be updated"
-  inputPath: testdata/progressing_addingMoreReplicas.yaml
+  inputPath: testdata/bluegreen/progressing_addingMoreReplicas.yaml
 - healthStatus:
     status: Progressing
     message: "Waiting for roll out to finish: old replicas are pending termination"
-  inputPath: testdata/progressing_killingOldReplicas.yaml
+  inputPath: testdata/bluegreen/progressing_killingOldReplicas.yaml
 - healthStatus:
     status: Progressing
     message: "Waiting for roll out to finish: updated replicas are still becoming available"
-  inputPath: testdata/progressing_waitingUntilAvailable.yaml
+  inputPath: testdata/bluegreen/progressing_waitingUntilAvailable.yaml
+#Canary
 - healthStatus:
-    status: Degraded
-    message: Rollout has missing field '.Spec.Strategy.Type'
-  inputPath: testdata/degraded_invalidSpec.yaml
+    status: Progressing
+    message: Waiting for rollout to finish steps
+  inputPath: testdata/canary/progressing_setWeightStep.yaml
+- healthStatus:
+    status: Suspended
+    message: Rollout is paused
+  inputPath: testdata/canary/suspended_pausedStep.yaml
+- healthStatus:
+    status: Healthy
+    message: The rollout has completed all steps
+  inputPath: testdata/canary/healthy_executedAllSteps.yaml
+- healthStatus:
+    status: Progressing
+    message: 'Waiting for roll out to finish: old replicas are pending termination'
+  inputPath: testdata/canary/progressing_noSteps.yaml
+- healthStatus:
+    status: Healthy
+    message: The rollout has completed canary deployment
+  inputPath: testdata/canary/healthy_noSteps.yaml

resource_customizations/argoproj.io/Rollout/testdata/canary/healthy_executedAllSteps.yaml

@@ -0,0 +1,73 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Rollout
+metadata:
+  annotations:
+    kubectl.kubernetes.io/last-applied-configuration: >
+      {"apiVersion":"argoproj.io/v1alpha1","kind":"Rollout","metadata":{"annotations":{},"labels":{"app.kubernetes.io/instance":"guestbook-canary","ksonnet.io/component":"guestbook-ui"},"name":"guestbook-canary","namespace":"default"},"spec":{"minReadySeconds":10,"replicas":5,"selector":{"matchLabels":{"app":"guestbook-canary"}},"strategy":{"canary":{"maxSurge":1,"maxUnavailable":0,"steps":[{"setWeight":20},{"pause":{"duration":30}},{"setWeight":40},{"pause":{}}]}},"template":{"metadata":{"labels":{"app":"guestbook-canary"}},"spec":{"containers":[{"image":"gcr.io/heptio-images/ks-guestbook-demo:0.1","name":"guestbook-canary","ports":[{"containerPort":80}]}]}}}}
+    rollout.argoproj.io/revision: '1'
+  clusterName: ''
+  creationTimestamp: '2019-05-01T21:55:30Z'
+  generation: 1
+  labels:
+    app.kubernetes.io/instance: guestbook-canary
+    ksonnet.io/component: guestbook-ui
+  name: guestbook-canary
+  namespace: default
+  resourceVersion: '955764'
+  selfLink: /apis/argoproj.io/v1alpha1/namespaces/default/rollouts/guestbook-canary
+  uid: d6105ccd-6c5b-11e9-b8d7-025000000001
+spec:
+  minReadySeconds: 10
+  replicas: 5
+  selector:
+    matchLabels:
+      app: guestbook-canary
+  strategy:
+    canary:
+      maxSurge: 1
+      maxUnavailable: 0
+      steps:
+        - setWeight: 20
+        - pause:
+            duration: 30
+        - setWeight: 40
+        - pause: {}
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: guestbook-canary
+    spec:
+      containers:
+        - image: 'gcr.io/heptio-images/ks-guestbook-demo:0.1'
+          name: guestbook-canary
+          ports:
+            - containerPort: 80
+          resources: {}
+status:
+  HPAReplicas: 5
+  availableReplicas: 5
+  blueGreen: {}
+  canary:
+    stableRS: 84ccfddd66
+  conditions:
+    - lastTransitionTime: '2019-05-01T21:55:30Z'
+      lastUpdateTime: '2019-05-01T21:55:58Z'
+      message: ReplicaSet "guestbook-canary-84ccfddd66" has successfully progressed.
+      reason: NewReplicaSetAvailable
+      status: 'True'
+      type: Progressing
+    - lastTransitionTime: '2019-05-01T21:55:58Z'
+      lastUpdateTime: '2019-05-01T21:55:58Z'
+      message: Rollout has minimum availability
+      reason: AvailableReason
+      status: 'True'
+      type: Available
+  currentPodHash: 84ccfddd66
+  currentStepHash: 5f8fbdf7bb
+  currentStepIndex: 4
+  observedGeneration: c45557fd9
+  readyReplicas: 5
+  replicas: 5
+  selector: app=guestbook-canary
+  updatedReplicas: 5

resource_customizations/argoproj.io/Rollout/testdata/canary/healthy_noSteps.yaml

@@ -0,0 +1,66 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Rollout
+metadata:
+  annotations:
+    kubectl.kubernetes.io/last-applied-configuration: >
+      {"apiVersion":"argoproj.io/v1alpha1","kind":"Rollout","metadata":{"annotations":{},"labels":{"app.kubernetes.io/instance":"guestbook-canary","ksonnet.io/component":"guestbook-ui"},"name":"guestbook-canary","namespace":"default"},"spec":{"minReadySeconds":10,"replicas":5,"selector":{"matchLabels":{"app":"guestbook-canary"}},"strategy":{"canary":{"maxSurge":1,"maxUnavailable":0,"steps":[{"setWeight":20},{"pause":{"duration":30}},{"setWeight":40},{"pause":{}}]}},"template":{"metadata":{"labels":{"app":"guestbook-canary"}},"spec":{"containers":[{"image":"gcr.io/heptio-images/ks-guestbook-demo:0.1","name":"guestbook-canary","ports":[{"containerPort":80}]}]}}}}
+    rollout.argoproj.io/revision: '2'
+  clusterName: ''
+  creationTimestamp: '2019-05-01T21:55:30Z'
+  generation: 1
+  labels:
+    app.kubernetes.io/instance: guestbook-canary
+    ksonnet.io/component: guestbook-ui
+  name: guestbook-canary
+  namespace: default
+  resourceVersion: '956205'
+  selfLink: /apis/argoproj.io/v1alpha1/namespaces/default/rollouts/guestbook-canary
+  uid: d6105ccd-6c5b-11e9-b8d7-025000000001
+spec:
+  minReadySeconds: 10
+  replicas: 5
+  selector:
+    matchLabels:
+      app: guestbook-canary
+  strategy:
+    canary:
+      maxSurge: 1
+      maxUnavailable: 0
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: guestbook-canary
+    spec:
+      containers:
+        - image: 'gcr.io/heptio-images/ks-guestbook-demo:0.2'
+          name: guestbook-canary
+          ports:
+            - containerPort: 80
+          resources: {}
+status:
+  HPAReplicas: 5
+  availableReplicas: 5
+  blueGreen: {}
+  canary:
+    stableRS: 567dd56d89
+  conditions:
+    - lastTransitionTime: '2019-05-01T22:00:16Z'
+      lastUpdateTime: '2019-05-01T22:00:16Z'
+      message: Rollout has minimum availability
+      reason: AvailableReason
+      status: 'True'
+      type: Available
+    - lastTransitionTime: '2019-05-01T21:55:30Z'
+      lastUpdateTime: '2019-05-01T22:00:16Z'
+      message: ReplicaSet "guestbook-canary-567dd56d89" has successfully progressed.
+      reason: NewReplicaSetAvailable
+      status: 'True'
+      type: Progressing
+  currentPodHash: 567dd56d89
+  currentStepHash: 6c9545789c
+  observedGeneration: 6886f85bff
+  readyReplicas: 5
+  replicas: 5
+  selector: app=guestbook-canary
+  updatedReplicas: 5