Skip to content

Commit

Permalink
feat: Create RolloutPaused condition (#1054)
Browse files Browse the repository at this point in the history 
* feat: Create RolloutPaused condition
  • Loading branch information
@khhirani
khhirani authored Apr 8, 2021
1 parent f594c74 commit 959a4a5
Show file tree
Hide file tree
Showing 8 changed files with 233 additions and 42 deletions.
2 changes: 2 additions & 0 deletions pkg/apis/rollouts/v1alpha1/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -639,6 +639,8 @@ const (
// RolloutReplicaFailure ReplicaFailure is added in a deployment when one of its pods
// fails to be created or deleted.
RolloutReplicaFailure RolloutConditionType = "ReplicaFailure"
// RolloutPaused means that rollout is in a paused state. It is still progressing at this point.
RolloutPaused RolloutConditionType = "Paused"
)

// RolloutCondition describes the state of a rollout at a certain point.
Expand Down
53 changes: 44 additions & 9 deletions rollout/analysis_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1443,7 +1443,11 @@ func TestDoNotCreateBackgroundAnalysisRunAfterInconclusiveRun(t *testing.T) {
Reason: v1alpha1.PauseReasonInconclusiveAnalysis,
StartTime: metav1.Now(),
}}
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")

progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

f.rolloutLister = append(f.rolloutLister, r2)
Expand All @@ -1454,6 +1458,13 @@ func TestDoNotCreateBackgroundAnalysisRunAfterInconclusiveRun(t *testing.T) {
f.run(getKey(r2, t))

patch := f.getPatchedRollout(patchIndex)

//expectedPatch := fmt.Sprintf(`{
// "status":{
// "conditions":[%s, %s],
// "observedGeneration": ""
// }
//}`, progressingConditionString, pausedConditionString)
assert.Equal(t, calculatePatch(r2, OnlyObservedGenerationPatch), patch)
}

Expand Down Expand Up @@ -1543,7 +1554,10 @@ func TestCreatePrePromotionAnalysisRun(t *testing.T) {
rs2PodHash := rs2.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]

r2 = updateBlueGreenRolloutStatus(r2, rs2PodHash, rs1PodHash, rs1PodHash, 1, 1, 2, 1, true, true)
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

previewSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs2PodHash}
Expand Down Expand Up @@ -1723,7 +1737,10 @@ func TestRolloutPrePromotionAnalysisBecomesInconclusive(t *testing.T) {
Name: ar.Name,
Status: v1alpha1.AnalysisPhaseRunning,
}
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs1PodHash}
Expand Down Expand Up @@ -1788,7 +1805,10 @@ func TestRolloutPrePromotionAnalysisSwitchServiceAfterSuccess(t *testing.T) {
Name: ar.Name,
Status: v1alpha1.AnalysisPhaseRunning,
}
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs1PodHash}
Expand Down Expand Up @@ -1851,7 +1871,10 @@ func TestRolloutPrePromotionAnalysisHonorAutoPromotionSeconds(t *testing.T) {
r2 = updateBlueGreenRolloutStatus(r2, "", rs1PodHash, rs1PodHash, 1, 1, 2, 1, true, true)
now := metav1.NewTime(metav1.Now().Add(-10 * time.Second))
r2.Status.PauseConditions[0].StartTime = now
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs1PodHash}
Expand Down Expand Up @@ -1914,7 +1937,10 @@ func TestRolloutPrePromotionAnalysisDoNothingOnInconclusiveAnalysis(t *testing.T
}
r2.Status.PauseConditions = append(r2.Status.PauseConditions, inconclusivePauseCondition)
r2.Status.ObservedGeneration = strconv.Itoa(int(r2.Generation))
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs1PodHash}
Expand Down Expand Up @@ -1957,7 +1983,10 @@ func TestAbortRolloutOnErrorPrePromotionAnalysis(t *testing.T) {
rs1PodHash := rs1.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]

r2 = updateBlueGreenRolloutStatus(r2, "", rs1PodHash, rs1PodHash, 1, 1, 2, 1, true, true)
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs1PodHash}
Expand Down Expand Up @@ -2122,7 +2151,10 @@ func TestPostPromotionAnalysisRunHandleInconclusive(t *testing.T) {
Reason: v1alpha1.PauseReasonInconclusiveAnalysis,
StartTime: metav1.Now(),
}}
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs2PodHash}
Expand Down Expand Up @@ -2175,7 +2207,10 @@ func TestAbortRolloutOnErrorPostPromotionAnalysis(t *testing.T) {
rs2PodHash := rs2.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]

r2 = updateBlueGreenRolloutStatus(r2, "", rs2PodHash, rs1PodHash, 1, 1, 2, 1, true, true)
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs2PodHash}
Expand Down
32 changes: 25 additions & 7 deletions rollout/bluegreen_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ func TestBlueGreenHandlePause(t *testing.T) {
"conditions": %s
}
}`
addedConditions := generateConditionsPatch(true, conditions.PausedRolloutReason, rs2, true, "")
addedConditions := generateConditionsPatchWithPause(true, conditions.PausedRolloutReason, rs2, true, "", true)
assert.Equal(t, calculatePatch(r2, fmt.Sprintf(expectedPatch, addedConditions)), patch)
})

Expand All @@ -266,7 +266,10 @@ func TestBlueGreenHandlePause(t *testing.T) {
rs2PodHash := rs2.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]

r2 = updateBlueGreenRolloutStatus(r2, rs2PodHash, rs1PodHash, rs1PodHash, 1, 1, 2, 1, true, true)
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

previewSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs2PodHash}
Expand Down Expand Up @@ -310,7 +313,10 @@ func TestBlueGreenHandlePause(t *testing.T) {
Reason: v1alpha1.PauseReasonInconclusiveAnalysis,
StartTime: now,
})
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

previewSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs2PodHash}
Expand Down Expand Up @@ -347,7 +353,10 @@ func TestBlueGreenHandlePause(t *testing.T) {
rs2PodHash := rs2.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]

r2 = updateBlueGreenRolloutStatus(r2, rs2PodHash, rs1PodHash, rs1PodHash, 1, 1, 2, 1, true, true)
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

previewSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs2PodHash}
Expand Down Expand Up @@ -385,7 +394,10 @@ func TestBlueGreenHandlePause(t *testing.T) {
before := metav1.NewTime(now.Add(-1 * time.Minute))
r2.Status.PauseConditions[0].StartTime = before
r2.Status.ControllerPause = true
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs1PodHash}
Expand Down Expand Up @@ -439,7 +451,10 @@ func TestBlueGreenHandlePause(t *testing.T) {
r2.Status.PauseConditions[0].StartTime = before
r2.Status.ControllerPause = true
r2.Spec.Paused = true
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs1PodHash}
Expand Down Expand Up @@ -731,7 +746,10 @@ func TestBlueGreenRolloutStatusHPAStatusFieldsActiveSelectorSet(t *testing.T) {

r2 = updateBlueGreenRolloutStatus(r2, rs2PodHash, rs1PodHash, rs1PodHash, 0, 0, 0, 0, true, false)
r2.Status.Selector = ""
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

f.rolloutLister = append(f.rolloutLister, r2)
Expand Down
45 changes: 32 additions & 13 deletions rollout/canary_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,9 @@ func TestCanaryRolloutNoProgressWhilePaused(t *testing.T) {
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

rs1 := newReplicaSetWithStatus(r1, 10, 10)
rs2 := newReplicaSetWithStatus(r2, 0, 0)

Expand Down Expand Up @@ -216,9 +219,12 @@ func TestCanaryRolloutUpdatePauseConditionWhilePaused(t *testing.T) {
r1 := newCanaryRollout("foo", 10, nil, steps, pointer.Int32Ptr(0), intstr.FromInt(1), intstr.FromInt(0))
r2 := bumpVersion(r1)

progressingCondition, _ := newProgressingCondition(conditions.ReplicaSetUpdatedReason, r2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

rs1 := newReplicaSetWithStatus(r1, 10, 10)
rs2 := newReplicaSetWithStatus(r2, 0, 0)

Expand All @@ -231,17 +237,10 @@ func TestCanaryRolloutUpdatePauseConditionWhilePaused(t *testing.T) {
f.objects = append(f.objects, r2)

addPausedConditionPatch := f.expectPatchRolloutAction(r2)
f.expectPatchRolloutAction(r2)
f.run(getKey(r2, t))

patch := f.getPatchedRollout(addPausedConditionPatch)
_, pausedCondition := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
expectedPatch := fmt.Sprintf(`{
"status": {
"conditions": [%s]
}
}`, pausedCondition)
assert.Equal(t, calculatePatch(r2, expectedPatch), patch)
assert.Equal(t, calculatePatch(r2, OnlyObservedGenerationPatch), patch)
}

func TestCanaryRolloutResetProgressDeadlineOnRetry(t *testing.T) {
Expand Down Expand Up @@ -885,6 +884,9 @@ func TestSyncRolloutWaitAddToQueue(t *testing.T) {
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

r2.Status.ObservedGeneration = strconv.Itoa(int(r2.Generation))
f.rolloutLister = append(f.rolloutLister, r2)
f.objects = append(f.objects, r2)
Expand Down Expand Up @@ -927,6 +929,9 @@ func TestSyncRolloutIgnoreWaitOutsideOfReconciliationPeriod(t *testing.T) {
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

f.rolloutLister = append(f.rolloutLister, r2)
f.objects = append(f.objects, r2)

Expand Down Expand Up @@ -964,7 +969,10 @@ func TestSyncRolloutWaitIncrementStepIndex(t *testing.T) {
f.replicaSetLister = append(f.replicaSetLister, rs1, rs2)

r2 = updateCanaryRolloutStatus(r2, rs1PodHash, 10, 1, 10, false)
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

earlier := metav1.Now()
Expand Down Expand Up @@ -1008,6 +1016,9 @@ func TestCanaryRolloutStatusHPAStatusFields(t *testing.T) {
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, r2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

rs1 := newReplicaSetWithStatus(r1, 4, 4)
rs1PodHash := rs1.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]
rs2 := newReplicaSetWithStatus(r2, 1, 1)
Expand Down Expand Up @@ -1249,7 +1260,10 @@ func TestCanaryRolloutScaleWhilePaused(t *testing.T) {

r2 = updateCanaryRolloutStatus(r2, rs1PodHash, 5, 0, 5, true)
r2.Spec.Replicas = pointer.Int32Ptr(10)
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs2, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

f.rolloutLister = append(f.rolloutLister, r2)
Expand Down Expand Up @@ -1343,7 +1357,9 @@ func TestNoResumeAfterPauseDurationIfUserPaused(t *testing.T) {
Reason: v1alpha1.PauseReasonCanaryPauseStep,
StartTime: overAMinuteAgo,
}}
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs1, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs1, "")
conditions.SetRolloutCondition(&r1.Status, progressingCondition)
pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r1.Status, pausedCondition)
r1.Spec.Paused = true
f.kubeobjects = append(f.kubeobjects, rs1)
Expand Down Expand Up @@ -1380,7 +1396,10 @@ func TestHandleNilNewRSOnScaleAndImageChange(t *testing.T) {
r2 := bumpVersion(r1)
r2.Spec.Replicas = pointer.Int32Ptr(3)
r2 = updateCanaryRolloutStatus(r2, rs1PodHash, 3, 0, 3, true)
pausedCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs1, "")
progressingCondition, _ := newProgressingCondition(conditions.PausedRolloutReason, rs1, "")
conditions.SetRolloutCondition(&r2.Status, progressingCondition)

pausedCondition, _ := newPausedCondition(true)
conditions.SetRolloutCondition(&r2.Status, pausedCondition)

f.kubeobjects = append(f.kubeobjects, rs1)
Expand Down
30 changes: 30 additions & 0 deletions rollout/controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,26 @@ func newReplicaSetWithStatus(r *v1alpha1.Rollout, replicas int, availableReplica
return rs
}

func newPausedCondition(isPaused bool) (v1alpha1.RolloutCondition, string) {
status := corev1.ConditionTrue
if !isPaused {
status = corev1.ConditionFalse
}
condition := v1alpha1.RolloutCondition{
LastTransitionTime: metav1.Now(),
LastUpdateTime: metav1.Now(),
Message: conditions.PausedRolloutMessage,
Reason: conditions.PausedRolloutReason,
Status: status,
Type: v1alpha1.RolloutPaused,
}
conditionBytes, err := json.Marshal(condition)
if err != nil {
panic(err)
}
return condition, string(conditionBytes)
}

func newProgressingCondition(reason string, resourceObj runtime.Object, optionalMessage string) (v1alpha1.RolloutCondition, string) {
status := corev1.ConditionTrue
msg := ""
Expand Down Expand Up @@ -277,6 +297,16 @@ func generateConditionsPatch(available bool, progressingReason string, progressi
return fmt.Sprintf("[%s, %s]", progressingCondition, availableCondition)
}

func generateConditionsPatchWithPause(available bool, progressingReason string, progressingResource runtime.Object, availableConditionFirst bool, progressingMessage string, isPaused bool) string {
_, availableCondition := newAvailableCondition(available)
_, progressingCondition := newProgressingCondition(progressingReason, progressingResource, progressingMessage)
_, pauseCondition := newPausedCondition(isPaused)
if availableConditionFirst {
return fmt.Sprintf("[%s, %s, %s]", availableCondition, progressingCondition, pauseCondition)
}
return fmt.Sprintf("[%s, %s, %s]", progressingCondition, pauseCondition, availableCondition)
}

// func updateBlueGreenRolloutStatus(r *v1alpha1.Rollout, preview, active string, availableReplicas, updatedReplicas, hpaReplicas int32, pause bool, available bool, progressingStatus string) *v1alpha1.Rollout {
func updateBlueGreenRolloutStatus(r *v1alpha1.Rollout, preview, active, stable string, availableReplicas, updatedReplicas, totalReplicas, hpaReplicas int32, pause bool, available bool) *v1alpha1.Rollout {
newRollout := updateBaseRolloutStatus(r, availableReplicas, updatedReplicas, totalReplicas, hpaReplicas)
Expand Down
Loading

0 comments on commit 959a4a5

Please sign in to comment.