From b857210b1928dbc4b3b5191462c879b86da21a17 Mon Sep 17 00:00:00 2001 From: Fokion Sotiropoulos Date: Mon, 19 Jun 2023 17:44:43 +0100 Subject: [PATCH 1/4] Update the skip as it was not working correctly It could not evaluate a variable that was defined on previous steps and during processing we should check that the failures are empty Signed-off-by: Fokion Sotiropoulos --- assertion_test.go | 13 +++++++++++++ process_testcase.go | 7 +++---- process_teststep.go | 2 +- tests/skip/skip.yml | 29 +++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 tests/skip/skip.yml diff --git a/assertion_test.go b/assertion_test.go index 4f1cd865..543195ea 100644 --- a/assertion_test.go +++ b/assertion_test.go @@ -1,6 +1,8 @@ package venom import ( + "context" + "github.com/ovh/venom/assertions" "reflect" "testing" ) @@ -22,3 +24,14 @@ func Test_splitAssertion(t *testing.T) { } } } + +func Test_parseAssertions(t *testing.T) { + vars := make(map[string]string) + vars["out"] = "2" + assertion, err := parseAssertions(context.Background(), "out ShouldEqual '2'", vars) + _ = assertions.ShouldNotBeNil(&assertion) + e := assertions.ShouldBeNil(err) + if e != nil { + t.Errorf("The err should be nil") + } +} diff --git a/process_testcase.go b/process_testcase.go index 4cb08825..67293b5f 100644 --- a/process_testcase.go +++ b/process_testcase.go @@ -333,7 +333,6 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe tsResult.End = time.Now() tsResult.Duration = tsResult.End.Sub(tsResult.Start).Seconds() - tc.testSteps = append(tc.testSteps, step) } @@ -440,13 +439,13 @@ func parseSkip(ctx context.Context, tc *TestCase, ts *TestStepResult, rawStep [] // Evaluate skip assertions if len(assertions.Skip) > 0 { - results, err := testConditionalStatement(ctx, tc, assertions.Skip, tc.Vars, fmt.Sprintf("skipping testcase %%q step #%d: %%v", stepNumber)) + failures, err := testConditionalStatement(ctx, tc, assertions.Skip, tc.computedVars, fmt.Sprintf("skipping testcase %%q step #%d: %%v", stepNumber)) if err != nil { Error(ctx, "unable to evaluate \"skip\" assertions: %v", err) return false, err } - if len(results) > 0 { - for _, s := range results { + if len(failures) == 0 { + for _, s := range failures { ts.Skipped = append(ts.Skipped, Skipped{Value: s}) Warn(ctx, s) } diff --git a/process_teststep.go b/process_teststep.go index 2273cf0c..69711bfa 100644 --- a/process_teststep.go +++ b/process_teststep.go @@ -112,7 +112,7 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, if assertRes.OK { break } - failures, err := testConditionalStatement(ctx, tc, e.RetryIf(), tsResult.ComputedVars, "") + failures, err := testConditionalStatement(ctx, tc, e.RetryIf(), AllVarsFromCtx(ctx), "") if err != nil { tsResult.appendError(fmt.Errorf("Error while evaluating retry condition: %v", err)) break diff --git a/tests/skip/skip.yml b/tests/skip/skip.yml new file mode 100644 index 00000000..20a92218 --- /dev/null +++ b/tests/skip/skip.yml @@ -0,0 +1,29 @@ +name : "Testing skip" +testcases: + - name : "skip something" + steps: + - info: + - Set a variable to 1 + - script : "echo '1'" + vars : + out: + from : result.systemout + - info: + - "do not skip it as the value of out is different to 2" + skip : + - out ShouldEqual '2' + - name: skip-this + skip: + - out ShouldEqual '1' + - script: "echo 'false'" + vars: + outAsBool: + from: result.systemout + - info: + - "Fail should be skipped" + skip : + - out ShouldEqual 1 + - info: + - "Fail should be skipped as false" + skip: + - outAsBool ShouldBeFalse From eed4e713b1a8ae877dead4c43f63074007cf70fb Mon Sep 17 00:00:00 2001 From: Fokion Sotiropoulos Date: Thu, 27 Jul 2023 19:04:04 +0100 Subject: [PATCH 2/4] adding the check in the testcase as well Signed-off-by: Fokion Sotiropoulos --- process_testcase.go | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/process_testcase.go b/process_testcase.go index 67293b5f..e9a79ab2 100644 --- a/process_testcase.go +++ b/process_testcase.go @@ -163,21 +163,23 @@ func (v *Venom) processSecrets(ctx context.Context, ts *TestSuite, tc *TestCase) } func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepResult) { - results, err := testConditionalStatement(ctx, tc, tc.Skip, tc.Vars, "skipping testcase %q: %v") - if err != nil { - Error(ctx, "unable to evaluate \"skip\" assertions: %v", err) - testStepResult := TestStepResult{} - testStepResult.appendError(err) - tc.TestStepResults = append(tc.TestStepResults, testStepResult) - return - } - if len(results) > 0 { - tc.Status = StatusSkip - for _, s := range results { - tc.Skipped = append(tc.Skipped, Skipped{Value: s}) - Warn(ctx, s) + if len(tc.Skip) > 0 { + results, err := testConditionalStatement(ctx, tc, tc.Skip, tc.Vars, "skipping testcase %q: %v") + if err != nil { + Error(ctx, "unable to evaluate \"skip\" assertions: %v", err) + testStepResult := TestStepResult{} + testStepResult.appendError(err) + tc.TestStepResults = append(tc.TestStepResults, testStepResult) + return + } + if len(results) == 0 { + tc.Status = StatusSkip + for _, s := range results { + tc.Skipped = append(tc.Skipped, Skipped{Value: s}) + Warn(ctx, s) + } + return } - return } var knowExecutors = map[string]struct{}{} From ef1ce9a21b886a06a3f9c4946d9175e2a3868b57 Mon Sep 17 00:00:00 2001 From: Fokion Sotiropoulos Date: Thu, 27 Jul 2023 19:10:55 +0100 Subject: [PATCH 3/4] adding one more test for the testcases --- tests/skip/skip-testcase.yml | 19 +++++++++++++++++++ tests/skip/skip.yml | 6 ++++++ 2 files changed, 25 insertions(+) create mode 100644 tests/skip/skip-testcase.yml diff --git a/tests/skip/skip-testcase.yml b/tests/skip/skip-testcase.yml new file mode 100644 index 00000000..cdf28a07 --- /dev/null +++ b/tests/skip/skip-testcase.yml @@ -0,0 +1,19 @@ +name : "Testing skip" +testcases: + - name : "producer" + steps: + - info: + - Set a variable to 1 + - script : "echo '1'" + vars : + out: + from : result.systemout + - name: "consumer" + skip: + - producer.out ShouldEqual 1 + step: + - info: + - "Fail should be skipped" + script: | + exit 1 + diff --git a/tests/skip/skip.yml b/tests/skip/skip.yml index 20a92218..b5aef33a 100644 --- a/tests/skip/skip.yml +++ b/tests/skip/skip.yml @@ -13,6 +13,8 @@ testcases: skip : - out ShouldEqual '2' - name: skip-this + script: | + exit 1 skip: - out ShouldEqual '1' - script: "echo 'false'" @@ -21,9 +23,13 @@ testcases: from: result.systemout - info: - "Fail should be skipped" + script: | + exit 1 skip : - out ShouldEqual 1 - info: - "Fail should be skipped as false" + script: | + exit 1 skip: - outAsBool ShouldBeFalse From 9b96a7949a906aa92b0a7d57eb409cc684df1801 Mon Sep 17 00:00:00 2001 From: Fokion Sotiropoulos Date: Fri, 28 Jul 2023 14:51:08 +0000 Subject: [PATCH 4/4] update retry in test step --- process_teststep.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/process_teststep.go b/process_teststep.go index 69711bfa..07f5647f 100644 --- a/process_teststep.go +++ b/process_teststep.go @@ -112,15 +112,17 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, if assertRes.OK { break } - failures, err := testConditionalStatement(ctx, tc, e.RetryIf(), AllVarsFromCtx(ctx), "") - if err != nil { - tsResult.appendError(fmt.Errorf("Error while evaluating retry condition: %v", err)) - break - } - if len(failures) > 0 { - failure := newFailure(ctx, *tc, stepNumber, rangedIndex, "", fmt.Errorf("retry conditions not fulfilled, skipping %d remaining retries", e.Retry()-tsResult.Retries)) - tsResult.Errors = append(tsResult.Errors, *failure) - break + if len(e.RetryIf()) > 0{ + failures, err := testConditionalStatement(ctx, tc, e.RetryIf(), AllVarsFromCtx(ctx), "") + if err != nil { + tsResult.appendError(fmt.Errorf("Error while evaluating retry condition: %v", err)) + break + } + if len(failures) == 0 { + failure := newFailure(ctx, *tc, stepNumber, rangedIndex, "", fmt.Errorf("retry conditions not fulfilled, skipping %d remaining retries", e.Retry()-tsResult.Retries)) + tsResult.Errors = append(tsResult.Errors, *failure) + break + } } }