Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: removed result from testcase variables and removed errors warning and info from non verbose runs #617

Merged
merged 2 commits into from
Feb 14, 2023
Merged

feat: removed result from testcase variables and removed errors warning and info from non verbose runs #617

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7bdc7d9
feat: removed ranged stdout on non-verbose runs
Jan 30, 2023
59761b3
feat: remove tc computedInfo, fix info not showing right variable val…
Jan 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 30 additions & 35 deletions process_testcase.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe
} else {
tsResult.Start = time.Now()
tsResult.Status = StatusRun
result := v.RunTestStep(ctx, e, tc, tsResult, stepNumber, rangedIndex, step)
v.RunTestStep(ctx, e, tc, tsResult, stepNumber, rangedIndex, step)
if len(tsResult.Errors) > 0 || !tsResult.AssertionsApplied.OK {
tsResult.Status = StatusFail
} else {
Expand All @@ -322,9 +322,6 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe
tsResult.End = time.Now()
tsResult.Duration = tsResult.End.Sub(tsResult.Start).Seconds()

mapResult := GetExecutorResult(result)
previousStepVars.AddAll(H(mapResult))

tc.testSteps = append(tc.testSteps, step)
}

Expand All @@ -340,17 +337,16 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe
if isRequired {
failure := newFailure(ctx, *tc, stepNumber, rangedIndex, "", fmt.Errorf("At least one required assertion failed, skipping remaining steps"))
tsResult.appendFailure(*failure)
v.printTestStepResult(tc, tsResult, tsIn, ranged, stepNumber, true)
v.printTestStepResult(tc, tsResult, tsIn, stepNumber, true)
return
}
v.printTestStepResult(tc, tsResult, tsIn, ranged, stepNumber, false)
v.printTestStepResult(tc, tsResult, tsIn, stepNumber, false)
continue
}
v.printTestStepResult(tc, tsResult, tsIn, ranged, stepNumber, false)
v.printTestStepResult(tc, tsResult, tsIn, stepNumber, false)

allVars := tc.Vars.Clone()
allVars.AddAll(tc.computedVars.Clone())
tsResult.ComputedVars = tc.computedVars.Clone()
allVars.AddAll(tsResult.ComputedVars.Clone())

assign, _, err := processVariableAssignments(ctx, tc.Name, allVars, rawStep)
if err != nil {
Expand All @@ -375,43 +371,42 @@ func (v *Venom) setTestStepName(ts *TestStepResult, e ExecutorRunner, step TestS
}
}
if ranged.Enabled {
if rangedIndex == 0 {
v.Print("\n")
}
name = fmt.Sprintf("%s (range=%s)", name, rangedData.Key)
}
ts.Name = name

if print || ranged.Enabled {
if print {
v.Print(" \t\t• %s", ts.Name)
}
}

// Print a single step result (if verbosity is enabled)
func (v *Venom) printTestStepResult(tc *TestCase, ts *TestStepResult, tsIn *TestStepResult, ranged Range, stepNumber int, mustAssertionFailed bool) {
fromUserExecutor := tsIn != nil
if fromUserExecutor {
func (v *Venom) printTestStepResult(tc *TestCase, ts *TestStepResult, tsIn *TestStepResult, stepNumber int, mustAssertionFailed bool) {
if tsIn != nil {
tsIn.appendFailure(ts.Errors...)
}
if ranged.Enabled || v.Verbose >= 1 {
if !fromUserExecutor { //Else print step status
if len(ts.Errors) > 0 {
v.Println(" %s", Red(StatusFail))
for _, f := range ts.Errors {
v.Println(" \t\t %s", Yellow(f.Value))
}
if mustAssertionFailed {
skipped := len(tc.RawTestSteps) - stepNumber - 1
if skipped == 1 {
v.Println(" \t\t %s", Gray(fmt.Sprintf("%d other step was skipped", skipped)))
} else {
v.Println(" \t\t %s", Gray(fmt.Sprintf("%d other steps were skipped", skipped)))
}
} else if v.Verbose >= 1 {
if len(ts.Errors) > 0 {
v.Println(" %s", Red(StatusFail))
for _, i := range ts.ComputedInfo {
v.Println(" \t\t %s %s", Cyan("[info]"), Cyan(i))
}
for _, f := range ts.Errors {
v.Println(" \t\t %s", Yellow(f.Value))
}
if mustAssertionFailed {
skipped := len(tc.RawTestSteps) - stepNumber - 1
if skipped == 1 {
v.Println(" \t\t %s", Gray(fmt.Sprintf("%d other step was skipped", skipped)))
} else {
v.Println(" \t\t %s", Gray(fmt.Sprintf("%d other steps were skipped", skipped)))
}
} else if ts.Status == StatusSkip {
v.Println(" %s", Gray(StatusSkip))
} else {
v.Println(" %s", Green(StatusPass))
}
} else if ts.Status == StatusSkip {
v.Println(" %s", Gray(StatusSkip))
} else {
v.Println(" %s", Green(StatusPass))
for _, i := range ts.ComputedInfo {
v.Println(" \t\t %s %s", Cyan("[info]"), Cyan(i))
}
}
}
Expand Down
12 changes: 5 additions & 7 deletions process_teststep.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ type dumpFile struct {
}

// RunTestStep executes a venom testcase is a venom context
func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, tsResult *TestStepResult, stepNumber int, rangedIndex int, step TestStep) interface{} {
func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, tsResult *TestStepResult, stepNumber int, rangedIndex int, step TestStep) {
ctx = context.WithValue(ctx, ContextKey("executor"), e.Name())

var assertRes AssertionsApplied
Expand Down Expand Up @@ -65,7 +65,8 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase,
filename := path.Join(oDir, fmt.Sprintf("%s.%s.step.%d.%d.dump.json", slug.Make(StringVarFromCtx(ctx, "venom.testsuite.shortName")), slug.Make(tc.Name), stepNumber, rangedIndex))

if err := os.WriteFile(filename, []byte(output), 0644); err != nil {
return fmt.Errorf("Error while creating file %s: %v", filename, err)
Error(ctx, "Error while creating file %s: %v", filename, err)
return
}
tc.computedVerbose = append(tc.computedVerbose, fmt.Sprintf("writing %s", filename))
}
Expand All @@ -92,7 +93,6 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase,
}
}
Info(ctx, info)
tc.computedInfo = append(tc.computedInfo, info)
tsResult.ComputedInfo = append(tsResult.ComputedInfo, info)
}

Expand All @@ -108,12 +108,12 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase,
}

tsResult.AssertionsApplied = assertRes
tc.computedVars.AddAll(H(mapResult))
tsResult.ComputedVars.AddAll(H(mapResult))

if assertRes.OK {
break
}
failures, err := testConditionalStatement(ctx, tc, e.RetryIf(), tc.computedVars, "")
failures, err := testConditionalStatement(ctx, tc, e.RetryIf(), tsResult.ComputedVars, "")
if err != nil {
tsResult.appendError(fmt.Errorf("Error while evaluating retry condition: %v", err))
break
Expand All @@ -133,8 +133,6 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase,

tsResult.Systemerr += assertRes.systemerr + "\n"
tsResult.Systemout += assertRes.systemout + "\n"

return result
}

func (v *Venom) runTestStepExecutor(ctx context.Context, e ExecutorRunner, tc *TestCase, ts *TestStepResult, step TestStep) (interface{}, error) {
Expand Down
18 changes: 10 additions & 8 deletions process_testsuite.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) {

// Verbose mode already reported tests status, so just print them when non-verbose
indent := ""
if hasRanged || verboseReport {
if verboseReport {
indent = "\t "
// If the testcase was entirely skipped, then the verbose mode will not have any output
// Print something to inform that the testcase was indeed processed although skipped
Expand All @@ -163,19 +163,21 @@ func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) {
}
}

for _, i := range tc.computedInfo {
v.Println("\t %s%s %s", indent, Cyan("[info]"), Cyan(i))
}

for _, i := range tc.computedVerbose {
v.PrintlnIndentedTrace(i, indent)
}

// Verbose mode already reported failures, so just print them when non-verbose
if !hasRanged && !verboseReport && hasFailure {
if !verboseReport && hasFailure {
for _, testStepResult := range tc.TestStepResults {
for _, f := range testStepResult.Errors {
v.Println("%s", Yellow(f.Value))
if len(testStepResult.ComputedInfo) > 0 || len(testStepResult.Errors) > 0 {
v.Println(" \t\t• %s", testStepResult.Name)
for _, f := range testStepResult.ComputedInfo {
v.Println(" \t\t %s", Cyan(f))
}
for _, f := range testStepResult.Errors {
v.Println(" \t\t %s", Yellow(f.Value))
}
}
}
}
Expand Down
1 change: 0 additions & 1 deletion types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,6 @@ type TestCase struct {
TestSuiteVars H `json:"-" yaml:"-"`

computedVars H `json:"-" yaml:"-"`
computedInfo []string `json:"-" yaml:"-"`
computedVerbose []string `json:"-" yaml:"-"`
IsExecutor bool `json:"-" yaml:"-"`
IsEvaluated bool `json:"-" yaml:"-"`
Expand Down
4 changes: 0 additions & 4 deletions types_executor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,10 +276,6 @@ func (v *Venom) RunUserExecutor(ctx context.Context, runner ExecutorRunner, tcIn
return nil, err
}

// re-inject info into executorRunner
b := runner.(*executor)
b.info = append(b.info, tc.computedInfo...)

var outputResult interface{}
if err := yaml.Unmarshal([]byte(outputS), &outputResult); err != nil {
return nil, errors.Wrapf(err, "unable to unmarshal")
Expand Down