Skip to content

Commit

Permalink
Support @test_throws just checking the error message (#41888)
Browse files Browse the repository at this point in the history 
With this PR,

    @test_throws "reducing over an empty collection" reduce(+, ())

allows you to check the displayed error message without caring about
the details of which specific Exception subtype is used.

The pattern-matching options are the same as for `@test_warn`.

Co-authored by: Jameson Nash <[email protected]>
Co-authored by: Takafumi Arakaki <[email protected]>
  • Loading branch information
@timholy
timholy authored Aug 17, 2021
1 parent b7f7708 commit a03392a
Show file tree
Hide file tree
Showing 3 changed files with 91 additions and 14 deletions.
3 changes: 3 additions & 0 deletions NEWS.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ New library functions
New library features
--------------------

* `@test_throws "some message" triggers_error()` can now be used to check whether the displayed error text
contains "some message" regardless of the specific exception type.
Regular expressions, lists of strings, and matching functions are also supported. ([#41888)

Standard library changes
------------------------
Expand Down
64 changes: 51 additions & 13 deletions stdlib/Test/src/Test.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,9 @@ struct Pass <: Result
data
value
source::Union{Nothing,LineNumberNode}
function Pass(test_type::Symbol, orig_expr, data, thrown, source=nothing)
return new(test_type, orig_expr, data, thrown isa String ? "String" : thrown, source)
message_only::Bool
function Pass(test_type::Symbol, orig_expr, data, thrown, source=nothing, message_only=false)
return new(test_type, orig_expr, data, thrown, source, message_only)
end
end

Expand All @@ -98,7 +99,11 @@ function Base.show(io::IO, t::Pass)
end
if t.test_type === :test_throws
# The correct type of exception was thrown
print(io, "\n Thrown: ", t.value isa String ? t.value : typeof(t.value))
if t.message_only
print(io, "\n Message: ", t.value)
else
print(io, "\n Thrown: ", typeof(t.value))
end
elseif t.test_type === :test && t.data !== nothing
# The test was an expression, so display the term-by-term
# evaluated version as well
Expand All @@ -118,12 +123,14 @@ struct Fail <: Result
data::Union{Nothing, String}
value::String
source::LineNumberNode
function Fail(test_type::Symbol, orig_expr, data, value, source::LineNumberNode)
message_only::Bool
function Fail(test_type::Symbol, orig_expr, data, value, source::LineNumberNode, message_only::Bool=false)
return new(test_type,
string(orig_expr),
data === nothing ? nothing : string(data),
string(isa(data, Type) ? typeof(value) : value),
source)
source,
message_only)
end
end

Expand All @@ -132,18 +139,24 @@ function Base.show(io::IO, t::Fail)
print(io, " at ")
printstyled(io, something(t.source.file, :none), ":", t.source.line, "\n"; bold=true, color=:default)
print(io, " Expression: ", t.orig_expr)
value, data = t.value, t.data
if t.test_type === :test_throws_wrong
# An exception was thrown, but it was of the wrong type
print(io, "\n Expected: ", t.data)
print(io, "\n Thrown: ", t.value)
if t.message_only
print(io, "\n Expected: ", data)
print(io, "\n Message: ", value)
else
print(io, "\n Expected: ", data)
print(io, "\n Thrown: ", value)
end
elseif t.test_type === :test_throws_nothing
# An exception was expected, but no exception was thrown
print(io, "\n Expected: ", t.data)
print(io, "\n Expected: ", data)
print(io, "\n No exception thrown")
elseif t.test_type === :test && t.data !== nothing
elseif t.test_type === :test && data !== nothing
# The test was an expression, so display the term-by-term
# evaluated version as well
print(io, "\n Evaluated: ", t.data)
print(io, "\n Evaluated: ", data)
end
end

Expand Down Expand Up @@ -238,6 +251,7 @@ function Serialization.serialize(s::Serialization.AbstractSerializer, t::Pass)
Serialization.serialize(s, t.data === nothing ? nothing : string(t.data))
Serialization.serialize(s, string(t.value))
Serialization.serialize(s, t.source === nothing ? nothing : t.source)
Serialization.serialize(s, t.message_only)
nothing
end

Expand Down Expand Up @@ -657,6 +671,8 @@ end
Tests that the expression `expr` throws `exception`.
The exception may specify either a type,
a string, regular expression, or list of strings occurring in the displayed error message,
a matching function,
or a value (which will be tested for equality by comparing fields).
Note that `@test_throws` does not support a trailing keyword form.
Expand All @@ -671,7 +687,18 @@ julia> @test_throws DimensionMismatch [1, 2, 3] + [1, 2]
Test Passed
Expression: [1, 2, 3] + [1, 2]
Thrown: DimensionMismatch
julia> @test_throws "Try sqrt(Complex" sqrt(-1)
Test Passed
Expression: sqrt(-1)
Message: "DomainError with -1.0:\\nsqrt will only return a complex result if called with a complex argument. Try sqrt(Complex(x))."
```
In the final example, instead of matching a single string it could alternatively have been performed with:
- `["Try", "Complex"]` (a list of strings)
- `r"Try sqrt\\([Cc]omplex"` (a regular expression)
- `str -> occursin("complex", str)` (a matching function)
"""
macro test_throws(extype, ex)
orig_ex = Expr(:inert, ex)
Expand All @@ -697,6 +724,7 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype)
if isa(result, Threw)
# Check that the right type of exception was thrown
success = false
message_only = false
exc = result.exception
# NB: Throwing LoadError from macroexpands is deprecated, but in order to limit
# the breakage in package tests we add extra logic here.
Expand All @@ -712,7 +740,7 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype)
else
isa(exc, extype)
end
else
elseif isa(extype, Exception) || !isa(exc, Exception)
if extype isa LoadError && !(exc isa LoadError) && typeof(extype.error) == typeof(exc)
extype = extype.error # deprecated
end
Expand All @@ -725,11 +753,21 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype)
end
end
end
else
message_only = true
exc = sprint(showerror, exc)
success = contains_warn(exc, extype)
exc = repr(exc)
if isa(extype, AbstractString)
extype = repr(extype)
elseif isa(extype, Function)
extype = "< match function >"
end
end
if success
testres = Pass(:test_throws, orig_expr, extype, exc, result.source)
testres = Pass(:test_throws, orig_expr, extype, exc, result.source, message_only)
else
testres = Fail(:test_throws_wrong, orig_expr, extype, exc, result.source)
testres = Fail(:test_throws_wrong, orig_expr, extype, exc, result.source, message_only)
end
else
testres = Fail(:test_throws_nothing, orig_expr, extype, nothing, result.source)
Expand Down
38 changes: 37 additions & 1 deletion stdlib/Test/test/runtests.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,16 @@ end
"Thrown: ErrorException")
@test endswith(sprint(show, @test_throws ErrorException("test") error("test")),
"Thrown: ErrorException")
@test endswith(sprint(show, @test_throws "a test" error("a test")),
"Message: \"a test\"")
@test occursin("Message: \"DomainError",
sprint(show, @test_throws r"sqrt\([Cc]omplex" sqrt(-1)))
@test endswith(sprint(show, @test_throws str->occursin("a t", str) error("a test")),
"Message: \"a test\"")
@test endswith(sprint(show, @test_throws ["BoundsError", "access", "1-element", "at index [2]"] [1][2]),
"Message: \"BoundsError: attempt to access 1-element Vector{$Int} at index [2]\"")
@test_throws "\"" throw("\"")
@test_throws Returns(false) throw(Returns(false))
end
# Test printing of Fail results
include("nothrow_testset.jl")
Expand Down Expand Up @@ -148,6 +158,11 @@ let fails = @testset NoThrowTestSet begin
@test contains(str1, str2)
# 22 - Fail - Type Comparison
@test typeof(1) <: typeof("julia")
# 23 - 26 - Fail - wrong message
@test_throws "A test" error("a test")
@test_throws r"sqrt\([Cc]omplx" sqrt(-1)
@test_throws str->occursin("a T", str) error("a test")
@test_throws ["BoundsError", "acess", "1-element", "at index [2]"] [1][2]
end
for fail in fails
@test fail isa Test.Fail
Expand Down Expand Up @@ -262,6 +277,27 @@ let fails = @testset NoThrowTestSet begin
@test occursin("Expression: typeof(1) <: typeof(\"julia\")", str)
@test occursin("Evaluated: $(typeof(1)) <: $(typeof("julia"))", str)
end

let str = sprint(show, fails[23])
@test occursin("Expected: \"A test\"", str)
@test occursin("Message: \"a test\"", str)
end

let str = sprint(show, fails[24])
@test occursin("Expected: r\"sqrt\\([Cc]omplx\"", str)
@test occursin(r"Message: .*Try sqrt\(Complex", str)
end

let str = sprint(show, fails[25])
@test occursin("Expected: < match function >", str)
@test occursin("Message: \"a test\"", str)
end

let str = sprint(show, fails[26])
@test occursin("Expected: [\"BoundsError\", \"acess\", \"1-element\", \"at index [2]\"]", str)
@test occursin(r"Message: \"BoundsError.* 1-element.*at index \[2\]", str)
end

end

let errors = @testset NoThrowTestSet begin
Expand Down Expand Up @@ -1202,4 +1238,4 @@ Test.finish(ts::PassInformationTestSet) = ts
@test ts.results[2].data == ErrorException
@test ts.results[2].value == ErrorException("Msg")
@test ts.results[2].source == LineNumberNode(test_throws_line_number, @__FILE__)
end
end

0 comments on commit a03392a

Please sign in to comment.