pass macro call location to macro-function (rebase)

NEWS.md

@@ -24,6 +24,16 @@ This section lists changes that do not have deprecation warnings.
   * `@__DIR__` returns the current working directory rather than `nothing` when not run
     from a file ([#21759]).
 
+  * `@__FILE__` and `@__DIR__` return information relative to the file that it was parsed from,
+    rather than from the task-local `SOURCE_PATH` global when it was expanded.
+
+  * All macros receive an extra argument `__source__::LineNumberNode` which describes the
+    parser location in the source file for the `@` of the macro call.
+    It can be accessed as a normal argument variable in the body of the macro.
+    This is implemented by inserting an extra leading argument into the
+    `Expr(:macrocall, :@name, LineNumberNode(...), args...)`
+    surface syntax. ([#21746])
+
   * Passing the same keyword argument multiple times is now a syntax error ([#16937]).
 
   * `getsockname` on a `TCPSocket` now returns the locally bound address and port

base/boot.jl

@@ -87,6 +87,7 @@
 
 #struct LineNumberNode
 #    line::Int
+#    file::Any # nominally Union{Symbol,Void}
 #end
 
 #struct LabelNode
@@ -281,7 +282,8 @@ _new(:GotoNode, :Int)
 _new(:NewvarNode, :SlotNumber)
 _new(:QuoteNode, :ANY)
 _new(:SSAValue, :Int)
-eval(:((::Type{LineNumberNode})(l::Int) = $(Expr(:new, :LineNumberNode, :l))))
+eval(:((::Type{LineNumberNode})(l::Int) = $(Expr(:new, :LineNumberNode, :l, nothing))))
+eval(:((::Type{LineNumberNode})(l::Int, f::ANY) = $(Expr(:new, :LineNumberNode, :l, :f))))
 eval(:((::Type{GlobalRef})(m::Module, s::Symbol) = $(Expr(:new, :GlobalRef, :m, :s))))
 eval(:((::Type{SlotNumber})(n::Int) = $(Expr(:new, :SlotNumber, :n))))
 eval(:((::Type{TypedSlot})(n::Int, t::ANY) = $(Expr(:new, :TypedSlot, :n, :t))))

base/docs/Docs.jl

@@ -83,9 +83,11 @@ function initmeta(m::Module = current_module())
 end
 
 function signature!(tv, expr::Expr)
-    if isexpr(expr, (:call, :macrocall))
+    is_macrocall = isexpr(expr, :macrocall)
+    if is_macrocall || isexpr(expr, :call)
         sig = :(Union{Tuple{}})
-        for arg in expr.args[2:end]
+        first_arg = is_macrocall ? 3 : 2 # skip function arguments
+        for arg in expr.args[first_arg:end]
             isexpr(arg, :parameters) && continue
             if isexpr(arg, :kw) # optional arg
                 push!(sig.args, :(Tuple{$(sig.args[end].args[2:end]...)}))
@@ -599,7 +601,7 @@ function __doc__!(meta, def, define)
         # the Base image). We just need to convert each `@__doc__` marker to an `@doc`.
         finddoc(def) do each
             each.head = :macrocall
-            each.args = [Symbol("@doc"), meta, each.args[end], define]
+            each.args = [Symbol("@doc"), nothing, meta, each.args[end], define] # TODO: forward line number info
         end
     else
         # `def` has already been defined during Base image gen so we just need to find and
@@ -642,7 +644,7 @@ const BINDING_HEADS = [:typealias, :const, :global, :(=)]  # deprecation: remove
 isquotedmacrocall(x) =
     isexpr(x, :copyast, 1) &&
     isa(x.args[1], QuoteNode) &&
-    isexpr(x.args[1].value, :macrocall, 1)
+    isexpr(x.args[1].value, :macrocall, 2)
 # Simple expressions / atoms the may be documented.
 isbasicdoc(x) = isexpr(x, :.) || isa(x, Union{QuoteNode, Symbol})
 is_signature(x) = isexpr(x, :call) || (isexpr(x, :(::), 2) && isexpr(x.args[1], :call)) || isexpr(x, :where)
@@ -730,7 +732,7 @@ function docm(ex)
         parsedoc(keywords[ex])
     elseif isa(ex, Union{Expr, Symbol})
         binding = esc(bindingexpr(namify(ex)))
-        if isexpr(ex, [:call, :macrocall])
+        if isexpr(ex, :call) || isexpr(ex, :macrocall)
             sig = esc(signature(ex))
             :($(doc)($binding, $sig))
         else
@@ -749,11 +751,21 @@ include("utils.jl")
 # Swap out the bootstrap macro with the real one.
 Core.atdoc!(docm)
 
+macro local_hygiene(expr)
+    # removes `esc` Exprs relative to the module argument to expand
+    # and resolves everything else relative to this (Doc) module
+    # this allows us to get good errors and backtraces
+    # from calling docm (by not using macros),
+    # while also getting macro-expansion correct (by using the macro-expander)
+    return expr
+end
 function loaddocs(docs)
+    unescape = GlobalRef(Docs, Symbol("@local_hygiene"))
     for (mod, ex, str, file, line) in docs
         data = Dict(:path => string(file), :linenumber => line)
         doc = docstr(str, data)
-        eval(mod, :(@doc($doc, $ex, false)))
+        docstring = eval(mod, Expr(:body, Expr(:return, Expr(:call, QuoteNode(docm), QuoteNode(doc), QuoteNode(ex), false)))) # expand the real @doc macro now (using a hack because macroexpand takes current-module as an implicit argument)
+        eval(mod, Expr(:macrocall, unescape, nothing, docstring))
     end
     empty!(docs)
 end

base/docs/basedocs.jl

@@ -644,13 +644,6 @@ to be set after construction. See `struct` and the manual for more information.
 """
 kw"mutable struct"
 
-"""
-    @__LINE__ -> Int
-
-`@__LINE__` expands to the line number of the call-site.
-"""
-kw"@__LINE__"
-
 """
     ans
 

base/docs/utils.jl

@@ -190,7 +190,7 @@ function repl(io::IO, s::Symbol)
         $(_repl(s))
     end
 end
-isregex(x) = isexpr(x, :macrocall, 2) && x.args[1] === Symbol("@r_str") && !isempty(x.args[2])
+isregex(x) = isexpr(x, :macrocall, 3) && x.args[1] === Symbol("@r_str") && !isempty(x.args[3])
 repl(io::IO, ex::Expr) = isregex(ex) ? :(apropos($io, $ex)) : _repl(ex)
 repl(io::IO, str::AbstractString) = :(apropos($io, $str))
 repl(io::IO, other) = :(@doc $(esc(other)))

base/exports.jl

@@ -1248,6 +1248,7 @@ export
     # parser internal
     @__FILE__,
     @__DIR__,
+    @__LINE__,
     @int128_str,
     @uint128_str,
     @big_str,

base/expr.jl

@@ -276,9 +276,16 @@ end
 
 remove_linenums!(ex) = ex
 function remove_linenums!(ex::Expr)
-    filter!(x->!((isa(x,Expr) && x.head === :line) || isa(x,LineNumberNode)), ex.args)
+    if ex.head === :body || ex.head === :block || ex.head === :quote
+        # remove line number expressions from metadata (not argument literal or inert) position
+        filter!(ex.args) do x
+            isa(x, Expr) && x.head === :line && return false
+            isa(x, LineNumberNode) && return false
+            return true
+        end
+    end
     for subex in ex.args
         remove_linenums!(subex)
     end
-    ex
+    return ex
 end

base/inference.jl

@@ -4140,39 +4140,38 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference
     end
 
     do_coverage = coverage_enabled()
-    if do_coverage
-        line = method.line
-        if !isempty(stmts) && isa(stmts[1], LineNumberNode)
-            line = (shift!(stmts)::LineNumberNode).line
+    line::Int = method.line
+    file = method.file
+    if !isempty(stmts)
+        if !do_coverage && all(inlining_ignore, stmts)
+            empty!(stmts)
+        elseif isa(stmts[1], LineNumberNode)
+            linenode = shift!(stmts)::LineNumberNode
+            line = linenode.line
+            isa(linenode.file, Symbol) && (file = linenode.file)
         end
+    end
+    if do_coverage
         # Check if we are switching module, which is necessary to catch user
         # code inlined into `Base` with `--code-coverage=user`.
         # Assume we are inlining directly into `enclosing` instead of another
         # function inlined in it
         mod = method.module
         if mod === sv.mod
-            unshift!(stmts, Expr(:meta, :push_loc, method.file,
+            unshift!(stmts, Expr(:meta, :push_loc, file,
                                  method.name, line))
         else
-            unshift!(stmts, Expr(:meta, :push_loc, method.file,
+            unshift!(stmts, Expr(:meta, :push_loc, file,
                                  method.name, line, mod))
         end
         push!(stmts, Expr(:meta, :pop_loc))
     elseif !isempty(stmts)
-        if all(inlining_ignore, stmts)
-            empty!(stmts)
+        unshift!(stmts, Expr(:meta, :push_loc, file,
+                             method.name, line))
+        if isa(stmts[end], LineNumberNode)
+            stmts[end] = Expr(:meta, :pop_loc)
         else
-            line::Int = method.line
-            if isa(stmts[1], LineNumberNode)
-                line = (shift!(stmts)::LineNumberNode).line
-            end
-            unshift!(stmts, Expr(:meta, :push_loc, method.file,
-                                 method.name, line))
-            if isa(stmts[end], LineNumberNode)
-                stmts[end] = Expr(:meta, :pop_loc)
-            else
-                push!(stmts, Expr(:meta, :pop_loc))
-            end
+            push!(stmts, Expr(:meta, :pop_loc))
         end
     end
     if !isempty(stmts) && !propagate_inbounds

base/interactiveutil.jl

@@ -349,7 +349,7 @@ function code_warntype(io::IO, f, t::ANY)
 end
 code_warntype(f, t::ANY) = code_warntype(STDOUT, f, t)
 
-typesof(args...) = Tuple{map(a->(isa(a,Type) ? Type{a} : typeof(a)), args)...}
+typesof(args...) = Tuple{Any[ Core.Typeof(a) for a in args ]...}
 
 gen_call_with_extracted_types(fcn, ex0::Symbol) = Expr(:call, fcn, Meta.quot(ex0))
 function gen_call_with_extracted_types(fcn, ex0)
@@ -371,9 +371,9 @@ function gen_call_with_extracted_types(fcn, ex0)
     exret = Expr(:none)
     is_macro = false
     ex = expand(ex0)
-    if isa(ex0, Expr) && ex0.head == :macrocall # Make @edit @time 1+2 edit the macro
+    if isa(ex0, Expr) && ex0.head == :macrocall # Make @edit @time 1+2 edit the macro by using the types of the *expressions*
         is_macro = true
-        exret = Expr(:call, fcn,  esc(ex0.args[1]), typesof(ex0.args[2:end]...))
+        exret = Expr(:call, fcn, esc(ex0.args[1]), Tuple{#=__source__=#LineNumberNode, Any[ Core.Typeof(a) for a in ex0.args[3:end] ]...})
     elseif !isa(ex, Expr)
         exret = Expr(:call, :error, "expression is not a function call or symbol")
     elseif ex.head == :call

base/loading.jl

@@ -538,24 +538,6 @@ function source_dir()
     p === nothing ? pwd() : dirname(p)
 end
 
-"""
-    @__FILE__ -> AbstractString
-
-`@__FILE__` expands to a string with the absolute file path of the file containing the
-macro. Returns `nothing` if run from a REPL or an empty string if evaluated by
-`julia -e <expr>`. Alternatively see [`PROGRAM_FILE`](@ref).
-"""
-macro __FILE__() source_path() end
-
-"""
-    @__DIR__ -> AbstractString
-
-`@__DIR__` expands to a string with the directory part of the absolute path of the file
-containing the macro. Returns the current working directory if run from a REPL or if
-evaluated by `julia -e <expr>`.
-"""
-macro __DIR__() source_dir() end
-
 include_from_node1(path::AbstractString) = include_from_node1(String(path))
 function include_from_node1(_path::String)
     path, prev = _include_dependency(_path)
@@ -823,3 +805,38 @@ function stale_cachefile(modpath::String, cachefile::String)
         close(io)
     end
 end
+
+"""
+    @__LINE__ -> Int
+
+`@__LINE__` expands to the line number of the location of the macrocall.
+Returns `0` if the line number could not be determined.
+"""
+macro __LINE__()
+    return __source__.line
+end
+
+"""
+    @__FILE__ -> AbstractString
+
+`@__FILE__` expands to a string with the path to the file containing the
+macrocall, or an empty string if evaluated by `julia -e <expr>`.
+Returns `nothing` if the macro was missing parser source information.
+Alternatively see [`PROGRAM_FILE`](@ref).
+"""
+macro __FILE__()
+    __source__.file === nothing && return nothing
+    return String(__source__.file)
+end
+
+"""
+    @__DIR__ -> AbstractString
+
+`@__DIR__` expands to a string with the absolute path to the directory of the file
+containing the macrocall.
+Returns the current working directory if run from a REPL or if evaluated by `julia -e <expr>`.
+"""
+macro __DIR__()
+    __source__.file === nothing && return nothing
+    return abspath(dirname(String(__source__.file)))
+end

base/math.jl

@@ -124,7 +124,7 @@ macro evalpoly(z, p...)
              :(s = muladd(x, x, y*y)),
              as...,
              :(muladd($ai, tt, $b)))
-    R = Expr(:macrocall, Symbol("@horner"), :tt, map(esc, p)...)
+    R = Expr(:macrocall, Symbol("@horner"), (), :tt, map(esc, p)...)
     :(let tt = $(esc(z))
           isa(tt, Complex) ? $C : $R
       end)

base/show.jl

@@ -505,10 +505,6 @@ const prec_decl = operator_precedence(:(::))
 is_expr(ex, head::Symbol)         = (isa(ex, Expr) && (ex.head == head))
 is_expr(ex, head::Symbol, n::Int) = is_expr(ex, head) && length(ex.args) == n
 
-is_linenumber(ex::LineNumberNode) = true
-is_linenumber(ex::Expr)           = (ex.head == :line)
-is_linenumber(ex)                 = false
-
 is_quoted(ex)            = false
 is_quoted(ex::QuoteNode) = true
 is_quoted(ex::Expr)      = is_expr(ex, :quote, 1) || is_expr(ex, :inert, 1)
@@ -538,18 +534,22 @@ end
 
 emphasize(io, str::AbstractString) = have_color ? print_with_color(Base.error_color(), io, str; bold = true) : print(io, uppercase(str))
 
-show_linenumber(io::IO, line)       = print(io," # line ",line,':')
-show_linenumber(io::IO, line, file) = print(io," # ", file,", line ",line,':')
+show_linenumber(io::IO, line)       = print(io, "#= line ", line, " =#")
+show_linenumber(io::IO, line, file) = print(io, "#= ", file, ":", line, " =#")
+show_linenumber(io::IO, line, file::Void) = show_linenumber(io, line)
 
 # show a block, e g if/for/etc
 function show_block(io::IO, head, args::Vector, body, indent::Int)
-    print(io, head, ' ')
-    show_list(io, args, ", ", indent)
+    print(io, head)
+    if !isempty(args)
+        print(io, ' ')
+        show_list(io, args, ", ", indent)
+    end
 
     ind = head === :module || head === :baremodule ? indent : indent + indent_width
     exs = (is_expr(body, :block) || is_expr(body, :body)) ? body.args : Any[body]
     for ex in exs
-        if !is_linenumber(ex); print(io, '\n', " "^ind); end
+        print(io, '\n', " "^ind)
         show_unquoted(io, ex, ind, -1)
     end
     print(io, '\n', " "^indent)
@@ -566,7 +566,7 @@ end
 # show an indented list
 function show_list(io::IO, items, sep, indent::Int, prec::Int=0, enclose_operators::Bool=false)
     n = length(items)
-    if n == 0; return end
+    n == 0 && return
     indent += indent_width
     first = true
     for item in items
@@ -613,7 +613,7 @@ end
 ## AST printing ##
 
 show_unquoted(io::IO, sym::Symbol, ::Int, ::Int)        = print(io, sym)
-show_unquoted(io::IO, ex::LineNumberNode, ::Int, ::Int) = show_linenumber(io, ex.line)
+show_unquoted(io::IO, ex::LineNumberNode, ::Int, ::Int) = show_linenumber(io, ex.line, ex.file)
 show_unquoted(io::IO, ex::LabelNode, ::Int, ::Int)      = print(io, ex.label, ": ")
 show_unquoted(io::IO, ex::GotoNode, ::Int, ::Int)       = print(io, "goto ", ex.label)
 show_unquoted(io::IO, ex::GlobalRef, ::Int, ::Int)      = print(io, ex.mod, '.', ex.name)
@@ -913,12 +913,20 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int)
         print(io, head, ' ')
         show_list(io, args, ", ", indent)
 
-    elseif head === :macrocall && nargs >= 1
+    elseif head === :macrocall && nargs >= 2
+        # first show the line number argument as a comment
+        if isa(args[2], LineNumberNode) || is_expr(args[2], :line)
+            print(io, args[2], ' ')
+        end
         # Use the functional syntax unless specifically designated with prec=-1
+        # and hide the line number argument from the argument list
         if prec >= 0
-            show_call(io, :call, ex.args[1], ex.args[2:end], indent)
+            show_call(io, :call, args[1], args[3:end], indent)
         else
-            show_list(io, args, ' ', indent)
+            show_args = Vector{Any}(length(args) - 1)
+            show_args[1] = args[1]
+            show_args[2:end] = args[3:end]
+            show_list(io, show_args, ' ', indent)
         end
 
     elseif head === :line && 1 <= nargs <= 2

doc/REQUIRE

@@ -1,3 +1,3 @@
 Compat 0.25.0 0.25.0+
 DocStringExtensions 0.3.3 0.3.3+
-Documenter 0.10.2 0.10.2+
+Documenter 0.10.3 0.10.3+