Skip to content

Commit

Permalink
layout: support pointers inlining into parents [NFCI] (#33886)
Browse files Browse the repository at this point in the history
Includes codegen support for immutable objects that contain pointers
appearing on stack (well, in registers, since LLVM support of
non-integral addrspace pointers inside aggregates in memory is poor),
and includes layout support, so that most (non-self-referential)
immutable objects can be stored inline inside their parent allocation.

Currently fully disabled, aside from some optimizations and improvements
to object_id / isa tests.

Co-Authored-By: Oscar Blumberg <oscar.blumberg@ens.fr>
  • Loading branch information
2 people authored and JeffBezanson committed Dec 17, 2019
1 parent fdcaa06 commit 630a551
Showing 25 changed files with 1,072 additions and 412 deletions.
79 changes: 44 additions & 35 deletions base/array.jl
Original file line number Diff line number Diff line change
@@ -158,7 +158,7 @@ size(a::Array{<:Any,N}) where {N} = (@_inline_meta; ntuple(M -> size(a, M), Val(

asize_from(a::Array, n) = n > ndims(a) ? () : (arraysize(a,n), asize_from(a, n+1)...)

allocatedinline(::Type{T}) where {T} = (@_pure_meta; ccall(:jl_array_store_unboxed, Cint, (Any,), T) != Cint(0))
allocatedinline(::Type{T}) where {T} = (@_pure_meta; ccall(:jl_stored_inline, Cint, (Any,), T) != Cint(0))

"""
Base.isbitsunion(::Type{T})
@@ -177,14 +177,20 @@ false
isbitsunion(u::Union) = allocatedinline(u)
isbitsunion(x) = false

function _unsetindex!(A::Array{T}, i::Int) where {T}
@inbounds function _unsetindex!(A::Array{T}, i::Int) where {T}
@boundscheck checkbounds(A, i)
t = @_gc_preserve_begin A
p = Ptr{Ptr{Cvoid}}(pointer(A, i))
if !allocatedinline(T)
t = @_gc_preserve_begin A
p = Ptr{Ptr{Cvoid}}(pointer(A))
unsafe_store!(p, C_NULL, i)
@_gc_preserve_end t
unsafe_store!(p, C_NULL)
elseif T isa DataType
if !datatype_pointerfree(T)
for j = 1:(Core.sizeof(T) ÷ Core.sizeof(Ptr{Cvoid}))
unsafe_store!(p, C_NULL, j)
end
end
end
@_gc_preserve_end t
return A
end

@@ -255,19 +261,41 @@ the same manner as C.
function unsafe_copyto!(dest::Array{T}, doffs, src::Array{T}, soffs, n) where T
t1 = @_gc_preserve_begin dest
t2 = @_gc_preserve_begin src
if isbitsunion(T)
destp = pointer(dest, doffs)
srcp = pointer(src, soffs)
if !allocatedinline(T)
ccall(:jl_array_ptr_copy, Cvoid, (Any, Ptr{Cvoid}, Any, Ptr{Cvoid}, Int),
dest, destp, src, srcp, n)
elseif isbitstype(T)
ccall(:memmove, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
pointer(dest, doffs), pointer(src, soffs), n * aligned_sizeof(T))
destp, srcp, n * aligned_sizeof(T))
elseif isbitsunion(T)
ccall(:memmove, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
destp, srcp, n * aligned_sizeof(T))
# copy selector bytes
ccall(:memmove, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), dest) + doffs - 1,
ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), src) + soffs - 1,
n)
elseif allocatedinline(T)
unsafe_copyto!(pointer(dest, doffs), pointer(src, soffs), n)
else
ccall(:jl_array_ptr_copy, Cvoid, (Any, Ptr{Cvoid}, Any, Ptr{Cvoid}, Int),
dest, pointer(dest, doffs), src, pointer(src, soffs), n)
# handle base-case: everything else above was just optimizations
@inbounds if destp < srcp || destp > srcp + n
for i = 1:n
if isassigned(src, soffs + i - 1)
dest[doffs + i - 1] = src[soffs + i - 1]
else
_unsetindex!(dest, doffs + i - 1)
end
end
else
for i = n:-1:1
if isassigned(src, soffs + i - 1)
dest[doffs + i - 1] = src[soffs + i - 1]
else
_unsetindex!(dest, doffs + i - 1)
end
end
end
end
@_gc_preserve_end t2
@_gc_preserve_end t1
@@ -1566,32 +1594,13 @@ function vcat(arrays::Vector{T}...) where T
n += length(a)
end
arr = Vector{T}(undef, n)
ptr = pointer(arr)
if isbitsunion(T)
selptr = ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), arr)
end
elsz = aligned_sizeof(T)
t = @_gc_preserve_begin arr
nd = 1
for a in arrays
na = length(a)
nba = na * elsz
if isbitsunion(T)
ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
ptr, a, nba)
# copy selector bytes
ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
selptr, ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), a), na)
selptr += na
elseif allocatedinline(T)
ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
ptr, a, nba)
else
ccall(:jl_array_ptr_copy, Cvoid, (Any, Ptr{Cvoid}, Any, Ptr{Cvoid}, Int),
arr, ptr, a, pointer(a), na)
end
ptr += nba
@assert nd + na <= 1 + length(arr) # Concurrent modification of arrays?
unsafe_copyto!(arr, nd, a, 1, na)
nd += na
end
@_gc_preserve_end t
return arr
end

1 change: 1 addition & 0 deletions base/reflection.jl
Original file line number Diff line number Diff line change
@@ -318,6 +318,7 @@ datatype_fieldtypes(x::DataType) = ccall(:jl_get_fieldtypes, Any, (Any,), x)
struct DataTypeLayout
nfields::UInt32
npointers::UInt32
firstptr::Int32
alignment::UInt32
# alignment : 9;
# haspadding : 1;
2 changes: 1 addition & 1 deletion base/refpointer.jl
Original file line number Diff line number Diff line change
@@ -74,7 +74,7 @@ RefArray(x::AbstractArray{T}, i::Int=1, roots::Nothing=nothing) where {T} = RefA
convert(::Type{Ref{T}}, x::AbstractArray{T}) where {T} = RefArray(x, 1)

function unsafe_convert(P::Type{Ptr{T}}, b::RefArray{T}) where T
if datatype_pointerfree(RefValue{T})
if allocatedinline(T)
p = pointer(b.x, b.i)
elseif isconcretetype(T) && T.mutable
p = pointer_from_objref(b.x[b.i])
2 changes: 1 addition & 1 deletion base/refvalue.jl
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@ RefValue(x::T) where {T} = RefValue{T}(x)
isassigned(x::RefValue) = isdefined(x, :x)

function unsafe_convert(P::Type{Ptr{T}}, b::RefValue{T}) where T
if datatype_pointerfree(RefValue{T})
if allocatedinline(T)
p = pointer_from_objref(b)
elseif isconcretetype(T) && T.mutable
p = pointer_from_objref(b.x)
70 changes: 47 additions & 23 deletions src/array.c
Original file line number Diff line number Diff line change
@@ -27,12 +27,6 @@ char *jl_array_typetagdata(jl_array_t *a) JL_NOTSAFEPOINT
return ((char*)jl_array_data(a)) + ((jl_array_ndims(a) == 1 ? (a->maxsize - a->offset) : jl_array_len(a)) * a->elsize) + a->offset;
}

JL_DLLEXPORT int jl_array_store_unboxed(jl_value_t *eltype) JL_NOTSAFEPOINT
{
size_t fsz = 0, al = 0;
return jl_islayout_inline(eltype, &fsz, &al);
}

STATIC_INLINE jl_value_t *jl_array_owner(jl_array_t *a JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT
{
if (a->flags.how == 3) {
@@ -53,7 +47,7 @@ size_t jl_arr_xtralloc_limit = 0;
#define MAXINTVAL (((size_t)-1)>>1)

static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
int isunboxed, int isunion, int elsz)
int isunboxed, int hasptr, int isunion, int elsz)
{
jl_ptls_t ptls = jl_get_ptls_states();
size_t i, tot, nel=1;
@@ -101,7 +95,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
// No allocation or safepoint allowed after this
a->flags.how = 0;
data = (char*)a + doffs;
if ((tot > 0 && !isunboxed) || isunion)
if (tot > 0 && (!isunboxed || hasptr || isunion)) // TODO: check for zeroinit
memset(data, 0, tot);
}
else {
@@ -113,7 +107,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
// No allocation or safepoint allowed after this
a->flags.how = 2;
jl_gc_track_malloced_array(ptls, a);
if (!isunboxed || isunion)
if (tot > 0 && (!isunboxed || hasptr || isunion)) // TODO: check for zeroinit
// need to zero out isbits union array selector bytes to ensure a valid type index
memset(data, 0, tot);
}
@@ -127,6 +121,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
#endif
a->flags.ndims = ndims;
a->flags.ptrarray = !isunboxed;
a->flags.hasptr = hasptr;
a->elsize = elsz;
a->flags.isshared = 0;
a->flags.isaligned = 1;
@@ -135,9 +130,12 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
a->nrows = nel;
a->maxsize = nel;
}
else if (a->flags.ndims != ndims) {
jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions");
}
else {
size_t *adims = &a->nrows;
for(i=0; i < ndims; i++)
for (i = 0; i < ndims; i++)
adims[i] = dims[i];
}

@@ -152,6 +150,7 @@ static inline jl_array_t *_new_array(jl_value_t *atype, uint32_t ndims, size_t *
jl_type_error_rt("Array", "element type", (jl_value_t*)jl_type_type, eltype);
int isunboxed = jl_islayout_inline(eltype, &elsz, &al);
int isunion = jl_is_uniontype(eltype);
int hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0);
if (!isunboxed) {
elsz = sizeof(void*);
al = elsz;
@@ -160,13 +159,13 @@ static inline jl_array_t *_new_array(jl_value_t *atype, uint32_t ndims, size_t *
elsz = LLT_ALIGN(elsz, al);
}

return _new_array_(atype, ndims, dims, isunboxed, isunion, elsz);
return _new_array_(atype, ndims, dims, isunboxed, hasptr, isunion, elsz);
}

jl_array_t *jl_new_array_for_deserialization(jl_value_t *atype, uint32_t ndims, size_t *dims,
int isunboxed, int isunion, int elsz)
int isunboxed, int hasptr, int isunion, int elsz)
{
return _new_array_(atype, ndims, dims, isunboxed, isunion, elsz);
return _new_array_(atype, ndims, dims, isunboxed, hasptr, isunion, elsz);
}

#ifndef JL_NDEBUG
@@ -224,10 +223,12 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data,
"reinterpret from alignment %d bytes to alignment %d bytes not allowed",
(int) oldalign, (int) align);
a->flags.ptrarray = 0;
a->flags.hasptr = data->flags.hasptr;
}
else {
a->elsize = sizeof(void*);
a->flags.ptrarray = 1;
a->flags.hasptr = 0;
}

// if data is itself a shared wrapper,
@@ -247,6 +248,9 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data,
a->nrows = l;
a->maxsize = l;
}
else if (a->flags.ndims != ndims) {
jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions");
}
else {
size_t *adims = &a->nrows;
size_t l = 1;
@@ -281,6 +285,7 @@ JL_DLLEXPORT jl_array_t *jl_string_to_array(jl_value_t *str)
a->flags.isaligned = 0;
a->elsize = 1;
a->flags.ptrarray = 0;
a->flags.hasptr = 0;
jl_array_data_owner(a) = str;
a->flags.how = 3;
a->flags.isshared = 1;
@@ -300,12 +305,12 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
jl_array_t *a;
jl_value_t *eltype = jl_tparam0(atype);

int isunboxed = jl_array_store_unboxed(eltype);
size_t elsz;
unsigned align;
int isunboxed = jl_stored_inline(eltype);
if (isunboxed && jl_is_uniontype(eltype))
jl_exceptionf(jl_argumenterror_type,
"unsafe_wrap: unspecified layout for union element type");
size_t elsz;
unsigned align;
if (isunboxed) {
elsz = jl_datatype_size(eltype);
align = jl_datatype_align(eltype);
@@ -328,6 +333,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
#endif
a->elsize = LLT_ALIGN(elsz, align);
a->flags.ptrarray = !isunboxed;
a->flags.hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0);
a->flags.ndims = 1;
a->flags.isshared = 1;
a->flags.isaligned = 0; // TODO: allow passing memalign'd buffers
@@ -366,12 +372,12 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
return jl_ptr_to_array_1d(atype, data, nel, own_buffer);
jl_value_t *eltype = jl_tparam0(atype);

int isunboxed = jl_array_store_unboxed(eltype);
size_t elsz;
unsigned align;
int isunboxed = jl_stored_inline(eltype);
if (isunboxed && jl_is_uniontype(eltype))
jl_exceptionf(jl_argumenterror_type,
"unsafe_wrap: unspecified layout for union element type");
size_t elsz;
unsigned align;
if (isunboxed) {
elsz = jl_datatype_size(eltype);
align = jl_datatype_align(eltype);
@@ -394,6 +400,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
#endif
a->elsize = LLT_ALIGN(elsz, align);
a->flags.ptrarray = !isunboxed;
a->flags.hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0);
a->flags.ndims = ndims;
a->offset = 0;
a->flags.isshared = 1;
@@ -408,6 +415,8 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
}

assert(ndims != 1); // handled above
if (a->flags.ndims != ndims)
jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions");
memcpy(&a->nrows, dims, ndims * sizeof(size_t));
return a;
}
@@ -559,8 +568,16 @@ JL_DLLEXPORT jl_value_t *jl_arrayref(jl_array_t *a, size_t i)

JL_DLLEXPORT int jl_array_isassigned(jl_array_t *a, size_t i)
{
if (a->flags.ptrarray)
if (a->flags.ptrarray) {
return ((jl_value_t**)jl_array_data(a))[i] != NULL;
}
else if (a->flags.hasptr) {
jl_datatype_t *eltype = (jl_datatype_t*)jl_tparam0(jl_typeof(a));
assert(eltype->layout->first_ptr >= 0);
jl_value_t **slot =
(jl_value_t**)(&((char*)a->data)[i*a->elsize] + eltype->layout->first_ptr);
return *slot != NULL;
}
return 1;
}

@@ -585,6 +602,8 @@ JL_DLLEXPORT void jl_arrayset(jl_array_t *a JL_ROOTING_ARGUMENT, jl_value_t *rhs
return;
}
jl_assign_bits(&((char*)a->data)[i * a->elsize], rhs);
if (a->flags.hasptr)
jl_gc_multi_wb(jl_array_owner(a), rhs);
}
else {
((jl_value_t**)a->data)[i] = rhs;
@@ -598,6 +617,11 @@ JL_DLLEXPORT void jl_arrayunset(jl_array_t *a, size_t i)
jl_bounds_error_int((jl_value_t*)a, i + 1);
if (a->flags.ptrarray)
((jl_value_t**)a->data)[i] = NULL;
else if (a->flags.hasptr) {
size_t elsize = a->elsize;
jl_assume(elsize >= sizeof(void*) && elsize % sizeof(void*) == 0);
memset(&((jl_value_t**)a->data)[i], 0, elsize);
}
}

// at this size and bigger, allocate resized array data with malloc directly
@@ -809,7 +833,7 @@ STATIC_INLINE void jl_array_grow_at_beg(jl_array_t *a, size_t idx, size_t inc,
#endif
a->nrows = newnrows;
a->data = newdata;
if (a->flags.ptrarray) {
if (a->flags.ptrarray || a->flags.hasptr) { // TODO: check for zeroinit
memset(newdata + idx * elsz, 0, nbinc);
}
else if (isbitsunion) {
@@ -890,7 +914,7 @@ STATIC_INLINE void jl_array_grow_at_end(jl_array_t *a, size_t idx,
a->length = newnrows;
#endif
a->nrows = newnrows;
if (a->flags.ptrarray) {
if (a->flags.ptrarray || a->flags.hasptr) { // TODO: check for zeroinit
memset(data + idx * elsz, 0, inc * elsz);
}
}
@@ -1129,7 +1153,7 @@ JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary)
int isunion = jl_is_uniontype(jl_tparam0(jl_typeof(ary)));
jl_array_t *new_ary = _new_array_(jl_typeof(ary), jl_array_ndims(ary),
&ary->nrows, !ary->flags.ptrarray,
isunion, elsz);
ary->flags.hasptr, isunion, elsz);
memcpy(new_ary->data, ary->data, len * elsz);
// ensure isbits union arrays copy their selector bytes correctly
if (jl_array_isbitsunion(ary))
Loading

0 comments on commit 630a551

Please sign in to comment.