diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index f6b45eb4466fb..b9a2230c8bc99 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -37,6 +37,7 @@ use crate::back::write::{ submit_post_lto_module_to_llvm, submit_pre_lto_module_to_llvm, ComputedLtoType, OngoingCodegen, }; use crate::common::{self, IntPredicate, RealPredicate, TypeKind}; +use crate::meth::load_vtable; use crate::mir::operand::OperandValue; use crate::mir::place::PlaceRef; use crate::traits::*; @@ -135,14 +136,8 @@ fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( if let Some(entry_idx) = vptr_entry_idx { let ptr_size = bx.data_layout().pointer_size; - let ptr_align = bx.data_layout().pointer_align.abi; let vtable_byte_offset = u64::try_from(entry_idx).unwrap() * ptr_size.bytes(); - let gep = bx.inbounds_ptradd(old_info, bx.const_usize(vtable_byte_offset)); - let new_vptr = bx.load(bx.type_ptr(), gep, ptr_align); - bx.nonnull_metadata(new_vptr); - // VTable loads are invariant. - bx.set_invariant_load(new_vptr); - new_vptr + load_vtable(bx, old_info, bx.type_ptr(), vtable_byte_offset, source, true) } else { old_info } diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs index ecc3b2b24f1cf..7eb0ecd12ffec 100644 --- a/compiler/rustc_codegen_ssa/src/meth.rs +++ b/compiler/rustc_codegen_ssa/src/meth.rs @@ -28,27 +28,9 @@ impl<'a, 'tcx> VirtualIndex { let llty = bx.fn_ptr_backend_type(fn_abi); let ptr_size = bx.data_layout().pointer_size; - let ptr_align = bx.data_layout().pointer_align.abi; let vtable_byte_offset = self.0 * ptr_size.bytes(); - if bx.cx().sess().opts.unstable_opts.virtual_function_elimination - && bx.cx().sess().lto() == Lto::Fat - { - let typeid = bx - .typeid_metadata(typeid_for_trait_ref(bx.tcx(), expect_dyn_trait_in_self(ty))) - .unwrap(); - let func = bx.type_checked_load(llvtable, vtable_byte_offset, typeid); - func - } else { - let gep = bx.inbounds_ptradd(llvtable, bx.const_usize(vtable_byte_offset)); - let ptr = bx.load(llty, gep, ptr_align); - // VTable loads are invariant. - bx.set_invariant_load(ptr); - if nonnull { - bx.nonnull_metadata(ptr); - } - ptr - } + load_vtable(bx, llvtable, llty, vtable_byte_offset, ty, nonnull) } pub(crate) fn get_optional_fn>( @@ -75,31 +57,27 @@ impl<'a, 'tcx> VirtualIndex { self, bx: &mut Bx, llvtable: Bx::Value, + ty: Ty<'tcx>, ) -> Bx::Value { // Load the data pointer from the object. debug!("get_int({:?}, {:?})", llvtable, self); let llty = bx.type_isize(); let ptr_size = bx.data_layout().pointer_size; - let ptr_align = bx.data_layout().pointer_align.abi; let vtable_byte_offset = self.0 * ptr_size.bytes(); - let gep = bx.inbounds_ptradd(llvtable, bx.const_usize(vtable_byte_offset)); - let ptr = bx.load(llty, gep, ptr_align); - // VTable loads are invariant. - bx.set_invariant_load(ptr); - ptr + load_vtable(bx, llvtable, llty, vtable_byte_offset, ty, false) } } /// This takes a valid `self` receiver type and extracts the principal trait -/// ref of the type. -fn expect_dyn_trait_in_self(ty: Ty<'_>) -> ty::PolyExistentialTraitRef<'_> { +/// ref of the type. Return `None` if there is no principal trait. +fn dyn_trait_in_self(ty: Ty<'_>) -> Option> { for arg in ty.peel_refs().walk() { if let GenericArgKind::Type(ty) = arg.unpack() && let ty::Dynamic(data, _, _) = ty.kind() { - return data.principal().expect("expected principal trait object"); + return data.principal(); } } @@ -138,3 +116,36 @@ pub(crate) fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>( cx.vtables().borrow_mut().insert((ty, trait_ref), vtable); vtable } + +/// Call this function whenever you need to load a vtable. +pub(crate) fn load_vtable<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + llvtable: Bx::Value, + llty: Bx::Type, + vtable_byte_offset: u64, + ty: Ty<'tcx>, + nonnull: bool, +) -> Bx::Value { + let ptr_align = bx.data_layout().pointer_align.abi; + + if bx.cx().sess().opts.unstable_opts.virtual_function_elimination + && bx.cx().sess().lto() == Lto::Fat + { + if let Some(trait_ref) = dyn_trait_in_self(ty) { + let typeid = bx.typeid_metadata(typeid_for_trait_ref(bx.tcx(), trait_ref)).unwrap(); + let func = bx.type_checked_load(llvtable, vtable_byte_offset, typeid); + return func; + } else if nonnull { + bug!("load nonnull value from a vtable without a principal trait") + } + } + + let gep = bx.inbounds_ptradd(llvtable, bx.const_usize(vtable_byte_offset)); + let ptr = bx.load(llty, gep, ptr_align); + // VTable loads are invariant. + bx.set_invariant_load(ptr); + if nonnull { + bx.nonnull_metadata(ptr); + } + ptr +} diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index 4acbc04c505e8..c7af81b77bdc9 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -126,7 +126,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { sym::vtable_align => ty::COMMON_VTABLE_ENTRIES_ALIGN, _ => bug!(), }; - let value = meth::VirtualIndex::from_index(idx).get_usize(bx, vtable); + let value = meth::VirtualIndex::from_index(idx).get_usize(bx, vtable, callee_ty); match name { // Size is always <= isize::MAX. sym::vtable_size => { diff --git a/compiler/rustc_codegen_ssa/src/size_of_val.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs index 3f3ae21035d5a..827b939217e82 100644 --- a/compiler/rustc_codegen_ssa/src/size_of_val.rs +++ b/compiler/rustc_codegen_ssa/src/size_of_val.rs @@ -28,9 +28,9 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( // Load size/align from vtable. let vtable = info.unwrap(); let size = meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_SIZE) - .get_usize(bx, vtable); + .get_usize(bx, vtable, t); let align = meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_ALIGN) - .get_usize(bx, vtable); + .get_usize(bx, vtable, t); // Size is always <= isize::MAX. let size_bound = bx.data_layout().ptr_sized_integer().signed_max() as u128; diff --git a/tests/crashes/123955.rs b/tests/crashes/123955.rs deleted file mode 100644 index fdd58c84794cf..0000000000000 --- a/tests/crashes/123955.rs +++ /dev/null @@ -1,6 +0,0 @@ -//@ known-bug: #123955 -//@ compile-flags: -Clto -Zvirtual-function-elimination -//@ only-x86_64 -pub fn main() { - _ = Box::new(()) as Box; -} diff --git a/tests/crashes/124092.rs b/tests/crashes/124092.rs deleted file mode 100644 index c03db384e76ea..0000000000000 --- a/tests/crashes/124092.rs +++ /dev/null @@ -1,7 +0,0 @@ -//@ known-bug: #124092 -//@ compile-flags: -Zvirtual-function-elimination=true -Clto=true -//@ only-x86_64 -const X: for<'b> fn(&'b ()) = |&()| (); -fn main() { - let dyn_debug = Box::new(X) as Box as Box; -} diff --git a/tests/ui/codegen/virtual-function-elimination.rs b/tests/ui/codegen/virtual-function-elimination.rs new file mode 100644 index 0000000000000..3cbeb1293e50a --- /dev/null +++ b/tests/ui/codegen/virtual-function-elimination.rs @@ -0,0 +1,17 @@ +//@ build-pass +//@ compile-flags: -Zvirtual-function-elimination=true -Clto=true +//@ only-x86_64 +//@ no-prefer-dynamic + +// issue #123955 +pub fn test0() { + _ = Box::new(()) as Box; +} + +// issue #124092 +const X: for<'b> fn(&'b ()) = |&()| (); +pub fn test1() { + let _dyn_debug = Box::new(X) as Box as Box; +} + +fn main() {}