diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index e9591c7c8dbc6..1e18b1232de76 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -213,14 +213,10 @@ impl AttrTokenStream { .into_iter() } AttrTokenTree::Attributes(data) => { - let mut outer_attrs = Vec::new(); - let mut inner_attrs = Vec::new(); - for attr in &data.attrs { - match attr.style { - crate::AttrStyle::Outer => outer_attrs.push(attr), - crate::AttrStyle::Inner => inner_attrs.push(attr), - } - } + let idx = data + .attrs + .partition_point(|attr| matches!(attr.style, crate::AttrStyle::Outer)); + let (outer_attrs, inner_attrs) = data.attrs.split_at(idx); let mut target_tokens: Vec<_> = data .tokens @@ -265,10 +261,10 @@ impl AttrTokenStream { "Failed to find trailing delimited group in: {target_tokens:?}" ); } - let mut flat: SmallVec<[_; 1]> = SmallVec::new(); + let mut flat: SmallVec<[_; 1]> = + SmallVec::with_capacity(target_tokens.len() + outer_attrs.len()); for attr in outer_attrs { - // FIXME: Make this more efficient - flat.extend(attr.tokens().0.clone().iter().cloned()); + flat.extend(attr.tokens().0.iter().cloned()); } flat.extend(target_tokens); flat.into_iter() diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs index fc0dba6b67bc3..9df3e4030fff6 100644 --- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs +++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs @@ -79,7 +79,7 @@ fn eval_body_using_ecx<'mir, 'tcx>( intern_const_alloc_recursive(ecx, intern_kind, &ret)?; // we leave alignment checks off, since this `ecx` will not be used for further evaluation anyway - debug!("eval_body_using_ecx done: {:?}", *ret); + debug!("eval_body_using_ecx done: {:?}", ret); Ok(ret) } @@ -147,7 +147,7 @@ pub(super) fn op_to_const<'tcx>( // We know `offset` is relative to the allocation, so we can use `into_parts`. let to_const_value = |mplace: &MPlaceTy<'_>| { debug!("to_const_value(mplace: {:?})", mplace); - match mplace.ptr.into_parts() { + match mplace.ptr().into_parts() { (Some(alloc_id), offset) => { let alloc = ecx.tcx.global_alloc(alloc_id).unwrap_memory(); ConstValue::ByRef { alloc, offset } @@ -370,7 +370,7 @@ pub fn eval_to_allocation_raw_provider<'tcx>( inner = true; } }; - let alloc_id = mplace.ptr.provenance.unwrap(); + let alloc_id = mplace.ptr().provenance.unwrap(); // Validation failed, report an error. This is always a hard error. if let Err(error) = validation { diff --git a/compiler/rustc_const_eval/src/const_eval/mod.rs b/compiler/rustc_const_eval/src/const_eval/mod.rs index ef31155215a1b..5327fa5ce39bd 100644 --- a/compiler/rustc_const_eval/src/const_eval/mod.rs +++ b/compiler/rustc_const_eval/src/const_eval/mod.rs @@ -30,7 +30,7 @@ pub(crate) fn const_caller_location( if intern_const_alloc_recursive(&mut ecx, InternKind::Constant, &loc_place).is_err() { bug!("intern_const_alloc_recursive should not error in this case") } - ConstValue::Scalar(Scalar::from_maybe_pointer(loc_place.ptr, &tcx)) + ConstValue::Scalar(Scalar::from_maybe_pointer(loc_place.ptr(), &tcx)) } // We forbid type-level constants that contain more than `VALTREE_MAX_NODES` nodes. diff --git a/compiler/rustc_const_eval/src/const_eval/valtrees.rs b/compiler/rustc_const_eval/src/const_eval/valtrees.rs index b15a65d67a3d3..572d7f9ac2fec 100644 --- a/compiler/rustc_const_eval/src/const_eval/valtrees.rs +++ b/compiler/rustc_const_eval/src/const_eval/valtrees.rs @@ -5,7 +5,7 @@ use crate::const_eval::CanAccessStatics; use crate::interpret::MPlaceTy; use crate::interpret::{ intern_const_alloc_recursive, ConstValue, ImmTy, Immediate, InternKind, MemPlaceMeta, - MemoryKind, Place, Projectable, Scalar, + MemoryKind, PlaceTy, Projectable, Scalar, }; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt}; @@ -318,7 +318,7 @@ fn valtree_into_mplace<'tcx>( let len_scalar = Scalar::from_target_usize(len as u64, &tcx); Immediate::ScalarPair( - Scalar::from_maybe_pointer((*pointee_place).ptr, &tcx), + Scalar::from_maybe_pointer(pointee_place.ptr(), &tcx), len_scalar, ) } @@ -383,5 +383,5 @@ fn valtree_into_mplace<'tcx>( } fn dump_place<'tcx>(ecx: &CompileTimeEvalContext<'tcx, 'tcx>, place: &MPlaceTy<'tcx>) { - trace!("{:?}", ecx.dump_place(Place::Ptr(**place))); + trace!("{:?}", ecx.dump_place(&PlaceTy::from(place.clone()))); } diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs index 1c90ce29b02e3..db1eaf5862179 100644 --- a/compiler/rustc_const_eval/src/interpret/eval_context.rs +++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs @@ -21,8 +21,8 @@ use rustc_target::abi::{call::FnAbi, Align, HasDataLayout, Size, TargetDataLayou use super::{ AllocId, GlobalId, Immediate, InterpErrorInfo, InterpResult, MPlaceTy, Machine, MemPlace, - MemPlaceMeta, Memory, MemoryKind, Operand, Place, PlaceTy, PointerArithmetic, Provenance, - Scalar, StackPopJump, + MemPlaceMeta, Memory, MemoryKind, Operand, Place, PlaceTy, Pointer, PointerArithmetic, + Projectable, Provenance, Scalar, StackPopJump, }; use crate::errors::{self, ErroneousConstUsed}; use crate::util; @@ -155,17 +155,26 @@ pub enum StackPopCleanup { } /// State of a local variable including a memoized layout -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct LocalState<'tcx, Prov: Provenance = AllocId> { - pub value: LocalValue, + value: LocalValue, /// Don't modify if `Some`, this is only used to prevent computing the layout twice. /// Avoids computing the layout of locals that are never actually initialized. - pub layout: Cell>>, + layout: Cell>>, +} + +impl std::fmt::Debug for LocalState<'_, Prov> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LocalState") + .field("value", &self.value) + .field("ty", &self.layout.get().map(|l| l.ty)) + .finish() + } } /// Current value of a local variable #[derive(Copy, Clone, Debug)] // Miri debug-prints these -pub enum LocalValue { +pub(super) enum LocalValue { /// This local is not currently alive, and cannot be used at all. Dead, /// A normal, live local. @@ -176,10 +185,27 @@ pub enum LocalValue { Live(Operand), } -impl<'tcx, Prov: Provenance + 'static> LocalState<'tcx, Prov> { +impl<'tcx, Prov: Provenance> LocalState<'tcx, Prov> { + pub fn make_live_uninit(&mut self) { + self.value = LocalValue::Live(Operand::Immediate(Immediate::Uninit)); + } + + /// This is a hack because Miri needs a way to visit all the provenance in a `LocalState` + /// without having a layout or `TyCtxt` available, and we want to keep the `Operand` type + /// private. + pub fn as_mplace_or_imm( + &self, + ) -> Option>, MemPlaceMeta), Immediate>> { + match self.value { + LocalValue::Dead => None, + LocalValue::Live(Operand::Indirect(mplace)) => Some(Left((mplace.ptr, mplace.meta))), + LocalValue::Live(Operand::Immediate(imm)) => Some(Right(imm)), + } + } + /// Read the local's value or error if the local is not yet live or not live anymore. #[inline(always)] - pub fn access(&self) -> InterpResult<'tcx, &Operand> { + pub(super) fn access(&self) -> InterpResult<'tcx, &Operand> { match &self.value { LocalValue::Dead => throw_ub!(DeadLocal), // could even be "invalid program"? LocalValue::Live(val) => Ok(val), @@ -189,10 +215,10 @@ impl<'tcx, Prov: Provenance + 'static> LocalState<'tcx, Prov> { /// Overwrite the local. If the local can be overwritten in place, return a reference /// to do so; otherwise return the `MemPlace` to consult instead. /// - /// Note: This may only be invoked from the `Machine::access_local_mut` hook and not from - /// anywhere else. You may be invalidating machine invariants if you do! + /// Note: Before calling this, call the `before_access_local_mut` machine hook! You may be + /// invalidating machine invariants otherwise! #[inline(always)] - pub fn access_mut(&mut self) -> InterpResult<'tcx, &mut Operand> { + pub(super) fn access_mut(&mut self) -> InterpResult<'tcx, &mut Operand> { match &mut self.value { LocalValue::Dead => throw_ub!(DeadLocal), // could even be "invalid program"? LocalValue::Live(val) => Ok(val), @@ -694,7 +720,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { &self, mplace: &MPlaceTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx, Option<(Size, Align)>> { - self.size_and_align_of(&mplace.meta, &mplace.layout) + self.size_and_align_of(&mplace.meta(), &mplace.layout) } #[instrument(skip(self, body, return_place, return_to_block), level = "debug")] @@ -826,7 +852,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { .expect("return place should always be live"); let dest = self.frame().return_place.clone(); let err = self.copy_op(&op, &dest, /*allow_transmute*/ true); - trace!("return value: {:?}", self.dump_place(*dest)); + trace!("return value: {:?}", self.dump_place(&dest)); // We delay actually short-circuiting on this error until *after* the stack frame is // popped, since we want this error to be attributed to the caller, whose type defines // this transmute. @@ -974,7 +1000,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } // Need to allocate some memory, since `Immediate::Uninit` cannot be unsized. let dest_place = self.allocate_dyn(layout, MemoryKind::Stack, meta)?; - Operand::Indirect(*dest_place) + Operand::Indirect(*dest_place.mplace()) } else { assert!(!meta.has_meta()); // we're dropping the metadata // Just make this an efficient immediate. @@ -1068,8 +1094,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } #[must_use] - pub fn dump_place(&self, place: Place) -> PlacePrinter<'_, 'mir, 'tcx, M> { - PlacePrinter { ecx: self, place } + pub fn dump_place( + &self, + place: &PlaceTy<'tcx, M::Provenance>, + ) -> PlacePrinter<'_, 'mir, 'tcx, M> { + PlacePrinter { ecx: self, place: *place.place() } } #[must_use] diff --git a/compiler/rustc_const_eval/src/interpret/intern.rs b/compiler/rustc_const_eval/src/interpret/intern.rs index 42950d1ffb0c8..562f7c610fd65 100644 --- a/compiler/rustc_const_eval/src/interpret/intern.rs +++ b/compiler/rustc_const_eval/src/interpret/intern.rs @@ -25,7 +25,7 @@ use rustc_ast::Mutability; use super::{ AllocId, Allocation, ConstAllocation, InterpCx, MPlaceTy, Machine, MemoryKind, PlaceTy, - ValueVisitor, + Projectable, ValueVisitor, }; use crate::const_eval; use crate::errors::{DanglingPtrInFinal, UnsupportedUntypedPointer}; @@ -177,7 +177,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::Memory if let ty::Dynamic(_, _, ty::Dyn) = tcx.struct_tail_erasing_lifetimes(referenced_ty, self.ecx.param_env).kind() { - let ptr = mplace.meta.unwrap_meta().to_pointer(&tcx)?; + let ptr = mplace.meta().unwrap_meta().to_pointer(&tcx)?; if let Some(alloc_id) = ptr.provenance { // Explicitly choose const mode here, since vtables are immutable, even // if the reference of the fat pointer is mutable. @@ -191,7 +191,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::Memory } // Check if we have encountered this pointer+layout combination before. // Only recurse for allocation-backed pointers. - if let Some(alloc_id) = mplace.ptr.provenance { + if let Some(alloc_id) = mplace.ptr().provenance { // Compute the mode with which we intern this. Our goal here is to make as many // statics as we can immutable so they can be placed in read-only memory by LLVM. let ref_mode = match self.mode { @@ -267,7 +267,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::Memory // If there is no provenance in this allocation, it does not contain references // that point to another allocation, and we can avoid the interning walk. - if let Some(alloc) = self.ecx.get_ptr_alloc(mplace.ptr, size, align)? { + if let Some(alloc) = self.ecx.get_ptr_alloc(mplace.ptr(), size, align)? { if !alloc.has_provenance() { return Ok(false); } @@ -353,7 +353,7 @@ pub fn intern_const_alloc_recursive< leftover_allocations, // The outermost allocation must exist, because we allocated it with // `Memory::allocate`. - ret.ptr.provenance.unwrap(), + ret.ptr().provenance.unwrap(), base_intern_mode, Some(ret.layout.ty), ); @@ -466,7 +466,7 @@ impl<'mir, 'tcx: 'mir, M: super::intern::CompileTimeMachine<'mir, 'tcx, !>> ) -> InterpResult<'tcx, ConstAllocation<'tcx>> { let dest = self.allocate(layout, MemoryKind::Stack)?; f(self, &dest.clone().into())?; - let mut alloc = self.memory.alloc_map.remove(&dest.ptr.provenance.unwrap()).unwrap().1; + let mut alloc = self.memory.alloc_map.remove(&dest.ptr().provenance.unwrap()).unwrap().1; alloc.mutability = Mutability::Not; Ok(self.tcx.mk_const_alloc(alloc)) } diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index d6ca6fe73a6e6..7feed74ceae17 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -466,7 +466,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { _ => return Ok(false), } - trace!("{:?}", self.dump_place(**dest)); + trace!("{:?}", self.dump_place(dest)); self.go_to_block(ret); Ok(true) } diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs index 91c07d73fceb5..9fda6b037c8ec 100644 --- a/compiler/rustc_const_eval/src/interpret/machine.rs +++ b/compiler/rustc_const_eval/src/interpret/machine.rs @@ -18,7 +18,7 @@ use crate::const_eval::CheckAlignment; use super::{ AllocBytes, AllocId, AllocRange, Allocation, ConstAllocation, FnArg, Frame, ImmTy, InterpCx, - InterpResult, MPlaceTy, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, Scalar, + InterpResult, MPlaceTy, MemoryKind, OpTy, PlaceTy, Pointer, Provenance, Scalar, }; /// Data returned by Machine::stack_pop, @@ -237,22 +237,22 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized { right: &ImmTy<'tcx, Self::Provenance>, ) -> InterpResult<'tcx, (Scalar, bool, Ty<'tcx>)>; - /// Called to write the specified `local` from the `frame`. + /// Called before writing the specified `local` of the `frame`. /// Since writing a ZST is not actually accessing memory or locals, this is never invoked /// for ZST reads. /// /// Due to borrow checker trouble, we indicate the `frame` as an index rather than an `&mut /// Frame`. - #[inline] - fn access_local_mut<'a>( - ecx: &'a mut InterpCx<'mir, 'tcx, Self>, - frame: usize, - local: mir::Local, - ) -> InterpResult<'tcx, &'a mut Operand> + #[inline(always)] + fn before_access_local_mut<'a>( + _ecx: &'a mut InterpCx<'mir, 'tcx, Self>, + _frame: usize, + _local: mir::Local, + ) -> InterpResult<'tcx> where 'tcx: 'mir, { - ecx.stack_mut()[frame].locals[local].access_mut() + Ok(()) } /// Called before a basic block terminator is executed. diff --git a/compiler/rustc_const_eval/src/interpret/mod.rs b/compiler/rustc_const_eval/src/interpret/mod.rs index b0b553c45d432..69eb22028fa3d 100644 --- a/compiler/rustc_const_eval/src/interpret/mod.rs +++ b/compiler/rustc_const_eval/src/interpret/mod.rs @@ -20,16 +20,21 @@ mod visitor; pub use rustc_middle::mir::interpret::*; // have all the `interpret` symbols in one place: here -pub use self::eval_context::{Frame, FrameInfo, InterpCx, LocalState, LocalValue, StackPopCleanup}; +pub use self::eval_context::{Frame, FrameInfo, InterpCx, StackPopCleanup}; pub use self::intern::{intern_const_alloc_recursive, InternKind}; pub use self::machine::{compile_time_machine, AllocMap, Machine, MayLeak, StackPopJump}; pub use self::memory::{AllocKind, AllocRef, AllocRefMut, FnVal, Memory, MemoryKind}; -pub use self::operand::{ImmTy, Immediate, OpTy, Operand, Readable}; -pub use self::place::{MPlaceTy, MemPlace, MemPlaceMeta, Place, PlaceTy, Writeable}; +pub use self::operand::{ImmTy, Immediate, OpTy, Readable}; +pub use self::place::{MPlaceTy, MemPlaceMeta, PlaceTy, Writeable}; pub use self::projection::Projectable; pub use self::terminator::FnArg; pub use self::validity::{CtfeValidationMode, RefTracking}; pub use self::visitor::ValueVisitor; +use self::{ + operand::Operand, + place::{MemPlace, Place}, +}; + pub(crate) use self::intrinsics::eval_nullary_intrinsic; use eval_context::{from_known_layout, mir_assign_valid_types}; diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index 8d948eb10a7ad..26dd9098bb5d6 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -88,7 +88,7 @@ impl Immediate { // ScalarPair needs a type to interpret, so we often have an immediate and a type together // as input for binary and cast operations. -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct ImmTy<'tcx, Prov: Provenance = AllocId> { imm: Immediate, pub layout: TyAndLayout<'tcx>, @@ -134,6 +134,12 @@ impl std::fmt::Display for ImmTy<'_, Prov> { } } +impl std::fmt::Debug for ImmTy<'_, Prov> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ImmTy").field("imm", &self.imm).field("ty", &self.layout.ty).finish() + } +} + impl<'tcx, Prov: Provenance> std::ops::Deref for ImmTy<'tcx, Prov> { type Target = Immediate; #[inline(always)] @@ -142,51 +148,6 @@ impl<'tcx, Prov: Provenance> std::ops::Deref for ImmTy<'tcx, Prov> { } } -/// An `Operand` is the result of computing a `mir::Operand`. It can be immediate, -/// or still in memory. The latter is an optimization, to delay reading that chunk of -/// memory and to avoid having to store arbitrary-sized data here. -#[derive(Copy, Clone, Debug)] -pub enum Operand { - Immediate(Immediate), - Indirect(MemPlace), -} - -#[derive(Clone, Debug)] -pub struct OpTy<'tcx, Prov: Provenance = AllocId> { - op: Operand, // Keep this private; it helps enforce invariants. - pub layout: TyAndLayout<'tcx>, - /// rustc does not have a proper way to represent the type of a field of a `repr(packed)` struct: - /// it needs to have a different alignment than the field type would usually have. - /// So we represent this here with a separate field that "overwrites" `layout.align`. - /// This means `layout.align` should never be used for an `OpTy`! - /// `None` means "alignment does not matter since this is a by-value operand" - /// (`Operand::Immediate`); this field is only relevant for `Operand::Indirect`. - /// Also CTFE ignores alignment anyway, so this is for Miri only. - pub align: Option, -} - -impl<'tcx, Prov: Provenance> std::ops::Deref for OpTy<'tcx, Prov> { - type Target = Operand; - #[inline(always)] - fn deref(&self) -> &Operand { - &self.op - } -} - -impl<'tcx, Prov: Provenance> From> for OpTy<'tcx, Prov> { - #[inline(always)] - fn from(mplace: MPlaceTy<'tcx, Prov>) -> Self { - OpTy { op: Operand::Indirect(*mplace), layout: mplace.layout, align: Some(mplace.align) } - } -} - -impl<'tcx, Prov: Provenance> From> for OpTy<'tcx, Prov> { - #[inline(always)] - fn from(val: ImmTy<'tcx, Prov>) -> Self { - OpTy { op: Operand::Immediate(val.imm), layout: val.layout, align: None } - } -} - impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> { #[inline] pub fn from_scalar(val: Scalar, layout: TyAndLayout<'tcx>) -> Self { @@ -319,7 +280,61 @@ impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for ImmTy<'tcx, Prov> { } } -impl<'tcx, Prov: Provenance + 'static> Projectable<'tcx, Prov> for OpTy<'tcx, Prov> { +/// An `Operand` is the result of computing a `mir::Operand`. It can be immediate, +/// or still in memory. The latter is an optimization, to delay reading that chunk of +/// memory and to avoid having to store arbitrary-sized data here. +#[derive(Copy, Clone, Debug)] +pub(super) enum Operand { + Immediate(Immediate), + Indirect(MemPlace), +} + +#[derive(Clone)] +pub struct OpTy<'tcx, Prov: Provenance = AllocId> { + op: Operand, // Keep this private; it helps enforce invariants. + pub layout: TyAndLayout<'tcx>, + /// rustc does not have a proper way to represent the type of a field of a `repr(packed)` struct: + /// it needs to have a different alignment than the field type would usually have. + /// So we represent this here with a separate field that "overwrites" `layout.align`. + /// This means `layout.align` should never be used for an `OpTy`! + /// `None` means "alignment does not matter since this is a by-value operand" + /// (`Operand::Immediate`); this field is only relevant for `Operand::Indirect`. + /// Also CTFE ignores alignment anyway, so this is for Miri only. + pub align: Option, +} + +impl std::fmt::Debug for OpTy<'_, Prov> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OpTy").field("op", &self.op).field("ty", &self.layout.ty).finish() + } +} + +impl<'tcx, Prov: Provenance> From> for OpTy<'tcx, Prov> { + #[inline(always)] + fn from(val: ImmTy<'tcx, Prov>) -> Self { + OpTy { op: Operand::Immediate(val.imm), layout: val.layout, align: None } + } +} + +impl<'tcx, Prov: Provenance> From> for OpTy<'tcx, Prov> { + #[inline(always)] + fn from(mplace: MPlaceTy<'tcx, Prov>) -> Self { + OpTy { + op: Operand::Indirect(*mplace.mplace()), + layout: mplace.layout, + align: Some(mplace.align), + } + } +} + +impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> { + #[inline(always)] + pub(super) fn op(&self) -> &Operand { + &self.op + } +} + +impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for OpTy<'tcx, Prov> { #[inline(always)] fn layout(&self) -> TyAndLayout<'tcx> { self.layout @@ -328,7 +343,7 @@ impl<'tcx, Prov: Provenance + 'static> Projectable<'tcx, Prov> for OpTy<'tcx, Pr #[inline] fn meta(&self) -> MemPlaceMeta { match self.as_mplace_or_imm() { - Left(mplace) => mplace.meta, + Left(mplace) => mplace.meta(), Right(_) => { debug_assert!(self.layout.is_sized(), "unsized immediates are not a thing"); MemPlaceMeta::None @@ -362,18 +377,19 @@ impl<'tcx, Prov: Provenance + 'static> Projectable<'tcx, Prov> for OpTy<'tcx, Pr } } +/// The `Readable` trait describes interpreter values that one can read from. pub trait Readable<'tcx, Prov: Provenance>: Projectable<'tcx, Prov> { fn as_mplace_or_imm(&self) -> Either, ImmTy<'tcx, Prov>>; } -impl<'tcx, Prov: Provenance + 'static> Readable<'tcx, Prov> for OpTy<'tcx, Prov> { +impl<'tcx, Prov: Provenance> Readable<'tcx, Prov> for OpTy<'tcx, Prov> { #[inline(always)] fn as_mplace_or_imm(&self) -> Either, ImmTy<'tcx, Prov>> { self.as_mplace_or_imm() } } -impl<'tcx, Prov: Provenance + 'static> Readable<'tcx, Prov> for MPlaceTy<'tcx, Prov> { +impl<'tcx, Prov: Provenance> Readable<'tcx, Prov> for MPlaceTy<'tcx, Prov> { #[inline(always)] fn as_mplace_or_imm(&self) -> Either, ImmTy<'tcx, Prov>> { Left(self.clone()) @@ -535,7 +551,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Turn the wide MPlace into a string (must already be dereferenced!) pub fn read_str(&self, mplace: &MPlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx, &str> { let len = mplace.len(self)?; - let bytes = self.read_bytes_ptr_strip_provenance(mplace.ptr, Size::from_bytes(len))?; + let bytes = self.read_bytes_ptr_strip_provenance(mplace.ptr(), Size::from_bytes(len))?; let str = std::str::from_utf8(bytes).map_err(|err| err_ub!(InvalidStr(err)))?; Ok(str) } @@ -630,7 +646,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { op = self.project(&op, elem)? } - trace!("eval_place_to_op: got {:?}", *op); + trace!("eval_place_to_op: got {:?}", op); // Sanity-check the type we ended up with. debug_assert!( mir_assign_valid_types( @@ -673,7 +689,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { self.eval_mir_constant(&c, Some(constant.span), layout)? } }; - trace!("{:?}: {:?}", mir_op, *op); + trace!("{:?}: {:?}", mir_op, op); Ok(op) } diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs index 65c5cd656cb83..d8ad82d3da0a1 100644 --- a/compiler/rustc_const_eval/src/interpret/place.rs +++ b/compiler/rustc_const_eval/src/interpret/place.rs @@ -51,7 +51,7 @@ impl MemPlaceMeta { } #[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)] -pub struct MemPlace { +pub(super) struct MemPlace { /// The pointer can be a pure integer, with the `None` provenance. pub ptr: Pointer>, /// Metadata for unsized places. Interpretation is up to the type. @@ -60,68 +60,6 @@ pub struct MemPlace { pub meta: MemPlaceMeta, } -/// A MemPlace with its layout. Constructing it is only possible in this module. -#[derive(Clone, Hash, Eq, PartialEq, Debug)] -pub struct MPlaceTy<'tcx, Prov: Provenance = AllocId> { - mplace: MemPlace, - pub layout: TyAndLayout<'tcx>, - /// rustc does not have a proper way to represent the type of a field of a `repr(packed)` struct: - /// it needs to have a different alignment than the field type would usually have. - /// So we represent this here with a separate field that "overwrites" `layout.align`. - /// This means `layout.align` should never be used for a `MPlaceTy`! - pub align: Align, -} - -impl<'tcx, Prov: Provenance> std::ops::Deref for MPlaceTy<'tcx, Prov> { - type Target = MemPlace; - #[inline(always)] - fn deref(&self) -> &MemPlace { - &self.mplace - } -} - -#[derive(Copy, Clone, Debug)] -pub enum Place { - /// A place referring to a value allocated in the `Memory` system. - Ptr(MemPlace), - - /// To support alloc-free locals, we are able to write directly to a local. The offset indicates - /// where in the local this place is located; if it is `None`, no projection has been applied. - /// Such projections are meaningful even if the offset is 0, since they can change layouts. - /// (Without that optimization, we'd just always be a `MemPlace`.) - /// Note that this only stores the frame index, not the thread this frame belongs to -- that is - /// implicit. This means a `Place` must never be moved across interpreter thread boundaries! - /// - /// This variant shall not be used for unsized types -- those must always live in memory. - Local { frame: usize, local: mir::Local, offset: Option }, -} - -#[derive(Clone, Debug)] -pub struct PlaceTy<'tcx, Prov: Provenance = AllocId> { - place: Place, // Keep this private; it helps enforce invariants. - pub layout: TyAndLayout<'tcx>, - /// rustc does not have a proper way to represent the type of a field of a `repr(packed)` struct: - /// it needs to have a different alignment than the field type would usually have. - /// So we represent this here with a separate field that "overwrites" `layout.align`. - /// This means `layout.align` should never be used for a `PlaceTy`! - pub align: Align, -} - -impl<'tcx, Prov: Provenance> std::ops::Deref for PlaceTy<'tcx, Prov> { - type Target = Place; - #[inline(always)] - fn deref(&self) -> &Place { - &self.place - } -} - -impl<'tcx, Prov: Provenance> From> for PlaceTy<'tcx, Prov> { - #[inline(always)] - fn from(mplace: MPlaceTy<'tcx, Prov>) -> Self { - PlaceTy { place: Place::Ptr(*mplace), layout: mplace.layout, align: mplace.align } - } -} - impl MemPlace { #[inline(always)] pub fn from_ptr(ptr: Pointer>) -> Self { @@ -165,6 +103,27 @@ impl MemPlace { } } +/// A MemPlace with its layout. Constructing it is only possible in this module. +#[derive(Clone, Hash, Eq, PartialEq)] +pub struct MPlaceTy<'tcx, Prov: Provenance = AllocId> { + mplace: MemPlace, + pub layout: TyAndLayout<'tcx>, + /// rustc does not have a proper way to represent the type of a field of a `repr(packed)` struct: + /// it needs to have a different alignment than the field type would usually have. + /// So we represent this here with a separate field that "overwrites" `layout.align`. + /// This means `layout.align` should never be used for a `MPlaceTy`! + pub align: Align, +} + +impl std::fmt::Debug for MPlaceTy<'_, Prov> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MPlaceTy") + .field("mplace", &self.mplace) + .field("ty", &self.layout.ty) + .finish() + } +} + impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> { /// Produces a MemPlace that works for ZST but nothing else. /// Conceptually this is a new allocation, but it doesn't actually create an allocation so you @@ -194,9 +153,29 @@ impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> { align: layout.align.abi, } } + + /// Adjust the provenance of the main pointer (metadata is unaffected). + pub fn map_provenance(self, f: impl FnOnce(Option) -> Option) -> Self { + MPlaceTy { mplace: self.mplace.map_provenance(f), ..self } + } + + #[inline(always)] + pub(super) fn mplace(&self) -> &MemPlace { + &self.mplace + } + + #[inline(always)] + pub fn ptr(&self) -> Pointer> { + self.mplace.ptr + } + + #[inline(always)] + pub fn to_ref(&self, cx: &impl HasDataLayout) -> Immediate { + self.mplace.to_ref(cx) + } } -impl<'tcx, Prov: Provenance + 'static> Projectable<'tcx, Prov> for MPlaceTy<'tcx, Prov> { +impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for MPlaceTy<'tcx, Prov> { #[inline(always)] fn layout(&self) -> TyAndLayout<'tcx> { self.layout @@ -204,7 +183,7 @@ impl<'tcx, Prov: Provenance + 'static> Projectable<'tcx, Prov> for MPlaceTy<'tcx #[inline(always)] fn meta(&self) -> MemPlaceMeta { - self.meta + self.mplace.meta } fn offset_with_meta<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>( @@ -229,7 +208,76 @@ impl<'tcx, Prov: Provenance + 'static> Projectable<'tcx, Prov> for MPlaceTy<'tcx } } -impl<'tcx, Prov: Provenance + 'static> Projectable<'tcx, Prov> for PlaceTy<'tcx, Prov> { +#[derive(Copy, Clone, Debug)] +pub(super) enum Place { + /// A place referring to a value allocated in the `Memory` system. + Ptr(MemPlace), + + /// To support alloc-free locals, we are able to write directly to a local. The offset indicates + /// where in the local this place is located; if it is `None`, no projection has been applied. + /// Such projections are meaningful even if the offset is 0, since they can change layouts. + /// (Without that optimization, we'd just always be a `MemPlace`.) + /// Note that this only stores the frame index, not the thread this frame belongs to -- that is + /// implicit. This means a `Place` must never be moved across interpreter thread boundaries! + /// + /// This variant shall not be used for unsized types -- those must always live in memory. + Local { frame: usize, local: mir::Local, offset: Option }, +} + +#[derive(Clone)] +pub struct PlaceTy<'tcx, Prov: Provenance = AllocId> { + place: Place, // Keep this private; it helps enforce invariants. + pub layout: TyAndLayout<'tcx>, + /// rustc does not have a proper way to represent the type of a field of a `repr(packed)` struct: + /// it needs to have a different alignment than the field type would usually have. + /// So we represent this here with a separate field that "overwrites" `layout.align`. + /// This means `layout.align` should never be used for a `PlaceTy`! + pub align: Align, +} + +impl std::fmt::Debug for PlaceTy<'_, Prov> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PlaceTy").field("place", &self.place).field("ty", &self.layout.ty).finish() + } +} + +impl<'tcx, Prov: Provenance> From> for PlaceTy<'tcx, Prov> { + #[inline(always)] + fn from(mplace: MPlaceTy<'tcx, Prov>) -> Self { + PlaceTy { place: Place::Ptr(mplace.mplace), layout: mplace.layout, align: mplace.align } + } +} + +impl<'tcx, Prov: Provenance> PlaceTy<'tcx, Prov> { + #[inline(always)] + pub(super) fn place(&self) -> &Place { + &self.place + } + + /// A place is either an mplace or some local. + #[inline(always)] + pub fn as_mplace_or_local( + &self, + ) -> Either, (usize, mir::Local, Option)> { + match self.place { + Place::Ptr(mplace) => Left(MPlaceTy { mplace, layout: self.layout, align: self.align }), + Place::Local { frame, local, offset } => Right((frame, local, offset)), + } + } + + #[inline(always)] + #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980) + pub fn assert_mem_place(&self) -> MPlaceTy<'tcx, Prov> { + self.as_mplace_or_local().left().unwrap_or_else(|| { + bug!( + "PlaceTy of type {} was a local when it was expected to be an MPlace", + self.layout.ty + ) + }) + } +} + +impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for PlaceTy<'tcx, Prov> { #[inline(always)] fn layout(&self) -> TyAndLayout<'tcx> { self.layout @@ -238,7 +286,7 @@ impl<'tcx, Prov: Provenance + 'static> Projectable<'tcx, Prov> for PlaceTy<'tcx, #[inline] fn meta(&self) -> MemPlaceMeta { match self.as_mplace_or_local() { - Left(mplace) => mplace.meta, + Left(mplace) => mplace.meta(), Right(_) => { debug_assert!(self.layout.is_sized(), "unsized locals should live in memory"); MemPlaceMeta::None @@ -286,11 +334,11 @@ impl<'tcx, Prov: Provenance + 'static> Projectable<'tcx, Prov> for PlaceTy<'tcx, impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> { #[inline(always)] pub fn as_mplace_or_imm(&self) -> Either, ImmTy<'tcx, Prov>> { - match **self { + match self.op() { Operand::Indirect(mplace) => { - Left(MPlaceTy { mplace, layout: self.layout, align: self.align.unwrap() }) + Left(MPlaceTy { mplace: *mplace, layout: self.layout, align: self.align.unwrap() }) } - Operand::Immediate(imm) => Right(ImmTy::from_immediate(imm, self.layout)), + Operand::Immediate(imm) => Right(ImmTy::from_immediate(*imm, self.layout)), } } @@ -306,30 +354,7 @@ impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> { } } -impl<'tcx, Prov: Provenance + 'static> PlaceTy<'tcx, Prov> { - /// A place is either an mplace or some local. - #[inline(always)] - pub fn as_mplace_or_local( - &self, - ) -> Either, (usize, mir::Local, Option)> { - match **self { - Place::Ptr(mplace) => Left(MPlaceTy { mplace, layout: self.layout, align: self.align }), - Place::Local { frame, local, offset } => Right((frame, local, offset)), - } - } - - #[inline(always)] - #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980) - pub fn assert_mem_place(&self) -> MPlaceTy<'tcx, Prov> { - self.as_mplace_or_local().left().unwrap_or_else(|| { - bug!( - "PlaceTy of type {} was a local when it was expected to be an MPlace", - self.layout.ty - ) - }) - } -} - +/// The `Weiteable` trait describes interpreter values that can be written to. pub trait Writeable<'tcx, Prov: Provenance>: Projectable<'tcx, Prov> { fn as_mplace_or_local( &self, @@ -341,7 +366,7 @@ pub trait Writeable<'tcx, Prov: Provenance>: Projectable<'tcx, Prov> { ) -> InterpResult<'tcx, MPlaceTy<'tcx, Prov>>; } -impl<'tcx, Prov: Provenance + 'static> Writeable<'tcx, Prov> for PlaceTy<'tcx, Prov> { +impl<'tcx, Prov: Provenance> Writeable<'tcx, Prov> for PlaceTy<'tcx, Prov> { #[inline(always)] fn as_mplace_or_local( &self, @@ -360,7 +385,7 @@ impl<'tcx, Prov: Provenance + 'static> Writeable<'tcx, Prov> for PlaceTy<'tcx, P } } -impl<'tcx, Prov: Provenance + 'static> Writeable<'tcx, Prov> for MPlaceTy<'tcx, Prov> { +impl<'tcx, Prov: Provenance> Writeable<'tcx, Prov> for MPlaceTy<'tcx, Prov> { #[inline(always)] fn as_mplace_or_local( &self, @@ -381,7 +406,7 @@ impl<'tcx, Prov: Provenance + 'static> Writeable<'tcx, Prov> for MPlaceTy<'tcx, // FIXME: Working around https://github.com/rust-lang/rust/issues/54385 impl<'mir, 'tcx: 'mir, Prov, M> InterpCx<'mir, 'tcx, M> where - Prov: Provenance + 'static, + Prov: Provenance, M: Machine<'mir, 'tcx, Provenance = Prov>, { /// Take a value, which represents a (thin or wide) reference, and make it a place. @@ -415,7 +440,7 @@ where &self, mplace: &MPlaceTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> { - let imm = mplace.to_ref(self); + let imm = mplace.mplace.to_ref(self); let layout = self.layout_of(Ty::new_mut_ptr(self.tcx.tcx, mplace.layout.ty))?; Ok(ImmTy::from_immediate(imm, layout)) } @@ -449,7 +474,7 @@ where .size_and_align_of_mplace(&mplace)? .unwrap_or((mplace.layout.size, mplace.layout.align.abi)); // Due to packed places, only `mplace.align` matters. - self.get_ptr_alloc(mplace.ptr, size, mplace.align) + self.get_ptr_alloc(mplace.ptr(), size, mplace.align) } #[inline] @@ -462,7 +487,7 @@ where .size_and_align_of_mplace(&mplace)? .unwrap_or((mplace.layout.size, mplace.layout.align.abi)); // Due to packed places, only `mplace.align` matters. - self.get_ptr_alloc_mut(mplace.ptr, size, mplace.align) + self.get_ptr_alloc_mut(mplace.ptr(), size, mplace.align) } /// Check if this mplace is dereferenceable and sufficiently aligned. @@ -473,7 +498,7 @@ where // Due to packed places, only `mplace.align` matters. let align = if M::enforce_alignment(self).should_check() { mplace.align } else { Align::ONE }; - self.check_ptr_access_align(mplace.ptr, size, align, CheckInAllocMsg::DerefTest)?; + self.check_ptr_access_align(mplace.ptr(), size, align, CheckInAllocMsg::DerefTest)?; Ok(()) } @@ -542,7 +567,7 @@ where place = self.project(&place, elem)? } - trace!("{:?}", self.dump_place(place.place)); + trace!("{:?}", self.dump_place(&place)); // Sanity-check the type we ended up with. debug_assert!( mir_assign_valid_types( @@ -618,7 +643,8 @@ where // just fall back to the indirect path. dest.force_mplace(self)? } else { - match M::access_local_mut(self, frame, local)? { + M::before_access_local_mut(self, frame, local)?; + match self.stack_mut()[frame].locals[local].access_mut()? { Operand::Immediate(local_val) => { // Local can be updated in-place. *local_val = src; @@ -738,7 +764,8 @@ where // FIXME: share the logic with `write_immediate_no_validate`. dest.force_mplace(self)? } else { - match M::access_local_mut(self, frame, local)? { + M::before_access_local_mut(self, frame, local)?; + match self.stack_mut()[frame].locals[local].access_mut()? { Operand::Immediate(local) => { *local = Immediate::Uninit; return Ok(()); @@ -832,7 +859,7 @@ where *src_val, src.layout(), dest_mem.align, - *dest_mem, + dest_mem.mplace, ) }; } @@ -859,7 +886,12 @@ where // (Or as the `Assign` docs put it, assignments "not producing primitives" must be // non-overlapping.) self.mem_copy( - src.ptr, src.align, dest.ptr, dest.align, dest_size, /*nonoverlapping*/ true, + src.ptr(), + src.align, + dest.ptr(), + dest.align, + dest_size, + /*nonoverlapping*/ true, ) } @@ -874,7 +906,8 @@ where ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> { let mplace = match place.place { Place::Local { frame, local, offset } => { - let whole_local = match M::access_local_mut(self, frame, local)? { + M::before_access_local_mut(self, frame, local)?; + let whole_local = match self.stack_mut()[frame].locals[local].access_mut()? { &mut Operand::Immediate(local_val) => { // We need to make an allocation. @@ -894,16 +927,16 @@ where local_val, local_layout, local_layout.align.abi, - *mplace, + mplace.mplace, )?; } M::after_local_allocated(self, frame, local, &mplace)?; // Now we can call `access_mut` again, asserting it goes well, and actually // overwrite things. This points to the entire allocation, not just the part // the place refers to, i.e. we do this before we apply `offset`. - *M::access_local_mut(self, frame, local).unwrap() = - Operand::Indirect(*mplace); - *mplace + *self.stack_mut()[frame].locals[local].access_mut().unwrap() = + Operand::Indirect(mplace.mplace); + mplace.mplace } &mut Operand::Indirect(mplace) => mplace, // this already was an indirect local }; @@ -1011,12 +1044,12 @@ where matches!(mplace.layout.ty.kind(), ty::Dynamic(_, _, ty::Dyn)), "`unpack_dyn_trait` only makes sense on `dyn*` types" ); - let vtable = mplace.meta.unwrap_meta().to_pointer(self)?; + let vtable = mplace.meta().unwrap_meta().to_pointer(self)?; let (ty, _) = self.get_ptr_vtable(vtable)?; let layout = self.layout_of(ty)?; let mplace = MPlaceTy { - mplace: MemPlace { meta: MemPlaceMeta::None, ..**mplace }, + mplace: MemPlace { meta: MemPlaceMeta::None, ..mplace.mplace }, layout, align: layout.align.abi, }; diff --git a/compiler/rustc_const_eval/src/interpret/projection.rs b/compiler/rustc_const_eval/src/interpret/projection.rs index c69321d109e49..6c720ac4a574d 100644 --- a/compiler/rustc_const_eval/src/interpret/projection.rs +++ b/compiler/rustc_const_eval/src/interpret/projection.rs @@ -89,7 +89,7 @@ pub trait Projectable<'tcx, Prov: Provenance>: Sized + std::fmt::Debug { } /// A type representing iteration over the elements of an array. -pub struct ArrayIterator<'tcx, 'a, Prov: Provenance + 'static, P: Projectable<'tcx, Prov>> { +pub struct ArrayIterator<'tcx, 'a, Prov: Provenance, P: Projectable<'tcx, Prov>> { base: &'a P, range: Range, stride: Size, @@ -97,9 +97,7 @@ pub struct ArrayIterator<'tcx, 'a, Prov: Provenance + 'static, P: Projectable<'t _phantom: PhantomData, // otherwise it says `Prov` is never used... } -impl<'tcx, 'a, Prov: Provenance + 'static, P: Projectable<'tcx, Prov>> - ArrayIterator<'tcx, 'a, Prov, P> -{ +impl<'tcx, 'a, Prov: Provenance, P: Projectable<'tcx, Prov>> ArrayIterator<'tcx, 'a, Prov, P> { /// Should be the same `ecx` on each call, and match the one used to create the iterator. pub fn next<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>( &mut self, @@ -113,7 +111,7 @@ impl<'tcx, 'a, Prov: Provenance + 'static, P: Projectable<'tcx, Prov>> // FIXME: Working around https://github.com/rust-lang/rust/issues/54385 impl<'mir, 'tcx: 'mir, Prov, M> InterpCx<'mir, 'tcx, M> where - Prov: Provenance + 'static, + Prov: Provenance, M: Machine<'mir, 'tcx, Provenance = Prov>, { /// Offset a pointer to project to a field of a struct/union. Unlike `place_field`, this is diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index 0740894a4ffa6..57f99bd8f617e 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -204,7 +204,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // avoid writing each operand individually and instead just make many copies // of the first element. let elem_size = first.layout.size; - let first_ptr = first.ptr; + let first_ptr = first.ptr(); let rest_ptr = first_ptr.offset(elem_size, self)?; // For the alignment of `rest_ptr`, we crucially do *not* use `first.align` as // that place might be more aligned than its type mandates (a `u8` array could @@ -305,7 +305,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } } - trace!("{:?}", self.dump_place(*dest)); + trace!("{:?}", self.dump_place(&dest)); Ok(()) } diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs index ca66c4cfb6312..bc4edf1c4b60b 100644 --- a/compiler/rustc_const_eval/src/interpret/terminator.rs +++ b/compiler/rustc_const_eval/src/interpret/terminator.rs @@ -793,7 +793,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { throw_ub_custom!(fluent::const_eval_dyn_star_call_vtable_mismatch); } - (vptr, dyn_ty, recv.ptr) + (vptr, dyn_ty, recv.ptr()) } else { // Doesn't have to be a `dyn Trait`, but the unsized tail must be `dyn Trait`. // (For that reason we also cannot use `unpack_dyn_trait`.) @@ -810,7 +810,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { assert!(receiver_place.layout.is_unsized()); // Get the required information from the vtable. - let vptr = receiver_place.meta.unwrap_meta().to_pointer(self)?; + let vptr = receiver_place.meta().unwrap_meta().to_pointer(self)?; let (dyn_ty, dyn_trait) = self.get_ptr_vtable(vptr)?; if dyn_trait != data.principal() { throw_ub_custom!(fluent::const_eval_dyn_call_vtable_mismatch); @@ -819,7 +819,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // It might be surprising that we use a pointer as the receiver even if this // is a by-val case; this works because by-val passing of an unsized `dyn // Trait` to a function is actually desugared to a pointer. - (vptr, dyn_ty, receiver_place.ptr) + (vptr, dyn_ty, receiver_place.ptr()) }; // Now determine the actual method to call. We can do that in two different ways and diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs index 0d08d6be91944..8e2b35fd5b619 100644 --- a/compiler/rustc_const_eval/src/interpret/validity.rs +++ b/compiler/rustc_const_eval/src/interpret/validity.rs @@ -360,7 +360,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' // Handle wide pointers. // Check metadata early, for better diagnostics if place.layout.is_unsized() { - self.check_wide_ptr_meta(place.meta, place.layout)?; + self.check_wide_ptr_meta(place.meta(), place.layout)?; } // Make sure this is dereferenceable and all. let size_and_align = try_validation!( @@ -379,7 +379,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' // Direct call to `check_ptr_access_align` checks alignment even on CTFE machines. try_validation!( self.ecx.check_ptr_access_align( - place.ptr, + place.ptr(), size, align, CheckInAllocMsg::InboundsTest, // will anyway be replaced by validity message @@ -414,7 +414,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' if let Some(ref_tracking) = self.ref_tracking.as_deref_mut() { // Proceed recursively even for ZST, no reason to skip them! // `!` is a ZST and we want to validate it. - if let Ok((alloc_id, _offset, _prov)) = self.ecx.ptr_try_get_alloc_id(place.ptr) { + if let Ok((alloc_id, _offset, _prov)) = self.ecx.ptr_try_get_alloc_id(place.ptr()) { // Let's see what kind of memory this points to. let alloc_kind = self.ecx.tcx.try_get_global_alloc(alloc_id); match alloc_kind { @@ -521,7 +521,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' let place = self.ecx.ref_to_mplace(&self.read_immediate(value, ExpectedKind::RawPtr)?)?; if place.layout.is_unsized() { - self.check_wide_ptr_meta(place.meta, place.layout)?; + self.check_wide_ptr_meta(place.meta(), place.layout)?; } Ok(true) } @@ -739,7 +739,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> let mplace = op.assert_mem_place(); // strings are unsized and hence never immediate let len = mplace.len(self.ecx)?; try_validation!( - self.ecx.read_bytes_ptr_strip_provenance(mplace.ptr, Size::from_bytes(len)), + self.ecx.read_bytes_ptr_strip_provenance(mplace.ptr(), Size::from_bytes(len)), self.path, Ub(InvalidUninitBytes(..)) => Uninit { expected: ExpectedKind::Str }, Unsup(ReadPointerAsInt(_)) => PointerAsInt { expected: ExpectedKind::Str } @@ -789,7 +789,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> // to reject those pointers, we just do not have the machinery to // talk about parts of a pointer. // We also accept uninit, for consistency with the slow path. - let alloc = self.ecx.get_ptr_alloc(mplace.ptr, size, mplace.align)?.expect("we already excluded size 0"); + let alloc = self.ecx.get_ptr_alloc(mplace.ptr(), size, mplace.align)?.expect("we already excluded size 0"); match alloc.get_bytes_strip_provenance() { // In the happy case, we needn't check anything else. diff --git a/compiler/rustc_const_eval/src/util/check_validity_requirement.rs b/compiler/rustc_const_eval/src/util/check_validity_requirement.rs index 2d1970791cacd..e9e0690f07df1 100644 --- a/compiler/rustc_const_eval/src/util/check_validity_requirement.rs +++ b/compiler/rustc_const_eval/src/util/check_validity_requirement.rs @@ -54,7 +54,7 @@ fn might_permit_raw_init_strict<'tcx>( if kind == ValidityRequirement::Zero { cx.write_bytes_ptr( - allocated.ptr, + allocated.ptr(), std::iter::repeat(0_u8).take(ty.layout.size().bytes_usize()), ) .expect("failed to write bytes for zero valid check"); diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 6af5c24c458a6..8f8f1d8c04feb 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -1202,7 +1202,11 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, Ret->ModuleMap[module->identifier] = mem_buffer; +#if LLVM_VERSION_GE(18, 0) + if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index)) { +#else if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) { +#endif LLVMRustSetLastError(toString(std::move(Err)).c_str()); return nullptr; } diff --git a/compiler/rustc_middle/src/mir/interpret/pointer.rs b/compiler/rustc_middle/src/mir/interpret/pointer.rs index 65d04919357f3..1c9ce1cb1308c 100644 --- a/compiler/rustc_middle/src/mir/interpret/pointer.rs +++ b/compiler/rustc_middle/src/mir/interpret/pointer.rs @@ -103,7 +103,7 @@ impl PointerArithmetic for T {} /// mostly opaque; the `Machine` trait extends it with some more operations that also have access to /// some global state. /// The `Debug` rendering is used to display bare provenance, and for the default impl of `fmt`. -pub trait Provenance: Copy + fmt::Debug { +pub trait Provenance: Copy + fmt::Debug + 'static { /// Says whether the `offset` field of `Pointer`s with this provenance is the actual physical address. /// - If `false`, the offset *must* be relative. This means the bytes representing a pointer are /// different from what the Abstract Machine prescribes, so the interpreter must prevent any diff --git a/compiler/rustc_mir_build/src/build/custom/parse.rs b/compiler/rustc_mir_build/src/build/custom/parse.rs index 60c4a041696bc..f83c62fd580b8 100644 --- a/compiler/rustc_mir_build/src/build/custom/parse.rs +++ b/compiler/rustc_mir_build/src/build/custom/parse.rs @@ -1,5 +1,6 @@ use rustc_index::IndexSlice; -use rustc_middle::{mir::*, thir::*, ty::Ty}; +use rustc_middle::ty::{self, Ty}; +use rustc_middle::{mir::*, thir::*}; use rustc_span::Span; use super::{PResult, ParseCtxt, ParseError}; @@ -159,6 +160,14 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> { ); self.parse_local_decls(local_decls.iter().copied())?; + let (debuginfo, rest) = parse_by_kind!(self, rest, _, "body with debuginfo", + ExprKind::Block { block } => { + let block = &self.thir[*block]; + (&block.stmts, block.expr.unwrap()) + }, + ); + self.parse_debuginfo(debuginfo.iter().copied())?; + let block_defs = parse_by_kind!(self, rest, _, "body with block defs", ExprKind::Block { block } => &self.thir[*block].stmts, ); @@ -195,6 +204,52 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> { Ok(()) } + fn parse_debuginfo(&mut self, stmts: impl Iterator) -> PResult<()> { + for stmt in stmts { + let stmt = &self.thir[stmt]; + let expr = match stmt.kind { + StmtKind::Let { span, .. } => { + return Err(ParseError { + span, + item_description: format!("{:?}", stmt), + expected: "debuginfo".to_string(), + }); + } + StmtKind::Expr { expr, .. } => expr, + }; + let span = self.thir[expr].span; + let (name, operand) = parse_by_kind!(self, expr, _, "debuginfo", + @call("mir_debuginfo", args) => { + (args[0], args[1]) + }, + ); + let name = parse_by_kind!(self, name, _, "debuginfo", + ExprKind::Literal { lit, neg: false } => lit, + ); + let Some(name) = name.node.str() else { + return Err(ParseError { + span, + item_description: format!("{:?}", name), + expected: "string".to_string(), + }); + }; + let operand = self.parse_operand(operand)?; + let value = match operand { + Operand::Constant(c) => VarDebugInfoContents::Const(*c), + Operand::Copy(p) | Operand::Move(p) => VarDebugInfoContents::Place(p), + }; + let dbginfo = VarDebugInfo { + name, + source_info: SourceInfo { span, scope: self.source_scope }, + argument_index: None, + value, + }; + self.body.var_debug_info.push(dbginfo); + } + + Ok(()) + } + fn parse_let_statement(&mut self, stmt_id: StmtId) -> PResult<(LocalVarId, Ty<'tcx>, Span)> { let pattern = match &self.thir[stmt_id].kind { StmtKind::Let { pattern, .. } => pattern, diff --git a/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs b/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs index 26662f5de459f..da5f2b1bcc99c 100644 --- a/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs +++ b/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs @@ -204,7 +204,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> { ) } - fn parse_operand(&self, expr_id: ExprId) -> PResult> { + pub fn parse_operand(&self, expr_id: ExprId) -> PResult> { parse_by_kind!(self, expr_id, expr, "operand", @call("mir_move", args) => self.parse_place(args[0]).map(Operand::Move), @call("mir_static", args) => self.parse_static(args[0]), diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index 0aaa3f75c8268..5f135e96980f1 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -22,8 +22,7 @@ use rustc_target::spec::abi::Abi as CallAbi; use crate::MirPass; use rustc_const_eval::interpret::{ self, compile_time_machine, AllocId, ConstAllocation, ConstValue, FnArg, Frame, ImmTy, - Immediate, InterpCx, InterpResult, LocalValue, MemoryKind, OpTy, PlaceTy, Pointer, Scalar, - StackPopCleanup, + Immediate, InterpCx, InterpResult, MemoryKind, OpTy, PlaceTy, Pointer, Scalar, StackPopCleanup, }; /// The maximum number of bytes that we'll allocate space for a local or the return value. @@ -225,11 +224,11 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> throw_machine_stop_str!("pointer arithmetic or comparisons aren't supported in ConstProp") } - fn access_local_mut<'a>( + fn before_access_local_mut<'a>( ecx: &'a mut InterpCx<'mir, 'tcx, Self>, frame: usize, local: Local, - ) -> InterpResult<'tcx, &'a mut interpret::Operand> { + ) -> InterpResult<'tcx> { assert_eq!(frame, 0); match ecx.machine.can_const_prop[local] { ConstPropMode::NoPropagation => { @@ -242,7 +241,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> } ConstPropMode::FullConstProp => {} } - ecx.machine.stack[frame].locals[local].access_mut() + Ok(()) } fn before_access_global( @@ -382,8 +381,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // mark those as live... We rely on `local_to_place`/`local_to_op` in the interpreter // stopping us before those unsized immediates can cause issues deeper in the // interpreter. - ecx.frame_mut().locals[local].value = - LocalValue::Live(interpret::Operand::Immediate(Immediate::Uninit)); + ecx.frame_mut().locals[local].make_live_uninit(); } ConstPropagator { ecx, tcx, param_env, local_decls: &dummy_body.local_decls } @@ -392,7 +390,11 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { fn get_const(&self, place: Place<'tcx>) -> Option> { let op = match self.ecx.eval_place_to_op(place, None) { Ok(op) => { - if matches!(*op, interpret::Operand::Immediate(Immediate::Uninit)) { + if op + .as_mplace_or_imm() + .right() + .is_some_and(|imm| matches!(*imm, Immediate::Uninit)) + { // Make sure nobody accidentally uses this value. return None; } @@ -415,8 +417,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { /// Remove `local` from the pool of `Locals`. Allows writing to them, /// but not reading from them anymore. fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) { - ecx.frame_mut().locals[local].value = - LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)); + ecx.frame_mut().locals[local].make_live_uninit(); ecx.machine.written_only_inside_own_block_locals.remove(&local); } @@ -743,7 +744,8 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { ) -> Option> { if let PlaceElem::Index(local) = elem && let Some(value) = self.get_const(local.into()) - && let interpret::Operand::Immediate(Immediate::Scalar(scalar)) = *value + && let Some(imm) = value.as_mplace_or_imm().right() + && let Immediate::Scalar(scalar) = *imm && let Ok(offset) = scalar.to_target_usize(&self.tcx) && let Some(min_length) = offset.checked_add(1) { diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs index dec79ddf58cf8..4b51beed09558 100644 --- a/compiler/rustc_mir_transform/src/const_prop_lint.rs +++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs @@ -7,7 +7,7 @@ use either::Left; use rustc_const_eval::interpret::Immediate; use rustc_const_eval::interpret::{ - self, InterpCx, InterpResult, LocalValue, MemoryKind, OpTy, Scalar, StackPopCleanup, + InterpCx, InterpResult, MemoryKind, OpTy, Scalar, StackPopCleanup, }; use rustc_const_eval::ReportErrorExt; use rustc_hir::def::DefKind; @@ -212,8 +212,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // mark those as live... We rely on `local_to_place`/`local_to_op` in the interpreter // stopping us before those unsized immediates can cause issues deeper in the // interpreter. - ecx.frame_mut().locals[local].value = - LocalValue::Live(interpret::Operand::Immediate(Immediate::Uninit)); + ecx.frame_mut().locals[local].make_live_uninit(); } ConstPropagator { @@ -236,7 +235,11 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { fn get_const(&self, place: Place<'tcx>) -> Option> { let op = match self.ecx.eval_place_to_op(place, None) { Ok(op) => { - if matches!(*op, interpret::Operand::Immediate(Immediate::Uninit)) { + if op + .as_mplace_or_imm() + .right() + .is_some_and(|imm| matches!(*imm, Immediate::Uninit)) + { // Make sure nobody accidentally uses this value. return None; } @@ -259,8 +262,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { /// Remove `local` from the pool of `Locals`. Allows writing to them, /// but not reading from them anymore. fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) { - ecx.frame_mut().locals[local].value = - LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)); + ecx.frame_mut().locals[local].make_live_uninit(); ecx.machine.written_only_inside_own_block_locals.remove(&local); } @@ -656,12 +658,12 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { } StatementKind::StorageLive(local) => { let frame = self.ecx.frame_mut(); - frame.locals[local].value = - LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)); + frame.locals[local].make_live_uninit(); } StatementKind::StorageDead(local) => { let frame = self.ecx.frame_mut(); - frame.locals[local].value = LocalValue::Dead; + // We don't actually track liveness, so the local remains live. But forget its value. + frame.locals[local].make_live_uninit(); } _ => {} } diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs index eafbaba9ef3b8..5a9c900f64347 100644 --- a/library/core/src/intrinsics.rs +++ b/library/core/src/intrinsics.rs @@ -1130,7 +1130,10 @@ extern "rust-intrinsic" { /// may lead to unexpected and unstable compilation results. This makes `transmute` **incredibly /// unsafe**. `transmute` should be the absolute last resort. /// - /// Transmuting pointers to integers in a `const` context is [undefined behavior][ub]. + /// Transmuting pointers *to* integers in a `const` context is [undefined behavior][ub], + /// unless the pointer was originally created *from* an integer. + /// (That includes this function specifically, integer-to-pointer casts, and helpers like [`invalid`][crate::ptr::invalid], + /// but also semantically-equivalent conversions such as punning through `repr(C)` union fields.) /// Any attempt to use the resulting value for integer operations will abort const-evaluation. /// (And even outside `const`, such transmutation is touching on many unspecified aspects of the /// Rust memory model and should be avoided. See below for alternatives.) diff --git a/library/core/src/intrinsics/mir.rs b/library/core/src/intrinsics/mir.rs index b99346b6ba608..cab195dad9b05 100644 --- a/library/core/src/intrinsics/mir.rs +++ b/library/core/src/intrinsics/mir.rs @@ -12,7 +12,8 @@ //! //! Typical usage will look like this: //! -//! ```rust +#![cfg_attr(bootstrap, doc = "```rust,ignore")] +#![cfg_attr(not(bootstrap), doc = "```rust")] //! #![feature(core_intrinsics, custom_mir)] //! #![allow(internal_features)] //! @@ -62,7 +63,8 @@ //! //! # Examples //! -//! ```rust +#![cfg_attr(bootstrap, doc = "```rust,ignore")] +#![cfg_attr(not(bootstrap), doc = "```rust")] //! #![feature(core_intrinsics, custom_mir)] //! #![allow(internal_features)] //! @@ -317,7 +319,8 @@ define!( /// /// # Examples /// - /// ```rust + #[cfg_attr(bootstrap, doc = "```rust,ignore")] + #[cfg_attr(not(bootstrap), doc = "```rust")] /// #![allow(internal_features)] /// #![feature(custom_mir, core_intrinsics)] /// @@ -361,6 +364,11 @@ define!( #[doc(hidden)] fn __internal_make_place(place: T) -> *mut T ); +define!( + "mir_debuginfo", + #[doc(hidden)] + fn __debuginfo(name: &'static str, s: T) +); /// Macro for generating custom MIR. /// @@ -371,6 +379,7 @@ pub macro mir { ( $(type RET = $ret_ty:ty ;)? $(let $local_decl:ident $(: $local_decl_ty:ty)? ;)* + $(debug $dbg_name:ident => $dbg_data:expr ;)* { $($entry:tt)* @@ -394,26 +403,32 @@ pub macro mir { $( let $local_decl $(: $local_decl_ty)? ; )* - ::core::intrinsics::mir::__internal_extract_let!($($entry)*); $( ::core::intrinsics::mir::__internal_extract_let!($($block)*); )* { - // Finally, the contents of the basic blocks - ::core::intrinsics::mir::__internal_remove_let!({ - {} - { $($entry)* } - }); + // Now debuginfo $( + __debuginfo(stringify!($dbg_name), $dbg_data); + )* + + { + // Finally, the contents of the basic blocks ::core::intrinsics::mir::__internal_remove_let!({ {} - { $($block)* } + { $($entry)* } }); - )* + $( + ::core::intrinsics::mir::__internal_remove_let!({ + {} + { $($block)* } + }); + )* - RET + RET + } } } }} diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index b97cb652076e4..c3edd3a9d7def 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -20,11 +20,19 @@ // FIXME: Fill me in with more detail when the interface settles //! This library is built on the assumption of a few existing symbols: //! -//! * `memcpy`, `memcmp`, `memset`, `strlen` - These are core memory routines which are -//! often generated by LLVM. Additionally, this library can make explicit -//! calls to these functions. Their signatures are the same as found in C. -//! These functions are often provided by the system libc, but can also be -//! provided by the [compiler-builtins crate](https://crates.io/crates/compiler_builtins). +//! * `memcpy`, `memmove`, `memset`, `memcmp`, `bcmp`, `strlen` - These are core memory routines +//! which are generated by Rust codegen backends. Additionally, this library can make explicit +//! calls to `strlen`. Their signatures are the same as found in C, but there are extra +//! assumptions about their semantics: For `memcpy`, `memmove`, `memset`, `memcmp`, and `bcmp`, if +//! the `n` parameter is 0, the function is assumed to not be UB. Furthermore, for `memcpy`, if +//! source and target pointer are equal, the function is assumed to not be UB. +//! (Note that these are [standard assumptions](https://reviews.llvm.org/D86993) among compilers.) +//! These functions are often provided by the system libc, but can also be provided by the +//! [compiler-builtins crate](https://crates.io/crates/compiler_builtins). +//! Note that the library does not guarantee that it will always make these assumptions, so Rust +//! user code directly calling the C functions should follow the C specification! The advice for +//! Rust user code is to call the functions provided by this library instead (such as +//! `ptr::copy`). //! //! * `rust_begin_panic` - This function takes four arguments, a //! `fmt::Arguments`, a `&'static str`, and two `u32`'s. These four arguments diff --git a/library/core/src/num/dec2flt/fpu.rs b/library/core/src/num/dec2flt/fpu.rs index 3806977f70ee4..8d62684f8d383 100644 --- a/library/core/src/num/dec2flt/fpu.rs +++ b/library/core/src/num/dec2flt/fpu.rs @@ -8,6 +8,17 @@ pub use fpu_precision::set_precision; // round to 80 bits causing double rounding to happen when values are eventually represented as // 32/64 bit float values. To overcome this, the FPU control word can be set so that the // computations are performed in the desired precision. +// +// Note that normally, it is Undefined Behavior to alter the FPU control word while Rust code runs. +// The compiler assumes that the control word is always in its default state. However, in this +// particular case the semantics with the altered control word are actually *more faithful* +// to Rust semantics than the default -- arguably it is all the code that runs *outside* of the scope +// of a `set_precision` guard that is wrong. +// In other words, we are only using this to work around . +// Sometimes killing UB with UB actually works... +// (If this is used to set 32bit precision, there is still a risk that the compiler moves some 64bit +// operation into the scope of the `set_precision` guard. So it's not like this is totally sound. +// But it's not really any less sound than the default state of 80bit precision...) #[cfg(all(target_arch = "x86", not(target_feature = "sse2")))] mod fpu_precision { use core::arch::asm; diff --git a/library/core/src/num/dec2flt/number.rs b/library/core/src/num/dec2flt/number.rs index 8589e2bbd4fac..2538991564ae4 100644 --- a/library/core/src/num/dec2flt/number.rs +++ b/library/core/src/num/dec2flt/number.rs @@ -51,6 +51,7 @@ impl Number { /// There is an exception: disguised fast-path cases, where we can shift /// powers-of-10 from the exponent to the significant digits. pub fn try_fast_path(&self) -> Option { + // Here we need to work around . // The fast path crucially depends on arithmetic being rounded to the correct number of bits // without any intermediate rounding. On x86 (without SSE or SSE2) this requires the precision // of the x87 FPU stack to be changed so that it directly rounds to 64/32 bit. diff --git a/src/tools/miri/src/borrow_tracker/stacked_borrows/mod.rs b/src/tools/miri/src/borrow_tracker/stacked_borrows/mod.rs index 0351f586872eb..a54bd32cd4097 100644 --- a/src/tools/miri/src/borrow_tracker/stacked_borrows/mod.rs +++ b/src/tools/miri/src/borrow_tracker/stacked_borrows/mod.rs @@ -615,7 +615,7 @@ trait EvalContextPrivExt<'mir: 'ecx, 'tcx: 'mir, 'ecx>: crate::MiriInterpCxExt<' ) -> InterpResult<'tcx, Option> { let this = self.eval_context_mut(); // Ensure we bail out if the pointer goes out-of-bounds (see miri#1050). - this.check_ptr_access_align(place.ptr, size, Align::ONE, CheckInAllocMsg::InboundsTest)?; + this.check_ptr_access_align(place.ptr(), size, Align::ONE, CheckInAllocMsg::InboundsTest)?; // It is crucial that this gets called on all code paths, to ensure we track tag creation. let log_creation = |this: &MiriInterpCx<'mir, 'tcx>, @@ -682,7 +682,7 @@ trait EvalContextPrivExt<'mir: 'ecx, 'tcx: 'mir, 'ecx>: crate::MiriInterpCxExt<' trace!( "reborrow of size 0: reference {:?} derived from {:?} (pointee {})", new_tag, - place.ptr, + place.ptr(), place.layout.ty, ); // Don't update any stacks for a zero-sized access; borrow stacks are per-byte and this @@ -692,7 +692,7 @@ trait EvalContextPrivExt<'mir: 'ecx, 'tcx: 'mir, 'ecx>: crate::MiriInterpCxExt<' // attempt to use it for a non-zero-sized access. // Dangling slices are a common case here; it's valid to get their length but with raw // pointer tagging for example all calls to get_unchecked on them are invalid. - if let Ok((alloc_id, base_offset, orig_tag)) = this.ptr_try_get_alloc_id(place.ptr) { + if let Ok((alloc_id, base_offset, orig_tag)) = this.ptr_try_get_alloc_id(place.ptr()) { log_creation(this, Some((alloc_id, base_offset, orig_tag)))?; // Still give it the new provenance, it got retagged after all. return Ok(Some(Provenance::Concrete { alloc_id, tag: new_tag })); @@ -700,11 +700,11 @@ trait EvalContextPrivExt<'mir: 'ecx, 'tcx: 'mir, 'ecx>: crate::MiriInterpCxExt<' // This pointer doesn't come with an AllocId. :shrug: log_creation(this, None)?; // Provenance unchanged. - return Ok(place.ptr.provenance); + return Ok(place.ptr().provenance); } } - let (alloc_id, base_offset, orig_tag) = this.ptr_get_alloc_id(place.ptr)?; + let (alloc_id, base_offset, orig_tag) = this.ptr_get_alloc_id(place.ptr())?; log_creation(this, Some((alloc_id, base_offset, orig_tag)))?; trace!( diff --git a/src/tools/miri/src/borrow_tracker/tree_borrows/mod.rs b/src/tools/miri/src/borrow_tracker/tree_borrows/mod.rs index 08b138c68aec8..8dd925a0ad80a 100644 --- a/src/tools/miri/src/borrow_tracker/tree_borrows/mod.rs +++ b/src/tools/miri/src/borrow_tracker/tree_borrows/mod.rs @@ -172,7 +172,7 @@ trait EvalContextPrivExt<'mir: 'ecx, 'tcx: 'mir, 'ecx>: crate::MiriInterpCxExt<' let this = self.eval_context_mut(); // Ensure we bail out if the pointer goes out-of-bounds (see miri#1050). this.check_ptr_access_align( - place.ptr, + place.ptr(), ptr_size, Align::ONE, CheckInAllocMsg::InboundsTest, @@ -197,7 +197,7 @@ trait EvalContextPrivExt<'mir: 'ecx, 'tcx: 'mir, 'ecx>: crate::MiriInterpCxExt<' }; trace!("Reborrow of size {:?}", ptr_size); - let (alloc_id, base_offset, parent_prov) = match this.ptr_try_get_alloc_id(place.ptr) { + let (alloc_id, base_offset, parent_prov) = match this.ptr_try_get_alloc_id(place.ptr()) { Ok(data) => { // Unlike SB, we *do* a proper retag for size 0 if can identify the allocation. // After all, the pointer may be lazily initialized outside this initial range. @@ -210,18 +210,18 @@ trait EvalContextPrivExt<'mir: 'ecx, 'tcx: 'mir, 'ecx>: crate::MiriInterpCxExt<' trace!( "reborrow of size 0: reference {:?} derived from {:?} (pointee {})", new_tag, - place.ptr, + place.ptr(), place.layout.ty, ); log_creation(this, None)?; // Keep original provenance. - return Ok(place.ptr.provenance); + return Ok(place.ptr().provenance); } }; log_creation(this, Some((alloc_id, base_offset, parent_prov)))?; let orig_tag = match parent_prov { - ProvenanceExtra::Wildcard => return Ok(place.ptr.provenance), // TODO: handle wildcard pointers + ProvenanceExtra::Wildcard => return Ok(place.ptr().provenance), // TODO: handle wildcard pointers ProvenanceExtra::Concrete(tag) => tag, }; diff --git a/src/tools/miri/src/concurrency/data_race.rs b/src/tools/miri/src/concurrency/data_race.rs index 37125337d6f0e..513cec1f6c374 100644 --- a/src/tools/miri/src/concurrency/data_race.rs +++ b/src/tools/miri/src/concurrency/data_race.rs @@ -1018,7 +1018,7 @@ trait EvalContextPrivExt<'mir, 'tcx: 'mir>: MiriInterpCxExt<'mir, 'tcx> { // be 8-aligned). let align = Align::from_bytes(place.layout.size.bytes()).unwrap(); this.check_ptr_access_align( - place.ptr, + place.ptr(), place.layout.size, align, CheckInAllocMsg::MemoryAccessTest, @@ -1031,7 +1031,7 @@ trait EvalContextPrivExt<'mir, 'tcx: 'mir>: MiriInterpCxExt<'mir, 'tcx> { // We avoid `get_ptr_alloc` since we do *not* want to run the access hooks -- the actual // access will happen later. let (alloc_id, _offset, _prov) = - this.ptr_try_get_alloc_id(place.ptr).expect("there are no zero-sized atomic accesses"); + this.ptr_try_get_alloc_id(place.ptr()).expect("there are no zero-sized atomic accesses"); if this.get_alloc_mutability(alloc_id)? == Mutability::Not { // FIXME: make this prettier, once these messages have separate title/span/help messages. throw_ub_format!( @@ -1135,7 +1135,7 @@ trait EvalContextPrivExt<'mir, 'tcx: 'mir>: MiriInterpCxExt<'mir, 'tcx> { if let Some(data_race) = &this.machine.data_race { if data_race.race_detecting() { let size = place.layout.size; - let (alloc_id, base_offset, _prov) = this.ptr_get_alloc_id(place.ptr)?; + let (alloc_id, base_offset, _prov) = this.ptr_get_alloc_id(place.ptr())?; // Load and log the atomic operation. // Note that atomic loads are possible even from read-only allocations, so `get_alloc_extra_mut` is not an option. let alloc_meta = this.get_alloc_extra(alloc_id)?.data_race.as_ref().unwrap(); @@ -1143,7 +1143,7 @@ trait EvalContextPrivExt<'mir, 'tcx: 'mir>: MiriInterpCxExt<'mir, 'tcx> { "Atomic op({}) with ordering {:?} on {:?} (size={})", description, &atomic, - place.ptr, + place.ptr(), size.bytes() ); @@ -1186,7 +1186,7 @@ trait EvalContextPrivExt<'mir, 'tcx: 'mir>: MiriInterpCxExt<'mir, 'tcx> { { log::trace!( "Updated atomic memory({:?}, size={}) to {:#?}", - place.ptr, + place.ptr(), size.bytes(), mem_clocks.atomic_ops ); diff --git a/src/tools/miri/src/concurrency/thread.rs b/src/tools/miri/src/concurrency/thread.rs index 9c11ad85aefd3..295d86fd24021 100644 --- a/src/tools/miri/src/concurrency/thread.rs +++ b/src/tools/miri/src/concurrency/thread.rs @@ -8,6 +8,7 @@ use std::task::Poll; use std::time::{Duration, SystemTime}; use log::trace; +use either::Either; use rustc_data_structures::fx::FxHashMap; use rustc_hir::def_id::DefId; @@ -259,8 +260,15 @@ impl VisitTags for Frame<'_, '_, Provenance, FrameExtra<'_>> { return_place.visit_tags(visit); // Locals. for local in locals.iter() { - if let LocalValue::Live(value) = &local.value { - value.visit_tags(visit); + match local.as_mplace_or_imm() { + None => {} + Some(Either::Left((ptr, meta))) => { + ptr.visit_tags(visit); + meta.visit_tags(visit); + } + Some(Either::Right(imm)) => { + imm.visit_tags(visit); + } } } diff --git a/src/tools/miri/src/concurrency/weak_memory.rs b/src/tools/miri/src/concurrency/weak_memory.rs index c1395468fee2e..6781b1f6dd382 100644 --- a/src/tools/miri/src/concurrency/weak_memory.rs +++ b/src/tools/miri/src/concurrency/weak_memory.rs @@ -481,7 +481,7 @@ pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>: place: &MPlaceTy<'tcx, Provenance>, ) -> InterpResult<'tcx> { let this = self.eval_context_ref(); - let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(place.ptr)?; + let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(place.ptr())?; if let crate::AllocExtra { weak_memory: Some(alloc_buffers), data_race: Some(alloc_clocks), @@ -512,7 +512,7 @@ pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>: init: Scalar, ) -> InterpResult<'tcx> { let this = self.eval_context_mut(); - let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(place.ptr)?; + let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(place.ptr())?; if let ( crate::AllocExtra { weak_memory: Some(alloc_buffers), .. }, crate::MiriMachine { data_race: Some(global), threads, .. }, @@ -539,7 +539,7 @@ pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>: ) -> InterpResult<'tcx, Scalar> { let this = self.eval_context_ref(); if let Some(global) = &this.machine.data_race { - let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(place.ptr)?; + let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(place.ptr())?; if let Some(alloc_buffers) = this.get_alloc_extra(alloc_id)?.weak_memory.as_ref() { if atomic == AtomicReadOrd::SeqCst { global.sc_read(&this.machine.threads); @@ -577,7 +577,7 @@ pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>: init: Scalar, ) -> InterpResult<'tcx> { let this = self.eval_context_mut(); - let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(dest.ptr)?; + let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(dest.ptr())?; if let ( crate::AllocExtra { weak_memory: Some(alloc_buffers), .. }, crate::MiriMachine { data_race: Some(global), threads, .. }, @@ -627,7 +627,7 @@ pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>: global.sc_read(&this.machine.threads); } let size = place.layout.size; - let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(place.ptr)?; + let (alloc_id, base_offset, ..) = this.ptr_get_alloc_id(place.ptr())?; if let Some(alloc_buffers) = this.get_alloc_extra(alloc_id)?.weak_memory.as_ref() { let buffer = alloc_buffers .get_or_create_store_buffer(alloc_range(base_offset, size), init)?; diff --git a/src/tools/miri/src/diagnostics.rs b/src/tools/miri/src/diagnostics.rs index 298c53675229b..4f249acda1df1 100644 --- a/src/tools/miri/src/diagnostics.rs +++ b/src/tools/miri/src/diagnostics.rs @@ -376,9 +376,9 @@ pub fn report_error<'tcx, 'mir>( for (i, frame) in ecx.active_thread_stack().iter().enumerate() { trace!("-------------------"); trace!("Frame {}", i); - trace!(" return: {:?}", *frame.return_place); + trace!(" return: {:?}", frame.return_place); for (i, local) in frame.locals.iter().enumerate() { - trace!(" local {}: {:?}", i, local.value); + trace!(" local {}: {:?}", i, local); } } diff --git a/src/tools/miri/src/eval.rs b/src/tools/miri/src/eval.rs index 88e7d5386db79..beb13ebdfe6fd 100644 --- a/src/tools/miri/src/eval.rs +++ b/src/tools/miri/src/eval.rs @@ -242,10 +242,7 @@ impl MainThreadState { }, Done => { // Figure out exit code. - let ret_place = MPlaceTy::from_aligned_ptr( - this.machine.main_fn_ret_place.unwrap().ptr, - this.machine.layouts.isize, - ); + let ret_place = this.machine.main_fn_ret_place.clone().unwrap(); let exit_code = this.read_target_isize(&ret_place)?; // Need to call this ourselves since we are not going to return to the scheduler // loop, and we want the main thread TLS to not show up as memory leaks. @@ -308,7 +305,7 @@ pub fn create_ecx<'mir, 'tcx: 'mir>( let arg_type = Ty::new_array(tcx, tcx.types.u8, size); let arg_place = ecx.allocate(ecx.layout_of(arg_type)?, MiriMemoryKind::Machine.into())?; - ecx.write_os_str_to_c_str(OsStr::new(arg), arg_place.ptr, size)?; + ecx.write_os_str_to_c_str(OsStr::new(arg), arg_place.ptr(), size)?; ecx.mark_immutable(&arg_place); argvs.push(arg_place.to_ref(&ecx)); } @@ -332,7 +329,7 @@ pub fn create_ecx<'mir, 'tcx: 'mir>( ecx.allocate(ecx.machine.layouts.isize, MiriMemoryKind::Machine.into())?; ecx.write_scalar(argc, &argc_place)?; ecx.mark_immutable(&argc_place); - ecx.machine.argc = Some(*argc_place); + ecx.machine.argc = Some(argc_place.ptr()); let argv_place = ecx.allocate( ecx.layout_of(Ty::new_imm_ptr(tcx, tcx.types.unit))?, @@ -340,7 +337,7 @@ pub fn create_ecx<'mir, 'tcx: 'mir>( )?; ecx.write_immediate(argv, &argv_place)?; ecx.mark_immutable(&argv_place); - ecx.machine.argv = Some(*argv_place); + ecx.machine.argv = Some(argv_place.ptr()); } // Store command line as UTF-16 for Windows `GetCommandLineW`. { @@ -351,7 +348,7 @@ pub fn create_ecx<'mir, 'tcx: 'mir>( Ty::new_array(tcx, tcx.types.u16, u64::try_from(cmd_utf16.len()).unwrap()); let cmd_place = ecx.allocate(ecx.layout_of(cmd_type)?, MiriMemoryKind::Machine.into())?; - ecx.machine.cmd_line = Some(*cmd_place); + ecx.machine.cmd_line = Some(cmd_place.ptr()); // Store the UTF-16 string. We just allocated so we know the bounds are fine. for (idx, &c) in cmd_utf16.iter().enumerate() { let place = ecx.project_field(&cmd_place, idx)?; @@ -364,7 +361,7 @@ pub fn create_ecx<'mir, 'tcx: 'mir>( // Return place (in static memory so that it does not count as leak). let ret_place = ecx.allocate(ecx.machine.layouts.isize, MiriMemoryKind::Machine.into())?; - ecx.machine.main_fn_ret_place = Some(*ret_place); + ecx.machine.main_fn_ret_place = Some(ret_place.clone()); // Call start function. match entry_type { diff --git a/src/tools/miri/src/helpers.rs b/src/tools/miri/src/helpers.rs index 844827889a7b2..0c7e82781476d 100644 --- a/src/tools/miri/src/helpers.rs +++ b/src/tools/miri/src/helpers.rs @@ -381,7 +381,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { // Store how far we proceeded into the place so far. Everything to the left of // this offset has already been handled, in the sense that the frozen parts // have had `action` called on them. - let start_addr = place.ptr.addr(); + let start_addr = place.ptr().addr(); let mut cur_addr = start_addr; // Called when we detected an `UnsafeCell` at the given offset and size. // Calls `action` and advances `cur_ptr`. @@ -413,7 +413,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { let mut visitor = UnsafeCellVisitor { ecx: this, unsafe_cell_action: |place| { - trace!("unsafe_cell_action on {:?}", place.ptr); + trace!("unsafe_cell_action on {:?}", place.ptr()); // We need a size to go on. let unsafe_cell_size = this .size_and_align_of_mplace(place)? @@ -422,7 +422,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { .unwrap_or_else(|| place.layout.size); // Now handle this `UnsafeCell`, unless it is empty. if unsafe_cell_size != Size::ZERO { - unsafe_cell_action(&place.ptr, unsafe_cell_size) + unsafe_cell_action(&place.ptr(), unsafe_cell_size) } else { Ok(()) } @@ -432,7 +432,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { } // The part between the end_ptr and the end of the place is also frozen. // So pretend there is a 0-sized `UnsafeCell` at the end. - unsafe_cell_action(&place.ptr.offset(size, this)?, Size::ZERO)?; + unsafe_cell_action(&place.ptr().offset(size, this)?, Size::ZERO)?; // Done! return Ok(()); @@ -994,10 +994,10 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { } /// Mark a machine allocation that was just created as immutable. - fn mark_immutable(&mut self, mplace: &MemPlace) { + fn mark_immutable(&mut self, mplace: &MPlaceTy<'tcx, Provenance>) { let this = self.eval_context_mut(); // This got just allocated, so there definitely is a pointer here. - let provenance = mplace.ptr.into_pointer_or_addr().unwrap().provenance; + let provenance = mplace.ptr().into_pointer_or_addr().unwrap().provenance; this.alloc_mark_immutable(provenance.get_alloc_id().unwrap()).unwrap(); } diff --git a/src/tools/miri/src/machine.rs b/src/tools/miri/src/machine.rs index 9d01a04bf4adb..80f8318044802 100644 --- a/src/tools/miri/src/machine.rs +++ b/src/tools/miri/src/machine.rs @@ -396,14 +396,14 @@ pub struct MiriMachine<'mir, 'tcx> { pub(crate) env_vars: EnvVars<'tcx>, /// Return place of the main function. - pub(crate) main_fn_ret_place: Option>, + pub(crate) main_fn_ret_place: Option>, /// Program arguments (`Option` because we can only initialize them after creating the ecx). /// These are *pointers* to argc/argv because macOS. /// We also need the full command line as one string because of Windows. - pub(crate) argc: Option>, - pub(crate) argv: Option>, - pub(crate) cmd_line: Option>, + pub(crate) argc: Option>>, + pub(crate) argv: Option>>, + pub(crate) cmd_line: Option>>, /// TLS state. pub(crate) tls: TlsData<'tcx>, @@ -670,7 +670,7 @@ impl<'mir, 'tcx> MiriMachine<'mir, 'tcx> { ) -> InterpResult<'tcx> { let place = this.allocate(val.layout, MiriMemoryKind::ExternStatic.into())?; this.write_immediate(*val, &place)?; - Self::add_extern_static(this, name, place.ptr); + Self::add_extern_static(this, name, place.ptr()); Ok(()) } @@ -686,7 +686,7 @@ impl<'mir, 'tcx> MiriMachine<'mir, 'tcx> { Self::add_extern_static( this, "environ", - this.machine.env_vars.environ.as_ref().unwrap().ptr, + this.machine.env_vars.environ.as_ref().unwrap().ptr(), ); // A couple zero-initialized pointer-sized extern statics. // Most of them are for weak symbols, which we all set to null (indicating that the @@ -703,7 +703,7 @@ impl<'mir, 'tcx> MiriMachine<'mir, 'tcx> { Self::add_extern_static( this, "environ", - this.machine.env_vars.environ.as_ref().unwrap().ptr, + this.machine.env_vars.environ.as_ref().unwrap().ptr(), ); } "android" => { @@ -1415,7 +1415,7 @@ impl<'mir, 'tcx> Machine<'mir, 'tcx> for MiriMachine<'mir, 'tcx> { local: mir::Local, mplace: &MPlaceTy<'tcx, Provenance>, ) -> InterpResult<'tcx> { - let Some(Provenance::Concrete { alloc_id, .. }) = mplace.ptr.provenance else { + let Some(Provenance::Concrete { alloc_id, .. }) = mplace.ptr().provenance else { panic!("after_local_allocated should only be called on fresh allocations"); }; let local_decl = &ecx.active_thread_stack()[frame].body.local_decls[local]; diff --git a/src/tools/miri/src/shims/backtrace.rs b/src/tools/miri/src/shims/backtrace.rs index a3297bf819f0a..c4c9f77fab7f9 100644 --- a/src/tools/miri/src/shims/backtrace.rs +++ b/src/tools/miri/src/shims/backtrace.rs @@ -89,7 +89,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { } this.write_immediate( - Immediate::new_slice(Scalar::from_maybe_pointer(alloc.ptr, this), len, this), + Immediate::new_slice(Scalar::from_maybe_pointer(alloc.ptr(), this), len, this), dest, )?; } diff --git a/src/tools/miri/src/shims/env.rs b/src/tools/miri/src/shims/env.rs index 3694ea51da6c6..154a7f6983343 100644 --- a/src/tools/miri/src/shims/env.rs +++ b/src/tools/miri/src/shims/env.rs @@ -459,7 +459,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { let place = this.project_field(&vars_place, idx)?; this.write_pointer(var, &place)?; } - this.write_pointer(vars_place.ptr, &this.machine.env_vars.environ.clone().unwrap())?; + this.write_pointer(vars_place.ptr(), &this.machine.env_vars.environ.clone().unwrap())?; Ok(()) } diff --git a/src/tools/miri/src/shims/foreign_items.rs b/src/tools/miri/src/shims/foreign_items.rs index 7996e72615f0e..47cbd4419f3dc 100644 --- a/src/tools/miri/src/shims/foreign_items.rs +++ b/src/tools/miri/src/shims/foreign_items.rs @@ -324,7 +324,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { // Second: functions that return immediately. match this.emulate_foreign_item_by_name(link_name, abi, args, dest)? { EmulateByNameResult::NeedsJumping => { - trace!("{:?}", this.dump_place(**dest)); + trace!("{:?}", this.dump_place(dest)); this.go_to_block(ret); } EmulateByNameResult::AlreadyJumped => (), diff --git a/src/tools/miri/src/shims/intrinsics/mod.rs b/src/tools/miri/src/shims/intrinsics/mod.rs index 8a84b4f51b7f2..ef9d071044818 100644 --- a/src/tools/miri/src/shims/intrinsics/mod.rs +++ b/src/tools/miri/src/shims/intrinsics/mod.rs @@ -62,7 +62,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { // The rest jumps to `ret` immediately. this.emulate_intrinsic_by_name(intrinsic_name, args, dest)?; - trace!("{:?}", this.dump_place(**dest)); + trace!("{:?}", this.dump_place(dest)); this.go_to_block(ret); Ok(()) } diff --git a/src/tools/miri/src/shims/os_str.rs b/src/tools/miri/src/shims/os_str.rs index a94b1ddcd95b9..f6e76545a4d8d 100644 --- a/src/tools/miri/src/shims/os_str.rs +++ b/src/tools/miri/src/shims/os_str.rs @@ -143,9 +143,9 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { let arg_type = Ty::new_array(this.tcx.tcx, this.tcx.types.u8, size); let arg_place = this.allocate(this.layout_of(arg_type).unwrap(), memkind)?; - let (written, _) = self.write_os_str_to_c_str(os_str, arg_place.ptr, size).unwrap(); + let (written, _) = self.write_os_str_to_c_str(os_str, arg_place.ptr(), size).unwrap(); assert!(written); - Ok(arg_place.ptr) + Ok(arg_place.ptr()) } /// Allocate enough memory to store the given `OsStr` as a null-terminated sequence of `u16`. @@ -160,9 +160,9 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { let arg_type = Ty::new_array(this.tcx.tcx, this.tcx.types.u16, size); let arg_place = this.allocate(this.layout_of(arg_type).unwrap(), memkind)?; let (written, _) = - self.write_os_str_to_wide_str(os_str, arg_place.ptr, size, /*truncate*/ false).unwrap(); + self.write_os_str_to_wide_str(os_str, arg_place.ptr(), size, /*truncate*/ false).unwrap(); assert!(written); - Ok(arg_place.ptr) + Ok(arg_place.ptr()) } /// Read a null-terminated sequence of bytes, and perform path separator conversion if needed. diff --git a/src/tools/miri/src/shims/unix/android/dlsym.rs b/src/tools/miri/src/shims/unix/android/dlsym.rs index b0c9d729c9d90..451bc0bd5e153 100644 --- a/src/tools/miri/src/shims/unix/android/dlsym.rs +++ b/src/tools/miri/src/shims/unix/android/dlsym.rs @@ -47,7 +47,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { } } - log::trace!("{:?}", this.dump_place(**dest)); + log::trace!("{:?}", this.dump_place(dest)); this.go_to_block(ret); Ok(()) } diff --git a/src/tools/miri/src/shims/unix/foreign_items.rs b/src/tools/miri/src/shims/unix/foreign_items.rs index 3a48019053563..865f01931f706 100644 --- a/src/tools/miri/src/shims/unix/foreign_items.rs +++ b/src/tools/miri/src/shims/unix/foreign_items.rs @@ -597,7 +597,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { this.write_pointer(buf, &pw_dir)?; if written { - this.write_pointer(pwd.ptr, &result)?; + this.write_pointer(pwd.ptr(), &result)?; this.write_null(dest)?; } else { this.write_null(&result)?; diff --git a/src/tools/miri/src/shims/unix/fs.rs b/src/tools/miri/src/shims/unix/fs.rs index dd2da3d22f286..8963dfb97a6e5 100644 --- a/src/tools/miri/src/shims/unix/fs.rs +++ b/src/tools/miri/src/shims/unix/fs.rs @@ -1439,7 +1439,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { let file_name = dir_entry.file_name(); // not a Path as there are no separators! let (name_fits, file_name_buf_len) = this.write_os_str_to_c_str( &file_name, - name_place.ptr, + name_place.ptr(), name_place.layout.size.bytes(), )?; let file_name_len = file_name_buf_len.checked_sub(1).unwrap(); diff --git a/src/tools/miri/src/shims/unix/linux/sync.rs b/src/tools/miri/src/shims/unix/linux/sync.rs index 0474c9fd90a5c..7d15abfbfb2d8 100644 --- a/src/tools/miri/src/shims/unix/linux/sync.rs +++ b/src/tools/miri/src/shims/unix/linux/sync.rs @@ -35,7 +35,7 @@ pub fn futex<'tcx>( let thread = this.get_active_thread(); // This is a vararg function so we have to bring our own type for this pointer. let addr = MPlaceTy::from_aligned_ptr(addr, this.machine.layouts.i32); - let addr_usize = addr.ptr.addr().bytes(); + let addr_usize = addr.ptr().addr().bytes(); let futex_private = this.eval_libc_i32("FUTEX_PRIVATE_FLAG"); let futex_wait = this.eval_libc_i32("FUTEX_WAIT"); @@ -90,7 +90,7 @@ pub fn futex<'tcx>( &this.read_immediate(&args[3])?, this.libc_ty_layout("timespec"), )?; - let timeout_time = if this.ptr_is_null(timeout.ptr)? { + let timeout_time = if this.ptr_is_null(timeout.ptr())? { None } else { let realtime = op & futex_realtime == futex_realtime; diff --git a/src/tools/miri/src/shims/unix/macos/dlsym.rs b/src/tools/miri/src/shims/unix/macos/dlsym.rs index 9177ecefe1207..63ad680de60fa 100644 --- a/src/tools/miri/src/shims/unix/macos/dlsym.rs +++ b/src/tools/miri/src/shims/unix/macos/dlsym.rs @@ -45,7 +45,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { } } - trace!("{:?}", this.dump_place(**dest)); + trace!("{:?}", this.dump_place(dest)); this.go_to_block(ret); Ok(()) } diff --git a/src/tools/miri/src/shims/unix/macos/foreign_items.rs b/src/tools/miri/src/shims/unix/macos/foreign_items.rs index f073daab8ed8e..3524fd70bcf85 100644 --- a/src/tools/miri/src/shims/unix/macos/foreign_items.rs +++ b/src/tools/miri/src/shims/unix/macos/foreign_items.rs @@ -91,7 +91,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { .environ .as_ref() .expect("machine must be initialized") - .ptr, + .ptr(), dest, )?; } @@ -113,14 +113,14 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { "_NSGetArgc" => { let [] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; this.write_pointer( - this.machine.argc.expect("machine must be initialized").ptr, + this.machine.argc.expect("machine must be initialized"), dest, )?; } "_NSGetArgv" => { let [] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; this.write_pointer( - this.machine.argv.expect("machine must be initialized").ptr, + this.machine.argv.expect("machine must be initialized"), dest, )?; } diff --git a/src/tools/miri/src/shims/windows/dlsym.rs b/src/tools/miri/src/shims/windows/dlsym.rs index 7e2051fc98a0e..e5afee35905d0 100644 --- a/src/tools/miri/src/shims/windows/dlsym.rs +++ b/src/tools/miri/src/shims/windows/dlsym.rs @@ -75,7 +75,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { } } - trace!("{:?}", this.dump_place(**dest)); + trace!("{:?}", this.dump_place(dest)); this.go_to_block(ret); Ok(()) } diff --git a/src/tools/miri/src/shims/windows/foreign_items.rs b/src/tools/miri/src/shims/windows/foreign_items.rs index cb90ed57ffe99..28999268ba052 100644 --- a/src/tools/miri/src/shims/windows/foreign_items.rs +++ b/src/tools/miri/src/shims/windows/foreign_items.rs @@ -190,7 +190,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { this.deref_pointer_as(system_info, this.windows_ty_layout("SYSTEM_INFO"))?; // Initialize with `0`. this.write_bytes_ptr( - system_info.ptr, + system_info.ptr(), iter::repeat(0u8).take(system_info.layout.size.bytes_usize()), )?; // Set selected fields. @@ -235,7 +235,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { "GetCommandLineW" => { let [] = this.check_shim(abi, Abi::System { unwind: false }, link_name, args)?; this.write_pointer( - this.machine.cmd_line.expect("machine must be initialized").ptr, + this.machine.cmd_line.expect("machine must be initialized"), dest, )?; } diff --git a/src/tools/miri/src/tag_gc.rs b/src/tools/miri/src/tag_gc.rs index cefdcc2b5b83d..3cccdd3635388 100644 --- a/src/tools/miri/src/tag_gc.rs +++ b/src/tools/miri/src/tag_gc.rs @@ -1,3 +1,5 @@ +use either::Either; + use rustc_data_structures::fx::FxHashSet; use crate::*; @@ -81,46 +83,33 @@ impl VisitTags for MemPlaceMeta { } } -impl VisitTags for MemPlace { +impl VisitTags for ImmTy<'_, Provenance> { fn visit_tags(&self, visit: &mut dyn FnMut(BorTag)) { - let MemPlace { ptr, meta } = self; - ptr.visit_tags(visit); - meta.visit_tags(visit); + (**self).visit_tags(visit) } } impl VisitTags for MPlaceTy<'_, Provenance> { fn visit_tags(&self, visit: &mut dyn FnMut(BorTag)) { - (**self).visit_tags(visit) - } -} - -impl VisitTags for Place { - fn visit_tags(&self, visit: &mut dyn FnMut(BorTag)) { - match self { - Place::Ptr(p) => p.visit_tags(visit), - Place::Local { .. } => { - // Will be visited as part of the stack frame. - } - } + self.ptr().visit_tags(visit); + self.meta().visit_tags(visit); } } impl VisitTags for PlaceTy<'_, Provenance> { fn visit_tags(&self, visit: &mut dyn FnMut(BorTag)) { - (**self).visit_tags(visit) + match self.as_mplace_or_local() { + Either::Left(mplace) => mplace.visit_tags(visit), + Either::Right(_) => (), + } } } -impl VisitTags for Operand { +impl VisitTags for OpTy<'_, Provenance> { fn visit_tags(&self, visit: &mut dyn FnMut(BorTag)) { - match self { - Operand::Immediate(imm) => { - imm.visit_tags(visit); - } - Operand::Indirect(p) => { - p.visit_tags(visit); - } + match self.as_mplace_or_imm() { + Either::Left(mplace) => mplace.visit_tags(visit), + Either::Right(imm) => imm.visit_tags(visit), } } } diff --git a/tests/mir-opt/building/custom/debuginfo.numbered.built.after.mir b/tests/mir-opt/building/custom/debuginfo.numbered.built.after.mir new file mode 100644 index 0000000000000..d863925371812 --- /dev/null +++ b/tests/mir-opt/building/custom/debuginfo.numbered.built.after.mir @@ -0,0 +1,11 @@ +// MIR for `numbered` after built + +fn numbered(_1: (u32, i32)) -> () { + debug first => (_1.0: u32); + debug second => (_1.0: u32); + let mut _0: (); + + bb0: { + return; + } +} diff --git a/tests/mir-opt/building/custom/debuginfo.pointee.built.after.mir b/tests/mir-opt/building/custom/debuginfo.pointee.built.after.mir new file mode 100644 index 0000000000000..86cced6f801f4 --- /dev/null +++ b/tests/mir-opt/building/custom/debuginfo.pointee.built.after.mir @@ -0,0 +1,10 @@ +// MIR for `pointee` after built + +fn pointee(_1: &mut Option) -> () { + debug foo => (((*_1) as variant#1).0: i32); + let mut _0: (); + + bb0: { + return; + } +} diff --git a/tests/mir-opt/building/custom/debuginfo.rs b/tests/mir-opt/building/custom/debuginfo.rs new file mode 100644 index 0000000000000..bfdc3d3eacd44 --- /dev/null +++ b/tests/mir-opt/building/custom/debuginfo.rs @@ -0,0 +1,71 @@ +#![feature(custom_mir, core_intrinsics)] + +extern crate core; +use core::intrinsics::mir::*; + +// EMIT_MIR debuginfo.pointee.built.after.mir +#[custom_mir(dialect = "built")] +fn pointee(opt: &mut Option) { + mir!( + debug foo => Field::(Variant(*opt, 1), 0); + { + Return() + } + ) +} + +// EMIT_MIR debuginfo.numbered.built.after.mir +#[custom_mir(dialect = "analysis", phase = "post-cleanup")] +fn numbered(i: (u32, i32)) { + mir!( + debug first => i.0; + debug second => i.0; + { + Return() + } + ) +} + +struct S { x: f32 } + +// EMIT_MIR debuginfo.structured.built.after.mir +#[custom_mir(dialect = "analysis", phase = "post-cleanup")] +fn structured(i: S) { + mir!( + debug x => i.x; + { + Return() + } + ) +} + +// EMIT_MIR debuginfo.variant.built.after.mir +#[custom_mir(dialect = "built")] +fn variant(opt: Option) { + mir!( + debug inner => Field::(Variant(opt, 1), 0); + { + Return() + } + ) +} + +// EMIT_MIR debuginfo.variant_deref.built.after.mir +#[custom_mir(dialect = "built")] +fn variant_deref(opt: Option<&i32>) { + mir!( + debug pointer => Field::<&i32>(Variant(opt, 1), 0); + debug deref => *Field::<&i32>(Variant(opt, 1), 0); + { + Return() + } + ) +} + +fn main() { + numbered((5, 6)); + structured(S { x: 5. }); + variant(Some(5)); + variant_deref(Some(&5)); + pointee(&mut Some(5)); +} diff --git a/tests/mir-opt/building/custom/debuginfo.structured.built.after.mir b/tests/mir-opt/building/custom/debuginfo.structured.built.after.mir new file mode 100644 index 0000000000000..d122b6bfe29a4 --- /dev/null +++ b/tests/mir-opt/building/custom/debuginfo.structured.built.after.mir @@ -0,0 +1,10 @@ +// MIR for `structured` after built + +fn structured(_1: S) -> () { + debug x => (_1.0: f32); + let mut _0: (); + + bb0: { + return; + } +} diff --git a/tests/mir-opt/building/custom/debuginfo.variant.built.after.mir b/tests/mir-opt/building/custom/debuginfo.variant.built.after.mir new file mode 100644 index 0000000000000..d173723fd8922 --- /dev/null +++ b/tests/mir-opt/building/custom/debuginfo.variant.built.after.mir @@ -0,0 +1,10 @@ +// MIR for `variant` after built + +fn variant(_1: Option) -> () { + debug inner => ((_1 as variant#1).0: i32); + let mut _0: (); + + bb0: { + return; + } +} diff --git a/tests/mir-opt/building/custom/debuginfo.variant_deref.built.after.mir b/tests/mir-opt/building/custom/debuginfo.variant_deref.built.after.mir new file mode 100644 index 0000000000000..37d5d1b2dfc0b --- /dev/null +++ b/tests/mir-opt/building/custom/debuginfo.variant_deref.built.after.mir @@ -0,0 +1,11 @@ +// MIR for `variant_deref` after built + +fn variant_deref(_1: Option<&i32>) -> () { + debug pointer => ((_1 as variant#1).0: &i32); + debug deref => (*((_1 as variant#1).0: &i32)); + let mut _0: (); + + bb0: { + return; + } +}