diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index 39d6b1267a593..18c2dd71ad3d0 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -1915,6 +1915,27 @@ impl<'tcx> Place<'tcx> { (base, proj) }) } + + /// Generates a new place by appending `more_projections` to the existing ones + /// and interning the result. + pub fn project_deeper(self, more_projections: &[PlaceElem<'tcx>], tcx: TyCtxt<'tcx>) -> Self { + if more_projections.is_empty() { + return self; + } + + let mut v: Vec>; + + let new_projections = if self.projection.is_empty() { + more_projections + } else { + v = Vec::with_capacity(self.projection.len() + more_projections.len()); + v.extend(self.projection); + v.extend(more_projections); + &v + }; + + Place { local: self.local, projection: tcx.intern_place_elems(new_projections) } + } } impl From for Place<'_> { @@ -2187,6 +2208,15 @@ impl<'tcx> Operand<'tcx> { Operand::Copy(_) | Operand::Move(_) => None, } } + + /// Gets the `ty::FnDef` from an operand if it's a constant function item. + /// + /// While this is unlikely in general, it's the normal case of what you'll + /// find as the `func` in a [`TerminatorKind::Call`]. + pub fn const_fn_def(&self) -> Option<(DefId, SubstsRef<'tcx>)> { + let const_ty = self.constant()?.literal.ty(); + if let ty::FnDef(def_id, substs) = *const_ty.kind() { Some((def_id, substs)) } else { None } + } } /////////////////////////////////////////////////////////////////////////// diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index da54ad06e048b..775d86153ced2 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -2380,6 +2380,57 @@ impl<'tcx> Ty<'tcx> { } } } + + /// Fast path helper for primitives which are always `Copy` and which + /// have a side-effect-free `Clone` impl. + /// + /// Returning true means the type is known to be pure and `Copy+Clone`. + /// Returning `false` means nothing -- could be `Copy`, might not be. + /// + /// This is mostly useful for optimizations, as there are the types + /// on which we can replace cloning with dereferencing. + pub fn is_trivially_pure_clone_copy(self) -> bool { + match self.kind() { + ty::Bool | ty::Char | ty::Never => true, + + // These aren't even `Clone` + ty::Str | ty::Slice(..) | ty::Foreign(..) | ty::Dynamic(..) => false, + + ty::Int(..) | ty::Uint(..) | ty::Float(..) => true, + + // The voldemort ZSTs are fine. + ty::FnDef(..) => true, + + ty::Array(element_ty, _len) => element_ty.is_trivially_pure_clone_copy(), + + // A 100-tuple isn't "trivial", so doing this only for reasonable sizes. + ty::Tuple(field_tys) => { + field_tys.len() <= 3 && field_tys.iter().all(Self::is_trivially_pure_clone_copy) + } + + // Sometimes traits aren't implemented for every ABI or arity, + // because we can't be generic over everything yet. + ty::FnPtr(..) => false, + + // Definitely absolutely not copy. + ty::Ref(_, _, hir::Mutability::Mut) => false, + + // Thin pointers & thin shared references are pure-clone-copy, but for + // anything with custom metadata it might be more complicated. + ty::Ref(_, _, hir::Mutability::Not) | ty::RawPtr(..) => false, + + ty::Generator(..) | ty::GeneratorWitness(..) => false, + + // Might be, but not "trivial" so just giving the safe answer. + ty::Adt(..) | ty::Closure(..) | ty::Opaque(..) => false, + + ty::Projection(..) | ty::Param(..) | ty::Infer(..) | ty::Error(..) => false, + + ty::Bound(..) | ty::Placeholder(..) => { + bug!("`is_trivially_pure_clone_copy` applied to unexpected type: {:?}", self); + } + } + } } /// Extra information about why we ended up with a particular variance. diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs index 04e766d16cc49..eb16d305d0a80 100644 --- a/compiler/rustc_middle/src/ty/util.rs +++ b/compiler/rustc_middle/src/ty/util.rs @@ -704,7 +704,7 @@ impl<'tcx> Ty<'tcx> { tcx_at: TyCtxtAt<'tcx>, param_env: ty::ParamEnv<'tcx>, ) -> bool { - tcx_at.is_copy_raw(param_env.and(self)) + self.is_trivially_pure_clone_copy() || tcx_at.is_copy_raw(param_env.and(self)) } /// Checks whether values of this type `T` have a size known at diff --git a/compiler/rustc_mir_transform/src/instcombine.rs b/compiler/rustc_mir_transform/src/instcombine.rs index 385fcc43496e3..d1c4a4b21d0a2 100644 --- a/compiler/rustc_mir_transform/src/instcombine.rs +++ b/compiler/rustc_mir_transform/src/instcombine.rs @@ -4,7 +4,7 @@ use crate::MirPass; use rustc_hir::Mutability; use rustc_middle::mir::{ BinOp, Body, Constant, LocalDecls, Operand, Place, ProjectionElem, Rvalue, SourceInfo, - StatementKind, UnOp, + Statement, StatementKind, Terminator, TerminatorKind, UnOp, }; use rustc_middle::ty::{self, TyCtxt}; @@ -29,6 +29,11 @@ impl<'tcx> MirPass<'tcx> for InstCombine { _ => {} } } + + ctx.combine_primitive_clone( + &mut block.terminator.as_mut().unwrap(), + &mut block.statements, + ); } } } @@ -130,4 +135,70 @@ impl<'tcx> InstCombineContext<'tcx, '_> { } } } + + fn combine_primitive_clone( + &self, + terminator: &mut Terminator<'tcx>, + statements: &mut Vec>, + ) { + let TerminatorKind::Call { func, args, destination, .. } = &mut terminator.kind + else { return }; + + // It's definitely not a clone if there are multiple arguments + if args.len() != 1 { + return; + } + + let Some((destination_place, destination_block)) = *destination + else { return }; + + // Only bother looking more if it's easy to know what we're calling + let Some((fn_def_id, fn_substs)) = func.const_fn_def() + else { return }; + + // Clone needs one subst, so we can cheaply rule out other stuff + if fn_substs.len() != 1 { + return; + } + + // These types are easily available from locals, so check that before + // doing DefId lookups to figure out what we're actually calling. + let arg_ty = args[0].ty(self.local_decls, self.tcx); + + let ty::Ref(_region, inner_ty, Mutability::Not) = *arg_ty.kind() + else { return }; + + if !inner_ty.is_trivially_pure_clone_copy() { + return; + } + + let trait_def_id = self.tcx.trait_of_item(fn_def_id); + if trait_def_id.is_none() || trait_def_id != self.tcx.lang_items().clone_trait() { + return; + } + + if !self.tcx.consider_optimizing(|| { + format!( + "InstCombine - Call: {:?} SourceInfo: {:?}", + (fn_def_id, fn_substs), + terminator.source_info + ) + }) { + return; + } + + let Some(arg_place) = args.pop().unwrap().place() + else { return }; + + statements.push(Statement { + source_info: terminator.source_info, + kind: StatementKind::Assign(box ( + destination_place, + Rvalue::Use(Operand::Copy( + arg_place.project_deeper(&[ProjectionElem::Deref], self.tcx), + )), + )), + }); + terminator.kind = TerminatorKind::Goto { target: destination_block }; + } } diff --git a/src/test/assembly/sparc-struct-abi.rs b/src/test/assembly/sparc-struct-abi.rs index dd8e6f614df11..6a898b2974a65 100644 --- a/src/test/assembly/sparc-struct-abi.rs +++ b/src/test/assembly/sparc-struct-abi.rs @@ -13,6 +13,7 @@ pub trait Sized {} #[lang = "copy"] pub trait Copy {} +impl Copy for f32 {} #[repr(C)] pub struct Franta { diff --git a/src/test/assembly/target-feature-multiple.rs b/src/test/assembly/target-feature-multiple.rs index 4c2073678b842..18d896e86b215 100644 --- a/src/test/assembly/target-feature-multiple.rs +++ b/src/test/assembly/target-feature-multiple.rs @@ -23,6 +23,7 @@ trait Sized {} #[lang = "copy"] trait Copy {} +impl Copy for u32 {} // Use of these requires target features to be enabled extern "unadjusted" { diff --git a/src/test/codegen/abi-sysv64.rs b/src/test/codegen/abi-sysv64.rs index bb910d573b338..dfc312279083d 100644 --- a/src/test/codegen/abi-sysv64.rs +++ b/src/test/codegen/abi-sysv64.rs @@ -13,6 +13,7 @@ trait Sized {} #[lang = "copy"] trait Copy {} +impl Copy for i64 {} // CHECK: define x86_64_sysvcc i64 @has_sysv64_abi #[no_mangle] diff --git a/src/test/codegen/abi-x86-interrupt.rs b/src/test/codegen/abi-x86-interrupt.rs index 119004d261d60..d612f603e4fea 100644 --- a/src/test/codegen/abi-x86-interrupt.rs +++ b/src/test/codegen/abi-x86-interrupt.rs @@ -13,6 +13,7 @@ trait Sized {} #[lang = "copy"] trait Copy {} +impl Copy for i64 {} // CHECK: define x86_intrcc i64 @has_x86_interrupt_abi #[no_mangle] diff --git a/src/test/codegen/frame-pointer.rs b/src/test/codegen/frame-pointer.rs index 367591dcb9617..f7c02d47939fe 100644 --- a/src/test/codegen/frame-pointer.rs +++ b/src/test/codegen/frame-pointer.rs @@ -17,7 +17,7 @@ trait Sized { } #[lang="copy"] trait Copy { } - +impl Copy for u32 {} // CHECK: define i32 @peach{{.*}}[[PEACH_ATTRS:\#[0-9]+]] { diff --git a/src/test/codegen/inline-hint.rs b/src/test/codegen/inline-hint.rs index ad41badf38169..d3ea1915a8b19 100644 --- a/src/test/codegen/inline-hint.rs +++ b/src/test/codegen/inline-hint.rs @@ -6,7 +6,7 @@ pub fn f() { let a = A; - let b = (0i32, 1i32, 2i32, 3i32); + let b = (0i32, 1i32, 2i32, 3 as *const i32); let c = || {}; a(String::new(), String::new()); @@ -21,7 +21,7 @@ struct A(String, String); // CHECK-NOT: inlinehint // CHECK-SAME: {{$}} -// CHECK: ; <(i32, i32, i32, i32) as core::clone::Clone>::clone +// CHECK: ; <(i32, i32, i32, *const i{{16|32|64}}) as core::clone::Clone>::clone // CHECK-NEXT: ; Function Attrs: inlinehint // CHECK: ; inline_hint::f::{closure#0} diff --git a/src/test/codegen/riscv-abi/riscv64-lp64-lp64f-lp64d-abi.rs b/src/test/codegen/riscv-abi/riscv64-lp64-lp64f-lp64d-abi.rs index faf81b5ae7605..7f0f678062a64 100644 --- a/src/test/codegen/riscv-abi/riscv64-lp64-lp64f-lp64d-abi.rs +++ b/src/test/codegen/riscv-abi/riscv64-lp64-lp64f-lp64d-abi.rs @@ -10,6 +10,14 @@ trait Sized {} #[lang = "copy"] trait Copy {} +impl Copy for bool {} +impl Copy for i8 {} +impl Copy for u8 {} +impl Copy for i32 {} +impl Copy for i64 {} +impl Copy for u64 {} +impl Copy for f32 {} +impl Copy for f64 {} // CHECK: define void @f_void() #[no_mangle] diff --git a/src/test/mir-opt/combine_clone_of_primitives.rs b/src/test/mir-opt/combine_clone_of_primitives.rs new file mode 100644 index 0000000000000..0972d2d68a115 --- /dev/null +++ b/src/test/mir-opt/combine_clone_of_primitives.rs @@ -0,0 +1,20 @@ +// compile-flags: -C opt-level=0 -Z inline_mir=no +// ignore-wasm32 compiled with panic=abort by default + +// EMIT_MIR combine_clone_of_primitives.{impl#0}-clone.InstCombine.diff + +#[derive(Clone)] +struct MyThing { + v: T, + i: u64, + a: [f32; 3], +} + +fn main() { + let x = MyThing:: { v: 2, i: 3, a: [0.0; 3] }; + let y = x.clone(); + + assert_eq!(y.v, 2); + assert_eq!(y.i, 3); + assert_eq!(y.a, [0.0; 3]); +} diff --git a/src/test/mir-opt/combine_clone_of_primitives.{impl#0}-clone.InstCombine.diff b/src/test/mir-opt/combine_clone_of_primitives.{impl#0}-clone.InstCombine.diff new file mode 100644 index 0000000000000..62e5da4902cb4 --- /dev/null +++ b/src/test/mir-opt/combine_clone_of_primitives.{impl#0}-clone.InstCombine.diff @@ -0,0 +1,80 @@ +- // MIR for `::clone` before InstCombine ++ // MIR for `::clone` after InstCombine + + fn ::clone(_1: &MyThing) -> MyThing { + debug self => _1; // in scope 0 at $DIR/combine_clone_of_primitives.rs:6:10: 6:15 + let mut _0: MyThing; // return place in scope 0 at $DIR/combine_clone_of_primitives.rs:6:10: 6:15 + let _2: &T; // in scope 0 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9 + let _3: &u64; // in scope 0 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11 + let _4: &[f32; 3]; // in scope 0 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16 + let mut _5: T; // in scope 0 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9 + let mut _6: &T; // in scope 0 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9 + let _7: &T; // in scope 0 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9 + let mut _8: u64; // in scope 0 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11 + let mut _9: &u64; // in scope 0 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11 + let _10: &u64; // in scope 0 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11 + let mut _11: [f32; 3]; // in scope 0 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16 + let mut _12: &[f32; 3]; // in scope 0 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16 + let _13: &[f32; 3]; // in scope 0 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16 + scope 1 { + debug __self_0_0 => _2; // in scope 1 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9 + debug __self_0_1 => _3; // in scope 1 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11 + debug __self_0_2 => _4; // in scope 1 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16 + } + + bb0: { + _2 = &((*_1).0: T); // scope 0 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9 + _3 = &((*_1).1: u64); // scope 0 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11 + _4 = &((*_1).2: [f32; 3]); // scope 0 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16 +- _7 = &(*_2); // scope 1 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9 +- _6 = &(*_7); // scope 1 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9 ++ _7 = _2; // scope 1 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9 ++ _6 = _7; // scope 1 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9 + _5 = ::clone(move _6) -> bb1; // scope 1 at $DIR/combine_clone_of_primitives.rs:8:5: 8:9 + // mir::Constant + // + span: $DIR/combine_clone_of_primitives.rs:8:5: 8:9 + // + literal: Const { ty: for<'r> fn(&'r T) -> T {::clone}, val: Value(Scalar()) } + } + + bb1: { +- _10 = &(*_3); // scope 1 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11 +- _9 = &(*_10); // scope 1 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11 +- _8 = ::clone(move _9) -> [return: bb2, unwind: bb4]; // scope 1 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11 +- // mir::Constant +- // + span: $DIR/combine_clone_of_primitives.rs:9:5: 9:11 +- // + literal: Const { ty: for<'r> fn(&'r u64) -> u64 {::clone}, val: Value(Scalar()) } ++ _10 = _3; // scope 1 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11 ++ _9 = _10; // scope 1 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11 ++ _8 = (*_9); // scope 1 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11 ++ goto -> bb2; // scope 1 at $DIR/combine_clone_of_primitives.rs:9:5: 9:11 + } + + bb2: { +- _13 = &(*_4); // scope 1 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16 +- _12 = &(*_13); // scope 1 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16 +- _11 = <[f32; 3] as Clone>::clone(move _12) -> [return: bb3, unwind: bb4]; // scope 1 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16 +- // mir::Constant +- // + span: $DIR/combine_clone_of_primitives.rs:10:5: 10:16 +- // + literal: Const { ty: for<'r> fn(&'r [f32; 3]) -> [f32; 3] {<[f32; 3] as Clone>::clone}, val: Value(Scalar()) } ++ _13 = _4; // scope 1 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16 ++ _12 = _13; // scope 1 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16 ++ _11 = (*_12); // scope 1 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16 ++ goto -> bb3; // scope 1 at $DIR/combine_clone_of_primitives.rs:10:5: 10:16 + } + + bb3: { + (_0.0: T) = move _5; // scope 1 at $DIR/combine_clone_of_primitives.rs:6:10: 6:15 + (_0.1: u64) = move _8; // scope 1 at $DIR/combine_clone_of_primitives.rs:6:10: 6:15 + (_0.2: [f32; 3]) = move _11; // scope 1 at $DIR/combine_clone_of_primitives.rs:6:10: 6:15 + return; // scope 0 at $DIR/combine_clone_of_primitives.rs:6:15: 6:15 + } + + bb4 (cleanup): { + drop(_5) -> bb5; // scope 1 at $DIR/combine_clone_of_primitives.rs:6:14: 6:15 + } + + bb5 (cleanup): { + resume; // scope 0 at $DIR/combine_clone_of_primitives.rs:6:10: 6:15 + } + } + diff --git a/src/test/ui/cmse-nonsecure/cmse-nonsecure-call/params-on-registers.rs b/src/test/ui/cmse-nonsecure/cmse-nonsecure-call/params-on-registers.rs index 9e4521df8c34f..bbc039bdf5c7b 100644 --- a/src/test/ui/cmse-nonsecure/cmse-nonsecure-call/params-on-registers.rs +++ b/src/test/ui/cmse-nonsecure/cmse-nonsecure-call/params-on-registers.rs @@ -7,6 +7,7 @@ pub trait Sized { } #[lang="copy"] pub trait Copy { } +impl Copy for u32 {} extern "rust-intrinsic" { pub fn transmute(e: T) -> U; diff --git a/src/test/ui/cmse-nonsecure/cmse-nonsecure-call/params-on-stack.rs b/src/test/ui/cmse-nonsecure/cmse-nonsecure-call/params-on-stack.rs index d5c67af2b4146..b8112b20a54c6 100644 --- a/src/test/ui/cmse-nonsecure/cmse-nonsecure-call/params-on-stack.rs +++ b/src/test/ui/cmse-nonsecure/cmse-nonsecure-call/params-on-stack.rs @@ -7,6 +7,7 @@ pub trait Sized { } #[lang="copy"] pub trait Copy { } +impl Copy for u32 {} extern "rust-intrinsic" { pub fn transmute(e: T) -> U; diff --git a/src/test/ui/cmse-nonsecure/cmse-nonsecure-entry/params-on-registers.rs b/src/test/ui/cmse-nonsecure/cmse-nonsecure-entry/params-on-registers.rs index 8cde9ba58b93b..5591a8a5864b5 100644 --- a/src/test/ui/cmse-nonsecure/cmse-nonsecure-entry/params-on-registers.rs +++ b/src/test/ui/cmse-nonsecure/cmse-nonsecure-entry/params-on-registers.rs @@ -7,6 +7,7 @@ trait Sized { } #[lang="copy"] trait Copy { } +impl Copy for u32 {} #[no_mangle] #[cmse_nonsecure_entry] diff --git a/src/test/ui/cmse-nonsecure/cmse-nonsecure-entry/params-on-stack.rs b/src/test/ui/cmse-nonsecure/cmse-nonsecure-entry/params-on-stack.rs index 9a1b0a38d5eac..39b41dac41f77 100644 --- a/src/test/ui/cmse-nonsecure/cmse-nonsecure-entry/params-on-stack.rs +++ b/src/test/ui/cmse-nonsecure/cmse-nonsecure-entry/params-on-stack.rs @@ -7,6 +7,7 @@ trait Sized { } #[lang="copy"] trait Copy { } +impl Copy for u32 {} #[no_mangle] #[cmse_nonsecure_entry]