Allow reifying intrinsics to fn pointers.

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -35,6 +35,7 @@ use rustc_middle::ty::{
 use rustc_span::def_id::CRATE_DEF_ID;
 use rustc_span::{Span, DUMMY_SP};
 use rustc_target::abi::VariantIdx;
+use rustc_target::spec::abi::Abi;
 use rustc_trait_selection::infer::InferCtxtExt as _;
 use rustc_trait_selection::opaque_types::{GenerateMemberConstraints, InferCtxtExt};
 use rustc_trait_selection::traits::error_reporting::InferCtxtExt as _;
@@ -323,6 +324,7 @@ fn translate_outlives_facts(typeck: &mut TypeChecker<'_, '_>) {
     }
 }
 
+#[track_caller]
 fn mirbug(tcx: TyCtxt<'_>, span: Span, msg: &str) {
     // We sometimes see MIR failures (notably predicate failures) due to
     // the fact that we check rvalue sized predicates here. So use `delay_span_bug`
@@ -2054,6 +2056,16 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                         // and hence may contain unnormalized results.
                         let fn_sig = self.normalize(fn_sig, location);
 
+                        // NOTE(eddyb) see comment on `prepare_fn_sig_for_reify`
+                        // in `rustc_typeck::check::coercion`.
+                        let fn_sig = fn_sig.map_bound(|mut sig| {
+                            if matches!(sig.abi, Abi::RustIntrinsic | Abi::PlatformIntrinsic) {
+                                sig.abi = Abi::Rust;
+                            }
+
+                            sig
+                        });
+
                         let ty_fn_ptr_from = tcx.mk_fn_ptr(fn_sig);
 
                         if let Err(terr) = self.eq_types(

compiler/rustc_infer/src/infer/error_reporting/mod.rs

@@ -2628,9 +2628,6 @@ impl<'tcx> ObligationCauseExt<'tcx> for ObligationCause<'tcx> {
                 TypeError::CyclicTy(ty) if ty.is_closure() || ty.is_generator() => {
                     Error0644("closure/generator type that references itself")
                 }
-                TypeError::IntrinsicCast => {
-                    Error0308("cannot coerce intrinsics to function pointers")
-                }
                 TypeError::ObjectUnsafeCoercion(did) => Error0038(*did),
                 _ => Error0308("mismatched types"),
             },

compiler/rustc_middle/src/ty/error.rs

@@ -66,7 +66,6 @@ pub enum TypeError<'tcx> {
     ObjectUnsafeCoercion(DefId),
     ConstMismatch(ExpectedFound<&'tcx ty::Const<'tcx>>),
 
-    IntrinsicCast,
     /// Safe `#[target_feature]` functions are not assignable to safe function pointers.
     TargetFeatureCast(DefId),
 }
@@ -197,7 +196,6 @@ impl<'tcx> fmt::Display for TypeError<'tcx> {
             ConstMismatch(ref values) => {
                 write!(f, "expected `{}`, found `{}`", values.expected, values.found)
             }
-            IntrinsicCast => write!(f, "cannot coerce intrinsics to function pointers"),
             TargetFeatureCast(_) => write!(
                 f,
                 "cannot coerce functions with `#[target_feature]` to safe function pointers"
@@ -229,7 +227,6 @@ impl<'tcx> TypeError<'tcx> {
             | ProjectionMismatched(_)
             | ExistentialMismatch(_)
             | ConstMismatch(_)
-            | IntrinsicCast
             | ObjectUnsafeCoercion(_) => true,
         }
     }

compiler/rustc_middle/src/ty/instance.rs

@@ -394,6 +394,10 @@ impl<'tcx> Instance<'tcx> {
                     debug!(" => fn pointer created for virtual call");
                     resolved.def = InstanceDef::ReifyShim(def_id);
                 }
+                InstanceDef::Intrinsic(def_id) => {
+                    debug!(" => fn pointer created for intrinsic call");
+                    resolved.def = InstanceDef::ReifyShim(def_id);
+                }
                 _ => {}
             }
 

compiler/rustc_middle/src/ty/structural_impls.rs

@@ -619,7 +619,6 @@ impl<'a, 'tcx> Lift<'tcx> for ty::error::TypeError<'a> {
             Sorts(x) => return tcx.lift(x).map(Sorts),
             ExistentialMismatch(x) => return tcx.lift(x).map(ExistentialMismatch),
             ConstMismatch(x) => return tcx.lift(x).map(ConstMismatch),
-            IntrinsicCast => IntrinsicCast,
             TargetFeatureCast(x) => TargetFeatureCast(x),
             ObjectUnsafeCoercion(x) => return tcx.lift(x).map(ObjectUnsafeCoercion),
         })

compiler/rustc_mir_transform/src/shim.rs

@@ -17,8 +17,8 @@ use std::iter;
 
 use crate::util::expand_aggregate;
 use crate::{
-    abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, remove_noop_landing_pads,
-    run_passes, simplify,
+    abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, lower_intrinsics,
+    remove_noop_landing_pads, run_passes, simplify,
 };
 use rustc_middle::mir::patch::MirPatch;
 use rustc_mir_dataflow::elaborate_drops::{self, DropElaborator, DropFlagMode, DropStyle};
@@ -82,6 +82,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<'
         &[&[
             &add_moves_for_packed_drops::AddMovesForPackedDrops,
             &remove_noop_landing_pads::RemoveNoopLandingPads,
+            &lower_intrinsics::LowerIntrinsics,
             &simplify::SimplifyCfg::new("make_shim"),
             &add_call_guards::CriticalCallEdges,
             &abort_unwinding_calls::AbortUnwindingCalls,

compiler/rustc_typeck/src/check/cast.rs

@@ -38,7 +38,6 @@ use rustc_hir::lang_items::LangItem;
 use rustc_middle::mir::Mutability;
 use rustc_middle::ty::adjustment::AllowTwoPhase;
 use rustc_middle::ty::cast::{CastKind, CastTy};
-use rustc_middle::ty::error::TypeError;
 use rustc_middle::ty::subst::SubstsRef;
 use rustc_middle::ty::{self, Ty, TypeAndMut, TypeFoldable};
 use rustc_session::lint;
@@ -700,9 +699,6 @@ impl<'a, 'tcx> CastCheck<'tcx> {
                             AllowTwoPhase::No,
                             None,
                         );
-                        if let Err(TypeError::IntrinsicCast) = res {
-                            return Err(CastError::IllegalCast);
-                        }
                         if res.is_err() {
                             return Err(CastError::NonScalar);
                         }

compiler/rustc_typeck/src/check/coercion.rs

@@ -84,6 +84,23 @@ impl<'a, 'tcx> Deref for Coerce<'a, 'tcx> {
 
 type CoerceResult<'tcx> = InferResult<'tcx, (Vec<Adjustment<'tcx>>, Ty<'tcx>)>;
 
+/// Make any adjustments necessary for a function signature to be compatible
+/// with reification to a `fn` pointer. In particular, intrinsics are imported
+/// using pseudo-ABIs (`extern "rust-intrinsic" {...}`) currently, but that's
+/// an implementation detail and any `fn` pointers that may be taken to them
+/// should be indistinguishable from those to regular Rust functions, in order
+/// to allow e.g. libcore public APIs to be replaced with intrinsics, without
+/// breaking code that was, explicitly or implicitly, creating `fn` pointers.
+// FIXME(eddyb) intrinsics shouldn't use pseudo-ABIs, but rather the  Rust ABI
+// and some other way to indicate that they are intrinsics (e.g. new attributes).
+fn prepare_fn_sig_for_reify(mut sig: ty::FnSig<'tcx>) -> ty::FnSig<'tcx> {
+    if matches!(sig.abi, Abi::RustIntrinsic | Abi::PlatformIntrinsic) {
+        sig.abi = Abi::Rust;
+    }
+
+    sig
+}
+
 /// Coercing a mutable reference to an immutable works, while
 /// coercing `&T` to `&mut T` should be forbidden.
 fn coerce_mutbls<'tcx>(
@@ -779,10 +796,9 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
         match b.kind() {
             ty::FnPtr(b_sig) => {
                 let a_sig = a.fn_sig(self.tcx);
-                // Intrinsics are not coercible to function pointers
-                if a_sig.abi() == Abi::RustIntrinsic || a_sig.abi() == Abi::PlatformIntrinsic {
-                    return Err(TypeError::IntrinsicCast);
-                }
+
+                // NOTE(eddyb) see comment on `prepare_fn_sig_for_reify`.
+                let a_sig = a_sig.map_bound(prepare_fn_sig_for_reify);
 
                 // Safe `#[target_feature]` functions are not assignable to safe fn pointers (RFC 2396).
                 if let ty::FnDef(def_id, _) = *a.kind() {
@@ -1043,14 +1059,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
         };
         if let (Some(a_sig), Some(b_sig)) = (a_sig, b_sig) {
-            // Intrinsics are not coercible to function pointers.
-            if a_sig.abi() == Abi::RustIntrinsic
-                || a_sig.abi() == Abi::PlatformIntrinsic
-                || b_sig.abi() == Abi::RustIntrinsic
-                || b_sig.abi() == Abi::PlatformIntrinsic
-            {
-                return Err(TypeError::IntrinsicCast);
-            }
+            // NOTE(eddyb) see comment on `prepare_fn_sig_for_reify`.
+            let a_sig = a_sig.map_bound(prepare_fn_sig_for_reify);
+            let b_sig = b_sig.map_bound(prepare_fn_sig_for_reify);
+
             // The signature must match.
             let a_sig = self.normalize_associated_types_in(new.span, a_sig);
             let b_sig = self.normalize_associated_types_in(new.span, b_sig);

src/test/mir-opt/lower_intrinsics.core.intrinsics-#1-discriminant_value.LowerIntrinsics.diff

@@ -0,0 +1,24 @@
+- // MIR for `discriminant_value` before LowerIntrinsics
++ // MIR for `discriminant_value` after LowerIntrinsics
+
+  fn discriminant_value(_1: &T) -> <T as DiscriminantKind>::Discriminant {
+      let mut _0: <T as std::marker::DiscriminantKind>::Discriminant; // return place in scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+
+      bb0: {
+-         _0 = discriminant_value::<T>(move _1) -> bb1; // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+-                                          // mir::Constant
+-                                          // + span: $SRC_DIR/core/src/intrinsics.rs:LL:COL
+-                                          // + literal: Const { ty: for<'r> extern "rust-intrinsic" fn(&'r T) -> <T as std::marker::DiscriminantKind>::Discriminant {std::intrinsics::discriminant_value::<T>}, val: Value(Scalar(<ZST>)) }
++         _0 = discriminant((*_1));        // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
++         goto -> bb1;                     // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+      }
+
+      bb1: {
+          return;                          // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+      }
+
+      bb2 (cleanup): {
+          resume;                          // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+      }
+  }
+

src/test/mir-opt/lower_intrinsics.core.intrinsics-#1-forget.LowerIntrinsics.diff

@@ -0,0 +1,24 @@
+- // MIR for `std::intrinsics::forget` before LowerIntrinsics
++ // MIR for `std::intrinsics::forget` after LowerIntrinsics
+
+  fn std::intrinsics::forget(_1: T) -> () {
+      let mut _0: ();                      // return place in scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+
+      bb0: {
+-         _0 = std::intrinsics::forget::<T>(move _1) -> bb1; // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+-                                          // mir::Constant
+-                                          // + span: $SRC_DIR/core/src/intrinsics.rs:LL:COL
+-                                          // + literal: Const { ty: extern "rust-intrinsic" fn(T) {std::intrinsics::forget::<T>}, val: Value(Scalar(<ZST>)) }
++         _0 = const ();                   // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
++         goto -> bb1;                     // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+      }
+
+      bb1: {
+          return;                          // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+      }
+
+      bb2 (cleanup): {
+          resume;                          // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+      }
+  }
+

src/test/mir-opt/lower_intrinsics.core.intrinsics-#1-size_of.LowerIntrinsics.diff

@@ -0,0 +1,24 @@
+- // MIR for `std::intrinsics::size_of` before LowerIntrinsics
++ // MIR for `std::intrinsics::size_of` after LowerIntrinsics
+
+  fn std::intrinsics::size_of() -> usize {
+      let mut _0: usize;                   // return place in scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+
+      bb0: {
+-         _0 = std::intrinsics::size_of::<T>() -> bb1; // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+-                                          // mir::Constant
+-                                          // + span: $SRC_DIR/core/src/intrinsics.rs:LL:COL
+-                                          // + literal: Const { ty: extern "rust-intrinsic" fn() -> usize {std::intrinsics::size_of::<T>}, val: Value(Scalar(<ZST>)) }
++         _0 = SizeOf(T);                  // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
++         goto -> bb1;                     // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+      }
+
+      bb1: {
+          return;                          // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+      }
+
+      bb2 (cleanup): {
+          resume;                          // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+      }
+  }
+

src/test/mir-opt/lower_intrinsics.core.intrinsics-#1-unreachable.LowerIntrinsics.diff

@@ -0,0 +1,23 @@
+- // MIR for `std::intrinsics::unreachable` before LowerIntrinsics
++ // MIR for `std::intrinsics::unreachable` after LowerIntrinsics
+
+  fn std::intrinsics::unreachable() -> ! {
+      let mut _0: !;                       // return place in scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+
+      bb0: {
+-         _0 = std::intrinsics::unreachable() -> bb1; // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+-                                          // mir::Constant
+-                                          // + span: $SRC_DIR/core/src/intrinsics.rs:LL:COL
+-                                          // + literal: Const { ty: unsafe extern "rust-intrinsic" fn() -> ! {std::intrinsics::unreachable}, val: Value(Scalar(<ZST>)) }
++         unreachable;                     // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+      }
+
+      bb1: {
+          return;                          // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+      }
+
+      bb2 (cleanup): {
+          resume;                          // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+      }
+  }
+

src/test/mir-opt/lower_intrinsics.core.intrinsics-#1-wrapping_add.LowerIntrinsics.diff

@@ -0,0 +1,24 @@
+- // MIR for `wrapping_add` before LowerIntrinsics
++ // MIR for `wrapping_add` after LowerIntrinsics
+
+  fn wrapping_add(_1: T, _2: T) -> T {
+      let mut _0: T;                       // return place in scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+
+      bb0: {
+-         _0 = wrapping_add::<T>(move _1, move _2) -> bb1; // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+-                                          // mir::Constant
+-                                          // + span: $SRC_DIR/core/src/intrinsics.rs:LL:COL
+-                                          // + literal: Const { ty: extern "rust-intrinsic" fn(T, T) -> T {std::intrinsics::wrapping_add::<T>}, val: Value(Scalar(<ZST>)) }
++         _0 = Add(move _1, move _2);      // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
++         goto -> bb1;                     // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+      }
+
+      bb1: {
+          return;                          // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+      }
+
+      bb2 (cleanup): {
+          resume;                          // scope 0 at $SRC_DIR/core/src/intrinsics.rs:LL:COL
+      }
+  }
+

src/test/mir-opt/lower_intrinsics.rs

@@ -76,3 +76,20 @@ pub fn discriminant<T>(t: T) {
     core::intrinsics::discriminant_value(&());
     core::intrinsics::discriminant_value(&E::B);
 }
+
+// Check that the MIR shims used for reifying intrinsics to `fn` pointers,
+// also go through the lowering pass.
+pub fn reify_intrinsics() -> impl Copy {
+    (
+// EMIT_MIR core.intrinsics-#1-wrapping_add.LowerIntrinsics.diff
+        core::intrinsics::wrapping_add::<u32> as unsafe fn(_, _) -> _,
+// EMIT_MIR core.intrinsics-#1-size_of.LowerIntrinsics.diff
+        core::intrinsics::size_of::<u8> as unsafe fn() -> _,
+// EMIT_MIR core.intrinsics-#1-unreachable.LowerIntrinsics.diff
+        core::intrinsics::unreachable as unsafe fn() -> !,
+// EMIT_MIR core.intrinsics-#1-forget.LowerIntrinsics.diff
+        core::intrinsics::forget::<E> as unsafe fn(_),
+// EMIT_MIR core.intrinsics-#1-discriminant_value.LowerIntrinsics.diff
+        core::intrinsics::discriminant_value::<E> as unsafe fn(_) -> _,
+    )
+}

src/test/ui/reify-intrinsic.rs

@@ -1,23 +1,38 @@
-// check-fail
+// run-pass
 
 #![feature(core_intrinsics, intrinsics)]
 
-fn a() {
-    let _: unsafe extern "rust-intrinsic" fn(isize) -> usize = std::mem::transmute;
-    //~^ ERROR cannot coerce
+// NOTE(eddyb) `#[inline(never)]` and returning `fn` pointers from functions is
+// done to force codegen (of the reification-to-`fn`-ptr shims around intrinsics).
+
+#[inline(never)]
+fn a() -> unsafe fn(isize) -> usize {
+    let f: unsafe fn(isize) -> usize = std::mem::transmute;
+    f
 }
 
-fn b() {
-    let _ = std::mem::transmute as unsafe extern "rust-intrinsic" fn(isize) -> usize;
-    //~^ ERROR casting
+#[inline(never)]
+fn b() -> unsafe fn(isize) -> usize {
+    let f = std::mem::transmute as unsafe fn(isize) -> usize;
+    f
 }
 
-fn c() {
-    let _ = [
+#[inline(never)]
+fn c() -> [fn(bool) -> bool; 2] {
+    let fs = [
         std::intrinsics::likely,
         std::intrinsics::unlikely,
-        //~^ ERROR cannot coerce
     ];
+    fs
 }
 
-fn main() {}
+fn main() {
+    unsafe {
+        assert_eq!(a()(-1), !0);
+        assert_eq!(b()(-1), !0);
+    }
+
+    let [likely_ptr, unlikely_ptr] = c();
+    assert!(likely_ptr(true));
+    assert!(unlikely_ptr(true));
+}