Reveal opaque types before drop elaboration

compiler/rustc_codegen_cranelift/src/base.rs

@@ -875,7 +875,7 @@ pub(crate) fn codegen_place<'tcx>(
             PlaceElem::Deref => {
                 cplace = cplace.place_deref(fx);
             }
-            PlaceElem::OpaqueCast(ty) => cplace = cplace.place_opaque_cast(fx, ty),
+            PlaceElem::OpaqueCast(ty) => bug!("encountered OpaqueCast({ty}) in codegen"),
             PlaceElem::Field(field, _ty) => {
                 cplace = cplace.place_field(fx, field);
             }

compiler/rustc_codegen_cranelift/src/value_and_place.rs

@@ -674,14 +674,6 @@ impl<'tcx> CPlace<'tcx> {
         }
     }
 
-    pub(crate) fn place_opaque_cast(
-        self,
-        fx: &mut FunctionCx<'_, '_, 'tcx>,
-        ty: Ty<'tcx>,
-    ) -> CPlace<'tcx> {
-        CPlace { inner: self.inner, layout: fx.layout_of(ty) }
-    }
-
     pub(crate) fn place_field(
         self,
         fx: &mut FunctionCx<'_, '_, 'tcx>,

compiler/rustc_codegen_ssa/src/mir/place.rs

@@ -463,7 +463,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 mir::ProjectionElem::Field(ref field, _) => {
                     cg_base.project_field(bx, field.index())
                 }
-                mir::ProjectionElem::OpaqueCast(ty) => cg_base.project_type(bx, ty),
+                mir::ProjectionElem::OpaqueCast(ty) => {
+                    bug!("encountered OpaqueCast({ty}) in codegen")
+                }
                 mir::ProjectionElem::Index(index) => {
                     let index = &mir::Operand::Copy(mir::Place::from(index));
                     let index = self.codegen_operand(bx, index);

compiler/rustc_const_eval/src/interpret/projection.rs

@@ -316,7 +316,9 @@ where
     {
         use rustc_middle::mir::ProjectionElem::*;
         Ok(match proj_elem {
-            OpaqueCast(ty) => base.transmute(self.layout_of(ty)?, self)?,
+            OpaqueCast(ty) => {
+                span_bug!(self.cur_span(), "OpaqueCast({ty}) encountered after borrowck")
+            }
             Field(field, _) => self.project_field(base, field.index())?,
             Downcast(_, variant) => self.project_downcast(base, variant)?,
             Deref => self.deref_pointer(&base.to_op(self)?)?.into(),

compiler/rustc_const_eval/src/transform/validate.rs

@@ -633,6 +633,14 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
         location: Location,
     ) {
         match elem {
+            ProjectionElem::OpaqueCast(ty)
+                if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) =>
+            {
+                self.fail(
+                    location,
+                    format!("explicit opaque type cast to `{ty}` after `RevealAll`"),
+                )
+            }
             ProjectionElem::Index(index) => {
                 let index_ty = self.body.local_decls[index].ty;
                 if index_ty != self.tcx.types.usize {

compiler/rustc_middle/src/mir/syntax.rs

@@ -139,6 +139,7 @@ pub enum RuntimePhase {
     /// * [`TerminatorKind::Yield`]
     /// * [`TerminatorKind::GeneratorDrop`]
     /// * [`Rvalue::Aggregate`] for any `AggregateKind` except `Array`
+    /// * [`PlaceElem::OpaqueCast`]
     ///
     /// And the following variants are allowed:
     /// * [`StatementKind::Retag`]

compiler/rustc_mir_dataflow/src/elaborate_drops.rs

@@ -194,6 +194,7 @@ where
     D: DropElaborator<'b, 'tcx>,
     'tcx: 'b,
 {
+    #[instrument(level = "trace", skip(self), ret)]
     fn place_ty(&self, place: Place<'tcx>) -> Ty<'tcx> {
         place.ty(self.elaborator.body(), self.tcx()).ty
     }
@@ -220,11 +221,9 @@ where
     //
     // FIXME: I think we should just control the flags externally,
     // and then we do not need this machinery.
+    #[instrument(level = "debug")]
     pub fn elaborate_drop(&mut self, bb: BasicBlock) {
-        debug!("elaborate_drop({:?}, {:?})", bb, self);
-        let style = self.elaborator.drop_style(self.path, DropFlagMode::Deep);
-        debug!("elaborate_drop({:?}, {:?}): live - {:?}", bb, self, style);
-        match style {
+        match self.elaborator.drop_style(self.path, DropFlagMode::Deep) {
             DropStyle::Dead => {
                 self.elaborator
                     .patch()

compiler/rustc_mir_transform/src/elaborate_drops.rs

@@ -170,6 +170,7 @@ impl<'a, 'tcx> DropElaborator<'a, 'tcx> for Elaborator<'a, '_, 'tcx> {
         self.ctxt.param_env()
     }
 
+    #[instrument(level = "debug", skip(self), ret)]
     fn drop_style(&self, path: Self::Path, mode: DropFlagMode) -> DropStyle {
         let ((maybe_live, maybe_dead), multipart) = match mode {
             DropFlagMode::Shallow => (self.ctxt.init_data.maybe_live_dead(path), false),

compiler/rustc_mir_transform/src/lib.rs

@@ -480,6 +480,7 @@ fn run_runtime_lowering_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
     let passes: &[&dyn MirPass<'tcx>] = &[
         // These next passes must be executed together
         &add_call_guards::CriticalCallEdges,
+        &reveal_all::RevealAll, // has to be done before drop elaboration, since we need to drop opaque types, too.
         &elaborate_drops::ElaborateDrops,
         // This will remove extraneous landing pads which are no longer
         // necessary as well as well as forcing any call in a non-unwinding
@@ -526,7 +527,6 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         body,
         &[
             &check_alignment::CheckAlignment,
-            &reveal_all::RevealAll, // has to be done before inlining, since inlined code is in RevealAll mode.
             &lower_slice_len::LowerSliceLenCalls, // has to be done before inlining, otherwise actual call will be almost always inlined. Also simple, so can just do first
             &unreachable_prop::UnreachablePropagation,
             &uninhabited_enum_branching::UninhabitedEnumBranching,

compiler/rustc_mir_transform/src/reveal_all.rs

@@ -8,16 +8,7 @@ use rustc_middle::ty::{self, Ty, TyCtxt};
 pub struct RevealAll;
 
 impl<'tcx> MirPass<'tcx> for RevealAll {
-    fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
-        sess.mir_opt_level() >= 3 || super::inline::Inline.is_enabled(sess)
-    }
-
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        // Do not apply this transformation to generators.
-        if body.generator.is_some() {
-            return;
-        }
-
         let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id());
         RevealAllVisitor { tcx, param_env }.visit_body_preserves_cfg(body);
     }
@@ -34,6 +25,29 @@ impl<'tcx> MutVisitor<'tcx> for RevealAllVisitor<'tcx> {
         self.tcx
     }
 
+    #[inline]
+    fn visit_place(
+        &mut self,
+        place: &mut Place<'tcx>,
+        _context: PlaceContext,
+        _location: Location,
+    ) {
+        // Performance optimization: don't reintern if there is no `OpaqueCast` to remove.
+        if place.projection.iter().all(|elem| !matches!(elem, ProjectionElem::OpaqueCast(_))) {
+            return;
+        }
+        // `OpaqueCast` projections are only needed if there are opaque types on which projections are performed.
+        // After the `RevealAll` pass, all opaque types are replaced with their hidden types, so we don't need these
+        // projections anymore.
+        place.projection = self.tcx.mk_place_elems(
+            &place
+                .projection
+                .into_iter()
+                .filter(|elem| !matches!(elem, ProjectionElem::OpaqueCast(_)))
+                .collect::<Vec<_>>(),
+        );
+    }
+
     #[inline]
     fn visit_constant(&mut self, constant: &mut ConstOperand<'tcx>, _: Location) {
         // We have to use `try_normalize_erasing_regions` here, since it's

tests/incremental/hashes/function_interfaces.rs

@@ -302,7 +302,7 @@ pub fn return_impl_trait() -> i32        {
 }
 
 #[cfg(not(any(cfail1,cfail4)))]
-#[rustc_clean(cfg = "cfail2", except = "hir_owner, hir_owner_nodes, typeck, fn_sig, optimized_mir")]
+#[rustc_clean(cfg = "cfail2", except = "hir_owner, hir_owner_nodes, typeck, fn_sig")]
 #[rustc_clean(cfg = "cfail3")]
 #[rustc_clean(cfg = "cfail5", except = "hir_owner, hir_owner_nodes, typeck, fn_sig, optimized_mir")]
 #[rustc_clean(cfg = "cfail6")]

tests/mir-opt/building/async_await.b-{closure#0}.generator_resume.0.mir

@@ -2,13 +2,16 @@
 /* generator_layout = GeneratorLayout {
     field_tys: {
         _0: GeneratorSavedTy {
-            ty: Alias(
-                Opaque,
-                AliasTy {
-                    args: [
-                    ],
-                    def_id: DefId(0:7 ~ async_await[ccf8]::a::{opaque#0}),
-                },
+            ty: Generator(
+                DefId(0:4 ~ async_await[ccf8]::a::{closure#0}),
+                [
+                std::future::ResumeTy,
+                (),
+                (),
+                GeneratorWitness(DefId(0:4 ~ async_await[ccf8]::a::{closure#0}), []),
+                (),
+                ],
+                Static,
             ),
             source_info: SourceInfo {
                 span: $DIR/async_await.rs:15:9: 15:14 (#8),
@@ -17,13 +20,16 @@
             ignore_for_traits: false,
         },
         _1: GeneratorSavedTy {
-            ty: Alias(
-                Opaque,
-                AliasTy {
-                    args: [
-                    ],
-                    def_id: DefId(0:7 ~ async_await[ccf8]::a::{opaque#0}),
-                },
+            ty: Generator(
+                DefId(0:4 ~ async_await[ccf8]::a::{closure#0}),
+                [
+                std::future::ResumeTy,
+                (),
+                (),
+                GeneratorWitness(DefId(0:4 ~ async_await[ccf8]::a::{closure#0}), []),
+                (),
+                ],
+                Static,
             ),
             source_info: SourceInfo {
                 span: $DIR/async_await.rs:16:9: 16:14 (#10),
@@ -49,30 +55,30 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body@$DIR/async_await.rs:14:18: 17:2}>,
     debug _task_context => _38;
     let mut _0: std::task::Poll<()>;
     let _3: ();
-    let mut _4: impl std::future::Future<Output = ()>;
-    let mut _5: impl std::future::Future<Output = ()>;
-    let mut _6: impl std::future::Future<Output = ()>;
+    let mut _4: {async fn body@$DIR/async_await.rs:11:14: 11:16};
+    let mut _5: {async fn body@$DIR/async_await.rs:11:14: 11:16};
+    let mut _6: {async fn body@$DIR/async_await.rs:11:14: 11:16};
     let mut _7: ();
     let _8: ();
     let mut _9: std::task::Poll<()>;
-    let mut _10: std::pin::Pin<&mut impl std::future::Future<Output = ()>>;
-    let mut _11: &mut impl std::future::Future<Output = ()>;
-    let mut _12: &mut impl std::future::Future<Output = ()>;
+    let mut _10: std::pin::Pin<&mut {async fn body@$DIR/async_await.rs:11:14: 11:16}>;
+    let mut _11: &mut {async fn body@$DIR/async_await.rs:11:14: 11:16};
+    let mut _12: &mut {async fn body@$DIR/async_await.rs:11:14: 11:16};
     let mut _13: &mut std::task::Context<'_>;
     let mut _14: &mut std::task::Context<'_>;
     let mut _15: &mut std::task::Context<'_>;
     let mut _16: isize;
     let mut _18: !;
     let mut _19: &mut std::task::Context<'_>;
     let mut _20: ();
-    let mut _21: impl std::future::Future<Output = ()>;
-    let mut _22: impl std::future::Future<Output = ()>;
-    let mut _23: impl std::future::Future<Output = ()>;
+    let mut _21: {async fn body@$DIR/async_await.rs:11:14: 11:16};
+    let mut _22: {async fn body@$DIR/async_await.rs:11:14: 11:16};
+    let mut _23: {async fn body@$DIR/async_await.rs:11:14: 11:16};
     let _24: ();
     let mut _25: std::task::Poll<()>;
-    let mut _26: std::pin::Pin<&mut impl std::future::Future<Output = ()>>;
-    let mut _27: &mut impl std::future::Future<Output = ()>;
-    let mut _28: &mut impl std::future::Future<Output = ()>;
+    let mut _26: std::pin::Pin<&mut {async fn body@$DIR/async_await.rs:11:14: 11:16}>;
+    let mut _27: &mut {async fn body@$DIR/async_await.rs:11:14: 11:16};
+    let mut _28: &mut {async fn body@$DIR/async_await.rs:11:14: 11:16};
     let mut _29: &mut std::task::Context<'_>;
     let mut _30: &mut std::task::Context<'_>;
     let mut _31: &mut std::task::Context<'_>;
@@ -84,7 +90,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body@$DIR/async_await.rs:14:18: 17:2}>,
     let mut _38: &mut std::task::Context<'_>;
     let mut _39: u32;
     scope 1 {
-        debug __awaitee => (((*(_1.0: &mut {async fn body@$DIR/async_await.rs:14:18: 17:2})) as variant#3).0: impl std::future::Future<Output = ()>);
+        debug __awaitee => (((*(_1.0: &mut {async fn body@$DIR/async_await.rs:14:18: 17:2})) as variant#3).0: {async fn body@$DIR/async_await.rs:11:14: 11:16});
         let _17: ();
         scope 2 {
         }
@@ -93,7 +99,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body@$DIR/async_await.rs:14:18: 17:2}>,
         }
     }
     scope 4 {
-        debug __awaitee => (((*(_1.0: &mut {async fn body@$DIR/async_await.rs:14:18: 17:2})) as variant#4).0: impl std::future::Future<Output = ()>);
+        debug __awaitee => (((*(_1.0: &mut {async fn body@$DIR/async_await.rs:14:18: 17:2})) as variant#4).0: {async fn body@$DIR/async_await.rs:11:14: 11:16});
         let _33: ();
         scope 5 {
         }
@@ -116,13 +122,13 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body@$DIR/async_await.rs:14:18: 17:2}>,
     }
 
     bb2: {
-        _4 = <impl Future<Output = ()> as IntoFuture>::into_future(move _5) -> [return: bb3, unwind unreachable];
+        _4 = <{async fn body@$DIR/async_await.rs:11:14: 11:16} as IntoFuture>::into_future(move _5) -> [return: bb3, unwind unreachable];
     }
 
     bb3: {
         StorageDead(_5);
         nop;
-        (((*(_1.0: &mut {async fn body@$DIR/async_await.rs:14:18: 17:2})) as variant#3).0: impl std::future::Future<Output = ()>) = move _4;
+        (((*(_1.0: &mut {async fn body@$DIR/async_await.rs:14:18: 17:2})) as variant#3).0: {async fn body@$DIR/async_await.rs:11:14: 11:16}) = move _4;
         goto -> bb4;
     }
 
@@ -132,9 +138,9 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body@$DIR/async_await.rs:14:18: 17:2}>,
         StorageLive(_10);
         StorageLive(_11);
         StorageLive(_12);
-        _12 = &mut (((*(_1.0: &mut {async fn body@$DIR/async_await.rs:14:18: 17:2})) as variant#3).0: impl std::future::Future<Output = ()>);
+        _12 = &mut (((*(_1.0: &mut {async fn body@$DIR/async_await.rs:14:18: 17:2})) as variant#3).0: {async fn body@$DIR/async_await.rs:11:14: 11:16});
         _11 = &mut (*_12);
-        _10 = Pin::<&mut impl Future<Output = ()>>::new_unchecked(move _11) -> [return: bb5, unwind unreachable];
+        _10 = Pin::<&mut {async fn body@$DIR/async_await.rs:11:14: 11:16}>::new_unchecked(move _11) -> [return: bb5, unwind unreachable];
     }
 
     bb5: {
@@ -150,7 +156,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body@$DIR/async_await.rs:14:18: 17:2}>,
     bb6: {
         _13 = &mut (*_14);
         StorageDead(_15);
-        _9 = <impl Future<Output = ()> as Future>::poll(move _10, move _13) -> [return: bb7, unwind unreachable];
+        _9 = <{async fn body@$DIR/async_await.rs:11:14: 11:16} as Future>::poll(move _10, move _13) -> [return: bb7, unwind unreachable];
     }
 
     bb7: {
@@ -187,7 +193,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body@$DIR/async_await.rs:14:18: 17:2}>,
         StorageDead(_12);
         StorageDead(_9);
         StorageDead(_8);
-        drop((((*(_1.0: &mut {async fn body@$DIR/async_await.rs:14:18: 17:2})) as variant#3).0: impl std::future::Future<Output = ()>)) -> [return: bb12, unwind unreachable];
+        drop((((*(_1.0: &mut {async fn body@$DIR/async_await.rs:14:18: 17:2})) as variant#3).0: {async fn body@$DIR/async_await.rs:11:14: 11:16})) -> [return: bb12, unwind unreachable];
     }
 
     bb11: {
@@ -212,13 +218,13 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body@$DIR/async_await.rs:14:18: 17:2}>,
     }
 
     bb14: {
-        _21 = <impl Future<Output = ()> as IntoFuture>::into_future(move _22) -> [return: bb15, unwind unreachable];
+        _21 = <{async fn body@$DIR/async_await.rs:11:14: 11:16} as IntoFuture>::into_future(move _22) -> [return: bb15, unwind unreachable];
     }
 
     bb15: {
         StorageDead(_22);
         nop;
-        (((*(_1.0: &mut {async fn body@$DIR/async_await.rs:14:18: 17:2})) as variant#4).0: impl std::future::Future<Output = ()>) = move _21;
+        (((*(_1.0: &mut {async fn body@$DIR/async_await.rs:14:18: 17:2})) as variant#4).0: {async fn body@$DIR/async_await.rs:11:14: 11:16}) = move _21;
         goto -> bb16;
     }
 
@@ -228,9 +234,9 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body@$DIR/async_await.rs:14:18: 17:2}>,
         StorageLive(_26);
         StorageLive(_27);
         StorageLive(_28);
-        _28 = &mut (((*(_1.0: &mut {async fn body@$DIR/async_await.rs:14:18: 17:2})) as variant#4).0: impl std::future::Future<Output = ()>);
+        _28 = &mut (((*(_1.0: &mut {async fn body@$DIR/async_await.rs:14:18: 17:2})) as variant#4).0: {async fn body@$DIR/async_await.rs:11:14: 11:16});
         _27 = &mut (*_28);
-        _26 = Pin::<&mut impl Future<Output = ()>>::new_unchecked(move _27) -> [return: bb17, unwind unreachable];
+        _26 = Pin::<&mut {async fn body@$DIR/async_await.rs:11:14: 11:16}>::new_unchecked(move _27) -> [return: bb17, unwind unreachable];
     }
 
     bb17: {
@@ -246,7 +252,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body@$DIR/async_await.rs:14:18: 17:2}>,
     bb18: {
         _29 = &mut (*_30);
         StorageDead(_31);
-        _25 = <impl Future<Output = ()> as Future>::poll(move _26, move _29) -> [return: bb19, unwind unreachable];
+        _25 = <{async fn body@$DIR/async_await.rs:11:14: 11:16} as Future>::poll(move _26, move _29) -> [return: bb19, unwind unreachable];
     }
 
     bb19: {
@@ -279,7 +285,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body@$DIR/async_await.rs:14:18: 17:2}>,
         StorageDead(_28);
         StorageDead(_25);
         StorageDead(_24);
-        drop((((*(_1.0: &mut {async fn body@$DIR/async_await.rs:14:18: 17:2})) as variant#4).0: impl std::future::Future<Output = ()>)) -> [return: bb23, unwind unreachable];
+        drop((((*(_1.0: &mut {async fn body@$DIR/async_await.rs:14:18: 17:2})) as variant#4).0: {async fn body@$DIR/async_await.rs:11:14: 11:16})) -> [return: bb23, unwind unreachable];
     }
 
     bb22: {