diff --git a/compiler/rustc_const_eval/src/interpret/cast.rs b/compiler/rustc_const_eval/src/interpret/cast.rs
index fb484fba9fd06..076415b2d1b2f 100644
--- a/compiler/rustc_const_eval/src/interpret/cast.rs
+++ b/compiler/rustc_const_eval/src/interpret/cast.rs
@@ -366,22 +366,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             }
             (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => {
                 assert_eq!(def_a, def_b);
-                if def_a.is_box() || def_b.is_box() {
-                    if !def_a.is_box() || !def_b.is_box() {
-                        span_bug!(
-                            self.cur_span(),
-                            "invalid unsizing between {:?} -> {:?}",
-                            src.layout.ty,
-                            cast_ty.ty
-                        );
-                    }
-                    return self.unsize_into_ptr(
-                        src,
-                        dest,
-                        src.layout.ty.boxed_ty(),
-                        cast_ty.ty.boxed_ty(),
-                    );
-                }
 
                 // unsizing of generic struct with pointer fields
                 // Example: `Arc<T>` -> `Arc<Trait>`
diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs
index 630281bb09254..905ab6cb578fc 100644
--- a/compiler/rustc_const_eval/src/interpret/validity.rs
+++ b/compiler/rustc_const_eval/src/interpret/validity.rs
@@ -594,7 +594,13 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
                 Ok(true)
             }
             ty::Adt(def, ..) if def.is_box() => {
-                self.check_safe_pointer(value, "box")?;
+                let unique = self.ecx.operand_field(value, 0)?;
+                let nonnull = self.ecx.operand_field(&unique, 0)?;
+                let ptr = self.ecx.operand_field(&nonnull, 0)?;
+                self.check_safe_pointer(&ptr, "box")?;
+
+                // Check other fields of Box
+                self.walk_value(value)?;
                 Ok(true)
             }
             ty::FnPtr(_sig) => {