diff --git a/tests/fail/unaligned_pointers/reference_to_packed.rs b/tests/fail/unaligned_pointers/reference_to_packed.rs
index a807200771..816b6ab9fb 100644
--- a/tests/fail/unaligned_pointers/reference_to_packed.rs
+++ b/tests/fail/unaligned_pointers/reference_to_packed.rs
@@ -1,7 +1,9 @@
 // This should fail even without validation/SB
 //@compile-flags: -Zmiri-disable-validation -Zmiri-disable-stacked-borrows
 
-#![allow(dead_code, unused_variables, unaligned_references)]
+#![allow(dead_code, unused_variables)]
+
+use std::{ptr, mem};
 
 #[repr(packed)]
 struct Foo {
@@ -9,11 +11,16 @@ struct Foo {
     y: i32,
 }
 
+unsafe fn raw_to_ref<'a, T>(x: *const T) -> &'a T {
+    mem::transmute(x)
+}
+
 fn main() {
     // Try many times as this might work by chance.
     for _ in 0..20 {
         let foo = Foo { x: 42, y: 99 };
-        let p = &foo.x;
-        let i = *p; //~ERROR: alignment 4 is required
+        // There seem to be implicit reborrows, which make the error already appear here
+        let p: &i32 = unsafe { raw_to_ref(ptr::addr_of!(foo.x)) }; //~ERROR: alignment 4 is required
+        let i = *p;
     }
 }
diff --git a/tests/fail/unaligned_pointers/reference_to_packed.stderr b/tests/fail/unaligned_pointers/reference_to_packed.stderr
index 6c2a3dca2d..7c246706db 100644
--- a/tests/fail/unaligned_pointers/reference_to_packed.stderr
+++ b/tests/fail/unaligned_pointers/reference_to_packed.stderr
@@ -1,8 +1,8 @@
 error: Undefined Behavior: accessing memory with alignment ALIGN, but alignment ALIGN is required
   --> $DIR/reference_to_packed.rs:LL:CC
    |
-LL |         let i = *p;
-   |                 ^^ accessing memory with alignment ALIGN, but alignment ALIGN is required
+LL |         let p: &i32 = unsafe { raw_to_ref(ptr::addr_of!(foo.x)) };
+   |                                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ accessing memory with alignment ALIGN, but alignment ALIGN is required
    |
    = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
    = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information