Rollup merge of rust-lang#97423 - m-ou-se:memory-ordering-intrinsics,…

… r=tmiasko

Simplify memory ordering intrinsics

This changes the names of the atomic intrinsics to always fully include their memory ordering arguments.

```diff
- atomic_cxchg
+ atomic_cxchg_seqcst_seqcst

- atomic_cxchg_acqrel
+ atomic_cxchg_acqrel_release

- atomic_cxchg_acqrel_failrelaxed
+ atomic_cxchg_acqrel_relaxed

// And so on.
```

- `seqcst` is no longer implied
- The failure ordering on chxchg is no longer implied in some cases, but now always explicitly part of the name.
- `release` is no longer shortened to just `rel`. That was especially confusing, since `relaxed` also starts with `rel`.
- `acquire` is no longer shortened to just `acq`, such that the names now all match the `std::sync::atomic::Ordering` variants exactly.
- This now allows for more combinations on the compare exchange operations, such as `atomic_cxchg_acquire_release`, which is necessary for rust-lang#68464.
- This PR only exposes the new possibilities through unstable intrinsics, but not yet through the stable API. That's for [a separate PR](rust-lang#98383) that requires an FCP.

Suffixes for operations with a single memory order:

| Order   | Before       | After      |
|---------|--------------|------------|
| Relaxed | `_relaxed`   | `_relaxed` |
| Acquire | `_acq`       | `_acquire` |
| Release | `_rel`       | `_release` |
| AcqRel  | `_acqrel`    | `_acqrel`  |
| SeqCst  | (none)       | `_seqcst`  |

Suffixes for compare-and-exchange operations with two memory orderings:

| Success | Failure | Before                   | After              |
|---------|---------|--------------------------|--------------------|
| Relaxed | Relaxed | `_relaxed`               | `_relaxed_relaxed` |
| Relaxed | Acquire | ❌                      | `_relaxed_acquire` |
| Relaxed | SeqCst  | ❌                      | `_relaxed_seqcst`  |
| Acquire | Relaxed | `_acq_failrelaxed`       | `_acquire_relaxed` |
| Acquire | Acquire | `_acq`                   | `_acquire_acquire` |
| Acquire | SeqCst  | ❌                      | `_acquire_seqcst`  |
| Release | Relaxed | `_rel`                   | `_release_relaxed` |
| Release | Acquire | ❌                      | `_release_acquire` |
| Release | SeqCst  | ❌                      | `_release_seqcst`  |
| AcqRel  | Relaxed | `_acqrel_failrelaxed`    | `_acqrel_relaxed`  |
| AcqRel  | Acquire | `_acqrel`                | `_acqrel_acquire`  |
| AcqRel  | SeqCst  | ❌                      | `_acqrel_seqcst`   |
| SeqCst  | Relaxed | `_failrelaxed`           | `_seqcst_relaxed`  |
| SeqCst  | Acquire | `_failacq`               | `_seqcst_acquire`  |
| SeqCst  | SeqCst  | (none)                   | `_seqcst_seqcst`   |

compiler/rustc_codegen_ssa/src/lib.rs

@@ -6,6 +6,7 @@
 #![feature(associated_type_bounds)]
 #![feature(strict_provenance)]
 #![feature(int_roundings)]
+#![feature(if_let_guard)]
 #![recursion_limit = "256"]
 #![allow(rustc::potential_query_instability)]
 

compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

@@ -376,32 +376,23 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             }
 
             // This requires that atomic intrinsics follow a specific naming pattern:
-            // "atomic_<operation>[_<ordering>]", and no ordering means SeqCst
-            name if name_str.starts_with("atomic_") => {
+            // "atomic_<operation>[_<ordering>]"
+            name if let Some(atomic) = name_str.strip_prefix("atomic_") => {
                 use crate::common::AtomicOrdering::*;
                 use crate::common::{AtomicRmwBinOp, SynchronizationScope};
 
-                let split: Vec<_> = name_str.split('_').collect();
-
-                let is_cxchg = split[1] == "cxchg" || split[1] == "cxchgweak";
-                let (order, failorder) = match split.len() {
-                    2 => (SequentiallyConsistent, SequentiallyConsistent),
-                    3 => match split[2] {
-                        "unordered" => (Unordered, Unordered),
-                        "relaxed" => (Relaxed, Relaxed),
-                        "acq" => (Acquire, Acquire),
-                        "rel" => (Release, Relaxed),
-                        "acqrel" => (AcquireRelease, Acquire),
-                        "failrelaxed" if is_cxchg => (SequentiallyConsistent, Relaxed),
-                        "failacq" if is_cxchg => (SequentiallyConsistent, Acquire),
-                        _ => bx.sess().fatal("unknown ordering in atomic intrinsic"),
-                    },
-                    4 => match (split[2], split[3]) {
-                        ("acq", "failrelaxed") if is_cxchg => (Acquire, Relaxed),
-                        ("acqrel", "failrelaxed") if is_cxchg => (AcquireRelease, Relaxed),
-                        _ => bx.sess().fatal("unknown ordering in atomic intrinsic"),
-                    },
-                    _ => bx.sess().fatal("Atomic intrinsic not in correct format"),
+                let Some((instruction, ordering)) = atomic.split_once('_') else {
+                    bx.sess().fatal("Atomic intrinsic missing memory ordering");
+                };
+
+                let parse_ordering = |bx: &Bx, s| match s {
+                    "unordered" => Unordered,
+                    "relaxed" => Relaxed,
+                    "acquire" => Acquire,
+                    "release" => Release,
+                    "acqrel" => AcquireRelease,
+                    "seqcst" => SequentiallyConsistent,
+                    _ => bx.sess().fatal("unknown ordering in atomic intrinsic"),
                 };
 
                 let invalid_monomorphization = |ty| {
@@ -416,11 +407,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     );
                 };
 
-                match split[1] {
+                match instruction {
                     "cxchg" | "cxchgweak" => {
+                        let Some((success, failure)) = ordering.split_once('_') else {
+                            bx.sess().fatal("Atomic compare-exchange intrinsic missing failure memory ordering");
+                        };
                         let ty = substs.type_at(0);
                         if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() {
-                            let weak = split[1] == "cxchgweak";
+                            let weak = instruction == "cxchgweak";
                             let mut dst = args[0].immediate();
                             let mut cmp = args[1].immediate();
                             let mut src = args[2].immediate();
@@ -432,7 +426,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                                 cmp = bx.ptrtoint(cmp, bx.type_isize());
                                 src = bx.ptrtoint(src, bx.type_isize());
                             }
-                            let pair = bx.atomic_cmpxchg(dst, cmp, src, order, failorder, weak);
+                            let pair = bx.atomic_cmpxchg(dst, cmp, src, parse_ordering(bx, success), parse_ordering(bx, failure), weak);
                             let val = bx.extract_value(pair, 0);
                             let success = bx.extract_value(pair, 1);
                             let val = bx.from_immediate(val);
@@ -460,11 +454,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                                 let llty = bx.type_isize();
                                 let ptr_llty = bx.type_ptr_to(llty);
                                 source = bx.pointercast(source, ptr_llty);
-                                let result = bx.atomic_load(llty, source, order, size);
+                                let result = bx.atomic_load(llty, source, parse_ordering(bx, ordering), size);
                                 // ... and then cast the result back to a pointer
                                 bx.inttoptr(result, bx.backend_type(layout))
                             } else {
-                                bx.atomic_load(bx.backend_type(layout), source, order, size)
+                                bx.atomic_load(bx.backend_type(layout), source, parse_ordering(bx, ordering), size)
                             }
                         } else {
                             return invalid_monomorphization(ty);
@@ -484,20 +478,20 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                                 ptr = bx.pointercast(ptr, ptr_llty);
                                 val = bx.ptrtoint(val, bx.type_isize());
                             }
-                            bx.atomic_store(val, ptr, order, size);
+                            bx.atomic_store(val, ptr, parse_ordering(bx, ordering), size);
                             return;
                         } else {
                             return invalid_monomorphization(ty);
                         }
                     }
 
                     "fence" => {
-                        bx.atomic_fence(order, SynchronizationScope::CrossThread);
+                        bx.atomic_fence(parse_ordering(bx, ordering), SynchronizationScope::CrossThread);
                         return;
                     }
 
                     "singlethreadfence" => {
-                        bx.atomic_fence(order, SynchronizationScope::SingleThread);
+                        bx.atomic_fence(parse_ordering(bx, ordering), SynchronizationScope::SingleThread);
                         return;
                     }
 
@@ -531,7 +525,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                                 ptr = bx.pointercast(ptr, ptr_llty);
                                 val = bx.ptrtoint(val, bx.type_isize());
                             }
-                            bx.atomic_rmw(atom_op, ptr, val, order)
+                            bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering))
                         } else {
                             return invalid_monomorphization(ty);
                         }

compiler/rustc_error_codes/src/error_codes/E0093.md

@@ -24,12 +24,12 @@ functions are defined in `compiler/rustc_codegen_llvm/src/intrinsic.rs` and in
 #![feature(intrinsics)]
 
 extern "rust-intrinsic" {
-    fn atomic_fence(); // ok!
+    fn atomic_fence_seqcst(); // ok!
 }
 
 fn main() {
     unsafe {
-        atomic_fence();
+        atomic_fence_seqcst();
     }
 }
 ```