Merge #17

17: Allow CAS failure ordering stronger than success ordering r=taiki-e a=taiki-e See rust-lang/rust#98383 and [p0418r2](https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0418r2.html). Co-authored-by: Taiki Endo <[email protected]>
taiki-e · Jun 23, 2022 · ba5f29e · ba5f29e
2 parents 1a23b74 + 0f55179
commit ba5f29e
Show file tree

Hide file tree

Showing 9 changed files with 266 additions and 120 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,8 @@ Note: In this file, do not use the hard wrap in the middle of a sentence for com
 
 ## [Unreleased]
 
+- Allow CAS failure ordering stronger than success ordering.
+
 ## [0.3.2] - 2022-06-19
 
 - Optimize x86_64 128-bit atomic load/store on Intel CPU with AVX. ([#16](https://github.com/taiki-e/portable-atomic/pull/16))

diff --git a/src/imp/atomic128/aarch64.rs b/src/imp/atomic128/aarch64.rs
@@ -325,8 +325,10 @@ unsafe fn atomic_compare_exchange(
     old: u128,
     new: u128,
     success: Ordering,
-    _failure: Ordering,
+    failure: Ordering,
 ) -> Result<u128, u128> {
+    let success = crate::utils::upgrade_success_ordering(success, failure);
+
     #[cfg(any(target_feature = "lse", portable_atomic_target_feature = "lse"))]
     // SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`.
     // cfg guarantee that the CPU supports FEAT_LSE.

diff --git a/src/imp/atomic128/powerpc64.rs b/src/imp/atomic128/powerpc64.rs
@@ -153,8 +153,11 @@ unsafe fn atomic_compare_exchange(
     old: u128,
     new: u128,
     success: Ordering,
-    _failure: Ordering,
+    failure: Ordering,
 ) -> Result<u128, u128> {
+    debug_assert!(dst as usize % 16 == 0);
+    let success = crate::utils::upgrade_success_ordering(success, failure);
+
     // SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`.
     let res = unsafe {
         let old = U128 { whole: old };

diff --git a/src/imp/atomic128/x86_64.rs b/src/imp/atomic128/x86_64.rs
@@ -154,6 +154,7 @@ unsafe fn cmpxchg16b(
 #[inline]
 unsafe fn _atomic_load_vmovdqa(src: *mut u128, _order: Ordering) -> u128 {
     debug_assert!(src as usize % 16 == 0);
+
     // SAFETY: the caller must uphold the safety contract for `_atomic_load_vmovdqa`.
     unsafe {
         let out: core::arch::x86_64::__m128;
@@ -170,6 +171,7 @@ unsafe fn _atomic_load_vmovdqa(src: *mut u128, _order: Ordering) -> u128 {
 #[inline]
 unsafe fn _atomic_store_vmovdqa(dst: *mut u128, val: u128, _order: Ordering) {
     debug_assert!(dst as usize % 16 == 0);
+
     // SAFETY: the caller must uphold the safety contract for `_atomic_store_vmovdqa`.
     unsafe {
         let val: core::arch::x86_64::__m128 = core::mem::transmute(val);
@@ -274,6 +276,7 @@ unsafe fn atomic_compare_exchange(
     success: Ordering,
     failure: Ordering,
 ) -> Result<u128, u128> {
+    let success = crate::utils::upgrade_success_ordering(success, failure);
     // SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`.
     let (res, ok) = unsafe { cmpxchg16b(dst, old, new, success, failure) };
     if ok {

diff --git a/src/imp/core_atomic.rs b/src/imp/core_atomic.rs
@@ -1,4 +1,7 @@
 // Wrap the standard library's atomic types in newtype.
+//
+// This is not a reexport, because we want to backport changes like
+// https://github.com/rust-lang/rust/pull/98383 to old compilers.
 
 #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
 #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
@@ -25,6 +28,34 @@ impl AtomicBool {
     pub(crate) fn into_inner(self) -> bool {
         self.inner.into_inner()
     }
+    #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
+    #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
+    #[inline]
+    pub(crate) fn compare_exchange(
+        &self,
+        current: bool,
+        new: bool,
+        success: Ordering,
+        failure: Ordering,
+    ) -> Result<bool, bool> {
+        // TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged.
+        let success = crate::utils::upgrade_success_ordering(success, failure);
+        self.inner.compare_exchange(current, new, success, failure)
+    }
+    #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
+    #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
+    #[inline]
+    pub(crate) fn compare_exchange_weak(
+        &self,
+        current: bool,
+        new: bool,
+        success: Ordering,
+        failure: Ordering,
+    ) -> Result<bool, bool> {
+        // TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged.
+        let success = crate::utils::upgrade_success_ordering(success, failure);
+        self.inner.compare_exchange_weak(current, new, success, failure)
+    }
 }
 impl core::ops::Deref for AtomicBool {
     type Target = core::sync::atomic::AtomicBool;
@@ -61,6 +92,34 @@ impl<T> AtomicPtr<T> {
     pub(crate) fn into_inner(self) -> *mut T {
         self.inner.into_inner()
     }
+    #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
+    #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
+    #[inline]
+    pub(crate) fn compare_exchange(
+        &self,
+        current: *mut T,
+        new: *mut T,
+        success: Ordering,
+        failure: Ordering,
+    ) -> Result<*mut T, *mut T> {
+        // TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged.
+        let success = crate::utils::upgrade_success_ordering(success, failure);
+        self.inner.compare_exchange(current, new, success, failure)
+    }
+    #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
+    #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
+    #[inline]
+    pub(crate) fn compare_exchange_weak(
+        &self,
+        current: *mut T,
+        new: *mut T,
+        success: Ordering,
+        failure: Ordering,
+    ) -> Result<*mut T, *mut T> {
+        // TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged.
+        let success = crate::utils::upgrade_success_ordering(success, failure);
+        self.inner.compare_exchange_weak(current, new, success, failure)
+    }
 }
 impl<T> core::ops::Deref for AtomicPtr<T> {
     type Target = core::sync::atomic::AtomicPtr<T>;
@@ -99,6 +158,46 @@ macro_rules! atomic_int {
             pub(crate) fn into_inner(self) -> $int_type {
                 self.inner.into_inner()
             }
+            #[cfg_attr(
+                portable_atomic_no_cfg_target_has_atomic,
+                cfg(not(portable_atomic_no_atomic_cas))
+            )]
+            #[cfg_attr(
+                not(portable_atomic_no_cfg_target_has_atomic),
+                cfg(target_has_atomic = "ptr")
+            )]
+            #[inline]
+            pub(crate) fn compare_exchange(
+                &self,
+                current: $int_type,
+                new: $int_type,
+                success: Ordering,
+                failure: Ordering,
+            ) -> Result<$int_type, $int_type> {
+                // TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged.
+                let success = crate::utils::upgrade_success_ordering(success, failure);
+                self.inner.compare_exchange(current, new, success, failure)
+            }
+            #[cfg_attr(
+                portable_atomic_no_cfg_target_has_atomic,
+                cfg(not(portable_atomic_no_atomic_cas))
+            )]
+            #[cfg_attr(
+                not(portable_atomic_no_cfg_target_has_atomic),
+                cfg(target_has_atomic = "ptr")
+            )]
+            #[inline]
+            pub(crate) fn compare_exchange_weak(
+                &self,
+                current: $int_type,
+                new: $int_type,
+                success: Ordering,
+                failure: Ordering,
+            ) -> Result<$int_type, $int_type> {
+                // TODO: add cfg once https://github.com/rust-lang/rust/pull/98383 merged.
+                let success = crate::utils::upgrade_success_ordering(success, failure);
+                self.inner.compare_exchange_weak(current, new, success, failure)
+            }
             #[cfg(portable_atomic_no_atomic_min_max)]
             #[cfg_attr(
                 portable_atomic_no_cfg_target_has_atomic,

diff --git a/src/lib.rs b/src/lib.rs
@@ -419,8 +419,7 @@ impl AtomicBool {
     /// `failure` describes the required ordering for the load operation that takes place when
     /// the comparison fails. Using [`Acquire`] as success ordering makes the store part
     /// of this operation [`Relaxed`], and using [`Release`] makes the successful load
-    /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
-    /// and must be equivalent to or weaker than the success ordering.
+    /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
     #[cfg_attr(
         portable_atomic_no_cfg_target_has_atomic,
         cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core))
@@ -454,8 +453,7 @@ impl AtomicBool {
     /// `failure` describes the required ordering for the load operation that takes place when
     /// the comparison fails. Using [`Acquire`] as success ordering makes the store part
     /// of this operation [`Relaxed`], and using [`Release`] makes the successful load
-    /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
-    /// and must be equivalent to or weaker than the success ordering.
+    /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
     #[cfg_attr(
         portable_atomic_no_cfg_target_has_atomic,
         cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core))
@@ -593,8 +591,7 @@ impl AtomicBool {
     /// Using [`Acquire`] as success ordering makes the store part of this
     /// operation [`Relaxed`], and using [`Release`] makes the final successful
     /// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`],
-    /// [`Acquire`] or [`Relaxed`] and must be equivalent to or weaker than the
-    /// success ordering.
+    /// [`Acquire`] or [`Relaxed`].
     #[cfg_attr(
         portable_atomic_no_cfg_target_has_atomic,
         cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core))
@@ -792,8 +789,7 @@ impl<T> AtomicPtr<T> {
     /// `failure` describes the required ordering for the load operation that takes place when
     /// the comparison fails. Using [`Acquire`] as success ordering makes the store part
     /// of this operation [`Relaxed`], and using [`Release`] makes the successful load
-    /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
-    /// and must be equivalent to or weaker than the success ordering.
+    /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
     #[cfg_attr(
         portable_atomic_no_cfg_target_has_atomic,
         cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core))
@@ -827,8 +823,7 @@ impl<T> AtomicPtr<T> {
     /// `failure` describes the required ordering for the load operation that takes place when
     /// the comparison fails. Using [`Acquire`] as success ordering makes the store part
     /// of this operation [`Relaxed`], and using [`Release`] makes the successful load
-    /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
-    /// and must be equivalent to or weaker than the success ordering.
+    /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
     #[cfg_attr(
         portable_atomic_no_cfg_target_has_atomic,
         cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core))
@@ -867,8 +862,7 @@ impl<T> AtomicPtr<T> {
     /// Using [`Acquire`] as success ordering makes the store part of this
     /// operation [`Relaxed`], and using [`Release`] makes the final successful
     /// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`],
-    /// [`Acquire`] or [`Relaxed`] and must be equivalent to or weaker than the
-    /// success ordering.
+    /// [`Acquire`] or [`Relaxed`].
     #[cfg_attr(
         portable_atomic_no_cfg_target_has_atomic,
         cfg(any(not(portable_atomic_no_atomic_cas), portable_atomic_unsafe_assume_single_core))
@@ -1087,8 +1081,7 @@ atomic instructions or locks will be used.
             /// `failure` describes the required ordering for the load operation that takes place when
             /// the comparison fails. Using [`Acquire`] as success ordering makes the store part
             /// of this operation [`Relaxed`], and using [`Release`] makes the successful load
-            /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
-            /// and must be equivalent to or weaker than the success ordering.
+            /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
             #[cfg_attr(
                 portable_atomic_no_cfg_target_has_atomic,
                 cfg(any(
@@ -1126,8 +1119,7 @@ atomic instructions or locks will be used.
             /// `failure` describes the required ordering for the load operation that takes place when
             /// the comparison fails. Using [`Acquire`] as success ordering makes the store part
             /// of this operation [`Relaxed`], and using [`Release`] makes the successful load
-            /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
-            /// and must be equivalent to or weaker than the success ordering.
+            /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
             #[cfg_attr(
                 portable_atomic_no_cfg_target_has_atomic,
                 cfg(any(
@@ -1322,8 +1314,7 @@ atomic instructions or locks will be used.
             ///
             /// Using [`Acquire`] as success ordering makes the store part
             /// of this operation [`Relaxed`], and using [`Release`] makes the final successful load
-            /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
-            /// and must be equivalent to or weaker than the success ordering.
+            /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
             #[cfg_attr(
                 portable_atomic_no_cfg_target_has_atomic,
                 cfg(any(
@@ -1585,8 +1576,7 @@ This type has the same in-memory representation as the underlying floating point
             /// `failure` describes the required ordering for the load operation that takes place when
             /// the comparison fails. Using [`Acquire`] as success ordering makes the store part
             /// of this operation [`Relaxed`], and using [`Release`] makes the successful load
-            /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
-            /// and must be equivalent to or weaker than the success ordering.
+            /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
             #[cfg_attr(
                 portable_atomic_no_cfg_target_has_atomic,
                 cfg(any(
@@ -1632,8 +1622,7 @@ This type has the same in-memory representation as the underlying floating point
             /// `failure` describes the required ordering for the load operation that takes place when
             /// the comparison fails. Using [`Acquire`] as success ordering makes the store part
             /// of this operation [`Relaxed`], and using [`Release`] makes the successful load
-            /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
-            /// and must be equivalent to or weaker than the success ordering.
+            /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
             #[cfg_attr(
                 portable_atomic_no_cfg_target_has_atomic,
                 cfg(any(
@@ -1734,8 +1723,7 @@ This type has the same in-memory representation as the underlying floating point
             ///
             /// Using [`Acquire`] as success ordering makes the store part
             /// of this operation [`Relaxed`], and using [`Release`] makes the final successful load
-            /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`]
-            /// and must be equivalent to or weaker than the success ordering.
+            /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].
             #[cfg_attr(
                 portable_atomic_no_cfg_target_has_atomic,
                 cfg(any(