Merge #2709 #2800 #2811

2709: ci: add Python 3.12-dev jobs r=davidhewitt a=davidhewitt

Separately to #2708, time to start testing 3.12 alphas on CI. I imagine this might also take a while for the packages to come available.

2800: allow `**kwargs` to take arguments which conflict with positional-only parameters r=davidhewitt a=davidhewitt

Closes #2799 

I did a little bit of refactoring at the same time, sorry! The bit which changes behaviour is that in the now-factored-out `handle_kwargs` method, if the keyword argument name is found in the positional-only range then the varkeywords handler is now given a chance to accept it. (Lines 387-390 in new `extract_argument.rs`.)

2811: adjust vectorcall symbols for pypy r=davidhewitt a=davidhewitt

Closes #2738

Co-authored-by: David Hewitt <[email protected]>

.github/workflows/ci.yml

@@ -141,6 +141,7 @@ jobs:
           "3.9",
           "3.10",
           "3.11",
+          "3.12-dev",
           "pypy-3.7",
           "pypy-3.8",
           "pypy-3.9"

newsfragments/2800.fixed.md

@@ -0,0 +1 @@
+Allow functions taking `**kwargs` to accept keyword arguments which share a name with a positional-only argument (as permitted by PEP 570).

newsfragments/2811.added.md

@@ -0,0 +1 @@
+Add FFI definitions `PyVectorcall_NARGS` and `PY_VECTORCALL_ARGUMENTS_OFFSET` for PyPy 3.8 and up.

newsfragments/2811.fixed.md

@@ -0,0 +1 @@
+Fix unresolved symbol for `PyObject_Vectorcall` on PyPy 3.9 and up.

pyo3-ffi/src/cpython/abstract_.rs

@@ -39,11 +39,11 @@ extern "C" {
     ) -> *mut PyObject;
 }
 
-#[cfg(all(Py_3_8, not(PyPy)))]
+#[cfg(all(Py_3_8))]
 const PY_VECTORCALL_ARGUMENTS_OFFSET: Py_ssize_t =
     1 << (8 * std::mem::size_of::<Py_ssize_t>() as Py_ssize_t - 1);
 
-#[cfg(all(Py_3_8, not(PyPy)))]
+#[cfg(all(Py_3_8))]
 #[inline(always)]
 pub unsafe fn PyVectorcall_NARGS(n: size_t) -> Py_ssize_t {
     assert!(n <= (PY_SSIZE_T_MAX as size_t));
@@ -103,6 +103,7 @@ pub unsafe fn PyObject_Vectorcall(
 extern "C" {
     #[cfg(all(PyPy, Py_3_8))]
     #[cfg_attr(not(Py_3_9), link_name = "_PyPyObject_Vectorcall")]
+    #[cfg_attr(Py_3_9, link_name = "PyPyObject_Vectorcall")]
     pub fn PyObject_Vectorcall(
         callable: *mut PyObject,
         args: *const *mut PyObject,

pyo3-ffi/src/cpython/unicodeobject.rs

@@ -1,6 +1,7 @@
 #[cfg(not(PyPy))]
 use crate::Py_hash_t;
 use crate::{PyObject, Py_UCS1, Py_UCS2, Py_UCS4, Py_UNICODE, Py_ssize_t};
+#[cfg(not(Py_3_12))]
 use libc::wchar_t;
 use std::os::raw::{c_char, c_int, c_uint, c_void};
 
@@ -48,6 +49,7 @@ pub struct PyASCIIObject {
     /// unsigned int ready:1;
     /// unsigned int :24;
     pub state: u32,
+    #[cfg(not(Py_3_12))]
     pub wstr: *mut wchar_t,
 }
 
@@ -56,6 +58,7 @@ pub struct PyASCIIObject {
 /// In addition, they are disabled on big-endian architectures to restrict this to most "common"
 /// platforms, which are at least tested on CI and appear to be sound.
 #[cfg(target_endian = "little")]
+#[cfg(not(Py_3_12))]
 impl PyASCIIObject {
     #[inline]
     pub unsafe fn interned(&self) -> c_uint {
@@ -83,11 +86,35 @@ impl PyASCIIObject {
     }
 }
 
+#[cfg(Py_3_12)]
+impl PyASCIIObject {
+    #[inline]
+    pub unsafe fn interned(&self) -> c_uint {
+        self.state & 1
+    }
+
+    #[inline]
+    pub unsafe fn kind(&self) -> c_uint {
+        (self.state >> 1) & 7
+    }
+
+    #[inline]
+    pub unsafe fn compact(&self) -> c_uint {
+        (self.state >> 4) & 1
+    }
+
+    #[inline]
+    pub unsafe fn ascii(&self) -> c_uint {
+        (self.state >> 5) & 1
+    }
+}
+
 #[repr(C)]
 pub struct PyCompactUnicodeObject {
     pub _base: PyASCIIObject,
     pub utf8_length: Py_ssize_t,
     pub utf8: *mut c_char,
+    #[cfg(not(Py_3_12))]
     pub wstr_length: Py_ssize_t,
 }
 
@@ -123,6 +150,7 @@ pub const SSTATE_INTERNED_IMMORTAL: c_uint = 2;
 #[cfg(target_endian = "little")]
 pub unsafe fn PyUnicode_IS_ASCII(op: *mut PyObject) -> c_uint {
     debug_assert!(crate::PyUnicode_Check(op) != 0);
+    #[cfg(not(Py_3_12))]
     debug_assert!(PyUnicode_IS_READY(op) != 0);
 
     (*(op as *mut PyASCIIObject)).ascii()
@@ -141,7 +169,7 @@ pub unsafe fn PyUnicode_IS_COMPACT_ASCII(op: *mut PyObject) -> c_uint {
 }
 
 #[cfg(not(Py_3_12))]
-#[cfg_attr(Py_3_10, deprecated(note = "Python 3.10"))]
+#[deprecated(note = "Removed in Python 3.12")]
 pub const PyUnicode_WCHAR_KIND: c_uint = 0;
 
 pub const PyUnicode_1BYTE_KIND: c_uint = 1;
@@ -170,6 +198,7 @@ pub unsafe fn PyUnicode_4BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS4 {
 #[cfg(target_endian = "little")]
 pub unsafe fn PyUnicode_KIND(op: *mut PyObject) -> c_uint {
     debug_assert!(crate::PyUnicode_Check(op) != 0);
+    #[cfg(not(Py_3_12))]
     debug_assert!(PyUnicode_IS_READY(op) != 0);
 
     (*(op as *mut PyASCIIObject)).kind()
@@ -213,19 +242,26 @@ pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void {
 #[cfg(target_endian = "little")]
 pub unsafe fn PyUnicode_GET_LENGTH(op: *mut PyObject) -> Py_ssize_t {
     debug_assert!(crate::PyUnicode_Check(op) != 0);
+    #[cfg(not(Py_3_12))]
     debug_assert!(PyUnicode_IS_READY(op) != 0);
 
     (*(op as *mut PyASCIIObject)).length
 }
 
 #[inline]
+#[cfg(not(Py_3_12))]
 #[cfg(target_endian = "little")]
 pub unsafe fn PyUnicode_IS_READY(op: *mut PyObject) -> c_uint {
     (*(op as *mut PyASCIIObject)).ready()
 }
 
+#[inline]
+#[cfg(Py_3_12)]
+pub unsafe fn PyUnicode_IS_READY(_: *mut PyObject) -> c_uint {
+    1
+}
+
 #[cfg(not(Py_3_12))]
-#[cfg_attr(Py_3_10, deprecated(note = "Python 3.10"))]
 #[inline]
 #[cfg(target_endian = "little")]
 pub unsafe fn PyUnicode_READY(op: *mut PyObject) -> c_int {
@@ -238,6 +274,12 @@ pub unsafe fn PyUnicode_READY(op: *mut PyObject) -> c_int {
     }
 }
 
+#[cfg(Py_3_12)]
+#[inline]
+pub unsafe fn PyUnicode_READY(_: *mut PyObject) -> c_int {
+    0
+}
+
 // skipped PyUnicode_MAX_CHAR_VALUE
 // skipped _PyUnicode_get_wstr_length
 // skipped PyUnicode_WSTR_LENGTH

pyo3-ffi/src/unicodeobject.rs

@@ -59,6 +59,8 @@ extern "C" {
     pub fn PyUnicode_AsUCS4Copy(unicode: *mut PyObject) -> *mut Py_UCS4;
     #[cfg_attr(PyPy, link_name = "PyPyUnicode_GetLength")]
     pub fn PyUnicode_GetLength(unicode: *mut PyObject) -> Py_ssize_t;
+    #[cfg(not(Py_3_12))]
+    #[deprecated(note = "Removed in Python 3.12")]
     #[cfg_attr(PyPy, link_name = "PyPyUnicode_GetSize")]
     pub fn PyUnicode_GetSize(unicode: *mut PyObject) -> Py_ssize_t;
     pub fn PyUnicode_ReadChar(unicode: *mut PyObject, index: Py_ssize_t) -> Py_UCS4;

src/ffi/tests.rs

@@ -3,7 +3,7 @@ use crate::{types::PyDict, AsPyPointer, IntoPy, Py, PyAny, Python};
 
 #[cfg(target_endian = "little")]
 use crate::types::PyString;
-#[cfg(target_endian = "little")]
+#[cfg(all(target_endian = "little", not(Py_3_12)))]
 use libc::wchar_t;
 
 #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons
@@ -119,6 +119,7 @@ fn ascii_object_bitfield() {
         #[cfg(not(PyPy))]
         hash: 0,
         state: 0,
+        #[cfg(not(Py_3_12))]
         wstr: std::ptr::null_mut() as *mut wchar_t,
     };
 
@@ -127,26 +128,58 @@ fn ascii_object_bitfield() {
         assert_eq!(o.kind(), 0);
         assert_eq!(o.compact(), 0);
         assert_eq!(o.ascii(), 0);
+        #[cfg(not(Py_3_12))]
         assert_eq!(o.ready(), 0);
 
-        for i in 0..4 {
-            o.state = i;
-            assert_eq!(o.interned(), i);
+        #[cfg(not(Py_3_12))]
+        {
+            for i in 0..4 {
+                o.state = i;
+                assert_eq!(o.interned(), i);
+            }
+
+            for i in 0..8 {
+                o.state = i << 2;
+                assert_eq!(o.kind(), i);
+            }
+
+            o.state = 0;
+            assert_eq!(o.compact(), 0);
+            o.state = 1 << 5;
+            assert_eq!(o.compact(), 1);
+
+            o.state = 0;
+            assert_eq!(o.ascii(), 0);
+            o.state = 1 << 6;
+            assert_eq!(o.ascii(), 1);
+
+            o.state = 0;
+            assert_eq!(o.ready(), 0);
+            o.state = 1 << 7;
+            assert_eq!(o.ready(), 1);
         }
 
-        for i in 0..8 {
-            o.state = i << 2;
-            assert_eq!(o.kind(), i);
+        #[cfg(Py_3_12)]
+        {
+            assert_eq!(o.interned(), 0);
+            o.state = 1;
+            assert_eq!(o.interned(), 1);
+
+            for i in 0..8 {
+                o.state = i << 1;
+                assert_eq!(o.kind(), i);
+            }
+
+            o.state = 0;
+            assert_eq!(o.compact(), 0);
+            o.state = 1 << 4;
+            assert_eq!(o.compact(), 1);
+
+            o.state = 0;
+            assert_eq!(o.ascii(), 0);
+            o.state = 1 << 5;
+            assert_eq!(o.ascii(), 1);
         }
-
-        o.state = 1 << 5;
-        assert_eq!(o.compact(), 1);
-
-        o.state = 1 << 6;
-        assert_eq!(o.ascii(), 1);
-
-        o.state = 1 << 7;
-        assert_eq!(o.ready(), 1);
     }
 }
 
@@ -167,6 +200,7 @@ fn ascii() {
             assert_eq!(ascii.kind(), PyUnicode_1BYTE_KIND);
             assert_eq!(ascii.compact(), 1);
             assert_eq!(ascii.ascii(), 1);
+            #[cfg(not(Py_3_12))]
             assert_eq!(ascii.ready(), 1);
 
             assert_eq!(PyUnicode_IS_ASCII(ptr), 1);
@@ -208,6 +242,7 @@ fn ucs4() {
             assert_eq!(ascii.kind(), PyUnicode_4BYTE_KIND);
             assert_eq!(ascii.compact(), 1);
             assert_eq!(ascii.ascii(), 0);
+            #[cfg(not(Py_3_12))]
             assert_eq!(ascii.ready(), 1);
 
             assert_eq!(PyUnicode_IS_ASCII(ptr), 0);