From f7edd35a9e223ed2f9c8f3281caf78284d647cc1 Mon Sep 17 00:00:00 2001 From: Nicholas Sim Date: Fri, 19 Feb 2021 19:43:11 +0800 Subject: [PATCH 1/6] ffi cleanup: abstract.h --- CHANGELOG.md | 1 + src/ffi/{objectabstract.rs => abstract_.rs} | 36 +++++++++++++++------ src/ffi/cpython/abstract_.rs | 15 +++------ src/ffi/mod.rs | 9 ++---- 4 files changed, 35 insertions(+), 26 deletions(-) rename src/ffi/{objectabstract.rs => abstract_.rs} (94%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e0e8f08083..e4d357714bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Remove FFI definition `PyCFunction_ClearFreeList` for Python 3.9 and later. [#1425](https://github.com/PyO3/pyo3/pull/1425) - `PYO3_CROSS_LIB_DIR` enviroment variable no long required when compiling for x86-64 Python from macOS arm64 and reverse. [#1428](https://github.com/PyO3/pyo3/pull/1428) - Fix FFI definition `_PyEval_RequestCodeExtraIndex` which took an argument of the wrong type. [#1429](https://github.com/PyO3/pyo3/pull/1429) +- Ensure `PyIter_Check` is available for Python < 3.8 under abi3. [#1436](https://github.com/PyO3/pyo3/pull/1436) ## [0.13.2] - 2021-02-12 ### Packaging diff --git a/src/ffi/objectabstract.rs b/src/ffi/abstract_.rs similarity index 94% rename from src/ffi/objectabstract.rs rename to src/ffi/abstract_.rs index c2403e1f526..8bef516f2c1 100644 --- a/src/ffi/objectabstract.rs +++ b/src/ffi/abstract_.rs @@ -51,6 +51,9 @@ extern "C" { ... ) -> *mut PyObject; + // skipped _PyObject_CallFunction_SizeT + // skipped _PyObject_CallMethod_SizeT + #[cfg_attr(PyPy, link_name = "PyPyObject_CallFunctionObjArgs")] pub fn PyObject_CallFunctionObjArgs(callable: *mut PyObject, ...) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyObject_CallMethodObjArgs")] @@ -84,16 +87,27 @@ extern "C" { pub fn PyObject_Format(obj: *mut PyObject, format_spec: *mut PyObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyObject_GetIter")] pub fn PyObject_GetIter(arg1: *mut PyObject) -> *mut PyObject; +} - // PyIter_Check for unlimited API is in cpython/abstract_.rs - #[cfg(any(all(Py_LIMITED_API, Py_3_8), PyPy))] - #[cfg_attr(PyPy, link_name = "PyPyIter_Check")] - pub fn PyIter_Check(obj: *mut PyObject) -> c_int; +#[cfg(not(any(all(Py_3_8, Py_LIMITED_API), PyPy)))] +pub unsafe fn PyIter_Check(o: *mut PyObject) -> c_int { + (match (*crate::ffi::Py_TYPE(o)).tp_iternext { + Some(tp_iternext) => { + tp_iternext as *const std::os::raw::c_void + != crate::ffi::_PyObject_NextNotImplemented as _ + } + None => false, + }) as c_int } extern "C" { + #[cfg(any(all(Py_3_8, Py_LIMITED_API), PyPy))] + #[cfg_attr(PyPy, link_name = "PyPyIter_Check")] + pub fn PyIter_Check(obj: *mut PyObject) -> c_int; + #[cfg_attr(PyPy, link_name = "PyPyIter_Next")] pub fn PyIter_Next(arg1: *mut PyObject) -> *mut PyObject; + // skipped non-limited / 3.10 PyIter_Send #[cfg_attr(PyPy, link_name = "PyPyNumber_Check")] pub fn PyNumber_Check(o: *mut PyObject) -> c_int; @@ -134,13 +148,9 @@ extern "C" { pub fn PyNumber_Xor(o1: *mut PyObject, o2: *mut PyObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyNumber_Or")] pub fn PyNumber_Or(o1: *mut PyObject, o2: *mut PyObject) -> *mut PyObject; - - #[cfg(PyPy)] - #[link_name = "PyPyIndex_Check"] - pub fn PyIndex_Check(o: *mut PyObject) -> c_int; } -#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[cfg(not(any(all(Py_3_8, Py_LIMITED_API), PyPy)))] #[inline] pub unsafe fn PyIndex_Check(o: *mut PyObject) -> c_int { let tp_as_number = (*Py_TYPE(o)).tp_as_number; @@ -148,6 +158,10 @@ pub unsafe fn PyIndex_Check(o: *mut PyObject) -> c_int { } extern "C" { + #[cfg(any(all(Py_3_8, Py_LIMITED_API), PyPy))] + #[link_name = "PyPyIndex_Check"] + pub fn PyIndex_Check(o: *mut PyObject) -> c_int; + #[cfg_attr(PyPy, link_name = "PyPyNumber_Index")] pub fn PyNumber_Index(o: *mut PyObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyNumber_AsSsize_t")] @@ -232,7 +246,9 @@ extern "C" { pub fn PySequence_List(o: *mut PyObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPySequence_Fast")] pub fn PySequence_Fast(o: *mut PyObject, m: *const c_char) -> *mut PyObject; - // TODO: PySequence_Fast macros + // skipped PySequenc_Fast_GET_SIZE + // skipped PySequenc_Fast_GET_ITEM + // skipped PySequenc_Fast_GET_ITEMS pub fn PySequence_Count(o: *mut PyObject, value: *mut PyObject) -> Py_ssize_t; #[cfg_attr(PyPy, link_name = "PyPySequence_Contains")] pub fn PySequence_Contains(seq: *mut PyObject, ob: *mut PyObject) -> c_int; diff --git a/src/ffi/cpython/abstract_.rs b/src/ffi/cpython/abstract_.rs index 3dd33c49c94..b86ed1a7008 100644 --- a/src/ffi/cpython/abstract_.rs +++ b/src/ffi/cpython/abstract_.rs @@ -266,16 +266,11 @@ extern "C" { pub fn PyBuffer_Release(view: *mut Py_buffer); } -#[inline] -#[cfg(not(any(all(Py_3_8, Py_LIMITED_API), PyPy)))] -pub unsafe fn PyIter_Check(o: *mut PyObject) -> c_int { - (match (*crate::ffi::Py_TYPE(o)).tp_iternext { - Some(tp_iternext) => { - tp_iternext as *const c_void != crate::ffi::_PyObject_NextNotImplemented as _ - } - None => false, - }) as c_int -} +// PyIter_Check defined in ffi/abstract_.rs +// PyIndex_Check defined in ffi/abstract_.rs +// Not defined here because this file is not compiled under the +// limited API, but the macros need to be defined for 3.6, 3.7 which +// predate the limited API changes. // skipped PySequence_ITEM diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 73fa2e3e547..70192783495 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -12,6 +12,7 @@ macro_rules! opaque_struct { }; } +pub use self::abstract_::*; pub use self::bltinmodule::*; pub use self::boolobject::*; pub use self::bytearrayobject::*; @@ -47,7 +48,6 @@ pub use self::methodobject::*; pub use self::modsupport::*; pub use self::moduleobject::*; pub use self::object::*; -pub use self::objectabstract::*; // FIXME: no matching objectabstract.h in cpython master pub use self::objimpl::*; pub use self::osmodule::*; pub use self::pyarena::*; @@ -76,8 +76,7 @@ pub use self::weakrefobject::*; #[cfg(not(Py_LIMITED_API))] pub use self::cpython::*; - -// skipped abstract.h +mod abstract_; // skipped asdl.h // skipped ast.h mod bltinmodule; @@ -200,8 +199,6 @@ mod pythonrun; // TODO some functions need to be moved to pylifecycle mod osmodule; mod sysmodule; // TODO supports PEP-384 only; needs adjustment for Python 3.3 and 3.5 -mod objectabstract; // TODO supports PEP-384 only; needs adjustment for Python 3.3 and 3.5 - // mod pyctype; TODO excluded by PEP-384 mod pystrtod; // TODO supports PEP-384 only; needs adjustment for Python 3.3 and 3.5 // mod pystrcmp; TODO nothing interesting for Rust? @@ -212,5 +209,5 @@ mod pystrtod; // TODO supports PEP-384 only; needs adjustment for Python 3.3 and // Additional headers that are not exported by Python.h pub mod structmember; // TODO supports PEP-384 only; needs adjustment for Python 3.3 and 3.5 -#[cfg(not(Py_LIMITED_API))] +#[cfg(all(Py_3_8, not(Py_LIMITED_API)))] mod cpython; From 24f091b8ff7f48a6e8ff40ab4d83ba7edfcf1645 Mon Sep 17 00:00:00 2001 From: Nicholas Sim Date: Fri, 19 Feb 2021 19:50:29 +0800 Subject: [PATCH 2/6] remove cfg for PyTryFrom impl for PyIterator --- CHANGELOG.md | 3 ++- src/types/iterator.rs | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4d357714bd..77cd2dda19e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add #[pyo3(from_py_with = "...")]` attribute for function arguments and struct fields to override the default from-Python conversion. [#1411](https://github.com/PyO3/pyo3/pull/1411) - Add FFI definition `PyCFunction_CheckExact` for Python 3.9 and later. [#1425](https://github.com/PyO3/pyo3/pull/1425) - Add FFI definition `Py_IS_TYPE`. [#1429](https://github.com/PyO3/pyo3/pull/1429) +- Implement `PyTryFrom` for `PyIterator` for Python 3.7 and earlier with the `abi3` feature. [#1436](https://github.com/PyO3/pyo3/pull/1436) ### Changed - Change `PyTimeAcces::get_fold()` to return a `bool` instead of a `u8`. [#1397](https://github.com/PyO3/pyo3/pull/1397) @@ -30,7 +31,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Remove FFI definition `PyCFunction_ClearFreeList` for Python 3.9 and later. [#1425](https://github.com/PyO3/pyo3/pull/1425) - `PYO3_CROSS_LIB_DIR` enviroment variable no long required when compiling for x86-64 Python from macOS arm64 and reverse. [#1428](https://github.com/PyO3/pyo3/pull/1428) - Fix FFI definition `_PyEval_RequestCodeExtraIndex` which took an argument of the wrong type. [#1429](https://github.com/PyO3/pyo3/pull/1429) -- Ensure `PyIter_Check` is available for Python < 3.8 under abi3. [#1436](https://github.com/PyO3/pyo3/pull/1436) +- Fix FFI definition `PyIter_Check` missing for Python 3.7 and earlier with the `abi3` feature. [#1436](https://github.com/PyO3/pyo3/pull/1436) ## [0.13.2] - 2021-02-12 ### Packaging diff --git a/src/types/iterator.rs b/src/types/iterator.rs index e9c0a674e6f..4c0c2502a73 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -67,8 +67,6 @@ impl<'p> Iterator for &'p PyIterator { } } -// PyIter_Check does not exist in the limited API until 3.8 -#[cfg(any(not(Py_LIMITED_API), Py_3_8))] impl<'v> PyTryFrom<'v> for PyIterator { fn try_from>(value: V) -> Result<&'v PyIterator, PyDowncastError<'v>> { let value = value.into(); From 13bfbd38b44ab1195ac98585218f063102b45efe Mon Sep 17 00:00:00 2001 From: Nicholas Sim Date: Fri, 19 Feb 2021 19:54:10 +0800 Subject: [PATCH 3/6] CHANGELOG: update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77cd2dda19e..3d0ec096a43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `PYO3_CROSS_LIB_DIR` enviroment variable no long required when compiling for x86-64 Python from macOS arm64 and reverse. [#1428](https://github.com/PyO3/pyo3/pull/1428) - Fix FFI definition `_PyEval_RequestCodeExtraIndex` which took an argument of the wrong type. [#1429](https://github.com/PyO3/pyo3/pull/1429) - Fix FFI definition `PyIter_Check` missing for Python 3.7 and earlier with the `abi3` feature. [#1436](https://github.com/PyO3/pyo3/pull/1436) +- Fix FFI definition `PyIndex_Check` missing with the `abi3` feature. [#1436](https://github.com/PyO3/pyo3/pull/1436) ## [0.13.2] - 2021-02-12 ### Packaging From ec894e3a33e403472c14a8a8b7035a391b39c76f Mon Sep 17 00:00:00 2001 From: Nicholas Sim Date: Fri, 19 Feb 2021 20:01:48 +0800 Subject: [PATCH 4/6] don't require Py_3_8 for ffi::cpython Some functions available in earlier versions were moved to this directory, but are still valid. --- src/ffi/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 70192783495..bc739b681db 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -209,5 +209,5 @@ mod pystrtod; // TODO supports PEP-384 only; needs adjustment for Python 3.3 and // Additional headers that are not exported by Python.h pub mod structmember; // TODO supports PEP-384 only; needs adjustment for Python 3.3 and 3.5 -#[cfg(all(Py_3_8, not(Py_LIMITED_API)))] +#[cfg(not(Py_LIMITED_API))] mod cpython; From 60848f94e44eb11e85811c129fc93ef8e736ccef Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Mon, 22 Feb 2021 23:10:54 +0000 Subject: [PATCH 5/6] ffi: fix pypy build --- src/ffi/pyerrors.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/ffi/pyerrors.rs b/src/ffi/pyerrors.rs index 4bd08d66d1d..f5b99fe1f2e 100644 --- a/src/ffi/pyerrors.rs +++ b/src/ffi/pyerrors.rs @@ -1,9 +1,5 @@ use crate::ffi::object::*; -#[cfg(PyPy)] -use crate::ffi::objectabstract::PyObject_CallFunction; use crate::ffi::pyport::Py_ssize_t; -#[cfg(PyPy)] -use std::ffi::CStr; use std::os::raw::{c_char, c_int}; #[repr(C)] @@ -162,9 +158,11 @@ pub unsafe fn PyUnicodeDecodeError_Create( end: Py_ssize_t, _reason: *const c_char, ) -> *mut PyObject { - return PyObject_CallFunction( + return crate::ffi::PyObject_CallFunction( PyExc_UnicodeDecodeError, - CStr::from_bytes_with_nul(b"sy#nns\0").unwrap().as_ptr(), + std::ffi::CStr::from_bytes_with_nul(b"sy#nns\0") + .unwrap() + .as_ptr(), encoding, object, length, From d278aaff5494cc9ee8b1439fd1ee90875a1a29af Mon Sep 17 00:00:00 2001 From: Nicholas Sim Date: Sun, 28 Feb 2021 13:16:02 +0800 Subject: [PATCH 6/6] Remove PyIter_Check, PyIndex_Check under abi3 for Python < 3.8 While these are defined as macros in the Python C API, they rely on access to the PyTypeObject structure, which is not part of the limited API for those versions. --- CHANGELOG.md | 2 -- src/ffi/abstract_.rs | 9 +++++++-- src/types/iterator.rs | 2 ++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d0ec096a43..13d263153c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add #[pyo3(from_py_with = "...")]` attribute for function arguments and struct fields to override the default from-Python conversion. [#1411](https://github.com/PyO3/pyo3/pull/1411) - Add FFI definition `PyCFunction_CheckExact` for Python 3.9 and later. [#1425](https://github.com/PyO3/pyo3/pull/1425) - Add FFI definition `Py_IS_TYPE`. [#1429](https://github.com/PyO3/pyo3/pull/1429) -- Implement `PyTryFrom` for `PyIterator` for Python 3.7 and earlier with the `abi3` feature. [#1436](https://github.com/PyO3/pyo3/pull/1436) ### Changed - Change `PyTimeAcces::get_fold()` to return a `bool` instead of a `u8`. [#1397](https://github.com/PyO3/pyo3/pull/1397) @@ -31,7 +30,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Remove FFI definition `PyCFunction_ClearFreeList` for Python 3.9 and later. [#1425](https://github.com/PyO3/pyo3/pull/1425) - `PYO3_CROSS_LIB_DIR` enviroment variable no long required when compiling for x86-64 Python from macOS arm64 and reverse. [#1428](https://github.com/PyO3/pyo3/pull/1428) - Fix FFI definition `_PyEval_RequestCodeExtraIndex` which took an argument of the wrong type. [#1429](https://github.com/PyO3/pyo3/pull/1429) -- Fix FFI definition `PyIter_Check` missing for Python 3.7 and earlier with the `abi3` feature. [#1436](https://github.com/PyO3/pyo3/pull/1436) - Fix FFI definition `PyIndex_Check` missing with the `abi3` feature. [#1436](https://github.com/PyO3/pyo3/pull/1436) ## [0.13.2] - 2021-02-12 diff --git a/src/ffi/abstract_.rs b/src/ffi/abstract_.rs index 8bef516f2c1..e4ee4eadfe9 100644 --- a/src/ffi/abstract_.rs +++ b/src/ffi/abstract_.rs @@ -89,7 +89,10 @@ extern "C" { pub fn PyObject_GetIter(arg1: *mut PyObject) -> *mut PyObject; } -#[cfg(not(any(all(Py_3_8, Py_LIMITED_API), PyPy)))] +// Defined as this macro in Python 3.6, 3.7 limited API, but relies on +// non-limited PyTypeObject. Don't expose this since it cannot be used. +#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[inline] pub unsafe fn PyIter_Check(o: *mut PyObject) -> c_int { (match (*crate::ffi::Py_TYPE(o)).tp_iternext { Some(tp_iternext) => { @@ -150,7 +153,9 @@ extern "C" { pub fn PyNumber_Or(o1: *mut PyObject, o2: *mut PyObject) -> *mut PyObject; } -#[cfg(not(any(all(Py_3_8, Py_LIMITED_API), PyPy)))] +// Defined as this macro in Python 3.6, 3.7 limited API, but relies on +// non-limited PyTypeObject. Don't expose this since it cannot be used. +#[cfg(not(any(Py_LIMITED_API, PyPy)))] #[inline] pub unsafe fn PyIndex_Check(o: *mut PyObject) -> c_int { let tp_as_number = (*Py_TYPE(o)).tp_as_number; diff --git a/src/types/iterator.rs b/src/types/iterator.rs index 4c0c2502a73..e9c0a674e6f 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -67,6 +67,8 @@ impl<'p> Iterator for &'p PyIterator { } } +// PyIter_Check does not exist in the limited API until 3.8 +#[cfg(any(not(Py_LIMITED_API), Py_3_8))] impl<'v> PyTryFrom<'v> for PyIterator { fn try_from>(value: V) -> Result<&'v PyIterator, PyDowncastError<'v>> { let value = value.into();