From e4edc6cdd7c63f4a94b830c4c492ea667155a2fe Mon Sep 17 00:00:00 2001 From: Rene Leveille Date: Sat, 8 Aug 2020 17:25:34 -0400 Subject: [PATCH 01/11] Load compilation options from _sysconfigdata_*.py file Following the discussion in #1077 this change allows the compilation script to load the configurations from a _sysconfigdata_ file in the library directory. This file is also provided on target systems in the same directory. At least on Manjaro Linux. Which could remove the need to run a python script at compile time for compiling the the host. I've also addressed the linking need for android in #1082. --- Cargo.toml | 4 ++ build.rs | 149 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 145 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f50846baa62..60298c6295b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,10 @@ assert_approx_eq = "1.1.0" trybuild = "1.0.23" rustversion = "1.0" +[build-dependencies] +walkdir = "~2.3" +regex = "~1.3" + [features] default = ["macros"] macros = ["ctor", "indoc", "inventory", "paste", "pyo3cls", "unindent"] diff --git a/build.rs b/build.rs index 2426fedfd2b..5825a6b93c8 100644 --- a/build.rs +++ b/build.rs @@ -134,10 +134,112 @@ fn fix_config_map(mut config_map: HashMap) -> HashMap Result<(InterpreterConfig, HashMap)> { - let python_include_dir = env::var("PYO3_CROSS_INCLUDE_DIR")?; - let python_include_dir = Path::new(&python_include_dir); +fn parse_sysconfigdata(config_path: impl AsRef) -> Result> { + let config_reader = BufReader::new(File::open(config_path)?); + let mut entries = HashMap::new(); + let entry_re = regex::Regex::new(r#"'([a-zA-Z_0-9]*)': ((?:"|')([\S ]*)(?:"|')|\d+)($|,$)"#)?; + let subsequent_re = regex::Regex::new(r#"\s+(?:"|')([\S ]*)(?:"|')($|,)"#)?; + let mut previous_finished = None; + for maybe_line in config_reader.lines() { + let line = maybe_line?; + if previous_finished.is_none() { + let captures = match entry_re.captures(&line) { + Some(c) => c, + None => continue, + }; + let key = captures[1].to_owned(); + let val = if let Some(val) = captures.get(3) { + val.as_str().to_owned() + } else { + captures[2].to_owned() + }; + if &captures[4] != "," && &captures[4] != "}" { + previous_finished = Some(key.clone()); + } + entries.insert(key, val); + } else if let Some(ref key) = previous_finished { + let captures = match subsequent_re.captures(&line) { + Some(c) => c, + None => continue, + }; + let prev = entries.remove(key).unwrap(); + entries.insert(key.clone(), prev + &captures[1]); + + if &captures[2] == "," || &captures[2] == "}" { + previous_finished = None; + } + } + } + Ok(entries) +} + +fn load_cross_compile_from_sysconfigdata( + python_include_dir: &Path, + python_lib_dir: &str, +) -> Result<(InterpreterConfig, HashMap)> { + // find info from sysconfig + // first find sysconfigdata file + let sysconfig_re = regex::Regex::new(r"_sysconfigdata_m?_linux_([a-z_\-0-9]*)?\.py$")?; + let mut walker = walkdir::WalkDir::new(&python_lib_dir).into_iter(); + let sysconfig_path = loop { + let entry = match walker.next() { + Some(Ok(entry)) => entry, + None => bail!("Could not find sysconfigdata file"), + _ => continue, + }; + let entry = entry.into_path(); + if sysconfig_re.is_match(entry.to_str().unwrap()) { + break entry; + } + }; + let config_map = parse_sysconfigdata(sysconfig_path)?; + + let shared = match config_map + .get("Py_ENABLE_SHARED") + .map(|x| x.as_str()) + .ok_or("Py_ENABLE_SHARED is not defined")? + { + "1" | "true" | "True" => true, + "0" | "false" | "False" => false, + _ => panic!("Py_ENABLE_SHARED must be a bool (1/true/True or 0/false/False"), + }; + + let (major, minor) = match config_map.get("VERSION") { + Some(s) => { + let split = s.split(".").collect::>(); + (split[0].parse::()?, split[1].parse::()?) + } + None => bail!("Could not find python version"), + }; + + let ld_version = match config_map.get("LDVERSION") { + Some(s) => s.clone(), + None => format!("{}.{}", major, minor), + }; + let python_version = PythonVersion { + major, + minor: Some(minor), + implementation: PythonInterpreterKind::CPython, + }; + + let interpreter_config = InterpreterConfig { + version: python_version, + libdir: Some(python_lib_dir.to_owned()), + shared, + ld_version, + base_prefix: "".to_string(), + executable: PathBuf::new(), + calcsize_pointer: None, + }; + + Ok((interpreter_config, fix_config_map(config_map))) +} + +fn load_cross_compile_from_headers( + python_include_dir: &Path, + python_lib_dir: &str, +) -> Result<(InterpreterConfig, HashMap)> { let patchlevel_defines = parse_header_defines(python_include_dir.join("patchlevel.h"))?; let major = match patchlevel_defines @@ -177,9 +279,9 @@ fn load_cross_compile_info() -> Result<(InterpreterConfig, HashMap Result<(InterpreterConfig, HashMap Result<(InterpreterConfig, HashMap)> { + let python_include_dir = Path::new(&python_include_dir); + // Try to configure from the sysconfigdata file which is more accurate for the information + // provided at python's compile time + match load_cross_compile_from_sysconfigdata(python_include_dir, &python_lib_dir) { + Ok(ret) => Ok(ret), + // If the config could not be loaded by sysconfigdata, failover to configuring from headers + Err(_) => load_cross_compile_from_headers(python_include_dir, &python_lib_dir), + } +} + /// Examine python's compile flags to pass to cfg by launching /// the interpreter and printing variables of interest from /// sysconfig.get_config_vars. @@ -567,10 +683,27 @@ fn main() -> Result<()> { // If you have troubles with your shell accepting '.' in a var name, // try using 'env' (sorry but this isn't our fault - it just has to // match the pkg-config package name, which is going to have a . in it). - let cross_compiling = - env::var("PYO3_CROSS_INCLUDE_DIR").is_ok() && env::var("PYO3_CROSS_LIB_DIR").is_ok(); + // + // Detecting if cross-compiling by checking if the target triple is different from the host + // rustc's triple. + let cross_compiling = env::var("TARGET") != env::var("HOST"); let (interpreter_config, mut config_map) = if cross_compiling { - load_cross_compile_info()? + // If cross compiling we need the path to the cross-compiled include dir and lib dir, else + // fail quickly and loudly + let python_include_dir = match env::var("PYO3_CROSS_INCLUDE_DIR") { + Ok(v) => v, + Err(_) => bail!( + "Must provide PYO3_CROSS_INCLUDE_DIR environment variable when cross-compiling" + ), + }; + let python_lib_dir = match env::var("PYO3_CROSS_LIB_DIR") { + Ok(v) => v, + Err(_) => { + bail!("Must provide PYO3_CROSS_LIB_DIR environment variable when cross-compiling") + } + }; + + load_cross_compile_info(python_include_dir, python_lib_dir)? } else { find_interpreter_and_get_config()? }; From 3bc951a5516f8acf8bea7974f2070e0ca9fcf27c Mon Sep 17 00:00:00 2001 From: Rene Leveille Date: Sat, 8 Aug 2020 17:41:16 -0400 Subject: [PATCH 02/11] make clippy happy and add to changelog --- CHANGELOG.md | 2 ++ build.rs | 11 +++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af51ac0da49..75eb50771cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `PyObject` is now just a type alias for `Py`. [#1063](https://github.com/PyO3/pyo3/pull/1063) - Implement `Send + Sync` for `PyErr`. `PyErr::new`, `PyErr::from_type`, `PyException::py_err` and `PyException::into` have had these bounds added to their arguments. [#1067](https://github.com/PyO3/pyo3/pull/1067) - Change `#[pyproto]` to return NotImplemented for operators for which Python can try a reversed operation. [1072](https://github.com/PyO3/pyo3/pull/1072) +- Change method for getting cross-compilation configurations using the file which provides the `sysconfig` module with its data, `_sysconfig_*.py` located in the lib directory. [1077](https://github.com/PyO3/pyo3/issues/1077) ### Removed - Remove `PyString::as_bytes`. [#1023](https://github.com/PyO3/pyo3/pull/1023) @@ -42,6 +43,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Allows `&Self` as a `#[pymethods]` argument again. [#1071](https://github.com/PyO3/pyo3/pull/1071) - Fix best-effort build against PyPy 3.6. #[1092](https://github.com/PyO3/pyo3/pull/1092) - Improve lifetime elision in `#[pyproto]`. [#1093](https://github.com/PyO3/pyo3/pull/1093) +- Linking against libpython when compiling for android even with `extension-module` set[#1082](https://github.com/PyO3/pyo3/issues/1082) ## [0.11.1] - 2020-06-30 ### Added diff --git a/build.rs b/build.rs index 5825a6b93c8..fb268986904 100644 --- a/build.rs +++ b/build.rs @@ -175,7 +175,6 @@ fn parse_sysconfigdata(config_path: impl AsRef) -> Result Result<(InterpreterConfig, HashMap)> { // find info from sysconfig @@ -207,7 +206,7 @@ fn load_cross_compile_from_sysconfigdata( let (major, minor) = match config_map.get("VERSION") { Some(s) => { - let split = s.split(".").collect::>(); + let split = s.split('.').collect::>(); (split[0].parse::()?, split[1].parse::()?) } None => bail!("Could not find python version"), @@ -237,9 +236,10 @@ fn load_cross_compile_from_sysconfigdata( } fn load_cross_compile_from_headers( - python_include_dir: &Path, + python_include_dir: &str, python_lib_dir: &str, ) -> Result<(InterpreterConfig, HashMap)> { + let python_include_dir = Path::new(&python_include_dir); let patchlevel_defines = parse_header_defines(python_include_dir.join("patchlevel.h"))?; let major = match patchlevel_defines @@ -294,13 +294,12 @@ fn load_cross_compile_info( python_include_dir: String, python_lib_dir: String, ) -> Result<(InterpreterConfig, HashMap)> { - let python_include_dir = Path::new(&python_include_dir); // Try to configure from the sysconfigdata file which is more accurate for the information // provided at python's compile time - match load_cross_compile_from_sysconfigdata(python_include_dir, &python_lib_dir) { + match load_cross_compile_from_sysconfigdata(&python_lib_dir) { Ok(ret) => Ok(ret), // If the config could not be loaded by sysconfigdata, failover to configuring from headers - Err(_) => load_cross_compile_from_headers(python_include_dir, &python_lib_dir), + Err(_) => load_cross_compile_from_headers(&python_include_dir, &python_lib_dir), } } From aef159d4ee71c1651827e5578438773cdedf0122 Mon Sep 17 00:00:00 2001 From: Rene Leveille Date: Sat, 8 Aug 2020 22:37:38 -0400 Subject: [PATCH 03/11] appy suggestions from PR --- CHANGELOG.md | 4 +- Cargo.toml | 7 +- build.rs | 186 ++++++++++++++++++++----- guide/src/building_and_distribution.md | 3 +- 4 files changed, 160 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75eb50771cd..9e7bc65f240 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `PyObject` is now just a type alias for `Py`. [#1063](https://github.com/PyO3/pyo3/pull/1063) - Implement `Send + Sync` for `PyErr`. `PyErr::new`, `PyErr::from_type`, `PyException::py_err` and `PyException::into` have had these bounds added to their arguments. [#1067](https://github.com/PyO3/pyo3/pull/1067) - Change `#[pyproto]` to return NotImplemented for operators for which Python can try a reversed operation. [1072](https://github.com/PyO3/pyo3/pull/1072) -- Change method for getting cross-compilation configurations using the file which provides the `sysconfig` module with its data, `_sysconfig_*.py` located in the lib directory. [1077](https://github.com/PyO3/pyo3/issues/1077) ### Removed - Remove `PyString::as_bytes`. [#1023](https://github.com/PyO3/pyo3/pull/1023) @@ -43,7 +42,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Allows `&Self` as a `#[pymethods]` argument again. [#1071](https://github.com/PyO3/pyo3/pull/1071) - Fix best-effort build against PyPy 3.6. #[1092](https://github.com/PyO3/pyo3/pull/1092) - Improve lifetime elision in `#[pyproto]`. [#1093](https://github.com/PyO3/pyo3/pull/1093) -- Linking against libpython when compiling for android even with `extension-module` set[#1082](https://github.com/PyO3/pyo3/issues/1082) +- Fix python configuration detection when cross-compiling. [1077](https://github.com/PyO3/pyo3/issues/1077) +- Link against libpython on android with `extension-module` set. [#1082](https://github.com/PyO3/pyo3/issues/1082) ## [0.11.1] - 2020-06-30 ### Added diff --git a/Cargo.toml b/Cargo.toml index 60298c6295b..b283ab13d76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,8 +32,8 @@ trybuild = "1.0.23" rustversion = "1.0" [build-dependencies] -walkdir = "~2.3" -regex = "~1.3" +walkdir = {version = "~2.3", optional = true } +regex = { version = "~1.3", optional = true } [features] default = ["macros"] @@ -54,6 +54,9 @@ extension-module = [] # are welcome. # abi3 = [] +# Use this feature when cross-compiling the library +cross-compile = ["walkdir", "regex"] + [workspace] members = [ "pyo3cls", diff --git a/build.rs b/build.rs index fb268986904..e7b0677726b 100644 --- a/build.rs +++ b/build.rs @@ -2,7 +2,7 @@ use std::{ collections::HashMap, convert::AsRef, env, fmt, - fs::File, + fs::{self, File}, io::{self, BufRead, BufReader}, path::{Path, PathBuf}, process::{Command, Stdio}, @@ -76,6 +76,57 @@ impl FromStr for PythonInterpreterKind { } } +struct PythonPaths { + lib_dir: String, + include_dir: Option, +} + +impl PythonPaths { + fn both() -> Result { + Ok(PythonPaths { + include_dir: Some(PythonPaths::validate_variable("PYO3_CROSS_INCLUDE_DIR")?), + ..PythonPaths::lib_only()? + }) + } + + fn lib_only() -> Result { + Ok(PythonPaths { + lib_dir: PythonPaths::validate_variable("PYO3_CROSS_LIB_DIR")?, + include_dir: None, + }) + } + + fn validate_variable(var: &str) -> Result { + let path = match env::var(var) { + Ok(v) => v, + Err(_) => bail!( + "Must provide {} environment variable when cross-compiling", + var + ), + }; + + if fs::metadata(&path).is_err() { + bail!("{} value of {} does not exist", var, path) + } + + Ok(path) + } +} + +fn cross_compiling() -> Result> { + if env::var("TARGET")? == env::var("HOST")? { + return Ok(None); + } + + if env::var("CARGO_CFG_TARGET_FAMILY")? == "windows" { + Ok(Some(PythonPaths::both()?)) + } else if cfg!(feature = "cross-compile") { + Ok(Some(PythonPaths::lib_only()?)) + } else { + bail!("Cross compiling PyO3 for a unix platform requires the cross-compile feature to be enabled") + } +} + /// A list of python interpreter compile-time preprocessor defines that /// we will pick up and pass to rustc via --cfg=py_sys_config={varname}; /// this allows using them conditional cfg attributes in the .rs files, so @@ -134,11 +185,57 @@ fn fix_config_map(mut config_map: HashMap) -> HashMap) -> Result> { let config_reader = BufReader::new(File::open(config_path)?); let mut entries = HashMap::new(); - let entry_re = regex::Regex::new(r#"'([a-zA-Z_0-9]*)': ((?:"|')([\S ]*)(?:"|')|\d+)($|,$)"#)?; - let subsequent_re = regex::Regex::new(r#"\s+(?:"|')([\S ]*)(?:"|')($|,)"#)?; + + let entry_re = regex::Regex::new(r#"'([a-zA-Z_0-9]*)': ((?:"|')([\S ]*)(?:"|')|\d+)($|,|})"#)?; + let subsequent_re = regex::Regex::new(r#"\s+(?:"|')([\S ]*)(?:"|')($|,|})"#)?; let mut previous_finished = None; for maybe_line in config_reader.lines() { let line = maybe_line?; @@ -174,13 +271,35 @@ fn parse_sysconfigdata(config_path: impl AsRef) -> Result Result<(InterpreterConfig, HashMap)> { - // find info from sysconfig - // first find sysconfigdata file - let sysconfig_re = regex::Regex::new(r"_sysconfigdata_m?_linux_([a-z_\-0-9]*)?\.py$")?; - let mut walker = walkdir::WalkDir::new(&python_lib_dir).into_iter(); + let sysconfig_re = + regex::Regex::new(r"_sysconfigdata_(?:u|d|m|)_[a-z0-9]+_([a-z_\-0-9]*)?\.py$")?; + let mut walker = walkdir::WalkDir::new(&python_paths.lib_dir).into_iter(); let sysconfig_path = loop { let entry = match walker.next() { Some(Ok(entry)) => entry, @@ -224,7 +343,7 @@ fn load_cross_compile_from_sysconfigdata( let interpreter_config = InterpreterConfig { version: python_version, - libdir: Some(python_lib_dir.to_owned()), + libdir: Some(python_paths.lib_dir.to_owned()), shared, ld_version, base_prefix: "".to_string(), @@ -236,9 +355,9 @@ fn load_cross_compile_from_sysconfigdata( } fn load_cross_compile_from_headers( - python_include_dir: &str, - python_lib_dir: &str, + python_paths: PythonPaths, ) -> Result<(InterpreterConfig, HashMap)> { + let python_include_dir = python_paths.include_dir.unwrap(); let python_include_dir = Path::new(&python_include_dir); let patchlevel_defines = parse_header_defines(python_include_dir.join("patchlevel.h"))?; @@ -279,7 +398,7 @@ fn load_cross_compile_from_headers( let interpreter_config = InterpreterConfig { version: python_version, - libdir: Some(python_lib_dir.to_owned()), + libdir: Some(python_paths.lib_dir.to_owned()), shared, ld_version: format!("{}.{}", major, minor), base_prefix: "".to_string(), @@ -290,17 +409,27 @@ fn load_cross_compile_from_headers( Ok((interpreter_config, fix_config_map(config_map))) } +#[allow(unused_variables)] fn load_cross_compile_info( - python_include_dir: String, - python_lib_dir: String, + python_paths: PythonPaths, ) -> Result<(InterpreterConfig, HashMap)> { - // Try to configure from the sysconfigdata file which is more accurate for the information - // provided at python's compile time - match load_cross_compile_from_sysconfigdata(&python_lib_dir) { - Ok(ret) => Ok(ret), - // If the config could not be loaded by sysconfigdata, failover to configuring from headers - Err(_) => load_cross_compile_from_headers(&python_include_dir, &python_lib_dir), + let target_family = env::var("CARGO_CFG_TARGET_FAMILY")?; + // Because compiling for windows on linux still includes the unix target family + if target_family == "unix" && cfg!(feature = "cross-compile") { + // Configure for unix platforms using the sysconfigdata file + #[cfg(feature = "cross-compile")] + { + return load_cross_compile_from_sysconfigdata(python_paths); + } + } else if target_family == "windows" { + // Must configure by headers on windows platform + return load_cross_compile_from_headers(python_paths); } + + // If you get here you were on unix without cross-compile capabilities + bail!( + "Cross compiling PyO3 for a unix platform requires the cross-compile feature to be enabled" + ); } /// Examine python's compile flags to pass to cfg by launching @@ -685,24 +814,11 @@ fn main() -> Result<()> { // // Detecting if cross-compiling by checking if the target triple is different from the host // rustc's triple. - let cross_compiling = env::var("TARGET") != env::var("HOST"); - let (interpreter_config, mut config_map) = if cross_compiling { + let (interpreter_config, mut config_map) = if let Some(paths) = cross_compiling()? { // If cross compiling we need the path to the cross-compiled include dir and lib dir, else // fail quickly and loudly - let python_include_dir = match env::var("PYO3_CROSS_INCLUDE_DIR") { - Ok(v) => v, - Err(_) => bail!( - "Must provide PYO3_CROSS_INCLUDE_DIR environment variable when cross-compiling" - ), - }; - let python_lib_dir = match env::var("PYO3_CROSS_LIB_DIR") { - Ok(v) => v, - Err(_) => { - bail!("Must provide PYO3_CROSS_LIB_DIR environment variable when cross-compiling") - } - }; - load_cross_compile_info(python_include_dir, python_lib_dir)? + load_cross_compile_info(paths)? } else { find_interpreter_and_get_config()? }; diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index 7c36fd3822f..d8d9c2798fa 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -51,8 +51,9 @@ See https://github.com/japaric/rust-cross for a primer on cross compiling Rust i After you've obtained the above, you can build a cross compiled PyO3 module by setting a few extra environment variables: -* `PYO3_CROSS_INCLUDE_DIR`: This variable must be set to the directory containing the headers for the target's Python interpreter. +* `PYO3_CROSS_INCLUDE_DIR`: This variable must be set to the directory containing the headers for the target's Python interpreter. It is only necessary if compiling for Windows platforms * `PYO3_CROSS_LIB_DIR`: This variable must be set to the directory containing the target's libpython DSO. +* If compiling for unix platforms, the `cross-compile` feature must be set. An example might look like the following (assuming your target's sysroot is at `/home/pyo3/cross/sysroot` and that your target is `armv7`): From afac788c1c662d63f7db4ce52eb518752e866ddc Mon Sep 17 00:00:00 2001 From: Rene Leveille Date: Mon, 17 Aug 2020 15:12:12 -0400 Subject: [PATCH 04/11] fix complexity of finding and parsing --- Cargo.toml | 7 -- build.rs | 283 ++++++++++++++++++++++------------------------------- 2 files changed, 117 insertions(+), 173 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b283ab13d76..f50846baa62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,10 +31,6 @@ assert_approx_eq = "1.1.0" trybuild = "1.0.23" rustversion = "1.0" -[build-dependencies] -walkdir = {version = "~2.3", optional = true } -regex = { version = "~1.3", optional = true } - [features] default = ["macros"] macros = ["ctor", "indoc", "inventory", "paste", "pyo3cls", "unindent"] @@ -54,9 +50,6 @@ extension-module = [] # are welcome. # abi3 = [] -# Use this feature when cross-compiling the library -cross-compile = ["walkdir", "regex"] - [workspace] members = [ "pyo3cls", diff --git a/build.rs b/build.rs index e7b0677726b..9378d0ce958 100644 --- a/build.rs +++ b/build.rs @@ -2,7 +2,7 @@ use std::{ collections::HashMap, convert::AsRef, env, fmt, - fs::{self, File}, + fs::{self, DirEntry, File}, io::{self, BufRead, BufReader}, path::{Path, PathBuf}, process::{Command, Stdio}, @@ -76,23 +76,27 @@ impl FromStr for PythonInterpreterKind { } } -struct PythonPaths { +struct CrossPython { lib_dir: String, include_dir: Option, + os: String, + arch: String, } -impl PythonPaths { +impl CrossPython { fn both() -> Result { - Ok(PythonPaths { - include_dir: Some(PythonPaths::validate_variable("PYO3_CROSS_INCLUDE_DIR")?), - ..PythonPaths::lib_only()? + Ok(CrossPython { + include_dir: Some(CrossPython::validate_variable("PYO3_CROSS_INCLUDE_DIR")?), + ..CrossPython::lib_only()? }) } fn lib_only() -> Result { - Ok(PythonPaths { - lib_dir: PythonPaths::validate_variable("PYO3_CROSS_LIB_DIR")?, + Ok(CrossPython { + lib_dir: CrossPython::validate_variable("PYO3_CROSS_LIB_DIR")?, include_dir: None, + os: env::var("CARGO_CFG_TARGET_OS").unwrap(), + arch: env::var("CARGO_CFG_TARGET_ARCH").unwrap(), }) } @@ -113,17 +117,15 @@ impl PythonPaths { } } -fn cross_compiling() -> Result> { +fn cross_compiling() -> Result> { if env::var("TARGET")? == env::var("HOST")? { return Ok(None); } if env::var("CARGO_CFG_TARGET_FAMILY")? == "windows" { - Ok(Some(PythonPaths::both()?)) - } else if cfg!(feature = "cross-compile") { - Ok(Some(PythonPaths::lib_only()?)) + Ok(Some(CrossPython::both()?)) } else { - bail!("Cross compiling PyO3 for a unix platform requires the cross-compile feature to be enabled") + Ok(Some(CrossPython::lib_only()?)) } } @@ -185,6 +187,36 @@ fn fix_config_map(mut config_map: HashMap) -> HashMap HashMap { + output + .lines() + .filter_map(|line| { + let mut i = line.splitn(2, ' '); + Some((i.next()?.into(), i.next()?.into())) + }) + .collect() +} + +fn as_bool(config: &HashMap, key: &str) -> Result { + match config + .get(key) + .map(|x| x.as_str()) + .ok_or(format!("{} is not defined", key))? + { + "1" | "true" | "True" => Ok(true), + "0" | "false" | "False" => Ok(false), + _ => Err(format!("{} must be a bool (1/true/True or 0/false/False", key).into()), + } +} + +fn as_numeric(config: &HashMap, key: &str) -> Result { + config + .get(key) + .ok_or(format!("{} is not defined", key))? + .parse::() + .map_err(|_| format!("Could not parse value of {}", key).into()) +} + /// Parse sysconfigdata file /// /// The sysconfigdata is basically a dictionary, and since we can't really use this library to read @@ -193,148 +225,89 @@ fn fix_config_map(mut config_map: HashMap) -> HashMap) -> Result> { - let config_reader = BufReader::new(File::open(config_path)?); - let mut entries = HashMap::new(); + let mut script = fs::read_to_string(config_path)?; + script += r#" +print("version_major", build_time_vars["VERSION"][0]) # 3 +print("version_minor", build_time_vars["VERSION"][2]) # E.g., 8 +if "WITH_THREAD" in build_time_vars: + print("WITH_THREAD", build_time_vars["WITH_THREAD"]) +if "Py_TRACE_REFS" in build_time_vars: + print("Py_TRACE_REFS", build_time_vars["Py_TRACE_REFS"]) +if "COUNT_ALLOCS" in build_time_vars: + print("COUNT_ALLOCS", build_time_vars["COUNT_ALLOCS"]) +if "Py_REF_DEBUG" in build_time_vars: + print("Py_REF_DEBUG", build_time_vars["Py_REF_DEBUG"]) +print("Py_DEBUG", build_time_vars["Py_DEBUG"]) +print("Py_ENABLE_SHARED", build_time_vars["Py_ENABLE_SHARED"]) +print("LDVERSION", build_time_vars["LDVERSION"]) +print("SIZEOF_VOID_P", build_time_vars["SIZEOF_VOID_P"]) +"#; + let output = run_python_script(&find_interpreter()?, &script)?; - let entry_re = regex::Regex::new(r#"'([a-zA-Z_0-9]*)': ((?:"|')([\S ]*)(?:"|')|\d+)($|,|})"#)?; - let subsequent_re = regex::Regex::new(r#"\s+(?:"|')([\S ]*)(?:"|')($|,|})"#)?; - let mut previous_finished = None; - for maybe_line in config_reader.lines() { - let line = maybe_line?; - if previous_finished.is_none() { - let captures = match entry_re.captures(&line) { - Some(c) => c, - None => continue, - }; - let key = captures[1].to_owned(); - let val = if let Some(val) = captures.get(3) { - val.as_str().to_owned() - } else { - captures[2].to_owned() - }; - if &captures[4] != "," && &captures[4] != "}" { - previous_finished = Some(key.clone()); - } - entries.insert(key, val); - } else if let Some(ref key) = previous_finished { - let captures = match subsequent_re.captures(&line) { - Some(c) => c, - None => continue, - }; - let prev = entries.remove(key).unwrap(); - entries.insert(key.clone(), prev + &captures[1]); - - if &captures[2] == "," || &captures[2] == "}" { - previous_finished = None; + Ok(parse_script_output(&output)) +} + +fn starts_with(entry: &DirEntry, pat: &str) -> bool { + let name = entry.file_name(); + name.to_string_lossy().starts_with(pat) +} +fn ends_with(entry: &DirEntry, pat: &str) -> bool { + let name = entry.file_name(); + name.to_string_lossy().ends_with(pat) +} + +fn find_sysconfigdata(path: impl AsRef, cross: &CrossPython) -> Option { + for f in fs::read_dir(path).expect("Path does not exist") { + return match f { + Ok(ref f) if starts_with(f, "_sysconfigdata") && ends_with(f, "py") => Some(f.path()), + Ok(ref f) if starts_with(f, "build") => find_sysconfigdata(f.path(), cross), + Ok(ref f) if starts_with(f, "lib.") => { + let name = f.file_name(); + // check if right target os + if !name.to_string_lossy().contains(if cross.os == "android" { + "linux" + } else { + &cross.os + }) { + println!("{:?}", f); + continue; + } + // Check if right arch + if !name.to_string_lossy().contains(&cross.arch) { + println!("{:?}", f); + continue; + } + find_sysconfigdata(f.path(), cross) } - } + _ => continue, + }; } - - Ok(entries) + None } /// Find cross compilation information from sysconfigdata file /// /// first find sysconfigdata file which follows the pattern [`_sysconfigdata_{abi}_{platform}_{multiarch}`][1] -/// -/// The ABI flags can be either u, d, or m according to PEP3148. Default flags became empty since -/// m was removed in python 3.8. (?:u|d|m|) a non capturing group for these flags -/// -/// platform follows the output from [sys.platform][2] -/// [a-z0-9]+ -/// -/// Multi-arch is the target triple. ([a-z_\-0-9]*)? capturing group which might not be present. -/// -/// # Examples -/// ```txt -/// _sysconfigdata__freebsd_.py -/// _sysconfigdata_m_linux_x86_64-linux-gnu.py -/// _sysconfigdata_d_darwin_x86_64-apple-darwin.py -/// _sysconfigdata_u_windows_i686-pc-windows-gnu.py -/// ``` +/// on python 3.6 or greater. On python 3.5 it is simply `_sysconfigdata.py`. /// /// [1]: https://github.com/python/cpython/blob/3.8/Lib/sysconfig.py#L348 -/// [2]: https://docs.python.org/3/library/sys.html#sys.platform -#[cfg(feature = "cross-compile")] fn load_cross_compile_from_sysconfigdata( - python_paths: PythonPaths, + python_paths: CrossPython, ) -> Result<(InterpreterConfig, HashMap)> { - let sysconfig_re = - regex::Regex::new(r"_sysconfigdata_(?:u|d|m|)_[a-z0-9]+_([a-z_\-0-9]*)?\.py$")?; - let mut walker = walkdir::WalkDir::new(&python_paths.lib_dir).into_iter(); - let sysconfig_path = loop { - let entry = match walker.next() { - Some(Ok(entry)) => entry, - None => bail!("Could not find sysconfigdata file"), - _ => continue, - }; - let entry = entry.into_path(); - if sysconfig_re.is_match(entry.to_str().unwrap()) { - break entry; - } - }; + let sysconfig_path = find_sysconfigdata(&python_paths.lib_dir, &python_paths) + .expect("_sysconfigdata*.py not found"); let config_map = parse_sysconfigdata(sysconfig_path)?; - let shared = match config_map - .get("Py_ENABLE_SHARED") - .map(|x| x.as_str()) - .ok_or("Py_ENABLE_SHARED is not defined")? - { - "1" | "true" | "True" => true, - "0" | "false" | "False" => false, - _ => panic!("Py_ENABLE_SHARED must be a bool (1/true/True or 0/false/False"), - }; - - let (major, minor) = match config_map.get("VERSION") { - Some(s) => { - let split = s.split('.').collect::>(); - (split[0].parse::()?, split[1].parse::()?) - } - None => bail!("Could not find python version"), - }; - + let shared = as_bool(&config_map, "Py_ENABLE_SHARED")?; + let major = as_numeric(&config_map, "version_major")?; + let minor = as_numeric(&config_map, "version_minor")?; let ld_version = match config_map.get("LDVERSION") { Some(s) => s.clone(), None => format!("{}.{}", major, minor), }; + let calcsize_pointer = as_numeric(&config_map, "SIZEOF_VOID_P").ok(); + let python_version = PythonVersion { major, minor: Some(minor), @@ -348,14 +321,14 @@ fn load_cross_compile_from_sysconfigdata( ld_version, base_prefix: "".to_string(), executable: PathBuf::new(), - calcsize_pointer: None, + calcsize_pointer, }; Ok((interpreter_config, fix_config_map(config_map))) } fn load_cross_compile_from_headers( - python_paths: PythonPaths, + python_paths: CrossPython, ) -> Result<(InterpreterConfig, HashMap)> { let python_include_dir = python_paths.include_dir.unwrap(); let python_include_dir = Path::new(&python_include_dir); @@ -386,15 +359,7 @@ fn load_cross_compile_from_headers( }; let config_map = parse_header_defines(python_include_dir.join("pyconfig.h"))?; - let shared = match config_map - .get("Py_ENABLE_SHARED") - .map(|x| x.as_str()) - .ok_or("Py_ENABLE_SHARED is not defined")? - { - "1" | "true" | "True" => true, - "0" | "false" | "False" => false, - _ => panic!("Py_ENABLE_SHARED must be a bool (1/true/True or 0/false/False"), - }; + let shared = as_bool(&config_map, "Py_ENABLE_SHARED")?; let interpreter_config = InterpreterConfig { version: python_version, @@ -411,25 +376,17 @@ fn load_cross_compile_from_headers( #[allow(unused_variables)] fn load_cross_compile_info( - python_paths: PythonPaths, + python_paths: CrossPython, ) -> Result<(InterpreterConfig, HashMap)> { let target_family = env::var("CARGO_CFG_TARGET_FAMILY")?; // Because compiling for windows on linux still includes the unix target family - if target_family == "unix" && cfg!(feature = "cross-compile") { + if target_family == "unix" { // Configure for unix platforms using the sysconfigdata file - #[cfg(feature = "cross-compile")] - { - return load_cross_compile_from_sysconfigdata(python_paths); - } - } else if target_family == "windows" { + return load_cross_compile_from_sysconfigdata(python_paths); + } else { // Must configure by headers on windows platform return load_cross_compile_from_headers(python_paths); } - - // If you get here you were on unix without cross-compile capabilities - bail!( - "Cross compiling PyO3 for a unix platform requires the cross-compile feature to be enabled" - ); } /// Examine python's compile flags to pass to cfg by launching @@ -690,13 +647,7 @@ print("executable", sys.executable) print("calcsize_pointer", struct.calcsize("P")) "#; let output = run_python_script(interpreter, script)?; - let map: HashMap = output - .lines() - .filter_map(|line| { - let mut i = line.splitn(2, ' '); - Some((i.next()?.into(), i.next()?.into())) - }) - .collect(); + let map: HashMap = parse_script_output(&output); Ok(InterpreterConfig { version: PythonVersion { major: map["version_major"].parse()?, From 0cd1e5bf19acdb1aefc64a5a0494e31179f804bb Mon Sep 17 00:00:00 2001 From: Rene Leveille Date: Mon, 17 Aug 2020 15:17:34 -0400 Subject: [PATCH 05/11] change documentation --- guide/src/building_and_distribution.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index d8d9c2798fa..5c37f47559a 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -45,21 +45,28 @@ Cross compiling PyO3 modules is relatively straightforward and requires a few pi * A toolchain for your target. * The appropriate options in your Cargo `.config` for the platform you're targeting and the toolchain you are using. * A Python interpreter that's already been compiled for your target. +* A Python interpreter that is built for your host and available through the `PATH`. * The headers that match the above interpreter. See https://github.com/japaric/rust-cross for a primer on cross compiling Rust in general. After you've obtained the above, you can build a cross compiled PyO3 module by setting a few extra environment variables: -* `PYO3_CROSS_INCLUDE_DIR`: This variable must be set to the directory containing the headers for the target's Python interpreter. It is only necessary if compiling for Windows platforms +* `PYO3_CROSS_INCLUDE_DIR`: This variable must be set to the directory containing the headers for the target's Python interpreter. **It is only necessary if targeting Windows platforms** * `PYO3_CROSS_LIB_DIR`: This variable must be set to the directory containing the target's libpython DSO. -* If compiling for unix platforms, the `cross-compile` feature must be set. An example might look like the following (assuming your target's sysroot is at `/home/pyo3/cross/sysroot` and that your target is `armv7`): ```sh -export PYO3_CROSS_INCLUDE_DIR="/home/pyo3/cross/sysroot/usr/include" export PYO3_CROSS_LIB_DIR="/home/pyo3/cross/sysroot/usr/lib" cargo build --target armv7-unknown-linux-gnueabihf ``` + +Or another example with the same sys root but building for windows: +```sh +export PYO3_CROSS_INCLUDE_DIR="/home/pyo3/cross/sysroot/usr/include" +export PYO3_CROSS_LIB_DIR="/home/pyo3/cross/sysroot/usr/lib" + +cargo build --target x86_64-pc-windows-gnu +``` \ No newline at end of file From 695be35289785580898b534f2b0c843bb51c4a54 Mon Sep 17 00:00:00 2001 From: Rene Leveille Date: Mon, 17 Aug 2020 15:23:24 -0400 Subject: [PATCH 06/11] make clippy happy --- build.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/build.rs b/build.rs index 9378d0ce958..f4959272ff2 100644 --- a/build.rs +++ b/build.rs @@ -270,12 +270,10 @@ fn find_sysconfigdata(path: impl AsRef, cross: &CrossPython) -> Option Date: Mon, 17 Aug 2020 18:14:05 -0400 Subject: [PATCH 07/11] apply suggestions from review --- build.rs | 126 ++++++++++++------------- guide/src/building_and_distribution.md | 2 +- 2 files changed, 60 insertions(+), 68 deletions(-) diff --git a/build.rs b/build.rs index f4959272ff2..b9c6e7cfaa0 100644 --- a/build.rs +++ b/build.rs @@ -76,56 +76,84 @@ impl FromStr for PythonInterpreterKind { } } -struct CrossPython { - lib_dir: String, - include_dir: Option, +trait GetPrimitive { + fn get_bool(&self, key: &str) -> Result; + fn get_numeric(&self, key: &str) -> Result; +} + +impl GetPrimitive for HashMap { + fn get_bool(&self, key: &str) -> Result { + match self + .get(key) + .map(|x| x.as_str()) + .ok_or(format!("{} is not defined", key))? + { + "1" | "true" | "True" => Ok(true), + "0" | "false" | "False" => Ok(false), + _ => Err(format!("{} must be a bool (1/true/True or 0/false/False", key).into()), + } + } + + fn get_numeric(&self, key: &str) -> Result { + self.get(key) + .ok_or(format!("{} is not defined", key))? + .parse::() + .map_err(|_| format!("Could not parse value of {}", key).into()) + } +} + +struct CrossCompileConfig { + lib_dir: PathBuf, + include_dir: Option, os: String, arch: String, } -impl CrossPython { +impl CrossCompileConfig { fn both() -> Result { - Ok(CrossPython { - include_dir: Some(CrossPython::validate_variable("PYO3_CROSS_INCLUDE_DIR")?), - ..CrossPython::lib_only()? + Ok(CrossCompileConfig { + include_dir: Some(CrossCompileConfig::validate_variable( + "PYO3_CROSS_INCLUDE_DIR", + )?), + ..CrossCompileConfig::lib_only()? }) } fn lib_only() -> Result { - Ok(CrossPython { - lib_dir: CrossPython::validate_variable("PYO3_CROSS_LIB_DIR")?, + Ok(CrossCompileConfig { + lib_dir: CrossCompileConfig::validate_variable("PYO3_CROSS_LIB_DIR")?, include_dir: None, os: env::var("CARGO_CFG_TARGET_OS").unwrap(), arch: env::var("CARGO_CFG_TARGET_ARCH").unwrap(), }) } - fn validate_variable(var: &str) -> Result { - let path = match env::var(var) { - Ok(v) => v, - Err(_) => bail!( + fn validate_variable(var: &str) -> Result { + let path = match env::var_os(var) { + Some(v) => v, + None => bail!( "Must provide {} environment variable when cross-compiling", var ), }; if fs::metadata(&path).is_err() { - bail!("{} value of {} does not exist", var, path) + bail!("{} value of {:?} does not exist", var, path) } - Ok(path) + Ok(path.into()) } } -fn cross_compiling() -> Result> { +fn cross_compiling() -> Result> { if env::var("TARGET")? == env::var("HOST")? { return Ok(None); } if env::var("CARGO_CFG_TARGET_FAMILY")? == "windows" { - Ok(Some(CrossPython::both()?)) + Ok(Some(CrossCompileConfig::both()?)) } else { - Ok(Some(CrossPython::lib_only()?)) + Ok(Some(CrossCompileConfig::lib_only()?)) } } @@ -197,26 +225,6 @@ fn parse_script_output(output: &str) -> HashMap { .collect() } -fn as_bool(config: &HashMap, key: &str) -> Result { - match config - .get(key) - .map(|x| x.as_str()) - .ok_or(format!("{} is not defined", key))? - { - "1" | "true" | "True" => Ok(true), - "0" | "false" | "False" => Ok(false), - _ => Err(format!("{} must be a bool (1/true/True or 0/false/False", key).into()), - } -} - -fn as_numeric(config: &HashMap, key: &str) -> Result { - config - .get(key) - .ok_or(format!("{} is not defined", key))? - .parse::() - .map_err(|_| format!("Could not parse value of {}", key).into()) -} - /// Parse sysconfigdata file /// /// The sysconfigdata is basically a dictionary, and since we can't really use this library to read @@ -257,7 +265,7 @@ fn ends_with(entry: &DirEntry, pat: &str) -> bool { name.to_string_lossy().ends_with(pat) } -fn find_sysconfigdata(path: impl AsRef, cross: &CrossPython) -> Option { +fn find_sysconfigdata(path: impl AsRef, cross: &CrossCompileConfig) -> Option { for f in fs::read_dir(path).expect("Path does not exist") { return match f { Ok(ref f) if starts_with(f, "_sysconfigdata") && ends_with(f, "py") => Some(f.path()), @@ -291,20 +299,20 @@ fn find_sysconfigdata(path: impl AsRef, cross: &CrossPython) -> Option Result<(InterpreterConfig, HashMap)> { let sysconfig_path = find_sysconfigdata(&python_paths.lib_dir, &python_paths) .expect("_sysconfigdata*.py not found"); let config_map = parse_sysconfigdata(sysconfig_path)?; - let shared = as_bool(&config_map, "Py_ENABLE_SHARED")?; - let major = as_numeric(&config_map, "version_major")?; - let minor = as_numeric(&config_map, "version_minor")?; + let shared = config_map.get_bool("Py_ENABLE_SHARED")?; + let major = config_map.get_numeric("version_major")?; + let minor = config_map.get_numeric("version_minor")?; let ld_version = match config_map.get("LDVERSION") { Some(s) => s.clone(), None => format!("{}.{}", major, minor), }; - let calcsize_pointer = as_numeric(&config_map, "SIZEOF_VOID_P").ok(); + let calcsize_pointer = config_map.get_numeric("SIZEOF_VOID_P").ok(); let python_version = PythonVersion { major, @@ -314,7 +322,7 @@ fn load_cross_compile_from_sysconfigdata( let interpreter_config = InterpreterConfig { version: python_version, - libdir: Some(python_paths.lib_dir), + libdir: python_paths.lib_dir.to_str().map(String::from), shared, ld_version, base_prefix: "".to_string(), @@ -326,29 +334,14 @@ fn load_cross_compile_from_sysconfigdata( } fn load_cross_compile_from_headers( - python_paths: CrossPython, + python_paths: CrossCompileConfig, ) -> Result<(InterpreterConfig, HashMap)> { let python_include_dir = python_paths.include_dir.unwrap(); let python_include_dir = Path::new(&python_include_dir); let patchlevel_defines = parse_header_defines(python_include_dir.join("patchlevel.h"))?; - let major = match patchlevel_defines - .get("PY_MAJOR_VERSION") - .map(|major| major.parse::()) - { - Some(Ok(major)) => major, - Some(Err(e)) => bail!("Failed to parse PY_MAJOR_VERSION: {}", e), - None => bail!("PY_MAJOR_VERSION undefined"), - }; - - let minor = match patchlevel_defines - .get("PY_MINOR_VERSION") - .map(|minor| minor.parse::()) - { - Some(Ok(minor)) => minor, - Some(Err(e)) => bail!("Failed to parse PY_MINOR_VERSION: {}", e), - None => bail!("PY_MINOR_VERSION undefined"), - }; + let major = patchlevel_defines.get_numeric("PY_MAJOR_VERSION")?; + let minor = patchlevel_defines.get_numeric("PY_MINOR_VERSION")?; let python_version = PythonVersion { major, @@ -357,11 +350,11 @@ fn load_cross_compile_from_headers( }; let config_map = parse_header_defines(python_include_dir.join("pyconfig.h"))?; - let shared = as_bool(&config_map, "Py_ENABLE_SHARED")?; + let shared = config_map.get_bool("Py_ENABLE_SHARED")?; let interpreter_config = InterpreterConfig { version: python_version, - libdir: Some(python_paths.lib_dir), + libdir: python_paths.lib_dir.to_str().map(String::from), shared, ld_version: format!("{}.{}", major, minor), base_prefix: "".to_string(), @@ -372,9 +365,8 @@ fn load_cross_compile_from_headers( Ok((interpreter_config, fix_config_map(config_map))) } -#[allow(unused_variables)] fn load_cross_compile_info( - python_paths: CrossPython, + python_paths: CrossCompileConfig, ) -> Result<(InterpreterConfig, HashMap)> { let target_family = env::var("CARGO_CFG_TARGET_FAMILY")?; // Because compiling for windows on linux still includes the unix target family diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index 5c37f47559a..f0dca9ccea3 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -45,7 +45,7 @@ Cross compiling PyO3 modules is relatively straightforward and requires a few pi * A toolchain for your target. * The appropriate options in your Cargo `.config` for the platform you're targeting and the toolchain you are using. * A Python interpreter that's already been compiled for your target. -* A Python interpreter that is built for your host and available through the `PATH`. +* A Python interpreter that is built for your host and available through the `PATH` or setting the [`PYO3_PYTHON`](#python-version) variable. * The headers that match the above interpreter. See https://github.com/japaric/rust-cross for a primer on cross compiling Rust in general. From 6ddae70a2b3cc8fcf9b3a0e7bfefe0bc5fa2026c Mon Sep 17 00:00:00 2001 From: Rene Leveille Date: Tue, 18 Aug 2020 13:23:48 -0400 Subject: [PATCH 08/11] fix changelog and added comment for finding sysconfigdata --- CHANGELOG.md | 4 ++-- build.rs | 43 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e7bc65f240..661fcdb2659 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,8 +42,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Allows `&Self` as a `#[pymethods]` argument again. [#1071](https://github.com/PyO3/pyo3/pull/1071) - Fix best-effort build against PyPy 3.6. #[1092](https://github.com/PyO3/pyo3/pull/1092) - Improve lifetime elision in `#[pyproto]`. [#1093](https://github.com/PyO3/pyo3/pull/1093) -- Fix python configuration detection when cross-compiling. [1077](https://github.com/PyO3/pyo3/issues/1077) -- Link against libpython on android with `extension-module` set. [#1082](https://github.com/PyO3/pyo3/issues/1082) +- Fix python configuration detection when cross-compiling. [#1095](https://github.com/PyO3/pyo3/pull/1095) +- Link against libpython on android with `extension-module` set. [#1095](https://github.com/PyO3/pyo3/pull/1095) ## [0.11.1] - 2020-06-30 ### Added diff --git a/build.rs b/build.rs index b9c6e7cfaa0..900a25e1a24 100644 --- a/build.rs +++ b/build.rs @@ -227,12 +227,9 @@ fn parse_script_output(output: &str) -> HashMap { /// Parse sysconfigdata file /// -/// The sysconfigdata is basically a dictionary, and since we can't really use this library to read -/// it for us, egg and chicken type thing, we parse it with some regex. Here there are two regex's. -/// The first one is to match an entry in the dictionary (`entry_re`). Then when the entry is on -/// multiple lines we use the second regex to capture the additional string entry and add it to the -/// previously captured key. We detect if this is a multi line entry by checking if the last capture -/// group contains either `,` or `}`. +/// The sysconfigdata is simply a dictionary containing all the build time variables used for the +/// python executable and library. Here it is read and added to a script to extract only what is +/// necessary. This necessitates a python interpreter for the host machine to work. fn parse_sysconfigdata(config_path: impl AsRef) -> Result> { let mut script = fs::read_to_string(config_path)?; script += r#" @@ -265,6 +262,37 @@ fn ends_with(entry: &DirEntry, pat: &str) -> bool { name.to_string_lossy().ends_with(pat) } +/// Finds the `_sysconfigdata*.py` file in the library path +/// +/// From the python source this file is always going to be located at `build/lib.{PLATFORM}-{PY_MINOR_VERSION}` +/// when built from source. The [exact line][1] is defined as: +/// +/// ```py +/// pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version_info[:2]) +/// ``` +/// +/// Where get_platform returns a kebab-case formated string containing the os, the architecture and +/// possibly the os' kernel version (not the case on linux). However, when installed using a package +/// manager, the `_sysconfigdata*.py` file is installed in the `${PREFIX}/lib/python3.Y/` directory. +/// The `_sysconfigdata*.py` is generally in a sub-directory of the location of `libpython3.Y.so`. +/// So we must find the file in the following possible locations: +/// +/// ```sh +/// # distribution from package manager, lib_dir should include lib/ +/// ${INSTALL_PREFIX}/lib/python3.Y/_sysconfigdata*.py +/// ${INSTALL_PREFIX}/lib/libpython3.Y.so +/// ${INSTALL_PREFIX}/lib/python3.Y/config-3.Y-${HOST_TRIPLE}/libpython3.Y.so +/// +/// # Built from source from host +/// ${CROSS_COMPILED_LOCATION}/build/lib.linux-x86_64-Y/_sysconfigdata*.py +/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so +/// +/// # if cross compiled, kernel release is only present on certain OS targets. +/// ${CROSS_COMPILED_LOCATION}/build/lib.{OS}(-{OS-KERNEL-RELEASE})?-{ARCH}-Y/_sysconfigdata*.py +/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so +/// ``` +/// +/// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389 fn find_sysconfigdata(path: impl AsRef, cross: &CrossCompileConfig) -> Option { for f in fs::read_dir(path).expect("Path does not exist") { return match f { @@ -665,9 +693,10 @@ fn configure(interpreter_config: &InterpreterConfig) -> Result { } check_target_architecture(interpreter_config)?; + let target_os = env::var_os("CARGO_CFG_TARGET_OS").unwrap(); let is_extension_module = env::var_os("CARGO_FEATURE_EXTENSION_MODULE").is_some(); - if !is_extension_module || cfg!(target_os = "windows") { + if !is_extension_module || target_os == "windows" || target_os == "android" { println!("{}", get_rustc_link_lib(&interpreter_config)?); if let Some(libdir) = &interpreter_config.libdir { println!("cargo:rustc-link-search=native={}", libdir); From 7beb2720757de2bf762334808b1ad82217620f8f Mon Sep 17 00:00:00 2001 From: Rene Leveille Date: Tue, 18 Aug 2020 22:05:29 -0400 Subject: [PATCH 09/11] find within general lib dir, with new optional env variable --- build.rs | 52 ++++++++++++++++++++------ guide/src/building_and_distribution.md | 11 +++++- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/build.rs b/build.rs index 900a25e1a24..a14a1af50b9 100644 --- a/build.rs +++ b/build.rs @@ -105,6 +105,7 @@ impl GetPrimitive for HashMap { struct CrossCompileConfig { lib_dir: PathBuf, include_dir: Option, + version: Option, os: String, arch: String, } @@ -125,6 +126,7 @@ impl CrossCompileConfig { include_dir: None, os: env::var("CARGO_CFG_TARGET_OS").unwrap(), arch: env::var("CARGO_CFG_TARGET_ARCH").unwrap(), + version: env::var_os("PYO3_PYTHON_VERSION").map(|s| s.into_string().unwrap()), }) } @@ -262,10 +264,10 @@ fn ends_with(entry: &DirEntry, pat: &str) -> bool { name.to_string_lossy().ends_with(pat) } -/// Finds the `_sysconfigdata*.py` file in the library path +/// Finds the `_sysconfigdata*.py` file in the library path. /// -/// From the python source this file is always going to be located at `build/lib.{PLATFORM}-{PY_MINOR_VERSION}` -/// when built from source. The [exact line][1] is defined as: +/// From the python source for `_sysconfigdata*.py` is always going to be located at +/// `build/lib.{PLATFORM}-{PY_MINOR_VERSION}` when built from source. The [exact line][1] is defined as: /// /// ```py /// pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version_info[:2]) @@ -293,11 +295,36 @@ fn ends_with(entry: &DirEntry, pat: &str) -> bool { /// ``` /// /// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389 -fn find_sysconfigdata(path: impl AsRef, cross: &CrossCompileConfig) -> Option { +fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result { + let mut sysconfig_paths = search_lib_dir(&cross.lib_dir, &cross); + if sysconfig_paths.len() == 0 { + bail!( + "Could not find either libpython.so or _sysconfigdata*.py in {}", + cross.lib_dir.display() + ); + } else if sysconfig_paths.len() > 1 { + bail!( + "Detected multiple possible python versions, please set the PYO3_PYTHON_VERSION \ + variable to the wanted version on your system\nsysconfigdata paths = {:?}", + sysconfig_paths + ) + } + + Ok(sysconfig_paths.remove(0)) +} + +/// recursive search for _sysconfigdata, returns all possibilities of sysconfigdata paths +fn search_lib_dir(path: impl AsRef, cross: &CrossCompileConfig) -> Vec { + let mut sysconfig_paths = vec![]; + let version_pat = if let Some(ref v) = cross.version { + format!("python{}", v) + } else { + "python3.".into() + }; for f in fs::read_dir(path).expect("Path does not exist") { - return match f { - Ok(ref f) if starts_with(f, "_sysconfigdata") && ends_with(f, "py") => Some(f.path()), - Ok(ref f) if starts_with(f, "build") => find_sysconfigdata(f.path(), cross), + let sysc = match f { + Ok(ref f) if starts_with(f, "_sysconfigdata") && ends_with(f, "py") => vec![f.path()], + Ok(ref f) if starts_with(f, "build") => search_lib_dir(f.path(), cross), Ok(ref f) if starts_with(f, "lib.") => { let name = f.file_name(); // check if right target os @@ -312,12 +339,14 @@ fn find_sysconfigdata(path: impl AsRef, cross: &CrossCompileConfig) -> Opt if !name.to_string_lossy().contains(&cross.arch) { continue; } - find_sysconfigdata(f.path(), cross) + search_lib_dir(f.path(), cross) } + Ok(ref f) if starts_with(f, &version_pat) => search_lib_dir(f.path(), cross), _ => continue, }; + sysconfig_paths.extend(sysc); } - None + sysconfig_paths } /// Find cross compilation information from sysconfigdata file @@ -329,8 +358,7 @@ fn find_sysconfigdata(path: impl AsRef, cross: &CrossCompileConfig) -> Opt fn load_cross_compile_from_sysconfigdata( python_paths: CrossCompileConfig, ) -> Result<(InterpreterConfig, HashMap)> { - let sysconfig_path = find_sysconfigdata(&python_paths.lib_dir, &python_paths) - .expect("_sysconfigdata*.py not found"); + let sysconfig_path = find_sysconfigdata(&python_paths)?; let config_map = parse_sysconfigdata(sysconfig_path)?; let shared = config_map.get_bool("Py_ENABLE_SHARED")?; @@ -350,7 +378,7 @@ fn load_cross_compile_from_sysconfigdata( let interpreter_config = InterpreterConfig { version: python_version, - libdir: python_paths.lib_dir.to_str().map(String::from), + libdir: python_paths.lib_dir.to_str().map(String::from), //libpython_path.to_str().map(String::from), shared, ld_version, base_prefix: "".to_string(), diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index f0dca9ccea3..8f908e8d01f 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -53,7 +53,8 @@ See https://github.com/japaric/rust-cross for a primer on cross compiling Rust i After you've obtained the above, you can build a cross compiled PyO3 module by setting a few extra environment variables: * `PYO3_CROSS_INCLUDE_DIR`: This variable must be set to the directory containing the headers for the target's Python interpreter. **It is only necessary if targeting Windows platforms** -* `PYO3_CROSS_LIB_DIR`: This variable must be set to the directory containing the target's libpython DSO. +* `PYO3_CROSS_LIB_DIR`: This variable must be set to the directory containing the target's libpython DSO and the associated `_sysconfigdata*.py` file. +* `PYO3_PYTHON_VERSION`: This variable must be set if there are multiple versions of python compiled for a unix machine. An example might look like the following (assuming your target's sysroot is at `/home/pyo3/cross/sysroot` and that your target is `armv7`): @@ -63,6 +64,14 @@ export PYO3_CROSS_LIB_DIR="/home/pyo3/cross/sysroot/usr/lib" cargo build --target armv7-unknown-linux-gnueabihf ``` +If there are multiple python versions at the cross lib directory and you cannot set a more precise location to include both the `libpython` DSO and `_sysconfigdata*.py` files, you can set the required version: +```sh +export PYO3_PYTHON_VERSION=3.8 +export PYO3_CROSS_LIB_DIR="/home/pyo3/cross/sysroot/usr/lib" + +cargo build --target armv7-unknown-linux-gnueabihf +``` + Or another example with the same sys root but building for windows: ```sh export PYO3_CROSS_INCLUDE_DIR="/home/pyo3/cross/sysroot/usr/include" From 455ec80a981f765202b22219681e7afc095129eb Mon Sep 17 00:00:00 2001 From: Rene Leveille Date: Tue, 18 Aug 2020 22:14:14 -0400 Subject: [PATCH 10/11] make clippy happy --- build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.rs b/build.rs index a14a1af50b9..b0f1e06f77b 100644 --- a/build.rs +++ b/build.rs @@ -297,7 +297,7 @@ fn ends_with(entry: &DirEntry, pat: &str) -> bool { /// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389 fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result { let mut sysconfig_paths = search_lib_dir(&cross.lib_dir, &cross); - if sysconfig_paths.len() == 0 { + if sysconfig_paths.is_empty() { bail!( "Could not find either libpython.so or _sysconfigdata*.py in {}", cross.lib_dir.display() From 441d7f52b18a01038d31b20d7ad1037550f5a4b9 Mon Sep 17 00:00:00 2001 From: Rene Leveille Date: Wed, 19 Aug 2020 12:11:14 -0400 Subject: [PATCH 11/11] apply symlink use case, change variable name --- build.rs | 11 ++++++++--- guide/src/building_and_distribution.md | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/build.rs b/build.rs index b0f1e06f77b..d96e526097b 100644 --- a/build.rs +++ b/build.rs @@ -126,7 +126,7 @@ impl CrossCompileConfig { include_dir: None, os: env::var("CARGO_CFG_TARGET_OS").unwrap(), arch: env::var("CARGO_CFG_TARGET_ARCH").unwrap(), - version: env::var_os("PYO3_PYTHON_VERSION").map(|s| s.into_string().unwrap()), + version: env::var_os("PYO3_CROSS_PYTHON_VERSION").map(|s| s.into_string().unwrap()), }) } @@ -296,7 +296,12 @@ fn ends_with(entry: &DirEntry, pat: &str) -> bool { /// /// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389 fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result { - let mut sysconfig_paths = search_lib_dir(&cross.lib_dir, &cross); + let sysconfig_paths = search_lib_dir(&cross.lib_dir, &cross); + let mut sysconfig_paths = sysconfig_paths + .iter() + .filter_map(|p| fs::canonicalize(p).ok()) + .collect::>(); + sysconfig_paths.dedup(); if sysconfig_paths.is_empty() { bail!( "Could not find either libpython.so or _sysconfigdata*.py in {}", @@ -378,7 +383,7 @@ fn load_cross_compile_from_sysconfigdata( let interpreter_config = InterpreterConfig { version: python_version, - libdir: python_paths.lib_dir.to_str().map(String::from), //libpython_path.to_str().map(String::from), + libdir: python_paths.lib_dir.to_str().map(String::from), shared, ld_version, base_prefix: "".to_string(), diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index 8f908e8d01f..e32806d8f90 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -54,7 +54,7 @@ After you've obtained the above, you can build a cross compiled PyO3 module by s * `PYO3_CROSS_INCLUDE_DIR`: This variable must be set to the directory containing the headers for the target's Python interpreter. **It is only necessary if targeting Windows platforms** * `PYO3_CROSS_LIB_DIR`: This variable must be set to the directory containing the target's libpython DSO and the associated `_sysconfigdata*.py` file. -* `PYO3_PYTHON_VERSION`: This variable must be set if there are multiple versions of python compiled for a unix machine. +* `PYO3_CROSS_PYTHON_VERSION`: This variable must be set if there are multiple versions of python compiled for a unix machine. An example might look like the following (assuming your target's sysroot is at `/home/pyo3/cross/sysroot` and that your target is `armv7`): @@ -66,7 +66,7 @@ cargo build --target armv7-unknown-linux-gnueabihf If there are multiple python versions at the cross lib directory and you cannot set a more precise location to include both the `libpython` DSO and `_sysconfigdata*.py` files, you can set the required version: ```sh -export PYO3_PYTHON_VERSION=3.8 +export PYO3_CROSS_PYTHON_VERSION=3.8 export PYO3_CROSS_LIB_DIR="/home/pyo3/cross/sysroot/usr/lib" cargo build --target armv7-unknown-linux-gnueabihf