Load compilation options from _sysconfigdata_*.py file

CHANGELOG.md

@@ -42,6 +42,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Allows `&Self` as a `#[pymethods]` argument again. [#1071](https://github.com/PyO3/pyo3/pull/1071)
 - Fix best-effort build against PyPy 3.6. #[1092](https://github.com/PyO3/pyo3/pull/1092)
 - Improve lifetime elision in `#[pyproto]`. [#1093](https://github.com/PyO3/pyo3/pull/1093)
+- Fix python configuration detection when cross-compiling. [1077](https://github.com/PyO3/pyo3/issues/1077)
+- Link against libpython on android with `extension-module` set. [#1082](https://github.com/PyO3/pyo3/issues/1082)
 
 ## [0.11.1] - 2020-06-30
 ### Added

build.rs

@@ -2,7 +2,7 @@ use std::{
     collections::HashMap,
     convert::AsRef,
     env, fmt,
-    fs::File,
+    fs::{self, DirEntry, File},
     io::{self, BufRead, BufReader},
     path::{Path, PathBuf},
     process::{Command, Stdio},
@@ -76,6 +76,59 @@ impl FromStr for PythonInterpreterKind {
     }
 }
 
+struct CrossPython {
+    lib_dir: String,
+    include_dir: Option<String>,
+    os: String,
+    arch: String,
+}
+
+impl CrossPython {
+    fn both() -> Result<Self> {
+        Ok(CrossPython {
+            include_dir: Some(CrossPython::validate_variable("PYO3_CROSS_INCLUDE_DIR")?),
+            ..CrossPython::lib_only()?
+        })
+    }
+
+    fn lib_only() -> Result<Self> {
+        Ok(CrossPython {
+            lib_dir: CrossPython::validate_variable("PYO3_CROSS_LIB_DIR")?,
+            include_dir: None,
+            os: env::var("CARGO_CFG_TARGET_OS").unwrap(),
+            arch: env::var("CARGO_CFG_TARGET_ARCH").unwrap(),
+        })
+    }
+
+    fn validate_variable(var: &str) -> Result<String> {
+        let path = match env::var(var) {
+            Ok(v) => v,
+            Err(_) => bail!(
+                "Must provide {} environment variable when cross-compiling",
+                var
+            ),
+        };
+
+        if fs::metadata(&path).is_err() {
+            bail!("{} value of {} does not exist", var, path)
+        }
+
+        Ok(path)
+    }
+}
+
+fn cross_compiling() -> Result<Option<CrossPython>> {
+    if env::var("TARGET")? == env::var("HOST")? {
+        return Ok(None);
+    }
+
+    if env::var("CARGO_CFG_TARGET_FAMILY")? == "windows" {
+        Ok(Some(CrossPython::both()?))
+    } else {
+        Ok(Some(CrossPython::lib_only()?))
+    }
+}
+
 /// A list of python interpreter compile-time preprocessor defines that
 /// we will pick up and pass to rustc via --cfg=py_sys_config={varname};
 /// this allows using them conditional cfg attributes in the .rs files, so
@@ -134,10 +187,149 @@ fn fix_config_map(mut config_map: HashMap<String, String>) -> HashMap<String, St
     config_map
 }
 
-fn load_cross_compile_info() -> Result<(InterpreterConfig, HashMap<String, String>)> {
-    let python_include_dir = env::var("PYO3_CROSS_INCLUDE_DIR")?;
-    let python_include_dir = Path::new(&python_include_dir);
+fn parse_script_output(output: &str) -> HashMap<String, String> {
+    output
+        .lines()
+        .filter_map(|line| {
+            let mut i = line.splitn(2, ' ');
+            Some((i.next()?.into(), i.next()?.into()))
+        })
+        .collect()
+}
+
+fn as_bool(config: &HashMap<String, String>, key: &str) -> Result<bool> {
+    match config
+        .get(key)
+        .map(|x| x.as_str())
+        .ok_or(format!("{} is not defined", key))?
+    {
+        "1" | "true" | "True" => Ok(true),
+        "0" | "false" | "False" => Ok(false),
+        _ => Err(format!("{} must be a bool (1/true/True or 0/false/False", key).into()),
+    }
+}
+
+fn as_numeric<T: FromStr>(config: &HashMap<String, String>, key: &str) -> Result<T> {
+    config
+        .get(key)
+        .ok_or(format!("{} is not defined", key))?
+        .parse::<T>()
+        .map_err(|_| format!("Could not parse value of {}", key).into())
+}
+
+/// Parse sysconfigdata file
+///
+/// The sysconfigdata is basically a dictionary, and since we can't really use this library to read
+/// it for us, egg and chicken type thing, we parse it with some regex. Here there are two regex's.
+/// The first one is to match an entry in the dictionary (`entry_re`). Then when the entry is on
+/// multiple lines we use the second regex to capture the additional string entry and add it to the
+/// previously captured key. We detect if this is a multi line entry by checking if the last capture
+/// group contains either `,` or `}`.
+fn parse_sysconfigdata(config_path: impl AsRef<Path>) -> Result<HashMap<String, String>> {
+    let mut script = fs::read_to_string(config_path)?;
+    script += r#"
+print("version_major", build_time_vars["VERSION"][0])  # 3
+print("version_minor", build_time_vars["VERSION"][2])  # E.g., 8
+if "WITH_THREAD" in build_time_vars:
+    print("WITH_THREAD", build_time_vars["WITH_THREAD"])
+if "Py_TRACE_REFS" in build_time_vars:
+    print("Py_TRACE_REFS", build_time_vars["Py_TRACE_REFS"])
+if "COUNT_ALLOCS" in build_time_vars:
+    print("COUNT_ALLOCS", build_time_vars["COUNT_ALLOCS"])
+if "Py_REF_DEBUG" in build_time_vars:
+    print("Py_REF_DEBUG", build_time_vars["Py_REF_DEBUG"])
+print("Py_DEBUG", build_time_vars["Py_DEBUG"])
+print("Py_ENABLE_SHARED", build_time_vars["Py_ENABLE_SHARED"])
+print("LDVERSION", build_time_vars["LDVERSION"])
+print("SIZEOF_VOID_P", build_time_vars["SIZEOF_VOID_P"])
+"#;
+    let output = run_python_script(&find_interpreter()?, &script)?;
+
+    Ok(parse_script_output(&output))
+}
+
+fn starts_with(entry: &DirEntry, pat: &str) -> bool {
+    let name = entry.file_name();
+    name.to_string_lossy().starts_with(pat)
+}
+fn ends_with(entry: &DirEntry, pat: &str) -> bool {
+    let name = entry.file_name();
+    name.to_string_lossy().ends_with(pat)
+}
 
+fn find_sysconfigdata(path: impl AsRef<Path>, cross: &CrossPython) -> Option<PathBuf> {
+    for f in fs::read_dir(path).expect("Path does not exist") {
+        return match f {
+            Ok(ref f) if starts_with(f, "_sysconfigdata") && ends_with(f, "py") => Some(f.path()),
+            Ok(ref f) if starts_with(f, "build") => find_sysconfigdata(f.path(), cross),
+            Ok(ref f) if starts_with(f, "lib.") => {
+                let name = f.file_name();
+                // check if right target os
+                if !name.to_string_lossy().contains(if cross.os == "android" {
+                    "linux"
+                } else {
+                    &cross.os
+                }) {
+                    continue;
+                }
+                // Check if right arch
+                if !name.to_string_lossy().contains(&cross.arch) {
+                    continue;
+                }
+                find_sysconfigdata(f.path(), cross)
+            }
+            _ => continue,
+        };
+    }
+    None
+}
+
+/// Find cross compilation information from sysconfigdata file
+///
+/// first find sysconfigdata file which follows the pattern [`_sysconfigdata_{abi}_{platform}_{multiarch}`][1]
+/// on python 3.6 or greater. On python 3.5 it is simply `_sysconfigdata.py`.
+///
+/// [1]: https://github.com/python/cpython/blob/3.8/Lib/sysconfig.py#L348
+fn load_cross_compile_from_sysconfigdata(
+    python_paths: CrossPython,
+) -> Result<(InterpreterConfig, HashMap<String, String>)> {
+    let sysconfig_path = find_sysconfigdata(&python_paths.lib_dir, &python_paths)
+        .expect("_sysconfigdata*.py not found");
+    let config_map = parse_sysconfigdata(sysconfig_path)?;
+
+    let shared = as_bool(&config_map, "Py_ENABLE_SHARED")?;
+    let major = as_numeric(&config_map, "version_major")?;
+    let minor = as_numeric(&config_map, "version_minor")?;
+    let ld_version = match config_map.get("LDVERSION") {
+        Some(s) => s.clone(),
+        None => format!("{}.{}", major, minor),
+    };
+    let calcsize_pointer = as_numeric(&config_map, "SIZEOF_VOID_P").ok();
+
+    let python_version = PythonVersion {
+        major,
+        minor: Some(minor),
+        implementation: PythonInterpreterKind::CPython,
+    };
+
+    let interpreter_config = InterpreterConfig {
+        version: python_version,
+        libdir: Some(python_paths.lib_dir),
+        shared,
+        ld_version,
+        base_prefix: "".to_string(),
+        executable: PathBuf::new(),
+        calcsize_pointer,
+    };
+
+    Ok((interpreter_config, fix_config_map(config_map)))
+}
+
+fn load_cross_compile_from_headers(
+    python_paths: CrossPython,
+) -> Result<(InterpreterConfig, HashMap<String, String>)> {
+    let python_include_dir = python_paths.include_dir.unwrap();
+    let python_include_dir = Path::new(&python_include_dir);
     let patchlevel_defines = parse_header_defines(python_include_dir.join("patchlevel.h"))?;
 
     let major = match patchlevel_defines
@@ -165,21 +357,13 @@ fn load_cross_compile_info() -> Result<(InterpreterConfig, HashMap<String, Strin
     };
 
     let config_map = parse_header_defines(python_include_dir.join("pyconfig.h"))?;
-    let shared = match config_map
-        .get("Py_ENABLE_SHARED")
-        .map(|x| x.as_str())
-        .ok_or("Py_ENABLE_SHARED is not defined")?
-    {
-        "1" | "true" | "True" => true,
-        "0" | "false" | "False" => false,
-        _ => panic!("Py_ENABLE_SHARED must be a bool (1/true/True or 0/false/False"),
-    };
+    let shared = as_bool(&config_map, "Py_ENABLE_SHARED")?;
 
     let interpreter_config = InterpreterConfig {
         version: python_version,
-        libdir: Some(env::var("PYO3_CROSS_LIB_DIR")?),
+        libdir: Some(python_paths.lib_dir),
         shared,
-        ld_version: "".to_string(),
+        ld_version: format!("{}.{}", major, minor),
         base_prefix: "".to_string(),
         executable: PathBuf::new(),
         calcsize_pointer: None,
@@ -188,6 +372,21 @@ fn load_cross_compile_info() -> Result<(InterpreterConfig, HashMap<String, Strin
     Ok((interpreter_config, fix_config_map(config_map)))
 }
 
+#[allow(unused_variables)]
+fn load_cross_compile_info(
+    python_paths: CrossPython,
+) -> Result<(InterpreterConfig, HashMap<String, String>)> {
+    let target_family = env::var("CARGO_CFG_TARGET_FAMILY")?;
+    // Because compiling for windows on linux still includes the unix target family
+    if target_family == "unix" {
+        // Configure for unix platforms using the sysconfigdata file
+        load_cross_compile_from_sysconfigdata(python_paths)
+    } else {
+        // Must configure by headers on windows platform
+        load_cross_compile_from_headers(python_paths)
+    }
+}
+
 /// Examine python's compile flags to pass to cfg by launching
 /// the interpreter and printing variables of interest from
 /// sysconfig.get_config_vars.
@@ -446,13 +645,7 @@ print("executable", sys.executable)
 print("calcsize_pointer", struct.calcsize("P"))
 "#;
     let output = run_python_script(interpreter, script)?;
-    let map: HashMap<String, String> = output
-        .lines()
-        .filter_map(|line| {
-            let mut i = line.splitn(2, ' ');
-            Some((i.next()?.into(), i.next()?.into()))
-        })
-        .collect();
+    let map: HashMap<String, String> = parse_script_output(&output);
     Ok(InterpreterConfig {
         version: PythonVersion {
             major: map["version_major"].parse()?,
@@ -567,10 +760,14 @@ fn main() -> Result<()> {
     // If you have troubles with your shell accepting '.' in a var name,
     // try using 'env' (sorry but this isn't our fault - it just has to
     // match the pkg-config package name, which is going to have a . in it).
-    let cross_compiling =
-        env::var("PYO3_CROSS_INCLUDE_DIR").is_ok() && env::var("PYO3_CROSS_LIB_DIR").is_ok();
-    let (interpreter_config, mut config_map) = if cross_compiling {
-        load_cross_compile_info()?
+    //
+    // Detecting if cross-compiling by checking if the target triple is different from the host
+    // rustc's triple.
+    let (interpreter_config, mut config_map) = if let Some(paths) = cross_compiling()? {
+        // If cross compiling we need the path to the cross-compiled include dir and lib dir, else
+        // fail quickly and loudly
+
+        load_cross_compile_info(paths)?
     } else {
         find_interpreter_and_get_config()?
     };

guide/src/building_and_distribution.md

@@ -45,20 +45,28 @@ Cross compiling PyO3 modules is relatively straightforward and requires a few pi
 * A toolchain for your target.
 * The appropriate options in your Cargo `.config` for the platform you're targeting and the toolchain you are using.
 * A Python interpreter that's already been compiled for your target.
+* A Python interpreter that is built for your host and available through the `PATH`.
 * The headers that match the above interpreter.
 
 See https://github.com/japaric/rust-cross for a primer on cross compiling Rust in general.
 
 After you've obtained the above, you can build a cross compiled PyO3 module by setting a few extra environment variables:
 
-* `PYO3_CROSS_INCLUDE_DIR`: This variable must be set to the directory containing the headers for the target's Python interpreter.
+* `PYO3_CROSS_INCLUDE_DIR`: This variable must be set to the directory containing the headers for the target's Python interpreter. **It is only necessary if targeting Windows platforms**
 * `PYO3_CROSS_LIB_DIR`: This variable must be set to the directory containing the target's libpython DSO.
 
 An example might look like the following (assuming your target's sysroot is at `/home/pyo3/cross/sysroot` and that your target is `armv7`):
 
 ```sh
-export PYO3_CROSS_INCLUDE_DIR="/home/pyo3/cross/sysroot/usr/include"
 export PYO3_CROSS_LIB_DIR="/home/pyo3/cross/sysroot/usr/lib"
 
 cargo build --target armv7-unknown-linux-gnueabihf
 ```
+
+Or another example with the same sys root but building for windows:
+```sh
+export PYO3_CROSS_INCLUDE_DIR="/home/pyo3/cross/sysroot/usr/include"
+export PYO3_CROSS_LIB_DIR="/home/pyo3/cross/sysroot/usr/lib"
+
+cargo build --target x86_64-pc-windows-gnu
+```