Flattened podman support commits

.github/install_latest_podman.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+__doc__="
+Based on code in: https://github.com/redhat-actions/podman-login/blob/main/.github/install_latest_podman.sh
+"
+# https://podman.io/getting-started/installation
+# shellcheck source=/dev/null
+. /etc/os-release
+echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
+curl -sSfL "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key" | sudo apt-key add -
+sudo apt-get update
+sudo apt-get -y upgrade
+sudo apt-get -y install podman

.github/workflows/test.yml

@@ -45,6 +45,13 @@ jobs:
       with:
         python-version: ${{ matrix.python_version }}
 
+    # Install podman on this CI instance for podman tests on linux
+    # Snippet from: https://github.com/redhat-actions/podman-login/blob/main/.github/workflows/example.yml
+    - name: Install latest podman
+      if: runner.os == 'Linux'
+      run: |
+        bash .github/install_latest_podman.sh
+
     - name: Install dependencies
       run: |
         python -m pip install ".[test]"

cibuildwheel/docker_container.py

@@ -25,6 +25,18 @@ class DockerContainer:
     A bash shell is running in the remote container. When `call()` is invoked,
     the command is relayed to the remote shell, and the results are streamed
     back to cibuildwheel.
+
+    TODO:
+        - [ ] Rename to Container as this now generalizes docker and podman?
+
+    Example:
+        >>> from cibuildwheel.docker_container import *  # NOQA
+        >>> docker_image = "quay.io/pypa/manylinux2014_x86_64:2020-05-17-2f8ac3b"
+        >>> # Test the default container
+        >>> with DockerContainer(docker_image=docker_image) as self:
+        ...     self.call(["echo", "hello world"])
+        ...     self.call(["cat", "/proc/1/cgroup"])
+        ...     print(self.get_environment())
     """
 
     UTILITY_PYTHON = "/opt/python/cp38-cp38/bin/python"
@@ -34,7 +46,13 @@ class DockerContainer:
     bash_stdout: IO[bytes]
 
     def __init__(
-        self, *, docker_image: str, simulate_32_bit: bool = False, cwd: Optional[PathOrStr] = None
+        self,
+        *,
+        docker_image: str,
+        simulate_32_bit: bool = False,
+        cwd: Optional[PathOrStr] = None,
+        container_engine: str = "docker",
+        env: Optional[Dict[str, str]] = None,
     ):
         if not docker_image:
             raise ValueError("Must have a non-empty docker image to run.")
@@ -43,10 +61,12 @@ def __init__(
         self.simulate_32_bit = simulate_32_bit
         self.cwd = cwd
         self.name: Optional[str] = None
+        self.container_engine = container_engine
+        self.env = env  # If specified, overwrite environment variables
 
     def __enter__(self) -> "DockerContainer":
+
         self.name = f"cibuildwheel-{uuid.uuid4()}"
-        cwd_args = ["-w", str(self.cwd)] if self.cwd else []
 
         # work-around for Travis-CI PPC64le Docker runs since 2021:
         # this avoids network splits
@@ -57,39 +77,49 @@ def __enter__(self) -> "DockerContainer":
             network_args = ["--network=host"]
 
         shell_args = ["linux32", "/bin/bash"] if self.simulate_32_bit else ["/bin/bash"]
+
         subprocess.run(
             [
-                "docker",
+                self.container_engine,
                 "create",
                 "--env=CIBUILDWHEEL",
                 f"--name={self.name}",
                 "--interactive",
-                "--volume=/:/host",  # ignored on CircleCI
                 *network_args,
-                *cwd_args,
+                # Z-flags is for SELinux
+                "--volume=/:/host:Z",  # ignored on CircleCI
                 self.docker_image,
                 *shell_args,
             ],
+            env=self.env,
             check=True,
         )
+
         self.process = subprocess.Popen(
             [
-                "docker",
+                self.container_engine,
                 "start",
                 "--attach",
                 "--interactive",
                 self.name,
             ],
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
+            env=self.env,
         )
 
         assert self.process.stdin and self.process.stdout
         self.bash_stdin = self.process.stdin
         self.bash_stdout = self.process.stdout
 
         # run a noop command to block until the container is responding
-        self.call(["/bin/true"])
+        self.call(["/bin/true"], cwd="")
+
+        if self.cwd:
+            # Although `docker create -w` does create the working dir if it
+            # does not exist, podman does not. Unfortunately I don't think
+            # there is a way to set the workdir on a running container.
+            self.call(["mkdir", "-p", str(self.cwd)], cwd="")
 
         return self
 
@@ -106,10 +136,15 @@ def __exit__(
         self.bash_stdin.close()
         self.bash_stdout.close()
 
+        self.process.wait()
+
         assert isinstance(self.name, str)
 
         subprocess.run(
-            ["docker", "rm", "--force", "-v", self.name], stdout=subprocess.DEVNULL, check=False
+            [self.container_engine, "rm", "--force", "-v", self.name],
+            stdout=subprocess.DEVNULL,
+            env=self.env,
+            check=False
         )
         self.name = None
 
@@ -121,29 +156,67 @@ def copy_into(self, from_path: Path, to_path: PurePath) -> None:
 
         if from_path.is_dir():
             self.call(["mkdir", "-p", to_path])
+            # NOTE: it may be necessary allow the user to exclude directories
+            # (e.g. --exclude-vcs-ignores --exclude='.cache') in the future.
+            # This is important if the oci images themselves are in the
+            # repo directory we are copying into the container.
             subprocess.run(
-                f"tar cf - . | docker exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
+                f"tar cf - . | {self.container_engine} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
                 shell=True,
                 check=True,
                 cwd=from_path,
+                env=self.env,
             )
         else:
             subprocess.run(
-                f'cat {shell_quote(from_path)} | docker exec -i {self.name} sh -c "cat > {shell_quote(to_path)}"',
+                f'cat {shell_quote(from_path)} | {self.container_engine} exec -i {self.name} sh -c "cat > {shell_quote(to_path)}"',
                 shell=True,
                 check=True,
+                env=self.env,
             )
 
     def copy_out(self, from_path: PurePath, to_path: Path) -> None:
         # note: we assume from_path is a dir
         to_path.mkdir(parents=True, exist_ok=True)
 
-        subprocess.run(
-            f"docker exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -",
-            shell=True,
-            check=True,
-            cwd=to_path,
-        )
+        if self.container_engine == "podman":
+            command = f"{self.container_engine} exec -i {self.name} tar -cC {shell_quote(from_path)} -f /tmp/output-{self.name}.tar ."
+            subprocess.run(
+                command,
+                shell=True,
+                check=True,
+                cwd=to_path,
+                env=self.env,
+            )
+
+            command = f"{self.container_engine} cp {self.name}:/tmp/output-{self.name}.tar output-{self.name}.tar"
+            subprocess.run(
+                command,
+                shell=True,
+                check=True,
+                cwd=to_path,
+                env=self.env,
+            )
+            command = f"tar -xvf output-{self.name}.tar"
+            subprocess.run(
+                command,
+                shell=True,
+                check=True,
+                cwd=to_path,
+                env=self.env,
+            )
+            os.unlink(to_path / f"output-{self.name}.tar")
+        elif self.container_engine == "docker":
+            command = f"{self.container_engine} exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -"
+            subprocess.run(
+                command,
+                shell=True,
+                check=True,
+                cwd=to_path,
+                env=self.env,
+            )
+        else:
+            raise KeyError(self.container_engine)
 
     def glob(self, path: PurePath, pattern: str) -> List[PurePath]:
         glob_pattern = os.path.join(str(path), pattern)
@@ -169,6 +242,11 @@ def call(
         cwd: Optional[PathOrStr] = None,
     ) -> str:
 
+        if cwd is None:
+            # Hack because podman won't let us start a container with our
+            # desired working dir
+            cwd = self.cwd
+
         chdir = f"cd {cwd}" if cwd else ""
         env_assignments = (
             " ".join(f"{shlex.quote(k)}={shlex.quote(v)}" for k, v in env.items())
@@ -254,6 +332,23 @@ def environment_executor(self, command: List[str], environment: Dict[str, str])
         # used as an EnvironmentExecutor to evaluate commands and capture output
         return self.call(command, env=environment, capture_output=True)
 
+    def debug_info(self) -> str:
+        if self.container_engine == "podman":
+            command = f"{self.container_engine} info --debug"
+        else:
+            command = f"{self.container_engine} info"
+        completed = subprocess.run(
+            command,
+            shell=True,
+            check=True,
+            cwd=self.cwd,
+            env=self.env,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+        )
+        output = str(completed.stdout, encoding="utf8", errors="surrogateescape")
+        return output
+
 
 def shell_quote(path: PurePath) -> str:
     return shlex.quote(str(path))

cibuildwheel/linux.py

@@ -320,12 +320,16 @@ def build_on_docker(
 
 
 def build(options: Options, tmp_path: Path) -> None:  # pylint: disable=unused-argument
+
+    build_opts = options.build_options(None)
     try:
         # check docker is installed
-        subprocess.run(["docker", "--version"], check=True, stdout=subprocess.DEVNULL)
+        subprocess.run(
+            [build_opts.container_engine, "--version"], check=True, stdout=subprocess.DEVNULL
+        )
     except subprocess.CalledProcessError:
         print(
-            "cibuildwheel: Docker not found. Docker is required to run Linux builds. "
+            f"cibuildwheel: {build_opts.container_engine} not found. An OCI exe like Docker or Podman is required to run Linux builds "
             "If you're building on Travis CI, add `services: [docker]` to your .travis.yml."
             "If you're building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml",
             file=sys.stderr,
@@ -355,6 +359,7 @@ def build(options: Options, tmp_path: Path) -> None:  # pylint: disable=unused-a
                 docker_image=build_step.docker_image,
                 simulate_32_bit=build_step.platform_tag.endswith("i686"),
                 cwd=container_project_path,
+                container_engine=build_opts.container_engine,
             ) as docker:
 
                 build_on_docker(

cibuildwheel/options.py

@@ -79,6 +79,7 @@ class BuildOptions(NamedTuple):
     test_extras: str
     build_verbosity: int
     build_frontend: BuildFrontend
+    container_engine: str
 
     @property
     def package_dir(self) -> Path:
@@ -422,6 +423,7 @@ def build_options(self, identifier: Optional[str]) -> BuildOptions:
             test_requires = self.reader.get("test-requires", sep=" ").split()
             test_extras = self.reader.get("test-extras", sep=",")
             build_verbosity_str = self.reader.get("build-verbosity")
+            container_engine = self.reader.get("container-engine")
 
             build_frontend: BuildFrontend
             if build_frontend_str == "build":
@@ -520,6 +522,7 @@ def build_options(self, identifier: Optional[str]) -> BuildOptions:
                 manylinux_images=manylinux_images or None,
                 musllinux_images=musllinux_images or None,
                 build_frontend=build_frontend,
+                container_engine=container_engine,
             )
 
     def check_for_invalid_configuration(self, identifiers: List[str]) -> None:

cibuildwheel/resources/defaults.toml

@@ -19,6 +19,8 @@ before-test = ""
 test-requires = []
 test-extras = []
 
+container-engine = "docker"
+
 manylinux-x86_64-image = "manylinux2014"
 manylinux-i686-image = "manylinux2014"
 manylinux-aarch64-image = "manylinux2014"

setup.py

@@ -14,6 +14,7 @@
         "pytest-timeout",
         "pytest-xdist",
         "build",
+        "toml",
     ],
     "bin": [
         "click",

unit_test/conftest.py

@@ -16,7 +16,7 @@ def pytest_configure(config):
 
 def pytest_collection_modifyitems(config, items):
     if config.getoption("--run-docker"):
-        # --run-docker given in cli: do not skip docker tests
+        # --run-docker given in cli: do not skip container tests
         return
     skip_docker = pytest.mark.skip(reason="need --run-docker option to run")
     for item in items: