[ci] use rayci to build windows wheels (ray-project#43447)

Context: this is a series of PR to move windows wheel to rayci (and deprecate .buildkite/windows_ci.sh). The mentioned script runs into issues when I try to upgrade it to python 3.11. Moving the script to rayci to uniform the window build environment.

- Reuse copy_build_artifacts script for windows. This will allow the wheel to be uploaded to buildkite artifact, in addition to the release s3 bucket
- Support volumes when creating a windows container. I do this by moving volumes to the base container class, so that both linux and windows container and initialize volumes. volumes is required in a later PR when I use the windows container to build windows wheel.
- Use rayci and windowsbuild wanda image to build windows wheel

Test:
- CI
- windows wheel are uploaded to buildkite artifact: https://buildkite.com/ray-project/premerge/builds/20211#018de8cf-3bc5-4c4b-aec8-593a4e89f311
- postmerge run: https://buildkite.com/ray-project/postmerge/builds/3123#018de8df-1d45-4c4d-b1e4-4781488e81fb/6-12182

Signed-off-by: can <[email protected]>

.buildkite/build.rayci.yml

@@ -55,8 +55,8 @@ steps:
     mount_windows_artifacts: true
     instance_type: windows
     commands:
-      - export BUILD_ONE_PYTHON_ONLY={{matrix}}
-      - bash /c/workdir/.buildkite/windows_ci.sh wheels
+      - bash ci/ray_ci/windows/install_tools.sh
+      - bazel run //ci/ray_ci:build_in_docker_windows -- wheel --python-version {{matrix}} --operating-system windows --upload
     matrix:
       - "3.8"
       - "3.9"

ci/build/copy_build_artifacts.sh

@@ -3,6 +3,11 @@
 set -exuo pipefail
 
 ARTIFACT_TYPE=${1:-wheel}
+if [[ "$OSTYPE" == "msys" ]]; then
+  ARTIFACT_MOUNT="/c/artifact-mount"
+else
+  ARTIFACT_MOUNT="/artifact-mount"
+fi
 
 if [[ "$ARTIFACT_TYPE" != "wheel" && "$ARTIFACT_TYPE" != "jar" ]]; then
   echo "Invalid artifact type: $ARTIFACT_TYPE"
@@ -12,22 +17,32 @@ fi
 if [[ "$ARTIFACT_TYPE" == "wheel" ]]; then
   BRANCH_DESTINATION="branch_wheels"
   MASTER_DESTINATION="wheels"
-  ARTIFACT_PATH=".whl"
+  if [[ "$OSTYPE" == "msys" ]]; then
+    ARTIFACT_PATH="python/dist"
+  else
+    ARTIFACT_PATH=".whl"
+  fi
 else
   BRANCH_DESTINATION="branch_jars"
   MASTER_DESTINATION="jars"
   ARTIFACT_PATH=".jar/linux"
 fi
 
+if [[ "$OSTYPE" == "msys" ]]; then
+  ARTIFACT_PATH="python/dist"
+  ARTIFACT_MOUNT="/c/artifact-mount"
+fi
+
 export PATH=/opt/python/cp38-cp38/bin:$PATH
 pip install -q aws_requests_auth boto3
 ./ci/env/env_info.sh
 
 # Sync the directory to buildkite artifacts
-rm -rf /artifact-mount/"$ARTIFACT_PATH" || true
-mkdir -p /artifact-mount/"$ARTIFACT_PATH"
-cp -r "$ARTIFACT_PATH" /artifact-mount/"$ARTIFACT_PATH"
-chmod -R 777 /artifact-mount/"$ARTIFACT_PATH"
+ARTIFACT_MOUNT_PATH="$ARTIFACT_MOUNT/$ARTIFACT_PATH"
+rm -rf "$ARTIFACT_MOUNT_PATH" || true
+mkdir -p "$ARTIFACT_MOUNT_PATH"
+cp -r "$ARTIFACT_PATH" "$ARTIFACT_MOUNT_PATH"
+chmod -R 777 "$ARTIFACT_MOUNT_PATH"
 
 # Upload to the wheels S3 bucket when running on postmerge pipeline.
 readonly PIPELINE_POSTMERGE="0189e759-8c96-4302-b6b5-b4274406bf89"

ci/ray_ci/BUILD.bazel

@@ -41,6 +41,12 @@ py_binary(
     deps = [":ray_ci_lib"],
 )
 
+py_binary(
+    name = "build_in_docker_windows",
+    srcs = ["build_in_docker_windows.py"],
+    deps = [":ray_ci_lib"],
+)
+
 py_test(
     name = "test_linux_container",
     size = "small",

ci/ray_ci/build_in_docker_windows.py

@@ -0,0 +1,4 @@
+from ci.ray_ci.builder import main
+
+if __name__ == "__main__":
+    main()

ci/ray_ci/builder.py

@@ -9,6 +9,7 @@
     ARCHITECTURE,
     BuilderContainer,
 )
+from ci.ray_ci.windows_builder_container import WindowsBuilderContainer
 from ci.ray_ci.docker_container import PLATFORM
 from ci.ray_ci.ray_docker_container import RayDockerContainer
 from ci.ray_ci.anyscale_docker_container import AnyscaleDockerContainer
@@ -50,6 +51,12 @@
     type=click.Choice(list(ARCHITECTURE)),
     help=("Platform to build the docker with"),
 )
+@click.option(
+    "--operating-system",
+    default="linux",
+    type=click.Choice(["linux", "windows"]),
+    help=("Operating system to run tests on"),
+)
 @click.option(
     "--canonical-tag",
     default=None,
@@ -70,6 +77,7 @@ def main(
     python_version: str,
     platform: List[str],
     architecture: str,
+    operating_system: str,
     canonical_tag: str,
     upload: bool,
 ) -> None:
@@ -79,7 +87,7 @@ def main(
     docker_login(_DOCKER_ECR_REPO.split("/")[0])
     if artifact_type == "wheel":
         logger.info(f"Building wheel for {python_version}")
-        build_wheel(python_version, build_type, architecture, upload)
+        build_wheel(python_version, build_type, architecture, operating_system, upload)
         return
 
     if artifact_type == "docker":
@@ -114,12 +122,18 @@ def main(
 
 
 def build_wheel(
-    python_version: str, build_type: str, architecture: str, upload: bool
+    python_version: str,
+    build_type: str,
+    architecture: str,
+    operating_system: str,
+    upload: bool,
 ) -> None:
     """
     Build a wheel artifact.
     """
-    BuilderContainer(python_version, build_type, architecture, upload).run()
+    if operating_system == "windows":
+        return WindowsBuilderContainer(python_version, upload).run()
+    return BuilderContainer(python_version, build_type, architecture, upload).run()
 
 
 def build_docker(

ci/ray_ci/container.py

@@ -20,6 +20,7 @@
     "BUILDKITE_COMMIT",
     "BUILDKITE_JOB_ID",
     "BUILDKITE_LABEL",
+    "BUILDKITE_BAZEL_CACHE_URL",
     "BUILDKITE_PIPELINE_ID",
 ]
 _RAYCI_BUILD_ID = os.environ.get("RAYCI_BUILD_ID", "unknown")
@@ -30,8 +31,14 @@ class Container(abc.ABC):
     A wrapper for running commands in ray ci docker container
     """
 
-    def __init__(self, docker_tag: str, envs: Optional[List[str]] = None) -> None:
+    def __init__(
+        self,
+        docker_tag: str,
+        volumes: Optional[List[str]] = None,
+        envs: Optional[List[str]] = None,
+    ) -> None:
         self.docker_tag = docker_tag
+        self.volumes = volumes or []
         self.envs = envs or []
         self.envs += _DOCKER_ENV
 
@@ -90,7 +97,7 @@ def get_run_command(
             command += ["--env", env]
         if network:
             command += ["--network", network]
-        for volume in volumes or []:
+        for volume in (volumes or []) + self.volumes:
             command += ["--volume", volume]
         return (
             command

ci/ray_ci/linux_container.py

@@ -20,8 +20,7 @@ def __init__(
         envs: Optional[List[str]] = None,
         tmp_filesystem: Optional[str] = None,
     ) -> None:
-        super().__init__(docker_tag, envs)
-        self.volumes = volumes or []
+        super().__init__(docker_tag, volumes, envs)
 
         if tmp_filesystem is not None:
             if tmp_filesystem != "tmpfs":
@@ -72,8 +71,6 @@ def get_run_command_extra_args(
                 "--mount",
                 f"type={self.tmp_filesystem},destination=/tmp",
             ]
-        for volume in self.volumes:
-            extra_args += ["--volume", volume]
         for cap in _DOCKER_CAP_ADD:
             extra_args += ["--cap-add", cap]
         if gpu_ids:

ci/ray_ci/test_windows_container.py

@@ -44,7 +44,7 @@ def _mock_subprocess(inputs: List[str], stdout, stderr) -> None:
 
 
 def test_get_run_command() -> None:
-    container = WindowsContainer("test")
+    container = WindowsContainer("test", volumes=["/hi:/hello"])
     envs = []
     for env in _DOCKER_ENV:
         envs.extend(["--env", env])
@@ -56,8 +56,12 @@ def test_get_run_command() -> None:
         "-i",
         "--rm",
         "--volume",
-        f"{artifact_mount_host}:" f"{artifact_mount_container}",
+        f"{artifact_mount_host}:{artifact_mount_container}",
     ] + envs + [
+        "--volume",
+        "/hi:/hello",
+        "--workdir",
+        "C:\\rayci",
         "029272617770.dkr.ecr.us-west-2.amazonaws.com/rayproject/citemp:unknown-test",
         "bash",
         "-c",

ci/ray_ci/windows_builder_container.py

@@ -0,0 +1,29 @@
+import os
+
+from ci.ray_ci.windows_container import WindowsContainer, WORKDIR
+
+
+class WindowsBuilderContainer(WindowsContainer):
+    def __init__(
+        self,
+        python_version: str,
+        upload: bool,
+    ) -> None:
+        super().__init__(
+            "windowsbuild",
+            volumes=[
+                f"{os.path.abspath(os.environ.get('RAYCI_CHECKOUT_DIR'))}:{WORKDIR}",
+            ],
+        )
+        self.python_version = python_version
+        self.upload = upload
+
+    def run(self) -> None:
+        cmds = [
+            "powershell ci/pipeline/fix-windows-container-networking.ps1",
+            f"export BUILD_ONE_PYTHON_ONLY={self.python_version}",
+            "./python/build-wheel-windows.sh",
+        ]
+        if self.upload:
+            cmds += ["./ci/build/copy_build_artifacts.sh"]
+        self.run_script(cmds)

ci/ray_ci/windows_container.py

@@ -6,6 +6,9 @@
 from ci.ray_ci.container import Container
 
 
+WORKDIR = "C:\\rayci"
+
+
 class WindowsContainer(Container):
     def install_ray(self, build_type: Optional[str] = None) -> List[str]:
         assert build_type is None, f"Windows does not support build types {build_type}"
@@ -38,7 +41,7 @@ def get_run_command_extra_args(
         gpu_ids: Optional[List[int]] = None,
     ) -> List[str]:
         assert not gpu_ids, "Windows does not support gpu ids"
-        return []
+        return ["--workdir", WORKDIR]
 
     def get_artifact_mount(self) -> Tuple[str, str]:
         return ("C:\\tmp\\artifacts", "C:\\artifact-mount")

ci/ray_ci/windows_tester_container.py

@@ -14,7 +14,7 @@ def __init__(
         network: Optional[str] = None,
         skip_ray_installation: bool = False,
     ) -> None:
-        WindowsContainer.__init__(self, docker_tag, test_envs)
+        WindowsContainer.__init__(self, docker_tag, envs=test_envs)
         TesterContainer.__init__(
             self,
             shard_count,

python/build-wheel-windows.sh

@@ -89,6 +89,18 @@ build_wheel_windows() {
   uninstall_ray || ray_uninstall_status=1
 
   local local_dir="python/dist"
+  {
+    echo "build --announce_rc";  
+    echo "build --config=ci";
+    echo "startup --output_user_root=c:/raytmp";
+    echo "build --remote_cache=${BUILDKITE_BAZEL_CACHE_URL}";
+  } >> ~/.bazelrc
+
+  if [[ "$BUILDKITE_PIPELINE_ID" == "0189942e-0876-4b8f-80a4-617f988ec59b" ]]; then
+    # Do not upload cache results for premerge pipeline
+    echo "build --remote_upload_local_results=false" >> ~/.bazelrc
+  fi
+
   for pyversion in "${PY_VERSIONS[@]}"; do
     if [[ "${BUILD_ONE_PYTHON_ONLY:-}" != "" && "${pyversion}" != "${BUILD_ONE_PYTHON_ONLY}" ]]; then
       continue