Merge pull request #294 from CosmWasm/multiarch-builders

Create universal library with for ARM and Intel
CosmWasm · Feb 23, 2022 · b431b63 · b431b63
2 parents 1624bde + f42579b
commit b431b63
Show file tree

Hide file tree

Showing 15 changed files with 211 additions and 79 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -11,6 +11,10 @@ insert_final_newline = true
 indent_style = space
 indent_size = 4
 
+[*.py]
+indent_style = space
+indent_size = 4
+
 [{Makefile,*.go}]
 indent_style = tab
 indent_size = 4
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
-target/
+# Please also see libwasmvm/.gitignore for the Rust-specific ignored files.
+
 **/*.rs.bk
 *.iml
 .idea
@@ -7,7 +8,6 @@ target/
 lib*.a
 
 # artifacts from compile tests
-artifacts/
 /demo
 tmp
 a.out
diff --git a/Makefile b/Makefile
@@ -1,9 +1,9 @@
 .PHONY: all build build-rust build-go test
 
 # Builds the Rust library libwasmvm
-BUILDERS_PREFIX := cosmwasm/go-ext-builder:0008
+BUILDERS_PREFIX := cosmwasm/go-ext-builder:0009
 # Contains a full Go dev environment in order to run Go tests on the built library
-ALPINE_TESTER := cosmwasm/go-ext-builder:0008-alpine
+ALPINE_TESTER := cosmwasm/go-ext-builder:0009-alpine
 
 USER_ID := $(shell id -u)
 USER_GROUP = $(shell id -g)
@@ -73,10 +73,10 @@ release-build-linux:
 
 # Creates a release build in a containerized build environment of the shared library for macOS (.dylib)
 release-build-macos:
-	rm -rf libwasmvm/target/release
-	docker run --rm -u $(USER_ID):$(USER_GROUP) -v $(shell pwd)/libwasmvm:/code $(BUILDERS_PREFIX)-cross
-	cp libwasmvm/target/x86_64-apple-darwin/release/deps/libwasmvm.dylib api
-	cp libwasmvm/bindings.h api
+	rm -rf libwasmvm/target/x86_64-apple-darwin/release
+	rm -rf libwasmvm/target/aarch64-apple-darwin/release
+	docker run --rm -u $(USER_ID):$(USER_GROUP) -v $(shell pwd)/libwasmvm:/code $(BUILDERS_PREFIX)-cross build_macos.sh
+	cp libwasmvm/artifacts/libwasmvm.dylib api
 	make update-bindings
 
 update-bindings:

diff --git a/README.md b/README.md
@@ -14,39 +14,45 @@ link with, and Go developers should just be able to import this directly.
 
 ## Supported Platforms
 
-Requires Rust 1.51+, Requires Go 1.17+
-
-Since this package includes a rust prebuilt dll, you cannot just import the go code,
-but need to be on a system that works with an existing dll. Currently this is Linux
-(tested on Ubuntu, Debian, and CentOS7) and MacOS. We have a build system for Windows,
-but it is [not supported][wasmer_support] by the Wasmer Singlepass backend which we rely upon.
-
-[wasmer_support]: https://docs.wasmer.io/ecosystem/wasmer/wasmer-features
-
-### Overview
-
-|               | [x86]               | [x86_64]            | [ARM32]              | [ARM64]              |
-| ------------- | ------------------- | ------------------- | -------------------- | -------------------- |
-| Linux (glibc) | ❌‍                 | ✅                  | ❌‍ <sub>[#53]</sub> | ❌‍ <sub>[#53]</sub> |
-| Linux (muslc) | ❌‍                 | ✅                  | ❌‍ <sub>[#53]</sub> | ❌‍ <sub>[#53]</sub> |
-| macOS         | ❌‍                 | ✅                  | ❌‍ <sub>[#53]</sub> | ❌‍ <sub>[#53]</sub> |
-| Windows       | ❌ <sub>[#28]</sub> | ❌ <sub>[#28]</sub> | ❌ <sub>[#28]</sub>  | ❌ <sub>[#28]</sub>  |
-
-[x86]: https://en.wikipedia.org/wiki/X86
-[x86_64]: https://en.wikipedia.org/wiki/X86-64
-[arm32]: https://en.wikipedia.org/wiki/AArch32
-[arm64]: https://en.wikipedia.org/wiki/AArch64
-[#28]: https://github.com/CosmWasm/wasmvm/issues/28
-[#53]: https://github.com/CosmWasm/wasmvm/issues/53
-
-✅ Supported and activly maintained.
-
-❌ Blocked by external dependency.
-
-🤷‍ Not supported because nobody cares so far. Feel free to look into it.
-
-This is all blocked on [wasmer support for singlepass backend](https://docs.wasmer.io/ecosystem/wasmer/wasmer-features#compiler-support-by-chipset).
-We can only move on these wasmvm issues when the upstream has support.
+Requires Rust 1.55+ and Go 1.17+.
+
+The Rust implementation of the VM is compiled to a library called libwasmvm. This is
+then linked to the Go code when the final binary is built. For that reason not all
+systems supported by Go are supported by this project.
+
+Linux (tested on Ubuntu, Debian, and CentOS7, Alpine) and macOS is supported.
+We are working on Windows (#288).
+
+[#288]: https://github.com/CosmWasm/wasmvm/pull/288
+
+### Builds of libwasmvm
+
+Our system currently supports the following builds. In general we can only support targets
+that are [supported by Wasmer's singlepass backend](https://docs.wasmer.io/ecosystem/wasmer/wasmer-features#compiler-support-by-chipset),
+which for example excludes all 32 bit systems.
+
+<!-- AUTO GENERATED BY libwasmvm_builds.py START -->
+
+| OS family       | Arch    | Linking | Supported            | Wasmer 2.2+                  | Note                                                                                                                                   |
+| --------------- | ------- | ------- | -------------------- | ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- |
+| Linux (glibc)   | x86_64  | shared  | ✅libwasmvm.so      | ✅libwasmvm.so              |                                                                                                                                        |
+| Linux (glibc)   | x86_64  | static  | 🚫                  | 🚫                          | Would link libwasmvm statically but glibc dynamically as static glibc linking is not recommended. Potentially interesting for Osmosis. |
+| Linux (glibc)   | aarch64 | shared  | 🚫                  | ✅libwasmvm.aarch64.so      |                                                                                                                                        |
+| Linux (glibc)   | aarch64 | static  | 🚫                  | 🚫                          |                                                                                                                                        |
+| Linux (musl)    | x86_64  | shared  | 🚫                  | 🚫                          | Possible but not needed                                                                                                                |
+| Linux (musl)    | x86_64  | static  | ✅libwasmvm_muslc.a | ✅libwasmvm_muslc.a         |                                                                                                                                        |
+| Linux (musl)    | aarch64 | shared  | 🚫                  | 🚫                          | Possible but not needed                                                                                                                |
+| Linux (musl)    | aarch64 | static  | 🚫                  | ✅libwasmvm_muslc.aarch64.a |                                                                                                                                        |
+| macOS           | x86_64  | shared  | ✅libwasmvm.dylib   | ✅libwasmvm.dylib           | Fat/universal library with multiple archs (#294)                                                                                       |
+| macOS           | x86_64  | static  | 🚫                  | 🚫                          |                                                                                                                                        |
+| macOS           | aarch64 | shared  | 🚫                  | ✅libwasmvm.dylib           | Fat/universal library with multiple archs (#294)                                                                                       |
+| macOS           | aarch64 | static  | 🚫                  | 🚫                          |                                                                                                                                        |
+| Windows (mingw) | x86_64  | shared  | 🏗wasmvm.dll         | 🏗wasmvm.dll                 | See #288                                                                                                                               |
+| Windows (mingw) | x86_64  | static  | 🚫                  | 🚫                          |                                                                                                                                        |
+| Windows (mingw) | aarch64 | shared  | 🚫                  | 🚫                          |                                                                                                                                        |
+| Windows (mingw) | aarch64 | static  | 🚫                  | 🚫                          |                                                                                                                                        |
+
+<!-- AUTO GENERATED BY libwasmvm_builds.py END -->
 
 ## Docs
 

diff --git a/builders/Dockerfile.alpine b/builders/Dockerfile.alpine
@@ -2,7 +2,7 @@
 # 1. Build the static Rust library
 # 2. Execute Go tests that use and test this library
 # For 2. we define the Go image here. For 1. we install Rust below.
-FROM golang:1.17.5-alpine
+FROM golang:1.17.7-alpine
 
 ENV RUSTUP_HOME=/usr/local/rustup \
     CARGO_HOME=/usr/local/cargo \

diff --git a/builders/Dockerfile.centos7 b/builders/Dockerfile.centos7
@@ -1,23 +1,21 @@
 FROM centos:centos7
 
-RUN yum -y update
-RUN yum -y install clang gcc gcc-c++ make wget
+RUN yum -y update \
+  && yum -y install clang gcc gcc-c++ make wget
 
 # GET FROM https://github.com/rust-lang/docker-rust-nightly
 ENV RUSTUP_HOME=/usr/local/rustup \
     CARGO_HOME=/usr/local/cargo \
     PATH=/usr/local/cargo/bin:$PATH
 
-
-RUN url="https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init"; \
-    wget "$url"; \
-    chmod +x rustup-init; \
-    ./rustup-init -y --no-modify-path --default-toolchain 1.55.0; \
-    rm rustup-init; \
-    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
-    rustup --version; \
-    cargo --version; \
-    rustc --version;
+RUN wget "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init" \
+  && chmod +x rustup-init \
+  && ./rustup-init -y --no-modify-path --default-toolchain 1.55.0 \
+  && rm rustup-init \
+  && chmod -R a+w $RUSTUP_HOME $CARGO_HOME \
+  && rustup --version \
+  && cargo --version \
+  && rustc --version
 
 # allow non-root user to download more deps later
 RUN chmod -R 777 /usr/local/cargo

diff --git a/builders/Dockerfile.cross b/builders/Dockerfile.cross
@@ -1,45 +1,52 @@
-FROM rust:1.55.0-buster
+FROM rust:1.55.0-bullseye
 
 # Install build dependencies
-RUN apt-get update
-RUN apt install -y clang gcc g++ zlib1g-dev libmpc-dev libmpfr-dev libgmp-dev
-RUN apt install -y build-essential cmake
-
-# add some llvm configs for later - how to cross-compile this in wasmer-llvm-backend???
-RUN echo deb http://deb.debian.org/debian buster-backports main >> /etc/apt/sources.list
-RUN apt-get update
-RUN apt install -y libllvm8 llvm-8 llvm-8-dev llvm-8-runtime
-ENV LLVM_SYS_80_PREFIX=/usr/lib/llvm-8
+RUN apt-get update \
+  && apt install -y clang gcc g++ zlib1g-dev libmpc-dev libmpfr-dev libgmp-dev build-essential cmake
 
 ## ADD MACOS SUPPORT
 
 WORKDIR /opt
 
-# Add macOS Rust target
-RUN rustup target add x86_64-apple-darwin
+# Add macOS Rust targets
+RUN rustup target add x86_64-apple-darwin aarch64-apple-darwin
 
 # Build osxcross
-RUN git clone https://github.com/tpoechtrager/osxcross
-RUN cd osxcross && \
-	wget -nc https://s3.dockerproject.org/darwin/v2/MacOSX10.10.sdk.tar.xz && \
-    mv MacOSX10.10.sdk.tar.xz tarballs/ && \
-    UNATTENDED=yes OSX_VERSION_MIN=10.10 ./build.sh
+# See https://github.com/tpoechtrager/osxcross/blob/master/build.sh#L31-L49 for SDK overview.
+#
+# SDK availability is tricky. There is 10.10 and 10.11. at
+#   - https://s3.dockerproject.org/darwin/v2/MacOSX10.10.sdk.tar.xz
+#   - https://s3.dockerproject.org/darwin/v2/MacOSX10.11.sdk.tar.xz
+# and we have https://github.com/phracker/MacOSX-SDKs/releases.
+# At some point we might want to use our own package.
+RUN git clone https://github.com/tpoechtrager/osxcross \
+  && cd osxcross \
+  # Don't change file name when downloading because osxcross auto-detects the version from the name
+  && wget -nc https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.3.sdk.tar.xz \
+  && mv MacOSX11.3.sdk.tar.xz tarballs/ \
+  && UNATTENDED=yes OSX_VERSION_MIN=10.10 ./build.sh \
+  # Cleanups before Docker layer is finalized
+  && rm -r tarballs/
 RUN chmod +rx /opt/osxcross
 RUN chmod +rx /opt/osxcross/target
 RUN chmod -R +rx /opt/osxcross/target/bin
 
+# RUN ls -l /opt/osxcross/target/bin
+RUN /opt/osxcross/target/bin/x86_64-apple-darwin20.4-clang --version
+RUN /opt/osxcross/target/bin/aarch64-apple-darwin20.4-clang --version
+
 # allow non-root user to download more deps later
 RUN chmod -R 777 /usr/local/cargo
 
 ## COPY BUILD SCRIPTS
 
 WORKDIR /code
 
-COPY guest/*.sh /opt/
-RUN chmod +x /opt/*.sh
+COPY guest/*.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/*.sh
 
 RUN mkdir /.cargo
 RUN chmod +rx /.cargo
 COPY guest/cargo-config /.cargo/config
 
-CMD ["/opt/build_macos.sh"]
+CMD ["bash", "-c", "echo 'Argument missing. Pass one build script (e.g. build_macos.sh) to docker run' && exit 1"]
diff --git a/builders/Makefile b/builders/Makefile
@@ -1,6 +1,6 @@
 # Versioned by a simple counter that is not bound to a specific CosmWasm version
 # See builders/README.md
-BUILDERS_PREFIX := cosmwasm/go-ext-builder:0008
+BUILDERS_PREFIX := cosmwasm/go-ext-builder:0009
 
 .PHONY: docker-image-centos7
 docker-image-centos7:

diff --git a/builders/README.md b/builders/README.md
@@ -14,6 +14,11 @@ can do the cross-compilation.
 
 ## Changelog
 
+**Version 0009:**
+
+- Let macOS build dylib files with both aarch64 and x86_64 code.
+- Update Go (for testing only) to 1.17.7.
+
 **Version 0008:**
 
 - Update Rust to 1.55.0 and Go (for testing only) to 1.17.5.

diff --git a/builders/guest/build_linux.sh b/builders/guest/build_linux.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+set -o errexit -o nounset -o pipefail
 
 # See https://github.com/CosmWasm/wasmvm/issues/222#issuecomment-880616953 for two approaches to
 # enable stripping through cargo (if that is desired).

diff --git a/builders/guest/build_macos.sh b/builders/guest/build_macos.sh
@@ -1,12 +1,24 @@
 #!/bin/bash
+set -o errexit -o nounset -o pipefail
 
 # ref: https://wapl.es/rust/2019/02/17/rust-cross-compile-linux-to-macos.html
 export PATH="/opt/osxcross/target/bin:$PATH"
 export LIBZ_SYS_STATIC=1
-export CC=o64-clang
-export CXX=o64-clang++
 
 # See https://github.com/CosmWasm/wasmvm/issues/222#issuecomment-880616953 for two approaches to
 # enable stripping through cargo (if that is desired).
 
+echo "Starting aarch64-apple-darwin build"
+export CC=aarch64-apple-darwin20.4-clang
+export CXX=aarch64-apple-darwin20.4-clang++
+cargo build --release --target aarch64-apple-darwin
+
+echo "Starting x86_64-apple-darwin build"
+export CC=o64-clang
+export CXX=o64-clang++
 cargo build --release --target x86_64-apple-darwin
+
+# Create a universal library with both archs
+lipo -output artifacts/libwasmvm.dylib -create \
+  target/x86_64-apple-darwin/release/deps/libwasmvm.dylib \
+  target/aarch64-apple-darwin/release/deps/libwasmvm.dylib
diff --git a/builders/guest/cargo-config b/builders/guest/cargo-config
@@ -1,3 +1,7 @@
 [target.x86_64-apple-darwin]
-linker = "x86_64-apple-darwin14-clang"
-ar = "x86_64-apple-darwin14-ar"
+linker = "x86_64-apple-darwin20.4-clang"
+ar = "x86_64-apple-darwin20.4-ar"
+
+[target.aarch64-apple-darwin]
+linker = "aarch64-apple-darwin20.4-clang"
+ar = "aarch64-apple-darwin20.4-ar"