feat(build): link openssl statically (#2984)

Signed-off-by: Alex Chi <[email protected]>

.config/hakari.toml

@@ -22,3 +22,6 @@ platforms = [
 
 # Write out exact versions rather than a semver range. (Defaults to false.)
 # exact-versions = true
+
+[traversal-excludes]
+workspace-members = ["workspace-static-link"]

.github/aws-config/Dockerfile

@@ -12,6 +12,5 @@ CMD bash
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update -y && apt-get install -yy ca-certificates lsb-release wget
-RUN apt-get install -yy curl lcov git unzip zip libssl-dev zlib1g-dev cmake build-essential
+RUN apt-get install -yy curl lcov git unzip zip zlib1g-dev cmake build-essential
 RUN apt-get install -yy pkg-config make golang psmisc
-

.github/workflows/main.yml

@@ -1,4 +1,3 @@
-
 # ================= THIS FILE IS AUTOMATICALLY GENERATED =================
 #
 # To edit this file, please refer to the instructions in CONTRIBUTING.md.
@@ -510,7 +509,7 @@ jobs:
           cargo fmt --all -- --check
       - name: Build Rust components
         run: |
-          cargo build -p risingwave_cmd_all -p risedev -p risingwave_regress_test --profile dev
+          cargo build -p risingwave_cmd_all -p risedev -p risingwave_regress_test --features static-link --profile dev
       - name: Compress RisingWave debug info
         run: |
           objcopy --compress-debug-sections=zlib-gnu target/debug/risingwave
@@ -582,7 +581,7 @@ jobs:
           cargo fmt --all -- --check
       - name: Build Rust components
         run: |
-          cargo build -p risingwave_cmd_all -p risedev -p risingwave_regress_test --profile release
+          cargo build -p risingwave_cmd_all -p risedev -p risingwave_regress_test --features static-link --profile release
       - name: Compress RisingWave debug info
         run: |
           objcopy --compress-debug-sections=zlib-gnu target/release/risingwave
@@ -641,7 +640,7 @@ jobs:
       - name: Run rust clippy check
         run: |
           # If new CI checks are added, the one with `--locked` must be run first.
-          cargo clippy --all-targets --all-features --locked -- -D warnings
+          cargo clippy --all-targets --features failpoints --locked -- -D warnings
       - name: Build documentation
         run: |
           cargo doc --document-private-items --no-deps
@@ -739,7 +738,6 @@ jobs:
           asset_path: risingwave-${{ github.ref_name }}-x86_64-unknown-linux.tar.gz
           asset_name: risingwave-${{ github.ref_name }}-x86_64-unknown-linux.tar.gz
           asset_content_type: application/tar+gzip
-
 # ================= THIS FILE IS AUTOMATICALLY GENERATED =================
 #
 # To edit this file, please refer to the instructions in CONTRIBUTING.md.

Cargo.toml

@@ -28,6 +28,7 @@ members = [
   "src/utils/logging",
   "src/utils/memcomparable",
   "src/utils/pgwire",
+  "src/utils/static-link",
   "src/utils/value-encoding",
   "src/workspace-hack",
 ]

Makefile.toml

@@ -345,7 +345,7 @@ script = """
 
 set -e
 
-for tool in cargo-llvm-cov cargo-nextest cargo-udeps cargo-hakari cargo-sort
+for tool in cargo-llvm-cov cargo-nextest cargo-udeps cargo-hakari cargo-sort cargo-make
 do
   echo "install: $(tput setaf 4)$tool$(tput sgr0)"
   cargo install $tool

ci/Dockerfile

@@ -4,7 +4,7 @@ ENV LANG en_US.utf8
 
 RUN apt-get update && \
     DEBIAN_FRONTEND=noninteractive apt-get -y install make build-essential cmake protobuf-compiler curl \
-    openssl libssl-dev pkg-config bash openjdk-11-jdk wget unzip git tmux lld
+    openssl libssl-dev libcurl4-openssl-dev pkg-config bash openjdk-11-jdk wget unzip git tmux lld
 
 SHELL ["/bin/bash", "-c"]
 

ci/scripts/build.sh

@@ -38,11 +38,14 @@ echo "--- Rust format check"
 cargo fmt --all -- --check
 
 echo "--- Build Rust components"
-cargo build -p risingwave_cmd_all -p risedev -p risingwave_regress_test --profile "$profile"
+cargo build -p risingwave_cmd_all -p risedev -p risingwave_regress_test --features static-link --profile "$profile"
 
 echo "--- Compress RisingWave debug info"
 objcopy --compress-debug-sections=zlib-gnu target/"$target"/risingwave
 
+echo "--- Show link info"
+ldd target/"$target"/risingwave
+
 echo "--- Upload artifacts"
 cp target/"$target"/risingwave ./risingwave-"$profile"
 cp target/"$target"/risedev-playground ./risedev-playground-"$profile"

ci/scripts/unit-test.sh

@@ -10,13 +10,13 @@ rustup default "$(cat ./rust-toolchain)" && rustup component add llvm-tools-prev
 cargo install cargo-llvm-cov
 
 echo "--- Run clippy check"
-cargo clippy --all-targets --all-features --locked -- -D warnings
+cargo clippy --all-targets --features failpoints --locked -- -D warnings
 
 echo "--- Build documentation"
 cargo doc --document-private-items --no-deps
 
 echo "--- Run unit tests with coverage"
-cargo llvm-cov nextest --lcov --output-path lcov.info --features=failpoints -- --no-fail-fast
+cargo llvm-cov nextest --lcov --output-path lcov.info --features failpoints -- --no-fail-fast
 
 echo "--- Run doctest"
 cargo test --doc

docker/Dockerfile

@@ -2,7 +2,7 @@ FROM ubuntu:22.04 as builder
 
 ENV LANG en_US.utf8
 
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install make build-essential cmake protobuf-compiler curl openssl libssl-dev pkg-config bash lld
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install make build-essential cmake protobuf-compiler curl pkg-config bash lld
 
 SHELL ["/bin/bash", "-c"]
 
@@ -24,12 +24,12 @@ RUN rustup self update \
 
 RUN cargo fetch
 RUN mkdir -p /risingwave/bin
-RUN cargo build -p risingwave_cmd -p risingwave_cmd_all --release && \
+RUN cargo build -p risingwave_cmd -p risingwave_cmd_all --release --features static-link && \
   mv /risingwave/target/release/{frontend,compute-node,meta-node,compactor,risingwave} /risingwave/bin/ && \
   cargo clean
 
 FROM ubuntu:22.04 as image-base
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install openssl libssl-dev ca-certificates && rm -rf /var/lib/{apt,dpkg,cache,log}/
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install ca-certificates && rm -rf /var/lib/{apt,dpkg,cache,log}/
 
 FROM image-base as frontend-node
 RUN mkdir -p /risingwave/bin

docker/aws/Dockerfile

@@ -1,5 +1,5 @@
 FROM ubuntu:22.04
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install openssl libssl-dev ca-certificates && rm -rf /var/lib/{apt,dpkg,cache,log}/
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install ca-certificates && rm -rf /var/lib/{apt,dpkg,cache,log}/
 
 RUN mkdir -p /risingwave/bin
 COPY risingwave /risingwave/bin/risingwave

docker/aws/aws-build.sh

@@ -10,7 +10,7 @@ if ! [[ "$DIR" =~ ^/ebs.* ]] ; then
 fi
 
 cd "$DIR/../.."
-cargo build -p risingwave_cmd_all --release
+cargo build -p risingwave_cmd_all --release --features static-link
 objcopy --compress-debug-sections=zlib-gnu target/release/risingwave "$DIR/risingwave"
 
 cd "$DIR"

docs/developer-guide.md

@@ -73,7 +73,7 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
 To install the dependencies on Debian-based Linux systems, run:
 
 ```shell
-sudo apt install make build-essential cmake protobuf-compiler curl openssl libssl-dev pkg-config postgresql-client tmux lld
+sudo apt install make build-essential cmake protobuf-compiler curl openssl libssl-dev libcurl4-openssl-dev pkg-config postgresql-client tmux lld
 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
 ```
 

src/cmd/Cargo.toml

@@ -3,6 +3,9 @@ name = "risingwave_cmd"
 version = "0.1.8"
 edition = "2021"
 
+[features]
+static-link = ["workspace-static-link"]
+
 [dependencies]
 clap = { version = "3", features = ["derive"] }
 log = { version = "0.4", features = ["release_max_level_info"] }
@@ -25,6 +28,7 @@ tokio = { version = "=0.2.0-alpha.3", package = "madsim-tokio", features = [
 ] }
 tracing = { version = "0.1", features = ["release_max_level_info"] }
 workspace-hack = { version = "0.1", path = "../workspace-hack" }
+workspace-static-link = { path = "../utils/static-link", optional = true, features = ["enabled"] }
 
 [[bin]]
 name = "frontend"

src/cmd_all/Cargo.toml

@@ -3,6 +3,9 @@ name = "risingwave_cmd_all"
 version = "0.1.8"
 edition = "2021"
 
+[features]
+static-link = ["workspace-static-link"]
+
 [dependencies]
 anyhow = "1"
 clap = { version = "3", features = ["derive"] }
@@ -27,6 +30,7 @@ tokio = { version = "=0.2.0-alpha.3", package = "madsim-tokio", features = [
 ] }
 tracing = { version = "0.1", features = ["release_max_level_info"] }
 workspace-hack = { version = "0.1", path = "../workspace-hack" }
+workspace-static-link = { path = "../utils/static-link", optional = true, features = ["enabled"] }
 
 [[bin]]
 name = "risingwave"

src/object_store/Cargo.toml

@@ -25,3 +25,4 @@ tokio = { version = "=0.2.0-alpha.3", package = "madsim-tokio", features = [
     "fs",
 ] }
 tracing = { version = "0.1" }
+workspace-hack = { version = "0.1", path = "../workspace-hack" }

src/risedevtool/Cargo.toml

@@ -12,7 +12,7 @@ console = "0.15"
 dialoguer = "0.10"
 enum-iterator = "1"
 indicatif = "0.16"
-isahc = "1"
+isahc = { version = "1", default-features = false, features = ["text-decoding"] }
 itertools = "0.10"
 kafka = { version = "0.9", default-features = false }
 regex = "1"

src/utils/logging/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 [dependencies]
 async-trait = "0.1"
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
-isahc = "1" # isahc is the http client used for tracing. Always set it as the same version as opentelemetry-jaeger's.
+isahc = { version = "1", default-features = false } # isahc is the http client used for tracing. Always set it as the same version as opentelemetry-jaeger's.
 madsim = "=0.2.0-alpha.3"
 opentelemetry = { version = "0.17", features = ["rt-tokio", "trace"] }
 opentelemetry-jaeger = { version = "0.16", features = [

src/utils/static-link/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "workspace-static-link"
+version = "0.1.8"
+edition = "2021"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[features]
+# Use a feature flag to control enable or not, otherwise `cargo test` will compile all dependencies.
+enabled = ["isahc", "openssl"]
+
+[dependencies]
+isahc = { version = "1", optional = true, default-features = false, features = ["static-ssl", "static-curl"] }
+openssl = { version = "0.10", optional = true, features = ["vendored"] }
+# workspace-hack = { version = "0.1", path = "../../workspace-hack" }
+# Don't add workspace-hack into this crate!

src/utils/static-link/src/lib.rs

@@ -0,0 +1,15 @@
+// Copyright 2022 Singularity Data
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! This crate includes dependencies that need to be statically-linked.

src/workspace-hack/Cargo.toml

@@ -19,8 +19,7 @@ bstr = { version = "0.2", features = ["lazy_static", "regex-automata", "serde",
 bytes = { version = "1", features = ["serde", "std"] }
 chrono = { version = "0.4", features = ["clock", "libc", "oldtime", "std", "time", "winapi"] }
 crossbeam-deque = { version = "0.8", features = ["crossbeam-epoch", "crossbeam-utils", "std"] }
-crossbeam-utils-c38e5c1d305a1b54 = { package = "crossbeam-utils", version = "0.8", features = ["lazy_static", "std"] }
-crossbeam-utils-ca01ad9e24f5d932 = { package = "crossbeam-utils", version = "0.7", features = ["lazy_static", "std"] }
+crossbeam-utils = { version = "0.7", features = ["lazy_static", "std"] }
 crypto-common = { version = "0.1", default-features = false, features = ["std"] }
 digest = { version = "0.10", features = ["alloc", "block-buffer", "core-api", "mac", "std", "subtle"] }
 either = { version = "1", features = ["use_std"] }
@@ -36,6 +35,7 @@ futures-util = { version = "0.3", features = ["alloc", "async-await", "async-awa
 hashbrown = { version = "0.11", features = ["ahash", "inline-more", "raw"] }
 hyper = { version = "0.14", features = ["client", "full", "h2", "http1", "http2", "runtime", "server", "socket2", "stream", "tcp"] }
 indexmap = { version = "1", default-features = false, features = ["std"] }
+isahc = { version = "1", default-features = false, features = ["encoding_rs", "mime", "text-decoding"] }
 libc = { version = "0.2", features = ["std"] }
 libz-sys = { version = "1", features = ["libc", "stock-zlib"] }
 lock_api = { version = "0.4", default-features = false, features = ["arc_lock"] }
@@ -73,8 +73,7 @@ bytes = { version = "1", features = ["serde", "std"] }
 cc = { version = "1", default-features = false, features = ["jobserver", "parallel"] }
 chrono = { version = "0.4", features = ["clock", "libc", "oldtime", "std", "time", "winapi"] }
 crossbeam-deque = { version = "0.8", features = ["crossbeam-epoch", "crossbeam-utils", "std"] }
-crossbeam-utils-c38e5c1d305a1b54 = { package = "crossbeam-utils", version = "0.8", features = ["lazy_static", "std"] }
-crossbeam-utils-ca01ad9e24f5d932 = { package = "crossbeam-utils", version = "0.7", features = ["lazy_static", "std"] }
+crossbeam-utils = { version = "0.7", features = ["lazy_static", "std"] }
 crypto-common = { version = "0.1", default-features = false, features = ["std"] }
 digest = { version = "0.10", features = ["alloc", "block-buffer", "core-api", "mac", "std", "subtle"] }
 either = { version = "1", features = ["use_std"] }
@@ -90,6 +89,7 @@ futures-util = { version = "0.3", features = ["alloc", "async-await", "async-awa
 hashbrown = { version = "0.11", features = ["ahash", "inline-more", "raw"] }
 hyper = { version = "0.14", features = ["client", "full", "h2", "http1", "http2", "runtime", "server", "socket2", "stream", "tcp"] }
 indexmap = { version = "1", default-features = false, features = ["std"] }
+isahc = { version = "1", default-features = false, features = ["encoding_rs", "mime", "text-decoding"] }
 libc = { version = "0.2", features = ["std"] }
 libz-sys = { version = "1", features = ["libc", "stock-zlib"] }
 lock_api = { version = "0.4", default-features = false, features = ["arc_lock"] }