[improve][pip] PIP-324: Alpine image (#22054)

.github/workflows/pulsar-ci.yaml

@@ -508,15 +508,10 @@ jobs:
           cd $HOME
           $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
 
-      - name: Pick ubuntu mirror for the docker image build
-        run: |
-          # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable
-          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror
-
       - name: Build java-test-image docker image
         run: |
           # build docker image
-          mvn -B -am -pl tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \
+          mvn -B -am -pl docker/pulsar,tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \
           -Dmaven.test.skip=true -Ddocker.squash=true -DskipSourceReleaseAssembly=true \
           -Dspotbugs.skip=true  -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
 
@@ -868,17 +863,11 @@ jobs:
           cd $HOME
           $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
 
-      - name: Pick ubuntu mirror for the docker image build
-        run: |
-          # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable
-          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror
-
       - name: Build latest-version-image docker image
         run: |
           # build docker image
           # include building of Connectors, Offloaders and server distros
           mvn -B -am -pl distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \
-          -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" -DIMAGE_JDK_MAJOR_VERSION="${IMAGE_JDK_MAJOR_VERSION}" \
           -Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \
           -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
 

build/build_java_test_image.sh

@@ -27,6 +27,5 @@ if [[ "$(docker version -f '{{.Server.Experimental}}' 2>/dev/null)" == "true" ]]
   SQUASH_PARAM="-Ddocker.squash=true"
 fi
 mvn -am -pl tests/docker-images/java-test-image -Pcore-modules,-main,integrationTests,docker \
-  -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \
   -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true $SQUASH_PARAM \
   "$@" install

build/pulsar_ci_tool.sh

@@ -46,59 +46,14 @@ function ci_print_thread_dumps() {
 
 # runs maven
 function _ci_mvn() {
-  mvn -B -ntp -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \
-        "$@"
+  mvn -B -ntp "$@"
 }
 
 # runs OWASP Dependency Check for all projects
 function ci_dependency_check() {
   _ci_mvn -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl '!pulsar-client-tools-test' "$@"
 }
 
-# Finds fastest up-to-date ubuntu mirror based on download speed
-function ci_find_fast_ubuntu_mirror() {
-  local ubuntu_release=${1:-"$(lsb_release -c 2>/dev/null | cut -f2 || echo "jammy")"}
-  local ubuntu_arch=${2:-"$(dpkg --print-architecture 2>/dev/null || echo "amd64")"}
-  {
-    # choose mirrors that are up-to-date by checking the Last-Modified header for
-    {
-      # randomly choose up to 10 mirrors using http:// protocol
-      # (https isn't supported in docker containers that don't have ca-certificates installed)
-      curl -s http://mirrors.ubuntu.com/mirrors.txt | grep '^http://' | shuf -n 10
-      # also consider Azure's Ubuntu mirror
-      echo http://azure.archive.ubuntu.com/ubuntu/
-    } | xargs -I {} sh -c "ubuntu_release=$ubuntu_release ubuntu_arch=$ubuntu_arch;"'echo "$(curl -m 5 -sI {}dists/${ubuntu_release}/Contents-${ubuntu_arch}.gz|sed s/\\r\$//|grep Last-Modified|awk -F": " "{ print \$2 }" | LANG=C date -f- -u +%s)" "{}"' | sort -rg | awk '{ if (NR==1) TS=$1; if ($1 == TS) print $2 }'
-  } | xargs -I {} sh -c 'echo `curl -r 0-102400 -m 5 -s -w %{speed_download} -o /dev/null {}ls-lR.gz` {}' \
-    |sort -g -r |head -1| awk '{ print $2  }'
-}
-
-function ci_pick_ubuntu_mirror() {
-  echo "Choosing fastest up-to-date ubuntu mirror based on download speed..."
-  UBUNTU_MIRROR=$(ci_find_fast_ubuntu_mirror)
-  if [ -z "$UBUNTU_MIRROR" ]; then
-      # fallback to no mirror
-      UBUNTU_MIRROR="http://archive.ubuntu.com/ubuntu/"
-      UBUNTU_SECURITY_MIRROR="http://security.ubuntu.com/ubuntu/"
-  else
-      UBUNTU_SECURITY_MIRROR="${UBUNTU_MIRROR}"
-  fi
-  OLD_MIRROR=$(cat /etc/apt/sources.list | grep '^deb ' | head -1 | awk '{ print $2 }')
-  echo "Picked '$UBUNTU_MIRROR'. Current mirror is '$OLD_MIRROR'."
-  if [[ "$OLD_MIRROR" != "$UBUNTU_MIRROR" ]]; then
-    sudo sed -i "s|$OLD_MIRROR|$UBUNTU_MIRROR|g" /etc/apt/sources.list
-    sudo apt-get update
-  fi
-  # set the chosen mirror also in the UBUNTU_MIRROR and UBUNTU_SECURITY_MIRROR environment variables
-  # that can be used by docker builds
-  export UBUNTU_MIRROR
-  export UBUNTU_SECURITY_MIRROR
-  # make environment variables available for later GitHub Actions steps
-  if [ -n "$GITHUB_ENV" ]; then
-    echo "UBUNTU_MIRROR=$UBUNTU_MIRROR" >> $GITHUB_ENV
-    echo "UBUNTU_SECURITY_MIRROR=$UBUNTU_SECURITY_MIRROR" >> $GITHUB_ENV
-  fi
-}
-
 # installs a tool executable if it's not found on the PATH
 function ci_install_tool() {
   local tool_executable=$1
@@ -108,7 +63,6 @@ function ci_install_tool() {
       echo "::group::Installing ${tool_package}"
       sudo apt-get -y install ${tool_package} >/dev/null || {
         echo "Installing the package failed. Switching the ubuntu mirror and retrying..."
-        ci_pick_ubuntu_mirror
         # retry after picking the ubuntu mirror
         sudo apt-get -y install ${tool_package}
       }

docker/glibc-package/Dockerfile

@@ -0,0 +1,79 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+ARG GLIBC_VERSION=2.38
+
+FROM ubuntu:22.04 as build
+ARG GLIBC_VERSION
+
+RUN apt-get -q update \
+	&& apt-get -qy install \
+		bison \
+		build-essential \
+		gawk \
+		gettext \
+		openssl \
+		python3 \
+		texinfo \
+		wget
+
+# Build GLibc
+RUN wget -qO- https://ftpmirror.gnu.org/libc/glibc-${GLIBC_VERSION}.tar.gz | tar zxf -
+RUN mkdir /glibc-build
+WORKDIR /glibc-build
+RUN /glibc-${GLIBC_VERSION}/configure \
+        			--prefix=/usr/glibc-compat \
+        			--libdir=/usr/glibc-compat/lib \
+        			--libexecdir=/usr/glibc-compat/lib \
+        			--enable-multi-arch \
+        			--enable-stack-protector=strong
+RUN make -j$(nproc)
+RUN make install
+RUN tar --dereference --hard-dereference -zcf /glibc-bin.tar.gz /usr/glibc-compat
+
+
+################################################
+## Build the APK package
+FROM alpine:3.19 as apk
+ARG GLIBC_VERSION
+
+RUN apk add abuild sudo build-base
+
+RUN mkdir /build
+WORKDIR build
+
+COPY --from=build /glibc-bin.tar.gz /build
+
+COPY ./scripts /build
+
+RUN echo "pkgver=\"${GLIBC_VERSION}\"" >> /build/APKBUILD
+RUN echo "sha512sums=\"$(sha512sum glibc-bin.tar.gz ld.so.conf)\"" >> /build/APKBUILD
+
+RUN abuild-keygen -a -i -n
+RUN abuild -F -c -r
+
+################################################
+## Last stage - Only leaves the packages
+FROM busybox
+ARG GLIBC_VERSION
+
+RUN mkdir -p /root/packages
+COPY --from=apk /root/packages/*/glibc-${GLIBC_VERSION}-r0.apk /root/packages
+COPY --from=apk /root/packages/*/glibc-bin-${GLIBC_VERSION}-r0.apk /root/packages

build/docker/README.md → docker/glibc-package/README.md

@@ -19,30 +19,21 @@
 
 -->
 
-This folder contains a Docker image that can used to compile the Pulsar C++ client library
-and website in a reproducible environment.
+# GLibc compatibility package
 
-```shell
-docker build -t pulsar-build .
-```
-
-The image is already available at https://hub.docker.com/r/apachepulsar/pulsar-build
-
-Example: `apachepulsar/pulsar-build:ubuntu-16.04`
-
-## Build and Publish pulsar-build image
+This directory includes the Docker scripts to build an image with GLibc compiled for Alpine Linux. 
 
-> Only committers have permissions on publishing pulsar images to `apachepulsar` docker hub.
+This is used to ensure plugins that are going to be used in the Pulsar image and that are depeding on GLibc, will
+still be working correctly in the Alpine Image. (eg: Netty Tc-Native and Kinesis Producer Library).
 
-### Build pulsar-build image
+This image only needs to be re-created when we want to upgrade to a newer version of GLibc.
 
+# Steps
 
-```shell
-docker build -t apachepulsar/pulsar-build:ubuntu-16.04 .
+1. Change the version in the Dockerfile for this directory.
+2. Rebuild the image and push it to Docker Hub:
 ```
-
-### Publish pulsar-build image
-
-```shell
-publish.sh
+docker buildx build --platform=linux/amd64,linux/arm64 -t apachepulsar/glibc-base:2.38 . --push
 ```
+
+The image tag is then used in `docker/pulsar/Dockerfile`.