[feat][build] Support ARM64-based docker images (apache#17733)

(cherry picked from commit 9a2aeb2)
Signed-off-by: Zixuan Liu <[email protected]>

docker/pulsar/Dockerfile

@@ -24,6 +24,7 @@ ARG PULSAR_TARBALL
 
 ADD ${PULSAR_TARBALL} /
 RUN mv /apache-pulsar-* /pulsar
+RUN rm -rf /pulsar/bin/*.cmd
 
 COPY scripts/apply-config-from-env.py /pulsar/bin
 COPY scripts/apply-config-from-env-with-prefix.py /pulsar/bin
@@ -72,7 +73,9 @@ RUN mkdir -p /etc/apt/keyrings \
      && echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list \
      && apt-get update \
      && apt-get -y dist-upgrade \
-     && apt-get -y install temurin-11-jdk
+     && apt-get -y install temurin-11-jdk \
+     && export ARCH=$(uname -m | sed -r 's/aarch64/arm64/g' |  awk '!/arm64/{$0="amd64"}1') \
+     && echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/temurin-11-jdk-$ARCH/conf/security/java.security
 
 # Cleanup apt
 RUN apt-get -y --purge autoremove \
@@ -91,13 +94,10 @@ RUN pip3 install pyyaml==5.4.1
 # 4. /pulsar - hadoop writes to this directory
 RUN mkdir /pulsar && chmod g+w /pulsar
 
-ENV JAVA_HOME /usr/lib/jvm/temurin-11-jdk-amd64
-RUN echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/temurin-11-jdk-amd64/conf/security/java.security
 ADD target/python-client/ /pulsar/pulsar-client
 
 ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE
 
-
 COPY --from=pulsar /pulsar /pulsar
 WORKDIR /pulsar
 

docker/pulsar/pom.xml

@@ -31,6 +31,7 @@
   <packaging>pom</packaging>
 
   <properties>
+    <pythonClientBuildArch>x86_64</pythonClientBuildArch>
     <skipBuildPythonClient>false</skipBuildPythonClient>
     <skipCopyPythonClients>false</skipCopyPythonClients>
   </properties>
@@ -79,7 +80,7 @@
                   <executable>${project.basedir}/../../pulsar-client-cpp/docker/build-wheels.sh</executable>
                   <arguments>
                     <!-- build python 3.8 -->
-                    <argument>3.8 cp38-cp38 manylinux2014 x86_64</argument>
+                    <argument>3.8 cp38-cp38 manylinux2014 ${pythonClientBuildArch}</argument>
                   </arguments>
                 </configuration>
               </execution>

docker/pulsar/scripts/install-pulsar-client.sh

@@ -20,6 +20,13 @@
 
 set -x
 
+# TODO: remove these lines once grpcio doesn't need to compile from source on ARM64 platform
+ARCH=$(uname -m | sed -r 's/aarch64/arm64/g' |  awk '!/arm64/{$0="amd64"}1')
+if [ "${ARCH}" == "arm64" ]; then
+  apt update
+  apt -y install build-essential python3-dev
+fi
+
 PYTHON_MAJOR_MINOR=$(python3 -V | sed -E 's/.* ([[:digit:]]+)\.([[:digit:]]+).*/\1\2/')
 WHEEL_FILE=$(ls /pulsar/pulsar-client | grep "cp${PYTHON_MAJOR_MINOR}")
 pip3 install /pulsar/pulsar-client/${WHEEL_FILE}[all]

tests/docker-images/java-test-image/Dockerfile

@@ -46,10 +46,9 @@ RUN mkdir -p /etc/apt/keyrings \
      && echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list \
      && apt-get update \
      && apt-get -y dist-upgrade \
-     && apt-get -y install temurin-11-jdk
-
-ENV JAVA_HOME /usr/lib/jvm/temurin-11-jdk-amd64
-RUN echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/temurin-11-jdk-amd64/conf/security/java.security
+     && apt-get -y install temurin-11-jdk \
+     && export ARCH=$(uname -m | sed -r 's/aarch64/arm64/g' |  awk '!/arm64/{$0="amd64"}1') \
+     && echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/temurin-11-jdk-$ARCH/conf/security/java.security
 
 # /pulsar/bin/watch-znode.py requires python3-kazoo
 # /pulsar/bin/pulsar-managed-ledger-admin requires python3-protobuf

tests/docker-images/latest-version-image/Dockerfile

@@ -29,13 +29,8 @@ RUN apt-get install -y procps curl git build-essential
 
 ENV GOLANG_VERSION 1.13.3
 
-RUN curl -sSL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz \
-		| tar -C /usr/local -xz
-
-# RUN wget https://dl.google.com/go/go1.13.3.linux-amd64.tar.gz && tar -xvf go1.13.3.linux-amd64.tar.gz && mv go /usr/local
-# RUN export GOROOT=/usr/local/go && export GOPATH=$HOME/go && export PATH=$GOPATH/bin:$GOROOT/bin:$PATH
-# RUN echo "export GOROOT=/usr/local/go" >> ~/.profile && echo "export GOPATH=$HOME/go" >> ~/.profile && echo "export PATH=$GOPATH/bin:$GOROOT/bin:$PATH" >> ~/.profile
-
+RUN export ARCH=$(uname -m | sed -r 's/aarch64/arm64/g' |  awk '!/arm64/{$0="amd64"}1') \
+     && curl -sSL https://golang.org/dl/go$GOLANG_VERSION.linux-$ARCH.tar.gz | tar -C /usr/local -xz
 ENV PATH /usr/local/go/bin:$PATH
 
 RUN mkdir -p /go/src /go/bin && chmod -R 777 /go