Improve bazel builds (#53)

Improve Bazel build system of the UDFClient - Change the configuration of compiler flags to achieve similar compiler optimization for the UDFClient than with the old build bash script - Activate link time optimization which might allow the compiler more optimizations than before - Remove unnecessary shared libraries for java, because these increased the loading time of the UDFClient - Simplify the rpath workaround by moving it to the java_repository.bzl - The build system can now build two variants of the executable, one with hard coded paths and one with a wrapper script which sets the paths via environment variables. The executable with hard coded paths only runs in the container environment, but has a few percent faster startup than the wrapper script. The wrapper script can be used local debugging and testing outside of the container environment.
exasol · Jul 25, 2019 · dd54f66 · dd54f66
1 parent df5e38a
commit dd54f66
Show file tree

Hide file tree

Showing 44 changed files with 1,028 additions and 148 deletions.
diff --git a/flavors/fancyr-EXASOL-6.0.0/flavor_base/base_test_build_run/Dockerfile b/flavors/fancyr-EXASOL-6.0.0/flavor_base/base_test_build_run/Dockerfile
@@ -12,12 +12,12 @@ RUN ldconfig
 RUN mkdir /exaudf_src /exaudf
 COPY src/ /exaudf_src/
 
-#ENV VERBOSE_BUILD "--subcommands --verbose_failures"
-
 WORKDIR /exaudf_src/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh -c dbg --define r=true //:exaudfclient"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config r --config slow-wrapper-py2"]
 RUN cp -r -L bazel-bin/* /exaudf
 
+RUN ./test_udfclient.sh /exaudf/exaudfclient.sh
+
 RUN mkdir /exasol_emulator
 COPY emulator/ /exasol_emulator
 COPY src/exaudflib/zmqcontainer.proto /exasol_emulator

diff --git a/flavors/fancyr-EXASOL-6.0.0/flavor_base/build_deps/packages/apt_get_packages b/flavors/fancyr-EXASOL-6.0.0/flavor_base/build_deps/packages/apt_get_packages
@@ -7,4 +7,5 @@ openjdk-8-jdk
 build-essential
 locales
 libpcre3-dev
-protobuf-compiler
+protobuf-compiler
+chrpath
diff --git a/flavors/fancyr-EXASOL-6.0.0/flavor_base/build_run/Dockerfile b/flavors/fancyr-EXASOL-6.0.0/flavor_base/build_run/Dockerfile
@@ -12,10 +12,10 @@ RUN ldconfig
 RUN mkdir /exaudf_src /exaudf
 COPY src/ /exaudf_src/
 
-#ENV VERBOSE_BUILD "--subcommands --verbose_failures"
-
 WORKDIR /exaudf_src/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh -c opt --define r=true //:exaudfclient"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config optimize-r --config fast-binary-py2"]
 RUN cp -r -L bazel-bin/* /exaudf
 
+RUN ./test_udfclient.sh /exaudf/exaudfclient
+
 COPY --from={{build_deps}} /build_info /build_info
diff --git a/flavors/fancyr-EXASOL-6.1.0/flavor_base/base_test_build_run/Dockerfile b/flavors/fancyr-EXASOL-6.1.0/flavor_base/base_test_build_run/Dockerfile
@@ -12,12 +12,12 @@ RUN ldconfig
 RUN mkdir /exaudf_src /exaudf
 COPY src/ /exaudf_src/
 
-#ENV VERBOSE_BUILD "--subcommands --verbose_failures"
-
 WORKDIR /exaudf_src/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh -c dbg --define r=true //:exaudfclient"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config r --config slow-wrapper-py2"]
 RUN cp -r -L bazel-bin/* /exaudf
 
+RUN ./test_udfclient.sh /exaudf/exaudfclient.sh
+
 RUN mkdir /exasol_emulator
 COPY emulator/ /exasol_emulator
 COPY src/exaudflib/zmqcontainer.proto /exasol_emulator

diff --git a/flavors/fancyr-EXASOL-6.1.0/flavor_base/build_deps/packages/apt_get_packages b/flavors/fancyr-EXASOL-6.1.0/flavor_base/build_deps/packages/apt_get_packages
@@ -7,4 +7,5 @@ openjdk-8-jdk
 build-essential
 locales
 libpcre3-dev
-protobuf-compiler
+protobuf-compiler
+chrpath
diff --git a/flavors/fancyr-EXASOL-6.1.0/flavor_base/build_run/Dockerfile b/flavors/fancyr-EXASOL-6.1.0/flavor_base/build_run/Dockerfile
@@ -12,10 +12,10 @@ RUN ldconfig
 RUN mkdir /exaudf_src /exaudf
 COPY src/ /exaudf_src/
 
-#ENV VERBOSE_BUILD "--subcommands --verbose_failures"
-
 WORKDIR /exaudf_src/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh -c opt --define r=true //:exaudfclient"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config optimize-r --config fast-binary-py2"]
 RUN cp -r -L bazel-bin/* /exaudf
 
+RUN ./test_udfclient.sh /exaudf/exaudfclient
+
 COPY --from={{build_deps}} /build_info /build_info
diff --git a/flavors/python3-ds-EXASOL-6.0.0/flavor_base/base_test_build_run/Dockerfile b/flavors/python3-ds-EXASOL-6.0.0/flavor_base/base_test_build_run/Dockerfile
@@ -12,12 +12,12 @@ RUN ldconfig
 RUN mkdir /exaudf_src /exaudf
 COPY src/ /exaudf_src/
 
-#ENV VERBOSE_BUILD "--subcommands --verbose_failures"
-
 WORKDIR /exaudf_src/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh -c dbg --define python=true //:exaudfclient_py3"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config slow-wrapper-py3"]
 RUN cp -r -L bazel-bin/* /exaudf
 
+RUN ./test_udfclient.sh /exaudf/exaudfclient_py3.sh
+
 RUN mkdir /exasol_emulator
 COPY emulator/ /exasol_emulator
 COPY src/exaudflib/zmqcontainer.proto /exasol_emulator

diff --git a/flavors/python3-ds-EXASOL-6.0.0/flavor_base/build_deps/packages/apt_get_packages b/flavors/python3-ds-EXASOL-6.0.0/flavor_base/build_deps/packages/apt_get_packages
@@ -7,4 +7,5 @@ openjdk-8-jdk
 build-essential
 locales
 libpcre3-dev
-protobuf-compiler
+protobuf-compiler
+chrpath
diff --git a/flavors/python3-ds-EXASOL-6.0.0/flavor_base/build_run/Dockerfile b/flavors/python3-ds-EXASOL-6.0.0/flavor_base/build_run/Dockerfile
@@ -12,10 +12,10 @@ RUN ldconfig
 RUN mkdir /exaudf_src /exaudf
 COPY src/ /exaudf_src/
 
-#ENV VERBOSE_BUILD "--subcommands --verbose_failures"
-
 WORKDIR /exaudf_src/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh -c opt --define python=true //:exaudfclient_py3"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config fast-binary-py3"]
 RUN cp -r -L bazel-bin/* /exaudf
 
+RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
+
 COPY --from={{build_deps}} /build_info /build_info
diff --git a/flavors/python3-ds-EXASOL-6.1.0/flavor_base/base_test_build_run/Dockerfile b/flavors/python3-ds-EXASOL-6.1.0/flavor_base/base_test_build_run/Dockerfile
@@ -12,12 +12,12 @@ RUN ldconfig
 RUN mkdir /exaudf_src /exaudf
 COPY src/ /exaudf_src/
 
-#ENV VERBOSE_BUILD "--subcommands --verbose_failures"
-
 WORKDIR /exaudf_src/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh -c dbg --define python=true //:exaudfclient_py3"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config slow-wrapper-py3"]
 RUN cp -r -L bazel-bin/* /exaudf
 
+RUN ./test_udfclient.sh /exaudf/exaudfclient_py3.sh
+
 RUN mkdir /exasol_emulator
 COPY emulator/ /exasol_emulator
 COPY src/exaudflib/zmqcontainer.proto /exasol_emulator

diff --git a/flavors/python3-ds-EXASOL-6.1.0/flavor_base/build_deps/packages/apt_get_packages b/flavors/python3-ds-EXASOL-6.1.0/flavor_base/build_deps/packages/apt_get_packages
@@ -7,4 +7,5 @@ openjdk-8-jdk
 build-essential
 locales
 libpcre3-dev
-protobuf-compiler
+protobuf-compiler
+chrpath
diff --git a/flavors/python3-ds-EXASOL-6.1.0/flavor_base/build_run/Dockerfile b/flavors/python3-ds-EXASOL-6.1.0/flavor_base/build_run/Dockerfile
@@ -12,10 +12,10 @@ RUN ldconfig
 RUN mkdir /exaudf_src /exaudf
 COPY src/ /exaudf_src/
 
-#ENV VERBOSE_BUILD "--subcommands --verbose_failures"
-
 WORKDIR /exaudf_src/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh -c opt --define python=true //:exaudfclient_py3"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config fast-binary-py3"]
 RUN cp -r -L bazel-bin/* /exaudf
 
+RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
+
 COPY --from={{build_deps}} /build_info /build_info
diff --git a/flavors/python3-ds-cuda-preview-EXASOL-6.1.0/flavor_base/base_test_build_run/Dockerfile b/flavors/python3-ds-cuda-preview-EXASOL-6.1.0/flavor_base/base_test_build_run/Dockerfile
@@ -12,12 +12,12 @@ RUN ldconfig
 RUN mkdir /exaudf_src /exaudf
 COPY src/ /exaudf_src/
 
-#ENV VERBOSE_BUILD "--subcommands --verbose_failures"
-
 WORKDIR /exaudf_src/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh -c dbg --define python=true //:exaudfclient_py3"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config slow-wrapper-py3"]
 RUN cp -r -L bazel-bin/* /exaudf
 
+RUN ./test_udfclient.sh /exaudf/exaudfclient_py3.sh
+
 RUN mkdir /exasol_emulator
 COPY emulator/ /exasol_emulator
 COPY src/exaudflib/zmqcontainer.proto /exasol_emulator

diff --git a/...ors/python3-ds-cuda-preview-EXASOL-6.1.0/flavor_base/build_deps/packages/apt_get_packages b/...ors/python3-ds-cuda-preview-EXASOL-6.1.0/flavor_base/build_deps/packages/apt_get_packages
@@ -7,4 +7,5 @@ openjdk-8-jdk
 build-essential
 locales
 libpcre3-dev
-protobuf-compiler
+protobuf-compiler
+chrpath
diff --git a/flavors/python3-ds-cuda-preview-EXASOL-6.1.0/flavor_base/build_run/Dockerfile b/flavors/python3-ds-cuda-preview-EXASOL-6.1.0/flavor_base/build_run/Dockerfile
@@ -12,10 +12,10 @@ RUN ldconfig
 RUN mkdir /exaudf_src /exaudf
 COPY src/ /exaudf_src/
 
-#ENV VERBOSE_BUILD "--subcommands --verbose_failures"
-
 WORKDIR /exaudf_src/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh -c opt --define python=true //:exaudfclient_py3"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config fast-binary-py3"]
 RUN cp -r -L bazel-bin/* /exaudf
 
+RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
+
 COPY --from={{build_deps}} /build_info /build_info
diff --git a/flavors/standard-EXASOL-6.0.0/flavor_base/base_test_build_run/Dockerfile b/flavors/standard-EXASOL-6.0.0/flavor_base/base_test_build_run/Dockerfile
@@ -12,12 +12,12 @@ RUN ldconfig
 RUN mkdir /exaudf_src /exaudf
 COPY src/ /exaudf_src/
 
-#ENV VERBOSE_BUILD "--subcommands --verbose_failures"
-
 WORKDIR /exaudf_src/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh -c dbg --define r=true --define python=true --define java=true //:exaudfclient"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config java --config r --config slow-wrapper-py2"]
 RUN cp -r -L bazel-bin/* /exaudf
 
+RUN ./test_udfclient.sh /exaudf/exaudfclient.sh
+
 RUN mkdir /exasol_emulator
 COPY emulator/ /exasol_emulator
 COPY src/exaudflib/zmqcontainer.proto /exasol_emulator

diff --git a/flavors/standard-EXASOL-6.0.0/flavor_base/build_deps/packages/apt_get_packages b/flavors/standard-EXASOL-6.0.0/flavor_base/build_deps/packages/apt_get_packages
@@ -7,4 +7,5 @@ openjdk-8-jdk
 build-essential
 locales
 libpcre3-dev
-protobuf-compiler
+protobuf-compiler
+chrpath
diff --git a/flavors/standard-EXASOL-6.0.0/flavor_base/build_run/Dockerfile b/flavors/standard-EXASOL-6.0.0/flavor_base/build_run/Dockerfile
@@ -9,13 +9,18 @@ COPY --from={{build_deps}} /env /env
 
 RUN ldconfig
 
-RUN mkdir /exaudf_src /exaudf
+RUN mkdir /exaudf_src /exaudf /exaudf/src
 COPY src/ /exaudf_src/
 
-#ENV VERBOSE_BUILD "--subcommands --verbose_failures"
+ENV VERBOSE_BUILD "--subcommands --verbose_failures"
 
 WORKDIR /exaudf_src/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh -c opt --define r=true --define python=true --define java=true //:exaudfclient"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config java --config optimize-r --config fast-binary-py2"]
 RUN cp -r -L bazel-bin/* /exaudf
 
+# These source files is required for the building of the protegrity client
+RUN cp debug_message.h exaudfclient.cc exaudflib/exaudflib.cc exaudflib/exaudflib.h exaudflib/zmqcontainer.proto /exaudf/src
+
+RUN ./test_udfclient.sh /exaudf/exaudfclient
+
 COPY --from={{build_deps}} /build_info /build_info
diff --git a/flavors/standard-EXASOL-6.1.0/flavor_base/base_test_build_run/Dockerfile b/flavors/standard-EXASOL-6.1.0/flavor_base/base_test_build_run/Dockerfile
@@ -12,12 +12,13 @@ RUN ldconfig
 RUN mkdir /exaudf_src /exaudf
 COPY src/ /exaudf_src/
 
-#ENV VERBOSE_BUILD "--subcommands --verbose_failures"
-
 WORKDIR /exaudf_src/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh -c dbg --define r=true --define python=true --define java=true //:exaudfclient //:exaudfclient_py3"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config java --config r --config slow-wrapper-both"]
 RUN cp -r -L bazel-bin/* /exaudf
 
+RUN ./test_udfclient.sh /exaudf/exaudfclient.sh
+RUN ./test_udfclient.sh /exaudf/exaudfclient_py3.sh
+
 RUN mkdir /exasol_emulator
 COPY emulator/ /exasol_emulator
 COPY src/exaudflib/zmqcontainer.proto /exasol_emulator

diff --git a/flavors/standard-EXASOL-6.1.0/flavor_base/build_deps/packages/apt_get_packages b/flavors/standard-EXASOL-6.1.0/flavor_base/build_deps/packages/apt_get_packages
@@ -7,4 +7,5 @@ openjdk-8-jdk
 build-essential
 locales
 libpcre3-dev
-protobuf-compiler
+protobuf-compiler
+chrpath
diff --git a/flavors/standard-EXASOL-6.1.0/flavor_base/build_run/Dockerfile b/flavors/standard-EXASOL-6.1.0/flavor_base/build_run/Dockerfile
@@ -9,13 +9,17 @@ COPY --from={{build_deps}} /env /env
 
 RUN ldconfig
 
-RUN mkdir /exaudf_src /exaudf
+RUN mkdir /exaudf_src /exaudf /exaudf/src 
 COPY src/ /exaudf_src/
 
-#ENV VERBOSE_BUILD "--subcommands --verbose_failures"
-
 WORKDIR /exaudf_src/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh -c opt --define r=true --define python=true --define java=true //:exaudfclient //:exaudfclient_py3"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config java --config optimize-r --config fast-binary-both"]
 RUN cp -r -L bazel-bin/* /exaudf
 
+# These source files is required for the building of the protegrity client
+RUN cp debug_message.h exaudfclient.cc exaudflib/exaudflib.cc exaudflib/exaudflib.h exaudflib/zmqcontainer.proto /exaudf/src
+
+RUN ./test_udfclient.sh /exaudf/exaudfclient
+RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
+
 COPY --from={{build_deps}} /build_info /build_info
diff --git a/src/.bazelrc b/src/.bazelrc
@@ -1,3 +1,18 @@
-startup --expand_configs_in_place
-build --force_pic --action_env=NUMPY_PREFIX --action_env=PYTHON2_PREFIX --action_env=PYTHON2_VERSION --action_env=PYTHON3_SYSPATH --action_env=PYTHON3_PREFIX --action_env=PYTHON3_VERSION --action_env=PYTHON2_SYSPATH --action_env=PROTOBUF_BIN --action_env=PROTOBUF_LIBRARY_PREFIX --action_env=PROTOBUF_INCLUDE_PREFIX --action_env=JAVA_PREFIX --cxxopt='-std=c++14'
-build:opt --copt=-O3 
+build --copt='-std=c++14' --force_pic --action_env=PROTOBUF_BIN --action_env=PROTOBUF_LIBRARY_PREFIX --action_env=PROTOBUF_INCLUDE_PREFIX 
+# TODO add environment variables for R libraries
+build:benchmark --define benchmark=true
+build:r --define r=true
+build:java  --define java=true --action_env=JAVA_PREFIX
+build:python --define python=true  --action_env=PYTHON2_SYSPATH --action_env=PYTHON2_PREFIX --action_env=PYTHON2_VERSION --action_env=NUMPY_PREFIX --action_env=PYTHON3_SYSPATH --action_env=PYTHON3_PREFIX --action_env=PYTHON3_VERSION
+build:fast-binary-py2 --copt='-DCUSTOM_LIBEXAUDFLIB_PATH="/exaudf/libexaudflib_complete.so"' //:exaudfclient
+build:fast-binary-py3 --copt='-DCUSTOM_LIBEXAUDFLIB_PATH="/exaudf/libexaudflib_complete.so"' //:exaudfclient_py3
+build:fast-binary-both --config=fast-binary-py2 --config=fast-binary-py3
+build:slow-wrapper-py2 //:exaudfclient_py2_bash_wrapper
+build:slow-wrapper-py3 //:exaudfclient_py3_bash_wrapper
+build:slow-wrapper-both --config=slow-wrapper-py2 --config=slow-wrapper-py3 
+build:verbose --copt='-v' --subcommands --verbose_failures --announce_rc
+#TODO test linkopts="-flto"
+build:optimize --copt="-g0" --copt="-DNDEBUG" --copt=-fstack-protector-strong --copt=-fomit-frame-pointer --copt=-ffunction-sections --copt=-fdata-sections --copt="-O3" --copt="-U_FORTIFY_SOURCE" --copt="-flto"
+build:optimize-r --define r-unroll-loops=true --config=r
+build:no-tty --curses=no --color=no
+build:debug-build --sandbox_debug --config=verbose