From da18c86cbfea1d8fe6940035f9103e15890d47ce Mon Sep 17 00:00:00 2001 From: Michael Legleux Date: Tue, 21 Mar 2023 02:18:01 -0700 Subject: [PATCH] Build packages with Ubuntu 18.04 Restores Ubuntu 18.04 packages Update docker images to use Conan --- Builds/CMake/RippledCompiler.cmake | 5 +- Builds/CMake/RippledRelease.cmake | 10 +- Builds/containers/gitlab-ci/pkgbuild.yml | 77 ++++++++- .../gitlab-ci/push_to_artifactory.sh | 2 +- .../containers/packaging/dpkg/build_dpkg.sh | 2 - Builds/containers/packaging/dpkg/debian/rules | 6 +- Builds/containers/ubuntu-builder/Dockerfile | 17 +- .../containers/ubuntu-builder/ubuntu_setup.sh | 154 +++++++----------- 8 files changed, 153 insertions(+), 120 deletions(-) diff --git a/Builds/CMake/RippledCompiler.cmake b/Builds/CMake/RippledCompiler.cmake index 9a96d647f45..2fbb6004333 100644 --- a/Builds/CMake/RippledCompiler.cmake +++ b/Builds/CMake/RippledCompiler.cmake @@ -124,7 +124,10 @@ else () # * static option set and # * NOT APPLE (AppleClang does not support static libc/c++) and # * NOT san (sanitizers typically don't work with static libc/c++) - $<$,$>,$>>:-static-libstdc++>) + $<$,$>,$>>: + -static-libstdc++ + -static-libgcc + >) endif () if (use_gold AND is_gcc) diff --git a/Builds/CMake/RippledRelease.cmake b/Builds/CMake/RippledRelease.cmake index 8e4dcfcd6d0..1c60ea80d7b 100644 --- a/Builds/CMake/RippledRelease.cmake +++ b/Builds/CMake/RippledRelease.cmake @@ -64,16 +64,16 @@ if (is_root_project) #[===================================================================[ dpkg #]===================================================================] - # currently use ubuntu 16.04 as a base b/c it has one of + # currently use ubuntu 18.04 as a base b/c it has one of # the lower versions of libc among ubuntu and debian releases. # we could change this in the future and build with some other deb # based system. add_custom_target (dpkg_container docker build --pull - --build-arg DIST_TAG=20.04 + --build-arg DIST_TAG=18.04 --build-arg GIT_COMMIT=${commit_hash} - -t rippleci/rippled-dpkg-builder:${container_label} + -t rippled-dpkg-builder:${container_label} $<$:--cache-from=${dpkg_cache_from}> -f ubuntu-builder/Dockerfile . WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/Builds/containers @@ -119,7 +119,7 @@ if (is_root_project) docker run -v ${CMAKE_CURRENT_SOURCE_DIR}:/opt/rippled_bld/pkg/rippled -v ${CMAKE_CURRENT_BINARY_DIR}/packages:/opt/rippled_bld/pkg/out - -t rippleci/rippled-dpkg-builder:${container_label} + -t rippled-dpkg-builder:${container_label} /bin/bash -c "cp -fpu rippled/Builds/containers/packaging/dpkg/build_dpkg.sh . && ./build_dpkg.sh" VERBATIM USES_TERMINAL @@ -159,7 +159,7 @@ if (is_root_project) add_custom_target (ci_container docker build --pull - --build-arg DIST_TAG=20.04 + --build-arg DIST_TAG=18.04 --build-arg GIT_COMMIT=${commit_hash} --build-arg CI_USE=true -t rippled-ci-builder:${container_label} diff --git a/Builds/containers/gitlab-ci/pkgbuild.yml b/Builds/containers/gitlab-ci/pkgbuild.yml index 8a602c84cdc..f35186acb8e 100644 --- a/Builds/containers/gitlab-ci/pkgbuild.yml +++ b/Builds/containers/gitlab-ci/pkgbuild.yml @@ -12,10 +12,10 @@ variables: # cmake build (container targets) and tagged/pushed so they # can be used here RPM_CONTAINER_TAG: "2023-02-13" - RPM_CONTAINER_NAME: "rippleci/rippled-rpm-builder" + RPM_CONTAINER_NAME: "rippled-rpm-builder" RPM_CONTAINER_FULLNAME: "${RPM_CONTAINER_NAME}:${RPM_CONTAINER_TAG}" - DPKG_CONTAINER_TAG: "2023-02-13" - DPKG_CONTAINER_NAME: "rippleci/rippled-dpkg-builder" + DPKG_CONTAINER_TAG: "2023-03-20" + DPKG_CONTAINER_NAME: "rippled-dpkg-builder" DPKG_CONTAINER_FULLNAME: "${DPKG_CONTAINER_NAME}:${DPKG_CONTAINER_TAG}" ARTIFACTORY_HOST: "artifactory.ops.ripple.com" ARTIFACTORY_HUB: "${ARTIFACTORY_HOST}:6555" @@ -190,7 +190,7 @@ rocky_8_smoketest: dependencies: - rpm_build image: - name: rockylinux/rockylinux:8 + name: artifactory.ops.ripple.com/rockylinux/rockylinux:8 <<: *run_local_smoketest fedora_37_smoketest: @@ -209,6 +209,14 @@ fedora_38_smoketest: name: artifactory.ops.ripple.com/fedora:38 <<: *run_local_smoketest +ubuntu_18_smoketest: + stage: smoketest + dependencies: + - dpkg_build + image: + name: artifactory.ops.ripple.com/ubuntu:18.04 + <<: *run_local_smoketest + ubuntu_20_smoketest: stage: smoketest dependencies: @@ -225,6 +233,14 @@ ubuntu_22_smoketest: name: artifactory.ops.ripple.com/ubuntu:22.04 <<: *run_local_smoketest +debian_10_smoketest: + stage: smoketest + dependencies: + - dpkg_build + image: + name: artifactory.ops.ripple.com/debian:10 + <<: *run_local_smoketest + debian_11_smoketest: stage: smoketest dependencies: @@ -329,12 +345,13 @@ centos_7_verify_repo_test: <<: *only_primary <<: *run_repo_smoketest + rocky_8_verify_repo_test: stage: verify_from_test variables: RPM_REPO: "rippled-rpm-test-mirror" image: - name: rockylinux/rockylinux:8 + name: artifactory.ops.ripple.com/rockylinux/rockylinux:8 dependencies: - rpm_sign <<: *only_primary @@ -362,6 +379,18 @@ fedora_38_verify_repo_test: <<: *only_primary <<: *run_repo_smoketest +ubuntu_18_verify_repo_test: + stage: verify_from_test + variables: + DISTRO: "bionic" + DEB_REPO: "rippled-deb-test-mirror" + image: + name: artifactory.ops.ripple.com/ubuntu:18.04 + dependencies: + - dpkg_sign + <<: *only_primary + <<: *run_repo_smoketest + ubuntu_20_verify_repo_test: stage: verify_from_test variables: @@ -386,6 +415,18 @@ ubuntu_22_verify_repo_test: <<: *only_primary <<: *run_repo_smoketest +debian_10_verify_repo_test: + stage: verify_from_test + variables: + DISTRO: "buster" + DEB_REPO: "rippled-deb-test-mirror" + image: + name: artifactory.ops.ripple.com/debian:10 + dependencies: + - dpkg_sign + <<: *only_primary + <<: *run_repo_smoketest + debian_11_verify_repo_test: stage: verify_from_test variables: @@ -468,7 +509,7 @@ rocky_8_verify_repo_prod: variables: RPM_REPO: "rippled-rpm" image: - name: rockylinux/rockylinux:8 + name: artifactory.ops.ripple.com/rockylinux/rockylinux:8 dependencies: - rpm_sign <<: *only_primary @@ -496,6 +537,18 @@ fedora_38_verify_repo_prod: <<: *only_primary <<: *run_repo_smoketest +ubuntu_18_verify_repo_prod: + stage: verify_from_prod + variables: + DISTRO: "bionic" + DEB_REPO: "rippled-deb" + image: + name: artifactory.ops.ripple.com/ubuntu:18.04 + dependencies: + - dpkg_sign + <<: *only_primary + <<: *run_repo_smoketest + ubuntu_20_verify_repo_prod: stage: verify_from_prod variables: @@ -520,6 +573,18 @@ ubuntu_22_verify_repo_prod: <<: *only_primary <<: *run_repo_smoketest +debian_10_verify_repo_prod: + stage: verify_from_prod + variables: + DISTRO: "buster" + DEB_REPO: "rippled-deb" + image: + name: artifactory.ops.ripple.com/debian:10 + dependencies: + - dpkg_sign + <<: *only_primary + <<: *run_repo_smoketest + debian_11_verify_repo_prod: stage: verify_from_prod variables: diff --git a/Builds/containers/gitlab-ci/push_to_artifactory.sh b/Builds/containers/gitlab-ci/push_to_artifactory.sh index 255f4247ae0..97a9e906ad6 100644 --- a/Builds/containers/gitlab-ci/push_to_artifactory.sh +++ b/Builds/containers/gitlab-ci/push_to_artifactory.sh @@ -20,7 +20,7 @@ RIPPLED_REPORTING_DBG_PKG=$(ls rippled-reporting-dbgsym_*.*deb) # TODO - where to upload src tgz? RIPPLED_SRC=$(ls rippled_*.orig.tar.gz) DEB_MATRIX=";deb.component=${COMPONENT};deb.architecture=amd64" -for dist in bullseye focal jammy; do +for dist in buster bullseye bionic focal jammy; do DEB_MATRIX="${DEB_MATRIX};deb.distribution=${dist}" done echo "{ \"debs\": {" > "${TOPDIR}/files.info" diff --git a/Builds/containers/packaging/dpkg/build_dpkg.sh b/Builds/containers/packaging/dpkg/build_dpkg.sh index fb67ea2ee10..3cfafe9e342 100755 --- a/Builds/containers/packaging/dpkg/build_dpkg.sh +++ b/Builds/containers/packaging/dpkg/build_dpkg.sh @@ -45,8 +45,6 @@ CHANGELOG # PATH must be preserved for our more modern cmake in /opt/local # TODO : consider allowing lintian to run in future ? export DH_BUILD_DDEBS=1 -export CC=gcc-11 -export CXX=g++-11 debuild --no-lintian --preserve-envvar PATH --preserve-env -us -uc rc=$?; if [[ $rc != 0 ]]; then error "error building dpkg" diff --git a/Builds/containers/packaging/dpkg/debian/rules b/Builds/containers/packaging/dpkg/debian/rules index 83af329cd6d..d62517d34b6 100755 --- a/Builds/containers/packaging/dpkg/debian/rules +++ b/Builds/containers/packaging/dpkg/debian/rules @@ -23,6 +23,8 @@ override_dh_auto_configure: conan install . \ --install-folder bld/rippled \ --build missing \ + --build boost \ + --build sqlite3 \ --settings build_type=Release cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \ @@ -38,8 +40,10 @@ override_dh_auto_configure: conan install . \ --install-folder bld/rippled-reporting \ --build missing \ + --build boost \ + --build sqlite3 \ + --build libuv \ --settings build_type=Release \ - --settings compiler.cppstd=17 \ --options reporting=True cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \ diff --git a/Builds/containers/ubuntu-builder/Dockerfile b/Builds/containers/ubuntu-builder/Dockerfile index 478d9ad7590..23723967fc2 100644 --- a/Builds/containers/ubuntu-builder/Dockerfile +++ b/Builds/containers/ubuntu-builder/Dockerfile @@ -1,22 +1,13 @@ -ARG DIST_TAG=20.04 +ARG DIST_TAG=18.04 FROM ubuntu:$DIST_TAG ARG GIT_COMMIT=unknown ARG CI_USE=false LABEL git-commit=$GIT_COMMIT -# install/setup prerequisites: -COPY ubuntu-builder/ubuntu_setup.sh /tmp/ -COPY shared/install_cmake.sh /tmp/ -RUN chmod +x /tmp/ubuntu_setup.sh && \ - chmod +x /tmp/install_cmake.sh -RUN /tmp/ubuntu_setup.sh +WORKDIR /root +COPY ubuntu-builder/ubuntu_setup.sh . +RUN ./ubuntu_setup.sh && rm ubuntu_setup.sh -RUN /tmp/install_cmake.sh 3.16.3 /opt/local/cmake-3.16 -RUN ln -s /opt/local/cmake-3.16 /opt/local/cmake -ENV PATH="/opt/local/cmake/bin:$PATH" - -# prep files for package building -RUN update-alternatives --set gcc /usr/bin/gcc-11 RUN mkdir -m 777 -p /opt/rippled_bld/pkg/ WORKDIR /opt/rippled_bld/pkg diff --git a/Builds/containers/ubuntu-builder/ubuntu_setup.sh b/Builds/containers/ubuntu-builder/ubuntu_setup.sh index 379ca61e2b1..cd8db75153d 100755 --- a/Builds/containers/ubuntu-builder/ubuntu_setup.sh +++ b/Builds/containers/ubuntu-builder/ubuntu_setup.sh @@ -1,104 +1,76 @@ #!/usr/bin/env bash -set -ex -source /etc/os-release +set -o errexit +set -o nounset +set -o xtrace -if [[ ${VERSION_ID} =~ ^20\. || ${VERSION_ID} =~ ^22\. ]] ; then - echo "setup for ${PRETTY_NAME}" -else - echo "${VERSION} not supported" - exit 1 -fi +# Parameters -export DEBIAN_FRONTEND="noninteractive" -echo "Acquire::Retries 3;" > /etc/apt/apt.conf.d/80-retries -echo "Acquire::http::Pipeline-Depth 0;" >> /etc/apt/apt.conf.d/80-retries -echo "Acquire::http::No-Cache true;" >> /etc/apt/apt.conf.d/80-retries -echo "Acquire::BrokenProxy true;" >> /etc/apt/apt.conf.d/80-retries -apt-get update -o Acquire::CompressionTypes::Order::=gz +gcc_version=${GCC_VERSION:-10} +cmake_version=${CMAKE_VERSION:-3.25.1} +conan_version=${CONAN_VERSION:-1.59} -apt-get -y update -apt-get -y install apt-utils -apt-get -y install software-properties-common wget curl ca-certificates -apt-get -y install python3-pip -apt-get -y upgrade -add-apt-repository -y ppa:ubuntu-toolchain-r/test -apt-get -y clean -apt-get -y update +apt update +# Iteratively build the list of packages to install so that we can interleave +# the lines with comments explaining their inclusion. +dependencies='' +# - to identify the Ubuntu version +dependencies+=' lsb-release' +# - for add-apt-repository +dependencies+=' software-properties-common' +# - to download CMake +dependencies+=' curl' +# - to build CMake +dependencies+=' libssl-dev' +# - Python headers for Boost.Python +dependencies+=' python3-dev' +# - to install Conan +dependencies+=' python3-pip' +# - to download rippled +dependencies+=' git' +# - CMake generators (but not CMake itself) +dependencies+=' make ninja-build' +apt install --yes ${dependencies} -apt-get -y --fix-missing install \ - make cmake ninja-build autoconf automake libtool pkg-config libtool \ - openssl libssl-dev \ - liblzma-dev libbz2-dev zlib1g-dev \ - libjemalloc-dev \ - gdb gdbserver \ - libstdc++6 \ - flex bison parallel \ - libicu-dev texinfo \ - java-common javacc \ - dpkg-dev debhelper devscripts fakeroot \ - debmake git-buildpackage dh-make gitpkg debsums gnupg \ - dh-buildinfo dh-make \ - apt-transport-https +add-apt-repository --yes ppa:ubuntu-toolchain-r/test +apt install --yes gcc-${gcc_version} g++-${gcc_version} \ + debhelper debmake debsums gnupg dh-buildinfo dh-make dh-systemd cmake \ + ninja-build zlib1g-dev make cmake ninja-build autoconf automake \ + pkg-config apt-transport-https -if [[ ${VERSION_ID} =~ ^20\. ]] ; then -apt-get install -y \ - dh-systemd -fi - -apt-get -y install gcc-11 g++-11 +# Give us nice unversioned aliases for gcc and company. update-alternatives --install \ - /usr/bin/gcc gcc /usr/bin/gcc-11 20 \ - --slave /usr/bin/g++ g++ /usr/bin/g++-11 \ - --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \ - --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-11 \ - --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 \ - --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \ - --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-11 \ - --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-11 + /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 \ + --slave /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} \ + --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-${gcc_version} \ + --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-${gcc_version} \ + --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-${gcc_version} \ + --slave /usr/bin/gcov gcov /usr/bin/gcov-${gcc_version} \ + --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-${gcc_version} \ + --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-${gcc_version} update-alternatives --auto gcc -update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 20 -update-alternatives --auto cpp - -wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - - -if [[ ${VERSION_ID} =~ ^20\. ]] ; then - cat << EOF > /etc/apt/sources.list.d/llvm.list -deb http://apt.llvm.org/focal/ llvm-toolchain-focal main -deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal main -deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main -deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main -deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main -deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main -EOF - apt-get -y install binutils clang-12 -fi - +# Download and unpack CMake. +cmake_slug="cmake-${cmake_version}" +curl --location --remote-name \ + "https://github.com/Kitware/CMake/releases/download/v${cmake_version}/${cmake_slug}.tar.gz" +tar xzf ${cmake_slug}.tar.gz +rm ${cmake_slug}.tar.gz -apt-get -y update -if [[ ${VERSION_ID} =~ ^20\. ]] ; then -for v in 12 14; do - apt-get -y install \ - clang-$v libclang-common-$v-dev libclang-$v-dev libllvm$v llvm-$v \ - llvm-$v-dev llvm-$v-runtime clang-format-$v python3-clang-$v \ - lld-$v libfuzzer-$v-dev libc++-$v-dev python-is-python3 - update-alternatives --install \ - /usr/bin/clang clang /usr/bin/clang-$v 40 \ - --slave /usr/bin/clang++ clang++ /usr/bin/clang++-$v \ - --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-$v \ - --slave /usr/bin/asan-symbolize asan-symbolize /usr/bin/asan_symbolize-$v \ - --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-$v \ - --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-$v \ - --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-$v \ - --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-$v \ - --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-$v - done -fi +# Build and install CMake. +cd ${cmake_slug} +./bootstrap --parallel=$(nproc) +make --jobs $(nproc) +make install +cd .. +rm --recursive --force ${cmake_slug} -pip install "conan<2" && \ - conan profile new default --detect && \ - conan profile update settings.compiler.cppstd=20 default && \ - conan profile update settings.compiler.libcxx=libstdc++11 default +# Install Conan. +pip3 install conan==${conan_version} -apt-get -y autoremove +conan profile new --detect gcc +conan profile update settings.compiler=gcc gcc +conan profile update settings.compiler.version=${gcc_version} gcc +conan profile update settings.compiler.libcxx=libstdc++11 gcc +conan profile update env.CC=/usr/bin/gcc gcc +conan profile update env.CXX=/usr/bin/g++ gcc