From c9c3e4149ac36f945c2b5c0585ac6b2ab1f25591 Mon Sep 17 00:00:00 2001 From: Bo Yao Date: Mon, 2 Mar 2020 16:38:04 -0800 Subject: [PATCH 1/8] fix coverage --- scripts/near-kcov-runtime.Dockerfile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/near-kcov-runtime.Dockerfile b/scripts/near-kcov-runtime.Dockerfile index 03c7b4049f2..c44f7d0f539 100644 --- a/scripts/near-kcov-runtime.Dockerfile +++ b/scripts/near-kcov-runtime.Dockerfile @@ -1,9 +1,19 @@ FROM amazonlinux:2 as builder RUN yum update -y && yum install -y \ - openssl-devel.x86_64 elfutils-libelf-devel libcurl-devel binutils-devel elfutils-devel zlib-devel git cmake3 ninja-build python3 \ + openssl-devel.x86_64 elfutils-libelf-devel libcurl-devel binutils-devel elfutils-devel zlib-devel \ + git gcc-c++ make ninja-build python3 tar wget \ && yum clean all && rm -rf /var/cache/yum +RUN wget https://cmake.org/files/v3.10/cmake-3.10.0.tar.gz && \ + tar -xvzf cmake-3.10.0.tar.gz && \ + cd cmake-3.10.0 && \ + ./bootstrap && \ + make && \ + make install && \ + cd .. && \ + rm -rf cmake* + RUN git clone https://github.com/SimonKagstrom/kcov.git WORKDIR /kcov From ccdff8c95b3924e69fad714a58f884f66ee598b7 Mon Sep 17 00:00:00 2001 From: Bo Yao Date: Mon, 2 Mar 2020 17:48:26 -0800 Subject: [PATCH 2/8] debug coverage --- scripts/parallel_coverage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/parallel_coverage.py b/scripts/parallel_coverage.py index 49b57124595..4ceec741da4 100644 --- a/scripts/parallel_coverage.py +++ b/scripts/parallel_coverage.py @@ -88,11 +88,13 @@ def merge_coverage(i, to_merge, j): with ThreadPoolExecutor(max_workers=workers()) as executor: while True: covs = glob.glob(f'{coverage_dir(i)}/*') + print('<<<>>>') + print(covs) if len(covs) == 1: break subprocess.check_output(f'mkdir -p {coverage_dir(i+1)}', shell=True) - cov_to_merge = list(grouper(covs, 2 )) + cov_to_merge = list(grouper(covs, 2)) if cov_to_merge[-1][-1] is None: # ensure the last to merge is not only one cov cov_to_merge[-2] += (cov_to_merge[-1][0],) From 826d2a7471a284f5ab7c3af97f1bdf69e6b2a04e Mon Sep 17 00:00:00 2001 From: Bo Yao Date: Mon, 2 Mar 2020 18:04:45 -0800 Subject: [PATCH 3/8] debug release --- scripts/master_docker_release.sh | 6 ++++++ scripts/parallel_coverage.py | 3 +++ 2 files changed, 9 insertions(+) create mode 100755 scripts/master_docker_release.sh diff --git a/scripts/master_docker_release.sh b/scripts/master_docker_release.sh new file mode 100755 index 00000000000..0e6a9700072 --- /dev/null +++ b/scripts/master_docker_release.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -euo pipefail + +make +docker tag nearcore nearprotocol/nearcore:master +docker push nearprotocol/nearcore:master \ No newline at end of file diff --git a/scripts/parallel_coverage.py b/scripts/parallel_coverage.py index 4ceec741da4..9a4ef531000 100644 --- a/scripts/parallel_coverage.py +++ b/scripts/parallel_coverage.py @@ -49,6 +49,9 @@ def coverage_dir(i): def merge_coverage(i, to_merge, j): p = subprocess.Popen(['kcov', '--merge', os.path.join(coverage_dir(i+1), str(j)), *to_merge], stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() + print(f'[[[]]] {i} {j}') + print(stdout) + print(stderr) return (p.returncode, stdout, stderr) From ffc9026af9616d668f8423ed0f9dc94795170b1e Mon Sep 17 00:00:00 2001 From: Bo Yao Date: Mon, 2 Mar 2020 18:53:10 -0800 Subject: [PATCH 4/8] debug cov --- scripts/parallel_coverage.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/parallel_coverage.py b/scripts/parallel_coverage.py index 9a4ef531000..159456c935f 100644 --- a/scripts/parallel_coverage.py +++ b/scripts/parallel_coverage.py @@ -49,9 +49,6 @@ def coverage_dir(i): def merge_coverage(i, to_merge, j): p = subprocess.Popen(['kcov', '--merge', os.path.join(coverage_dir(i+1), str(j)), *to_merge], stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() - print(f'[[[]]] {i} {j}') - print(stdout) - print(stderr) return (p.returncode, stdout, stderr) @@ -108,8 +105,11 @@ def merge_coverage(i, to_merge, j): j+=1 futures.append(executor.submit(merge_coverage, i, cov, j)) - for _ in as_completed(futures): - pass + for f in as_completed(futures): + r = f.result() + print('[[[]]]') + print(r[1]) + print(r[2]) i+=1 From b49c7b250a6eed40e5f9d72c022816c054a8ad64 Mon Sep 17 00:00:00 2001 From: Bo Yao Date: Tue, 3 Mar 2020 11:02:23 -0800 Subject: [PATCH 5/8] update badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1e1a0e3a6b7..6390fe0cbf1 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ## NEAR Protocol - scalable and usable blockchain -![Build status](https://gitlab.com/near-protocol/nearcore/badges/master/pipeline.svg) +[![Build status](https://badge.buildkite.com/a81147cb62c585cc434459eedd1d25e521453120ead9ee6c64.svg)](https://buildkite.com/nearprotocol/nearcore) [![codecov](https://codecov.io/gh/nearprotocol/nearcore/branch/master/graph/badge.svg)](https://codecov.io/gh/nearprotocol/nearcore) [![dependency status](https://deps.rs/repo/github/nearprotocol/nearcore/status.svg)](https://deps.rs/repo/github/nearprotocol/nearcore) [![Join the community on Spectrum](https://withspectrum.github.io/badge/badge.svg)](https://spectrum.chat/near) From 203303e37dab195b2bcf14d5f0ef2e385493f009 Mon Sep 17 00:00:00 2001 From: Bo Yao Date: Tue, 3 Mar 2020 11:28:25 -0800 Subject: [PATCH 6/8] remove debug in cov --- scripts/parallel_coverage.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scripts/parallel_coverage.py b/scripts/parallel_coverage.py index 159456c935f..91ae04e6498 100644 --- a/scripts/parallel_coverage.py +++ b/scripts/parallel_coverage.py @@ -88,8 +88,6 @@ def merge_coverage(i, to_merge, j): with ThreadPoolExecutor(max_workers=workers()) as executor: while True: covs = glob.glob(f'{coverage_dir(i)}/*') - print('<<<>>>') - print(covs) if len(covs) == 1: break subprocess.check_output(f'mkdir -p {coverage_dir(i+1)}', shell=True) @@ -106,10 +104,7 @@ def merge_coverage(i, to_merge, j): futures.append(executor.submit(merge_coverage, i, cov, j)) for f in as_completed(futures): - r = f.result() - print('[[[]]]') - print(r[1]) - print(r[2]) + pass i+=1 From 6157db05e087f579ac103ff6ebf50dda3897ef6b Mon Sep 17 00:00:00 2001 From: Bo Yao Date: Tue, 3 Mar 2020 12:17:06 -0800 Subject: [PATCH 7/8] update docker release script --- scripts/docker-release.sh | 27 +++++++++------------------ scripts/master_docker_release.sh | 6 ------ 2 files changed, 9 insertions(+), 24 deletions(-) delete mode 100755 scripts/master_docker_release.sh diff --git a/scripts/docker-release.sh b/scripts/docker-release.sh index 0a98e49dc7a..5f661f4b930 100755 --- a/scripts/docker-release.sh +++ b/scripts/docker-release.sh @@ -1,22 +1,13 @@ #!/bin/bash -function make_sure { - read -p "$1. Are you sure [y/n]? " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]] - then - exit 1 - fi -} +set -euo pipefail -branch=$(git rev-parse --abbrev-ref HEAD) -if [ "${branch}" != "master" ]; then - make_sure "Not in master branch" -fi - -if [ -n "$(git status -s)" ]; then - make_sure "There's untracked files or uncommitted changes" -fi +branch=${BUILDKITE_BRANCH} +commit=${BUILDKITE_COMMIT} make -docker tag nearcore nearprotocol/nearcore:latest -docker push nearprotocol/nearcore:latest \ No newline at end of file +# Here we don't check master, beta and stable criteria, they have to be checked in buildkite pipeline +# before this script. And nearprotocol/nearcore:latest must be manually tagged from a verified stable. +docker tag nearcore nearprotocol/nearcore:${branch}-${commit} +docker tag nearcore nearprotocol/nearcore:${branch} +docker push nearprotocol/nearcore:${branch}-${commit} +docker push nearprotocol/nearcore:${branch} diff --git a/scripts/master_docker_release.sh b/scripts/master_docker_release.sh deleted file mode 100755 index 0e6a9700072..00000000000 --- a/scripts/master_docker_release.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -euo pipefail - -make -docker tag nearcore nearprotocol/nearcore:master -docker push nearprotocol/nearcore:master \ No newline at end of file From bec1b293583973e818f7d67d655c39efa868206d Mon Sep 17 00:00:00 2001 From: Bo Yao Date: Wed, 4 Mar 2020 18:32:38 -0800 Subject: [PATCH 8/8] add hint what image was pushed --- scripts/docker-release.sh | 1 + scripts/upload_release.sh | 16 ---------------- 2 files changed, 1 insertion(+), 16 deletions(-) delete mode 100755 scripts/upload_release.sh diff --git a/scripts/docker-release.sh b/scripts/docker-release.sh index 5f661f4b930..a7f9341c345 100755 --- a/scripts/docker-release.sh +++ b/scripts/docker-release.sh @@ -9,5 +9,6 @@ make # before this script. And nearprotocol/nearcore:latest must be manually tagged from a verified stable. docker tag nearcore nearprotocol/nearcore:${branch}-${commit} docker tag nearcore nearprotocol/nearcore:${branch} +set -x docker push nearprotocol/nearcore:${branch}-${commit} docker push nearprotocol/nearcore:${branch} diff --git a/scripts/upload_release.sh b/scripts/upload_release.sh deleted file mode 100755 index 0acc60ecdd4..00000000000 --- a/scripts/upload_release.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -if [[ -z "${GITLAB_CI}" ]]; then - echo "This script only works in CI" - exit 1 -fi - -# file name example: near-staging-a449f839907f060e75f4448e8f30c7bf7f7a951b -near_binary_file=near-${CI_COMMIT_REF_NAME}-$(git rev-parse HEAD) -near_latest=near-${CI_COMMIT_REF_NAME}-latest -cp target/release/near ${near_binary_file}-release -cp target/debug/near ${near_binary_file}-debug -/snap/bin/gsutil cp ${near_binary_file}-release gs://nearprotocol_nearcore_release -/snap/bin/gsutil cp gs://nearprotocol_nearcore_release/${near_binary_file}-release gs://nearprotocol_nearcore_release/${near_latest}-release -/snap/bin/gsutil cp ${near_binary_file}-debug gs://nearprotocol_nearcore_release -/snap/bin/gsutil cp gs://nearprotocol_nearcore_release/${near_binary_file}-debug gs://nearprotocol_nearcore_release/${near_latest}-debug