Merge pull request #299 from gnes-ai/add-benchmark-to-cd

feat(cd): adding benchmark to cd pipeline

.drone-cd.yml

@@ -54,14 +54,69 @@ steps:
   - export MSG_CONTENT=""
   - ./shell/push-wechatwork.sh
 
+- name: notify the start on benchmark work
+  image: byrnedo/alpine-curl
+  environment:
+    BOT_URL:
+      from_secret: BOT_URL
+  commands:
+  - export MSG_LINK=${DRONE_BUILD_LINK}
+  - export MSG_TITLE="🏎️🐎 Start to benchmark the new master \`$DRONE_SOURCE_BRANCH\`(\`${DRONE_BUILD_NUMBER}\`)"
+  - export MSG_CONTENT="click the link below to see the status"
+  - ./shell/push-wechatwork.sh
+
+- name: build and push docker images
+  image: gnes/dind
+  privileged: true
+  volumes:
+  - name: docker
+    path: /var/run/docker.sock
+  - name: cache
+    path: /workspace
+  environment:
+    TCLOUD_USER:
+      from_secret: TCLOUD_USER
+    TCLOUD_PWD:
+      from_secret: TCLOUD_PWD
+    HUB_USER:
+      from_secret: HUB_USER
+    HUB_PWD:
+      from_secret: HUB_PWD
+    BADGE_WEBHOOK:
+      from_secret: BADGE_WEBHOOK
+    BOT_URL:
+      from_secret: BOT_URL
+    GITHUB_USER:
+      from_secret: GITHUB_USER
+    GITHUB_PWD:
+      from_secret: GITHUB_PWD
+    DO_BENCHMARK:
+      from_secret: DO_BENCHMARK
+    BENCHMARK_DIR: /workspace/benchmark/
+    SHARED_HOST_PATH: /tmp/
+  commands:
+  - ./benchmark-eval.sh
+  - docker system prune -a -f
+
+- name: notify the sucess on wechat work
+  image: byrnedo/alpine-curl
+  environment:
+    BOT_URL:
+      from_secret: BOT_URL
+  commands:
+  - export MSG_LINK="https://github.com/gnes-ai/benchmark"
+  - export MSG_TITLE="✅🏍️ New benchmark is updated!"
+  - export MSG_CONTENT=""
+  - ./shell/push-wechatwork.sh
+
 - name: notify the failure on wechat work
   image: byrnedo/alpine-curl
   environment:
     BOT_URL:
       from_secret: BOT_URL
   commands:
   - export MSG_LINK=$DRONE_BUILD_LINK
-  - export MSG_TITLE="❌🎁 fail to build docker image!"
+  - export MSG_TITLE="❌🎁🏍️ fail to build and benchmark the docker image!"
   - export MSG_CONTENT="please inform [$DRONE_COMMIT_AUTHOR]($DRONE_COMMIT_AUTHOR_EMAIL) to modify and fix [\`$DRONE_SOURCE_BRANCH\`]($DRONE_COMMIT_LINK). click the link below to see the details."
   - ./shell/push-wechatwork.sh
   when:
@@ -78,6 +133,9 @@ volumes:
 - name: docker
   host:
     path: /var/run/docker.sock
+- name: cache
+  host:
+    path: /tmp/
 
 ---
 
@@ -136,14 +194,69 @@ steps:
   - export MSG_CONTENT=""
   - ./shell/push-wechatwork.sh
 
+- name: notify the start on benchmark work
+  image: byrnedo/alpine-curl
+  environment:
+    BOT_URL:
+      from_secret: BOT_URL
+  commands:
+  - export MSG_LINK=${DRONE_BUILD_LINK}
+  - export MSG_TITLE="🏎️🐎 Start to benchmark the new master \`$DRONE_SOURCE_BRANCH\`(\`${DRONE_TAG}\`)"
+  - export MSG_CONTENT="click the link below to see the status"
+  - ./shell/push-wechatwork.sh
+
+- name: build and push docker images
+  image: gnes/dind
+  privileged: true
+  volumes:
+  - name: docker
+    path: /var/run/docker.sock
+  - name: cache
+    path: /workspace
+  environment:
+    TCLOUD_USER:
+      from_secret: TCLOUD_USER
+    TCLOUD_PWD:
+      from_secret: TCLOUD_PWD
+    HUB_USER:
+      from_secret: HUB_USER
+    HUB_PWD:
+      from_secret: HUB_PWD
+    BADGE_WEBHOOK:
+      from_secret: BADGE_WEBHOOK
+    BOT_URL:
+      from_secret: BOT_URL
+    GITHUB_USER:
+      from_secret: GITHUB_USER
+    GITHUB_PWD:
+      from_secret: GITHUB_PWD
+    DO_BENCHMARK:
+      from_secret: DO_BENCHMARK
+    BENCHMARK_DIR: /workspace/benchmark/
+    SHARED_HOST_PATH: /tmp/
+  commands:
+  - ./benchmark-eval.sh
+  - docker system prune -a -f
+
+- name: notify the sucess on wechat work
+  image: byrnedo/alpine-curl
+  environment:
+    BOT_URL:
+      from_secret: BOT_URL
+  commands:
+  - export MSG_LINK="https://github.com/gnes-ai/benchmark"
+  - export MSG_TITLE="✅🏍️ New benchmark is updated!"
+  - export MSG_CONTENT=""
+  - ./shell/push-wechatwork.sh
+
 - name: notify the failure on wechat work
   image: byrnedo/alpine-curl
   environment:
     BOT_URL:
       from_secret: BOT_URL
   commands:
   - export MSG_LINK=$DRONE_BUILD_LINK
-  - export MSG_TITLE="❌🎁 fail to build docker image!"
+  - export MSG_TITLE="❌🎁🏍️ fail to build and benchmark the docker image!"
   - export MSG_CONTENT="[tag link](https://github.com/gnes-ai/gnes/tree/${DRONE_TAG}) please inform [$DRONE_COMMIT_AUTHOR]($DRONE_COMMIT_AUTHOR_EMAIL) to modify and fix [\`$DRONE_SOURCE_BRANCH\`]($DRONE_COMMIT_LINK). click the link below to see the details."
   - ./shell/push-wechatwork.sh
   when:
@@ -157,4 +270,7 @@ trigger:
 volumes:
 - name: docker
   host:
-    path: /var/run/docker.sock
+    path: /var/run/docker.sock
+- name: cache
+  host:
+    path: /tmp/

Dockerfiles/dind.Dockerfile

@@ -1,5 +1,5 @@
 FROM docker:git
 
-RUN apk update && apk upgrade && apk --no-cache add curl bash
+RUN apk update && apk upgrade && apk --no-cache add curl bash make
 
 CMD ["bash"]

README.md

@@ -648,6 +648,11 @@ The official documentation of GNES is hosted on [doc.gnes.ai](https://doc.gnes.a
 
 - Migrating from [`bert-as-service`](https://github.com/hanxiao/bert-as-service)
 
+<h2 align="center">Benchmark</h2>
+
+We have setup [this repository](https://github.com/gnes-ai/benchmark) to track the network latency over different GNES versions. As a part of CICD pipeline, this repo gets automatically updated when the [GNES master](https://github.com/gnes-ai/gnes) is updated or a new [GNES version is released](https://github.com/gnes-ai/gnes/releases).
+
+
 <h2 align="center">Contributing</h2>
 
 Thanks for your interest in contributing! GNES always welcome the contribution from the open-source community, individual committers and other partners. Without you, GNES can't be successful.

benchmark-eval.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# prevent duplicate evaluation over multiple CICD machines
+if [[ -z "${DO_BENCHMARK}" ]]; then
+    printf "\$DO_BENCHMARK not set, exit\n"
+    exit 1
+fi
+
+cd /workspace && rm -rf benchmark && git clone https://github.com/gnes-ai/benchmark.git && cd benchmark
+
+git remote rm origin && git remote add origin https://$GITHUB_USER:$GITHUB_PWD@github.com/gnes-ai/benchmark.git
+git config --global user.email "[email protected]" && git config --global user.name "Han Xiao"
+
+if [[ -z "${DRONE_TAG}" ]]; then
+    VER_TAG="latest"
+else
+    VER_TAG=${DRONE_TAG}
+fi
+
+export GNES_IMG_TAG=${VER_TAG}-alpine
+
+docker swarm init
+make pull && make build
+
+for EXP_ID in 1 2 3 4 ;
+do
+    export GNES_BENCHMARK_ID=$EXP_ID
+    make clean && make test d=500 b=10 s=1000000 && make clean
+    make wait t=40
+done
+
+# faster for debugging CICD
+#for EXP_ID in 1 2 ;
+#do
+#    export GNES_BENCHMARK_ID=$EXP_ID
+#    make pull && make build && make test d=100 b=10 s=10 && make clean
+#    make wait t=20
+#done
+
+cat $BENCHMARK_DIR/.data/history*.json
+make run_summary
+cat $BENCHMARK_DIR/README.md
+
+git add -u && git status && git commit -m "auto eval the benchmark due to gnes update (${DRONE_TAG})"
+git push --set-upstream origin master