From 0a59d37d623a8331f74ba4bd1c02650e6eaaaddf Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Wed, 23 Oct 2019 14:56:51 +0800 Subject: [PATCH 01/17] fix(ci): orange ci fixing and improve --- .orange-ci.yml | 83 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 30 deletions(-) diff --git a/.orange-ci.yml b/.orange-ci.yml index f148315a..d6c3e3d6 100644 --- a/.orange-ci.yml +++ b/.orange-ci.yml @@ -1,36 +1,36 @@ -ok-stage: &ok_stage - - name: git comment - jobs: - - name: make comment - script: echo "looks good to me 👍" > comment.txt - - name: submit comment - type: git:comment - options: - fromFile: comment.txt - - name: make notification - script: echo "# ✅ 由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH分支刚刚测试通过!\n[点击查看](https://git.code.oa.com/ai-innersource/nes/)" > notify.md - - name: message - type: wework:message - options: - robot: f5ee526b-c6dc-4ef1-97c0-ac14594e57b0 - fromFile: notify.md +gnes-commit-lint-pipe: &bp0 + docker: + image: node:alpine + stages: + - name: do commit lint + script: + - npm install --global --save-dev @commitlint/config-conventional @commitlint/cli --registry=http://r.tnpm.oa.com + - "echo \"module.exports = {extends: ['@commitlint/config-conventional']}\" > commitlint.config.js" + - echo $DRONE_COMMIT_MESSAGE | commitlint install-deps: &install_deps - name: install dependencies jobs: - - name: setup cache to local - script: mkdir -p .cache && ln -snf $(pwd)/.cache $HOME/.cache - - name: show cache folders - script: printf "XDG_CACHE_HOME=$XDG_CACHE_HOME\nPIP_DOWNLOAD_CACHE=$PIP_DOWNLOAD_CACHE\n" - - name: show cpu flag - script: cat /proc/cpuinfo | grep flags + - name: setup ci environment + script: + - mkdir -p .cache && ln -snf $(pwd)/.cache $HOME/.cache + - export PIP_DOWNLOAD_CACHE=${pwd}/.cache/pip_download_cache && export XDG_CACHE_HOME=${pwd}/.cache/pip + - printf "XDG_CACHE_HOME=$XDG_CACHE_HOME\nPIP_DOWNLOAD_CACHE=$PIP_DOWNLOAD_CACHE\n" + - cat /proc/cpuinfo | grep flags + - "python -c 'import tensorflow as tf; print(tf.__version__); a=tf.constant(1, tf.int32); print(tf.Session().run(a))'" + - nvcc --version - name: install project deps env: INDEX_URL: http://mirrors.oa.com/pypi/web/simple/ TRUST_DOM: mirrors.oa.com - script: "pip install -e .[all] -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" - - name: check tensorflow works correctly - script: "python -c 'import tensorflow as tf; print(tf.__version__); a=tf.constant(1, tf.int32); print(tf.Session().run(a))'" + script: + - pip install cffi==1.12.3 + - "pip install -e .[all] -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" + - pip uninstall -y torchvision + - pip install https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl + - "python -c 'import torchvision; print(torchvision.__version__)'" + - "python -c 'import torch; print(torch.__version__)'" + - python -m unittest tests/*.py -v pylint: &pylint - name: check code style @@ -40,20 +40,39 @@ unit-test: &unit_test - name: run unit test script: python -m unittest tests/*.py -basic-pipeline: &bp1 +notify-stage: ¬ify_stage + - name: git comment + jobs: + - name: make comment + script: echo "🍊 looks good to me 👍" > comment.txt + - name: submit comment + type: git:comment + options: + fromFile: comment.txt + - name: make notification + script: echo "$SUCCESS_MSG" > notify.md + - name: message + type: wework:message + options: + robot: f5ee526b-c6dc-4ef1-97c0-ac14594e57b0 + fromFile: notify.md + + +gnes-unit-test-pipe: &bp1 network: idc-ai-sse4 docker: - image: docker.oa.com:8080/public/ailab-faiss-bert-elmo-openai-w2v:latest + image: gnes/ci-base env: GNES_ENV_SET: orange-ci - PIP_DOWNLOAD_CACHE: ${pwd}.cache/pip_download_cache - XDG_CACHE_HOME: ${pwd}.cache/pip + PIP_DOWNLOAD_CACHE: ${pwd}/.cache/pip_download_cache + XDG_CACHE_HOME: ${pwd}/.cache/pip + SUCCESS_MSG: "# ✅😃 All tests passed, good job! \n由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH分支刚刚测试通过!\n[点击查看]($ORANGE_EVENT_URL)" cacheFrom: .orange-ci.cache stages: - <<: *install_deps - <<: *pylint - <<: *unit_test - - <<: *ok_stage + - <<: *notify_stage notify-pipeline1: &np1 network: devnet @@ -68,6 +87,9 @@ notify-pipeline1: &np1 - hanhxiao - larryjfyan - madwang + - leonjiang + - raccoonliu + - kegokang type: markdown fromFile: notify.md - name: message @@ -154,6 +176,7 @@ docker-pipeline2: &dp2 master: merge_request: + - <<: *bp0 - <<: *bp1 push: - <<: *np1 From 094f583ae8e5d7953a0d45f0f228d3810d45c3ba Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Wed, 23 Oct 2019 15:00:06 +0800 Subject: [PATCH 02/17] fix(ci): orange ci fixing and improve --- .orange-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.orange-ci.yml b/.orange-ci.yml index d6c3e3d6..8322fd99 100644 --- a/.orange-ci.yml +++ b/.orange-ci.yml @@ -6,7 +6,7 @@ gnes-commit-lint-pipe: &bp0 script: - npm install --global --save-dev @commitlint/config-conventional @commitlint/cli --registry=http://r.tnpm.oa.com - "echo \"module.exports = {extends: ['@commitlint/config-conventional']}\" > commitlint.config.js" - - echo $DRONE_COMMIT_MESSAGE | commitlint + - echo $ORANGE_LATEST_COMMIT_MESSAGE | commitlint install-deps: &install_deps - name: install dependencies From 6d962916477432cc0ef999e2f7c66773940802e6 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Wed, 23 Oct 2019 15:35:06 +0800 Subject: [PATCH 03/17] fix(ci): orange ci fixing and improve --- .orange-ci.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.orange-ci.yml b/.orange-ci.yml index 8322fd99..fac38c44 100644 --- a/.orange-ci.yml +++ b/.orange-ci.yml @@ -1,6 +1,9 @@ gnes-commit-lint-pipe: &bp0 docker: image: node:alpine + network: idc-hk + wework: + title: "🍊⌛🖊️ Check commit lint" stages: - name: do commit lint script: @@ -24,7 +27,7 @@ install-deps: &install_deps INDEX_URL: http://mirrors.oa.com/pypi/web/simple/ TRUST_DOM: mirrors.oa.com script: - - pip install cffi==1.12.3 + - "pip install cffi==1.12.3 ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" - "pip install -e .[all] -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" - pip uninstall -y torchvision - pip install https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl @@ -59,14 +62,16 @@ notify-stage: ¬ify_stage gnes-unit-test-pipe: &bp1 - network: idc-ai-sse4 + network: idc-hk + wework: + title: "🍊⌛🏗 Install GNES and do unit test" docker: image: gnes/ci-base env: GNES_ENV_SET: orange-ci PIP_DOWNLOAD_CACHE: ${pwd}/.cache/pip_download_cache XDG_CACHE_HOME: ${pwd}/.cache/pip - SUCCESS_MSG: "# ✅😃 All tests passed, good job! \n由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH分支刚刚测试通过!\n[点击查看]($ORANGE_EVENT_URL)" + SUCCESS_MSG: "# 🍊✅😃 All tests passed, good job! \n由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH分支刚刚测试通过!\n[点击查看]($ORANGE_EVENT_URL)" cacheFrom: .orange-ci.cache stages: - <<: *install_deps From b6554b7628936833fd142373220c0d3531f2d6b5 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Wed, 23 Oct 2019 15:36:28 +0800 Subject: [PATCH 04/17] fix(ci): orange ci fixing and improve --- .orange-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.orange-ci.yml b/.orange-ci.yml index fac38c44..757a7c79 100644 --- a/.orange-ci.yml +++ b/.orange-ci.yml @@ -2,14 +2,15 @@ gnes-commit-lint-pipe: &bp0 docker: image: node:alpine network: idc-hk - wework: - title: "🍊⌛🖊️ Check commit lint" + wework: + title: "🍊⌛🖊️ Check commit lint" stages: - name: do commit lint script: - npm install --global --save-dev @commitlint/config-conventional @commitlint/cli --registry=http://r.tnpm.oa.com - "echo \"module.exports = {extends: ['@commitlint/config-conventional']}\" > commitlint.config.js" - echo $ORANGE_LATEST_COMMIT_MESSAGE | commitlint + - <<: *notify_stage install-deps: &install_deps - name: install dependencies From 4ee8ee7dc255892ebf9551862da0d183f68c4589 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Wed, 23 Oct 2019 15:36:50 +0800 Subject: [PATCH 05/17] fix(ci): orange ci fixing and improve --- .orange-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.orange-ci.yml b/.orange-ci.yml index 757a7c79..86570ba5 100644 --- a/.orange-ci.yml +++ b/.orange-ci.yml @@ -10,7 +10,6 @@ gnes-commit-lint-pipe: &bp0 - npm install --global --save-dev @commitlint/config-conventional @commitlint/cli --registry=http://r.tnpm.oa.com - "echo \"module.exports = {extends: ['@commitlint/config-conventional']}\" > commitlint.config.js" - echo $ORANGE_LATEST_COMMIT_MESSAGE | commitlint - - <<: *notify_stage install-deps: &install_deps - name: install dependencies From 76d52b344bc823c614ca211ea79b5e50f44d98ec Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Wed, 23 Oct 2019 15:48:09 +0800 Subject: [PATCH 06/17] fix(ci): orange ci fixing and improve --- .orange-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.orange-ci.yml b/.orange-ci.yml index 86570ba5..74c701aa 100644 --- a/.orange-ci.yml +++ b/.orange-ci.yml @@ -62,7 +62,7 @@ notify-stage: ¬ify_stage gnes-unit-test-pipe: &bp1 - network: idc-hk + network: idc-ai-sse4 wework: title: "🍊⌛🏗 Install GNES and do unit test" docker: From 04b7b1c836f0be32adac10a275a6ccdc58adb010 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Wed, 23 Oct 2019 16:29:09 +0800 Subject: [PATCH 07/17] fix(ci): orange ci fixing and improve --- .orange-ci.yml | 145 +++++++++++++++++++++++++++++-------------------- 1 file changed, 85 insertions(+), 60 deletions(-) diff --git a/.orange-ci.yml b/.orange-ci.yml index 74c701aa..dffdc4f1 100644 --- a/.orange-ci.yml +++ b/.orange-ci.yml @@ -1,67 +1,75 @@ -gnes-commit-lint-pipe: &bp0 - docker: - image: node:alpine - network: idc-hk - wework: - title: "🍊⌛🖊️ Check commit lint" - stages: - - name: do commit lint - script: - - npm install --global --save-dev @commitlint/config-conventional @commitlint/cli --registry=http://r.tnpm.oa.com - - "echo \"module.exports = {extends: ['@commitlint/config-conventional']}\" > commitlint.config.js" - - echo $ORANGE_LATEST_COMMIT_MESSAGE | commitlint - -install-deps: &install_deps - - name: install dependencies - jobs: - - name: setup ci environment - script: - - mkdir -p .cache && ln -snf $(pwd)/.cache $HOME/.cache - - export PIP_DOWNLOAD_CACHE=${pwd}/.cache/pip_download_cache && export XDG_CACHE_HOME=${pwd}/.cache/pip - - printf "XDG_CACHE_HOME=$XDG_CACHE_HOME\nPIP_DOWNLOAD_CACHE=$PIP_DOWNLOAD_CACHE\n" - - cat /proc/cpuinfo | grep flags - - "python -c 'import tensorflow as tf; print(tf.__version__); a=tf.constant(1, tf.int32); print(tf.Session().run(a))'" - - nvcc --version - - name: install project deps - env: - INDEX_URL: http://mirrors.oa.com/pypi/web/simple/ - TRUST_DOM: mirrors.oa.com - script: - - "pip install cffi==1.12.3 ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" - - "pip install -e .[all] -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" - - pip uninstall -y torchvision - - pip install https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl - - "python -c 'import torchvision; print(torchvision.__version__)'" - - "python -c 'import torch; print(torch.__version__)'" - - python -m unittest tests/*.py -v - -pylint: &pylint +# basic stages + +stage-install-dep: &stage_install_deps + - name: setup ci environment + script: + - mkdir -p .cache && ln -snf $(pwd)/.cache $HOME/.cache + - export PIP_DOWNLOAD_CACHE=${pwd}/.cache/pip_download_cache && export XDG_CACHE_HOME=${pwd}/.cache/pip + - printf "XDG_CACHE_HOME=$XDG_CACHE_HOME\nPIP_DOWNLOAD_CACHE=$PIP_DOWNLOAD_CACHE\n" + - cat /proc/cpuinfo | grep flags + - "python -c 'import tensorflow as tf; print(tf.__version__); a=tf.constant(1, tf.int32); print(tf.Session().run(a))'" + - nvcc --version + - name: install project deps + env: + INDEX_URL: http://mirrors.tencent.com/pypi/simple/ + TRUST_DOM: mirrors.tencent.com + script: + - "pip install cffi==1.12.3 coverage codecov ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" + - "pip install -e .[all] -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" + - pip uninstall -y torchvision + - pip install https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl + - "python -c 'import torchvision; print(torchvision.__version__)'" + - "python -c 'import torch; print(torch.__version__)'" + +stage-pylint: &stage_pylint - name: check code style script: pylint gnes/**/*.py --exit-zero -unit-test: &unit_test +stage-unit-test: &stage_unit_test - name: run unit test - script: python -m unittest tests/*.py + script: python -m unittest tests/*.py -v -notify-stage: ¬ify_stage - - name: git comment - jobs: - - name: make comment - script: echo "🍊 looks good to me 👍" > comment.txt - - name: submit comment - type: git:comment - options: - fromFile: comment.txt - - name: make notification - script: echo "$SUCCESS_MSG" > notify.md - - name: message - type: wework:message - options: - robot: f5ee526b-c6dc-4ef1-97c0-ac14594e57b0 - fromFile: notify.md +stage-notify-done: &stage_notify + - name: make comment + script: echo "🍊 looks good to me 👍" > comment.txt + - name: submit comment + type: git:comment + options: + fromFile: comment.txt + - name: make notification + script: echo "$SUCCESS_MSG" > notify.md + - name: message + type: wework:message + options: + robot: f5ee526b-c6dc-4ef1-97c0-ac14594e57b0 + fromFile: notify.md + +stage-commit-lint: &stage_commit_lint + - name: do commit lint + script: + - npm install --global --save-dev @commitlint/config-conventional @commitlint/cli --registry=http://r.tnpm.oa.com + - "echo \"module.exports = {extends: ['@commitlint/config-conventional']}\" > commitlint.config.js" + - echo $ORANGE_LATEST_COMMIT_MESSAGE | commitlint + +stage-coverity: &stage_coverity + - name: do coverity + script: + - coverage run -m unittest discover -s tests/ + - coverage report --skip-covered + - codecov --token $CODECOV_TOKEN +# lint -gnes-unit-test-pipe: &bp1 +pipe-lint: &bp0 + docker: + image: node:alpine + network: idc-hk + wework: + title: "🍊⌛🖊️ Check commit lint" + stages: + - <<: *commit_lint + +pipe-unit-test: &bp1 network: idc-ai-sse4 wework: title: "🍊⌛🏗 Install GNES and do unit test" @@ -74,10 +82,27 @@ gnes-unit-test-pipe: &bp1 SUCCESS_MSG: "# 🍊✅😃 All tests passed, good job! \n由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH分支刚刚测试通过!\n[点击查看]($ORANGE_EVENT_URL)" cacheFrom: .orange-ci.cache stages: - - <<: *install_deps - - <<: *pylint + - <<: *stage_install_deps + - <<: *stage_pylint + - <<: *stage_coverity + - <<: *stage_notify + +pipe-coverity: &cp1 + network: idc-ai-sse4 + wework: + title: "🍊⌛🏗 Do coverity check on the new master" + docker: + image: gnes/ci-base + env: + GNES_ENV_SET: orange-ci + PIP_DOWNLOAD_CACHE: ${pwd}/.cache/pip_download_cache + XDG_CACHE_HOME: ${pwd}/.cache/pip + SUCCESS_MSG: "# 🍊✅😃 All tests passed, good job! \n由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH分支刚刚测试通过!\n[点击查看]($ORANGE_EVENT_URL)" + cacheFrom: .orange-ci.cache + stages: + - <<: *stage_install_deps - <<: *unit_test - - <<: *notify_stage + - <<: *stage_notify notify-pipeline1: &np1 network: devnet From d05f9c2a0e180a75f2443f8fc9c1794cafd512dc Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Wed, 23 Oct 2019 16:29:53 +0800 Subject: [PATCH 08/17] fix(ci): orange ci fixing and improve --- .orange-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.orange-ci.yml b/.orange-ci.yml index dffdc4f1..40b755a0 100644 --- a/.orange-ci.yml +++ b/.orange-ci.yml @@ -1,4 +1,4 @@ -# basic stages +# BASIC STAGES stage-install-dep: &stage_install_deps - name: setup ci environment @@ -67,7 +67,7 @@ pipe-lint: &bp0 wework: title: "🍊⌛🖊️ Check commit lint" stages: - - <<: *commit_lint + - <<: *stage_commit_lint pipe-unit-test: &bp1 network: idc-ai-sse4 From e7700bdbe4cf590acc0fb10444092225f5bfc17a Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Wed, 23 Oct 2019 16:30:14 +0800 Subject: [PATCH 09/17] fix(ci): orange ci fixing and improve --- .orange-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.orange-ci.yml b/.orange-ci.yml index 40b755a0..582c44ab 100644 --- a/.orange-ci.yml +++ b/.orange-ci.yml @@ -101,7 +101,7 @@ pipe-coverity: &cp1 cacheFrom: .orange-ci.cache stages: - <<: *stage_install_deps - - <<: *unit_test + - <<: *stage_unit_test - <<: *stage_notify notify-pipeline1: &np1 From db65505d5523f17f0a21961553d3c7cbd5d54541 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Wed, 23 Oct 2019 16:34:28 +0800 Subject: [PATCH 10/17] fix(ci): orange ci fixing and improve --- .orange-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.orange-ci.yml b/.orange-ci.yml index 582c44ab..b6c61647 100644 --- a/.orange-ci.yml +++ b/.orange-ci.yml @@ -83,8 +83,7 @@ pipe-unit-test: &bp1 cacheFrom: .orange-ci.cache stages: - <<: *stage_install_deps - - <<: *stage_pylint - - <<: *stage_coverity + - <<: *stage_unit_test - <<: *stage_notify pipe-coverity: &cp1 @@ -97,11 +96,12 @@ pipe-coverity: &cp1 GNES_ENV_SET: orange-ci PIP_DOWNLOAD_CACHE: ${pwd}/.cache/pip_download_cache XDG_CACHE_HOME: ${pwd}/.cache/pip - SUCCESS_MSG: "# 🍊✅😃 All tests passed, good job! \n由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH分支刚刚测试通过!\n[点击查看]($ORANGE_EVENT_URL)" + SUCCESS_MSG: "# 🍊✅😃 Report is successfully updated!" + envFrom: https://git.code.oa.com/gnes/gnes-secret/blob/master/env cacheFrom: .orange-ci.cache stages: - <<: *stage_install_deps - - <<: *stage_unit_test + - <<: *stage_coverity - <<: *stage_notify notify-pipeline1: &np1 From c9a9cdeea544b03925fb130fe057249bfb32a913 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Wed, 23 Oct 2019 16:38:37 +0800 Subject: [PATCH 11/17] fix(ci): orange ci fixing and improve --- .orange-ci.yml | 68 ++++++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/.orange-ci.yml b/.orange-ci.yml index b6c61647..bcadc452 100644 --- a/.orange-ci.yml +++ b/.orange-ci.yml @@ -1,25 +1,26 @@ # BASIC STAGES stage-install-dep: &stage_install_deps - - name: setup ci environment - script: - - mkdir -p .cache && ln -snf $(pwd)/.cache $HOME/.cache - - export PIP_DOWNLOAD_CACHE=${pwd}/.cache/pip_download_cache && export XDG_CACHE_HOME=${pwd}/.cache/pip - - printf "XDG_CACHE_HOME=$XDG_CACHE_HOME\nPIP_DOWNLOAD_CACHE=$PIP_DOWNLOAD_CACHE\n" - - cat /proc/cpuinfo | grep flags - - "python -c 'import tensorflow as tf; print(tf.__version__); a=tf.constant(1, tf.int32); print(tf.Session().run(a))'" - - nvcc --version - - name: install project deps - env: - INDEX_URL: http://mirrors.tencent.com/pypi/simple/ - TRUST_DOM: mirrors.tencent.com - script: - - "pip install cffi==1.12.3 coverage codecov ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" - - "pip install -e .[all] -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" - - pip uninstall -y torchvision - - pip install https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl - - "python -c 'import torchvision; print(torchvision.__version__)'" - - "python -c 'import torch; print(torch.__version__)'" + jobs: + - name: setup ci environment + script: + - mkdir -p .cache && ln -snf $(pwd)/.cache $HOME/.cache + - export PIP_DOWNLOAD_CACHE=${pwd}/.cache/pip_download_cache && export XDG_CACHE_HOME=${pwd}/.cache/pip + - printf "XDG_CACHE_HOME=$XDG_CACHE_HOME\nPIP_DOWNLOAD_CACHE=$PIP_DOWNLOAD_CACHE\n" + - cat /proc/cpuinfo | grep flags + - "python -c 'import tensorflow as tf; print(tf.__version__); a=tf.constant(1, tf.int32); print(tf.Session().run(a))'" + - nvcc --version + - name: install project deps + env: + INDEX_URL: http://mirrors.tencent.com/pypi/simple/ + TRUST_DOM: mirrors.tencent.com + script: + - "pip install cffi==1.12.3 coverage codecov ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" + - "pip install -e .[all] -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" + - pip uninstall -y torchvision + - pip install https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl + - "python -c 'import torchvision; print(torchvision.__version__)'" + - "python -c 'import torch; print(torch.__version__)'" stage-pylint: &stage_pylint - name: check code style @@ -30,19 +31,22 @@ stage-unit-test: &stage_unit_test script: python -m unittest tests/*.py -v stage-notify-done: &stage_notify - - name: make comment - script: echo "🍊 looks good to me 👍" > comment.txt - - name: submit comment - type: git:comment - options: - fromFile: comment.txt - - name: make notification - script: echo "$SUCCESS_MSG" > notify.md - - name: message - type: wework:message - options: - robot: f5ee526b-c6dc-4ef1-97c0-ac14594e57b0 - fromFile: notify.md + jobs: + job1: + - name: make comment + script: echo "🍊 looks good to me 👍" > comment.txt + - name: submit comment + type: git:comment + options: + fromFile: comment.txt + job2: + - name: make notification + script: echo "$SUCCESS_MSG" > notify.md + - name: message + type: wework:message + options: + robot: f5ee526b-c6dc-4ef1-97c0-ac14594e57b0 + fromFile: notify.md stage-commit-lint: &stage_commit_lint - name: do commit lint From 715edf7b56ba804b6b97b9d28978b6d7262f059d Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Wed, 23 Oct 2019 16:41:20 +0800 Subject: [PATCH 12/17] fix(ci): orange ci fixing and improve --- .orange-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.orange-ci.yml b/.orange-ci.yml index bcadc452..a4a1fc4f 100644 --- a/.orange-ci.yml +++ b/.orange-ci.yml @@ -12,11 +12,11 @@ stage-install-dep: &stage_install_deps - nvcc --version - name: install project deps env: - INDEX_URL: http://mirrors.tencent.com/pypi/simple/ + INDEX_URL: https://mirrors.tencent.com/pypi/simple/ TRUST_DOM: mirrors.tencent.com script: - - "pip install cffi==1.12.3 coverage codecov ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" - - "pip install -e .[all] -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" + - "pip install cffi==1.12.3 coverage codecov ${INDEX_URL} -i ${INDEX_URL}" + - "pip install -e .[all] -i ${INDEX_URL}" - pip uninstall -y torchvision - pip install https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl - "python -c 'import torchvision; print(torchvision.__version__)'" From 2e91d68b932dbdd7501905accbce188113e6bb01 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Wed, 23 Oct 2019 16:43:32 +0800 Subject: [PATCH 13/17] fix(ci): orange ci fixing and improve --- .orange-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.orange-ci.yml b/.orange-ci.yml index a4a1fc4f..03b01e77 100644 --- a/.orange-ci.yml +++ b/.orange-ci.yml @@ -15,8 +15,8 @@ stage-install-dep: &stage_install_deps INDEX_URL: https://mirrors.tencent.com/pypi/simple/ TRUST_DOM: mirrors.tencent.com script: - - "pip install cffi==1.12.3 coverage codecov ${INDEX_URL} -i ${INDEX_URL}" - - "pip install -e .[all] -i ${INDEX_URL}" + - "pip install cffi==1.12.3 coverage codecov -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" + - "pip install -e .[all] -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" - pip uninstall -y torchvision - pip install https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl - "python -c 'import torchvision; print(torchvision.__version__)'" From 91503f0c908ace3d7eb2e40e1a200062c1d51e00 Mon Sep 17 00:00:00 2001 From: Han Xiao Date: Thu, 24 Oct 2019 10:47:34 +0800 Subject: [PATCH 14/17] docs(flow): fix formatting in flow docs --- .orange-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.orange-ci.yml b/.orange-ci.yml index 03b01e77..f75d1d6f 100644 --- a/.orange-ci.yml +++ b/.orange-ci.yml @@ -78,7 +78,7 @@ pipe-unit-test: &bp1 wework: title: "🍊⌛🏗 Install GNES and do unit test" docker: - image: gnes/ci-base + image: ccr.ccs.tencentyun.com/gnes/ci-base env: GNES_ENV_SET: orange-ci PIP_DOWNLOAD_CACHE: ${pwd}/.cache/pip_download_cache @@ -95,7 +95,7 @@ pipe-coverity: &cp1 wework: title: "🍊⌛🏗 Do coverity check on the new master" docker: - image: gnes/ci-base + image: ccr.ccs.tencentyun.com/gnes/ci-base env: GNES_ENV_SET: orange-ci PIP_DOWNLOAD_CACHE: ${pwd}/.cache/pip_download_cache From efc1e2a03742a203838d458f4c9b01f4f44366d9 Mon Sep 17 00:00:00 2001 From: Han Xiao Date: Thu, 24 Oct 2019 11:28:53 +0800 Subject: [PATCH 15/17] docs(flow): fix formatting in flow docs --- .orange-ci.yml | 135 +++++++++++++++++++++++++------------------------ 1 file changed, 69 insertions(+), 66 deletions(-) diff --git a/.orange-ci.yml b/.orange-ci.yml index f75d1d6f..a80c43d8 100644 --- a/.orange-ci.yml +++ b/.orange-ci.yml @@ -1,26 +1,27 @@ # BASIC STAGES stage-install-dep: &stage_install_deps - jobs: - - name: setup ci environment - script: - - mkdir -p .cache && ln -snf $(pwd)/.cache $HOME/.cache - - export PIP_DOWNLOAD_CACHE=${pwd}/.cache/pip_download_cache && export XDG_CACHE_HOME=${pwd}/.cache/pip - - printf "XDG_CACHE_HOME=$XDG_CACHE_HOME\nPIP_DOWNLOAD_CACHE=$PIP_DOWNLOAD_CACHE\n" - - cat /proc/cpuinfo | grep flags - - "python -c 'import tensorflow as tf; print(tf.__version__); a=tf.constant(1, tf.int32); print(tf.Session().run(a))'" - - nvcc --version - - name: install project deps - env: - INDEX_URL: https://mirrors.tencent.com/pypi/simple/ - TRUST_DOM: mirrors.tencent.com - script: - - "pip install cffi==1.12.3 coverage codecov -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" - - "pip install -e .[all] -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" - - pip uninstall -y torchvision - - pip install https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl - - "python -c 'import torchvision; print(torchvision.__version__)'" - - "python -c 'import torch; print(torch.__version__)'" + - name: install dependencies + jobs: + - name: setup ci environment + script: + - mkdir -p .cache && ln -snf $(pwd)/.cache $HOME/.cache + - export PIP_DOWNLOAD_CACHE=${pwd}/.cache/pip_download_cache && export XDG_CACHE_HOME=${pwd}/.cache/pip + - printf "XDG_CACHE_HOME=$XDG_CACHE_HOME\nPIP_DOWNLOAD_CACHE=$PIP_DOWNLOAD_CACHE\n" + - cat /proc/cpuinfo | grep flags + - "python -c 'import tensorflow as tf; print(tf.__version__); a=tf.constant(1, tf.int32); print(tf.Session().run(a))'" + - nvcc --version + - name: install project deps + env: + INDEX_URL: https://mirrors.tencent.com/pypi/simple/ + TRUST_DOM: mirrors.tencent.com + script: + - "pip install cffi==1.12.3 coverage codecov -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" + - "pip install -e .[all] -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" + - pip uninstall -y torchvision + - pip install https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl + - "python -c 'import torchvision; print(torchvision.__version__)'" + - "python -c 'import torch; print(torch.__version__)'" stage-pylint: &stage_pylint - name: check code style @@ -31,21 +32,46 @@ stage-unit-test: &stage_unit_test script: python -m unittest tests/*.py -v stage-notify-done: &stage_notify - jobs: - job1: - - name: make comment - script: echo "🍊 looks good to me 👍" > comment.txt - - name: submit comment - type: git:comment - options: - fromFile: comment.txt - job2: + - name: notify the success + jobs: + job1: + - name: make comment + script: echo "🍊 looks good to me 👍" > comment.txt + - name: submit comment + type: git:comment + options: + fromFile: comment.txt + job2: + - name: make notification + script: echo "$SUCCESS_MSG" > notify.md + - name: message + type: wework:message + options: + robot: $BOT_ID + fromFile: notify.md + +stage-notify-all: &stage_notify_all + - name: notfiy the success to all + jobs: - name: make notification - script: echo "$SUCCESS_MSG" > notify.md + script: echo "# 🍊🚀 GNES的Master分支刚刚更新了!\n此次更新由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH合并后触发,[点击查看]($ORANGE_EVENT_URL)" > notify.md + - name: send mail + type: mail:send + options: + title: 🍊🚀 GNES的Master分支刚刚更新! + to: + - hanhxiao + - larryjfyan + - madwang + - leonjiang + - raccoonliu + - kegokang + type: markdown + fromFile: notify.md - name: message type: wework:message options: - robot: f5ee526b-c6dc-4ef1-97c0-ac14594e57b0 + robot: $BOT_ID fromFile: notify.md stage-commit-lint: &stage_commit_lint @@ -64,7 +90,7 @@ stage-coverity: &stage_coverity # lint -pipe-lint: &bp0 +pipe-lint: &pipe_lint docker: image: node:alpine network: idc-hk @@ -73,7 +99,7 @@ pipe-lint: &bp0 stages: - <<: *stage_commit_lint -pipe-unit-test: &bp1 +pipe-unit-test: &pipe_unit_test network: idc-ai-sse4 wework: title: "🍊⌛🏗 Install GNES and do unit test" @@ -90,7 +116,7 @@ pipe-unit-test: &bp1 - <<: *stage_unit_test - <<: *stage_notify -pipe-coverity: &cp1 +pipe-coverity: &pipe_coverity network: idc-ai-sse4 wework: title: "🍊⌛🏗 Do coverity check on the new master" @@ -101,36 +127,14 @@ pipe-coverity: &cp1 PIP_DOWNLOAD_CACHE: ${pwd}/.cache/pip_download_cache XDG_CACHE_HOME: ${pwd}/.cache/pip SUCCESS_MSG: "# 🍊✅😃 Report is successfully updated!" - envFrom: https://git.code.oa.com/gnes/gnes-secret/blob/master/env + envFrom: https://git.code.oa.com/gnes/secret-keys/blob/master/keys cacheFrom: .orange-ci.cache stages: - <<: *stage_install_deps - <<: *stage_coverity - <<: *stage_notify + - <<: *stage_notify_all -notify-pipeline1: &np1 - network: devnet - stages: - - name: make notification - script: echo "# 🚀 NES的Master分支刚刚更新了!\n此次更新由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH合并后触发,[点击查看](https://git.code.oa.com/ai-innersource/nes/)" > notify.md - - name: send mail - type: mail:send - options: - title: 🚀 NES的Master分支刚刚更新! - to: - - hanhxiao - - larryjfyan - - madwang - - leonjiang - - raccoonliu - - kegokang - type: markdown - fromFile: notify.md - - name: message - type: wework:message - options: - robot: f5ee526b-c6dc-4ef1-97c0-ac14594e57b0 - fromFile: notify.md docker-stage: &docker_stage - name: docker push @@ -164,7 +168,7 @@ release-stage: &release_stage1 - name: message type: wework:message options: - robot: f5ee526b-c6dc-4ef1-97c0-ac14594e57b0 + robot: $BOT_ID fromFile: notify.md release-stage2: &release_stage2 @@ -181,14 +185,14 @@ release-stage2: &release_stage2 - name: message type: wework:message options: - robot: 33f48450-4e17-4e99-b476-b1c97ea233c6 + robot: $BOT_ID fromFile: notify.md docker-pipeline1: &dp1 network: devnet services: - docker - envFrom: https://git.code.oa.com/hanhxiao/nes-keys/blob/master/env + envFrom: https://git.code.oa.com/gnes/secret-keys/blob/master/keys env: PROJ_NAME: aipd-gnes GIT_TAG: master @@ -200,7 +204,7 @@ docker-pipeline2: &dp2 network: devnet services: - docker - envFrom: https://git.code.oa.com/hanhxiao/nes-keys/blob/master/env + envFrom: https://git.code.oa.com/gnes/secret-keys/blob/master/keys env: PROJ_NAME: aipd-gnes GIT_TAG: $ORANGE_BRANCH @@ -210,11 +214,10 @@ docker-pipeline2: &dp2 master: merge_request: - - <<: *bp0 - - <<: *bp1 + - <<: *pipe_lint + - <<: *pipe_unit_test push: - - <<: *np1 - - <<: *dp1 + - <<: *pipe_coverity "**": tag_push: From 12cd69c04e080ea1b4e84eae90e177442c412469 Mon Sep 17 00:00:00 2001 From: Han Xiao Date: Thu, 24 Oct 2019 12:15:14 +0800 Subject: [PATCH 16/17] docs(flow): fix formatting in flow docs --- .orange-ci.yml | 94 ++++++++++---------------------------------------- 1 file changed, 18 insertions(+), 76 deletions(-) diff --git a/.orange-ci.yml b/.orange-ci.yml index a80c43d8..db4edb24 100644 --- a/.orange-ci.yml +++ b/.orange-ci.yml @@ -1,4 +1,5 @@ -# BASIC STAGES +################### +# STAGES stage-install-dep: &stage_install_deps - name: install dependencies @@ -54,11 +55,11 @@ stage-notify-all: &stage_notify_all - name: notfiy the success to all jobs: - name: make notification - script: echo "# 🍊🚀 GNES的Master分支刚刚更新了!\n此次更新由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH合并后触发,[点击查看]($ORANGE_EVENT_URL)" > notify.md + script: echo "$SUCCESS_ALL_MSG" > notify.md - name: send mail type: mail:send options: - title: 🍊🚀 GNES的Master分支刚刚更新! + title: $SUCCESS_ALL_MSG to: - hanhxiao - larryjfyan @@ -88,7 +89,14 @@ stage-coverity: &stage_coverity - coverage report --skip-covered - codecov --token $CODECOV_TOKEN -# lint + +stage-docker: &stage_docker + - name: docker push + script: + - ./docker-build.sh + +################### +# PIPELINES pipe-lint: &pipe_lint docker: @@ -127,6 +135,7 @@ pipe-coverity: &pipe_coverity PIP_DOWNLOAD_CACHE: ${pwd}/.cache/pip_download_cache XDG_CACHE_HOME: ${pwd}/.cache/pip SUCCESS_MSG: "# 🍊✅😃 Report is successfully updated!" + SUCCESS_ALL_MSG: "# 🍊🚀 GNES的Master分支刚刚更新了!\n此次更新由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH合并后触发,[点击查看]($ORANGE_EVENT_URL)" envFrom: https://git.code.oa.com/gnes/secret-keys/blob/master/keys cacheFrom: .orange-ci.cache stages: @@ -135,82 +144,14 @@ pipe-coverity: &pipe_coverity - <<: *stage_notify - <<: *stage_notify_all - -docker-stage: &docker_stage - - name: docker push - jobs: - - name: docker login - script: docker login -u $DOCKER_USER -p $DOCKER_PWD docker.oa.com:8080 - - name: push docker - script: export DOCKER_NAMESPACE="docker.oa.com:8080/public" && bash docker-push.sh $GIT_TAG - - name: make notification - script: export DOCKER_NAMESPACE="docker.oa.com:8080/public" && echo "# 🗳️ NES容器的$GIT_TAG版本刚刚更新至$DOCKER_NAMESPACE!\n运行此版本:\ndocker pull $DOCKER_NAMESPACE/$PROJ_NAME:$GIT_TAG && docker run -v /data1/cips/data:/ext_data -it $DOCKER_NAMESPACE/$PROJ_NAME:$GIT_TAG bash" >>notify.md - - name: docker login - script: docker login -u $TCLOUD_USER -p $TCLOUD_PWD ccr.ccs.tencentyun.com - - name: push docker - script: export DOCKER_NAMESPACE="ccr.ccs.tencentyun.com/gnes" && bash docker-push.sh $GIT_TAG - - name: make notification - script: export DOCKER_NAMESPACE="ccr.ccs.tencentyun.com/gnes" && echo "# 🗳️ NES容器的$GIT_TAG版本刚刚更新至$DOCKER_NAMESPACE!\n运行此版本:\ndocker pull $DOCKER_NAMESPACE/$PROJ_NAME:$GIT_TAG && docker run -v /data1/cips/data:/ext_data -it $DOCKER_NAMESPACE/$PROJ_NAME:$GIT_TAG bash" >> notify.md - -release-stage: &release_stage1 - - name: notify about release - jobs: - - name: send mail - type: mail:send - options: - title: 🗳️ NES容器的$GIT_TAG刚刚更新! - to: - - hanhxiao - - larryjfyan - - madwang - type: markdown - fromFile: notify.md - - name: message - type: wework:message - options: - robot: $BOT_ID - fromFile: notify.md - -release-stage2: &release_stage2 - - name: notify about release - jobs: - - name: send mail - type: mail:send - options: - title: 🗳️ NES容器的$GIT_TAG刚刚更新! - to: - - g_AIPD_SC_RELEVANCE - type: markdown - fromFile: notify.md - - name: message - type: wework:message - options: - robot: $BOT_ID - fromFile: notify.md - -docker-pipeline1: &dp1 - network: devnet - services: - - docker - envFrom: https://git.code.oa.com/gnes/secret-keys/blob/master/keys - env: - PROJ_NAME: aipd-gnes - GIT_TAG: master - stages: - - <<: *docker_stage - - <<: *release_stage1 - -docker-pipeline2: &dp2 +pipe-docker: &pipe_docker network: devnet services: - docker envFrom: https://git.code.oa.com/gnes/secret-keys/blob/master/keys - env: - PROJ_NAME: aipd-gnes - GIT_TAG: $ORANGE_BRANCH stages: - - <<: *docker_stage - - <<: *release_stage2 + - <<: *stage_docker + - <<: *stage_notify_all master: merge_request: @@ -218,7 +159,8 @@ master: - <<: *pipe_unit_test push: - <<: *pipe_coverity + - <<: *pipe_docker "**": tag_push: - - <<: *dp2 \ No newline at end of file + - <<: *pipe_docker \ No newline at end of file From 388bd2faeb51975f507d04909c883f0ff0b7655a Mon Sep 17 00:00:00 2001 From: Han Xiao Date: Thu, 24 Oct 2019 12:49:55 +0800 Subject: [PATCH 17/17] fix(base): fix socket linger --- gnes/service/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gnes/service/base.py b/gnes/service/base.py index 1e114dcb..4d7dd254 100644 --- a/gnes/service/base.py +++ b/gnes/service/base.py @@ -141,6 +141,7 @@ def build_socket(ctx: 'zmq.Context', host: str, port: int, SocketType.PAIR_BIND: lambda: ctx.socket(zmq.PAIR), SocketType.PAIR_CONNECT: lambda: ctx.socket(zmq.PAIR) }[socket_type]() + sock.setsockopt(zmq.LINGER, 0) if socket_type.is_bind: if use_ipc: