diff --git a/.orange-ci.yml b/.orange-ci.yml index f148315a..db4edb24 100644 --- a/.orange-ci.yml +++ b/.orange-ci.yml @@ -1,164 +1,166 @@ -ok-stage: &ok_stage - - name: git comment - jobs: - - name: make comment - script: echo "looks good to me 👍" > comment.txt - - name: submit comment - type: git:comment - options: - fromFile: comment.txt - - name: make notification - script: echo "# ✅ 由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH分支刚刚测试通过!\n[点击查看](https://git.code.oa.com/ai-innersource/nes/)" > notify.md - - name: message - type: wework:message - options: - robot: f5ee526b-c6dc-4ef1-97c0-ac14594e57b0 - fromFile: notify.md +################### +# STAGES -install-deps: &install_deps +stage-install-dep: &stage_install_deps - name: install dependencies jobs: - - name: setup cache to local - script: mkdir -p .cache && ln -snf $(pwd)/.cache $HOME/.cache - - name: show cache folders - script: printf "XDG_CACHE_HOME=$XDG_CACHE_HOME\nPIP_DOWNLOAD_CACHE=$PIP_DOWNLOAD_CACHE\n" - - name: show cpu flag - script: cat /proc/cpuinfo | grep flags + - name: setup ci environment + script: + - mkdir -p .cache && ln -snf $(pwd)/.cache $HOME/.cache + - export PIP_DOWNLOAD_CACHE=${pwd}/.cache/pip_download_cache && export XDG_CACHE_HOME=${pwd}/.cache/pip + - printf "XDG_CACHE_HOME=$XDG_CACHE_HOME\nPIP_DOWNLOAD_CACHE=$PIP_DOWNLOAD_CACHE\n" + - cat /proc/cpuinfo | grep flags + - "python -c 'import tensorflow as tf; print(tf.__version__); a=tf.constant(1, tf.int32); print(tf.Session().run(a))'" + - nvcc --version - name: install project deps env: - INDEX_URL: http://mirrors.oa.com/pypi/web/simple/ - TRUST_DOM: mirrors.oa.com - script: "pip install -e .[all] -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" - - name: check tensorflow works correctly - script: "python -c 'import tensorflow as tf; print(tf.__version__); a=tf.constant(1, tf.int32); print(tf.Session().run(a))'" + INDEX_URL: https://mirrors.tencent.com/pypi/simple/ + TRUST_DOM: mirrors.tencent.com + script: + - "pip install cffi==1.12.3 coverage codecov -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" + - "pip install -e .[all] -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}" + - pip uninstall -y torchvision + - pip install https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl + - "python -c 'import torchvision; print(torchvision.__version__)'" + - "python -c 'import torch; print(torch.__version__)'" -pylint: &pylint +stage-pylint: &stage_pylint - name: check code style script: pylint gnes/**/*.py --exit-zero -unit-test: &unit_test +stage-unit-test: &stage_unit_test - name: run unit test - script: python -m unittest tests/*.py - -basic-pipeline: &bp1 - network: idc-ai-sse4 - docker: - image: docker.oa.com:8080/public/ailab-faiss-bert-elmo-openai-w2v:latest - env: - GNES_ENV_SET: orange-ci - PIP_DOWNLOAD_CACHE: ${pwd}.cache/pip_download_cache - XDG_CACHE_HOME: ${pwd}.cache/pip - cacheFrom: .orange-ci.cache - stages: - - <<: *install_deps - - <<: *pylint - - <<: *unit_test - - <<: *ok_stage + script: python -m unittest tests/*.py -v -notify-pipeline1: &np1 - network: devnet - stages: - - name: make notification - script: echo "# 🚀 NES的Master分支刚刚更新了!\n此次更新由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH合并后触发,[点击查看](https://git.code.oa.com/ai-innersource/nes/)" > notify.md - - name: send mail - type: mail:send - options: - title: 🚀 NES的Master分支刚刚更新! - to: - - hanhxiao - - larryjfyan - - madwang - type: markdown - fromFile: notify.md - - name: message - type: wework:message - options: - robot: f5ee526b-c6dc-4ef1-97c0-ac14594e57b0 - fromFile: notify.md - -docker-stage: &docker_stage - - name: docker push +stage-notify-done: &stage_notify + - name: notify the success jobs: - - name: docker login - script: docker login -u $DOCKER_USER -p $DOCKER_PWD docker.oa.com:8080 - - name: push docker - script: export DOCKER_NAMESPACE="docker.oa.com:8080/public" && bash docker-push.sh $GIT_TAG - - name: make notification - script: export DOCKER_NAMESPACE="docker.oa.com:8080/public" && echo "# 🗳️ NES容器的$GIT_TAG版本刚刚更新至$DOCKER_NAMESPACE!\n运行此版本:\ndocker pull $DOCKER_NAMESPACE/$PROJ_NAME:$GIT_TAG && docker run -v /data1/cips/data:/ext_data -it $DOCKER_NAMESPACE/$PROJ_NAME:$GIT_TAG bash" >>notify.md - - name: docker login - script: docker login -u $TCLOUD_USER -p $TCLOUD_PWD ccr.ccs.tencentyun.com - - name: push docker - script: export DOCKER_NAMESPACE="ccr.ccs.tencentyun.com/gnes" && bash docker-push.sh $GIT_TAG - - name: make notification - script: export DOCKER_NAMESPACE="ccr.ccs.tencentyun.com/gnes" && echo "# 🗳️ NES容器的$GIT_TAG版本刚刚更新至$DOCKER_NAMESPACE!\n运行此版本:\ndocker pull $DOCKER_NAMESPACE/$PROJ_NAME:$GIT_TAG && docker run -v /data1/cips/data:/ext_data -it $DOCKER_NAMESPACE/$PROJ_NAME:$GIT_TAG bash" >> notify.md + job1: + - name: make comment + script: echo "🍊 looks good to me 👍" > comment.txt + - name: submit comment + type: git:comment + options: + fromFile: comment.txt + job2: + - name: make notification + script: echo "$SUCCESS_MSG" > notify.md + - name: message + type: wework:message + options: + robot: $BOT_ID + fromFile: notify.md -release-stage: &release_stage1 - - name: notify about release +stage-notify-all: &stage_notify_all + - name: notfiy the success to all jobs: + - name: make notification + script: echo "$SUCCESS_ALL_MSG" > notify.md - name: send mail type: mail:send options: - title: 🗳️ NES容器的$GIT_TAG刚刚更新! + title: $SUCCESS_ALL_MSG to: - hanhxiao - larryjfyan - madwang + - leonjiang + - raccoonliu + - kegokang type: markdown fromFile: notify.md - name: message type: wework:message options: - robot: f5ee526b-c6dc-4ef1-97c0-ac14594e57b0 + robot: $BOT_ID fromFile: notify.md -release-stage2: &release_stage2 - - name: notify about release - jobs: - - name: send mail - type: mail:send - options: - title: 🗳️ NES容器的$GIT_TAG刚刚更新! - to: - - g_AIPD_SC_RELEVANCE - type: markdown - fromFile: notify.md - - name: message - type: wework:message - options: - robot: 33f48450-4e17-4e99-b476-b1c97ea233c6 - fromFile: notify.md +stage-commit-lint: &stage_commit_lint + - name: do commit lint + script: + - npm install --global --save-dev @commitlint/config-conventional @commitlint/cli --registry=http://r.tnpm.oa.com + - "echo \"module.exports = {extends: ['@commitlint/config-conventional']}\" > commitlint.config.js" + - echo $ORANGE_LATEST_COMMIT_MESSAGE | commitlint -docker-pipeline1: &dp1 - network: devnet - services: - - docker - envFrom: https://git.code.oa.com/hanhxiao/nes-keys/blob/master/env +stage-coverity: &stage_coverity + - name: do coverity + script: + - coverage run -m unittest discover -s tests/ + - coverage report --skip-covered + - codecov --token $CODECOV_TOKEN + + +stage-docker: &stage_docker + - name: docker push + script: + - ./docker-build.sh + +################### +# PIPELINES + +pipe-lint: &pipe_lint + docker: + image: node:alpine + network: idc-hk + wework: + title: "🍊⌛🖊️ Check commit lint" + stages: + - <<: *stage_commit_lint + +pipe-unit-test: &pipe_unit_test + network: idc-ai-sse4 + wework: + title: "🍊⌛🏗 Install GNES and do unit test" + docker: + image: ccr.ccs.tencentyun.com/gnes/ci-base env: - PROJ_NAME: aipd-gnes - GIT_TAG: master + GNES_ENV_SET: orange-ci + PIP_DOWNLOAD_CACHE: ${pwd}/.cache/pip_download_cache + XDG_CACHE_HOME: ${pwd}/.cache/pip + SUCCESS_MSG: "# 🍊✅😃 All tests passed, good job! \n由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH分支刚刚测试通过!\n[点击查看]($ORANGE_EVENT_URL)" + cacheFrom: .orange-ci.cache stages: - - <<: *docker_stage - - <<: *release_stage1 + - <<: *stage_install_deps + - <<: *stage_unit_test + - <<: *stage_notify -docker-pipeline2: &dp2 +pipe-coverity: &pipe_coverity + network: idc-ai-sse4 + wework: + title: "🍊⌛🏗 Do coverity check on the new master" + docker: + image: ccr.ccs.tencentyun.com/gnes/ci-base + env: + GNES_ENV_SET: orange-ci + PIP_DOWNLOAD_CACHE: ${pwd}/.cache/pip_download_cache + XDG_CACHE_HOME: ${pwd}/.cache/pip + SUCCESS_MSG: "# 🍊✅😃 Report is successfully updated!" + SUCCESS_ALL_MSG: "# 🍊🚀 GNES的Master分支刚刚更新了!\n此次更新由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH合并后触发,[点击查看]($ORANGE_EVENT_URL)" + envFrom: https://git.code.oa.com/gnes/secret-keys/blob/master/keys + cacheFrom: .orange-ci.cache + stages: + - <<: *stage_install_deps + - <<: *stage_coverity + - <<: *stage_notify + - <<: *stage_notify_all + +pipe-docker: &pipe_docker network: devnet services: - docker - envFrom: https://git.code.oa.com/hanhxiao/nes-keys/blob/master/env - env: - PROJ_NAME: aipd-gnes - GIT_TAG: $ORANGE_BRANCH + envFrom: https://git.code.oa.com/gnes/secret-keys/blob/master/keys stages: - - <<: *docker_stage - - <<: *release_stage2 + - <<: *stage_docker + - <<: *stage_notify_all master: merge_request: - - <<: *bp1 + - <<: *pipe_lint + - <<: *pipe_unit_test push: - - <<: *np1 - - <<: *dp1 + - <<: *pipe_coverity + - <<: *pipe_docker "**": tag_push: - - <<: *dp2 \ No newline at end of file + - <<: *pipe_docker \ No newline at end of file diff --git a/gnes/service/base.py b/gnes/service/base.py index 1e114dcb..4d7dd254 100644 --- a/gnes/service/base.py +++ b/gnes/service/base.py @@ -141,6 +141,7 @@ def build_socket(ctx: 'zmq.Context', host: str, port: int, SocketType.PAIR_BIND: lambda: ctx.socket(zmq.PAIR), SocketType.PAIR_CONNECT: lambda: ctx.socket(zmq.PAIR) }[socket_type]() + sock.setsockopt(zmq.LINGER, 0) if socket_type.is_bind: if use_ipc: