Merge pull request #356 from gnes-ai/fix-orange-ci

ci(orange): fix orange ci settings
gnes-ai · Oct 24, 2019 · b4d2c8c · b4d2c8c
2 parents 5de329d + 388bd2f
commit b4d2c8c
Show file tree

Hide file tree

Showing 2 changed files with 124 additions and 121 deletions.
diff --git a/.orange-ci.yml b/.orange-ci.yml
@@ -1,164 +1,166 @@
-ok-stage: &ok_stage
-  - name: git comment
-    jobs:
-      - name: make comment
-        script: echo "looks good to me 👍" > comment.txt
-      - name: submit comment
-        type: git:comment
-        options:
-          fromFile: comment.txt
-      - name: make notification
-        script: echo "# ✅ 由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH分支刚刚测试通过！\n[点击查看](https://git.code.oa.com/ai-innersource/nes/)" > notify.md
-      - name: message
-        type: wework:message
-        options:
-          robot: f5ee526b-c6dc-4ef1-97c0-ac14594e57b0
-          fromFile: notify.md
+###################
+# STAGES
 
-install-deps: &install_deps
+stage-install-dep: &stage_install_deps
   - name: install dependencies
     jobs:
-      - name: setup cache to local
-        script: mkdir -p .cache && ln -snf $(pwd)/.cache $HOME/.cache
-      - name: show cache folders
-        script: printf "XDG_CACHE_HOME=$XDG_CACHE_HOME\nPIP_DOWNLOAD_CACHE=$PIP_DOWNLOAD_CACHE\n"
-      - name: show cpu flag
-        script: cat /proc/cpuinfo | grep flags
+      - name: setup ci environment
+        script:
+          - mkdir -p .cache && ln -snf $(pwd)/.cache $HOME/.cache
+          - export PIP_DOWNLOAD_CACHE=${pwd}/.cache/pip_download_cache && export XDG_CACHE_HOME=${pwd}/.cache/pip
+          - printf "XDG_CACHE_HOME=$XDG_CACHE_HOME\nPIP_DOWNLOAD_CACHE=$PIP_DOWNLOAD_CACHE\n"
+          - cat /proc/cpuinfo | grep flags
+          - "python -c 'import tensorflow as tf; print(tf.__version__); a=tf.constant(1, tf.int32); print(tf.Session().run(a))'"
+          - nvcc --version
       - name: install project deps
         env:
-          INDEX_URL: http://mirrors.oa.com/pypi/web/simple/
-          TRUST_DOM: mirrors.oa.com
-        script: "pip install -e .[all] -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}"
-      - name: check tensorflow works correctly
-        script: "python -c 'import tensorflow as tf; print(tf.__version__); a=tf.constant(1, tf.int32); print(tf.Session().run(a))'"
+          INDEX_URL: https://mirrors.tencent.com/pypi/simple/
+          TRUST_DOM: mirrors.tencent.com
+        script:
+          - "pip install cffi==1.12.3 coverage codecov -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}"
+          - "pip install -e .[all] -i ${INDEX_URL} --extra-index-url ${INDEX_URL} --trusted-host ${TRUST_DOM}"
+          - pip uninstall -y torchvision
+          - pip install https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
+          - "python -c 'import torchvision; print(torchvision.__version__)'"
+          - "python -c 'import torch; print(torch.__version__)'"
 
-pylint: &pylint
+stage-pylint: &stage_pylint
   - name: check code style
     script: pylint gnes/**/*.py --exit-zero
 
-unit-test: &unit_test
+stage-unit-test: &stage_unit_test
   - name: run unit test
-    script: python -m unittest tests/*.py
-
-basic-pipeline: &bp1
-  network: idc-ai-sse4
-  docker:
-    image: docker.oa.com:8080/public/ailab-faiss-bert-elmo-openai-w2v:latest
-  env:
-    GNES_ENV_SET: orange-ci
-    PIP_DOWNLOAD_CACHE: ${pwd}.cache/pip_download_cache
-    XDG_CACHE_HOME: ${pwd}.cache/pip
-  cacheFrom: .orange-ci.cache
-  stages:
-    - <<: *install_deps
-    - <<: *pylint
-    - <<: *unit_test
-    - <<: *ok_stage
+    script: python -m unittest tests/*.py -v
 
-notify-pipeline1: &np1
-  network: devnet
-  stages:
-    - name: make notification
-      script: echo "# 🚀 NES的Master分支刚刚更新了！\n此次更新由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH合并后触发，[点击查看](https://git.code.oa.com/ai-innersource/nes/)" > notify.md
-    - name: send mail
-      type: mail:send
-      options:
-        title: 🚀 NES的Master分支刚刚更新！
-        to:
-          - hanhxiao
-          - larryjfyan
-          - madwang
-        type: markdown
-        fromFile: notify.md
-    - name: message
-      type: wework:message
-      options:
-        robot: f5ee526b-c6dc-4ef1-97c0-ac14594e57b0
-        fromFile: notify.md
-
-docker-stage: &docker_stage
-  - name: docker push
+stage-notify-done: &stage_notify
+  - name: notify the success
     jobs:
-      - name: docker login
-        script: docker login -u $DOCKER_USER -p $DOCKER_PWD docker.oa.com:8080
-      - name: push docker
-        script: export DOCKER_NAMESPACE="docker.oa.com:8080/public" && bash docker-push.sh $GIT_TAG
-      - name: make notification
-        script: export DOCKER_NAMESPACE="docker.oa.com:8080/public" && echo "# 🗳️ NES容器的$GIT_TAG版本刚刚更新至$DOCKER_NAMESPACE！\n运行此版本：\ndocker pull $DOCKER_NAMESPACE/$PROJ_NAME:$GIT_TAG && docker run -v /data1/cips/data:/ext_data -it $DOCKER_NAMESPACE/$PROJ_NAME:$GIT_TAG bash" >>notify.md
-      - name: docker login
-        script: docker login -u $TCLOUD_USER -p $TCLOUD_PWD ccr.ccs.tencentyun.com
-      - name: push docker
-        script: export DOCKER_NAMESPACE="ccr.ccs.tencentyun.com/gnes" && bash docker-push.sh $GIT_TAG
-      - name: make notification
-        script: export DOCKER_NAMESPACE="ccr.ccs.tencentyun.com/gnes" && echo "# 🗳️ NES容器的$GIT_TAG版本刚刚更新至$DOCKER_NAMESPACE！\n运行此版本：\ndocker pull $DOCKER_NAMESPACE/$PROJ_NAME:$GIT_TAG && docker run -v /data1/cips/data:/ext_data -it $DOCKER_NAMESPACE/$PROJ_NAME:$GIT_TAG bash" >> notify.md
+      job1:
+        - name: make comment
+          script: echo "🍊 looks good to me 👍" > comment.txt
+        - name: submit comment
+          type: git:comment
+          options:
+            fromFile: comment.txt
+      job2:
+        - name: make notification
+          script: echo "$SUCCESS_MSG" > notify.md
+        - name: message
+          type: wework:message
+          options:
+            robot: $BOT_ID
+            fromFile: notify.md
 
-release-stage: &release_stage1
-  - name: notify about release
+stage-notify-all: &stage_notify_all
+  - name: notfiy the success to all
     jobs:
+      - name: make notification
+        script: echo "$SUCCESS_ALL_MSG" > notify.md
       - name: send mail
         type: mail:send
         options:
-          title: 🗳️ NES容器的$GIT_TAG刚刚更新！
+          title: $SUCCESS_ALL_MSG
           to:
             - hanhxiao
             - larryjfyan
             - madwang
+            - leonjiang
+            - raccoonliu
+            - kegokang
           type: markdown
           fromFile: notify.md
       - name: message
         type: wework:message
         options:
-          robot: f5ee526b-c6dc-4ef1-97c0-ac14594e57b0
+          robot: $BOT_ID
           fromFile: notify.md
 
-release-stage2: &release_stage2
-  - name: notify about release
-    jobs:
-      - name: send mail
-        type: mail:send
-        options:
-          title: 🗳️ NES容器的$GIT_TAG刚刚更新！
-          to:
-            - g_AIPD_SC_RELEVANCE
-          type: markdown
-          fromFile: notify.md
-      - name: message
-        type: wework:message
-        options:
-          robot: 33f48450-4e17-4e99-b476-b1c97ea233c6
-          fromFile: notify.md
+stage-commit-lint: &stage_commit_lint
+  - name: do commit lint
+    script:
+      - npm install --global --save-dev @commitlint/config-conventional @commitlint/cli --registry=http://r.tnpm.oa.com
+      - "echo \"module.exports = {extends: ['@commitlint/config-conventional']}\" > commitlint.config.js"
+      - echo $ORANGE_LATEST_COMMIT_MESSAGE | commitlint
 
-docker-pipeline1: &dp1
-  network: devnet
-  services:
-    - docker
-  envFrom: https://git.code.oa.com/hanhxiao/nes-keys/blob/master/env
+stage-coverity: &stage_coverity
+  - name: do coverity
+    script:
+      - coverage run -m unittest discover -s tests/
+      - coverage report --skip-covered
+      - codecov --token $CODECOV_TOKEN
+
+
+stage-docker: &stage_docker
+  - name: docker push
+    script:
+      - ./docker-build.sh
+
+###################
+# PIPELINES
+
+pipe-lint: &pipe_lint
+  docker:
+    image: node:alpine
+  network: idc-hk
+  wework:
+    title: "🍊⌛🖊️ Check commit lint"
+  stages:
+    - <<: *stage_commit_lint
+
+pipe-unit-test: &pipe_unit_test
+  network: idc-ai-sse4
+  wework:
+    title: "🍊⌛🏗 Install GNES and do unit test"
+  docker:
+    image: ccr.ccs.tencentyun.com/gnes/ci-base
   env:
-    PROJ_NAME: aipd-gnes
-    GIT_TAG: master
+    GNES_ENV_SET: orange-ci
+    PIP_DOWNLOAD_CACHE: ${pwd}/.cache/pip_download_cache
+    XDG_CACHE_HOME: ${pwd}/.cache/pip
+    SUCCESS_MSG: "# 🍊✅😃 All tests passed, good job! \n由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH分支刚刚测试通过！\n[点击查看]($ORANGE_EVENT_URL)"
+  cacheFrom: .orange-ci.cache
   stages:
-    - <<: *docker_stage
-    - <<: *release_stage1
+    - <<: *stage_install_deps
+    - <<: *stage_unit_test
+    - <<: *stage_notify
 
-docker-pipeline2: &dp2
+pipe-coverity: &pipe_coverity
+  network: idc-ai-sse4
+  wework:
+    title: "🍊⌛🏗 Do coverity check on the new master"
+  docker:
+    image: ccr.ccs.tencentyun.com/gnes/ci-base
+  env:
+    GNES_ENV_SET: orange-ci
+    PIP_DOWNLOAD_CACHE: ${pwd}/.cache/pip_download_cache
+    XDG_CACHE_HOME: ${pwd}/.cache/pip
+    SUCCESS_MSG: "# 🍊✅😃 Report is successfully updated!"
+    SUCCESS_ALL_MSG: "# 🍊🚀 GNES的Master分支刚刚更新了！\n此次更新由$ORANGE_BUILD_USER提交的$ORANGE_MERGE_REQUEST_BRANCH合并后触发，[点击查看]($ORANGE_EVENT_URL)"
+  envFrom: https://git.code.oa.com/gnes/secret-keys/blob/master/keys
+  cacheFrom: .orange-ci.cache
+  stages:
+    - <<: *stage_install_deps
+    - <<: *stage_coverity
+    - <<: *stage_notify
+    - <<: *stage_notify_all
+
+pipe-docker: &pipe_docker
   network: devnet
   services:
     - docker
-  envFrom: https://git.code.oa.com/hanhxiao/nes-keys/blob/master/env
-  env:
-    PROJ_NAME: aipd-gnes
-    GIT_TAG: $ORANGE_BRANCH
+  envFrom: https://git.code.oa.com/gnes/secret-keys/blob/master/keys
   stages:
-    - <<: *docker_stage
-    - <<: *release_stage2
+    - <<: *stage_docker
+    - <<: *stage_notify_all
 
 master:
   merge_request:
-    - <<: *bp1
+    - <<: *pipe_lint
+    - <<: *pipe_unit_test
   push:
-    - <<: *np1
-    - <<: *dp1
+    - <<: *pipe_coverity
+    - <<: *pipe_docker
 
 "**":
   tag_push:
-    - <<: *dp2
+    - <<: *pipe_docker
diff --git a/gnes/service/base.py b/gnes/service/base.py
@@ -141,6 +141,7 @@ def build_socket(ctx: 'zmq.Context', host: str, port: int,
         SocketType.PAIR_BIND: lambda: ctx.socket(zmq.PAIR),
         SocketType.PAIR_CONNECT: lambda: ctx.socket(zmq.PAIR)
     }[socket_type]()
+    sock.setsockopt(zmq.LINGER, 0)
 
     if socket_type.is_bind:
         if use_ipc: