Merge pull request #4 from gnes-ai/ci-improve

ci(wechat-work): optimize wechat notification

.drone.yml

@@ -5,18 +5,24 @@ clone:
   depth: 50
 
 steps:
+
 - name: notify the start on wechat work
-  image: gnes/ci-base
+  image: byrnedo/alpine-curl
   environment:
-    BOT_ID:
-      from_secret: gnes-wechat-botid
+    BOT_URL:
+      from_secret: BOT_URL
+  commands:
+  - export MSG_LINK=${DRONE_BUILD_LINK}
+  - export MSG_TITLE="⌛🏗 Start a CI pipeline \`$DRONE_SOURCE_BRANCH\`(\`${DRONE_BUILD_NUMBER}\`)"
+  - export MSG_CONTENT="submit by [$DRONE_COMMIT_AUTHOR]($DRONE_COMMIT_AUTHOR_EMAIL), click the link below to see the status"
+  - ./shell/push-wechatwork.sh
+
+- name: check commit style
+  image: node:alpine
   commands:
-  - "export MSG_CONTENT=\"🍽 Start a CI pipeline for $DRONE_SOURCE_BRANCH submit by $DRONE_COMMIT_AUTHOR \\n > [View]($DRONE_REPO_LINK)\""
-  - "echo '{\"msgtype\": \"markdown\", \"markdown\": {\"content\": \"'$MSG_CONTENT'\"}}' > notify.md"
-  - >
-    curl -s 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key='$BOT_ID \
-       -H 'Content-Type: application/json' \
-       --data-binary "@notify.md" > /dev/null
+  - npm install --global --save-dev @commitlint/config-conventional @commitlint/cli
+  - "echo \"module.exports = {extends: ['@commitlint/config-conventional']}\" > commitlint.config.js"
+  - echo $DRONE_COMMIT_MESSAGE | commitlint
 
 - name: setup ci environment
   image: gnes/ci-base
@@ -36,31 +42,29 @@ steps:
   - pylint gnes/**/*.py --exit-zero
   - python -m unittest tests/*.py
 
+
 - name: notify the sucess on wechat work
-  image: gnes/ci-base
+  image: byrnedo/alpine-curl
   environment:
-    BOT_ID:
-      from_secret: gnes-wechat-botid
+    BOT_URL:
+      from_secret: BOT_URL
   commands:
-  - "export MSG_CONTENT=\"✅ Looks good to me. All tests passed!\\n > Author: $DRONE_COMMIT_AUTHOR \\n > Branch: $DRONE_SOURCE_BRANCH \\n > [View]($DRONE_REPO_LINK)\""
-  - "echo '{\"msgtype\": \"markdown\", \"markdown\": {\"content\": \"'$MSG_CONTENT'\"}}' > notify.md"
-  - >
-    curl -s 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key='$BOT_ID \
-       -H 'Content-Type: application/json' \
-       --data-binary "@notify.md" > /dev/null
+  - export MSG_LINK=$DRONE_REPO_LINK
+  - export MSG_TITLE="✅😃 All tests passed, good job! \`$DRONE_SOURCE_BRANCH\`(\`${DRONE_BUILD_NUMBER}\`)"
+  - export MSG_CONTENT="the branch \`$DRONE_SOURCE_BRANCH\` submit by [$DRONE_COMMIT_AUTHOR]($DRONE_COMMIT_AUTHOR_EMAIL) is ready to merge to master"
+  - ./shell/push-wechatwork.sh
+
 
 - name: notify the failure on wechat work
-  image: gnes/ci-base
+  image: byrnedo/alpine-curl
   environment:
-    BOT_ID:
-      from_secret: gnes-wechat-botid
+    BOT_URL:
+      from_secret: BOT_URL
   commands:
-  - "export MSG_CONTENT=\"❌ CI pipeline failed!\\n > Author: $DRONE_COMMIT_AUTHOR \\n > Branch: $DRONE_SOURCE_BRANCH \\n > [View]($DRONE_REPO_LINK)\""
-  - "echo '{\"msgtype\": \"markdown\", \"markdown\": {\"content\": \"'$MSG_CONTENT'\"}}' > notify.md"
-  - >
-    curl -s 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key='$BOT_ID \
-       -H 'Content-Type: application/json' \
-       --data-binary "@notify.md" > /dev/null
+  - export MSG_LINK=$DRONE_BUILD_LINK
+  - export MSG_TITLE="❌😥 CI pipeline \`$DRONE_SOURCE_BRANCH\`(\`${DRONE_BUILD_NUMBER}\`) is failed!"
+  - export MSG_CONTENT="please inform [$DRONE_COMMIT_AUTHOR]($DRONE_COMMIT_AUTHOR_EMAIL) to modify and fix [\`$DRONE_SOURCE_BRANCH\`]($DRONE_COMMIT_LINK). click the link below to see the details."
+  - ./shell/push-wechatwork.sh
   when:
     status:
     - failure
@@ -69,5 +73,4 @@ trigger:
   branch:
   - master
   event:
-  - pull_request
-  - push
+  - pull_request

CONTRIBUTING.md

@@ -0,0 +1,32 @@
+# Contributing to GNES
+
+## Commit Message and Pull Request Naming
+
+To help everyone with understanding the commit history of GNES, we employ [`commitlint`](https://commitlint.js.org/#/) in the CI pipeline to enforce the commit styles. Specifically, our convention is:
+
+```text
+type(scope?): subject
+```
+
+where `type` is one of the following:
+
+- build
+- ci
+- chore
+- docs
+- feat
+- fix
+- perf
+- refactor
+- revert
+- style
+- test
+
+`scope` is optional, represents the module your commit working on.
+
+`subject` explains the commit.
+
+As an example, a commit that implements a new encoder should be phrased as:
+```text
+feat(encoder): add new inceptionV3 as image encoder
+``` 

README.md

@@ -17,6 +17,9 @@
 <a href='https://doc.gnes.ai/'>
     <img src='https://readthedocs.org/projects/gnes/badge/?version=latest' alt='Documentation Status' />
 </a>
+<a href="https://www.codacy.com/app/gnes-ai/gnes?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=gnes-ai/gnes&amp;utm_campaign=Badge_Grade">
+    <img src="https://api.codacy.com/project/badge/Grade/a9ce545b9f3846ba954bcd449e090984"/>
+</a>
 <a href='https://github.com/gnes-ai/gnes/blob/master/LICENSE'>
     <img alt="PyPI - License" src="https://img.shields.io/pypi/l/gnes.svg">
 </a>

gnes/preprocessor/image/simple.py

@@ -74,7 +74,8 @@ def _get_all_sliding_window(self, img: 'np.ndarray') -> List['np.ndarray']:
             writeable=False
         )
         expanded_input = expanded_input.reshape((-1, self.window_size, self.window_size, 3))
-        return [np.array(Image.fromarray(img).resize((self.target_img_size, self.target_img_size))) for img in expanded_input]
+        return [np.array(Image.fromarray(img).resize((self.target_img_size, self.target_img_size))) for img in
+                expanded_input]
 
 
 class VanillaSlidingPreprocessor(BaseSlidingPreprocessor):

gnes/preprocessor/video/base.py

@@ -19,6 +19,3 @@
 
 class BaseVideoPreprocessor(BasePreprocessor):
     doc_type = gnes_pb2.Document.VIDEO
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)

gnes/preprocessor/video/ffmpeg.py

@@ -13,21 +13,22 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 import io
+import subprocess as sp
 from typing import List
+
 import numpy as np
 from PIL import Image
-import imagehash
+
 from .base import BaseVideoPreprocessor
 from ...proto import gnes_pb2, array2blob
-import subprocess as sp
 
 
 class FFmpegPreprocessor(BaseVideoPreprocessor):
 
     def __init__(self,
-                 duplicate_rm=True,
-                 use_phash_weight=False,
-                 phash_thresh=5,
+                 duplicate_rm: bool = True,
+                 use_phash_weight: bool = False,
+                 phash_thresh: int = 5,
                  *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.phash_thresh = phash_thresh
@@ -94,6 +95,7 @@ def apply(self, doc: 'gnes_pb2.Document') -> None:
 
     @staticmethod
     def phash(image_bytes: bytes):
+        import imagehash
         return imagehash.phash(Image.open(io.BytesIO(image_bytes)))
 
     @staticmethod
@@ -103,7 +105,7 @@ def pic_weight(image_array: List[np.ndarray]) -> List[float]:
         n_channel = image_array[0].shape[-1]
         for i in range(len(image_array)):
             # calcualte the variance of histgram of pixels
-            weight[i] = sum([np.histogram(image_array[i][:,:,_])[0].var()
+            weight[i] = sum([np.histogram(image_array[i][:, :, _])[0].var()
                              for _ in range(n_channel)])
         weight = weight / weight.sum()
 
@@ -119,7 +121,7 @@ def duplicate_rm_hash(self, image_list: List[bytes]) -> List[bytes]:
             if len(ret) >= 1:
                 # only keep images with high phash diff
                 # comparing only last kept 9 pics
-                for j in range(1, min(len(ret)+1, 9)):
+                for j in range(1, min(len(ret) + 1, 9)):
                     dist = abs(ret[-j][1] - h)
                     if dist < self.phash_thresh:
                         flag = 0

shell/push-wechatwork.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+
+TEMPLATE="{\"msgtype\": \"markdown\", \"markdown\": {\"content\": \"# $MSG_TITLE\n > $MSG_CONTENT\n\n > 🔗 [$MSG_LINK]($MSG_LINK)\"}}"
+
+echo $TEMPLATE | curl -s $BOT_URL -H 'Content-Type: application/json' --data-binary "@-"