Merge branch 'master' into chore-bumping-version

gnes/cli/parser.py

@@ -173,15 +173,19 @@ def set_loadable_service_parser(parser=None):
     return parser
 
 
-def set_preprocessor_service_parser(parser=None):
+# shortcut to keep consistent
+set_encoder_parser = set_loadable_service_parser
+
+
+def set_preprocessor_parser(parser=None):
     if not parser:
         parser = set_base_parser()
     set_loadable_service_parser(parser)
     parser.set_defaults(read_only=True)
     return parser
 
 
-def set_router_service_parser(parser=None):
+def set_router_parser(parser=None):
     if not parser:
         parser = set_base_parser()
     set_loadable_service_parser(parser)
@@ -191,7 +195,7 @@ def set_router_service_parser(parser=None):
     return parser
 
 
-def set_indexer_service_parser(parser=None):
+def set_indexer_parser(parser=None):
     from ..service.base import SocketType
 
     if not parser:
@@ -260,7 +264,7 @@ def set_frontend_parser(parser=None):
     return parser
 
 
-def set_cli_client_parser(parser=None):
+def set_client_cli_parser(parser=None):
     import sys
     if not parser:
         parser = set_base_parser()
@@ -289,7 +293,7 @@ def set_cli_client_parser(parser=None):
     return parser
 
 
-def set_benchmark_client_parser(parser=None):
+def set_client_benchmark_parser(parser=None):
     if not parser:
         parser = set_base_parser()
     _set_grpc_parser(parser)
@@ -304,7 +308,7 @@ def set_benchmark_client_parser(parser=None):
     return parser
 
 
-def set_http_service_parser(parser=None):
+def set_client_http_parser(parser=None):
     if not parser:
         parser = set_base_parser()
     _set_grpc_parser(parser)
@@ -330,20 +334,20 @@ def get_main_parser():
 
     # microservices
     set_frontend_parser(sp.add_parser('frontend', help='start a frontend service'))
-    set_loadable_service_parser(sp.add_parser('encode', help='start an encoder service'))
-    set_indexer_service_parser(sp.add_parser('index', help='start an indexer service'))
-    set_router_service_parser(sp.add_parser('route', help='start a router service'))
-    set_preprocessor_service_parser(sp.add_parser('preprocess', help='start a preprocessor service'))
+    set_encoder_parser(sp.add_parser('encode', help='start an encoder service'))
+    set_indexer_parser(sp.add_parser('index', help='start an indexer service'))
+    set_router_parser(sp.add_parser('route', help='start a router service'))
+    set_preprocessor_parser(sp.add_parser('preprocess', help='start a preprocessor service'))
     set_grpc_service_parser(sp.add_parser('grpc', help='start a general purpose grpc service'))
 
     pp = sp.add_parser('client', help='start a GNES client of the selected type')
     spp = pp.add_subparsers(dest='client', title='GNES client sub-commands',
                             description='use "gnes client [sub-command] --help" '
                                         'to get detailed information about each client sub-command')
     # clients
-    set_http_service_parser(spp.add_parser('http', help='start a client that allows HTTP requests as input'))
-    set_cli_client_parser(spp.add_parser('cli', help='start a client that allows stdin as input'))
-    set_benchmark_client_parser(spp.add_parser('benchmark', help='start a client for benchmark and unittest'))
+    set_client_http_parser(spp.add_parser('http', help='start a client that allows HTTP requests as input'))
+    set_client_cli_parser(spp.add_parser('cli', help='start a client that allows stdin as input'))
+    set_client_benchmark_parser(spp.add_parser('benchmark', help='start a client for benchmark and unittest'))
 
     # others
     set_composer_flask_parser(sp.add_parser('compose', help='start a GNES Board to visualize YAML configs'))

gnes/composer/base.py

@@ -25,8 +25,8 @@
 
 from .. import __version__
 from ..cli.parser import set_frontend_parser, \
-    set_router_service_parser, set_loadable_service_parser, set_preprocessor_service_parser, \
-    set_indexer_service_parser
+    set_router_parser, set_loadable_service_parser, set_preprocessor_parser, \
+    set_indexer_parser
 from ..helper import set_logger
 from ..service.base import SocketType
 
@@ -44,10 +44,10 @@ class YamlComposer:
 
     comp2args = {
         'Encoder': set_loadable_service_parser().parse_args(['--yaml_path', 'BaseEncoder']),
-        'Router': set_router_service_parser().parse_args(['--yaml_path', 'BaseRouter']),
-        'Indexer': set_indexer_service_parser().parse_args(['--yaml_path', 'BaseIndexer']),
+        'Router': set_router_parser().parse_args(['--yaml_path', 'BaseRouter']),
+        'Indexer': set_indexer_parser().parse_args(['--yaml_path', 'BaseIndexer']),
         'Frontend': set_frontend_parser().parse_args([]),
-        'Preprocessor': set_preprocessor_service_parser().parse_args(['--yaml_path', 'BasePreprocessor'])
+        'Preprocessor': set_preprocessor_parser().parse_args(['--yaml_path', 'BasePreprocessor'])
     }
 
     class Layer:

gnes/helper.py

@@ -13,8 +13,8 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-
 import fcntl
+import importlib.util
 import logging
 import os
 import sys
@@ -504,19 +504,43 @@ def load_contrib_module():
 
         if contrib:
             default_logger.info(
-                'find value in $GNES_CONTRIB_MODULE=%s, will try to load these modules from external' % contrib)
+                'find a value in $GNES_CONTRIB_MODULE=%s, will try to load these modules from external' % contrib)
             for c in contrib.split(','):
                 if ':' in c:
                     _name, _path = c.split(':')
-                    spec = importlib.util.spec_from_file_location('gnes.contrib', _path)
-                    foo = importlib.util.module_from_spec(spec)
-                    spec.loader.exec_module(foo)
-                    m = getattr(foo, _name)
+                    m = PathImporter.add_modules(_path)
                     modules.append(m)
                     default_logger.info('successfully register %s class, you can now use it via yaml.' % m)
         return modules
 
 
+class PathImporter:
+
+    @staticmethod
+    def _get_module_name(absolute_path):
+        module_name = os.path.basename(absolute_path)
+        module_name = module_name.replace('.py', '')
+        return module_name
+
+    @staticmethod
+    def add_modules(*paths):
+        for p in paths:
+            if not os.path.exists(p):
+                raise FileNotFoundError('cannot import module from %s, file not exist')
+            module, spec = PathImporter._path_import(p)
+            sys.modules[spec.name] = module
+        return module
+
+    @staticmethod
+    def _path_import(absolute_path):
+        module_name = PathImporter._get_module_name(absolute_path)
+        spec = importlib.util.spec_from_file_location(module_name, absolute_path)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        sys.modules[spec.name] = module
+        return module, spec
+
+
 profile_logger = set_logger('PROFILE')
 default_logger = set_logger('GNES')
 profiling = time_profile

gnes/preprocessor/__init__.py

@@ -27,7 +27,7 @@
     'VanillaSlidingPreprocessor': 'image.sliding_window',
     'WeightedSlidingPreprocessor': 'image.sliding_window',
     'SegmentPreprocessor': 'image.segmentation',
-    'BaseUnaryPreprocessor': 'base',
+    'UnaryPreprocessor': 'base',
     'ResizeChunkPreprocessor': 'image.resize',
     'BaseVideoPreprocessor': 'video.base',
     'FFmpegPreprocessor': 'video.ffmpeg',

gnes/preprocessor/base.py

@@ -14,12 +14,9 @@
 #  limitations under the License.
 
 
-import ctypes
 import io
-import random
 
 import numpy as np
-from PIL import Image
 
 from ..base import TrainableBase, CompositionalTrainableBase
 from ..proto import gnes_pb2, array2blob
@@ -57,7 +54,7 @@ def train(self, data, *args, **kwargs):
                 data = be.apply(data, *args, **kwargs)
 
 
-class BaseUnaryPreprocessor(BasePreprocessor):
+class UnaryPreprocessor(BasePreprocessor):
     is_trained = True
 
     def __init__(self, doc_type: int, *args, **kwargs):
@@ -79,6 +76,7 @@ def raw_to_chunk(self, chunk: 'gnes_pb2.Chunk', raw_bytes: bytes):
         if self.doc_type == gnes_pb2.Document.TEXT:
             chunk.text = raw_bytes.decode()
         elif self.doc_type == gnes_pb2.Document.IMAGE:
+            from PIL import Image
             img = np.array(Image.open(io.BytesIO(raw_bytes)))
             chunk.blob.CopyFrom(array2blob(img))
         elif self.doc_type == gnes_pb2.Document.VIDEO:

gnes/service/base.py

@@ -250,7 +250,7 @@ def dump(self):
                 self._model.dump()
                 self.logger.info('dumping finished!')
         else:
-            self.logger.warning('dumping is not allowed as "read_only" is set to true.')
+            self.logger.info('pass dumping as "read_only" set to true.')
 
     def message_handler(self, msg: 'gnes_pb2.Message', out_sck, ctrl_sck):
         try:

gnes/service/grpc.py

@@ -14,13 +14,10 @@
 #  limitations under the License.
 
 
-import importlib.util
-import os
-import sys
-
 import grpc
 
 from .base import BaseService as BS, MessageHandler
+from ..helper import PathImporter
 from ..proto import gnes_pb2
 
 
@@ -33,7 +30,7 @@ def post_init(self):
             options=[('grpc.max_send_message_length', self.args.max_message_size * 1024 * 1024),
                      ('grpc.max_receive_message_length', self.args.max_message_size * 1024 * 1024)])
 
-        foo = self.PathImport().add_modules(self.args.pb2_path, self.args.pb2_grpc_path)
+        foo = PathImporter.add_modules(self.args.pb2_path, self.args.pb2_grpc_path)
 
         # build stub
         self.stub = getattr(foo, self.args.stub_name)(self.channel)
@@ -45,28 +42,3 @@ def close(self):
     @handler.register(NotImplementedError)
     def _handler_default(self, msg: 'gnes_pb2.Message'):
         yield getattr(self.stub, self.args.api_name)(msg)
-
-    class PathImport:
-
-        @staticmethod
-        def get_module_name(absolute_path):
-            module_name = os.path.basename(absolute_path)
-            module_name = module_name.replace('.py', '')
-            return module_name
-
-        def add_modules(self, pb2_path, pb2_grpc_path):
-            (module, spec) = self.path_import(pb2_path)
-            sys.modules[spec.name] = module
-
-            (module, spec) = self.path_import(pb2_grpc_path)
-            sys.modules[spec.name] = module
-
-            return module
-
-        def path_import(self, absolute_path):
-            module_name = self.get_module_name(absolute_path)
-            spec = importlib.util.spec_from_file_location(module_name, absolute_path)
-            module = importlib.util.module_from_spec(spec)
-            spec.loader.exec_module(module)
-            sys.modules[spec.name] = module
-            return module, spec

setup.py

@@ -1,3 +1,18 @@
+#  Tencent is pleased to support the open source community by making GNES available.
+#
+#  Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 from os import path
 
 from setuptools import setup, find_packages

tests/test_audio_preprocessor.py

@@ -1,7 +1,7 @@
 import os
 import unittest
 
-from gnes.cli.parser import set_preprocessor_service_parser, _set_client_parser
+from gnes.cli.parser import set_preprocessor_parser, _set_client_parser
 from gnes.client.base import ZmqClient
 from gnes.proto import gnes_pb2, RequestGenerator, blob2array
 from gnes.service.preprocessor import PreprocessorService
@@ -17,14 +17,14 @@ def setUp(self):
                             for _ in os.listdir(self.video_path)]
 
     def test_video_preprocessor_service_empty(self):
-        args = set_preprocessor_service_parser().parse_args([
+        args = set_preprocessor_parser().parse_args([
             '--yaml_path', self.yml_path
         ])
         with PreprocessorService(args):
             pass
 
     def test_video_preprocessor_service_realdata(self):
-        args = set_preprocessor_service_parser().parse_args([
+        args = set_preprocessor_parser().parse_args([
             '--yaml_path', self.yml_path
         ])
 

tests/test_contrib_module.py

@@ -1,40 +1,45 @@
 import os
-import sys
 import unittest.mock
 
 dirname = os.path.dirname(__file__)
 module_path = os.path.join(dirname, 'contrib', 'dummy_contrib.py')
 cls_name = 'FooContribEncoder'
 
+
 @unittest.SkipTest
 class TestContribModule(unittest.TestCase):
     def setUp(self):
         self.yaml_path = os.path.join(os.path.dirname(__file__),
                                       'contrib', 'dummy.yml')
+        self.dump_yaml_path = os.path.join(os.path.dirname(__file__),
+                                           'dummy-dump.yml')
 
     def tearDown(self):
         # reload gnes module on every unit test
-        for mod in list(sys.modules.keys()):
-            if mod.startswith('gnes.'):
-                del (sys.modules[mod])
+        if os.path.exists(self.dump_yaml_path):
+            os.remove(self.dump_yaml_path)
 
-    @unittest.mock.patch.dict(os.environ, {'GNES_CONTRIB_MODULE': '%s:%s' % (cls_name, module_path)})
     def test_load_contrib(self):
-
+        os.environ['GNES_CONTRIB_MODULE'] = '%s:%s' % (cls_name, module_path)
         from gnes.encoder.base import BaseEncoder, BaseTextEncoder
         a = BaseEncoder.load_yaml(self.yaml_path)
         self.assertIsInstance(a, BaseTextEncoder)
         self.assertEqual(a.encode([]), 'hello 531')
+        a.dump()
+        a.dump_yaml(self.dump_yaml_path)
+        b = BaseEncoder.load_yaml(self.dump_yaml_path)
+        self.assertIsInstance(b, BaseTextEncoder)
+        self.assertEqual(b.encode([]), 'hello 531')
 
-    @unittest.mock.patch.dict(os.environ, {'GNES_CONTRIB_MODULE': '%s:%s' % ('blah', module_path)})
     def test_bad_name(self):
+        os.environ['GNES_CONTRIB_MODULE'] = '%s:%s' % ('blah', module_path)
         try:
             from gnes.encoder.base import BaseEncoder
         except AttributeError:
             pass
 
-    @unittest.mock.patch.dict(os.environ, {'GNES_CONTRIB_MODULE': '%s:%s' % (cls_name, 'blah')})
     def test_bad_path(self):
+        os.environ['GNES_CONTRIB_MODULE'] = '%s:%s' % (cls_name, 'blah')
         try:
             from gnes.encoder.base import BaseEncoder
         except AttributeError:

tests/test_image_encoder.py

@@ -4,7 +4,7 @@
 import zipfile
 
 from gnes.encoder.image.base import BasePytorchEncoder
-from gnes.preprocessor.base import BaseUnaryPreprocessor, PipelinePreprocessor
+from gnes.preprocessor.base import UnaryPreprocessor, PipelinePreprocessor
 from gnes.preprocessor.image.resize import ResizeChunkPreprocessor
 from gnes.preprocessor.image.sliding_window import VanillaSlidingPreprocessor
 from gnes.proto import gnes_pb2, blob2array
@@ -21,7 +21,7 @@ def img_process_for_test(dirname):
 
     test_img_all_preprocessor = []
     pipline_prep1 = PipelinePreprocessor()
-    pipline_prep1.components = lambda: [BaseUnaryPreprocessor(doc_type=gnes_pb2.Document.IMAGE),
+    pipline_prep1.components = lambda: [UnaryPreprocessor(doc_type=gnes_pb2.Document.IMAGE),
                                         ResizeChunkPreprocessor()]
     for preprocessor in [pipline_prep1,
                          VanillaSlidingPreprocessor()]: