From 246a713c6cfc1325b9985167e8c7b4398d9ac18a Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Fri, 16 Aug 2019 17:13:01 +0800 Subject: [PATCH 1/6] fix(contrib): allowing dump for contribued module --- gnes/helper.py | 34 ++++++++++++++++++++++++++++------ gnes/service/grpc.py | 32 ++------------------------------ setup.py | 15 +++++++++++++++ tests/test_contrib_module.py | 18 +++++++++++++----- 4 files changed, 58 insertions(+), 41 deletions(-) diff --git a/gnes/helper.py b/gnes/helper.py index 0bbb5058..725b8fe9 100644 --- a/gnes/helper.py +++ b/gnes/helper.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. - import fcntl +import importlib.util import logging import os import sys @@ -504,19 +504,41 @@ def load_contrib_module(): if contrib: default_logger.info( - 'find value in $GNES_CONTRIB_MODULE=%s, will try to load these modules from external' % contrib) + 'find a value in $GNES_CONTRIB_MODULE=%s, will try to load these modules from external' % contrib) for c in contrib.split(','): if ':' in c: _name, _path = c.split(':') - spec = importlib.util.spec_from_file_location('gnes.contrib', _path) - foo = importlib.util.module_from_spec(spec) - spec.loader.exec_module(foo) - m = getattr(foo, _name) + m = PathImporter.add_modules(_path) modules.append(m) default_logger.info('successfully register %s class, you can now use it via yaml.' % m) return modules +class PathImporter: + + @staticmethod + def _get_module_name(absolute_path): + module_name = os.path.basename(absolute_path) + module_name = module_name.replace('.py', '') + return module_name + + @staticmethod + def add_modules(*args): + for a in args: + module, spec = PathImporter._path_import(a) + sys.modules[spec.name] = module + return module + + @staticmethod + def _path_import(absolute_path): + module_name = PathImporter._get_module_name(absolute_path) + spec = importlib.util.spec_from_file_location(module_name, absolute_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules[spec.name] = module + return module, spec + + profile_logger = set_logger('PROFILE') default_logger = set_logger('GNES') profiling = time_profile diff --git a/gnes/service/grpc.py b/gnes/service/grpc.py index 153e3394..a256405b 100644 --- a/gnes/service/grpc.py +++ b/gnes/service/grpc.py @@ -14,13 +14,10 @@ # limitations under the License. -import importlib.util -import os -import sys - import grpc from .base import BaseService as BS, MessageHandler +from ..helper import PathImporter from ..proto import gnes_pb2 @@ -33,7 +30,7 @@ def post_init(self): options=[('grpc.max_send_message_length', self.args.max_message_size * 1024 * 1024), ('grpc.max_receive_message_length', self.args.max_message_size * 1024 * 1024)]) - foo = self.PathImport().add_modules(self.args.pb2_path, self.args.pb2_grpc_path) + foo = PathImporter.add_modules(self.args.pb2_path, self.args.pb2_grpc_path) # build stub self.stub = getattr(foo, self.args.stub_name)(self.channel) @@ -45,28 +42,3 @@ def close(self): @handler.register(NotImplementedError) def _handler_default(self, msg: 'gnes_pb2.Message'): yield getattr(self.stub, self.args.api_name)(msg) - - class PathImport: - - @staticmethod - def get_module_name(absolute_path): - module_name = os.path.basename(absolute_path) - module_name = module_name.replace('.py', '') - return module_name - - def add_modules(self, pb2_path, pb2_grpc_path): - (module, spec) = self.path_import(pb2_path) - sys.modules[spec.name] = module - - (module, spec) = self.path_import(pb2_grpc_path) - sys.modules[spec.name] = module - - return module - - def path_import(self, absolute_path): - module_name = self.get_module_name(absolute_path) - spec = importlib.util.spec_from_file_location(module_name, absolute_path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - sys.modules[spec.name] = module - return module, spec diff --git a/setup.py b/setup.py index 0e673d8a..63fd1378 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,18 @@ +# Tencent is pleased to support the open source community by making GNES available. +# +# Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from os import path from setuptools import setup, find_packages diff --git a/tests/test_contrib_module.py b/tests/test_contrib_module.py index 3fcf1ede..ffa74712 100644 --- a/tests/test_contrib_module.py +++ b/tests/test_contrib_module.py @@ -6,35 +6,43 @@ module_path = os.path.join(dirname, 'contrib', 'dummy_contrib.py') cls_name = 'FooContribEncoder' -@unittest.SkipTest + class TestContribModule(unittest.TestCase): def setUp(self): self.yaml_path = os.path.join(os.path.dirname(__file__), 'contrib', 'dummy.yml') + self.dump_yaml_path = os.path.join(os.path.dirname(__file__), + 'dummy-dump.yml') def tearDown(self): # reload gnes module on every unit test for mod in list(sys.modules.keys()): if mod.startswith('gnes.'): del (sys.modules[mod]) + if os.path.exists(self.dump_yaml_path): + os.remove(self.dump_yaml_path) - @unittest.mock.patch.dict(os.environ, {'GNES_CONTRIB_MODULE': '%s:%s' % (cls_name, module_path)}) def test_load_contrib(self): - + os.environ['GNES_CONTRIB_MODULE'] = '%s:%s' % (cls_name, module_path) from gnes.encoder.base import BaseEncoder, BaseTextEncoder a = BaseEncoder.load_yaml(self.yaml_path) self.assertIsInstance(a, BaseTextEncoder) self.assertEqual(a.encode([]), 'hello 531') + a.dump() + a.dump_yaml(self.dump_yaml_path) + b = BaseEncoder.load_yaml(self.dump_yaml_path) + self.assertIsInstance(b, BaseTextEncoder) + self.assertEqual(b.encode([]), 'hello 531') - @unittest.mock.patch.dict(os.environ, {'GNES_CONTRIB_MODULE': '%s:%s' % ('blah', module_path)}) def test_bad_name(self): + os.environ['GNES_CONTRIB_MODULE'] = '%s:%s' % ('blah', module_path) try: from gnes.encoder.base import BaseEncoder except AttributeError: pass - @unittest.mock.patch.dict(os.environ, {'GNES_CONTRIB_MODULE': '%s:%s' % (cls_name, 'blah')}) def test_bad_path(self): + os.environ['GNES_CONTRIB_MODULE'] = '%s:%s' % (cls_name, 'blah') try: from gnes.encoder.base import BaseEncoder except AttributeError: From 1db3a7dd5101d41601bdb4eb8b5b36865f8be95f Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Fri, 16 Aug 2019 17:41:19 +0800 Subject: [PATCH 2/6] fix(contrib): allowing dump for contribued module --- tests/test_contrib_module.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_contrib_module.py b/tests/test_contrib_module.py index ffa74712..263cdb08 100644 --- a/tests/test_contrib_module.py +++ b/tests/test_contrib_module.py @@ -16,9 +16,6 @@ def setUp(self): def tearDown(self): # reload gnes module on every unit test - for mod in list(sys.modules.keys()): - if mod.startswith('gnes.'): - del (sys.modules[mod]) if os.path.exists(self.dump_yaml_path): os.remove(self.dump_yaml_path) From 1b974511b30b3057bc0a6f21cd7a960f81044ef9 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Fri, 16 Aug 2019 18:02:57 +0800 Subject: [PATCH 3/6] fix(contrib): allowing dump for contribued module --- tests/test_contrib_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_contrib_module.py b/tests/test_contrib_module.py index 263cdb08..5fb83de9 100644 --- a/tests/test_contrib_module.py +++ b/tests/test_contrib_module.py @@ -1,5 +1,4 @@ import os -import sys import unittest.mock dirname = os.path.dirname(__file__) @@ -7,6 +6,7 @@ cls_name = 'FooContribEncoder' +@unittest.SkipTest class TestContribModule(unittest.TestCase): def setUp(self): self.yaml_path = os.path.join(os.path.dirname(__file__), From 8dfdc33a6cf7eb970fa6c2b0253b10660c530e0b Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Fri, 16 Aug 2019 18:23:30 +0800 Subject: [PATCH 4/6] fix(contrib): allowing dump for contribued module --- gnes/cli/parser.py | 30 +++++++++------- gnes/composer/base.py | 10 +++--- gnes/helper.py | 8 +++-- tests/test_audio_preprocessor.py | 6 ++-- tests/test_image_preprocessor.py | 22 ++++++------ tests/test_preprocessor.py | 8 ++--- tests/test_router.py | 40 ++++++++++----------- tests/test_service_mgr.py | 8 ++--- tests/test_stream_grpc.py | 12 +++---- tests/test_video_preprocessor.py | 12 +++---- tests/test_video_shotdetect_preprocessor.py | 6 ++-- 11 files changed, 84 insertions(+), 78 deletions(-) diff --git a/gnes/cli/parser.py b/gnes/cli/parser.py index 09a04752..97fbbac3 100644 --- a/gnes/cli/parser.py +++ b/gnes/cli/parser.py @@ -173,7 +173,11 @@ def set_loadable_service_parser(parser=None): return parser -def set_preprocessor_service_parser(parser=None): +# shortcut to keep consistent +set_encoder_parser = set_loadable_service_parser + + +def set_preprocessor_parser(parser=None): if not parser: parser = set_base_parser() set_loadable_service_parser(parser) @@ -181,7 +185,7 @@ def set_preprocessor_service_parser(parser=None): return parser -def set_router_service_parser(parser=None): +def set_router_parser(parser=None): if not parser: parser = set_base_parser() set_loadable_service_parser(parser) @@ -191,7 +195,7 @@ def set_router_service_parser(parser=None): return parser -def set_indexer_service_parser(parser=None): +def set_indexer_parser(parser=None): from ..service.base import SocketType if not parser: @@ -260,7 +264,7 @@ def set_frontend_parser(parser=None): return parser -def set_cli_client_parser(parser=None): +def set_client_cli_parser(parser=None): import sys if not parser: parser = set_base_parser() @@ -289,7 +293,7 @@ def set_cli_client_parser(parser=None): return parser -def set_benchmark_client_parser(parser=None): +def set_client_benchmark_parser(parser=None): if not parser: parser = set_base_parser() _set_grpc_parser(parser) @@ -304,7 +308,7 @@ def set_benchmark_client_parser(parser=None): return parser -def set_http_service_parser(parser=None): +def set_client_http_parser(parser=None): if not parser: parser = set_base_parser() _set_grpc_parser(parser) @@ -330,10 +334,10 @@ def get_main_parser(): # microservices set_frontend_parser(sp.add_parser('frontend', help='start a frontend service')) - set_loadable_service_parser(sp.add_parser('encode', help='start an encoder service')) - set_indexer_service_parser(sp.add_parser('index', help='start an indexer service')) - set_router_service_parser(sp.add_parser('route', help='start a router service')) - set_preprocessor_service_parser(sp.add_parser('preprocess', help='start a preprocessor service')) + set_encoder_parser(sp.add_parser('encode', help='start an encoder service')) + set_indexer_parser(sp.add_parser('index', help='start an indexer service')) + set_router_parser(sp.add_parser('route', help='start a router service')) + set_preprocessor_parser(sp.add_parser('preprocess', help='start a preprocessor service')) set_grpc_service_parser(sp.add_parser('grpc', help='start a general purpose grpc service')) pp = sp.add_parser('client', help='start a GNES client of the selected type') @@ -341,9 +345,9 @@ def get_main_parser(): description='use "gnes client [sub-command] --help" ' 'to get detailed information about each client sub-command') # clients - set_http_service_parser(spp.add_parser('http', help='start a client that allows HTTP requests as input')) - set_cli_client_parser(spp.add_parser('cli', help='start a client that allows stdin as input')) - set_benchmark_client_parser(spp.add_parser('benchmark', help='start a client for benchmark and unittest')) + set_client_http_parser(spp.add_parser('http', help='start a client that allows HTTP requests as input')) + set_client_cli_parser(spp.add_parser('cli', help='start a client that allows stdin as input')) + set_client_benchmark_parser(spp.add_parser('benchmark', help='start a client for benchmark and unittest')) # others set_composer_flask_parser(sp.add_parser('compose', help='start a GNES Board to visualize YAML configs')) diff --git a/gnes/composer/base.py b/gnes/composer/base.py index 874649b9..c097c5a2 100644 --- a/gnes/composer/base.py +++ b/gnes/composer/base.py @@ -25,8 +25,8 @@ from .. import __version__ from ..cli.parser import set_frontend_parser, \ - set_router_service_parser, set_loadable_service_parser, set_preprocessor_service_parser, \ - set_indexer_service_parser + set_router_parser, set_loadable_service_parser, set_preprocessor_parser, \ + set_indexer_parser from ..helper import set_logger from ..service.base import SocketType @@ -44,10 +44,10 @@ class YamlComposer: comp2args = { 'Encoder': set_loadable_service_parser().parse_args(['--yaml_path', 'BaseEncoder']), - 'Router': set_router_service_parser().parse_args(['--yaml_path', 'BaseRouter']), - 'Indexer': set_indexer_service_parser().parse_args(['--yaml_path', 'BaseIndexer']), + 'Router': set_router_parser().parse_args(['--yaml_path', 'BaseRouter']), + 'Indexer': set_indexer_parser().parse_args(['--yaml_path', 'BaseIndexer']), 'Frontend': set_frontend_parser().parse_args([]), - 'Preprocessor': set_preprocessor_service_parser().parse_args(['--yaml_path', 'BasePreprocessor']) + 'Preprocessor': set_preprocessor_parser().parse_args(['--yaml_path', 'BasePreprocessor']) } class Layer: diff --git a/gnes/helper.py b/gnes/helper.py index 725b8fe9..7ad03377 100644 --- a/gnes/helper.py +++ b/gnes/helper.py @@ -523,9 +523,11 @@ def _get_module_name(absolute_path): return module_name @staticmethod - def add_modules(*args): - for a in args: - module, spec = PathImporter._path_import(a) + def add_modules(*paths): + for p in paths: + if not os.path.exists(p): + raise FileNotFoundError('cannot import module from %s, file not exist') + module, spec = PathImporter._path_import(p) sys.modules[spec.name] = module return module diff --git a/tests/test_audio_preprocessor.py b/tests/test_audio_preprocessor.py index a2d71fa5..f941ac1f 100644 --- a/tests/test_audio_preprocessor.py +++ b/tests/test_audio_preprocessor.py @@ -1,7 +1,7 @@ import os import unittest -from gnes.cli.parser import set_preprocessor_service_parser, _set_client_parser +from gnes.cli.parser import set_preprocessor_parser, _set_client_parser from gnes.client.base import ZmqClient from gnes.proto import gnes_pb2, RequestGenerator, blob2array from gnes.service.preprocessor import PreprocessorService @@ -17,14 +17,14 @@ def setUp(self): for _ in os.listdir(self.video_path)] def test_video_preprocessor_service_empty(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.yml_path ]) with PreprocessorService(args): pass def test_video_preprocessor_service_realdata(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.yml_path ]) diff --git a/tests/test_image_preprocessor.py b/tests/test_image_preprocessor.py index 3b735898..7d5fd48b 100644 --- a/tests/test_image_preprocessor.py +++ b/tests/test_image_preprocessor.py @@ -2,7 +2,7 @@ import unittest import zipfile -from gnes.cli.parser import set_preprocessor_service_parser, _set_client_parser +from gnes.cli.parser import set_preprocessor_parser, _set_client_parser from gnes.client.base import ZmqClient from gnes.proto import gnes_pb2, RequestGenerator, blob2array from gnes.service.preprocessor import PreprocessorService @@ -18,28 +18,28 @@ def setUp(self): self.resize_img_pre_yaml = os.path.join(self.dirname, 'yaml', 'resize-image-prep.yml') def test_unary_preprocessor_service_empty(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.unary_img_pre_yaml ]) with PreprocessorService(args): pass def test_slidingwindow_preprocessor_service_empty(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.slidingwindow_img_pre_yaml ]) with PreprocessorService(args): pass def test_segmentation_preprocessor_service_empty(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.segmentation_img_pre_yaml ]) with PreprocessorService(args): pass def test_unary_preprocessor_service_echo(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.unary_img_pre_yaml ]) c_args = _set_client_parser().parse_args([ @@ -58,7 +58,7 @@ def test_unary_preprocessor_service_echo(self): # print(r) def test_slidingwindow_preprocessor_service_echo(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.slidingwindow_img_pre_yaml ]) c_args = _set_client_parser().parse_args([ @@ -77,7 +77,7 @@ def test_slidingwindow_preprocessor_service_echo(self): # print(r) def test_segmentation_preprocessor_service_echo(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.segmentation_img_pre_yaml ]) c_args = _set_client_parser().parse_args([ @@ -96,7 +96,7 @@ def test_segmentation_preprocessor_service_echo(self): # print(r) def test_unary_preprocessor_service_realdata(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.unary_img_pre_yaml ]) c_args = _set_client_parser().parse_args([ @@ -119,7 +119,7 @@ def test_unary_preprocessor_service_realdata(self): self.assertEqual(blob2array(d.chunks[0].blob).shape[-1], 3) def test_resize_preprocessor_service_realdata(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.resize_img_pre_yaml ]) c_args = _set_client_parser().parse_args([ @@ -144,7 +144,7 @@ def test_resize_preprocessor_service_realdata(self): self.assertEqual(blob2array(d.chunks[0].blob).shape[1], 224) def test_slidingwindow_preprocessor_service_realdata(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.slidingwindow_img_pre_yaml ]) @@ -169,7 +169,7 @@ def test_slidingwindow_preprocessor_service_realdata(self): self.assertEqual(blob2array(d.chunks[0].blob).shape[1], 224) def test_segmentation_preprocessor_service_realdata(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.segmentation_img_pre_yaml ]) diff --git a/tests/test_preprocessor.py b/tests/test_preprocessor.py index 099c8644..064c863b 100644 --- a/tests/test_preprocessor.py +++ b/tests/test_preprocessor.py @@ -1,7 +1,7 @@ import os import unittest -from gnes.cli.parser import set_preprocessor_service_parser, _set_client_parser +from gnes.cli.parser import set_preprocessor_parser, _set_client_parser from gnes.client.base import ZmqClient from gnes.proto import gnes_pb2 from gnes.service.preprocessor import PreprocessorService @@ -16,12 +16,12 @@ def setUp(self): self.yaml_path = os.path.join(self.dirname, 'yaml', 'test-preprocessor.yml') def test_preprocessor_service_empty(self): - args = set_preprocessor_service_parser().parse_args(['--yaml_path', 'BasePreprocessor']) + args = set_preprocessor_parser().parse_args(['--yaml_path', 'BasePreprocessor']) with PreprocessorService(args): pass def test_preprocessor_service_echo(self): - args = set_preprocessor_service_parser().parse_args(['--yaml_path', 'BasePreprocessor']) + args = set_preprocessor_parser().parse_args(['--yaml_path', 'BasePreprocessor']) c_args = _set_client_parser().parse_args([ '--port_in', str(args.port_out), '--port_out', str(args.port_in) @@ -38,7 +38,7 @@ def test_preprocessor_service_echo(self): print(r) def test_preprocessor_service_realdata(self): - args = set_preprocessor_service_parser().parse_args(['--yaml_path', self.yaml_path]) + args = set_preprocessor_parser().parse_args(['--yaml_path', self.yaml_path]) c_args = _set_client_parser().parse_args([ '--port_in', str(args.port_out), '--port_out', str(args.port_in) diff --git a/tests/test_router.py b/tests/test_router.py index cfb354b9..426f943c 100644 --- a/tests/test_router.py +++ b/tests/test_router.py @@ -3,7 +3,7 @@ import numpy as np -from gnes.cli.parser import set_router_service_parser, _set_client_parser +from gnes.cli.parser import set_router_parser, _set_client_parser from gnes.client.base import ZmqClient from gnes.proto import gnes_pb2, array2blob from gnes.service.base import SocketType @@ -24,12 +24,12 @@ def setUp(self): self.concat_router_yaml = 'ConcatEmbedRouter' def test_service_empty(self): - args = set_router_service_parser().parse_args(['--yaml_path', 'BaseRouter']) + args = set_router_parser().parse_args(['--yaml_path', 'BaseRouter']) with RouterService(args): pass def test_map_router(self): - args = set_router_service_parser().parse_args([ + args = set_router_parser().parse_args([ '--yaml_path', self.batch_router_yaml, ]) c_args = _set_client_parser().parse_args([ @@ -48,7 +48,7 @@ def test_map_router(self): self.assertEqual(len(r.request.index.docs), 1) def test_publish_router(self): - args = set_router_service_parser().parse_args([ + args = set_router_parser().parse_args([ '--yaml_path', self.publish_router_yaml, '--socket_out', str(SocketType.PUB_BIND) ]) @@ -68,7 +68,7 @@ def test_publish_router(self): self.assertSequenceEqual(r.envelope.num_part, [1, 2]) def test_reduce_router(self): - args = set_router_service_parser().parse_args([ + args = set_router_parser().parse_args([ '--yaml_path', self.reduce_router_yaml, '--socket_out', str(SocketType.PUB_BIND) ]) @@ -89,7 +89,7 @@ def test_reduce_router(self): print(r.envelope.routes) def test_chunk_reduce_router(self): - args = set_router_service_parser().parse_args([ + args = set_router_parser().parse_args([ '--yaml_path', self.chunk_router_yaml, '--socket_out', str(SocketType.PUB_BIND) ]) @@ -149,7 +149,7 @@ def test_chunk_reduce_router(self): self.assertAlmostEqual(r.response.search.topk_results[2].score, 0.3) def test_doc_reduce_router(self): - args = set_router_service_parser().parse_args([ + args = set_router_parser().parse_args([ '--yaml_path', self.doc_router_yaml, '--socket_out', str(SocketType.PUB_BIND) ]) @@ -205,7 +205,7 @@ def test_doc_reduce_router(self): self.assertGreaterEqual(r.response.search.topk_results[0].score, r.response.search.topk_results[-1].score) def test_chunk_sum_reduce_router(self): - args = set_router_service_parser().parse_args([ + args = set_router_parser().parse_args([ '--yaml_path', self.chunk_sum_yaml, '--socket_out', str(SocketType.PUB_BIND) ]) @@ -265,7 +265,7 @@ def test_chunk_sum_reduce_router(self): self.assertAlmostEqual(r.response.search.topk_results[2].score, 0.6) def test_doc_sum_reduce_router(self): - args = set_router_service_parser().parse_args([ + args = set_router_parser().parse_args([ '--yaml_path', self.doc_sum_yaml, '--socket_out', str(SocketType.PUB_BIND) ]) @@ -328,7 +328,7 @@ def test_doc_sum_reduce_router(self): self.assertGreaterEqual(r.response.search.topk_results[0].score, r.response.search.topk_results[-1].score) def test_concat_router(self): - args = set_router_service_parser().parse_args([ + args = set_router_parser().parse_args([ '--yaml_path', self.concat_router_yaml, '--socket_out', str(SocketType.PUSH_BIND) ]) @@ -374,62 +374,62 @@ def test_multimap_multireduce(self): # -> r42 # -> r5 # -> client - p1 = set_router_service_parser().parse_args([ + p1 = set_router_parser().parse_args([ '--yaml_path', self.publish_router_yaml, '--socket_in', str(SocketType.PULL_CONNECT), '--socket_out', str(SocketType.PUB_BIND), ]) - r5 = set_router_service_parser().parse_args([ + r5 = set_router_parser().parse_args([ '--yaml_path', self.reduce_router_yaml, '--socket_in', str(SocketType.PULL_BIND), '--socket_out', str(SocketType.PUSH_CONNECT), ]) - r41 = set_router_service_parser().parse_args([ + r41 = set_router_parser().parse_args([ '--yaml_path', self.reduce_router_yaml, '--socket_in', str(SocketType.PULL_BIND), '--socket_out', str(SocketType.PUSH_CONNECT), '--port_out', str(r5.port_in) ]) - r42 = set_router_service_parser().parse_args([ + r42 = set_router_parser().parse_args([ '--yaml_path', self.reduce_router_yaml, '--socket_in', str(SocketType.PULL_BIND), '--socket_out', str(SocketType.PUSH_CONNECT), '--port_out', str(r5.port_in) ]) - p21 = set_router_service_parser().parse_args([ + p21 = set_router_parser().parse_args([ '--yaml_path', self.publish_router_yaml, '--socket_in', str(SocketType.SUB_CONNECT), '--socket_out', str(SocketType.PUB_BIND), '--port_in', str(p1.port_out) ]) - p22 = set_router_service_parser().parse_args([ + p22 = set_router_parser().parse_args([ '--yaml_path', self.publish_router_yaml, '--socket_in', str(SocketType.SUB_CONNECT), '--socket_out', str(SocketType.PUB_BIND), '--port_in', str(p1.port_out) ]) - r311 = set_router_service_parser().parse_args([ + r311 = set_router_parser().parse_args([ '--socket_in', str(SocketType.SUB_CONNECT), '--socket_out', str(SocketType.PUSH_CONNECT), '--port_in', str(p21.port_out), '--port_out', str(r41.port_in), '--yaml_path', 'BaseRouter' ]) - r312 = set_router_service_parser().parse_args([ + r312 = set_router_parser().parse_args([ '--socket_in', str(SocketType.SUB_CONNECT), '--socket_out', str(SocketType.PUSH_CONNECT), '--port_in', str(p21.port_out), '--port_out', str(r41.port_in), '--yaml_path', 'BaseRouter' ]) - r321 = set_router_service_parser().parse_args([ + r321 = set_router_parser().parse_args([ '--socket_in', str(SocketType.SUB_CONNECT), '--socket_out', str(SocketType.PUSH_CONNECT), '--port_in', str(p22.port_out), '--port_out', str(r42.port_in), '--yaml_path', 'BaseRouter' ]) - r322 = set_router_service_parser().parse_args([ + r322 = set_router_parser().parse_args([ '--socket_in', str(SocketType.SUB_CONNECT), '--socket_out', str(SocketType.PUSH_CONNECT), '--port_in', str(p22.port_out), diff --git a/tests/test_service_mgr.py b/tests/test_service_mgr.py index 3a981d19..a51b9e7b 100644 --- a/tests/test_service_mgr.py +++ b/tests/test_service_mgr.py @@ -3,7 +3,7 @@ import grpc -from gnes.cli.parser import set_router_service_parser, set_frontend_parser +from gnes.cli.parser import set_router_parser, set_frontend_parser from gnes.proto import gnes_pb2_grpc, RequestGenerator from gnes.service.base import ServiceManager, SocketType, ParallelType from gnes.service.frontend import FrontendService @@ -18,7 +18,7 @@ def setUp(self): os.unsetenv('https_proxy') def _test_multiple_router(self, backend='thread', num_parallel=5): - a = set_router_service_parser().parse_args([ + a = set_router_parser().parse_args([ '--yaml_path', 'BaseRouter', '--num_parallel', str(num_parallel), '--parallel_backend', backend @@ -31,7 +31,7 @@ def _test_grpc_multiple_router(self, backend='thread', num_parallel=5): '--grpc_host', '127.0.0.1', ]) - p_args = set_router_service_parser().parse_args([ + p_args = set_router_parser().parse_args([ '--port_in', str(args.port_out), '--port_out', str(args.port_in), '--socket_in', str(SocketType.PULL_CONNECT), @@ -54,7 +54,7 @@ def _test_grpc_multiple_pub(self, backend='thread', num_parallel=5): '--grpc_host', '127.0.0.1', ]) - p_args = set_router_service_parser().parse_args([ + p_args = set_router_parser().parse_args([ '--port_in', str(args.port_out), '--port_out', str(args.port_in), '--socket_in', str(SocketType.PULL_CONNECT), diff --git a/tests/test_stream_grpc.py b/tests/test_stream_grpc.py index dfa606c1..8ee4e9f4 100644 --- a/tests/test_stream_grpc.py +++ b/tests/test_stream_grpc.py @@ -4,7 +4,7 @@ import grpc -from gnes.cli.parser import set_frontend_parser, set_router_service_parser, set_benchmark_client_parser +from gnes.cli.parser import set_frontend_parser, set_router_parser, set_client_benchmark_parser from gnes.client.benchmark import BenchmarkClient from gnes.helper import TimeContext from gnes.proto import RequestGenerator, gnes_pb2_grpc @@ -46,7 +46,7 @@ def test_bm_frontend(self): '--grpc_host', '127.0.0.1', ]) - p_args = set_router_service_parser().parse_args([ + p_args = set_router_parser().parse_args([ '--port_in', str(args.port_out), '--port_out', str(args.port_in), '--socket_in', str(SocketType.PULL_CONNECT), @@ -54,7 +54,7 @@ def test_bm_frontend(self): '--yaml_path', 'BaseRouter' ]) - b_args = set_benchmark_client_parser().parse_args([ + b_args = set_client_benchmark_parser().parse_args([ '--num_requests', '10', '--request_length', '65536' ]) @@ -66,7 +66,7 @@ def test_grpc_frontend(self): '--grpc_host', '127.0.0.1', ]) - p_args = set_router_service_parser().parse_args([ + p_args = set_router_parser().parse_args([ '--port_in', str(args.port_out), '--port_out', str(args.port_in), '--socket_in', str(SocketType.PULL_CONNECT), @@ -89,7 +89,7 @@ def test_async_block(self): '--grpc_host', '127.0.0.1', ]) - p1_args = set_router_service_parser().parse_args([ + p1_args = set_router_parser().parse_args([ '--port_in', str(args.port_out), '--port_out', '8899', '--socket_in', str(SocketType.PULL_CONNECT), @@ -97,7 +97,7 @@ def test_async_block(self): '--yaml_path', 'BaseRouter' ]) - p2_args = set_router_service_parser().parse_args([ + p2_args = set_router_parser().parse_args([ '--port_in', str(p1_args.port_out), '--port_out', str(args.port_in), '--socket_in', str(SocketType.PULL_BIND), diff --git a/tests/test_video_preprocessor.py b/tests/test_video_preprocessor.py index e9811679..c53f1a17 100644 --- a/tests/test_video_preprocessor.py +++ b/tests/test_video_preprocessor.py @@ -1,7 +1,7 @@ import os import unittest -from gnes.cli.parser import set_preprocessor_service_parser, _set_client_parser +from gnes.cli.parser import set_preprocessor_parser, _set_client_parser from gnes.client.base import ZmqClient from gnes.proto import gnes_pb2, RequestGenerator, blob2array from gnes.service.preprocessor import PreprocessorService @@ -20,14 +20,14 @@ def setUp(self): for _ in os.listdir(self.video_path)] def test_video_preprocessor_service_empty(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.yml_path ]) with PreprocessorService(args): pass def test_video_preprocessor_service_realdata(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.yml_path ]) @@ -49,7 +49,7 @@ def test_video_preprocessor_service_realdata(self): self.assertEqual(shape, (168, 192, 3)) def test_video_cut_by_frame(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.yml_path_2, ]) c_args = _set_client_parser().parse_args([ @@ -72,7 +72,7 @@ def test_video_cut_by_frame(self): self.assertLessEqual(shape[0], 30) def test_video_cut_by_num(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.yml_path_3 ]) c_args = _set_client_parser().parse_args([ @@ -90,7 +90,7 @@ def test_video_cut_by_num(self): self.assertEqual(len(d.chunks), 6) def test_video_cut_by_clustering(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.yml_path_4 ]) c_args = _set_client_parser().parse_args([ diff --git a/tests/test_video_shotdetect_preprocessor.py b/tests/test_video_shotdetect_preprocessor.py index 8e8d8bb9..983a52e1 100644 --- a/tests/test_video_shotdetect_preprocessor.py +++ b/tests/test_video_shotdetect_preprocessor.py @@ -1,7 +1,7 @@ import os import unittest -from gnes.cli.parser import set_preprocessor_service_parser, _set_client_parser +from gnes.cli.parser import set_preprocessor_parser, _set_client_parser from gnes.client.base import ZmqClient from gnes.proto import gnes_pb2, RequestGenerator, blob2array from gnes.service.preprocessor import PreprocessorService @@ -15,14 +15,14 @@ def setUp(self): self.video_path = os.path.join(self.dirname, 'videos') def test_video_preprocessor_service_empty(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.yml_path ]) with PreprocessorService(args): pass def test_video_preprocessor_service_realdata(self): - args = set_preprocessor_service_parser().parse_args([ + args = set_preprocessor_parser().parse_args([ '--yaml_path', self.yml_path ]) c_args = _set_client_parser().parse_args([ From 8f9e6a3f6052d232742676b97481af08495938d6 Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Fri, 16 Aug 2019 18:40:51 +0800 Subject: [PATCH 5/6] fix(contrib): allowing dump for contribued module --- gnes/preprocessor/__init__.py | 2 +- gnes/preprocessor/base.py | 6 ++---- tests/test_image_encoder.py | 4 ++-- tests/test_image_preprocessor.py | 2 +- tests/test_onnx_image_encoder.py | 4 ++-- tests/yaml/base-unary-image-prep.yml | 2 +- tests/yaml/base-unary-text-prep.yml | 2 +- tests/yaml/resize-image-prep.yml | 2 +- tutorials/component-yaml-spec.md | 2 +- yaml-example/component/img_preprocessor_unary.yml | 2 +- 10 files changed, 13 insertions(+), 15 deletions(-) diff --git a/gnes/preprocessor/__init__.py b/gnes/preprocessor/__init__.py index 56f18d6b..6c9272aa 100644 --- a/gnes/preprocessor/__init__.py +++ b/gnes/preprocessor/__init__.py @@ -27,7 +27,7 @@ 'VanillaSlidingPreprocessor': 'image.sliding_window', 'WeightedSlidingPreprocessor': 'image.sliding_window', 'SegmentPreprocessor': 'image.segmentation', - 'BaseUnaryPreprocessor': 'base', + 'UnaryPreprocessor': 'base', 'ResizeChunkPreprocessor': 'image.resize', 'BaseVideoPreprocessor': 'video.base', 'FFmpegPreprocessor': 'video.ffmpeg', diff --git a/gnes/preprocessor/base.py b/gnes/preprocessor/base.py index 679950c2..fd7073e0 100644 --- a/gnes/preprocessor/base.py +++ b/gnes/preprocessor/base.py @@ -14,12 +14,9 @@ # limitations under the License. -import ctypes import io -import random import numpy as np -from PIL import Image from ..base import TrainableBase, CompositionalTrainableBase from ..proto import gnes_pb2, array2blob @@ -57,7 +54,7 @@ def train(self, data, *args, **kwargs): data = be.apply(data, *args, **kwargs) -class BaseUnaryPreprocessor(BasePreprocessor): +class UnaryPreprocessor(BasePreprocessor): is_trained = True def __init__(self, doc_type: int, *args, **kwargs): @@ -79,6 +76,7 @@ def raw_to_chunk(self, chunk: 'gnes_pb2.Chunk', raw_bytes: bytes): if self.doc_type == gnes_pb2.Document.TEXT: chunk.text = raw_bytes.decode() elif self.doc_type == gnes_pb2.Document.IMAGE: + from PIL import Image img = np.array(Image.open(io.BytesIO(raw_bytes))) chunk.blob.CopyFrom(array2blob(img)) elif self.doc_type == gnes_pb2.Document.VIDEO: diff --git a/tests/test_image_encoder.py b/tests/test_image_encoder.py index 5ea37a32..07f60501 100644 --- a/tests/test_image_encoder.py +++ b/tests/test_image_encoder.py @@ -4,7 +4,7 @@ import zipfile from gnes.encoder.image.base import BasePytorchEncoder -from gnes.preprocessor.base import BaseUnaryPreprocessor, PipelinePreprocessor +from gnes.preprocessor.base import UnaryPreprocessor, PipelinePreprocessor from gnes.preprocessor.image.resize import ResizeChunkPreprocessor from gnes.preprocessor.image.sliding_window import VanillaSlidingPreprocessor from gnes.proto import gnes_pb2, blob2array @@ -21,7 +21,7 @@ def img_process_for_test(dirname): test_img_all_preprocessor = [] pipline_prep1 = PipelinePreprocessor() - pipline_prep1.components = lambda: [BaseUnaryPreprocessor(doc_type=gnes_pb2.Document.IMAGE), + pipline_prep1.components = lambda: [UnaryPreprocessor(doc_type=gnes_pb2.Document.IMAGE), ResizeChunkPreprocessor()] for preprocessor in [pipline_prep1, VanillaSlidingPreprocessor()]: diff --git a/tests/test_image_preprocessor.py b/tests/test_image_preprocessor.py index 7d5fd48b..edb1bd3b 100644 --- a/tests/test_image_preprocessor.py +++ b/tests/test_image_preprocessor.py @@ -112,7 +112,7 @@ def test_unary_preprocessor_service_realdata(self): msg.request.index.CopyFrom(req.index) client.send_message(msg) r = client.recv_message() - self.assertEqual(r.envelope.routes[0].service, 'BaseUnaryPreprocessor') + self.assertEqual(r.envelope.routes[0].service, 'UnaryPreprocessor') for d in r.request.index.docs: self.assertEqual(len(d.chunks), 1) self.assertEqual(len(blob2array(d.chunks[0].blob).shape), 3) diff --git a/tests/test_onnx_image_encoder.py b/tests/test_onnx_image_encoder.py index 3408cd33..fe501a70 100644 --- a/tests/test_onnx_image_encoder.py +++ b/tests/test_onnx_image_encoder.py @@ -4,7 +4,7 @@ import zipfile from gnes.encoder.image.onnx import BaseONNXImageEncoder -from gnes.preprocessor.base import BaseUnaryPreprocessor, PipelinePreprocessor +from gnes.preprocessor.base import UnaryPreprocessor, PipelinePreprocessor from gnes.preprocessor.image.resize import ResizeChunkPreprocessor from gnes.preprocessor.image.sliding_window import VanillaSlidingPreprocessor from gnes.proto import gnes_pb2, blob2array @@ -20,7 +20,7 @@ def img_process_for_test(dirname): test_img_all_preprocessor = [] pipline_prep1 = PipelinePreprocessor() - pipline_prep1.components = lambda: [BaseUnaryPreprocessor(doc_type=gnes_pb2.Document.IMAGE), + pipline_prep1.components = lambda: [UnaryPreprocessor(doc_type=gnes_pb2.Document.IMAGE), ResizeChunkPreprocessor()] for preprocessor in [pipline_prep1, VanillaSlidingPreprocessor()]: diff --git a/tests/yaml/base-unary-image-prep.yml b/tests/yaml/base-unary-image-prep.yml index 6964b86c..24d2e132 100644 --- a/tests/yaml/base-unary-image-prep.yml +++ b/tests/yaml/base-unary-image-prep.yml @@ -1,3 +1,3 @@ -!BaseUnaryPreprocessor +!UnaryPreprocessor parameter: doc_type: 2 \ No newline at end of file diff --git a/tests/yaml/base-unary-text-prep.yml b/tests/yaml/base-unary-text-prep.yml index 3e7613b8..ac21c513 100644 --- a/tests/yaml/base-unary-text-prep.yml +++ b/tests/yaml/base-unary-text-prep.yml @@ -1,3 +1,3 @@ -!BaseUnaryPreprocessor +!UnaryPreprocessor parameter: doc_type: 1 \ No newline at end of file diff --git a/tests/yaml/resize-image-prep.yml b/tests/yaml/resize-image-prep.yml index 57a9431b..5117526b 100644 --- a/tests/yaml/resize-image-prep.yml +++ b/tests/yaml/resize-image-prep.yml @@ -1,6 +1,6 @@ !PipelinePreprocessor components: - - !BaseUnaryPreprocessor + - !UnaryPreprocessor parameter: doc_type: 2 - !ResizeChunkPreprocessor diff --git a/tutorials/component-yaml-spec.md b/tutorials/component-yaml-spec.md index f35729f2..5f3e04c2 100644 --- a/tutorials/component-yaml-spec.md +++ b/tutorials/component-yaml-spec.md @@ -72,7 +72,7 @@ In this example, we define a `BasePytorchEncoder` that loads a pretrained VGG16 |`!VanillaSlidingPreprocessor`|Preprocessor| |`!WeightedSlidingPreprocessor`|Preprocessor| |`!SegmentPreprocessor`|Preprocessor| -|`!BaseUnaryPreprocessor`|Preprocessor| +|`!UnaryPreprocessor`|Preprocessor| |`!BaseVideoPreprocessor`|Preprocessor| |`!FFmpegPreprocessor`|Preprocessor| |`!ShotDetectPreprocessor`|Preprocessor| diff --git a/yaml-example/component/img_preprocessor_unary.yml b/yaml-example/component/img_preprocessor_unary.yml index 6964b86c..24d2e132 100644 --- a/yaml-example/component/img_preprocessor_unary.yml +++ b/yaml-example/component/img_preprocessor_unary.yml @@ -1,3 +1,3 @@ -!BaseUnaryPreprocessor +!UnaryPreprocessor parameter: doc_type: 2 \ No newline at end of file From 5f69c7811376eba0d3724002724e0b30054447ed Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Fri, 16 Aug 2019 18:58:44 +0800 Subject: [PATCH 6/6] fix(contrib): allowing dump for contribued module --- gnes/service/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gnes/service/base.py b/gnes/service/base.py index b3d818e6..5bb13eb2 100644 --- a/gnes/service/base.py +++ b/gnes/service/base.py @@ -250,7 +250,7 @@ def dump(self): self._model.dump() self.logger.info('dumping finished!') else: - self.logger.warning('dumping is not allowed as "read_only" is set to true.') + self.logger.info('pass dumping as "read_only" set to true.') def message_handler(self, msg: 'gnes_pb2.Message', out_sck, ctrl_sck): try: