From 00c25f3981b214910d9a0c38fb6be3b64995960c Mon Sep 17 00:00:00 2001 From: hanhxiao Date: Thu, 12 Sep 2019 12:27:45 +0800 Subject: [PATCH] fix(parser): remove limite on message size --- gnes/cli/parser.py | 4 ++-- gnes/client/benchmark.py | 4 ++-- gnes/client/http.py | 11 ++++++++--- gnes/service/frontend.py | 4 ++-- gnes/service/grpc.py | 4 ++-- setup.py | 7 ++++--- 6 files changed, 20 insertions(+), 14 deletions(-) diff --git a/gnes/cli/parser.py b/gnes/cli/parser.py index 93b64176..b4f9f315 100644 --- a/gnes/cli/parser.py +++ b/gnes/cli/parser.py @@ -240,8 +240,8 @@ def _set_grpc_parser(parser=None): type=int, default=8800, help='host port of the grpc service') - parser.add_argument('--max_message_size', type=int, default=100, - help='maximum send and receive size for grpc server in (MB)') + parser.add_argument('--max_message_size', type=int, default=-1, + help='maximum send and receive size for grpc server in bytes, -1 means unlimited') return parser diff --git a/gnes/client/benchmark.py b/gnes/client/benchmark.py index 8993f63c..09265c66 100644 --- a/gnes/client/benchmark.py +++ b/gnes/client/benchmark.py @@ -27,8 +27,8 @@ def __init__(self, args): with grpc.insecure_channel( '%s:%d' % (args.grpc_host, args.grpc_port), - options=[('grpc.max_send_message_length', args.max_message_size * 1024 * 1024), - ('grpc.max_receive_message_length', args.max_message_size * 1024 * 1024)]) as channel: + options=[('grpc.max_send_message_length', args.max_message_size), + ('grpc.max_receive_message_length', args.max_message_size)]) as channel: stub = gnes_pb2_grpc.GnesRPCStub(channel) id = 0 diff --git a/gnes/client/http.py b/gnes/client/http.py index c41fcedc..43e21dca 100644 --- a/gnes/client/http.py +++ b/gnes/client/http.py @@ -31,7 +31,12 @@ def __init__(self, args=None): self.logger = set_logger(self.__class__.__name__, self.args.verbose) def start(self): - from aiohttp import web + try: + from aiohttp import web + except ImportError: + self.logger.error('can not import aiohttp, it is not installed correctly. please do ' + '"pip install gnes[aiohttp]"') + return loop = asyncio.get_event_loop() executor = ThreadPoolExecutor(max_workers=self.args.max_workers) @@ -94,8 +99,8 @@ def stub_call(req): with grpc.insecure_channel( '%s:%s' % (self.args.grpc_host, self.args.grpc_port), - options=[('grpc.max_send_message_length', self.args.max_message_size * 1024 * 1024), - ('grpc.max_receive_message_length', self.args.max_message_size * 1024 * 1024), + options=[('grpc.max_send_message_length', self.args.max_message_size), + ('grpc.max_receive_message_length', self.args.max_message_size), ('grpc.keepalive_timeout_ms', 100 * 1000)]) as channel: stub = gnes_pb2_grpc.GnesRPCStub(channel) loop.run_until_complete(init(loop)) diff --git a/gnes/service/frontend.py b/gnes/service/frontend.py index efd6831e..6f227f13 100644 --- a/gnes/service/frontend.py +++ b/gnes/service/frontend.py @@ -31,8 +31,8 @@ def __init__(self, args): self.logger = set_logger(self.__class__.__name__, args.verbose) self.server = grpc.server( ThreadPoolExecutor(max_workers=args.max_concurrency), - options=[('grpc.max_send_message_length', args.max_message_size * 1024 * 1024), - ('grpc.max_receive_message_length', args.max_message_size * 1024 * 1024)]) + options=[('grpc.max_send_message_length', args.max_message_size), + ('grpc.max_receive_message_length', args.max_message_size)]) self.logger.info('start a frontend with %d workers' % args.max_concurrency) gnes_pb2_grpc.add_GnesRPCServicer_to_server(self._Servicer(args), self.server) diff --git a/gnes/service/grpc.py b/gnes/service/grpc.py index da2df279..6e369ced 100644 --- a/gnes/service/grpc.py +++ b/gnes/service/grpc.py @@ -27,8 +27,8 @@ class GRPCService(BS): def post_init(self): self.channel = grpc.insecure_channel( '%s:%s' % (self.args.grpc_host, self.args.grpc_port), - options=[('grpc.max_send_message_length', self.args.max_message_size * 1024 * 1024), - ('grpc.max_receive_message_length', self.args.max_message_size * 1024 * 1024)]) + options=[('grpc.max_send_message_length', self.args.max_message_size), + ('grpc.max_receive_message_length', self.args.max_message_size)]) m = PathImporter.add_modules(self.args.pb2_path, self.args.pb2_grpc_path) diff --git a/setup.py b/setup.py index 4030ce71..ac697948 100644 --- a/setup.py +++ b/setup.py @@ -52,8 +52,7 @@ 'protobuf', 'grpcio', 'ruamel.yaml>=0.15.89', - 'pyzmq>=17.1.0', - 'aiohttp'] + 'pyzmq>=17.1.0'] # using pip install gnes[xx] is depreciated # extras_dep is kept for legacy issue, will be removed soon @@ -73,7 +72,9 @@ 'onnx': ['onnxruntime'], 'audio': ['librosa>=0.7.0'], 'scipy': ['scipy', 'sklearn'], - 'flask': ['flask'] + 'flask': ['flask'], + 'aiohttp': ['aiohttp'], + 'http': ['flask', 'aiohttp'] }