From fce6f1ab1a1d8e8b09c65cd8878d06b7043fd658 Mon Sep 17 00:00:00 2001 From: felix Date: Wed, 17 Jul 2019 12:24:09 +0800 Subject: [PATCH 1/6] use form to upload file in http post request for image/vidao query --- gnes/client/http.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/gnes/client/http.py b/gnes/client/http.py index 85d5ca2c..ece5738b 100644 --- a/gnes/client/http.py +++ b/gnes/client/http.py @@ -39,13 +39,30 @@ def start(self): async def general_handler(request, parser, *args, **kwargs): try: - data = await asyncio.wait_for(request.json(), 10) + data = dict() + # # Option 1: uploading streaming chunk data + # data = b"" + # async for chunk in request.content.iter_any(): + # data += chunk + # self.logger.info("received %d content" % len(data)) + + # Option 2: uploading via Multipart-Encoded File + post_data = await request.post() + if 'query' in post_data.keys(): + _file = post_data.get('query') + self.logger.info("query request from input file: %s" % _file.filename) + data['query'] = _file.file.read() + elif 'docs' in post_data.keys(): + files = post_data.getall("docs") + self.logger.info("index request from input files: %d files" % len(files)) + data['docs'] = [_file.file.read() for _file in files] + self.logger.info('data received, beigin processing') resp = await loop.run_in_executor( executor, stub_call, - parser([d.encode() for d in data.get('docs')] if hasattr(data, 'docs') - else data.get('query').encode(), *args, **kwargs)) + parser([d for d in data.get('docs')] if hasattr(data, 'docs') + else data.get('query'), *args, **kwargs)) self.logger.info('handling finished, will send to user') return web.Response(body=json.dumps({'result': resp, 'meta': None}, ensure_ascii=False), status=200, From c7c76f3fa6aff0fbf88290be2b7a810e6286f2ac Mon Sep 17 00:00:00 2001 From: felix Date: Wed, 17 Jul 2019 12:24:09 +0800 Subject: [PATCH 2/6] use form to upload file in http post request for image/vidao query --- gnes/client/http.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/gnes/client/http.py b/gnes/client/http.py index 85d5ca2c..ece5738b 100644 --- a/gnes/client/http.py +++ b/gnes/client/http.py @@ -39,13 +39,30 @@ def start(self): async def general_handler(request, parser, *args, **kwargs): try: - data = await asyncio.wait_for(request.json(), 10) + data = dict() + # # Option 1: uploading streaming chunk data + # data = b"" + # async for chunk in request.content.iter_any(): + # data += chunk + # self.logger.info("received %d content" % len(data)) + + # Option 2: uploading via Multipart-Encoded File + post_data = await request.post() + if 'query' in post_data.keys(): + _file = post_data.get('query') + self.logger.info("query request from input file: %s" % _file.filename) + data['query'] = _file.file.read() + elif 'docs' in post_data.keys(): + files = post_data.getall("docs") + self.logger.info("index request from input files: %d files" % len(files)) + data['docs'] = [_file.file.read() for _file in files] + self.logger.info('data received, beigin processing') resp = await loop.run_in_executor( executor, stub_call, - parser([d.encode() for d in data.get('docs')] if hasattr(data, 'docs') - else data.get('query').encode(), *args, **kwargs)) + parser([d for d in data.get('docs')] if hasattr(data, 'docs') + else data.get('query'), *args, **kwargs)) self.logger.info('handling finished, will send to user') return web.Response(body=json.dumps({'result': resp, 'meta': None}, ensure_ascii=False), status=200, From 1229f5c986a3a340f3ad7c2e48de55faa4fbaaa0 Mon Sep 17 00:00:00 2001 From: felix Date: Wed, 17 Jul 2019 16:51:32 +0800 Subject: [PATCH 3/6] rename image preprocessor py file --- gnes/preprocessor/__init__.py | 4 ++-- .../image/{slidingWindow.py => sliding_window.py} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename gnes/preprocessor/image/{slidingWindow.py => sliding_window.py} (100%) diff --git a/gnes/preprocessor/__init__.py b/gnes/preprocessor/__init__.py index e5b976d2..269a3541 100644 --- a/gnes/preprocessor/__init__.py +++ b/gnes/preprocessor/__init__.py @@ -23,8 +23,8 @@ 'TextPreprocessor': 'text.simple', 'BaseImagePreprocessor': 'image.base', 'BaseTextPreprocessor': 'text.base', - 'VanillaSlidingPreprocessor': 'image.simple', - 'WeightedSlidingPreprocessor': 'image.simple', + 'VanillaSlidingPreprocessor': 'image.sliding_window', + 'WeightedSlidingPreprocessor': 'image.sliding_window', 'SegmentPreprocessor': 'image.segmentation', 'BaseSingletonPreprocessor': 'base', 'BaseVideoPreprocessor': 'video.base', diff --git a/gnes/preprocessor/image/slidingWindow.py b/gnes/preprocessor/image/sliding_window.py similarity index 100% rename from gnes/preprocessor/image/slidingWindow.py rename to gnes/preprocessor/image/sliding_window.py From 3836020aca4dbf9d3a650ca8c75907c2803db477 Mon Sep 17 00:00:00 2001 From: felix Date: Wed, 17 Jul 2019 17:53:00 +0800 Subject: [PATCH 4/6] fix(preprocessor): fix preprocessor service handler function name error --- gnes/service/preprocessor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gnes/service/preprocessor.py b/gnes/service/preprocessor.py index 50f2ecce..fb98c294 100644 --- a/gnes/service/preprocessor.py +++ b/gnes/service/preprocessor.py @@ -27,15 +27,15 @@ def post_init(self): self._model = self.load_model(BasePreprocessor) @handler.register(gnes_pb2.Request.TrainRequest) - def _handler_train_index(self, msg: 'gnes_pb2.Message'): + def _handler_train(self, msg: 'gnes_pb2.Message'): for d in msg.request.train.docs: self._model.apply(d) @handler.register(gnes_pb2.Request.IndexRequest) - def _handler_train_index(self, msg: 'gnes_pb2.Message'): + def _handler_index(self, msg: 'gnes_pb2.Message'): for d in msg.request.index.docs: self._model.apply(d) @handler.register(gnes_pb2.Request.QueryRequest) - def _handler_train_index(self, msg: 'gnes_pb2.Message'): + def _handler_query(self, msg: 'gnes_pb2.Message'): self._model.apply(msg.request.search.query) From 38eca0ceb7102d7076f7e3854b0154d85a2cd9c0 Mon Sep 17 00:00:00 2001 From: felix Date: Wed, 17 Jul 2019 19:40:37 +0800 Subject: [PATCH 5/6] fix(grpc): change grpc client message size limit --- gnes/client/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnes/client/http.py b/gnes/client/http.py index ece5738b..e85efaf4 100644 --- a/gnes/client/http.py +++ b/gnes/client/http.py @@ -98,8 +98,8 @@ def stub_call(req): with grpc.insecure_channel( '%s:%s' % (self.args.grpc_host, self.args.grpc_port), - options=[('grpc.max_send_message_length', 50 * 1024 * 1024), - ('grpc.max_receive_message_length', 50 * 1024 * 1024), + options=[('grpc.max_send_message_length', 100 * 1024 * 1024), + ('grpc.max_receive_message_length', 100 * 1024 * 1024), ('grpc.keepalive_timeout_ms', 100 * 1000)]) as channel: stub = gnes_pb2_grpc.GnesRPCStub(channel) loop.run_until_complete(init(loop)) From 44b1a0c976f6232708393c8c967d627da1d7df09 Mon Sep 17 00:00:00 2001 From: felix Date: Wed, 17 Jul 2019 19:49:44 +0800 Subject: [PATCH 6/6] fix unittest --- tests/test_image_encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_image_encoder.py b/tests/test_image_encoder.py index 23da2760..842e3e01 100644 --- a/tests/test_image_encoder.py +++ b/tests/test_image_encoder.py @@ -5,7 +5,7 @@ import numpy as np from gnes.encoder.image.base import BasePytorchEncoder -from gnes.preprocessor.image.slidingWindow import VanillaSlidingPreprocessor +from gnes.preprocessor.image.sliding_window import VanillaSlidingPreprocessor from gnes.preprocessor.base import BaseSingletonPreprocessor from gnes.proto import gnes_pb2, blob2array