Skip to content
This repository has been archived by the owner on Feb 22, 2020. It is now read-only.

Commit

Permalink
Merge pull request #28 from gnes-ai/fix_http_client
Browse files Browse the repository at this point in the history
fix(http): fix http client for image query
  • Loading branch information
Larryjianfeng authored Jul 17, 2019
2 parents 968e35c + 44b1a0c commit f2667bb
Show file tree
Hide file tree
Showing 5 changed files with 28 additions and 11 deletions.
27 changes: 22 additions & 5 deletions gnes/client/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,30 @@ def start(self):

async def general_handler(request, parser, *args, **kwargs):
try:
data = await asyncio.wait_for(request.json(), 10)
data = dict()
# # Option 1: uploading streaming chunk data
# data = b""
# async for chunk in request.content.iter_any():
# data += chunk
# self.logger.info("received %d content" % len(data))

# Option 2: uploading via Multipart-Encoded File
post_data = await request.post()
if 'query' in post_data.keys():
_file = post_data.get('query')
self.logger.info("query request from input file: %s" % _file.filename)
data['query'] = _file.file.read()
elif 'docs' in post_data.keys():
files = post_data.getall("docs")
self.logger.info("index request from input files: %d files" % len(files))
data['docs'] = [_file.file.read() for _file in files]

self.logger.info('data received, beigin processing')
resp = await loop.run_in_executor(
executor,
stub_call,
parser([d.encode() for d in data.get('docs')] if hasattr(data, 'docs')
else data.get('query').encode(), *args, **kwargs))
parser([d for d in data.get('docs')] if hasattr(data, 'docs')
else data.get('query'), *args, **kwargs))
self.logger.info('handling finished, will send to user')
return web.Response(body=json.dumps({'result': resp, 'meta': None}, ensure_ascii=False),
status=200,
Expand Down Expand Up @@ -81,8 +98,8 @@ def stub_call(req):

with grpc.insecure_channel(
'%s:%s' % (self.args.grpc_host, self.args.grpc_port),
options=[('grpc.max_send_message_length', 50 * 1024 * 1024),
('grpc.max_receive_message_length', 50 * 1024 * 1024),
options=[('grpc.max_send_message_length', 100 * 1024 * 1024),
('grpc.max_receive_message_length', 100 * 1024 * 1024),
('grpc.keepalive_timeout_ms', 100 * 1000)]) as channel:
stub = gnes_pb2_grpc.GnesRPCStub(channel)
loop.run_until_complete(init(loop))
Expand Down
4 changes: 2 additions & 2 deletions gnes/preprocessor/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
'TextPreprocessor': 'text.simple',
'BaseImagePreprocessor': 'image.base',
'BaseTextPreprocessor': 'text.base',
'VanillaSlidingPreprocessor': 'image.simple',
'WeightedSlidingPreprocessor': 'image.simple',
'VanillaSlidingPreprocessor': 'image.sliding_window',
'WeightedSlidingPreprocessor': 'image.sliding_window',
'SegmentPreprocessor': 'image.segmentation',
'BaseSingletonPreprocessor': 'base',
'BaseVideoPreprocessor': 'video.base',
Expand Down
6 changes: 3 additions & 3 deletions gnes/service/preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,15 @@ def post_init(self):
self._model = self.load_model(BasePreprocessor)

@handler.register(gnes_pb2.Request.TrainRequest)
def _handler_train_index(self, msg: 'gnes_pb2.Message'):
def _handler_train(self, msg: 'gnes_pb2.Message'):
for d in msg.request.train.docs:
self._model.apply(d)

@handler.register(gnes_pb2.Request.IndexRequest)
def _handler_train_index(self, msg: 'gnes_pb2.Message'):
def _handler_index(self, msg: 'gnes_pb2.Message'):
for d in msg.request.index.docs:
self._model.apply(d)

@handler.register(gnes_pb2.Request.QueryRequest)
def _handler_train_index(self, msg: 'gnes_pb2.Message'):
def _handler_query(self, msg: 'gnes_pb2.Message'):
self._model.apply(msg.request.search.query)
2 changes: 1 addition & 1 deletion tests/test_image_encoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import numpy as np

from gnes.encoder.image.base import BasePytorchEncoder
from gnes.preprocessor.image.slidingWindow import VanillaSlidingPreprocessor
from gnes.preprocessor.image.sliding_window import VanillaSlidingPreprocessor
from gnes.preprocessor.base import BaseSingletonPreprocessor
from gnes.proto import gnes_pb2, blob2array

Expand Down

0 comments on commit f2667bb

Please sign in to comment.