From adcded5ab265bbe0ed4bbb64e22bab03a7c07c61 Mon Sep 17 00:00:00 2001 From: Sami Jaghouar Date: Mon, 2 May 2022 15:52:33 +0200 Subject: [PATCH 1/6] feat: add monitoring of internal part --- server/clip_server/torch-flow.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/clip_server/torch-flow.yml b/server/clip_server/torch-flow.yml index e7ba01c5a..3b73679d5 100644 --- a/server/clip_server/torch-flow.yml +++ b/server/clip_server/torch-flow.yml @@ -2,6 +2,7 @@ jtype: Flow version: '1' with: port: 51000 + monitoring: True executors: - name: clip_t uses: @@ -9,3 +10,5 @@ executors: metas: py_modules: - executors/clip_torch.py + monitoring: true + port_monitoring: 9091 From a18cfad0d7d1d743eca810180286dc3b83022271 Mon Sep 17 00:00:00 2001 From: numb3r3 Date: Thu, 26 May 2022 12:02:49 +0800 Subject: [PATCH 2/6] fix: rebase to main --- server/clip_server/executors/clip_onnx.py | 18 ++++++++---- server/clip_server/executors/clip_tensorrt.py | 24 ++++++++++------ server/clip_server/executors/clip_torch.py | 28 +++++++++++++------ server/clip_server/onnx-flow.yml | 5 +++- server/clip_server/tensorrt-flow.yml | 5 +++- server/clip_server/torch-flow.yml | 1 + server/setup.py | 1 + 7 files changed, 59 insertions(+), 23 deletions(-) diff --git a/server/clip_server/executors/clip_onnx.py b/server/clip_server/executors/clip_onnx.py index b080118d8..14bfe8a7f 100644 --- a/server/clip_server/executors/clip_onnx.py +++ b/server/clip_server/executors/clip_onnx.py @@ -13,7 +13,7 @@ ) from clip_server.model import clip from clip_server.model.clip_onnx import CLIPOnnxModel -from jina import Executor, requests, DocumentArray +from jina import Executor, requests, DocumentArray, monitor class CLIPEncoder(Executor): @@ -78,6 +78,16 @@ def __init__( self._model.start_sessions(sess_options=sess_options, providers=providers) + @monitor() + def _preproc_images(self, docs: 'DocumentArray'): + return preproc_image( + docs, preprocess_fn=self._preprocess_tensor, return_np=True + ) + + @monitor() + def _preproc_texts(self, docs: 'DocumentArray'): + return preproc_text(docs, return_np=True) + @requests(on='/rank') async def rank(self, docs: 'DocumentArray', parameters: Dict, **kwargs): await self.encode(docs['@r,m']) @@ -94,9 +104,7 @@ async def encode(self, docs: 'DocumentArray', **kwargs): # for image if _img_da: for minibatch, _contents in _img_da.map_batch( - partial( - preproc_image, preprocess_fn=self._preprocess_tensor, return_np=True - ), + self._preproc_images, batch_size=self._minibatch_size, pool=self._pool, ): @@ -112,7 +120,7 @@ async def encode(self, docs: 'DocumentArray', **kwargs): # for text if _txt_da: for minibatch, _contents in _txt_da.map_batch( - partial(preproc_text, return_np=True), + self._preproc_texts, batch_size=self._minibatch_size, pool=self._pool, ): diff --git a/server/clip_server/executors/clip_tensorrt.py b/server/clip_server/executors/clip_tensorrt.py index fa1fc62f9..e1099116e 100644 --- a/server/clip_server/executors/clip_tensorrt.py +++ b/server/clip_server/executors/clip_tensorrt.py @@ -11,7 +11,7 @@ ) from clip_server.model import clip from clip_server.model.clip_trt import CLIPTensorRTModel -from jina import Executor, requests, DocumentArray +from jina import Executor, requests, DocumentArray, monitor class CLIPEncoder(Executor): @@ -46,6 +46,19 @@ def __init__( self._model.start_engines() + @monitor() + def _preproc_images(self, docs: 'DocumentArray'): + return preproc_image( + docs, + preprocess_fn=self._preprocess_tensor, + device=self._device, + return_np=False, + ) + + @monitor() + def _preproc_texts(self, docs: 'DocumentArray'): + return preproc_text(docs, device=self._device, return_np=False) + @requests(on='/rank') async def rank(self, docs: 'DocumentArray', parameters: Dict, **kwargs): await self.encode(docs['@r,m']) @@ -62,12 +75,7 @@ async def encode(self, docs: 'DocumentArray', **kwargs): # for image if _img_da: for minibatch, _contents in _img_da.map_batch( - partial( - preproc_image, - preprocess_fn=self._preprocess_tensor, - device=self._device, - return_np=False, - ), + self._preproc_images, batch_size=self._minibatch_size, pool=self._pool, ): @@ -89,7 +97,7 @@ async def encode(self, docs: 'DocumentArray', **kwargs): # for text if _txt_da: for minibatch, _contents in _txt_da.map_batch( - partial(preproc_text, device=self._device, return_np=False), + self._preproc_texts, batch_size=self._minibatch_size, pool=self._pool, ): diff --git a/server/clip_server/executors/clip_torch.py b/server/clip_server/executors/clip_torch.py index e356f448b..a09119bec 100644 --- a/server/clip_server/executors/clip_torch.py +++ b/server/clip_server/executors/clip_torch.py @@ -13,7 +13,7 @@ set_rank, ) from clip_server.model import clip -from jina import Executor, requests, DocumentArray +from jina import Executor, requests, DocumentArray, monitor class CLIPEncoder(Executor): @@ -59,8 +59,22 @@ def __init__( self._pool = ThreadPool(processes=num_worker_preprocess) + @monitor() + def _preproc_images(self, docs: 'DocumentArray'): + return preproc_image( + docs, + preprocess_fn=self._preprocess_tensor, + device=self._device, + return_np=False, + ) + + @monitor() + def _preproc_texts(self, docs: 'DocumentArray'): + return preproc_text(docs, device=self._device, return_np=False) + @requests(on='/rank') async def rank(self, docs: 'DocumentArray', parameters: Dict, **kwargs): + self._rank_docs_counter.inc(len(docs)) await self.encode(docs['@r,m']) @@ -73,16 +87,14 @@ async def encode(self, docs: 'DocumentArray', **kwargs): for d in docs: split_img_txt_da(d, _img_da, _txt_da) + self._img_docs_counter.inc(len(_img_da)) + self._txt_docs_counter.inc(len(_txt_da)) + with torch.inference_mode(): # for image if _img_da: for minibatch, _contents in _img_da.map_batch( - partial( - preproc_image, - preprocess_fn=self._preprocess_tensor, - device=self._device, - return_np=False, - ), + self._preproc_images, batch_size=self._minibatch_size, pool=self._pool, ): @@ -104,7 +116,7 @@ async def encode(self, docs: 'DocumentArray', **kwargs): # for text if _txt_da: for minibatch, _contents in _txt_da.map_batch( - partial(preproc_text, device=self._device, return_np=False), + self._preproc_texts, batch_size=self._minibatch_size, pool=self._pool, ): diff --git a/server/clip_server/onnx-flow.yml b/server/clip_server/onnx-flow.yml index e02be2f77..d4bffef8a 100644 --- a/server/clip_server/onnx-flow.yml +++ b/server/clip_server/onnx-flow.yml @@ -2,6 +2,8 @@ jtype: Flow version: '1' with: port: 51000 + monitoring: True + port_monitoring: 9090 executors: - name: clip_o uses: @@ -9,4 +11,5 @@ executors: metas: py_modules: - executors/clip_onnx.py - + monitoring: true + port_monitoring: 9091 diff --git a/server/clip_server/tensorrt-flow.yml b/server/clip_server/tensorrt-flow.yml index fb1910981..f8e2393a6 100644 --- a/server/clip_server/tensorrt-flow.yml +++ b/server/clip_server/tensorrt-flow.yml @@ -2,6 +2,8 @@ jtype: Flow version: '1' with: port: 51000 + monitoring: True + port_monitoring: 9090 executors: - name: clip_r uses: @@ -9,4 +11,5 @@ executors: metas: py_modules: - executors/clip_tensorrt.py - + monitoring: true + port_monitoring: 9091 diff --git a/server/clip_server/torch-flow.yml b/server/clip_server/torch-flow.yml index 3b73679d5..a1dfb5d6e 100644 --- a/server/clip_server/torch-flow.yml +++ b/server/clip_server/torch-flow.yml @@ -3,6 +3,7 @@ version: '1' with: port: 51000 monitoring: True + port_monitoring: 9090 executors: - name: clip_t uses: diff --git a/server/setup.py b/server/setup.py index 9dc283592..ea44f4782 100644 --- a/server/setup.py +++ b/server/setup.py @@ -49,6 +49,7 @@ 'jina>=3.3.21', 'docarray>=0.11.0', 'packaging', + 'prometheus-client', ], extras_require={ 'onnx': [ From 7008ca461523b011fbf74085c2b1b6af26e3c2ea Mon Sep 17 00:00:00 2001 From: numb3r3 Date: Thu, 26 May 2022 12:06:11 +0800 Subject: [PATCH 3/6] fix: bump jina 3.4.0 --- server/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/setup.py b/server/setup.py index ea44f4782..adbb971e7 100644 --- a/server/setup.py +++ b/server/setup.py @@ -46,7 +46,7 @@ 'torch', 'regex', 'torchvision', - 'jina>=3.3.21', + 'jina>=3.4.0', 'docarray>=0.11.0', 'packaging', 'prometheus-client', From 3a0c105654a9cf32e82246f233084e581e530880 Mon Sep 17 00:00:00 2001 From: numb3r3 Date: Thu, 26 May 2022 12:34:39 +0800 Subject: [PATCH 4/6] fix: pb dependency --- server/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/setup.py b/server/setup.py index adbb971e7..add2747ec 100644 --- a/server/setup.py +++ b/server/setup.py @@ -42,6 +42,7 @@ zip_safe=False, setup_requires=['setuptools>=18.0', 'wheel'], install_requires=[ + 'protobuf>=3.13.0,<=3.20.1', 'ftfy', 'torch', 'regex', From 782d7bf3e9281d3e4c896839c92e4ecb16189374 Mon Sep 17 00:00:00 2001 From: numb3r3 Date: Thu, 26 May 2022 12:44:18 +0800 Subject: [PATCH 5/6] fix: revert rank monitor --- server/clip_server/executors/clip_torch.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/clip_server/executors/clip_torch.py b/server/clip_server/executors/clip_torch.py index a09119bec..5a31f56bb 100644 --- a/server/clip_server/executors/clip_torch.py +++ b/server/clip_server/executors/clip_torch.py @@ -74,8 +74,6 @@ def _preproc_texts(self, docs: 'DocumentArray'): @requests(on='/rank') async def rank(self, docs: 'DocumentArray', parameters: Dict, **kwargs): - self._rank_docs_counter.inc(len(docs)) - await self.encode(docs['@r,m']) set_rank(docs) From 5f6c1999f9d596bf53924458fb4bf8227d69f671 Mon Sep 17 00:00:00 2001 From: numb3r3 Date: Thu, 26 May 2022 12:50:14 +0800 Subject: [PATCH 6/6] fix: revert custom counter --- server/clip_server/executors/clip_torch.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/clip_server/executors/clip_torch.py b/server/clip_server/executors/clip_torch.py index 5a31f56bb..a51eb9ee6 100644 --- a/server/clip_server/executors/clip_torch.py +++ b/server/clip_server/executors/clip_torch.py @@ -85,9 +85,6 @@ async def encode(self, docs: 'DocumentArray', **kwargs): for d in docs: split_img_txt_da(d, _img_da, _txt_da) - self._img_docs_counter.inc(len(_img_da)) - self._txt_docs_counter.inc(len(_txt_da)) - with torch.inference_mode(): # for image if _img_da: