From 8e88b7d893904bb3ec070ab3eb88dc7de3e2809f Mon Sep 17 00:00:00 2001 From: numb3r3 Date: Wed, 1 Jun 2022 11:27:41 +0800 Subject: [PATCH 1/4] fix: turnoff monitor in default yaml config --- server/clip_server/onnx-flow.yml | 4 ---- server/clip_server/tensorrt-flow.yml | 4 ---- server/clip_server/torch-flow.yml | 4 ---- 3 files changed, 12 deletions(-) diff --git a/server/clip_server/onnx-flow.yml b/server/clip_server/onnx-flow.yml index d4bffef8a..3368c2188 100644 --- a/server/clip_server/onnx-flow.yml +++ b/server/clip_server/onnx-flow.yml @@ -2,8 +2,6 @@ jtype: Flow version: '1' with: port: 51000 - monitoring: True - port_monitoring: 9090 executors: - name: clip_o uses: @@ -11,5 +9,3 @@ executors: metas: py_modules: - executors/clip_onnx.py - monitoring: true - port_monitoring: 9091 diff --git a/server/clip_server/tensorrt-flow.yml b/server/clip_server/tensorrt-flow.yml index f8e2393a6..5b9c0dc27 100644 --- a/server/clip_server/tensorrt-flow.yml +++ b/server/clip_server/tensorrt-flow.yml @@ -2,8 +2,6 @@ jtype: Flow version: '1' with: port: 51000 - monitoring: True - port_monitoring: 9090 executors: - name: clip_r uses: @@ -11,5 +9,3 @@ executors: metas: py_modules: - executors/clip_tensorrt.py - monitoring: true - port_monitoring: 9091 diff --git a/server/clip_server/torch-flow.yml b/server/clip_server/torch-flow.yml index a1dfb5d6e..e7ba01c5a 100644 --- a/server/clip_server/torch-flow.yml +++ b/server/clip_server/torch-flow.yml @@ -2,8 +2,6 @@ jtype: Flow version: '1' with: port: 51000 - monitoring: True - port_monitoring: 9090 executors: - name: clip_t uses: @@ -11,5 +9,3 @@ executors: metas: py_modules: - executors/clip_torch.py - monitoring: true - port_monitoring: 9091 From 696a80ac51c030a66bf9eaf563f9ca7251fc6b0a Mon Sep 17 00:00:00 2001 From: numb3r3 Date: Wed, 1 Jun 2022 11:38:27 +0800 Subject: [PATCH 2/4] fix: monitor encoding perf --- server/clip_server/executors/clip_onnx.py | 13 ++++++- server/clip_server/executors/clip_tensorrt.py | 38 ++++++++++++------- server/clip_server/executors/clip_torch.py | 27 +++++++------ 3 files changed, 50 insertions(+), 28 deletions(-) diff --git a/server/clip_server/executors/clip_onnx.py b/server/clip_server/executors/clip_onnx.py index 14bfe8a7f..29b066a66 100644 --- a/server/clip_server/executors/clip_onnx.py +++ b/server/clip_server/executors/clip_onnx.py @@ -88,6 +88,14 @@ def _preproc_images(self, docs: 'DocumentArray'): def _preproc_texts(self, docs: 'DocumentArray'): return preproc_text(docs, return_np=True) + @monitor() + def _encode_images(self, docs: 'DocumentArray'): + docs.embeddings = self._model.encode_image(docs.tensors) + + @monitor() + def _encode_texts(self, docs: 'DocumentArray'): + docs.embeddings = self._model.encode_text(docs.tensors) + @requests(on='/rank') async def rank(self, docs: 'DocumentArray', parameters: Dict, **kwargs): await self.encode(docs['@r,m']) @@ -108,7 +116,8 @@ async def encode(self, docs: 'DocumentArray', **kwargs): batch_size=self._minibatch_size, pool=self._pool, ): - minibatch.embeddings = self._model.encode_image(minibatch.tensors) + self._encode_images(minibatch) + # recover original content try: _ = iter(_contents) @@ -124,7 +133,7 @@ async def encode(self, docs: 'DocumentArray', **kwargs): batch_size=self._minibatch_size, pool=self._pool, ): - minibatch.embeddings = self._model.encode_text(minibatch.tensors) + self._encode_texts(minibatch) # recover original content try: _ = iter(_contents) diff --git a/server/clip_server/executors/clip_tensorrt.py b/server/clip_server/executors/clip_tensorrt.py index e1099116e..1c55cd068 100644 --- a/server/clip_server/executors/clip_tensorrt.py +++ b/server/clip_server/executors/clip_tensorrt.py @@ -59,6 +59,26 @@ def _preproc_images(self, docs: 'DocumentArray'): def _preproc_texts(self, docs: 'DocumentArray'): return preproc_text(docs, device=self._device, return_np=False) + @monitor() + def _encode_images(self, docs: 'DocumentArray'): + docs.embeddings = ( + self._model.encode_image(docs.tensors) + .detach() + .cpu() + .numpy() + .astype(np.float32) + ) + + @monitor() + def _encode_texts(self, docs: 'DocumentArray'): + docs.embeddings = ( + self._model.encode_text(docs.tensors) + .detach() + .cpu() + .numpy() + .astype(np.float32) + ) + @requests(on='/rank') async def rank(self, docs: 'DocumentArray', parameters: Dict, **kwargs): await self.encode(docs['@r,m']) @@ -79,13 +99,8 @@ async def encode(self, docs: 'DocumentArray', **kwargs): batch_size=self._minibatch_size, pool=self._pool, ): - minibatch.embeddings = ( - self._model.encode_image(minibatch.tensors) - .detach() - .cpu() - .numpy() - .astype(np.float32) - ) + self._encode_images(minibatch) + # recover original content try: _ = iter(_contents) @@ -101,13 +116,8 @@ async def encode(self, docs: 'DocumentArray', **kwargs): batch_size=self._minibatch_size, pool=self._pool, ): - minibatch.embeddings = ( - self._model.encode_text(minibatch.tensors) - .detach() - .cpu() - .numpy() - .astype(np.float32) - ) + self._encode_texts(minibatch) + # recover original content try: _ = iter(_contents) diff --git a/server/clip_server/executors/clip_torch.py b/server/clip_server/executors/clip_torch.py index a51eb9ee6..0a770d587 100644 --- a/server/clip_server/executors/clip_torch.py +++ b/server/clip_server/executors/clip_torch.py @@ -72,6 +72,18 @@ def _preproc_images(self, docs: 'DocumentArray'): def _preproc_texts(self, docs: 'DocumentArray'): return preproc_text(docs, device=self._device, return_np=False) + @monitor() + def _encode_images(self, docs: 'DocumentArray'): + docs.embeddings = ( + self._model.encode_image(docs.tensors).cpu().numpy().astype(np.float32) + ) + + @monitor() + def _encode_texts(self, docs: 'DocumentArray'): + docs.embeddings = ( + self._model.encode_text(docs.tensors).cpu().numpy().astype(np.float32) + ) + @requests(on='/rank') async def rank(self, docs: 'DocumentArray', parameters: Dict, **kwargs): await self.encode(docs['@r,m']) @@ -93,12 +105,8 @@ async def encode(self, docs: 'DocumentArray', **kwargs): batch_size=self._minibatch_size, pool=self._pool, ): - minibatch.embeddings = ( - self._model.encode_image(minibatch.tensors) - .cpu() - .numpy() - .astype(np.float32) - ) + + self._encode_images(minibatch) # recover original content try: @@ -115,12 +123,7 @@ async def encode(self, docs: 'DocumentArray', **kwargs): batch_size=self._minibatch_size, pool=self._pool, ): - minibatch.embeddings = ( - self._model.encode_text(minibatch.tensors) - .cpu() - .numpy() - .astype(np.float32) - ) + self._encode_texts(minibatch) # recover original content try: From 3dce99da3407aaf8bd60a7027232ea15e7e84b69 Mon Sep 17 00:00:00 2001 From: numb3r3 Date: Wed, 1 Jun 2022 11:51:31 +0800 Subject: [PATCH 3/4] fix: better monitor name --- server/clip_server/executors/clip_onnx.py | 8 ++++---- server/clip_server/executors/clip_tensorrt.py | 8 ++++---- server/clip_server/executors/clip_torch.py | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/server/clip_server/executors/clip_onnx.py b/server/clip_server/executors/clip_onnx.py index 29b066a66..a6a365a74 100644 --- a/server/clip_server/executors/clip_onnx.py +++ b/server/clip_server/executors/clip_onnx.py @@ -78,21 +78,21 @@ def __init__( self._model.start_sessions(sess_options=sess_options, providers=providers) - @monitor() + @monitor(name='preprocess_images') def _preproc_images(self, docs: 'DocumentArray'): return preproc_image( docs, preprocess_fn=self._preprocess_tensor, return_np=True ) - @monitor() + @monitor(name='preprocess_texts') def _preproc_texts(self, docs: 'DocumentArray'): return preproc_text(docs, return_np=True) - @monitor() + @monitor(name='encode_images') def _encode_images(self, docs: 'DocumentArray'): docs.embeddings = self._model.encode_image(docs.tensors) - @monitor() + @monitor(name='encode_texts') def _encode_texts(self, docs: 'DocumentArray'): docs.embeddings = self._model.encode_text(docs.tensors) diff --git a/server/clip_server/executors/clip_tensorrt.py b/server/clip_server/executors/clip_tensorrt.py index 1c55cd068..b424fc93e 100644 --- a/server/clip_server/executors/clip_tensorrt.py +++ b/server/clip_server/executors/clip_tensorrt.py @@ -46,7 +46,7 @@ def __init__( self._model.start_engines() - @monitor() + @monitor(name='preprocess_images') def _preproc_images(self, docs: 'DocumentArray'): return preproc_image( docs, @@ -55,11 +55,11 @@ def _preproc_images(self, docs: 'DocumentArray'): return_np=False, ) - @monitor() + @monitor(name='preprocess_texts') def _preproc_texts(self, docs: 'DocumentArray'): return preproc_text(docs, device=self._device, return_np=False) - @monitor() + @monitor(name='encode_images') def _encode_images(self, docs: 'DocumentArray'): docs.embeddings = ( self._model.encode_image(docs.tensors) @@ -69,7 +69,7 @@ def _encode_images(self, docs: 'DocumentArray'): .astype(np.float32) ) - @monitor() + @monitor(name='encode_texts') def _encode_texts(self, docs: 'DocumentArray'): docs.embeddings = ( self._model.encode_text(docs.tensors) diff --git a/server/clip_server/executors/clip_torch.py b/server/clip_server/executors/clip_torch.py index 0a770d587..1cbe268dd 100644 --- a/server/clip_server/executors/clip_torch.py +++ b/server/clip_server/executors/clip_torch.py @@ -59,7 +59,7 @@ def __init__( self._pool = ThreadPool(processes=num_worker_preprocess) - @monitor() + @monitor(name='preprocess_images') def _preproc_images(self, docs: 'DocumentArray'): return preproc_image( docs, @@ -68,17 +68,17 @@ def _preproc_images(self, docs: 'DocumentArray'): return_np=False, ) - @monitor() + @monitor(name='preprocess_texts') def _preproc_texts(self, docs: 'DocumentArray'): return preproc_text(docs, device=self._device, return_np=False) - @monitor() + @monitor(name='encode_images') def _encode_images(self, docs: 'DocumentArray'): docs.embeddings = ( self._model.encode_image(docs.tensors).cpu().numpy().astype(np.float32) ) - @monitor() + @monitor(name='encode_texts') def _encode_texts(self, docs: 'DocumentArray'): docs.embeddings = ( self._model.encode_text(docs.tensors).cpu().numpy().astype(np.float32) From ef115659b7f5c9e70ea28192e82bc35218d88d08 Mon Sep 17 00:00:00 2001 From: numb3r3 Date: Wed, 1 Jun 2022 15:44:52 +0800 Subject: [PATCH 4/4] fix: address comments --- server/clip_server/executors/clip_onnx.py | 8 ++++---- server/clip_server/executors/clip_tensorrt.py | 8 ++++---- server/clip_server/executors/clip_torch.py | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/server/clip_server/executors/clip_onnx.py b/server/clip_server/executors/clip_onnx.py index a6a365a74..6afaef9c0 100644 --- a/server/clip_server/executors/clip_onnx.py +++ b/server/clip_server/executors/clip_onnx.py @@ -78,21 +78,21 @@ def __init__( self._model.start_sessions(sess_options=sess_options, providers=providers) - @monitor(name='preprocess_images') + @monitor(name='preprocess_images_seconds') def _preproc_images(self, docs: 'DocumentArray'): return preproc_image( docs, preprocess_fn=self._preprocess_tensor, return_np=True ) - @monitor(name='preprocess_texts') + @monitor(name='preprocess_texts_seconds') def _preproc_texts(self, docs: 'DocumentArray'): return preproc_text(docs, return_np=True) - @monitor(name='encode_images') + @monitor(name='encode_images_seconds') def _encode_images(self, docs: 'DocumentArray'): docs.embeddings = self._model.encode_image(docs.tensors) - @monitor(name='encode_texts') + @monitor(name='encode_texts_seconds') def _encode_texts(self, docs: 'DocumentArray'): docs.embeddings = self._model.encode_text(docs.tensors) diff --git a/server/clip_server/executors/clip_tensorrt.py b/server/clip_server/executors/clip_tensorrt.py index b424fc93e..a05df304a 100644 --- a/server/clip_server/executors/clip_tensorrt.py +++ b/server/clip_server/executors/clip_tensorrt.py @@ -46,7 +46,7 @@ def __init__( self._model.start_engines() - @monitor(name='preprocess_images') + @monitor(name='preprocess_images_seconds') def _preproc_images(self, docs: 'DocumentArray'): return preproc_image( docs, @@ -55,11 +55,11 @@ def _preproc_images(self, docs: 'DocumentArray'): return_np=False, ) - @monitor(name='preprocess_texts') + @monitor(name='preprocess_texts_seconds') def _preproc_texts(self, docs: 'DocumentArray'): return preproc_text(docs, device=self._device, return_np=False) - @monitor(name='encode_images') + @monitor(name='encode_images_seconds') def _encode_images(self, docs: 'DocumentArray'): docs.embeddings = ( self._model.encode_image(docs.tensors) @@ -69,7 +69,7 @@ def _encode_images(self, docs: 'DocumentArray'): .astype(np.float32) ) - @monitor(name='encode_texts') + @monitor(name='encode_texts_seconds') def _encode_texts(self, docs: 'DocumentArray'): docs.embeddings = ( self._model.encode_text(docs.tensors) diff --git a/server/clip_server/executors/clip_torch.py b/server/clip_server/executors/clip_torch.py index 1cbe268dd..2648efa32 100644 --- a/server/clip_server/executors/clip_torch.py +++ b/server/clip_server/executors/clip_torch.py @@ -59,7 +59,7 @@ def __init__( self._pool = ThreadPool(processes=num_worker_preprocess) - @monitor(name='preprocess_images') + @monitor(name='preprocess_images_seconds') def _preproc_images(self, docs: 'DocumentArray'): return preproc_image( docs, @@ -68,17 +68,17 @@ def _preproc_images(self, docs: 'DocumentArray'): return_np=False, ) - @monitor(name='preprocess_texts') + @monitor(name='preprocess_texts_seconds') def _preproc_texts(self, docs: 'DocumentArray'): return preproc_text(docs, device=self._device, return_np=False) - @monitor(name='encode_images') + @monitor(name='encode_images_seconds') def _encode_images(self, docs: 'DocumentArray'): docs.embeddings = ( self._model.encode_image(docs.tensors).cpu().numpy().astype(np.float32) ) - @monitor(name='encode_texts') + @monitor(name='encode_texts_seconds') def _encode_texts(self, docs: 'DocumentArray'): docs.embeddings = ( self._model.encode_text(docs.tensors).cpu().numpy().astype(np.float32)