From b62470caf6b410c2814d66575deccf652410012b Mon Sep 17 00:00:00 2001 From: ZiniuYu Date: Thu, 28 Jul 2022 13:34:51 +0800 Subject: [PATCH 1/8] chore: remove clip_hg --- .github/workflows/cd.yml | 1 - .github/workflows/ci.yml | 1 - server/clip_server/executors/clip_hg.py | 241 ------------------------ server/clip_server/hg-flow.yml | 12 -- server/setup.py | 1 - tests/conftest.py | 28 +-- tests/test_hg.py | 25 --- 7 files changed, 2 insertions(+), 307 deletions(-) delete mode 100644 server/clip_server/executors/clip_hg.py delete mode 100644 server/clip_server/hg-flow.yml delete mode 100644 tests/test_hg.py diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index ad96912d2..9dcfd0a66 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -42,7 +42,6 @@ jobs: python -m pip install wheel pip install --no-cache-dir "client/[test]" pip install --no-cache-dir "server/[onnx]" - pip install --no-cache-dir "server/[huggingface]" - name: Test id: test run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa1bf9f28..31305b570 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,7 +114,6 @@ jobs: python -m pip install wheel pytest pytest-cov pip install --no-cache-dir "client/[test]" pip install --no-cache-dir "server/[onnx]" - pip install --no-cache-dir "server/[huggingface]" - name: Test id: test run: | diff --git a/server/clip_server/executors/clip_hg.py b/server/clip_server/executors/clip_hg.py deleted file mode 100644 index a5edab62e..000000000 --- a/server/clip_server/executors/clip_hg.py +++ /dev/null @@ -1,241 +0,0 @@ -import os -import warnings -from multiprocessing.pool import ThreadPool -from typing import Dict, Optional -import numpy as np -import torch -from transformers import CLIPFeatureExtractor, CLIPModel, CLIPTokenizer -from clip_server.executors.helper import ( - split_img_txt_da, - set_rank, -) -from jina import Executor, requests, DocumentArray - - -class CLIPEncoder(Executor): - def __init__( - self, - pretrained_model_name_or_path: str = 'openai/clip-vit-base-patch32', - finetuned_checkpoint_path: Optional[str] = None, - base_feature_extractor: Optional[str] = None, - base_tokenizer_model: Optional[str] = None, - preprocessing: bool = True, - max_length: int = 77, - device: str = 'cpu', - num_worker_preprocess: int = 4, - minibatch_size: int = 32, - traversal_paths: str = '@r', - *args, - **kwargs, - ): - """ - :param pretrained_model_name_or_path: Can be either: - - A string, the model id of a pretrained CLIP model hosted - inside a model repo on huggingface.co, e.g., - 'openai/clip-vit-base-patch32' - - A path to a directory containing model weights saved, e.g., - ./my_model_directory/ - :param finetuned_checkpoint_path: If set, the pretrained model weights will be replaced with weights - loading from the given checkpoint. - :param base_feature_extractor: Base feature extractor for images. - Defaults to ``pretrained_model_name_or_path`` if None. - :param base_tokenizer_model: Base tokenizer model. - Defaults to ``pretrained_model_name_or_path`` if None. - :param preprocessing: Whether to use the `base_feature_extractor` - on images (tensors) before encoding them. If you disable this, you must - ensure that the images you pass in have the correct format, see the - ``encode`` method for details. - :param max_length: Max length argument for the tokenizer. All CLIP models - use 77 as the max length. - :param device: Pytorch device to put the model on, e.g. 'cpu', 'cuda', - 'cuda:1'. - :param num_worker_preprocess: Number of cpu processes used in preprocessing step. - :param minibatch_size: Default batch size for encoding, used if the - batch size is not passed as a parameter with the request. - :param traversal_paths: Default traversal paths for encoding, used if - the traversal path is not passed as a parameter with the request. - """ - super().__init__(*args, **kwargs) - self._minibatch_size = minibatch_size - - self._preprocessing = preprocessing - self._max_length = max_length - self._traversal_paths = traversal_paths - - # self.device = device - if not device: - self._device = 'cuda' if torch.cuda.is_available() else 'cpu' - else: - self._device = device - - if not self._device.startswith('cuda') and ( - 'OMP_NUM_THREADS' not in os.environ - and hasattr(self.runtime_args, 'replicas') - ): - replicas = getattr(self.runtime_args, 'replicas', 1) - num_threads = max(1, torch.get_num_threads() // replicas) - if num_threads < 2: - warnings.warn( - f'Too many replicas ({replicas}) vs too few threads {num_threads} may result in ' - f'sub-optimal performance.' - ) - - # NOTE: make sure to set the threads right after the torch import, - # and `torch.set_num_threads` always take precedence over environment variables `OMP_NUM_THREADS`. - # For more details, please see https://pytorch.org/docs/stable/generated/torch.set_num_threads.html - torch.set_num_threads(max(num_threads, 1)) - torch.set_num_interop_threads(1) - - self._vision_preprocessor = CLIPFeatureExtractor.from_pretrained( - base_feature_extractor or pretrained_model_name_or_path - ) - self._tokenizer = CLIPTokenizer.from_pretrained( - base_tokenizer_model or pretrained_model_name_or_path - ) - self._model = CLIPModel.from_pretrained(pretrained_model_name_or_path) - - if finetuned_checkpoint_path: - if finetuned_checkpoint_path.startswith( - 'https://' - ) or finetuned_checkpoint_path.startswith('http://'): - state_dict = torch.hub.load_state_dict_from_url( - finetuned_checkpoint_path, map_location='cpu', progress=True - ) - else: - state_dict = torch.load(finetuned_checkpoint_path, map_location='cpu') - self._model.load_state_dict(state_dict) - - self._model.eval().to(self._device) - self._pool = ThreadPool(processes=num_worker_preprocess) - - def _preproc_images(self, docs: 'DocumentArray'): - with self.monitor( - name='preprocess_images_seconds', - documentation='images preprocess time in seconds', - ): - if self._preprocessing: - tensors_batch = [] - - for d in docs: - content = d.content - - if d.blob: - d.convert_blob_to_image_tensor() - elif d.tensor is None and d.uri: - # in case user uses HTTP protocol and send data via curl not using .blob (base64), but in .uri - d.load_uri_to_image_tensor() - - tensors_batch.append(d.tensor) - - # recover content - d.content = content - - batch_data = self._vision_preprocessor( - images=tensors_batch, - return_tensors='pt', - ) - batch_data = { - k: v.type(torch.float32).to(self._device) - for k, v in batch_data.items() - } - - else: - batch_data = { - 'pixel_values': torch.tensor( - docs.tensors, dtype=torch.float32, device=self._device - ) - } - - return docs, batch_data - - def _preproc_texts(self, docs: 'DocumentArray'): - with self.monitor( - name='preprocess_texts_seconds', - documentation='texts preprocess time in seconds', - ): - batch_data = self._tokenizer( - docs.texts, - max_length=self._max_length, - padding='longest', - truncation=True, - return_tensors='pt', - ) - batch_data = {k: v.to(self._device) for k, v in batch_data.items()} - return docs, batch_data - - @requests(on='/rank') - async def rank(self, docs: 'DocumentArray', parameters: Dict, **kwargs): - await self.encode(docs['@r,m']) - - set_rank(docs) - - @requests - async def encode(self, docs: DocumentArray, parameters: Dict = {}, **kwargs): - """ - Encode all documents with `text` or image content using the corresponding CLIP - encoder. Store the embeddings in the `embedding` attribute. - - :param docs: Documents sent to the encoder. The image docs must have - ``tensor`` of the - shape ``Height x Width x 3``. By default, the input ``tensor`` must - be an ``ndarray`` with ``dtype=uint8`` or ``dtype=float32``. - If you set ``preprocessing=True`` when creating this encoder, - then the ``tensor`` arrays should have the shape ``[H, W, 3]``, and be in - the RGB color format with ``dtype=uint8``. - If you set ``preprocessing=False`` when creating this encoder, - then you need to ensure that the images you pass in are already - pre-processed. This means that they are all the same size (for batching) - - the CLIP model was trained on images of the size ``224 x 224``, and that - they are of the shape ``[3, H, W]`` with ``dtype=float32``. They should - also be normalized (values between 0 and 1). - :param parameters: A dictionary that contains parameters to control encoding. - The accepted keys are ``traversal_paths`` and ``minibatch_size`` - in their - absence their corresponding default values are used. - """ - - traversal_paths = parameters.get('traversal_paths', self._traversal_paths) - minibatch_size = parameters.get('minibatch_size', self._minibatch_size) - - _img_da = DocumentArray() - _txt_da = DocumentArray() - for d in docs[traversal_paths]: - split_img_txt_da(d, _img_da, _txt_da) - - with torch.inference_mode(): - # for image - if _img_da: - for minibatch, batch_data in _img_da.map_batch( - self._preproc_images, - batch_size=minibatch_size, - pool=self._pool, - ): - with self.monitor( - name='encode_images_seconds', - documentation='images encode time in seconds', - ): - minibatch.embeddings = ( - self._model.get_image_features(**batch_data) - .cpu() - .numpy() - .astype(np.float32) - ) - - # for text - if _txt_da: - for minibatch, batch_data in _txt_da.map_batch( - self._preproc_texts, - batch_size=minibatch_size, - pool=self._pool, - ): - with self.monitor( - name='encode_texts_seconds', - documentation='texts encode time in seconds', - ): - minibatch.embeddings = ( - self._model.get_text_features(**batch_data) - .cpu() - .numpy() - .astype(np.float32) - ) - - return docs diff --git a/server/clip_server/hg-flow.yml b/server/clip_server/hg-flow.yml deleted file mode 100644 index bf8120a4e..000000000 --- a/server/clip_server/hg-flow.yml +++ /dev/null @@ -1,12 +0,0 @@ -jtype: Flow -version: '1' -with: - port: 51000 -executors: - - name: clip_h - uses: - jtype: CLIPEncoder - metas: - py_modules: - - executors/clip_hg.py - replicas: 1 \ No newline at end of file diff --git a/server/setup.py b/server/setup.py index 90117cfb6..fb17f35a7 100644 --- a/server/setup.py +++ b/server/setup.py @@ -58,7 +58,6 @@ ] + (['onnxruntime-gpu>=1.8.0'] if sys.platform != 'darwin' else []), 'tensorrt': ['nvidia-tensorrt'], - 'huggingface': ['transformers>=4.16.2'], }, classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/tests/conftest.py b/tests/conftest.py index cc7feea03..a172d288f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,15 +16,13 @@ def random_port(): return random_port -@pytest.fixture(scope='session', params=['onnx', 'torch', 'hg', 'onnx_custom']) +@pytest.fixture(scope='session', params=['onnx', 'torch', 'onnx_custom']) def make_flow(port_generator, request): if request.param != 'onnx_custom': if request.param == 'onnx': from clip_server.executors.clip_onnx import CLIPEncoder - elif request.param == 'torch': - from clip_server.executors.clip_torch import CLIPEncoder else: - from clip_server.executors.clip_hg import CLIPEncoder + from clip_server.executors.clip_torch import CLIPEncoder f = Flow(port=port_generator()).add(name=request.param, uses=CLIPEncoder) else: @@ -56,25 +54,3 @@ def make_trt_flow(port_generator, request): f = Flow(port=port_generator()).add(name=request.param, uses=CLIPEncoder) with f: yield f - - -@pytest.fixture(scope='session', params=['hg']) -def make_hg_flow(port_generator, request): - from clip_server.executors.clip_hg import CLIPEncoder - - f = Flow(port=port_generator()).add(name=request.param, uses=CLIPEncoder) - with f: - yield f - - -@pytest.fixture(scope='session', params=['hg']) -def make_hg_flow_no_default(port_generator, request): - from clip_server.executors.clip_hg import CLIPEncoder - - f = Flow(port=port_generator()).add( - name=request.param, - uses=CLIPEncoder, - uses_with={'preprocessing': False}, - ) - with f: - yield f diff --git a/tests/test_hg.py b/tests/test_hg.py deleted file mode 100644 index b35858d49..000000000 --- a/tests/test_hg.py +++ /dev/null @@ -1,25 +0,0 @@ -import os - -import pytest -import numpy as np -from docarray import Document, DocumentArray -from jina import Flow - -from clip_client.client import Client - - -@pytest.mark.parametrize( - 'inputs', - [ - DocumentArray( - [ - Document(tensor=np.ones((3, 224, 224), dtype=np.float32)), - Document(tensor=np.ones((3, 224, 224), dtype=np.float32)), - ] - ), - ], -) -def test_batch_no_preprocessing(make_hg_flow_no_default, inputs, port_generator): - c = Client(server=f'grpc://0.0.0.0:{make_hg_flow_no_default.port}') - r = c.encode(inputs if not callable(inputs) else inputs()) - assert r.embeddings.shape == (len(r), 512) From a2ccb89339c06cda0319eb991b56d022e6efe0b1 Mon Sep 17 00:00:00 2001 From: ZiniuYu Date: Thu, 28 Jul 2022 13:54:56 +0800 Subject: [PATCH 2/8] fix: add dependency for m-clip --- server/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/setup.py b/server/setup.py index fb17f35a7..fd0ff4690 100644 --- a/server/setup.py +++ b/server/setup.py @@ -47,6 +47,7 @@ 'torch', 'regex', 'torchvision', + 'transformers', 'jina>=3.6.0', 'prometheus-client', 'open_clip_torch>=1.3.0', From a0a1af2169c50ba9be1ceb268a5681a9622d7fd0 Mon Sep 17 00:00:00 2001 From: ZiniuYu Date: Thu, 28 Jul 2022 14:01:23 +0800 Subject: [PATCH 3/8] fix: transformers version --- server/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/setup.py b/server/setup.py index fd0ff4690..8651a4619 100644 --- a/server/setup.py +++ b/server/setup.py @@ -47,7 +47,7 @@ 'torch', 'regex', 'torchvision', - 'transformers', + 'transformers>=4.16.2', 'jina>=3.6.0', 'prometheus-client', 'open_clip_torch>=1.3.0', From e8b60440268d70de2767d02bfa38acfa0656d02f Mon Sep 17 00:00:00 2001 From: ZiniuYu Date: Thu, 28 Jul 2022 14:35:05 +0800 Subject: [PATCH 4/8] fix: modify dependency key --- .github/workflows/cd.yml | 1 + .github/workflows/ci.yml | 1 + server/setup.py | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 9dcfd0a66..ec560c1ec 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -42,6 +42,7 @@ jobs: python -m pip install wheel pip install --no-cache-dir "client/[test]" pip install --no-cache-dir "server/[onnx]" + pip install --no-cache-dir "server/[mclip]" - name: Test id: test run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31305b570..1637a5b40 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,6 +114,7 @@ jobs: python -m pip install wheel pytest pytest-cov pip install --no-cache-dir "client/[test]" pip install --no-cache-dir "server/[onnx]" + pip install --no-cache-dir "server/[mclip]" - name: Test id: test run: | diff --git a/server/setup.py b/server/setup.py index 8651a4619..f6326a7ab 100644 --- a/server/setup.py +++ b/server/setup.py @@ -47,7 +47,6 @@ 'torch', 'regex', 'torchvision', - 'transformers>=4.16.2', 'jina>=3.6.0', 'prometheus-client', 'open_clip_torch>=1.3.0', @@ -59,6 +58,7 @@ ] + (['onnxruntime-gpu>=1.8.0'] if sys.platform != 'darwin' else []), 'tensorrt': ['nvidia-tensorrt'], + 'mclip': ['transformers>=4.16.2'], }, classifiers=[ 'Development Status :: 5 - Production/Stable', From 3821b3081ada0e5ab46e77b70c0900cdf7448825 Mon Sep 17 00:00:00 2001 From: Ziniu Yu Date: Fri, 29 Jul 2022 10:36:47 +0800 Subject: [PATCH 5/8] fix: update requirements Co-authored-by: felix-wang <35718120+numb3r3@users.noreply.github.com> --- server/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/setup.py b/server/setup.py index f6326a7ab..1c7251b52 100644 --- a/server/setup.py +++ b/server/setup.py @@ -58,7 +58,7 @@ ] + (['onnxruntime-gpu>=1.8.0'] if sys.platform != 'darwin' else []), 'tensorrt': ['nvidia-tensorrt'], - 'mclip': ['transformers>=4.16.2'], + 'transformers': ['transformers>=4.16.2'], }, classifiers=[ 'Development Status :: 5 - Production/Stable', From f0478fdfd546f3996539f2d6d468bb1da78711c0 Mon Sep 17 00:00:00 2001 From: Ziniu Yu Date: Fri, 29 Jul 2022 10:36:55 +0800 Subject: [PATCH 6/8] fix: update requirements Co-authored-by: felix-wang <35718120+numb3r3@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1637a5b40..039d28793 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,7 +114,7 @@ jobs: python -m pip install wheel pytest pytest-cov pip install --no-cache-dir "client/[test]" pip install --no-cache-dir "server/[onnx]" - pip install --no-cache-dir "server/[mclip]" + pip install --no-cache-dir "server/[transformers]" - name: Test id: test run: | From 28cae4647f2f4aeab9e2fb49d6f830bd4e3734ff Mon Sep 17 00:00:00 2001 From: Ziniu Yu Date: Fri, 29 Jul 2022 10:37:01 +0800 Subject: [PATCH 7/8] fix: update requirements Co-authored-by: felix-wang <35718120+numb3r3@users.noreply.github.com> --- .github/workflows/cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index ec560c1ec..dc5b5b388 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -42,7 +42,7 @@ jobs: python -m pip install wheel pip install --no-cache-dir "client/[test]" pip install --no-cache-dir "server/[onnx]" - pip install --no-cache-dir "server/[mclip]" + pip install --no-cache-dir "server/[transformers]" - name: Test id: test run: | From e21bf7057abef5784a0711968ce2bf4bf7aa8ff4 Mon Sep 17 00:00:00 2001 From: ZiniuYu Date: Fri, 29 Jul 2022 15:19:13 +0800 Subject: [PATCH 8/8] fix: tmp set grpcio version --- server/setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/setup.py b/server/setup.py index 1c7251b52..fd1e1ffad 100644 --- a/server/setup.py +++ b/server/setup.py @@ -50,6 +50,9 @@ 'jina>=3.6.0', 'prometheus-client', 'open_clip_torch>=1.3.0', + 'grpcio>=1.46.0,<1.48.0', # tmp requires + 'grpcio-reflection>=1.46.0,<1.48.0', # tmp requires + 'grpcio-health-checking>=1.46.0,<1.48.0', # tmp requires ], extras_require={ 'onnx': [