diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38d773dc0..ab7d9b26c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,7 +136,7 @@ jobs: fail_ci_if_error: false token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos - gpu-test: + trt-gpu-test: needs: prep-testbed runs-on: [self-hosted, x64, gpu, linux] strategy: @@ -173,6 +173,58 @@ jobs: -v -s -m "gpu" ./tests/test_tensorrt.py pytest --suppress-no-test-exit-code --cov=clip_client --cov=clip_server --cov-report=xml \ -v -s -m "gpu" ./tests/test_simple.py + echo "::set-output name=codecov_flag::cas" + timeout-minutes: 30 + env: + # fix re-initialized torch runtime error on cuda device + JINA_MP_START_METHOD: spawn + - name: Check codecov file + id: check_files + uses: andstor/file-existence-action@v1 + with: + files: "coverage.xml" + - name: Upload coverage from test to Codecov + uses: codecov/codecov-action@v3 + if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.7' + with: + file: coverage.xml + name: gpu-related-codecov + flags: ${{ steps.test.outputs.codecov_flag }} + fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos + + gpu-model-test: + needs: prep-testbed + runs-on: [ self-hosted, x64, gpu, linux ] + strategy: + fail-fast: false + matrix: + python-version: [ 3.7 ] + steps: + - uses: actions/checkout@v2 + with: + # For coverage builds fetch the whole history + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Prepare enviroment + run: | + python -m pip install --upgrade pip + python -m pip install wheel pytest pytest-cov nvidia-pyindex + pip install -e "client/[test]" + pip install -e "server/[onnx]" + pip install -e "server/[transformers]" + { + pip install -e "server/[flash-attn]" + } || { + echo "flash attention was not installed." + } + pip install --no-cache-dir "server/[cn_clip]" + - name: Test + id: test + run: | pytest --suppress-no-test-exit-code --cov=clip_client --cov=clip_server --cov-report=xml \ -v -s -m "gpu" ./tests/test_model.py echo "::set-output name=codecov_flag::cas" @@ -197,7 +249,7 @@ jobs: # just for blocking the merge until all parallel core-test are successful success-all-test: - needs: [commit-lint, core-test, gpu-test] + needs: [commit-lint, core-test, trt-gpu-test, gpu-model-test] if: always() runs-on: ubuntu-latest steps: diff --git a/docs/user-guides/server.md b/docs/user-guides/server.md index ec0384d9a..4a1d5a574 100644 --- a/docs/user-guides/server.md +++ b/docs/user-guides/server.md @@ -591,9 +591,21 @@ The build argument `--build-arg GROUP_ID=$(id -g ${USER}) --build-arg USER_ID=$( ### Run +````{tab} PyTorch ```bash docker run -p 51009:51000 -v $HOME/.cache:/home/cas/.cache --gpus all jinaai/clip-server ``` +```` +````{tab} ONNX +```bash +docker run -p 51009:51000 -v $HOME/.cache:/home/cas/.cache --gpus all jinaai/clip-server:master-onnx onnx-flow.yml +``` +```` +````{tab} TensorRT +```bash +docker run -p 51009:51000 -v $HOME/.cache:/home/cas/.cache --gpus all jinaai/clip-server:master-tensorrt tensorrt-flow.yml +``` +```` Here, `51009` is the public port on the host and `51000` is the {ref}`in-container port defined inside YAML`. The argument `-v $HOME/.cache:/home/cas/.cache` leverages host's cache and prevents you to download the same model next time on start. @@ -601,11 +613,23 @@ Due to the limitation of the terminal inside Docker container, you will **not** To pass a YAML config from the host, one can do: +````{tab} PyTorch ```bash cat my.yml | docker run -i -p 51009:51000 -v $HOME/.cache:/home/cas/.cache --gpus all jinaai/clip-server -i ``` +```` +````{tab} ONNX +```bash +cat my.yml | docker run -i -p 51009:51000 -v $HOME/.cache:/home/cas/.cache --gpus all jinaai/clip-server:master-onnx -i +``` +```` +````{tab} TensorRT +```bash +cat my.yml | docker run -i -p 51009:51000 -v $HOME/.cache:/home/cas/.cache --gpus all jinaai/clip-server:master-tensorrt -i +``` +```` -The CLI usage is the same {ref}`as described here `. +The CLI usage is the same {ref}`as described here `. ```{tip} You can enable debug logging via: `docker run --env JINA_LOG_LEVEL=debug ...` diff --git a/server/setup.py b/server/setup.py index 7d850fabf..b59f8b6a1 100644 --- a/server/setup.py +++ b/server/setup.py @@ -51,12 +51,17 @@ ], extras_require={ 'onnx': [ - 'onnxruntime', 'onnx', 'onnxmltools', ] - + (['onnxruntime-gpu>=1.8.0'] if sys.platform != 'darwin' else []), - 'tensorrt': ['nvidia-tensorrt'], + + ( + ['onnxruntime-gpu<=1.13.1'] + if sys.platform != 'darwin' + else ['onnxruntime<=1.13.1'] + ), + 'tensorrt': [ + 'nvidia-tensorrt==8.4.1.5', + ], 'transformers': ['transformers>=4.16.2'], 'search': ['annlite>=0.3.10'], 'flash-attn': ['flash-attn'], diff --git a/tests/test_asyncio.py b/tests/test_asyncio.py index cf3618be4..67bbbd2e7 100644 --- a/tests/test_asyncio.py +++ b/tests/test_asyncio.py @@ -26,7 +26,7 @@ async def test_async_encode(make_flow): DocumentArray( [ Document( - uri='https://docarray.jina.ai/_static/favicon.png', + uri='https://clip-as-service.jina.ai/_static/favicon.png', text='hello, world', ), ] diff --git a/tests/test_helper.py b/tests/test_helper.py index acce92ad2..3494d0583 100644 --- a/tests/test_helper.py +++ b/tests/test_helper.py @@ -32,10 +32,10 @@ def test_numpy_softmax(shape, axis): Document(text='goodbye, world'), Document( text='hello, world', - uri='https://docarray.jina.ai/_static/favicon.png', + uri='https://clip-as-service.jina.ai/_static/favicon.png', ), Document( - uri='https://docarray.jina.ai/_static/favicon.png', + uri='https://clip-as-service.jina.ai/_static/favicon.png', ), ] ), @@ -47,14 +47,14 @@ def test_numpy_softmax(shape, axis): Document(text='hello, world'), Document(tensor=np.array([0, 1, 2])), Document( - uri='https://docarray.jina.ai/_static/favicon.png' + uri='https://clip-as-service.jina.ai/_static/favicon.png' ).load_uri_to_blob(), Document( tensor=np.array([0, 1, 2]), - uri='https://docarray.jina.ai/_static/favicon.png', + uri='https://clip-as-service.jina.ai/_static/favicon.png', ), Document( - uri='https://docarray.jina.ai/_static/favicon.png', + uri='https://clip-as-service.jina.ai/_static/favicon.png', ), ] ), @@ -64,7 +64,7 @@ def test_numpy_softmax(shape, axis): DocumentArray( [ Document(text='hello, world'), - Document(uri='https://docarray.jina.ai/_static/favicon.png'), + Document(uri='https://clip-as-service.jina.ai/_static/favicon.png'), ] ), (1, 1), @@ -86,7 +86,7 @@ def test_split_img_txt_da(inputs): DocumentArray( [ Document( - uri='https://docarray.jina.ai/_static/favicon.png', + uri='https://clip-as-service.jina.ai/_static/favicon.png', ).load_uri_to_blob(), ] ) diff --git a/tests/test_ranker.py b/tests/test_ranker.py index 70f8165b3..450f66530 100644 --- a/tests/test_ranker.py +++ b/tests/test_ranker.py @@ -68,14 +68,14 @@ async def test_torch_executor_rank_text2imgs(encoder_class): [ [ Document( - uri='https://docarray.jina.ai/_static/favicon.png', + uri='https://clip-as-service.jina.ai/_static/favicon.png', matches=[ Document(text='hello, world'), Document(text='goodbye, world'), ], ), Document( - uri='https://docarray.jina.ai/_static/favicon.png', + uri='https://clip-as-service.jina.ai/_static/favicon.png', matches=[ Document(text='hello, world'), Document(text='goodbye, world'), @@ -85,14 +85,14 @@ async def test_torch_executor_rank_text2imgs(encoder_class): DocumentArray( [ Document( - uri='https://docarray.jina.ai/_static/favicon.png', + uri='https://clip-as-service.jina.ai/_static/favicon.png', matches=[ Document(text='hello, world'), Document(text='goodbye, world'), ], ), Document( - uri='https://docarray.jina.ai/_static/favicon.png', + uri='https://clip-as-service.jina.ai/_static/favicon.png', matches=[ Document(text='hello, world'), Document(text='goodbye, world'), @@ -102,7 +102,7 @@ async def test_torch_executor_rank_text2imgs(encoder_class): ), lambda: ( Document( - uri='https://docarray.jina.ai/_static/favicon.png', + uri='https://clip-as-service.jina.ai/_static/favicon.png', matches=[ Document(text='hello, world'), Document(text='goodbye, world'), @@ -115,7 +115,9 @@ async def test_torch_executor_rank_text2imgs(encoder_class): Document( text='hello, world', matches=[ - Document(uri='https://docarray.jina.ai/_static/favicon.png'), + Document( + uri='https://clip-as-service.jina.ai/_static/favicon.png' + ), Document( uri=f'{os.path.dirname(os.path.abspath(__file__))}/img/00000.jpg' ), @@ -144,7 +146,7 @@ def test_docarray_inputs(make_flow, inputs): [ [ Document( - uri='https://docarray.jina.ai/_static/favicon.png', + uri='https://clip-as-service.jina.ai/_static/favicon.png', matches=[ Document(text='hello, world'), Document(text='goodbye, world'), @@ -154,7 +156,7 @@ def test_docarray_inputs(make_flow, inputs): DocumentArray( [ Document( - uri='https://docarray.jina.ai/_static/favicon.png', + uri='https://clip-as-service.jina.ai/_static/favicon.png', matches=[ Document(text='hello, world'), Document(text='goodbye, world'), @@ -164,7 +166,7 @@ def test_docarray_inputs(make_flow, inputs): ), lambda: ( Document( - uri='https://docarray.jina.ai/_static/favicon.png', + uri='https://clip-as-service.jina.ai/_static/favicon.png', matches=[ Document(text='hello, world'), Document(text='goodbye, world'), @@ -177,7 +179,9 @@ def test_docarray_inputs(make_flow, inputs): Document( text='hello, world', matches=[ - Document(uri='https://docarray.jina.ai/_static/favicon.png'), + Document( + uri='https://clip-as-service.jina.ai/_static/favicon.png' + ), Document( uri=f'{os.path.dirname(os.path.abspath(__file__))}/img/00000.jpg' ), diff --git a/tests/test_search.py b/tests/test_search.py index 52dd15e4c..b9645e96e 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -15,7 +15,7 @@ lambda: (Document(text='hello, world') for _ in range(10)), DocumentArray( [ - Document(uri='https://docarray.jina.ai/_static/favicon.png'), + Document(uri='https://clip-as-service.jina.ai/_static/favicon.png'), Document( uri=f'{os.path.dirname(os.path.abspath(__file__))}/img/00000.jpg' ), @@ -52,7 +52,7 @@ def test_index_search(make_search_flow, inputs, limit): lambda: (Document(text='hello, world') for _ in range(10)), DocumentArray( [ - Document(uri='https://docarray.jina.ai/_static/favicon.png'), + Document(uri='https://clip-as-service.jina.ai/_static/favicon.png'), Document( uri=f'{os.path.dirname(os.path.abspath(__file__))}/img/00000.jpg' ), diff --git a/tests/test_server.py b/tests/test_server.py index b5046baf0..56efa742f 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -10,9 +10,9 @@ def test_server_download(tmpdir): download_model( - url='https://docarray.jina.ai/_static/favicon.png', + url='https://clip-as-service.jina.ai/_static/favicon.png', target_folder=tmpdir, - md5sum='66ea4817d73514888dcf6c7d2b00016d', + md5sum='43104e468ddd23c55bc662d84c87a7f8', with_resume=False, ) target_path = os.path.join(tmpdir, 'favicon.png') @@ -27,20 +27,20 @@ def test_server_download(tmpdir): os.remove(target_path) download_model( - url='https://docarray.jina.ai/_static/favicon.png', + url='https://clip-as-service.jina.ai/_static/favicon.png', target_folder=tmpdir, - md5sum='66ea4817d73514888dcf6c7d2b00016d', + md5sum='43104e468ddd23c55bc662d84c87a7f8', with_resume=True, ) assert os.path.getsize(target_path) == file_size assert not os.path.exists(part_path) -@pytest.mark.parametrize('md5', ['ABC', None, '66ea4817d73514888dcf6c7d2b00016d']) +@pytest.mark.parametrize('md5', ['ABC', None, '43104e468ddd23c55bc662d84c87a7f8']) def test_server_download_md5(tmpdir, md5): if md5 != 'ABC': download_model( - url='https://docarray.jina.ai/_static/favicon.png', + url='https://clip-as-service.jina.ai/_static/favicon.png', target_folder=tmpdir, md5sum=md5, with_resume=False, @@ -48,7 +48,7 @@ def test_server_download_md5(tmpdir, md5): else: with pytest.raises(Exception): download_model( - url='https://docarray.jina.ai/_static/favicon.png', + url='https://clip-as-service.jina.ai/_static/favicon.png', target_folder=tmpdir, md5sum=md5, with_resume=False, @@ -58,7 +58,7 @@ def test_server_download_md5(tmpdir, md5): def test_server_download_not_regular_file(tmpdir): with pytest.raises(Exception): download_model( - url='https://docarray.jina.ai/_static/favicon.png', + url='https://clip-as-service.jina.ai/_static/favicon.png', target_folder=tmpdir, md5sum='', with_resume=False, @@ -87,7 +87,7 @@ def test_make_onnx_flow_wrong_name_path(): 'image_uri', [ f'{os.path.dirname(os.path.abspath(__file__))}/img/00000.jpg', - 'https://docarray.jina.ai/_static/favicon.png', + 'https://clip-as-service.jina.ai/_static/favicon.png', ], ) @pytest.mark.parametrize('size', [224, 288, 384, 448]) diff --git a/tests/test_simple.py b/tests/test_simple.py index 8611395f4..8dab51322 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -35,7 +35,7 @@ def test_protocols(port_generator, protocol, jit, pytestconfig): ('hello, world', 'goodbye, world'), lambda: ('hello, world' for _ in range(10)), [ - 'https://docarray.jina.ai/_static/favicon.png', + 'https://clip-as-service.jina.ai/_static/favicon.png', f'{os.path.dirname(os.path.abspath(__file__))}/img/00000.jpg', 'hello, world', ], @@ -58,7 +58,7 @@ def test_plain_inputs(make_flow, inputs): lambda: (Document(text='hello, world') for _ in range(10)), DocumentArray( [ - Document(uri='https://docarray.jina.ai/_static/favicon.png'), + Document(uri='https://clip-as-service.jina.ai/_static/favicon.png'), Document( uri=f'{os.path.dirname(os.path.abspath(__file__))}/img/00000.jpg' ), @@ -90,7 +90,7 @@ def test_docarray_inputs(make_flow, inputs): DocumentArray( [ Document( - uri='https://docarray.jina.ai/_static/favicon.png', + uri='https://clip-as-service.jina.ai/_static/favicon.png', text='hello, world', ), ] @@ -117,7 +117,7 @@ def test_docarray_preserve_original_inputs(make_flow, inputs): DocumentArray( [ Document( - uri='https://docarray.jina.ai/_static/favicon.png', + uri='https://clip-as-service.jina.ai/_static/favicon.png', text='hello, world', ), ] diff --git a/tests/test_tensorrt.py b/tests/test_tensorrt.py index 7752073bb..395413f0e 100644 --- a/tests/test_tensorrt.py +++ b/tests/test_tensorrt.py @@ -17,7 +17,7 @@ lambda: (Document(text='hello, world') for _ in range(10)), DocumentArray( [ - Document(uri='https://docarray.jina.ai/_static/favicon.png'), + Document(uri='https://clip-as-service.jina.ai/_static/favicon.png'), Document( uri=f'{os.path.dirname(os.path.abspath(__file__))}/img/00000.jpg' ), @@ -47,13 +47,13 @@ def test_docarray_inputs(make_trt_flow, inputs): 'd', [ Document( - uri='https://docarray.jina.ai/_static/favicon.png', + uri='https://clip-as-service.jina.ai/_static/favicon.png', matches=[Document(text='hello, world'), Document(text='goodbye, world')], ), Document( text='hello, world', matches=[ - Document(uri='https://docarray.jina.ai/_static/favicon.png'), + Document(uri='https://clip-as-service.jina.ai/_static/favicon.png'), Document( uri=f'{os.path.dirname(os.path.abspath(__file__))}/img/00000.jpg' ),