Merge pull request #89 from weaviate/fix/ntlk-vulnerability #175
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Main | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - '**' | |
| paths-ignore: | |
| - README.md | |
| - LICENSE | |
| pull_request: | |
| jobs: | |
| build-image: | |
| name: Build image | |
| runs-on: ubuntu-latest-8-cores | |
| strategy: | |
| matrix: | |
| include: | |
| - model_name: distilbert-base-uncased | |
| model_tag_name: distilbert-base-uncased | |
| onnx_runtime: false | |
| - model_name: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 | |
| model_tag_name: sentence-transformers-paraphrase-multilingual-MiniLM-L12-v2 | |
| onnx_runtime: false | |
| - model_name: sentence-transformers/multi-qa-MiniLM-L6-cos-v1 | |
| model_tag_name: sentence-transformers-multi-qa-MiniLM-L6-cos-v1 | |
| onnx_runtime: false | |
| - model_name: sentence-transformers/multi-qa-mpnet-base-cos-v1 | |
| model_tag_name: sentence-transformers-multi-qa-mpnet-base-cos-v1 | |
| onnx_runtime: false | |
| - model_name: sentence-transformers/all-mpnet-base-v2 | |
| model_tag_name: sentence-transformers-all-mpnet-base-v2 | |
| onnx_runtime: false | |
| - model_name: sentence-transformers/all-MiniLM-L12-v2 | |
| model_tag_name: sentence-transformers-all-MiniLM-L12-v2 | |
| onnx_runtime: false | |
| - model_name: sentence-transformers/paraphrase-multilingual-mpnet-base-v2 | |
| model_tag_name: sentence-transformers-paraphrase-multilingual-mpnet-base-v2 | |
| onnx_runtime: false | |
| - model_name: sentence-transformers/all-MiniLM-L6-v2 | |
| model_tag_name: sentence-transformers-all-MiniLM-L6-v2 | |
| onnx_runtime: false | |
| - model_name: sentence-transformers/multi-qa-distilbert-cos-v1 | |
| model_tag_name: sentence-transformers-multi-qa-distilbert-cos-v1 | |
| onnx_runtime: false | |
| - model_name: sentence-transformers/gtr-t5-base | |
| model_tag_name: sentence-transformers-gtr-t5-base | |
| onnx_runtime: false | |
| - model_name: sentence-transformers/gtr-t5-large | |
| model_tag_name: sentence-transformers-gtr-t5-large | |
| onnx_runtime: false | |
| - model_name: sentence-transformers/sentence-t5-base | |
| model_tag_name: sentence-transformers-sentence-t5-base | |
| onnx_runtime: false | |
| - model_name: vblagoje/dpr-ctx_encoder-single-lfqa-wiki | |
| model_tag_name: vblagoje-dpr-ctx_encoder-single-lfqa-wiki | |
| onnx_runtime: false | |
| - model_name: vblagoje/dpr-question_encoder-single-lfqa-wiki | |
| model_tag_name: vblagoje-dpr-question_encoder-single-lfqa-wiki | |
| onnx_runtime: false | |
| - model_name: facebook/dpr-ctx_encoder-single-nq-base | |
| model_tag_name: facebook-dpr-ctx_encoder-single-nq-base | |
| onnx_runtime: false | |
| - model_name: facebook/dpr-question_encoder-single-nq-base | |
| model_tag_name: facebook-dpr-question_encoder-single-nq-base | |
| onnx_runtime: false | |
| - model_name: google/flan-t5-base | |
| model_tag_name: google-flan-t5-base | |
| onnx_runtime: false | |
| - model_name: google/flan-t5-large | |
| model_tag_name: google-flan-t5-large | |
| onnx_runtime: false | |
| - model_name: biu-nlp/abstract-sim-sentence | |
| model_tag_name: biu-nlp-abstract-sim-sentence | |
| onnx_runtime: false | |
| - model_name: biu-nlp/abstract-sim-query | |
| model_tag_name: biu-nlp-abstract-sim-query | |
| onnx_runtime: false | |
| - model_name: BAAI/bge-small-en-v1.5 | |
| model_tag_name: baai-bge-small-en-v1.5 | |
| onnx_runtime: false | |
| - model_name: BAAI/bge-base-en-v1.5 | |
| model_tag_name: baai-bge-base-en-v1.5 | |
| onnx_runtime: false | |
| - model_name: BAAI/bge-small-en-v1.5 | |
| model_tag_name: baai-bge-small-en-v1.5 | |
| onnx_runtime: true | |
| - model_name: BAAI/bge-base-en-v1.5 | |
| model_tag_name: baai-bge-base-en-v1.5 | |
| onnx_runtime: true | |
| - model_name: BAAI/bge-m3 | |
| model_tag_name: baai-bge-m3 | |
| onnx_runtime: true | |
| - model_name: sentence-transformers/all-MiniLM-L6-v2 | |
| model_tag_name: sentence-transformers-all-MiniLM-L6-v2 | |
| onnx_runtime: true | |
| - model_name: mixedbread-ai/mxbai-embed-large-v1 | |
| model_tag_name: mixedbread-ai-mxbai-embed-large-v1 | |
| onnx_runtime: false | |
| use_sentence_transformers_vectorizer: true | |
| - model_name: mixedbread-ai/mxbai-embed-large-v1 | |
| model_tag_name: mixedbread-ai-mxbai-embed-large-v1 | |
| onnx_runtime: true | |
| - model_name: Snowflake/snowflake-arctic-embed-xs | |
| model_tag_name: snowflake-snowflake-arctic-embed-xs | |
| onnx_runtime: false | |
| - model_name: Snowflake/snowflake-arctic-embed-xs | |
| model_tag_name: snowflake-snowflake-arctic-embed-xs | |
| onnx_runtime: true | |
| - model_name: Snowflake/snowflake-arctic-embed-s | |
| model_tag_name: snowflake-snowflake-arctic-embed-s | |
| onnx_runtime: false | |
| - model_name: Snowflake/snowflake-arctic-embed-s | |
| model_tag_name: snowflake-snowflake-arctic-embed-s | |
| onnx_runtime: true | |
| - model_name: Snowflake/snowflake-arctic-embed-m | |
| model_tag_name: snowflake-snowflake-arctic-embed-m | |
| onnx_runtime: false | |
| - model_name: Snowflake/snowflake-arctic-embed-m | |
| model_tag_name: snowflake-snowflake-arctic-embed-m | |
| onnx_runtime: true | |
| - model_name: Snowflake/snowflake-arctic-embed-l | |
| model_tag_name: snowflake-snowflake-arctic-embed-l | |
| onnx_runtime: false | |
| - model_name: Snowflake/snowflake-arctic-embed-l | |
| model_tag_name: snowflake-snowflake-arctic-embed-l | |
| onnx_runtime: true | |
| - model_name: Snowflake/snowflake-arctic-embed-m-v1.5 | |
| model_tag_name: snowflake-snowflake-arctic-embed-m-v1.5 | |
| onnx_runtime: false | |
| - model_name: Snowflake/snowflake-arctic-embed-m-v1.5 | |
| model_tag_name: snowflake-snowflake-arctic-embed-m-v1.5 | |
| onnx_runtime: true | |
| env: | |
| LOCAL_REPO: transformers-inference | |
| REMOTE_REPO: semitechnologies/transformers-inference | |
| MODEL_NAME: ${{matrix.model_name}} | |
| MODEL_TAG_NAME: ${{matrix.model_tag_name}} | |
| ONNX_RUNTIME: ${{matrix.onnx_runtime}} | |
| USE_SENTENCE_TRANSFORMERS_VECTORIZER: ${{matrix.use_sentence_transformers_vectorizer}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| cache: 'pip' # caching pip dependencies | |
| - name: Login to Docker Hub | |
| if: ${{ !github.event.pull_request.head.repo.fork }} # no PRs from fork | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{secrets.DOCKER_USERNAME}} | |
| password: ${{secrets.DOCKER_PASSWORD}} | |
| - name: Build and test | |
| run: | | |
| cicd/build.sh | |
| cicd/test.sh | |
| - name: Deploy | |
| if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') | |
| env: | |
| DOCKER_USERNAME: ${{secrets.DOCKER_USERNAME}} | |
| DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} | |
| run: | | |
| cicd/docker_push.sh | |
| build-custom-image: | |
| name: Build custom image | |
| runs-on: ubuntu-latest-8-cores | |
| env: | |
| LOCAL_REPO: transformers-inference | |
| REMOTE_REPO: semitechnologies/transformers-inference | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| - name: Login to Docker Hub | |
| if: ${{ !github.event.pull_request.head.repo.fork }} # no PRs from fork | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{secrets.DOCKER_USERNAME}} | |
| password: ${{secrets.DOCKER_PASSWORD}} | |
| - name: Build custom image | |
| env: | |
| DOCKER_USERNAME: ${{secrets.DOCKER_USERNAME}} | |
| DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} | |
| run: | | |
| cicd/build_custom_base.sh | |
| - name: Deploy | |
| if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') | |
| env: | |
| DOCKER_USERNAME: ${{secrets.DOCKER_USERNAME}} | |
| DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} | |
| run: | | |
| cicd/docker_push_custom_base.sh |