daily_run_test #4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: daily_run_test | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '56 16 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CONDA_ENV: opencompass_regression | |
| PIP_CACHE_PATH: /cpfs01/user/qa-llm-cicd/.cache/pip | |
| USERSPACE_PREFIX: /cpfs01/user/qa-llm-cicd | |
| HF_CACHE_PATH: /cpfs01/shared/public/public_hdd/llmeval/model_weights/hf_hub | |
| DATEASET_CACHE_PATH: /cpfs01/shared/public/public_hdd/llmeval/llm-evaluation-datasets | |
| HF_DATASETS_OFFLINE: 1 | |
| TRANSFORMERS_OFFLINE: 1 | |
| HF_HUB_OFFLINE: 1 | |
| jobs: | |
| daily_run_test: | |
| runs-on: self-hosted | |
| environment: 'prod' | |
| timeout-minutes: 240 #4hours | |
| steps: | |
| - name: Clone repository | |
| uses: actions/checkout@v2 | |
| - name: Prepare - create conda env and install torch | |
| run: | | |
| eval "$(conda shell.bash hook)" | |
| conda create -y --name ${{env.CONDA_ENV}} python=3.10 | |
| conda activate ${{env.CONDA_ENV}} | |
| pip install /cpfs01/user/qa-llm-cicd/packages/lmdeploy-0.5.0+cu118-cp310-cp310-manylinux2014_x86_64.whl --cache-dir ${{env.PIP_CACHE_PATH}} --index-url https://download.pytorch.org/whl/cu118 | |
| pip install /cpfs01/user/qa-llm-cicd/packages/vllm-0.5.2+cu118-cp310-cp310-manylinux1_x86_64.whl --cache-dir ${{env.PIP_CACHE_PATH}} | |
| FLASH_ATTENTION_FORCE_BUILD=TRUE pip install /cpfs01/user/qa-llm-cicd/packages/flash_attn-2.5.8+cu118torch2.2cxx11abiFALSE-cp310-cp310-linux_x86_64.whl --index-url https://download.pytorch.org/whl/cu118 | |
| pip install torch==2.2.2 torchvision==0.17.2 torchaudio==2.2.2 --cache-dir ${{env.PIP_CACHE_PATH}} --index-url https://download.pytorch.org/whl/cu118 | |
| pip install xformers==0.0.25.post1 --cache-dir ${{env.PIP_CACHE_PATH}} --extra-index-url https://download.pytorch.org/whl/cu118 | |
| conda info --envs | |
| - name: Prepare - Pip install code | |
| run: | | |
| eval "$(conda shell.bash hook)" | |
| conda activate ${{env.CONDA_ENV}} | |
| pip install -e . --cache-dir ${{env.PIP_CACHE_PATH}} | |
| pip install human_eval transformers protobuf pytest --cache-dir ${{env.PIP_CACHE_PATH}} | |
| conda info --envs | |
| - name: Prepare - prepare data and hf model | |
| run: | | |
| ln -s ${{env.DATEASET_CACHE_PATH}} data | |
| rm -rf ~/.cache/huggingface/hub -f && mkdir ~/.cache -p && mkdir ~/.cache/huggingface -p | |
| ln -s ${{env.HF_CACHE_PATH}} ~/.cache/huggingface/hub | |
| - name: Run chat model test | |
| run: | | |
| eval "$(conda shell.bash hook)" | |
| conda activate ${{env.CONDA_ENV}} | |
| conda info --envs | |
| python3 run.py .github/scripts/eval_regression_chat.py --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/chat --reuse | |
| rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/chat/*/summary regression_result_daily | |
| python -m pytest -m chat -s -v --color=yes .github/scripts/oc_score_assert.py | |
| - name: Run base model test | |
| run: | | |
| eval "$(conda shell.bash hook)" | |
| conda activate ${{env.CONDA_ENV}} | |
| conda info --envs | |
| python3 run.py .github/scripts/eval_regression_base.py --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/base --reuse | |
| rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/base/*/summary regression_result_daily | |
| python -m pytest -m base -s -v --color=yes .github/scripts/oc_score_assert.py | |
| - name: Run command testcase | |
| run: | | |
| eval "$(conda shell.bash hook)" | |
| conda activate ${{env.CONDA_ENV}} | |
| conda info --envs | |
| export from_tf=TRUE | |
| python tools/list_configs.py internlm2_5 mmlu | |
| python run.py --models hf_internlm2_5_7b --datasets race_ppl --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd1 --reuse | |
| rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd1/*/summary regression_result_daily | |
| python -m pytest -m case1 -s -v --color=yes .github/scripts/oc_score_assert.py | |
| python run.py --models hf_internlm2_5_7b_chat --datasets race_gen -a lmdeploy --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd2 --reuse | |
| rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd2/*/summary regression_result_daily | |
| python -m pytest -m case2 -s -v --color=yes .github/scripts/oc_score_assert.py | |
| python run.py --datasets race_ppl --hf-type base --hf-path internlm/internlm2_5-7b --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd3 --reuse | |
| rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd3/*/summary regression_result_daily | |
| python -m pytest -m case3 -s -v --color=yes .github/scripts/oc_score_assert.py | |
| python run.py --datasets race_gen --hf-type chat --hf-path internlm/internlm2_5-7b-chat --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd4 --reuse | |
| rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd4/*/summary regression_result_daily | |
| python -m pytest -m case4 -s -v --color=yes .github/scripts/oc_score_assert.py | |
| - name: Remove Conda Env | |
| if: always() | |
| run: | | |
| rm -rf regression_result_daily | |
| eval "$(conda shell.bash hook)" | |
| conda env remove -y --name ${{env.CONDA_ENV}} | |
| conda info --envs | |
| notify_to_feishu: | |
| if: ${{ always() && !cancelled() && contains(needs.*.result, 'failure') && (github.ref_name == 'develop' || github.ref_name == 'main') }} | |
| needs: [daily_run_test] | |
| environment: 'prod' | |
| timeout-minutes: 5 | |
| runs-on: self-hosted | |
| steps: | |
| - name: notify | |
| run: | | |
| curl -X POST -H "Content-Type: application/json" -d '{"msg_type":"post","content":{"post":{"zh_cn":{"title":"Opencompass- Daily test failed","content":[[{"tag":"text","text":"branch: ${{github.ref_name}}, run action: ${{github.workflow}} failed. "},{"tag":"a","text":"Please click here for details ","href":"https://github.com/'${{ github.repository }}'/actions/runs/'${GITHUB_RUN_ID}'"},{"tag":"at","user_id":"'${{ secrets.USER_ID }}'"}]]}}}}' ${{ secrets.WEBHOOK_URL }} |