From 34930281ebe88d719e209fbeaebc7512db3eb4b3 Mon Sep 17 00:00:00 2001 From: PeixuanZuo <94887879+PeixuanZuo@users.noreply.github.com> Date: Thu, 20 Oct 2022 12:08:57 +0800 Subject: [PATCH] [ROCm] Fix azcopy issue on ROCm ci pipeline (#13365) ### Description Use SAS Token to fix error` failed to perform copy command due to error: no SAS token or OAuth token is present and the resource is not public` Generate SAS Token of target data, add it into Key vault, and use it as Pipeline Variable. ### Motivation and Context Co-authored-by: peixuanzuo --- orttraining/tools/ci_test/download_azure_blob_archive.py | 8 +++++++- .../orttraining-linux-gpu-amd-e2e-test-ci-pipeline.yml | 7 ++++--- .../azure-pipelines/orttraining-pai-ci-pipeline.yml | 4 +++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/orttraining/tools/ci_test/download_azure_blob_archive.py b/orttraining/tools/ci_test/download_azure_blob_archive.py index 32743f63281df..6fa875a1d2373 100755 --- a/orttraining/tools/ci_test/download_azure_blob_archive.py +++ b/orttraining/tools/ci_test/download_azure_blob_archive.py @@ -57,7 +57,13 @@ def main(): with tempfile.TemporaryDirectory() as temp_dir, get_azcopy() as azcopy_path: archive_path = os.path.join(temp_dir, "archive.zip") print("Downloading archive from '{}'...".format(args.azure_blob_url)) - _download(azcopy_path, args.azure_blob_url, archive_path) + + azure_blob_url = args.azure_blob_url + azure_blob_sas_token = os.getenv("AZURE_BLOB_SAS_TOKEN", None) + if azure_blob_sas_token and azure_blob_sas_token != "": + azure_blob_url = azure_blob_url + "?" + azure_blob_sas_token + + _download(azcopy_path, azure_blob_url, archive_path) if args.archive_sha256_digest: _check_file_sha256_digest(archive_path, args.archive_sha256_digest) print("Extracting to '{}'...".format(args.target_dir)) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-amd-e2e-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-amd-e2e-test-ci-pipeline.yml index bc75073c13ca5..8207a336880ff 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-amd-e2e-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-amd-e2e-test-ci-pipeline.yml @@ -20,11 +20,12 @@ jobs: eval "$('/home/ciagent/conda/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" echo "Selecting GPU based on HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES" displayName: 'Initialize environment' - + # update these if the E2E test data changes - script: |- + export AZURE_BLOB_SAS_TOKEN="$(onnxruntimetestdata-storage-training-container-sas-token)" python orttraining/tools/ci_test/download_azure_blob_archive.py \ - --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \ + --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip \ --target_dir training_e2e_test_data \ --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9 displayName: 'Download onnxruntime_training_data.zip data' @@ -66,7 +67,7 @@ jobs: --gpu_sku MI100_32G displayName: 'Run C++ BERT-L performance test' condition: succeededOrFailed() # ensure all tests are run - + - script: |- python orttraining/tools/ci_test/run_convergence_test.py \ --binary_dir build/RelWithDebInfo \ diff --git a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml index 1e19afabb178b..3a4e13e5f3796 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml @@ -278,11 +278,13 @@ jobs: - task: CmdLine@2 inputs: script: |- + export AZURE_BLOB_SAS_TOKEN="$(onnxruntimetestdata-storage-training-container-sas-token)" python orttraining/tools/ci_test/download_azure_blob_archive.py \ - --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \ + --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip \ --target_dir training_e2e_test_data \ --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9 condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed + retryCountOnTaskFailure: 2 displayName: 'Download onnxruntime_training_data.zip data' - task: CmdLine@2