Fix BOM issue Generate-DocIndex.ps1 (#41768)

Co-authored-by: Daniel Jurek <[email protected]>
Azure · Feb 5, 2024 · 37f2513 · 37f2513
1 parent 817c6cc
commit 37f2513
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 44 deletions.
diff --git a/eng/common/docgeneration/Generate-DocIndex.ps1 b/eng/common/docgeneration/Generate-DocIndex.ps1
@@ -10,50 +10,33 @@ Param (
 )
 . "${PSScriptRoot}\..\scripts\common.ps1"
 
-# Given the github io blob storage url and language regex,
-# the helper function will return a list of artifact names.
-function Get-BlobStorage-Artifacts($blobStorageUrl, $blobDirectoryRegex, $blobArtifactsReplacement) {
+# Fetch a list of "artifacts" from blob storage corresponding to the given
+# language (-storagePrefix). Remove the prefix from the path names to arrive at
+# an "artifact" name.
+function Get-BlobStorage-Artifacts(
+  $blobDirectoryRegex,
+  $blobArtifactsReplacement,
+  $storageAccountName,
+  $storageContainerName,
+  $storagePrefix
+) {
     LogDebug "Reading artifact from storage blob ..."
-    $returnedArtifacts = @()
-    $pageToken = ""
-    Do {
-      $resp = ""
-      if (!$pageToken) {
-        # First page call.
-        $resp = Invoke-RestMethod -Method Get -Uri $blobStorageUrl
-      }
-      else {
-        # Next page call
-        $blobStorageUrlPageToken = $blobStorageUrl + "&marker=$pageToken"
-        $resp = Invoke-RestMethod -Method Get -Uri $blobStorageUrlPageToken
-      }
-      # Convert to xml documents.
-      $xmlDoc = [xml](removeBomFromString $resp)
-      foreach ($elem in $xmlDoc.EnumerationResults.Blobs.BlobPrefix) {
-        # What service return like "dotnet/Azure.AI.Anomalydetector/", needs to fetch out "Azure.AI.Anomalydetector"
-        $artifact = $elem.Name -replace $blobDirectoryRegex, $blobArtifactsReplacement
-        $returnedArtifacts += $artifact
-      }
-      # Fetch page token
-      $pageToken = $xmlDoc.EnumerationResults.NextMarker
-    } while ($pageToken)
-    return $returnedArtifacts
-  }
-
-# The sequence of Bom bytes differs by different encoding.
-# The helper function here is only to strip the utf-8 encoding system as it is used by blob storage list api.
-# Return the original string if not in BOM utf-8 sequence.
-function RemoveBomFromString([string]$bomAwareString) {
-    if ($bomAwareString.length -le 3) {
-        return $bomAwareString
-    }
-    $bomPatternByteArray = [byte[]] (0xef, 0xbb, 0xbf)
-    # The default encoding for powershell is ISO-8859-1, so converting bytes with the encoding.
-    $bomAwareBytes = [Text.Encoding]::GetEncoding(28591).GetBytes($bomAwareString.Substring(0, 3))
-    if (@(Compare-Object $bomPatternByteArray $bomAwareBytes -SyncWindow 0).Length -eq 0) {
-        return $bomAwareString.Substring(3)
-    }
-    return $bomAwareString
+    # "--only-show-errors" suppresses warnings about the fact that the az CLI is not authenticated
+    # "--query '[].name'" returns a list of only blob names
+    # "--num-results *" handles pagination so the caller does not have to
+    $artifacts = az storage blob list `
+        --account-name $storageAccountName `
+        --container-name $storageContainerName `
+        --prefix $storagePrefix `
+        --delimiter / `
+        --only-show-errors `
+        --query '[].name' `
+        --num-results * | ConvertFrom-Json
+    LogDebug "Number of artifacts found: $($artifacts.Length)"
+
+    # example: "python/azure-storage-blob" -> "azure-storage-blob"
+    $artifacts = $artifacts.ForEach({ $_ -replace $blobDirectoryRegex, $blobArtifactsReplacement })
+    return $artifacts
 }
 
 function Get-TocMapping {

diff --git a/eng/common/pipelines/templates/jobs/docindex.yml b/eng/common/pipelines/templates/jobs/docindex.yml
@@ -1,7 +1,7 @@
 jobs:
   - job: CreateDocIndex
     pool:
-      vmImage: windows-2022
+      name: azsdk-pool-mms-win-2022-general
     steps:
       - task: UsePythonVersion@0
         displayName: 'Use Python 3.9'