From e3b88e414a2895c963fb6a2f3ccf6a6cc7095e8d Mon Sep 17 00:00:00 2001 From: Abdelhalim Khouas <42040066+abdelhalimkhouas@users.noreply.github.com> Date: Wed, 4 Sep 2024 08:01:00 +0300 Subject: [PATCH] [Feature] [WRS-1849] Publish connectors to DEV Connector Hub storage (#49) * update pr merge action * add publish test action * change version for testing * use old version * fix changed connectors step * test change 1 * fix action * test change 2 * change fetch-depth * test change 3 * fix condition * test change 4 * ignore version bump commit * test change 5 * use correct path to copy from * test change 6 * list containers * test change 7 * change connection string * test change 8 * add version validation * use separate step to validate version * fix workflow file * fix workflow file * log versions * remove log * test pr validation * re-order steps * test validation * test validation 2 * remove change * update publish script to run only on changed connector * update workflow to handle single connector change * test 12 * use correct github env variable * test 13 * use correct connection string * test 14 * fix script * test 15 * update pr merge action * remove test action * remove test comment * make supportedAuth required * remove change * skip build and publish if no connector was changed --- .github/workflows/pr-merge.yml | 35 ++++++++++++++++--- .github/workflows/pr-validation.yml | 25 +++++++++++++ scripts/publish.js | 13 ++++--- .../resources/schemas/connector-config.json | 2 +- src/connector-cli/src/core/types/gen-types.ts | 12 +++---- 5 files changed, 71 insertions(+), 16 deletions(-) diff --git a/.github/workflows/pr-merge.yml b/.github/workflows/pr-merge.yml index 98b97f4..32ab30c 100644 --- a/.github/workflows/pr-merge.yml +++ b/.github/workflows/pr-merge.yml @@ -15,21 +15,25 @@ jobs: node-version: [20] steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} registry-url: 'https://npm.pkg.github.com' scope: '@chili-publish' + - name: Install dependencies run: yarn env: NODE_AUTH_TOKEN: ${{ secrets.PACKAGE_SECRET }} + - name: Download index.json uses: azure/CLI@v1 with: inlineScript: | - container_name="sdk" + container_name="connector-marketplace" blob_name="connector-repo/index.json" destination_file="existing/index.json" connection_string="${{ secrets.AZURE_CDN_STUDIO_DEV_CONNECTION_STRING }}" @@ -46,19 +50,42 @@ jobs: else echo "Blob does not exist." fi + + - name: Determine changed connectors + run: | + if git rev-parse --verify HEAD^ >/dev/null 2>&1; then + echo "Diffing HEAD^ HEAD" + CHANGED_CONNECTOR=$(git diff --name-only HEAD^ HEAD | grep '^src/connectors/' | cut -d/ -f3 | sort | uniq) + else + echo "Diffing HEAD" + CHANGED_CONNECTOR=$(git diff --name-only HEAD | grep '^src/connectors/' | cut -d/ -f3 | sort | uniq) + fi + echo "Changed directories: $CHANGED_CONNECTOR" + echo "CHANGED_CONNECTOR=$CHANGED_CONNECTOR" >> $GITHUB_ENV + - name: Build Cli run: yarn run build-cli + - name: Build and Publish connectors - run: node scripts/publish.js - - name: copy to upload folder + if: env.CHANGED_CONNECTOR != '' + run: | + echo "Building and publishing $CHANGED_CONNECTOR" + node scripts/publish.js $CHANGED_CONNECTOR + + - name: Copy to upload folder + if: env.CHANGED_CONNECTOR != '' run: mkdir -p upload/connector-repo && cp -r publish/* upload/connector-repo + - name: Upload artifacts + if: env.CHANGED_CONNECTOR != '' uses: actions/upload-artifact@v2 with: name: connectors path: upload/connector-repo + - name: Upload to Azure + if: env.CHANGED_CONNECTOR != '' uses: azure/CLI@v1 with: inlineScript: | - az storage blob upload-batch -d sdk -s upload/ --connection-string "${{ secrets.AZURE_CDN_STUDIO_DEV_CONNECTION_STRING }}" --overwrite true + az storage blob upload-batch -d connector-marketplace -s upload/ --connection-string "${{ secrets.AZURE_ST_MARKETPLACE_DEV_CONNECTION_STRING }}" --overwrite true diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 8334099..d9d6766 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -11,6 +11,8 @@ jobs: steps: - name: Check out code uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Get changed files id: files @@ -40,6 +42,29 @@ jobs: fi done echo "::set-output name=connector::$connector_changed" + + - name: Ensure connector version is bumped + run: | + connector_changed="" + for changed_file in ${{ steps.files.outputs.all }} + do + # Extract the first tree segments of the path (e.g., src/connectors/connector-name) + connector_path=$(echo "$changed_file" | cut -d'/' -f1-3) + if [[ "$connector_path" == src/connectors/* ]] + then + # Extract the connector name (e.g., connector-name) + connector_name=$(echo "$connector_path" | cut -d'/' -f3) + if [[ -z "$connector_changed" ]] + then + MAIN_VERSION=$(git show origin/main:src/connectors/$connector_name/package.json | jq -r .version) + PR_VERSION=$(jq -r .version src/connectors/$connector_name/package.json) + if [ "$MAIN_VERSION" = "$PR_VERSION" ]; then + echo "Error: The version in src/connectors/$connector_name/package.json is the same as in the main branch. Please bump the version" + exit 1 + fi + fi + fi + done - name: Add labels uses: actions-ecosystem/action-add-labels@v1 diff --git a/scripts/publish.js b/scripts/publish.js index 88243bf..95e5070 100644 --- a/scripts/publish.js +++ b/scripts/publish.js @@ -15,14 +15,15 @@ const {execSync} = require('child_process'); const repoRoot = findRepoRoot(__dirname); console.log(`Found repo root at ${repoRoot}`); -const connectorsDir = path.join(repoRoot, 'src', 'connectors'); -console.log(`Found connectors at ${connectorsDir}`); -fs.readdirSync(connectorsDir).forEach(file => { - const dirPath = path.join(connectorsDir, file); + +const connectorsToProcess = process.argv.slice(2); +connectorsToProcess.forEach(file => { + const dirPath = path.join(repoRoot, 'src', 'connectors', file); console.log(`Checking ${dirPath}`); + if (fs.statSync(dirPath).isDirectory()) { console.log(`Processing ${dirPath}`); - var connectorJson = processConnector(dirPath); + const connectorJson = processConnector(dirPath); // Write the JSON object to /publish/.json const publishDir = path.join(repoRoot, 'publish'); if (!fs.existsSync(publishDir)) { @@ -33,6 +34,8 @@ fs.readdirSync(connectorsDir).forEach(file => { JSON.stringify(connectorJson, null, 2), ); console.log(`Successfully processed ${connectorJson.name}`); + } else { + console.log(`${dirPath} is not a directory. Skipping.`); } }); diff --git a/src/connector-cli/resources/schemas/connector-config.json b/src/connector-cli/resources/schemas/connector-config.json index 8799951..66c16ee 100644 --- a/src/connector-cli/resources/schemas/connector-config.json +++ b/src/connector-cli/resources/schemas/connector-config.json @@ -30,5 +30,5 @@ } }, "additionalProperties": false, - "required": ["type", "options"] + "required": ["type", "options", "supportedAuth"] } diff --git a/src/connector-cli/src/core/types/gen-types.ts b/src/connector-cli/src/core/types/gen-types.ts index b2a9cb0..4aeba13 100644 --- a/src/connector-cli/src/core/types/gen-types.ts +++ b/src/connector-cli/src/core/types/gen-types.ts @@ -72,11 +72,11 @@ export interface StaticKey { } export interface ConnectorConfig { - iconUrl?: string; - mappings?: { [key: string]: any }; - options: { [key: string]: any }; - supportedAuth?: SupportedAuth[]; - type: Type; + iconUrl?: string; + mappings?: { [key: string]: any }; + options: { [key: string]: any }; + supportedAuth: SupportedAuth[]; + type: Type; } export enum SupportedAuth { @@ -342,7 +342,7 @@ const typeMap: any = { { json: "iconUrl", js: "iconUrl", typ: u(undefined, "") }, { json: "mappings", js: "mappings", typ: u(undefined, m("any")) }, { json: "options", js: "options", typ: m("any") }, - { json: "supportedAuth", js: "supportedAuth", typ: u(undefined, a(r("SupportedAuth"))) }, + { json: "supportedAuth", js: "supportedAuth", typ: a(r("SupportedAuth")) }, { json: "type", js: "type", typ: r("Type") }, ], false), "TokenEndpointAuthMethodsSupported": [