[feat][CI] Add solution to reduce GitHub Actions resource consumption

To reduce resource consumption, PRs would require approval before all tests would be run.

PR authors would be required to run tests in their own forks to make sure that tests pass and
before the PR is approved. Alternatively Apache Pulsar committers can add a ready-to-test label
to the PR to make it eligible for testing.

This way PR authors would have CI feedback without causing resource shortage to apache/pulsar CI.

Please check the proposal email on dev mailing list: https://lists.apache.org/thread/gwfmxmxlhtsjn17sxxc367jcs4pcwofz

.github/PULL_REQUEST_TEMPLATE.md

@@ -76,3 +76,25 @@ This change added tests and can be verified as follows:
 
 - [ ] `doc-complete`
 (Docs have been already added)
+
+### Matching PR in forked repository
+
+PR in forked repository: <!-- ENTER URL HERE 
+
+After opening this PR, the build in apache/pulsar will fail and instructions will
+be provided for opening a PR in the PR author's forked repository.
+
+apache/pulsar pull requests should be first tested in your own fork since the 
+apache/pulsar CI based on GitHub Actions has constrained resources and quota.
+GitHub Actions provides separate quota for pull requests that are executed in 
+a forked repository.
+
+The tests will be run in the forked repository until all PR review comments have
+been handled, the tests pass and the PR is approved by a reviewer.
+
+-->
+
+
+
+
+

.github/workflows/ci-cpp-build.yaml

@@ -38,7 +38,6 @@ jobs:
     outputs:
       docs_only: ${{ needs.changed_files_job.outputs.docs_only }}
       cpp_only: ${{ needs.changed_files_job.outputs.cpp_only }}
-      changed_tests: ${{ steps.changes.outputs.tests_files }}
     steps:
       - name: checkout
         uses: actions/checkout@v2
@@ -54,6 +53,12 @@ jobs:
         id: check_changes
         run: |
           echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
+          echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}"
+
+      - name: Check if the PR has been approved for testing
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
+        run: |
+          build/pulsar_ci_tool.sh check_ready_to_test
 
   cpp-build-centos7:
     needs: changed_files_job

.github/workflows/ci-go-functions.yaml

@@ -41,7 +41,6 @@ jobs:
     outputs:
       docs_only: ${{ steps.check_changes.outputs.docs_only }}
       cpp_only: ${{ steps.check_changes.outputs.cpp_only }}
-      changed_tests: ${{ steps.changes.outputs.tests_files }}
     steps:
       - name: checkout
         uses: actions/checkout@v2
@@ -57,6 +56,12 @@ jobs:
         id: check_changes
         run: |
           echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
+          echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}"
+
+      - name: Check if the PR has been approved for testing
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
+        run: |
+          build/pulsar_ci_tool.sh check_ready_to_test
 
   check-style:
     needs: changed_files_job

.github/workflows/pulsar-ci-flaky.yaml

@@ -60,6 +60,11 @@ jobs:
           echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
           echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}"
 
+      - name: Check if the PR has been approved for testing
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
+        run: |
+          build/pulsar_ci_tool.sh check_ready_to_test
+
   build-and-test:
     needs: changed_files_job
     name: Flaky tests suite

.github/workflows/pulsar-ci.yaml

@@ -60,6 +60,11 @@ jobs:
           echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
           echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}"
 
+      - name: Check if the PR has been approved for testing
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
+        run: |
+          build/pulsar_ci_tool.sh check_ready_to_test
+
   build-and-license-check:
     needs: changed_files_job
     name: Build and License check

build/pulsar_ci_tool.sh

@@ -154,6 +154,59 @@ function ci_move_test_reports() {
   )
 }
 
+function ci_check_ready_to_test() {
+  if [[ -z "$GITHUB_EVENT_PATH" ]]; then
+    >&2 echo "GITHUB_EVENT_PATH isn't set"
+    return 1
+  fi
+  # check ready-to-test label
+  if ! yq eval -e '.pull_request.labels[] | .name | select(. == "ready-to-test")' "$GITHUB_EVENT_PATH" &> /dev/null; then
+    # check if the PR has been approved
+    PR_NUM=$(jq -r '.pull_request.number' "${GITHUB_EVENT_PATH}")
+    REPO_FULL_NAME=$(jq -r '.repository.full_name' "${GITHUB_EVENT_PATH}")
+    REPO_NAME=$(basename "${REPO_FULL_NAME}")
+    REPO_OWNER=$(dirname "${REPO_FULL_NAME}")
+    # use graphql query to find out reviewDecision
+    PR_REVIEW_DECISION=$(curl -s -H "Authorization: bearer $GITHUB_TOKEN" -X POST -d '{"query": "query { repository(name: \"'${REPO_NAME}'\", owner: \"'${REPO_OWNER}'\") { pullRequest(number: '${PR_NUM}') { reviewDecision } } }"}' https://api.github.com/graphql |jq -r '.data.repository.pullRequest.reviewDecision')
+    echo "Review decision for PR #${PR_NUM} in repository ${REPO_OWNER}/${REPO_NAME} is ${PR_REVIEW_DECISION}"
+    if [[ "$PR_REVIEW_DECISION" != "APPROVED" ]]; then
+      FORK_REPO_URL=$(jq -r '.pull_request.head.repo.html_url' "$GITHUB_EVENT_PATH")
+      PR_BRANCH_LABEL=$(jq -r '.pull_request.head.label' "$GITHUB_EVENT_PATH")
+      PR_URL=$(jq -r '.pull_request.html_url' "$GITHUB_EVENT_PATH")
+      >&2 tee -a "$GITHUB_STEP_SUMMARY" <<EOF
+
+# Instructions for proceeding with the pull request:
+
+apache/pulsar pull requests should be first tested in your own fork since the apache/pulsar CI based on
+GitHub Actions has constrained resources and quota. GitHub Actions provides separate quota for
+pull requests that are executed in a forked repository.
+
+1. Go to ${FORK_REPO_URL} and ensure that your branch is up to date with https://github.com/apache/pulsar
+   Sync your fork if it's behind.
+2. Open a pull request to your own fork. You can use this link to create the pull request in
+   your own fork:
+   ${FORK_REPO_URL}/compare/master...${PR_BRANCH_LABEL}?expand=1
+3. Edit the description of the pull request ${PR_URL} and add the link to the PR that you opened to your own fork
+   so that the reviewer can verify that tests pass in your own fork.
+4. Ensure that tests pass in your own fork. Your own fork will be used to run the tests during the PR review
+   and any changes made during the review. You as a PR author are responsible for following up on test failures.
+   Please report any flaky tests as new issues at https://github.com/apache/pulsar/issues
+   after checking that the flaky test isn't already reported.
+5. When the PR is approved, it will be possible to restart the Pulsar CI workflow within apache/pulsar
+   repository by adding a comment "/pulsarbot rerun-failure-checks" to the PR.
+   An alternative for the PR approval is to add a ready-to-test label to the PR. This can be done
+   by Apache Pulsar committers.
+6. When tests pass on the apache/pulsar side, the PR can be merged by a Apache Pulsar Committer.
+
+If you have any trouble, please join the #contributor channel on Pulsar Slack to get real-time support.
+Instructions for joining Pulsar Slack: https://pulsar.apache.org/community#section-discussions
+
+EOF
+      exit 1
+    fi
+  fi
+}
+
 if [ -z "$1" ]; then
   echo "usage: $0 [ci_tool_function_name]"
   echo "Available ci tool functions:"