Updated Migration Scripts (#50)

* reimplement README.yaml modifications in python

* fix dropped comment

* log PRs

* incremental improvements

* refactor to use script instead of git-xargs

* update readme

* add new migration for github settings

Makefile

@@ -1,21 +1,7 @@
-export DOCKER_ORG ?= cloudposse
-export DOCKER_IMAGE ?= $(DOCKER_ORG)/terraform-root-modules
-export DOCKER_TAG ?= latest
-export DOCKER_IMAGE_NAME ?= $(DOCKER_IMAGE):$(DOCKER_TAG)
-export DOCKER_BUILD_FLAGS = 
-export README_DEPS ?= docs/targets.md docs/terraform.md
+export README_DEPS ?= docs/targets.md
 -include $(shell curl -sSL -o .build-harness "https://cloudposse.tools/build-harness"; echo .build-harness)
 
 all: init deps build install run
 
 deps:
 	@exit 0
-
-build:
-	@make --no-print-directory docker:build
-
-push:
-	docker push $(DOCKER_IMAGE)
-
-run:
-	docker run -it ${DOCKER_IMAGE_NAME} sh

migrate/Makefile

@@ -1,4 +1,3 @@
 deps:
 	brew install gh yq
 	pip3 install yamlfix
-

migrate/git-xargs.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+
+# Due to just endless problems working around `git-xargs` limitations, challenges with rate limits,
+# resorting to a basic shell script that does the same thing, but with more control and less magic and no parallelism.
+set -e
+
+migration=$1
+batch=$2
+
+# Validate usage
+if [[ -z "$migration" || -z "$batch" ]]; then
+    echo "Usage: $0 <migration_branch> <batch>"
+    exit 1
+fi
+curdir=$(pwd)
+migration_branch="migration/${migration}"
+migration_dir="migrations/${migration}"
+repos="${migration_dir}/${batch}"
+
+# Test if the repo file exists
+if [[ ! -f "$repos" ]]; then
+  echo "Repo file for ${batch} batch does not exist: $repos"
+  exit 1
+fi
+
+if [[ ! -d "migrations/${migration}" ]]; then
+  echo "Migration does not exist: $migration"
+  exit 1
+fi
+
+while IFS= read -r repo; do
+  if [[ $repo == \#* ]]; then
+    continue
+  fi
+  cd "${curdir}" 
+  repo_name=$(basename "$repo" .git)
+  repo_owner=$(basename "$(dirname "$repo")")
+  repo_url="[email protected]:${repo_owner}/${repo_name}.git"
+  tmp_dir="tmp/$repo_owner/$repo_name"
+
+  echo "Processing $repo_url"
+  if [[ ! -d "$tmp_dir" ]]; then
+    mkdir -p $(dirname "$tmp_dir")
+    git clone "$repo_url" "${tmp_dir}"
+  fi
+
+  cd "${tmp_dir}" || exit
+
+  # Identify the default branch
+  default_branch=$(git remote show origin | grep 'HEAD branch' | cut -d' ' -f5)
+
+  git checkout "${default_branch}"
+  # Ensure our branch is reset to match the remote default branch, so that we can apply the migration cleanly
+  git reset --hard origin/${default_branch}
+  git pull origin "${default_branch}"
+
+  if git rev-parse --verify "${migration_branch}" >/dev/null 2>&1; then
+    echo "Deleting existing ${migration_branch} branch"
+    git branch -D "${migration_branch}"
+  fi
+  echo "Creating ${migration_branch} branch"
+  git checkout -b "${migration_branch}"
+
+  # Always start clean
+  git clean -fxd
+
+  export XARGS_DRY_RUN=${XARGS_DRY_RUN:-true}
+  export XARGS_REPO_NAME="${repo_name}"
+  export XARGS_REPO_OWNER="${repo_owner}"
+  export MIGRATE_PATH="${curdir}"
+  echo "Performing migration..."
+  $curdir/run.sh "${migration}"
+  if [[ $? -ne 0 ]]; then
+    echo "Migration failed for ${repo}"
+    exit 1
+  fi
+
+done < "${repos}"

migrate/lib/common.sh

@@ -3,14 +3,47 @@ function title() {
 }
 
 function error() {
-    printf "“❌ \e[31mError: %s\e[0m\n" "$*"
+    printf "“❌ \e[31mError: %s\e[0m\n" "$*" >&2
     exit 1
 }
 
 function info() {
-    printf "✅︎ %s\n" "$*"
+    printf "✅︎ %s\n" "$*" >&2
 }
 
+function gh() {
+  while true; do
+    # Check the GitHub API rate limit
+    rate_limit=$(command gh api rate_limit | jq '.resources.core.remaining')
+
+    # Print the current rate limit
+    info "GitHub API rate limit: $rate_limit remaining"
+
+    # If the rate limit is sufficient, break the loop
+    if (( rate_limit > 10 )); then
+      break
+    fi
+
+    # Sleep for 60 seconds before checking the rate limit again
+    info "Rate limit too low, sleeping for 60 seconds..."
+    sleep 60
+  done
+
+  # Call the gh command with the provided arguments
+  command gh "$@"
+  exit_code=$?
+  # Generate a random number between 500 and 3000 (representing milliseconds)
+  random_milliseconds=$(( RANDOM % 2500 + 500 ))
+
+  # Convert milliseconds to seconds
+  random_seconds=$(echo "scale=3; $random_milliseconds / 1000" | bc)
+
+  # Sleep for the random amount of time
+  sleep $random_seconds
+  return $exit_code
+}
+
+
 function install() {
     local source=${1}
     local destination=${2:-$source}
@@ -66,11 +99,11 @@ function create_labels() {
 
     IFS=',' read -ra required_labels <<< "$LABELS"
 
-    local existing_labels=$(gh label list)
+    local existing_labels=$(gh label list --json name  --jq '.[].name')
 
     for label in "${required_labels[@]}"; do
         if ! echo "$existing_labels" | grep -q "$label"; then
-            gh label create $label -c '#b60205'
+            gh label create "$label" -c '#b60205'
             info "Created label [$label]"
         fi
     done

migrate/lib/github-settings.sh

@@ -16,13 +16,25 @@ function install_github_settings() {
         info "Creating $settings"
         echo "_extends: .github" > $settings
     fi
+    # Fetch the current name and description of the repo and update the settings file
+    local repo_description=$(gh repo view --json description --jq '.description')
+    local repo_homepage=$(gh repo view --json homepageUrl --jq '.homepageUrl')
+    local repo_topics=$(gh api repos/{owner}/{repo}/topics --jq '.names | join(", ")')
+
+    yq -ei ".repository.name = \"$XARGS_REPO_NAME\"" $settings
+    yq -ei ".repository.description = \"$repo_description\"" $settings
+    if [ -z "$repo_homepage" ]; then
+        yq -ei ".repository.homepage = \"https://cloudposse.com/accelerate\"" $settings
+    else
+        yq -ei ".repository.homepage = \"$repo_homepage\"" $settings
+    fi
+
+    yq -ei ".repository.topics = \"$repo_topics\"" $settings
 
     # finally, let's sort the file so _extends is at the top.
     yq -ei 'sort_keys(.)' $settings
-
-    # Format the YAML for humans
-    yamlfix -c ${MIGRATE_PATH}/yamlfix.yml $settings
-
+    sed -i '' '/# Upstream changes/d' $settings
+    yq -ei '(._extends | key) head_comment="Upstream changes from _extends are only recognized when modifications are made to this file in the default branch."' $settings
+
     git add $settings
-
 }

migrate/lib/github.sh

@@ -0,0 +1,5 @@
+
+function delete_branch_protection() {
+    local default_branch=$(git remote show origin | grep 'HEAD branch' | cut -d' ' -f5)
+    gh api -X DELETE /repos/${XARGS_REPO_OWNER}/${XARGS_REPO_NAME}/branches/${default_branch}/protection
+}

migrate/lib/mergify.sh

@@ -4,6 +4,19 @@ function install_mergify() {
     local config=".github/mergify.yml"
 
     mkdir -p $(dirname $config)
+
+    case "${REPO_TYPE}" in
+        "terraform-provider")
+            rm -f $config
+            ;;
+        "terraform-module")
+            rm -f $config
+            ;;
+        "github-action")
+            rm -f $config
+            ;;
+    esac
+
     if [ -f $config ]; then
         info "Mergify config already installed"
         # Ensure it's always extending from the .github repo