Update build scripts to build from any named docs fork (knative#242)

* make /community optional - default to knative repo * move webhook payload output to bottom of log * add logic to determine and use repo name of any fork * extract and use repo name from specified fork * Avoid Host key verification failed.
RichieEscarez · Mar 6, 2021 · 1554cdd · 1554cdd
1 parent 47c2f49
commit 1554cdd
Show file tree

Hide file tree

Showing 4 changed files with 97 additions and 42 deletions.
diff --git a/scripts/build.sh b/scripts/build.sh
@@ -27,14 +27,16 @@ set -e
 source scripts/docs-version-settings.sh
 # Use default repo and branch from docs-version-settings.sh
 BRANCH="$DEFAULTBRANCH"
-FORK="$DEFAULTFORK"
+FORK="$DEFAULTREPO"
+REPO="$DEFAULTORG"
 
 # Set build default values
 BUILDENVIRONMENT="production"
 BUILDALLRELEASES="true"
 BUILDSINGLEBRANCH="false"
 LOCALBUILD="false"
 PRBUILD="false"
+WEBHOOK="false"
 
 # Manually specify your fork and branch for all builds.
 #
@@ -55,8 +57,11 @@ while getopts f:b:a: arg; do
   case $arg in
     f)
       echo 'FORK:' "${OPTARG}"
-      # Set specified knative/docs repo fork
+      # The GitHub repo name of the knative/docs fork to builb.
+      # Example: myrepo/forkname
       FORK="${OPTARG}"
+      # Extract the repo name
+      REPO=$(echo "$FORK" | sed -e 's/\.*\/.*//')
       ;;
     b)
       echo 'BRANCH:' "${OPTARG}"
@@ -67,9 +72,9 @@ while getopts f:b:a: arg; do
       echo 'BUILDING ALL RELEASES'
       # True by default. If set to "false" , the build does not clone nor build
       # the docs releases from other branches.
-      # REQUIRED: If you specify a fork ($FORK), all of the same branches 
+      # REQUIRED: If you specify a fork ($FORK), all of the same branches
       # (with the same branch names) that are built in knative.dev must
-      # also exist and be available in that $FORK (ie, 'release-0.X'). 
+      # also exist and be available in that $FORK (ie, 'release-0.X').
       # See /config/production/params.toml for the list of the branches
       # their names that are currently built in knative.dev.
       BUILDALLRELEASES="${OPTARG}"
@@ -80,10 +85,10 @@ done
 # If a webhook triggered the build, get repo fork and branch name
 if [ "$INCOMING_HOOK_BODY" ] || [ "$INCOMING_HOOK_TITLE" ] || [ "$INCOMING_HOOK_URL" ]
 then
+  WEBHOOK="true"
   echo '------ BUILD REQUEST FROM KNATIVE/DOCS WEBHOOK ------'
+
   echo 'Webhook Title:' "$INCOMING_HOOK_TITLE"
-  echo 'Webhook URL:' "$INCOMING_HOOK_URL"
-  echo 'Webhook Body:' "$INCOMING_HOOK_BODY"
 
   # If webhook is from a "PULL REQUEST" event
   if echo "$INCOMING_HOOK_BODY" | grep -q -m 1 '\"pull_request\"'
@@ -96,11 +101,13 @@ then
     PULL_REQUEST=$(echo "$INCOMING_HOOK_BODY" | grep -o -m 1 '\"number\"\:.*\,\"pull_request\"' | sed -e 's/\"number\"\://;s/\,\"pull_request\"//' || true)
     # Retrieve the fork and branch from PR webhook
     FORK_BRANCH=$(echo "$INCOMING_HOOK_BODY" | grep -o -m 1 '\"label\"\:\".*\"\,\"ref\"' | sed -e 's/\"label\"\:\"knative\:.*//;s/\"label\"\:\"//;s/\"\,\"ref\".*//' || true)
-    # Extract just the fork name
-    FORK=$(echo "$FORK_BRANCH" | sed -e 's/\:.*//')
+    # Extract just the repo name
+    REPO=$(echo "$FORK_BRANCH" | sed -e 's/\:.*//')
+    # Retrieve the repo fork name from PR webhook
+    FORK=$(echo "$INCOMING_HOOK_BODY" | grep -o -m 1 '\"full_name\"\:\".*\"\,\"private\"' | sed -e 's/\"full_name\"\:\"knative\/.*//;s/\"full_name\"\:\"//;s/\"\,\"private\".*//' || true)
     # If PR was merged, just run default build and deploy production site (www.knative.dev)
     MERGEDPR=$(echo "$INCOMING_HOOK_BODY" | grep -o '\"merged\"\:true\,' || : )
-    if [ "$MERGEDPR" = "true" ] 
+    if [ "$MERGEDPR" = "true" ]
     then
       # For merged PR, do not get branch name (use default: "latest knative release branch")
       echo '------ PR' "$PULL_REQUEST" 'MERGED ------'
@@ -114,7 +121,7 @@ then
   else
     # Webhook from "PUSH event"
     # If the event was from someone's fork, then get their branchname
-    if [ "$FORK" != "knative" ]
+    if [ "$REPO" != "knative" ]
     then
       BRANCH=$(echo "$INCOMING_HOOK_BODY" | grep -o -m 1 ':"refs\/heads\/.*\"\,\"before\"' | sed -e 's/.*:\"refs\/heads\///;s/\"\,\"before\".*//' || true)
       # Use "Staging" environment settings (config/staging)
@@ -131,15 +138,15 @@ echo 'Build environment:' "$BUILDENVIRONMENT"
 if [ "$PRBUILD" = "true" ]
 then
 # Builds only the content from the PR
-echo 'Building docs from PR#' "$PULL_REQUEST" 
+echo 'Building docs from PR#' "$PULL_REQUEST"
 else
 # The Netlify $PULL_REQUEST variable doesnt like the use of our multiple repos: Always returns false if we dont override it(see above)
 echo 'Pull Request:' "$PULL_REQUEST"
 fi
 # Only display these values when building other user's forks
-if [ "$FORK" != "knative" ]
+if [ "$REPO" != "knative" ]
 then
-echo 'Building From Fork:' "$FORK"
+echo 'Building From:' "$FORK"
 echo 'Using Branch:' "$BRANCH"
 fi
 echo 'Commit HEAD:' "$HEAD"
@@ -162,6 +169,14 @@ fi
 # Process the source files
 source scripts/processsourcefiles.sh
 
+# If from a WEBHOOK, show payload
+if [ "$WEBHOOK" = "true" ]
+then
+  echo '------ WEBHOOK DETAILS ------'
+  echo 'Webhook URL:' "$INCOMING_HOOK_URL"
+  echo 'Webhook Body:' "$INCOMING_HOOK_BODY"
+fi
+
 # BUILD MARKDOWN
 # Start HUGO build
 cd themes/docsy && git submodule update -f --init && cd ../.. && hugo --environment "$BUILDENVIRONMENT"

diff --git a/scripts/docs-version-settings.sh b/scripts/docs-version-settings.sh
@@ -16,4 +16,5 @@ NUMOFVERSIONS="3"
 OLDESTVERSION=$((LATESTVERSION-NUMOFVERSIONS))
 
 # An optional value that you can locally override for local builds/testing
-DEFAULTFORK="knative"
+DEFAULTORG="knative"
+DEFAULTREPO="$DEFAULTORG/docs"
diff --git a/scripts/localbuild.sh b/scripts/localbuild.sh
@@ -15,7 +15,7 @@ set -e
 #    2. Add it to your `PATH`. For example, add the following line to your `~/.bash_profile`:
 #      `PATH="/usr/local/opt/gnu-sed/libexec/gnubin:$PATH"`
 #
-# 3. Optional: Install PostCSS if you want to change the sites CSS and need to build those changes locally.
+# 3. Install PostCSS. Needed to build the site locally.
 #    https://www.docsy.dev/docs/getting-started/#install-postcss
 #
 # 4. Clone the knative/docs repo:
@@ -24,10 +24,18 @@ set -e
 # 5. Clone the knative/website repo, including the Docsy theme submodule:
 #    `git clone --recurse-submodules https://github.com/knative/website.git`
 #
-# 6. From the root of the knative/website clone, run:
+#     Note: These repos must be cloned into the same folder and use the same
+#     names ('docs', 'website', 'community')
+#
+# 6. Optional: Clone the knative/community repo.
+#    `git clone https://github.com/knative/community.git`
+#
+# 7. From the root of the knative/website clone, run:
 #    `scripts/localbuild.sh`
 #
-# 7. If you change content in your knative/docs repo clone, you rebuild your local
+#     See all command options below (ie. build from your remote fork, etc).
+#
+# 8. If you change content in your knative/docs repo clone, you rebuild your local
 #    site by stopping the localhost (CTRL C) and running `scripts/localbuild.sh` again.
 #
 # By default, the command locally runs a Hugo build of using your local knative/website and
@@ -44,13 +52,15 @@ set -e
 source scripts/docs-version-settings.sh
 # Use default repo and branch from docs-version-settings.sh
 BRANCH="$DEFAULTBRANCH"
-FORK="$DEFAULTFORK"
+FORK="$DEFAULTREPO"
+REPO="$DEFAULTORG"
 
 # Set local build default values
 BUILDENVIRONMENT="local"
 BUILDALLRELEASES="false"
 BUILDSINGLEBRANCH="false"
 PRBUILD="false"
+LOCALBUILD="true"
 
 # Default Hugo build options
 # disable Hugo server
@@ -66,7 +76,7 @@ LIVERELOAD=" --watch=false --disableLiveReload"
 #
 #     USAGE: Append the -f repofork and/or the -b branchname to the command.
 #            Example:
-#                    ./scripts/build.sh -f repofork -b branchname -s
+#                    ./scripts/build.sh -f repofork -b branchname -s true
 #
 # (2) Run a complete local build of the knative.dev site. Clones all the content
 #     from knative/docs repo, including all branches.
@@ -96,9 +106,9 @@ LIVERELOAD=" --watch=false --disableLiveReload"
 #
 #    - Build content from specified fork and branch:
 #      - Build any branch from your fork or from someones in a PR
-#      ./scripts/localbuild.sh -f REPOFORK -b BRANCHNAME
+#      ./scripts/localbuild.sh -f REPO/FORK -b BRANCHNAME
 #
-#    - Locally build a specific branch from $FORK ($DEFAULTFORK):
+#    - Locally build a specific branch from knative/docs:
 #      ./scripts/localbuild.sh -b BRANCHNAME
 #
 #    - Combine other -s or -a flags. Example:
@@ -110,10 +120,13 @@ while getopts "f:b:a:s:" arg; do
 	  echo '--- BUILDING FROM ---'
       echo 'FORK:' "${OPTARG}"
       # Build remote content locally
-      # Set the GitHub repo name of your knative/docs fork you want built.
+      # Set the GitHub repo name of your knative/docs fork that you want built.
+      # Example: myrepo/forkname
       FORK="${OPTARG}"
       # Retrieve content from remote repo
       BUILDSINGLEBRANCH="true"
+      # Extract the repo name
+      REPO=$(echo "$FORK" | sed -e 's/\.*\/.*//')
       ;;
     b)
       echo 'USING BRANCH:' "${OPTARG}"
@@ -124,7 +137,7 @@ while getopts "f:b:a:s:" arg; do
       BUILDSINGLEBRANCH="true"
       ;;
     a)
-      echo 'BUILDING ALL RELEASES FROM KNATIVE/DOCS'
+      echo 'BUILDING ALL RELEASES FROM' "$FORK"
       # If 'true', all knative/docs branches are built to mimic a
       # "production" build.
       # REQUIRED: If you specify a fork ($FORK), all of the same branches

diff --git a/scripts/processsourcefiles.sh b/scripts/processsourcefiles.sh
@@ -15,54 +15,56 @@ if [ "$BUILDALLRELEASES" = "true" ]
 then
 # PRODUCTION BUILD (ALL RELEASES)
 # Full build for knative.dev (config/production). Contributors can also use this for personal builds.
-  echo '------ BUILDING ALL DOC RELEASES ------'
+  echo '------ BUILDING ALL DOCS BRANCHES/RELEASES ------'
   # Build Knative docs from:
-  # - https://github.com/"$FORK"/docs
-  # - https://github.com/knative/community
+  # - https://github.com/"$FORK"
+  # - https://github.com/"$REPO"/community
 
-  # Build all branches (assumes $FORK contains all docs versions)
+  # Build all branches (assumes $FORK contains all docs branches)
   echo '------ Cloning Community and Pre-release docs (master) ------'
   # MASTER
-  echo 'Getting blog posts and community owned samples from knative/docs master branch'
-  git clone --quiet -b master https://github.com/"$FORK"/docs.git content/en
+  echo 'Getting blog posts and community owned samples from the master branch of' "$FORK"
+  git clone --quiet -b master https://github.com/"$FORK".git content/en
   echo 'Getting pre-release development docs from master branch'
   # Move "pre-release" docs content into the 'development' folder:
   mv content/en/docs content/en/development
   # DOCS BRANCHES
   echo '------ Cloning all docs releases ------'
-  # Get versions of released docs from their branches in "$FORK"/docs
+  # Get versions of released docs from their branches in "$FORK"
   # Versions are defined in website/scripts/docs-version-settings.sh
   # If this is a PR build, then build that content as the latest release (assume PR preview builds are always from "latest")
-  echo 'Getting the archived docs releases from branches in:' "$FORK"'/docs'
+  echo 'Getting the archived docs releases from branches in:' "$FORK"
   r=$OLDESTVERSION
   while [[ $r -le $LATESTVERSION ]]
   do
     CLONE="temp/release-0.${r}"
     echo 'Getting docs from: release-0.'"${r}"
-    git clone --quiet -b "release-0.${r}" "https://github.com/${FORK}/docs.git" "$CLONE"
+    git clone --quiet -b "release-0.${r}" "https://github.com/${FORK}.git" "$CLONE"
     if [ "$r" = "$LATESTVERSION" ]
     then
-      echo 'The /docs/ section is built from:' "$FORK"'/release-0.'"${r}"
+      echo 'The /docs/ section is built from the' '/release-0.'"${r}" 'branch of' "$FORK"
       mv "$CLONE"/docs content/en/docs
     else
       # Only use the "docs" folder from all branches/releases
       mv "$CLONE"/docs content/en/v0."$r"-docs
     fi
     (( r = r + 1 ))
   done
+  LOCALBUILD="false"
 
 elif [ "$BUILDSINGLEBRANCH" = "true" ]
 then
 # SINGLE REMOTE BRANCH BUILD
 # Build only the content from $FORK and $BRANCH
-  echo '------ BUILDING CONENT FROM REMOTE ------'
-  echo 'The /docs/ section is built from the' "$BRANCH" 'branch of' "$FORK"'/docs'
-  git clone --quiet -b "$BRANCH" https://github.com/"$FORK"/docs.git content/en
+  echo '------ BUILDING SINGLE REMOTE BRANCH ------'
+  echo 'The /docs/ section is built from the' "$BRANCH" 'branch of' "$FORK"
+  git clone --quiet -b "$BRANCH" https://github.com/"$FORK".git content/en
+  LOCALBUILD="false"
 else
 # DEFAULT: LOCAL BUILD
-# Assumes that knative/docs and knative/website are cloned to the same directory.
+# Assumes that knative/docs, /community, and /website are cloned to the same directory.
   LOCALBUILD="true"
-  echo '------ BUILDING ONLY FROM YOUR LOCAL KNATIVE/DOCS CLONE ------'
+  echo '------ BUILDING YOUR LOCAL KNATIVE/DOCS CLONE ------'
   echo 'Copying local clone of knative/docs into the /docs folder under:'
   pwd
   cp -r ../docs content/en/
@@ -74,12 +76,36 @@ else
   fi
 fi
 
+# Build knative/community
 if [ "$LOCALBUILD" = "false" ]
 then
-  echo '------ Cloning contributor docs ------'
-  # COMMUNITY
-  echo 'Getting Knative contributor guidelines from the master branch of' "$FORK"'/community'
-  git clone --quiet -b master https://github.com/"$FORK"/community.git temp/community
+  echo ' - ----- Cloning contributor docs ------'
+  echo 'Getting Knative community and contributor guidelines'
+  if ! [ "$BUILDENVIRONMENT" = "production" ]
+  then
+    # Try to build /community from remote or PR
+    # Check if $FORK includes the /community (remote and PR builds)
+    if ! (git ls-remote --quiet [email protected]:"$REPO"/community.git)
+    then
+      echo 'No /community fork found in' "$REPO"
+      DEFAULTCOMMUNITY="true"
+    else
+      # Build /community from $REPO fork
+      echo 'Building from the master branch of' "$REPO"'/community'
+      git clone --quiet -b master https://github.com/"$REPO"/community.git temp/community
+      DEFAULTCOMMUNITY="false"
+    fi
+  else
+    DEFAULTCOMMUNITY="true"
+  fi
+
+  # Build knative/community for production builds and by default
+  if [ "$DEFAULTCOMMUNITY" = "true" ]
+  then
+    echo 'Building from the master branch of' "$DEFAULTORG"'/community'
+    # Default to knative/community
+    git clone --quiet -b master https://github.com/"$DEFAULTORG"/community.git temp/community
+  fi
   # Move files into existing "contributing" folder
   mv temp/community/* content/en/community/contributing
 fi