Support Yarn 2

CHANGELOG.md

@@ -4,7 +4,19 @@
 - update docs URL when node modules are checked into git ([#794](https://github.com/heroku/heroku-buildpack-nodejs/pull/794))
 - move disabling yarn caching YARN_CACHE env var ([#796](https://github.com/heroku/heroku-buildpack-nodejs/pull/796))
 - use meta_get "node-package-manager" for reporting ([#802](https://github.com/heroku/heroku-buildpack-nodejs/pull/802))
-- change YARN_CACHE to USE_YARN_CACHE env var ([#803](https://github.com/heroku/heroku-buildpack-nodejs/pull/803))
+- change YARN_CACHE to USE_YARN_CACHE env var ([#803](https://github.com/heroku/heroku-buildpack-nodejs/pull/ 803))
+- create a pull request to merge Yarn 2 support ([#795](https://github.com/heroku/heroku-buildpack-nodejs/pull/795))
+
+### feature: Yarn 2 support
+- Add warning for detection of Yarn 2 usage with caching ([#755](https://github.com/heroku/heroku-buildpack-nodejs/pull/755))
+- Add warning for Yarn 2 usage, warn user of ignoring engine from package.json ([#761](https://github.com/heroku/heroku-buildpack-nodejs/pull/761))
+- Add warning for if .npmrc file exists when using Yarn 2 ([#764](https://github.com/heroku/heroku-buildpack-nodejs/pull/764))
+- Add warning for if .yarnrc file exists when using Yarn 2 ([#768](https://github.com/heroku/heroku-buildpack-nodejs/pull/768))
+- Add checks for Yarn 2 ([#759](https://github.com/heroku/heroku-buildpack-nodejs/pull/759))
+- Turn off production and ignore-script flags for Yarn 2 and add global vendored yarn alias ([775](https://github.com/heroku/heroku-buildpack-nodejs/pull/775))
+- Set `$YARN` to false when `$YARN2` is true ([#776](https://github.com/heroku/heroku-buildpack-nodejs/pull/776))
+- Skip writing to the cache and pruning of the node modules when Yarn 2 is used ([#785](https://github.com/heroku/heroku-buildpack-nodejs/pull/785))
+- Test use of packages in `.yarn/release` directory ([#786](https://github.com/heroku/heroku-buildpack-nodejs/pull/786))
 
 ## v172 (2020-05-28)
 - display yarn engine log to build output when engine is provided in monorepo ([#771](https://github.com/heroku/heroku-buildpack-nodejs/pull/771))

README.md

@@ -107,3 +107,36 @@ make heroku-20
 The tests are run via the vendored
 [shunit2](https://github.com/kward/shunit2)
 test framework.
+
+### Debugging
+
+To display the logged build outputs to assist with debugging, use the "echo" and "cat" commands. For example:
+
+```sh
+test() {
+  local log_file var
+
+  var="testtest"
+  log_file=$(mktemp)
+  echo "this is the log file" > "$log_file"
+  echo "test log file" >> "$log_file"
+
+  # use `echo` and `cat` for printing variables and reading files respectively
+  echo $var
+  cat $log_file
+
+  # some cases when debugging is necessary
+  assertEquals "$var" "testtest"
+  assertFileContains "test log file" "$log_file"
+}
+```
+
+Running the test above would produce:
+
+```log
+testtest
+this is the log file
+test log file
+```
+
+The test output writes to `$STD_OUT`, so you can use `cat $STD_OUT` to read output.

bin/compile

@@ -97,6 +97,7 @@ meta_set "build-step" "init"
 [ -e "$BUILD_DIR/node_modules" ] && PREBUILD=true || PREBUILD=false
 [ -f "$BUILD_DIR/yarn.lock" ] && YARN=true || YARN=false
 [ -f "$BUILD_DIR/package-lock.json" ] && NPM_LOCK=true || NPM_LOCK=false
+YARN2=$(detect_yarn2 "$YARN" "$BUILD_DIR")
 
 ### Save build info
 features_init "nodejs" "$BUILD_DIR" "$CACHE_DIR" "$BP_DIR/features"
@@ -110,7 +111,6 @@ fail_dot_heroku_node "$BUILD_DIR"
 fail_invalid_package_json "$BUILD_DIR"
 fail_multiple_lockfiles "$BUILD_DIR"
 fail_iojs_unsupported "$BUILD_DIR"
-fail_yarn2_unsupported "$YARN" "$BUILD_DIR"
 warn_prebuilt_modules "$BUILD_DIR"
 warn_missing_package_json "$BUILD_DIR"
 
@@ -141,6 +141,38 @@ create_build_env
 [ ! "$NPM_CONFIG_CACHE" ] && NPM_CONFIG_CACHE=$(mktemp -d -t npmcache.XXXXX)
 export YARN_CACHE_FOLDER NPM_CONFIG_CACHE
 
+### Configure vendored package manager
+export YARN2_PATH
+
+if [[ "$YARN2" == "true" ]]; then
+  if [[ "$NODE_MODULES_CACHE" == "true" ]]; then
+    warn "
+      WARNING- You are using Yarn 2. The buildpack won't cache, because Yarn 2 does not provide node modules.
+      "
+  fi
+
+  if [[ -f "$BUILD_DIR/.npmrc" ]]; then
+    warn "There is an existing .npmrc file that will not be used. All configurations are read from the .yarnrc.yml file."
+  fi
+
+  if [[ -f "$BUILD_DIR/.yarnrc" ]]; then
+    warn "There is an existing .yarnrc file that will not be used. All configurations are read from the .yarnrc.yml file."
+  fi
+
+  ### Configure Yarn 2
+  # fail for no yarnrc.yml
+  fail_missing_yarnrc_yml "$BUILD_DIR"
+
+  # get yarn_path
+  YARN2_PATH=$(get_yarn_path "$BUILD_DIR")
+
+  # fail for no yarnPath in rc
+  fail_missing_yarn_path "$BUILD_DIR" "$YARN2_PATH"
+
+  # fail for missing yarn in .yarn/releases
+  fail_missing_yarn_vendor "$BUILD_DIR" "$YARN2_PATH"
+fi
+
 if [[ $(features_get "cache-native-yarn-cache") == "false" ]]; then
   warn "You're currently using the heroku-buildpack-features file to configure. Using Yarn Cache has moved behind an environment variable. Use 'heroku config:set USE_YARN_CACHE=false' to set the variable."
   USE_YARN_CACHE=false
@@ -153,6 +185,10 @@ install_bins() {
   npm_engine=$(read_json "$BUILD_DIR/package.json" ".engines.npm")
   yarn_engine=$(read_json "$BUILD_DIR/package.json" ".engines.yarn")
 
+  if [[ "$YARN2" == "true" && -n "$yarn_engine" ]]; then
+    warn "You should be using a vendored version of Yarn. Heroku will ignore the engine from the package.json."
+  fi
+
   meta_set "node-version-request" "$node_engine"
   meta_set "npm-version-request" "$npm_engine"
   meta_set "yarn-version-request" "$yarn_engine"
@@ -193,6 +229,10 @@ install_bins() {
   fi
 
   warn_old_npm
+
+  if [[ $YARN2 == true ]]; then
+    YARN=false
+  fi
 }
 
 header "Installing binaries" | output "$LOG_FILE"
@@ -253,8 +293,9 @@ build_dependencies() {
 
   cache_status="$(get_cache_status "$CACHE_DIR")"
   start=$(nowms)
-
-  if $YARN; then
+  if [[ "$YARN2" == "true" ]]; then
+    echo "Using Yarn 2, skipping installations."
+  elif $YARN; then
     yarn_node_modules "$BUILD_DIR"
   elif $PREBUILD; then
     echo "Prebuild detected (node_modules already exists)"
@@ -309,7 +350,11 @@ prune_devdependencies() {
 #   and in the new feature
 #   and we're using the default cache directories
 # then save off the cache after we prune out devDepenencies
-if [[ "$YARN" == "true" ]] && [[ "$USE_YARN_CACHE" == "true" ]] && [[ "$(get_cache_directories "$BUILD_DIR")" == "" ]]; then
+if [[ "$YARN2" == "true" ]]; then
+  echo "Using Yarn 2, writing to the cache and pruning of the node modules will be skipped." | output "$LOG_FILE"
+  meta_set "build-step" "prune-dependencies"
+  meta_set "build-step" "save-cache"
+elif [[ "$YARN" == "true" ]] && [[ "$USE_YARN_CACHE" == "true" ]] && [[ "$(get_cache_directories "$BUILD_DIR")" == "" ]]; then
   meta_set "build-step" "prune-dependencies"
   header "Pruning devDependencies" | output "$LOG_FILE"
   prune_devdependencies | output "$LOG_FILE"

lib/binaries.sh

@@ -62,7 +62,12 @@ install_yarn() {
     tar xzf /tmp/yarn.tar.gz -C "$dir" --strip 1
   fi
   chmod +x "$dir"/bin/*
-  echo "Installed yarn $(yarn --version)"
+
+  if $YARN2; then
+    echo "Using yarn $(yarn --version)"
+  else
+    echo "Installed yarn $(yarn --version)"
+  fi
 }
 
 install_nodejs() {

lib/dependencies.sh

@@ -27,7 +27,7 @@ run_if_present() {
   script=$(read_json "$build_dir/package.json" ".scripts[\"$script_name\"]")
 
   if [[ "$has_script_name" == "true" ]]; then
-    if $YARN; then
+    if $YARN || $YARN2; then
       echo "Running $script_name (yarn)"
       # yarn will throw an error if the script is an empty string, so check for this case
       if [[ -n "$script" ]]; then

lib/failure.sh

@@ -114,11 +114,7 @@ fail_iojs_unsupported() {
 }
 
 fail_yarn2_unsupported() {
-  local uses_yarn="$1"
-  local build_dir="$2"
-  local uses_yarn2
-
-  uses_yarn2=$(detect_yarn2 "$uses_yarn" "$build_dir")
+  local uses_yarn2="$1"
 
   if [[ "$uses_yarn2" == "true" ]]; then
     mcount "failures.yarn2-unsupported"
@@ -365,6 +361,78 @@ fail_invalid_semver() {
   fi
 }
 
+# Yarn 2 failures
+
+fail_missing_yarnrc_yml() {
+  local build_dir="$1"
+
+  if [[ ! -f "$build_dir/.yarnrc.yml" ]]; then
+    mcount "failures.missing-yarnrc-yml"
+    meta_set "failure" "missing-yarnrc-yml"
+    header "Build failed"
+    warn "The 'yarnrc.yml' file is not found
+
+      It looks like the 'yarnrc.yml' file is missing from this project. Please
+      make sure this file is checked into version control and made available to
+      Heroku.
+
+      To generate 'yarnrc.yml', make sure Yarn 2 is installed on your local
+      machine and set the version in your project directory with:
+
+       $ yarn set version berry
+      "
+    fail
+  fi
+}
+
+fail_missing_yarn_path() {
+  local build_dir="$1"
+  local yarn_path="$2"
+
+  if [[ "$yarn_path" == "" ]]; then
+    mcount "failures.missing-yarn-path"
+    meta_set "failure" "missing-yarn-path"
+    header "Build failed"
+    warn "The 'yarnPath' could not be read from the 'yarnrc.yml' file
+
+      It looks like 'yarnrc.yml' is missing the 'yarnPath' value, which is needed
+      to identify the location of yarn for this build.
+
+      To regenerate 'yarnrc.yml' with the 'yarnPath' value set, make sure Yarn 2
+      is installed on your local machine and set the version in your project
+      directory with:
+
+       $ yarn set version berry
+      "
+    fail
+  fi
+}
+
+fail_missing_yarn_vendor() {
+  local build_dir="$1"
+  local yarn_path="$2"
+
+  if [[ ! -f "$build_dir/$yarn_path" ]]; then
+    mcount "failures.missing-yarn-vendor"
+    meta_set "failure" "missing-yarn-vendor"
+    header "Build failed"
+    warn "Yarn was not found
+
+      It looks like yarn is missing from $yarn_path, which is needed to continue
+      this build on Heroku. Yarn 2 recommends vendoring Yarn under the '.yarn/releases'
+      directory, so remember to check the '.yarn' directory into version control
+      to use during builds.
+
+      To generate the '.yarn' directory correctly, make sure Yarn 2 is installed
+      on your local machine and run the following in your project directory:
+
+       $ yarn install
+       $ yarn set version berry
+      "
+    fail
+  fi
+}
+
 log_other_failures() {
   local log_file="$1"
   if grep -qi "sh: 1: .*: not found" "$log_file"; then

lib/yarn2.sh

@@ -19,3 +19,8 @@ detect_yarn2() {
     echo "false"
   fi
 }
+
+get_yarn_path() {
+  local build_dir="$1"
+  $YQ r "$build_dir/.yarnrc.yml" yarnPath 2>&1
+}