From fab461a042da2817c9ed8bc6ea8260e313be6c02 Mon Sep 17 00:00:00 2001 From: Joyce Liu <12664976+yt3liu@users.noreply.github.com> Date: Mon, 11 Mar 2019 11:30:42 -0700 Subject: [PATCH] Fail fast in releash.sh when testing (#331) (#559) --- scripts/README.md | 2 + scripts/presubmit-tests.sh | 11 ++++- scripts/release.sh | 1 + test/unit/presubmit-tests.sh | 82 +++++++++++++++++++++++++++++++----- 4 files changed, 84 insertions(+), 12 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index b0d5f2502c..1d14a55f71 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -33,6 +33,8 @@ This is a helper script to run the presubmit tests. To use it: - `DISABLE_MD_LINTING`: Disable linting markdown files, defaults to 0 (false). - `DISABLE_MD_LINK_CHECK`: Disable checking links in markdown files, defaults to 0 (false). + - `PRESUBMIT_TEST_FAIL_FAST`: Fail the presubmit test immediately if a test fails, + defaults to 0 (false). 1. [optional] Define the functions `pre_build_tests()` and/or `post_build_tests()`. These functions will be called before or after the diff --git a/scripts/presubmit-tests.sh b/scripts/presubmit-tests.sh index e3c024f825..b10a8515cf 100755 --- a/scripts/presubmit-tests.sh +++ b/scripts/presubmit-tests.sh @@ -22,6 +22,7 @@ source $(dirname ${BASH_SOURCE})/library.sh # Custom configuration of presubmit tests readonly DISABLE_MD_LINTING=${DISABLE_MD_LINTING:-0} readonly DISABLE_MD_LINK_CHECK=${DISABLE_MD_LINK_CHECK:-0} +readonly PRESUBMIT_TEST_FAIL_FAST=${PRESUBMIT_TEST_FAIL_FAST:-0} # Extensions or file patterns that don't require presubmit tests. readonly NO_PRESUBMIT_FILES=(\.png \.gitignore \.gitattributes ^OWNERS ^OWNERS_ALIASES ^AUTHORS) @@ -317,8 +318,14 @@ function main() { fi run_build_tests || failed=1 - run_unit_tests || failed=1 - run_integration_tests || failed=1 + # If PRESUBMIT_TEST_FAIL_FAST is set to true, don't run unit tests if build tests failed + if (( ! PRESUBMIT_TEST_FAIL_FAST )) || (( ! failed )); then + run_unit_tests || failed=1 + fi + # If PRESUBMIT_TEST_FAIL_FAST is set to true, don't run integration tests if build/unit tests failed + if (( ! PRESUBMIT_TEST_FAIL_FAST )) || (( ! failed )); then + run_integration_tests || failed=1 + fi exit ${failed} } diff --git a/scripts/release.sh b/scripts/release.sh index c35541d1b6..e51f4216ef 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -69,6 +69,7 @@ function publish_yamls() { # These are global environment variables. SKIP_TESTS=0 +PRESUBMIT_TEST_FAIL_FAST=1 TAG_RELEASE=0 PUBLISH_RELEASE=0 PUBLISH_TO_GITHUB=0 diff --git a/test/unit/presubmit-tests.sh b/test/unit/presubmit-tests.sh index 336e84c08e..1b2bef809c 100755 --- a/test/unit/presubmit-tests.sh +++ b/test/unit/presubmit-tests.sh @@ -19,28 +19,32 @@ [[ -z "${PULL_PULL_SHA:-}" ]] && PULL_PULL_SHA=456 [[ -z "${ARTIFATCS:-}" ]] && ARTIFACTS=/tmp -source $(dirname $0)/../../scripts/presubmit-tests.sh source $(dirname $0)/test-helper.sh set -e -# Mock external tools for testing purposes. +function init_test_env() { + source $(dirname $0)/../../scripts/presubmit-tests.sh -function list_changed_files() { - echo "foobar.go" -} + # Mock external tools for testing purposes. + function list_changed_files() { + echo "foobar.go" + } -function markdown-link-check() { - return 0 -} + function markdown-link-check() { + return 0 + } -function mdl() { - return 0 + function mdl() { + return 0 + } } # Helper functions. function mock_presubmit_runners() { + RETURN_CODE="${1:-0}" + RAN_BUILD_TESTS=0 RAN_UNIT_TESTS=0 RAN_INTEGRATION_TESTS=0 @@ -52,30 +56,39 @@ function mock_presubmit_runners() { POST_INTEGRATION_TESTS=0 function pre_build_tests() { PRE_BUILD_TESTS=1 + return ${RETURN_CODE} } function build_tests() { RAN_BUILD_TESTS=1 + return ${RETURN_CODE} } function post_build_tests() { POST_BUILD_TESTS=1 + return ${RETURN_CODE} } function pre_unit_tests() { PRE_UNIT_TESTS=1 + return ${RETURN_CODE} } function unit_tests() { RAN_UNIT_TESTS=1 + return ${RETURN_CODE} } function post_unit_tests() { POST_UNIT_TESTS=1 + return ${RETURN_CODE} } function pre_integration_tests() { PRE_INTEGRATION_TESTS=1 + return ${RETURN_CODE} } function integration_tests() { RAN_INTEGRATION_TESTS=1 + return ${RETURN_CODE} } function post_integration_tests() { POST_INTEGRATION_TESTS=1 + return ${RETURN_CODE} } } @@ -117,6 +130,51 @@ function test_custom_runners_basic() { } } +function test_custom_runners_fail_slow() { + mock_presubmit_runners ${FAILURE} + unset pre_build_tests + unset post_build_tests + unset pre_unit_tests + unset post_unit_tests + unset pre_integration_tests + unset post_integration_tests + function check_results() { + (( ! PRE_BUILD_TESTS )) || test_failed "Pre build tests did run" + (( RAN_BUILD_TESTS )) || test_failed "Build tests did not run" + (( ! POST_BUILD_TESTS )) || test_failed "Post build tests did run" + (( ! PRE_UNIT_TESTS )) || test_failed "Pre unit tests did run" + (( RAN_UNIT_TESTS )) || test_failed "Unit tests did not run" + (( ! POST_UNIT_TESTS )) || test_failed "Post unit tests did run" + (( ! PRE_INTEGRATION_TESTS )) || test_failed "Pre integration tests did run" + (( RAN_INTEGRATION_TESTS )) || test_failed "Custom integration tests did not run" + (( ! POST_INTEGRATION_TESTS )) || test_failed "Post integration tests did run" + echo "Test failed slow" + } +} + +function test_custom_runners_fail_fast() { + PRESUBMIT_TEST_FAIL_FAST=1 + mock_presubmit_runners ${FAILURE} + unset pre_build_tests + unset post_build_tests + unset pre_unit_tests + unset post_unit_tests + unset pre_integration_tests + unset post_integration_tests + function check_results() { + (( ! PRE_BUILD_TESTS )) || test_failed "Pre build tests did run" + (( RAN_BUILD_TESTS )) || test_failed "Build tests did not run" + (( ! POST_BUILD_TESTS )) || test_failed "Post build tests did run" + (( ! PRE_UNIT_TESTS )) || test_failed "Pre unit tests did run" + (( ! RAN_UNIT_TESTS )) || test_failed "Unit tests did run" + (( ! POST_UNIT_TESTS )) || test_failed "Post unit tests did run" + (( ! PRE_INTEGRATION_TESTS )) || test_failed "Pre integration tests did run" + (( ! RAN_INTEGRATION_TESTS )) || test_failed "Custom integration tests did run" + (( ! POST_INTEGRATION_TESTS )) || test_failed "Post integration tests did run" + echo "Test failed fast" + } +} + function run_markdown_build_tests() { function list_changed_files() { echo "README.md" @@ -125,6 +183,7 @@ function run_markdown_build_tests() { } function run_main() { + init_test_env # Keep current EXIT trap, used by `test_function` local current_trap="$(trap -p EXIT | cut -d\' -f2)" trap -- "${current_trap};check_results" EXIT @@ -135,9 +194,12 @@ echo ">> Testing custom test runners" test_function ${SUCCESS} "Test passed" call_function_pre test_custom_runners_all run_main test_function ${SUCCESS} "Test passed" call_function_pre test_custom_runners_basic run_main +test_function ${FAILURE} "Test failed slow" call_function_pre test_custom_runners_fail_slow run_main +test_function ${FAILURE} "Test failed fast" call_function_pre test_custom_runners_fail_fast run_main echo ">> Testing default test runners" +init_test_env test_function ${SUCCESS} "BUILD TESTS PASSED" main --build-tests test_function ${SUCCESS} "BUILD TESTS PASSED" call_function_pre run_markdown_build_tests