diff --git a/Makefile b/Makefile index a8a87150bf..0f2711fd34 100644 --- a/Makefile +++ b/Makefile @@ -166,4 +166,4 @@ controller-gen: command -v controller-gen &> /dev/null || (cd tools && go install sigs.k8s.io/controller-tools/cmd/controller-gen) deploy.kind: - DOCKER_PUSH=n KIND_LOAD_IMAGE=y FORCE_REDEPLOY=y ./scripts/deploy-kubefed.sh $(IMAGE_NAME) + KIND_LOAD_IMAGE=y FORCE_REDEPLOY=y ./scripts/deploy-kubefed.sh $(IMAGE_NAME) diff --git a/docs/environments/kind.md b/docs/environments/kind.md index 4fdc4f45d0..3e926ff2d1 100644 --- a/docs/environments/kind.md +++ b/docs/environments/kind.md @@ -5,12 +5,7 @@ - [`kind` - `k`ubernetes `in` `d`ocker](#kind---kubernetes-in-docker) - [Download and Install kind](#download-and-install-kind) - [Create Clusters](#create-clusters) - - [Create Insecure Container Registry](#create-insecure-container-registry) - - [Configure Insecure Container Registry](#configure-insecure-container-registry) - - [Run Script](#run-script) - [Delete Clusters](#delete-clusters) - - [Delete Insecure Container Registry](#delete-insecure-container-registry) - - [Run Script](#run-script-1) @@ -35,69 +30,6 @@ will be installed. Your `GOBIN` directory should be at `$(go env GOPATH)/bin`: You can proceed to create clusters once you have `kind` available in your path. -### Create Insecure Container Registry - -Please answer the following question to determine if you need to set up an -insecure container registry on your host: - -1. Are you planning on creating container images locally without pushing to a - public container registry such as `quay.io`. For example, you can build your - own custom image e.g. `172.17.0.1:5000/:`, as part of your - development workflow and push to this container registry . See the - [development guide](/docs/development.md#test-your-changes) for more - examples. - -If you answered yes, then you will need to create an insecure container -registry. Creating a container registry is necessary if you want your kind -clusters to pull images that you built locally on your host without pushing -them to a public container registry. See the [docker -docs](https://docs.docker.com/registry) for more details. - -In order to create an insecure container registry, you can pass the -`CREATE_INSECURE_REGISTRY` flag to `create-clusters.sh` as follows: - -```bash -CREATE_INSECURE_REGISTRY=y ./scripts/create-clusters.sh -``` - -### Configure Insecure Container Registry - -Please answer the following questions to determine if you need to configure an -insecure container registry on your host: - -1. Is this the first time you're running the `create-clusters.sh` script? -2. Does your docker daemon need to be configured for an insecure container - registry? - -If you answered yes to both of these questions, then you will need to configure -the docker daemon on your host for an insecure container registry. The reason -for an insecure registry is to simplify the container registry setup by not -enabling TLS. **This only needs to be done once for a particular host**. -See the [docker docs](https://docs.docker.com/registry) for more details. - -In order to configure an insecure container registry, you can pass the -`CONFIGURE_INSECURE_REGISTRY_HOST` flag to `create-clusters.sh` as shown below. The -default container registry host is `172.17.0.1:5000` and needs to match -the IP address of the default docker bridge on your host, typically -`172.17.0.1`. If you would like to change this then set the -`CONTAINER_REGISTRY_HOST=":"` flag. - -```bash -CONFIGURE_INSECURE_REGISTRY_HOST=y ./scripts/create-clusters.sh -``` - -This will automatically create the necessary dockerd daemon config and reload -the docker daemon for you. Keep in mind that it will **not** do this for you -if a config already exists, or your docker daemon is already configured with an -`--insecure-registry` command line option. - -If you would like to manually make the changes to your docker daemon instead, -add `172.17.0.1:5000` as an insecure registry host and reload or restart your -docker daemon. See the [docker -docs](https://docs.docker.com/registry/insecure/) for more details. - -### Run Script - Run the following command to create `2` `kind` clusters: ```bash @@ -113,17 +45,6 @@ NUM_CLUSTERS= ./scripts/create-clusters.sh ## Delete Clusters -### Delete Insecure Container Registry - -Specify the `DELETE_INSECURE_REGISTRY` flag if you set up an insecure container -registry and would like to have it deleted. - -```bash -DELETE_INSECURE_REGISTRY=y ./scripts/delete-clusters.sh -``` - -### Run Script - Run the following command to delete `2` `kind` clusters: ```bash diff --git a/scripts/create-clusters.sh b/scripts/create-clusters.sh index ca573df2ef..014ddc2cfb 100755 --- a/scripts/create-clusters.sh +++ b/scripts/create-clusters.sh @@ -21,90 +21,12 @@ set -o errexit set -o nounset set -o pipefail -source "$(dirname "${BASH_SOURCE}")/util.sh" -CREATE_INSECURE_REGISTRY="${CREATE_INSECURE_REGISTRY:-}" -CONFIGURE_INSECURE_REGISTRY_HOST="${CONFIGURE_INSECURE_REGISTRY_HOST:-}" -CONFIGURE_INSECURE_REGISTRY_CLUSTER="${CONFIGURE_INSECURE_REGISTRY_CLUSTER-y}" -CONTAINER_REGISTRY_HOST="${CONTAINER_REGISTRY_HOST:-172.17.0.1:5000}" +# shellcheck source=util.sh +source "${BASH_SOURCE%/*}/util.sh" NUM_CLUSTERS="${NUM_CLUSTERS:-2}" KIND_IMAGE="${KIND_IMAGE:-}" KIND_TAG="${KIND_TAG:-}" -docker_daemon_config="/etc/docker/daemon.json" -containerd_config="/etc/containerd/config.toml" -OS=`uname` - -function create-insecure-registry() { - # Run insecure registry as container - docker run -d -p 5000:5000 --restart=always --name registry registry:2 -} - -function configure-insecure-registry() { - local err= - if sudo test -f "${docker_daemon_config}"; then - if sudo grep -q "\"insecure-registries\": \[\"${CONTAINER_REGISTRY_HOST}\"\]" ${docker_daemon_config}; then - return 0 - elif sudo grep -q "\"insecure-registries\": " ${docker_daemon_config}; then - echo < /dev/null - docker rm registry &> /dev/null - fi - return 1 - fi - - configure-insecure-registry-and-reload "sudo bash -c" $(pgrep dockerd) ${docker_daemon_config} -} - -function configure-insecure-registry-and-reload() { - local cmd_context="${1}" # context to run command e.g. sudo, docker exec - local docker_pid="${2}" - local config_file="${3}" - ${cmd_context} "$(insecure-registry-config-cmd ${config_file})" - ${cmd_context} "$(reload-daemon-cmd "${docker_pid}")" -} - -function insecure-registry-config-cmd() { - local config_file="${1}" - case $config_file in - $docker_daemon_config) - echo "cat < ${docker_daemon_config} -{ - \"insecure-registries\": [\"${CONTAINER_REGISTRY_HOST}\"] -} -EOF -" - ;; - $containerd_config) - if [ "${CONFIGURE_INSECURE_REGISTRY_CLUSTER}" ]; then - echo -e "containerdConfigPatches:\n- |-\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"${CONTAINER_REGISTRY_HOST}\"]\n endpoint = [\"http://${CONTAINER_REGISTRY_HOST}\"]\n" - else - echo "" - fi - ;; - *) - echo "Sorry, config insecure registy is not supported for $config_file" - ;; - esac -} - -function reload-daemon-cmd() { - echo "kill -s SIGHUP ${1}" -} +OS="$(uname)" function create-clusters() { local num_clusters=${1} @@ -115,19 +37,15 @@ function create-clusters() { elif [[ "${KIND_TAG}" ]]; then image_arg="--image=kindest/node:${KIND_TAG}" fi - for i in $(seq ${num_clusters}); do - kind create cluster --name "cluster${i}" --config - "${image_arg}" << EOF -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -$(insecure-registry-config-cmd ${containerd_config}) -EOF - fixup-cluster ${i} + for i in $(seq "${num_clusters}"); do + kind create cluster --name "cluster${i}" "${image_arg}" + fixup-cluster "${i}" echo done echo "Waiting for clusters to be ready" - check-clusters-ready ${num_clusters} + check-clusters-ready "${num_clusters}" } function fixup-cluster() { @@ -135,7 +53,8 @@ function fixup-cluster() { if [ "$OS" != "Darwin" ];then # Set container IP address as kube API endpoint in order for clusters to reach kube API servers in other clusters. - local docker_ip=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "cluster${i}-control-plane") + local docker_ip + docker_ip=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "cluster${i}-control-plane") kubectl config set-cluster "kind-cluster${i}" --server="https://${docker_ip}:6443" fi @@ -144,23 +63,13 @@ function fixup-cluster() { } function check-clusters-ready() { - for i in $(seq ${1}); do + for i in $(seq "${1}"); do util::wait-for-condition 'ok' "kubectl --context cluster${i} get --raw=/healthz &> /dev/null" 120 done } -if [[ "${CREATE_INSECURE_REGISTRY}" ]]; then - echo "Creating container registry on host" - create-insecure-registry -fi - -if [[ "${CONFIGURE_INSECURE_REGISTRY_HOST}" ]]; then - echo "Configuring container registry on host" - configure-insecure-registry -fi - echo "Creating ${NUM_CLUSTERS} clusters" -create-clusters ${NUM_CLUSTERS} +create-clusters "${NUM_CLUSTERS}" kubectl config use-context cluster1 echo "Complete" diff --git a/scripts/delete-clusters.sh b/scripts/delete-clusters.sh index d5e5e1f487..041ff0e094 100755 --- a/scripts/delete-clusters.sh +++ b/scripts/delete-clusters.sh @@ -21,33 +21,22 @@ set -o errexit set -o nounset set -o pipefail -DELETE_INSECURE_REGISTRY="${DELETE_INSECURE_REGISTRY:-}" NUM_CLUSTERS="${NUM_CLUSTERS:-2}" -function delete-insecure-registry() { - docker kill registry &> /dev/null || return 0 - docker rm registry &> /dev/null || true -} - function delete-clusters() { local num_clusters=${1} - for i in $(seq ${num_clusters}); do + for i in $(seq "${num_clusters}"); do # The context name has been changed when creating clusters by 'create-cluster.sh'. # This will result in the context can't be removed by kind when deleting a cluster. # So, we need to change context name back and let kind take care about it. kubectl config rename-context "cluster${i}" "kind-cluster${i}" - kind delete cluster --name cluster${i} + kind delete cluster --name "cluster${i}" done } -if [[ "${DELETE_INSECURE_REGISTRY}" ]]; then - echo "Deleting container registry on host" - delete-insecure-registry -fi - echo "Deleting ${NUM_CLUSTERS} clusters" -delete-clusters ${NUM_CLUSTERS} +delete-clusters "${NUM_CLUSTERS}" echo "Complete" diff --git a/scripts/deploy-kubefed.sh b/scripts/deploy-kubefed.sh index d2e09c87e1..ddfcf4d10a 100755 --- a/scripts/deploy-kubefed.sh +++ b/scripts/deploy-kubefed.sh @@ -35,7 +35,8 @@ set -o errexit set -o nounset set -o pipefail -source "$(dirname "${BASH_SOURCE}")/util.sh" +# shellcheck source=util.sh +source "${BASH_SOURCE%/*}/util.sh" function deploy-with-helm() { local repository=${IMAGE_NAME%/*} @@ -45,10 +46,10 @@ function deploy-with-helm() { local cmd if [[ "${NAMESPACED}" ]]; then - cmd="$(helm-deploy-cmd kubefed-${NS} ${NS} ${repository} ${image} ${tag})" + cmd="$(helm-deploy-cmd "kubefed-${NS}" "${NS}" "${repository}" "${image}" "${tag}")" cmd="${cmd} --set global.scope=Namespaced" else - cmd="$(helm-deploy-cmd kubefed ${NS} ${repository} ${image} ${tag})" + cmd="$(helm-deploy-cmd kubefed "${NS}" "${repository}" "${image}" "${tag}")" fi if [[ "${IMAGE_PULL_POLICY:-}" ]]; then @@ -57,8 +58,8 @@ function deploy-with-helm() { ${cmd} - deployment-image-as-expected ${NS} kubefed-admission-webhook admission-webhook ${repository}/${image}:${tag} - deployment-image-as-expected ${NS} kubefed-controller-manager controller-manager ${repository}/${image}:${tag} + deployment-image-as-expected "${NS}" kubefed-admission-webhook admission-webhook "${repository}/${image}:${tag}" + deployment-image-as-expected "${NS}" kubefed-controller-manager controller-manager "${repository}/${image}:${tag}" } function helm-deploy-cmd { @@ -86,7 +87,8 @@ function helm-deploy-cmd { } function kubefed-admission-webhook-ready() { - local readyReplicas=$(kubectl -n ${1} get deployments.apps kubefed-admission-webhook -o jsonpath='{.status.readyReplicas}') + local readyReplicas + readyReplicas=$(kubectl -n "${1}" get deployments.apps kubefed-admission-webhook -o jsonpath='{.status.readyReplicas}') [[ "${readyReplicas}" -ge "1" ]] } @@ -96,7 +98,8 @@ function deployment-image-as-expected() { local container="${3}" local expected_image="${4}" - local deployed_image="$(kubectl -n ${namespace} get deployment ${deployment} -o jsonpath='{.spec.template.spec.containers[?(@.name=="'"${container}"'")].image}')" + local deployed_image + deployed_image="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.spec.template.spec.containers[?(@.name=="'"${container}"'")].image}')" [[ "${deployed_image}" == "${expected_image}" ]] } @@ -105,7 +108,7 @@ function check-command-installed() { command -v "${cmdName}" >/dev/null 2>&1 || { - echo "${cmdName} command not found. Please download dependencies using $(dirname ${BASH_SOURCE})/download-binaries.sh and install it in your PATH." >&2 + echo "${cmdName} command not found. Please download dependencies using ${BASH_SOURCE%/*}/download-binaries.sh and install it in your PATH." >&2 exit 1 } } @@ -121,7 +124,7 @@ else USE_LATEST= fi -KF_NS_ARGS="--kubefed-namespace=${NS} " +KF_NS_ARGS="--kubefed-namespace=${NS}" if [[ -z "${IMAGE_NAME}" ]]; then >&2 echo "Usage: $0 [join-cluster]... @@ -133,7 +136,6 @@ Example: docker.io//kubefed:test If intending to use the docker hub as the container registry to push the KubeFed image to, make sure to login to the local docker daemon to ensure credentials are available for push: - $ docker login --username should be the kubeconfig context name for the additional cluster to join. @@ -153,19 +155,14 @@ check-command-installed helm # Build KubeFed binaries and image if [[ "${USE_LATEST:-}" != "y" ]]; then cd "$(dirname "$0")/.." - make container IMAGE_NAME=${IMAGE_NAME} + make container IMAGE_NAME="${IMAGE_NAME}" cd - - # Use DOCKER_PUSH=n ./scripts/deploy-kubefed.sh to skip docker - # push on container image when not using latest image. - if [[ "${DOCKER_PUSH:-y}" == "y" ]]; then - docker push ${IMAGE_NAME} - fi fi -# Use KIND_LOAD_IMAGE=y DOCKER_PUSH= ./scripts/deploy-kubefed.sh to load +# Use KIND_LOAD_IMAGE=y ./scripts/deploy-kubefed.sh to load # the built docker image into kind before deploying. if [[ "${KIND_LOAD_IMAGE:-}" == "y" ]]; then - kind load docker-image ${IMAGE_NAME} + kind load docker-image "${IMAGE_NAME}" --name="${KIND_CLUSTER_NAME:-}" fi cd "$(dirname "$0")/.." @@ -177,8 +174,8 @@ deploy-with-helm # Join the host cluster CONTEXT="$(kubectl config current-context)" -./bin/kubefedctl join "${CONTEXT}" --host-cluster-context "${CONTEXT}" --v=2 ${KF_NS_ARGS} --error-on-existing=false +./bin/kubefedctl join "${CONTEXT}" --host-cluster-context "${CONTEXT}" --v=2 "${KF_NS_ARGS}" --error-on-existing=false for c in ${JOIN_CLUSTERS}; do - ./bin/kubefedctl join "${c}" --host-cluster-context "${CONTEXT}" --v=2 ${KF_NS_ARGS} --error-on-existing=false + ./bin/kubefedctl join "${c}" --host-cluster-context "${CONTEXT}" --v=2 "${KF_NS_ARGS}" --error-on-existing=false done diff --git a/scripts/download-e2e-binaries.sh b/scripts/download-e2e-binaries.sh index 6ca3f57f4e..3b0530b9c5 100755 --- a/scripts/download-e2e-binaries.sh +++ b/scripts/download-e2e-binaries.sh @@ -35,10 +35,11 @@ dest_dir="${root_dir}/bin" mkdir -p "${dest_dir}" # kind +platform="$(uname -s|tr A-Z a-z)" kind_version="v0.9.0" kind_path="${dest_dir}/kind" -kind_url="https://github.com/kubernetes-sigs/kind/releases/download/${kind_version}/kind-linux-amd64" -curl -Lo "${kind_path}" "${kind_url}" && chmod +x "${kind_path}" +kind_url="https://github.com/kubernetes-sigs/kind/releases/download/${kind_version}/kind-${platform}-amd64" +curl -fLo "${kind_path}" "${kind_url}" && chmod +x "${kind_path}" # Pull the busybox image (used in tests of workload types) docker pull busybox diff --git a/scripts/pre-commit.sh b/scripts/pre-commit.sh index 725c2ed2ba..899b4dd26e 100755 --- a/scripts/pre-commit.sh +++ b/scripts/pre-commit.sh @@ -20,8 +20,9 @@ set -o errexit set -o nounset set -o pipefail -source "$(dirname "${BASH_SOURCE}")/util.sh" -ROOT_DIR="$(cd "$(dirname "$0")/.." ; pwd)" +# shellcheck source=util.sh +source "${BASH_SOURCE%/*}/util.sh" +ROOT_DIR="$(cd "${BASH_SOURCE%/*}/.." ; pwd)" TEMP_DIR="$(mktemp -d)" MAKE_CMD="make -C ${ROOT_DIR}" OS="$(go env GOOS)" @@ -30,7 +31,6 @@ PLATFORM="${OS}-${ARCH}" NUM_CLUSTERS="${NUM_CLUSTERS:-2}" JOIN_CLUSTERS="${JOIN_CLUSTERS:-}" DOWNLOAD_BINARIES="${DOWNLOAD_BINARIES:-}" -CONTAINER_REGISTRY_HOST="${CONTAINER_REGISTRY_HOST:-172.17.0.1:5000}" COMMON_TEST_ARGS="-kubeconfig=${HOME}/.kube/config -ginkgo.v -single-call-timeout=1m -ginkgo.trace -ginkgo.randomizeAllSpecs" E2E_TEST_CMD="${TEMP_DIR}/e2e-${PLATFORM} ${COMMON_TEST_ARGS}" # Disable limited scope in-memory controllers to ensure the controllers in the @@ -45,31 +45,19 @@ function build-binaries() { ${MAKE_CMD} e2e # Copying the test binary to ${TEMP_DIR} to ensure # there's no dependency on the static files in the path - cp ${ROOT_DIR}/bin/e2e-${PLATFORM} ${TEMP_DIR} + cp "${ROOT_DIR}/bin/e2e-${PLATFORM}" "${TEMP_DIR}" } function download-dependencies() { - if [[ -z "${DOWNLOAD_BINARIES}" ]]; then - return + if [[ "${DOWNLOAD_BINARIES:-}" == "y" ]]; then + ./scripts/download-binaries.sh fi - - ./scripts/download-binaries.sh } function run-unit-tests() { KUBEBUILDER_ASSETS=${ROOT_DIR}/bin ${MAKE_CMD} test } -function join-cluster-list() { - if [[ -z "${JOIN_CLUSTERS}" ]]; then - for i in $(seq 2 ${NUM_CLUSTERS}); do - JOIN_CLUSTERS+="cluster${i} " - done - export JOIN_CLUSTERS=$(echo ${JOIN_CLUSTERS} | sed 's/ $//') - fi - echo "${JOIN_CLUSTERS}" -} - function run-e2e-tests() { ${E2E_TEST_CMD} } @@ -98,7 +86,7 @@ function check-git-state() { return fi echo "ERROR: the working tree is dirty:" - for line in "${output}"; do + for line in ${output}; do echo "${line}" done git diff @@ -148,14 +136,17 @@ run-unit-tests echo "Downloading e2e test dependencies" ./scripts/download-e2e-binaries.sh -CREATE_INSECURE_REGISTRY=y CONFIGURE_INSECURE_REGISTRY_HOST=y \ - KIND_TAG="v1.19.1@sha256:98cf5288864662e37115e362b23e4369c8c4a408f99cbc06e58ac30ddc721600" ./scripts/create-clusters.sh +KIND_TAG="v1.19.1@sha256:98cf5288864662e37115e362b23e4369c8c4a408f99cbc06e58ac30ddc721600" ./scripts/create-clusters.sh -# Initialize list of clusters to join -join-cluster-list > /dev/null +declare -a join_cluster_list=() +if [[ -z "${JOIN_CLUSTERS}" ]]; then + for i in $(seq 2 "${NUM_CLUSTERS}"); do + join_cluster_list+=("cluster${i}") + done +fi echo "Deploying cluster-scoped kubefed" -./scripts/deploy-kubefed.sh ${CONTAINER_REGISTRY_HOST}/kubefed:e2e $(join-cluster-list) +KIND_CLUSTER_NAME=cluster1 KIND_LOAD_IMAGE=y ./scripts/deploy-kubefed.sh local/kubefed:e2e "${join_cluster_list[@]-}" echo "Running e2e tests against cluster-scoped kubefed" run-e2e-tests @@ -180,7 +171,7 @@ echo "Deleting cluster-scoped kubefed" ./scripts/delete-kubefed.sh echo "Deploying namespace-scoped kubefed" -KUBEFED_NAMESPACE=foo NAMESPACED=y ./scripts/deploy-kubefed.sh ${CONTAINER_REGISTRY_HOST}/kubefed:e2e $(join-cluster-list) +KUBEFED_NAMESPACE=foo NAMESPACED=y ./scripts/deploy-kubefed.sh local/kubefed:e2e "${join_cluster_list[@]}" echo "Running go e2e tests with namespace-scoped kubefed" run-namespaced-e2e-tests