diff --git a/.github/workflows/ce-provision-test-gitlab.yml b/.github/workflows/ce-provision-test-gitlab.yml index 4d7f226d9..bd50e7233 100644 --- a/.github/workflows/ce-provision-test-gitlab.yml +++ b/.github/workflows/ce-provision-test-gitlab.yml @@ -1,42 +1,42 @@ name: Run GitLab server test build -# Run this workflow every time a new commit pushed to your repository -on: pull_request +# Run this workflow nightly and every time a new commit pushed to your repository +on: + schedule: + - cron: '30 4 * * *' + pull_request: jobs: # Set the job key. The key is displayed as the job name # when a job name is not provided test-gitlab: + if: ${{ github.event.pull_request.head.ref != 'documentation' }} # Name the Job - name: Run tests against Ansible code base + name: Build a GitLab server with ce-provision # Set the type of machine to run on runs-on: ubuntu-latest - steps: - # Checks out a copy of your repository on the ubuntu-latest machine - - name: Checkout code - if: ${{ github.event.pull_request.head.ref != 'documentation' }} - uses: actions/checkout@v2 + # Use our ce-dev Debian base container + container: + image: codeenigma/ce-dev:2.x + volumes: + - ${{ github.workspace }}:/home/controller - # Installs the ce-dev stack - - name: Install ce-dev - if: ${{ github.event.pull_request.head.ref != 'documentation' }} + steps: + - name: Install ce-provision run: | - cd /tmp - wget https://golang.org/dl/go1.15.8.linux-amd64.tar.gz - sudo tar -C /usr/local -xzf go1.15.8.linux-amd64.tar.gz - export PATH=$PATH:/usr/local/go/bin - git clone https://github.com/FiloSottile/mkcert && cd mkcert - go build -ldflags "-X main.Version=$(git describe --tags)" - sudo mv ./mkcert /usr/local/bin && cd ../ - sudo chmod +x /usr/local/bin/mkcert - rm -Rf mkcert - curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/${{ github.event.pull_request.base.ref }}/install.sh | /bin/sh -s -- --platform linux + /usr/bin/curl -LO https://raw.githubusercontent.com/codeenigma/ce-provision/${{ github.event.pull_request.head.ref }}/install.sh + /usr/bin/chmod +x ./install.sh + /usr/bin/sudo ./install.sh --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --docker --no-firewall - # Uses the ce-dev stack to run a test provision - - name: Run a test provision - if: ${{ github.event.pull_request.head.ref != 'documentation' }} + # Run a GitLab server provision + - name: Prepare Git repos on disk run: | - git clone --branch ${{ github.event.pull_request.base.ref }} https://github.com/codeenigma/ce-dev-ce-provision-config.git config - /bin/bash ce-dev/ansible/test.sh --examples gitlab --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} - shell: bash + /usr/bin/git config --global --add safe.directory /home/controller/ce-provision + /usr/bin/git config --global --add safe.directory /home/controller/ce-provision/config + + - name: Start SSHD + run: /usr/sbin/sshd& + + - name: Provision a test GitLab server + run: /usr/bin/su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/gitlab/ci.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 9b1c15b32..54421cd40 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -1,60 +1,42 @@ name: Run web server test build -# Run this workflow every time a new commit pushed to your repository -on: pull_request +# Run this workflow nightly and every time a new commit pushed to your repository +on: + schedule: + - cron: '30 4 * * *' + pull_request: jobs: # Set the job key. The key is displayed as the job name # when a job name is not provided test-web: + if: ${{ github.event.pull_request.head.ref != 'documentation' }} # Name the Job - name: Run tests against Ansible code base + name: Build a web server with ce-provision # Set the type of machine to run on runs-on: ubuntu-latest # Use our ce-dev Debian base container container: image: codeenigma/ce-dev:2.x + volumes: + - ${{ github.workspace }}:/home/controller steps: - # Checks out a copy of your repository on the ubuntu-latest machine - #- name: Checkout code - # if: ${{ github.event.pull_request.head.ref != 'documentation' }} - # uses: actions/checkout@v2 - - # Installs ce-provision - name: Install ce-provision - if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | - curl -LO https://raw.githubusercontent.com/codeenigma/ce-provision/${{ github.event.pull_request.head.ref }}/install.sh - chmod +x ./install.sh - sudo ./install.sh --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --docker + /usr/bin/curl -LO https://raw.githubusercontent.com/codeenigma/ce-provision/${{ github.event.pull_request.head.ref }}/install.sh + /usr/bin/chmod +x ./install.sh + /usr/bin/sudo ./install.sh --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --docker --no-firewall # Run a web server provision - - name: Provision a test web server - if: ${{ github.event.pull_request.head.ref != 'documentation' }} + - name: Prepare Git repos on disk run: | - /bin/sh /home/runner/ce-provision/scripts/provision.sh --python-interpreter /home/runner/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/runner/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force + /usr/bin/git config --global --add safe.directory /home/controller/ce-provision + /usr/bin/git config --global --add safe.directory /home/controller/ce-provision/config - # Installs the ce-dev stack - #- name: Install ce-dev - # if: ${{ github.event.pull_request.head.ref != 'documentation' }} - # run: | - # cd /tmp - # wget https://golang.org/dl/go1.15.8.linux-amd64.tar.gz - # sudo tar -C /usr/local -xzf go1.15.8.linux-amd64.tar.gz - # export PATH=$PATH:/usr/local/go/bin - # git clone https://github.com/FiloSottile/mkcert && cd mkcert - # go build -ldflags "-X main.Version=$(git describe --tags)" - # sudo mv ./mkcert /usr/local/bin && cd ../ - # sudo chmod +x /usr/local/bin/mkcert - # rm -Rf mkcert - # curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/${{ github.event.pull_request.base.ref }}/install.sh | /bin/sh -s -- --platform linux + - name: Start SSHD + run: /usr/sbin/sshd& - # Uses the ce-dev stack to run a test provision - #- name: Run a test provision - # if: ${{ github.event.pull_request.head.ref != 'documentation' }} - # run: | - # git clone --branch ${{ github.event.pull_request.base.ref }} https://github.com/codeenigma/ce-dev-ce-provision-config.git config - # /bin/bash ce-dev/ansible/test.sh --examples web --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} - # shell: bash + - name: Provision a test web server + run: /usr/bin/su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/ci.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" diff --git a/ce-dev/ansible/plays/gitlab/ci.yml b/ce-dev/ansible/plays/gitlab/ci.yml new file mode 100644 index 000000000..eb61f611f --- /dev/null +++ b/ce-dev/ansible/plays/gitlab/ci.yml @@ -0,0 +1,37 @@ +--- +- hosts: localhost + become: true + + vars: + project_name: gitlab + is_local: true + _ce_provision_base_dir: /home/ce-dev/ce-provision + _init: + force_play: true + vars_dirs: + - "{{ _ce_provision_base_dir }}/ce-dev/ansible/vars/_common" + - "{{ _ce_provision_base_dir }}/ce-dev/ansible/vars/{{ project_name }}" + + tasks: + - ansible.builtin.import_role: + name: _init + - ansible.builtin.import_role: + name: debian/user_provision + - ansible.builtin.import_role: + name: _meta/common_base + - ansible.builtin.import_role: + name: debian/ce_deploy + - ansible.builtin.import_role: + name: aws/aws_credentials + - ansible.builtin.import_role: + name: debian/gitlab + - ansible.builtin.import_role: + name: debian/gitlab_runner + - ansible.builtin.import_role: + name: debian/ssh_server + - ansible.builtin.import_role: + name: debian/sops + - ansible.builtin.import_role: + name: debian/gpg_key + - ansible.builtin.import_role: + name: _exit diff --git a/ce-dev/ansible/plays/web/ci.yml b/ce-dev/ansible/plays/web/ci.yml new file mode 100644 index 000000000..94bd2f49f --- /dev/null +++ b/ce-dev/ansible/plays/web/ci.yml @@ -0,0 +1,22 @@ +--- +# Spin up a "web" instance. +- hosts: localhost + become: true + + vars: + project_name: web + is_local: true + _ce_provision_base_dir: /home/ce-dev/ce-provision + _init: + force_play: true + vars_dirs: + - "{{ _ce_provision_base_dir }}/ce-dev/ansible/vars/_common" + - "{{ _ce_provision_base_dir }}/ce-dev/ansible/vars/{{ project_name }}" + + tasks: + - ansible.builtin.import_role: + name: _init + - ansible.builtin.import_role: + name: _meta/webserver + - ansible.builtin.import_role: + name: _exit diff --git a/ce-dev/ansible/vars/gitlab/gitlab_runner.yml b/ce-dev/ansible/vars/gitlab/gitlab_runner.yml new file mode 100644 index 000000000..721b65ab0 --- /dev/null +++ b/ce-dev/ansible/vars/gitlab/gitlab_runner.yml @@ -0,0 +1,2 @@ +gitlab_runner: + restart: false # no systemd in CI containers diff --git a/docs/roles/_init.md b/docs/roles/_init.md index 0f40180e3..e619e5656 100644 --- a/docs/roles/_init.md +++ b/docs/roles/_init.md @@ -9,7 +9,10 @@ This is meant to ALWAYS be included as the first task of a play. If you include ## Default variables ```yaml --- -_ce_provision_username: "{% if is_local is defined and is_local %}ce-dev{% else %}controller{% endif %}" +# Set this variable to true to tell ce-provision it is running in a container. +is_local: false + +_ce_provision_username: "{% if is_local %}ce-dev{% else %}controller{% endif %}" _venv_path: "/home/{{ _ce_provision_username }}/ce-python" _venv_command: /usr/bin/python3 -m venv _venv_install_username: "{{ _ce_provision_username }}" @@ -25,8 +28,8 @@ _init: # This is used to detect if the playbook must re-run or not. vars_dirs: [] force_play: false - lock_file: /tmp/ce-provision-lock - deploy_lock_file: /tmp/ce-deploy-lock # must match lock_file in ce-deploy + lock_file: /tmp/ce-provision-lock # set to an empty string to disable locking behaviour + deploy_lock_file: /tmp/ce-deploy-lock # must match lock_file in ce-deploy, set to an empty string to disable locking behaviour ce_provision_version: 2.x # Outputted by the _init role at the start of plays install_ansible: true # set to false to not install Ansible in a venv diff --git a/docs/roles/debian/aws_efs_client.md b/docs/roles/debian/aws_efs_client.md index 0711f04d9..5fc392261 100644 --- a/docs/roles/debian/aws_efs_client.md +++ b/docs/roles/debian/aws_efs_client.md @@ -46,7 +46,7 @@ _mount_state: present aws_efs_client: aws_profile: example # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set region: eu-west-1 # AWS region name - can be substituted for "{{ _aws_region }}" if set - version: 2.1.0 # version of AWS EFS utils to use + version: "{{ '1.35.0' if ansible_distribution_major_version | int < 12 else '2.1.0' }}" # 2.1.0 requires libssl v3 which is absent on Debian < 12 by default. build_suffix: "-1_all" # sometimes there is a suffix appended to the package name, e.g. `amazon-efs-utils-1.35.0-1_all.deb` deb_url: "" # provide an alternative location for the .deb package # See https://docs.ansible.com/ansible/latest/modules/mount_module.html diff --git a/docs/roles/debian/ce_deploy.md b/docs/roles/debian/ce_deploy.md index 90aa38931..743cbf8cd 100644 --- a/docs/roles/debian/ce_deploy.md +++ b/docs/roles/debian/ce_deploy.md @@ -8,7 +8,7 @@ Installs Code Enigma's deploy stack on a server. ```yaml --- _ce_deploy: - username: "{% if is_local is defined and is_local %}ce-dev{% else %}deploy{% endif %}" + username: "{% if is_local %}ce-dev{% else %}deploy{% endif %}" ce_deploy: # These are usually set in the _init role using _venv_path, _venv_command and _venv_install_username but can be overridden. diff --git a/docs/roles/debian/gitlab.md b/docs/roles/debian/gitlab.md index c02282fd2..f4b11638b 100644 --- a/docs/roles/debian/gitlab.md +++ b/docs/roles/debian/gitlab.md @@ -38,6 +38,7 @@ gitlab: ssl: # @see the 'ssl' role. Note that domain is autopopulated from server_name above. enabled: false # manual SSL handling disabled by default handling: selfsigned + replace_existing: false # Linux setup linux_user: git linux_group: git diff --git a/docs/roles/debian/php-fpm.md b/docs/roles/debian/php-fpm.md index 063e97276..443ffbf87 100644 --- a/docs/roles/debian/php-fpm.md +++ b/docs/roles/debian/php-fpm.md @@ -23,7 +23,7 @@ php: # It is important to scale up processes on bigger servers, so that more # requests can be handled. Double the number of vCPUs is a good default. # Can be between 5 and 64. - max_children: "{{ [5, [ansible_facts.ansible_processor_nproc * 2, 64] | min] | max }}" + max_children: "{{ [5, [(ansible_facts.ansible_processor_nproc | default(1)) * 2, 64] | min] | max }}" # Fallback in case ansible_processor_nproc is not gathered before tasks start_servers: 2 min_spare_servers: 1 max_spare_servers: 3 diff --git a/install.sh b/install.sh index 1de23eae4..7ca8822d0 100755 --- a/install.sh +++ b/install.sh @@ -15,6 +15,7 @@ usage(){ /usr/bin/echo '--user: Ansible controller user (default: controller)' /usr/bin/echo '--config: Git URL to your ce-provision Ansible config repository (default: https://github.com/codeenigma/ce-provision-config-example.git)' /usr/bin/echo '--config-branch: branch of your Ansible config repository to use (default: 1.x)' + /usr/bin/echo '--no-firewall: skip installing iptables with ports 22, 80 and 443 open' /usr/bin/echo '--gitlab: install GitLab CE on this server (default: no, set to desired GitLab address to install, e.g. gitlab.example.com)' /usr/bin/echo '--letsencrypt: try to create an SSL certificate with LetsEncrypt (requires DNS pointing at this server for provided GitLab URL)' /usr/bin/echo '--aws: enable AWS support' @@ -49,6 +50,9 @@ parse_options(){ "--letsencrypt") LE_SUPPORT="yes" ;; + "--no-firewall") + FIREWALL="false" + ;; "--aws") AWS_SUPPORT="true" ;; @@ -71,9 +75,11 @@ CONFIG_REPO="https://github.com/codeenigma/ce-provision-config-example.git" CONFIG_REPO_BRANCH="1.x" GITLAB_URL="no" LE_SUPPORT="no" +FIREWALL="true" AWS_SUPPORT="false" IS_LOCAL="false" SERVER_HOSTNAME=$(hostname) +ANSIBLE_COMMAND="" # Parse options. parse_options "$@" @@ -90,6 +96,7 @@ if [ "$(id -u)" -ne 0 ] fi # Check we are using a compatible Linux distribution. +/usr/bin/echo "-------------------------------------------------" if [ "$ID" != "debian" ]; then if [ "$ID_LIKE" != "debian" ]; then /usr/bin/echo "ce-provision only supports Debian Linux and derivatives." @@ -132,7 +139,7 @@ fi git ca-certificates git-lfs \ openssh-client nfs-common stunnel4 \ python3-venv python3-debian \ - zip unzip gzip tar dnsutils + zip unzip gzip tar dnsutils net-tools /usr/bin/echo "-------------------------------------------------" # Install Ansible in a Python virtual environment. @@ -152,6 +159,8 @@ fi /usr/bin/echo "-------------------------------------------------" if [ ! -d "/home/$CONTROLLER_USER/ce-provision" ]; then /usr/bin/su - "$CONTROLLER_USER" -c "git clone --branch $VERSION https://github.com/codeenigma/ce-provision.git /home/$CONTROLLER_USER/ce-provision" + /usr/bin/su - "$CONTROLLER_USER" -c "git clone --branch $CONFIG_REPO_BRANCH $CONFIG_REPO /home/$CONTROLLER_USER/ce-provision/config" + /usr/bin/su - "$CONTROLLER_USER" -c "/usr/bin/ln -s /home/$CONTROLLER_USER/ce-provision/config/ansible.cfg /home/$CONTROLLER_USER/ce-provision/ansible.cfg" else /usr/bin/echo "ce-provision directory at /home/$CONTROLLER_USER/ce-provision already exists. Skipping." /usr/bin/echo "-------------------------------------------------" @@ -168,9 +177,13 @@ fi - name: Install ce-provision. ansible.builtin.import_role: name: debian/ce_provision + - name: Configure controller user. + ansible.builtin.import_role: + name: debian/user_provision EOL # Create vars file. /bin/cat >"/home/$CONTROLLER_USER/ce-provision/vars.yml" << EOL +--- _domain_name: ${SERVER_HOSTNAME} _ce_provision_data_dir: /home/${CONTROLLER_USER}/ce-provision/data _ce_provision_username: ${CONTROLLER_USER} @@ -205,6 +218,22 @@ ce_provision: enabled: true command: "/home/${CONTROLLER_USER}/ce-python/bin/ansible-galaxy collection install --force" on_calendar: "Mon *-*-* 04:00:00" +user_provision: + username: "${CONTROLLER_USER}" + home: "/home/${CONTROLLER_USER}" + create: false + create_home: false + update_password: always + utility_username: "${CONTROLLER_USER}" + utility_host: localhost + sudoer: true + groups: + - bypass2fa + ssh_keys: + - "{{ lookup('file', '/home/${CONTROLLER_USER}/ce-provision/data/localhost/home/${CONTROLLER_USER}/.ssh/id_ecdsa.pub') }}" + ssh_private_keys: [] + known_hosts: [] + known_hosts_hash: true firewall_config: purge: true firewall_state: started @@ -224,18 +253,24 @@ firewall_config: - "80" - "443" EOL + # Tell Ansible this is a Docker container if [ "$IS_LOCAL" = "true" ]; then - /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL, ansible_galaxy.extra_params: --force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" + ANSIBLE_COMMAND="ansible-playbook --extra-vars \"{is_local: $IS_LOCAL}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" else - /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{ansible_galaxy.extra_params: --force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" + ANSIBLE_COMMAND="ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" fi +# Configure ce-provision +/usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/$ANSIBLE_COMMAND" /usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" + +# Install firewall +if [ "$FIREWALL" = "true" ]; then # Create playbook for firewall. -/usr/bin/echo "-------------------------------------------------" -/usr/bin/echo "Install firewall." -/usr/bin/echo "-------------------------------------------------" -/bin/cat >"/home/$CONTROLLER_USER/ce-provision/provision.yml" << EOL + /usr/bin/echo "-------------------------------------------------" + /usr/bin/echo "Install firewall." + /usr/bin/echo "-------------------------------------------------" + /bin/cat >"/home/$CONTROLLER_USER/ce-provision/provision.yml" << EOL --- - hosts: "localhost" become: true @@ -246,8 +281,13 @@ fi ansible.builtin.import_role: name: debian/firewall_config EOL -/usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" -/usr/bin/echo "-------------------------------------------------" + /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" + /usr/bin/echo "-------------------------------------------------" +else + /usr/bin/echo "-------------------------------------------------" + /usr/bin/echo "Skipping firewall." + /usr/bin/echo "-------------------------------------------------" +fi # Install GitLab if [ "$GITLAB_URL" != "no" ]; then @@ -270,6 +310,8 @@ if [ "$GITLAB_URL" != "no" ]; then EOL # Create vars file. /bin/cat >"/home/$CONTROLLER_USER/ce-provision/vars.yml" << EOL +--- +_domain_name: ${SERVER_HOSTNAME} gitlab_runner: apt_origin: "origin=packages.gitlab.com/runner/gitlab-runner,codename=\${distro_codename},label=gitlab-runner" # used by apt_unattended_upgrades apt_signed_by: https://packages.gitlab.com/runner/gitlab-runner/gpgkey @@ -363,6 +405,9 @@ else /usr/bin/echo "GitLab not requested. Skipping." /usr/bin/echo "-------------------------------------------------" fi -/usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/vars.yml" -/usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" +# Tidy up if not a container +if [ "$IS_LOCAL" = "false" ]; then + /usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/vars.yml" + /usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" +fi /usr/bin/echo "DONE." diff --git a/roles/_exit/tasks/main.yml b/roles/_exit/tasks/main.yml index e6a844f4d..51d676278 100644 --- a/roles/_exit/tasks/main.yml +++ b/roles/_exit/tasks/main.yml @@ -53,3 +53,4 @@ ansible.builtin.file: path: "{{ _init.lock_file }}" state: absent + when: _init.lock_file | length > 0 diff --git a/roles/_init/README.md b/roles/_init/README.md index 0f40180e3..e619e5656 100644 --- a/roles/_init/README.md +++ b/roles/_init/README.md @@ -9,7 +9,10 @@ This is meant to ALWAYS be included as the first task of a play. If you include ## Default variables ```yaml --- -_ce_provision_username: "{% if is_local is defined and is_local %}ce-dev{% else %}controller{% endif %}" +# Set this variable to true to tell ce-provision it is running in a container. +is_local: false + +_ce_provision_username: "{% if is_local %}ce-dev{% else %}controller{% endif %}" _venv_path: "/home/{{ _ce_provision_username }}/ce-python" _venv_command: /usr/bin/python3 -m venv _venv_install_username: "{{ _ce_provision_username }}" @@ -25,8 +28,8 @@ _init: # This is used to detect if the playbook must re-run or not. vars_dirs: [] force_play: false - lock_file: /tmp/ce-provision-lock - deploy_lock_file: /tmp/ce-deploy-lock # must match lock_file in ce-deploy + lock_file: /tmp/ce-provision-lock # set to an empty string to disable locking behaviour + deploy_lock_file: /tmp/ce-deploy-lock # must match lock_file in ce-deploy, set to an empty string to disable locking behaviour ce_provision_version: 2.x # Outputted by the _init role at the start of plays install_ansible: true # set to false to not install Ansible in a venv diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 97a5ad1bf..5c2d85d42 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -1,5 +1,8 @@ --- -_ce_provision_username: "{% if is_local is defined and is_local %}ce-dev{% else %}controller{% endif %}" +# Set this variable to true to tell ce-provision it is running in a container. +is_local: false + +_ce_provision_username: "{% if is_local %}ce-dev{% else %}controller{% endif %}" _venv_path: "/home/{{ _ce_provision_username }}/ce-python" _venv_command: /usr/bin/python3 -m venv _venv_install_username: "{{ _ce_provision_username }}" @@ -15,8 +18,8 @@ _init: # This is used to detect if the playbook must re-run or not. vars_dirs: [] force_play: false - lock_file: /tmp/ce-provision-lock - deploy_lock_file: /tmp/ce-deploy-lock # must match lock_file in ce-deploy + lock_file: /tmp/ce-provision-lock # set to an empty string to disable locking behaviour + deploy_lock_file: /tmp/ce-deploy-lock # must match lock_file in ce-deploy, set to an empty string to disable locking behaviour ce_provision_version: 2.x # Outputted by the _init role at the start of plays install_ansible: true # set to false to not install Ansible in a venv diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index 3207c13f4..ac996f250 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -8,10 +8,13 @@ - name: Check for a ce-deploy lock file. ansible.builtin.stat: path: "{{ _init.deploy_lock_file }}" + when: _init.deploy_lock_file | length > 0 register: _ce_deploy_lock - name: Abort if ce-deploy lock file exists. - when: _ce_deploy_lock.stat.exists is defined and _ce_deploy_lock.stat.exists + when: + - _init.deploy_lock_file | length > 0 + - _ce_deploy_lock.stat.exists is defined and _ce_deploy_lock.stat.exists block: - name: Abort if ce-deploy lock file is found. ansible.builtin.debug: @@ -31,6 +34,7 @@ path: "{{ _init.lock_file }}" state: touch mode: 0644 + when: _init.lock_file | length > 0 # Load Linux services into ansible_facts.services. - name: Populate service facts diff --git a/roles/debian/ansible_galaxy/tasks/main.yml b/roles/debian/ansible_galaxy/tasks/main.yml index 5dde4a6df..106943f1d 100644 --- a/roles/debian/ansible_galaxy/tasks/main.yml +++ b/roles/debian/ansible_galaxy/tasks/main.yml @@ -13,7 +13,6 @@ - name: Install ansible-galaxy roles and/or collections. when: - _galaxy_requirements.stat.exists - - not is_local block: - name: Set up the ansible-galaxy command. ansible.builtin.set_fact: diff --git a/roles/debian/apt_repository/tasks/main.yml b/roles/debian/apt_repository/tasks/main.yml index df016422b..cef05098e 100644 --- a/roles/debian/apt_repository/tasks/main.yml +++ b/roles/debian/apt_repository/tasks/main.yml @@ -66,6 +66,7 @@ - apt_repository.signed_by is defined - apt_repository.signed_by | length > 0 - apt_repository.signed_by is url # https://docs.ansible.com/ansible/latest/collections/ansible/builtin/url_test.html + - not is_local block: - name: Create script to refresh APT repository key. ansible.builtin.template: diff --git a/roles/debian/aws_efs_client/README.md b/roles/debian/aws_efs_client/README.md index 0711f04d9..5fc392261 100644 --- a/roles/debian/aws_efs_client/README.md +++ b/roles/debian/aws_efs_client/README.md @@ -46,7 +46,7 @@ _mount_state: present aws_efs_client: aws_profile: example # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set region: eu-west-1 # AWS region name - can be substituted for "{{ _aws_region }}" if set - version: 2.1.0 # version of AWS EFS utils to use + version: "{{ '1.35.0' if ansible_distribution_major_version | int < 12 else '2.1.0' }}" # 2.1.0 requires libssl v3 which is absent on Debian < 12 by default. build_suffix: "-1_all" # sometimes there is a suffix appended to the package name, e.g. `amazon-efs-utils-1.35.0-1_all.deb` deb_url: "" # provide an alternative location for the .deb package # See https://docs.ansible.com/ansible/latest/modules/mount_module.html diff --git a/roles/debian/ce_deploy/README.md b/roles/debian/ce_deploy/README.md index 90aa38931..743cbf8cd 100644 --- a/roles/debian/ce_deploy/README.md +++ b/roles/debian/ce_deploy/README.md @@ -8,7 +8,7 @@ Installs Code Enigma's deploy stack on a server. ```yaml --- _ce_deploy: - username: "{% if is_local is defined and is_local %}ce-dev{% else %}deploy{% endif %}" + username: "{% if is_local %}ce-dev{% else %}deploy{% endif %}" ce_deploy: # These are usually set in the _init role using _venv_path, _venv_command and _venv_install_username but can be overridden. diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index 2999edae9..b4af9748b 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -1,6 +1,6 @@ --- _ce_deploy: - username: "{% if is_local is defined and is_local %}ce-dev{% else %}deploy{% endif %}" + username: "{% if is_local %}ce-dev{% else %}deploy{% endif %}" ce_deploy: # These are usually set in the _init role using _venv_path, _venv_command and _venv_install_username but can be overridden. diff --git a/roles/debian/duplicity/tasks/main.yml b/roles/debian/duplicity/tasks/main.yml index 4a06b5857..7f7182e90 100644 --- a/roles/debian/duplicity/tasks/main.yml +++ b/roles/debian/duplicity/tasks/main.yml @@ -101,7 +101,7 @@ - name: Copy include-exclude filelist. ansible.builtin.template: src: include-exclude-filelist.j2 - dest: "{{ duplicity.install_dir }}/etc/{{ dir.name }}-include-filelist" + dest: "{{ duplicity.install_dir }}/etc/{{ dir.name }}-include-exclude-filelist" owner: root group: root mode: 0644 diff --git a/roles/debian/gitlab/README.md b/roles/debian/gitlab/README.md index c02282fd2..f4b11638b 100644 --- a/roles/debian/gitlab/README.md +++ b/roles/debian/gitlab/README.md @@ -38,6 +38,7 @@ gitlab: ssl: # @see the 'ssl' role. Note that domain is autopopulated from server_name above. enabled: false # manual SSL handling disabled by default handling: selfsigned + replace_existing: false # Linux setup linux_user: git linux_group: git diff --git a/roles/debian/gitlab/defaults/main.yml b/roles/debian/gitlab/defaults/main.yml index f09d68f18..e2ed2ff05 100644 --- a/roles/debian/gitlab/defaults/main.yml +++ b/roles/debian/gitlab/defaults/main.yml @@ -25,6 +25,7 @@ gitlab: ssl: # @see the 'ssl' role. Note that domain is autopopulated from server_name above. enabled: false # manual SSL handling disabled by default handling: selfsigned + replace_existing: false # Linux setup linux_user: git linux_group: git diff --git a/roles/debian/gitlab/tasks/main.yml b/roles/debian/gitlab/tasks/main.yml index f98820592..00b55e162 100644 --- a/roles/debian/gitlab/tasks/main.yml +++ b/roles/debian/gitlab/tasks/main.yml @@ -87,20 +87,19 @@ src: gitlab-config.rb.j2 dest: /etc/gitlab/gitlab-config.rb -- name: Stop Gitlab. - ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl stop - -- name: Reconfigure Gitlab. - ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl reconfigure - - name: Let Gitlab know it's on Docker. ansible.builtin.copy: content: "gitlab-docker" dest: "/opt/gitlab/embedded/service/gitlab-rails/INSTALLATION_TYPE" mode: "0666" - when: - - is_local is defined - - is_local + when: is_local + +- name: Stop Gitlab. + ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl stop + +- name: Reconfigure Gitlab. + ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl reconfigure + when: not is_local - name: Copy startup script in place. ansible.builtin.template: @@ -110,9 +109,7 @@ group: root mode: "0555" force: true - when: - - is_local is defined - - is_local + when: is_local - name: Trigger overrides ansible.builtin.include_role: @@ -125,12 +122,12 @@ - name: Manually restart Gitlab/Docker. ansible.builtin.command: "/bin/sh /opt/gitlab-init.sh" - when: - - is_local is defined - - is_local + when: is_local - name: Ensure GitLab is started. ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl start +# @TODO - this task fails in CI with GitHub Actions because PostGreSQL isn't running - name: Run the GitLab configuration script for config that cannot be set in gitlab.rb. - ansible.builtin.command: "gitlab-rails runner /etc/gitlab/gitlab-config.rb" + ansible.builtin.command: /opt/gitlab/bin/gitlab-rails runner /etc/gitlab/gitlab-config.rb + when: not is_local diff --git a/roles/debian/gitlab/templates/gitlab-config.rb.j2 b/roles/debian/gitlab/templates/gitlab-config.rb.j2 index d3e1690f1..4a0619be9 100644 --- a/roles/debian/gitlab/templates/gitlab-config.rb.j2 +++ b/roles/debian/gitlab/templates/gitlab-config.rb.j2 @@ -1,9 +1,6 @@ # Disable DSA keys ApplicationSetting.last.update(dsa_key_restriction: -1) -# Disable ECDSA keys -ApplicationSetting.last.update(ecdsa_key_restriction: -1) - # Enforce at least 2048 bits for RSA keys ApplicationSetting.last.update(rsa_key_restriction: 2048) @@ -24,4 +21,3 @@ ApplicationSetting.last.update(signup_enabled: false) # Disable standard sign-in dialogue Gitlab::CurrentSettings.update!(password_authentication_enabled_for_web: false) {% endif %} - diff --git a/roles/debian/gitlab_runner/tasks/main.yml b/roles/debian/gitlab_runner/tasks/main.yml index 745daadc9..017dd4afc 100644 --- a/roles/debian/gitlab_runner/tasks/main.yml +++ b/roles/debian/gitlab_runner/tasks/main.yml @@ -138,7 +138,7 @@ become: true - name: Restart gitlab-runner service. - ansible.builtin.service: + ansible.builtin.systemd_service: name: gitlab-runner state: restarted daemon_reload: true diff --git a/roles/debian/jitsi/tasks/main.yml b/roles/debian/jitsi/tasks/main.yml index ec485f6ba..cef38260b 100644 --- a/roles/debian/jitsi/tasks/main.yml +++ b/roles/debian/jitsi/tasks/main.yml @@ -27,7 +27,7 @@ DefaultTasksMax=65000 - name: Force systemd to reread configs. - ansible.builtin.systemd: + ansible.builtin.systemd_service: daemon_reload: true - name: Write interactive hostname value for automated installation. diff --git a/roles/debian/locales/tasks/main.yml b/roles/debian/locales/tasks/main.yml index 7cdf73af4..e6e8af3bf 100644 --- a/roles/debian/locales/tasks/main.yml +++ b/roles/debian/locales/tasks/main.yml @@ -33,4 +33,4 @@ - name: Set timezone. community.general.timezone: name: "{{ locales.timezone }}" - when: not is_local is defined or not is_local + when: not is_local diff --git a/roles/debian/mysql_server_mariadb/tasks/main.yml b/roles/debian/mysql_server_mariadb/tasks/main.yml index f76d88ca7..f463d95e8 100644 --- a/roles/debian/mysql_server_mariadb/tasks/main.yml +++ b/roles/debian/mysql_server_mariadb/tasks/main.yml @@ -33,12 +33,12 @@ - name: Update MySQL root password. ansible.builtin.shell: > mysql -NBe 'GRANT ALL ON *.* TO "root"@"%" IDENTIFIED BY "root" WITH GRANT OPTION;' - when: is_local is defined and is_local + when: is_local - name: Update MySQL ce-dev password. ansible.builtin.shell: > mysql -NBe 'GRANT ALL ON *.* TO "ce-dev"@"%" IDENTIFIED BY "ce-dev" WITH GRANT OPTION;' - when: is_local is defined and is_local + when: is_local - name: Ensure mysql can write data. ansible.builtin.file: diff --git a/roles/debian/mysql_server_oracle_ce/tasks/main.yml b/roles/debian/mysql_server_oracle_ce/tasks/main.yml index b427af88c..f42b70f5b 100644 --- a/roles/debian/mysql_server_oracle_ce/tasks/main.yml +++ b/roles/debian/mysql_server_oracle_ce/tasks/main.yml @@ -77,12 +77,12 @@ - name: Update MySQL root password. ansible.builtin.shell: > mysql -NBe 'GRANT ALL ON *.* TO "root"@"%" IDENTIFIED BY "root" WITH GRANT OPTION;' - when: is_local is defined and is_local + when: is_local - name: Update MySQL ce-dev password. ansible.builtin.shell: > mysql -NBe 'GRANT ALL ON *.* TO "ce-dev"@"%" IDENTIFIED BY "ce-dev" WITH GRANT OPTION;' - when: is_local is defined and is_local + when: is_local - name: Ensure MySQL can write data. ansible.builtin.file: diff --git a/roles/debian/nginx/templates/drupal10.j2 b/roles/debian/nginx/templates/drupal10.j2 index fee75d6c5..a4dfa5449 100644 --- a/roles/debian/nginx/templates/drupal10.j2 +++ b/roles/debian/nginx/templates/drupal10.j2 @@ -81,7 +81,7 @@ location ~ ^/(index|cron|modules\/statistics\/statistics|core\/modules\/statisti try_files @phpprocess @phpprocess; } # Allow install/update for local stack. -{% if is_local is defined and is_local %} +{% if is_local %} location ~ ^/(install|update)\.php$ { try_files @phpprocess @phpprocess; } diff --git a/roles/debian/nginx/templates/drupal_common.j2 b/roles/debian/nginx/templates/drupal_common.j2 index 770203509..3e406dd6d 100644 --- a/roles/debian/nginx/templates/drupal_common.j2 +++ b/roles/debian/nginx/templates/drupal_common.j2 @@ -72,7 +72,7 @@ location ~ ^/(index|cron|modules\/statistics\/statistics|core\/modules\/statisti try_files @phpprocess @phpprocess; } # Allow install/update for local stack. -{% if is_local is defined and is_local %} +{% if is_local %} location ~ ^/(install|update)\.php$ { try_files @phpprocess @phpprocess; } diff --git a/roles/debian/nginx/templates/mautic.j2 b/roles/debian/nginx/templates/mautic.j2 index 92845e808..76de23a5c 100644 --- a/roles/debian/nginx/templates/mautic.j2 +++ b/roles/debian/nginx/templates/mautic.j2 @@ -60,7 +60,7 @@ location ~ /(addons|plugins)/.*/Assets/ { } # Allow index_dev/upgrade for local stack. -{% if is_local is defined and is_local %} +{% if is_local %} location ~ ^/(index_dev|upgrade)\.php$ { try_files @phpprocess @phpprocess; } diff --git a/roles/debian/php-fpm/README.md b/roles/debian/php-fpm/README.md index 063e97276..443ffbf87 100644 --- a/roles/debian/php-fpm/README.md +++ b/roles/debian/php-fpm/README.md @@ -23,7 +23,7 @@ php: # It is important to scale up processes on bigger servers, so that more # requests can be handled. Double the number of vCPUs is a good default. # Can be between 5 and 64. - max_children: "{{ [5, [ansible_facts.ansible_processor_nproc * 2, 64] | min] | max }}" + max_children: "{{ [5, [(ansible_facts.ansible_processor_nproc | default(1)) * 2, 64] | min] | max }}" # Fallback in case ansible_processor_nproc is not gathered before tasks start_servers: 2 min_spare_servers: 1 max_spare_servers: 3 diff --git a/roles/debian/php_blackfire/defaults/main.yml b/roles/debian/php_blackfire/defaults/main.yml index 323e94a55..0e0e5c8a9 100644 --- a/roles/debian/php_blackfire/defaults/main.yml +++ b/roles/debian/php_blackfire/defaults/main.yml @@ -6,3 +6,5 @@ blackfire: agent_server_token: "" agent_client_id: "" agent_client_token: "" + agent_cli_config_path: /home/ce-dev/.blackfire.ini + agent_cli_config_owner: ce-dev diff --git a/roles/debian/php_blackfire/tasks/main.yml b/roles/debian/php_blackfire/tasks/main.yml index d7e513f0f..e093d607c 100644 --- a/roles/debian/php_blackfire/tasks/main.yml +++ b/roles/debian/php_blackfire/tasks/main.yml @@ -67,13 +67,13 @@ - name: Generate agent CLI configuration. ansible.builtin.template: src: blackfire.ini.j2 - dest: "/home/vagrant/.blackfire.ini" - owner: vagrant - group: vagrant + dest: "{{ blackfire.agent_cli_config_path }}" + owner: "{{ blackfire.agent_cli_config_owner }}" + group: "{{ blackfire.agent_cli_config_owner }}" mode: 0644 when: - blackfire.enable - - is_local is defined and is_local + - is_local - name: Trigger overrides ansible.builtin.include_role: diff --git a/roles/debian/postfix/tasks/main.yml b/roles/debian/postfix/tasks/main.yml index 83c7ecec6..94d433c28 100644 --- a/roles/debian/postfix/tasks/main.yml +++ b/roles/debian/postfix/tasks/main.yml @@ -99,7 +99,6 @@ state: directory when: - postfix.ce_dev_delivery_mode == "host" - - is_local is defined - is_local - name: Configure procmail to NULL. @@ -110,9 +109,7 @@ group: root mode: "0644" force: true - when: - - is_local is defined - - is_local + when: is_local - name: Configure procmail to host directory. ansible.builtin.template: @@ -124,7 +121,6 @@ force: true when: - postfix.ce_dev_delivery_mode == "host" - - is_local is defined - is_local - name: Configure procmail to local. @@ -133,7 +129,6 @@ state: absent when: - postfix.ce_dev_delivery_mode == "local" - - is_local is defined - is_local # Needed for Docker. diff --git a/roles/debian/postfix/templates/main.cf.j2 b/roles/debian/postfix/templates/main.cf.j2 index e79389db4..6addd3c23 100644 --- a/roles/debian/postfix/templates/main.cf.j2 +++ b/roles/debian/postfix/templates/main.cf.j2 @@ -52,7 +52,7 @@ smtpd_tls_key_file = {{ postfix.ssl.smtp_tls_key_file }} smtpd_tls_CApath = {{ postfix.ssl.smtp_tls_CApath }} smtpd_tls_CAfile = {{ postfix.ssl.smtp_tls_CAfile }} {% endif %} -{% if (is_local is defined) and is_local %} +{% if is_local %} # Force all mail to ce-dev user. virtual_alias_domains = "" virtual_alias_maps = static:ce-dev diff --git a/roles/debian/user_deploy/defaults/main.yml b/roles/debian/user_deploy/defaults/main.yml index dc22f8164..1d83a8cd7 100644 --- a/roles/debian/user_deploy/defaults/main.yml +++ b/roles/debian/user_deploy/defaults/main.yml @@ -1,5 +1,5 @@ --- -_user_deploy_username: "{% if is_local is defined and is_local %}ce-dev{% else %}deploy{% endif %}" +_user_deploy_username: "{% if is_local %}ce-dev{% else %}deploy{% endif %}" user_deploy: # This sets both username and main group. # If you are using ce-deploy to deploy code this must match the `deploy_user` variable diff --git a/roles/debian/user_provision/defaults/main.yml b/roles/debian/user_provision/defaults/main.yml index 7d07a8cc2..f10896c32 100644 --- a/roles/debian/user_provision/defaults/main.yml +++ b/roles/debian/user_provision/defaults/main.yml @@ -1,5 +1,5 @@ --- -_user_provision_username: "{% if is_local is defined and is_local %}ce-dev{% else %}controller{% endif %}" +_user_provision_username: "{% if is_local %}ce-dev{% else %}controller{% endif %}" user_provision: # This sets both username and main group. username: "{{ _user_provision_username }}"