Fix rolling restart by waiting for consul leave and after restart

README.md

@@ -839,8 +839,8 @@ _Consul Enterprise Only (requires that CONSUL_ENTERPRISE is set to true)_
  - Restarts consul node one by one to avoid service interruption on existing cluster (Unix platforms only).
  - Default value: *false*
 
-### `consul_rolling_restart_delay_sec`
- - Adds a delay between node restart (Linux platforms only).
+ ### `consul_rolling_restart_delay_sec`
+ - Adds a delay between consul leave and node restart (Linux platforms only).
  - Default value: 5
 
 #### Custom Configuration Section

defaults/main.yml

@@ -50,6 +50,7 @@ consul_log_path: "{{ lookup('env', 'CONSUL_LOG_PATH') | default('/var/log/consul
 consul_log_file: "{{ lookup('env', 'CONSUL_LOG_FILE') | default('consul.log', true) }}"
 consul_run_path: /run/consul
 consul_binary: "{{ consul_bin_path }}/consul"
+consul_pid_file: "{{ consul_run_path }}/consul.pid"
 
 ### System user and group
 consul_manage_user: true

handlers/reload_consul_conf.yml

@@ -3,6 +3,6 @@
 # Cannot use `consul reload` because it requires the HTTP API to be bound to a non-loopback interface
 
 - name: Reload consul configuration on unix # noqa no-changed-when
-  ansible.builtin.command: pkill --pidfile "{{ consul_run_path }}/consul.pid" --signal SIGHUP
+  ansible.builtin.command: pkill --pidfile "{{ consul_pid_file }}" --signal SIGHUP
   when: ansible_os_family != "Windows"
   listen: reload consul configuration

tasks/leave_restart_consul.yml

@@ -1,9 +1,23 @@
 ---
 
-- name: Consul leave
+- name: Check if consul is running
   delegate_to: "{{ rolling_restart_host }}"
-  ansible.builtin.command: "{{ consul_binary }} leave {% if consul_acl_enable %} -token {{ consul_acl_master_token }} {% endif %} -http-addr {{ consul_addresses.http }}:{{ consul_ports.http }}"
-  changed_when: true
+  stat:
+    path: "{{ consul_pid_file }}"
+  register: consul_pid
+
+- block:
+  - name: Consul leave
+    delegate_to: "{{ rolling_restart_host }}"
+    ansible.builtin.command:
+      cmd: "{{ consul_binary }} leave {% if consul_acl_enable %} -token {{ consul_acl_master_token }} {% endif %} -http-addr {{ consul_addresses.http }}:{{ consul_ports.http }}"
+    changed_when: true
+
+  - name: Give the cluster some time to settle
+    ansible.builtin.pause:
+      seconds: "{{ consul_rolling_restart_delay_sec }}"
+    when: consul_rolling_restart_delay_sec > 0
+  when: consul_pid.stat.exists
 
 - name: Restart consul on Unix
   delegate_to: "{{ rolling_restart_host }}"
@@ -13,7 +27,12 @@
     # Needed to force SysV service manager on Docker for Molecule tests
     use: "{{ ansible_service_mgr }}"
 
-- name: Wait for service availability
-  ansible.builtin.pause:
-    seconds: "{{ consul_rolling_restart_delay_sec }}"
-  when: consul_rolling_restart_delay_sec > 0
+- name: Assert that consul service is running
+  delegate_to: "{{ rolling_restart_host }}"
+  ansible.builtin.command:
+    cmd: "{{ consul_binary }} info {% if consul_acl_enable %} -token {{ consul_acl_master_token }} {% endif %} -http-addr {{ consul_addresses.http }}:{{ consul_ports.http }}"
+  changed_when: false
+  register: consul_info
+  retries: 20
+  delay: 1
+  until: "consul_info.rc == 0"

templates/consul_bsdinit.j2

@@ -30,7 +30,7 @@ consul_start() {
     for user in ${consul_users}; do
         mkdir {{ consul_run_path }}
         chown -R "{{ consul_user }}:{{ consul_group }}" {{ consul_run_path }}
-        su -m "${user}" -c "{{ consul_bin_path }}/consul agent -config-file={{ consul_config_path }}/config.json -config-dir={{ consul_configd_path }} -pid-file={{ consul_run_path }}/consul.pid&"
+        su -m "${user}" -c "{{ consul_bin_path }}/consul agent -config-file={{ consul_config_path }}/config.json -config-dir={{ consul_configd_path }} -pid-file={{ consul_pid_file }}&"
     done
 }
 

templates/consul_launchctl.plist.j2

@@ -24,7 +24,7 @@
     <string>agent</string>
     <string>-config-file={{ consul_config_path }}/config.json</string>
     <string>-config-dir={{ consul_configd_path }}</string>
-    <string>-pid-file={{ consul_run_path }}/consul.pid</string>
+    <string>-pid-file={{ consul_pid_file }}</string>
   </array>
   <key>RunAtLoad</key>
   <true/>

templates/consul_smf_manifest.j2

@@ -23,7 +23,7 @@
             <method_credential user="{{ consul_user }}" group="{{ consul_group }}"/>
         </method_context>
 
-        <exec_method type="method" name="start" exec="{{ consul_bin_path }}/consul agent -config-file={{ consul_config_path}}/config.json -config-dir={{ consul_configd_path }} -pid-file={{ consul_run_path }}/consul.pid" timeout_seconds="60"/>
+        <exec_method type="method" name="start" exec="{{ consul_bin_path }}/consul agent -config-file={{ consul_config_path}}/config.json -config-dir={{ consul_configd_path }} -pid-file={{ consul_pid_file }}" timeout_seconds="60"/>
 
         <exec_method type="method" name="stop" exec=":kill" timeout_seconds="60"/>
 

templates/consul_systemd.service.j2

@@ -16,7 +16,7 @@ After=network-online.target
 [Service]
 User={{ consul_user }}
 Group={{ consul_group }}
-PIDFile={{ consul_run_path }}/consul.pid
+PIDFile={{ consul_pid_file }}
 PermissionsStartOnly=true
 {% if consul_ui_legacy %}
 Environment=CONSUL_UI_LEGACY=true
@@ -26,7 +26,7 @@ ExecStartPre=/bin/chown -R {{ consul_user }}:{{ consul_group }} {{ consul_run_pa
 ExecStart={{ consul_bin_path }}/consul agent \
     -config-file={{ consul_config_path }}/config.json \
     -config-dir={{ consul_configd_path}} \
-    -pid-file={{ consul_run_path }}/consul.pid
+    -pid-file={{ consul_pid_file }}
 ExecReload=/bin/kill -HUP $MAINPID
 KillMode=process
 KillSignal=SIGTERM

templates/consul_sysvinit.j2

@@ -3,7 +3,7 @@
 # chkconfig:   2345 95 95
 # description: Consul service discovery framework
 # processname: consul
-# pidfile: {{ consul_run_path }}/consul.pid
+# pidfile: {{ consul_pid_file }}
 
 {% if ansible_distribution == "Ubuntu" %}
 . /lib/lsb/init-functions
@@ -14,7 +14,6 @@
 CONSUL={{ consul_bin_path }}/consul
 CONFIG={{ consul_config_path }}/config.json
 CONFIGD={{ consul_configd_path }}
-PID_FILE={{ consul_run_path }}/consul.pid
 LOCK_FILE=/var/lock/subsys/consul
 {% if consul_ui_legacy %}
 CONSUL_UI_LEGACY=true
@@ -29,13 +28,13 @@ mkrundir() {
         chown {{ consul_user }} {{ consul_run_path }}
 }
 
-KILLPROC_OPT="-p ${PID_FILE}"
+KILLPROC_OPT="-p {{ consul_pid_file }}"
 mkpidfile() {
         mkrundir
-        [ ! -f "${PID_FILE}" ] && pidofproc "${CONSUL}" > "${PID_FILE}"
+        [ ! -f "{{ consul_pid_file }}" ] && pidofproc "${CONSUL}" > "{{ consul_pid_file }}"
         chown -R {{ consul_user }} {{ consul_run_path }}
         if [ $? -ne 0 ] ; then
-            rm "${PID_FILE}"
+            rm "{{ consul_pid_file }}"
             KILLPROC_OPT=""
         fi
 }
@@ -44,10 +43,10 @@ start() {
         echo -n "Starting consul: "
         mkrundir
         mkpidfile
-        # [ -f "${PID_FILE}" ] && rm "${PID_FILE}"
+        # [ -f "{{ consul_pid_file }}" ] && rm "{{ consul_pid_file }}"
         daemon --user={{ consul_user }} \
-            --pidfile="${PID_FILE}" \
-            "${CONSUL}" agent -config-file="${CONFIG}" -config-dir="${CONFIGD}" -pid-file="${PID_FILE}" &
+            --pidfile="{{ consul_pid_file }}" \
+            "${CONSUL}" agent -config-file="${CONFIG}" -config-dir="${CONFIGD}" -pid-file="{{ consul_pid_file }}" &
         retcode=$?
         touch ${LOCK_FILE}
         return "${retcode}"
@@ -63,7 +62,7 @@ stop() {
         killproc "${KILLPROC_OPT}" "${CONSUL}" -SIGTERM
 
         retcode=$?
-        rm -f "${LOCK_FILE}" "${PID_FILE}"
+        rm -f "${LOCK_FILE}" "{{ consul_pid_file }}"
         return "${retcode}"
 }