shuffle domains before renewal, to remedy the process being interrupted

Makefile

@@ -50,9 +50,11 @@ install: check-dependencies
 	install -m 644 lib/resty/auto-ssl/storage_adapters/file.lua $(INST_LUADIR)/resty/auto-ssl/storage_adapters/file.lua
 	install -m 644 lib/resty/auto-ssl/storage_adapters/redis.lua $(INST_LUADIR)/resty/auto-ssl/storage_adapters/redis.lua
 	install -d $(INST_LUADIR)/resty/auto-ssl/utils
+	install -m 644 lib/resty/auto-ssl/utils/random_seed.lua $(INST_LUADIR)/resty/auto-ssl/utils/random_seed.lua
+	install -m 644 lib/resty/auto-ssl/utils/run_command.lua $(INST_LUADIR)/resty/auto-ssl/utils/run_command.lua
 	install -m 644 lib/resty/auto-ssl/utils/shell_execute.lua $(INST_LUADIR)/resty/auto-ssl/utils/shell_execute.lua
+	install -m 644 lib/resty/auto-ssl/utils/shuffle_table.lua $(INST_LUADIR)/resty/auto-ssl/utils/shuffle_table.lua
 	install -m 644 lib/resty/auto-ssl/utils/start_sockproc.lua $(INST_LUADIR)/resty/auto-ssl/utils/start_sockproc.lua
-	install -m 644 lib/resty/auto-ssl/utils/run_command.lua $(INST_LUADIR)/resty/auto-ssl/utils/run_command.lua
 	install -d $(INST_LUADIR)/resty/auto-ssl/vendor
 	install -m 644 lib/resty/auto-ssl/vendor/shell.lua $(INST_LUADIR)/resty/auto-ssl/vendor/shell.lua
 	install -d $(INST_BINDIR)/resty-auto-ssl

lib/resty/auto-ssl/init_master.lua

@@ -1,3 +1,4 @@
+require "resty.auto-ssl.utils.random_seed"
 local resty_random = require "resty.random"
 local run_command = require "resty.auto-ssl.utils.run_command"
 local str = require "resty.string"

lib/resty/auto-ssl/init_worker.lua

@@ -1,3 +1,4 @@
+local random_seed = require "resty.auto-ssl.utils.random_seed"
 local renewal_job = require "resty.auto-ssl.jobs.renewal"
 local run_command = require "resty.auto-ssl.utils.run_command"
 local start_sockproc = require "resty.auto-ssl.utils.start_sockproc"
@@ -13,6 +14,11 @@ return function(auto_ssl_instance)
     ngx.log(ngx.ERR, "auto-ssl: failed to create letsencrypt/locks dir: ", mkdir_locks_err)
   end
 
+  -- random_seed was called during the "init" master phase, but we want to
+  -- ensure each worker process's random seed is different, so force another
+  -- call in the init_worker phase.
+  random_seed()
+
   -- Startup sockproc. This background process allows for non-blocking shell
   -- commands with resty.shell.
   --

lib/resty/auto-ssl/jobs/renewal.lua

@@ -1,5 +1,6 @@
 local lock = require "resty.lock"
 local run_command = require "resty.auto-ssl.utils.run_command"
+local shuffle_table = require "resty.auto-ssl.utils.shuffle_table"
 local ssl_provider = require "resty.auto-ssl.ssl_providers.lets_encrypt"
 
 local _M = {}
@@ -141,6 +142,12 @@ local function renew_all_domains(auto_ssl_instance)
   if domains_err then
     ngx.log(ngx.ERR, "auto-ssl: failed to fetch all certificate domains: ", domains_err)
   else
+    -- Randomize the renewal order so that if nginx is reloaded during renewals
+    -- or rate limits are hit, the renewals are attempted in a different order
+    -- each time (which may allow things to eventually succeed over multiple
+    -- renewal attempts).
+    shuffle_table(domains)
+
     for _, domain in ipairs(domains) do
       renew_check_cert(auto_ssl_instance, storage, domain)
     end

lib/resty/auto-ssl/utils/random_seed.lua

@@ -0,0 +1,24 @@
+local random_bytes = require("resty.random").bytes
+
+-- Seed math.randomseed based on random bytes from openssl. This is better than
+-- relying on something like os.time(), since that can result in identical
+-- seeds for multiple workers started up at the same time.
+--
+-- Based on https://github.com/bungle/lua-resty-random/blob/17b604f7f7dd217557ca548fc1a9a0d373386480/lib/resty/random.lua#L49-L52
+local function seed()
+  local num_bytes = 4
+  local random = random_bytes(num_bytes, true)
+  if not random then
+    ngx.log(ngx.WARN, "Could not generate cryptographically secure random data for seeding math.randomseed. Falling back to non-secure random data.")
+    random = random_bytes(num_bytes, false)
+  end
+
+  local byte1, byte2, byte3, byte4 = string.byte(random, 1, 4)
+  local randomseed = byte1 * 0x1000000 + byte2 * 0x10000 + byte3 * 0x100 + byte4
+
+  return math.randomseed(randomseed)
+end
+
+seed()
+
+return seed

lib/resty/auto-ssl/utils/shuffle_table.lua

@@ -0,0 +1,7 @@
+return function(tbl)
+  for i = #tbl, 1, -1 do
+    local rand = math.random(i)
+    tbl[i], tbl[rand] = tbl[rand], tbl[i]
+  end
+  return tbl
+end