adding mongoose_subdomain_utils

some notes to the changes:
* mod_event_pusher_push:maybe_enable_node/5:
   - first of all, previous implementation had problems with federation pub-sub servers.
   - also, having dynamic domains enabled, it might be too expensive to get all the domains.
   - and that dynamic domains are added in the lazy manner to the routing table
* rework of the virtual_pubsub_hosts option processing/validation:
   - @hosts@ template was a bit confusing, people were asking about the difference to @host@ template all the time.
   - it is still possible to configure manually all the sub-hosts, if there's such a need.
   - now processing of the virtual_pubsub_hosts is done in the same way as other host templates
   - there is no need in storing separate normalized_virtual_pubsub_hosts option, so we can remove gen_mod:set_module_opt/4 call

big_tests/tests/distributed_helper.erl

@@ -176,3 +176,5 @@ mongooseim_script(Node, Cmd, Args, Config) ->
     CtlCmd = script_path(Node, Config, "mongooseim"),
     ejabberdctl_helper:run(CtlCmd, [Cmd | Args]).
 
+subhost_pattern(SubhostTemplate) ->
+    rpc(mim(), mongoose_subdomain_utils, make_subdomain_pattern, [SubhostTemplate]).

big_tests/tests/domain_isolation_SUITE.erl

@@ -4,7 +4,7 @@
 -include_lib("eunit/include/eunit.hrl").
 
 -compile(export_all).
--import(distributed_helper, [mim/0, require_rpc_nodes/1, rpc/4]).
+-import(distributed_helper, [mim/0, require_rpc_nodes/1, rpc/4, subhost_pattern/1]).
 
 suite() ->
     require_rpc_nodes([mim]).
@@ -32,7 +32,7 @@ end_per_suite(Config) ->
 
 init_per_testcase(TestcaseName, Config) ->
     Host = domain(),
-    MucHost = binary_to_list(<<"muclight.", Host/binary>>),
+    MucHost = subhost_pattern(<<"muclight.", Host/binary>>),
     dynamic_modules:start(Host, mod_domain_isolation, [{extra_domains, [MucHost]}]),
     dynamic_modules:start(Host, mod_muc_light, [{host, MucHost}]),
     escalus:init_per_testcase(TestcaseName, Config).

big_tests/tests/domain_removal_SUITE.erl

@@ -14,7 +14,7 @@
          mam_muc_removal/1,
          inbox_removal/1]).
 
--import(distributed_helper, [mim/0, rpc/4]).
+-import(distributed_helper, [mim/0, rpc/4, subhost_pattern/1]).
 
 -include("mam_helper.hrl").
 -include_lib("escalus/include/escalus.hrl").
@@ -68,8 +68,8 @@ end_per_group(_Groupname, Config) ->
     ok.
 
 group_to_modules(mam_removal) ->
-    MH = muc_light_helper:muc_host(),
-    [{mod_mam_meta, [{backend, rdbms}, {pm, []}, {muc, [{host, MH}]}]},
+    HostPattern = subhost_pattern(muc_light_helper:muc_host()),
+    [{mod_mam_meta, [{backend, rdbms}, {pm, []}, {muc, [{host, HostPattern}]}]},
      {mod_muc_light, []}];
 group_to_modules(inbox_removal) ->
     [{mod_inbox, []}].

big_tests/tests/gdpr_SUITE.erl

@@ -60,8 +60,7 @@
 
 -import(ejabberdctl_helper, [ejabberdctl/3]).
 
--import(distributed_helper, [mim/0,
-                             rpc/4]).
+-import(distributed_helper, [mim/0, subhost_pattern/1, rpc/4]).
 
 -import(muc_light_helper, [room_bin_jid/1]).
 
@@ -354,7 +353,7 @@ groupchat_module(muc) ->
     [];
 groupchat_module(muclight) ->
     [{mod_muc_light,
-     [{host, binary_to_list(?MUCLIGHTHOST)},
+     [{host, subhost_pattern(?MUCLIGHTHOST)},
       {backend, mongoose_helper:mnesia_or_rdbms_backend()},
       {rooms_in_rosters, true}]}].
 
@@ -368,28 +367,32 @@ mam_required_modules(CN, Backend) when CN =:= remove_mam_pm;
                      {pm, [{archive_groupchats, false}]}]}];
 mam_required_modules(CN, Backend) when CN =:= retrieve_mam_pm_and_muc_light_dont_interfere;
                                        CN =:= retrieve_mam_muc_light ->
+    HostPattern = subhost_pattern("muclight.@HOST@"),
     [{mod_mam_meta, [{backend, Backend},
                      {pm, [{archive_groupchats, false}]},
-                     {muc, [{host, "muclight.@HOST@"}]}]},
-     {mod_muc_light, [{host, "muclight.@HOST@"}]}];
+                     {muc, [{host, HostPattern}]}]},
+     {mod_muc_light, [{host, HostPattern}]}];
 mam_required_modules(retrieve_mam_pm_and_muc_light_interfere, Backend) ->
+    HostPattern = subhost_pattern("muclight.@HOST@"),
     [{mod_mam_meta, [{backend, Backend},
                      {rdbms_message_format, simple}, %% ignored for any other than rdbms backend
                      simple, %% used only by cassandra backend
                      {pm, [{archive_groupchats, true}]},
-                     {muc, [{host, "muclight.@HOST@"}]}]},
-     {mod_muc_light, [{host, "muclight.@HOST@"}]}];
+                     {muc, [{host, HostPattern}]}]},
+     {mod_muc_light, [{host, HostPattern}]}];
 mam_required_modules(CN, Backend) when CN =:= retrieve_mam_muc_private_msg;
                                        CN =:= retrieve_mam_muc ->
+    HostPattern = subhost_pattern("muc.@HOST@"),
     [{mod_mam_meta, [{backend, Backend},
                      {pm, [{archive_groupchats, false}]},
-                     {muc, [{host, "muc.@HOST@"}]}]},
-     {mod_muc, [{host, "muc.@HOST@"}]}];
+                     {muc, [{host, HostPattern}]}]},
+     {mod_muc, [{host, HostPattern}]}];
 mam_required_modules(retrieve_mam_muc_store_pm, Backend) ->
+    HostPattern = subhost_pattern("muc.@HOST@"),
     [{mod_mam_meta, [{backend, Backend},
                      {pm, [{archive_groupchats, true}]},
-                     {muc, [{host, "muc.@HOST@"}]}]},
-     {mod_muc, [{host, "muc.@HOST@"}]}].
+                     {muc, [{host, HostPattern}]}]},
+     {mod_muc, [{host, HostPattern}]}].
 
 pick_enabled_backend() ->
     BackendsList = [
@@ -408,9 +411,10 @@ offline_required_modules() ->
 pubsub_required_modules() ->
     pubsub_required_modules([<<"flat">>, <<"pep">>, <<"push">>]).
 pubsub_required_modules(Plugins) ->
+    HostPattern = subhost_pattern("pubsub.@HOST@"),
     [{mod_caps, []}, {mod_pubsub, [
                                    {backend, mongoose_helper:mnesia_or_rdbms_backend()},
-                                   {host, "pubsub.@HOST@"},
+                                   {host, HostPattern},
                                    {nodetree, <<"tree">>},
                                    {plugins, Plugins}
                                   ]

big_tests/tests/inbox_helper.erl

@@ -71,6 +71,7 @@
 
 -import(muc_helper, [foreach_recipient/2]).
 -import(muc_light_helper, [lbin/1]).
+-import(distributed_helper, [subhost_pattern/1]).
 
 -define(NS_ESL_INBOX, <<"erlang-solutions.com:xmpp:inbox:0">>).
 -define(NS_ESL_INBOX_CONVERSATION, <<"erlang-solutions.com:xmpp:inbox:0#conversation">>).
@@ -132,7 +133,7 @@ skip_or_run_inbox_tests(TestCases) ->
 
 required_modules() ->
     [
-     {mod_muc_light, [{host, binary_to_list(muclight_config_domain())},
+     {mod_muc_light, [{host, subhost_pattern(muclight_config_domain())},
                       {backend, rdbms}]},
      {mod_inbox, inbox_opts()}
     ].

big_tests/tests/mam_SUITE.erl

@@ -119,6 +119,7 @@
 
 -import(distributed_helper, [mim/0,
                              require_rpc_nodes/1,
+                             subhost_pattern/1,
                              rpc/4]).
 
 -import(muc_helper,
@@ -677,56 +678,61 @@ init_modules(rdbms, muc_light, Config) ->
     init_module(host(), mod_mam_muc_rdbms_arch, []),
     Config1;
 init_modules(BT = riak_timed_yz_buckets, muc_light, Config) ->
-    dynamic_modules:start(host(), mod_muc_light, [{host, binary_to_list(muc_light_host())}]),
+    dynamic_modules:start(host(), mod_muc_light, [{host, subhost_pattern(muc_light_host())}]),
     init_modules(BT, generic, [{muc_domain, "muclight.@HOST@"} | Config]);
 init_modules(BT = cassandra, muc_light, config) ->
     init_modules_for_muc_light(BT, config);
 init_modules(cassandra, muc_all, Config) ->
     init_module(host(), mod_mam_muc_cassandra_arch, []),
-    init_module(host(), mod_mam_muc, [{host, muc_domain(Config)}]),
+    init_module(host(), mod_mam_muc, [{host, subhost_pattern(muc_domain(Config))}]),
     Config;
 init_modules(BT = elasticsearch, muc_light, config) ->
     init_modules_for_muc_light(BT, config);
 init_modules(elasticsearch, muc_all, Config) ->
     init_module(host(), mod_mam_muc_elasticsearch_arch, []),
-    init_module(host(), mod_mam_muc, [{host, muc_domain(Config)}]),
+    init_module(host(), mod_mam_muc, [{host, subhost_pattern(muc_domain(Config))}]),
     Config;
 init_modules(rdbms, C, Config) when C =:= muc_all;
                                     C =:= muc_disabled_retraction ->
     init_module(host(), mod_mam_muc_rdbms_arch, []),
     init_module(host(), mod_mam_rdbms_prefs, [muc]),
     init_module(host(), mod_mam_rdbms_user, [muc, pm]),
-    init_module(host(), mod_mam_muc, [{host, muc_domain(Config)}] ++ addin_mam_options(C, Config)),
+    init_module(host(), mod_mam_muc, [{host, subhost_pattern(muc_domain(Config))}] ++
+                                     addin_mam_options(C, Config)),
     Config;
 init_modules(rdbms_simple, C, Config) when C =:= muc_all;
                                            C =:= muc_disabled_retraction ->
     init_module(host(), mod_mam_muc_rdbms_arch, []),
     init_module(host(), mod_mam_rdbms_prefs, [muc]),
     init_module(host(), mod_mam_rdbms_user, [muc, pm]),
-    init_module(host(), mod_mam_muc, [{host, muc_domain(Config)}] ++ addin_mam_options(C, Config)),
+    init_module(host(), mod_mam_muc, [{host, subhost_pattern(muc_domain(Config))}] ++
+                                     addin_mam_options(C, Config)),
     Config;
 init_modules(rdbms_async_pool, C, Config) when C =:= muc_all;
                                                C =:= muc_disabled_retraction ->
     init_module(host(), mod_mam_muc_rdbms_arch, [no_writer]),
     init_module(host(), mod_mam_muc_rdbms_async_pool_writer, [{flush_interval, 1}]), %% 1ms
     init_module(host(), mod_mam_rdbms_prefs, [muc]),
     init_module(host(), mod_mam_rdbms_user, [muc, pm]),
-    init_module(host(), mod_mam_muc, [{host, muc_domain(Config)}] ++ addin_mam_options(C, Config)),
+    init_module(host(), mod_mam_muc, [{host, subhost_pattern(muc_domain(Config))}] ++
+                                     addin_mam_options(C, Config)),
     Config;
 init_modules(rdbms_mnesia, C, Config) when C =:= muc_all;
                                            C =:= muc_disabled_retraction ->
     init_module(host(), mod_mam_muc_rdbms_arch, []),
     init_module(host(), mod_mam_mnesia_prefs, [muc]),
     init_module(host(), mod_mam_rdbms_user, [muc, pm]),
-    init_module(host(), mod_mam_muc, [{host, muc_domain(Config)}] ++ addin_mam_options(C, Config)),
+    init_module(host(), mod_mam_muc, [{host, subhost_pattern(muc_domain(Config))}] ++
+                                     addin_mam_options(C, Config)),
     Config;
 init_modules(rdbms_cache, C, Config) when C =:= muc_all;
                                           C =:= muc_disabled_retraction ->
     init_module(host(), mod_mam_muc_rdbms_arch, []),
     init_module(host(), mod_mam_rdbms_prefs, [muc]),
     init_module(host(), mod_mam_rdbms_user, [muc, pm]),
     init_module(host(), mod_mam_cache_user, [muc]),
-    init_module(host(), mod_mam_muc, [{host, muc_domain(Config)}] ++ addin_mam_options(C, Config)),
+    init_module(host(), mod_mam_muc, [{host, subhost_pattern(muc_domain(Config))}] ++
+                                     addin_mam_options(C, Config)),
     Config;
 init_modules(rdbms_async_cache, C, Config) when C =:= muc_all;
                                                 C =:= muc_disabled_retraction ->
@@ -735,15 +741,17 @@ init_modules(rdbms_async_cache, C, Config) when C =:= muc_all;
     init_module(host(), mod_mam_rdbms_prefs, [muc]),
     init_module(host(), mod_mam_rdbms_user, [muc, pm]),
     init_module(host(), mod_mam_cache_user, [muc]),
-    init_module(host(), mod_mam_muc, [{host, muc_domain(Config)}] ++ addin_mam_options(C, Config)),
+    init_module(host(), mod_mam_muc, [{host, subhost_pattern(muc_domain(Config))}] ++
+                                     addin_mam_options(C, Config)),
     Config;
 init_modules(rdbms_mnesia_muc_cache, C, Config) when C =:= muc_all;
                                                      C =:= muc_disabled_retraction ->
     init_module(host(), mod_mam_muc_rdbms_arch, []),
     init_module(host(), mod_mam_mnesia_prefs, [muc]),
     init_module(host(), mod_mam_rdbms_user, [muc, pm]),
     init_module(host(), mod_mam_muc_cache_user, [muc]),
-    init_module(host(), mod_mam_muc, [{host, muc_domain(Config)}] ++ addin_mam_options(C, Config)),
+    init_module(host(), mod_mam_muc, [{host, subhost_pattern(muc_domain(Config))}] ++
+                                     addin_mam_options(C, Config)),
     Config;
 init_modules(rdbms_mnesia_muc_cache, _, _Config) ->
     skip;
@@ -753,7 +761,8 @@ init_modules(rdbms_mnesia_cache, C, Config) when C =:= muc_all;
     init_module(host(), mod_mam_mnesia_prefs, [muc]),
     init_module(host(), mod_mam_rdbms_user, [muc, pm]),
     init_module(host(), mod_mam_cache_user, [muc]),
-    init_module(host(), mod_mam_muc, [{host, muc_domain(Config)}] ++ addin_mam_options(C, Config)),
+    init_module(host(), mod_mam_muc, [{host, subhost_pattern(muc_domain(Config))}] ++
+                                     addin_mam_options(C, Config)),
     Config;
 init_modules(BackendType, muc_light, Config) ->
     Config1 = init_modules_for_muc_light(BackendType, Config),
@@ -780,8 +789,8 @@ init_modules(riak_timed_yz_buckets, C, Config) ->
     init_module(host(), mod_mam_mnesia_prefs, [pm, muc,
                                                {archive_key, mam_archive_key_server_user}]),
     init_module(host(), mod_mam, addin_mam_options(C, Config)),
-    init_module(host(), mod_mam_muc,
-                [{host, muc_domain(Config)}] ++ addin_mam_options(C, Config)),
+    init_module(host(), mod_mam_muc, [{host, subhost_pattern(muc_domain(Config))}] ++
+                                     addin_mam_options(C, Config)),
     Config;
 init_modules(cassandra, C, Config) ->
     init_module(host(), mod_mam_cassandra_arch, [pm]),
@@ -840,7 +849,7 @@ rdbms_simple_opts() ->
     [{db_jid_format, mam_jid_rfc}, {db_message_format, mam_message_xml}].
 
 init_modules_for_muc_light(BackendType, Config) ->
-    dynamic_modules:start(host(), mod_muc_light, [{host, binary_to_list(muc_light_host())}]),
+    dynamic_modules:start(host(), mod_muc_light, [{host, subhost_pattern(muc_light_host())}]),
     Config1 = init_modules(BackendType, muc_all, [{muc_domain, "muclight.@HOST@"} | Config]),
     init_modules(BackendType, pm, [{archive_groupchats, false} | Config1]).
 

big_tests/tests/mam_send_message_SUITE.erl

@@ -22,6 +22,7 @@
 
 -import(distributed_helper, [mim/0,
                              require_rpc_nodes/1,
+                             subhost_pattern/1,
                              rpc/4]).
 
 -include("mam_helper.hrl").
@@ -75,7 +76,7 @@ end_per_group(_Groupname, Config) ->
     ok.
 
 group_to_modules(send_message) ->
-    MH = muc_light_helper:muc_host(),
+    MH = subhost_pattern(muc_light_helper:muc_host()),
     [{mod_mam_meta, [{backend, rdbms}, {pm, []}, {muc, [{host, MH}]},
                      {send_message, mam_send_message_example}]},
      {mod_muc_light, []},

big_tests/tests/muc_SUITE.erl

@@ -25,6 +25,7 @@
 -include("assert_received_match.hrl").
 
 -import(distributed_helper, [mim/0,
+                             subhost_pattern/1,
                              rpc/4]).
 
 -import(muc_helper,
@@ -378,10 +379,11 @@ init_per_group(G, Config) when G =:= http_auth_no_server;
 init_per_group(hibernation, Config) ->
     case mam_helper:backend() of
         rdbms ->
-    dynamic_modules:start(domain(), mod_mam_muc_rdbms_arch, [muc]),
-    dynamic_modules:start(domain(), mod_mam_rdbms_prefs, [muc]),
-    dynamic_modules:start(domain(), mod_mam_rdbms_user, [pm, muc]),
-            dynamic_modules:start(domain(), mod_mam_muc, [{host, "muc.@HOST@"}]);
+            dynamic_modules:start(domain(), mod_mam_muc_rdbms_arch, [muc]),
+            dynamic_modules:start(domain(), mod_mam_rdbms_prefs, [muc]),
+            dynamic_modules:start(domain(), mod_mam_rdbms_user, [pm, muc]),
+            HostPattern = subhost_pattern("muc.@HOST@"),
+            dynamic_modules:start(domain(), mod_mam_muc, [{host, HostPattern}]);
         _ ->
             ok
     end,
@@ -397,7 +399,7 @@ init_per_group(_GroupName, Config) ->
 
 required_modules(http_auth) ->
     [{mod_muc, [
-                {host, "muc.@HOST@"},
+                {host, subhost_pattern("muc.@HOST@")},
                 {access, muc},
                 {access_create, muc_create},
                 {http_auth_pool, muc_http_auth_test},
@@ -530,7 +532,7 @@ init_per_testcase(CaseName, Config) when CaseName =:= disco_features_with_mam;
                                          CaseName =:= disco_info_with_mam ->
     dynamic_modules:start(domain(), mod_mam_muc,
                           [{backend, rdbms},
-                           {host, binary_to_list(muc_host())}]),
+                           {host, subhost_pattern(muc_host())}]),
     escalus:init_per_testcase(CaseName, Config);
 
 init_per_testcase(CaseName, Config) ->

big_tests/tests/muc_helper.erl

@@ -6,6 +6,7 @@
 -include_lib("escalus/include/escalus_xmlns.hrl").
 
 -import(distributed_helper, [mim/0,
+                             subhost_pattern/1,
                              rpc/4]).
 
 -type verify_fun() :: fun((Incoming :: #xmlel{}) -> any()).
@@ -47,7 +48,7 @@ load_muc(Host) ->
     Backend = muc_backend(),
     %% TODO refactoring. "localhost" should be passed as a parameter
     dynamic_modules:start(<<"localhost">>, mod_muc,
-                          [{host, binary_to_list(Host)},
+                          [{host, subhost_pattern(Host)},
                            {backend, Backend},
                            {hibernate_timeout, 2000},
                            {hibernated_room_check_interval, 1000},

big_tests/tests/muc_light_SUITE.erl

@@ -74,6 +74,7 @@
 
 -import(escalus_ejabberd, [rpc/3]).
 -import(muc_helper, [foreach_occupant/3, foreach_recipient/2]).
+-import(distributed_helper, [subhost_pattern/1]).
 -import(muc_light_helper, [
                            bin_aff_users/1,
                            gc_message_verify_fun/3,
@@ -199,7 +200,7 @@ suite() ->
 init_per_suite(Config) ->
     Host = ct:get_config({hosts, mim, domain}),
     {ok, _} = dynamic_modules:start(Host, mod_muc_light,
-                                    [{host, binary_to_list(?MUCHOST)},
+                                    [{host, subhost_pattern(?MUCHOST)},
                                      {backend, mongoose_helper:mnesia_or_rdbms_backend()},
                                      {rooms_in_rosters, true}]),
     Config1 = escalus:init_per_suite(Config),
@@ -236,7 +237,7 @@ init_per_testcase(CaseName, Config) when CaseName =:= disco_features_with_mam;
     set_default_mod_config(),
     dynamic_modules:start(domain(), mod_mam_muc,
                           [{backend, rdbms},
-                           {host, binary_to_list(?MUCHOST)}]),
+                           {host, subhost_pattern(?MUCHOST)}]),
     escalus:init_per_testcase(CaseName, Config);
 init_per_testcase(disco_rooms_rsm, Config) ->
     set_default_mod_config(),

big_tests/tests/muc_light_http_api_SUITE.erl

@@ -27,7 +27,7 @@
 -include_lib("exml/include/exml.hrl").
 
 -import(muc_light_helper, [stanza_create_room/3]).
-
+-import(distributed_helper, [subhost_pattern/1]).
 
 %%--------------------------------------------------------------------
 %% Suite configuration
@@ -63,7 +63,7 @@ negative_response() ->
 
 init_per_suite(Config) ->
     dynamic_modules:start(<<"localhost">>, mod_muc_light,
-        [{host, binary_to_list(muc_light_domain())},
+        [{host, subhost_pattern(muc_light_domain())},
          {rooms_in_rosters, true},
          {backend, mongoose_helper:mnesia_or_rdbms_backend()}]),
     escalus:init_per_suite(Config).