Expand pools taking new host_config ones

src/wpool/mongoose_wpool.erl

@@ -27,19 +27,20 @@
          get_pool_settings/3, get_pools/0, stats/3]).
 
 -export([start_sup_pool/3]).
--export([start_configured_pools/0, start_configured_pools/1, start_configured_pools/2]).
+-export([start_configured_pools/0, start_configured_pools/1,
+         start_configured_pools/2, start_configured_pools/3]).
 -export([is_configured/1]).
 -export([make_pool_name/3]).
 -export([call_start_callback/2]).
 
 %% Mostly for tests
--export([expand_pools/2]).
+-export([expand_pools/3]).
 
--ignore_xref([call/2, cast/2, cast/3, expand_pools/2, get_worker/2,
+-ignore_xref([call/2, cast/2, cast/3, expand_pools/3, get_worker/2,
               send_request/2, send_request/3, send_request/4, send_request/5,
               is_configured/2, is_configured/1, is_configured/1, start/2, start/3,
-              start/5, start_configured_pools/1, start_configured_pools/2, stats/3,
-              stop/1, stop/2]).
+              start/5, start_configured_pools/1, start_configured_pools/2, start_configured_pools/3,
+              stats/3, stop/1, stop/2]).
 
 -type pool_type() :: redis | http | rdbms | cassandra | elastic | generic | rabbit | ldap.
 
@@ -112,7 +113,11 @@ start_configured_pools(PoolsIn) ->
     start_configured_pools(PoolsIn, ?ALL_HOST_TYPES).
 
 start_configured_pools(PoolsIn, HostTypes) ->
-    Pools = expand_pools(PoolsIn, HostTypes),
+    HostTypeSpecific = get_host_type_specific_pools(HostTypes),
+    start_configured_pools(PoolsIn, HostTypeSpecific, HostTypes).
+
+start_configured_pools(PoolsIn, HostTypeSpecific, HostTypes) ->
+    Pools = expand_pools(PoolsIn, HostTypeSpecific, HostTypes),
     [call_callback(init, PoolType, []) || PoolType <- get_unique_types(PoolsIn)],
     [start(Pool) || Pool <- Pools].
 
@@ -348,19 +353,25 @@ make_callback_module_name(PoolType) ->
     Name = "mongoose_wpool_" ++ atom_to_list(PoolType),
     list_to_atom(Name).
 
--spec expand_pools([pool_map_in()], [mongooseim:host_type()]) -> [pool_map()].
-expand_pools(Pools, HostTypes) ->
-    %% First we select only pools for a specific vhost
-    HostSpecific = [{Type, HT, Tag} || #{type := Type, scope := HT, tag := Tag} <- Pools, is_binary(HT)],
-    %% Then we expand all pools with `host_type` as HostType parameter but using host_type specific configs
-    %% if they were provided
+-spec get_host_type_specific_pools([mongooseim:host_type()]) -> [pool_map_in()].
+get_host_type_specific_pools(HostTypes) ->
+    lists:append([ mongoose_config:get_opt({outgoing_pools, HostType}, [])
+                   || HostType <- HostTypes ]).
+
+-spec expand_pools([pool_map_in()], [pool_map_in()], [mongooseim:host_type()]) -> [pool_map()].
+expand_pools(Pools, PerHostType, HostTypes) ->
+    %% First we select only vhost/host_type specific pools
+    HostSpecific = [ {Type, HT, Tag}
+                     || #{type := Type, scope := HT, tag := Tag} <- PerHostType ],
+    %% Then we expand all pools with `host_type` as scope
+    %% but using host_type specific configs if they were provided
     F = fun(M = #{type := PoolType, scope := host_type, tag := Tag}) ->
                 [M#{scope => HostType} || HostType <- HostTypes,
                                           not lists:member({PoolType, HostType, Tag}, HostSpecific)];
            (Other) -> [Other]
         end,
     Pools1 = lists:flatmap(F, Pools),
-    lists:map(fun prepare_pool_map/1, Pools1).
+    lists:map(fun prepare_pool_map/1, PerHostType ++ Pools1).
 
 -spec prepare_pool_map(pool_map_in()) -> pool_map().
 prepare_pool_map(Pool = #{scope := HT, opts := Opts}) ->

test/auth_http_SUITE.erl

@@ -77,7 +77,7 @@ init_per_suite(Config) ->
               % This would be started via outgoing_pools in normal case
               Pool = config([outgoing_pools, http, auth], pool_opts()),
               HostTypes = [?HOST_TYPE, <<"another host type">>],
-              mongoose_wpool:start_configured_pools([Pool], HostTypes),
+              mongoose_wpool:start_configured_pools([Pool], [], HostTypes),
               mongoose_wpool_http:init(),
               ejabberd_auth_http:start(?HOST_TYPE)
       end),

test/ejabberd_sm_SUITE.erl

@@ -78,7 +78,7 @@ init_redis_group(true, Config) ->
                   mongoose_wpool:ensure_started(),
                   % This would be started via outgoing_pools in normal case
                   Pool = default_config([outgoing_pools, redis, default]),
-                  mongoose_wpool:start_configured_pools([Pool], []),
+                  mongoose_wpool:start_configured_pools([Pool], [], []),
                   Self ! ready,
                   receive stop -> ok end
           end),

test/http_client_SUITE.erl

@@ -57,7 +57,7 @@ end_per_suite(_Config) ->
 
 init_per_testcase(TC, Config) ->
     Pool = config([outgoing_pools, http, pool()], pool_opts(TC)),
-    mongoose_wpool:start_configured_pools([Pool], [<<"a.com">>]),
+    mongoose_wpool:start_configured_pools([Pool], [], [<<"a.com">>]),
     Config.
 
 pool_opts(request_timeout_test) ->

test/mongoose_wpool_SUITE.erl

@@ -137,37 +137,40 @@ generic_pools_are_started_for_all_vhosts(_C) ->
                  ordsets:from_list(mongoose_wpool:get_pools())).
 
 host_specific_pools_are_preserved(_C) ->
-    Pools = [#{type => generic, scope => host_type, tag => default, opts => #{}, conn_opts => #{}},
-             #{type => generic, scope => <<"b.com">>, tag => default,
-               opts => #{workers => 12}, conn_opts => #{}}],
-    Expanded = mongoose_wpool:expand_pools(Pools, [<<"a.com">>, <<"b.com">>, <<"c.eu">>]),
-    ?assertMatch([#{type := generic, host_type := <<"a.com">>, tag := default,
-                    opts := [], conn_opts := #{}},
-                  #{type := generic, host_type := <<"c.eu">>, tag := default,
-                    opts := [], conn_opts := #{}},
-                  #{type := generic, host_type := <<"b.com">>, tag := default,
-                    opts := [{workers, 12}], conn_opts := #{}}],
-                 Expanded).
+    Pools = [#{type => generic, scope => host_type, tag => default, opts => #{}, conn_opts => #{}}],
+    HostSpecific = [#{type => generic, scope => <<"b.com">>, tag => default,
+                      opts => #{workers => 12}, conn_opts => #{}}],
+    Expanded = mongoose_wpool:expand_pools(
+                 Pools, HostSpecific, [<<"a.com">>, <<"b.com">>, <<"c.eu">>]),
+    Expected = lists:sort([#{type => generic, host_type => <<"a.com">>, tag => default,
+                             opts => [], conn_opts => #{}},
+                           #{type => generic, host_type => <<"c.eu">>, tag => default,
+                             opts => [], conn_opts => #{}},
+                           #{type => generic, host_type => <<"b.com">>, tag => default,
+                             opts => [{workers, 12}], conn_opts => #{}}]),
+    ?assertMatch(Expected, lists:sort(Expanded)).
 
 pools_for_different_tag_are_expanded_with_host_specific_config_preserved(_C) ->
     Pools = [#{type => generic, scope => host_type, tag => default, opts => #{}, conn_opts => #{}},
-             #{type => generic, scope => <<"b.com">>, tag => default,
-               opts => #{workers => 12}, conn_opts => #{}},
-             #{type => generic, scope => host_type, tag => other_tag, opts => #{}, conn_opts => #{}}],
-    Expanded = mongoose_wpool:expand_pools(Pools, [<<"a.com">>, <<"b.com">>, <<"c.eu">>]),
-    ?assertMatch([#{type := generic, host_type := <<"a.com">>, tag := default,
-                    opts := [], conn_opts := #{}},
-                  #{type := generic, host_type := <<"c.eu">>, tag := default,
-                    opts := [], conn_opts := #{}},
-                  #{type := generic, host_type := <<"b.com">>, tag := default,
-                    opts := [{workers, 12}], conn_opts := #{}},
-                  #{type := generic, host_type := <<"a.com">>, tag := other_tag,
-                    opts := [], conn_opts := #{}},
-                  #{type := generic, host_type := <<"b.com">>, tag := other_tag,
-                    opts := [], conn_opts := #{}},
-                  #{type := generic, host_type := <<"c.eu">>, tag := other_tag,
-                    opts := [], conn_opts := #{}}],
-                 Expanded).
+             #{type => generic, scope => host_type, tag => other_tag,
+               opts => #{}, conn_opts => #{}}],
+    HostSpecific = [#{type => generic, scope => <<"b.com">>, tag => default,
+                      opts => #{workers => 12}, conn_opts => #{}}],
+    Expanded = mongoose_wpool:expand_pools(
+                 Pools, HostSpecific, [<<"a.com">>, <<"b.com">>, <<"c.eu">>]),
+    Expected = lists:sort([#{type => generic, host_type => <<"a.com">>, tag => default,
+                             opts => [], conn_opts => #{}},
+                           #{type => generic, host_type => <<"c.eu">>, tag => default,
+                             opts => [], conn_opts => #{}},
+                           #{type => generic, host_type => <<"b.com">>, tag => default,
+                             opts => [{workers, 12}], conn_opts => #{}},
+                           #{type => generic, host_type => <<"a.com">>, tag => other_tag,
+                             opts => [], conn_opts => #{}},
+                           #{type => generic, host_type => <<"b.com">>, tag => other_tag,
+                             opts => [], conn_opts => #{}},
+                           #{type => generic, host_type => <<"c.eu">>, tag => other_tag,
+                             opts => [], conn_opts => #{}}]),
+    ?assertMatch(Expected, lists:sort(Expanded)).
 
 global_pool_is_used_by_default(_C) ->
     Pools = [#{type => generic, scope => global, tag => default, opts => #{}, conn_opts => #{}},