Merge pull request #3108 from esl/multi-tenancy-tests

Multi tenancy tests

.circleci/config.yml

@@ -202,14 +202,18 @@ orbs:
             type: boolean
             description: Erlang distribution with TLS enabled
             default: false
+          spec:
+            type: string
+            description: Test spec file to use
+            default: default.spec
         environment:
           MIX_ENV: test
           PRESET: <<parameters.preset>>
           DB: <<parameters.db>>
           TLS_DIST: <<parameters.tls_dist>>
           ELASTICSEARCH_VERSION: 5.6.9
           CASSANDRA_VERSION: 3.9
-          TEST_SPEC: mam.spec
+          TESTSPEC: <<parameters.spec>>
           REDIS_VERSION: 3.2.10
         steps:
           - checkout
@@ -466,6 +470,17 @@ workflows:
           requires:
             - otp_23
           filters: *all_tags
+      # ============= DYNAMIC DOMAINS =============
+      - mim/big_tests:
+          name: dynamic_domains
+          otp_package: 23.0.3-1
+          spec: dynamic_domains.spec
+          preset: pgsql_mnesia
+          db: pgsql
+          context: mongooseim-org
+          requires:
+            - otp_23
+          filters: *all_tags
       # ============= 1 VERSION OLDER TESTS =============
       - mim/big_tests:
           name: ldap_mnesia_22

big_tests/dynamic_domains.config

@@ -0,0 +1,41 @@
+%% Options defined here are used when testing dynamic domains, see 'dynamic_domains.spec'
+%% They take precedence over 'test.config'
+
+{hosts, [{mim,  [{node, mongooseim@localhost},
+                 {domain, <<"domain.example.com">>},
+                 {secondary_domain, <<"domain.example.org">>},
+                 {dynamic_domains, [<<"domain.example.com">>, <<"domain.example.org">>]},
+                 {vars, "mim1"},
+                 {cluster, mim},
+                 {s2s_port, 5269},
+                 {incoming_s2s_port, 5269},
+                 {metrics_rest_port, 5288},
+                 {c2s_port, 5222},
+                 {c2s_tls_port, 5223},
+                 {cowboy_port, 5280},
+                 {cowboy_secure_port, 5285},
+                 {http_api_client_endpoint_port, 8089},
+                 {service_port, 8888},
+                 {kicking_service_port, 8666},
+                 {hidden_service_port, 8189},
+                 {gd_endpoint_port, 5555},
+                 {http_notifications_port, 8000}]}
+        ]}.
+
+{escalus_users, [
+    {alice, [
+        {username, <<"alicE">>},
+        {server, <<"domain.example.com">>},
+        {host, <<"localhost">>},
+        {password, <<"matygrysa">>}]},
+    {alice_bis, [
+        {username, <<"alicE">>},
+        {server, <<"domain.example.org">>},
+        {host, <<"localhost">>},
+        {password, <<"matygrysa">>}]},
+    {bob, [
+        {username, <<"bOb">>},
+        {server, <<"domain.example.com">>},
+        {host, <<"localhost">>},
+        {password, <<"makrolika">>}]}
+]}.

big_tests/dynamic_domains.spec

@@ -0,0 +1,30 @@
+%% Spec examples:
+%%
+%%   {suites, "tests", amp_SUITE}.
+%%   {groups, "tests", amp_SUITE, [discovery]}.
+%%   {groups, "tests", amp_SUITE, [discovery], {cases, [stream_feature_test]}}.
+%%   {cases, "tests", amp_SUITE, [stream_feature_test]}.
+%%
+%% For more info see:
+%% http://www.erlang.org/doc/apps/common_test/run_test_chapter.html#test_specifications
+{include, "tests"}.
+
+{suites, "tests", domain_isolation_SUITE}.
+
+{config, ["dynamic_domains.config", "test.config"]}.
+{logdir, "ct_report"}.
+
+%% ct_tty_hook will log CT failures to TTY verbosely
+%% ct_mongoose_hook will:
+%% * log suite start/end events in the MongooseIM console
+%% * ensure preset value is passed to ct Config
+%% * check server's purity after SUITE
+{ct_hooks, [ct_groups_summary_hook, ct_tty_hook, ct_mongoose_hook, ct_progress_hook,
+            ct_mim_config_hook,
+            ct_markdown_errors_hook,
+            {ct_mongoose_log_hook, [ejabberd_node, ejabberd_cookie]},
+            {ct_mongoose_log_hook, [ejabberd2_node, ejabberd_cookie]}
+           ]}.
+
+%% To enable printing group and case enters on server side
+%%{ct_hooks, [{ct_mongoose_hook, [print_group, print_case]}]}.

big_tests/run_common_test.erl

@@ -162,9 +162,9 @@ save_count(Test, Configs) ->
     file:write_file("/tmp/ct_count", integer_to_list(Repeat*Times)).
 
 run_test(Test, PresetsToRun, CoverOpts) ->
-    prepare_cover(Test, CoverOpts),
+    {ConfigFiles, Props} = get_ct_config(Test),
+    prepare_cover(Props, CoverOpts),
     error_logger:info_msg("Presets to run ~p", [PresetsToRun]),
-    {ConfigFile, Props} = get_ct_config(Test),
     case get_presets(Props) of
         {ok, Presets} ->
             Presets1 = case PresetsToRun of
@@ -183,15 +183,15 @@ run_test(Test, PresetsToRun, CoverOpts) ->
             error_logger:info_msg("Starting test of ~p configurations: ~n~p~n",
                                   [Length, Names]),
             Zip = lists:zip(lists:seq(1, Length), Presets1),
-            R = [ run_config_test(Preset, Test, N, Length) || {N, Preset} <- Zip ],
+            R = [ run_config_test(Props, Preset, Test, N, Length) || {N, Preset} <- Zip ],
             save_count(Test, Presets1),
-            analyze_coverage(Test, CoverOpts),
+            analyze_coverage(Props, CoverOpts),
             R;
         {error, not_found} ->
-            error_logger:info_msg("Presets were not found in the config file ~ts",
-                                  [ConfigFile]),
+            error_logger:info_msg("Presets were not found in the config files ~ts",
+                                  [ConfigFiles]),
             R = do_run_quick_test(Test, CoverOpts),
-            analyze_coverage(Test, CoverOpts),
+            analyze_coverage(Props, CoverOpts),
             R
     end.
 
@@ -211,12 +211,10 @@ get_presets(Props) ->
 get_ct_config(Opts) ->
     Spec = proplists:get_value(spec, Opts),
     Props = read_file(Spec),
-    ConfigFile = case proplists:lookup(config, Props) of
-        {config, [Config]} -> Config;
-        _                  -> "test.config"
-    end,
-    {ok, ConfigProps} = handle_file_error(ConfigFile, file:consult(ConfigFile)),
-    {ConfigFile, ConfigProps}.
+    ConfigFiles = proplists:get_value(config, Props, ["test.config"]),
+    % Apply the files in reverse, like ct will do when running the tests
+    ConfigProps = merge_vars([read_file(File) || File <- lists:reverse(ConfigFiles)]),
+    {ConfigFiles, ConfigProps}.
 
 preset_names(Presets) ->
     [Preset||{Preset, _} <- Presets].
@@ -234,8 +232,8 @@ do_run_quick_test(Test, CoverOpts) ->
             [{ok, {Ok, Failed, UserSkipped, AutoSkipped}}]
     end.
 
-run_config_test({Name, Variables}, Test, N, Tests) ->
-    enable_preset(Name, Variables, Test, N, Tests),
+run_config_test(Props, {Name, Variables}, Test, N, Tests) ->
+    enable_preset(Props, Name, Variables, N, Tests),
     load_test_modules(Test),
     Result = ct:run_test([{label, Name} | Test]),
     case Result of
@@ -245,12 +243,12 @@ run_config_test({Name, Variables}, Test, N, Tests) ->
             {ok, {Ok, Failed, UserSkipped, AutoSkipped}}
     end.
 
-enable_preset(Name, PresetVars, Test, N, Tests) ->
+enable_preset(Props, Name, PresetVars, N, Tests) ->
     %% TODO: Do this with a multicall, otherwise it's not as fast as possible (not parallelized).
     %%       A multicall requires the function to be defined on the other side, though.
     Rs = [ maybe_enable_preset_on_node(host_node(H), PresetVars,
                                        host_vars(H), host_name(H))
-           || H <- get_hosts_to_enable_preset(Test) ],
+           || H <- get_hosts_to_enable_preset(Props) ],
     [ok] = lists:usort(Rs),
     error_logger:info_msg("Configuration ~p of ~p: ~p started.~n",
                           [N, Tests, Name]).
@@ -324,21 +322,21 @@ call(Node, M, F, A) ->
             Result
     end.
 
-prepare_cover(Test, true) ->
+prepare_cover(Props, true) ->
     io:format("Preparing cover~n"),
-    prepare(Test);
+    prepare(Props);
 prepare_cover(_, _) ->
     ok.
 
-analyze_coverage(Test, true) ->
-    analyze(Test, true);
-analyze_coverage(Test, ModuleList) when is_list(ModuleList) ->
-    analyze(Test, ModuleList);
+analyze_coverage(Props, true) ->
+    analyze(Props, true);
+analyze_coverage(Props, ModuleList) when is_list(ModuleList) ->
+    analyze(Props, ModuleList);
 analyze_coverage(_, _) ->
     ok.
 
-prepare(Test) ->
-    Nodes = get_mongoose_nodes(Test),
+prepare(Props) ->
+    Nodes = get_mongoose_nodes(Props),
     maybe_compile_cover(Nodes).
 
 maybe_compile_cover([]) ->
@@ -364,14 +362,14 @@ maybe_compile_cover(Nodes) ->
     report_progress("~nCover compilation took ~ts~n", [microseconds_to_string(Time)]),
     ok.
 
-analyze(Test, CoverOpts) ->
+analyze(Props, CoverOpts) ->
     io:format("Coverage analyzing~n"),
-    Nodes = get_mongoose_nodes(Test),
-    analyze(Test, CoverOpts, Nodes).
+    Nodes = get_mongoose_nodes(Props),
+    analyze(Props, CoverOpts, Nodes).
 
-analyze(_Test, _CoverOpts, []) ->
+analyze(_Props, _CoverOpts, []) ->
     ok;
-analyze(_Test, CoverOpts, Nodes) ->
+analyze(_Props, CoverOpts, Nodes) ->
     deduplicate_cover_server_console_prints(),
     %% Import small tests cover
     Files = filelib:wildcard(repo_dir() ++ "/_build/**/cover/*.coverdata"),
@@ -433,19 +431,19 @@ make_html(Modules) ->
     file:write(File, row("Summary", CSum, NCSum, percent(CSum, NCSum), "#")),
     file:close(File).
 
-get_hosts_to_enable_preset(Test) ->
-    Hosts = get_all_hosts(Test),
-    %% We apply preset options to `mim` and `reg` clusters
-    Clusters = group_by(fun host_cluster/1, Hosts),
-    dict:fetch(mim, Clusters) ++ dict:fetch(reg, Clusters).
+get_hosts_to_enable_preset(Props) ->
+    [Host || Host <- get_all_hosts(Props), should_enable_preset(host_cluster(Host))].
+
+should_enable_preset(mim) -> true;
+should_enable_preset(reg) -> true;
+should_enable_preset(_) -> false.
 
-get_all_hosts(Test) ->
-    {_File, Props} = get_ct_config(Test),
+get_all_hosts(Props) ->
     {hosts, Hosts} = lists:keyfind(hosts, 1, Props),
     Hosts.
 
-get_mongoose_nodes(Test) ->
-    [ host_node(H) || H <- get_all_hosts(Test), is_test_host_enabled(host_name(H)) ].
+get_mongoose_nodes(Props) ->
+    [ host_node(H) || H <- get_all_hosts(Props), is_test_host_enabled(host_name(H)) ].
 
 percent(0, _) -> 0;
 percent(C, NC) when C /= 0; NC /= 0 -> round(C / (NC+C) * 100);
@@ -545,12 +543,6 @@ exit_code({_, _, _, _}) ->
 print(Handle, Fmt, Args) ->
     io:format(Handle, Fmt, Args).
 
-%% Source: https://gist.github.com/jbpotonnier/1310406
-group_by(F, L) ->
-    lists:foldr(fun ({K, V}, D) -> dict:append(K, V, D) end,
-                dict:new(),
-                [ {F(X), X} || X <- L ]).
-
 host_cluster(Host) -> host_param(cluster, Host).
 host_node(Host)    -> host_param(node, Host).
 host_vars(Host)    -> host_param(vars, Host).

big_tests/src/ct_mongoose_hook.erl

@@ -42,6 +42,7 @@ init(_Id, Opts) ->
     Unfolded = proplists:unfold(Opts),
     PrintGroup = proplists:get_value(print_group, Unfolded, false),
     PrintCase = proplists:get_value(print_case, Unfolded, false),
+    domain_helper:insert_configured_domains(),
     {ok, #state{print_group = PrintGroup, print_case = PrintCase }}.
 
 %% @doc Called before init_per_suite is called.
@@ -108,6 +109,7 @@ on_tc_skip(_TC, _Reason, State) ->
 
 %% @doc Called when the scope of the CTH is done
 terminate(_State) ->
+    domain_helper:delete_configured_domains(),
     ok.
 
 maybe_print_on_server(false, _, _, _) ->

big_tests/tests/domain_helper.erl

@@ -0,0 +1,28 @@
+-module(domain_helper).
+
+-export([insert_configured_domains/0,
+         delete_configured_domains/0,
+         insert_domain/2,
+         delete_domain/2]).
+
+-import(distributed_helper, [get_or_fail/1, rpc/4]).
+
+insert_configured_domains() ->
+    for_each_configured_domain(fun insert_domain/2).
+
+delete_configured_domains() ->
+    for_each_configured_domain(fun delete_domain/2).
+
+insert_domain(Node, Domain) ->
+    ok = rpc(Node, mongoose_domain_core, insert, [Domain, host_type(), dummy_source]).
+
+delete_domain(Node, Domain) ->
+    ok = rpc(Node, mongoose_domain_core, delete, [Domain]).
+
+for_each_configured_domain(F) ->
+    [F(#{node => proplists:get_value(node, Opts)}, Domain) ||
+        {_, Opts} <- ct:get_config(hosts),
+        Domain <- proplists:get_value(dynamic_domains, Opts, [])].
+
+host_type() ->
+    <<"test type">>. %% preconfigured in the toml file

big_tests/tests/dynamic_domains_pm_SUITE.erl

@@ -71,13 +71,10 @@ auth_domain_removal_is_triggered_on_hook(_Config) ->
 
 %% helper functions
 insert_domains(Nodes, Domains) ->
-    Source = dummy_source, %% can be anything, we don't care about it
-    [ok = rpc(Node, mongoose_domain_core, insert, [Domain, ?HOST_TYPE, Source]) ||
-        Node <- Nodes, Domain <- Domains].
+    [domain_helper:insert_domain(Node, Domain) || Node <- Nodes, Domain <- Domains].
 
 remove_domains(Nodes, Domains) ->
-    [ok = rpc(Node, mongoose_domain_core, delete, [Domain]) ||
-        Node <- Nodes, Domain <- Domains].
+    [domain_helper:delete_domain(Node, Domain) || Node <- Nodes, Domain <- Domains].
 
 cluster_nodes([], Config) -> Config;
 cluster_nodes([Node | T], Config) ->

tools/test-runner-complete.sh

@@ -37,7 +37,7 @@ fi
 try_to_load_bash_completion_using_brew
 
 _run_all_tests() {
-  printf "%s\n" "${COMP_WORDS[@]}" > /tmp/test-runner-last-competion
+  printf "%s\n" "${COMP_WORDS[@]}" > /tmp/test-runner-last-completion
   # Make COMP_WORDS, without using colon as a breaker
   # To see all breakers, check COMP_WORDBREAKS variable
   # It's needed for bash only, not zsh
@@ -57,7 +57,9 @@ _run_all_tests() {
   done
 
   cur=${COMP_WORDS[$COMP_CWORD]}
-  opt=${COMP_WORDS[$pos]}
+
+  # Last option and its arguments before 'cur'
+  opt=${COMP_WORDS[@]:$pos:$(($COMP_CWORD-$pos))}
 
   # If current is an option - we don't care about option expansion
   case "$cur" in
@@ -79,6 +81,10 @@ _run_all_tests() {
     ARRAY=( $(./tools/test-runner.sh --list-presets) )
     COMPREPLY=( $( compgen -W "${ARRAY[*]} --" -- $cur ) );;
 
+    --spec) # no '*' here because only one spec is allowed
+    ARRAY=( $(./tools/test-runner.sh --list-specs) )
+    COMPREPLY=( $( compgen -W "${ARRAY[*]} --" -- $cur ) );;
+
     --dev-nodes*)
     ARRAY=( $(./tools/test-runner.sh --list-dev-nodes) )
     COMPREPLY=( $( compgen -W "${ARRAY[*]} --" -- $cur ) );;
@@ -103,6 +109,7 @@ _run_all_tests() {
     SUITES=$(./tools/test_runner/list_suites.sh $LIST_SUITES_ARGS)
 
     COMPREPLY=( $( compgen -W '--db --preset \
+                          --spec \
                           --dev-nodes \
                           --test-hosts \
                           --one-node \