Merge pull request #3766 from esl/fix-overlay-vars

Fix and simplify overlays

Makefile

@@ -16,7 +16,6 @@ clean:
 	-rm -rf asngen
 	-rm configure.out
 	-rm rel/configure.vars.config
-	-rm rel/vars-toml.config
 
 # REBAR_CT_EXTRA_ARGS comes from a test runner
 ct:
@@ -27,7 +26,7 @@ ct:
 eunit:
 	@$(RUN) $(REBAR) eunit
 
-rel: certs configure.out rel/vars-toml.config
+rel: certs configure.out rel/configure.vars.config
 	. ./configure.out && $(REBAR) as prod release
 
 shell: certs etc/mongooseim.cfg
@@ -40,9 +39,6 @@ rock:
 	elif [ "$(BRANCH)" ]; then tools/rock_changed.sh $(BRANCH); \
 	else tools/rock_changed.sh; fi
 
-rel/vars-toml.config: rel/vars-toml.config.in rel/configure.vars.config
-	cat $^ > $@
-
 ## Don't allow these files to go out of sync!
 configure.out rel/configure.vars.config:
 	./tools/configure with-all without-jingle-sip

big_tests/run_common_test.erl

@@ -278,22 +278,20 @@ is_test_host_enabled(HostName) ->
 enable_preset_on_node(Node, PresetVars, HostVarsFilePrefix) ->
     {ok, Cwd} = call(Node, file, get_cwd, []),
     TemplatePath = filename:join([repo_dir(), "rel", "files", "mongooseim.toml"]),
-    DefaultVarsPath = filename:join([repo_dir(), "rel", "vars-toml.config"]),
     NodeVarsPath = filename:join([repo_dir(), "rel", HostVarsFilePrefix ++ ".vars-toml.config"]),
 
     {ok, Template} = handle_file_error(TemplatePath, file:read_file(TemplatePath)),
-    {ok, DefaultVars} = handle_file_error(DefaultVarsPath, file:consult(DefaultVarsPath)),
-    {ok, NodeVars} = handle_file_error(NodeVarsPath, file:consult(NodeVarsPath)),
+    NodeVars = read_vars(NodeVarsPath),
 
-    TemplatedConfig = template_config(Template, [DefaultVars, NodeVars, PresetVars]),
+    TemplatedConfig = template_config(Template, NodeVars ++ PresetVars),
     CfgPath = filename:join([Cwd, "etc", "mongooseim.toml"]),
     ok = call(Node, file, write_file, [CfgPath, TemplatedConfig]),
     call(Node, application, stop, [mongooseim]),
     call(Node, application, start, [mongooseim]),
     ok.
 
-template_config(Template, Vars) ->
-    MergedVars = ensure_binary_strings(merge_vars(Vars)),
+template_config(Template, RawVars) ->
+    MergedVars = ensure_binary_strings(maps:from_list(RawVars)),
     %% Render twice to replace variables in variables
     Tmp = bbmustache:render(Template, MergedVars, [{key_type, atom}]),
     bbmustache:render(Tmp, MergedVars, [{key_type, atom}]).
@@ -305,11 +303,20 @@ merge_vars([Vars1, Vars2|Rest]) ->
     merge_vars([Vars|Rest]);
 merge_vars([Vars]) -> Vars.
 
+read_vars(File) ->
+    {ok, Terms} = handle_file_error(File, file:consult(File)),
+    lists:flatmap(fun({Key, Val}) ->
+                          [{Key, Val}];
+                     (IncludedFile) when is_list(IncludedFile) ->
+                          Path = filename:join(filename:dirname(File), IncludedFile),
+                          read_vars(Path)
+                  end, Terms).
+
 %% bbmustache tries to iterate over lists, so we need to make them binaries
 ensure_binary_strings(Vars) ->
-    lists:map(fun({dbs, V}) -> {dbs, V};
-                 ({K, V}) when is_list(V) -> {K, list_to_binary(V)};
-                 ({K, V}) -> {K, V}
+    maps:map(fun(dbs, V) -> V;
+                (_K, V) when is_list(V) -> list_to_binary(V);
+                (_K, V) -> V
               end, Vars).
 
 call(Node, M, F, A) ->

big_tests/tests/ejabberd_node_utils.erl

@@ -154,41 +154,40 @@ modify_config_file(CfgVarsToChange, Config) ->
       ConfigVariable :: atom(),
       Value :: string().
 modify_config_file(Host, VarsToChange, Config, Format) ->
-    VarsFile = vars_file(Format),
     NodeVarsFile = ct:get_config({hosts, Host, vars}, Config) ++ "." ++ vars_file(Format),
     TemplatePath = config_template_path(Config, Format),
-    DefaultVarsPath = config_vars_path(VarsFile, Config),
     NodeVarsPath = config_vars_path(NodeVarsFile, Config),
 
     {ok, Template} = file:read_file(TemplatePath),
-    {ok, DefaultVars} = file:consult(DefaultVarsPath),
-    {ok, NodeVars} = file:consult(NodeVarsPath),
+    NodeVars = read_vars(NodeVarsPath),
     PresetVars = preset_vars(Config, Format),
 
-    TemplatedConfig = template_config(Template, [DefaultVars, NodeVars, PresetVars, VarsToChange]),
+    TemplatedConfig = template_config(Template, NodeVars ++ PresetVars ++ VarsToChange),
 
     RPCSpec = distributed_helper:Host(),
     NewCfgPath = update_config_path(RPCSpec, Format),
     ok = ejabberd_node_utils:call_fun(RPCSpec, file, write_file, [NewCfgPath, TemplatedConfig]).
 
+read_vars(File) ->
+    {ok, Terms} = file:consult(File),
+    lists:flatmap(fun({Key, Val}) ->
+                          [{Key, Val}];
+                     (IncludedFile) when is_list(IncludedFile) ->
+                          Path = filename:join(filename:dirname(File), IncludedFile),
+                          read_vars(Path)
+                  end, Terms).
+
 template_config(Template, Vars) ->
-    MergedVars = ensure_binary_strings(merge_vars(Vars)),
+    MergedVars = ensure_binary_strings(maps:from_list(Vars)),
     %% Render twice to replace variables in variables
     Tmp = bbmustache:render(Template, MergedVars, [{key_type, atom}]),
     bbmustache:render(Tmp, MergedVars, [{key_type, atom}]).
 
-merge_vars([Vars1, Vars2|Rest]) ->
-    Vars = lists:foldl(fun ({Var, Val}, Acc) ->
-                               lists:keystore(Var, 1, Acc, {Var, Val})
-                       end, Vars1, Vars2),
-    merge_vars([Vars|Rest]);
-merge_vars([Vars]) -> Vars.
-
 %% bbmustache tries to iterate over lists, so we need to make them binaries
 ensure_binary_strings(Vars) ->
-    lists:map(fun({dbs, V}) -> {dbs, V};
-                 ({K, V}) when is_list(V) -> {K, list_to_binary(V)};
-                 ({K, V}) -> {K, V}
+    maps:map(fun(dbs, V) -> V;
+                (_K, V) when is_list(V) -> list_to_binary(V);
+                (_K, V) -> V
               end, Vars).
 
 update_config_path(RPCSpec, Format) ->

rebar.config

@@ -164,15 +164,15 @@
                                  {overlay, [{template, "rel/files/mongooseim.toml", "etc/mongooseim.toml"}]} ]},
                                  {erl_opts, [{d, 'PROD_NODE'}]} ]},
              %% development nodes
-             {mim1,    [{relx, [ {overlay_vars, ["rel/vars-toml.config", "rel/mim1.vars-toml.config"]},
+             {mim1,    [{relx, [ {overlay_vars, "rel/mim1.vars-toml.config"},
                                  {overlay, [{template, "rel/files/mongooseim.toml", "etc/mongooseim.toml"}]} ]}]},
-             {mim2,    [{relx, [ {overlay_vars, ["rel/vars-toml.config", "rel/mim2.vars-toml.config"]},
+             {mim2,    [{relx, [ {overlay_vars, "rel/mim2.vars-toml.config"},
                                  {overlay, [{template, "rel/files/mongooseim.toml", "etc/mongooseim.toml"}]} ]}]},
-             {mim3,    [{relx, [ {overlay_vars, ["rel/vars-toml.config", "rel/mim3.vars-toml.config"]},
+             {mim3,    [{relx, [ {overlay_vars, "rel/mim3.vars-toml.config"},
                                  {overlay, [{template, "rel/files/mongooseim.toml", "etc/mongooseim.toml"}]} ]}]},
-             {fed1,    [{relx, [ {overlay_vars, ["rel/vars-toml.config", "rel/fed1.vars-toml.config"]},
+             {fed1,    [{relx, [ {overlay_vars, "rel/fed1.vars-toml.config"},
                                  {overlay, [{template, "rel/files/mongooseim.toml", "etc/mongooseim.toml"}]} ]}]},
-             {reg1,    [{relx, [ {overlay_vars, ["rel/vars-toml.config", "rel/reg1.vars-toml.config"]},
+             {reg1,    [{relx, [ {overlay_vars, "rel/reg1.vars-toml.config"},
                                  {overlay, [{template, "rel/files/mongooseim.toml", "etc/mongooseim.toml"}]} ]}]},
              {test,    [{extra_src_dirs, [{"test", [{recursive, true}]}]}]}
             ]}.

rel/fed1.vars-toml.config

@@ -1,3 +1,5 @@
+"./vars-toml.config".
+
 {node_name, "fed1@localhost"}.
 
 {c2s_port, 5242}.

rel/mim1.vars-toml.config

@@ -1,3 +1,5 @@
+"./vars-toml.config".
+
 {c2s_tls_port, 5223}.
 {outgoing_s2s_port, 5299}.
 {service_port, 8888}.

rel/mim2.vars-toml.config

@@ -1,3 +1,5 @@
+"./vars-toml.config".
+
 {node_name, "ejabberd2@localhost"}.
 
 {c2s_port, 5232}.

rel/mim3.vars-toml.config

@@ -1,3 +1,5 @@
+"./vars-toml.config".
+
 {node_name, "mongooseim3@localhost"}.
 
 {c2s_port, 5262}.

rel/reg1.vars-toml.config

@@ -1,3 +1,5 @@
+"./vars-toml.config".
+
 {node_name, "reg1@localhost"}.
 
 {c2s_port, 5252}.

rel/vars-toml.config.in → rel/vars-toml.config

@@ -55,7 +55,9 @@
 {s2s_certfile, "\"priv/ssl/fake_server.pem\""}.
 {all_metrics_are_global, "false"}.
 
-%% Defined in Makefile by appending configure.vars.config
+"./configure.vars.config".
+
+%% Defined by appending configure.vars.config
 %% Uncomment for manual release generation.
 %{mongooseim_runner_user, ""}.
 %{mongooseim_script_dir, "$(cd ${0%/*} && pwd)"}.

tools/test_runner/apply_templates.erl

@@ -20,21 +20,24 @@ main([NodeAtom, BuildDirAtom]) ->
 
 
 overlay_vars(Node) ->
-    Vars = consult_map("rel/vars-toml.config"),
-    NodeVars = consult_map("rel/" ++ atom_to_list(Node) ++ ".vars-toml.config"),
-    %% NodeVars overrides Vars
-    ensure_binary_strings(maps:merge(Vars, NodeVars)).
+    File = "rel/" ++ atom_to_list(Node) ++ ".vars-toml.config",
+    ensure_binary_strings(maps:from_list(read_vars(File))).
+
+read_vars(File) ->
+    {ok, Terms} = file:consult(File),
+    lists:flatmap(fun({Key, Val}) ->
+                          [{Key, Val}];
+                     (IncludedFile) when is_list(IncludedFile) ->
+                          Path = filename:join(filename:dirname(File), IncludedFile),
+                          read_vars(Path)
+                  end, Terms).
 
 %% bbmustache tries to iterate over lists, so we need to make them binaries
 ensure_binary_strings(Vars) ->
     maps:map(fun(_K, V) when is_list(V) -> list_to_binary(V);
                 (_K, V) -> V
              end, Vars).
 
-consult_map(File) ->
-    {ok, Vars} = file:consult(File),
-    maps:from_list(Vars).
-
 %% Based on rebar.config overlay section
 templates(RelDir) ->
     simple_templates(RelDir) ++ erts_templates(RelDir).