Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cleaning logic for discovery_nodes table - using database timestamp #4049

Merged
merged 5 commits into from
Sep 4, 2023
Merged

Cleaning logic for discovery_nodes table - using database timestamp #4049

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2ba064e
Cleaning discovery_nodes from inactive nodes
arcusfelis Jun 29, 2023
d6223aa
Use 3 hours TTL for nodes in disco
arcusfelis Aug 30, 2023
dc613b0
Remove dead code
arcusfelis Sep 1, 2023
aca43ba
Use meck to test cets_disco_SUITE:rdbms_backend_supports_auto_cleanin…
arcusfelis Sep 1, 2023
d1dade1
Update CETS dependency
arcusfelis Sep 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 45 additions & 18 deletions big_tests/tests/cets_disco_SUITE.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
-module(cets_disco_SUITE).
-compile([export_all, nowarn_export_all]).

-import(distributed_helper, [mim/0, rpc/4]).
-import(distributed_helper, [mim/0, mim2/0, rpc/4]).
chrzaszcz marked this conversation as resolved.
Show resolved Hide resolved
-include_lib("common_test/include/ct.hrl").

%%--------------------------------------------------------------------
Expand Down Expand Up @@ -46,9 +46,15 @@ init_per_group(_, Config) ->
end_per_group(_, Config) ->
Config.

init_per_testcase(rdbms_backend_supports_auto_cleaning = CaseName, Config) ->
mock_timestamp(mim(), month_ago()) ++
escalus:init_per_testcase(CaseName, Config);
init_per_testcase(CaseName, Config) ->
escalus:init_per_testcase(CaseName, Config).

end_per_testcase(rdbms_backend_supports_auto_cleaning = CaseName, Config) ->
unmock_timestamp(mim()),
escalus:end_per_testcase(CaseName, Config);
end_per_testcase(CaseName, Config) ->
escalus:end_per_testcase(CaseName, Config).

Expand All @@ -67,44 +73,65 @@ rdbms_backend(_Config) ->
CN = <<"big_test">>,
Opts1 = #{cluster_name => CN, node_name_to_insert => <<"test1">>},
Opts2 = #{cluster_name => CN, node_name_to_insert => <<"test2">>},
State1 = disco_init(Opts1),
disco_get_nodes(State1),
State2 = disco_init(Opts2),
{{ok, Nodes}, State2_2} = disco_get_nodes(State2),
State1 = disco_init(mim(), Opts1),
disco_get_nodes(mim(), State1),
State2 = disco_init(mim2(), Opts2),
{{ok, Nodes}, State2_2} = disco_get_nodes(mim2(), State2),
%% "test2" node can see "test1"
true = lists:member(test1, Nodes),
{{ok, _}, State2_3} = disco_get_nodes(State2_2),
{{ok, _}, State2_3} = disco_get_nodes(mim2(), State2_2),
%% Check that we follow the right code branch
#{last_query_info := #{already_registered := true}} = State2_3.

rdbms_backend_supports_auto_cleaning(_Config) ->
rdbms_backend_supports_auto_cleaning(Config) ->
ensure_mocked(Config),
CN = <<"big_test2">>,
Opts1 = #{cluster_name => CN, node_name_to_insert => <<"test1">>, override_timestamp => month_ago()},
Opts1 = #{cluster_name => CN, node_name_to_insert => <<"test1">>},
Opts2 = #{cluster_name => CN, node_name_to_insert => <<"test2">>},
%% test1 row is written
State1 = disco_init(Opts1),
{_, State1_2} = disco_get_nodes(State1),
{{ok, Nodes1}, _} = disco_get_nodes(State1_2),
%% test1 row is written with an old (mocked) timestamp
State1 = disco_init(mim(), Opts1),
{_, State1_2} = disco_get_nodes(mim(), State1),
{{ok, Nodes1}, State1_3} = disco_get_nodes(mim(), State1_2),
Timestamp = proplists:get_value(mocked_timestamp, Config),
#{last_query_info := #{timestamp := Timestamp}} = State1_3,
%% It is in DB
true = lists:member(test1, Nodes1),
%% test2 would clean test1 registration
State2 = disco_init(Opts2),
{{ok, Nodes2}, State2_2} = disco_get_nodes(State2),
%% We don't mock on mim2 node, so timestamps would differ
State2 = disco_init(mim2(), Opts2),
{{ok, Nodes2}, State2_2} = disco_get_nodes(mim2(), State2),
false = lists:member(test1, Nodes2),
#{last_query_info := #{run_cleaning_result := {removed, [test1]}}} = State2_2.

%%--------------------------------------------------------------------
%% Helpers
%%--------------------------------------------------------------------

disco_init(Opts) ->
rpc(mim(), mongoose_cets_discovery_rdbms, init, [Opts]).
disco_init(Node, Opts) ->
rpc(Node, mongoose_cets_discovery_rdbms, init, [Opts]).

disco_get_nodes(State) ->
rpc(mim(), mongoose_cets_discovery_rdbms, get_nodes, [State]).
disco_get_nodes(Node, State) ->
rpc(Node, mongoose_cets_discovery_rdbms, get_nodes, [State]).

timestamp() ->
os:system_time(second).

month_ago() ->
timestamp() - timer:hours(24 * 30) div 1000.

mock_timestamp(Node, Timestamp) ->
ok = rpc(Node, meck, new, [mongoose_rdbms_timestamp, [passthrough, no_link]]),
ok = rpc(Node, meck, expect, [mongoose_rdbms_timestamp, select, 0, Timestamp]),
%% Ensure that we mock
EnsureMocked = fun() ->
Timestamp = rpc(Node, mongoose_rdbms_timestamp, select, [])
end,
EnsureMocked(),
Copy link
Member

@chrzaszcz chrzaszcz Sep 1, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why execute and ignore result? Seems to do nothing.

Copy link
Contributor Author

@arcusfelis arcusfelis Sep 1, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am matching Timestamp variable here.
So, EnsureMocked will crash if select() returns something different.

Copy link
Contributor Author

@arcusfelis arcusfelis Sep 1, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I.e. if meck fails to mock for some reason, we would get:

=== ERROR! init_per_testcase crashed!
	Location: [{cets_disco_SUITE,'-mock_timestamp/2-fun-0-',[127](file:///Users/mikhailuvarov/erlang/esl/MongooseIM/big_tests/ct_report/[email protected]_17.25.22/big_tests.tests.cets_disco_SUITE.logs/run.2023-09-01_17.25.23/cets_disco_suite.src.html#127)},
              {cets_disco_SUITE,mock_timestamp,[129](file:///Users/mikhailuvarov/erlang/esl/MongooseIM/big_tests/ct_report/[email protected]_17.25.22/big_tests.tests.cets_disco_SUITE.logs/run.2023-09-01_17.25.23/cets_disco_suite.src.html#129)},
              {cets_disco_SUITE,init_per_testcase,[50](file:///Users/mikhailuvarov/erlang/esl/MongooseIM/big_tests/ct_report/[email protected]_17.25.22/big_tests.tests.cets_disco_SUITE.logs/run.2023-09-01_17.25.23/cets_disco_suite.src.html#50)},
              {test_server,do_init_per_testcase,1552},
              {test_server,run_test_case_eval1,1253},
              {test_server,run_test_case_eval,1223}]
	Reason: {{badmatch,1693581924},
 [{cets_disco_SUITE,'-mock_timestamp/2-fun-0-',2,
                    [{file,"cets_disco_SUITE.erl"},{line,127}]},
  {cets_disco_SUITE,mock_timestamp,2,
                    [{file,"cets_disco_SUITE.erl"},{line,129}]},
  {cets_disco_SUITE,init_per_testcase,2,
                    [{file,"cets_disco_SUITE.erl"},{line,50}]},
  {test_server,do_init_per_testcase,2,[{file,"test_server.erl"},{line,1552}]},
  {test_server,run_test_case_eval1,6,[{file,"test_server.erl"},{line,1253}]},
  {test_server,run_test_case_eval,9,[{file,"test_server.erl"},{line,1223}]}]}

I could remove EnsureMocked though, it is just to verify and mocking is working.

[{ensure_mocked, EnsureMocked}, {mocked_timestamp, Timestamp}].

ensure_mocked(Config) ->
EnsureMocked = proplists:get_value(ensure_mocked, Config),
EnsureMocked().

unmock_timestamp(Node) ->
ok = rpc(Node, meck, unload, [mongoose_rdbms_timestamp]).
8 changes: 3 additions & 5 deletions src/mongoose_cets_discovery_rdbms.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
-endif.

-type opts() :: #{cluster_name => binary(), node_name_to_insert => binary(), last_query_info => map(),
expire_time := non_neg_integer(), override_timestamp := integer()}.
expire_time := non_neg_integer()}.
-type state() :: opts().

-spec init(opts()) -> state().
Expand Down Expand Up @@ -52,8 +52,7 @@ is_rdbms_running() ->

try_register(ClusterName, NodeBin, State) when is_binary(NodeBin), is_binary(ClusterName) ->
prepare(),
%% We could pass the timestamp as an argument in tests
Timestamp = maps:get(override_timestamp, State, timestamp()),
Timestamp = timestamp(),
Node = binary_to_atom(NodeBin),
{selected, Rows} = select(ClusterName),
Zipped = [{binary_to_atom(DbNodeBin), Num, TS} || {DbNodeBin, Num, TS} <- Rows],
Expand Down Expand Up @@ -92,8 +91,7 @@ run_cleaning(ClusterName, Timestamp, Rows, State) ->
[] ->
{skip, nothing_expired};
_ ->
[delete_node_from_db(ClusterName, DbNodeBin)
|| {DbNodeBin, _Num, _TS} <- Expired],
[delete_node_from_db(ClusterName, DbNodeBin) || {DbNodeBin, _Num, _TS} <- Expired],
?LOG_WARNING(#{what => cets_expired_nodes,
text => <<"Expired nodes are detected in discovery_nodes table">>,
expired_nodes => ExpiredNodes}),
Expand Down