WIP

big_tests/tests/accounts_SUITE.erl

@@ -189,20 +189,22 @@ unregister(Config) ->
     assert_event(auth_unregister_user, escalus_users:get_jid(Config, UserSpec)).
 
 already_registered(Config) ->
-    escalus_fresh:story(Config, [{alice, 1}], fun(Alice) ->
-        escalus:send(Alice, escalus_stanza:get_registration_fields()),
-        Stanza = escalus:wait_for_stanza(Alice),
-        escalus:assert(is_iq_result, Stanza),
-        true = has_registered_element(Stanza)
+    escalus_fresh:story(Config, [{alice, 1}], fun already_registered_story/1).
+
+already_registered_story(Alice) ->
+    AliceJid = escalus_utils:get_short_jid(Alice),
+    assert_event(auth_register_user, AliceJid), % one event expected
+    escalus:send(Alice, escalus_stanza:get_registration_fields()),
+    Stanza = escalus:wait_for_stanza(Alice),
+    escalus:assert(is_iq_result, Stanza),
+    true = has_registered_element(Stanza),
+    assert_event(auth_register_user, AliceJid). % still one event - nothing new
 
-                                              end).
 registration_conflict(Config) ->
     [Alice] = escalus_users:get_users([alice]),
     {ok, result, _Stanza} = escalus_users:create_user(Config, Alice),
     {ok, conflict, _Raw} = escalus_users:create_user(Config, Alice).
 
-
-
 admin_notify(Config) ->
     [{Name1, UserSpec1}, {Name2, UserSpec2}] = escalus_users:get_users([alice, bob]),
     [{_, AdminSpec}] = escalus_users:get_users([admin]),
@@ -457,5 +459,6 @@ disable_watcher(Config) ->
 
 assert_event(EventName, BinJid) ->
     #jid{luser = LUser, lserver = LServer} = jid:from_binary(BinJid),
-    instrument_helper:assert(EventName, #{host_type => host_type()},
-                             fun(M) -> M =:= #{count => 1, user => LUser, server => LServer} end).
+    instrument_helper:assert_one(
+      EventName, #{host_type => host_type()},
+      fun(M) -> M =:= #{count => 1, user => LUser, server => LServer} end).

big_tests/tests/anonymous_SUITE.erl

@@ -89,7 +89,7 @@ connection_is_registered_with_login(Config) ->
         true = F(),
         escalus_connection:kill(Anna),
         mongoose_helper:wait_until(F, false),
-        assert_event(auth_anonymous_register_user, JID)
+        assert_event(auth_anonymous_unregister_user, JID)
     end).
 
 messages_story(Config) ->
@@ -110,5 +110,6 @@ host_type() ->
     domain_helper:anonymous_host_type().
 
 assert_event(EventName, #jid{luser = LUser, lserver = LServer}) ->
-    instrument_helper:assert(EventName, #{host_type => host_type()},
-                             fun(M) -> M =:= #{count => 1, user => LUser, server => LServer} end).
+    instrument_helper:assert_one(
+      EventName, #{host_type => host_type()},
+      fun(M) -> M =:= #{count => 1, user => LUser, server => LServer} end).

big_tests/tests/domain_isolation_SUITE.erl

@@ -157,7 +157,7 @@ verify_alice_has_no_pending_messages(Alice, Bob) ->
 assert_stanza_dropped(Sender, Recipient, Stanza) ->
     SenderJid = jid:from_binary(escalus_utils:get_jid(Sender)),
     RecipientJid = jid:from_binary(escalus_utils:get_jid(Recipient)),
-    instrument_helper:assert(
+    instrument_helper:assert_one(
       router_stanza_dropped, #{host_type => host_type()},
       fun(#{count := 1, from_jid := From, to_jid := To, stanza := DroppedStanza}) ->
               From =:= SenderJid andalso To =:= RecipientJid andalso DroppedStanza =:= Stanza

big_tests/tests/instrument_cets_SUITE.erl

@@ -42,15 +42,14 @@ end_per_group(_, _Config) ->
 init_per_testcase(_, Config) ->
     Config.
 
-check_instrumentation(Config) ->
-    instrument_helper:wait_for_new(cets_info, #{}),
-    instrument_helper:assert(cets_info, #{}, fun(Res) ->
-            %% Values are integers
-            lists:all(fun(Name) -> is_integer(maps:get(Name, Res)) end, instrumentation_metrics_names())
-            andalso
-            %% Check that there are no unknown fields
-            [] =:= maps:keys(maps:without(instrumentation_metrics_names(), Res))
-        end).
+check_instrumentation(_Config) ->
+    MeasurementsList = instrument_helper:wait_for_new(cets_info, #{}),
+    instrument_helper:assert(cets_info, #{}, MeasurementsList, fun check_info/1, 1).
+
+%% Check that values are integers and there are no unknown fields
+check_info(Res) ->
+    lists:all(fun(Name) -> is_integer(maps:get(Name, Res)) end, instrumentation_metrics_names())
+        andalso #{} =:= maps:without(instrumentation_metrics_names(), Res).
 
 instrumentation_metrics_names() ->
     [available_nodes,

big_tests/tests/instrument_helper.erl

@@ -5,7 +5,7 @@
 
 -export([declared_events/1, declared_events/2,
          start/1, start/2, stop/0,
-         assert/3, assert/4, filter/2,
+         assert/3, assert/4, assert/5, assert_one/3, assert_none/3, filter/2,
          assert_not_emitted/1, assert_not_emitted/2,
          wait_for/2, wait_for_new/2,
          lookup/2, take/2]).
@@ -20,6 +20,8 @@
 -type event_name() :: atom().
 -type labels() :: #{atom() => term()}.
 -type measurements() :: #{atom() => term()}.
+-type check_fun() :: fun((measurements()) -> boolean()).
+-type event_count() :: non_neg_integer() | positive.
 
 %% API
 
@@ -59,32 +61,48 @@ stop() ->
     verify_unlogged((Untested -- Logged) -- Negative),
     verify_logged_but_untested((Logged -- Tested) -- Negative).
 
--spec assert(event_name(), labels(), fun((measurements()) -> boolean())) -> ok.
+-spec assert(event_name(), labels(), check_fun()) -> ok.
 assert(EventName, Labels, CheckF) ->
-    assert(EventName, Labels, lookup(EventName, Labels), CheckF).
+    assert(EventName, Labels, CheckF, positive).
+
+-spec assert_one(event_name(), labels(), check_fun()) -> ok.
+assert_one(EventName, Labels, CheckF) ->
+    assert(EventName, Labels, CheckF, 1).
+
+-spec assert_none(event_name(), labels(), check_fun()) -> ok.
+assert_none(EventName, Labels, CheckF) ->
+    assert(EventName, Labels, CheckF, 0).
+
+-spec assert(event_name(), labels(), check_fun(), event_count()) -> ok;
+            (event_name(), labels(), [measurements()], check_fun()) -> ok.
+assert(EventName, Labels, CheckF, Count) when is_function(CheckF) ->
+    assert(EventName, Labels, lookup(EventName, Labels), CheckF, Count);
+assert(EventName, Labels, MeasurementsList, CheckF) when is_function(CheckF) ->
+    assert(EventName, Labels, MeasurementsList, CheckF, positive).
 
 %% @doc `CheckF' can return a boolean or fail with `function_clause', which means `false'.
 %% This is for convenience - you only have to code one clause.
--spec assert(event_name(), labels(), [measurements()], fun((measurements()) -> boolean())) -> ok.
-assert(EventName, Labels, MeasurementsList, CheckF) ->
+-spec assert(event_name(), labels(), [measurements()], check_fun(), event_count()) -> ok.
+assert(EventName, Labels, MeasurementsList, CheckF, ExpectedCount)
+  when ExpectedCount =:= positive;
+       is_integer(ExpectedCount), ExpectedCount >= 0 ->
     case filter(CheckF, MeasurementsList) of
-        [] ->
-            ct:log("All measurements for event ~p with labels ~p:~n~p",
-                   [EventName, Labels, MeasurementsList]),
-            ct:fail("No instrumentation events matched");
-        Filtered ->
+        [] when ExpectedCount =:= 0 ->
+            ok; % don't mark anything as tested
+        Filtered when ExpectedCount =:= positive;
+                      ExpectedCount =:= length(Filtered) ->
             ct:log("Matching measurements for event ~p with labels ~p:~n~p",
                    [EventName, Labels, Filtered]),
-            event_tested(EventName, Labels)
+            event_tested(EventName, Labels);
+        Filtered ->
+            ct:log("All measurements for event ~p with labels ~p:~n~p",
+                   [EventName, Labels, MeasurementsList]),
+            ct:fail("Incorrect number of instrumentation events - matched: ~p, expected: ~p",
+                    [length(Filtered), ExpectedCount])
     end.
 
 assert_not_emitted(EventName, Labels) ->
-    case lookup(EventName, Labels) of
-        [] ->
-            ok;
-        Events ->
-            ct:fail("Measurements emitted but should not ~p", [Events])
-    end.
+    assert_none(EventName, Labels, fun(_) -> true end).
 
 assert_not_emitted(Events) ->
     [assert_not_emitted(Event, Label) || {Event, Label} <- Events].

big_tests/tests/mam_SUITE.erl

@@ -113,7 +113,8 @@
          assert_lookup_event/2,
          assert_flushed_event_if_async/2,
          assert_dropped_iq_event/2,
-         assert_event_with_jid/2
+         assert_event_with_jid/2,
+         assert_no_event_with_jid/2
         ]).
 
 -import(muc_light_helper,
@@ -3395,11 +3396,13 @@ muc_prefs_set_request(ConfigIn) ->
 muc_prefs_set_request_not_an_owner(ConfigIn) ->
     F = fun(Config, _Alice, Bob) ->
         Room = ?config(room, Config),
+        RoomAddr = room_address(Room),
         escalus:send(Bob, stanza_to_room(stanza_prefs_set_request(<<"roster">>,
                                                                     [<<"[email protected]">>],
                                                                     [<<"[email protected]">>],
                                                                     mam_ns_binary()), Room)),
-        escalus:assert(is_error, [<<"cancel">>, <<"not-allowed">>], escalus:wait_for_stanza(Bob))
+        escalus:assert(is_error, [<<"cancel">>, <<"not-allowed">>], escalus:wait_for_stanza(Bob)),
+        assert_no_event_with_jid(mod_mam_muc_get_prefs, RoomAddr)
     end,
     RoomOpts = [{persistent, true}],
     UserSpecs = [{alice, 1}, {bob, 1}],

big_tests/tests/mam_helper.erl

@@ -1414,27 +1414,30 @@ assert_list_size(N, List) when N =:= length(List) -> List.
 %% Assertions for instrumentation events
 
 assert_archive_message_event(EventName, BinJid) ->
-    assert_event(EventName,
-                 fun(#{count := 1, time := T, params := #{local_jid := LocalJid}}) ->
-                         eq_bjid(LocalJid, BinJid) andalso pos_int(T)
-                 end).
+    instrument_helper:assert_one(
+      EventName, labels(),
+      fun(#{count := 1, time := T, params := #{local_jid := LocalJid}}) ->
+              eq_bjid(LocalJid, BinJid) andalso pos_int(T)
+      end).
 
 assert_lookup_event(EventName, BinJid) ->
-    assert_event(EventName,
-                 fun(#{count := 1, size := 1, time := T, params := #{caller_jid := CallerJid}}) ->
-                         eq_bjid(CallerJid, BinJid) andalso pos_int(T)
-                 end).
+    instrument_helper:assert_one(
+      EventName, labels(),
+      fun(#{count := 1, size := 1, time := T, params := #{caller_jid := CallerJid}}) ->
+              eq_bjid(CallerJid, BinJid) andalso pos_int(T)
+      end).
 
 %% The event might originate from a different test case running in parallel,
 %% but there is no easy way around it other than adding all flushed messages to measurements.
 assert_flushed_event_if_async(EventName, Config) ->
     case ?config(configuration, Config) of
         C when C =:= rdbms_async_pool;
                C =:= rdbms_async_cache ->
-            assert_event(EventName,
-                         fun(#{count := Count, time := T, time_per_message := T1}) ->
-                                 pos_int(Count) andalso pos_int(T) andalso pos_int(T1) andalso T >= T1
-                         end);
+            instrument_helper:assert(
+              EventName, labels(),
+              fun(#{count := Count, time := T, time_per_message := T1}) ->
+                      pos_int(Count) andalso pos_int(T) andalso pos_int(T1) andalso T >= T1
+              end);
         _ ->
             ok
     end.
@@ -1444,18 +1447,21 @@ assert_dropped_iq_event(Config, BinJid) ->
                     undefined -> mod_mam_pm_dropped_iq;
                     _ -> mod_mam_muc_dropped_iq
                 end,
-    assert_event(EventName, fun(#{acc := #{stanza := #{from_jid := FromJid}}}) ->
-                                    eq_bjid(FromJid, BinJid)
-                            end).
+    instrument_helper:assert_one(
+      EventName, labels(),
+      fun(#{acc := #{stanza := #{from_jid := FromJid}}}) -> eq_bjid(FromJid, BinJid) end).
 
 assert_event_with_jid(EventName, BinJid) ->
-    assert_event(EventName, fun(#{count := 1, jid := Jid}) -> eq_bjid(Jid, BinJid) end).
+    assert_events_with_jid(EventName, BinJid, 1).
 
-assert_dropped_msg_event(EventName) ->
-    assert_event(EventName, fun(#{count := 1}) -> true end).
+assert_no_event_with_jid(EventName, BinJid) ->
+    assert_events_with_jid(EventName, BinJid, 0).
 
-assert_event(EventName, F) ->
-    instrument_helper:assert(EventName, #{host_type => host_type()}, F).
+assert_events_with_jid(EventName, BinJid, Count) ->
+    instrument_helper:assert(EventName, labels(),
+                             fun(#{count := 1, jid := Jid}) -> eq_bjid(Jid, BinJid) end, Count).
+
+labels() -> #{host_type => host_type()}.
 
 pos_int(T) -> is_integer(T) andalso T > 0.
 

big_tests/tests/metrics_c2s_SUITE.erl

@@ -80,8 +80,8 @@ login_story(Alice) ->
 
     %% Note: The first two events might originate from other tests because of unknown JID.
     %% It is acceptable, because the goal is to check that they are emitted when users log in.
-    assert_event(out, fun(#xmlel{name = Name}) -> Name =:= <<"stream:features">> end),
-    assert_event(in, fun(#xmlel{name = Name}) -> Name =:= <<"auth">> end),
+    assert_events(out, fun(#xmlel{name = Name}) -> Name =:= <<"stream:features">> end),
+    assert_events(in, fun(#xmlel{name = Name}) -> Name =:= <<"auth">> end),
 
     assert_event(out, AliceBareJid, #{},
                  fun(#xmlel{name = Name}) -> Name =:= <<"success">> end),
@@ -189,11 +189,11 @@ has_child(SubElName, El) ->
 
 assert_event(Dir, ClientOrJid, Measurements) ->
     Jid = jid:from_binary(escalus_utils:get_jid(ClientOrJid)),
-    instrument_helper:assert(
+    instrument_helper:assert_one(
       event_name(Dir), #{host_type => host_type()},
       fun(M) -> M =:= Measurements#{jid => Jid, count => 1} end).
 
-assert_event(Dir, CheckElFun) ->
+assert_events(Dir, CheckElFun) ->
     instrument_helper:assert(
       event_name(Dir), #{host_type => host_type()},
       fun(M = #{element := El}) ->
@@ -202,7 +202,7 @@ assert_event(Dir, CheckElFun) ->
 
 assert_event(Dir, ClientOrJid, Measurements, CheckElFun) ->
     Jid = jid:from_binary(escalus_utils:get_jid(ClientOrJid)),
-    instrument_helper:assert(
+    instrument_helper:assert_one(
       event_name(Dir), #{host_type => host_type()},
       fun(M = #{element := El}) ->
                maps:remove(element, M) =:= Measurements#{jid => Jid, count => 1}
@@ -217,7 +217,7 @@ event_name(in) -> c2s_element_in.
 assert_message_bounced_event(Sender, Recipient) ->
     FromJid = jid:from_binary(escalus_utils:get_jid(Sender)),
     ToJid = jid:from_binary(escalus_utils:get_jid(Recipient)),
-    instrument_helper:assert(
+    instrument_helper:assert_one(
       sm_message_bounced, #{host_type => host_type()},
       fun(#{count := 1, from_jid := From, to_jid := To}) ->
                From =:= FromJid andalso To =:= ToJid

big_tests/tests/metrics_roster_SUITE.erl

@@ -301,7 +301,7 @@ declared_backend_events() ->
 %% This works only for get_roster and get_subscription_lists because of the function arguments
 assert_backend_event(Client, Function) ->
     ClientJid = jid:from_binary(escalus_utils:get_short_jid(Client)),
-    instrument_helper:assert(
+    instrument_helper:assert_one(
       backend_mod(), #{host_type => host_type(), function => Function},
       fun(#{count := 1, time := T, args := [_, User, Server]}) when T > 0 ->
               ClientJid =:= jid:make_bare(User, Server)

big_tests/tests/metrics_session_SUITE.erl

@@ -121,21 +121,20 @@ session_unique(Config) ->
 assert_sm_login_event(Client) ->
     JID = jid:from_binary(escalus_client:full_jid(Client)),
     F = fun(M) -> M =:= #{logins => 1, count => 1, jid => JID} end,
-    instrument_helper:assert(sm_session, #{host_type => host_type()}, F).
+    instrument_helper:assert_one(sm_session, #{host_type => host_type()}, F).
 
 assert_no_sm_login_event(UserSpec) ->
     LUser = jid:nodeprep(proplists:get_value(username, UserSpec)),
-    LoginEvents = instrument_helper:lookup(sm_session, #{host_type => host_type()}),
     F = fun(#{jid := JID}) -> jid:luser(JID) =:= LUser end,
-    ?assertEqual([], instrument_helper:filter(F, LoginEvents)).
+    instrument_helper:assert_none(sm_session, #{host_type => host_type()}, F).
 
 assert_sm_logout_event(Client) ->
     JID = jid:from_binary(escalus_client:full_jid(Client)),
     F = fun(M) -> M =:= #{logouts => 1, count => -1, jid => JID} end,
-    instrument_helper:assert(sm_session, #{host_type => host_type()}, F).
+    instrument_helper:assert_one(sm_session, #{host_type => host_type()}, F).
 
 assert_c2s_auth_failed(UserSpec) ->
     Server = proplists:get_value(server, UserSpec),
     UserName = proplists:get_value(username, UserSpec),
     F = fun(M) -> M =:= #{count => 1, server => Server, username => UserName} end,
-    instrument_helper:assert(c2s_auth_failed, #{host_type => host_type()}, F).
+    instrument_helper:assert_one(c2s_auth_failed, #{host_type => host_type()}, F).

big_tests/tests/mod_event_pusher_http_SUITE.erl

@@ -116,23 +116,28 @@ simple_message(Config) ->
     {_, _} = binary:match(Body, <<"alice">>),
     {_, _} = binary:match(Body, <<"Simple">>),
     AliceName = escalus_client:username(Alice),
-    instrument_helper:assert(mod_event_pusher_http_sent, #{host_type => host_type()},
-                             fun(#{count := 1, sender := S, response_code := <<"200">>, response_time := T}) ->
-                                 ?assert(T > 0), AliceName =:= S end).
+    instrument_helper:assert_one(
+      mod_event_pusher_http_sent, #{host_type => host_type()},
+      fun(#{count := 1, sender := S, response_code := <<"200">>, response_time := T}) ->
+              ?assert(T > 0), AliceName =:= S
+      end).
 
 simple_message_no_listener(Config) ->
     Alice = do_simple_message(Config, <<"Hi, NoListener!">>),
     AliceName = escalus_client:username(Alice),
-    instrument_helper:assert(mod_event_pusher_http_sent, #{host_type => host_type()},
-                             fun(#{failure_count := 1, sender := S}) -> AliceName =:= S end).
+    instrument_helper:assert_one(
+      mod_event_pusher_http_sent, #{host_type => host_type()},
+      fun(#{failure_count := 1, sender := S}) -> AliceName =:= S end).
 
 
 simple_message_failing_listener(Config) ->
     Alice = do_simple_message(Config, <<"Hi, Failing!">>),
     AliceName = escalus_client:username(Alice),
-    instrument_helper:assert(mod_event_pusher_http_sent, #{host_type => host_type()},
-                             fun(#{count := 1, sender := S, response_code := <<"500">>, response_time := T}) ->
-                                 ?assert(T > 0), AliceName =:= S end).
+    instrument_helper:assert_one(
+      mod_event_pusher_http_sent, #{host_type => host_type()},
+      fun(#{count := 1, sender := S, response_code := <<"500">>, response_time := T}) ->
+              ?assert(T > 0), AliceName =:= S
+      end).
 
 do_simple_message(Config0, Msg) ->
     Config = escalus_fresh:create_users(Config0, [{alice, 1}, {bob, 1}]),