Merge pull request #3544 from esl/markers/removals

big_tests/default.spec

@@ -78,6 +78,7 @@
 {suites, "tests", service_mongoose_system_metrics_SUITE}.
 {suites, "tests", shared_roster_SUITE}.
 {suites, "tests", sic_SUITE}.
+{suites, "tests", smart_markers_SUITE}.
 {suites, "tests", sm_SUITE}.
 {suites, "tests", users_api_SUITE}.
 {suites, "tests", vcard_SUITE}.

big_tests/dynamic_domains.spec

@@ -125,6 +125,7 @@
 
 {suites, "tests", sic_SUITE}.
 
+{suites, "tests", smart_markers_SUITE}.
 {suites, "tests", sm_SUITE}.
 {suites, "tests", users_api_SUITE}.
 {suites, "tests", vcard_SUITE}.

big_tests/tests/domain_removal_SUITE.erl

@@ -12,7 +12,8 @@
 -include_lib("common_test/include/ct.hrl").
 
 all() ->
-    [{group, auth_removal},
+    [
+     {group, auth_removal},
      {group, cache_removal},
      {group, mam_removal},
      {group, inbox_removal},
@@ -21,8 +22,10 @@ all() ->
      {group, private_removal},
      {group, roster_removal},
      {group, offline_removal},
+     {group, markers_removal},
      {group, vcard_removal},
-     {group, last_removal}].
+     {group, last_removal}
+    ].
 
 groups() ->
     [
@@ -37,6 +40,7 @@ groups() ->
      {private_removal, [], [private_removal]},
      {roster_removal, [], [roster_removal]},
      {offline_removal, [], [offline_removal]},
+     {markers_removal, [], [markers_removal]},
      {vcard_removal, [], [vcard_removal]},
      {last_removal, [], [last_removal]}
     ].
@@ -98,6 +102,8 @@ group_to_modules(roster_removal) ->
     [{mod_roster, [{backend, rdbms}]}];
 group_to_modules(offline_removal) ->
     [{mod_offline, [{backend, rdbms}]}];
+group_to_modules(markers_removal) ->
+    [{mod_smart_markers, [{backend, rdbms}]}];
 group_to_modules(vcard_removal) ->
     [{mod_vcard, config_parser_helper:mod_config(mod_vcard, #{backend => rdbms})}];
 group_to_modules(last_removal) ->
@@ -290,6 +296,27 @@ offline_removal(Config) ->
         ?assertMatch({ok, []}, rpc(mim(), mod_offline_rdbms, fetch_messages, [host_type(), BobJid]))
     end).
 
+markers_removal(Config) ->
+    escalus:fresh_story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
+        Body = <<"Hello Bob!">>,
+        MsgId = escalus_stanza:id(),
+        Msg = escalus_stanza:set_id(escalus_stanza:chat_to(Bob, Body), MsgId),
+        escalus:send(Alice, Msg),
+        escalus:wait_for_stanza(Bob),
+        ChatMarker = escalus_stanza:chat_marker(Alice, <<"displayed">>, MsgId),
+        escalus:send(Bob, ChatMarker),
+        escalus:wait_for_stanza(Alice),
+        mongoose_helper:wait_until(
+          fun() -> 1 =< mongoose_helper:generic_count(mod_smart_markers) end, true),
+        % check messages in DB
+        AliceJid = jid:from_binary(escalus_client:full_jid(Alice)),
+        ?assertMatch([_], rpc(mim(), mod_smart_markers_backend, get_chat_markers,
+                                    [host_type(), AliceJid, undefined, 0])),
+        run_remove_domain(),
+        ?assertMatch([], rpc(mim(), mod_smart_markers_backend, get_chat_markers,
+                                   [host_type(), AliceJid, undefined, 0]))
+    end).
+
 roster_removal(Config) ->
     escalus:fresh_story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
         %% add contact

big_tests/tests/mongoose_helper.erl

@@ -19,7 +19,8 @@
          total_privacy_items/0,
          total_private_items/0,
          total_vcard_items/0,
-         total_roster_items/0]).
+         total_roster_items/0,
+         generic_count/1]).
 
 -export([clear_last_activity/2,
          clear_caps_cache/1]).
@@ -188,6 +189,7 @@ generic_count_per_host_type(HostType, Module) ->
             generic_count_backend(B)
     end.
 
+generic_count_backend(mod_smart_markers_rdbms) -> count_rdbms(<<"smart_markers">>);
 generic_count_backend(mod_offline_mnesia) -> count_wildpattern(offline_msg);
 generic_count_backend(mod_offline_rdbms) -> count_rdbms(<<"offline_message">>);
 generic_count_backend(mod_offline_riak) -> count_riak(<<"offline">>);
@@ -222,7 +224,7 @@ count_wildpattern(Table) ->
 count_rdbms(Table) ->
     {selected, [{N}]} =
         rpc(mim(), mongoose_rdbms, sql_query,
-            [<<"localhost">>, [<<"select count(*) from ", Table/binary, " ;">>]]),
+            [domain_helper:host_type(), [<<"select count(*) from ", Table/binary, " ;">>]]),
     count_to_integer(N).
 
 count_to_integer(N) when is_binary(N) ->

big_tests/tests/smart_markers_SUITE.erl

@@ -0,0 +1,191 @@
+-module(smart_markers_SUITE).
+-compile([export_all, nowarn_export_all]).
+
+-include("muc_light.hrl").
+
+-import(distributed_helper, [mim/0, rpc/4, subhost_pattern/1]).
+-import(domain_helper, [host_type/0]).
+
+%%% Suite configuration
+all() ->
+    case (not ct_helper:is_ct_running())
+          orelse mongoose_helper:is_rdbms_enabled(host_type()) of
+        true -> all_cases();
+        false -> {skip, require_rdbms}
+    end.
+
+all_cases() ->
+    [
+     {group, one2one},
+     {group, muclight}
+    ].
+
+groups() ->
+    [
+     {one2one, [],
+      [
+       marker_is_stored,
+       remove_markers_when_removed_user
+      ]},
+     {muclight, [],
+      [
+       marker_is_stored_for_room,
+       marker_is_removed_when_user_leaves_room,
+       markers_are_removed_when_room_is_removed
+      ]}
+    ].
+
+suite() ->
+    escalus:suite().
+
+init_per_suite(Config) ->
+    escalus:init_per_suite(Config).
+
+end_per_suite(Config) ->
+    escalus:end_per_suite(Config).
+
+init_per_group(GroupName, Config) ->
+    ok = dynamic_modules:ensure_modules(host_type(), group_to_module(GroupName)),
+    Config.
+
+group_to_module(one2one) ->
+    [{mod_smart_markers, [{backend, rdbms}]}];
+group_to_module(muclight) ->
+    [{mod_smart_markers, [{backend, rdbms}]},
+     {mod_muc_light,
+      [{host, subhost_pattern(muc_light_helper:muc_host_pattern())},
+       {backend, rdbms}]}].
+
+end_per_group(muclight, Config) ->
+    muc_light_helper:clear_db(host_type()),
+    end_per_group(generic, Config);
+end_per_group(_, Config) ->
+    escalus_fresh:clean(),
+    dynamic_modules:restore_modules(Config),
+    Config.
+
+init_per_testcase(Name, Config) ->
+    escalus:init_per_testcase(Name, Config).
+end_per_testcase(Name, Config) ->
+    escalus:end_per_testcase(Name, Config).
+
+%%% tests
+marker_is_stored(Config) ->
+    escalus:fresh_story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
+        send_message_respond_marker(Alice, Bob),
+        AliceJid = jid:from_binary(escalus_client:full_jid(Alice)),
+        BobJid = jid:from_binary(escalus_client:full_jid(Bob)),
+        mongoose_helper:wait_until(
+          fun() -> length(fetch_markers_for_users(BobJid, AliceJid)) > 0 end, true)
+    end).
+
+remove_markers_when_removed_user(Config) ->
+    escalus:fresh_story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
+        Body = <<"Hello Bob!">>,
+        MsgId = escalus_stanza:id(),
+        Msg = escalus_stanza:set_id(escalus_stanza:chat_to(Bob, Body), MsgId),
+        escalus:send(Alice, Msg),
+        escalus:wait_for_stanza(Bob),
+        ChatMarker = escalus_stanza:chat_marker(Alice, <<"displayed">>, MsgId),
+        escalus:send(Bob, ChatMarker),
+        escalus:wait_for_stanza(Alice),
+        AliceJid = jid:from_binary(escalus_client:full_jid(Alice)),
+        BobJid = jid:from_binary(escalus_client:full_jid(Bob)),
+        mongoose_helper:wait_until(fun() -> length(fetch_markers_for_users(BobJid, AliceJid)) > 0 end, true),
+        unregister_user(Bob),
+        mongoose_helper:wait_until(fun() -> length(fetch_markers_for_users(BobJid, AliceJid)) end, 0)
+    end).
+
+marker_is_stored_for_room(Config) ->
+    escalus:fresh_story(Config, [{alice, 1}, {bob, 1}, {kate, 1}],
+                        fun(Alice, Bob, Kate) ->
+        Users = [Alice, Bob, Kate],
+        RoomId = create_room(Alice, [Bob, Kate], Config),
+        RoomBinJid = muc_light_helper:room_bin_jid(RoomId),
+        one_marker_in_room(Users, RoomBinJid, Alice, Bob),
+        BobJid = jid:from_binary(escalus_client:full_jid(Bob)),
+        mongoose_helper:wait_until(
+          fun() -> length(fetch_markers_for_users(BobJid, jid:from_binary(RoomBinJid))) > 0 end, true)
+    end).
+
+marker_is_removed_when_user_leaves_room(Config) ->
+    escalus:fresh_story(Config, [{alice, 1}, {bob, 1}],
+                        fun(Alice, Bob) ->
+        Users = [Alice, Bob],
+        RoomId = create_room(Alice, [Bob], Config),
+        RoomBinJid = muc_light_helper:room_bin_jid(RoomId),
+        RoomJid = jid:from_binary(RoomBinJid),
+        one_marker_in_room(Users, RoomBinJid, Alice, Bob),
+        BobJid = jid:from_binary(escalus_client:full_jid(Bob)),
+        mongoose_helper:wait_until(
+          fun() -> length(fetch_markers_for_users(BobJid, RoomJid)) > 0 end, true),
+        % Remove Bob from the room
+        muc_light_helper:user_leave(RoomId, Bob, [Alice]),
+        mongoose_helper:wait_until(
+          fun() -> length(fetch_markers_for_users(BobJid, RoomJid)) end, 0)
+    end).
+
+markers_are_removed_when_room_is_removed(Config) ->
+    escalus:fresh_story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
+        Users = [Alice, Bob],
+        RoomId = create_room(Alice, [Bob], Config),
+        RoomBinJid = muc_light_helper:room_bin_jid(RoomId),
+        RoomJid = jid:from_binary(RoomBinJid),
+        one_marker_in_room(Users, RoomBinJid, Alice, Bob),
+        BobJid = jid:from_binary(escalus_client:full_jid(Bob)),
+        mongoose_helper:wait_until(
+          fun() -> length(fetch_markers_for_users(BobJid, RoomJid)) > 0 end, true),
+        %% The room is then deleted
+        delete_room(Alice, Users, RoomBinJid),
+        [ begin
+              Jid = jid:from_binary(escalus_client:full_jid(User)),
+              mongoose_helper:wait_until(
+                fun() -> length(fetch_markers_for_users(Jid, RoomJid)) end, 0)
+          end || User <- Users]
+    end).
+
+%%% helpers
+fetch_markers_for_users(From, To) ->
+    MRs = rpc(mim(), mod_smart_markers_backend, get_chat_markers,
+              [host_type(), To, undefined, 0]),
+    [MR || #{from := FR} = MR <- MRs, jid:are_bare_equal(From, FR)].
+
+create_room(Owner, Members, Config) ->
+    RoomId = muc_helper:fresh_room_name(),
+    MucHost = muc_light_helper:muc_host(),
+    muc_light_helper:create_room(RoomId, MucHost, Owner, Members, Config, muc_light_helper:ver(1)),
+    RoomId.
+
+delete_room(Owner, Users, RoomBinJid) ->
+    Destroy = escalus_stanza:to(escalus_stanza:iq_set(?NS_MUC_LIGHT_DESTROY, []), RoomBinJid),
+    escalus:send(Owner, Destroy),
+    AffUsersChanges = [{User, none} || User <- Users ],
+    muc_light_helper:verify_aff_bcast([], AffUsersChanges, [?NS_MUC_LIGHT_DESTROY]),
+    escalus:assert(is_iq_result, escalus:wait_for_stanza(Owner)).
+
+one_marker_in_room(Users, RoomBinJid, Writer, Marker) ->
+        MsgId = escalus_stanza:id(),
+        Msg = escalus_stanza:set_id(
+                escalus_stanza:groupchat_to(RoomBinJid, <<"Hello">>), MsgId),
+        escalus:send(Writer, Msg),
+        [ escalus:wait_for_stanza(User) || User <- Users],
+        ChatMarker = escalus_stanza:setattr(
+                       escalus_stanza:chat_marker(RoomBinJid, <<"displayed">>, MsgId),
+                       <<"type">>, <<"groupchat">>),
+        escalus:send(Marker, ChatMarker),
+        [ escalus:wait_for_stanza(User) || User <- Users],
+        MsgId.
+
+send_message_respond_marker(MsgWriter, MarkerAnswerer) ->
+    Body = <<"Hello">>,
+    MsgId = escalus_stanza:id(),
+    Msg = escalus_stanza:set_id(escalus_stanza:chat_to(MarkerAnswerer, Body), MsgId),
+    escalus:send(MsgWriter, Msg),
+    escalus:wait_for_stanza(MarkerAnswerer),
+    ChatMarker = escalus_stanza:chat_marker(MsgWriter, <<"displayed">>, MsgId),
+    escalus:send(MarkerAnswerer, ChatMarker),
+    escalus:wait_for_stanza(MsgWriter).
+
+unregister_user(Client) ->
+    Jid = jid:from_binary(escalus_client:short_jid(Client)),
+    rpc(mim(), ejabberd_auth, remove_user, [Jid]).

doc/migrations/5.0.0_5.1.0.md

@@ -45,3 +45,6 @@ now becomes
   async_writer.batch_size = 100
   muc.async_writer.enabled = false
 ```
+
+## Smart markers
+There's an experimental and undocumented module called `mod_smart_markers`, that had a default table in the RDBMS schema, which you probably never used (or shouldn't have, as it was undocumented). If you rely on this table, the column `from_jid` has been split in `from_luser` and `lserver`, in order to support the `remove_domain` callback for the dynamic domains functionality. You might need to migrate it, or simply drop the previously defined table and recreate the new one.

priv/mssql2012.sql

@@ -686,9 +686,10 @@ CREATE TABLE muc_registered(
 -- from_jid, to_jid and thread have 250 characters in MySQL
 -- but here we are limited by index size (900 bytes)
 CREATE TABLE smart_markers (
-    from_jid NVARCHAR(150) NOT NULL,
-    to_jid NVARCHAR(150) NOT NULL,
-    thread NVARCHAR(145) NOT NULL,
+    lserver NVARCHAR(100) NOT NULL,
+    luser NVARCHAR(150) NOT NULL,
+    to_jid NVARCHAR(250) NOT NULL,
+    thread NVARCHAR(80) NOT NULL,
     -- chat marker types:
     -- 'R' - received
     -- 'D' - displayed
@@ -697,7 +698,8 @@ CREATE TABLE smart_markers (
     msg_id NVARCHAR(250) NOT NULL,
     timestamp BIGINT NOT NULL,
     CONSTRAINT pk_smart_markers PRIMARY KEY CLUSTERED(
-            from_jid ASC,
+            lserver ASC,
+            luser ASC,
             to_jid ASC,
             thread ASC,
             type ASC

priv/mysql.sql

@@ -493,16 +493,17 @@ CREATE TABLE mongoose_cluster_id (
 );
 
 CREATE TABLE smart_markers (
-    from_jid VARCHAR(250) NOT NULL,
-    to_jid VARCHAR(250) NOT NULL,
-    thread VARCHAR(250) NOT NULL,
+    lserver VARBINARY(255) NOT NULL,
+    luser VARBINARY(1023) NOT NULL,
+    to_jid VARBINARY(1542) NOT NULL,
+    thread VARBINARY(250) NOT NULL,
     -- 'R' - received
     -- 'D' - displayed
     -- 'A' - acknowledged
     type ENUM('R', 'D', 'A') NOT NULL,
-    msg_id VARCHAR(250) NOT NULL,
+    msg_id VARBINARY(250) NOT NULL,
     timestamp BIGINT NOT NULL,
-    PRIMARY KEY(from_jid, to_jid, thread, type)
+    PRIMARY KEY(lserver, luser, to_jid, thread, type)
 ) CHARACTER SET utf8mb4;
 
 CREATE INDEX i_smart_markers USING BTREE ON smart_markers(to_jid, thread);

priv/pg.sql

@@ -455,13 +455,14 @@ CREATE TABLE mongoose_cluster_id (
 CREATE TYPE chat_marker_type AS ENUM('R', 'D', 'A');
 
 CREATE TABLE smart_markers (
-    from_jid VARCHAR(250) NOT NULL,
+    lserver VARCHAR(250) NOT NULL,
+    luser VARCHAR(250) NOT NULL,
     to_jid VARCHAR(250) NOT NULL,
     thread VARCHAR(250) NOT NULL,
     type chat_marker_type NOT NULL,
     msg_id VARCHAR(250) NOT NULL,
     timestamp BIGINT NOT NULL,
-    PRIMARY KEY(from_jid, to_jid, thread, type)
+    PRIMARY KEY(lserver, luser, to_jid, thread, type)
 );
 
 CREATE INDEX i_smart_markers ON smart_markers(to_jid, thread);