Add CETS backend for mod_keystore

big_tests/test.config

@@ -248,6 +248,7 @@
       {stream_management_backend, cets},
       {muc_online_backend, cets},
       {jingle_sip_backend, cets},
+      {keystore_backend, cets},
       {auth_method, "rdbms"},
       {internal_databases, "[internal_databases.cets]
   cluster_name = \"{{cluster_name}}\""},

big_tests/tests/graphql_token_SUITE.erl

@@ -82,9 +82,10 @@ end_per_suite(Config) ->
     dynamic_modules:restore_modules(Config),
     escalus:end_per_suite(Config).
 
-required_modules() ->
+required_modules(Config) ->
     KeyOpts = #{keys => #{token_secret => ram,
-                         provision_pre_shared => ram}},
+                         provision_pre_shared => ram},
+                backend => ct_helper:get_preset_var(Config, keystore_backend, mnesia)},
     KeyStoreOpts = config_parser_helper:mod_config(mod_keystore, KeyOpts),
     [{mod_keystore, KeyStoreOpts},
      {mod_auth_token, auth_token_opts()}].
@@ -112,7 +113,7 @@ init_per_group(user, Config) ->
 
 ensure_token_started(Config) ->
     HostType = domain_helper:host_type(),
-    dynamic_modules:ensure_modules(HostType, required_modules()),
+    dynamic_modules:ensure_modules(HostType, required_modules(Config)),
     Config.
 
 ensure_token_stopped(Config) ->

big_tests/tests/oauth_SUITE.erl

@@ -83,7 +83,7 @@ init_per_suite(Config0) ->
         true ->
             HostType = domain_helper:host_type(),
             Config = dynamic_modules:save_modules(HostType, Config0),
-            dynamic_modules:ensure_modules(HostType, required_modules()),
+            dynamic_modules:ensure_modules(HostType, required_modules(Config)),
             escalus:init_per_suite(Config);
         false ->
             {skip, "RDBMS not available"}
@@ -459,8 +459,9 @@ serialize(ServerSideToken) ->
 to_lower(B) when is_binary(B) ->
     list_to_binary(string:to_lower(binary_to_list(B))).
 
-required_modules() ->
-    KeyOpts = #{keys => #{token_secret => ram,
+required_modules(Config) ->
+    KeyOpts = #{backend => ct_helper:get_preset_var(Config, keystore_backend, mnesia),
+                keys => #{token_secret => ram,
                          %% This is a hack for tests! As the name implies,
                          %% a pre-shared key should be read from a file stored
                          %% on disk. This way it can be shared with trusted 3rd

rebar.config

@@ -80,7 +80,7 @@
   {cache_tab, "1.0.30"},
   {segmented_cache, "0.3.0"},
   {worker_pool, "6.0.1"},
-  {cets, {git, "https://github.com/esl/cets.git", {branch, "main"}}},
+  {cets, {git, "https://github.com/esl/cets.git", {branch, "insert-new-or-lookup"}}},
 
   %%% HTTP tools
   {graphql, {git, "https://github.com/esl/graphql-erlang.git", {branch, "master"}}},

rebar.lock

@@ -8,7 +8,7 @@
  {<<"certifi">>,{pkg,<<"certifi">>,<<"2.9.0">>},1},
  {<<"cets">>,
   {git,"https://github.com/esl/cets.git",
-       {ref,"e3ad43f3836ea457bcb54e2f8266e9d7c32b014f"}},
+       {ref,"72b01d2dd97c193e60365f0e7e53d30d0876ba1b"}},
   0},
  {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.9.0">>},0},
  {<<"cowboy_swagger">>,{pkg,<<"cowboy_swagger">>,<<"2.5.1">>},0},

src/mod_keystore.erl

@@ -57,6 +57,7 @@ start(HostType, Opts) ->
 -spec stop(mongooseim:host_type()) -> ok.
 stop(HostType) ->
     clear_keystore_ets(HostType),
+    mod_keystore_backend:stop(HostType),
     ok.
 
 -spec hooks(mongooseim:host_type()) -> gen_hook:hook_list().

src/mod_keystore_backend.erl

@@ -2,11 +2,12 @@
 %% the backend modules (i.e. mod_keystore_mnesia...).
 -module(mod_keystore_backend).
 
--export([init/2, init_ram_key/2, get_key/1]).
+-export([init/2, stop/1, init_ram_key/2, get_key/1]).
 
 -define(MAIN_MODULE, mod_keystore).
 
 -callback init(mongooseim:host_type(), gen_mod:module_opts()) -> ok.
+-callback stop(mongooseim:host_type()) -> ok.
 
 %% Cluster members race to decide whose key gets stored in the distributed database.
 %% That's why ProposedKey (the key this cluster member tries to propagate to other nodes)
@@ -24,6 +25,11 @@ init(HostType, Opts) ->
     Args = [HostType, Opts],
     mongoose_backend:call(HostType, ?MAIN_MODULE, ?FUNCTION_NAME, Args).
 
+-spec stop(mongooseim:host_type()) -> ok.
+stop(HostType) ->
+    Args = [HostType],
+    mongoose_backend:call(HostType, ?MAIN_MODULE, ?FUNCTION_NAME, Args).
+
 %% Cluster members race to decide whose key gets stored in the distributed database.
 %% That's why ProposedKey (the key this cluster member tries to propagate to other nodes)
 %% might not be the same as ActualKey (key of the member who will have won the race).

src/mod_keystore_cets.erl

@@ -0,0 +1,52 @@
+-module(mod_keystore_cets).
+-behaviour(mod_keystore_backend).
+
+-export([init/2,
+         stop/1,
+         init_ram_key/1,
+         get_key/1]).
+
+%% CETS callbacks
+-export([handle_conflict/2]).
+
+-ignore_xref([get_key/1, init/2, init_ram_key/1]).
+
+-include("mod_keystore.hrl").
+-include("mongoose_logger.hrl").
+
+table_name(HostType) ->
+    binary_to_atom(<<"cets_keystore_", HostType/binary>>).
+
+-spec init(mongooseim:host_type(), gen_mod:module_opts()) -> ok.
+init(HostType, _Opts) ->
+    %% There is no logic to remove keys.
+    %% Separate table per HostType (so we could remove the table once the module is unloaded).
+    Tab = table_name(HostType),
+    cets:start(Tab, #{handle_conflict => fun ?MODULE:handle_conflict/2}),
+    cets_discovery:add_table(mongoose_cets_discovery, Tab),
+    ok.
+
+-spec stop(mongooseim:host_type()) -> ok.
+stop(HostType) ->
+    Tab = table_name(HostType),
+    cets_discovery:delete_table(mongoose_cets_discovery, Tab),
+    cets:stop(Tab),
+    ok.
+
+%% We need to choose one key consistently.
+handle_conflict(Rec1, Rec2) ->
+    max(Rec1, Rec2).
+
+-spec init_ram_key(ProposedKey) -> Result when
+      ProposedKey :: mod_keystore:key(),
+      Result :: {ok, ActualKey} | {error, init_ram_key_failed},
+      ActualKey :: mod_keystore:key().
+init_ram_key(#key{id = Id = {_, HostType}, key = PropKey}) ->
+    Tab = table_name(HostType),
+    {_, [{Id, Key}]} = cets:insert_new_or_lookup(Tab, {Id, PropKey}),
+    {ok, #key{id = Id, key = Key}}.
+
+-spec get_key(Id :: mod_keystore:key_id()) -> mod_keystore:key_list().
+get_key(Id = {_, HostType}) ->
+    Tab = table_name(HostType),
+    ets:lookup(Tab, Id).

src/mod_keystore_mnesia.erl

@@ -2,6 +2,7 @@
 -behaviour(mod_keystore_backend).
 
 -export([init/2,
+         stop/1,
          init_ram_key/1,
          get_key/1]).
 
@@ -18,6 +19,10 @@ init(_HostType, _Opts) ->
     mnesia:add_table_copy(key, node(), ram_copies),
     ok.
 
+-spec stop(mongooseim:host_type()) -> ok.
+stop(_HostType) ->
+    ok.
+
 -spec init_ram_key(ProposedKey) -> Result when
       ProposedKey :: mod_keystore:key(),
       Result :: {ok, ActualKey} | {error, any()},