WIP

big_tests/tests/instrument_SUITE.erl

@@ -0,0 +1,133 @@
+%%% @doc Instrumentation tests that don't fit into the XMPP-specific test suites
+
+-module(instrument_SUITE).
+-include_lib("eunit/include/eunit.hrl").
+
+-compile([export_all, nowarn_export_all]).
+
+-import(distributed_helper, [mim/0, mim2/0, rpc/4]).
+-import(domain_helper, [host_type/1]).
+-import(mongooseimctl_helper, [rpc_call/3]).
+
+all() ->
+    [{group, cets},
+     {group, system}].
+
+groups() ->
+    [{cets, [], cets_cases()},
+     {system, [parallel], system_cases()}].
+
+cets_cases() ->
+    [cets_info].
+
+system_cases() ->
+    [system_up_time,
+     system_tcp_ports,
+     system_process_queue_lengths,
+     system_info,
+     system_memory,
+     system_dist_data].
+
+init_per_suite(Config) ->
+    Config1 = mongoose_helper:backup_and_set_config_option(
+                Config, [instrumentation, probe_interval], 1),
+    restart_probes(),
+    Config1.
+
+end_per_suite(Config) ->
+    mongoose_helper:restore_config_option(Config, [instrumentation, probe_interval]),
+    restart_probes().
+
+init_per_group(cets, Config) ->
+    case rpc_call(mongoose_config, lookup_opt, [[internal_databases, cets]]) of
+        {error, not_found} ->
+            {skip, "CETS is not configured"};
+        {ok, _} ->
+            instrument_helper:start([{cets_info, #{}}]),
+            Config
+    end;
+init_per_group(system, Config) ->
+    instrument_helper:start(instrument_helper:declared_events(mongoose_system_probe, [])),
+    Config.
+
+end_per_group(_, _Config) ->
+    instrument_helper:stop().
+
+init_per_testcase(_, Config) ->
+    Config.
+
+cets_info(_Config) ->
+    instrument_helper:wait_and_assert_new(cets_info, #{}, fun check_cets_info/1).
+
+%% Check that values are integers and there are no unknown fields
+check_cets_info(Res) ->
+    lists:all(fun(Name) -> is_integer(maps:get(Name, Res)) end, cets_metrics_names())
+        andalso #{} =:= maps:without(cets_metrics_names(), Res).
+
+cets_metrics_names() ->
+    [available_nodes,
+     unavailable_nodes,
+     joined_nodes,
+     discovered_nodes,
+     discovery_works,
+     remote_nodes_without_disco,
+     remote_nodes_with_unknown_tables,
+     remote_unknown_tables,
+     remote_nodes_with_missing_tables,
+     remote_missing_tables,
+     conflict_nodes,
+     conflict_tables].
+
+system_up_time(_Config) ->
+    instrument_helper:wait_and_assert_new(system_up_time, #{},
+                                          fun(#{seconds := Seconds}) -> Seconds > 0 end).
+
+system_tcp_ports(_Config) ->
+    instrument_helper:wait_and_assert_new(system_tcp_ports, #{},
+                                          fun(#{count := Count}) -> Count > 0 end).
+
+system_process_queue_lengths(_Config) ->
+    instrument_helper:wait_and_assert_new(system_process_queue_lengths, #{},
+                                          fun(#{total := Total}) -> Total >= 0 end).
+
+system_info(_Config) ->
+    instrument_helper:wait_and_assert_new(system_info, #{}, fun check_system_info/1).
+
+%% There should be at least one process, port and ETS, and limits shouldn't be reached.
+check_system_info(#{port_count := PortCount, port_limit := PortLimit,
+                    process_count := ProcessCount, process_limit := ProcessLimit,
+                    ets_count := ETSCount, ets_limit := ETSLimit})
+  when PortCount > 0, PortLimit > PortCount,
+       ProcessCount > 0, ProcessLimit > ProcessCount,
+       ETSCount > 0, ETSLimit > ETSCount ->
+    true.
+
+system_memory(_Config) ->
+    instrument_helper:wait_and_assert_new(system_memory, #{}, fun check_system_memory/1).
+
+%% There should be all types of memory consumption, with the constraints explained
+%% in the docs for erlang:memory/0.
+check_system_memory(
+  #{total := Total, atom := Atom, atom_used := AtomUsed, binary := Binary, code := Code, ets := ETS,
+    processes := Processes, processes_used := ProcessesUsed, system := System})
+  when Total =:= Processes + System, Atom >= AtomUsed, AtomUsed > 0, Binary > 0, Code > 0, ETS > 0,
+       Processes >= ProcessesUsed, System >= Atom + Binary + ETS ->
+    true.
+
+system_dist_data(_Config) ->
+    instrument_helper:wait_and_assert_new(system_dist_data, #{}, fun check_system_dist_data/1).
+
+%% There should be data sent and received already, and 2 connections at least (mim2, mim3).
+check_system_dist_data(
+  #{connections := Connections, recv_oct := RecvOct, recv_cnt := RecvCnt, recv_max := RecvMax,
+    send_oct := SendOct, send_cnt := SendCnt, send_max := SendMax, send_pend := SendPend})
+  when Connections >= 2, RecvOct > RecvCnt, RecvCnt > 0, RecvMax > 0,
+       SendOct > SendCnt, SendCnt > 0, SendMax > 0, SendPend >= 0 ->
+    true.
+
+restart_probes() ->
+    lists:foreach(fun restart/1, [mongoose_instrument_probe_cets, mongoose_system_probe]).
+
+restart(Module) ->
+    rpc_call(Module, stop, []),
+    rpc_call(Module, start, []).

src/ejabberd_app.erl

@@ -65,6 +65,7 @@ do_start() ->
     mongoose_logs:set_global_loglevel(mongoose_config:get_opt(loglevel)),
     mongoose_deprecations:start(),
     {ok, _} = Sup = ejabberd_sup:start_link(),
+    mongoose_system_probe:start(),
     mongoose_router:start(),
     mongoose_wpool:ensure_started(),
     mongoose_wpool:start_configured_pools(),
@@ -98,6 +99,7 @@ prep_stop(State) ->
     mongoose_metrics:remove_all_metrics(),
     mongoose_graphql_commands:stop(),
     mongoose_router:stop(),
+    mongoose_system_probe:stop(),
     State.
 
 %% All the processes were killed when this function is called

src/instrument/mongoose_instrument.erl

@@ -56,7 +56,7 @@
 -optional_callbacks([config_spec/0, start/0, stop/0]).
 
 -export_type([event_name/0, labels/0, label_key/0, label_value/0, config/0, measurements/0,
-              spec/0, handlers/0, metric_name/0, metric_type/0, probe_config/0]).
+              spec/0, handlers/0, metrics/0, metric_name/0, metric_type/0, probe_config/0]).
 
 %% API
 

src/metrics/mongoose_metrics.erl

@@ -56,8 +56,6 @@
 -spec init() -> ok.
 init() ->
     prepare_prefixes(),
-    create_vm_metrics(),
-    create_global_metrics(?GLOBAL_COUNTERS),
     create_data_metrics().
 
 -spec init_mongooseim_metrics() -> ok.
@@ -124,7 +122,7 @@ get_dist_data_stats() ->
 
 -spec get_up_time() -> {value, integer()}.
 get_up_time() ->
-    {value, erlang:round(element(1, erlang:statistics(wall_clock))/1000)}.
+    {value, round(element(1, erlang:statistics(wall_clock))/1000)}.
 
 -spec get_mnesia_running_db_nodes_count() -> {value, non_neg_integer()}.
 get_mnesia_running_db_nodes_count() ->
@@ -228,12 +226,6 @@ remove_metric({Name, _, _}) ->
 create_global_metrics(Metrics) ->
     lists:foreach(fun({Metric, Spec}) -> ensure_metric(global, Metric, Spec) end, Metrics).
 
-create_vm_metrics() ->
-    lists:foreach(fun({Metric, FunSpec, DataPoints}) ->
-                          FunSpecTuple = list_to_tuple(FunSpec ++ [DataPoints]),
-                          catch ensure_metric(global, Metric, FunSpecTuple)
-                  end, ?VM_STATS).
-
 ensure_metric(HostType, Metric, Type, ShortType) when is_atom(Metric) ->
     ensure_metric(HostType, [Metric], Type, ShortType);
 
@@ -266,9 +258,7 @@ do_create_metric(PrefixedMetric, ExometerType, ExometerOpts) ->
 
 create_data_metrics() ->
     lists:foreach(fun(Metric) -> ensure_metric(global, Metric, spiral) end,
-        ?GLOBAL_SPIRALS),
-    lists:foreach(fun({Metric, Spec}) -> ensure_metric(global, Metric, Spec) end,
-        ?DATA_FUN_METRICS).
+        ?GLOBAL_SPIRALS).
 
 start_metrics_subscriptions(Reporter, MetricPrefix, Interval) ->
     [subscribe_metric(Reporter, Metric, Interval)

src/metrics/mongoose_metrics_definitions.hrl

@@ -13,32 +13,13 @@
            [{callback_module, Module},
             {sample_interval, ?REPORT_INTERVAL}]}}).
 
--define(GLOBAL_COUNTERS,
-        [{nodeUpTime,
-          {function, mongoose_metrics, get_up_time, [],
-           tagged, [value]}},
-         ?PROBE(tcpPortsUsed, mongoose_metrics_probe_tcp),
-         ?PROBE(processQueueLengths, mongoose_metrics_probe_queues)
-        ]
-).
-
 -define(MNESIA_COUNTERS,
         [{clusterSize,
           {function, mongoose_metrics, get_mnesia_running_db_nodes_count, [],
            tagged, [value]}}
         ]
 ).
 
--define(VM_STATS, [{[erlang, system_info], [function, erlang, system_info, ['$dp'], value],
-                    [port_count, port_limit, process_count, process_limit, ets_limit]},
-                   {[erlang, memory], [function, erlang, memory, ['$dp'], value],
-                    [total, processes_used, atom_used, binary, ets, system]}]).
-
--define(DATA_FUN_METRICS,
-        [{[data, dist],
-          {function, mongoose_metrics, get_dist_data_stats, [], proplist, [connections | ?INET_STATS]}}]).
-
-
 -define(INET_STATS, [recv_oct,
                      recv_cnt,
                      recv_max,

test/mongooseim_metrics_SUITE.erl

@@ -1,9 +1,7 @@
 -module(mongooseim_metrics_SUITE).
 
--include_lib("exml/include/exml.hrl").
 -include_lib("proper/include/proper.hrl").
 -include_lib("eunit/include/eunit.hrl").
--include("jlib.hrl").
 -include_lib("common_test/include/ct.hrl").
 
 -compile([export_all, nowarn_export_all]).