Merge pull request #1821 from rabbitmq/lrb-update-sysmon_handler

Update to sysmon_handler 1.1.0

Author: lukebakken

priv/schema/rabbit.schema

@@ -1363,95 +1363,152 @@ end}.
 %% that are overly busy. Processes with large heaps or that take a
 %% long time to garbage collect will count toward this threshold.
 {mapping, "sysmon_handler.thresholds.busy_processes", "sysmon_handler.process_limit", [
-  {default, 30},
-  {datatype, integer},
-  hidden
+    {datatype, integer},
+    hidden
 ]}.
 
+{translation, "sysmon_handler.process_limit",
+    fun(Conf) ->
+            case cuttlefish:conf_get("sysmon_handler.thresholds.busy_processes", Conf, undefined) of
+                undefined ->
+                    cuttlefish:unset();
+                Int when is_integer(Int) ->
+                    Int;
+                _ ->
+                    cuttlefish:invalid("should be a non-negative integer")
+            end
+    end
+}.
+
 %% @doc The threshold at which to warn about the number of ports that
 %% are overly busy. Ports with full input buffers count toward this
 %% threshold.
 {mapping, "sysmon_handler.thresholds.busy_ports", "sysmon_handler.port_limit", [
-  {default, 2},
   {datatype, integer},
   hidden
 ]}.
 
+{translation, "sysmon_handler.port_limit",
+    fun(Conf) ->
+            case cuttlefish:conf_get("sysmon_handler.thresholds.busy_ports", Conf, undefined) of
+                undefined ->
+                    cuttlefish:unset();
+                Int when is_integer(Int) ->
+                    Int;
+                _ ->
+                    cuttlefish:invalid("should be a non-negative integer")
+            end
+    end
+}.
+
 %% @doc A process will become busy when it exceeds this amount of time
 %% doing garbage collection.
-%%
-%% NOTE: Enabling this setting can cause performance problems on
-%% multi-core systems.
 %% @see sysmon_handler.thresholds.busy_processes
 {mapping, "sysmon_handler.triggers.process.garbage_collection", "sysmon_handler.gc_ms_limit", [
-  {default, off},
   {datatype, [{atom, off},
               {duration, ms}]},
   hidden
 ]}.
 
 {translation, "sysmon_handler.gc_ms_limit",
- fun(Conf) ->
-     case cuttlefish:conf_get("sysmon_handler.triggers.process.garbage_collection", Conf) of
-         off -> 0;
-         Int -> Int
-     end
- end}.
+    fun(Conf) ->
+            case cuttlefish:conf_get("sysmon_handler.triggers.process.garbage_collection", Conf, undefined) of
+                undefined ->
+                    cuttlefish:unset();
+                off ->
+                    0;
+                Int when is_integer(Int) ->
+                    Int;
+                _ ->
+                    cuttlefish:invalid("should be a non-negative integer")
+            end
+    end
+}.
 
 %% @doc A process will become busy when it exceeds this amount of time
 %% during a single process scheduling & execution cycle.
 {mapping, "sysmon_handler.triggers.process.long_scheduled_execution", "sysmon_handler.schedule_ms_limit", [
-  {default, off},
   {datatype, [{atom, off},
               {duration, ms}]},
   hidden
 ]}.
 
 {translation, "sysmon_handler.schedule_ms_limit",
- fun(Conf) ->
-     case cuttlefish:conf_get("sysmon_handler.triggers.process.long_scheduled_execution", Conf) of
-         off -> 0;
-         Int -> Int
-     end
- end}.
+    fun(Conf) ->
+            case cuttlefish:conf_get("sysmon_handler.triggers.process.long_scheduled_execution", Conf, undefined) of
+                undefined ->
+                    cuttlefish:unset();
+                off ->
+                    0;
+                Int when is_integer(Int) ->
+                    Int;
+                _ ->
+                    cuttlefish:invalid("should be a non-negative integer")
+            end
+    end
+}.
 
 %% @doc A process will become busy when its heap exceeds this size.
 %% @see sysmon_handler.thresholds.busy_processes
 {mapping, "sysmon_handler.triggers.process.heap_size", "sysmon_handler.heap_word_limit", [
-  {default, "160444000"},
-  {datatype, [bytesize, {atom, off}]},
+  {datatype, [{atom, off},
+              bytesize]},
   hidden
 ]}.
 
 {translation, "sysmon_handler.heap_word_limit",
- fun(Conf) ->
-    case cuttlefish:conf_get("sysmon_handler.triggers.process.heap_size", Conf) of
-        off -> 0;
-        Bytes ->
-            WordSize = erlang:system_info(wordsize),
-            Bytes div WordSize
+    fun(Conf) ->
+            case cuttlefish:conf_get("sysmon_handler.triggers.process.heap_size", Conf, undefined) of
+                undefined ->
+                    cuttlefish:unset();
+                off ->
+                    0;
+                Bytes when is_integer(Bytes) ->
+                    WordSize = erlang:system_info(wordsize),
+                    Bytes div WordSize;
+                _ ->
+                    cuttlefish:invalid("should be a non-negative integer")
+            end
     end
-end}.
+}.
 
 %% @doc Whether ports with full input buffers will be counted as
 %% busy. Ports can represent open files or network sockets.
 %% @see sysmon_handler.thresholds.busy_ports
 {mapping, "sysmon_handler.triggers.port", "sysmon_handler.busy_port", [
-  {default, on},
   {datatype, flag},
   hidden
 ]}.
 
+{translation, "sysmon_handler.busy_port",
+    fun(Conf) ->
+            case cuttlefish:conf_get("sysmon_handler.triggers.port", Conf, undefined) of
+                undefined ->
+                    cuttlefish:unset();
+                Val -> Val
+            end
+    end
+}.
+
 %% @doc Whether distribution ports with full input buffers will be
 %% counted as busy. Distribution ports connect Erlang nodes within a
 %% single cluster.
 %% @see sysmon_handler.thresholds.busy_ports
 {mapping, "sysmon_handler.triggers.distribution_port", "sysmon_handler.busy_dist_port", [
-  {default, on},
   {datatype, flag},
   hidden
 ]}.
 
+{translation, "sysmon_handler.busy_dist_port",
+    fun(Conf) ->
+            case cuttlefish:conf_get("sysmon_handler.triggers.distribution_port", Conf, undefined) of
+                undefined ->
+                    cuttlefish:unset();
+                Val -> Val
+            end
+    end
+}.
+
 % ===============================
 % Validators
 % ===============================

src/rabbit.erl

@@ -523,6 +523,7 @@ start_apps(Apps) ->
 
 start_apps(Apps, RestartTypes) ->
     app_utils:load_applications(Apps),
+    ensure_sysmon_handler_app_config(),
     ConfigEntryDecoder = case application:get_env(rabbit, config_entry_decoder) of
         undefined ->
             [];
@@ -553,7 +554,6 @@ start_apps(Apps, RestartTypes) ->
         PassPhrase
     },
     decrypt_config(Apps, Algo),
-
     OrderedApps = app_utils:app_dependency_order(Apps, false),
     case lists:member(rabbit, Apps) of
         false -> rabbit_boot_steps:run_boot_steps(Apps); %% plugin activation
@@ -563,6 +563,30 @@ start_apps(Apps, RestartTypes) ->
                                       handle_app_error(could_not_start),
                                       RestartTypes).
 
+%% rabbitmq/rabbitmq-server#952
+%% This function is to be called after configuration has been optionally generated
+%% and the sysmon_handler application loaded, but not started. It will ensure that
+%% sane defaults are used for configuration settings that haven't been set by the
+%% user
+ensure_sysmon_handler_app_config() ->
+    Defaults = [
+                {process_limit, 100},
+                {port_limit, 100},
+                {gc_ms_limit, 0},
+                {schedule_ms_limit, 0},
+                {heap_word_limit, 10485760},
+                {busy_port, false},
+                {busy_dist_port, true}
+               ],
+    lists:foreach(fun({K, V}) ->
+                          case application:get_env(sysmon_handler, K) of
+                              undefined ->
+                                  application:set_env(sysmon_handler, K, V);
+                              _ ->
+                                  ok
+                          end
+                  end, Defaults).
+
 %% This function retrieves the correct IoDevice for requesting
 %% input. The problem with using the default IoDevice is that
 %% the Erlang shell prevents us from getting the input.

src/rabbit_sysmon_handler.erl

@@ -91,6 +91,11 @@ handle_event({monitor, PidOrPort, Type, Info}, State=#state{timer_ref=TimerRef})
     {Fmt, Args} = format_pretty_proc_or_port_info(PidOrPort),
     rabbit_log:warning("~p ~w ~w " ++ Fmt ++ " ~w", [?MODULE, Type, PidOrPort] ++ Args ++ [Info]),
     {ok, State#state{timer_ref=NewTimerRef}};
+handle_event({suppressed, Type, Info}, State=#state{timer_ref=TimerRef}) ->
+    %% Reset the inactivity timeout
+    NewTimerRef = reset_timer(TimerRef),
+    rabbit_log:debug("~p encountered a suppressed event of type ~w: ~w", [?MODULE, Type, Info]),
+    {ok, State#state{timer_ref=NewTimerRef}};
 handle_event(Event, State=#state{timer_ref=TimerRef}) ->
     NewTimerRef = reset_timer(TimerRef),
     rabbit_log:warning("~p unhandled event: ~p", [?MODULE, Event]),