fix ct

Author: Dave Parfitt

src/oc_erchef/apps/oc_chef_wm/itest/chef_wm_actions_queue_monitoring_SUITE.erl

@@ -34,43 +34,58 @@
 -define(CLIENT_AUTHZ_ID, <<"00000000000000000000000000000003">>).
 -define(ACTIONS_EXCHANGE, <<"actions">>).
 
-
-
+-define(Q_SETTING(K, V),
+        chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(K,V)).
 
 init_per_suite(InitialConfig) ->
-    [
-              {test_vhost, "/testing_vhost"},
-              {test_password, "chef123"},
-              {max_length, 10}]
-      ++ InitialConfig.
+    UseFakeRabbit =
+         case os:getenv("TRAVIS") of
+             false -> false;
+             _ -> true
+         end,
+    [{test_vhost, "/testing_vhost"},
+     {test_password, "chef123"},
+     {max_length, 10},
+     {use_fake_rabbit, UseFakeRabbit}]
+        ++ InitialConfig.
 
 end_per_suite(_Config) ->
     ok.
 
 
 init_per_testcase(_,  Config) ->
     % ensure rabbit starts from a clean slate
-    teardown_rabbit(Config),
+    case ?config(use_fake_rabbit, Config) of
+        false -> teardown_rabbit(Config);
+        true -> ok
+    end,
+
     Config.
 
 end_per_testcase(_, Config) ->
+    case ?config(use_fake_rabbit, Config) of
+        true ->
+            catch(exit(whereis(fake_rabbit), kill)),
+            meck:unload(bunnyc),
+                meck:unload(chef_wm_rabbitmq_management);
+        false -> ok
+    end,
+
     % leave rabbit up for troubleshooting failed tests
-    %teardown_rabbit(Config),
     setup_helper:base_end_per_suite(Config),
     Config.
 
+all() -> [basic_queue_monitor, queue_full_dont_start].
+
+
+
 
-all() ->
-    % don't run tests in Travis yet
-    case os:getenv("TRAVIS") of
-        false -> [basic_queue_monitor,
-                  queue_full_dont_start];
-        true -> []
-    end.
 
 basic_queue_monitor(Config) ->
     default_env_with_queue_monitor(Config),
+
     setup_rabbit(Config),
+
     _FinalConfig = setup_chef(Config),
     lager:start(),
     lager:set_loglevel(lager_console_backend, debug),
@@ -139,7 +154,6 @@ basic_queue_monitor(Config) ->
     get_alaska_msg(Config),
     get_alaska_msg(Config),
 
-
     wait_until_not_at_capacity(),
     Status6 = chef_wm_actions_queue_monitoring:status(),
     ?assert(proplists:get_value(check_count, Status6) > 0),
@@ -152,8 +166,20 @@ basic_queue_monitor(Config) ->
 
 queue_full_dont_start(Config) ->
     default_env_with_queue_monitor(Config),
+    MaxLength = ?config(max_length, Config),
+
     setup_rabbit(Config),
 
+    %% one more meck for this test
+    case ?config(use_fake_rabbit, Config) of
+        true ->
+            meck:expect(chef_wm_rabbitmq_management, sync_check_queue_at_capacity,
+                        fun(_Vhost, _Queue) ->
+                                {MaxLength, MaxLength, true}
+                        end);
+        false -> ok
+    end,
+
     ibrowse:start(),
     [ begin
           ct:pal("Publishing msg ~p to queue~n", [X]),
@@ -168,7 +194,8 @@ queue_full_dont_start(Config) ->
     catch Error:Reason ->
         ct:pal("Erchef failed to start (this is expected): ~p~n~p~n", [Error, Reason])
     end,
-    ok.
+
+ok.
 
 wait_until_not_at_capacity() ->
     wait_until(fun () ->
@@ -233,22 +260,22 @@ default_rabbit_config(Config) ->
             {queue_length_monitor_queue, alaska },
             {queue_length_monitor_millis, 30000 },
             {queue_length_monitor_timeout_millis, 5000 },
-            {drop_on_full_capacity, true }
+            {drop_on_full_capacity, true },
+            {prevent_startup_on_full_capacity, true}
         ]}
     ].
 
 
-%-define(Q_SETTING(K, V),
-%        chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(K,V)).
 
 default_env_with_queue_monitor(Config) ->
     application:set_env(oc_chef_wm, rabbitmq, default_rabbit_config(Config)),
 
-    chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(queue_length_monitor_enabled, true),
-    chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(queue_length_monitor_vhost, ?config(test_vhost, Config)),
-    chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(queue_length_monitor_queue, "alaska"),
-    chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(queue_length_monitor_millis, 2000),
-    chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(drop_on_full_capacity, true),
+    ?Q_SETTING(queue_length_monitor_enabled, true),
+    ?Q_SETTING(queue_length_monitor_vhost, ?config(test_vhost, Config)),
+    ?Q_SETTING(queue_length_monitor_queue, "alaska"),
+    ?Q_SETTING(queue_length_monitor_millis, 2000),
+    ?Q_SETTING(drop_on_full_capacity, true),
+    ?Q_SETTING(prevent_startup_on_full_capacity, true),
 
     application:set_env(oc_chef_wm, enable_actions, true),
     application:set_env(oc_chef_wm, actions_host, "127.0.0.1"),
@@ -279,6 +306,56 @@ setup_chef(InitialConfig) ->
 
 
 setup_rabbit(Config) ->
+    UseFakeRabbit = ?config(use_fake_rabbit, Config),
+    case UseFakeRabbit of
+        true -> setup_rabbit_fake(Config);
+        false -> setup_rabbit_real(Config)
+    end.
+
+teardown_rabbit(Config) ->
+    UseFakeRabbit = ?config(use_fake_rabbit, Config),
+    case UseFakeRabbit of
+      true -> ok;
+      false ->
+        Vhost = ?config(test_vhost, Config),
+        rabbitmqctl(["delete_vhost", to_str("\"~p\"", [Vhost])])
+    end.
+
+
+setup_rabbit_fake(Config) ->
+   start_fake_rabbit(?config(max_length, Config)),
+
+    meck:new(bunnyc),
+    meck:expect(bunnyc, start_link, fun(_, _, _, _) ->
+                                               {ok, self()}
+                                       end),
+
+    meck:expect(bunnyc, publish,
+                fun(_Server, _RoutingKey, _PersistentData) ->
+                    %io:format(user, "CAUGHT BUNNYC:PUBLISH ~p ~p ~p~n",
+                    %          [Server, RoutingKey, PersistentData]),
+                    fake_rabbit_enqueue(),
+                    ok
+                end),
+
+
+    meck:new(chef_wm_rabbitmq_management, [passthrough]),
+
+
+    meck:expect(chef_wm_rabbitmq_management, get_max_length,
+                fun(_Vhost) ->
+                        ?config(max_length, Config)
+                end),
+
+    meck:expect(chef_wm_rabbitmq_management, get_current_length,
+                fun(_Vhost, _Queue) ->
+                        fake_rabbit_current_length()
+                end),
+    ok.
+
+
+
+setup_rabbit_real(Config) ->
     ibrowse:start(),
     Vhost = ?config(test_vhost, Config),
     Password = ?config(test_password, Config),
@@ -295,7 +372,10 @@ setup_rabbit(Config) ->
     all_permissions("/", "actions"),
     all_permissions(Vhost, "actions"),
 
+    %% This is a lie, as rabbitmq needs to be restarted after the
+    %% plugin is installed
     cmd(["rabbitmq-plugins", "enable", "rabbitmq_management"]),
+
     rabbitmqctl(["set_user_tags", "rabbitmgmt", "administrator"]),
 
     MaxLength = ?config(max_length, Config),
@@ -321,28 +401,39 @@ setup_rabbit(Config) ->
 
 
 put_alaska_msg(Config) ->
-    Password = ?config(test_password, Config),
-    User = "rabbitmgmt",
-    Vhost = ?config(test_vhost, Config),
-    Path = escape_to_str("/api/exchanges/~s/actions/publish", Vhost),
-    Body = "{\"properties\":{},\"routing_key\":\"my key\",\"payload\":\"my body\",\"payload_encoding\":\"string\"}",
-    rabbitmq_mgmt(Path, post, Body, User, Password).
+    UseFakeRabbit = ?config(use_fake_rabbit, Config),
+    case UseFakeRabbit of
+        true ->
+            fake_rabbit_enqueue();
+        false ->
+            Password = ?config(test_password, Config),
+            User = "rabbitmgmt",
+            Vhost = ?config(test_vhost, Config),
+            Path = escape_to_str("/api/exchanges/~s/actions/publish", Vhost),
+            Body = "{\"properties\":{},\"routing_key\":\"my key\",\"payload\":\"my body\",\"payload_encoding\":\"string\"}",
+            rabbitmq_mgmt(Path, post, Body, User, Password)
+    end.
 
 
 get_alaska_msg(Config) ->
     Password = ?config(test_password, Config),
-    User = "rabbitmgmt",
-    Vhost = ?config(test_vhost, Config),
-    Path = escape_to_str("/api/queues/~s/alaska/get", Vhost),
-    Body = "{\"count\":1,\"requeue\":false,\"encoding\":\"auto\",\"truncate\":50000}",
-    Url = "http://localhost:15672" ++ Path,
-    % this request must use accept */*
-    ibrowse:send_req(Url,
-                     [{"accept", "*/*"},
-                      {"content-type", "application/json"}],
-                     post,
-                     Body,
-                     [{basic_auth, {User, Password}}]).
+    UseFakeRabbit = ?config(use_fake_rabbit, Config),
+    case UseFakeRabbit of
+        true -> fake_rabbit_dequeue();
+        false ->
+            User = "rabbitmgmt",
+            Vhost = ?config(test_vhost, Config),
+            Path = escape_to_str("/api/queues/~s/alaska/get", Vhost),
+            Body = "{\"count\":1,\"requeue\":false,\"encoding\":\"auto\",\"truncate\":50000}",
+            Url = "http://localhost:15672" ++ Path,
+            % this request must use accept */*
+            ibrowse:send_req(Url,
+                            [{"accept", "*/*"},
+                            {"content-type", "application/json"}],
+                            post,
+                            Body,
+                            [{basic_auth, {User, Password}}])
+    end.
 
 % make an http call to the rabbitmq mgmt console
 rabbitmq_mgmt(Path, Method, Body, User, Password) ->
@@ -354,9 +445,7 @@ rabbitmq_mgmt(Path, Method, Body, User, Password) ->
                      Body,
                      [{basic_auth, {User, Password}}]).
 
-teardown_rabbit(Config) ->
-    Vhost = ?config(test_vhost, Config),
-    rabbitmqctl(["delete_vhost", to_str("\"~p\"", [Vhost])]).
+
 
 cmd(List) ->
     Command = space_join(List),
@@ -384,7 +473,6 @@ space_join(L) ->
 
 
 make_data_bag(Requestor, Id) ->
-    %io:format(user, "Create data bag foobar~p~n", [Id]),
     DataBag =  to_str("{\"name\":\"foobar~p\"}",[Id]),
     Url = "http://localhost:8000/organizations/testorg/data",
     ibrowse:send_req(Url, [{"x-ops-userid", binary_to_list(Requestor)},
@@ -414,7 +502,6 @@ context() ->
     chef_db:make_context(?API_MIN_VER, <<"AB">>, no_header).
 
 
-%% TODO: move to chef_test lib
 rabbitmqctl(Subcmds) ->
     cmd(["rabbitmqctl"] ++ Subcmds).
 
@@ -429,3 +516,35 @@ to_str(P, Q) ->
 escape_to_str(Str, Val) ->
   lists:flatten(io_lib:format(Str, [http_uri:encode(Val)])).
 
+
+
+% spawn a process that keeps a count of enqueue/dequeues
+fake_rabbit(Max, Count) ->
+    receive
+        inc -> case Count == Max of
+                   true -> fake_rabbit(Max, Count);
+                   false -> fake_rabbit(Max, Count + 1)
+               end;
+        dec -> fake_rabbit(Max, Count -1);
+        {current_count, Pid} -> Pid ! {current_count, Count},
+                                fake_rabbit(Max, Count);
+        die -> ok
+    end.
+
+
+start_fake_rabbit(MaxLength) ->
+    Pid = spawn(?MODULE, fake_rabbit, [MaxLength, 0]),
+    register(fake_rabbit, Pid).
+
+fake_rabbit_enqueue() ->
+    whereis(fake_rabbit) ! inc.
+
+fake_rabbit_dequeue() ->
+    whereis(fake_rabbit) ! dec.
+
+fake_rabbit_current_length() ->
+    whereis(fake_rabbit) ! {current_count, self()},
+    receive
+        {current_count, N} -> N
+    end.
+

src/oc_erchef/apps/oc_chef_wm/src/chef_wm_rabbitmq_management.erl

@@ -97,6 +97,7 @@ set_rabbit_management_setting(Key, NewVal) ->
 set_rabbit_queue_monitor_setting(Key, NewVal) ->
     set_app_value(oc_chef_wm, rabbitmq, monitoring, Key, NewVal).
 
+%% value MUST already exist in the dict
 set_app_value(App, ConfigKey, SubSectionKey, Prop, NewValue) ->
     Cfg = envy:get(App, ConfigKey, [], any),
     NewCfg =
@@ -147,7 +148,7 @@ get_max_length(Vhost) ->
         {ok, "200", _, MaxLengthJson} ->
             parse_max_length_response(MaxLengthJson);
         {error, {conn_failed,_}} ->
-            lager:info("Can't connect to RabbitMQ management console"),
+            lager:info("Can't connect to RabbitMQ management console to fetch max-length"),
             undefined;
         {ok, "404", _, _} ->
             lager:info("RabbitMQ max-length policy not set"),
@@ -165,7 +166,7 @@ get_current_length(Vhost, Queue) ->
     CurrentResult = rabbit_mgmt_server_request(mk_current_length_path(Vhost)),
     case CurrentResult of
         {error, {conn_failed,_}} ->
-            lager:info("Can't connect to RabbitMQ management console"),
+            lager:info("Can't connect to RabbitMQ management console to fetch current length"),
             undefined;
         {ok, "200", _, CurrentStatusJson} ->
             parse_current_length_response(CurrentStatusJson, Queue);
@@ -252,7 +253,9 @@ check_current_queue_state(Vhost, Queue, DroppedSinceLastCheck) ->
                                                 {max_length(), reset_dropped_since_last_check} |
                                                 {max_length(), current_length(), queue_at_capacity()}.
 check_current_queue_length(Vhost, Queue, MaxLength, DroppedSinceLastCheck) ->
-    CurrentLength = get_current_length(Vhost, Queue),
+    % use ?MODULE here so I can use meck in integrations testing
+    % https://github.com/eproxus/meck/issues/142
+    CurrentLength = ?MODULE:get_current_length(Vhost, Queue),
     case CurrentLength of
         undefined ->
             % a queue doesn't appear to be bound to the