Merge pull request #183 from rabbitmq/replica-handle-nodedown

Replica: Handle nodedowns when starting replica reader.

Author: kjnilsson

src/osiris_replica.erl

@@ -531,7 +531,12 @@ handle_info({'EXIT', RRPid, Info},
                                 replica_reader_pid = RRPid}} = State) ->
     %% any replica reader exit is troublesome and requires the replica to also
     %% terminate
-    ?ERROR_(Name, "replica reader ~w exited with ~w", [RRPid, Info]),
+    case lists:member(Info, [normal, shutdown]) of
+        true ->
+            ?DEBUG_(Name, "replica reader ~w exited with ~w", [RRPid, Info]);
+        false ->
+            ?ERROR_(Name, "replica reader ~w exited with ~w", [RRPid, Info])
+    end,
     {stop, {shutdown, Info}, State};
 handle_info({'EXIT', Ref, normal},
             #?MODULE{cfg = #cfg{name = Name}} = State) ->

src/osiris_replica_reader.erl

@@ -67,19 +67,26 @@ stop(Pid) ->
     gen_server:cast(Pid, stop).
 
 start(Node, ReplicaReaderConf) when is_map(ReplicaReaderConf) ->
-    supervisor:start_child({osiris_replica_reader_sup, Node},
-                           #{id => make_ref(),
-                             start =>
+    try
+        supervisor:start_child({osiris_replica_reader_sup, Node},
+                               #{id => make_ref(),
+                                 start =>
                                  {osiris_replica_reader, start_link,
                                   [ReplicaReaderConf]},
-                             %% replica readers should never be
-                             %% restarted by their sups
-                             %% instead they need to be re-started
-                             %% by their replica
-                             restart => temporary,
-                             shutdown => 5000,
-                             type => worker,
-                             modules => [osiris_replica_reader]}).
+                                 %% replica readers should never be
+                                 %% restarted by their sups
+                                 %% instead they need to be re-started
+                                 %% by their replica
+                                 restart => temporary,
+                                 shutdown => 5000,
+                                 type => worker,
+                                 modules => [osiris_replica_reader]})
+    catch
+        exit:{{nodedown, _} = Res, _Stack} ->
+              {error, Res};
+        exit:{noproc = Res, _Stack} ->
+              {error, Res}
+    end.
 
 %%%===================================================================
 %%% gen_server callbacks

test/osiris_SUITE.erl

@@ -85,7 +85,8 @@ all_tests() ->
      single_node_reader_counters,
      cluster_reader_counters,
      combine_ips_hosts_test,
-     empty_last_segment].
+     empty_last_segment,
+     replica_reader_nodedown_noproc].
 
 %% Isolated to avoid test interference
 ipv6_tests() ->
@@ -1895,6 +1896,16 @@ empty_last_segment(Config) ->
     ?assert(erlang:is_process_alive(Leader2)),
     ok.
 
+replica_reader_nodedown_noproc(_Config) ->
+    %% unit test to ensure we handle down nodes gracefully.
+    {error, {nodedown, 'banana@fruit'}} =
+        osiris_replica_reader:start('banana@fruit', #{}),
+
+    _ = application:stop(osiris),
+    {error, noproc} =
+        osiris_replica_reader:start(node(), #{}),
+    ok.
+
 %% Utility
 
 write_n(Pid, N, Written) ->