Replica: Handle nodedowns and noproc when starting replica reader.

It is possible that the writer's node could go down in between
querying the writer for it's log overview and then starting
the replica reader. This commit handles that potentially common
scenarion more gracefully.

Author: kjnilsson

src/osiris_replica_reader.erl

@@ -67,19 +67,26 @@ stop(Pid) ->
     gen_server:cast(Pid, stop).
 
 start(Node, ReplicaReaderConf) when is_map(ReplicaReaderConf) ->
-    supervisor:start_child({osiris_replica_reader_sup, Node},
-                           #{id => make_ref(),
-                             start =>
+    try
+        supervisor:start_child({osiris_replica_reader_sup, Node},
+                               #{id => make_ref(),
+                                 start =>
                                  {osiris_replica_reader, start_link,
                                   [ReplicaReaderConf]},
-                             %% replica readers should never be
-                             %% restarted by their sups
-                             %% instead they need to be re-started
-                             %% by their replica
-                             restart => temporary,
-                             shutdown => 5000,
-                             type => worker,
-                             modules => [osiris_replica_reader]}).
+                                 %% replica readers should never be
+                                 %% restarted by their sups
+                                 %% instead they need to be re-started
+                                 %% by their replica
+                                 restart => temporary,
+                                 shutdown => 5000,
+                                 type => worker,
+                                 modules => [osiris_replica_reader]})
+    catch
+        exit:{{nodedown, _} = Res, _Stack} ->
+              {error, Res};
+        exit:{noproc = Res, _Stack} ->
+              {error, Res}
+    end.
 
 %%%===================================================================
 %%% gen_server callbacks

test/osiris_SUITE.erl

@@ -85,7 +85,8 @@ all_tests() ->
      single_node_reader_counters,
      cluster_reader_counters,
      combine_ips_hosts_test,
-     empty_last_segment].
+     empty_last_segment,
+     replica_reader_nodedown_noproc].
 
 %% Isolated to avoid test interference
 ipv6_tests() ->
@@ -1895,6 +1896,16 @@ empty_last_segment(Config) ->
     ?assert(erlang:is_process_alive(Leader2)),
     ok.
 
+replica_reader_nodedown_noproc(_Config) ->
+    %% unit test to ensure we handle down nodes gracefully.
+    {error, {nodedown, 'banana@fruit'}} =
+        osiris_replica_reader:start('banana@fruit', #{}),
+
+    _ = application:stop(osiris),
+    {error, noproc} =
+        osiris_replica_reader:start(node(), #{}),
+    ok.
+
 %% Utility
 
 write_n(Pid, N, Written) ->