23
23
-define (CONSUMER_TAG_PID (Tag , Pid ),
24
24
# consumer {cfg = # consumer_cfg {tag = Tag ,
25
25
pid = Pid }}).
26
+ -define (NON_EMPTY_MAP (M ), M when map_size (M ) > 0 ).
27
+ -define (EMPTY_MAP , M when map_size (M ) == 0 ).
26
28
27
29
-export ([
28
30
% % ra_machine callbacks
76
78
make_purge /0 ,
77
79
make_purge_nodes /1 ,
78
80
make_update_config /1 ,
81
+ make_eval_consumer_timeouts /1 ,
79
82
make_garbage_collection /0
80
83
]).
81
84
119
122
-record (purge_nodes , {nodes :: [node ()]}).
120
123
-record (update_config , {config :: config ()}).
121
124
-record (garbage_collection , {}).
122
- % -record(eval_consumer_timeouts, {consumer_keys :: [consumer_key()]}).
125
+ -record (eval_consumer_timeouts , {consumer_keys :: [consumer_key ()]}).
123
126
124
127
-opaque protocol () ::
125
128
# enqueue {} |
@@ -238,9 +241,31 @@ apply(Meta, #settle{msg_ids = MsgIds,
238
241
% % find_consumer/2 returns the actual consumer key even if
239
242
% % if id was passed instead for example
240
243
complete_and_checkout (Meta , MsgIds , ConsumerKey ,
241
- Con0 , [], State );
244
+ reactivate_timed_out (Con0 ),
245
+ [], State );
242
246
_ ->
243
- {State , ok }
247
+ {State , {error , invalid_consumer_key }}
248
+ end ;
249
+ apply (#{index := Idx ,
250
+ system_time := Ts } = Meta , # defer {msg_ids = MsgIds ,
251
+ consumer_key = Key },
252
+ #? STATE {consumers = Consumers } = State0 ) ->
253
+ case find_consumer (Key , Consumers ) of
254
+ {ConsumerKey , # consumer {checked_out = Checked0 } = Con0 } ->
255
+ Checked = maps :map (fun (MsgId , ? C_MSG (_At , Msg ) = Orig ) ->
256
+ case lists :member (MsgId , MsgIds ) of
257
+ true ->
258
+ ? C_MSG (Ts , Msg );
259
+ false ->
260
+ Orig
261
+ end
262
+ end , Checked0 ),
263
+ Con = reactivate_timed_out (Con0 # consumer {checked_out = Checked }),
264
+ State1 = State0 #? STATE {consumers = Consumers #{ConsumerKey => Con }},
265
+ {State , Ret , Effs } = checkout (Meta , State0 , State1 , []),
266
+ update_smallest_raft_index (Idx , Ret , State , Effs );
267
+ _ ->
268
+ {State0 , {error , invalid_consumer_key }}
244
269
end ;
245
270
apply (Meta , # discard {consumer_key = ConsumerKey ,
246
271
msg_ids = MsgIds },
@@ -263,19 +288,25 @@ apply(Meta, #discard{consumer_key = ConsumerKey,
263
288
{DlxState , Effects } = rabbit_fifo_dlx :discard (DiscardMsgs , rejected ,
264
289
DLH , DlxState0 ),
265
290
State = State0 #? STATE {dlx = DlxState },
266
- complete_and_checkout (Meta , MsgIds , ConsumerKey , Con , Effects , State );
291
+ complete_and_checkout (Meta , MsgIds , ConsumerKey ,
292
+ reactivate_timed_out (Con ),
293
+ Effects , State );
267
294
_ ->
268
- {State0 , ok }
295
+ {State0 , { error , invalid_consumer_key } }
269
296
end ;
270
297
apply (Meta , # return {consumer_key = ConsumerKey ,
271
298
msg_ids = MsgIds },
272
- #? STATE {consumers = Cons0 } = State ) ->
299
+ #? STATE {consumers = Cons0 } = State0 ) ->
273
300
case find_consumer (ConsumerKey , Cons0 ) of
274
- {ActualConsumerKey , # consumer {checked_out = Checked0 }} ->
301
+ {ActualConsumerKey , # consumer {checked_out = Checked0 } = Con0 } ->
302
+
303
+ State = State0 #? MODULE {consumers =
304
+ Cons0 #{ActualConsumerKey =>
305
+ reactivate_timed_out (Con0 )}},
275
306
Returned = maps :with (MsgIds , Checked0 ),
276
307
return (Meta , ActualConsumerKey , Returned , [], State );
277
308
_ ->
278
- {State , ok }
309
+ {State0 , { error , invalid_consumer_key } }
279
310
end ;
280
311
apply (#{index := Idx } = Meta ,
281
312
# requeue {consumer_key = ConsumerKey ,
@@ -731,12 +762,51 @@ apply(#{index := IncomingRaftIdx} = Meta, {dlx, _} = Cmd,
731
762
State1 = State0 #? STATE {dlx = DlxState },
732
763
{State , ok , Effects } = checkout (Meta , State0 , State1 , Effects0 ),
733
764
update_smallest_raft_index (IncomingRaftIdx , State , Effects );
765
+ apply (Meta , # eval_consumer_timeouts {consumer_keys = CKeys }, State ) ->
766
+ eval_consumer_timeouts (Meta , CKeys , State );
734
767
apply (_Meta , Cmd , State ) ->
735
768
% % handle unhandled commands gracefully
736
769
rabbit_log :debug (" rabbit_fifo: unhandled command ~W " , [Cmd , 10 ]),
737
770
{State , ok , []}.
738
771
739
- convert_v3_to_v4 (#{system_time := Ts }, # rabbit_fifo {consumers = Consumers0 } = StateV3 ) ->
772
+ eval_consumer_timeouts (#{system_time := Ts } = Meta , CKeys ,
773
+ #? STATE {cfg = # cfg {consumer_strategy = competing },
774
+ consumers = Consumers0 } = State0 ) ->
775
+ ToCheck = maps :with (CKeys , Consumers0 ),
776
+ {State , Effects } =
777
+ maps :fold (
778
+ fun (Ckey , # consumer {cfg = # consumer_cfg {},
779
+ status = up ,
780
+ checked_out = Ch } = C0 ,
781
+ {#? STATE {consumers = Cons } = S0 , E0 } = Acc ) ->
782
+ case maps :filter (fun (_MsgId , ? C_MSG (At , _ )) ->
783
+ (At + ? CONSUMER_LOCK_MS ) < Ts
784
+ end , Ch ) of
785
+ ? EMPTY_MAP ->
786
+ Acc ;
787
+ ? NON_EMPTY_MAP (ToReturn ) ->
788
+ % % there are timed out messages,
789
+ % % update consumer state to `timed_out'
790
+ % % TODO: only if current status us `up'
791
+ C1 = C0 # consumer {status = timed_out },
792
+ S1 = S0 #? STATE {consumers = Cons #{Ckey => C1 }},
793
+ {S , E1 } = maps :fold (
794
+ fun (MsgId , ? C_MSG (_At , Msg ), {S2 , E1 }) ->
795
+ return_one (Meta , MsgId , Msg ,
796
+ S2 , E1 , Ckey )
797
+ end , {S1 , E0 }, ToReturn ),
798
+ E = [{consumer_timeout , Ckey , maps :keys (ToReturn )} | E1 ],
799
+ C = maps :get (Ckey , S #? STATE .consumers ),
800
+ {update_or_remove_con (Meta , Ckey , C , S ), E }
801
+ end ;
802
+ (_Ckey , _Con , Acc ) ->
803
+ Acc
804
+ end , {State0 , []}, ToCheck ),
805
+
806
+ {State , ok , Effects }.
807
+
808
+ convert_v3_to_v4 (#{system_time := Ts },
809
+ # rabbit_fifo {consumers = Consumers0 } = StateV3 ) ->
740
810
Consumers = maps :map (
741
811
fun (_CKey , # consumer {checked_out = Ch0 } = C ) ->
742
812
Ch = maps :map (
@@ -965,18 +1035,27 @@ which_module(2) -> rabbit_fifo_v3;
965
1035
which_module (3 ) -> rabbit_fifo_v3 ;
966
1036
which_module (4 ) -> ? STATE .
967
1037
968
- - define (AUX , aux_v2 ).
1038
+ - define (AUX , aux_v3 ).
969
1039
970
1040
- record (aux_gc , {last_raft_idx = 0 :: ra :index ()}).
971
1041
- record (aux , {name :: atom (),
972
1042
capacity :: term (),
973
1043
gc = # aux_gc {} :: # aux_gc {}}).
1044
+ - record (aux_v2 , {name :: atom (),
1045
+ last_decorators_state :: term (),
1046
+ capacity :: term (),
1047
+ gc = # aux_gc {} :: # aux_gc {},
1048
+ tick_pid :: undefined | pid (),
1049
+ cache = #{} :: map ()}).
974
1050
- record (? AUX , {name :: atom (),
975
1051
last_decorators_state :: term (),
976
1052
capacity :: term (),
977
1053
gc = # aux_gc {} :: # aux_gc {},
978
1054
tick_pid :: undefined | pid (),
979
- cache = #{} :: map ()}).
1055
+ cache = #{} :: map (),
1056
+ last_consumer_timeout_check :: milliseconds (),
1057
+ reserved_1 ,
1058
+ reserved_2 }).
980
1059
981
1060
init_aux (Name ) when is_atom (Name ) ->
982
1061
% % TODO: catch specific exception throw if table already exists
@@ -985,15 +1064,36 @@ init_aux(Name) when is_atom(Name) ->
985
1064
{write_concurrency , true }]),
986
1065
Now = erlang :monotonic_time (micro_seconds ),
987
1066
#? AUX {name = Name ,
1067
+ last_consumer_timeout_check = erlang :system_time (millisecond ),
988
1068
capacity = {inactive , Now , 1 , 1.0 }}.
989
1069
990
1070
handle_aux (RaftState , Tag , Cmd , # aux {name = Name ,
991
1071
capacity = Cap ,
992
- gc = Gc }, RaAux ) ->
1072
+ gc = Gc
1073
+ }, RaAux ) ->
993
1074
% % convert aux state to new version
994
1075
Aux = #? AUX {name = Name ,
995
1076
capacity = Cap ,
996
- gc = Gc },
1077
+ gc = Gc ,
1078
+ last_consumer_timeout_check = erlang :system_time (millisecond )
1079
+ },
1080
+ handle_aux (RaftState , Tag , Cmd , Aux , RaAux );
1081
+ handle_aux (RaftState , Tag , Cmd , # aux_v2 {name = Name ,
1082
+ last_decorators_state = LDS ,
1083
+ capacity = Cap ,
1084
+ gc = Gc ,
1085
+ tick_pid = TickPid ,
1086
+ cache = Cache
1087
+ }, RaAux ) ->
1088
+ % % convert aux state to new version
1089
+ Aux = #? AUX {name = Name ,
1090
+ last_decorators_state = LDS ,
1091
+ capacity = Cap ,
1092
+ gc = Gc ,
1093
+ tick_pid = TickPid ,
1094
+ cache = Cache ,
1095
+ last_consumer_timeout_check = erlang :system_time (millisecond )
1096
+ },
997
1097
handle_aux (RaftState , Tag , Cmd , Aux , RaAux );
998
1098
handle_aux (_RaftState , cast , {# return {msg_ids = MsgIds ,
999
1099
consumer_key = Key } = Ret , Corr , Pid },
@@ -1028,7 +1128,8 @@ handle_aux(_RaftState, cast, {#return{msg_ids = MsgIds,
1028
1128
{no_reply , Aux0 , RaAux0 , [{append , Ret , {notify , Corr , Pid }}]}
1029
1129
end ;
1030
1130
handle_aux (leader , _ , {handle_tick , [QName , Overview0 , Nodes ]},
1031
- #? AUX {tick_pid = Pid } = Aux , RaAux ) ->
1131
+ #? AUX {tick_pid = Pid ,
1132
+ last_consumer_timeout_check = LastCheck } = Aux , RaAux ) ->
1032
1133
Overview = Overview0 #{members_info => ra_aux :members_info (RaAux )},
1033
1134
NewPid =
1034
1135
case process_is_alive (Pid ) of
@@ -1040,8 +1141,17 @@ handle_aux(leader, _, {handle_tick, [QName, Overview0, Nodes]},
1040
1141
% % Active TICK pid, do nothing
1041
1142
Pid
1042
1143
end ,
1043
- % % TODO: check consumer timeouts
1044
- {no_reply , Aux #? AUX {tick_pid = NewPid }, RaAux };
1144
+ % % check consumer timeouts
1145
+ Now = erlang :system_time (millisecond ),
1146
+ case Now - LastCheck > 1000 of
1147
+ true ->
1148
+ % % check if there are any consumer checked out message that have
1149
+ % % timed out.
1150
+ {no_reply , Aux #? AUX {tick_pid = NewPid ,
1151
+ last_consumer_timeout_check = Now }, RaAux };
1152
+ false ->
1153
+ {no_reply , Aux #? AUX {tick_pid = NewPid }, RaAux }
1154
+ end ;
1045
1155
handle_aux (_ , _ , {get_checked_out , ConsumerKey , MsgIds }, Aux0 , RaAux0 ) ->
1046
1156
#? STATE {cfg = # cfg {},
1047
1157
consumers = Consumers } = ra_aux :machine_state (RaAux0 ),
@@ -1724,38 +1834,21 @@ maybe_enqueue(RaftIdx, Ts, From, MsgSeqNo, RawMsg, Effects0,
1724
1834
return (#{index := IncomingRaftIdx } = Meta ,
1725
1835
ConsumerKey , Returned , Effects0 , State0 ) ->
1726
1836
{State1 , Effects1 } = maps :fold (
1727
- fun (MsgId , { _At , Msg } , {S0 , E0 }) ->
1837
+ fun (MsgId , ? C_MSG ( _At , Msg ) , {S0 , E0 }) ->
1728
1838
return_one (Meta , MsgId , Msg ,
1729
1839
S0 , E0 , ConsumerKey )
1730
1840
end , {State0 , Effects0 }, Returned ),
1731
1841
State2 = case State1 #? STATE .consumers of
1732
1842
#{ConsumerKey := Con } ->
1733
- update_or_remove_con (Meta , ConsumerKey , Con , State1 );
1843
+ update_or_remove_con (Meta , ConsumerKey ,
1844
+ Con , State1 );
1734
1845
_ ->
1735
1846
State1
1736
1847
end ,
1737
1848
{State , ok , Effects } = checkout (Meta , State0 , State2 , Effects1 ),
1738
1849
update_smallest_raft_index (IncomingRaftIdx , State , Effects ).
1739
1850
1740
1851
% used to process messages that are finished
1741
- complete (Meta , ConsumerKey , [MsgId ],
1742
- # consumer {checked_out = Checked0 } = Con0 ,
1743
- #? STATE {ra_indexes = Indexes0 ,
1744
- msg_bytes_checkout = BytesCheckout ,
1745
- messages_total = Tot } = State0 ) ->
1746
- case maps :take (MsgId , Checked0 ) of
1747
- {? C_MSG (_ , Idx , Hdr ), Checked } ->
1748
- SettledSize = get_header (size , Hdr ),
1749
- Indexes = rabbit_fifo_index :delete (Idx , Indexes0 ),
1750
- Con = Con0 # consumer {checked_out = Checked ,
1751
- credit = increase_credit (Con0 , 1 )},
1752
- State1 = update_or_remove_con (Meta , ConsumerKey , Con , State0 ),
1753
- State1 #? STATE {ra_indexes = Indexes ,
1754
- msg_bytes_checkout = BytesCheckout - SettledSize ,
1755
- messages_total = Tot - 1 };
1756
- error ->
1757
- State0
1758
- end ;
1759
1852
complete (Meta , ConsumerKey , MsgIds ,
1760
1853
# consumer {checked_out = Checked0 } = Con0 ,
1761
1854
#? STATE {ra_indexes = Indexes0 ,
@@ -2507,6 +2600,10 @@ make_purge_nodes(Nodes) ->
2507
2600
make_update_config (Config ) ->
2508
2601
# update_config {config = Config }.
2509
2602
2603
+ - spec make_eval_consumer_timeouts ([consumer_key ()]) -> protocol ().
2604
+ make_eval_consumer_timeouts (Keys ) when is_list (Keys ) ->
2605
+ # eval_consumer_timeouts {consumer_keys = Keys }.
2606
+
2510
2607
add_bytes_drop (Header ,
2511
2608
#? STATE {msg_bytes_enqueue = Enqueue } = State ) ->
2512
2609
Size = get_header (size , Header ),
@@ -2791,3 +2888,8 @@ maps_search(Pred, {K, V, I}) ->
2791
2888
end ;
2792
2889
maps_search (Pred , Map ) when is_map (Map ) ->
2793
2890
maps_search (Pred , maps :next (maps :iterator (Map ))).
2891
+
2892
+ reactivate_timed_out (# consumer {status = timed_out } = C ) ->
2893
+ C # consumer {status = up };
2894
+ reactivate_timed_out (C ) ->
2895
+ C .
0 commit comments