@@ -49,13 +49,11 @@ def perform_job(job_model: JobModel, connection: ConnectionType) -> Any: # noqa
49
49
loop = asyncio .new_event_loop ()
50
50
coro_result = loop .run_until_complete (result )
51
51
result = coro_result
52
- if job_model .success_callback :
53
- job_model .success_callback (job_model , connection , result )
52
+ job_model .call_success_callback (job_model , connection , result )
54
53
return result
55
54
except Exception as e :
56
55
logger .error (f"Job { job_model .name } failed with exception: { e } " , exc_info = True )
57
- if job_model .failure_callback :
58
- job_model .failure_callback (job_model , connection , * sys .exc_info ())
56
+ job_model .call_failure_callback (job_model , connection , * sys .exc_info ())
59
57
raise
60
58
finally :
61
59
assert job_model is _job_stack .pop ()
@@ -121,43 +119,27 @@ def clean_registries(self, timestamp: Optional[float] = None) -> None:
121
119
self .connection , before_score
122
120
)
123
121
124
- with self .connection .pipeline () as pipeline :
125
- for job_name , job_score in started_jobs :
126
- job = JobModel .get (job_name , connection = self .connection )
127
- if job is None or job .failure_callback is None or job_score + job .timeout > before_score :
128
- continue
122
+ for job_name , job_score in started_jobs :
123
+ job = JobModel .get (job_name , connection = self .connection )
124
+ if job is None or not job .has_failure_callback or job_score + job .timeout > before_score :
125
+ continue
129
126
130
- logger .debug (f"Running failure callbacks for { job .name } " )
131
- try :
132
- job .failure_callback (job , self .connection , traceback .extract_stack ())
133
- except Exception : # noqa
134
- logger .exception (f"Job { self .name } : error while executing failure callback" )
135
- raise
127
+ logger .debug (f"Running failure callbacks for { job .name } " )
128
+ try :
129
+ job .call_failure_callback (job , self .connection , traceback .extract_stack ())
130
+ except Exception : # noqa
131
+ logger .exception (f"Job { self .name } : error while executing failure callback" )
132
+ raise
136
133
137
- else :
138
- logger .warning (
139
- f"Queue cleanup: Moving job to { self .failed_job_registry .key } (due to AbandonedJobError)"
140
- )
141
- exc_string = (
142
- f"Moved to { self .failed_job_registry .key } , due to AbandonedJobError, at { datetime .now ()} "
143
- )
144
- job .status = JobStatus .FAILED
145
- score = current_timestamp () + SCHEDULER_CONFIG .DEFAULT_FAILURE_TTL
146
- Result .create (
147
- connection = pipeline ,
148
- job_name = job .name ,
149
- worker_name = job .worker_name ,
150
- _type = ResultType .FAILED ,
151
- ttl = SCHEDULER_CONFIG .DEFAULT_FAILURE_TTL ,
152
- exc_string = exc_string ,
153
- )
154
- self .failed_job_registry .add (pipeline , job .name , score )
155
- job .expire (connection = pipeline , ttl = SCHEDULER_CONFIG .DEFAULT_FAILURE_TTL )
156
- job .save (connection = pipeline )
134
+ else :
135
+ logger .warning (
136
+ f"Queue cleanup: Moving job to { self .failed_job_registry .key } (due to AbandonedJobError)"
137
+ )
138
+ exc_string = f"Moved to { self .failed_job_registry .key } , due to AbandonedJobError, at { datetime .now ()} "
139
+ self .job_handle_failure (JobStatus .FAILED , job , exc_string )
157
140
158
141
for registry in self .REGISTRIES .values ():
159
142
getattr (self , registry ).cleanup (connection = self .connection , timestamp = before_score )
160
- pipeline .execute ()
161
143
162
144
def first_queued_job_name (self ) -> Optional [str ]:
163
145
return self .queued_job_registry .get_first ()
@@ -258,37 +240,35 @@ def create_and_enqueue_job(
258
240
raise TypeError (f"Invalid type for when=`{ when } `" )
259
241
return job_model
260
242
261
- def job_handle_success (
262
- self , job : JobModel , result : Any , job_info_ttl : int , result_ttl : int , connection : ConnectionType
263
- ) -> None :
243
+ def job_handle_success (self , job : JobModel , result : Any , job_info_ttl : int , result_ttl : int ) -> None :
264
244
"""Saves and cleanup job after successful execution"""
265
245
job .after_execution (
266
246
job_info_ttl ,
267
247
JobStatus .FINISHED ,
268
248
prev_registry = self .active_job_registry ,
269
249
new_registry = self .finished_job_registry ,
270
- connection = connection ,
250
+ connection = self . connection ,
271
251
)
272
252
Result .create (
273
- connection ,
253
+ self . connection ,
274
254
job_name = job .name ,
275
255
worker_name = job .worker_name ,
276
256
_type = ResultType .SUCCESSFUL ,
277
257
return_value = result ,
278
258
ttl = result_ttl ,
279
259
)
280
260
281
- def job_handle_failure (self , status : JobStatus , job : JobModel , exc_string : str , connection : ConnectionType ) -> None :
261
+ def job_handle_failure (self , status : JobStatus , job : JobModel , exc_string : str ) -> None :
282
262
# Does not set job status since the job might be stopped
283
263
job .after_execution (
284
264
SCHEDULER_CONFIG .DEFAULT_FAILURE_TTL ,
285
265
status ,
286
266
prev_registry = self .active_job_registry ,
287
267
new_registry = self .failed_job_registry ,
288
- connection = connection ,
268
+ connection = self . connection ,
289
269
)
290
270
Result .create (
291
- connection ,
271
+ self . connection ,
292
272
job .name ,
293
273
job .worker_name ,
294
274
ResultType .FAILED ,
@@ -301,19 +281,11 @@ def run_sync(self, job: JobModel) -> JobModel:
301
281
job .prepare_for_execution ("sync" , self .active_job_registry , self .connection )
302
282
try :
303
283
result = perform_job (job , self .connection )
304
-
305
- with self .connection .pipeline () as pipeline :
306
- self .job_handle_success (
307
- job , result = result , job_info_ttl = job .job_info_ttl , result_ttl = job .success_ttl , connection = pipeline
308
- )
309
-
310
- pipeline .execute ()
284
+ self .job_handle_success (job , result = result , job_info_ttl = job .job_info_ttl , result_ttl = job .success_ttl )
311
285
except Exception as e : # noqa
312
286
logger .warning (f"Job { job .name } failed with exception: { e } " )
313
- with self .connection .pipeline () as pipeline :
314
- exc_string = "" .join (traceback .format_exception (* sys .exc_info ()))
315
- self .job_handle_failure (JobStatus .FAILED , job , exc_string , pipeline )
316
- pipeline .execute ()
287
+ exc_string = "" .join (traceback .format_exception (* sys .exc_info ()))
288
+ self .job_handle_failure (JobStatus .FAILED , job , exc_string )
317
289
return job
318
290
319
291
@classmethod
0 commit comments