I have a Chord. After the chord, we chain another task. If one of the tasks in that group of that chord raises an exception, I get a mess of an exception that looks like a bug in celery or django_celery_results.
I am using amqp as my task queue and django_celery_results as my results backend.
My Tasks:
@shared_task(bind=True)
def debug_task(self):
print(f'Request: debug_task')
@shared_task(bind=True)
def debug_A(self, i):
print(f'Doing A{i}')
@shared_task(bind=True)
def debug_broke(self):
print(f'OH NO')
raise Exception("Head Aspload")
@shared_task(bind=True)
def debug_finisher(self):
print(f'OK, We did it all')
def launch():
from celery import group
g = [debug_A.si(i) for i in range(5)]
g.append(debug_broke.si())
r = group(g) | debug_finisher.si() | debug_A.si(-1)
r.apply_async()
The results when I run this:
[2023-03-08 18:19:52,428: INFO/MainProcess] Task topclans.worker.debug_A[c652e39a-8cf5-4ac8-924d-3bce32c190f8] received
[2023-03-08 18:19:52,429: WARNING/ForkPoolWorker-1] Doing A0
[2023-03-08 18:19:52,434: INFO/MainProcess] Task topclans.worker.debug_A[c1b8ed9c-a9e0-4960-8b9b-1ecd5b2254a3] received
[2023-03-08 18:19:52,436: INFO/MainProcess] Task topclans.worker.debug_A[a4478640-a50a-42a2-a8f0-8e6b84abab90] received
[2023-03-08 18:19:52,439: INFO/MainProcess] Task topclans.worker.debug_A[d4c9d249-98dd-4071-ab03-70a350c7d171] received
[2023-03-08 18:19:52,439: INFO/MainProcess] Task topclans.worker.debug_A[41c4bdb0-8993-40b0-90bd-2d6c642cb518] received
[2023-03-08 18:19:52,462: INFO/ForkPoolWorker-1] Task topclans.worker.debug_A[c652e39a-8cf5-4ac8-924d-3bce32c190f8] succeeded in 0.03368515009060502s: None
[2023-03-08 18:19:52,465: INFO/MainProcess] Task topclans.worker.debug_broke[de4d83d9-1f5b-4f02-ad9e-1a3edbedd2f8] received
[2023-03-08 18:19:52,466: WARNING/ForkPoolWorker-1] Doing A1
[2023-03-08 18:19:52,481: INFO/ForkPoolWorker-1] Task topclans.worker.debug_A[c1b8ed9c-a9e0-4960-8b9b-1ecd5b2254a3] succeeded in 0.015777542954310775s: None
[2023-03-08 18:19:52,483: WARNING/ForkPoolWorker-1] Doing A2
[2023-03-08 18:19:52,500: INFO/ForkPoolWorker-1] Task topclans.worker.debug_A[a4478640-a50a-42a2-a8f0-8e6b84abab90] succeeded in 0.017103158170357347s: None
[2023-03-08 18:19:52,501: WARNING/ForkPoolWorker-1] Doing A3
[2023-03-08 18:19:52,515: INFO/ForkPoolWorker-1] Task topclans.worker.debug_A[d4c9d249-98dd-4071-ab03-70a350c7d171] succeeded in 0.013990010134875774s: None
[2023-03-08 18:19:52,516: WARNING/ForkPoolWorker-1] Doing A4
[2023-03-08 18:19:52,535: INFO/ForkPoolWorker-1] Task topclans.worker.debug_A[41c4bdb0-8993-40b0-90bd-2d6c642cb518] succeeded in 0.019201028160750866s: None
[2023-03-08 18:19:52,536: WARNING/ForkPoolWorker-1] OH NO
[2023-03-08 18:19:52,563: ERROR/ForkPoolWorker-1] Chord '3834f510-0652-407f-a589-6d9514e78974' raised: Exception('Head Aspload')
Traceback (most recent call last):
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/app/trace.py", line 451, in trace_task
R = retval = fun(*args, **kwargs)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/app/trace.py", line 734, in __protected_call__
return self.run(*args, **kwargs)
File "/mnt/code/wxp-git/wxp/src/topclans/worker/__init__.py", line 31, in debug_broke
raise Exception("Head Aspload")
Exception: Head Aspload
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django_celery_results/backends/database.py", line 276, in trigger_callback
ret = j(timeout=app.conf.result_chord_join_timeout, propagate=True)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/result.py", line 747, in join
value = result.get(
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/result.py", line 220, in get
self.maybe_throw(callback=callback)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/result.py", line 336, in maybe_throw
self.throw(value, self._to_remote_traceback(tb))
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/result.py", line 329, in throw
self.on_ready.throw(*args, **kwargs)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/vine/promises.py", line 234, in throw
reraise(type(exc), exc, tb)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/vine/utils.py", line 30, in reraise
raise value
Exception: Head Aspload
[2023-03-08 18:19:52,573: WARNING/ForkPoolWorker-1] /home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/app/trace.py:660: RuntimeWarning: Exception raised outside body: IntegrityError('null value in column "task_id" violates not-null constraint\nDETAIL: Failing row contains (77, null, FAILURE, application/x-python-serialize, binary, gASVRAAAAAAAAACMEWNlbGVyeS5leGNlcHRpb25zlIwKQ2hvcmRFcnJvcpSTlIwZ..., 2023-03-08 18:19:52.569411+00, null, gASVEQAAAAAAAAB9lIwIY2hpbGRyZW6UXZRzLg==, null, null, null, null, 2023-03-08 18:19:52.569405+00, null).\n'):
Traceback (most recent call last):
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/app/trace.py", line 451, in trace_task
R = retval = fun(*args, **kwargs)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/app/trace.py", line 734, in __protected_call__
return self.run(*args, **kwargs)
File "/mnt/code/wxp-git/wxp/src/topclans/worker/__init__.py", line 31, in debug_broke
raise Exception("Head Aspload")
Exception: Head Aspload
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django_celery_results/backends/database.py", line 276, in trigger_callback
ret = j(timeout=app.conf.result_chord_join_timeout, propagate=True)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/result.py", line 747, in join
value = result.get(
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/result.py", line 220, in get
self.maybe_throw(callback=callback)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/result.py", line 336, in maybe_throw
self.throw(value, self._to_remote_traceback(tb))
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/result.py", line 329, in throw
self.on_ready.throw(*args, **kwargs)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/vine/promises.py", line 234, in throw
reraise(type(exc), exc, tb)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/vine/utils.py", line 30, in reraise
raise value
Exception: Head Aspload
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/query.py", line 581, in get_or_create
return self.get(**kwargs), False
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/query.py", line 435, in get
raise self.model.DoesNotExist(
django_celery_results.models.TaskResult.DoesNotExist: TaskResult matching query does not exist.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
psycopg2.errors.NotNullViolation: null value in column "task_id" violates not-null constraint
DETAIL: Failing row contains (77, null, FAILURE, application/x-python-serialize, binary, gASVRAAAAAAAAACMEWNlbGVyeS5leGNlcHRpb25zlIwKQ2hvcmRFcnJvcpSTlIwZ..., 2023-03-08 18:19:52.569411+00, null, gASVEQAAAAAAAAB9lIwIY2hpbGRyZW6UXZRzLg==, null, null, null, null, 2023-03-08 18:19:52.569405+00, null).
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/app/trace.py", line 468, in trace_task
I, R, state, retval = on_error(task_request, exc, uuid)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/app/trace.py", line 379, in on_error
R = I.handle_error_state(
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/app/trace.py", line 178, in handle_error_state
return {
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/app/trace.py", line 225, in handle_failure
task.backend.mark_as_failure(
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/backends/base.py", line 177, in mark_as_failure
self.on_chord_part_return(request, state, exc)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django_celery_results/backends/database.py", line 258, in on_chord_part_return
trigger_callback(
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django_celery_results/backends/database.py", line 284, in trigger_callback
app.backend.chord_error_from_stack(callback, ChordError(reason))
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/backends/base.py", line 309, in chord_error_from_stack
return backend.fail_from_current_stack(callback.id, exc=exc)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/backends/base.py", line 316, in fail_from_current_stack
self.mark_as_failure(task_id, exc, exception_info.traceback)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/backends/base.py", line 172, in mark_as_failure
self.store_result(task_id, exc, state,
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/backends/base.py", line 528, in store_result
self._store_result(task_id, result, state, traceback,
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django_celery_results/backends/database.py", line 132, in _store_result
self.TaskModel._default_manager.store_result(**task_props)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django_celery_results/managers.py", line 43, in _inner
return fun(*args, **kwargs)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django_celery_results/managers.py", line 168, in store_result
obj, created = self.using(using).get_or_create(task_id=task_id,
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/query.py", line 588, in get_or_create
return self.create(**params), True
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/query.py", line 453, in create
obj.save(force_insert=True, using=self.db)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/base.py", line 739, in save
self.save_base(using=using, force_insert=force_insert,
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/base.py", line 776, in save_base
updated = self._save_table(
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/base.py", line 881, in _save_table
results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/base.py", line 919, in _do_insert
return manager._insert(
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/query.py", line 1270, in _insert
return query.get_compiler(using=using).execute_sql(returning_fields)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1416, in execute_sql
cursor.execute(sql, params)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/backends/utils.py", line 98, in execute
return super().execute(sql, params)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/backends/utils.py", line 66, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/backends/utils.py", line 75, in _execute_with_wrappers
return executor(sql, params, many, context)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
django.db.utils.IntegrityError: null value in column "task_id" violates not-null constraint
DETAIL: Failing row contains (77, null, FAILURE, application/x-python-serialize, binary, gASVRAAAAAAAAACMEWNlbGVyeS5leGNlcHRpb25zlIwKQ2hvcmRFcnJvcpSTlIwZ..., 2023-03-08 18:19:52.569411+00, null, gASVEQAAAAAAAAB9lIwIY2hpbGRyZW6UXZRzLg==, null, null, null, null, 2023-03-08 18:19:52.569405+00, null).
warn(RuntimeWarning(
[2023-03-08 18:19:52,595: WARNING/ForkPoolWorker-1] Can't find ChordCounter for Group 3834f510-0652-407f-a589-6d9514e78974
[2023-03-08 18:19:52,596: ERROR/ForkPoolWorker-1] Task topclans.worker.debug_broke[de4d83d9-1f5b-4f02-ad9e-1a3edbedd2f8] raised unexpected: IntegrityError('null value in column "task_id" violates not-null constraint\nDETAIL: Failing row contains (77, null, FAILURE, application/x-python-serialize, binary, gASVRAAAAAAAAACMEWNlbGVyeS5leGNlcHRpb25zlIwKQ2hvcmRFcnJvcpSTlIwZ..., 2023-03-08 18:19:52.569411+00, null, gASVEQAAAAAAAAB9lIwIY2hpbGRyZW6UXZRzLg==, null, null, null, null, 2023-03-08 18:19:52.569405+00, null).\n')
Traceback (most recent call last):
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/app/trace.py", line 451, in trace_task
R = retval = fun(*args, **kwargs)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/app/trace.py", line 734, in __protected_call__
return self.run(*args, **kwargs)
File "/mnt/code/wxp-git/wxp/src/topclans/worker/__init__.py", line 31, in debug_broke
raise Exception("Head Aspload")
Exception: Head Aspload
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django_celery_results/backends/database.py", line 276, in trigger_callback
ret = j(timeout=app.conf.result_chord_join_timeout, propagate=True)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/result.py", line 747, in join
value = result.get(
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/result.py", line 220, in get
self.maybe_throw(callback=callback)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/result.py", line 336, in maybe_throw
self.throw(value, self._to_remote_traceback(tb))
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/result.py", line 329, in throw
self.on_ready.throw(*args, **kwargs)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/vine/promises.py", line 234, in throw
reraise(type(exc), exc, tb)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/vine/utils.py", line 30, in reraise
raise value
Exception: Head Aspload
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/query.py", line 581, in get_or_create
return self.get(**kwargs), False
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/query.py", line 435, in get
raise self.model.DoesNotExist(
django_celery_results.models.TaskResult.DoesNotExist: TaskResult matching query does not exist.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
psycopg2.errors.NotNullViolation: null value in column "task_id" violates not-null constraint
DETAIL: Failing row contains (77, null, FAILURE, application/x-python-serialize, binary, gASVRAAAAAAAAACMEWNlbGVyeS5leGNlcHRpb25zlIwKQ2hvcmRFcnJvcpSTlIwZ..., 2023-03-08 18:19:52.569411+00, null, gASVEQAAAAAAAAB9lIwIY2hpbGRyZW6UXZRzLg==, null, null, null, null, 2023-03-08 18:19:52.569405+00, null).
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/app/trace.py", line 468, in trace_task
I, R, state, retval = on_error(task_request, exc, uuid)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/app/trace.py", line 379, in on_error
R = I.handle_error_state(
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/app/trace.py", line 178, in handle_error_state
return {
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/app/trace.py", line 225, in handle_failure
task.backend.mark_as_failure(
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/backends/base.py", line 177, in mark_as_failure
self.on_chord_part_return(request, state, exc)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django_celery_results/backends/database.py", line 258, in on_chord_part_return
trigger_callback(
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django_celery_results/backends/database.py", line 284, in trigger_callback
app.backend.chord_error_from_stack(callback, ChordError(reason))
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/backends/base.py", line 309, in chord_error_from_stack
return backend.fail_from_current_stack(callback.id, exc=exc)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/backends/base.py", line 316, in fail_from_current_stack
self.mark_as_failure(task_id, exc, exception_info.traceback)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/backends/base.py", line 172, in mark_as_failure
self.store_result(task_id, exc, state,
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/celery/backends/base.py", line 528, in store_result
self._store_result(task_id, result, state, traceback,
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django_celery_results/backends/database.py", line 132, in _store_result
self.TaskModel._default_manager.store_result(**task_props)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django_celery_results/managers.py", line 43, in _inner
return fun(*args, **kwargs)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django_celery_results/managers.py", line 168, in store_result
obj, created = self.using(using).get_or_create(task_id=task_id,
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/query.py", line 588, in get_or_create
return self.create(**params), True
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/query.py", line 453, in create
obj.save(force_insert=True, using=self.db)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/base.py", line 739, in save
self.save_base(using=using, force_insert=force_insert,
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/base.py", line 776, in save_base
updated = self._save_table(
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/base.py", line 881, in _save_table
results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/base.py", line 919, in _do_insert
return manager._insert(
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/query.py", line 1270, in _insert
return query.get_compiler(using=using).execute_sql(returning_fields)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1416, in execute_sql
cursor.execute(sql, params)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/backends/utils.py", line 98, in execute
return super().execute(sql, params)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/backends/utils.py", line 66, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/backends/utils.py", line 75, in _execute_with_wrappers
return executor(sql, params, many, context)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/home/matthew/venvs/wxp/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
django.db.utils.IntegrityError: null value in column "task_id" violates not-null constraint
DETAIL: Failing row contains (77, null, FAILURE, application/x-python-serialize, binary, gASVRAAAAAAAAACMEWNlbGVyeS5leGNlcHRpb25zlIwKQ2hvcmRFcnJvcpSTlIwZ..., 2023-03-08 18:19:52.569411+00, null, gASVEQAAAAAAAAB9lIwIY2hpbGRyZW6UXZRzLg==, null, null, null, null, 2023-03-08 18:19:52.569405+00, null).
It's these cascades of database errors that bother me. Am I doing somthing wrong? Or is this a bug in something?
If I just have the chord and don't chain another task, this seems to works fine. If I don't have the exception raised, that also works even with the extra chain.