feat(trigger): DPLAN-0112 #4 — per-handler failure counting + auto-disable
_fire_to_handlers now tracks consecutive failures per handler. After 5 consecutive failures, the handler is auto-disabled (skipped) and a CRITICAL log is emitted. Success resets the failure count. Disabled handlers re-enable on process restart (in-memory tracking). Prevents broken handlers from flooding logs forever. 370 tests passing. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
24568eb208
commit
59490757ca
@@ -48,6 +48,9 @@ class Trigger:
|
||||
_deferred_queue = [] # Queue for events fired during handling
|
||||
_draining_deferred = False # Prevents nested deferred processing
|
||||
_log_watcher_started = False # Lazy-start flag for log watcher
|
||||
_handler_failures = {} # handler -> consecutive failure count
|
||||
_disabled_handlers = set() # handlers auto-disabled after repeated failures
|
||||
_HANDLER_FAILURE_THRESHOLD = 5 # consecutive failures before auto-disable
|
||||
|
||||
@classmethod
|
||||
def _ensure_initialized(cls):
|
||||
@@ -99,10 +102,22 @@ class Trigger:
|
||||
data = dict(data) # Copy to avoid mutating caller's dict
|
||||
data['fire_event'] = cls.fire
|
||||
for handler in handlers:
|
||||
if handler in cls._disabled_handlers:
|
||||
continue
|
||||
try:
|
||||
handler(**data)
|
||||
cls._handler_failures.pop(handler, None) # Reset on success
|
||||
except Exception as e:
|
||||
logger.error(f"[TRIGGER] Handler error for {event}: {e}")
|
||||
count = cls._handler_failures.get(handler, 0) + 1
|
||||
cls._handler_failures[handler] = count
|
||||
if count >= cls._HANDLER_FAILURE_THRESHOLD:
|
||||
cls._disabled_handlers.add(handler)
|
||||
logger.error(
|
||||
f"[TRIGGER] Handler {getattr(handler, '__name__', handler)} "
|
||||
f"disabled after {count} consecutive failures"
|
||||
)
|
||||
else:
|
||||
logger.error(f"[TRIGGER] Handler error for {event}: {e}")
|
||||
|
||||
@classmethod
|
||||
def _drain_deferred(cls) -> None:
|
||||
|
||||
Reference in New Issue
Block a user