From 9c7b275466179fbfb0ae0525fe7df7e21574f6d1 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 11 Jul 2026 13:13:14 -0400 Subject: [PATCH 1/6] fix(app): tolerate queue items deleted while running Deleting a queue item (or clearing the queue) while it is executing is a supported operation: the delete/clear paths remove the DB row immediately and rely on the processor noticing the cancel event afterwards. The workflow call lifecycle re-fetched the queue item and its parents unconditionally after the session runner returned, raising SessionQueueItemNotFoundError and killing the processor thread when the row was already gone. Treat a missing queue item or parent as 'deleted mid-run' and return quietly instead, matching the existing handling in _on_after_run_session. Co-Authored-By: Claude Fable 5 --- .../workflow_call_runtime.py | 39 +++++++++--- .../services/test_workflow_call_runtime.py | 8 +++ .../app/services/workflow_call_test_utils.py | 63 ++++++++++++++++++- 3 files changed, 100 insertions(+), 10 deletions(-) diff --git a/invokeai/app/services/session_processor/workflow_call_runtime.py b/invokeai/app/services/session_processor/workflow_call_runtime.py index dba16650487..cd7f2fd3dfc 100644 --- a/invokeai/app/services/session_processor/workflow_call_runtime.py +++ b/invokeai/app/services/session_processor/workflow_call_runtime.py @@ -8,7 +8,11 @@ ) from invokeai.app.invocations.workflow_return import WorkflowReturnOutput from invokeai.app.services.session_processor.workflow_call_batch import build_child_workflow_session_results -from invokeai.app.services.session_queue.session_queue_common import NodeFieldValue, SessionQueueItem +from invokeai.app.services.session_queue.session_queue_common import ( + NodeFieldValue, + SessionQueueItem, + SessionQueueItemNotFoundError, +) from invokeai.app.services.shared.graph import GraphExecutionState if TYPE_CHECKING: @@ -175,15 +179,19 @@ def fail_waiting_workflow_call(self, queue_item: SessionQueueItem, error_message error_traceback=error_message, ) - def _get_parent_queue_item(self, child_queue_item: SessionQueueItem) -> SessionQueueItem: + def _get_parent_queue_item(self, child_queue_item: SessionQueueItem) -> SessionQueueItem | None: parent_item_id = child_queue_item.parent_item_id if parent_item_id is None: raise ValueError("Child workflow queue item is missing parent_item_id metadata.") - return self._session_runner._services.session_queue.get_queue_item(parent_item_id) + try: + return self._session_runner._services.session_queue.get_queue_item(parent_item_id) + except SessionQueueItemNotFoundError: + # The parent may have been deleted while the child was running (e.g. the queue was cleared). + return None def _resume_parent_from_completed_child(self, child_queue_item: SessionQueueItem) -> None: parent_queue_item = self._get_parent_queue_item(child_queue_item) - if parent_queue_item.status in ("completed", "failed", "canceled"): + if parent_queue_item is None or parent_queue_item.status in ("completed", "failed", "canceled"): return try: output = self.get_child_workflow_return_output(child_queue_item.session) @@ -200,7 +208,12 @@ def _resume_parent_from_completed_child(self, child_queue_item: SessionQueueItem exclude_item_ids={child_queue_item.item_id}, ) self.fail_waiting_workflow_call(parent_queue_item, str(e)) - parent_queue_item = self._session_runner._services.session_queue.get_queue_item(parent_queue_item.item_id) + try: + parent_queue_item = self._session_runner._services.session_queue.get_queue_item( + parent_queue_item.item_id + ) + except SessionQueueItemNotFoundError: + return if getattr(parent_queue_item, "parent_item_id", None) is not None: self._fail_parent_from_failed_child(parent_queue_item) return @@ -229,7 +242,7 @@ def _resume_parent_from_completed_child(self, child_queue_item: SessionQueueItem def _fail_parent_from_failed_child(self, child_queue_item: SessionQueueItem) -> None: parent_queue_item = self._get_parent_queue_item(child_queue_item) - if parent_queue_item.status in ("completed", "failed", "canceled"): + if parent_queue_item is None or parent_queue_item.status in ("completed", "failed", "canceled"): return waiting_frame = parent_queue_item.session.waiting_workflow_call if waiting_frame is None: @@ -244,19 +257,27 @@ def _fail_parent_from_failed_child(self, child_queue_item: SessionQueueItem) -> f"The selected saved workflow '{waiting_frame.workflow_id}' failed during child execution." ) self.fail_waiting_workflow_call(parent_queue_item, child_error_message) - parent_queue_item = self._session_runner._services.session_queue.get_queue_item(parent_queue_item.item_id) + try: + parent_queue_item = self._session_runner._services.session_queue.get_queue_item(parent_queue_item.item_id) + except SessionQueueItemNotFoundError: + return if getattr(parent_queue_item, "parent_item_id", None) is not None: self._fail_parent_from_failed_child(parent_queue_item) def _cancel_parent_from_canceled_child(self, child_queue_item: SessionQueueItem) -> None: parent_queue_item = self._get_parent_queue_item(child_queue_item) - if parent_queue_item.status == "canceled": + if parent_queue_item is None or parent_queue_item.status == "canceled": return self._session_runner._services.session_queue.cancel_queue_item(parent_queue_item.item_id) def run_queue_item(self, queue_item: SessionQueueItem) -> None: self._session_runner.run(queue_item) - updated_queue_item = self._session_runner._services.session_queue.get_queue_item(queue_item.item_id) + try: + updated_queue_item = self._session_runner._services.session_queue.get_queue_item(queue_item.item_id) + except SessionQueueItemNotFoundError: + # The queue item was deleted while it was running (e.g. the queue was cleared or the current item + # was deleted). There is no parent lifecycle work to do for a deleted item. + return if getattr(updated_queue_item, "parent_item_id", None) is None: return if updated_queue_item.status == "completed": diff --git a/tests/app/services/test_workflow_call_runtime.py b/tests/app/services/test_workflow_call_runtime.py index 1edaaaae007..8cb1b566830 100644 --- a/tests/app/services/test_workflow_call_runtime.py +++ b/tests/app/services/test_workflow_call_runtime.py @@ -39,6 +39,14 @@ def test_run_does_not_fail_canceled_parent_after_child_return_error(monkeypatch) workflow_call_tests.test_run_does_not_fail_canceled_parent_after_child_return_error(monkeypatch) +def test_run_queue_item_tolerates_queue_item_deleted_mid_run(monkeypatch) -> None: + workflow_call_tests.test_run_queue_item_tolerates_queue_item_deleted_mid_run(monkeypatch) + + +def test_run_queue_item_tolerates_parent_deleted_while_child_runs(monkeypatch) -> None: + workflow_call_tests.test_run_queue_item_tolerates_parent_deleted_while_child_runs(monkeypatch) + + def test_workflow_call_coordinator_builds_child_queue_item_with_relationship_metadata(monkeypatch) -> None: workflow_call_tests.test_workflow_call_coordinator_builds_child_queue_item_with_relationship_metadata(monkeypatch) diff --git a/tests/app/services/workflow_call_test_utils.py b/tests/app/services/workflow_call_test_utils.py index cbe94707e93..4ea8abeda19 100644 --- a/tests/app/services/workflow_call_test_utils.py +++ b/tests/app/services/workflow_call_test_utils.py @@ -23,6 +23,7 @@ WorkflowCallCoordinator, WorkflowCallQueueLifecycle, ) +from invokeai.app.services.session_queue.session_queue_common import SessionQueueItemNotFoundError from invokeai.app.services.shared.graph import Graph, GraphExecutionState, WorkflowCallFrame from invokeai.app.services.workflow_records.workflow_records_common import WorkflowCategory from tests.dangerously_run_function_in_subprocess import dangerously_run_function_in_subprocess @@ -913,7 +914,10 @@ def _ensure_queue_item(self, item_id: int, session): return queue_item def get_queue_item(self, item_id: int): - return self.items[item_id] + queue_item = self.items.get(item_id) + if queue_item is None: + raise SessionQueueItemNotFoundError(f"No queue item with id {item_id}") + return queue_item def set_queue_item_session(self, item_id: int, session): queue_item = self._ensure_queue_item(item_id, session) @@ -1510,6 +1514,63 @@ def test_workflow_call_queue_lifecycle_resumes_parent_from_completed_child( assert parent_outputs[0].values == {"result": [3]} +def test_run_queue_item_tolerates_queue_item_deleted_mid_run(monkeypatch: pytest.MonkeyPatch) -> None: + session_queue = _DummySessionQueue() + runner, events, _workflow_records = _build_workflow_runner(monkeypatch, session_queue=session_queue) + lifecycle = WorkflowCallQueueLifecycle(runner) + + queue_item = SimpleNamespace( + item_id=1, + status="in_progress", + session=None, + session_id="session-id", + parent_item_id=None, + ) + session_queue.add_queue_item(queue_item) + + # Simulate the queue item being deleted while it is running (e.g. the queue was cleared or the + # current item was deleted via the API). + def run_and_delete(item) -> None: + session_queue.delete_queue_items_by_id([item.item_id]) + + monkeypatch.setattr(runner, "run", run_and_delete) + + lifecycle.run_queue_item(queue_item) + + assert session_queue.deleted_item_ids == [1] + assert session_queue.completed_item_ids == [] + assert session_queue.failed_item_ids == [] + assert events.errors == [] + + +def test_run_queue_item_tolerates_parent_deleted_while_child_runs(monkeypatch: pytest.MonkeyPatch) -> None: + for child_terminal_status in ("completed", "failed", "canceled"): + session_queue = _DummySessionQueue() + runner, events, _workflow_records = _build_workflow_runner(monkeypatch, session_queue=session_queue) + lifecycle = WorkflowCallQueueLifecycle(runner) + + child_queue_item = SimpleNamespace( + item_id=100, + status="in_progress", + session=None, + session_id="child-session-id", + parent_item_id=1, + ) + session_queue.add_queue_item(child_queue_item) + + # The parent (item id 1) does not exist in the queue - it was deleted while the child was running. + def run_to_terminal_status(item, status=child_terminal_status) -> None: + item.status = status + + monkeypatch.setattr(runner, "run", run_to_terminal_status) + + lifecycle.run_queue_item(child_queue_item) + + assert session_queue.canceled_item_ids == [] + assert session_queue.failed_item_ids == [] + assert events.errors == [] + + def test_workflow_call_coordinator_cleans_up_enqueued_children_when_boundary_setup_fails( monkeypatch: pytest.MonkeyPatch, ) -> None: From 6d5c398f72b72d099275ece5a3daf46511be0696 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 11 Jul 2026 13:13:22 -0400 Subject: [PATCH 2/6] fix(ui): shift+x hotkey cancels current queue item instead of deleting it Commit 5baa4bd916 moved the visible queue controls back to cancelation semantics but the cancelQueueItem hotkey was missed and still called the delete hook. Deleting removes the DB row while the item is still executing, which raced the session processor. Align the hotkey with the CancelCurrentQueueItemIconButton it mirrors. Co-Authored-By: Claude Fable 5 --- .../web/src/common/hooks/useGlobalHotkeys.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/invokeai/frontend/web/src/common/hooks/useGlobalHotkeys.ts b/invokeai/frontend/web/src/common/hooks/useGlobalHotkeys.ts index dd43c0b0947..070a4866a50 100644 --- a/invokeai/frontend/web/src/common/hooks/useGlobalHotkeys.ts +++ b/invokeai/frontend/web/src/common/hooks/useGlobalHotkeys.ts @@ -1,8 +1,8 @@ import { useAppStore } from 'app/store/storeHooks'; import { useDeleteImageModalApi } from 'features/deleteImageModal/store/state'; import { selectSelection } from 'features/gallery/store/gallerySelectors'; +import { useCancelCurrentQueueItem } from 'features/queue/hooks/useCancelCurrentQueueItem'; import { useClearQueue } from 'features/queue/hooks/useClearQueue'; -import { useDeleteCurrentQueueItem } from 'features/queue/hooks/useDeleteCurrentQueueItem'; import { useInvoke } from 'features/queue/hooks/useInvoke'; import { useRegisteredHotkeys } from 'features/system/components/HotkeysModal/useHotkeyData'; import { navigationApi } from 'features/ui/layouts/navigation-api'; @@ -37,17 +37,19 @@ export const useGlobalHotkeys = () => { dependencies: [queue], }); - const deleteCurrentQueueItem = useDeleteCurrentQueueItem(); + const cancelCurrentQueueItem = useCancelCurrentQueueItem(); useRegisteredHotkeys({ id: 'cancelQueueItem', category: 'app', - callback: deleteCurrentQueueItem.trigger, + callback: () => { + cancelCurrentQueueItem.trigger(); + }, options: { - enabled: !deleteCurrentQueueItem.isDisabled && !deleteCurrentQueueItem.isLoading, + enabled: !cancelCurrentQueueItem.isDisabled && !cancelCurrentQueueItem.isLoading, preventDefault: true, }, - dependencies: [deleteCurrentQueueItem], + dependencies: [cancelCurrentQueueItem], }); const clearQueue = useClearQueue(); From b9ad1f2a988e1b080ee365c6be6309052ff62d20 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sat, 11 Jul 2026 22:14:19 -0400 Subject: [PATCH 3/6] fix(ui): reset progress bar when the current queue item is canceled Canceling the current item clears $lastProgressEvent on the terminal queue_item_status_changed event, but a denoise step callback can race the cancelation and emit invocation_progress afterwards, repopulating the progress bar which then sticks at its last value. The coordinator's state machine is designed to reject such trailing events, but the terminal status handler deleted the item's state entry at exactly the moment it was needed. Keep the entry instead - item ids are never reused (retries insert new rows) and the LRU cache bounds memory. Also reset the progress bar on queue_cleared: clearing the queue deletes the in-progress item without emitting any per-item terminal status event, so nothing else clears it on that path. Co-Authored-By: Claude Fable 5 --- .../src/services/events/setEventListeners.tsx | 3 ++ .../workflowExecutionCoordinator.test.ts | 35 +++++++++++++++++++ .../events/workflowExecutionCoordinator.ts | 5 ++- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/invokeai/frontend/web/src/services/events/setEventListeners.tsx b/invokeai/frontend/web/src/services/events/setEventListeners.tsx index 30f705d670e..d40a8eb1298 100644 --- a/invokeai/frontend/web/src/services/events/setEventListeners.tsx +++ b/invokeai/frontend/web/src/services/events/setEventListeners.tsx @@ -538,6 +538,9 @@ export const setEventListeners = ({ socket, store, setIsConnected }: SetEventLis socket.on('queue_cleared', (data) => { log.debug({ data }, 'Queue cleared'); + // Clearing the queue deletes the in-progress item without emitting a per-item terminal status + // event, so the progress bar must be reset here. + $lastProgressEvent.set(null); dispatch( queueApi.util.invalidateTags([ 'SessionQueueStatus', diff --git a/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.test.ts b/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.test.ts index 00c48c8a373..c8c256f679d 100644 --- a/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.test.ts +++ b/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.test.ts @@ -71,6 +71,28 @@ const buildInvocationStartedEvent = ( ...overrides, }) as S['InvocationStartedEvent']; +const buildInvocationProgressEvent = ( + overrides: Partial = {} +): S['InvocationProgressEvent'] => + ({ + queue_id: 'default', + item_id: 1, + batch_id: 'batch-1', + origin: null, + destination: null, + user_id: 'user-1', + session_id: 'session-1', + invocation_source_id: 'node-1', + invocation: { + id: 'prepared-node-1', + type: 'test_node', + }, + message: 'denoising', + percentage: 0.5, + image: null, + ...overrides, + }) as S['InvocationProgressEvent']; + const buildInvocationCompleteEvent = ( overrides: Partial = {} ): S['InvocationCompleteEvent'] => @@ -286,6 +308,19 @@ describe(createWorkflowExecutionCoordinator.name, () => { expect(queueItemRequests.size).toBe(1); }); + it('rejects trailing invocation progress after a queue item is canceled', () => { + const { coordinator } = createCoordinatorHarness(); + + coordinator.onQueueItemStatusChanged(buildQueueStatusEvent({ item_id: 1, status: 'in_progress', origin: null })); + expect(coordinator.onInvocationProgress(buildInvocationProgressEvent({ item_id: 1 }))).toBe(true); + + coordinator.onQueueItemStatusChanged(buildQueueStatusEvent({ item_id: 1, status: 'canceled', origin: null })); + + // A denoise step callback may race the cancelation and emit progress after the terminal status + // event. It must not be applied - it would repopulate the progress bar after it was cleared. + expect(coordinator.onInvocationProgress(buildInvocationProgressEvent({ item_id: 1 }))).toBe(false); + }); + it('still clears canvas workflow integration processing on late invocation errors', () => { const { clearCanvasWorkflowIntegrationProcessing, coordinator } = createCoordinatorHarness(); diff --git a/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.ts b/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.ts index fceb117d87d..c6f9bdb804a 100644 --- a/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.ts +++ b/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.ts @@ -59,7 +59,10 @@ export const createWorkflowExecutionCoordinator = (deps: WorkflowExecutionCoordi req?.abort?.(); req?.unsubscribe?.(); pendingWorkflowReconciliationRequests.delete(itemId); - workflowExecutionStates.delete(itemId); + // The workflow execution state entry is intentionally kept. A canceled queue item can emit a + // few trailing invocation events (e.g. a denoise step callback racing the cancelation), and the + // retained terminal state is what rejects them. Item ids are never reused, and the LRU cache + // bounds memory. clearCompletedInvocationKeysForQueueItem(deps.completedInvocationKeysByItemId, itemId); }; From 6fdcd80ec0221e631bfecb63ed0db0127628ef7a Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sun, 12 Jul 2026 10:36:48 -0400 Subject: [PATCH 4/6] fix(app): close TOCTOU races in parent lifecycle transitions Per review: guarding only the parent lookups left windows where the parent row is deleted after get_queue_item succeeds but before the subsequent mutation (set_queue_item_session, complete_queue_item, resume_queue_item, cancel_queue_item, or the fail path via _on_node_error). Those mutations re-read the row and raise SessionQueueItemNotFoundError, which returned to the session processor as the same non-fatal error this PR eliminates. Catch the exception around the three parent transitions in run_queue_item: any item vanishing mid-transition means the chain is being torn down and there is nothing left to update. The dummy session queue now supports simulating deletion-after-lookup (mutations raise, lookups succeed), with one test per transition path. Co-Authored-By: Claude Fable 5 --- .../workflow_call_runtime.py | 18 ++- .../services/test_workflow_call_runtime.py | 12 ++ .../app/services/workflow_call_test_utils.py | 106 ++++++++++++++++++ 3 files changed, 130 insertions(+), 6 deletions(-) diff --git a/invokeai/app/services/session_processor/workflow_call_runtime.py b/invokeai/app/services/session_processor/workflow_call_runtime.py index cd7f2fd3dfc..47576475101 100644 --- a/invokeai/app/services/session_processor/workflow_call_runtime.py +++ b/invokeai/app/services/session_processor/workflow_call_runtime.py @@ -280,9 +280,15 @@ def run_queue_item(self, queue_item: SessionQueueItem) -> None: return if getattr(updated_queue_item, "parent_item_id", None) is None: return - if updated_queue_item.status == "completed": - self._resume_parent_from_completed_child(updated_queue_item) - elif updated_queue_item.status == "failed": - self._fail_parent_from_failed_child(updated_queue_item) - elif updated_queue_item.status == "canceled": - self._cancel_parent_from_canceled_child(updated_queue_item) + try: + if updated_queue_item.status == "completed": + self._resume_parent_from_completed_child(updated_queue_item) + elif updated_queue_item.status == "failed": + self._fail_parent_from_failed_child(updated_queue_item) + elif updated_queue_item.status == "canceled": + self._cancel_parent_from_canceled_child(updated_queue_item) + except SessionQueueItemNotFoundError: + # An item in the parent chain was deleted after we looked it up but before we mutated it + # (the queue mutations re-read the row and raise when it has disappeared). The chain is + # being torn down; there is nothing left to update. + return diff --git a/tests/app/services/test_workflow_call_runtime.py b/tests/app/services/test_workflow_call_runtime.py index 8cb1b566830..6f8a469a896 100644 --- a/tests/app/services/test_workflow_call_runtime.py +++ b/tests/app/services/test_workflow_call_runtime.py @@ -47,6 +47,18 @@ def test_run_queue_item_tolerates_parent_deleted_while_child_runs(monkeypatch) - workflow_call_tests.test_run_queue_item_tolerates_parent_deleted_while_child_runs(monkeypatch) +def test_run_queue_item_tolerates_parent_deleted_before_completed_parent_mutation(monkeypatch) -> None: + workflow_call_tests.test_run_queue_item_tolerates_parent_deleted_before_completed_parent_mutation(monkeypatch) + + +def test_run_queue_item_tolerates_parent_deleted_before_failed_parent_mutation(monkeypatch) -> None: + workflow_call_tests.test_run_queue_item_tolerates_parent_deleted_before_failed_parent_mutation(monkeypatch) + + +def test_run_queue_item_tolerates_parent_deleted_before_canceled_parent_mutation(monkeypatch) -> None: + workflow_call_tests.test_run_queue_item_tolerates_parent_deleted_before_canceled_parent_mutation(monkeypatch) + + def test_workflow_call_coordinator_builds_child_queue_item_with_relationship_metadata(monkeypatch) -> None: workflow_call_tests.test_workflow_call_coordinator_builds_child_queue_item_with_relationship_metadata(monkeypatch) diff --git a/tests/app/services/workflow_call_test_utils.py b/tests/app/services/workflow_call_test_utils.py index 4ea8abeda19..8923d050bd9 100644 --- a/tests/app/services/workflow_call_test_utils.py +++ b/tests/app/services/workflow_call_test_utils.py @@ -888,6 +888,14 @@ def __init__(self) -> None: self.deleted_item_ids: list[int] = [] self.pending_count = 0 self.fail_enqueue_after: int | None = None + # Item ids whose mutation methods raise SessionQueueItemNotFoundError, simulating the row + # being deleted after a successful get_queue_item lookup. Like the SQLite implementation's + # mutations, which re-read the row and raise when it has disappeared. + self.not_found_item_ids: set[int] = set() + + def _raise_if_not_found(self, item_id: int) -> None: + if item_id in self.not_found_item_ids: + raise SessionQueueItemNotFoundError(f"No queue item with id {item_id}") def add_queue_item(self, queue_item): self.items[queue_item.item_id] = queue_item @@ -920,30 +928,42 @@ def get_queue_item(self, item_id: int): return queue_item def set_queue_item_session(self, item_id: int, session): + self._raise_if_not_found(item_id) queue_item = self._ensure_queue_item(item_id, session) queue_item.session = session self.session_updates.append((item_id, session)) return queue_item def suspend_queue_item(self, item_id: int): + self._raise_if_not_found(item_id) queue_item = self._ensure_queue_item(item_id, None) queue_item.status = "waiting" self.waiting_item_ids.append(item_id) return queue_item def resume_queue_item(self, item_id: int): + self._raise_if_not_found(item_id) queue_item = self._ensure_queue_item(item_id, None) queue_item.status = "pending" self.resumed_item_ids.append(item_id) return queue_item def complete_queue_item(self, item_id: int): + self._raise_if_not_found(item_id) queue_item = self._ensure_queue_item(item_id, None) queue_item.status = "completed" self.completed_item_ids.append(item_id) return queue_item + def cancel_queue_item(self, item_id: int): + self._raise_if_not_found(item_id) + queue_item = self._ensure_queue_item(item_id, None) + queue_item.status = "canceled" + self.canceled_item_ids.append(item_id) + return queue_item + def fail_queue_item(self, item_id: int, error_type: str, error_message: str, error_traceback: str): + self._raise_if_not_found(item_id) queue_item = self._ensure_queue_item(item_id, None) queue_item.status = "failed" queue_item.error_type = error_type @@ -1571,6 +1591,92 @@ def run_to_terminal_status(item, status=child_terminal_status) -> None: assert events.errors == [] +def _setup_suspended_workflow_call_parent(monkeypatch: pytest.MonkeyPatch): + """Runs a workflow-call parent to its suspension point: parent (item 1) is waiting, child (item 100) is pending.""" + session_queue = _DummySessionQueue() + runner, events, _workflow_records = _build_workflow_runner(monkeypatch, session_queue=session_queue) + lifecycle = WorkflowCallQueueLifecycle(runner) + + graph = Graph() + graph.add_node(CallSavedWorkflowInvocation(id="call-node", workflow_id="workflow-a")) + session = GraphExecutionState(graph=graph) + queue_item = type( + "QueueItem", + (), + { + "item_id": 1, + "status": "in_progress", + "session": session, + "session_id": "session-id", + "user_id": "user-1", + "queue_id": "default", + "batch_id": "batch-1", + }, + )() + session_queue.add_queue_item(queue_item) + lifecycle.run_queue_item(queue_item) + assert session_queue.waiting_item_ids == [1] + assert session_queue.enqueued_child_item_ids == [100] + return session_queue, runner, lifecycle, events + + +def test_run_queue_item_tolerates_parent_deleted_before_completed_parent_mutation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + session_queue, runner, lifecycle, _events = _setup_suspended_workflow_call_parent(monkeypatch) + + # The parent is deleted after the child's completion handler has looked it up: get_queue_item + # still succeeds, but the parent mutations (set_queue_item_session et al.) raise. + session_queue.not_found_item_ids.add(1) + + child_queue_item = session_queue.dequeue() + assert child_queue_item is not None + lifecycle.run_queue_item(child_queue_item) + + assert 100 in session_queue.completed_item_ids + assert 1 not in session_queue.completed_item_ids + assert session_queue.resumed_item_ids == [] + + +def test_run_queue_item_tolerates_parent_deleted_before_failed_parent_mutation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + session_queue, runner, lifecycle, _events = _setup_suspended_workflow_call_parent(monkeypatch) + + session_queue.not_found_item_ids.add(1) + + child_queue_item = session_queue.dequeue() + assert child_queue_item is not None + + def run_to_failed(item) -> None: + item.status = "failed" + item.error_message = "child failed" + + monkeypatch.setattr(runner, "run", run_to_failed) + lifecycle.run_queue_item(child_queue_item) + + assert 1 not in session_queue.failed_item_ids + + +def test_run_queue_item_tolerates_parent_deleted_before_canceled_parent_mutation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + session_queue, runner, lifecycle, _events = _setup_suspended_workflow_call_parent(monkeypatch) + + session_queue.not_found_item_ids.add(1) + + child_queue_item = session_queue.dequeue() + assert child_queue_item is not None + + def run_to_canceled(item) -> None: + item.status = "canceled" + + monkeypatch.setattr(runner, "run", run_to_canceled) + lifecycle.run_queue_item(child_queue_item) + + assert session_queue.canceled_item_ids == [] + + def test_workflow_call_coordinator_cleans_up_enqueued_children_when_boundary_setup_fails( monkeypatch: pytest.MonkeyPatch, ) -> None: From be3984d1c456097eb8d6167203de983de48ce088 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sun, 12 Jul 2026 10:36:48 -0400 Subject: [PATCH 5/6] fix(ui): reject trailing invocation events after queue_cleared Per review: clearing the queue reset the progress bar but did not mark the deleted in-progress item terminal in the workflowExecutionCoordinator, so a trailing invocation_progress event racing the clear was accepted and repopulated the bar - recreating the frozen progress bar on the clear path. The coordinator now exposes onQueueCleared, which aborts pending reconciliations and marks every tracked non-terminal item canceled so trailing events are rejected, and the queue_cleared listener calls it before resetting the progress store. Co-Authored-By: Claude Fable 5 --- .../src/services/events/setEventListeners.tsx | 4 +++- .../workflowExecutionCoordinator.test.ts | 18 ++++++++++++++++++ .../events/workflowExecutionCoordinator.ts | 15 +++++++++++++++ .../services/events/workflowExecutionState.ts | 3 ++- 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/invokeai/frontend/web/src/services/events/setEventListeners.tsx b/invokeai/frontend/web/src/services/events/setEventListeners.tsx index d40a8eb1298..11dabfc4601 100644 --- a/invokeai/frontend/web/src/services/events/setEventListeners.tsx +++ b/invokeai/frontend/web/src/services/events/setEventListeners.tsx @@ -539,7 +539,9 @@ export const setEventListeners = ({ socket, store, setIsConnected }: SetEventLis socket.on('queue_cleared', (data) => { log.debug({ data }, 'Queue cleared'); // Clearing the queue deletes the in-progress item without emitting a per-item terminal status - // event, so the progress bar must be reset here. + // event, so the progress bar must be reset here — and the coordinator must mark its tracked + // items terminal so a trailing invocation_progress event cannot repopulate the bar. + workflowExecutionCoordinator.onQueueCleared(); $lastProgressEvent.set(null); dispatch( queueApi.util.invalidateTags([ diff --git a/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.test.ts b/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.test.ts index c8c256f679d..f3dcd178003 100644 --- a/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.test.ts +++ b/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.test.ts @@ -321,6 +321,24 @@ describe(createWorkflowExecutionCoordinator.name, () => { expect(coordinator.onInvocationProgress(buildInvocationProgressEvent({ item_id: 1 }))).toBe(false); }); + it('rejects trailing invocation progress after the queue is cleared', () => { + const { coordinator, queueItemRequests } = createCoordinatorHarness(); + + coordinator.onQueueItemStatusChanged(buildQueueStatusEvent({ item_id: 1, status: 'in_progress', origin: null })); + expect(coordinator.onInvocationProgress(buildInvocationProgressEvent({ item_id: 1 }))).toBe(true); + + // Clearing the queue deletes items without emitting per-item terminal status events, so the + // coordinator is told directly. A workflow item with a pending reconciliation is also tracked + // to verify the reconciliation is aborted. + coordinator.onQueueItemStatusChanged( + buildQueueStatusEvent({ item_id: 2, status: 'completed', origin: 'workflows' }) + ); + coordinator.onQueueCleared(); + + expect(coordinator.onInvocationProgress(buildInvocationProgressEvent({ item_id: 1 }))).toBe(false); + expect(queueItemRequests.get(2)?.abort).toHaveBeenCalled(); + }); + it('still clears canvas workflow integration processing on late invocation errors', () => { const { clearCanvasWorkflowIntegrationProcessing, coordinator } = createCoordinatorHarness(); diff --git a/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.ts b/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.ts index c6f9bdb804a..8b85b055acf 100644 --- a/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.ts +++ b/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.ts @@ -15,6 +15,7 @@ import { } from 'services/events/nodeExecutionState'; import { createWorkflowExecutionState, + isTerminalQueueStatus, transitionWorkflowExecutionState, type WorkflowExecutionState, } from 'services/events/workflowExecutionState'; @@ -74,6 +75,19 @@ export const createWorkflowExecutionCoordinator = (deps: WorkflowExecutionCoordi pendingWorkflowReconciliationRequests.clear(); }; + const onQueueCleared = () => { + // Clearing the queue deletes its items without emitting per-item terminal status events, so the + // tracked items must be marked terminal here for trailing invocation events to be rejected. + cancelPendingWorkflowReconciliations(); + for (const itemId of [...workflowExecutionStates.keys()]) { + const state = workflowExecutionStates.get(itemId); + if (!state || isTerminalQueueStatus(state.queueStatus)) { + continue; + } + workflowExecutionStates.set(itemId, { ...state, queueStatus: 'canceled' }); + } + }; + const reconcileWorkflowQueueItemResults = (itemId: number, status: TerminalQueueStatus) => { const req = deps.reconcileQueueItem(itemId); pendingWorkflowReconciliationRequests.set(itemId, req); @@ -248,6 +262,7 @@ export const createWorkflowExecutionCoordinator = (deps: WorkflowExecutionCoordi onInvocationError, onInvocationProgress, onInvocationStarted, + onQueueCleared, onQueueItemStatusChanged, }; }; diff --git a/invokeai/frontend/web/src/services/events/workflowExecutionState.ts b/invokeai/frontend/web/src/services/events/workflowExecutionState.ts index bb571447ef0..76e679315d8 100644 --- a/invokeai/frontend/web/src/services/events/workflowExecutionState.ts +++ b/invokeai/frontend/web/src/services/events/workflowExecutionState.ts @@ -35,7 +35,8 @@ type WorkflowExecutionTransition = { const TERMINAL_QUEUE_STATUSES = new Set(['completed', 'failed', 'canceled']); const TERMINAL_INVOCATION_STATUSES = new Set(['completed', 'failed']); -const isTerminalQueueStatus = (status: QueueStatus | null) => status !== null && TERMINAL_QUEUE_STATUSES.has(status); +export const isTerminalQueueStatus = (status: QueueStatus | null) => + status !== null && TERMINAL_QUEUE_STATUSES.has(status); export const createWorkflowExecutionState = (): WorkflowExecutionState => ({ itemId: null, From ca09c0e3f604f2689b4d18ee06bd8775dfcae420 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sun, 12 Jul 2026 17:02:14 -0400 Subject: [PATCH 6/6] fix(app,ui): scope queue_cleared to the clearing user in multiuser mode A user-scoped queue clear (non-admin in multiuser mode) deletes only the requesting user's rows, but QueueClearedEvent carried no user identity and was broadcast to the whole queue room. Every client's coordinator treated it as its own clear: pending reconciliations of unaffected users' terminal items were aborted and never restarted, and their tracked non-terminal items were marked canceled, so completed sibling outputs went stale and subsequent events were rejected. QueueClearedEvent now carries the clearing scope (user_id; None means all users' items were deleted). Routing follows the existing multiuser convention: an unscoped clear is still broadcast to the queue room; a scoped clear goes in full to the owner + admins with a sanitized (user_id="redacted") companion to the rest of the queue room, so other clients refresh their queue tags without treating the clear as their own. The coordinator applies a clear only when it is unscoped or scoped to the current user, and the listener resets the progress bar only when the clear applied locally. Regression tests: coordinator (user B's in-flight reconciliation completes and its in-progress item keeps accepting events across user A's scoped clear; a clear scoped to the current user still applies) and socket routing (scoped clear full-to-owner/admin plus sanitized companion with owner/admin sids skipped). Co-Authored-By: Claude Fable 5 --- invokeai/app/api/sockets.py | 37 +++++++++++-- invokeai/app/services/events/events_base.py | 6 +-- invokeai/app/services/events/events_common.py | 9 +++- .../session_queue/session_queue_sqlite.py | 2 +- invokeai/frontend/web/openapi.json | 15 +++++- .../frontend/web/src/services/api/schema.ts | 6 +++ .../src/services/events/setEventListeners.tsx | 12 +++-- .../workflowExecutionCoordinator.test.ts | 53 ++++++++++++++++++- .../events/workflowExecutionCoordinator.ts | 13 ++++- .../routers/test_multiuser_authorization.py | 52 ++++++++++++++++-- 10 files changed, 184 insertions(+), 21 deletions(-) diff --git a/invokeai/app/api/sockets.py b/invokeai/app/api/sockets.py index 70ca97740cf..bdaf641c1c1 100644 --- a/invokeai/app/api/sockets.py +++ b/invokeai/app/api/sockets.py @@ -306,7 +306,10 @@ async def _handle_queue_event(self, event: FastAPIEvent[QueueEventBase]): per-session side effects. InvocationEventBase events stay private (owner + admins only). RecallParametersUpdatedEvent - is also private. QueueClearedEvent has no user identity and is broadcast to the queue room. + is also private. QueueClearedEvent is broadcast to the queue room when unscoped (an admin or + single-user clear that deleted every user's items); a user-scoped clear goes full to + owner + admins with a sanitized companion to the rest of the queue room, so other users + refresh their queue lists without treating the clear as their own. IMPORTANT: Check InvocationEventBase BEFORE QueueItemEventBase since InvocationEventBase inherits from QueueItemEventBase. The order of isinstance checks matters! @@ -445,9 +448,37 @@ async def _handle_queue_event(self, event: FastAPIEvent[QueueEventBase]): f"Emitted private queue_items_retried event to user rooms {event_data.user_ids} and admin room" ) + # QueueClearedEvent: an unscoped clear (user_id=None — admin or single-user mode) + # deleted every user's items, so everyone gets the full event. A user-scoped clear + # only deleted that user's rows: full event to owner+admin (single emit to a room + # list so a socket in both rooms receives it once), sanitized companion to the rest + # of the queue room so their queue lists and badge counts refetch without treating + # the clear as their own. + elif isinstance(event_data, QueueClearedEvent): + if event_data.user_id is None: + await self._sio.emit( + event=event_name, data=event_data.model_dump(mode="json"), room=event_data.queue_id + ) + logger.debug(f"Emitted unscoped queue_cleared to all subscribers in queue {event_data.queue_id}") + else: + user_room = f"user:{event_data.user_id}" + await self._sio.emit( + event=event_name, data=event_data.model_dump(mode="json"), room=[user_room, "admin"] + ) + sanitized = event_data.model_copy(update={"user_id": "redacted"}) + await self._sio.emit( + event=event_name, + data=sanitized.model_dump(mode="json"), + room=event_data.queue_id, + skip_sid=self._owner_and_admin_sids(event_data.user_id), + ) + logger.debug( + f"Emitted queue_cleared: full to {user_room}+admin, sanitized to queue {event_data.queue_id}" + ) + else: - # For remaining queue events (e.g. QueueClearedEvent) that do not - # carry user identity, emit to all subscribers in the queue room. + # For remaining queue events that do not carry user identity, + # emit to all subscribers in the queue room. await self._sio.emit( event=event_name, data=event_data.model_dump(mode="json"), room=event_data.queue_id ) diff --git a/invokeai/app/services/events/events_base.py b/invokeai/app/services/events/events_base.py index 0e1f71c2bc7..65be86f8399 100644 --- a/invokeai/app/services/events/events_base.py +++ b/invokeai/app/services/events/events_base.py @@ -113,9 +113,9 @@ def emit_queue_items_retried( """Emitted when a list of queue items are retried""" self.dispatch(QueueItemsRetriedEvent.build(retry_result, user_ids, retried_item_ids_by_user)) - def emit_queue_cleared(self, queue_id: str) -> None: - """Emitted when a queue is cleared""" - self.dispatch(QueueClearedEvent.build(queue_id)) + def emit_queue_cleared(self, queue_id: str, user_id: str | None = None) -> None: + """Emitted when a queue is cleared. `user_id` scopes the clear to one user's items; None means all.""" + self.dispatch(QueueClearedEvent.build(queue_id, user_id)) def emit_recall_parameters_updated(self, queue_id: str, user_id: str, parameters: dict) -> None: """Emitted when recall parameters are updated""" diff --git a/invokeai/app/services/events/events_common.py b/invokeai/app/services/events/events_common.py index 005b0e5435f..15b48bb4665 100644 --- a/invokeai/app/services/events/events_common.py +++ b/invokeai/app/services/events/events_common.py @@ -331,9 +331,14 @@ class QueueClearedEvent(QueueEventBase): __event_name__ = "queue_cleared" + user_id: str | None = Field( + default=None, + description="The ID of the user whose queue items were cleared, or None if all users' items were cleared", + ) + @classmethod - def build(cls, queue_id: str) -> "QueueClearedEvent": - return cls(queue_id=queue_id) + def build(cls, queue_id: str, user_id: str | None = None) -> "QueueClearedEvent": + return cls(queue_id=queue_id, user_id=user_id) class WorkflowEventBase(EventBase): diff --git a/invokeai/app/services/session_queue/session_queue_sqlite.py b/invokeai/app/services/session_queue/session_queue_sqlite.py index 53079322899..51980d68f3b 100644 --- a/invokeai/app/services/session_queue/session_queue_sqlite.py +++ b/invokeai/app/services/session_queue/session_queue_sqlite.py @@ -524,7 +524,7 @@ def clear(self, queue_id: str, user_id: Optional[str] = None) -> ClearResult: """, tuple(params), ) - self.__invoker.services.events.emit_queue_cleared(queue_id) + self.__invoker.services.events.emit_queue_cleared(queue_id, user_id) return ClearResult(deleted=count) def delete_queue_items_by_id(self, item_ids: list[int]) -> None: diff --git a/invokeai/frontend/web/openapi.json b/invokeai/frontend/web/openapi.json index 52ca3ecfcd5..d66ec1f7a01 100644 --- a/invokeai/frontend/web/openapi.json +++ b/invokeai/frontend/web/openapi.json @@ -59604,9 +59604,22 @@ "description": "The ID of the queue", "title": "Queue Id", "type": "string" + }, + "user_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the user whose queue items were cleared, or None if all users' items were cleared", + "title": "User Id" } }, - "required": ["timestamp", "queue_id"], + "required": ["timestamp", "queue_id", "user_id"], "title": "QueueClearedEvent", "type": "object" }, diff --git a/invokeai/frontend/web/src/services/api/schema.ts b/invokeai/frontend/web/src/services/api/schema.ts index 3f0f07c07c2..2cec6cbaa8b 100644 --- a/invokeai/frontend/web/src/services/api/schema.ts +++ b/invokeai/frontend/web/src/services/api/schema.ts @@ -25206,6 +25206,12 @@ export type components = { * @description The ID of the queue */ queue_id: string; + /** + * User Id + * @description The ID of the user whose queue items were cleared, or None if all users' items were cleared + * @default null + */ + user_id: string | null; }; /** * QueueItemStatusChangedEvent diff --git a/invokeai/frontend/web/src/services/events/setEventListeners.tsx b/invokeai/frontend/web/src/services/events/setEventListeners.tsx index 11dabfc4601..ab16dedbaa5 100644 --- a/invokeai/frontend/web/src/services/events/setEventListeners.tsx +++ b/invokeai/frontend/web/src/services/events/setEventListeners.tsx @@ -4,6 +4,7 @@ import { socketConnected } from 'app/store/middleware/listenerMiddleware/listene import type { AppStore } from 'app/store/store'; import { parseify } from 'common/util/serialize'; import { isNil, round } from 'es-toolkit/compat'; +import { selectCurrentUser } from 'features/auth/store/authSlice'; import { getDefaultRefImageConfig } from 'features/controlLayers/hooks/addLayerHooks'; import { allEntitiesDeleted, controlLayerRecalled } from 'features/controlLayers/store/canvasSlice'; import { canvasWorkflowIntegrationProcessingCompleted } from 'features/controlLayers/store/canvasWorkflowIntegrationSlice'; @@ -71,6 +72,7 @@ export const setEventListeners = ({ socket, store, setIsConnected }: SetEventLis clearCanvasWorkflowIntegrationProcessing: () => dispatch(canvasWorkflowIntegrationProcessingCompleted()), completedInvocationKeysByItemId, getAllNodeExecutionStates: () => $nodeExecutionStates.get(), + getCurrentUserId: () => selectCurrentUser(getState())?.user_id ?? null, getNodeExecutionState: (nodeId) => $nodeExecutionStates.get()[nodeId], logReconciliationError: (error, itemId) => { log.debug({ error: parseify(error) }, `Unable to reconcile workflow queue item ${itemId}`); @@ -540,9 +542,13 @@ export const setEventListeners = ({ socket, store, setIsConnected }: SetEventLis log.debug({ data }, 'Queue cleared'); // Clearing the queue deletes the in-progress item without emitting a per-item terminal status // event, so the progress bar must be reset here — and the coordinator must mark its tracked - // items terminal so a trailing invocation_progress event cannot repopulate the bar. - workflowExecutionCoordinator.onQueueCleared(); - $lastProgressEvent.set(null); + // items terminal so a trailing invocation_progress event cannot repopulate the bar. The + // coordinator reports whether the clear applied to this client's items: a user-scoped clear + // by another user (multiuser mode) leaves them untouched, and only the queue tags below need + // refreshing. + if (workflowExecutionCoordinator.onQueueCleared(data)) { + $lastProgressEvent.set(null); + } dispatch( queueApi.util.invalidateTags([ 'SessionQueueStatus', diff --git a/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.test.ts b/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.test.ts index f3dcd178003..7aebda7de87 100644 --- a/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.test.ts +++ b/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.test.ts @@ -138,6 +138,13 @@ const buildInvocationErrorEvent = (overrides: Partial ...overrides, }) as S['InvocationErrorEvent']; +const buildQueueClearedEvent = (overrides: Partial = {}): S['QueueClearedEvent'] => + ({ + queue_id: 'default', + user_id: null, + ...overrides, + }) as S['QueueClearedEvent']; + const buildQueueItem = (status: S['SessionQueueItem']['status']): S['SessionQueueItem'] => ({ item_id: 1, @@ -165,7 +172,7 @@ const buildQueueItem = (status: S['SessionQueueItem']['status']): S['SessionQueu }, }) as unknown as S['SessionQueueItem']; -const createCoordinatorHarness = () => { +const createCoordinatorHarness = (currentUserId: string | null = 'user-1') => { const completedInvocationKeysByItemId = new Map>(); const nodeExecutionStates: Record = {}; const onInvocationComplete = vi.fn(); @@ -177,6 +184,7 @@ const createCoordinatorHarness = () => { clearCanvasWorkflowIntegrationProcessing, completedInvocationKeysByItemId, getAllNodeExecutionStates: () => nodeExecutionStates, + getCurrentUserId: () => currentUserId, getNodeExecutionState: (nodeId) => nodeExecutionStates[nodeId], logReconciliationError, onInvocationComplete, @@ -333,12 +341,53 @@ describe(createWorkflowExecutionCoordinator.name, () => { coordinator.onQueueItemStatusChanged( buildQueueStatusEvent({ item_id: 2, status: 'completed', origin: 'workflows' }) ); - coordinator.onQueueCleared(); + expect(coordinator.onQueueCleared(buildQueueClearedEvent({ user_id: null }))).toBe(true); + + expect(coordinator.onInvocationProgress(buildInvocationProgressEvent({ item_id: 1 }))).toBe(false); + expect(queueItemRequests.get(2)?.abort).toHaveBeenCalled(); + }); + + it('applies a queue clear scoped to the current user', () => { + const { coordinator, queueItemRequests } = createCoordinatorHarness('user-1'); + + coordinator.onQueueItemStatusChanged(buildQueueStatusEvent({ item_id: 1, status: 'in_progress', origin: null })); + expect(coordinator.onInvocationProgress(buildInvocationProgressEvent({ item_id: 1 }))).toBe(true); + coordinator.onQueueItemStatusChanged( + buildQueueStatusEvent({ item_id: 2, status: 'completed', origin: 'workflows' }) + ); + + expect(coordinator.onQueueCleared(buildQueueClearedEvent({ user_id: 'user-1' }))).toBe(true); expect(coordinator.onInvocationProgress(buildInvocationProgressEvent({ item_id: 1 }))).toBe(false); expect(queueItemRequests.get(2)?.abort).toHaveBeenCalled(); }); + it("does not disturb this user's items when another user's scoped clear is broadcast", async () => { + // In multiuser mode a user-scoped clear only deletes that user's rows, but the queue_cleared + // event is broadcast to every queue subscriber (sanitized to user_id="redacted" for + // non-owner, non-admin recipients). This client's items were not touched: its in-flight + // reconciliation must complete and its in-progress item must keep accepting events. + const { coordinator, nodeExecutionStates, queueItemRequests } = createCoordinatorHarness('user-b'); + + coordinator.onQueueItemStatusChanged( + buildQueueStatusEvent({ item_id: 1, status: 'completed', origin: 'workflows' }) + ); + coordinator.onQueueItemStatusChanged(buildQueueStatusEvent({ item_id: 2, status: 'in_progress', origin: null })); + expect(coordinator.onInvocationProgress(buildInvocationProgressEvent({ item_id: 2 }))).toBe(true); + + expect(coordinator.onQueueCleared(buildQueueClearedEvent({ user_id: 'redacted' }))).toBe(false); + + expect(queueItemRequests.get(1)?.abort).not.toHaveBeenCalled(); + expect(coordinator.onInvocationProgress(buildInvocationProgressEvent({ item_id: 2 }))).toBe(true); + + queueItemRequests.get(1)?.resolve(buildQueueItem('completed')); + await Promise.resolve(); + await Promise.resolve(); + + expect(nodeExecutionStates['node-1']?.status).toBe(zNodeStatus.enum.COMPLETED); + expect(nodeExecutionStates['node-1']?.outputs).toHaveLength(1); + }); + it('still clears canvas workflow integration processing on late invocation errors', () => { const { clearCanvasWorkflowIntegrationProcessing, coordinator } = createCoordinatorHarness(); diff --git a/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.ts b/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.ts index 8b85b055acf..1a96a6affdc 100644 --- a/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.ts +++ b/invokeai/frontend/web/src/services/events/workflowExecutionCoordinator.ts @@ -32,6 +32,7 @@ type WorkflowExecutionCoordinatorDeps = { clearCanvasWorkflowIntegrationProcessing: () => void; completedInvocationKeysByItemId: Map>; getAllNodeExecutionStates: () => Record; + getCurrentUserId: () => string | null; getNodeExecutionState: (nodeId: string) => NodeExecutionState | undefined; logReconciliationError: (error: unknown, itemId: number) => void; onInvocationComplete: (data: S['InvocationCompleteEvent']) => void; @@ -75,7 +76,16 @@ export const createWorkflowExecutionCoordinator = (deps: WorkflowExecutionCoordi pendingWorkflowReconciliationRequests.clear(); }; - const onQueueCleared = () => { + const onQueueCleared = (data: S['QueueClearedEvent']): boolean => { + // A user-scoped clear (multiuser mode) deletes only that user's queue items, but the event is + // broadcast to every queue subscriber (sanitized to user_id="redacted" for non-owner, + // non-admin recipients). Another user's clear leaves this client's items untouched, so it must + // not mark them canceled or abort their pending reconciliations. An unscoped clear + // (user_id=null — an admin or single-user clear) deletes everything and always applies. + const clearedUserId = data.user_id ?? null; + if (clearedUserId !== null && clearedUserId !== deps.getCurrentUserId()) { + return false; + } // Clearing the queue deletes its items without emitting per-item terminal status events, so the // tracked items must be marked terminal here for trailing invocation events to be rejected. cancelPendingWorkflowReconciliations(); @@ -86,6 +96,7 @@ export const createWorkflowExecutionCoordinator = (deps: WorkflowExecutionCoordi } workflowExecutionStates.set(itemId, { ...state, queueStatus: 'canceled' }); } + return true; }; const reconcileWorkflowQueueItemResults = (itemId: number, status: TerminalQueueStatus) => { diff --git a/tests/app/routers/test_multiuser_authorization.py b/tests/app/routers/test_multiuser_authorization.py index f1ae8defa1a..5a7e673b18f 100644 --- a/tests/app/routers/test_multiuser_authorization.py +++ b/tests/app/routers/test_multiuser_authorization.py @@ -2060,9 +2060,9 @@ def test_queue_items_retried_payload_is_filtered_per_owner_room(self, socketio: assert admin_calls[0].kwargs["data"]["user_ids"] == ["owner-123", "owner-456"] assert admin_calls[0].kwargs["data"]["retried_item_ids_by_user"] == {"owner-123": [10], "owner-456": [20]} - def test_queue_cleared_still_broadcast(self, socketio: Any) -> None: - """QueueClearedEvent does not carry user identity and should still be broadcast - to all queue subscribers — this is a sanity check that we haven't over-scoped.""" + def test_unscoped_queue_cleared_still_broadcast(self, socketio: Any) -> None: + """An unscoped QueueClearedEvent (user_id=None — an admin or single-user clear that + deleted every user's items) should still be broadcast to all queue subscribers.""" import asyncio from unittest.mock import AsyncMock @@ -2075,8 +2075,50 @@ def test_queue_cleared_still_broadcast(self, socketio: Any) -> None: asyncio.run(socketio._handle_queue_event(("queue_cleared", event))) - rooms_emitted_to = [call.kwargs.get("room") for call in mock_emit.call_args_list] - assert "default" in rooms_emitted_to + assert len(mock_emit.call_args_list) == 1 + assert mock_emit.call_args_list[0].kwargs.get("room") == "default" + assert mock_emit.call_args_list[0].kwargs.get("data")["user_id"] is None + + def test_user_scoped_queue_cleared_routed_privately(self, socketio: Any) -> None: + """A user-scoped QueueClearedEvent only deleted that user's rows. The full event must + go to the owner + admin rooms; the rest of the queue room gets a sanitized companion + (user_id="redacted") so their queue lists refresh without treating the clear as their + own — otherwise another user's clear would abort their in-flight reconciliations and + mark their tracked items canceled.""" + import asyncio + from unittest.mock import AsyncMock + + from invokeai.app.services.events.events_common import QueueClearedEvent + + event = QueueClearedEvent.build(queue_id="default", user_id="owner-xyz") + + socketio._socket_users["sid-owner"] = {"user_id": "owner-xyz", "is_admin": False} + socketio._socket_users["sid-admin"] = {"user_id": "admin-1", "is_admin": True} + socketio._socket_users["sid-other"] = {"user_id": "other-user", "is_admin": False} + + mock_emit = AsyncMock() + socketio._sio.emit = mock_emit + + asyncio.run(socketio._handle_queue_event(("queue_cleared", event))) + + emits = [ + (c.kwargs.get("room"), c.kwargs.get("data"), c.kwargs.get("skip_sid")) for c in mock_emit.call_args_list + ] + + # Full event goes to owner + admin rooms in a single emit (room list deduplicates a + # socket that is in both rooms) + private_emits = [(p, s) for r, p, s in emits if r == ["user:owner-xyz", "admin"]] + assert len(private_emits) == 1 + assert private_emits[0][0]["user_id"] == "owner-xyz" + + # Sanitized companion goes to the queue room, skipping the owner's and admins' sids + queue_emits = [(p, s) for r, p, s in emits if r == "default"] + assert len(queue_emits) == 1, "expected exactly one sanitized emit to queue room" + sanitized_payload, skip_sid = queue_emits[0] + assert sanitized_payload["user_id"] == "redacted" + assert "sid-owner" in skip_sid + assert "sid-admin" in skip_sid + assert "sid-other" not in skip_sid def test_recall_parameters_emitted_once_to_owner_and_admin_rooms(self, socketio: Any) -> None: """RecallParametersUpdatedEvent must be delivered to the owner + admin rooms