Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions conversations/namespaces/chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -702,8 +702,13 @@ async def send_callback(message: str):
try:
data = json.loads(message) if isinstance(message, str) else message
await sio.emit('message', data, room=f'conversation_{conv_id}', namespace='/chat')
except Exception as e:
logger.debug(f"Send callback failed (client may have disconnected): {e}")
except ConnectionError:
logger.debug("Client disconnected while emitting chat message")
except Exception:
logger.exception(
"Failed to emit chat message",
extra={"conversation_id": conv_id},
)

return send_callback

Expand Down
32 changes: 21 additions & 11 deletions conversations/services/message_coordinator.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,11 +121,10 @@ async def send(self, data: Dict[str, Any]):
if self.send_callback:
try:
await self.send_callback(json.dumps(camelize(data)))
except Exception as e:
# Log but don't raise - client may have disconnected
logger.debug(
f"Failed to send WebSocket message (client may have disconnected): {type(e).__name__}"
)
except ConnectionError:
logger.debug("Client disconnected while sending WebSocket message")
except Exception:
logger.exception("Failed to send WebSocket message")

async def send_error(
self, error_code: str, error_message: str, details: Optional[Dict] = None
Expand Down Expand Up @@ -815,9 +814,13 @@ async def stream_ai_response(
)
elif result.tool_calls_made:
# Edge case: tools ran but the model produced no text.
logger.warning(
logger.error(
"[MessageCoordinator] Tools ran but no response was generated; "
"finalizing with fallback message."
"finalizing with fallback message.",
extra={
"stream_failure": "empty_response",
"message_id": message_obj.id,
},
)
await self._finalize_message(
message_obj=message_obj,
Expand All @@ -832,9 +835,13 @@ async def stream_ai_response(
# Some provider streams can close cleanly without yielding a
# token or an exception. Treat that as a completed failure,
# otherwise the original empty placeholder remains forever.
logger.warning(
logger.error(
"[MessageCoordinator] Provider stream ended without text "
"or tool calls; finalizing with retry guidance."
"or tool calls; finalizing with retry guidance.",
extra={
"stream_failure": "empty_response",
"message_id": message_obj.id,
},
)
await self._finalize_message(
message_obj=message_obj,
Expand Down Expand Up @@ -1027,8 +1034,11 @@ def _save_usage_breakdown(
try:
message_obj.usage_details = usage_breakdown
message_obj.save(update_fields=["usage_details"])
except Exception as exc:
logger.warning("Failed to save usage breakdown: %s", exc)
except Exception:
logger.exception(
"Failed to save usage breakdown",
extra={"message_id": message_obj.id},
)

async def _finalize_message(
self,
Expand Down
13 changes: 7 additions & 6 deletions conversations/services/tool_loop_binding.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,9 @@
from channels.db import database_sync_to_async
from django.utils import timezone

from conversations.constants import (DEFAULT_AI_SENDER_NAME, SenderType,
ToolCallOrigin)
from conversations.constants import DEFAULT_AI_SENDER_NAME, SenderType, ToolCallOrigin
from conversations.models import MessageToolCall
from conversations.services.websocket_response_service import \
WebSocketResponseService
from conversations.services.websocket_response_service import WebSocketResponseService
from core.services.llm_helpers.retrieval_targets import ChatRetrievalTarget
from core.services.tool_loop.binding import ArtifactHost
from core.services.tool_loop.persistence import serialize_persisted_result
Expand Down Expand Up @@ -87,8 +85,11 @@ def save_tool_call(
executed_at=timezone.now(),
round_index=round_index,
)
except Exception as exc:
logger.error("Failed to save MessageToolCall: %s", exc)
except Exception:
logger.exception(
"Failed to save MessageToolCall",
extra={"message_id": self._message.id},
)


class ChatStreamSink:
Expand Down
13 changes: 12 additions & 1 deletion conversations/services/tool_loop_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -315,12 +315,23 @@ async def run(
# already read, usage already billed, and tool work already
# persisted must survive a stalled provider stream.
result.timed_out = True
logger.warning(
# Error-level logging creates a Sentry event even though the
# timeout is handled so partial output can be preserved.
logger.error(
"[ToolLoopService] %s (round %s, turn %s) — "
"finishing with the partial turn",
exc,
round_index,
turn_key,
extra={
"stream_failure": "timeout",
"model": prepared.llm.identifier,
"provider": prepared.llm.provider,
"turn_id": turn_key,
"round_index": round_index,
"idle_timeout_seconds": self.stream_idle_timeout_seconds,
"has_partial_text": bool(text_accum),
},
)
break
except asyncio.CancelledError:
Expand Down
22 changes: 16 additions & 6 deletions conversations/services/web_search_source_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
"""

import logging
from typing import List, Dict, Any
from typing import Any, Dict, List

from channels.db import database_sync_to_async

from conversations.models import Message, WebSearchSource
Expand Down Expand Up @@ -77,12 +78,17 @@ def save_sources(
provider=provider[:20],
)
saved_count += 1
except Exception as e:
logger.warning(f"Failed to save web search source {url}: {e}")
except Exception:
logger.exception(
"Failed to save web search source",
extra={"message_id": message.id},
)
continue

if saved_count > 0:
logger.info(f"Saved {saved_count} web search sources for message {message.id}")
logger.info(
f"Saved {saved_count} web search sources for message {message.id}"
)

return saved_count

Expand All @@ -100,7 +106,11 @@ def delete_sources_for_message(message: Message) -> int:
Returns:
Number of sources deleted
"""
deleted_count, _ = WebSearchSource.active_objects.filter(message=message).delete()
deleted_count, _ = WebSearchSource.active_objects.filter(
message=message
).delete()
if deleted_count > 0:
logger.info(f"Deleted {deleted_count} web search sources for message {message.id}")
logger.info(
f"Deleted {deleted_count} web search sources for message {message.id}"
)
return deleted_count
125 changes: 125 additions & 0 deletions conversations/test_chat_secondary_reporting.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
"""Secondary failures remain recoverable and emit diagnosable Sentry events."""

from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock, patch

import sentry_sdk
from django.test import SimpleTestCase
from sentry_sdk.integrations.logging import LoggingIntegration

from conversations.namespaces.chat import ChatNamespace
from conversations.services.message_coordinator import MessageCoordinator
from conversations.services.tool_loop_binding import ChatToolLoopStore
from conversations.services.web_search_source_service import WebSearchSourceService
from core.services.claude_service import ClaudeService
from core.services.custom_llm_service import CustomLLMService
from core.services.gemini_service import GeminiService
from core.services.openai_service import OpenAIService


class SecondaryFailureReportingTests(SimpleTestCase):
def setUp(self):
self.events = []
self.client = sentry_sdk.Client(
dsn="https://public@example.com/1",
transport=self.events.append,
default_integrations=False,
integrations=[LoggingIntegration()],
)
self.scope = sentry_sdk.new_scope()
self.scope.__enter__().set_client(self.client)
self.addCleanup(self.client.close)
self.addCleanup(self.scope.__exit__, None, None, None)

def assert_exception_reported(self):
self.assertEqual(len(self.events), 1)
event = self.events.pop()
self.assertEqual(event["level"], "error")
self.assertEqual(event["exception"]["values"][-1]["type"], "RuntimeError")
self.assertIn("stacktrace", event["exception"]["values"][-1])
return event

async def test_usage_save_failure_is_reported_without_interrupting_chat(self):
coordinator = MessageCoordinator.__new__(MessageCoordinator)
message = SimpleNamespace(
id=42, save=Mock(side_effect=RuntimeError("DB failure"))
)
await coordinator._save_usage_breakdown(message, [{"round_index": 1}])
self.assertEqual(self.assert_exception_reported()["extra"]["message_id"], 42)

async def test_citation_save_failure_is_reported(self):
with patch(
"conversations.services.web_search_source_service.WebSearchSource.active_objects.create",
side_effect=RuntimeError("DB failure"),
):
count = await WebSearchSourceService.save_sources(
SimpleNamespace(id=42), [{"url": "https://example.com"}]
)
self.assertEqual(count, 0)
self.assert_exception_reported()

async def test_provider_cleanup_failures_are_reported(self):
for provider in (GeminiService, ClaudeService, OpenAIService, CustomLLMService):
with self.subTest(provider=provider.__name__):
service = provider.__new__(provider)
client = SimpleNamespace(
close=AsyncMock(side_effect=RuntimeError("Close failure")),
aio=SimpleNamespace(
aclose=AsyncMock(side_effect=RuntimeError("Close failure"))
),
)
if provider is CustomLLMService:
service.client = client
else:
service._client = client
await service.close()
self.assert_exception_reported()

async def test_unexpected_send_failure_is_reported_but_disconnect_is_not(self):
coordinator = MessageCoordinator.__new__(MessageCoordinator)
for error in (RuntimeError("Send failure"), ConnectionError("Disconnected")):
coordinator.send_callback = AsyncMock(side_effect=error)
await coordinator.send({"type": "message"})
if isinstance(error, ConnectionError):
self.assertEqual(self.events, [])
else:
self.assert_exception_reported()

async def test_socket_emit_failure_is_reported_once(self):
callback = ChatNamespace()._create_send_callback("socket", "conversation")
coordinator = MessageCoordinator.__new__(MessageCoordinator)
coordinator.send_callback = callback
with patch(
"conversations.namespaces.chat.sio.emit",
side_effect=RuntimeError("Emit failure"),
):
await coordinator.send({"type": "message"})
self.assert_exception_reported()

async def test_tool_call_save_failure_includes_traceback(self):
store = ChatToolLoopStore(SimpleNamespace(id=42))
with patch(
"conversations.services.tool_loop_binding.MessageToolCall.objects.create",
side_effect=RuntimeError("DB failure"),
):
await store.save_tool_call(
call=SimpleNamespace(id="call", name="tool"),
server_slug="server",
origin="mcp",
arguments={},
raw_result={},
is_error=False,
error="",
round_index=1,
execution_time_ms=1,
)
self.assert_exception_reported()

async def test_gemini_stream_failure_includes_traceback_and_recovery_text(self):
service = GeminiService.__new__(GeminiService)
service._prepare_messages = Mock(return_value=[])
service._create_stream = AsyncMock(side_effect=RuntimeError("Provider failure"))
chunks = [chunk async for chunk in service.stream_chat_completion([])]
self.assertEqual(len(chunks), 1)
self.assertTrue(chunks[0].text.startswith("Error:"))
self.assert_exception_reported()
Loading
Loading