feat: add error handling for bedrock on server (#698)

Co-authored-by: Mindy Long <[email protected]>

letta/errors.py

@@ -62,6 +62,20 @@ class LLMError(LettaError):
     pass
 
 
+class BedrockPermissionError(LettaError):
+    """Exception raised for errors in the Bedrock permission process."""
+
+    def __init__(self, message="User does not have access to the Bedrock model with the specified ID."):
+        super().__init__(message=message)
+
+
+class BedrockError(LettaError):
+    """Exception raised for errors in the Bedrock process."""
+
+    def __init__(self, message="Error with Bedrock model."):
+        super().__init__(message=message)
+
+
 class LLMJSONParsingError(LettaError):
     """Exception raised for errors in the JSON parsing process."""
 

letta/llm_api/anthropic.py

@@ -3,7 +3,9 @@
 from typing import List, Optional, Tuple, Union
 
 import anthropic
+from anthropic import PermissionDeniedError
 
+from letta.errors import BedrockError, BedrockPermissionError
 from letta.llm_api.aws_bedrock import get_bedrock_client
 from letta.schemas.message import Message
 from letta.schemas.openai.chat_completion_request import ChatCompletionRequest, Tool
@@ -414,5 +416,10 @@ def anthropic_bedrock_chat_completions_request(
     client = get_bedrock_client()
 
     # Make the request
-    response = client.messages.create(**data)
-    return convert_anthropic_response_to_chatcompletion(response=response, inner_thoughts_xml_tag=inner_thoughts_xml_tag)
+    try:
+        response = client.messages.create(**data)
+        return convert_anthropic_response_to_chatcompletion(response=response, inner_thoughts_xml_tag=inner_thoughts_xml_tag)
+    except PermissionDeniedError:
+        raise BedrockPermissionError(f"User does not have access to the Bedrock model with the specified ID. {data['model']}")
+    except Exception as e:
+        raise BedrockError(f"Bedrock error: {e}")

letta/llm_api/aws_bedrock.py

@@ -1,5 +1,5 @@
 import os
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List
 
 from anthropic import AnthropicBedrock
 
@@ -37,7 +37,7 @@ def get_bedrock_client():
     return bedrock
 
 
-def bedrock_get_model_list(region_name: str, model_provider: Optional[str] = None, output_modality: str = "TEXT") -> List[dict]:
+def bedrock_get_model_list(region_name: str) -> List[dict]:
     """
     Get list of available models from Bedrock.
 
@@ -53,8 +53,8 @@ def bedrock_get_model_list(region_name: str, model_provider: Optional[str] = Non
 
     try:
         bedrock = boto3.client("bedrock", region_name=region_name)
-        response = bedrock.list_foundation_models(byProvider=model_provider, byOutputModality=output_modality.upper())
-        return response["modelSummaries"]
+        response = bedrock.list_inference_profiles()
+        return response["inferenceProfileSummaries"]
     except Exception as e:
         print(f"Error getting model list: {str(e)}")
         raise e

letta/schemas/providers.py

@@ -708,39 +708,24 @@ class AnthropicBedrockProvider(Provider):
     def list_llm_models(self):
         from letta.llm_api.aws_bedrock import bedrock_get_model_list
 
-        models = bedrock_get_model_list(self.aws_region, model_provider="anthropic")
+        models = bedrock_get_model_list(self.aws_region)
 
         configs = []
         for model_summary in models:
-            model_id = model_summary["modelId"]
+            model_arn = model_summary["inferenceProfileArn"]
             configs.append(
                 LLMConfig(
-                    model=model_id,
+                    model=model_arn,
                     model_endpoint_type=self.name,
                     model_endpoint=None,
-                    context_window=self.get_model_context_window(model_id),
+                    context_window=self.get_model_context_window(model_arn),
+                    handle=self.get_handle(model_arn),
                 )
             )
         return configs
 
     def list_embedding_models(self):
-        from letta.llm_api.aws_bedrock import bedrock_get_model_list
-
-        # Will return nothing
-        models = bedrock_get_model_list(self.aws_region, model_provider="anthropic", output_modality="EMBEDDING")
-
-        configs = []
-        for model_summary in models:
-            model_id = model_summary["modelId"]
-            configs.append(
-                EmbeddingConfig(
-                    model=model_id,
-                    model_endpoint_type=self.name,
-                    model_endpoint=None,
-                    context_window=self.get_model_context_window(model_id),
-                )
-            )
-        return configs
+        return []
 
     def get_model_context_window(self, model_name: str) -> Optional[int]:
         # Context windows for Claude models

letta/server/rest_api/app.py

@@ -13,7 +13,7 @@
 
 from letta.__init__ import __version__
 from letta.constants import ADMIN_PREFIX, API_PREFIX, OPENAI_API_PREFIX
-from letta.errors import LettaAgentNotFoundError, LettaUserNotFoundError
+from letta.errors import BedrockPermissionError, LettaAgentNotFoundError, LettaUserNotFoundError
 from letta.log import get_logger
 from letta.orm.errors import DatabaseTimeoutError, ForeignKeyConstraintViolationError, NoResultFound, UniqueConstraintViolationError
 from letta.schemas.letta_message import create_letta_message_union_schema
@@ -208,6 +208,19 @@ async def agent_not_found_handler(request: Request, exc: LettaAgentNotFoundError
     async def user_not_found_handler(request: Request, exc: LettaUserNotFoundError):
         return JSONResponse(status_code=404, content={"detail": "User not found"})
 
+    @app.exception_handler(BedrockPermissionError)
+    async def bedrock_permission_error_handler(request, exc: BedrockPermissionError):
+        return JSONResponse(
+            status_code=403,
+            content={
+                "error": {
+                    "type": "bedrock_permission_denied",
+                    "message": "Unable to access the required AI model. Please check your Bedrock permissions or contact support.",
+                    "details": {"model_arn": exc.model_arn, "reason": str(exc)},
+                }
+            },
+        )
+
     settings.cors_origins.append("https://app.letta.com")
 
     if (os.getenv("LETTA_SERVER_SECURE") == "true") or "--secure" in sys.argv: