From 393e22d09f42748a02435abf864d3ccbf6ef1646 Mon Sep 17 00:00:00 2001 From: mlong93 <35275280+mlong93@users.noreply.github.com> Date: Fri, 17 Jan 2025 17:43:33 -0800 Subject: [PATCH] feat: add error handling for bedrock on server (#698) Co-authored-by: Mindy Long --- letta/errors.py | 14 ++++++++++++++ letta/llm_api/anthropic.py | 11 +++++++++-- letta/llm_api/aws_bedrock.py | 8 ++++---- letta/schemas/providers.py | 27 ++++++--------------------- letta/server/rest_api/app.py | 15 ++++++++++++++- 5 files changed, 47 insertions(+), 28 deletions(-) diff --git a/letta/errors.py b/letta/errors.py index 2c4703c0dd..818daebf01 100644 --- a/letta/errors.py +++ b/letta/errors.py @@ -62,6 +62,20 @@ class LLMError(LettaError): pass +class BedrockPermissionError(LettaError): + """Exception raised for errors in the Bedrock permission process.""" + + def __init__(self, message="User does not have access to the Bedrock model with the specified ID."): + super().__init__(message=message) + + +class BedrockError(LettaError): + """Exception raised for errors in the Bedrock process.""" + + def __init__(self, message="Error with Bedrock model."): + super().__init__(message=message) + + class LLMJSONParsingError(LettaError): """Exception raised for errors in the JSON parsing process.""" diff --git a/letta/llm_api/anthropic.py b/letta/llm_api/anthropic.py index 3fd197003c..87adfc5ad4 100644 --- a/letta/llm_api/anthropic.py +++ b/letta/llm_api/anthropic.py @@ -3,7 +3,9 @@ from typing import List, Optional, Tuple, Union import anthropic +from anthropic import PermissionDeniedError +from letta.errors import BedrockError, BedrockPermissionError from letta.llm_api.aws_bedrock import get_bedrock_client from letta.schemas.message import Message from letta.schemas.openai.chat_completion_request import ChatCompletionRequest, Tool @@ -414,5 +416,10 @@ def anthropic_bedrock_chat_completions_request( client = get_bedrock_client() # Make the request - response = client.messages.create(**data) - return convert_anthropic_response_to_chatcompletion(response=response, inner_thoughts_xml_tag=inner_thoughts_xml_tag) + try: + response = client.messages.create(**data) + return convert_anthropic_response_to_chatcompletion(response=response, inner_thoughts_xml_tag=inner_thoughts_xml_tag) + except PermissionDeniedError: + raise BedrockPermissionError(f"User does not have access to the Bedrock model with the specified ID. {data['model']}") + except Exception as e: + raise BedrockError(f"Bedrock error: {e}") diff --git a/letta/llm_api/aws_bedrock.py b/letta/llm_api/aws_bedrock.py index a527f917bf..b3077e4199 100644 --- a/letta/llm_api/aws_bedrock.py +++ b/letta/llm_api/aws_bedrock.py @@ -1,5 +1,5 @@ import os -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List from anthropic import AnthropicBedrock @@ -37,7 +37,7 @@ def get_bedrock_client(): return bedrock -def bedrock_get_model_list(region_name: str, model_provider: Optional[str] = None, output_modality: str = "TEXT") -> List[dict]: +def bedrock_get_model_list(region_name: str) -> List[dict]: """ Get list of available models from Bedrock. @@ -53,8 +53,8 @@ def bedrock_get_model_list(region_name: str, model_provider: Optional[str] = Non try: bedrock = boto3.client("bedrock", region_name=region_name) - response = bedrock.list_foundation_models(byProvider=model_provider, byOutputModality=output_modality.upper()) - return response["modelSummaries"] + response = bedrock.list_inference_profiles() + return response["inferenceProfileSummaries"] except Exception as e: print(f"Error getting model list: {str(e)}") raise e diff --git a/letta/schemas/providers.py b/letta/schemas/providers.py index ce05e22ac8..b57343112c 100644 --- a/letta/schemas/providers.py +++ b/letta/schemas/providers.py @@ -708,39 +708,24 @@ class AnthropicBedrockProvider(Provider): def list_llm_models(self): from letta.llm_api.aws_bedrock import bedrock_get_model_list - models = bedrock_get_model_list(self.aws_region, model_provider="anthropic") + models = bedrock_get_model_list(self.aws_region) configs = [] for model_summary in models: - model_id = model_summary["modelId"] + model_arn = model_summary["inferenceProfileArn"] configs.append( LLMConfig( - model=model_id, + model=model_arn, model_endpoint_type=self.name, model_endpoint=None, - context_window=self.get_model_context_window(model_id), + context_window=self.get_model_context_window(model_arn), + handle=self.get_handle(model_arn), ) ) return configs def list_embedding_models(self): - from letta.llm_api.aws_bedrock import bedrock_get_model_list - - # Will return nothing - models = bedrock_get_model_list(self.aws_region, model_provider="anthropic", output_modality="EMBEDDING") - - configs = [] - for model_summary in models: - model_id = model_summary["modelId"] - configs.append( - EmbeddingConfig( - model=model_id, - model_endpoint_type=self.name, - model_endpoint=None, - context_window=self.get_model_context_window(model_id), - ) - ) - return configs + return [] def get_model_context_window(self, model_name: str) -> Optional[int]: # Context windows for Claude models diff --git a/letta/server/rest_api/app.py b/letta/server/rest_api/app.py index b4bd35eca5..316bf0ad14 100644 --- a/letta/server/rest_api/app.py +++ b/letta/server/rest_api/app.py @@ -13,7 +13,7 @@ from letta.__init__ import __version__ from letta.constants import ADMIN_PREFIX, API_PREFIX, OPENAI_API_PREFIX -from letta.errors import LettaAgentNotFoundError, LettaUserNotFoundError +from letta.errors import BedrockPermissionError, LettaAgentNotFoundError, LettaUserNotFoundError from letta.log import get_logger from letta.orm.errors import DatabaseTimeoutError, ForeignKeyConstraintViolationError, NoResultFound, UniqueConstraintViolationError from letta.schemas.letta_message import create_letta_message_union_schema @@ -208,6 +208,19 @@ async def agent_not_found_handler(request: Request, exc: LettaAgentNotFoundError async def user_not_found_handler(request: Request, exc: LettaUserNotFoundError): return JSONResponse(status_code=404, content={"detail": "User not found"}) + @app.exception_handler(BedrockPermissionError) + async def bedrock_permission_error_handler(request, exc: BedrockPermissionError): + return JSONResponse( + status_code=403, + content={ + "error": { + "type": "bedrock_permission_denied", + "message": "Unable to access the required AI model. Please check your Bedrock permissions or contact support.", + "details": {"model_arn": exc.model_arn, "reason": str(exc)}, + } + }, + ) + settings.cors_origins.append("https://app.letta.com") if (os.getenv("LETTA_SERVER_SECURE") == "true") or "--secure" in sys.argv: