Merge pull request #1664 from denizekiz/VisionModel2

fix: Seperated imageModelProvider and imageVisionModelProvider for ImageDescriptioServices
elizaOS · Jan 2, 2025 · dac0c5b · dac0c5b
2 parents 3b7d9e3 + 3d824d9
commit dac0c5b
Show file tree

Hide file tree

Showing 5 changed files with 39 additions and 1 deletion.
diff --git a/docs/api/classes/AgentRuntime.md b/docs/api/classes/AgentRuntime.md
@@ -235,6 +235,13 @@ The model to use for generateText.
 
 The model to use for generateImage.
 
+
+### imageVisionModelProvider
+
+> **imageVisionModelProvider**: [`ModelProviderName`](../enumerations/ModelProviderName.md)
+
+The model to use for describing inputImages.
+
 #### Implementation of
 
 [`IAgentRuntime`](../interfaces/IAgentRuntime.md).[`imageModelProvider`](../interfaces/IAgentRuntime.md#imageModelProvider)

diff --git a/docs/api/type-aliases/Character.md b/docs/api/type-aliases/Character.md
@@ -44,6 +44,17 @@ Model provider to use
 
 Image model provider to use, if different from modelProvider
 
+
+
+### imageVisionModelProvider?
+
+> `optional` **imageVisionModelProvider**: [`ModelProviderName`](../enumerations/ModelProviderName.md)
+
+Image Vision model provider to use, if different from modelProvider
+
+imageVisionModelProvider
+
+
 ### modelEndpointOverride?
 
 > `optional` **modelEndpointOverride**: `string`

diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts
@@ -103,6 +103,12 @@ export class AgentRuntime implements IAgentRuntime {
      */
     imageModelProvider: ModelProviderName;
 
+
+     /**
+     * The model to use for describing images.
+     */
+    imageVisionModelProvider: ModelProviderName;
+
     /**
      * Fetch function to use
      * Some environments may not have access to the global fetch function and need a custom fetch override.
@@ -324,6 +330,16 @@ export class AgentRuntime implements IAgentRuntime {
             this.imageModelProvider
         );
 
+        this.imageVisionModelProvider =
+        this.character.imageVisionModelProvider ?? this.modelProvider;
+
+        elizaLogger.info("Selected model provider:", this.modelProvider);
+         elizaLogger.info(
+            "Selected image model provider:",
+            this.imageVisionModelProvider
+         );
+
+
         // Validate model provider
         if (!Object.values(ModelProviderName).includes(this.modelProvider)) {
             elizaLogger.error("Invalid model provider:", this.modelProvider);

diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts
@@ -687,6 +687,10 @@ export type Character = {
     /** Image model provider to use, if different from modelProvider */
     imageModelProvider?: ModelProviderName;
 
+
+    /** Image Vision model provider to use, if different from modelProvider */
+    imageVisionModelProvider?: ModelProviderName;
+
     /** Optional model endpoint override */
     modelEndpointOverride?: string;
 
@@ -1100,6 +1104,7 @@ export interface IAgentRuntime {
     token: string | null;
     modelProvider: ModelProviderName;
     imageModelProvider: ModelProviderName;
+    imageVisionModelProvider: ModelProviderName;
     character: Character;
     providers: Provider[];
     actions: Action[];

diff --git a/packages/plugin-node/src/services/image.ts b/packages/plugin-node/src/services/image.ts
@@ -211,7 +211,6 @@ export class ImageDescriptionService
                     this.runtime.imageModelProvider === ModelProviderName.OPENAI
                     ? models[this.runtime.imageModelProvider].endpoint
                     : "https://api.openai.com/v1";
-
                 const response = await fetch(endpoint + "/chat/completions", {
                     method: "POST",
                     headers: {