From fff51a95cf8b087445350e54adae8bab1e931e86 Mon Sep 17 00:00:00 2001 From: Matthew Khouzam Date: Fri, 8 Nov 2024 08:44:25 -0500 Subject: [PATCH 1/2] feat: offer 'disableStreaming' for OpenAI models Custom OpenAI models can now be configured with 'disableStreaming: true' to indicate that streaming shall not be used. This is especially useful for models which do not support streaming at all. Co-authored-by: Matthew Khouzam --- .../openai-frontend-application-contribution.ts | 17 +++++++++++++---- .../ai-openai/src/browser/openai-preferences.ts | 6 ++++++ .../common/openai-language-models-manager.ts | 4 ++++ .../ai-openai/src/node/openai-language-model.ts | 10 ++++++---- .../node/openai-language-models-manager-impl.ts | 5 ++++- 5 files changed, 33 insertions(+), 9 deletions(-) diff --git a/packages/ai-openai/src/browser/openai-frontend-application-contribution.ts b/packages/ai-openai/src/browser/openai-frontend-application-contribution.ts index b16f80aa0ef90..7129244499afb 100644 --- a/packages/ai-openai/src/browser/openai-frontend-application-contribution.ts +++ b/packages/ai-openai/src/browser/openai-frontend-application-contribution.ts @@ -63,8 +63,13 @@ export class OpenAiFrontendApplicationContribution implements FrontendApplicatio const newModels = createCustomModelDescriptionsFromPreferences(event.newValue); const modelsToRemove = oldModels.filter(model => !newModels.some(newModel => newModel.id === model.id)); - const modelsToAddOrUpdate = newModels.filter(newModel => !oldModels.some(model => - model.id === newModel.id && model.model === newModel.model && model.url === newModel.url && model.apiKey === newModel.apiKey)); + const modelsToAddOrUpdate = newModels.filter(newModel => + !oldModels.some(model => + model.id === newModel.id && + model.model === newModel.model && + model.url === newModel.url && + model.apiKey === newModel.apiKey && + model.disableStreaming === newModel.disableStreaming)); this.manager.removeLanguageModels(...modelsToRemove.map(model => model.id)); this.manager.createOrUpdateLanguageModels(...modelsToAddOrUpdate); @@ -74,11 +79,14 @@ export class OpenAiFrontendApplicationContribution implements FrontendApplicatio } } +const openAIModelsWithoutStreaming = ['o1-preview']; + function createOpenAIModelDescription(modelId: string): OpenAiModelDescription { return { id: `openai/${modelId}`, model: modelId, - apiKey: true + apiKey: true, + disableStreaming: openAIModelsWithoutStreaming.includes(modelId) }; } @@ -93,7 +101,8 @@ function createCustomModelDescriptionsFromPreferences(preferences: Partial string | undefined, public url: string | undefined) { } + constructor(public readonly id: string, public model: string, public disableStreaming: boolean, public apiKey: () => string | undefined, public url: string | undefined) { } async request(request: LanguageModelRequest, cancellationToken?: CancellationToken): Promise { const openai = this.initializeOpenAi(); @@ -152,8 +154,8 @@ export class OpenAiModel implements LanguageModel { }; } - protected isNonStreamingModel(model: string): boolean { - return ['o1-preview'].includes(model); + protected isNonStreamingModel(_model: string): boolean { + return this.disableStreaming; } protected supportsStructuredOutput(): boolean { diff --git a/packages/ai-openai/src/node/openai-language-models-manager-impl.ts b/packages/ai-openai/src/node/openai-language-models-manager-impl.ts index cfc81ba3b8adb..942a4c83cbdc4 100644 --- a/packages/ai-openai/src/node/openai-language-models-manager-impl.ts +++ b/packages/ai-openai/src/node/openai-language-models-manager-impl.ts @@ -57,9 +57,12 @@ export class OpenAiLanguageModelsManagerImpl implements OpenAiLanguageModelsMana } model.url = modelDescription.url; model.model = modelDescription.model; + model.disableStreaming = modelDescription.disableStreaming; model.apiKey = apiKeyProvider; } else { - this.languageModelRegistry.addLanguageModels([new OpenAiModel(modelDescription.id, modelDescription.model, apiKeyProvider, modelDescription.url)]); + this.languageModelRegistry.addLanguageModels([ + new OpenAiModel(modelDescription.id, modelDescription.model, modelDescription.disableStreaming, apiKeyProvider, modelDescription.url) + ]); } } } From d63f83a1bdcf39cdfa805d6c9159a20a4461c38c Mon Sep 17 00:00:00 2001 From: Stefan Dirix Date: Fri, 22 Nov 2024 17:21:45 +0100 Subject: [PATCH 2/2] Switch disableStreaming with enableStreaming --- .../browser/openai-frontend-application-contribution.ts | 8 ++++---- packages/ai-openai/src/browser/openai-preferences.ts | 6 +++--- .../src/common/openai-language-models-manager.ts | 2 +- packages/ai-openai/src/node/openai-language-model.ts | 6 +++--- .../src/node/openai-language-models-manager-impl.ts | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/ai-openai/src/browser/openai-frontend-application-contribution.ts b/packages/ai-openai/src/browser/openai-frontend-application-contribution.ts index 7129244499afb..3f917c77ed1ee 100644 --- a/packages/ai-openai/src/browser/openai-frontend-application-contribution.ts +++ b/packages/ai-openai/src/browser/openai-frontend-application-contribution.ts @@ -69,7 +69,7 @@ export class OpenAiFrontendApplicationContribution implements FrontendApplicatio model.model === newModel.model && model.url === newModel.url && model.apiKey === newModel.apiKey && - model.disableStreaming === newModel.disableStreaming)); + model.enableStreaming === newModel.enableStreaming)); this.manager.removeLanguageModels(...modelsToRemove.map(model => model.id)); this.manager.createOrUpdateLanguageModels(...modelsToAddOrUpdate); @@ -79,14 +79,14 @@ export class OpenAiFrontendApplicationContribution implements FrontendApplicatio } } -const openAIModelsWithoutStreaming = ['o1-preview']; +const openAIModelsWithDisabledStreaming = ['o1-preview']; function createOpenAIModelDescription(modelId: string): OpenAiModelDescription { return { id: `openai/${modelId}`, model: modelId, apiKey: true, - disableStreaming: openAIModelsWithoutStreaming.includes(modelId) + enableStreaming: !openAIModelsWithDisabledStreaming.includes(modelId) }; } @@ -102,7 +102,7 @@ function createCustomModelDescriptionsFromPreferences(preferences: Partial string | undefined, public url: string | undefined) { } + constructor(public readonly id: string, public model: string, public enableStreaming: boolean, public apiKey: () => string | undefined, public url: string | undefined) { } async request(request: LanguageModelRequest, cancellationToken?: CancellationToken): Promise { const openai = this.initializeOpenAi(); @@ -155,7 +155,7 @@ export class OpenAiModel implements LanguageModel { } protected isNonStreamingModel(_model: string): boolean { - return this.disableStreaming; + return !this.enableStreaming; } protected supportsStructuredOutput(): boolean { diff --git a/packages/ai-openai/src/node/openai-language-models-manager-impl.ts b/packages/ai-openai/src/node/openai-language-models-manager-impl.ts index 942a4c83cbdc4..4ccf77b0cc9f9 100644 --- a/packages/ai-openai/src/node/openai-language-models-manager-impl.ts +++ b/packages/ai-openai/src/node/openai-language-models-manager-impl.ts @@ -57,11 +57,11 @@ export class OpenAiLanguageModelsManagerImpl implements OpenAiLanguageModelsMana } model.url = modelDescription.url; model.model = modelDescription.model; - model.disableStreaming = modelDescription.disableStreaming; + model.enableStreaming = modelDescription.enableStreaming; model.apiKey = apiKeyProvider; } else { this.languageModelRegistry.addLanguageModels([ - new OpenAiModel(modelDescription.id, modelDescription.model, modelDescription.disableStreaming, apiKeyProvider, modelDescription.url) + new OpenAiModel(modelDescription.id, modelDescription.model, modelDescription.enableStreaming, apiKeyProvider, modelDescription.url) ]); } }