diff --git a/.changeset/slow-llamas-read.md b/.changeset/slow-llamas-read.md new file mode 100644 index 000000000..99d824b84 --- /dev/null +++ b/.changeset/slow-llamas-read.md @@ -0,0 +1,9 @@ +--- +'@livekit/agents': patch +'@livekit/agents-plugin-baseten': patch +'@livekit/agents-plugin-google': patch +'@livekit/agents-plugin-mistralai': patch +'@livekit/agents-plugin-openai': patch +--- + +fix(llm): raise the default streaming request timeout to 30s while keeping it configurable through connection options diff --git a/agents/src/inference/llm.ts b/agents/src/inference/llm.ts index 3434e496c..a12b44106 100644 --- a/agents/src/inference/llm.ts +++ b/agents/src/inference/llm.ts @@ -5,7 +5,7 @@ import OpenAI from 'openai'; import { APIConnectionError, APIStatusError, APITimeoutError } from '../_exceptions.js'; import * as llm from '../llm/index.js'; import type { APIConnectOptions } from '../types.js'; -import { DEFAULT_API_CONNECT_OPTIONS } from '../types.js'; +import { DEFAULT_LLM_API_CONNECT_OPTIONS } from '../types.js'; import { type Expand, toError } from '../utils.js'; import { type AnyString, @@ -250,7 +250,7 @@ export class LLM extends llm.LLM { chat({ chatCtx, toolCtx, - connOptions = DEFAULT_API_CONNECT_OPTIONS, + connOptions = DEFAULT_LLM_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, inferenceClass, diff --git a/agents/src/types.ts b/agents/src/types.ts index c66c95cf9..1454596a4 100644 --- a/agents/src/types.ts +++ b/agents/src/types.ts @@ -25,6 +25,11 @@ export const DEFAULT_API_CONNECT_OPTIONS: APIConnectOptions = { timeoutMs: 10000, }; +export const DEFAULT_LLM_API_CONNECT_OPTIONS: APIConnectOptions = { + ...DEFAULT_API_CONNECT_OPTIONS, + timeoutMs: 30000, +}; + /** * Return the interval for the given number of retries. * The first retry is immediate, and then uses specified retryIntervalMs. @@ -65,7 +70,7 @@ export interface ResolvedSessionConnectOptions { export const DEFAULT_SESSION_CONNECT_OPTIONS: ResolvedSessionConnectOptions = { sttConnOptions: DEFAULT_API_CONNECT_OPTIONS, - llmConnOptions: DEFAULT_API_CONNECT_OPTIONS, + llmConnOptions: DEFAULT_LLM_API_CONNECT_OPTIONS, ttsConnOptions: DEFAULT_API_CONNECT_OPTIONS, maxUnrecoverableErrors: 3, }; diff --git a/agents/src/voice/testing/fake_llm.ts b/agents/src/voice/testing/fake_llm.ts index ad3a1bf16..be36bcf2f 100644 --- a/agents/src/voice/testing/fake_llm.ts +++ b/agents/src/voice/testing/fake_llm.ts @@ -5,7 +5,7 @@ import type { ChatContext } from '../../llm/chat_context.js'; import { FunctionCall } from '../../llm/chat_context.js'; import { LLMStream as BaseLLMStream, LLM, type LLMStream } from '../../llm/llm.js'; import type { ToolChoice, ToolContext } from '../../llm/tool_context.js'; -import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../../types.js'; +import { type APIConnectOptions, DEFAULT_LLM_API_CONNECT_OPTIONS } from '../../types.js'; import { delay } from '../../utils.js'; export interface FakeLLMResponse { @@ -39,7 +39,7 @@ export class FakeLLM extends LLM { chat({ chatCtx, toolCtx, - connOptions = DEFAULT_API_CONNECT_OPTIONS, + connOptions = DEFAULT_LLM_API_CONNECT_OPTIONS, }: { chatCtx: ChatContext; toolCtx?: ToolContext; diff --git a/plugins/baseten/src/llm.ts b/plugins/baseten/src/llm.ts index 4039b7cac..7aac551f2 100644 --- a/plugins/baseten/src/llm.ts +++ b/plugins/baseten/src/llm.ts @@ -7,7 +7,7 @@ * Configures the OpenAI plugin to work with Baseten's OpenAI-compatible API */ import type { APIConnectOptions } from '@livekit/agents'; -import { DEFAULT_API_CONNECT_OPTIONS, inference, llm } from '@livekit/agents'; +import { DEFAULT_LLM_API_CONNECT_OPTIONS, inference, llm } from '@livekit/agents'; import { OpenAI } from 'openai'; import type { BasetenLLMOptions } from './types.js'; @@ -73,7 +73,7 @@ export class OpenAILLM extends llm.LLM { chat({ chatCtx, toolCtx, - connOptions = DEFAULT_API_CONNECT_OPTIONS, + connOptions = DEFAULT_LLM_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, extraKwargs, diff --git a/plugins/google/src/llm.ts b/plugins/google/src/llm.ts index ebd8b1f42..25c1f9aa9 100644 --- a/plugins/google/src/llm.ts +++ b/plugins/google/src/llm.ts @@ -7,7 +7,7 @@ import type { APIConnectOptions } from '@livekit/agents'; import { APIConnectionError, APIStatusError, - DEFAULT_API_CONNECT_OPTIONS, + DEFAULT_LLM_API_CONNECT_OPTIONS, llm, shortuuid, } from '@livekit/agents'; @@ -186,7 +186,7 @@ export class LLM extends llm.LLM { chat({ chatCtx, toolCtx, - connOptions = DEFAULT_API_CONNECT_OPTIONS, + connOptions = DEFAULT_LLM_API_CONNECT_OPTIONS, toolChoice, extraKwargs, geminiTools, diff --git a/plugins/mistralai/src/llm.ts b/plugins/mistralai/src/llm.ts index f0c07f7a2..c5de7caab 100644 --- a/plugins/mistralai/src/llm.ts +++ b/plugins/mistralai/src/llm.ts @@ -5,7 +5,7 @@ import type { APIConnectOptions } from '@livekit/agents'; import { APIConnectionError, APIStatusError, - DEFAULT_API_CONNECT_OPTIONS, + DEFAULT_LLM_API_CONNECT_OPTIONS, llm, shortuuid, } from '@livekit/agents'; @@ -118,7 +118,7 @@ export class LLM extends llm.LLM { chat({ chatCtx, toolCtx, - connOptions = DEFAULT_API_CONNECT_OPTIONS, + connOptions = DEFAULT_LLM_API_CONNECT_OPTIONS, toolChoice, extraKwargs, }: { diff --git a/plugins/openai/src/llm.ts b/plugins/openai/src/llm.ts index 434f51f05..8ab79a4df 100644 --- a/plugins/openai/src/llm.ts +++ b/plugins/openai/src/llm.ts @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 import type { APIConnectOptions } from '@livekit/agents'; -import { DEFAULT_API_CONNECT_OPTIONS, inference, llm } from '@livekit/agents'; +import { DEFAULT_LLM_API_CONNECT_OPTIONS, inference, llm } from '@livekit/agents'; import { AzureOpenAI, OpenAI } from 'openai'; import type { CerebrasChatModels, @@ -469,7 +469,7 @@ export class LLM extends llm.LLM { chat({ chatCtx, toolCtx, - connOptions = DEFAULT_API_CONNECT_OPTIONS, + connOptions = DEFAULT_LLM_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, extraKwargs, diff --git a/plugins/openai/src/responses/llm.ts b/plugins/openai/src/responses/llm.ts index 494a05c87..0e1358c29 100644 --- a/plugins/openai/src/responses/llm.ts +++ b/plugins/openai/src/responses/llm.ts @@ -6,7 +6,7 @@ import { APIConnectionError, APIStatusError, APITimeoutError, - DEFAULT_API_CONNECT_OPTIONS, + DEFAULT_LLM_API_CONNECT_OPTIONS, llm, log, toError, @@ -78,7 +78,7 @@ class ResponsesHttpLLM extends llm.LLM { override chat({ chatCtx, toolCtx, - connOptions = DEFAULT_API_CONNECT_OPTIONS, + connOptions = DEFAULT_LLM_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, extraKwargs, @@ -411,7 +411,7 @@ export class LLM extends llm.LLM { override chat({ chatCtx, toolCtx, - connOptions = DEFAULT_API_CONNECT_OPTIONS, + connOptions = DEFAULT_LLM_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, extraKwargs, diff --git a/plugins/openai/src/ws/llm.ts b/plugins/openai/src/ws/llm.ts index 1dd6483c6..9fe77931e 100644 --- a/plugins/openai/src/ws/llm.ts +++ b/plugins/openai/src/ws/llm.ts @@ -7,7 +7,7 @@ import { APIStatusError, APITimeoutError, ConnectionPool, - DEFAULT_API_CONNECT_OPTIONS, + DEFAULT_LLM_API_CONNECT_OPTIONS, llm, stream, toError, @@ -232,7 +232,7 @@ export class WSLLM extends llm.LLM { chat({ chatCtx, toolCtx, - connOptions = DEFAULT_API_CONNECT_OPTIONS, + connOptions = DEFAULT_LLM_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, extraKwargs,