import { BedrockRuntimeClient, StopReason as BedrockStopReason, CachePointType, ConversationRole, ConverseStreamCommand, ImageFormat, ToolResultStatus, } from "@aws-sdk/client-bedrock-runtime"; import { calculateCost } from "../models.js"; import { AssistantMessageEventStream } from "../utils/event-stream.js"; import { parseStreamingJson } from "../utils/json-parse.js"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.js"; import { adjustMaxTokensForThinking, buildBaseOptions, clampReasoning } from "./simple-options.js"; import { transformMessages } from "./transform-messages.js"; export const streamBedrock = (model, context, options = {}) => { const stream = new AssistantMessageEventStream(); (async () => { const output = { role: "assistant", content: [], api: "bedrock-converse-stream", provider: model.provider, model: model.id, usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, }, stopReason: "stop", timestamp: Date.now(), }; const blocks = output.content; const config = { region: options.region, profile: options.profile, }; // in Node.js/Bun environment only if (typeof process !== "undefined" && (process.versions?.node || process.versions?.bun)) { config.region = config.region || process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION; if (process.env.HTTP_PROXY || process.env.HTTPS_PROXY || process.env.NO_PROXY || process.env.http_proxy || process.env.https_proxy || process.env.no_proxy) { const nodeHttpHandler = await import("@smithy/node-http-handler"); const proxyAgent = await import("proxy-agent"); const agent = new proxyAgent.ProxyAgent(); // Bedrock runtime uses NodeHttp2Handler by default since v3.798.0, which is based // on `http2` module and has no support for http agent. // Use NodeHttpHandler to support http agent. config.requestHandler = new nodeHttpHandler.NodeHttpHandler({ httpAgent: agent, httpsAgent: agent, }); } } config.region = config.region || "us-east-1"; try { const client = new BedrockRuntimeClient(config); const commandInput = { modelId: model.id, messages: convertMessages(context, model), system: buildSystemPrompt(context.systemPrompt, model), inferenceConfig: { maxTokens: options.maxTokens, temperature: options.temperature }, toolConfig: convertToolConfig(context.tools, options.toolChoice), additionalModelRequestFields: buildAdditionalModelRequestFields(model, options), }; options?.onPayload?.(commandInput); const command = new ConverseStreamCommand(commandInput); const response = await client.send(command, { abortSignal: options.signal }); for await (const item of response.stream) { if (item.messageStart) { if (item.messageStart.role !== ConversationRole.ASSISTANT) { throw new Error("Unexpected assistant message start but got user message start instead"); } stream.push({ type: "start", partial: output }); } else if (item.contentBlockStart) { handleContentBlockStart(item.contentBlockStart, blocks, output, stream); } else if (item.contentBlockDelta) { handleContentBlockDelta(item.contentBlockDelta, blocks, output, stream); } else if (item.contentBlockStop) { handleContentBlockStop(item.contentBlockStop, blocks, output, stream); } else if (item.messageStop) { output.stopReason = mapStopReason(item.messageStop.stopReason); } else if (item.metadata) { handleMetadata(item.metadata, model, output); } else if (item.internalServerException) { throw new Error(`Internal server error: ${item.internalServerException.message}`); } else if (item.modelStreamErrorException) { throw new Error(`Model stream error: ${item.modelStreamErrorException.message}`); } else if (item.validationException) { throw new Error(`Validation error: ${item.validationException.message}`); } else if (item.throttlingException) { throw new Error(`Throttling error: ${item.throttlingException.message}`); } else if (item.serviceUnavailableException) { throw new Error(`Service unavailable: ${item.serviceUnavailableException.message}`); } } if (options.signal?.aborted) { throw new Error("Request was aborted"); } if (output.stopReason === "error" || output.stopReason === "aborted") { throw new Error("An unknown error occurred"); } stream.push({ type: "done", reason: output.stopReason, message: output }); stream.end(); } catch (error) { for (const block of output.content) { delete block.index; delete block.partialJson; } output.stopReason = options.signal?.aborted ? "aborted" : "error"; output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error); stream.push({ type: "error", reason: output.stopReason, error: output }); stream.end(); } })(); return stream; }; export const streamSimpleBedrock = (model, context, options) => { const base = buildBaseOptions(model, options, undefined); if (!options?.reasoning) { return streamBedrock(model, context, { ...base, reasoning: undefined }); } if (model.id.includes("anthropic.claude") || model.id.includes("anthropic/claude")) { const adjusted = adjustMaxTokensForThinking(base.maxTokens || 0, model.maxTokens, options.reasoning, options.thinkingBudgets); return streamBedrock(model, context, { ...base, maxTokens: adjusted.maxTokens, reasoning: options.reasoning, thinkingBudgets: { ...(options.thinkingBudgets || {}), [clampReasoning(options.reasoning)]: adjusted.thinkingBudget, }, }); } return streamBedrock(model, context, { ...base, reasoning: options.reasoning, thinkingBudgets: options.thinkingBudgets, }); }; function handleContentBlockStart(event, blocks, output, stream) { const index = event.contentBlockIndex; const start = event.start; if (start?.toolUse) { const block = { type: "toolCall", id: start.toolUse.toolUseId || "", name: start.toolUse.name || "", arguments: {}, partialJson: "", index, }; output.content.push(block); stream.push({ type: "toolcall_start", contentIndex: blocks.length - 1, partial: output }); } } function handleContentBlockDelta(event, blocks, output, stream) { const contentBlockIndex = event.contentBlockIndex; const delta = event.delta; let index = blocks.findIndex((b) => b.index === contentBlockIndex); let block = blocks[index]; if (delta?.text !== undefined) { // If no text block exists yet, create one, as `handleContentBlockStart` is not sent for text blocks if (!block) { const newBlock = { type: "text", text: "", index: contentBlockIndex }; output.content.push(newBlock); index = blocks.length - 1; block = blocks[index]; stream.push({ type: "text_start", contentIndex: index, partial: output }); } if (block.type === "text") { block.text += delta.text; stream.push({ type: "text_delta", contentIndex: index, delta: delta.text, partial: output }); } } else if (delta?.toolUse && block?.type === "toolCall") { block.partialJson = (block.partialJson || "") + (delta.toolUse.input || ""); block.arguments = parseStreamingJson(block.partialJson); stream.push({ type: "toolcall_delta", contentIndex: index, delta: delta.toolUse.input || "", partial: output }); } else if (delta?.reasoningContent) { let thinkingBlock = block; let thinkingIndex = index; if (!thinkingBlock) { const newBlock = { type: "thinking", thinking: "", thinkingSignature: "", index: contentBlockIndex }; output.content.push(newBlock); thinkingIndex = blocks.length - 1; thinkingBlock = blocks[thinkingIndex]; stream.push({ type: "thinking_start", contentIndex: thinkingIndex, partial: output }); } if (thinkingBlock?.type === "thinking") { if (delta.reasoningContent.text) { thinkingBlock.thinking += delta.reasoningContent.text; stream.push({ type: "thinking_delta", contentIndex: thinkingIndex, delta: delta.reasoningContent.text, partial: output, }); } if (delta.reasoningContent.signature) { thinkingBlock.thinkingSignature = (thinkingBlock.thinkingSignature || "") + delta.reasoningContent.signature; } } } } function handleMetadata(event, model, output) { if (event.usage) { output.usage.input = event.usage.inputTokens || 0; output.usage.output = event.usage.outputTokens || 0; output.usage.cacheRead = event.usage.cacheReadInputTokens || 0; output.usage.cacheWrite = event.usage.cacheWriteInputTokens || 0; output.usage.totalTokens = event.usage.totalTokens || output.usage.input + output.usage.output; calculateCost(model, output.usage); } } function handleContentBlockStop(event, blocks, output, stream) { const index = blocks.findIndex((b) => b.index === event.contentBlockIndex); const block = blocks[index]; if (!block) return; delete block.index; switch (block.type) { case "text": stream.push({ type: "text_end", contentIndex: index, content: block.text, partial: output }); break; case "thinking": stream.push({ type: "thinking_end", contentIndex: index, content: block.thinking, partial: output }); break; case "toolCall": block.arguments = parseStreamingJson(block.partialJson); delete block.partialJson; stream.push({ type: "toolcall_end", contentIndex: index, toolCall: block, partial: output }); break; } } /** * Check if the model supports prompt caching. * Supported: Claude 3.5 Haiku, Claude 3.7 Sonnet, Claude 4.x models */ function supportsPromptCaching(model) { const id = model.id.toLowerCase(); // Claude 4.x models (opus-4, sonnet-4, haiku-4) if (id.includes("claude") && (id.includes("-4-") || id.includes("-4."))) return true; // Claude 3.7 Sonnet if (id.includes("claude-3-7-sonnet")) return true; // Claude 3.5 Haiku if (id.includes("claude-3-5-haiku")) return true; return false; } /** * Check if the model supports thinking signatures in reasoningContent. * Only Anthropic Claude models support the signature field. * Other models (OpenAI, Qwen, Minimax, Moonshot, etc.) reject it with: * "This model doesn't support the reasoningContent.reasoningText.signature field" */ function supportsThinkingSignature(model) { const id = model.id.toLowerCase(); return id.includes("anthropic.claude") || id.includes("anthropic/claude"); } function buildSystemPrompt(systemPrompt, model) { if (!systemPrompt) return undefined; const blocks = [{ text: sanitizeSurrogates(systemPrompt) }]; // Add cache point for supported Claude models if (supportsPromptCaching(model)) { blocks.push({ cachePoint: { type: CachePointType.DEFAULT } }); } return blocks; } function normalizeToolCallId(id) { const sanitized = id.replace(/[^a-zA-Z0-9_-]/g, "_"); return sanitized.length > 64 ? sanitized.slice(0, 64) : sanitized; } function convertMessages(context, model) { const result = []; const transformedMessages = transformMessages(context.messages, model, normalizeToolCallId); for (let i = 0; i < transformedMessages.length; i++) { const m = transformedMessages[i]; switch (m.role) { case "user": result.push({ role: ConversationRole.USER, content: typeof m.content === "string" ? [{ text: sanitizeSurrogates(m.content) }] : m.content.map((c) => { switch (c.type) { case "text": return { text: sanitizeSurrogates(c.text) }; case "image": return { image: createImageBlock(c.mimeType, c.data) }; default: throw new Error("Unknown user content type"); } }), }); break; case "assistant": { // Skip assistant messages with empty content (e.g., from aborted requests) // Bedrock rejects messages with empty content arrays if (m.content.length === 0) { continue; } const contentBlocks = []; for (const c of m.content) { switch (c.type) { case "text": // Skip empty text blocks if (c.text.trim().length === 0) continue; contentBlocks.push({ text: sanitizeSurrogates(c.text) }); break; case "toolCall": contentBlocks.push({ toolUse: { toolUseId: c.id, name: c.name, input: c.arguments }, }); break; case "thinking": // Skip empty thinking blocks if (c.thinking.trim().length === 0) continue; // Only Anthropic models support the signature field in reasoningText. // For other models, we omit the signature to avoid errors like: // "This model doesn't support the reasoningContent.reasoningText.signature field" if (supportsThinkingSignature(model)) { contentBlocks.push({ reasoningContent: { reasoningText: { text: sanitizeSurrogates(c.thinking), signature: c.thinkingSignature }, }, }); } else { contentBlocks.push({ reasoningContent: { reasoningText: { text: sanitizeSurrogates(c.thinking) }, }, }); } break; default: throw new Error("Unknown assistant content type"); } } // Skip if all content blocks were filtered out if (contentBlocks.length === 0) { continue; } result.push({ role: ConversationRole.ASSISTANT, content: contentBlocks, }); break; } case "toolResult": { // Collect all consecutive toolResult messages into a single user message // Bedrock requires all tool results to be in one message const toolResults = []; // Add current tool result with all content blocks combined toolResults.push({ toolResult: { toolUseId: m.toolCallId, content: m.content.map((c) => c.type === "image" ? { image: createImageBlock(c.mimeType, c.data) } : { text: sanitizeSurrogates(c.text) }), status: m.isError ? ToolResultStatus.ERROR : ToolResultStatus.SUCCESS, }, }); // Look ahead for consecutive toolResult messages let j = i + 1; while (j < transformedMessages.length && transformedMessages[j].role === "toolResult") { const nextMsg = transformedMessages[j]; toolResults.push({ toolResult: { toolUseId: nextMsg.toolCallId, content: nextMsg.content.map((c) => c.type === "image" ? { image: createImageBlock(c.mimeType, c.data) } : { text: sanitizeSurrogates(c.text) }), status: nextMsg.isError ? ToolResultStatus.ERROR : ToolResultStatus.SUCCESS, }, }); j++; } // Skip the messages we've already processed i = j - 1; result.push({ role: ConversationRole.USER, content: toolResults, }); break; } default: throw new Error("Unknown message role"); } } // Add cache point to the last user message for supported Claude models if (supportsPromptCaching(model) && result.length > 0) { const lastMessage = result[result.length - 1]; if (lastMessage.role === ConversationRole.USER && lastMessage.content) { lastMessage.content.push({ cachePoint: { type: CachePointType.DEFAULT } }); } } return result; } function convertToolConfig(tools, toolChoice) { if (!tools?.length || toolChoice === "none") return undefined; const bedrockTools = tools.map((tool) => ({ toolSpec: { name: tool.name, description: tool.description, inputSchema: { json: tool.parameters }, }, })); let bedrockToolChoice; switch (toolChoice) { case "auto": bedrockToolChoice = { auto: {} }; break; case "any": bedrockToolChoice = { any: {} }; break; default: if (toolChoice?.type === "tool") { bedrockToolChoice = { tool: { name: toolChoice.name } }; } } return { tools: bedrockTools, toolChoice: bedrockToolChoice }; } function mapStopReason(reason) { switch (reason) { case BedrockStopReason.END_TURN: case BedrockStopReason.STOP_SEQUENCE: return "stop"; case BedrockStopReason.MAX_TOKENS: case BedrockStopReason.MODEL_CONTEXT_WINDOW_EXCEEDED: return "length"; case BedrockStopReason.TOOL_USE: return "toolUse"; default: return "error"; } } function buildAdditionalModelRequestFields(model, options) { if (!options.reasoning || !model.reasoning) { return undefined; } if (model.id.includes("anthropic.claude")) { const defaultBudgets = { minimal: 1024, low: 2048, medium: 8192, high: 16384, xhigh: 16384, // Claude doesn't support xhigh, clamp to high }; // Custom budgets override defaults (xhigh not in ThinkingBudgets, use high) const level = options.reasoning === "xhigh" ? "high" : options.reasoning; const budget = options.thinkingBudgets?.[level] ?? defaultBudgets[options.reasoning]; const result = { thinking: { type: "enabled", budget_tokens: budget, }, }; if (options.interleavedThinking) { result.anthropic_beta = ["interleaved-thinking-2025-05-14"]; } return result; } return undefined; } function createImageBlock(mimeType, data) { let format; switch (mimeType) { case "image/jpeg": case "image/jpg": format = ImageFormat.JPEG; break; case "image/png": format = ImageFormat.PNG; break; case "image/gif": format = ImageFormat.GIF; break; case "image/webp": format = ImageFormat.WEBP; break; default: throw new Error(`Unknown image type: ${mimeType}`); } const binaryString = atob(data); const bytes = new Uint8Array(binaryString.length); for (let i = 0; i < binaryString.length; i++) { bytes[i] = binaryString.charCodeAt(i); } return { source: { bytes }, format }; } //# sourceMappingURL=amazon-bedrock.js.map