511 lines
22 KiB
JavaScript
511 lines
22 KiB
JavaScript
import { BedrockRuntimeClient, StopReason as BedrockStopReason, CachePointType, ConversationRole, ConverseStreamCommand, ImageFormat, ToolResultStatus, } from "@aws-sdk/client-bedrock-runtime";
|
|
import { calculateCost } from "../models.js";
|
|
import { AssistantMessageEventStream } from "../utils/event-stream.js";
|
|
import { parseStreamingJson } from "../utils/json-parse.js";
|
|
import { sanitizeSurrogates } from "../utils/sanitize-unicode.js";
|
|
import { adjustMaxTokensForThinking, buildBaseOptions, clampReasoning } from "./simple-options.js";
|
|
import { transformMessages } from "./transform-messages.js";
|
|
export const streamBedrock = (model, context, options = {}) => {
|
|
const stream = new AssistantMessageEventStream();
|
|
(async () => {
|
|
const output = {
|
|
role: "assistant",
|
|
content: [],
|
|
api: "bedrock-converse-stream",
|
|
provider: model.provider,
|
|
model: model.id,
|
|
usage: {
|
|
input: 0,
|
|
output: 0,
|
|
cacheRead: 0,
|
|
cacheWrite: 0,
|
|
totalTokens: 0,
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
},
|
|
stopReason: "stop",
|
|
timestamp: Date.now(),
|
|
};
|
|
const blocks = output.content;
|
|
const config = {
|
|
region: options.region,
|
|
profile: options.profile,
|
|
};
|
|
// in Node.js/Bun environment only
|
|
if (typeof process !== "undefined" && (process.versions?.node || process.versions?.bun)) {
|
|
config.region = config.region || process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION;
|
|
if (process.env.HTTP_PROXY ||
|
|
process.env.HTTPS_PROXY ||
|
|
process.env.NO_PROXY ||
|
|
process.env.http_proxy ||
|
|
process.env.https_proxy ||
|
|
process.env.no_proxy) {
|
|
const nodeHttpHandler = await import("@smithy/node-http-handler");
|
|
const proxyAgent = await import("proxy-agent");
|
|
const agent = new proxyAgent.ProxyAgent();
|
|
// Bedrock runtime uses NodeHttp2Handler by default since v3.798.0, which is based
|
|
// on `http2` module and has no support for http agent.
|
|
// Use NodeHttpHandler to support http agent.
|
|
config.requestHandler = new nodeHttpHandler.NodeHttpHandler({
|
|
httpAgent: agent,
|
|
httpsAgent: agent,
|
|
});
|
|
}
|
|
}
|
|
config.region = config.region || "us-east-1";
|
|
try {
|
|
const client = new BedrockRuntimeClient(config);
|
|
const commandInput = {
|
|
modelId: model.id,
|
|
messages: convertMessages(context, model),
|
|
system: buildSystemPrompt(context.systemPrompt, model),
|
|
inferenceConfig: { maxTokens: options.maxTokens, temperature: options.temperature },
|
|
toolConfig: convertToolConfig(context.tools, options.toolChoice),
|
|
additionalModelRequestFields: buildAdditionalModelRequestFields(model, options),
|
|
};
|
|
options?.onPayload?.(commandInput);
|
|
const command = new ConverseStreamCommand(commandInput);
|
|
const response = await client.send(command, { abortSignal: options.signal });
|
|
for await (const item of response.stream) {
|
|
if (item.messageStart) {
|
|
if (item.messageStart.role !== ConversationRole.ASSISTANT) {
|
|
throw new Error("Unexpected assistant message start but got user message start instead");
|
|
}
|
|
stream.push({ type: "start", partial: output });
|
|
}
|
|
else if (item.contentBlockStart) {
|
|
handleContentBlockStart(item.contentBlockStart, blocks, output, stream);
|
|
}
|
|
else if (item.contentBlockDelta) {
|
|
handleContentBlockDelta(item.contentBlockDelta, blocks, output, stream);
|
|
}
|
|
else if (item.contentBlockStop) {
|
|
handleContentBlockStop(item.contentBlockStop, blocks, output, stream);
|
|
}
|
|
else if (item.messageStop) {
|
|
output.stopReason = mapStopReason(item.messageStop.stopReason);
|
|
}
|
|
else if (item.metadata) {
|
|
handleMetadata(item.metadata, model, output);
|
|
}
|
|
else if (item.internalServerException) {
|
|
throw new Error(`Internal server error: ${item.internalServerException.message}`);
|
|
}
|
|
else if (item.modelStreamErrorException) {
|
|
throw new Error(`Model stream error: ${item.modelStreamErrorException.message}`);
|
|
}
|
|
else if (item.validationException) {
|
|
throw new Error(`Validation error: ${item.validationException.message}`);
|
|
}
|
|
else if (item.throttlingException) {
|
|
throw new Error(`Throttling error: ${item.throttlingException.message}`);
|
|
}
|
|
else if (item.serviceUnavailableException) {
|
|
throw new Error(`Service unavailable: ${item.serviceUnavailableException.message}`);
|
|
}
|
|
}
|
|
if (options.signal?.aborted) {
|
|
throw new Error("Request was aborted");
|
|
}
|
|
if (output.stopReason === "error" || output.stopReason === "aborted") {
|
|
throw new Error("An unknown error occurred");
|
|
}
|
|
stream.push({ type: "done", reason: output.stopReason, message: output });
|
|
stream.end();
|
|
}
|
|
catch (error) {
|
|
for (const block of output.content) {
|
|
delete block.index;
|
|
delete block.partialJson;
|
|
}
|
|
output.stopReason = options.signal?.aborted ? "aborted" : "error";
|
|
output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error);
|
|
stream.push({ type: "error", reason: output.stopReason, error: output });
|
|
stream.end();
|
|
}
|
|
})();
|
|
return stream;
|
|
};
|
|
export const streamSimpleBedrock = (model, context, options) => {
|
|
const base = buildBaseOptions(model, options, undefined);
|
|
if (!options?.reasoning) {
|
|
return streamBedrock(model, context, { ...base, reasoning: undefined });
|
|
}
|
|
if (model.id.includes("anthropic.claude") || model.id.includes("anthropic/claude")) {
|
|
const adjusted = adjustMaxTokensForThinking(base.maxTokens || 0, model.maxTokens, options.reasoning, options.thinkingBudgets);
|
|
return streamBedrock(model, context, {
|
|
...base,
|
|
maxTokens: adjusted.maxTokens,
|
|
reasoning: options.reasoning,
|
|
thinkingBudgets: {
|
|
...(options.thinkingBudgets || {}),
|
|
[clampReasoning(options.reasoning)]: adjusted.thinkingBudget,
|
|
},
|
|
});
|
|
}
|
|
return streamBedrock(model, context, {
|
|
...base,
|
|
reasoning: options.reasoning,
|
|
thinkingBudgets: options.thinkingBudgets,
|
|
});
|
|
};
|
|
function handleContentBlockStart(event, blocks, output, stream) {
|
|
const index = event.contentBlockIndex;
|
|
const start = event.start;
|
|
if (start?.toolUse) {
|
|
const block = {
|
|
type: "toolCall",
|
|
id: start.toolUse.toolUseId || "",
|
|
name: start.toolUse.name || "",
|
|
arguments: {},
|
|
partialJson: "",
|
|
index,
|
|
};
|
|
output.content.push(block);
|
|
stream.push({ type: "toolcall_start", contentIndex: blocks.length - 1, partial: output });
|
|
}
|
|
}
|
|
function handleContentBlockDelta(event, blocks, output, stream) {
|
|
const contentBlockIndex = event.contentBlockIndex;
|
|
const delta = event.delta;
|
|
let index = blocks.findIndex((b) => b.index === contentBlockIndex);
|
|
let block = blocks[index];
|
|
if (delta?.text !== undefined) {
|
|
// If no text block exists yet, create one, as `handleContentBlockStart` is not sent for text blocks
|
|
if (!block) {
|
|
const newBlock = { type: "text", text: "", index: contentBlockIndex };
|
|
output.content.push(newBlock);
|
|
index = blocks.length - 1;
|
|
block = blocks[index];
|
|
stream.push({ type: "text_start", contentIndex: index, partial: output });
|
|
}
|
|
if (block.type === "text") {
|
|
block.text += delta.text;
|
|
stream.push({ type: "text_delta", contentIndex: index, delta: delta.text, partial: output });
|
|
}
|
|
}
|
|
else if (delta?.toolUse && block?.type === "toolCall") {
|
|
block.partialJson = (block.partialJson || "") + (delta.toolUse.input || "");
|
|
block.arguments = parseStreamingJson(block.partialJson);
|
|
stream.push({ type: "toolcall_delta", contentIndex: index, delta: delta.toolUse.input || "", partial: output });
|
|
}
|
|
else if (delta?.reasoningContent) {
|
|
let thinkingBlock = block;
|
|
let thinkingIndex = index;
|
|
if (!thinkingBlock) {
|
|
const newBlock = { type: "thinking", thinking: "", thinkingSignature: "", index: contentBlockIndex };
|
|
output.content.push(newBlock);
|
|
thinkingIndex = blocks.length - 1;
|
|
thinkingBlock = blocks[thinkingIndex];
|
|
stream.push({ type: "thinking_start", contentIndex: thinkingIndex, partial: output });
|
|
}
|
|
if (thinkingBlock?.type === "thinking") {
|
|
if (delta.reasoningContent.text) {
|
|
thinkingBlock.thinking += delta.reasoningContent.text;
|
|
stream.push({
|
|
type: "thinking_delta",
|
|
contentIndex: thinkingIndex,
|
|
delta: delta.reasoningContent.text,
|
|
partial: output,
|
|
});
|
|
}
|
|
if (delta.reasoningContent.signature) {
|
|
thinkingBlock.thinkingSignature =
|
|
(thinkingBlock.thinkingSignature || "") + delta.reasoningContent.signature;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
function handleMetadata(event, model, output) {
|
|
if (event.usage) {
|
|
output.usage.input = event.usage.inputTokens || 0;
|
|
output.usage.output = event.usage.outputTokens || 0;
|
|
output.usage.cacheRead = event.usage.cacheReadInputTokens || 0;
|
|
output.usage.cacheWrite = event.usage.cacheWriteInputTokens || 0;
|
|
output.usage.totalTokens = event.usage.totalTokens || output.usage.input + output.usage.output;
|
|
calculateCost(model, output.usage);
|
|
}
|
|
}
|
|
function handleContentBlockStop(event, blocks, output, stream) {
|
|
const index = blocks.findIndex((b) => b.index === event.contentBlockIndex);
|
|
const block = blocks[index];
|
|
if (!block)
|
|
return;
|
|
delete block.index;
|
|
switch (block.type) {
|
|
case "text":
|
|
stream.push({ type: "text_end", contentIndex: index, content: block.text, partial: output });
|
|
break;
|
|
case "thinking":
|
|
stream.push({ type: "thinking_end", contentIndex: index, content: block.thinking, partial: output });
|
|
break;
|
|
case "toolCall":
|
|
block.arguments = parseStreamingJson(block.partialJson);
|
|
delete block.partialJson;
|
|
stream.push({ type: "toolcall_end", contentIndex: index, toolCall: block, partial: output });
|
|
break;
|
|
}
|
|
}
|
|
/**
|
|
* Check if the model supports prompt caching.
|
|
* Supported: Claude 3.5 Haiku, Claude 3.7 Sonnet, Claude 4.x models
|
|
*/
|
|
function supportsPromptCaching(model) {
|
|
const id = model.id.toLowerCase();
|
|
// Claude 4.x models (opus-4, sonnet-4, haiku-4)
|
|
if (id.includes("claude") && (id.includes("-4-") || id.includes("-4.")))
|
|
return true;
|
|
// Claude 3.7 Sonnet
|
|
if (id.includes("claude-3-7-sonnet"))
|
|
return true;
|
|
// Claude 3.5 Haiku
|
|
if (id.includes("claude-3-5-haiku"))
|
|
return true;
|
|
return false;
|
|
}
|
|
/**
|
|
* Check if the model supports thinking signatures in reasoningContent.
|
|
* Only Anthropic Claude models support the signature field.
|
|
* Other models (OpenAI, Qwen, Minimax, Moonshot, etc.) reject it with:
|
|
* "This model doesn't support the reasoningContent.reasoningText.signature field"
|
|
*/
|
|
function supportsThinkingSignature(model) {
|
|
const id = model.id.toLowerCase();
|
|
return id.includes("anthropic.claude") || id.includes("anthropic/claude");
|
|
}
|
|
function buildSystemPrompt(systemPrompt, model) {
|
|
if (!systemPrompt)
|
|
return undefined;
|
|
const blocks = [{ text: sanitizeSurrogates(systemPrompt) }];
|
|
// Add cache point for supported Claude models
|
|
if (supportsPromptCaching(model)) {
|
|
blocks.push({ cachePoint: { type: CachePointType.DEFAULT } });
|
|
}
|
|
return blocks;
|
|
}
|
|
function normalizeToolCallId(id) {
|
|
const sanitized = id.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
return sanitized.length > 64 ? sanitized.slice(0, 64) : sanitized;
|
|
}
|
|
function convertMessages(context, model) {
|
|
const result = [];
|
|
const transformedMessages = transformMessages(context.messages, model, normalizeToolCallId);
|
|
for (let i = 0; i < transformedMessages.length; i++) {
|
|
const m = transformedMessages[i];
|
|
switch (m.role) {
|
|
case "user":
|
|
result.push({
|
|
role: ConversationRole.USER,
|
|
content: typeof m.content === "string"
|
|
? [{ text: sanitizeSurrogates(m.content) }]
|
|
: m.content.map((c) => {
|
|
switch (c.type) {
|
|
case "text":
|
|
return { text: sanitizeSurrogates(c.text) };
|
|
case "image":
|
|
return { image: createImageBlock(c.mimeType, c.data) };
|
|
default:
|
|
throw new Error("Unknown user content type");
|
|
}
|
|
}),
|
|
});
|
|
break;
|
|
case "assistant": {
|
|
// Skip assistant messages with empty content (e.g., from aborted requests)
|
|
// Bedrock rejects messages with empty content arrays
|
|
if (m.content.length === 0) {
|
|
continue;
|
|
}
|
|
const contentBlocks = [];
|
|
for (const c of m.content) {
|
|
switch (c.type) {
|
|
case "text":
|
|
// Skip empty text blocks
|
|
if (c.text.trim().length === 0)
|
|
continue;
|
|
contentBlocks.push({ text: sanitizeSurrogates(c.text) });
|
|
break;
|
|
case "toolCall":
|
|
contentBlocks.push({
|
|
toolUse: { toolUseId: c.id, name: c.name, input: c.arguments },
|
|
});
|
|
break;
|
|
case "thinking":
|
|
// Skip empty thinking blocks
|
|
if (c.thinking.trim().length === 0)
|
|
continue;
|
|
// Only Anthropic models support the signature field in reasoningText.
|
|
// For other models, we omit the signature to avoid errors like:
|
|
// "This model doesn't support the reasoningContent.reasoningText.signature field"
|
|
if (supportsThinkingSignature(model)) {
|
|
contentBlocks.push({
|
|
reasoningContent: {
|
|
reasoningText: { text: sanitizeSurrogates(c.thinking), signature: c.thinkingSignature },
|
|
},
|
|
});
|
|
}
|
|
else {
|
|
contentBlocks.push({
|
|
reasoningContent: {
|
|
reasoningText: { text: sanitizeSurrogates(c.thinking) },
|
|
},
|
|
});
|
|
}
|
|
break;
|
|
default:
|
|
throw new Error("Unknown assistant content type");
|
|
}
|
|
}
|
|
// Skip if all content blocks were filtered out
|
|
if (contentBlocks.length === 0) {
|
|
continue;
|
|
}
|
|
result.push({
|
|
role: ConversationRole.ASSISTANT,
|
|
content: contentBlocks,
|
|
});
|
|
break;
|
|
}
|
|
case "toolResult": {
|
|
// Collect all consecutive toolResult messages into a single user message
|
|
// Bedrock requires all tool results to be in one message
|
|
const toolResults = [];
|
|
// Add current tool result with all content blocks combined
|
|
toolResults.push({
|
|
toolResult: {
|
|
toolUseId: m.toolCallId,
|
|
content: m.content.map((c) => c.type === "image"
|
|
? { image: createImageBlock(c.mimeType, c.data) }
|
|
: { text: sanitizeSurrogates(c.text) }),
|
|
status: m.isError ? ToolResultStatus.ERROR : ToolResultStatus.SUCCESS,
|
|
},
|
|
});
|
|
// Look ahead for consecutive toolResult messages
|
|
let j = i + 1;
|
|
while (j < transformedMessages.length && transformedMessages[j].role === "toolResult") {
|
|
const nextMsg = transformedMessages[j];
|
|
toolResults.push({
|
|
toolResult: {
|
|
toolUseId: nextMsg.toolCallId,
|
|
content: nextMsg.content.map((c) => c.type === "image"
|
|
? { image: createImageBlock(c.mimeType, c.data) }
|
|
: { text: sanitizeSurrogates(c.text) }),
|
|
status: nextMsg.isError ? ToolResultStatus.ERROR : ToolResultStatus.SUCCESS,
|
|
},
|
|
});
|
|
j++;
|
|
}
|
|
// Skip the messages we've already processed
|
|
i = j - 1;
|
|
result.push({
|
|
role: ConversationRole.USER,
|
|
content: toolResults,
|
|
});
|
|
break;
|
|
}
|
|
default:
|
|
throw new Error("Unknown message role");
|
|
}
|
|
}
|
|
// Add cache point to the last user message for supported Claude models
|
|
if (supportsPromptCaching(model) && result.length > 0) {
|
|
const lastMessage = result[result.length - 1];
|
|
if (lastMessage.role === ConversationRole.USER && lastMessage.content) {
|
|
lastMessage.content.push({ cachePoint: { type: CachePointType.DEFAULT } });
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
function convertToolConfig(tools, toolChoice) {
|
|
if (!tools?.length || toolChoice === "none")
|
|
return undefined;
|
|
const bedrockTools = tools.map((tool) => ({
|
|
toolSpec: {
|
|
name: tool.name,
|
|
description: tool.description,
|
|
inputSchema: { json: tool.parameters },
|
|
},
|
|
}));
|
|
let bedrockToolChoice;
|
|
switch (toolChoice) {
|
|
case "auto":
|
|
bedrockToolChoice = { auto: {} };
|
|
break;
|
|
case "any":
|
|
bedrockToolChoice = { any: {} };
|
|
break;
|
|
default:
|
|
if (toolChoice?.type === "tool") {
|
|
bedrockToolChoice = { tool: { name: toolChoice.name } };
|
|
}
|
|
}
|
|
return { tools: bedrockTools, toolChoice: bedrockToolChoice };
|
|
}
|
|
function mapStopReason(reason) {
|
|
switch (reason) {
|
|
case BedrockStopReason.END_TURN:
|
|
case BedrockStopReason.STOP_SEQUENCE:
|
|
return "stop";
|
|
case BedrockStopReason.MAX_TOKENS:
|
|
case BedrockStopReason.MODEL_CONTEXT_WINDOW_EXCEEDED:
|
|
return "length";
|
|
case BedrockStopReason.TOOL_USE:
|
|
return "toolUse";
|
|
default:
|
|
return "error";
|
|
}
|
|
}
|
|
function buildAdditionalModelRequestFields(model, options) {
|
|
if (!options.reasoning || !model.reasoning) {
|
|
return undefined;
|
|
}
|
|
if (model.id.includes("anthropic.claude")) {
|
|
const defaultBudgets = {
|
|
minimal: 1024,
|
|
low: 2048,
|
|
medium: 8192,
|
|
high: 16384,
|
|
xhigh: 16384, // Claude doesn't support xhigh, clamp to high
|
|
};
|
|
// Custom budgets override defaults (xhigh not in ThinkingBudgets, use high)
|
|
const level = options.reasoning === "xhigh" ? "high" : options.reasoning;
|
|
const budget = options.thinkingBudgets?.[level] ?? defaultBudgets[options.reasoning];
|
|
const result = {
|
|
thinking: {
|
|
type: "enabled",
|
|
budget_tokens: budget,
|
|
},
|
|
};
|
|
if (options.interleavedThinking) {
|
|
result.anthropic_beta = ["interleaved-thinking-2025-05-14"];
|
|
}
|
|
return result;
|
|
}
|
|
return undefined;
|
|
}
|
|
function createImageBlock(mimeType, data) {
|
|
let format;
|
|
switch (mimeType) {
|
|
case "image/jpeg":
|
|
case "image/jpg":
|
|
format = ImageFormat.JPEG;
|
|
break;
|
|
case "image/png":
|
|
format = ImageFormat.PNG;
|
|
break;
|
|
case "image/gif":
|
|
format = ImageFormat.GIF;
|
|
break;
|
|
case "image/webp":
|
|
format = ImageFormat.WEBP;
|
|
break;
|
|
default:
|
|
throw new Error(`Unknown image type: ${mimeType}`);
|
|
}
|
|
const binaryString = atob(data);
|
|
const bytes = new Uint8Array(binaryString.length);
|
|
for (let i = 0; i < binaryString.length; i++) {
|
|
bytes[i] = binaryString.charCodeAt(i);
|
|
}
|
|
return { source: { bytes }, format };
|
|
}
|
|
//# sourceMappingURL=amazon-bedrock.js.map
|