import { g as resolveStateDir } from "./paths-BDd7_JUB.js"; import { n as resolveAgentConfig, r as resolveAgentDir, s as resolveAgentWorkspaceDir } from "./agent-scope-CrgUOY3f.js"; import { t as createSubsystemLogger } from "./subsystem-46MXi6Ip.js"; import { h as resolveUserPath, i as clampNumber$1, r as clampInt, v as sleep, x as truncateUtf16Safe } from "./utils-Dg0Xbl6w.js"; import { M as resolveApiKeyForProvider, j as requireApiKey } from "./model-selection-Cp1maz7B.js"; import { t as isTruthyEnvValue } from "./env-DOcCob95.js"; import { a as resolveSessionTranscriptsDirForAgent } from "./paths-50eo6DV6.js"; import { n as onSessionTranscriptUpdate } from "./transcript-events-fUhPZcB5.js"; import { a as ensureDir, c as listMemoryFiles, i as cosineSimilarity, l as normalizeExtraMemoryPaths, n as buildFileEntry, o as hashText, r as chunkMarkdown, s as isMemoryPath, t as requireNodeSqlite, u as parseEmbedding } from "./sqlite-Dz6S6ijV.js"; import os from "node:os"; import path from "node:path"; import fs from "node:fs"; import fs$1 from "node:fs/promises"; import { randomUUID } from "node:crypto"; import chokidar from "chokidar"; //#region src/agents/memory-search.ts const DEFAULT_OPENAI_MODEL = "text-embedding-3-small"; const DEFAULT_GEMINI_MODEL = "gemini-embedding-001"; const DEFAULT_CHUNK_TOKENS = 400; const DEFAULT_CHUNK_OVERLAP = 80; const DEFAULT_WATCH_DEBOUNCE_MS = 1500; const DEFAULT_SESSION_DELTA_BYTES = 1e5; const DEFAULT_SESSION_DELTA_MESSAGES = 50; const DEFAULT_MAX_RESULTS = 6; const DEFAULT_MIN_SCORE = .35; const DEFAULT_HYBRID_ENABLED = true; const DEFAULT_HYBRID_VECTOR_WEIGHT = .7; const DEFAULT_HYBRID_TEXT_WEIGHT = .3; const DEFAULT_HYBRID_CANDIDATE_MULTIPLIER = 4; const DEFAULT_CACHE_ENABLED = true; const DEFAULT_SOURCES = ["memory"]; function normalizeSources(sources, sessionMemoryEnabled) { const normalized = /* @__PURE__ */ new Set(); const input = sources?.length ? sources : DEFAULT_SOURCES; for (const source of input) { if (source === "memory") normalized.add("memory"); if (source === "sessions" && sessionMemoryEnabled) normalized.add("sessions"); } if (normalized.size === 0) normalized.add("memory"); return Array.from(normalized); } function resolveStorePath(agentId, raw) { const stateDir = resolveStateDir(process.env, os.homedir); const fallback = path.join(stateDir, "memory", `${agentId}.sqlite`); if (!raw) return fallback; return resolveUserPath(raw.includes("{agentId}") ? raw.replaceAll("{agentId}", agentId) : raw); } function mergeConfig(defaults, overrides, agentId) { const enabled = overrides?.enabled ?? defaults?.enabled ?? true; const sessionMemory = overrides?.experimental?.sessionMemory ?? defaults?.experimental?.sessionMemory ?? false; const provider = overrides?.provider ?? defaults?.provider ?? "auto"; const defaultRemote = defaults?.remote; const overrideRemote = overrides?.remote; const includeRemote = Boolean(overrideRemote?.baseUrl || overrideRemote?.apiKey || overrideRemote?.headers || defaultRemote?.baseUrl || defaultRemote?.apiKey || defaultRemote?.headers) || provider === "openai" || provider === "gemini" || provider === "auto"; const batch = { enabled: overrideRemote?.batch?.enabled ?? defaultRemote?.batch?.enabled ?? true, wait: overrideRemote?.batch?.wait ?? defaultRemote?.batch?.wait ?? true, concurrency: Math.max(1, overrideRemote?.batch?.concurrency ?? defaultRemote?.batch?.concurrency ?? 2), pollIntervalMs: overrideRemote?.batch?.pollIntervalMs ?? defaultRemote?.batch?.pollIntervalMs ?? 2e3, timeoutMinutes: overrideRemote?.batch?.timeoutMinutes ?? defaultRemote?.batch?.timeoutMinutes ?? 60 }; const remote = includeRemote ? { baseUrl: overrideRemote?.baseUrl ?? defaultRemote?.baseUrl, apiKey: overrideRemote?.apiKey ?? defaultRemote?.apiKey, headers: overrideRemote?.headers ?? defaultRemote?.headers, batch } : void 0; const fallback = overrides?.fallback ?? defaults?.fallback ?? "none"; const modelDefault = provider === "gemini" ? DEFAULT_GEMINI_MODEL : provider === "openai" ? DEFAULT_OPENAI_MODEL : void 0; const model = overrides?.model ?? defaults?.model ?? modelDefault ?? ""; const local = { modelPath: overrides?.local?.modelPath ?? defaults?.local?.modelPath, modelCacheDir: overrides?.local?.modelCacheDir ?? defaults?.local?.modelCacheDir }; const sources = normalizeSources(overrides?.sources ?? defaults?.sources, sessionMemory); const rawPaths = [...defaults?.extraPaths ?? [], ...overrides?.extraPaths ?? []].map((value) => value.trim()).filter(Boolean); const extraPaths = Array.from(new Set(rawPaths)); const vector = { enabled: overrides?.store?.vector?.enabled ?? defaults?.store?.vector?.enabled ?? true, extensionPath: overrides?.store?.vector?.extensionPath ?? defaults?.store?.vector?.extensionPath }; const store = { driver: overrides?.store?.driver ?? defaults?.store?.driver ?? "sqlite", path: resolveStorePath(agentId, overrides?.store?.path ?? defaults?.store?.path), vector }; const chunking = { tokens: overrides?.chunking?.tokens ?? defaults?.chunking?.tokens ?? DEFAULT_CHUNK_TOKENS, overlap: overrides?.chunking?.overlap ?? defaults?.chunking?.overlap ?? DEFAULT_CHUNK_OVERLAP }; const sync = { onSessionStart: overrides?.sync?.onSessionStart ?? defaults?.sync?.onSessionStart ?? true, onSearch: overrides?.sync?.onSearch ?? defaults?.sync?.onSearch ?? true, watch: overrides?.sync?.watch ?? defaults?.sync?.watch ?? true, watchDebounceMs: overrides?.sync?.watchDebounceMs ?? defaults?.sync?.watchDebounceMs ?? DEFAULT_WATCH_DEBOUNCE_MS, intervalMinutes: overrides?.sync?.intervalMinutes ?? defaults?.sync?.intervalMinutes ?? 0, sessions: { deltaBytes: overrides?.sync?.sessions?.deltaBytes ?? defaults?.sync?.sessions?.deltaBytes ?? DEFAULT_SESSION_DELTA_BYTES, deltaMessages: overrides?.sync?.sessions?.deltaMessages ?? defaults?.sync?.sessions?.deltaMessages ?? DEFAULT_SESSION_DELTA_MESSAGES } }; const query = { maxResults: overrides?.query?.maxResults ?? defaults?.query?.maxResults ?? DEFAULT_MAX_RESULTS, minScore: overrides?.query?.minScore ?? defaults?.query?.minScore ?? DEFAULT_MIN_SCORE }; const hybrid = { enabled: overrides?.query?.hybrid?.enabled ?? defaults?.query?.hybrid?.enabled ?? DEFAULT_HYBRID_ENABLED, vectorWeight: overrides?.query?.hybrid?.vectorWeight ?? defaults?.query?.hybrid?.vectorWeight ?? DEFAULT_HYBRID_VECTOR_WEIGHT, textWeight: overrides?.query?.hybrid?.textWeight ?? defaults?.query?.hybrid?.textWeight ?? DEFAULT_HYBRID_TEXT_WEIGHT, candidateMultiplier: overrides?.query?.hybrid?.candidateMultiplier ?? defaults?.query?.hybrid?.candidateMultiplier ?? DEFAULT_HYBRID_CANDIDATE_MULTIPLIER }; const cache = { enabled: overrides?.cache?.enabled ?? defaults?.cache?.enabled ?? DEFAULT_CACHE_ENABLED, maxEntries: overrides?.cache?.maxEntries ?? defaults?.cache?.maxEntries }; const overlap = clampNumber$1(chunking.overlap, 0, Math.max(0, chunking.tokens - 1)); const minScore = clampNumber$1(query.minScore, 0, 1); const vectorWeight = clampNumber$1(hybrid.vectorWeight, 0, 1); const textWeight = clampNumber$1(hybrid.textWeight, 0, 1); const sum = vectorWeight + textWeight; const normalizedVectorWeight = sum > 0 ? vectorWeight / sum : DEFAULT_HYBRID_VECTOR_WEIGHT; const normalizedTextWeight = sum > 0 ? textWeight / sum : DEFAULT_HYBRID_TEXT_WEIGHT; const candidateMultiplier = clampInt(hybrid.candidateMultiplier, 1, 20); const deltaBytes = clampInt(sync.sessions.deltaBytes, 0, Number.MAX_SAFE_INTEGER); const deltaMessages = clampInt(sync.sessions.deltaMessages, 0, Number.MAX_SAFE_INTEGER); return { enabled, sources, extraPaths, provider, remote, experimental: { sessionMemory }, fallback, model, local, store, chunking: { tokens: Math.max(1, chunking.tokens), overlap }, sync: { ...sync, sessions: { deltaBytes, deltaMessages } }, query: { ...query, minScore, hybrid: { enabled: Boolean(hybrid.enabled), vectorWeight: normalizedVectorWeight, textWeight: normalizedTextWeight, candidateMultiplier } }, cache: { enabled: Boolean(cache.enabled), maxEntries: typeof cache.maxEntries === "number" && Number.isFinite(cache.maxEntries) ? Math.max(1, Math.floor(cache.maxEntries)) : void 0 } }; } function resolveMemorySearchConfig(cfg, agentId) { const defaults = cfg.agents?.defaults?.memorySearch; const overrides = resolveAgentConfig(cfg, agentId)?.memorySearch; const resolved = mergeConfig(defaults, overrides, agentId); if (!resolved.enabled) return null; return resolved; } //#endregion //#region src/memory/batch-gemini.ts const GEMINI_BATCH_MAX_REQUESTS = 5e4; const debugEmbeddings$1 = isTruthyEnvValue(process.env.OPENCLAW_DEBUG_MEMORY_EMBEDDINGS); const log$2 = createSubsystemLogger("memory/embeddings"); const debugLog$1 = (message, meta) => { if (!debugEmbeddings$1) return; const suffix = meta ? ` ${JSON.stringify(meta)}` : ""; log$2.raw(`${message}${suffix}`); }; function getGeminiBaseUrl(gemini) { return gemini.baseUrl?.replace(/\/$/, "") ?? ""; } function getGeminiHeaders(gemini, params) { const headers = gemini.headers ? { ...gemini.headers } : {}; if (params.json) { if (!headers["Content-Type"] && !headers["content-type"]) headers["Content-Type"] = "application/json"; } else { delete headers["Content-Type"]; delete headers["content-type"]; } return headers; } function getGeminiUploadUrl(baseUrl) { if (baseUrl.includes("/v1beta")) return baseUrl.replace(/\/v1beta\/?$/, "/upload/v1beta"); return `${baseUrl.replace(/\/$/, "")}/upload`; } function splitGeminiBatchRequests(requests) { if (requests.length <= GEMINI_BATCH_MAX_REQUESTS) return [requests]; const groups = []; for (let i = 0; i < requests.length; i += GEMINI_BATCH_MAX_REQUESTS) groups.push(requests.slice(i, i + GEMINI_BATCH_MAX_REQUESTS)); return groups; } function buildGeminiUploadBody(params) { const boundary = `openclaw-${hashText(params.displayName)}`; const jsonPart = JSON.stringify({ file: { displayName: params.displayName, mimeType: "application/jsonl" } }); const delimiter = `--${boundary}\r\n`; const closeDelimiter = `--${boundary}--\r\n`; const parts = [ `${delimiter}Content-Type: application/json; charset=UTF-8\r\n\r\n${jsonPart}\r\n`, `${delimiter}Content-Type: application/jsonl; charset=UTF-8\r\n\r\n${params.jsonl}\r\n`, closeDelimiter ]; return { body: new Blob([parts.join("")], { type: "multipart/related" }), contentType: `multipart/related; boundary=${boundary}` }; } async function submitGeminiBatch(params) { const baseUrl = getGeminiBaseUrl(params.gemini); const uploadPayload = buildGeminiUploadBody({ jsonl: params.requests.map((request) => JSON.stringify({ key: request.custom_id, request: { content: request.content, task_type: request.taskType } })).join("\n"), displayName: `memory-embeddings-${hashText(String(Date.now()))}` }); const uploadUrl = `${getGeminiUploadUrl(baseUrl)}/files?uploadType=multipart`; debugLog$1("memory embeddings: gemini batch upload", { uploadUrl, baseUrl, requests: params.requests.length }); const fileRes = await fetch(uploadUrl, { method: "POST", headers: { ...getGeminiHeaders(params.gemini, { json: false }), "Content-Type": uploadPayload.contentType }, body: uploadPayload.body }); if (!fileRes.ok) { const text = await fileRes.text(); throw new Error(`gemini batch file upload failed: ${fileRes.status} ${text}`); } const filePayload = await fileRes.json(); const fileId = filePayload.name ?? filePayload.file?.name; if (!fileId) throw new Error("gemini batch file upload failed: missing file id"); const batchBody = { batch: { displayName: `memory-embeddings-${params.agentId}`, inputConfig: { file_name: fileId } } }; const batchEndpoint = `${baseUrl}/${params.gemini.modelPath}:asyncBatchEmbedContent`; debugLog$1("memory embeddings: gemini batch create", { batchEndpoint, fileId }); const batchRes = await fetch(batchEndpoint, { method: "POST", headers: getGeminiHeaders(params.gemini, { json: true }), body: JSON.stringify(batchBody) }); if (batchRes.ok) return await batchRes.json(); const text = await batchRes.text(); if (batchRes.status === 404) throw new Error("gemini batch create failed: 404 (asyncBatchEmbedContent not available for this model/baseUrl). Disable remote.batch.enabled or switch providers."); throw new Error(`gemini batch create failed: ${batchRes.status} ${text}`); } async function fetchGeminiBatchStatus(params) { const statusUrl = `${getGeminiBaseUrl(params.gemini)}/${params.batchName.startsWith("batches/") ? params.batchName : `batches/${params.batchName}`}`; debugLog$1("memory embeddings: gemini batch status", { statusUrl }); const res = await fetch(statusUrl, { headers: getGeminiHeaders(params.gemini, { json: true }) }); if (!res.ok) { const text = await res.text(); throw new Error(`gemini batch status failed: ${res.status} ${text}`); } return await res.json(); } async function fetchGeminiFileContent(params) { const downloadUrl = `${getGeminiBaseUrl(params.gemini)}/${params.fileId.startsWith("files/") ? params.fileId : `files/${params.fileId}`}:download`; debugLog$1("memory embeddings: gemini batch download", { downloadUrl }); const res = await fetch(downloadUrl, { headers: getGeminiHeaders(params.gemini, { json: true }) }); if (!res.ok) { const text = await res.text(); throw new Error(`gemini batch file content failed: ${res.status} ${text}`); } return await res.text(); } function parseGeminiBatchOutput(text) { if (!text.trim()) return []; return text.split("\n").map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line)); } async function waitForGeminiBatch(params) { const start = Date.now(); let current = params.initial; while (true) { const status = current ?? await fetchGeminiBatchStatus({ gemini: params.gemini, batchName: params.batchName }); const state = status.state ?? "UNKNOWN"; if ([ "SUCCEEDED", "COMPLETED", "DONE" ].includes(state)) { const outputFileId = status.outputConfig?.file ?? status.outputConfig?.fileId ?? status.metadata?.output?.responsesFile; if (!outputFileId) throw new Error(`gemini batch ${params.batchName} completed without output file`); return { outputFileId }; } if ([ "FAILED", "CANCELLED", "CANCELED", "EXPIRED" ].includes(state)) { const message = status.error?.message ?? "unknown error"; throw new Error(`gemini batch ${params.batchName} ${state}: ${message}`); } if (!params.wait) throw new Error(`gemini batch ${params.batchName} still ${state}; wait disabled`); if (Date.now() - start > params.timeoutMs) throw new Error(`gemini batch ${params.batchName} timed out after ${params.timeoutMs}ms`); params.debug?.(`gemini batch ${params.batchName} ${state}; waiting ${params.pollIntervalMs}ms`); await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs)); current = void 0; } } async function runWithConcurrency$1(tasks, limit) { if (tasks.length === 0) return []; const resolvedLimit = Math.max(1, Math.min(limit, tasks.length)); const results = Array.from({ length: tasks.length }); let next = 0; let firstError = null; const workers = Array.from({ length: resolvedLimit }, async () => { while (true) { if (firstError) return; const index = next; next += 1; if (index >= tasks.length) return; try { results[index] = await tasks[index](); } catch (err) { firstError = err; return; } } }); await Promise.allSettled(workers); if (firstError) throw firstError; return results; } async function runGeminiEmbeddingBatches(params) { if (params.requests.length === 0) return /* @__PURE__ */ new Map(); const groups = splitGeminiBatchRequests(params.requests); const byCustomId = /* @__PURE__ */ new Map(); const tasks = groups.map((group, groupIndex) => async () => { const batchInfo = await submitGeminiBatch({ gemini: params.gemini, requests: group, agentId: params.agentId }); const batchName = batchInfo.name ?? ""; if (!batchName) throw new Error("gemini batch create failed: missing batch name"); params.debug?.("memory embeddings: gemini batch created", { batchName, state: batchInfo.state, group: groupIndex + 1, groups: groups.length, requests: group.length }); if (!params.wait && batchInfo.state && ![ "SUCCEEDED", "COMPLETED", "DONE" ].includes(batchInfo.state)) throw new Error(`gemini batch ${batchName} submitted; enable remote.batch.wait to await completion`); const completed = batchInfo.state && [ "SUCCEEDED", "COMPLETED", "DONE" ].includes(batchInfo.state) ? { outputFileId: batchInfo.outputConfig?.file ?? batchInfo.outputConfig?.fileId ?? batchInfo.metadata?.output?.responsesFile ?? "" } : await waitForGeminiBatch({ gemini: params.gemini, batchName, wait: params.wait, pollIntervalMs: params.pollIntervalMs, timeoutMs: params.timeoutMs, debug: params.debug, initial: batchInfo }); if (!completed.outputFileId) throw new Error(`gemini batch ${batchName} completed without output file`); const outputLines = parseGeminiBatchOutput(await fetchGeminiFileContent({ gemini: params.gemini, fileId: completed.outputFileId })); const errors = []; const remaining = new Set(group.map((request) => request.custom_id)); for (const line of outputLines) { const customId = line.key ?? line.custom_id ?? line.request_id; if (!customId) continue; remaining.delete(customId); if (line.error?.message) { errors.push(`${customId}: ${line.error.message}`); continue; } if (line.response?.error?.message) { errors.push(`${customId}: ${line.response.error.message}`); continue; } const embedding = line.embedding?.values ?? line.response?.embedding?.values ?? []; if (embedding.length === 0) { errors.push(`${customId}: empty embedding`); continue; } byCustomId.set(customId, embedding); } if (errors.length > 0) throw new Error(`gemini batch ${batchName} failed: ${errors.join("; ")}`); if (remaining.size > 0) throw new Error(`gemini batch ${batchName} missing ${remaining.size} embedding responses`); }); params.debug?.("memory embeddings: gemini batch submit", { requests: params.requests.length, groups: groups.length, wait: params.wait, concurrency: params.concurrency, pollIntervalMs: params.pollIntervalMs, timeoutMs: params.timeoutMs }); await runWithConcurrency$1(tasks, params.concurrency); return byCustomId; } //#endregion //#region src/infra/retry.ts const DEFAULT_RETRY_CONFIG = { attempts: 3, minDelayMs: 300, maxDelayMs: 3e4, jitter: 0 }; const asFiniteNumber = (value) => typeof value === "number" && Number.isFinite(value) ? value : void 0; const clampNumber = (value, fallback, min, max) => { const next = asFiniteNumber(value); if (next === void 0) return fallback; const floor = typeof min === "number" ? min : Number.NEGATIVE_INFINITY; const ceiling = typeof max === "number" ? max : Number.POSITIVE_INFINITY; return Math.min(Math.max(next, floor), ceiling); }; function resolveRetryConfig(defaults = DEFAULT_RETRY_CONFIG, overrides) { const attempts = Math.max(1, Math.round(clampNumber(overrides?.attempts, defaults.attempts, 1))); const minDelayMs = Math.max(0, Math.round(clampNumber(overrides?.minDelayMs, defaults.minDelayMs, 0))); return { attempts, minDelayMs, maxDelayMs: Math.max(minDelayMs, Math.round(clampNumber(overrides?.maxDelayMs, defaults.maxDelayMs, 0))), jitter: clampNumber(overrides?.jitter, defaults.jitter, 0, 1) }; } function applyJitter(delayMs, jitter) { if (jitter <= 0) return delayMs; const offset = (Math.random() * 2 - 1) * jitter; return Math.max(0, Math.round(delayMs * (1 + offset))); } async function retryAsync(fn, attemptsOrOptions = 3, initialDelayMs = 300) { if (typeof attemptsOrOptions === "number") { const attempts = Math.max(1, Math.round(attemptsOrOptions)); let lastErr; for (let i = 0; i < attempts; i += 1) try { return await fn(); } catch (err) { lastErr = err; if (i === attempts - 1) break; await sleep(initialDelayMs * 2 ** i); } throw lastErr ?? /* @__PURE__ */ new Error("Retry failed"); } const options = attemptsOrOptions; const resolved = resolveRetryConfig(DEFAULT_RETRY_CONFIG, options); const maxAttempts = resolved.attempts; const minDelayMs = resolved.minDelayMs; const maxDelayMs = Number.isFinite(resolved.maxDelayMs) && resolved.maxDelayMs > 0 ? resolved.maxDelayMs : Number.POSITIVE_INFINITY; const jitter = resolved.jitter; const shouldRetry = options.shouldRetry ?? (() => true); let lastErr; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) try { return await fn(); } catch (err) { lastErr = err; if (attempt >= maxAttempts || !shouldRetry(err, attempt)) break; const retryAfterMs = options.retryAfterMs?.(err); const baseDelay = typeof retryAfterMs === "number" && Number.isFinite(retryAfterMs) ? Math.max(retryAfterMs, minDelayMs) : minDelayMs * 2 ** (attempt - 1); let delay = Math.min(baseDelay, maxDelayMs); delay = applyJitter(delay, jitter); delay = Math.min(Math.max(delay, minDelayMs), maxDelayMs); options.onRetry?.({ attempt, maxAttempts, delayMs: delay, err, label: options.label }); await sleep(delay); } throw lastErr ?? /* @__PURE__ */ new Error("Retry failed"); } //#endregion //#region src/memory/batch-openai.ts const OPENAI_BATCH_ENDPOINT = "/v1/embeddings"; const OPENAI_BATCH_COMPLETION_WINDOW = "24h"; const OPENAI_BATCH_MAX_REQUESTS = 5e4; function getOpenAiBaseUrl(openAi) { return openAi.baseUrl?.replace(/\/$/, "") ?? ""; } function getOpenAiHeaders(openAi, params) { const headers = openAi.headers ? { ...openAi.headers } : {}; if (params.json) { if (!headers["Content-Type"] && !headers["content-type"]) headers["Content-Type"] = "application/json"; } else { delete headers["Content-Type"]; delete headers["content-type"]; } return headers; } function splitOpenAiBatchRequests(requests) { if (requests.length <= OPENAI_BATCH_MAX_REQUESTS) return [requests]; const groups = []; for (let i = 0; i < requests.length; i += OPENAI_BATCH_MAX_REQUESTS) groups.push(requests.slice(i, i + OPENAI_BATCH_MAX_REQUESTS)); return groups; } async function submitOpenAiBatch(params) { const baseUrl = getOpenAiBaseUrl(params.openAi); const jsonl = params.requests.map((request) => JSON.stringify(request)).join("\n"); const form = new FormData(); form.append("purpose", "batch"); form.append("file", new Blob([jsonl], { type: "application/jsonl" }), `memory-embeddings.${hashText(String(Date.now()))}.jsonl`); const fileRes = await fetch(`${baseUrl}/files`, { method: "POST", headers: getOpenAiHeaders(params.openAi, { json: false }), body: form }); if (!fileRes.ok) { const text = await fileRes.text(); throw new Error(`openai batch file upload failed: ${fileRes.status} ${text}`); } const filePayload = await fileRes.json(); if (!filePayload.id) throw new Error("openai batch file upload failed: missing file id"); return await (await retryAsync(async () => { const res = await fetch(`${baseUrl}/batches`, { method: "POST", headers: getOpenAiHeaders(params.openAi, { json: true }), body: JSON.stringify({ input_file_id: filePayload.id, endpoint: OPENAI_BATCH_ENDPOINT, completion_window: OPENAI_BATCH_COMPLETION_WINDOW, metadata: { source: "openclaw-memory", agent: params.agentId } }) }); if (!res.ok) { const text = await res.text(); const err = /* @__PURE__ */ new Error(`openai batch create failed: ${res.status} ${text}`); err.status = res.status; throw err; } return res; }, { attempts: 3, minDelayMs: 300, maxDelayMs: 2e3, jitter: .2, shouldRetry: (err) => { const status = err.status; return status === 429 || typeof status === "number" && status >= 500; } })).json(); } async function fetchOpenAiBatchStatus(params) { const baseUrl = getOpenAiBaseUrl(params.openAi); const res = await fetch(`${baseUrl}/batches/${params.batchId}`, { headers: getOpenAiHeaders(params.openAi, { json: true }) }); if (!res.ok) { const text = await res.text(); throw new Error(`openai batch status failed: ${res.status} ${text}`); } return await res.json(); } async function fetchOpenAiFileContent(params) { const baseUrl = getOpenAiBaseUrl(params.openAi); const res = await fetch(`${baseUrl}/files/${params.fileId}/content`, { headers: getOpenAiHeaders(params.openAi, { json: true }) }); if (!res.ok) { const text = await res.text(); throw new Error(`openai batch file content failed: ${res.status} ${text}`); } return await res.text(); } function parseOpenAiBatchOutput(text) { if (!text.trim()) return []; return text.split("\n").map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line)); } async function readOpenAiBatchError(params) { try { const first = parseOpenAiBatchOutput(await fetchOpenAiFileContent({ openAi: params.openAi, fileId: params.errorFileId })).find((line) => line.error?.message || line.response?.body?.error); return first?.error?.message ?? (typeof first?.response?.body?.error?.message === "string" ? first?.response?.body?.error?.message : void 0); } catch (err) { const message = err instanceof Error ? err.message : String(err); return message ? `error file unavailable: ${message}` : void 0; } } async function waitForOpenAiBatch(params) { const start = Date.now(); let current = params.initial; while (true) { const status = current ?? await fetchOpenAiBatchStatus({ openAi: params.openAi, batchId: params.batchId }); const state = status.status ?? "unknown"; if (state === "completed") { if (!status.output_file_id) throw new Error(`openai batch ${params.batchId} completed without output file`); return { outputFileId: status.output_file_id, errorFileId: status.error_file_id ?? void 0 }; } if ([ "failed", "expired", "cancelled", "canceled" ].includes(state)) { const detail = status.error_file_id ? await readOpenAiBatchError({ openAi: params.openAi, errorFileId: status.error_file_id }) : void 0; const suffix = detail ? `: ${detail}` : ""; throw new Error(`openai batch ${params.batchId} ${state}${suffix}`); } if (!params.wait) throw new Error(`openai batch ${params.batchId} still ${state}; wait disabled`); if (Date.now() - start > params.timeoutMs) throw new Error(`openai batch ${params.batchId} timed out after ${params.timeoutMs}ms`); params.debug?.(`openai batch ${params.batchId} ${state}; waiting ${params.pollIntervalMs}ms`); await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs)); current = void 0; } } async function runWithConcurrency(tasks, limit) { if (tasks.length === 0) return []; const resolvedLimit = Math.max(1, Math.min(limit, tasks.length)); const results = Array.from({ length: tasks.length }); let next = 0; let firstError = null; const workers = Array.from({ length: resolvedLimit }, async () => { while (true) { if (firstError) return; const index = next; next += 1; if (index >= tasks.length) return; try { results[index] = await tasks[index](); } catch (err) { firstError = err; return; } } }); await Promise.allSettled(workers); if (firstError) throw firstError; return results; } async function runOpenAiEmbeddingBatches(params) { if (params.requests.length === 0) return /* @__PURE__ */ new Map(); const groups = splitOpenAiBatchRequests(params.requests); const byCustomId = /* @__PURE__ */ new Map(); const tasks = groups.map((group, groupIndex) => async () => { const batchInfo = await submitOpenAiBatch({ openAi: params.openAi, requests: group, agentId: params.agentId }); if (!batchInfo.id) throw new Error("openai batch create failed: missing batch id"); params.debug?.("memory embeddings: openai batch created", { batchId: batchInfo.id, status: batchInfo.status, group: groupIndex + 1, groups: groups.length, requests: group.length }); if (!params.wait && batchInfo.status !== "completed") throw new Error(`openai batch ${batchInfo.id} submitted; enable remote.batch.wait to await completion`); const completed = batchInfo.status === "completed" ? { outputFileId: batchInfo.output_file_id ?? "", errorFileId: batchInfo.error_file_id ?? void 0 } : await waitForOpenAiBatch({ openAi: params.openAi, batchId: batchInfo.id, wait: params.wait, pollIntervalMs: params.pollIntervalMs, timeoutMs: params.timeoutMs, debug: params.debug, initial: batchInfo }); if (!completed.outputFileId) throw new Error(`openai batch ${batchInfo.id} completed without output file`); const outputLines = parseOpenAiBatchOutput(await fetchOpenAiFileContent({ openAi: params.openAi, fileId: completed.outputFileId })); const errors = []; const remaining = new Set(group.map((request) => request.custom_id)); for (const line of outputLines) { const customId = line.custom_id; if (!customId) continue; remaining.delete(customId); if (line.error?.message) { errors.push(`${customId}: ${line.error.message}`); continue; } const response = line.response; if ((response?.status_code ?? 0) >= 400) { const message = response?.body?.error?.message ?? (typeof response?.body === "string" ? response.body : void 0) ?? "unknown error"; errors.push(`${customId}: ${message}`); continue; } const embedding = (response?.body?.data ?? [])[0]?.embedding ?? []; if (embedding.length === 0) { errors.push(`${customId}: empty embedding`); continue; } byCustomId.set(customId, embedding); } if (errors.length > 0) throw new Error(`openai batch ${batchInfo.id} failed: ${errors.join("; ")}`); if (remaining.size > 0) throw new Error(`openai batch ${batchInfo.id} missing ${remaining.size} embedding responses`); }); params.debug?.("memory embeddings: openai batch submit", { requests: params.requests.length, groups: groups.length, wait: params.wait, concurrency: params.concurrency, pollIntervalMs: params.pollIntervalMs, timeoutMs: params.timeoutMs }); await runWithConcurrency(tasks, params.concurrency); return byCustomId; } //#endregion //#region src/memory/embeddings-gemini.ts const DEFAULT_GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"; const DEFAULT_GEMINI_EMBEDDING_MODEL = "gemini-embedding-001"; const debugEmbeddings = isTruthyEnvValue(process.env.OPENCLAW_DEBUG_MEMORY_EMBEDDINGS); const log$1 = createSubsystemLogger("memory/embeddings"); const debugLog = (message, meta) => { if (!debugEmbeddings) return; const suffix = meta ? ` ${JSON.stringify(meta)}` : ""; log$1.raw(`${message}${suffix}`); }; function resolveRemoteApiKey(remoteApiKey) { const trimmed = remoteApiKey?.trim(); if (!trimmed) return; if (trimmed === "GOOGLE_API_KEY" || trimmed === "GEMINI_API_KEY") return process.env[trimmed]?.trim(); return trimmed; } function normalizeGeminiModel(model) { const trimmed = model.trim(); if (!trimmed) return DEFAULT_GEMINI_EMBEDDING_MODEL; const withoutPrefix = trimmed.replace(/^models\//, ""); if (withoutPrefix.startsWith("gemini/")) return withoutPrefix.slice(7); if (withoutPrefix.startsWith("google/")) return withoutPrefix.slice(7); return withoutPrefix; } function normalizeGeminiBaseUrl(raw) { const trimmed = raw.replace(/\/+$/, ""); const openAiIndex = trimmed.indexOf("/openai"); if (openAiIndex > -1) return trimmed.slice(0, openAiIndex); return trimmed; } function buildGeminiModelPath(model) { return model.startsWith("models/") ? model : `models/${model}`; } async function createGeminiEmbeddingProvider(options) { const client = await resolveGeminiEmbeddingClient(options); const baseUrl = client.baseUrl.replace(/\/$/, ""); const embedUrl = `${baseUrl}/${client.modelPath}:embedContent`; const batchUrl = `${baseUrl}/${client.modelPath}:batchEmbedContents`; const embedQuery = async (text) => { if (!text.trim()) return []; const res = await fetch(embedUrl, { method: "POST", headers: client.headers, body: JSON.stringify({ content: { parts: [{ text }] }, taskType: "RETRIEVAL_QUERY" }) }); if (!res.ok) { const payload = await res.text(); throw new Error(`gemini embeddings failed: ${res.status} ${payload}`); } return (await res.json()).embedding?.values ?? []; }; const embedBatch = async (texts) => { if (texts.length === 0) return []; const requests = texts.map((text) => ({ model: client.modelPath, content: { parts: [{ text }] }, taskType: "RETRIEVAL_DOCUMENT" })); const res = await fetch(batchUrl, { method: "POST", headers: client.headers, body: JSON.stringify({ requests }) }); if (!res.ok) { const payload = await res.text(); throw new Error(`gemini embeddings failed: ${res.status} ${payload}`); } const payload = await res.json(); const embeddings = Array.isArray(payload.embeddings) ? payload.embeddings : []; return texts.map((_, index) => embeddings[index]?.values ?? []); }; return { provider: { id: "gemini", model: client.model, embedQuery, embedBatch }, client }; } async function resolveGeminiEmbeddingClient(options) { const remote = options.remote; const remoteApiKey = resolveRemoteApiKey(remote?.apiKey); const remoteBaseUrl = remote?.baseUrl?.trim(); const apiKey = remoteApiKey ? remoteApiKey : requireApiKey(await resolveApiKeyForProvider({ provider: "google", cfg: options.config, agentDir: options.agentDir }), "google"); const providerConfig = options.config.models?.providers?.google; const rawBaseUrl = remoteBaseUrl || providerConfig?.baseUrl?.trim() || DEFAULT_GEMINI_BASE_URL; const baseUrl = normalizeGeminiBaseUrl(rawBaseUrl); const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers); const headers = { "Content-Type": "application/json", "x-goog-api-key": apiKey, ...headerOverrides }; const model = normalizeGeminiModel(options.model); const modelPath = buildGeminiModelPath(model); debugLog("memory embeddings: gemini client", { rawBaseUrl, baseUrl, model, modelPath, embedEndpoint: `${baseUrl}/${modelPath}:embedContent`, batchEndpoint: `${baseUrl}/${modelPath}:batchEmbedContents` }); return { baseUrl, headers, model, modelPath }; } //#endregion //#region src/memory/embeddings-openai.ts const DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small"; const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1"; function normalizeOpenAiModel(model) { const trimmed = model.trim(); if (!trimmed) return DEFAULT_OPENAI_EMBEDDING_MODEL; if (trimmed.startsWith("openai/")) return trimmed.slice(7); return trimmed; } async function createOpenAiEmbeddingProvider(options) { const client = await resolveOpenAiEmbeddingClient(options); const url = `${client.baseUrl.replace(/\/$/, "")}/embeddings`; const embed = async (input) => { if (input.length === 0) return []; const res = await fetch(url, { method: "POST", headers: client.headers, body: JSON.stringify({ model: client.model, input }) }); if (!res.ok) { const text = await res.text(); throw new Error(`openai embeddings failed: ${res.status} ${text}`); } return ((await res.json()).data ?? []).map((entry) => entry.embedding ?? []); }; return { provider: { id: "openai", model: client.model, embedQuery: async (text) => { const [vec] = await embed([text]); return vec ?? []; }, embedBatch: embed }, client }; } async function resolveOpenAiEmbeddingClient(options) { const remote = options.remote; const remoteApiKey = remote?.apiKey?.trim(); const remoteBaseUrl = remote?.baseUrl?.trim(); const apiKey = remoteApiKey ? remoteApiKey : requireApiKey(await resolveApiKeyForProvider({ provider: "openai", cfg: options.config, agentDir: options.agentDir }), "openai"); const providerConfig = options.config.models?.providers?.openai; const baseUrl = remoteBaseUrl || providerConfig?.baseUrl?.trim() || DEFAULT_OPENAI_BASE_URL; const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers); return { baseUrl, headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, ...headerOverrides }, model: normalizeOpenAiModel(options.model) }; } //#endregion //#region src/memory/node-llama.ts async function importNodeLlamaCpp() { return import("node-llama-cpp"); } //#endregion //#region src/memory/embeddings.ts function sanitizeAndNormalizeEmbedding(vec) { const sanitized = vec.map((value) => Number.isFinite(value) ? value : 0); const magnitude = Math.sqrt(sanitized.reduce((sum, value) => sum + value * value, 0)); if (magnitude < 1e-10) return sanitized; return sanitized.map((value) => value / magnitude); } const DEFAULT_LOCAL_MODEL = "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf"; function canAutoSelectLocal(options) { const modelPath = options.local?.modelPath?.trim(); if (!modelPath) return false; if (/^(hf:|https?:)/i.test(modelPath)) return false; const resolved = resolveUserPath(modelPath); try { return fs.statSync(resolved).isFile(); } catch { return false; } } function isMissingApiKeyError(err) { return formatError(err).includes("No API key found for provider"); } async function createLocalEmbeddingProvider(options) { const modelPath = options.local?.modelPath?.trim() || DEFAULT_LOCAL_MODEL; const modelCacheDir = options.local?.modelCacheDir?.trim(); const { getLlama, resolveModelFile, LlamaLogLevel } = await importNodeLlamaCpp(); let llama = null; let embeddingModel = null; let embeddingContext = null; const ensureContext = async () => { if (!llama) llama = await getLlama({ logLevel: LlamaLogLevel.error }); if (!embeddingModel) { const resolved = await resolveModelFile(modelPath, modelCacheDir || void 0); embeddingModel = await llama.loadModel({ modelPath: resolved }); } if (!embeddingContext) embeddingContext = await embeddingModel.createEmbeddingContext(); return embeddingContext; }; return { id: "local", model: modelPath, embedQuery: async (text) => { const embedding = await (await ensureContext()).getEmbeddingFor(text); return sanitizeAndNormalizeEmbedding(Array.from(embedding.vector)); }, embedBatch: async (texts) => { const ctx = await ensureContext(); return await Promise.all(texts.map(async (text) => { const embedding = await ctx.getEmbeddingFor(text); return sanitizeAndNormalizeEmbedding(Array.from(embedding.vector)); })); } }; } async function createEmbeddingProvider(options) { const requestedProvider = options.provider; const fallback = options.fallback; const createProvider = async (id) => { if (id === "local") return { provider: await createLocalEmbeddingProvider(options) }; if (id === "gemini") { const { provider, client } = await createGeminiEmbeddingProvider(options); return { provider, gemini: client }; } const { provider, client } = await createOpenAiEmbeddingProvider(options); return { provider, openAi: client }; }; const formatPrimaryError = (err, provider) => provider === "local" ? formatLocalSetupError(err) : formatError(err); if (requestedProvider === "auto") { const missingKeyErrors = []; let localError = null; if (canAutoSelectLocal(options)) try { return { ...await createProvider("local"), requestedProvider }; } catch (err) { localError = formatLocalSetupError(err); } for (const provider of ["openai", "gemini"]) try { return { ...await createProvider(provider), requestedProvider }; } catch (err) { const message = formatPrimaryError(err, provider); if (isMissingApiKeyError(err)) { missingKeyErrors.push(message); continue; } throw new Error(message, { cause: err }); } const details = [...missingKeyErrors, localError].filter(Boolean); if (details.length > 0) throw new Error(details.join("\n\n")); throw new Error("No embeddings provider available."); } try { return { ...await createProvider(requestedProvider), requestedProvider }; } catch (primaryErr) { const reason = formatPrimaryError(primaryErr, requestedProvider); if (fallback && fallback !== "none" && fallback !== requestedProvider) try { return { ...await createProvider(fallback), requestedProvider, fallbackFrom: requestedProvider, fallbackReason: reason }; } catch (fallbackErr) { throw new Error(`${reason}\n\nFallback to ${fallback} failed: ${formatError(fallbackErr)}`, { cause: fallbackErr }); } throw new Error(reason, { cause: primaryErr }); } } function formatError(err) { if (err instanceof Error) return err.message; return String(err); } function isNodeLlamaCppMissing(err) { if (!(err instanceof Error)) return false; if (err.code === "ERR_MODULE_NOT_FOUND") return err.message.includes("node-llama-cpp"); return false; } function formatLocalSetupError(err) { const detail = formatError(err); const missing = isNodeLlamaCppMissing(err); return [ "Local embeddings unavailable.", missing ? "Reason: optional dependency node-llama-cpp is missing (or failed to install)." : detail ? `Reason: ${detail}` : void 0, missing && detail ? `Detail: ${detail}` : null, "To enable local embeddings:", "1) Use Node 22 LTS (recommended for installs/updates)", missing ? "2) Reinstall OpenClaw (this should install node-llama-cpp): npm i -g openclaw@latest" : null, "3) If you use pnpm: pnpm approve-builds (select node-llama-cpp), then pnpm rebuild node-llama-cpp", "Or set agents.defaults.memorySearch.provider = \"openai\" (remote)." ].filter(Boolean).join("\n"); } //#endregion //#region src/memory/hybrid.ts function buildFtsQuery(raw) { const tokens = raw.match(/[A-Za-z0-9_]+/g)?.map((t) => t.trim()).filter(Boolean) ?? []; if (tokens.length === 0) return null; return tokens.map((t) => `"${t.replaceAll("\"", "")}"`).join(" AND "); } function bm25RankToScore(rank) { return 1 / (1 + (Number.isFinite(rank) ? Math.max(0, rank) : 999)); } function mergeHybridResults(params) { const byId = /* @__PURE__ */ new Map(); for (const r of params.vector) byId.set(r.id, { id: r.id, path: r.path, startLine: r.startLine, endLine: r.endLine, source: r.source, snippet: r.snippet, vectorScore: r.vectorScore, textScore: 0 }); for (const r of params.keyword) { const existing = byId.get(r.id); if (existing) { existing.textScore = r.textScore; if (r.snippet && r.snippet.length > 0) existing.snippet = r.snippet; } else byId.set(r.id, { id: r.id, path: r.path, startLine: r.startLine, endLine: r.endLine, source: r.source, snippet: r.snippet, vectorScore: 0, textScore: r.textScore }); } return Array.from(byId.values()).map((entry) => { const score = params.vectorWeight * entry.vectorScore + params.textWeight * entry.textScore; return { path: entry.path, startLine: entry.startLine, endLine: entry.endLine, score, snippet: entry.snippet, source: entry.source }; }).toSorted((a, b) => b.score - a.score); } //#endregion //#region src/memory/manager-search.ts const vectorToBlob$1 = (embedding) => Buffer.from(new Float32Array(embedding).buffer); async function searchVector(params) { if (params.queryVec.length === 0 || params.limit <= 0) return []; if (await params.ensureVectorReady(params.queryVec.length)) return params.db.prepare(`SELECT c.id, c.path, c.start_line, c.end_line, c.text, c.source, vec_distance_cosine(v.embedding, ?) AS dist FROM ${params.vectorTable} v\n JOIN chunks c ON c.id = v.id\n WHERE c.model = ?${params.sourceFilterVec.sql}\n ORDER BY dist ASC\n LIMIT ?`).all(vectorToBlob$1(params.queryVec), params.providerModel, ...params.sourceFilterVec.params, params.limit).map((row) => ({ id: row.id, path: row.path, startLine: row.start_line, endLine: row.end_line, score: 1 - row.dist, snippet: truncateUtf16Safe(row.text, params.snippetMaxChars), source: row.source })); return listChunks({ db: params.db, providerModel: params.providerModel, sourceFilter: params.sourceFilterChunks }).map((chunk) => ({ chunk, score: cosineSimilarity(params.queryVec, chunk.embedding) })).filter((entry) => Number.isFinite(entry.score)).toSorted((a, b) => b.score - a.score).slice(0, params.limit).map((entry) => ({ id: entry.chunk.id, path: entry.chunk.path, startLine: entry.chunk.startLine, endLine: entry.chunk.endLine, score: entry.score, snippet: truncateUtf16Safe(entry.chunk.text, params.snippetMaxChars), source: entry.chunk.source })); } function listChunks(params) { return params.db.prepare(`SELECT id, path, start_line, end_line, text, embedding, source FROM chunks WHERE model = ?${params.sourceFilter.sql}`).all(params.providerModel, ...params.sourceFilter.params).map((row) => ({ id: row.id, path: row.path, startLine: row.start_line, endLine: row.end_line, text: row.text, embedding: parseEmbedding(row.embedding), source: row.source })); } async function searchKeyword(params) { if (params.limit <= 0) return []; const ftsQuery = params.buildFtsQuery(params.query); if (!ftsQuery) return []; return params.db.prepare(`SELECT id, path, source, start_line, end_line, text,\n bm25(${params.ftsTable}) AS rank\n FROM ${params.ftsTable}\n WHERE ${params.ftsTable} MATCH ? AND model = ?${params.sourceFilter.sql}\n ORDER BY rank ASC\n LIMIT ?`).all(ftsQuery, params.providerModel, ...params.sourceFilter.params, params.limit).map((row) => { const textScore = params.bm25RankToScore(row.rank); return { id: row.id, path: row.path, startLine: row.start_line, endLine: row.end_line, score: textScore, textScore, snippet: truncateUtf16Safe(row.text, params.snippetMaxChars), source: row.source }; }); } //#endregion //#region src/memory/memory-schema.ts function ensureMemoryIndexSchema(params) { params.db.exec(` CREATE TABLE IF NOT EXISTS meta ( key TEXT PRIMARY KEY, value TEXT NOT NULL ); `); params.db.exec(` CREATE TABLE IF NOT EXISTS files ( path TEXT PRIMARY KEY, source TEXT NOT NULL DEFAULT 'memory', hash TEXT NOT NULL, mtime INTEGER NOT NULL, size INTEGER NOT NULL ); `); params.db.exec(` CREATE TABLE IF NOT EXISTS chunks ( id TEXT PRIMARY KEY, path TEXT NOT NULL, source TEXT NOT NULL DEFAULT 'memory', start_line INTEGER NOT NULL, end_line INTEGER NOT NULL, hash TEXT NOT NULL, model TEXT NOT NULL, text TEXT NOT NULL, embedding TEXT NOT NULL, updated_at INTEGER NOT NULL ); `); params.db.exec(` CREATE TABLE IF NOT EXISTS ${params.embeddingCacheTable} ( provider TEXT NOT NULL, model TEXT NOT NULL, provider_key TEXT NOT NULL, hash TEXT NOT NULL, embedding TEXT NOT NULL, dims INTEGER, updated_at INTEGER NOT NULL, PRIMARY KEY (provider, model, provider_key, hash) ); `); params.db.exec(`CREATE INDEX IF NOT EXISTS idx_embedding_cache_updated_at ON ${params.embeddingCacheTable}(updated_at);`); let ftsAvailable = false; let ftsError; if (params.ftsEnabled) try { params.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${params.ftsTable} USING fts5(\n text,\n id UNINDEXED,\n path UNINDEXED,\n source UNINDEXED,\n model UNINDEXED,\n start_line UNINDEXED,\n end_line UNINDEXED\n);`); ftsAvailable = true; } catch (err) { const message = err instanceof Error ? err.message : String(err); ftsAvailable = false; ftsError = message; } ensureColumn(params.db, "files", "source", "TEXT NOT NULL DEFAULT 'memory'"); ensureColumn(params.db, "chunks", "source", "TEXT NOT NULL DEFAULT 'memory'"); params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks(path);`); params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source);`); return { ftsAvailable, ...ftsError ? { ftsError } : {} }; } function ensureColumn(db, table, column, definition) { if (db.prepare(`PRAGMA table_info(${table})`).all().some((row) => row.name === column)) return; db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`); } //#endregion //#region src/memory/sqlite-vec.ts async function loadSqliteVecExtension(params) { try { const sqliteVec = await import("sqlite-vec"); const resolvedPath = params.extensionPath?.trim() ? params.extensionPath.trim() : void 0; const extensionPath = resolvedPath ?? sqliteVec.getLoadablePath(); params.db.enableLoadExtension(true); if (resolvedPath) params.db.loadExtension(extensionPath); else sqliteVec.load(params.db); return { ok: true, extensionPath }; } catch (err) { return { ok: false, error: err instanceof Error ? err.message : String(err) }; } } //#endregion //#region src/memory/manager.ts const META_KEY = "memory_index_meta_v1"; const SNIPPET_MAX_CHARS = 700; const VECTOR_TABLE = "chunks_vec"; const FTS_TABLE = "chunks_fts"; const EMBEDDING_CACHE_TABLE = "embedding_cache"; const SESSION_DIRTY_DEBOUNCE_MS = 5e3; const EMBEDDING_BATCH_MAX_TOKENS = 8e3; const EMBEDDING_APPROX_CHARS_PER_TOKEN = 1; const EMBEDDING_INDEX_CONCURRENCY = 4; const EMBEDDING_RETRY_MAX_ATTEMPTS = 3; const EMBEDDING_RETRY_BASE_DELAY_MS = 500; const EMBEDDING_RETRY_MAX_DELAY_MS = 8e3; const BATCH_FAILURE_LIMIT = 2; const SESSION_DELTA_READ_CHUNK_BYTES = 64 * 1024; const VECTOR_LOAD_TIMEOUT_MS = 3e4; const EMBEDDING_QUERY_TIMEOUT_REMOTE_MS = 6e4; const EMBEDDING_QUERY_TIMEOUT_LOCAL_MS = 5 * 6e4; const EMBEDDING_BATCH_TIMEOUT_REMOTE_MS = 2 * 6e4; const EMBEDDING_BATCH_TIMEOUT_LOCAL_MS = 10 * 6e4; const log = createSubsystemLogger("memory"); const INDEX_CACHE = /* @__PURE__ */ new Map(); const vectorToBlob = (embedding) => Buffer.from(new Float32Array(embedding).buffer); var MemoryIndexManager = class MemoryIndexManager { static async get(params) { const { cfg, agentId } = params; const settings = resolveMemorySearchConfig(cfg, agentId); if (!settings) return null; const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId); const key = `${agentId}:${workspaceDir}:${JSON.stringify(settings)}`; const existing = INDEX_CACHE.get(key); if (existing) return existing; const manager = new MemoryIndexManager({ cacheKey: key, cfg, agentId, workspaceDir, settings, providerResult: await createEmbeddingProvider({ config: cfg, agentDir: resolveAgentDir(cfg, agentId), provider: settings.provider, remote: settings.remote, model: settings.model, fallback: settings.fallback, local: settings.local }) }); INDEX_CACHE.set(key, manager); return manager; } constructor(params) { this.batchFailureCount = 0; this.batchFailureLock = Promise.resolve(); this.vectorReady = null; this.watcher = null; this.watchTimer = null; this.sessionWatchTimer = null; this.sessionUnsubscribe = null; this.intervalTimer = null; this.closed = false; this.dirty = false; this.sessionsDirty = false; this.sessionsDirtyFiles = /* @__PURE__ */ new Set(); this.sessionPendingFiles = /* @__PURE__ */ new Set(); this.sessionDeltas = /* @__PURE__ */ new Map(); this.sessionWarm = /* @__PURE__ */ new Set(); this.syncing = null; this.cacheKey = params.cacheKey; this.cfg = params.cfg; this.agentId = params.agentId; this.workspaceDir = params.workspaceDir; this.settings = params.settings; this.provider = params.providerResult.provider; this.requestedProvider = params.providerResult.requestedProvider; this.fallbackFrom = params.providerResult.fallbackFrom; this.fallbackReason = params.providerResult.fallbackReason; this.openAi = params.providerResult.openAi; this.gemini = params.providerResult.gemini; this.sources = new Set(params.settings.sources); this.db = this.openDatabase(); this.providerKey = this.computeProviderKey(); this.cache = { enabled: params.settings.cache.enabled, maxEntries: params.settings.cache.maxEntries }; this.fts = { enabled: params.settings.query.hybrid.enabled, available: false }; this.ensureSchema(); this.vector = { enabled: params.settings.store.vector.enabled, available: null, extensionPath: params.settings.store.vector.extensionPath }; const meta = this.readMeta(); if (meta?.vectorDims) this.vector.dims = meta.vectorDims; this.ensureWatcher(); this.ensureSessionListener(); this.ensureIntervalSync(); this.dirty = this.sources.has("memory"); this.batch = this.resolveBatchConfig(); } async warmSession(sessionKey) { if (!this.settings.sync.onSessionStart) return; const key = sessionKey?.trim() || ""; if (key && this.sessionWarm.has(key)) return; this.sync({ reason: "session-start" }).catch((err) => { log.warn(`memory sync failed (session-start): ${String(err)}`); }); if (key) this.sessionWarm.add(key); } async search(query, opts) { this.warmSession(opts?.sessionKey); if (this.settings.sync.onSearch && (this.dirty || this.sessionsDirty)) this.sync({ reason: "search" }).catch((err) => { log.warn(`memory sync failed (search): ${String(err)}`); }); const cleaned = query.trim(); if (!cleaned) return []; const minScore = opts?.minScore ?? this.settings.query.minScore; const maxResults = opts?.maxResults ?? this.settings.query.maxResults; const hybrid = this.settings.query.hybrid; const candidates = Math.min(200, Math.max(1, Math.floor(maxResults * hybrid.candidateMultiplier))); const keywordResults = hybrid.enabled ? await this.searchKeyword(cleaned, candidates).catch(() => []) : []; const queryVec = await this.embedQueryWithTimeout(cleaned); const vectorResults = queryVec.some((v) => v !== 0) ? await this.searchVector(queryVec, candidates).catch(() => []) : []; if (!hybrid.enabled) return vectorResults.filter((entry) => entry.score >= minScore).slice(0, maxResults); return this.mergeHybridResults({ vector: vectorResults, keyword: keywordResults, vectorWeight: hybrid.vectorWeight, textWeight: hybrid.textWeight }).filter((entry) => entry.score >= minScore).slice(0, maxResults); } async searchVector(queryVec, limit) { return (await searchVector({ db: this.db, vectorTable: VECTOR_TABLE, providerModel: this.provider.model, queryVec, limit, snippetMaxChars: SNIPPET_MAX_CHARS, ensureVectorReady: async (dimensions) => await this.ensureVectorReady(dimensions), sourceFilterVec: this.buildSourceFilter("c"), sourceFilterChunks: this.buildSourceFilter() })).map((entry) => entry); } buildFtsQuery(raw) { return buildFtsQuery(raw); } async searchKeyword(query, limit) { if (!this.fts.enabled || !this.fts.available) return []; const sourceFilter = this.buildSourceFilter(); return (await searchKeyword({ db: this.db, ftsTable: FTS_TABLE, providerModel: this.provider.model, query, limit, snippetMaxChars: SNIPPET_MAX_CHARS, sourceFilter, buildFtsQuery: (raw) => this.buildFtsQuery(raw), bm25RankToScore })).map((entry) => entry); } mergeHybridResults(params) { return mergeHybridResults({ vector: params.vector.map((r) => ({ id: r.id, path: r.path, startLine: r.startLine, endLine: r.endLine, source: r.source, snippet: r.snippet, vectorScore: r.score })), keyword: params.keyword.map((r) => ({ id: r.id, path: r.path, startLine: r.startLine, endLine: r.endLine, source: r.source, snippet: r.snippet, textScore: r.textScore })), vectorWeight: params.vectorWeight, textWeight: params.textWeight }).map((entry) => entry); } async sync(params) { if (this.syncing) return this.syncing; this.syncing = this.runSync(params).finally(() => { this.syncing = null; }); return this.syncing; } async readFile(params) { const rawPath = params.relPath.trim(); if (!rawPath) throw new Error("path required"); const absPath = path.isAbsolute(rawPath) ? path.resolve(rawPath) : path.resolve(this.workspaceDir, rawPath); const relPath = path.relative(this.workspaceDir, absPath).replace(/\\/g, "/"); const allowedWorkspace = relPath.length > 0 && !relPath.startsWith("..") && !path.isAbsolute(relPath) && isMemoryPath(relPath); let allowedAdditional = false; if (!allowedWorkspace && this.settings.extraPaths.length > 0) { const additionalPaths = normalizeExtraMemoryPaths(this.workspaceDir, this.settings.extraPaths); for (const additionalPath of additionalPaths) try { const stat = await fs$1.lstat(additionalPath); if (stat.isSymbolicLink()) continue; if (stat.isDirectory()) { if (absPath === additionalPath || absPath.startsWith(`${additionalPath}${path.sep}`)) { allowedAdditional = true; break; } continue; } if (stat.isFile()) { if (absPath === additionalPath && absPath.endsWith(".md")) { allowedAdditional = true; break; } } } catch {} } if (!allowedWorkspace && !allowedAdditional) throw new Error("path required"); if (!absPath.endsWith(".md")) throw new Error("path required"); const stat = await fs$1.lstat(absPath); if (stat.isSymbolicLink() || !stat.isFile()) throw new Error("path required"); const content = await fs$1.readFile(absPath, "utf-8"); if (!params.from && !params.lines) return { text: content, path: relPath }; const lines = content.split("\n"); const start = Math.max(1, params.from ?? 1); const count = Math.max(1, params.lines ?? lines.length); return { text: lines.slice(start - 1, start - 1 + count).join("\n"), path: relPath }; } status() { const sourceFilter = this.buildSourceFilter(); const files = this.db.prepare(`SELECT COUNT(*) as c FROM files WHERE 1=1${sourceFilter.sql}`).get(...sourceFilter.params); const chunks = this.db.prepare(`SELECT COUNT(*) as c FROM chunks WHERE 1=1${sourceFilter.sql}`).get(...sourceFilter.params); const sourceCounts = (() => { const sources = Array.from(this.sources); if (sources.length === 0) return []; const bySource = /* @__PURE__ */ new Map(); for (const source of sources) bySource.set(source, { files: 0, chunks: 0 }); const fileRows = this.db.prepare(`SELECT source, COUNT(*) as c FROM files WHERE 1=1${sourceFilter.sql} GROUP BY source`).all(...sourceFilter.params); for (const row of fileRows) { const entry = bySource.get(row.source) ?? { files: 0, chunks: 0 }; entry.files = row.c ?? 0; bySource.set(row.source, entry); } const chunkRows = this.db.prepare(`SELECT source, COUNT(*) as c FROM chunks WHERE 1=1${sourceFilter.sql} GROUP BY source`).all(...sourceFilter.params); for (const row of chunkRows) { const entry = bySource.get(row.source) ?? { files: 0, chunks: 0 }; entry.chunks = row.c ?? 0; bySource.set(row.source, entry); } return sources.map((source) => Object.assign({ source }, bySource.get(source))); })(); return { backend: "builtin", files: files?.c ?? 0, chunks: chunks?.c ?? 0, dirty: this.dirty || this.sessionsDirty, workspaceDir: this.workspaceDir, dbPath: this.settings.store.path, provider: this.provider.id, model: this.provider.model, requestedProvider: this.requestedProvider, sources: Array.from(this.sources), extraPaths: this.settings.extraPaths, sourceCounts, cache: this.cache.enabled ? { enabled: true, entries: this.db.prepare(`SELECT COUNT(*) as c FROM ${EMBEDDING_CACHE_TABLE}`).get()?.c ?? 0, maxEntries: this.cache.maxEntries } : { enabled: false, maxEntries: this.cache.maxEntries }, fts: { enabled: this.fts.enabled, available: this.fts.available, error: this.fts.loadError }, fallback: this.fallbackReason ? { from: this.fallbackFrom ?? "local", reason: this.fallbackReason } : void 0, vector: { enabled: this.vector.enabled, available: this.vector.available ?? void 0, extensionPath: this.vector.extensionPath, loadError: this.vector.loadError, dims: this.vector.dims }, batch: { enabled: this.batch.enabled, failures: this.batchFailureCount, limit: BATCH_FAILURE_LIMIT, wait: this.batch.wait, concurrency: this.batch.concurrency, pollIntervalMs: this.batch.pollIntervalMs, timeoutMs: this.batch.timeoutMs, lastError: this.batchFailureLastError, lastProvider: this.batchFailureLastProvider } }; } async probeVectorAvailability() { if (!this.vector.enabled) return false; return this.ensureVectorReady(); } async probeEmbeddingAvailability() { try { await this.embedBatchWithRetry(["ping"]); return { ok: true }; } catch (err) { return { ok: false, error: err instanceof Error ? err.message : String(err) }; } } async close() { if (this.closed) return; this.closed = true; if (this.watchTimer) { clearTimeout(this.watchTimer); this.watchTimer = null; } if (this.sessionWatchTimer) { clearTimeout(this.sessionWatchTimer); this.sessionWatchTimer = null; } if (this.intervalTimer) { clearInterval(this.intervalTimer); this.intervalTimer = null; } if (this.watcher) { await this.watcher.close(); this.watcher = null; } if (this.sessionUnsubscribe) { this.sessionUnsubscribe(); this.sessionUnsubscribe = null; } this.db.close(); INDEX_CACHE.delete(this.cacheKey); } async ensureVectorReady(dimensions) { if (!this.vector.enabled) return false; if (!this.vectorReady) this.vectorReady = this.withTimeout(this.loadVectorExtension(), VECTOR_LOAD_TIMEOUT_MS, `sqlite-vec load timed out after ${Math.round(VECTOR_LOAD_TIMEOUT_MS / 1e3)}s`); let ready = false; try { ready = await this.vectorReady; } catch (err) { const message = err instanceof Error ? err.message : String(err); this.vector.available = false; this.vector.loadError = message; this.vectorReady = null; log.warn(`sqlite-vec unavailable: ${message}`); return false; } if (ready && typeof dimensions === "number" && dimensions > 0) this.ensureVectorTable(dimensions); return ready; } async loadVectorExtension() { if (this.vector.available !== null) return this.vector.available; if (!this.vector.enabled) { this.vector.available = false; return false; } try { const resolvedPath = this.vector.extensionPath?.trim() ? resolveUserPath(this.vector.extensionPath) : void 0; const loaded = await loadSqliteVecExtension({ db: this.db, extensionPath: resolvedPath }); if (!loaded.ok) throw new Error(loaded.error ?? "unknown sqlite-vec load error"); this.vector.extensionPath = loaded.extensionPath; this.vector.available = true; return true; } catch (err) { const message = err instanceof Error ? err.message : String(err); this.vector.available = false; this.vector.loadError = message; log.warn(`sqlite-vec unavailable: ${message}`); return false; } } ensureVectorTable(dimensions) { if (this.vector.dims === dimensions) return; if (this.vector.dims && this.vector.dims !== dimensions) this.dropVectorTable(); this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${VECTOR_TABLE} USING vec0(\n id TEXT PRIMARY KEY,\n embedding FLOAT[${dimensions}]\n)`); this.vector.dims = dimensions; } dropVectorTable() { try { this.db.exec(`DROP TABLE IF EXISTS ${VECTOR_TABLE}`); } catch (err) { const message = err instanceof Error ? err.message : String(err); log.debug(`Failed to drop ${VECTOR_TABLE}: ${message}`); } } buildSourceFilter(alias) { const sources = Array.from(this.sources); if (sources.length === 0) return { sql: "", params: [] }; return { sql: ` AND ${alias ? `${alias}.source` : "source"} IN (${sources.map(() => "?").join(", ")})`, params: sources }; } openDatabase() { const dbPath = resolveUserPath(this.settings.store.path); return this.openDatabaseAtPath(dbPath); } openDatabaseAtPath(dbPath) { ensureDir(path.dirname(dbPath)); const { DatabaseSync } = requireNodeSqlite(); return new DatabaseSync(dbPath, { allowExtension: this.settings.store.vector.enabled }); } seedEmbeddingCache(sourceDb) { if (!this.cache.enabled) return; try { const rows = sourceDb.prepare(`SELECT provider, model, provider_key, hash, embedding, dims, updated_at FROM ${EMBEDDING_CACHE_TABLE}`).all(); if (!rows.length) return; const insert = this.db.prepare(`INSERT INTO ${EMBEDDING_CACHE_TABLE} (provider, model, provider_key, hash, embedding, dims, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(provider, model, provider_key, hash) DO UPDATE SET embedding=excluded.embedding, dims=excluded.dims, updated_at=excluded.updated_at`); this.db.exec("BEGIN"); for (const row of rows) insert.run(row.provider, row.model, row.provider_key, row.hash, row.embedding, row.dims, row.updated_at); this.db.exec("COMMIT"); } catch (err) { try { this.db.exec("ROLLBACK"); } catch {} throw err; } } async swapIndexFiles(targetPath, tempPath) { const backupPath = `${targetPath}.backup-${randomUUID()}`; await this.moveIndexFiles(targetPath, backupPath); try { await this.moveIndexFiles(tempPath, targetPath); } catch (err) { await this.moveIndexFiles(backupPath, targetPath); throw err; } await this.removeIndexFiles(backupPath); } async moveIndexFiles(sourceBase, targetBase) { for (const suffix of [ "", "-wal", "-shm" ]) { const source = `${sourceBase}${suffix}`; const target = `${targetBase}${suffix}`; try { await fs$1.rename(source, target); } catch (err) { if (err.code !== "ENOENT") throw err; } } } async removeIndexFiles(basePath) { await Promise.all([ "", "-wal", "-shm" ].map((suffix) => fs$1.rm(`${basePath}${suffix}`, { force: true }))); } ensureSchema() { const result = ensureMemoryIndexSchema({ db: this.db, embeddingCacheTable: EMBEDDING_CACHE_TABLE, ftsTable: FTS_TABLE, ftsEnabled: this.fts.enabled }); this.fts.available = result.ftsAvailable; if (result.ftsError) { this.fts.loadError = result.ftsError; log.warn(`fts unavailable: ${result.ftsError}`); } } ensureWatcher() { if (!this.sources.has("memory") || !this.settings.sync.watch || this.watcher) return; const additionalPaths = normalizeExtraMemoryPaths(this.workspaceDir, this.settings.extraPaths).map((entry) => { try { return fs.lstatSync(entry).isSymbolicLink() ? null : entry; } catch { return null; } }).filter((entry) => Boolean(entry)); const watchPaths = new Set([ path.join(this.workspaceDir, "MEMORY.md"), path.join(this.workspaceDir, "memory.md"), path.join(this.workspaceDir, "memory"), ...additionalPaths ]); this.watcher = chokidar.watch(Array.from(watchPaths), { ignoreInitial: true, awaitWriteFinish: { stabilityThreshold: this.settings.sync.watchDebounceMs, pollInterval: 100 } }); const markDirty = () => { this.dirty = true; this.scheduleWatchSync(); }; this.watcher.on("add", markDirty); this.watcher.on("change", markDirty); this.watcher.on("unlink", markDirty); } ensureSessionListener() { if (!this.sources.has("sessions") || this.sessionUnsubscribe) return; this.sessionUnsubscribe = onSessionTranscriptUpdate((update) => { if (this.closed) return; const sessionFile = update.sessionFile; if (!this.isSessionFileForAgent(sessionFile)) return; this.scheduleSessionDirty(sessionFile); }); } scheduleSessionDirty(sessionFile) { this.sessionPendingFiles.add(sessionFile); if (this.sessionWatchTimer) return; this.sessionWatchTimer = setTimeout(() => { this.sessionWatchTimer = null; this.processSessionDeltaBatch().catch((err) => { log.warn(`memory session delta failed: ${String(err)}`); }); }, SESSION_DIRTY_DEBOUNCE_MS); } async processSessionDeltaBatch() { if (this.sessionPendingFiles.size === 0) return; const pending = Array.from(this.sessionPendingFiles); this.sessionPendingFiles.clear(); let shouldSync = false; for (const sessionFile of pending) { const delta = await this.updateSessionDelta(sessionFile); if (!delta) continue; const bytesThreshold = delta.deltaBytes; const messagesThreshold = delta.deltaMessages; const bytesHit = bytesThreshold <= 0 ? delta.pendingBytes > 0 : delta.pendingBytes >= bytesThreshold; const messagesHit = messagesThreshold <= 0 ? delta.pendingMessages > 0 : delta.pendingMessages >= messagesThreshold; if (!bytesHit && !messagesHit) continue; this.sessionsDirtyFiles.add(sessionFile); this.sessionsDirty = true; delta.pendingBytes = bytesThreshold > 0 ? Math.max(0, delta.pendingBytes - bytesThreshold) : 0; delta.pendingMessages = messagesThreshold > 0 ? Math.max(0, delta.pendingMessages - messagesThreshold) : 0; shouldSync = true; } if (shouldSync) this.sync({ reason: "session-delta" }).catch((err) => { log.warn(`memory sync failed (session-delta): ${String(err)}`); }); } async updateSessionDelta(sessionFile) { const thresholds = this.settings.sync.sessions; if (!thresholds) return null; let stat; try { stat = await fs$1.stat(sessionFile); } catch { return null; } const size = stat.size; let state = this.sessionDeltas.get(sessionFile); if (!state) { state = { lastSize: 0, pendingBytes: 0, pendingMessages: 0 }; this.sessionDeltas.set(sessionFile, state); } const deltaBytes = Math.max(0, size - state.lastSize); if (deltaBytes === 0 && size === state.lastSize) return { deltaBytes: thresholds.deltaBytes, deltaMessages: thresholds.deltaMessages, pendingBytes: state.pendingBytes, pendingMessages: state.pendingMessages }; if (size < state.lastSize) { state.lastSize = size; state.pendingBytes += size; if (thresholds.deltaMessages > 0 && (thresholds.deltaBytes <= 0 || state.pendingBytes < thresholds.deltaBytes)) state.pendingMessages += await this.countNewlines(sessionFile, 0, size); } else { state.pendingBytes += deltaBytes; if (thresholds.deltaMessages > 0 && (thresholds.deltaBytes <= 0 || state.pendingBytes < thresholds.deltaBytes)) state.pendingMessages += await this.countNewlines(sessionFile, state.lastSize, size); state.lastSize = size; } this.sessionDeltas.set(sessionFile, state); return { deltaBytes: thresholds.deltaBytes, deltaMessages: thresholds.deltaMessages, pendingBytes: state.pendingBytes, pendingMessages: state.pendingMessages }; } async countNewlines(absPath, start, end) { if (end <= start) return 0; const handle = await fs$1.open(absPath, "r"); try { let offset = start; let count = 0; const buffer = Buffer.alloc(SESSION_DELTA_READ_CHUNK_BYTES); while (offset < end) { const toRead = Math.min(buffer.length, end - offset); const { bytesRead } = await handle.read(buffer, 0, toRead, offset); if (bytesRead <= 0) break; for (let i = 0; i < bytesRead; i += 1) if (buffer[i] === 10) count += 1; offset += bytesRead; } return count; } finally { await handle.close(); } } resetSessionDelta(absPath, size) { const state = this.sessionDeltas.get(absPath); if (!state) return; state.lastSize = size; state.pendingBytes = 0; state.pendingMessages = 0; } isSessionFileForAgent(sessionFile) { if (!sessionFile) return false; const sessionsDir = resolveSessionTranscriptsDirForAgent(this.agentId); const resolvedFile = path.resolve(sessionFile); const resolvedDir = path.resolve(sessionsDir); return resolvedFile.startsWith(`${resolvedDir}${path.sep}`); } ensureIntervalSync() { const minutes = this.settings.sync.intervalMinutes; if (!minutes || minutes <= 0 || this.intervalTimer) return; const ms = minutes * 60 * 1e3; this.intervalTimer = setInterval(() => { this.sync({ reason: "interval" }).catch((err) => { log.warn(`memory sync failed (interval): ${String(err)}`); }); }, ms); } scheduleWatchSync() { if (!this.sources.has("memory") || !this.settings.sync.watch) return; if (this.watchTimer) clearTimeout(this.watchTimer); this.watchTimer = setTimeout(() => { this.watchTimer = null; this.sync({ reason: "watch" }).catch((err) => { log.warn(`memory sync failed (watch): ${String(err)}`); }); }, this.settings.sync.watchDebounceMs); } shouldSyncSessions(params, needsFullReindex = false) { if (!this.sources.has("sessions")) return false; if (params?.force) return true; const reason = params?.reason; if (reason === "session-start" || reason === "watch") return false; if (needsFullReindex) return true; return this.sessionsDirty && this.sessionsDirtyFiles.size > 0; } async syncMemoryFiles(params) { const files = await listMemoryFiles(this.workspaceDir, this.settings.extraPaths); const fileEntries = await Promise.all(files.map(async (file) => buildFileEntry(file, this.workspaceDir))); log.debug("memory sync: indexing memory files", { files: fileEntries.length, needsFullReindex: params.needsFullReindex, batch: this.batch.enabled, concurrency: this.getIndexConcurrency() }); const activePaths = new Set(fileEntries.map((entry) => entry.path)); if (params.progress) { params.progress.total += fileEntries.length; params.progress.report({ completed: params.progress.completed, total: params.progress.total, label: this.batch.enabled ? "Indexing memory files (batch)..." : "Indexing memory files…" }); } const tasks = fileEntries.map((entry) => async () => { const record = this.db.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`).get(entry.path, "memory"); if (!params.needsFullReindex && record?.hash === entry.hash) { if (params.progress) { params.progress.completed += 1; params.progress.report({ completed: params.progress.completed, total: params.progress.total }); } return; } await this.indexFile(entry, { source: "memory" }); if (params.progress) { params.progress.completed += 1; params.progress.report({ completed: params.progress.completed, total: params.progress.total }); } }); await this.runWithConcurrency(tasks, this.getIndexConcurrency()); const staleRows = this.db.prepare(`SELECT path FROM files WHERE source = ?`).all("memory"); for (const stale of staleRows) { if (activePaths.has(stale.path)) continue; this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(stale.path, "memory"); try { this.db.prepare(`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`).run(stale.path, "memory"); } catch {} this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(stale.path, "memory"); if (this.fts.enabled && this.fts.available) try { this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`).run(stale.path, "memory", this.provider.model); } catch {} } } async syncSessionFiles(params) { const files = await this.listSessionFiles(); const activePaths = new Set(files.map((file) => this.sessionPathForFile(file))); const indexAll = params.needsFullReindex || this.sessionsDirtyFiles.size === 0; log.debug("memory sync: indexing session files", { files: files.length, indexAll, dirtyFiles: this.sessionsDirtyFiles.size, batch: this.batch.enabled, concurrency: this.getIndexConcurrency() }); if (params.progress) { params.progress.total += files.length; params.progress.report({ completed: params.progress.completed, total: params.progress.total, label: this.batch.enabled ? "Indexing session files (batch)..." : "Indexing session files…" }); } const tasks = files.map((absPath) => async () => { if (!indexAll && !this.sessionsDirtyFiles.has(absPath)) { if (params.progress) { params.progress.completed += 1; params.progress.report({ completed: params.progress.completed, total: params.progress.total }); } return; } const entry = await this.buildSessionEntry(absPath); if (!entry) { if (params.progress) { params.progress.completed += 1; params.progress.report({ completed: params.progress.completed, total: params.progress.total }); } return; } const record = this.db.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`).get(entry.path, "sessions"); if (!params.needsFullReindex && record?.hash === entry.hash) { if (params.progress) { params.progress.completed += 1; params.progress.report({ completed: params.progress.completed, total: params.progress.total }); } this.resetSessionDelta(absPath, entry.size); return; } await this.indexFile(entry, { source: "sessions", content: entry.content }); this.resetSessionDelta(absPath, entry.size); if (params.progress) { params.progress.completed += 1; params.progress.report({ completed: params.progress.completed, total: params.progress.total }); } }); await this.runWithConcurrency(tasks, this.getIndexConcurrency()); const staleRows = this.db.prepare(`SELECT path FROM files WHERE source = ?`).all("sessions"); for (const stale of staleRows) { if (activePaths.has(stale.path)) continue; this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(stale.path, "sessions"); try { this.db.prepare(`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`).run(stale.path, "sessions"); } catch {} this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(stale.path, "sessions"); if (this.fts.enabled && this.fts.available) try { this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`).run(stale.path, "sessions", this.provider.model); } catch {} } } createSyncProgress(onProgress) { const state = { completed: 0, total: 0, label: void 0, report: (update) => { if (update.label) state.label = update.label; const label = update.total > 0 && state.label ? `${state.label} ${update.completed}/${update.total}` : state.label; onProgress({ completed: update.completed, total: update.total, label }); } }; return state; } async runSync(params) { const progress = params?.progress ? this.createSyncProgress(params.progress) : void 0; if (progress) progress.report({ completed: progress.completed, total: progress.total, label: "Loading vector extension…" }); const vectorReady = await this.ensureVectorReady(); const meta = this.readMeta(); const needsFullReindex = params?.force || !meta || meta.model !== this.provider.model || meta.provider !== this.provider.id || meta.providerKey !== this.providerKey || meta.chunkTokens !== this.settings.chunking.tokens || meta.chunkOverlap !== this.settings.chunking.overlap || vectorReady && !meta?.vectorDims; try { if (needsFullReindex) { await this.runSafeReindex({ reason: params?.reason, force: params?.force, progress: progress ?? void 0 }); return; } const shouldSyncMemory = this.sources.has("memory") && (params?.force || needsFullReindex || this.dirty); const shouldSyncSessions = this.shouldSyncSessions(params, needsFullReindex); if (shouldSyncMemory) { await this.syncMemoryFiles({ needsFullReindex, progress: progress ?? void 0 }); this.dirty = false; } if (shouldSyncSessions) { await this.syncSessionFiles({ needsFullReindex, progress: progress ?? void 0 }); this.sessionsDirty = false; this.sessionsDirtyFiles.clear(); } else if (this.sessionsDirtyFiles.size > 0) this.sessionsDirty = true; else this.sessionsDirty = false; } catch (err) { const reason = err instanceof Error ? err.message : String(err); if (this.shouldFallbackOnError(reason) && await this.activateFallbackProvider(reason)) { await this.runSafeReindex({ reason: params?.reason ?? "fallback", force: true, progress: progress ?? void 0 }); return; } throw err; } } shouldFallbackOnError(message) { return /embedding|embeddings|batch/i.test(message); } resolveBatchConfig() { const batch = this.settings.remote?.batch; return { enabled: Boolean(batch?.enabled && (this.openAi && this.provider.id === "openai" || this.gemini && this.provider.id === "gemini")), wait: batch?.wait ?? true, concurrency: Math.max(1, batch?.concurrency ?? 2), pollIntervalMs: batch?.pollIntervalMs ?? 2e3, timeoutMs: (batch?.timeoutMinutes ?? 60) * 60 * 1e3 }; } async activateFallbackProvider(reason) { const fallback = this.settings.fallback; if (!fallback || fallback === "none" || fallback === this.provider.id) return false; if (this.fallbackFrom) return false; const fallbackFrom = this.provider.id; const fallbackModel = fallback === "gemini" ? DEFAULT_GEMINI_EMBEDDING_MODEL : fallback === "openai" ? DEFAULT_OPENAI_EMBEDDING_MODEL : this.settings.model; const fallbackResult = await createEmbeddingProvider({ config: this.cfg, agentDir: resolveAgentDir(this.cfg, this.agentId), provider: fallback, remote: this.settings.remote, model: fallbackModel, fallback: "none", local: this.settings.local }); this.fallbackFrom = fallbackFrom; this.fallbackReason = reason; this.provider = fallbackResult.provider; this.openAi = fallbackResult.openAi; this.gemini = fallbackResult.gemini; this.providerKey = this.computeProviderKey(); this.batch = this.resolveBatchConfig(); log.warn(`memory embeddings: switched to fallback provider (${fallback})`, { reason }); return true; } async runSafeReindex(params) { const dbPath = resolveUserPath(this.settings.store.path); const tempDbPath = `${dbPath}.tmp-${randomUUID()}`; const tempDb = this.openDatabaseAtPath(tempDbPath); const originalDb = this.db; let originalDbClosed = false; const originalState = { ftsAvailable: this.fts.available, ftsError: this.fts.loadError, vectorAvailable: this.vector.available, vectorLoadError: this.vector.loadError, vectorDims: this.vector.dims, vectorReady: this.vectorReady }; const restoreOriginalState = () => { if (originalDbClosed) this.db = this.openDatabaseAtPath(dbPath); else this.db = originalDb; this.fts.available = originalState.ftsAvailable; this.fts.loadError = originalState.ftsError; this.vector.available = originalDbClosed ? null : originalState.vectorAvailable; this.vector.loadError = originalState.vectorLoadError; this.vector.dims = originalState.vectorDims; this.vectorReady = originalDbClosed ? null : originalState.vectorReady; }; this.db = tempDb; this.vectorReady = null; this.vector.available = null; this.vector.loadError = void 0; this.vector.dims = void 0; this.fts.available = false; this.fts.loadError = void 0; this.ensureSchema(); let nextMeta = null; try { this.seedEmbeddingCache(originalDb); const shouldSyncMemory = this.sources.has("memory"); const shouldSyncSessions = this.shouldSyncSessions({ reason: params.reason, force: params.force }, true); if (shouldSyncMemory) { await this.syncMemoryFiles({ needsFullReindex: true, progress: params.progress }); this.dirty = false; } if (shouldSyncSessions) { await this.syncSessionFiles({ needsFullReindex: true, progress: params.progress }); this.sessionsDirty = false; this.sessionsDirtyFiles.clear(); } else if (this.sessionsDirtyFiles.size > 0) this.sessionsDirty = true; else this.sessionsDirty = false; nextMeta = { model: this.provider.model, provider: this.provider.id, providerKey: this.providerKey, chunkTokens: this.settings.chunking.tokens, chunkOverlap: this.settings.chunking.overlap }; if (this.vector.available && this.vector.dims) nextMeta.vectorDims = this.vector.dims; this.writeMeta(nextMeta); this.pruneEmbeddingCacheIfNeeded(); this.db.close(); originalDb.close(); originalDbClosed = true; await this.swapIndexFiles(dbPath, tempDbPath); this.db = this.openDatabaseAtPath(dbPath); this.vectorReady = null; this.vector.available = null; this.vector.loadError = void 0; this.ensureSchema(); this.vector.dims = nextMeta.vectorDims; } catch (err) { try { this.db.close(); } catch {} await this.removeIndexFiles(tempDbPath); restoreOriginalState(); throw err; } } resetIndex() { this.db.exec(`DELETE FROM files`); this.db.exec(`DELETE FROM chunks`); if (this.fts.enabled && this.fts.available) try { this.db.exec(`DELETE FROM ${FTS_TABLE}`); } catch {} this.dropVectorTable(); this.vector.dims = void 0; this.sessionsDirtyFiles.clear(); } readMeta() { const row = this.db.prepare(`SELECT value FROM meta WHERE key = ?`).get(META_KEY); if (!row?.value) return null; try { return JSON.parse(row.value); } catch { return null; } } writeMeta(meta) { const value = JSON.stringify(meta); this.db.prepare(`INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`).run(META_KEY, value); } async listSessionFiles() { const dir = resolveSessionTranscriptsDirForAgent(this.agentId); try { return (await fs$1.readdir(dir, { withFileTypes: true })).filter((entry) => entry.isFile()).map((entry) => entry.name).filter((name) => name.endsWith(".jsonl")).map((name) => path.join(dir, name)); } catch { return []; } } sessionPathForFile(absPath) { return path.join("sessions", path.basename(absPath)).replace(/\\/g, "/"); } normalizeSessionText(value) { return value.replace(/\s*\n+\s*/g, " ").replace(/\s+/g, " ").trim(); } extractSessionText(content) { if (typeof content === "string") { const normalized = this.normalizeSessionText(content); return normalized ? normalized : null; } if (!Array.isArray(content)) return null; const parts = []; for (const block of content) { if (!block || typeof block !== "object") continue; const record = block; if (record.type !== "text" || typeof record.text !== "string") continue; const normalized = this.normalizeSessionText(record.text); if (normalized) parts.push(normalized); } if (parts.length === 0) return null; return parts.join(" "); } async buildSessionEntry(absPath) { try { const stat = await fs$1.stat(absPath); const lines = (await fs$1.readFile(absPath, "utf-8")).split("\n"); const collected = []; for (const line of lines) { if (!line.trim()) continue; let record; try { record = JSON.parse(line); } catch { continue; } if (!record || typeof record !== "object" || record.type !== "message") continue; const message = record.message; if (!message || typeof message.role !== "string") continue; if (message.role !== "user" && message.role !== "assistant") continue; const text = this.extractSessionText(message.content); if (!text) continue; const label = message.role === "user" ? "User" : "Assistant"; collected.push(`${label}: ${text}`); } const content = collected.join("\n"); return { path: this.sessionPathForFile(absPath), absPath, mtimeMs: stat.mtimeMs, size: stat.size, hash: hashText(content), content }; } catch (err) { log.debug(`Failed reading session file ${absPath}: ${String(err)}`); return null; } } estimateEmbeddingTokens(text) { if (!text) return 0; return Math.ceil(text.length / EMBEDDING_APPROX_CHARS_PER_TOKEN); } buildEmbeddingBatches(chunks) { const batches = []; let current = []; let currentTokens = 0; for (const chunk of chunks) { const estimate = this.estimateEmbeddingTokens(chunk.text); if (current.length > 0 && currentTokens + estimate > EMBEDDING_BATCH_MAX_TOKENS) { batches.push(current); current = []; currentTokens = 0; } if (current.length === 0 && estimate > EMBEDDING_BATCH_MAX_TOKENS) { batches.push([chunk]); continue; } current.push(chunk); currentTokens += estimate; } if (current.length > 0) batches.push(current); return batches; } loadEmbeddingCache(hashes) { if (!this.cache.enabled) return /* @__PURE__ */ new Map(); if (hashes.length === 0) return /* @__PURE__ */ new Map(); const unique = []; const seen = /* @__PURE__ */ new Set(); for (const hash of hashes) { if (!hash) continue; if (seen.has(hash)) continue; seen.add(hash); unique.push(hash); } if (unique.length === 0) return /* @__PURE__ */ new Map(); const out = /* @__PURE__ */ new Map(); const baseParams = [ this.provider.id, this.provider.model, this.providerKey ]; const batchSize = 400; for (let start = 0; start < unique.length; start += batchSize) { const batch = unique.slice(start, start + batchSize); const placeholders = batch.map(() => "?").join(", "); const rows = this.db.prepare(`SELECT hash, embedding FROM ${EMBEDDING_CACHE_TABLE}\n WHERE provider = ? AND model = ? AND provider_key = ? AND hash IN (${placeholders})`).all(...baseParams, ...batch); for (const row of rows) out.set(row.hash, parseEmbedding(row.embedding)); } return out; } upsertEmbeddingCache(entries) { if (!this.cache.enabled) return; if (entries.length === 0) return; const now = Date.now(); const stmt = this.db.prepare(`INSERT INTO ${EMBEDDING_CACHE_TABLE} (provider, model, provider_key, hash, embedding, dims, updated_at)\n VALUES (?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT(provider, model, provider_key, hash) DO UPDATE SET\n embedding=excluded.embedding,\n dims=excluded.dims,\n updated_at=excluded.updated_at`); for (const entry of entries) { const embedding = entry.embedding ?? []; stmt.run(this.provider.id, this.provider.model, this.providerKey, entry.hash, JSON.stringify(embedding), embedding.length, now); } } pruneEmbeddingCacheIfNeeded() { if (!this.cache.enabled) return; const max = this.cache.maxEntries; if (!max || max <= 0) return; const count = this.db.prepare(`SELECT COUNT(*) as c FROM ${EMBEDDING_CACHE_TABLE}`).get()?.c ?? 0; if (count <= max) return; const excess = count - max; this.db.prepare(`DELETE FROM ${EMBEDDING_CACHE_TABLE}\n WHERE rowid IN (\n SELECT rowid FROM ${EMBEDDING_CACHE_TABLE}\n ORDER BY updated_at ASC\n LIMIT ?\n )`).run(excess); } async embedChunksInBatches(chunks) { if (chunks.length === 0) return []; const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash)); const embeddings = Array.from({ length: chunks.length }, () => []); const missing = []; for (let i = 0; i < chunks.length; i += 1) { const chunk = chunks[i]; const hit = chunk?.hash ? cached.get(chunk.hash) : void 0; if (hit && hit.length > 0) embeddings[i] = hit; else if (chunk) missing.push({ index: i, chunk }); } if (missing.length === 0) return embeddings; const missingChunks = missing.map((m) => m.chunk); const batches = this.buildEmbeddingBatches(missingChunks); const toCache = []; let cursor = 0; for (const batch of batches) { const batchEmbeddings = await this.embedBatchWithRetry(batch.map((chunk) => chunk.text)); for (let i = 0; i < batch.length; i += 1) { const item = missing[cursor + i]; const embedding = batchEmbeddings[i] ?? []; if (item) { embeddings[item.index] = embedding; toCache.push({ hash: item.chunk.hash, embedding }); } } cursor += batch.length; } this.upsertEmbeddingCache(toCache); return embeddings; } computeProviderKey() { if (this.provider.id === "openai" && this.openAi) { const entries = Object.entries(this.openAi.headers).filter(([key]) => key.toLowerCase() !== "authorization").toSorted(([a], [b]) => a.localeCompare(b)).map(([key, value]) => [key, value]); return hashText(JSON.stringify({ provider: "openai", baseUrl: this.openAi.baseUrl, model: this.openAi.model, headers: entries })); } if (this.provider.id === "gemini" && this.gemini) { const entries = Object.entries(this.gemini.headers).filter(([key]) => { const lower = key.toLowerCase(); return lower !== "authorization" && lower !== "x-goog-api-key"; }).toSorted(([a], [b]) => a.localeCompare(b)).map(([key, value]) => [key, value]); return hashText(JSON.stringify({ provider: "gemini", baseUrl: this.gemini.baseUrl, model: this.gemini.model, headers: entries })); } return hashText(JSON.stringify({ provider: this.provider.id, model: this.provider.model })); } async embedChunksWithBatch(chunks, entry, source) { if (this.provider.id === "openai" && this.openAi) return this.embedChunksWithOpenAiBatch(chunks, entry, source); if (this.provider.id === "gemini" && this.gemini) return this.embedChunksWithGeminiBatch(chunks, entry, source); return this.embedChunksInBatches(chunks); } async embedChunksWithOpenAiBatch(chunks, entry, source) { const openAi = this.openAi; if (!openAi) return this.embedChunksInBatches(chunks); if (chunks.length === 0) return []; const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash)); const embeddings = Array.from({ length: chunks.length }, () => []); const missing = []; for (let i = 0; i < chunks.length; i += 1) { const chunk = chunks[i]; const hit = chunk?.hash ? cached.get(chunk.hash) : void 0; if (hit && hit.length > 0) embeddings[i] = hit; else if (chunk) missing.push({ index: i, chunk }); } if (missing.length === 0) return embeddings; const requests = []; const mapping = /* @__PURE__ */ new Map(); for (const item of missing) { const chunk = item.chunk; const customId = hashText(`${source}:${entry.path}:${chunk.startLine}:${chunk.endLine}:${chunk.hash}:${item.index}`); mapping.set(customId, { index: item.index, hash: chunk.hash }); requests.push({ custom_id: customId, method: "POST", url: OPENAI_BATCH_ENDPOINT, body: { model: this.openAi?.model ?? this.provider.model, input: chunk.text } }); } const batchResult = await this.runBatchWithFallback({ provider: "openai", run: async () => await runOpenAiEmbeddingBatches({ openAi, agentId: this.agentId, requests, wait: this.batch.wait, concurrency: this.batch.concurrency, pollIntervalMs: this.batch.pollIntervalMs, timeoutMs: this.batch.timeoutMs, debug: (message, data) => log.debug(message, { ...data, source, chunks: chunks.length }) }), fallback: async () => await this.embedChunksInBatches(chunks) }); if (Array.isArray(batchResult)) return batchResult; const byCustomId = batchResult; const toCache = []; for (const [customId, embedding] of byCustomId.entries()) { const mapped = mapping.get(customId); if (!mapped) continue; embeddings[mapped.index] = embedding; toCache.push({ hash: mapped.hash, embedding }); } this.upsertEmbeddingCache(toCache); return embeddings; } async embedChunksWithGeminiBatch(chunks, entry, source) { const gemini = this.gemini; if (!gemini) return this.embedChunksInBatches(chunks); if (chunks.length === 0) return []; const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash)); const embeddings = Array.from({ length: chunks.length }, () => []); const missing = []; for (let i = 0; i < chunks.length; i += 1) { const chunk = chunks[i]; const hit = chunk?.hash ? cached.get(chunk.hash) : void 0; if (hit && hit.length > 0) embeddings[i] = hit; else if (chunk) missing.push({ index: i, chunk }); } if (missing.length === 0) return embeddings; const requests = []; const mapping = /* @__PURE__ */ new Map(); for (const item of missing) { const chunk = item.chunk; const customId = hashText(`${source}:${entry.path}:${chunk.startLine}:${chunk.endLine}:${chunk.hash}:${item.index}`); mapping.set(customId, { index: item.index, hash: chunk.hash }); requests.push({ custom_id: customId, content: { parts: [{ text: chunk.text }] }, taskType: "RETRIEVAL_DOCUMENT" }); } const batchResult = await this.runBatchWithFallback({ provider: "gemini", run: async () => await runGeminiEmbeddingBatches({ gemini, agentId: this.agentId, requests, wait: this.batch.wait, concurrency: this.batch.concurrency, pollIntervalMs: this.batch.pollIntervalMs, timeoutMs: this.batch.timeoutMs, debug: (message, data) => log.debug(message, { ...data, source, chunks: chunks.length }) }), fallback: async () => await this.embedChunksInBatches(chunks) }); if (Array.isArray(batchResult)) return batchResult; const byCustomId = batchResult; const toCache = []; for (const [customId, embedding] of byCustomId.entries()) { const mapped = mapping.get(customId); if (!mapped) continue; embeddings[mapped.index] = embedding; toCache.push({ hash: mapped.hash, embedding }); } this.upsertEmbeddingCache(toCache); return embeddings; } async embedBatchWithRetry(texts) { if (texts.length === 0) return []; let attempt = 0; let delayMs = EMBEDDING_RETRY_BASE_DELAY_MS; while (true) try { const timeoutMs = this.resolveEmbeddingTimeout("batch"); log.debug("memory embeddings: batch start", { provider: this.provider.id, items: texts.length, timeoutMs }); return await this.withTimeout(this.provider.embedBatch(texts), timeoutMs, `memory embeddings batch timed out after ${Math.round(timeoutMs / 1e3)}s`); } catch (err) { const message = err instanceof Error ? err.message : String(err); if (!this.isRetryableEmbeddingError(message) || attempt >= EMBEDDING_RETRY_MAX_ATTEMPTS) throw err; const waitMs = Math.min(EMBEDDING_RETRY_MAX_DELAY_MS, Math.round(delayMs * (1 + Math.random() * .2))); log.warn(`memory embeddings rate limited; retrying in ${waitMs}ms`); await new Promise((resolve) => setTimeout(resolve, waitMs)); delayMs *= 2; attempt += 1; } } isRetryableEmbeddingError(message) { return /(rate[_ ]limit|too many requests|429|resource has been exhausted|5\d\d|cloudflare)/i.test(message); } resolveEmbeddingTimeout(kind) { const isLocal = this.provider.id === "local"; if (kind === "query") return isLocal ? EMBEDDING_QUERY_TIMEOUT_LOCAL_MS : EMBEDDING_QUERY_TIMEOUT_REMOTE_MS; return isLocal ? EMBEDDING_BATCH_TIMEOUT_LOCAL_MS : EMBEDDING_BATCH_TIMEOUT_REMOTE_MS; } async embedQueryWithTimeout(text) { const timeoutMs = this.resolveEmbeddingTimeout("query"); log.debug("memory embeddings: query start", { provider: this.provider.id, timeoutMs }); return await this.withTimeout(this.provider.embedQuery(text), timeoutMs, `memory embeddings query timed out after ${Math.round(timeoutMs / 1e3)}s`); } async withTimeout(promise, timeoutMs, message) { if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) return await promise; let timer = null; const timeoutPromise = new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(message)), timeoutMs); }); try { return await Promise.race([promise, timeoutPromise]); } finally { if (timer) clearTimeout(timer); } } async runWithConcurrency(tasks, limit) { if (tasks.length === 0) return []; const resolvedLimit = Math.max(1, Math.min(limit, tasks.length)); const results = Array.from({ length: tasks.length }); let next = 0; let firstError = null; const workers = Array.from({ length: resolvedLimit }, async () => { while (true) { if (firstError) return; const index = next; next += 1; if (index >= tasks.length) return; try { results[index] = await tasks[index](); } catch (err) { firstError = err; return; } } }); await Promise.allSettled(workers); if (firstError) throw firstError; return results; } async withBatchFailureLock(fn) { let release; const wait = this.batchFailureLock; this.batchFailureLock = new Promise((resolve) => { release = resolve; }); await wait; try { return await fn(); } finally { release(); } } async resetBatchFailureCount() { await this.withBatchFailureLock(async () => { if (this.batchFailureCount > 0) log.debug("memory embeddings: batch recovered; resetting failure count"); this.batchFailureCount = 0; this.batchFailureLastError = void 0; this.batchFailureLastProvider = void 0; }); } async recordBatchFailure(params) { return await this.withBatchFailureLock(async () => { if (!this.batch.enabled) return { disabled: true, count: this.batchFailureCount }; const increment = params.forceDisable ? BATCH_FAILURE_LIMIT : Math.max(1, params.attempts ?? 1); this.batchFailureCount += increment; this.batchFailureLastError = params.message; this.batchFailureLastProvider = params.provider; const disabled = params.forceDisable || this.batchFailureCount >= BATCH_FAILURE_LIMIT; if (disabled) this.batch.enabled = false; return { disabled, count: this.batchFailureCount }; }); } isBatchTimeoutError(message) { return /timed out|timeout/i.test(message); } async runBatchWithTimeoutRetry(params) { try { return await params.run(); } catch (err) { const message = err instanceof Error ? err.message : String(err); if (this.isBatchTimeoutError(message)) { log.warn(`memory embeddings: ${params.provider} batch timed out; retrying once`); try { return await params.run(); } catch (retryErr) { retryErr.batchAttempts = 2; throw retryErr; } } throw err; } } async runBatchWithFallback(params) { if (!this.batch.enabled) return await params.fallback(); try { const result = await this.runBatchWithTimeoutRetry({ provider: params.provider, run: params.run }); await this.resetBatchFailureCount(); return result; } catch (err) { const message = err instanceof Error ? err.message : String(err); const attempts = err.batchAttempts ?? 1; const forceDisable = /asyncBatchEmbedContent not available/i.test(message); const failure = await this.recordBatchFailure({ provider: params.provider, message, attempts, forceDisable }); const suffix = failure.disabled ? "disabling batch" : "keeping batch enabled"; log.warn(`memory embeddings: ${params.provider} batch failed (${failure.count}/${BATCH_FAILURE_LIMIT}); ${suffix}; falling back to non-batch embeddings: ${message}`); return await params.fallback(); } } getIndexConcurrency() { return this.batch.enabled ? this.batch.concurrency : EMBEDDING_INDEX_CONCURRENCY; } async indexFile(entry, options) { const chunks = chunkMarkdown(options.content ?? await fs$1.readFile(entry.absPath, "utf-8"), this.settings.chunking).filter((chunk) => chunk.text.trim().length > 0); const embeddings = this.batch.enabled ? await this.embedChunksWithBatch(chunks, entry, options.source) : await this.embedChunksInBatches(chunks); const sample = embeddings.find((embedding) => embedding.length > 0); const vectorReady = sample ? await this.ensureVectorReady(sample.length) : false; const now = Date.now(); if (vectorReady) try { this.db.prepare(`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`).run(entry.path, options.source); } catch {} if (this.fts.enabled && this.fts.available) try { this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`).run(entry.path, options.source, this.provider.model); } catch {} this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(entry.path, options.source); for (let i = 0; i < chunks.length; i++) { const chunk = chunks[i]; const embedding = embeddings[i] ?? []; const id = hashText(`${options.source}:${entry.path}:${chunk.startLine}:${chunk.endLine}:${chunk.hash}:${this.provider.model}`); this.db.prepare(`INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET hash=excluded.hash, model=excluded.model, text=excluded.text, embedding=excluded.embedding, updated_at=excluded.updated_at`).run(id, entry.path, options.source, chunk.startLine, chunk.endLine, chunk.hash, this.provider.model, chunk.text, JSON.stringify(embedding), now); if (vectorReady && embedding.length > 0) { try { this.db.prepare(`DELETE FROM ${VECTOR_TABLE} WHERE id = ?`).run(id); } catch {} this.db.prepare(`INSERT INTO ${VECTOR_TABLE} (id, embedding) VALUES (?, ?)`).run(id, vectorToBlob(embedding)); } if (this.fts.enabled && this.fts.available) this.db.prepare(`INSERT INTO ${FTS_TABLE} (text, id, path, source, model, start_line, end_line)\n VALUES (?, ?, ?, ?, ?, ?, ?)`).run(chunk.text, id, entry.path, options.source, this.provider.model, chunk.startLine, chunk.endLine); } this.db.prepare(`INSERT INTO files (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?) ON CONFLICT(path) DO UPDATE SET source=excluded.source, hash=excluded.hash, mtime=excluded.mtime, size=excluded.size`).run(entry.path, options.source, entry.hash, entry.mtimeMs, entry.size); } }; //#endregion export { resolveMemorySearchConfig as i, resolveRetryConfig as n, retryAsync as r, MemoryIndexManager as t };