2868 lines
103 KiB
JavaScript
2868 lines
103 KiB
JavaScript
import { A as truncateUtf16Safe, D as sleep, g as clampNumber$1, h as clampInt, l as createSubsystemLogger, w as resolveUserPath } from "./exec-CjyVHUuE.js";
|
|
import { n as resolveAgentConfig, r as resolveAgentDir, s as resolveAgentWorkspaceDir } from "./agent-scope-qNPdmst1.js";
|
|
import { s as resolveStateDir } from "./paths-BFxmmTT5.js";
|
|
import { G as isTruthyEnvValue, v as requireApiKey, y as resolveApiKeyForProvider } from "./model-selection-Bo7pocNu.js";
|
|
import { i as resolveSessionTranscriptsDirForAgent } from "./paths-Bkjq_KcG.js";
|
|
import { n as onSessionTranscriptUpdate } from "./transcript-events-DOI14YeV.js";
|
|
import { a as ensureDir, c as listMemoryFiles, i as cosineSimilarity, l as normalizeExtraMemoryPaths, n as buildFileEntry, o as hashText, r as chunkMarkdown, s as isMemoryPath, t as requireNodeSqlite, u as parseEmbedding } from "./sqlite-CqVsQPIl.js";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import fs from "node:fs";
|
|
import fs$1 from "node:fs/promises";
|
|
import { randomUUID } from "node:crypto";
|
|
import chokidar from "chokidar";
|
|
|
|
//#region src/infra/retry.ts
|
|
const DEFAULT_RETRY_CONFIG = {
|
|
attempts: 3,
|
|
minDelayMs: 300,
|
|
maxDelayMs: 3e4,
|
|
jitter: 0
|
|
};
|
|
const asFiniteNumber = (value) => typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
const clampNumber = (value, fallback, min, max) => {
|
|
const next = asFiniteNumber(value);
|
|
if (next === void 0) return fallback;
|
|
const floor = typeof min === "number" ? min : Number.NEGATIVE_INFINITY;
|
|
const ceiling = typeof max === "number" ? max : Number.POSITIVE_INFINITY;
|
|
return Math.min(Math.max(next, floor), ceiling);
|
|
};
|
|
function resolveRetryConfig(defaults = DEFAULT_RETRY_CONFIG, overrides) {
|
|
const attempts = Math.max(1, Math.round(clampNumber(overrides?.attempts, defaults.attempts, 1)));
|
|
const minDelayMs = Math.max(0, Math.round(clampNumber(overrides?.minDelayMs, defaults.minDelayMs, 0)));
|
|
return {
|
|
attempts,
|
|
minDelayMs,
|
|
maxDelayMs: Math.max(minDelayMs, Math.round(clampNumber(overrides?.maxDelayMs, defaults.maxDelayMs, 0))),
|
|
jitter: clampNumber(overrides?.jitter, defaults.jitter, 0, 1)
|
|
};
|
|
}
|
|
function applyJitter(delayMs, jitter) {
|
|
if (jitter <= 0) return delayMs;
|
|
const offset = (Math.random() * 2 - 1) * jitter;
|
|
return Math.max(0, Math.round(delayMs * (1 + offset)));
|
|
}
|
|
async function retryAsync(fn, attemptsOrOptions = 3, initialDelayMs = 300) {
|
|
if (typeof attemptsOrOptions === "number") {
|
|
const attempts = Math.max(1, Math.round(attemptsOrOptions));
|
|
let lastErr;
|
|
for (let i = 0; i < attempts; i += 1) try {
|
|
return await fn();
|
|
} catch (err) {
|
|
lastErr = err;
|
|
if (i === attempts - 1) break;
|
|
await sleep(initialDelayMs * 2 ** i);
|
|
}
|
|
throw lastErr ?? /* @__PURE__ */ new Error("Retry failed");
|
|
}
|
|
const options = attemptsOrOptions;
|
|
const resolved = resolveRetryConfig(DEFAULT_RETRY_CONFIG, options);
|
|
const maxAttempts = resolved.attempts;
|
|
const minDelayMs = resolved.minDelayMs;
|
|
const maxDelayMs = Number.isFinite(resolved.maxDelayMs) && resolved.maxDelayMs > 0 ? resolved.maxDelayMs : Number.POSITIVE_INFINITY;
|
|
const jitter = resolved.jitter;
|
|
const shouldRetry = options.shouldRetry ?? (() => true);
|
|
let lastErr;
|
|
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) try {
|
|
return await fn();
|
|
} catch (err) {
|
|
lastErr = err;
|
|
if (attempt >= maxAttempts || !shouldRetry(err, attempt)) break;
|
|
const retryAfterMs = options.retryAfterMs?.(err);
|
|
const baseDelay = typeof retryAfterMs === "number" && Number.isFinite(retryAfterMs) ? Math.max(retryAfterMs, minDelayMs) : minDelayMs * 2 ** (attempt - 1);
|
|
let delay = Math.min(baseDelay, maxDelayMs);
|
|
delay = applyJitter(delay, jitter);
|
|
delay = Math.min(Math.max(delay, minDelayMs), maxDelayMs);
|
|
options.onRetry?.({
|
|
attempt,
|
|
maxAttempts,
|
|
delayMs: delay,
|
|
err,
|
|
label: options.label
|
|
});
|
|
await sleep(delay);
|
|
}
|
|
throw lastErr ?? /* @__PURE__ */ new Error("Retry failed");
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/agents/memory-search.ts
|
|
const DEFAULT_OPENAI_MODEL = "text-embedding-3-small";
|
|
const DEFAULT_GEMINI_MODEL = "gemini-embedding-001";
|
|
const DEFAULT_CHUNK_TOKENS = 400;
|
|
const DEFAULT_CHUNK_OVERLAP = 80;
|
|
const DEFAULT_WATCH_DEBOUNCE_MS = 1500;
|
|
const DEFAULT_SESSION_DELTA_BYTES = 1e5;
|
|
const DEFAULT_SESSION_DELTA_MESSAGES = 50;
|
|
const DEFAULT_MAX_RESULTS = 6;
|
|
const DEFAULT_MIN_SCORE = .35;
|
|
const DEFAULT_HYBRID_ENABLED = true;
|
|
const DEFAULT_HYBRID_VECTOR_WEIGHT = .7;
|
|
const DEFAULT_HYBRID_TEXT_WEIGHT = .3;
|
|
const DEFAULT_HYBRID_CANDIDATE_MULTIPLIER = 4;
|
|
const DEFAULT_CACHE_ENABLED = true;
|
|
const DEFAULT_SOURCES = ["memory"];
|
|
function normalizeSources(sources, sessionMemoryEnabled) {
|
|
const normalized = /* @__PURE__ */ new Set();
|
|
const input = sources?.length ? sources : DEFAULT_SOURCES;
|
|
for (const source of input) {
|
|
if (source === "memory") normalized.add("memory");
|
|
if (source === "sessions" && sessionMemoryEnabled) normalized.add("sessions");
|
|
}
|
|
if (normalized.size === 0) normalized.add("memory");
|
|
return Array.from(normalized);
|
|
}
|
|
function resolveStorePath(agentId, raw) {
|
|
const stateDir = resolveStateDir(process.env, os.homedir);
|
|
const fallback = path.join(stateDir, "memory", `${agentId}.sqlite`);
|
|
if (!raw) return fallback;
|
|
return resolveUserPath(raw.includes("{agentId}") ? raw.replaceAll("{agentId}", agentId) : raw);
|
|
}
|
|
function mergeConfig(defaults, overrides, agentId) {
|
|
const enabled = overrides?.enabled ?? defaults?.enabled ?? true;
|
|
const sessionMemory = overrides?.experimental?.sessionMemory ?? defaults?.experimental?.sessionMemory ?? false;
|
|
const provider = overrides?.provider ?? defaults?.provider ?? "auto";
|
|
const defaultRemote = defaults?.remote;
|
|
const overrideRemote = overrides?.remote;
|
|
const includeRemote = Boolean(overrideRemote?.baseUrl || overrideRemote?.apiKey || overrideRemote?.headers || defaultRemote?.baseUrl || defaultRemote?.apiKey || defaultRemote?.headers) || provider === "openai" || provider === "gemini" || provider === "auto";
|
|
const batch = {
|
|
enabled: overrideRemote?.batch?.enabled ?? defaultRemote?.batch?.enabled ?? true,
|
|
wait: overrideRemote?.batch?.wait ?? defaultRemote?.batch?.wait ?? true,
|
|
concurrency: Math.max(1, overrideRemote?.batch?.concurrency ?? defaultRemote?.batch?.concurrency ?? 2),
|
|
pollIntervalMs: overrideRemote?.batch?.pollIntervalMs ?? defaultRemote?.batch?.pollIntervalMs ?? 2e3,
|
|
timeoutMinutes: overrideRemote?.batch?.timeoutMinutes ?? defaultRemote?.batch?.timeoutMinutes ?? 60
|
|
};
|
|
const remote = includeRemote ? {
|
|
baseUrl: overrideRemote?.baseUrl ?? defaultRemote?.baseUrl,
|
|
apiKey: overrideRemote?.apiKey ?? defaultRemote?.apiKey,
|
|
headers: overrideRemote?.headers ?? defaultRemote?.headers,
|
|
batch
|
|
} : void 0;
|
|
const fallback = overrides?.fallback ?? defaults?.fallback ?? "none";
|
|
const modelDefault = provider === "gemini" ? DEFAULT_GEMINI_MODEL : provider === "openai" ? DEFAULT_OPENAI_MODEL : void 0;
|
|
const model = overrides?.model ?? defaults?.model ?? modelDefault ?? "";
|
|
const local = {
|
|
modelPath: overrides?.local?.modelPath ?? defaults?.local?.modelPath,
|
|
modelCacheDir: overrides?.local?.modelCacheDir ?? defaults?.local?.modelCacheDir
|
|
};
|
|
const sources = normalizeSources(overrides?.sources ?? defaults?.sources, sessionMemory);
|
|
const rawPaths = [...defaults?.extraPaths ?? [], ...overrides?.extraPaths ?? []].map((value) => value.trim()).filter(Boolean);
|
|
const extraPaths = Array.from(new Set(rawPaths));
|
|
const vector = {
|
|
enabled: overrides?.store?.vector?.enabled ?? defaults?.store?.vector?.enabled ?? true,
|
|
extensionPath: overrides?.store?.vector?.extensionPath ?? defaults?.store?.vector?.extensionPath
|
|
};
|
|
const store = {
|
|
driver: overrides?.store?.driver ?? defaults?.store?.driver ?? "sqlite",
|
|
path: resolveStorePath(agentId, overrides?.store?.path ?? defaults?.store?.path),
|
|
vector
|
|
};
|
|
const chunking = {
|
|
tokens: overrides?.chunking?.tokens ?? defaults?.chunking?.tokens ?? DEFAULT_CHUNK_TOKENS,
|
|
overlap: overrides?.chunking?.overlap ?? defaults?.chunking?.overlap ?? DEFAULT_CHUNK_OVERLAP
|
|
};
|
|
const sync = {
|
|
onSessionStart: overrides?.sync?.onSessionStart ?? defaults?.sync?.onSessionStart ?? true,
|
|
onSearch: overrides?.sync?.onSearch ?? defaults?.sync?.onSearch ?? true,
|
|
watch: overrides?.sync?.watch ?? defaults?.sync?.watch ?? true,
|
|
watchDebounceMs: overrides?.sync?.watchDebounceMs ?? defaults?.sync?.watchDebounceMs ?? DEFAULT_WATCH_DEBOUNCE_MS,
|
|
intervalMinutes: overrides?.sync?.intervalMinutes ?? defaults?.sync?.intervalMinutes ?? 0,
|
|
sessions: {
|
|
deltaBytes: overrides?.sync?.sessions?.deltaBytes ?? defaults?.sync?.sessions?.deltaBytes ?? DEFAULT_SESSION_DELTA_BYTES,
|
|
deltaMessages: overrides?.sync?.sessions?.deltaMessages ?? defaults?.sync?.sessions?.deltaMessages ?? DEFAULT_SESSION_DELTA_MESSAGES
|
|
}
|
|
};
|
|
const query = {
|
|
maxResults: overrides?.query?.maxResults ?? defaults?.query?.maxResults ?? DEFAULT_MAX_RESULTS,
|
|
minScore: overrides?.query?.minScore ?? defaults?.query?.minScore ?? DEFAULT_MIN_SCORE
|
|
};
|
|
const hybrid = {
|
|
enabled: overrides?.query?.hybrid?.enabled ?? defaults?.query?.hybrid?.enabled ?? DEFAULT_HYBRID_ENABLED,
|
|
vectorWeight: overrides?.query?.hybrid?.vectorWeight ?? defaults?.query?.hybrid?.vectorWeight ?? DEFAULT_HYBRID_VECTOR_WEIGHT,
|
|
textWeight: overrides?.query?.hybrid?.textWeight ?? defaults?.query?.hybrid?.textWeight ?? DEFAULT_HYBRID_TEXT_WEIGHT,
|
|
candidateMultiplier: overrides?.query?.hybrid?.candidateMultiplier ?? defaults?.query?.hybrid?.candidateMultiplier ?? DEFAULT_HYBRID_CANDIDATE_MULTIPLIER
|
|
};
|
|
const cache = {
|
|
enabled: overrides?.cache?.enabled ?? defaults?.cache?.enabled ?? DEFAULT_CACHE_ENABLED,
|
|
maxEntries: overrides?.cache?.maxEntries ?? defaults?.cache?.maxEntries
|
|
};
|
|
const overlap = clampNumber$1(chunking.overlap, 0, Math.max(0, chunking.tokens - 1));
|
|
const minScore = clampNumber$1(query.minScore, 0, 1);
|
|
const vectorWeight = clampNumber$1(hybrid.vectorWeight, 0, 1);
|
|
const textWeight = clampNumber$1(hybrid.textWeight, 0, 1);
|
|
const sum = vectorWeight + textWeight;
|
|
const normalizedVectorWeight = sum > 0 ? vectorWeight / sum : DEFAULT_HYBRID_VECTOR_WEIGHT;
|
|
const normalizedTextWeight = sum > 0 ? textWeight / sum : DEFAULT_HYBRID_TEXT_WEIGHT;
|
|
const candidateMultiplier = clampInt(hybrid.candidateMultiplier, 1, 20);
|
|
const deltaBytes = clampInt(sync.sessions.deltaBytes, 0, Number.MAX_SAFE_INTEGER);
|
|
const deltaMessages = clampInt(sync.sessions.deltaMessages, 0, Number.MAX_SAFE_INTEGER);
|
|
return {
|
|
enabled,
|
|
sources,
|
|
extraPaths,
|
|
provider,
|
|
remote,
|
|
experimental: { sessionMemory },
|
|
fallback,
|
|
model,
|
|
local,
|
|
store,
|
|
chunking: {
|
|
tokens: Math.max(1, chunking.tokens),
|
|
overlap
|
|
},
|
|
sync: {
|
|
...sync,
|
|
sessions: {
|
|
deltaBytes,
|
|
deltaMessages
|
|
}
|
|
},
|
|
query: {
|
|
...query,
|
|
minScore,
|
|
hybrid: {
|
|
enabled: Boolean(hybrid.enabled),
|
|
vectorWeight: normalizedVectorWeight,
|
|
textWeight: normalizedTextWeight,
|
|
candidateMultiplier
|
|
}
|
|
},
|
|
cache: {
|
|
enabled: Boolean(cache.enabled),
|
|
maxEntries: typeof cache.maxEntries === "number" && Number.isFinite(cache.maxEntries) ? Math.max(1, Math.floor(cache.maxEntries)) : void 0
|
|
}
|
|
};
|
|
}
|
|
function resolveMemorySearchConfig(cfg, agentId) {
|
|
const defaults = cfg.agents?.defaults?.memorySearch;
|
|
const overrides = resolveAgentConfig(cfg, agentId)?.memorySearch;
|
|
const resolved = mergeConfig(defaults, overrides, agentId);
|
|
if (!resolved.enabled) return null;
|
|
return resolved;
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/memory/batch-gemini.ts
|
|
const GEMINI_BATCH_MAX_REQUESTS = 5e4;
|
|
const debugEmbeddings$1 = isTruthyEnvValue(process.env.OPENCLAW_DEBUG_MEMORY_EMBEDDINGS);
|
|
const log$2 = createSubsystemLogger("memory/embeddings");
|
|
const debugLog$1 = (message, meta) => {
|
|
if (!debugEmbeddings$1) return;
|
|
const suffix = meta ? ` ${JSON.stringify(meta)}` : "";
|
|
log$2.raw(`${message}${suffix}`);
|
|
};
|
|
function getGeminiBaseUrl(gemini) {
|
|
return gemini.baseUrl?.replace(/\/$/, "") ?? "";
|
|
}
|
|
function getGeminiHeaders(gemini, params) {
|
|
const headers = gemini.headers ? { ...gemini.headers } : {};
|
|
if (params.json) {
|
|
if (!headers["Content-Type"] && !headers["content-type"]) headers["Content-Type"] = "application/json";
|
|
} else {
|
|
delete headers["Content-Type"];
|
|
delete headers["content-type"];
|
|
}
|
|
return headers;
|
|
}
|
|
function getGeminiUploadUrl(baseUrl) {
|
|
if (baseUrl.includes("/v1beta")) return baseUrl.replace(/\/v1beta\/?$/, "/upload/v1beta");
|
|
return `${baseUrl.replace(/\/$/, "")}/upload`;
|
|
}
|
|
function splitGeminiBatchRequests(requests) {
|
|
if (requests.length <= GEMINI_BATCH_MAX_REQUESTS) return [requests];
|
|
const groups = [];
|
|
for (let i = 0; i < requests.length; i += GEMINI_BATCH_MAX_REQUESTS) groups.push(requests.slice(i, i + GEMINI_BATCH_MAX_REQUESTS));
|
|
return groups;
|
|
}
|
|
function buildGeminiUploadBody(params) {
|
|
const boundary = `openclaw-${hashText(params.displayName)}`;
|
|
const jsonPart = JSON.stringify({ file: {
|
|
displayName: params.displayName,
|
|
mimeType: "application/jsonl"
|
|
} });
|
|
const delimiter = `--${boundary}\r\n`;
|
|
const closeDelimiter = `--${boundary}--\r\n`;
|
|
const parts = [
|
|
`${delimiter}Content-Type: application/json; charset=UTF-8\r\n\r\n${jsonPart}\r\n`,
|
|
`${delimiter}Content-Type: application/jsonl; charset=UTF-8\r\n\r\n${params.jsonl}\r\n`,
|
|
closeDelimiter
|
|
];
|
|
return {
|
|
body: new Blob([parts.join("")], { type: "multipart/related" }),
|
|
contentType: `multipart/related; boundary=${boundary}`
|
|
};
|
|
}
|
|
async function submitGeminiBatch(params) {
|
|
const baseUrl = getGeminiBaseUrl(params.gemini);
|
|
const uploadPayload = buildGeminiUploadBody({
|
|
jsonl: params.requests.map((request) => JSON.stringify({
|
|
key: request.custom_id,
|
|
request: {
|
|
content: request.content,
|
|
task_type: request.taskType
|
|
}
|
|
})).join("\n"),
|
|
displayName: `memory-embeddings-${hashText(String(Date.now()))}`
|
|
});
|
|
const uploadUrl = `${getGeminiUploadUrl(baseUrl)}/files?uploadType=multipart`;
|
|
debugLog$1("memory embeddings: gemini batch upload", {
|
|
uploadUrl,
|
|
baseUrl,
|
|
requests: params.requests.length
|
|
});
|
|
const fileRes = await fetch(uploadUrl, {
|
|
method: "POST",
|
|
headers: {
|
|
...getGeminiHeaders(params.gemini, { json: false }),
|
|
"Content-Type": uploadPayload.contentType
|
|
},
|
|
body: uploadPayload.body
|
|
});
|
|
if (!fileRes.ok) {
|
|
const text = await fileRes.text();
|
|
throw new Error(`gemini batch file upload failed: ${fileRes.status} ${text}`);
|
|
}
|
|
const filePayload = await fileRes.json();
|
|
const fileId = filePayload.name ?? filePayload.file?.name;
|
|
if (!fileId) throw new Error("gemini batch file upload failed: missing file id");
|
|
const batchBody = { batch: {
|
|
displayName: `memory-embeddings-${params.agentId}`,
|
|
inputConfig: { file_name: fileId }
|
|
} };
|
|
const batchEndpoint = `${baseUrl}/${params.gemini.modelPath}:asyncBatchEmbedContent`;
|
|
debugLog$1("memory embeddings: gemini batch create", {
|
|
batchEndpoint,
|
|
fileId
|
|
});
|
|
const batchRes = await fetch(batchEndpoint, {
|
|
method: "POST",
|
|
headers: getGeminiHeaders(params.gemini, { json: true }),
|
|
body: JSON.stringify(batchBody)
|
|
});
|
|
if (batchRes.ok) return await batchRes.json();
|
|
const text = await batchRes.text();
|
|
if (batchRes.status === 404) throw new Error("gemini batch create failed: 404 (asyncBatchEmbedContent not available for this model/baseUrl). Disable remote.batch.enabled or switch providers.");
|
|
throw new Error(`gemini batch create failed: ${batchRes.status} ${text}`);
|
|
}
|
|
async function fetchGeminiBatchStatus(params) {
|
|
const statusUrl = `${getGeminiBaseUrl(params.gemini)}/${params.batchName.startsWith("batches/") ? params.batchName : `batches/${params.batchName}`}`;
|
|
debugLog$1("memory embeddings: gemini batch status", { statusUrl });
|
|
const res = await fetch(statusUrl, { headers: getGeminiHeaders(params.gemini, { json: true }) });
|
|
if (!res.ok) {
|
|
const text = await res.text();
|
|
throw new Error(`gemini batch status failed: ${res.status} ${text}`);
|
|
}
|
|
return await res.json();
|
|
}
|
|
async function fetchGeminiFileContent(params) {
|
|
const downloadUrl = `${getGeminiBaseUrl(params.gemini)}/${params.fileId.startsWith("files/") ? params.fileId : `files/${params.fileId}`}:download`;
|
|
debugLog$1("memory embeddings: gemini batch download", { downloadUrl });
|
|
const res = await fetch(downloadUrl, { headers: getGeminiHeaders(params.gemini, { json: true }) });
|
|
if (!res.ok) {
|
|
const text = await res.text();
|
|
throw new Error(`gemini batch file content failed: ${res.status} ${text}`);
|
|
}
|
|
return await res.text();
|
|
}
|
|
function parseGeminiBatchOutput(text) {
|
|
if (!text.trim()) return [];
|
|
return text.split("\n").map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line));
|
|
}
|
|
async function waitForGeminiBatch(params) {
|
|
const start = Date.now();
|
|
let current = params.initial;
|
|
while (true) {
|
|
const status = current ?? await fetchGeminiBatchStatus({
|
|
gemini: params.gemini,
|
|
batchName: params.batchName
|
|
});
|
|
const state = status.state ?? "UNKNOWN";
|
|
if ([
|
|
"SUCCEEDED",
|
|
"COMPLETED",
|
|
"DONE"
|
|
].includes(state)) {
|
|
const outputFileId = status.outputConfig?.file ?? status.outputConfig?.fileId ?? status.metadata?.output?.responsesFile;
|
|
if (!outputFileId) throw new Error(`gemini batch ${params.batchName} completed without output file`);
|
|
return { outputFileId };
|
|
}
|
|
if ([
|
|
"FAILED",
|
|
"CANCELLED",
|
|
"CANCELED",
|
|
"EXPIRED"
|
|
].includes(state)) {
|
|
const message = status.error?.message ?? "unknown error";
|
|
throw new Error(`gemini batch ${params.batchName} ${state}: ${message}`);
|
|
}
|
|
if (!params.wait) throw new Error(`gemini batch ${params.batchName} still ${state}; wait disabled`);
|
|
if (Date.now() - start > params.timeoutMs) throw new Error(`gemini batch ${params.batchName} timed out after ${params.timeoutMs}ms`);
|
|
params.debug?.(`gemini batch ${params.batchName} ${state}; waiting ${params.pollIntervalMs}ms`);
|
|
await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs));
|
|
current = void 0;
|
|
}
|
|
}
|
|
async function runWithConcurrency$1(tasks, limit) {
|
|
if (tasks.length === 0) return [];
|
|
const resolvedLimit = Math.max(1, Math.min(limit, tasks.length));
|
|
const results = Array.from({ length: tasks.length });
|
|
let next = 0;
|
|
let firstError = null;
|
|
const workers = Array.from({ length: resolvedLimit }, async () => {
|
|
while (true) {
|
|
if (firstError) return;
|
|
const index = next;
|
|
next += 1;
|
|
if (index >= tasks.length) return;
|
|
try {
|
|
results[index] = await tasks[index]();
|
|
} catch (err) {
|
|
firstError = err;
|
|
return;
|
|
}
|
|
}
|
|
});
|
|
await Promise.allSettled(workers);
|
|
if (firstError) throw firstError;
|
|
return results;
|
|
}
|
|
async function runGeminiEmbeddingBatches(params) {
|
|
if (params.requests.length === 0) return /* @__PURE__ */ new Map();
|
|
const groups = splitGeminiBatchRequests(params.requests);
|
|
const byCustomId = /* @__PURE__ */ new Map();
|
|
const tasks = groups.map((group, groupIndex) => async () => {
|
|
const batchInfo = await submitGeminiBatch({
|
|
gemini: params.gemini,
|
|
requests: group,
|
|
agentId: params.agentId
|
|
});
|
|
const batchName = batchInfo.name ?? "";
|
|
if (!batchName) throw new Error("gemini batch create failed: missing batch name");
|
|
params.debug?.("memory embeddings: gemini batch created", {
|
|
batchName,
|
|
state: batchInfo.state,
|
|
group: groupIndex + 1,
|
|
groups: groups.length,
|
|
requests: group.length
|
|
});
|
|
if (!params.wait && batchInfo.state && ![
|
|
"SUCCEEDED",
|
|
"COMPLETED",
|
|
"DONE"
|
|
].includes(batchInfo.state)) throw new Error(`gemini batch ${batchName} submitted; enable remote.batch.wait to await completion`);
|
|
const completed = batchInfo.state && [
|
|
"SUCCEEDED",
|
|
"COMPLETED",
|
|
"DONE"
|
|
].includes(batchInfo.state) ? { outputFileId: batchInfo.outputConfig?.file ?? batchInfo.outputConfig?.fileId ?? batchInfo.metadata?.output?.responsesFile ?? "" } : await waitForGeminiBatch({
|
|
gemini: params.gemini,
|
|
batchName,
|
|
wait: params.wait,
|
|
pollIntervalMs: params.pollIntervalMs,
|
|
timeoutMs: params.timeoutMs,
|
|
debug: params.debug,
|
|
initial: batchInfo
|
|
});
|
|
if (!completed.outputFileId) throw new Error(`gemini batch ${batchName} completed without output file`);
|
|
const outputLines = parseGeminiBatchOutput(await fetchGeminiFileContent({
|
|
gemini: params.gemini,
|
|
fileId: completed.outputFileId
|
|
}));
|
|
const errors = [];
|
|
const remaining = new Set(group.map((request) => request.custom_id));
|
|
for (const line of outputLines) {
|
|
const customId = line.key ?? line.custom_id ?? line.request_id;
|
|
if (!customId) continue;
|
|
remaining.delete(customId);
|
|
if (line.error?.message) {
|
|
errors.push(`${customId}: ${line.error.message}`);
|
|
continue;
|
|
}
|
|
if (line.response?.error?.message) {
|
|
errors.push(`${customId}: ${line.response.error.message}`);
|
|
continue;
|
|
}
|
|
const embedding = line.embedding?.values ?? line.response?.embedding?.values ?? [];
|
|
if (embedding.length === 0) {
|
|
errors.push(`${customId}: empty embedding`);
|
|
continue;
|
|
}
|
|
byCustomId.set(customId, embedding);
|
|
}
|
|
if (errors.length > 0) throw new Error(`gemini batch ${batchName} failed: ${errors.join("; ")}`);
|
|
if (remaining.size > 0) throw new Error(`gemini batch ${batchName} missing ${remaining.size} embedding responses`);
|
|
});
|
|
params.debug?.("memory embeddings: gemini batch submit", {
|
|
requests: params.requests.length,
|
|
groups: groups.length,
|
|
wait: params.wait,
|
|
concurrency: params.concurrency,
|
|
pollIntervalMs: params.pollIntervalMs,
|
|
timeoutMs: params.timeoutMs
|
|
});
|
|
await runWithConcurrency$1(tasks, params.concurrency);
|
|
return byCustomId;
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/memory/batch-openai.ts
|
|
const OPENAI_BATCH_ENDPOINT = "/v1/embeddings";
|
|
const OPENAI_BATCH_COMPLETION_WINDOW = "24h";
|
|
const OPENAI_BATCH_MAX_REQUESTS = 5e4;
|
|
function getOpenAiBaseUrl(openAi) {
|
|
return openAi.baseUrl?.replace(/\/$/, "") ?? "";
|
|
}
|
|
function getOpenAiHeaders(openAi, params) {
|
|
const headers = openAi.headers ? { ...openAi.headers } : {};
|
|
if (params.json) {
|
|
if (!headers["Content-Type"] && !headers["content-type"]) headers["Content-Type"] = "application/json";
|
|
} else {
|
|
delete headers["Content-Type"];
|
|
delete headers["content-type"];
|
|
}
|
|
return headers;
|
|
}
|
|
function splitOpenAiBatchRequests(requests) {
|
|
if (requests.length <= OPENAI_BATCH_MAX_REQUESTS) return [requests];
|
|
const groups = [];
|
|
for (let i = 0; i < requests.length; i += OPENAI_BATCH_MAX_REQUESTS) groups.push(requests.slice(i, i + OPENAI_BATCH_MAX_REQUESTS));
|
|
return groups;
|
|
}
|
|
async function submitOpenAiBatch(params) {
|
|
const baseUrl = getOpenAiBaseUrl(params.openAi);
|
|
const jsonl = params.requests.map((request) => JSON.stringify(request)).join("\n");
|
|
const form = new FormData();
|
|
form.append("purpose", "batch");
|
|
form.append("file", new Blob([jsonl], { type: "application/jsonl" }), `memory-embeddings.${hashText(String(Date.now()))}.jsonl`);
|
|
const fileRes = await fetch(`${baseUrl}/files`, {
|
|
method: "POST",
|
|
headers: getOpenAiHeaders(params.openAi, { json: false }),
|
|
body: form
|
|
});
|
|
if (!fileRes.ok) {
|
|
const text = await fileRes.text();
|
|
throw new Error(`openai batch file upload failed: ${fileRes.status} ${text}`);
|
|
}
|
|
const filePayload = await fileRes.json();
|
|
if (!filePayload.id) throw new Error("openai batch file upload failed: missing file id");
|
|
return await (await retryAsync(async () => {
|
|
const res = await fetch(`${baseUrl}/batches`, {
|
|
method: "POST",
|
|
headers: getOpenAiHeaders(params.openAi, { json: true }),
|
|
body: JSON.stringify({
|
|
input_file_id: filePayload.id,
|
|
endpoint: OPENAI_BATCH_ENDPOINT,
|
|
completion_window: OPENAI_BATCH_COMPLETION_WINDOW,
|
|
metadata: {
|
|
source: "openclaw-memory",
|
|
agent: params.agentId
|
|
}
|
|
})
|
|
});
|
|
if (!res.ok) {
|
|
const text = await res.text();
|
|
const err = /* @__PURE__ */ new Error(`openai batch create failed: ${res.status} ${text}`);
|
|
err.status = res.status;
|
|
throw err;
|
|
}
|
|
return res;
|
|
}, {
|
|
attempts: 3,
|
|
minDelayMs: 300,
|
|
maxDelayMs: 2e3,
|
|
jitter: .2,
|
|
shouldRetry: (err) => {
|
|
const status = err.status;
|
|
return status === 429 || typeof status === "number" && status >= 500;
|
|
}
|
|
})).json();
|
|
}
|
|
async function fetchOpenAiBatchStatus(params) {
|
|
const baseUrl = getOpenAiBaseUrl(params.openAi);
|
|
const res = await fetch(`${baseUrl}/batches/${params.batchId}`, { headers: getOpenAiHeaders(params.openAi, { json: true }) });
|
|
if (!res.ok) {
|
|
const text = await res.text();
|
|
throw new Error(`openai batch status failed: ${res.status} ${text}`);
|
|
}
|
|
return await res.json();
|
|
}
|
|
async function fetchOpenAiFileContent(params) {
|
|
const baseUrl = getOpenAiBaseUrl(params.openAi);
|
|
const res = await fetch(`${baseUrl}/files/${params.fileId}/content`, { headers: getOpenAiHeaders(params.openAi, { json: true }) });
|
|
if (!res.ok) {
|
|
const text = await res.text();
|
|
throw new Error(`openai batch file content failed: ${res.status} ${text}`);
|
|
}
|
|
return await res.text();
|
|
}
|
|
function parseOpenAiBatchOutput(text) {
|
|
if (!text.trim()) return [];
|
|
return text.split("\n").map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line));
|
|
}
|
|
async function readOpenAiBatchError(params) {
|
|
try {
|
|
const first = parseOpenAiBatchOutput(await fetchOpenAiFileContent({
|
|
openAi: params.openAi,
|
|
fileId: params.errorFileId
|
|
})).find((line) => line.error?.message || line.response?.body?.error);
|
|
return first?.error?.message ?? (typeof first?.response?.body?.error?.message === "string" ? first?.response?.body?.error?.message : void 0);
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
return message ? `error file unavailable: ${message}` : void 0;
|
|
}
|
|
}
|
|
async function waitForOpenAiBatch(params) {
|
|
const start = Date.now();
|
|
let current = params.initial;
|
|
while (true) {
|
|
const status = current ?? await fetchOpenAiBatchStatus({
|
|
openAi: params.openAi,
|
|
batchId: params.batchId
|
|
});
|
|
const state = status.status ?? "unknown";
|
|
if (state === "completed") {
|
|
if (!status.output_file_id) throw new Error(`openai batch ${params.batchId} completed without output file`);
|
|
return {
|
|
outputFileId: status.output_file_id,
|
|
errorFileId: status.error_file_id ?? void 0
|
|
};
|
|
}
|
|
if ([
|
|
"failed",
|
|
"expired",
|
|
"cancelled",
|
|
"canceled"
|
|
].includes(state)) {
|
|
const detail = status.error_file_id ? await readOpenAiBatchError({
|
|
openAi: params.openAi,
|
|
errorFileId: status.error_file_id
|
|
}) : void 0;
|
|
const suffix = detail ? `: ${detail}` : "";
|
|
throw new Error(`openai batch ${params.batchId} ${state}${suffix}`);
|
|
}
|
|
if (!params.wait) throw new Error(`openai batch ${params.batchId} still ${state}; wait disabled`);
|
|
if (Date.now() - start > params.timeoutMs) throw new Error(`openai batch ${params.batchId} timed out after ${params.timeoutMs}ms`);
|
|
params.debug?.(`openai batch ${params.batchId} ${state}; waiting ${params.pollIntervalMs}ms`);
|
|
await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs));
|
|
current = void 0;
|
|
}
|
|
}
|
|
async function runWithConcurrency(tasks, limit) {
|
|
if (tasks.length === 0) return [];
|
|
const resolvedLimit = Math.max(1, Math.min(limit, tasks.length));
|
|
const results = Array.from({ length: tasks.length });
|
|
let next = 0;
|
|
let firstError = null;
|
|
const workers = Array.from({ length: resolvedLimit }, async () => {
|
|
while (true) {
|
|
if (firstError) return;
|
|
const index = next;
|
|
next += 1;
|
|
if (index >= tasks.length) return;
|
|
try {
|
|
results[index] = await tasks[index]();
|
|
} catch (err) {
|
|
firstError = err;
|
|
return;
|
|
}
|
|
}
|
|
});
|
|
await Promise.allSettled(workers);
|
|
if (firstError) throw firstError;
|
|
return results;
|
|
}
|
|
async function runOpenAiEmbeddingBatches(params) {
|
|
if (params.requests.length === 0) return /* @__PURE__ */ new Map();
|
|
const groups = splitOpenAiBatchRequests(params.requests);
|
|
const byCustomId = /* @__PURE__ */ new Map();
|
|
const tasks = groups.map((group, groupIndex) => async () => {
|
|
const batchInfo = await submitOpenAiBatch({
|
|
openAi: params.openAi,
|
|
requests: group,
|
|
agentId: params.agentId
|
|
});
|
|
if (!batchInfo.id) throw new Error("openai batch create failed: missing batch id");
|
|
params.debug?.("memory embeddings: openai batch created", {
|
|
batchId: batchInfo.id,
|
|
status: batchInfo.status,
|
|
group: groupIndex + 1,
|
|
groups: groups.length,
|
|
requests: group.length
|
|
});
|
|
if (!params.wait && batchInfo.status !== "completed") throw new Error(`openai batch ${batchInfo.id} submitted; enable remote.batch.wait to await completion`);
|
|
const completed = batchInfo.status === "completed" ? {
|
|
outputFileId: batchInfo.output_file_id ?? "",
|
|
errorFileId: batchInfo.error_file_id ?? void 0
|
|
} : await waitForOpenAiBatch({
|
|
openAi: params.openAi,
|
|
batchId: batchInfo.id,
|
|
wait: params.wait,
|
|
pollIntervalMs: params.pollIntervalMs,
|
|
timeoutMs: params.timeoutMs,
|
|
debug: params.debug,
|
|
initial: batchInfo
|
|
});
|
|
if (!completed.outputFileId) throw new Error(`openai batch ${batchInfo.id} completed without output file`);
|
|
const outputLines = parseOpenAiBatchOutput(await fetchOpenAiFileContent({
|
|
openAi: params.openAi,
|
|
fileId: completed.outputFileId
|
|
}));
|
|
const errors = [];
|
|
const remaining = new Set(group.map((request) => request.custom_id));
|
|
for (const line of outputLines) {
|
|
const customId = line.custom_id;
|
|
if (!customId) continue;
|
|
remaining.delete(customId);
|
|
if (line.error?.message) {
|
|
errors.push(`${customId}: ${line.error.message}`);
|
|
continue;
|
|
}
|
|
const response = line.response;
|
|
if ((response?.status_code ?? 0) >= 400) {
|
|
const message = response?.body?.error?.message ?? (typeof response?.body === "string" ? response.body : void 0) ?? "unknown error";
|
|
errors.push(`${customId}: ${message}`);
|
|
continue;
|
|
}
|
|
const embedding = (response?.body?.data ?? [])[0]?.embedding ?? [];
|
|
if (embedding.length === 0) {
|
|
errors.push(`${customId}: empty embedding`);
|
|
continue;
|
|
}
|
|
byCustomId.set(customId, embedding);
|
|
}
|
|
if (errors.length > 0) throw new Error(`openai batch ${batchInfo.id} failed: ${errors.join("; ")}`);
|
|
if (remaining.size > 0) throw new Error(`openai batch ${batchInfo.id} missing ${remaining.size} embedding responses`);
|
|
});
|
|
params.debug?.("memory embeddings: openai batch submit", {
|
|
requests: params.requests.length,
|
|
groups: groups.length,
|
|
wait: params.wait,
|
|
concurrency: params.concurrency,
|
|
pollIntervalMs: params.pollIntervalMs,
|
|
timeoutMs: params.timeoutMs
|
|
});
|
|
await runWithConcurrency(tasks, params.concurrency);
|
|
return byCustomId;
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/memory/embeddings-gemini.ts
|
|
const DEFAULT_GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
|
|
const DEFAULT_GEMINI_EMBEDDING_MODEL = "gemini-embedding-001";
|
|
const debugEmbeddings = isTruthyEnvValue(process.env.OPENCLAW_DEBUG_MEMORY_EMBEDDINGS);
|
|
const log$1 = createSubsystemLogger("memory/embeddings");
|
|
const debugLog = (message, meta) => {
|
|
if (!debugEmbeddings) return;
|
|
const suffix = meta ? ` ${JSON.stringify(meta)}` : "";
|
|
log$1.raw(`${message}${suffix}`);
|
|
};
|
|
function resolveRemoteApiKey(remoteApiKey) {
|
|
const trimmed = remoteApiKey?.trim();
|
|
if (!trimmed) return;
|
|
if (trimmed === "GOOGLE_API_KEY" || trimmed === "GEMINI_API_KEY") return process.env[trimmed]?.trim();
|
|
return trimmed;
|
|
}
|
|
function normalizeGeminiModel(model) {
|
|
const trimmed = model.trim();
|
|
if (!trimmed) return DEFAULT_GEMINI_EMBEDDING_MODEL;
|
|
const withoutPrefix = trimmed.replace(/^models\//, "");
|
|
if (withoutPrefix.startsWith("gemini/")) return withoutPrefix.slice(7);
|
|
if (withoutPrefix.startsWith("google/")) return withoutPrefix.slice(7);
|
|
return withoutPrefix;
|
|
}
|
|
function normalizeGeminiBaseUrl(raw) {
|
|
const trimmed = raw.replace(/\/+$/, "");
|
|
const openAiIndex = trimmed.indexOf("/openai");
|
|
if (openAiIndex > -1) return trimmed.slice(0, openAiIndex);
|
|
return trimmed;
|
|
}
|
|
function buildGeminiModelPath(model) {
|
|
return model.startsWith("models/") ? model : `models/${model}`;
|
|
}
|
|
async function createGeminiEmbeddingProvider(options) {
|
|
const client = await resolveGeminiEmbeddingClient(options);
|
|
const baseUrl = client.baseUrl.replace(/\/$/, "");
|
|
const embedUrl = `${baseUrl}/${client.modelPath}:embedContent`;
|
|
const batchUrl = `${baseUrl}/${client.modelPath}:batchEmbedContents`;
|
|
const embedQuery = async (text) => {
|
|
if (!text.trim()) return [];
|
|
const res = await fetch(embedUrl, {
|
|
method: "POST",
|
|
headers: client.headers,
|
|
body: JSON.stringify({
|
|
content: { parts: [{ text }] },
|
|
taskType: "RETRIEVAL_QUERY"
|
|
})
|
|
});
|
|
if (!res.ok) {
|
|
const payload = await res.text();
|
|
throw new Error(`gemini embeddings failed: ${res.status} ${payload}`);
|
|
}
|
|
return (await res.json()).embedding?.values ?? [];
|
|
};
|
|
const embedBatch = async (texts) => {
|
|
if (texts.length === 0) return [];
|
|
const requests = texts.map((text) => ({
|
|
model: client.modelPath,
|
|
content: { parts: [{ text }] },
|
|
taskType: "RETRIEVAL_DOCUMENT"
|
|
}));
|
|
const res = await fetch(batchUrl, {
|
|
method: "POST",
|
|
headers: client.headers,
|
|
body: JSON.stringify({ requests })
|
|
});
|
|
if (!res.ok) {
|
|
const payload = await res.text();
|
|
throw new Error(`gemini embeddings failed: ${res.status} ${payload}`);
|
|
}
|
|
const payload = await res.json();
|
|
const embeddings = Array.isArray(payload.embeddings) ? payload.embeddings : [];
|
|
return texts.map((_, index) => embeddings[index]?.values ?? []);
|
|
};
|
|
return {
|
|
provider: {
|
|
id: "gemini",
|
|
model: client.model,
|
|
embedQuery,
|
|
embedBatch
|
|
},
|
|
client
|
|
};
|
|
}
|
|
async function resolveGeminiEmbeddingClient(options) {
|
|
const remote = options.remote;
|
|
const remoteApiKey = resolveRemoteApiKey(remote?.apiKey);
|
|
const remoteBaseUrl = remote?.baseUrl?.trim();
|
|
const apiKey = remoteApiKey ? remoteApiKey : requireApiKey(await resolveApiKeyForProvider({
|
|
provider: "google",
|
|
cfg: options.config,
|
|
agentDir: options.agentDir
|
|
}), "google");
|
|
const providerConfig = options.config.models?.providers?.google;
|
|
const rawBaseUrl = remoteBaseUrl || providerConfig?.baseUrl?.trim() || DEFAULT_GEMINI_BASE_URL;
|
|
const baseUrl = normalizeGeminiBaseUrl(rawBaseUrl);
|
|
const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers);
|
|
const headers = {
|
|
"Content-Type": "application/json",
|
|
"x-goog-api-key": apiKey,
|
|
...headerOverrides
|
|
};
|
|
const model = normalizeGeminiModel(options.model);
|
|
const modelPath = buildGeminiModelPath(model);
|
|
debugLog("memory embeddings: gemini client", {
|
|
rawBaseUrl,
|
|
baseUrl,
|
|
model,
|
|
modelPath,
|
|
embedEndpoint: `${baseUrl}/${modelPath}:embedContent`,
|
|
batchEndpoint: `${baseUrl}/${modelPath}:batchEmbedContents`
|
|
});
|
|
return {
|
|
baseUrl,
|
|
headers,
|
|
model,
|
|
modelPath
|
|
};
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/memory/embeddings-openai.ts
|
|
const DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
|
|
const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
|
|
function normalizeOpenAiModel(model) {
|
|
const trimmed = model.trim();
|
|
if (!trimmed) return DEFAULT_OPENAI_EMBEDDING_MODEL;
|
|
if (trimmed.startsWith("openai/")) return trimmed.slice(7);
|
|
return trimmed;
|
|
}
|
|
async function createOpenAiEmbeddingProvider(options) {
|
|
const client = await resolveOpenAiEmbeddingClient(options);
|
|
const url = `${client.baseUrl.replace(/\/$/, "")}/embeddings`;
|
|
const embed = async (input) => {
|
|
if (input.length === 0) return [];
|
|
const res = await fetch(url, {
|
|
method: "POST",
|
|
headers: client.headers,
|
|
body: JSON.stringify({
|
|
model: client.model,
|
|
input
|
|
})
|
|
});
|
|
if (!res.ok) {
|
|
const text = await res.text();
|
|
throw new Error(`openai embeddings failed: ${res.status} ${text}`);
|
|
}
|
|
return ((await res.json()).data ?? []).map((entry) => entry.embedding ?? []);
|
|
};
|
|
return {
|
|
provider: {
|
|
id: "openai",
|
|
model: client.model,
|
|
embedQuery: async (text) => {
|
|
const [vec] = await embed([text]);
|
|
return vec ?? [];
|
|
},
|
|
embedBatch: embed
|
|
},
|
|
client
|
|
};
|
|
}
|
|
async function resolveOpenAiEmbeddingClient(options) {
|
|
const remote = options.remote;
|
|
const remoteApiKey = remote?.apiKey?.trim();
|
|
const remoteBaseUrl = remote?.baseUrl?.trim();
|
|
const apiKey = remoteApiKey ? remoteApiKey : requireApiKey(await resolveApiKeyForProvider({
|
|
provider: "openai",
|
|
cfg: options.config,
|
|
agentDir: options.agentDir
|
|
}), "openai");
|
|
const providerConfig = options.config.models?.providers?.openai;
|
|
const baseUrl = remoteBaseUrl || providerConfig?.baseUrl?.trim() || DEFAULT_OPENAI_BASE_URL;
|
|
const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers);
|
|
return {
|
|
baseUrl,
|
|
headers: {
|
|
"Content-Type": "application/json",
|
|
Authorization: `Bearer ${apiKey}`,
|
|
...headerOverrides
|
|
},
|
|
model: normalizeOpenAiModel(options.model)
|
|
};
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/memory/node-llama.ts
|
|
async function importNodeLlamaCpp() {
|
|
return import("node-llama-cpp");
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/memory/embeddings.ts
|
|
function sanitizeAndNormalizeEmbedding(vec) {
|
|
const sanitized = vec.map((value) => Number.isFinite(value) ? value : 0);
|
|
const magnitude = Math.sqrt(sanitized.reduce((sum, value) => sum + value * value, 0));
|
|
if (magnitude < 1e-10) return sanitized;
|
|
return sanitized.map((value) => value / magnitude);
|
|
}
|
|
const DEFAULT_LOCAL_MODEL = "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf";
|
|
function canAutoSelectLocal(options) {
|
|
const modelPath = options.local?.modelPath?.trim();
|
|
if (!modelPath) return false;
|
|
if (/^(hf:|https?:)/i.test(modelPath)) return false;
|
|
const resolved = resolveUserPath(modelPath);
|
|
try {
|
|
return fs.statSync(resolved).isFile();
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
function isMissingApiKeyError(err) {
|
|
return formatError(err).includes("No API key found for provider");
|
|
}
|
|
async function createLocalEmbeddingProvider(options) {
|
|
const modelPath = options.local?.modelPath?.trim() || DEFAULT_LOCAL_MODEL;
|
|
const modelCacheDir = options.local?.modelCacheDir?.trim();
|
|
const { getLlama, resolveModelFile, LlamaLogLevel } = await importNodeLlamaCpp();
|
|
let llama = null;
|
|
let embeddingModel = null;
|
|
let embeddingContext = null;
|
|
const ensureContext = async () => {
|
|
if (!llama) llama = await getLlama({ logLevel: LlamaLogLevel.error });
|
|
if (!embeddingModel) {
|
|
const resolved = await resolveModelFile(modelPath, modelCacheDir || void 0);
|
|
embeddingModel = await llama.loadModel({ modelPath: resolved });
|
|
}
|
|
if (!embeddingContext) embeddingContext = await embeddingModel.createEmbeddingContext();
|
|
return embeddingContext;
|
|
};
|
|
return {
|
|
id: "local",
|
|
model: modelPath,
|
|
embedQuery: async (text) => {
|
|
const embedding = await (await ensureContext()).getEmbeddingFor(text);
|
|
return sanitizeAndNormalizeEmbedding(Array.from(embedding.vector));
|
|
},
|
|
embedBatch: async (texts) => {
|
|
const ctx = await ensureContext();
|
|
return await Promise.all(texts.map(async (text) => {
|
|
const embedding = await ctx.getEmbeddingFor(text);
|
|
return sanitizeAndNormalizeEmbedding(Array.from(embedding.vector));
|
|
}));
|
|
}
|
|
};
|
|
}
|
|
async function createEmbeddingProvider(options) {
|
|
const requestedProvider = options.provider;
|
|
const fallback = options.fallback;
|
|
const createProvider = async (id) => {
|
|
if (id === "local") return { provider: await createLocalEmbeddingProvider(options) };
|
|
if (id === "gemini") {
|
|
const { provider, client } = await createGeminiEmbeddingProvider(options);
|
|
return {
|
|
provider,
|
|
gemini: client
|
|
};
|
|
}
|
|
const { provider, client } = await createOpenAiEmbeddingProvider(options);
|
|
return {
|
|
provider,
|
|
openAi: client
|
|
};
|
|
};
|
|
const formatPrimaryError = (err, provider) => provider === "local" ? formatLocalSetupError(err) : formatError(err);
|
|
if (requestedProvider === "auto") {
|
|
const missingKeyErrors = [];
|
|
let localError = null;
|
|
if (canAutoSelectLocal(options)) try {
|
|
return {
|
|
...await createProvider("local"),
|
|
requestedProvider
|
|
};
|
|
} catch (err) {
|
|
localError = formatLocalSetupError(err);
|
|
}
|
|
for (const provider of ["openai", "gemini"]) try {
|
|
return {
|
|
...await createProvider(provider),
|
|
requestedProvider
|
|
};
|
|
} catch (err) {
|
|
const message = formatPrimaryError(err, provider);
|
|
if (isMissingApiKeyError(err)) {
|
|
missingKeyErrors.push(message);
|
|
continue;
|
|
}
|
|
throw new Error(message, { cause: err });
|
|
}
|
|
const details = [...missingKeyErrors, localError].filter(Boolean);
|
|
if (details.length > 0) throw new Error(details.join("\n\n"));
|
|
throw new Error("No embeddings provider available.");
|
|
}
|
|
try {
|
|
return {
|
|
...await createProvider(requestedProvider),
|
|
requestedProvider
|
|
};
|
|
} catch (primaryErr) {
|
|
const reason = formatPrimaryError(primaryErr, requestedProvider);
|
|
if (fallback && fallback !== "none" && fallback !== requestedProvider) try {
|
|
return {
|
|
...await createProvider(fallback),
|
|
requestedProvider,
|
|
fallbackFrom: requestedProvider,
|
|
fallbackReason: reason
|
|
};
|
|
} catch (fallbackErr) {
|
|
throw new Error(`${reason}\n\nFallback to ${fallback} failed: ${formatError(fallbackErr)}`, { cause: fallbackErr });
|
|
}
|
|
throw new Error(reason, { cause: primaryErr });
|
|
}
|
|
}
|
|
function formatError(err) {
|
|
if (err instanceof Error) return err.message;
|
|
return String(err);
|
|
}
|
|
function isNodeLlamaCppMissing(err) {
|
|
if (!(err instanceof Error)) return false;
|
|
if (err.code === "ERR_MODULE_NOT_FOUND") return err.message.includes("node-llama-cpp");
|
|
return false;
|
|
}
|
|
function formatLocalSetupError(err) {
|
|
const detail = formatError(err);
|
|
const missing = isNodeLlamaCppMissing(err);
|
|
return [
|
|
"Local embeddings unavailable.",
|
|
missing ? "Reason: optional dependency node-llama-cpp is missing (or failed to install)." : detail ? `Reason: ${detail}` : void 0,
|
|
missing && detail ? `Detail: ${detail}` : null,
|
|
"To enable local embeddings:",
|
|
"1) Use Node 22 LTS (recommended for installs/updates)",
|
|
missing ? "2) Reinstall OpenClaw (this should install node-llama-cpp): npm i -g openclaw@latest" : null,
|
|
"3) If you use pnpm: pnpm approve-builds (select node-llama-cpp), then pnpm rebuild node-llama-cpp",
|
|
"Or set agents.defaults.memorySearch.provider = \"openai\" (remote)."
|
|
].filter(Boolean).join("\n");
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/memory/hybrid.ts
|
|
function buildFtsQuery(raw) {
|
|
const tokens = raw.match(/[A-Za-z0-9_]+/g)?.map((t) => t.trim()).filter(Boolean) ?? [];
|
|
if (tokens.length === 0) return null;
|
|
return tokens.map((t) => `"${t.replaceAll("\"", "")}"`).join(" AND ");
|
|
}
|
|
function bm25RankToScore(rank) {
|
|
return 1 / (1 + (Number.isFinite(rank) ? Math.max(0, rank) : 999));
|
|
}
|
|
function mergeHybridResults(params) {
|
|
const byId = /* @__PURE__ */ new Map();
|
|
for (const r of params.vector) byId.set(r.id, {
|
|
id: r.id,
|
|
path: r.path,
|
|
startLine: r.startLine,
|
|
endLine: r.endLine,
|
|
source: r.source,
|
|
snippet: r.snippet,
|
|
vectorScore: r.vectorScore,
|
|
textScore: 0
|
|
});
|
|
for (const r of params.keyword) {
|
|
const existing = byId.get(r.id);
|
|
if (existing) {
|
|
existing.textScore = r.textScore;
|
|
if (r.snippet && r.snippet.length > 0) existing.snippet = r.snippet;
|
|
} else byId.set(r.id, {
|
|
id: r.id,
|
|
path: r.path,
|
|
startLine: r.startLine,
|
|
endLine: r.endLine,
|
|
source: r.source,
|
|
snippet: r.snippet,
|
|
vectorScore: 0,
|
|
textScore: r.textScore
|
|
});
|
|
}
|
|
return Array.from(byId.values()).map((entry) => {
|
|
const score = params.vectorWeight * entry.vectorScore + params.textWeight * entry.textScore;
|
|
return {
|
|
path: entry.path,
|
|
startLine: entry.startLine,
|
|
endLine: entry.endLine,
|
|
score,
|
|
snippet: entry.snippet,
|
|
source: entry.source
|
|
};
|
|
}).toSorted((a, b) => b.score - a.score);
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/memory/manager-search.ts
|
|
const vectorToBlob$1 = (embedding) => Buffer.from(new Float32Array(embedding).buffer);
|
|
async function searchVector(params) {
|
|
if (params.queryVec.length === 0 || params.limit <= 0) return [];
|
|
if (await params.ensureVectorReady(params.queryVec.length)) return params.db.prepare(`SELECT c.id, c.path, c.start_line, c.end_line, c.text,
|
|
c.source,
|
|
vec_distance_cosine(v.embedding, ?) AS dist
|
|
FROM ${params.vectorTable} v\n JOIN chunks c ON c.id = v.id\n WHERE c.model = ?${params.sourceFilterVec.sql}\n ORDER BY dist ASC\n LIMIT ?`).all(vectorToBlob$1(params.queryVec), params.providerModel, ...params.sourceFilterVec.params, params.limit).map((row) => ({
|
|
id: row.id,
|
|
path: row.path,
|
|
startLine: row.start_line,
|
|
endLine: row.end_line,
|
|
score: 1 - row.dist,
|
|
snippet: truncateUtf16Safe(row.text, params.snippetMaxChars),
|
|
source: row.source
|
|
}));
|
|
return listChunks({
|
|
db: params.db,
|
|
providerModel: params.providerModel,
|
|
sourceFilter: params.sourceFilterChunks
|
|
}).map((chunk) => ({
|
|
chunk,
|
|
score: cosineSimilarity(params.queryVec, chunk.embedding)
|
|
})).filter((entry) => Number.isFinite(entry.score)).toSorted((a, b) => b.score - a.score).slice(0, params.limit).map((entry) => ({
|
|
id: entry.chunk.id,
|
|
path: entry.chunk.path,
|
|
startLine: entry.chunk.startLine,
|
|
endLine: entry.chunk.endLine,
|
|
score: entry.score,
|
|
snippet: truncateUtf16Safe(entry.chunk.text, params.snippetMaxChars),
|
|
source: entry.chunk.source
|
|
}));
|
|
}
|
|
function listChunks(params) {
|
|
return params.db.prepare(`SELECT id, path, start_line, end_line, text, embedding, source
|
|
FROM chunks
|
|
WHERE model = ?${params.sourceFilter.sql}`).all(params.providerModel, ...params.sourceFilter.params).map((row) => ({
|
|
id: row.id,
|
|
path: row.path,
|
|
startLine: row.start_line,
|
|
endLine: row.end_line,
|
|
text: row.text,
|
|
embedding: parseEmbedding(row.embedding),
|
|
source: row.source
|
|
}));
|
|
}
|
|
async function searchKeyword(params) {
|
|
if (params.limit <= 0) return [];
|
|
const ftsQuery = params.buildFtsQuery(params.query);
|
|
if (!ftsQuery) return [];
|
|
return params.db.prepare(`SELECT id, path, source, start_line, end_line, text,\n bm25(${params.ftsTable}) AS rank\n FROM ${params.ftsTable}\n WHERE ${params.ftsTable} MATCH ? AND model = ?${params.sourceFilter.sql}\n ORDER BY rank ASC\n LIMIT ?`).all(ftsQuery, params.providerModel, ...params.sourceFilter.params, params.limit).map((row) => {
|
|
const textScore = params.bm25RankToScore(row.rank);
|
|
return {
|
|
id: row.id,
|
|
path: row.path,
|
|
startLine: row.start_line,
|
|
endLine: row.end_line,
|
|
score: textScore,
|
|
textScore,
|
|
snippet: truncateUtf16Safe(row.text, params.snippetMaxChars),
|
|
source: row.source
|
|
};
|
|
});
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/memory/memory-schema.ts
|
|
function ensureMemoryIndexSchema(params) {
|
|
params.db.exec(`
|
|
CREATE TABLE IF NOT EXISTS meta (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT NOT NULL
|
|
);
|
|
`);
|
|
params.db.exec(`
|
|
CREATE TABLE IF NOT EXISTS files (
|
|
path TEXT PRIMARY KEY,
|
|
source TEXT NOT NULL DEFAULT 'memory',
|
|
hash TEXT NOT NULL,
|
|
mtime INTEGER NOT NULL,
|
|
size INTEGER NOT NULL
|
|
);
|
|
`);
|
|
params.db.exec(`
|
|
CREATE TABLE IF NOT EXISTS chunks (
|
|
id TEXT PRIMARY KEY,
|
|
path TEXT NOT NULL,
|
|
source TEXT NOT NULL DEFAULT 'memory',
|
|
start_line INTEGER NOT NULL,
|
|
end_line INTEGER NOT NULL,
|
|
hash TEXT NOT NULL,
|
|
model TEXT NOT NULL,
|
|
text TEXT NOT NULL,
|
|
embedding TEXT NOT NULL,
|
|
updated_at INTEGER NOT NULL
|
|
);
|
|
`);
|
|
params.db.exec(`
|
|
CREATE TABLE IF NOT EXISTS ${params.embeddingCacheTable} (
|
|
provider TEXT NOT NULL,
|
|
model TEXT NOT NULL,
|
|
provider_key TEXT NOT NULL,
|
|
hash TEXT NOT NULL,
|
|
embedding TEXT NOT NULL,
|
|
dims INTEGER,
|
|
updated_at INTEGER NOT NULL,
|
|
PRIMARY KEY (provider, model, provider_key, hash)
|
|
);
|
|
`);
|
|
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_embedding_cache_updated_at ON ${params.embeddingCacheTable}(updated_at);`);
|
|
let ftsAvailable = false;
|
|
let ftsError;
|
|
if (params.ftsEnabled) try {
|
|
params.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${params.ftsTable} USING fts5(\n text,\n id UNINDEXED,\n path UNINDEXED,\n source UNINDEXED,\n model UNINDEXED,\n start_line UNINDEXED,\n end_line UNINDEXED\n);`);
|
|
ftsAvailable = true;
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
ftsAvailable = false;
|
|
ftsError = message;
|
|
}
|
|
ensureColumn(params.db, "files", "source", "TEXT NOT NULL DEFAULT 'memory'");
|
|
ensureColumn(params.db, "chunks", "source", "TEXT NOT NULL DEFAULT 'memory'");
|
|
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks(path);`);
|
|
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source);`);
|
|
return {
|
|
ftsAvailable,
|
|
...ftsError ? { ftsError } : {}
|
|
};
|
|
}
|
|
function ensureColumn(db, table, column, definition) {
|
|
if (db.prepare(`PRAGMA table_info(${table})`).all().some((row) => row.name === column)) return;
|
|
db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/memory/sqlite-vec.ts
|
|
async function loadSqliteVecExtension(params) {
|
|
try {
|
|
const sqliteVec = await import("sqlite-vec");
|
|
const resolvedPath = params.extensionPath?.trim() ? params.extensionPath.trim() : void 0;
|
|
const extensionPath = resolvedPath ?? sqliteVec.getLoadablePath();
|
|
params.db.enableLoadExtension(true);
|
|
if (resolvedPath) params.db.loadExtension(extensionPath);
|
|
else sqliteVec.load(params.db);
|
|
return {
|
|
ok: true,
|
|
extensionPath
|
|
};
|
|
} catch (err) {
|
|
return {
|
|
ok: false,
|
|
error: err instanceof Error ? err.message : String(err)
|
|
};
|
|
}
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/memory/manager.ts
|
|
const META_KEY = "memory_index_meta_v1";
|
|
const SNIPPET_MAX_CHARS = 700;
|
|
const VECTOR_TABLE = "chunks_vec";
|
|
const FTS_TABLE = "chunks_fts";
|
|
const EMBEDDING_CACHE_TABLE = "embedding_cache";
|
|
const SESSION_DIRTY_DEBOUNCE_MS = 5e3;
|
|
const EMBEDDING_BATCH_MAX_TOKENS = 8e3;
|
|
const EMBEDDING_APPROX_CHARS_PER_TOKEN = 1;
|
|
const EMBEDDING_INDEX_CONCURRENCY = 4;
|
|
const EMBEDDING_RETRY_MAX_ATTEMPTS = 3;
|
|
const EMBEDDING_RETRY_BASE_DELAY_MS = 500;
|
|
const EMBEDDING_RETRY_MAX_DELAY_MS = 8e3;
|
|
const BATCH_FAILURE_LIMIT = 2;
|
|
const SESSION_DELTA_READ_CHUNK_BYTES = 64 * 1024;
|
|
const VECTOR_LOAD_TIMEOUT_MS = 3e4;
|
|
const EMBEDDING_QUERY_TIMEOUT_REMOTE_MS = 6e4;
|
|
const EMBEDDING_QUERY_TIMEOUT_LOCAL_MS = 5 * 6e4;
|
|
const EMBEDDING_BATCH_TIMEOUT_REMOTE_MS = 2 * 6e4;
|
|
const EMBEDDING_BATCH_TIMEOUT_LOCAL_MS = 10 * 6e4;
|
|
const log = createSubsystemLogger("memory");
|
|
const INDEX_CACHE = /* @__PURE__ */ new Map();
|
|
const vectorToBlob = (embedding) => Buffer.from(new Float32Array(embedding).buffer);
|
|
var MemoryIndexManager = class MemoryIndexManager {
|
|
static async get(params) {
|
|
const { cfg, agentId } = params;
|
|
const settings = resolveMemorySearchConfig(cfg, agentId);
|
|
if (!settings) return null;
|
|
const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId);
|
|
const key = `${agentId}:${workspaceDir}:${JSON.stringify(settings)}`;
|
|
const existing = INDEX_CACHE.get(key);
|
|
if (existing) return existing;
|
|
const manager = new MemoryIndexManager({
|
|
cacheKey: key,
|
|
cfg,
|
|
agentId,
|
|
workspaceDir,
|
|
settings,
|
|
providerResult: await createEmbeddingProvider({
|
|
config: cfg,
|
|
agentDir: resolveAgentDir(cfg, agentId),
|
|
provider: settings.provider,
|
|
remote: settings.remote,
|
|
model: settings.model,
|
|
fallback: settings.fallback,
|
|
local: settings.local
|
|
})
|
|
});
|
|
INDEX_CACHE.set(key, manager);
|
|
return manager;
|
|
}
|
|
constructor(params) {
|
|
this.batchFailureCount = 0;
|
|
this.batchFailureLock = Promise.resolve();
|
|
this.vectorReady = null;
|
|
this.watcher = null;
|
|
this.watchTimer = null;
|
|
this.sessionWatchTimer = null;
|
|
this.sessionUnsubscribe = null;
|
|
this.intervalTimer = null;
|
|
this.closed = false;
|
|
this.dirty = false;
|
|
this.sessionsDirty = false;
|
|
this.sessionsDirtyFiles = /* @__PURE__ */ new Set();
|
|
this.sessionPendingFiles = /* @__PURE__ */ new Set();
|
|
this.sessionDeltas = /* @__PURE__ */ new Map();
|
|
this.sessionWarm = /* @__PURE__ */ new Set();
|
|
this.syncing = null;
|
|
this.cacheKey = params.cacheKey;
|
|
this.cfg = params.cfg;
|
|
this.agentId = params.agentId;
|
|
this.workspaceDir = params.workspaceDir;
|
|
this.settings = params.settings;
|
|
this.provider = params.providerResult.provider;
|
|
this.requestedProvider = params.providerResult.requestedProvider;
|
|
this.fallbackFrom = params.providerResult.fallbackFrom;
|
|
this.fallbackReason = params.providerResult.fallbackReason;
|
|
this.openAi = params.providerResult.openAi;
|
|
this.gemini = params.providerResult.gemini;
|
|
this.sources = new Set(params.settings.sources);
|
|
this.db = this.openDatabase();
|
|
this.providerKey = this.computeProviderKey();
|
|
this.cache = {
|
|
enabled: params.settings.cache.enabled,
|
|
maxEntries: params.settings.cache.maxEntries
|
|
};
|
|
this.fts = {
|
|
enabled: params.settings.query.hybrid.enabled,
|
|
available: false
|
|
};
|
|
this.ensureSchema();
|
|
this.vector = {
|
|
enabled: params.settings.store.vector.enabled,
|
|
available: null,
|
|
extensionPath: params.settings.store.vector.extensionPath
|
|
};
|
|
const meta = this.readMeta();
|
|
if (meta?.vectorDims) this.vector.dims = meta.vectorDims;
|
|
this.ensureWatcher();
|
|
this.ensureSessionListener();
|
|
this.ensureIntervalSync();
|
|
this.dirty = this.sources.has("memory");
|
|
this.batch = this.resolveBatchConfig();
|
|
}
|
|
async warmSession(sessionKey) {
|
|
if (!this.settings.sync.onSessionStart) return;
|
|
const key = sessionKey?.trim() || "";
|
|
if (key && this.sessionWarm.has(key)) return;
|
|
this.sync({ reason: "session-start" }).catch((err) => {
|
|
log.warn(`memory sync failed (session-start): ${String(err)}`);
|
|
});
|
|
if (key) this.sessionWarm.add(key);
|
|
}
|
|
async search(query, opts) {
|
|
this.warmSession(opts?.sessionKey);
|
|
if (this.settings.sync.onSearch && (this.dirty || this.sessionsDirty)) this.sync({ reason: "search" }).catch((err) => {
|
|
log.warn(`memory sync failed (search): ${String(err)}`);
|
|
});
|
|
const cleaned = query.trim();
|
|
if (!cleaned) return [];
|
|
const minScore = opts?.minScore ?? this.settings.query.minScore;
|
|
const maxResults = opts?.maxResults ?? this.settings.query.maxResults;
|
|
const hybrid = this.settings.query.hybrid;
|
|
const candidates = Math.min(200, Math.max(1, Math.floor(maxResults * hybrid.candidateMultiplier)));
|
|
const keywordResults = hybrid.enabled ? await this.searchKeyword(cleaned, candidates).catch(() => []) : [];
|
|
const queryVec = await this.embedQueryWithTimeout(cleaned);
|
|
const vectorResults = queryVec.some((v) => v !== 0) ? await this.searchVector(queryVec, candidates).catch(() => []) : [];
|
|
if (!hybrid.enabled) return vectorResults.filter((entry) => entry.score >= minScore).slice(0, maxResults);
|
|
return this.mergeHybridResults({
|
|
vector: vectorResults,
|
|
keyword: keywordResults,
|
|
vectorWeight: hybrid.vectorWeight,
|
|
textWeight: hybrid.textWeight
|
|
}).filter((entry) => entry.score >= minScore).slice(0, maxResults);
|
|
}
|
|
async searchVector(queryVec, limit) {
|
|
return (await searchVector({
|
|
db: this.db,
|
|
vectorTable: VECTOR_TABLE,
|
|
providerModel: this.provider.model,
|
|
queryVec,
|
|
limit,
|
|
snippetMaxChars: SNIPPET_MAX_CHARS,
|
|
ensureVectorReady: async (dimensions) => await this.ensureVectorReady(dimensions),
|
|
sourceFilterVec: this.buildSourceFilter("c"),
|
|
sourceFilterChunks: this.buildSourceFilter()
|
|
})).map((entry) => entry);
|
|
}
|
|
buildFtsQuery(raw) {
|
|
return buildFtsQuery(raw);
|
|
}
|
|
async searchKeyword(query, limit) {
|
|
if (!this.fts.enabled || !this.fts.available) return [];
|
|
const sourceFilter = this.buildSourceFilter();
|
|
return (await searchKeyword({
|
|
db: this.db,
|
|
ftsTable: FTS_TABLE,
|
|
providerModel: this.provider.model,
|
|
query,
|
|
limit,
|
|
snippetMaxChars: SNIPPET_MAX_CHARS,
|
|
sourceFilter,
|
|
buildFtsQuery: (raw) => this.buildFtsQuery(raw),
|
|
bm25RankToScore
|
|
})).map((entry) => entry);
|
|
}
|
|
mergeHybridResults(params) {
|
|
return mergeHybridResults({
|
|
vector: params.vector.map((r) => ({
|
|
id: r.id,
|
|
path: r.path,
|
|
startLine: r.startLine,
|
|
endLine: r.endLine,
|
|
source: r.source,
|
|
snippet: r.snippet,
|
|
vectorScore: r.score
|
|
})),
|
|
keyword: params.keyword.map((r) => ({
|
|
id: r.id,
|
|
path: r.path,
|
|
startLine: r.startLine,
|
|
endLine: r.endLine,
|
|
source: r.source,
|
|
snippet: r.snippet,
|
|
textScore: r.textScore
|
|
})),
|
|
vectorWeight: params.vectorWeight,
|
|
textWeight: params.textWeight
|
|
}).map((entry) => entry);
|
|
}
|
|
async sync(params) {
|
|
if (this.syncing) return this.syncing;
|
|
this.syncing = this.runSync(params).finally(() => {
|
|
this.syncing = null;
|
|
});
|
|
return this.syncing;
|
|
}
|
|
async readFile(params) {
|
|
const rawPath = params.relPath.trim();
|
|
if (!rawPath) throw new Error("path required");
|
|
const absPath = path.isAbsolute(rawPath) ? path.resolve(rawPath) : path.resolve(this.workspaceDir, rawPath);
|
|
const relPath = path.relative(this.workspaceDir, absPath).replace(/\\/g, "/");
|
|
const allowedWorkspace = relPath.length > 0 && !relPath.startsWith("..") && !path.isAbsolute(relPath) && isMemoryPath(relPath);
|
|
let allowedAdditional = false;
|
|
if (!allowedWorkspace && this.settings.extraPaths.length > 0) {
|
|
const additionalPaths = normalizeExtraMemoryPaths(this.workspaceDir, this.settings.extraPaths);
|
|
for (const additionalPath of additionalPaths) try {
|
|
const stat = await fs$1.lstat(additionalPath);
|
|
if (stat.isSymbolicLink()) continue;
|
|
if (stat.isDirectory()) {
|
|
if (absPath === additionalPath || absPath.startsWith(`${additionalPath}${path.sep}`)) {
|
|
allowedAdditional = true;
|
|
break;
|
|
}
|
|
continue;
|
|
}
|
|
if (stat.isFile()) {
|
|
if (absPath === additionalPath && absPath.endsWith(".md")) {
|
|
allowedAdditional = true;
|
|
break;
|
|
}
|
|
}
|
|
} catch {}
|
|
}
|
|
if (!allowedWorkspace && !allowedAdditional) throw new Error("path required");
|
|
if (!absPath.endsWith(".md")) throw new Error("path required");
|
|
const stat = await fs$1.lstat(absPath);
|
|
if (stat.isSymbolicLink() || !stat.isFile()) throw new Error("path required");
|
|
const content = await fs$1.readFile(absPath, "utf-8");
|
|
if (!params.from && !params.lines) return {
|
|
text: content,
|
|
path: relPath
|
|
};
|
|
const lines = content.split("\n");
|
|
const start = Math.max(1, params.from ?? 1);
|
|
const count = Math.max(1, params.lines ?? lines.length);
|
|
return {
|
|
text: lines.slice(start - 1, start - 1 + count).join("\n"),
|
|
path: relPath
|
|
};
|
|
}
|
|
status() {
|
|
const sourceFilter = this.buildSourceFilter();
|
|
const files = this.db.prepare(`SELECT COUNT(*) as c FROM files WHERE 1=1${sourceFilter.sql}`).get(...sourceFilter.params);
|
|
const chunks = this.db.prepare(`SELECT COUNT(*) as c FROM chunks WHERE 1=1${sourceFilter.sql}`).get(...sourceFilter.params);
|
|
const sourceCounts = (() => {
|
|
const sources = Array.from(this.sources);
|
|
if (sources.length === 0) return [];
|
|
const bySource = /* @__PURE__ */ new Map();
|
|
for (const source of sources) bySource.set(source, {
|
|
files: 0,
|
|
chunks: 0
|
|
});
|
|
const fileRows = this.db.prepare(`SELECT source, COUNT(*) as c FROM files WHERE 1=1${sourceFilter.sql} GROUP BY source`).all(...sourceFilter.params);
|
|
for (const row of fileRows) {
|
|
const entry = bySource.get(row.source) ?? {
|
|
files: 0,
|
|
chunks: 0
|
|
};
|
|
entry.files = row.c ?? 0;
|
|
bySource.set(row.source, entry);
|
|
}
|
|
const chunkRows = this.db.prepare(`SELECT source, COUNT(*) as c FROM chunks WHERE 1=1${sourceFilter.sql} GROUP BY source`).all(...sourceFilter.params);
|
|
for (const row of chunkRows) {
|
|
const entry = bySource.get(row.source) ?? {
|
|
files: 0,
|
|
chunks: 0
|
|
};
|
|
entry.chunks = row.c ?? 0;
|
|
bySource.set(row.source, entry);
|
|
}
|
|
return sources.map((source) => Object.assign({ source }, bySource.get(source)));
|
|
})();
|
|
return {
|
|
backend: "builtin",
|
|
files: files?.c ?? 0,
|
|
chunks: chunks?.c ?? 0,
|
|
dirty: this.dirty || this.sessionsDirty,
|
|
workspaceDir: this.workspaceDir,
|
|
dbPath: this.settings.store.path,
|
|
provider: this.provider.id,
|
|
model: this.provider.model,
|
|
requestedProvider: this.requestedProvider,
|
|
sources: Array.from(this.sources),
|
|
extraPaths: this.settings.extraPaths,
|
|
sourceCounts,
|
|
cache: this.cache.enabled ? {
|
|
enabled: true,
|
|
entries: this.db.prepare(`SELECT COUNT(*) as c FROM ${EMBEDDING_CACHE_TABLE}`).get()?.c ?? 0,
|
|
maxEntries: this.cache.maxEntries
|
|
} : {
|
|
enabled: false,
|
|
maxEntries: this.cache.maxEntries
|
|
},
|
|
fts: {
|
|
enabled: this.fts.enabled,
|
|
available: this.fts.available,
|
|
error: this.fts.loadError
|
|
},
|
|
fallback: this.fallbackReason ? {
|
|
from: this.fallbackFrom ?? "local",
|
|
reason: this.fallbackReason
|
|
} : void 0,
|
|
vector: {
|
|
enabled: this.vector.enabled,
|
|
available: this.vector.available ?? void 0,
|
|
extensionPath: this.vector.extensionPath,
|
|
loadError: this.vector.loadError,
|
|
dims: this.vector.dims
|
|
},
|
|
batch: {
|
|
enabled: this.batch.enabled,
|
|
failures: this.batchFailureCount,
|
|
limit: BATCH_FAILURE_LIMIT,
|
|
wait: this.batch.wait,
|
|
concurrency: this.batch.concurrency,
|
|
pollIntervalMs: this.batch.pollIntervalMs,
|
|
timeoutMs: this.batch.timeoutMs,
|
|
lastError: this.batchFailureLastError,
|
|
lastProvider: this.batchFailureLastProvider
|
|
}
|
|
};
|
|
}
|
|
async probeVectorAvailability() {
|
|
if (!this.vector.enabled) return false;
|
|
return this.ensureVectorReady();
|
|
}
|
|
async probeEmbeddingAvailability() {
|
|
try {
|
|
await this.embedBatchWithRetry(["ping"]);
|
|
return { ok: true };
|
|
} catch (err) {
|
|
return {
|
|
ok: false,
|
|
error: err instanceof Error ? err.message : String(err)
|
|
};
|
|
}
|
|
}
|
|
async close() {
|
|
if (this.closed) return;
|
|
this.closed = true;
|
|
if (this.watchTimer) {
|
|
clearTimeout(this.watchTimer);
|
|
this.watchTimer = null;
|
|
}
|
|
if (this.sessionWatchTimer) {
|
|
clearTimeout(this.sessionWatchTimer);
|
|
this.sessionWatchTimer = null;
|
|
}
|
|
if (this.intervalTimer) {
|
|
clearInterval(this.intervalTimer);
|
|
this.intervalTimer = null;
|
|
}
|
|
if (this.watcher) {
|
|
await this.watcher.close();
|
|
this.watcher = null;
|
|
}
|
|
if (this.sessionUnsubscribe) {
|
|
this.sessionUnsubscribe();
|
|
this.sessionUnsubscribe = null;
|
|
}
|
|
this.db.close();
|
|
INDEX_CACHE.delete(this.cacheKey);
|
|
}
|
|
async ensureVectorReady(dimensions) {
|
|
if (!this.vector.enabled) return false;
|
|
if (!this.vectorReady) this.vectorReady = this.withTimeout(this.loadVectorExtension(), VECTOR_LOAD_TIMEOUT_MS, `sqlite-vec load timed out after ${Math.round(VECTOR_LOAD_TIMEOUT_MS / 1e3)}s`);
|
|
let ready = false;
|
|
try {
|
|
ready = await this.vectorReady;
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
this.vector.available = false;
|
|
this.vector.loadError = message;
|
|
this.vectorReady = null;
|
|
log.warn(`sqlite-vec unavailable: ${message}`);
|
|
return false;
|
|
}
|
|
if (ready && typeof dimensions === "number" && dimensions > 0) this.ensureVectorTable(dimensions);
|
|
return ready;
|
|
}
|
|
async loadVectorExtension() {
|
|
if (this.vector.available !== null) return this.vector.available;
|
|
if (!this.vector.enabled) {
|
|
this.vector.available = false;
|
|
return false;
|
|
}
|
|
try {
|
|
const resolvedPath = this.vector.extensionPath?.trim() ? resolveUserPath(this.vector.extensionPath) : void 0;
|
|
const loaded = await loadSqliteVecExtension({
|
|
db: this.db,
|
|
extensionPath: resolvedPath
|
|
});
|
|
if (!loaded.ok) throw new Error(loaded.error ?? "unknown sqlite-vec load error");
|
|
this.vector.extensionPath = loaded.extensionPath;
|
|
this.vector.available = true;
|
|
return true;
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
this.vector.available = false;
|
|
this.vector.loadError = message;
|
|
log.warn(`sqlite-vec unavailable: ${message}`);
|
|
return false;
|
|
}
|
|
}
|
|
ensureVectorTable(dimensions) {
|
|
if (this.vector.dims === dimensions) return;
|
|
if (this.vector.dims && this.vector.dims !== dimensions) this.dropVectorTable();
|
|
this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${VECTOR_TABLE} USING vec0(\n id TEXT PRIMARY KEY,\n embedding FLOAT[${dimensions}]\n)`);
|
|
this.vector.dims = dimensions;
|
|
}
|
|
dropVectorTable() {
|
|
try {
|
|
this.db.exec(`DROP TABLE IF EXISTS ${VECTOR_TABLE}`);
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
log.debug(`Failed to drop ${VECTOR_TABLE}: ${message}`);
|
|
}
|
|
}
|
|
buildSourceFilter(alias) {
|
|
const sources = Array.from(this.sources);
|
|
if (sources.length === 0) return {
|
|
sql: "",
|
|
params: []
|
|
};
|
|
return {
|
|
sql: ` AND ${alias ? `${alias}.source` : "source"} IN (${sources.map(() => "?").join(", ")})`,
|
|
params: sources
|
|
};
|
|
}
|
|
openDatabase() {
|
|
const dbPath = resolveUserPath(this.settings.store.path);
|
|
return this.openDatabaseAtPath(dbPath);
|
|
}
|
|
openDatabaseAtPath(dbPath) {
|
|
ensureDir(path.dirname(dbPath));
|
|
const { DatabaseSync } = requireNodeSqlite();
|
|
return new DatabaseSync(dbPath, { allowExtension: this.settings.store.vector.enabled });
|
|
}
|
|
seedEmbeddingCache(sourceDb) {
|
|
if (!this.cache.enabled) return;
|
|
try {
|
|
const rows = sourceDb.prepare(`SELECT provider, model, provider_key, hash, embedding, dims, updated_at FROM ${EMBEDDING_CACHE_TABLE}`).all();
|
|
if (!rows.length) return;
|
|
const insert = this.db.prepare(`INSERT INTO ${EMBEDDING_CACHE_TABLE} (provider, model, provider_key, hash, embedding, dims, updated_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(provider, model, provider_key, hash) DO UPDATE SET
|
|
embedding=excluded.embedding,
|
|
dims=excluded.dims,
|
|
updated_at=excluded.updated_at`);
|
|
this.db.exec("BEGIN");
|
|
for (const row of rows) insert.run(row.provider, row.model, row.provider_key, row.hash, row.embedding, row.dims, row.updated_at);
|
|
this.db.exec("COMMIT");
|
|
} catch (err) {
|
|
try {
|
|
this.db.exec("ROLLBACK");
|
|
} catch {}
|
|
throw err;
|
|
}
|
|
}
|
|
async swapIndexFiles(targetPath, tempPath) {
|
|
const backupPath = `${targetPath}.backup-${randomUUID()}`;
|
|
await this.moveIndexFiles(targetPath, backupPath);
|
|
try {
|
|
await this.moveIndexFiles(tempPath, targetPath);
|
|
} catch (err) {
|
|
await this.moveIndexFiles(backupPath, targetPath);
|
|
throw err;
|
|
}
|
|
await this.removeIndexFiles(backupPath);
|
|
}
|
|
async moveIndexFiles(sourceBase, targetBase) {
|
|
for (const suffix of [
|
|
"",
|
|
"-wal",
|
|
"-shm"
|
|
]) {
|
|
const source = `${sourceBase}${suffix}`;
|
|
const target = `${targetBase}${suffix}`;
|
|
try {
|
|
await fs$1.rename(source, target);
|
|
} catch (err) {
|
|
if (err.code !== "ENOENT") throw err;
|
|
}
|
|
}
|
|
}
|
|
async removeIndexFiles(basePath) {
|
|
await Promise.all([
|
|
"",
|
|
"-wal",
|
|
"-shm"
|
|
].map((suffix) => fs$1.rm(`${basePath}${suffix}`, { force: true })));
|
|
}
|
|
ensureSchema() {
|
|
const result = ensureMemoryIndexSchema({
|
|
db: this.db,
|
|
embeddingCacheTable: EMBEDDING_CACHE_TABLE,
|
|
ftsTable: FTS_TABLE,
|
|
ftsEnabled: this.fts.enabled
|
|
});
|
|
this.fts.available = result.ftsAvailable;
|
|
if (result.ftsError) {
|
|
this.fts.loadError = result.ftsError;
|
|
log.warn(`fts unavailable: ${result.ftsError}`);
|
|
}
|
|
}
|
|
ensureWatcher() {
|
|
if (!this.sources.has("memory") || !this.settings.sync.watch || this.watcher) return;
|
|
const additionalPaths = normalizeExtraMemoryPaths(this.workspaceDir, this.settings.extraPaths).map((entry) => {
|
|
try {
|
|
return fs.lstatSync(entry).isSymbolicLink() ? null : entry;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}).filter((entry) => Boolean(entry));
|
|
const watchPaths = new Set([
|
|
path.join(this.workspaceDir, "MEMORY.md"),
|
|
path.join(this.workspaceDir, "memory.md"),
|
|
path.join(this.workspaceDir, "memory"),
|
|
...additionalPaths
|
|
]);
|
|
this.watcher = chokidar.watch(Array.from(watchPaths), {
|
|
ignoreInitial: true,
|
|
awaitWriteFinish: {
|
|
stabilityThreshold: this.settings.sync.watchDebounceMs,
|
|
pollInterval: 100
|
|
}
|
|
});
|
|
const markDirty = () => {
|
|
this.dirty = true;
|
|
this.scheduleWatchSync();
|
|
};
|
|
this.watcher.on("add", markDirty);
|
|
this.watcher.on("change", markDirty);
|
|
this.watcher.on("unlink", markDirty);
|
|
}
|
|
ensureSessionListener() {
|
|
if (!this.sources.has("sessions") || this.sessionUnsubscribe) return;
|
|
this.sessionUnsubscribe = onSessionTranscriptUpdate((update) => {
|
|
if (this.closed) return;
|
|
const sessionFile = update.sessionFile;
|
|
if (!this.isSessionFileForAgent(sessionFile)) return;
|
|
this.scheduleSessionDirty(sessionFile);
|
|
});
|
|
}
|
|
scheduleSessionDirty(sessionFile) {
|
|
this.sessionPendingFiles.add(sessionFile);
|
|
if (this.sessionWatchTimer) return;
|
|
this.sessionWatchTimer = setTimeout(() => {
|
|
this.sessionWatchTimer = null;
|
|
this.processSessionDeltaBatch().catch((err) => {
|
|
log.warn(`memory session delta failed: ${String(err)}`);
|
|
});
|
|
}, SESSION_DIRTY_DEBOUNCE_MS);
|
|
}
|
|
async processSessionDeltaBatch() {
|
|
if (this.sessionPendingFiles.size === 0) return;
|
|
const pending = Array.from(this.sessionPendingFiles);
|
|
this.sessionPendingFiles.clear();
|
|
let shouldSync = false;
|
|
for (const sessionFile of pending) {
|
|
const delta = await this.updateSessionDelta(sessionFile);
|
|
if (!delta) continue;
|
|
const bytesThreshold = delta.deltaBytes;
|
|
const messagesThreshold = delta.deltaMessages;
|
|
const bytesHit = bytesThreshold <= 0 ? delta.pendingBytes > 0 : delta.pendingBytes >= bytesThreshold;
|
|
const messagesHit = messagesThreshold <= 0 ? delta.pendingMessages > 0 : delta.pendingMessages >= messagesThreshold;
|
|
if (!bytesHit && !messagesHit) continue;
|
|
this.sessionsDirtyFiles.add(sessionFile);
|
|
this.sessionsDirty = true;
|
|
delta.pendingBytes = bytesThreshold > 0 ? Math.max(0, delta.pendingBytes - bytesThreshold) : 0;
|
|
delta.pendingMessages = messagesThreshold > 0 ? Math.max(0, delta.pendingMessages - messagesThreshold) : 0;
|
|
shouldSync = true;
|
|
}
|
|
if (shouldSync) this.sync({ reason: "session-delta" }).catch((err) => {
|
|
log.warn(`memory sync failed (session-delta): ${String(err)}`);
|
|
});
|
|
}
|
|
async updateSessionDelta(sessionFile) {
|
|
const thresholds = this.settings.sync.sessions;
|
|
if (!thresholds) return null;
|
|
let stat;
|
|
try {
|
|
stat = await fs$1.stat(sessionFile);
|
|
} catch {
|
|
return null;
|
|
}
|
|
const size = stat.size;
|
|
let state = this.sessionDeltas.get(sessionFile);
|
|
if (!state) {
|
|
state = {
|
|
lastSize: 0,
|
|
pendingBytes: 0,
|
|
pendingMessages: 0
|
|
};
|
|
this.sessionDeltas.set(sessionFile, state);
|
|
}
|
|
const deltaBytes = Math.max(0, size - state.lastSize);
|
|
if (deltaBytes === 0 && size === state.lastSize) return {
|
|
deltaBytes: thresholds.deltaBytes,
|
|
deltaMessages: thresholds.deltaMessages,
|
|
pendingBytes: state.pendingBytes,
|
|
pendingMessages: state.pendingMessages
|
|
};
|
|
if (size < state.lastSize) {
|
|
state.lastSize = size;
|
|
state.pendingBytes += size;
|
|
if (thresholds.deltaMessages > 0 && (thresholds.deltaBytes <= 0 || state.pendingBytes < thresholds.deltaBytes)) state.pendingMessages += await this.countNewlines(sessionFile, 0, size);
|
|
} else {
|
|
state.pendingBytes += deltaBytes;
|
|
if (thresholds.deltaMessages > 0 && (thresholds.deltaBytes <= 0 || state.pendingBytes < thresholds.deltaBytes)) state.pendingMessages += await this.countNewlines(sessionFile, state.lastSize, size);
|
|
state.lastSize = size;
|
|
}
|
|
this.sessionDeltas.set(sessionFile, state);
|
|
return {
|
|
deltaBytes: thresholds.deltaBytes,
|
|
deltaMessages: thresholds.deltaMessages,
|
|
pendingBytes: state.pendingBytes,
|
|
pendingMessages: state.pendingMessages
|
|
};
|
|
}
|
|
async countNewlines(absPath, start, end) {
|
|
if (end <= start) return 0;
|
|
const handle = await fs$1.open(absPath, "r");
|
|
try {
|
|
let offset = start;
|
|
let count = 0;
|
|
const buffer = Buffer.alloc(SESSION_DELTA_READ_CHUNK_BYTES);
|
|
while (offset < end) {
|
|
const toRead = Math.min(buffer.length, end - offset);
|
|
const { bytesRead } = await handle.read(buffer, 0, toRead, offset);
|
|
if (bytesRead <= 0) break;
|
|
for (let i = 0; i < bytesRead; i += 1) if (buffer[i] === 10) count += 1;
|
|
offset += bytesRead;
|
|
}
|
|
return count;
|
|
} finally {
|
|
await handle.close();
|
|
}
|
|
}
|
|
resetSessionDelta(absPath, size) {
|
|
const state = this.sessionDeltas.get(absPath);
|
|
if (!state) return;
|
|
state.lastSize = size;
|
|
state.pendingBytes = 0;
|
|
state.pendingMessages = 0;
|
|
}
|
|
isSessionFileForAgent(sessionFile) {
|
|
if (!sessionFile) return false;
|
|
const sessionsDir = resolveSessionTranscriptsDirForAgent(this.agentId);
|
|
const resolvedFile = path.resolve(sessionFile);
|
|
const resolvedDir = path.resolve(sessionsDir);
|
|
return resolvedFile.startsWith(`${resolvedDir}${path.sep}`);
|
|
}
|
|
ensureIntervalSync() {
|
|
const minutes = this.settings.sync.intervalMinutes;
|
|
if (!minutes || minutes <= 0 || this.intervalTimer) return;
|
|
const ms = minutes * 60 * 1e3;
|
|
this.intervalTimer = setInterval(() => {
|
|
this.sync({ reason: "interval" }).catch((err) => {
|
|
log.warn(`memory sync failed (interval): ${String(err)}`);
|
|
});
|
|
}, ms);
|
|
}
|
|
scheduleWatchSync() {
|
|
if (!this.sources.has("memory") || !this.settings.sync.watch) return;
|
|
if (this.watchTimer) clearTimeout(this.watchTimer);
|
|
this.watchTimer = setTimeout(() => {
|
|
this.watchTimer = null;
|
|
this.sync({ reason: "watch" }).catch((err) => {
|
|
log.warn(`memory sync failed (watch): ${String(err)}`);
|
|
});
|
|
}, this.settings.sync.watchDebounceMs);
|
|
}
|
|
shouldSyncSessions(params, needsFullReindex = false) {
|
|
if (!this.sources.has("sessions")) return false;
|
|
if (params?.force) return true;
|
|
const reason = params?.reason;
|
|
if (reason === "session-start" || reason === "watch") return false;
|
|
if (needsFullReindex) return true;
|
|
return this.sessionsDirty && this.sessionsDirtyFiles.size > 0;
|
|
}
|
|
async syncMemoryFiles(params) {
|
|
const files = await listMemoryFiles(this.workspaceDir, this.settings.extraPaths);
|
|
const fileEntries = await Promise.all(files.map(async (file) => buildFileEntry(file, this.workspaceDir)));
|
|
log.debug("memory sync: indexing memory files", {
|
|
files: fileEntries.length,
|
|
needsFullReindex: params.needsFullReindex,
|
|
batch: this.batch.enabled,
|
|
concurrency: this.getIndexConcurrency()
|
|
});
|
|
const activePaths = new Set(fileEntries.map((entry) => entry.path));
|
|
if (params.progress) {
|
|
params.progress.total += fileEntries.length;
|
|
params.progress.report({
|
|
completed: params.progress.completed,
|
|
total: params.progress.total,
|
|
label: this.batch.enabled ? "Indexing memory files (batch)..." : "Indexing memory files…"
|
|
});
|
|
}
|
|
const tasks = fileEntries.map((entry) => async () => {
|
|
const record = this.db.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`).get(entry.path, "memory");
|
|
if (!params.needsFullReindex && record?.hash === entry.hash) {
|
|
if (params.progress) {
|
|
params.progress.completed += 1;
|
|
params.progress.report({
|
|
completed: params.progress.completed,
|
|
total: params.progress.total
|
|
});
|
|
}
|
|
return;
|
|
}
|
|
await this.indexFile(entry, { source: "memory" });
|
|
if (params.progress) {
|
|
params.progress.completed += 1;
|
|
params.progress.report({
|
|
completed: params.progress.completed,
|
|
total: params.progress.total
|
|
});
|
|
}
|
|
});
|
|
await this.runWithConcurrency(tasks, this.getIndexConcurrency());
|
|
const staleRows = this.db.prepare(`SELECT path FROM files WHERE source = ?`).all("memory");
|
|
for (const stale of staleRows) {
|
|
if (activePaths.has(stale.path)) continue;
|
|
this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(stale.path, "memory");
|
|
try {
|
|
this.db.prepare(`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`).run(stale.path, "memory");
|
|
} catch {}
|
|
this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(stale.path, "memory");
|
|
if (this.fts.enabled && this.fts.available) try {
|
|
this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`).run(stale.path, "memory", this.provider.model);
|
|
} catch {}
|
|
}
|
|
}
|
|
async syncSessionFiles(params) {
|
|
const files = await this.listSessionFiles();
|
|
const activePaths = new Set(files.map((file) => this.sessionPathForFile(file)));
|
|
const indexAll = params.needsFullReindex || this.sessionsDirtyFiles.size === 0;
|
|
log.debug("memory sync: indexing session files", {
|
|
files: files.length,
|
|
indexAll,
|
|
dirtyFiles: this.sessionsDirtyFiles.size,
|
|
batch: this.batch.enabled,
|
|
concurrency: this.getIndexConcurrency()
|
|
});
|
|
if (params.progress) {
|
|
params.progress.total += files.length;
|
|
params.progress.report({
|
|
completed: params.progress.completed,
|
|
total: params.progress.total,
|
|
label: this.batch.enabled ? "Indexing session files (batch)..." : "Indexing session files…"
|
|
});
|
|
}
|
|
const tasks = files.map((absPath) => async () => {
|
|
if (!indexAll && !this.sessionsDirtyFiles.has(absPath)) {
|
|
if (params.progress) {
|
|
params.progress.completed += 1;
|
|
params.progress.report({
|
|
completed: params.progress.completed,
|
|
total: params.progress.total
|
|
});
|
|
}
|
|
return;
|
|
}
|
|
const entry = await this.buildSessionEntry(absPath);
|
|
if (!entry) {
|
|
if (params.progress) {
|
|
params.progress.completed += 1;
|
|
params.progress.report({
|
|
completed: params.progress.completed,
|
|
total: params.progress.total
|
|
});
|
|
}
|
|
return;
|
|
}
|
|
const record = this.db.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`).get(entry.path, "sessions");
|
|
if (!params.needsFullReindex && record?.hash === entry.hash) {
|
|
if (params.progress) {
|
|
params.progress.completed += 1;
|
|
params.progress.report({
|
|
completed: params.progress.completed,
|
|
total: params.progress.total
|
|
});
|
|
}
|
|
this.resetSessionDelta(absPath, entry.size);
|
|
return;
|
|
}
|
|
await this.indexFile(entry, {
|
|
source: "sessions",
|
|
content: entry.content
|
|
});
|
|
this.resetSessionDelta(absPath, entry.size);
|
|
if (params.progress) {
|
|
params.progress.completed += 1;
|
|
params.progress.report({
|
|
completed: params.progress.completed,
|
|
total: params.progress.total
|
|
});
|
|
}
|
|
});
|
|
await this.runWithConcurrency(tasks, this.getIndexConcurrency());
|
|
const staleRows = this.db.prepare(`SELECT path FROM files WHERE source = ?`).all("sessions");
|
|
for (const stale of staleRows) {
|
|
if (activePaths.has(stale.path)) continue;
|
|
this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(stale.path, "sessions");
|
|
try {
|
|
this.db.prepare(`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`).run(stale.path, "sessions");
|
|
} catch {}
|
|
this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(stale.path, "sessions");
|
|
if (this.fts.enabled && this.fts.available) try {
|
|
this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`).run(stale.path, "sessions", this.provider.model);
|
|
} catch {}
|
|
}
|
|
}
|
|
createSyncProgress(onProgress) {
|
|
const state = {
|
|
completed: 0,
|
|
total: 0,
|
|
label: void 0,
|
|
report: (update) => {
|
|
if (update.label) state.label = update.label;
|
|
const label = update.total > 0 && state.label ? `${state.label} ${update.completed}/${update.total}` : state.label;
|
|
onProgress({
|
|
completed: update.completed,
|
|
total: update.total,
|
|
label
|
|
});
|
|
}
|
|
};
|
|
return state;
|
|
}
|
|
async runSync(params) {
|
|
const progress = params?.progress ? this.createSyncProgress(params.progress) : void 0;
|
|
if (progress) progress.report({
|
|
completed: progress.completed,
|
|
total: progress.total,
|
|
label: "Loading vector extension…"
|
|
});
|
|
const vectorReady = await this.ensureVectorReady();
|
|
const meta = this.readMeta();
|
|
const needsFullReindex = params?.force || !meta || meta.model !== this.provider.model || meta.provider !== this.provider.id || meta.providerKey !== this.providerKey || meta.chunkTokens !== this.settings.chunking.tokens || meta.chunkOverlap !== this.settings.chunking.overlap || vectorReady && !meta?.vectorDims;
|
|
try {
|
|
if (needsFullReindex) {
|
|
await this.runSafeReindex({
|
|
reason: params?.reason,
|
|
force: params?.force,
|
|
progress: progress ?? void 0
|
|
});
|
|
return;
|
|
}
|
|
const shouldSyncMemory = this.sources.has("memory") && (params?.force || needsFullReindex || this.dirty);
|
|
const shouldSyncSessions = this.shouldSyncSessions(params, needsFullReindex);
|
|
if (shouldSyncMemory) {
|
|
await this.syncMemoryFiles({
|
|
needsFullReindex,
|
|
progress: progress ?? void 0
|
|
});
|
|
this.dirty = false;
|
|
}
|
|
if (shouldSyncSessions) {
|
|
await this.syncSessionFiles({
|
|
needsFullReindex,
|
|
progress: progress ?? void 0
|
|
});
|
|
this.sessionsDirty = false;
|
|
this.sessionsDirtyFiles.clear();
|
|
} else if (this.sessionsDirtyFiles.size > 0) this.sessionsDirty = true;
|
|
else this.sessionsDirty = false;
|
|
} catch (err) {
|
|
const reason = err instanceof Error ? err.message : String(err);
|
|
if (this.shouldFallbackOnError(reason) && await this.activateFallbackProvider(reason)) {
|
|
await this.runSafeReindex({
|
|
reason: params?.reason ?? "fallback",
|
|
force: true,
|
|
progress: progress ?? void 0
|
|
});
|
|
return;
|
|
}
|
|
throw err;
|
|
}
|
|
}
|
|
shouldFallbackOnError(message) {
|
|
return /embedding|embeddings|batch/i.test(message);
|
|
}
|
|
resolveBatchConfig() {
|
|
const batch = this.settings.remote?.batch;
|
|
return {
|
|
enabled: Boolean(batch?.enabled && (this.openAi && this.provider.id === "openai" || this.gemini && this.provider.id === "gemini")),
|
|
wait: batch?.wait ?? true,
|
|
concurrency: Math.max(1, batch?.concurrency ?? 2),
|
|
pollIntervalMs: batch?.pollIntervalMs ?? 2e3,
|
|
timeoutMs: (batch?.timeoutMinutes ?? 60) * 60 * 1e3
|
|
};
|
|
}
|
|
async activateFallbackProvider(reason) {
|
|
const fallback = this.settings.fallback;
|
|
if (!fallback || fallback === "none" || fallback === this.provider.id) return false;
|
|
if (this.fallbackFrom) return false;
|
|
const fallbackFrom = this.provider.id;
|
|
const fallbackModel = fallback === "gemini" ? DEFAULT_GEMINI_EMBEDDING_MODEL : fallback === "openai" ? DEFAULT_OPENAI_EMBEDDING_MODEL : this.settings.model;
|
|
const fallbackResult = await createEmbeddingProvider({
|
|
config: this.cfg,
|
|
agentDir: resolveAgentDir(this.cfg, this.agentId),
|
|
provider: fallback,
|
|
remote: this.settings.remote,
|
|
model: fallbackModel,
|
|
fallback: "none",
|
|
local: this.settings.local
|
|
});
|
|
this.fallbackFrom = fallbackFrom;
|
|
this.fallbackReason = reason;
|
|
this.provider = fallbackResult.provider;
|
|
this.openAi = fallbackResult.openAi;
|
|
this.gemini = fallbackResult.gemini;
|
|
this.providerKey = this.computeProviderKey();
|
|
this.batch = this.resolveBatchConfig();
|
|
log.warn(`memory embeddings: switched to fallback provider (${fallback})`, { reason });
|
|
return true;
|
|
}
|
|
async runSafeReindex(params) {
|
|
const dbPath = resolveUserPath(this.settings.store.path);
|
|
const tempDbPath = `${dbPath}.tmp-${randomUUID()}`;
|
|
const tempDb = this.openDatabaseAtPath(tempDbPath);
|
|
const originalDb = this.db;
|
|
let originalDbClosed = false;
|
|
const originalState = {
|
|
ftsAvailable: this.fts.available,
|
|
ftsError: this.fts.loadError,
|
|
vectorAvailable: this.vector.available,
|
|
vectorLoadError: this.vector.loadError,
|
|
vectorDims: this.vector.dims,
|
|
vectorReady: this.vectorReady
|
|
};
|
|
const restoreOriginalState = () => {
|
|
if (originalDbClosed) this.db = this.openDatabaseAtPath(dbPath);
|
|
else this.db = originalDb;
|
|
this.fts.available = originalState.ftsAvailable;
|
|
this.fts.loadError = originalState.ftsError;
|
|
this.vector.available = originalDbClosed ? null : originalState.vectorAvailable;
|
|
this.vector.loadError = originalState.vectorLoadError;
|
|
this.vector.dims = originalState.vectorDims;
|
|
this.vectorReady = originalDbClosed ? null : originalState.vectorReady;
|
|
};
|
|
this.db = tempDb;
|
|
this.vectorReady = null;
|
|
this.vector.available = null;
|
|
this.vector.loadError = void 0;
|
|
this.vector.dims = void 0;
|
|
this.fts.available = false;
|
|
this.fts.loadError = void 0;
|
|
this.ensureSchema();
|
|
let nextMeta = null;
|
|
try {
|
|
this.seedEmbeddingCache(originalDb);
|
|
const shouldSyncMemory = this.sources.has("memory");
|
|
const shouldSyncSessions = this.shouldSyncSessions({
|
|
reason: params.reason,
|
|
force: params.force
|
|
}, true);
|
|
if (shouldSyncMemory) {
|
|
await this.syncMemoryFiles({
|
|
needsFullReindex: true,
|
|
progress: params.progress
|
|
});
|
|
this.dirty = false;
|
|
}
|
|
if (shouldSyncSessions) {
|
|
await this.syncSessionFiles({
|
|
needsFullReindex: true,
|
|
progress: params.progress
|
|
});
|
|
this.sessionsDirty = false;
|
|
this.sessionsDirtyFiles.clear();
|
|
} else if (this.sessionsDirtyFiles.size > 0) this.sessionsDirty = true;
|
|
else this.sessionsDirty = false;
|
|
nextMeta = {
|
|
model: this.provider.model,
|
|
provider: this.provider.id,
|
|
providerKey: this.providerKey,
|
|
chunkTokens: this.settings.chunking.tokens,
|
|
chunkOverlap: this.settings.chunking.overlap
|
|
};
|
|
if (this.vector.available && this.vector.dims) nextMeta.vectorDims = this.vector.dims;
|
|
this.writeMeta(nextMeta);
|
|
this.pruneEmbeddingCacheIfNeeded();
|
|
this.db.close();
|
|
originalDb.close();
|
|
originalDbClosed = true;
|
|
await this.swapIndexFiles(dbPath, tempDbPath);
|
|
this.db = this.openDatabaseAtPath(dbPath);
|
|
this.vectorReady = null;
|
|
this.vector.available = null;
|
|
this.vector.loadError = void 0;
|
|
this.ensureSchema();
|
|
this.vector.dims = nextMeta.vectorDims;
|
|
} catch (err) {
|
|
try {
|
|
this.db.close();
|
|
} catch {}
|
|
await this.removeIndexFiles(tempDbPath);
|
|
restoreOriginalState();
|
|
throw err;
|
|
}
|
|
}
|
|
resetIndex() {
|
|
this.db.exec(`DELETE FROM files`);
|
|
this.db.exec(`DELETE FROM chunks`);
|
|
if (this.fts.enabled && this.fts.available) try {
|
|
this.db.exec(`DELETE FROM ${FTS_TABLE}`);
|
|
} catch {}
|
|
this.dropVectorTable();
|
|
this.vector.dims = void 0;
|
|
this.sessionsDirtyFiles.clear();
|
|
}
|
|
readMeta() {
|
|
const row = this.db.prepare(`SELECT value FROM meta WHERE key = ?`).get(META_KEY);
|
|
if (!row?.value) return null;
|
|
try {
|
|
return JSON.parse(row.value);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
writeMeta(meta) {
|
|
const value = JSON.stringify(meta);
|
|
this.db.prepare(`INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`).run(META_KEY, value);
|
|
}
|
|
async listSessionFiles() {
|
|
const dir = resolveSessionTranscriptsDirForAgent(this.agentId);
|
|
try {
|
|
return (await fs$1.readdir(dir, { withFileTypes: true })).filter((entry) => entry.isFile()).map((entry) => entry.name).filter((name) => name.endsWith(".jsonl")).map((name) => path.join(dir, name));
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
sessionPathForFile(absPath) {
|
|
return path.join("sessions", path.basename(absPath)).replace(/\\/g, "/");
|
|
}
|
|
normalizeSessionText(value) {
|
|
return value.replace(/\s*\n+\s*/g, " ").replace(/\s+/g, " ").trim();
|
|
}
|
|
extractSessionText(content) {
|
|
if (typeof content === "string") {
|
|
const normalized = this.normalizeSessionText(content);
|
|
return normalized ? normalized : null;
|
|
}
|
|
if (!Array.isArray(content)) return null;
|
|
const parts = [];
|
|
for (const block of content) {
|
|
if (!block || typeof block !== "object") continue;
|
|
const record = block;
|
|
if (record.type !== "text" || typeof record.text !== "string") continue;
|
|
const normalized = this.normalizeSessionText(record.text);
|
|
if (normalized) parts.push(normalized);
|
|
}
|
|
if (parts.length === 0) return null;
|
|
return parts.join(" ");
|
|
}
|
|
async buildSessionEntry(absPath) {
|
|
try {
|
|
const stat = await fs$1.stat(absPath);
|
|
const lines = (await fs$1.readFile(absPath, "utf-8")).split("\n");
|
|
const collected = [];
|
|
for (const line of lines) {
|
|
if (!line.trim()) continue;
|
|
let record;
|
|
try {
|
|
record = JSON.parse(line);
|
|
} catch {
|
|
continue;
|
|
}
|
|
if (!record || typeof record !== "object" || record.type !== "message") continue;
|
|
const message = record.message;
|
|
if (!message || typeof message.role !== "string") continue;
|
|
if (message.role !== "user" && message.role !== "assistant") continue;
|
|
const text = this.extractSessionText(message.content);
|
|
if (!text) continue;
|
|
const label = message.role === "user" ? "User" : "Assistant";
|
|
collected.push(`${label}: ${text}`);
|
|
}
|
|
const content = collected.join("\n");
|
|
return {
|
|
path: this.sessionPathForFile(absPath),
|
|
absPath,
|
|
mtimeMs: stat.mtimeMs,
|
|
size: stat.size,
|
|
hash: hashText(content),
|
|
content
|
|
};
|
|
} catch (err) {
|
|
log.debug(`Failed reading session file ${absPath}: ${String(err)}`);
|
|
return null;
|
|
}
|
|
}
|
|
estimateEmbeddingTokens(text) {
|
|
if (!text) return 0;
|
|
return Math.ceil(text.length / EMBEDDING_APPROX_CHARS_PER_TOKEN);
|
|
}
|
|
buildEmbeddingBatches(chunks) {
|
|
const batches = [];
|
|
let current = [];
|
|
let currentTokens = 0;
|
|
for (const chunk of chunks) {
|
|
const estimate = this.estimateEmbeddingTokens(chunk.text);
|
|
if (current.length > 0 && currentTokens + estimate > EMBEDDING_BATCH_MAX_TOKENS) {
|
|
batches.push(current);
|
|
current = [];
|
|
currentTokens = 0;
|
|
}
|
|
if (current.length === 0 && estimate > EMBEDDING_BATCH_MAX_TOKENS) {
|
|
batches.push([chunk]);
|
|
continue;
|
|
}
|
|
current.push(chunk);
|
|
currentTokens += estimate;
|
|
}
|
|
if (current.length > 0) batches.push(current);
|
|
return batches;
|
|
}
|
|
loadEmbeddingCache(hashes) {
|
|
if (!this.cache.enabled) return /* @__PURE__ */ new Map();
|
|
if (hashes.length === 0) return /* @__PURE__ */ new Map();
|
|
const unique = [];
|
|
const seen = /* @__PURE__ */ new Set();
|
|
for (const hash of hashes) {
|
|
if (!hash) continue;
|
|
if (seen.has(hash)) continue;
|
|
seen.add(hash);
|
|
unique.push(hash);
|
|
}
|
|
if (unique.length === 0) return /* @__PURE__ */ new Map();
|
|
const out = /* @__PURE__ */ new Map();
|
|
const baseParams = [
|
|
this.provider.id,
|
|
this.provider.model,
|
|
this.providerKey
|
|
];
|
|
const batchSize = 400;
|
|
for (let start = 0; start < unique.length; start += batchSize) {
|
|
const batch = unique.slice(start, start + batchSize);
|
|
const placeholders = batch.map(() => "?").join(", ");
|
|
const rows = this.db.prepare(`SELECT hash, embedding FROM ${EMBEDDING_CACHE_TABLE}\n WHERE provider = ? AND model = ? AND provider_key = ? AND hash IN (${placeholders})`).all(...baseParams, ...batch);
|
|
for (const row of rows) out.set(row.hash, parseEmbedding(row.embedding));
|
|
}
|
|
return out;
|
|
}
|
|
upsertEmbeddingCache(entries) {
|
|
if (!this.cache.enabled) return;
|
|
if (entries.length === 0) return;
|
|
const now = Date.now();
|
|
const stmt = this.db.prepare(`INSERT INTO ${EMBEDDING_CACHE_TABLE} (provider, model, provider_key, hash, embedding, dims, updated_at)\n VALUES (?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT(provider, model, provider_key, hash) DO UPDATE SET\n embedding=excluded.embedding,\n dims=excluded.dims,\n updated_at=excluded.updated_at`);
|
|
for (const entry of entries) {
|
|
const embedding = entry.embedding ?? [];
|
|
stmt.run(this.provider.id, this.provider.model, this.providerKey, entry.hash, JSON.stringify(embedding), embedding.length, now);
|
|
}
|
|
}
|
|
pruneEmbeddingCacheIfNeeded() {
|
|
if (!this.cache.enabled) return;
|
|
const max = this.cache.maxEntries;
|
|
if (!max || max <= 0) return;
|
|
const count = this.db.prepare(`SELECT COUNT(*) as c FROM ${EMBEDDING_CACHE_TABLE}`).get()?.c ?? 0;
|
|
if (count <= max) return;
|
|
const excess = count - max;
|
|
this.db.prepare(`DELETE FROM ${EMBEDDING_CACHE_TABLE}\n WHERE rowid IN (\n SELECT rowid FROM ${EMBEDDING_CACHE_TABLE}\n ORDER BY updated_at ASC\n LIMIT ?\n )`).run(excess);
|
|
}
|
|
async embedChunksInBatches(chunks) {
|
|
if (chunks.length === 0) return [];
|
|
const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash));
|
|
const embeddings = Array.from({ length: chunks.length }, () => []);
|
|
const missing = [];
|
|
for (let i = 0; i < chunks.length; i += 1) {
|
|
const chunk = chunks[i];
|
|
const hit = chunk?.hash ? cached.get(chunk.hash) : void 0;
|
|
if (hit && hit.length > 0) embeddings[i] = hit;
|
|
else if (chunk) missing.push({
|
|
index: i,
|
|
chunk
|
|
});
|
|
}
|
|
if (missing.length === 0) return embeddings;
|
|
const missingChunks = missing.map((m) => m.chunk);
|
|
const batches = this.buildEmbeddingBatches(missingChunks);
|
|
const toCache = [];
|
|
let cursor = 0;
|
|
for (const batch of batches) {
|
|
const batchEmbeddings = await this.embedBatchWithRetry(batch.map((chunk) => chunk.text));
|
|
for (let i = 0; i < batch.length; i += 1) {
|
|
const item = missing[cursor + i];
|
|
const embedding = batchEmbeddings[i] ?? [];
|
|
if (item) {
|
|
embeddings[item.index] = embedding;
|
|
toCache.push({
|
|
hash: item.chunk.hash,
|
|
embedding
|
|
});
|
|
}
|
|
}
|
|
cursor += batch.length;
|
|
}
|
|
this.upsertEmbeddingCache(toCache);
|
|
return embeddings;
|
|
}
|
|
computeProviderKey() {
|
|
if (this.provider.id === "openai" && this.openAi) {
|
|
const entries = Object.entries(this.openAi.headers).filter(([key]) => key.toLowerCase() !== "authorization").toSorted(([a], [b]) => a.localeCompare(b)).map(([key, value]) => [key, value]);
|
|
return hashText(JSON.stringify({
|
|
provider: "openai",
|
|
baseUrl: this.openAi.baseUrl,
|
|
model: this.openAi.model,
|
|
headers: entries
|
|
}));
|
|
}
|
|
if (this.provider.id === "gemini" && this.gemini) {
|
|
const entries = Object.entries(this.gemini.headers).filter(([key]) => {
|
|
const lower = key.toLowerCase();
|
|
return lower !== "authorization" && lower !== "x-goog-api-key";
|
|
}).toSorted(([a], [b]) => a.localeCompare(b)).map(([key, value]) => [key, value]);
|
|
return hashText(JSON.stringify({
|
|
provider: "gemini",
|
|
baseUrl: this.gemini.baseUrl,
|
|
model: this.gemini.model,
|
|
headers: entries
|
|
}));
|
|
}
|
|
return hashText(JSON.stringify({
|
|
provider: this.provider.id,
|
|
model: this.provider.model
|
|
}));
|
|
}
|
|
async embedChunksWithBatch(chunks, entry, source) {
|
|
if (this.provider.id === "openai" && this.openAi) return this.embedChunksWithOpenAiBatch(chunks, entry, source);
|
|
if (this.provider.id === "gemini" && this.gemini) return this.embedChunksWithGeminiBatch(chunks, entry, source);
|
|
return this.embedChunksInBatches(chunks);
|
|
}
|
|
async embedChunksWithOpenAiBatch(chunks, entry, source) {
|
|
const openAi = this.openAi;
|
|
if (!openAi) return this.embedChunksInBatches(chunks);
|
|
if (chunks.length === 0) return [];
|
|
const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash));
|
|
const embeddings = Array.from({ length: chunks.length }, () => []);
|
|
const missing = [];
|
|
for (let i = 0; i < chunks.length; i += 1) {
|
|
const chunk = chunks[i];
|
|
const hit = chunk?.hash ? cached.get(chunk.hash) : void 0;
|
|
if (hit && hit.length > 0) embeddings[i] = hit;
|
|
else if (chunk) missing.push({
|
|
index: i,
|
|
chunk
|
|
});
|
|
}
|
|
if (missing.length === 0) return embeddings;
|
|
const requests = [];
|
|
const mapping = /* @__PURE__ */ new Map();
|
|
for (const item of missing) {
|
|
const chunk = item.chunk;
|
|
const customId = hashText(`${source}:${entry.path}:${chunk.startLine}:${chunk.endLine}:${chunk.hash}:${item.index}`);
|
|
mapping.set(customId, {
|
|
index: item.index,
|
|
hash: chunk.hash
|
|
});
|
|
requests.push({
|
|
custom_id: customId,
|
|
method: "POST",
|
|
url: OPENAI_BATCH_ENDPOINT,
|
|
body: {
|
|
model: this.openAi?.model ?? this.provider.model,
|
|
input: chunk.text
|
|
}
|
|
});
|
|
}
|
|
const batchResult = await this.runBatchWithFallback({
|
|
provider: "openai",
|
|
run: async () => await runOpenAiEmbeddingBatches({
|
|
openAi,
|
|
agentId: this.agentId,
|
|
requests,
|
|
wait: this.batch.wait,
|
|
concurrency: this.batch.concurrency,
|
|
pollIntervalMs: this.batch.pollIntervalMs,
|
|
timeoutMs: this.batch.timeoutMs,
|
|
debug: (message, data) => log.debug(message, {
|
|
...data,
|
|
source,
|
|
chunks: chunks.length
|
|
})
|
|
}),
|
|
fallback: async () => await this.embedChunksInBatches(chunks)
|
|
});
|
|
if (Array.isArray(batchResult)) return batchResult;
|
|
const byCustomId = batchResult;
|
|
const toCache = [];
|
|
for (const [customId, embedding] of byCustomId.entries()) {
|
|
const mapped = mapping.get(customId);
|
|
if (!mapped) continue;
|
|
embeddings[mapped.index] = embedding;
|
|
toCache.push({
|
|
hash: mapped.hash,
|
|
embedding
|
|
});
|
|
}
|
|
this.upsertEmbeddingCache(toCache);
|
|
return embeddings;
|
|
}
|
|
async embedChunksWithGeminiBatch(chunks, entry, source) {
|
|
const gemini = this.gemini;
|
|
if (!gemini) return this.embedChunksInBatches(chunks);
|
|
if (chunks.length === 0) return [];
|
|
const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash));
|
|
const embeddings = Array.from({ length: chunks.length }, () => []);
|
|
const missing = [];
|
|
for (let i = 0; i < chunks.length; i += 1) {
|
|
const chunk = chunks[i];
|
|
const hit = chunk?.hash ? cached.get(chunk.hash) : void 0;
|
|
if (hit && hit.length > 0) embeddings[i] = hit;
|
|
else if (chunk) missing.push({
|
|
index: i,
|
|
chunk
|
|
});
|
|
}
|
|
if (missing.length === 0) return embeddings;
|
|
const requests = [];
|
|
const mapping = /* @__PURE__ */ new Map();
|
|
for (const item of missing) {
|
|
const chunk = item.chunk;
|
|
const customId = hashText(`${source}:${entry.path}:${chunk.startLine}:${chunk.endLine}:${chunk.hash}:${item.index}`);
|
|
mapping.set(customId, {
|
|
index: item.index,
|
|
hash: chunk.hash
|
|
});
|
|
requests.push({
|
|
custom_id: customId,
|
|
content: { parts: [{ text: chunk.text }] },
|
|
taskType: "RETRIEVAL_DOCUMENT"
|
|
});
|
|
}
|
|
const batchResult = await this.runBatchWithFallback({
|
|
provider: "gemini",
|
|
run: async () => await runGeminiEmbeddingBatches({
|
|
gemini,
|
|
agentId: this.agentId,
|
|
requests,
|
|
wait: this.batch.wait,
|
|
concurrency: this.batch.concurrency,
|
|
pollIntervalMs: this.batch.pollIntervalMs,
|
|
timeoutMs: this.batch.timeoutMs,
|
|
debug: (message, data) => log.debug(message, {
|
|
...data,
|
|
source,
|
|
chunks: chunks.length
|
|
})
|
|
}),
|
|
fallback: async () => await this.embedChunksInBatches(chunks)
|
|
});
|
|
if (Array.isArray(batchResult)) return batchResult;
|
|
const byCustomId = batchResult;
|
|
const toCache = [];
|
|
for (const [customId, embedding] of byCustomId.entries()) {
|
|
const mapped = mapping.get(customId);
|
|
if (!mapped) continue;
|
|
embeddings[mapped.index] = embedding;
|
|
toCache.push({
|
|
hash: mapped.hash,
|
|
embedding
|
|
});
|
|
}
|
|
this.upsertEmbeddingCache(toCache);
|
|
return embeddings;
|
|
}
|
|
async embedBatchWithRetry(texts) {
|
|
if (texts.length === 0) return [];
|
|
let attempt = 0;
|
|
let delayMs = EMBEDDING_RETRY_BASE_DELAY_MS;
|
|
while (true) try {
|
|
const timeoutMs = this.resolveEmbeddingTimeout("batch");
|
|
log.debug("memory embeddings: batch start", {
|
|
provider: this.provider.id,
|
|
items: texts.length,
|
|
timeoutMs
|
|
});
|
|
return await this.withTimeout(this.provider.embedBatch(texts), timeoutMs, `memory embeddings batch timed out after ${Math.round(timeoutMs / 1e3)}s`);
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
if (!this.isRetryableEmbeddingError(message) || attempt >= EMBEDDING_RETRY_MAX_ATTEMPTS) throw err;
|
|
const waitMs = Math.min(EMBEDDING_RETRY_MAX_DELAY_MS, Math.round(delayMs * (1 + Math.random() * .2)));
|
|
log.warn(`memory embeddings rate limited; retrying in ${waitMs}ms`);
|
|
await new Promise((resolve) => setTimeout(resolve, waitMs));
|
|
delayMs *= 2;
|
|
attempt += 1;
|
|
}
|
|
}
|
|
isRetryableEmbeddingError(message) {
|
|
return /(rate[_ ]limit|too many requests|429|resource has been exhausted|5\d\d|cloudflare)/i.test(message);
|
|
}
|
|
resolveEmbeddingTimeout(kind) {
|
|
const isLocal = this.provider.id === "local";
|
|
if (kind === "query") return isLocal ? EMBEDDING_QUERY_TIMEOUT_LOCAL_MS : EMBEDDING_QUERY_TIMEOUT_REMOTE_MS;
|
|
return isLocal ? EMBEDDING_BATCH_TIMEOUT_LOCAL_MS : EMBEDDING_BATCH_TIMEOUT_REMOTE_MS;
|
|
}
|
|
async embedQueryWithTimeout(text) {
|
|
const timeoutMs = this.resolveEmbeddingTimeout("query");
|
|
log.debug("memory embeddings: query start", {
|
|
provider: this.provider.id,
|
|
timeoutMs
|
|
});
|
|
return await this.withTimeout(this.provider.embedQuery(text), timeoutMs, `memory embeddings query timed out after ${Math.round(timeoutMs / 1e3)}s`);
|
|
}
|
|
async withTimeout(promise, timeoutMs, message) {
|
|
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) return await promise;
|
|
let timer = null;
|
|
const timeoutPromise = new Promise((_, reject) => {
|
|
timer = setTimeout(() => reject(new Error(message)), timeoutMs);
|
|
});
|
|
try {
|
|
return await Promise.race([promise, timeoutPromise]);
|
|
} finally {
|
|
if (timer) clearTimeout(timer);
|
|
}
|
|
}
|
|
async runWithConcurrency(tasks, limit) {
|
|
if (tasks.length === 0) return [];
|
|
const resolvedLimit = Math.max(1, Math.min(limit, tasks.length));
|
|
const results = Array.from({ length: tasks.length });
|
|
let next = 0;
|
|
let firstError = null;
|
|
const workers = Array.from({ length: resolvedLimit }, async () => {
|
|
while (true) {
|
|
if (firstError) return;
|
|
const index = next;
|
|
next += 1;
|
|
if (index >= tasks.length) return;
|
|
try {
|
|
results[index] = await tasks[index]();
|
|
} catch (err) {
|
|
firstError = err;
|
|
return;
|
|
}
|
|
}
|
|
});
|
|
await Promise.allSettled(workers);
|
|
if (firstError) throw firstError;
|
|
return results;
|
|
}
|
|
async withBatchFailureLock(fn) {
|
|
let release;
|
|
const wait = this.batchFailureLock;
|
|
this.batchFailureLock = new Promise((resolve) => {
|
|
release = resolve;
|
|
});
|
|
await wait;
|
|
try {
|
|
return await fn();
|
|
} finally {
|
|
release();
|
|
}
|
|
}
|
|
async resetBatchFailureCount() {
|
|
await this.withBatchFailureLock(async () => {
|
|
if (this.batchFailureCount > 0) log.debug("memory embeddings: batch recovered; resetting failure count");
|
|
this.batchFailureCount = 0;
|
|
this.batchFailureLastError = void 0;
|
|
this.batchFailureLastProvider = void 0;
|
|
});
|
|
}
|
|
async recordBatchFailure(params) {
|
|
return await this.withBatchFailureLock(async () => {
|
|
if (!this.batch.enabled) return {
|
|
disabled: true,
|
|
count: this.batchFailureCount
|
|
};
|
|
const increment = params.forceDisable ? BATCH_FAILURE_LIMIT : Math.max(1, params.attempts ?? 1);
|
|
this.batchFailureCount += increment;
|
|
this.batchFailureLastError = params.message;
|
|
this.batchFailureLastProvider = params.provider;
|
|
const disabled = params.forceDisable || this.batchFailureCount >= BATCH_FAILURE_LIMIT;
|
|
if (disabled) this.batch.enabled = false;
|
|
return {
|
|
disabled,
|
|
count: this.batchFailureCount
|
|
};
|
|
});
|
|
}
|
|
isBatchTimeoutError(message) {
|
|
return /timed out|timeout/i.test(message);
|
|
}
|
|
async runBatchWithTimeoutRetry(params) {
|
|
try {
|
|
return await params.run();
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
if (this.isBatchTimeoutError(message)) {
|
|
log.warn(`memory embeddings: ${params.provider} batch timed out; retrying once`);
|
|
try {
|
|
return await params.run();
|
|
} catch (retryErr) {
|
|
retryErr.batchAttempts = 2;
|
|
throw retryErr;
|
|
}
|
|
}
|
|
throw err;
|
|
}
|
|
}
|
|
async runBatchWithFallback(params) {
|
|
if (!this.batch.enabled) return await params.fallback();
|
|
try {
|
|
const result = await this.runBatchWithTimeoutRetry({
|
|
provider: params.provider,
|
|
run: params.run
|
|
});
|
|
await this.resetBatchFailureCount();
|
|
return result;
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
const attempts = err.batchAttempts ?? 1;
|
|
const forceDisable = /asyncBatchEmbedContent not available/i.test(message);
|
|
const failure = await this.recordBatchFailure({
|
|
provider: params.provider,
|
|
message,
|
|
attempts,
|
|
forceDisable
|
|
});
|
|
const suffix = failure.disabled ? "disabling batch" : "keeping batch enabled";
|
|
log.warn(`memory embeddings: ${params.provider} batch failed (${failure.count}/${BATCH_FAILURE_LIMIT}); ${suffix}; falling back to non-batch embeddings: ${message}`);
|
|
return await params.fallback();
|
|
}
|
|
}
|
|
getIndexConcurrency() {
|
|
return this.batch.enabled ? this.batch.concurrency : EMBEDDING_INDEX_CONCURRENCY;
|
|
}
|
|
async indexFile(entry, options) {
|
|
const chunks = chunkMarkdown(options.content ?? await fs$1.readFile(entry.absPath, "utf-8"), this.settings.chunking).filter((chunk) => chunk.text.trim().length > 0);
|
|
const embeddings = this.batch.enabled ? await this.embedChunksWithBatch(chunks, entry, options.source) : await this.embedChunksInBatches(chunks);
|
|
const sample = embeddings.find((embedding) => embedding.length > 0);
|
|
const vectorReady = sample ? await this.ensureVectorReady(sample.length) : false;
|
|
const now = Date.now();
|
|
if (vectorReady) try {
|
|
this.db.prepare(`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`).run(entry.path, options.source);
|
|
} catch {}
|
|
if (this.fts.enabled && this.fts.available) try {
|
|
this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`).run(entry.path, options.source, this.provider.model);
|
|
} catch {}
|
|
this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(entry.path, options.source);
|
|
for (let i = 0; i < chunks.length; i++) {
|
|
const chunk = chunks[i];
|
|
const embedding = embeddings[i] ?? [];
|
|
const id = hashText(`${options.source}:${entry.path}:${chunk.startLine}:${chunk.endLine}:${chunk.hash}:${this.provider.model}`);
|
|
this.db.prepare(`INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(id) DO UPDATE SET
|
|
hash=excluded.hash,
|
|
model=excluded.model,
|
|
text=excluded.text,
|
|
embedding=excluded.embedding,
|
|
updated_at=excluded.updated_at`).run(id, entry.path, options.source, chunk.startLine, chunk.endLine, chunk.hash, this.provider.model, chunk.text, JSON.stringify(embedding), now);
|
|
if (vectorReady && embedding.length > 0) {
|
|
try {
|
|
this.db.prepare(`DELETE FROM ${VECTOR_TABLE} WHERE id = ?`).run(id);
|
|
} catch {}
|
|
this.db.prepare(`INSERT INTO ${VECTOR_TABLE} (id, embedding) VALUES (?, ?)`).run(id, vectorToBlob(embedding));
|
|
}
|
|
if (this.fts.enabled && this.fts.available) this.db.prepare(`INSERT INTO ${FTS_TABLE} (text, id, path, source, model, start_line, end_line)\n VALUES (?, ?, ?, ?, ?, ?, ?)`).run(chunk.text, id, entry.path, options.source, this.provider.model, chunk.startLine, chunk.endLine);
|
|
}
|
|
this.db.prepare(`INSERT INTO files (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)
|
|
ON CONFLICT(path) DO UPDATE SET
|
|
source=excluded.source,
|
|
hash=excluded.hash,
|
|
mtime=excluded.mtime,
|
|
size=excluded.size`).run(entry.path, options.source, entry.hash, entry.mtimeMs, entry.size);
|
|
}
|
|
};
|
|
|
|
//#endregion
|
|
export { retryAsync as i, resolveMemorySearchConfig as n, resolveRetryConfig as r, MemoryIndexManager as t }; |