整理项目记忆与Agent RAG入口

迁移项目共享记忆到 docs/project-memory，保留 .hermes 仅作为工具目录新增 Agent 本地 RAG 索引与上下文包检索脚本记录 RAG 依赖只安装到 .rag/runtime 并加入忽略规则同步文档与检查脚本中的项目记忆路径
2026-06-16 16:06:54 +08:00
parent a51e63415f
commit 15a527d7f4
29 changed files with 738 additions and 97 deletions
--- a/scripts/check-visual-novel-vn11-negative-scan.mjs
+++ b/scripts/check-visual-novel-vn11-negative-scan.mjs
@@ -7,7 +7,7 @@ const reportPath = join(repoRoot, '.tmp', 'VN11_NEGATIVE_SCAN_REPORT_2026-05-07.

 const documentTargets = [
  'docs',
-  '.hermes/shared-memory',
+  'docs/project-memory/shared-memory',
 ];

 const visualNovelImplementationTargets = [
@@ -202,7 +202,7 @@ const reportLines = [
  '## 扫描范围',
  '',
  '- 视觉小说工程代码：视觉小说前端、service、shared contracts、Rust contracts、module、api-server、SpacetimeDB schema 与 facade 路径',
-  '- 文档与共享记忆：`docs/`、`.hermes/shared-memory/`',
+  '- 文档与共享记忆：`docs/`、`docs/project-memory/shared-memory/`',
  '- 外部平台误入复核：视觉小说前端、service、shared contracts、Rust contracts、module、api-server、SpacetimeDB schema 与 facade 路径',
  '',
  '## 扫描结论',
--- a/scripts/rag/README.md
+++ b/scripts/rag/README.md
@@ -0,0 +1,83 @@
+# 本地 RAG
+
+本目录提供项目文档的本地 RAG 索引脚本，主要供 Agent 在执行任务前后检索项目上下文使用。它不是新的人工阅读入口；开发者仍按 `AGENTS.md`、`docs/README.md` 和 `docs/project-memory/` 阅读项目文档。
+
+项目默认不安装 RAG 运行时依赖，也不把 LanceDB、Transformers.js 或本地模型写入根 `package.json`。
+
+## 运行时依赖
+
+RAG 运行时依赖安装在 gitignored 的 `.rag/runtime/`，模型缓存和向量库也都在 `.rag/` 下。
+
+Agent 需要启用 RAG 检索时，应先询问用户是否安装本地依赖。用户确认后执行：
+
+```bash
+mkdir -p .rag/runtime
+npm init -y --prefix .rag/runtime
+npm install --prefix .rag/runtime @lancedb/lancedb@0.30.0 @huggingface/transformers@4.2.0
+```
+
+不要把这些依赖加入根 `package.json`。
+
+## 索引
+
+首次运行会下载本地 embedding 模型到 `.rag/models/`。默认模型为 `Xenova/multilingual-e5-small`，适合中英文混合文档。
+
+```bash
+npm run rag:index
+```
+
+小样本 smoke：
+
+```bash
+npm run rag:index -- --limit-files 3
+```
+
+只查看分片，不加载模型：
+
+```bash
+npm run rag:index -- --limit-files 3 --dry-run
+```
+
+## 搜索
+
+默认输出 Agent 上下文包，包含来源、分数、候选上下文和使用规则：
+
+```bash
+npm run rag:search -- --query "SpacetimeDB schema 变更默认值" --limit 8
+```
+
+可限制上下文包大小：
+
+```bash
+npm run rag:search -- --query "SpacetimeDB schema 变更默认值" --limit 8 --max-chars 8000
+```
+
+可输出结构化格式，便于 Agent 或其它工具解析：
+
+```bash
+npm run rag:search -- --query "SpacetimeDB schema 变更默认值" --format json
+npm run rag:search -- --query "SpacetimeDB schema 变更默认值" --format jsonl
+```
+
+如只想看短摘要，可使用旧式文本结果：
+
+```bash
+npm run rag:search -- --query "SpacetimeDB schema 变更默认值" --format text
+```
+
+Agent 使用规则：
+
+- 把 RAG 输出视为候选上下文，不直接当作最终事实。
+- 需要精确改代码或文档时，仍要打开对应源文件核对。
+- 来源冲突时，以当前代码和最新 `docs/` 为准。
+
+## 索引范围
+
+索引范围由 `scripts/rag/rag-config.json` 配置，默认包含：
+
+- `AGENTS.md`
+- `CONTEXT.md`，如果存在
+- `docs/project-memory/`
+- `docs/`
+
+`.hermes/` 是 Hermes 工具目录，不作为项目知识库索引源。
--- a/scripts/rag/index-docs.mjs
+++ b/scripts/rag/index-docs.mjs
@@ -0,0 +1,68 @@
+import { mkdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+import {
+  buildChunkId,
+  chunkText,
+  createEmbedder,
+  extractTitle,
+  hasFlag,
+  listSourceFiles,
+  loadRagRuntime,
+  parseLimitFiles,
+  readConfig,
+  repoRoot,
+} from './rag-utils.mjs';
+
+const config = readConfig();
+const limitFiles = parseLimitFiles(process.argv);
+const dryRun = hasFlag(process.argv, '--dry-run');
+
+const files = listSourceFiles(config, limitFiles);
+const rows = [];
+
+for (const file of files) {
+  const text = readFileSync(file.path, 'utf8');
+  const title = extractTitle(text, file.rel);
+  for (const chunk of chunkText(text, config.chunk ?? {})) {
+    rows.push({
+      id: buildChunkId(file.rel, chunk.index),
+      path: file.rel,
+      title,
+      chunk_index: chunk.index,
+      source_weight: file.weight,
+      text: chunk.text,
+    });
+  }
+}
+
+console.log(`[rag:index] source files=${files.length}, chunks=${rows.length}`);
+
+if (dryRun) {
+  for (const row of rows.slice(0, 10)) {
+    console.log(`- ${row.id} ${row.title}`);
+  }
+  process.exit(0);
+}
+
+if (rows.length === 0) {
+  throw new Error('No RAG chunks found.');
+}
+
+const { lancedb, transformers } = await loadRagRuntime(config);
+const embed = await createEmbedder(transformers, config.model);
+
+for (let index = 0; index < rows.length; index += 1) {
+  rows[index].vector = await embed(rows[index].text, 'passage');
+  if ((index + 1) % 25 === 0 || index + 1 === rows.length) {
+    console.log(`[rag:index] embedded ${index + 1}/${rows.length}`);
+  }
+}
+
+mkdirSync(join(repoRoot, config.databaseDir), { recursive: true });
+const db = await lancedb.connect(join(repoRoot, config.databaseDir));
+await db.createTable(config.tableName, rows, { mode: 'overwrite' });
+
+console.log(
+  `[rag:index] wrote table=${config.tableName}, db=${config.databaseDir}, model=${config.model}`,
+);
--- a/scripts/rag/rag-config.json
+++ b/scripts/rag/rag-config.json
@@ -0,0 +1,46 @@
+{
+  "runtimeDir": ".rag/runtime",
+  "databaseDir": ".rag/lancedb",
+  "modelCacheDir": ".rag/models",
+  "tableName": "project_docs",
+  "model": "Xenova/multilingual-e5-small",
+  "chunk": {
+    "maxChars": 1600,
+    "overlapChars": 220
+  },
+  "sources": [
+    {
+      "path": "AGENTS.md",
+      "weight": 1.4
+    },
+    {
+      "path": "CONTEXT.md",
+      "weight": 1.3,
+      "optional": true
+    },
+    {
+      "path": "docs/project-memory",
+      "weight": 1.35
+    },
+    {
+      "path": "docs",
+      "weight": 1.0
+    }
+  ],
+  "exclude": [
+    ".git/",
+    ".rag/",
+    ".hermes/",
+    ".codegraph/",
+    ".app/",
+    "node_modules/",
+    "dist/",
+    "build/",
+    "coverage/",
+    "logs/",
+    "output/",
+    "server-rs/target/",
+    "server-rs/target-",
+    "tmp/"
+  ]
+}
--- a/scripts/rag/rag-utils.mjs
+++ b/scripts/rag/rag-utils.mjs
@@ -0,0 +1,221 @@
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { dirname, extname, join, relative, resolve } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+export const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
+export const configPath = join(repoRoot, 'scripts/rag/rag-config.json');
+
+export function readConfig() {
+  return JSON.parse(readFileSync(configPath, 'utf8'));
+}
+
+export function normalizePath(filePath) {
+  return filePath.replace(/\\/gu, '/');
+}
+
+export function repoRelative(filePath) {
+  return normalizePath(relative(repoRoot, filePath));
+}
+
+export function resolveRepoPath(filePath) {
+  return resolve(repoRoot, filePath);
+}
+
+export function getRuntimeNodeModules(config) {
+  return join(repoRoot, config.runtimeDir, 'node_modules');
+}
+
+export function assertLocalRuntime(config) {
+  const runtimeModules = getRuntimeNodeModules(config);
+  const hasLance = existsSync(join(runtimeModules, '@lancedb/lancedb'));
+  const hasTransformers = existsSync(join(runtimeModules, '@huggingface/transformers'));
+
+  if (hasLance && hasTransformers) {
+    return runtimeModules;
+  }
+
+  throw new Error(
+    [
+      '本地 RAG 运行时依赖尚未安装。',
+      '按项目约定，RAG 依赖不进入根 package.json，也不默认安装。',
+      '需要启用 RAG 时，Agent 必须先询问用户，然后在本地 gitignored 目录安装：',
+      '',
+      `  mkdir -p ${config.runtimeDir}`,
+      `  npm init -y --prefix ${config.runtimeDir}`,
+      `  npm install --prefix ${config.runtimeDir} @lancedb/lancedb@0.30.0 @huggingface/transformers@4.2.0`,
+      '',
+      `当前检查目录：${runtimeModules}`,
+    ].join('\n'),
+  );
+}
+
+export async function loadRagRuntime(config) {
+  const runtimeModules = assertLocalRuntime(config);
+  const lancedb = await import(
+    pathToFileURL(join(runtimeModules, '@lancedb/lancedb/dist/index.js')).href
+  );
+  const transformers = await import(
+    pathToFileURL(
+      join(runtimeModules, '@huggingface/transformers/dist/transformers.node.mjs'),
+    ).href
+  );
+
+  transformers.env.cacheDir = join(repoRoot, config.modelCacheDir);
+  transformers.env.useFSCache = true;
+  transformers.env.allowRemoteModels = true;
+
+  return { lancedb, transformers };
+}
+
+export function listSourceFiles(config, limitFiles = Number.POSITIVE_INFINITY) {
+  const excluded = config.exclude ?? [];
+  const files = [];
+  const seen = new Set();
+
+  for (const source of config.sources ?? []) {
+    const sourcePath = resolveRepoPath(source.path);
+    if (!existsSync(sourcePath)) {
+      if (!source.optional) {
+        throw new Error(`RAG source not found: ${source.path}`);
+      }
+      continue;
+    }
+
+    for (const filePath of walkTextFiles(sourcePath, excluded)) {
+      const rel = repoRelative(filePath);
+      if (seen.has(rel)) {
+        continue;
+      }
+      seen.add(rel);
+      files.push({ path: filePath, rel, weight: source.weight ?? 1 });
+      if (files.length >= limitFiles) {
+        return files;
+      }
+    }
+  }
+
+  return files;
+}
+
+function walkTextFiles(targetPath, excluded) {
+  const stat = statSync(targetPath);
+  if (stat.isFile()) {
+    return shouldReadFile(targetPath, excluded) ? [targetPath] : [];
+  }
+
+  const files = [];
+  const walk = (dir) => {
+    for (const name of readdirSync(dir)) {
+      const child = join(dir, name);
+      const rel = `${repoRelative(child)}${statSync(child).isDirectory() ? '/' : ''}`;
+      if (excluded.some((prefix) => rel.startsWith(prefix))) {
+        continue;
+      }
+
+      const childStat = statSync(child);
+      if (childStat.isDirectory()) {
+        walk(child);
+      } else if (shouldReadFile(child, excluded)) {
+        files.push(child);
+      }
+    }
+  };
+  walk(targetPath);
+  return files.sort((a, b) => repoRelative(a).localeCompare(repoRelative(b)));
+}
+
+function shouldReadFile(filePath, excluded) {
+  const rel = repoRelative(filePath);
+  if (excluded.some((prefix) => rel.startsWith(prefix))) {
+    return false;
+  }
+  if (rel === 'AGENTS.md' || rel === 'CONTEXT.md' || rel.endsWith('/README.md')) {
+    return true;
+  }
+  return new Set(['.md', '.txt']).has(extname(filePath).toLowerCase());
+}
+
+export function chunkText(text, options) {
+  const maxChars = options.maxChars ?? 1600;
+  const overlapChars = options.overlapChars ?? 220;
+  const normalized = text.replace(/\r\n?/gu, '\n').trim();
+  if (!normalized) {
+    return [];
+  }
+
+  const blocks = normalized.split(/\n(?=#{1,6}\s+)/u);
+  const chunks = [];
+  let current = '';
+
+  const pushCurrent = () => {
+    const trimmed = current.trim();
+    if (trimmed) {
+      chunks.push(trimmed);
+    }
+    current = '';
+  };
+
+  for (const block of blocks) {
+    if ((current.length + block.length + 2) <= maxChars) {
+      current = current ? `${current}\n\n${block}` : block;
+      continue;
+    }
+    pushCurrent();
+    if (block.length <= maxChars) {
+      current = block;
+      continue;
+    }
+    for (let start = 0; start < block.length; start += Math.max(1, maxChars - overlapChars)) {
+      chunks.push(block.slice(start, start + maxChars).trim());
+    }
+  }
+  pushCurrent();
+
+  return chunks.map((chunk, index) => ({ index, text: chunk }));
+}
+
+export function buildChunkId(filePath, chunkIndex) {
+  return `${filePath}#${chunkIndex}`;
+}
+
+export function extractTitle(text, fallback) {
+  const title = text.match(/^#\s+(.+)$/mu)?.[1]?.trim();
+  return title || fallback;
+}
+
+export async function createEmbedder(transformers, model) {
+  const extractor = await transformers.pipeline('feature-extraction', model);
+
+  return async function embed(text, type) {
+    const prefix = type === 'query' ? 'query: ' : 'passage: ';
+    const output = await extractor(`${prefix}${text}`, {
+      pooling: 'mean',
+      normalize: true,
+    });
+    return Array.from(output.data, Number);
+  };
+}
+
+export function parseLimitFiles(argv) {
+  const value = readArg(argv, '--limit-files');
+  if (!value) {
+    return Number.POSITIVE_INFINITY;
+  }
+  const parsed = Number(value);
+  if (!Number.isInteger(parsed) || parsed <= 0) {
+    throw new Error(`Invalid --limit-files value: ${value}`);
+  }
+  return parsed;
+}
+
+export function readArg(argv, name, fallback = undefined) {
+  const index = argv.indexOf(name);
+  if (index === -1) {
+    return fallback;
+  }
+  return argv[index + 1] ?? fallback;
+}
+
+export function hasFlag(argv, name) {
+  return argv.includes(name);
+}
--- a/scripts/rag/search-docs.mjs
+++ b/scripts/rag/search-docs.mjs
@@ -0,0 +1,195 @@
+import { join } from 'node:path';
+
+import {
+  createEmbedder,
+  hasFlag,
+  loadRagRuntime,
+  readArg,
+  readConfig,
+  repoRoot,
+} from './rag-utils.mjs';
+
+const config = readConfig();
+const query = readArg(process.argv, '--query') ?? process.argv.slice(2).join(' ');
+const limit = Number(readArg(process.argv, '--limit', '8'));
+const maxChars = Number(readArg(process.argv, '--max-chars', '12000'));
+const format = readArg(process.argv, '--format', 'context');
+const includeText = !hasFlag(process.argv, '--no-text');
+
+if (!query) {
+  throw new Error(
+    'Usage: node scripts/rag/search-docs.mjs --query "搜索内容" [--limit 8] [--format context|json|jsonl|text] [--max-chars 12000]',
+  );
+}
+
+if (!['context', 'json', 'jsonl', 'text'].includes(format)) {
+  throw new Error(`Unsupported --format value: ${format}`);
+}
+
+if (!Number.isFinite(limit) || limit <= 0 || !Number.isInteger(limit)) {
+  throw new Error(`Invalid --limit value: ${limit}`);
+}
+
+if (!Number.isFinite(maxChars) || maxChars <= 0 || !Number.isInteger(maxChars)) {
+  throw new Error(`Invalid --max-chars value: ${maxChars}`);
+}
+
+const { lancedb, transformers } = await loadRagRuntime(config);
+const embed = await createEmbedder(transformers, config.model);
+const queryVector = await embed(query, 'query');
+
+const db = await lancedb.connect(join(repoRoot, config.databaseDir));
+const table = await db.openTable(config.tableName);
+const rawResults = await table
+  .vectorSearch(queryVector)
+  .select(['id', 'path', 'title', 'chunk_index', 'source_weight', 'text', '_distance'])
+  .limit(Math.max(limit * 3, limit))
+  .toArray();
+
+const results = rawResults
+  .map((row) => ({
+    ...row,
+    score: (1 / (1 + Number(row._distance ?? 0))) * Number(row.source_weight ?? 1),
+  }))
+  .sort((a, b) => b.score - a.score)
+  .slice(0, limit);
+
+const payload = buildAgentPayload(query, results, {
+  model: config.model,
+  tableName: config.tableName,
+  maxChars,
+  includeText,
+});
+
+if (format === 'json') {
+  console.log(JSON.stringify(payload, null, 2));
+} else if (format === 'jsonl') {
+  for (const result of payload.results) {
+    console.log(JSON.stringify(result));
+  }
+} else if (format === 'text') {
+  printTextResults(payload.results);
+} else {
+  console.log(formatContextPack(payload));
+}
+
+function buildAgentPayload(searchQuery, rows, options) {
+  const outputRows = [];
+  let remainingChars = options.maxChars;
+
+  for (const [index, row] of rows.entries()) {
+    const source = `${row.path}#${row.chunk_index}`;
+    const text = String(row.text ?? '').trim();
+    const result = {
+      rank: index + 1,
+      id: row.id,
+      source,
+      path: row.path,
+      title: row.title,
+      chunkIndex: Number(row.chunk_index),
+      score: Number(row.score),
+      distance: Number(row._distance ?? 0),
+      sourceWeight: Number(row.source_weight ?? 1),
+    };
+
+    if (options.includeText) {
+      const capped = capText(text, Math.max(0, remainingChars));
+      result.text = capped.text;
+      result.truncated = capped.truncated;
+      remainingChars -= result.text.length;
+    }
+
+    outputRows.push(result);
+  }
+
+  return {
+    kind: 'genarrative-rag-context',
+    query: searchQuery,
+    generatedAt: new Date().toISOString(),
+    model: options.model,
+    table: options.tableName,
+    maxChars: options.maxChars,
+    remainingChars,
+    resultCount: outputRows.length,
+    usage: [
+      'This context pack is primarily for Agent consumption.',
+      'Use sources as candidate context and inspect authoritative files before editing when exact line-level changes matter.',
+      'Prefer docs/project-memory and current docs over stale historical notes when sources conflict.',
+    ],
+    results: outputRows,
+  };
+}
+
+function capText(text, budget) {
+  if (budget <= 0) {
+    return { text: '', truncated: text.length > 0 };
+  }
+  if (text.length <= budget) {
+    return { text, truncated: false };
+  }
+  return { text: `${text.slice(0, Math.max(0, budget - 18)).trimEnd()}\n[TRUNCATED]`, truncated: true };
+}
+
+function formatContextPack(payload) {
+  const lines = [
+    '# Genarrative RAG Context',
+    '',
+    `query: ${payload.query}`,
+    `model: ${payload.model}`,
+    `results: ${payload.resultCount}`,
+    `maxChars: ${payload.maxChars}`,
+    '',
+    '## Agent Usage',
+    '',
+    '- This context pack is primarily for Agent consumption.',
+    '- Treat sources as candidate context; inspect authoritative files before exact edits.',
+    '- If sources conflict, prefer current code and current docs over stale historical notes.',
+    '',
+    '## Sources',
+    '',
+  ];
+
+  for (const result of payload.results) {
+    lines.push(
+      `${result.rank}. ${result.source} score=${result.score.toFixed(4)} distance=${result.distance.toFixed(4)} title=${result.title}`,
+    );
+  }
+
+  lines.push('', '## Context', '');
+
+  for (const result of payload.results) {
+    const fence = buildMarkdownFence(result.text ?? '');
+    lines.push(
+      `### [${result.rank}] ${result.title}`,
+      '',
+      `source: ${result.source}`,
+      `score: ${result.score.toFixed(4)}`,
+      '',
+      `${fence}text`,
+      result.text ?? '',
+      fence,
+      '',
+    );
+  }
+
+  return lines.join('\n');
+}
+
+function buildMarkdownFence(text) {
+  const longest = Math.max(3, ...Array.from(text.matchAll(/`+/gu), (match) => match[0].length));
+  return '`'.repeat(longest + 1);
+}
+
+function printTextResults(rows) {
+  for (const result of rows) {
+    const preview = String(result.text ?? '').replace(/\s+/gu, ' ').slice(0, 260);
+    console.log(
+      [
+        `${result.rank}. ${result.source}`,
+        `   title: ${result.title}`,
+        `   score: ${result.score.toFixed(4)} distance: ${result.distance.toFixed(4)}`,
+        `   ${preview}`,
+      ].join('\n'),
+    );
+  }
+}