CI/CD 四件与两份文档入库 —— debt.md 早已写「已接」而文件一直没跟踪

.gitea/workflows/ci.yml · tools/ci.mjs · tools/cd.mjs · docs/部署.md:推主干跑五道门禁 + 从空库起的集成测试 + 三份构建,全绿后部署到 runner 那台机(机密一律 secrets.*,机器相关一律 vars.*,文件里不写死)。debt.md 在 9848e2f 就点名了这几个文件,远端却从未有过 .gitea。 docs/keycloak-backend-guide.md:ADR-012 引为出处的后端接入手册(只有 clientId 与「有/没有」, 无密钥)。docs/数据域字段说明-E04-I01.md:设计中心数据域那一屏的字段说明。 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01712YeSo5fTfkCSPZHBiniq
2026-08-26 14:34:19 +08:00
parent 0f1e53d114
commit f52edd5383
6 changed files with 2743 additions and 0 deletions
@@ -0,0 +1,486 @@
+#!/usr/bin/env node
+/**
+ * CD 的八件小事(只在 Gitea Actions 的部署段用,本地开发不需要):
+ *
+ *   preflight          配置齐不齐、密钥换没换、外部命令在不在、目录写不写得动
+ *   backup             pg_dump 生产库(迁移没有 down,这是唯一的回滚手段)
+ *   db-ensure          生产库不在就建 + 灌 db/init
+ *   migrate-verify     已应用迁移条数 == journal 条数
+ *   activities-plan    判「首次播种 / 装活动包 / 什么都不做 / 当场红」
+ *   activities-verify  回查 sys.activity_package
+ *   health             轮询 /api/health,**验 body**
+ *   prune              releases 只留最近几版
+ *
+ * 三条设计律,每条都对着一个「信号全绿而事情没办成」的真实机制:
+ *   ① `install-activities` 的所有失败路径都是「打印一行然后 continue」,**永远 exit 0**
+ *      —— 所以装完必须回查库,不能信它的退出码(activities-verify)。
+ *   ② 活动包版本没升时 `savePackage` 是 onConflictDoNothing,新代码静默丢失而日志照样说成功
+ *      —— 所以版本没变但产物内容变了要**当场红**(activities-plan)。
+ *   ③ `/api/health` 在库挂掉时**照样返 200**,只是 body 里 `db:'down'`
+ *      —— 所以探活验 body 不验状态码(health)。
+ *
+ * 不新增依赖:pg 是根 devDependency(§12.5#3);共用的连库与灌 init 从 ci.mjs 取,不另抄一份。
+ */
+import { createHash } from 'node:crypto';
+import { spawn, spawnSync } from 'node:child_process';
+import {
+  accessSync,
+  appendFileSync,
+  constants,
+  createWriteStream,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  readdirSync,
+  realpathSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { createGzip } from 'node:zlib';
+import { pipeline } from 'node:stream/promises';
+import { applyInit, connect, die, withDatabase } from './ci.mjs';
+
+const root = dirname(dirname(fileURLToPath(import.meta.url)));
+
+/** 路径类配置(Gitea Variables)与机密(Gitea Secrets)—— 少一条就别往下走 */
+const VARS = ['DEPLOY_DIR', 'PM2_APP', 'WEB_ROOT'];
+const SECRETS = ['PROD_DATABASE_URL', 'JCAI_SECRET_KEY', 'JCAI_SEED_PASSWORD', 'JCAI_DEEPSEEK_API_KEY'];
+
+function need(name) {
+  const value = process.env[name];
+  if (value === undefined || value.trim() === '') die(`缺 ${name} —— 在 Gitea 仓库设置里补上再部署`);
+  return value;
+}
+
+function emit(line) {
+  process.stdout.write(`${line}\n`);
+}
+
+/** 写进 $GITHUB_ENV 供后续步骤读;本地跑没有这个文件就把该导的那行摆出来 */
+function exportEnv(key, value) {
+  const file = process.env['GITHUB_ENV'];
+  if (file === undefined) {
+    emit(`export ${key}='${value}'`);
+    return;
+  }
+  appendFileSync(file, `${key}=${value}\n`);
+}
+
+/**
+ * 命令在不在 PATH 上 —— 借 shell 的 `command -v`,别自己拼 PATH 去猜。
+ * 走 `sh -c` 而不是 `{ shell: true }`:后者在 node 22 上每次都吐一条
+ * 「传 args 给 shell 有安全风险」的弃用告警,而 CI 日志里那种噪音会让人不信别的话。
+ * 参数来自本文件里写死的名单,不接外部输入。
+ */
+function which(command) {
+  return spawnSync('sh', ['-c', `command -v ${command}`], { stdio: 'ignore' }).status === 0;
+}
+
+/* ── 生产库地址 ─────────────────────────────────────────────────────── */
+
+function prodUrl() {
+  return need('PROD_DATABASE_URL');
+}
+
+function adminUrlOf(url) {
+  return withDatabase(url, 'postgres');
+}
+
+function databaseNameOf(url) {
+  return new URL(url).pathname.replace(/^\//, '');
+}
+
+async function databaseExists(url) {
+  const admin = await connect(adminUrlOf(url));
+  try {
+    const found = await admin.query('select 1 from pg_database where datname = $1', [databaseNameOf(url)]);
+    return found.rowCount > 0;
+  } finally {
+    await admin.end();
+  }
+}
+
+/* ── preflight ─────────────────────────────────────────────────────── */
+
+async function preflight() {
+  const missing = [...VARS, ...SECRETS].filter((name) => {
+    const value = process.env[name];
+    return value === undefined || value.trim() === '';
+  });
+  if (missing.length > 0) {
+    die(
+      `以下配置还没设(一次说全,免得你来回补):\n` +
+        missing.map((name) => `  · ${name}${VARS.includes(name) ? '(Variables)' : '(Secrets)'}`).join('\n'),
+    );
+  }
+
+  const key = Buffer.from(need('JCAI_SECRET_KEY'), 'base64');
+  if (key.length !== 32) {
+    die(`JCAI_SECRET_KEY 必须是 32 字节的 base64(现在解出来 ${key.length} 字节)—— openssl rand -base64 32`);
+  }
+
+  const deployDir = need('DEPLOY_DIR');
+  const webRoot = need('WEB_ROOT');
+  mkdirSync(join(deployDir, 'releases'), { recursive: true });
+  mkdirSync(join(deployDir, 'backups'), { recursive: true });
+  mkdirSync(join(deployDir, 'data', 'attachments'), { recursive: true });
+  for (const [label, path] of [['DEPLOY_DIR', deployDir], ['WEB_ROOT', webRoot]]) {
+    if (!existsSync(path)) die(`${label} 不存在:${path} —— 先在主机上建好并让 runner 用户写得动`);
+    try {
+      accessSync(path, constants.W_OK);
+    } catch {
+      die(`${label} 写不动:${path} —— runner 用户没有写权限(nginx 的目录常常是 root 的)`);
+    }
+  }
+
+  const tools = ['pm2', 'rsync'].filter((command) => !which(command));
+  if (tools.length > 0) {
+    die(
+      `这几个命令不在 runner 用户的 PATH 上:${tools.join(' / ')}\n` +
+        '(pm2 常装在某个 node 版本管理器的 shim 里,而非交互 shell 读不到 ~/.zshrc)',
+    );
+  }
+  if (!which('pg_dump') && !(which('docker') && (process.env['PG_CONTAINER'] ?? '') !== '')) {
+    die(
+      '备份没有着落:PATH 上没有 pg_dump,也没给 PG_CONTAINER(库在容器里时用它 docker exec)。\n' +
+        '你选的是「自动跑迁移,先备份」—— 而迁移没有 down,备份是唯一的回滚手段,不能悄悄跳过。',
+    );
+  }
+
+  // 密钥指纹:换了这把钥匙 = 库里所有凭证密文全部解不开,而且不在启动时炸,是第一次真调模型才炸
+  const fingerprint = createHash('sha256').update(key).digest('hex').slice(0, 16);
+  const marker = join(deployDir, '.secret-fingerprint');
+  if (existsSync(marker)) {
+    const known = readFileSync(marker, 'utf8').trim();
+    if (known !== fingerprint) {
+      die(
+        `JCAI_SECRET_KEY 与上一次部署的不是同一把(${known} → ${fingerprint})。\n` +
+          'sys.llm_instance.api_key_cipher 与 sys.tool.connection_cipher 都是用它封的 ——\n' +
+          '换了就全部解不开,而且要等第一次真调模型才炸。\n' +
+          `确实要换,先把 ${marker} 删掉,并准备重配所有凭证。`,
+      );
+    }
+  } else {
+    writeFileSync(marker, `${fingerprint}\n`, 'utf8');
+    emit(`记下密钥指纹:${fingerprint}(以后每次部署都比对)`);
+  }
+
+  emit('部署前置检查通过 ✔');
+}
+
+/* ── backup ────────────────────────────────────────────────────────── */
+
+async function backup() {
+  const url = prodUrl();
+  if (!(await databaseExists(url))) {
+    emit(`生产库 ${databaseNameOf(url)} 还不存在 —— 这是首次部署,没有东西可备份,跳过`);
+    return;
+  }
+
+  const deployDir = need('DEPLOY_DIR');
+  const dir = join(deployDir, 'backups');
+  mkdirSync(dir, { recursive: true });
+  const parsed = new URL(url);
+  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const sha = (process.env['GITHUB_SHA'] ?? 'local').slice(0, 8);
+  const file = join(dir, `${stamp}-${sha}.sql.gz`);
+
+  const database = databaseNameOf(url);
+  const user = decodeURIComponent(parsed.username);
+  const env = { ...process.env, PGPASSWORD: decodeURIComponent(parsed.password) };
+
+  // 口令一律走环境,不进 argv(argv 在 ps 里对同机其他用户可见)
+  const [command, args] =
+    which('pg_dump')
+      ? ['pg_dump', ['-h', parsed.hostname, '-p', parsed.port === '' ? '5432' : parsed.port, '-U', user, '-d', database, '--no-owner', '--no-privileges']]
+      : ['docker', ['exec', '-e', 'PGPASSWORD', '-i', need('PG_CONTAINER'), 'pg_dump', '-U', user, '-d', database, '--no-owner', '--no-privileges']];
+
+  const child = spawn(command, args, { env, stdio: ['ignore', 'pipe', 'pipe'] });
+  let stderr = '';
+  child.stderr.on('data', (chunk) => {
+    stderr += String(chunk);
+  });
+  const finished = new Promise((resolve, reject) => {
+    child.on('error', reject);
+    child.on('close', (code) => (code === 0 ? resolve() : reject(new Error(`${command} 退出码 ${code}\n${stderr}`))));
+  });
+
+  try {
+    await Promise.all([pipeline(child.stdout, createGzip(), createWriteStream(file)), finished]);
+  } catch (error) {
+    die(`备份失败:${error instanceof Error ? error.message : String(error)}`);
+  }
+
+  emit(`已备份 → ${file}(${(statSync(file).size / 1024).toFixed(0)} KB)`);
+  keepNewest(dir, Number(process.env['KEEP_BACKUPS'] ?? 10), '备份');
+}
+
+/* ── db-ensure ─────────────────────────────────────────────────────── */
+
+async function dbEnsure() {
+  const url = prodUrl();
+  const name = databaseNameOf(url);
+  if (await databaseExists(url)) {
+    emit(`生产库 ${name} 已在 ✔`);
+    return;
+  }
+  const admin = await connect(adminUrlOf(url));
+  try {
+    await admin.query(`create database "${name}"`);
+  } catch (error) {
+    die(
+      `建库失败:${error instanceof Error ? error.message : String(error)}\n` +
+        'PROD_DATABASE_URL 里那个账号需要 CREATEDB 权限,或者你先手工把库建好。',
+    );
+  } finally {
+    await admin.end();
+  }
+  emit(`已建生产库 ${name}`);
+  // 空库不灌 db/init 迁不动:0026 往 ledger.__object_meta 建表,而迁移链只会替自己建 sys
+  await applyInit(url);
+}
+
+/* ── migrate-verify ────────────────────────────────────────────────── */
+
+async function migrateVerify() {
+  const journal = JSON.parse(readFileSync(join(root, 'db', 'migrations', 'meta', '_journal.json'), 'utf8'));
+  const expected = Array.isArray(journal.entries) ? journal.entries.length : 0;
+  const client = await connect(prodUrl());
+  let applied;
+  try {
+    const result = await client.query('select count(*)::int as n from sys.__drizzle_migrations');
+    applied = result.rows[0].n;
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    // 记账表不存在 = 迁移那一步压根没跑成(而它可能仍然打印了「完成」并 exit 0)
+    if (detail.includes('__drizzle_migrations')) {
+      die('库里连 sys.__drizzle_migrations 都没有 —— 迁移那一步根本没跑成。先看上一步的输出。');
+    }
+    die(`查迁移记账失败:${detail}`);
+  } finally {
+    await client.end();
+  }
+  if (applied !== expected) {
+    die(
+      `迁移没到位:库里 ${applied} 条,db/migrations 里 ${expected} 条。\n` +
+        'drizzle 只比「最后一条的时间戳」当水位线,不逐条比 hash ——\n' +
+        '新迁移在 _journal.json 里的 when 若 ≤ 水位线,会被永久静默跳过,而 migrate 照样打印「完成」。\n' +
+        '查:select max(created_at) from sys.__drizzle_migrations,与新增那条的 when 比一比。',
+    );
+  }
+  emit(`迁移到位 ✔ ${applied}/${expected} 条`);
+}
+
+/* ── 活动包 ────────────────────────────────────────────────────────── */
+
+/** 装哪几个包是 seed-activities.ts 说了算 —— 读它,不在这儿另存一份名单 */
+function realPackages() {
+  const source = readFileSync(join(root, 'apps', 'api', 'src', 'app', 'seed-activities.ts'), 'utf8');
+  const matched = /const REAL_PACKAGES = \[([^\]]*)\]/.exec(source);
+  if (matched === null) {
+    die('读不出 REAL_PACKAGES —— seed-activities.ts 的写法变了,来改 tools/cd.mjs(别让它拿一份过期名单继续跑)');
+  }
+  const names = [...matched[1].matchAll(/'([^']+)'/g)].map((one) => one[1]);
+  if (names.length === 0) die('REAL_PACKAGES 是空的 —— 那就没有活动包要装,确认这是不是你要的');
+  return names;
+}
+
+function distOf(pkg) {
+  const dir = join(root, 'activities', pkg, 'dist');
+  const manifestPath = join(dir, 'manifest.json');
+  const entryPath = join(dir, 'index.js');
+  if (!existsSync(manifestPath) || !existsSync(entryPath)) {
+    die(`活动包 ${pkg} 没有构建产物(${dir})—— 先跑 pnpm activities:build`);
+  }
+  const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
+  return {
+    activityId: manifest.identity.activityId,
+    version: manifest.identity.packageVersion,
+    artifact: readFileSync(entryPath, 'utf8'),
+  };
+}
+
+async function activitiesPlan() {
+  const client = await connect(prodUrl());
+  let staffCount;
+  const decisions = [];
+  try {
+    const staff = await client.query('select count(*)::int as n from sys.staff');
+    staffCount = staff.rows[0].n;
+
+    for (const pkg of realPackages()) {
+      const dist = distOf(pkg);
+      const row = await client.query(
+        'select artifact from sys.activity_package where activity_id = $1 and version = $2',
+        [dist.activityId, dist.version],
+      );
+      if (row.rowCount === 0) decisions.push({ pkg, dist, verdict: 'install' });
+      else if (row.rows[0].artifact === dist.artifact) decisions.push({ pkg, dist, verdict: 'skip' });
+      else decisions.push({ pkg, dist, verdict: 'conflict' });
+    }
+  } finally {
+    await client.end();
+  }
+
+  const conflicts = decisions.filter((one) => one.verdict === 'conflict');
+  if (conflicts.length > 0) {
+    die(
+      `这几个活动包**版本没升但内容变了**:\n` +
+        conflicts.map((one) => `  · ${one.pkg}@${one.dist.version}`).join('\n') +
+        '\n安装恒新增:同版本重装是 no-op(onConflictDoNothing),库里还是旧产物,\n' +
+        '而日志会照样打印「活动包已装」、退出码 0 —— 新代码静默不上线。\n' +
+        '改了内容就升 manifest 里的 packageVersion(CLAUDE.md 铁规矩 7)。',
+    );
+  }
+
+  const first = staffCount === 0;
+  const toInstall = decisions.filter((one) => one.verdict === 'install');
+
+  if (first) {
+    emit('库里一个员工都没有 → 判为**首次部署**,走 seed.js 播一次种(此后永不再跑 seed)');
+    exportEnv('FIRST_DEPLOY', '1');
+    exportEnv('ACTIVITIES_INSTALL', '0');
+    return;
+  }
+
+  exportEnv('FIRST_DEPLOY', '0');
+  exportEnv('ACTIVITIES_INSTALL', toInstall.length > 0 ? '1' : '0');
+  if (toInstall.length === 0) {
+    emit('活动包版本与库里一致、内容也一致 → 不装(顺带避开它会覆盖样例主数据、并发出 data 事件)');
+    return;
+  }
+  emit(`要装的活动包:${toInstall.map((one) => `${one.pkg}@${one.dist.version}`).join(' / ')}`);
+}
+
+async function activitiesVerify() {
+  const client = await connect(prodUrl());
+  const bad = [];
+  try {
+    for (const pkg of realPackages()) {
+      const dist = distOf(pkg);
+      const row = await client.query(
+        'select artifact from sys.activity_package where activity_id = $1 and version = $2',
+        [dist.activityId, dist.version],
+      );
+      if (row.rowCount === 0) bad.push(`${pkg}@${dist.version}:库里根本没有这一版`);
+      else if (row.rows[0].artifact !== dist.artifact) bad.push(`${pkg}@${dist.version}:库里那一版的产物与刚构建的不一致`);
+      else emit(`  ✔ ${pkg}@${dist.version}`);
+    }
+  } finally {
+    await client.end();
+  }
+  if (bad.length > 0) {
+    die(
+      `活动包没装成:\n${bad.map((one) => `  · ${one}`).join('\n')}\n` +
+        'install-activities 的所有失败路径都是「打印一行然后 continue」,**永远 exit 0** ——\n' +
+        '所以这道回查才是唯一说得出实情的地方。看它的日志里有没有「本次跳过」。',
+    );
+  }
+  emit('活动包已在库里 ✔');
+}
+
+/* ── health ────────────────────────────────────────────────────────── */
+
+async function health() {
+  const url = process.env['HEALTH_URL'] ?? `http://127.0.0.1:${process.env['PORT'] ?? '8787'}/api/health`;
+  const deadline = Date.now() + Number(process.env['HEALTH_TIMEOUT_MS'] ?? 60_000);
+  let last = '还没连上';
+  while (Date.now() < deadline) {
+    try {
+      const response = await fetch(url);
+      const body = await response.json();
+      // 库挂了它照样返 200,只是 body 里 db:'down' —— 所以验 body,不验状态码
+      if (body.ok === true && body.db === 'up') {
+        emit(`探活通过 ✔ ${url} → ok=true db=up version=${body.version}`);
+        return;
+      }
+      last = `HTTP ${response.status} ${JSON.stringify(body)}`;
+    } catch (error) {
+      last = error instanceof Error ? error.message : String(error);
+    }
+    await new Promise((resolve) => setTimeout(resolve, 2000));
+  }
+  die(
+    `探活没过:${url}\n最后一次看到的是:${last}\n` +
+      '两种常见死法:①进程压根没起来(入口守卫遇软链会静默 exit 0 —— pm2 必须用 releases 的真实路径起);\n' +
+      '②起来了但连不上库(body 里 db:"down",而 HTTP 仍是 200)。先看 pm2 logs。',
+  );
+}
+
+/* ── prune ─────────────────────────────────────────────────────────── */
+
+function keepNewest(dir, keep, label) {
+  const entries = readdirSync(dir)
+    .map((name) => ({ name, path: join(dir, name) }))
+    .map((one) => ({ ...one, at: statSync(one.path).mtimeMs }))
+    .sort((a, b) => b.at - a.at);
+  const drop = entries.slice(keep);
+  for (const one of drop) rmSync(one.path, { recursive: true, force: true });
+  emit(`${label}:留 ${Math.min(keep, entries.length)} 份,清掉 ${drop.length} 份`);
+}
+
+function prune() {
+  const deployDir = need('DEPLOY_DIR');
+  const dir = join(deployDir, 'releases');
+  if (!existsSync(dir)) return;
+  const keep = Number(process.env['KEEP_RELEASES'] ?? 5);
+  const currentLink = join(deployDir, 'current');
+  const current = existsSync(currentLink) ? realpathSync(currentLink) : undefined;
+
+  const entries = readdirSync(dir)
+    .map((name) => join(dir, name))
+    .map((path) => ({ path, at: statSync(path).mtimeMs }))
+    .sort((a, b) => b.at - a.at);
+
+  let kept = 0;
+  let dropped = 0;
+  for (const one of entries) {
+    // 在跑的那一版永远不删,哪怕它排到了第 6 位
+    if (kept < keep || one.path === current) {
+      kept += 1;
+      continue;
+    }
+    rmSync(one.path, { recursive: true, force: true });
+    dropped += 1;
+  }
+  emit(`发布目录:留 ${kept} 版,清掉 ${dropped} 版`);
+}
+
+/* ── 分发 ──────────────────────────────────────────────────────────── */
+
+const isMain = process.argv[1] !== undefined && pathToFileURL(process.argv[1]).href === import.meta.url;
+const [command] = process.argv.slice(2);
+
+if (isMain) {
+  switch (command) {
+    case 'preflight':
+      await preflight();
+      break;
+    case 'backup':
+      await backup();
+      break;
+    case 'db-ensure':
+      await dbEnsure();
+      break;
+    case 'migrate-verify':
+      await migrateVerify();
+      break;
+    case 'activities-plan':
+      await activitiesPlan();
+      break;
+    case 'activities-verify':
+      await activitiesVerify();
+      break;
+    case 'health':
+      await health();
+      break;
+    case 'prune':
+      prune();
+      break;
+    default:
+      die('用法:node tools/cd.mjs <preflight|backup|db-ensure|migrate-verify|activities-plan|activities-verify|health|prune>');
+  }
+}