forked from ChaiHao/BPOS-MVP
度量八指标全部接真:DMN 覆盖率与单位成本不再恒为 0
两处度量是空的,查下来都是「没人记」而不是「算错了」: ① **DMN 覆盖率恒 0**:判据只认标题里的「求值:」,把跑得最多的**纯工具步**整个漏掉。 DMN 决策表与机检正是落在纯工具/工具步上。改为按步类型分: 确定性求值(pure-tool / tool)÷ 判断总数(再加 LLM 步)——「临场搬进确定」的进度表。 顺带把**步类型显式写进留痕**:判据该由引擎给出,不靠解析标题后缀。 现读数 66%。 ② **单位成本恒 0**:任务消耗只记步数,token / 活跃时长 / 调人次数**从来没人累加** —— 于是八指标里的单位成本永远是 0,预算熔断与打扰上限也形同虚设。 · tick 计这一拍真正花掉的时间(挂起等人的不算活跃,那是世界等待); · flow 的 LLM 步与 loop 的每一回合都把 token 计入; · 每发一张卡计一次调人。 现读数:A2 单轮 12k token、A3 3.4k;单位成本 73 token 每放行单。 另修:执行卡的 ask 优先取 `card.ask`(此前只认 `reason`,新卡面格式下会退化成标题)。 第四轮端到端复跑:整条链闭环,消耗数据真实。 gates: tsc / eslint / depcruise(336 模块)/ api-report / 429 测试 全绿 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01KeDBx2PHLavbURXB8LJJ7m
This commit is contained in:
@@ -51,6 +51,8 @@ export type AgentRunnerDeps = {
|
||||
readonly gateway: ModelGateway;
|
||||
/** 活动声明的自定义模型名 */
|
||||
readonly modelOf: (taskId: TaskId) => Promise<string>;
|
||||
/** token 回报:计入任务预算 —— 不计,预算熔断就永远不会响 */
|
||||
readonly onUsage: (taskId: TaskId, tokens: number) => Promise<void>;
|
||||
};
|
||||
|
||||
export function createAgentRunner(deps: AgentRunnerDeps): {
|
||||
@@ -95,6 +97,7 @@ export function createAgentRunner(deps: AgentRunnerDeps): {
|
||||
taskId,
|
||||
label: 'loop-turn',
|
||||
});
|
||||
await deps.onUsage(taskId, result.usage.inputTokens + result.usage.outputTokens);
|
||||
return normalize(result.object, enabled);
|
||||
} catch (error) {
|
||||
/*
|
||||
|
||||
@@ -45,18 +45,29 @@ export function wireDeskElements(deps: DeskElementDeps): { minute: MinuteService
|
||||
clock: deps.clock,
|
||||
});
|
||||
|
||||
/** 归侧日志行 → 度量事实。判断类与断点由日志标题的确定形态识别,不另设埋点 */
|
||||
const toLogFact = (taskId: string, line: { at: string; side: LogFact['side']; title: string }): LogFact => ({
|
||||
taskId,
|
||||
at: line.at,
|
||||
side: line.side,
|
||||
title: line.title,
|
||||
isHumanToolCall: line.title.startsWith('发出:'),
|
||||
// DMN / 机检的确定性求值在日志里是「求值:」;LLM 的判断落在「调用:」
|
||||
isDeterministicJudgement: line.title.startsWith('求值:'),
|
||||
isJudgement: line.title.startsWith('求值:') || line.title.startsWith('调用:'),
|
||||
isBreakpoint: line.title.includes('断点') || line.title.includes('交集为空'),
|
||||
});
|
||||
/**
|
||||
* 归侧日志行 → 度量事实。零埋点:判据全部取自结构化日志本身。
|
||||
*
|
||||
* **判断次数**包含两类:确定性求值(DMN 决策表 / 机检 —— 落在纯工具与工具步)
|
||||
* 与临场判断(LLM 步、loop 的语义性前提评估)。
|
||||
* DMN 覆盖率 = 前者 ÷ 两者之和 ——「临场搬进确定」的进度表。
|
||||
* 早先只认标题里的「求值:」,把跑得最多的纯工具步整个漏掉了,覆盖率恒为 0。
|
||||
*/
|
||||
const toLogFact = (taskId: string, line: { at: string; side: LogFact['side']; title: string }): LogFact => {
|
||||
const deterministic =
|
||||
line.title.startsWith('求值:') || line.title.includes('(pure-tool)') || line.title.includes('(tool)');
|
||||
const situational = line.title.includes('(llm)');
|
||||
return {
|
||||
taskId,
|
||||
at: line.at,
|
||||
side: line.side,
|
||||
title: line.title,
|
||||
isHumanToolCall: line.title.startsWith('发出:'),
|
||||
isDeterministicJudgement: deterministic,
|
||||
isJudgement: deterministic || situational,
|
||||
isBreakpoint: line.title.includes('断点') || line.title.includes('交集为空'),
|
||||
};
|
||||
};
|
||||
|
||||
const report = createReportService({
|
||||
store: createReportStore(deps.db),
|
||||
|
||||
@@ -270,6 +270,7 @@ export function wireEngines(deps: EngineWiringDeps): EngineWiring {
|
||||
criteria: [{ label: '成果齐备', level: 'L5', pass: false }],
|
||||
fields: [],
|
||||
});
|
||||
await taskStore.addConsumption(input.taskId, { humanCalls: 1 });
|
||||
return { id: issued.order.id };
|
||||
}
|
||||
const fields = Array.isArray(input.card['fields'])
|
||||
@@ -278,9 +279,11 @@ export function wireEngines(deps: EngineWiringDeps): EngineWiring {
|
||||
const issued = await deps.humanTool.issueExecution(context, {
|
||||
title,
|
||||
basis: basis.length > 0 ? basis : [{ k: '来源', v: '步序声明' }],
|
||||
ask: String(input.card['reason'] ?? title),
|
||||
ask: String(input.card['ask'] ?? input.card['reason'] ?? title),
|
||||
fields,
|
||||
});
|
||||
// 打扰率的计量对象:每发一张卡就是调了一次人 —— 不计就等于没有上限
|
||||
await taskStore.addConsumption(input.taskId, { humanCalls: 1 });
|
||||
return { id: issued.order.id };
|
||||
},
|
||||
};
|
||||
@@ -348,6 +351,10 @@ export function wireEngines(deps: EngineWiringDeps): EngineWiring {
|
||||
stepNo: input.stepNo,
|
||||
label: 'flow-step',
|
||||
});
|
||||
// token 计入任务预算 —— 单位成本与预算熔断都靠它,不计就等于没有预算
|
||||
await taskStore.addConsumption(input.taskId, {
|
||||
tokens: result.usage.inputTokens + result.usage.outputTokens,
|
||||
});
|
||||
// 解析与校验归引擎(它才知道这一步声明了什么);这里只如实上交结果与原文
|
||||
return {
|
||||
ok: true,
|
||||
@@ -373,6 +380,8 @@ export function wireEngines(deps: EngineWiringDeps): EngineWiring {
|
||||
data: taskData,
|
||||
agent: createAgentRunner({
|
||||
gateway: deps.modelGateway,
|
||||
// loop 的每一回合都要问模型 —— 不计 token,预算熔断就永远不会响
|
||||
onUsage: (taskId, tokens) => taskStore.addConsumption(taskId, { tokens }).then(() => undefined),
|
||||
modelOf: async (taskId) => {
|
||||
const { activityId } = await taskContext.of(taskId);
|
||||
const manifest = await deps.registry.manifestOf(activityId);
|
||||
|
||||
Reference in New Issue
Block a user