/** * S-FEEL 子集的词法分析 —— 纯函数(编码规范 §8.3;定义域 OS 03)。 * * 子集范围:比较、逻辑、算术、字符串与日期函数,**无外部调用,静态可检**。 */ export const TOKEN_KINDS = [ 'number', 'string', 'name', 'punct', 'eof', ] as const; export type TokenKind = (typeof TOKEN_KINDS)[number]; export type Token = { readonly kind: TokenKind; readonly text: string; readonly value?: number | string; readonly pos: number; }; export class SfeelSyntaxError extends Error { constructor( message: string, readonly pos: number, ) { super(`S-FEEL 语法错误(第 ${pos} 字符):${message}`); this.name = 'SfeelSyntaxError'; } } /** 多字符符号先于单字符匹配 */ const PUNCT = ['<=', '>=', '!=', '==', '..', '<', '>', '=', '+', '-', '*', '/', '(', ')', '[', ']', ',', '.'] as const; const NAME_START = /[A-Za-z_一-龥]/; const NAME_PART = /[A-Za-z0-9_一-龥]/; export function tokenize(source: string): Token[] { const tokens: Token[] = []; let i = 0; while (i < source.length) { const ch = source[i]!; if (/\s/.test(ch)) { i += 1; continue; } // 字符串:单引号或双引号,支持 \" \\ \n \t if (ch === '"' || ch === "'") { const quote = ch; const start = i; i += 1; let value = ''; while (i < source.length && source[i] !== quote) { if (source[i] === '\\') { const next = source[i + 1]; value += next === 'n' ? '\n' : next === 't' ? '\t' : next === '\\' ? '\\' : next === quote ? quote : (next ?? ''); i += 2; continue; } value += source[i]; i += 1; } if (i >= source.length) throw new SfeelSyntaxError('字符串未闭合', start); i += 1; tokens.push({ kind: 'string', text: source.slice(start, i), value, pos: start }); continue; } // 数字:整数、小数;不支持科学计数(子集封顶) if (/[0-9]/.test(ch)) { const start = i; while (i < source.length && /[0-9]/.test(source[i]!)) i += 1; // `..` 是区间符号,不是小数点 if (source[i] === '.' && source[i + 1] !== '.') { i += 1; while (i < source.length && /[0-9]/.test(source[i]!)) i += 1; } const text = source.slice(start, i); tokens.push({ kind: 'number', text, value: Number(text), pos: start }); continue; } // 名字:标识符与关键字;允许空格分隔的多词函数名(如 `starts with`)在 parser 里合并 if (NAME_START.test(ch)) { const start = i; while (i < source.length && NAME_PART.test(source[i]!)) i += 1; tokens.push({ kind: 'name', text: source.slice(start, i), pos: start }); continue; } const punct = PUNCT.find((p) => source.startsWith(p, i)); if (punct !== undefined) { tokens.push({ kind: 'punct', text: punct, pos: i }); i += punct.length; continue; } throw new SfeelSyntaxError(`无法识别的字符 ${JSON.stringify(ch)}`, i); } tokens.push({ kind: 'eof', text: '', pos: source.length }); return tokens; }