update
This commit is contained in:
275
tests/unit/main/cjk-search.test.ts
Normal file
275
tests/unit/main/cjk-search.test.ts
Normal file
@@ -0,0 +1,275 @@
|
||||
// tests/unit/main/cjk-search.test.ts
|
||||
//
|
||||
// 守 r196「中文全文搜索失效」契约。
|
||||
//
|
||||
// 现象:FTS5 的 unicode61 分词器把**连续的中文串**当成一个 token
|
||||
// (unicode61 不区分文字系统 —— 汉字是 Unicode 字母)。
|
||||
// 实测(内存库 + unicode61 remove_diacritics 2):
|
||||
// 搜「学习」 → 0 行
|
||||
// 搜「机制」 → 0 行
|
||||
// 搜「注意力」→ 只有恰好在整串开头的那行能命中
|
||||
// 也就是中文笔记的全文搜索基本不可用(英文搜索正常,掩盖了问题)。
|
||||
//
|
||||
// 修法(索引侧 + 查询侧对称):
|
||||
// 索引侧:写 FTS 时在汉字之间插 ZWSP(U+200B)⇒ 每个汉字独立成 token;
|
||||
// 查询侧:中文查询改用 FTS5 短语("学 习")⇒ 要求 token 相邻;
|
||||
// 存量:迁移 0009 用 cjk_seg() 重分词。
|
||||
//
|
||||
// 本测锁四件事:
|
||||
// 1. 分词纯函数行为(含幂等 —— 迁移重跑不能累积 ZWSP);
|
||||
// 2. 查询串构造(拉丁行为**不变**,中文改短语);
|
||||
// 3. 端到端:真 SQLite + 真 FTS5 —— 中文子串能搜到、片段干净、
|
||||
// 拉丁前缀匹配没被改坏;
|
||||
// 4. 迁移 0009:存量(未分词)行在迁移后**能**被中文搜到。
|
||||
import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import type { NoteResource, ResourceId } from '../../../src/shared/types';
|
||||
|
||||
type Db = import('better-sqlite3').Database;
|
||||
|
||||
// 主进程 db 句柄由 sqlite.ts 的 getDB 提供 —— 测试里换成内存库。
|
||||
const { dbRef } = vi.hoisted(() => ({ dbRef: { current: null as Db | null } }));
|
||||
vi.mock('../../../src/main/db/sqlite', () => ({
|
||||
getDB: () => {
|
||||
if (!dbRef.current) throw new Error('test db not initialized');
|
||||
return dbRef.current;
|
||||
},
|
||||
}));
|
||||
|
||||
const {
|
||||
ZWSP,
|
||||
segmentCjk,
|
||||
stripZwsp,
|
||||
hasCjk,
|
||||
isCjkChar,
|
||||
segmentQueryToken,
|
||||
} = await import('../../../src/main/db/cjk-segment');
|
||||
const { sanitizeFtsQuery, upsertResource, searchResources } = await import(
|
||||
'../../../src/main/db/resources-repo'
|
||||
);
|
||||
const { applyMigrations } = await import('../../../src/main/db/migrator');
|
||||
|
||||
const MIGRATIONS_DIR = fileURLToPath(
|
||||
new URL('../../../src/main/db/migrations', import.meta.url),
|
||||
);
|
||||
|
||||
async function openDb(): Promise<Db> {
|
||||
const mod = (await import('better-sqlite3')) as unknown as { default: new (p: string) => Db };
|
||||
return new mod.default(':memory:');
|
||||
}
|
||||
|
||||
function note(id: string, title: string): NoteResource {
|
||||
return {
|
||||
kind: 'note',
|
||||
id: id as ResourceId,
|
||||
title,
|
||||
relPath: `${id}.md`,
|
||||
tags: [],
|
||||
size: 1,
|
||||
mtime: 1,
|
||||
indexedAt: 1,
|
||||
};
|
||||
}
|
||||
|
||||
/** 走真实写路径(upsertResource)插入一条中文笔记。 */
|
||||
function insertNote(id: string, title: string, body: string): void {
|
||||
upsertResource(note(id, title), body, `/abs/${id}.md`);
|
||||
}
|
||||
|
||||
describe('CJK 分词纯函数(r196)', () => {
|
||||
it('汉字之间插入 ZWSP', () => {
|
||||
expect(segmentCjk('深度学习')).toBe(
|
||||
['深', '度', '学', '习'].join(ZWSP),
|
||||
);
|
||||
});
|
||||
|
||||
it('CJK↔拉丁边界也断开(否则整串是一个 token,两边都搜不到)', () => {
|
||||
expect(segmentCjk('Attention机制')).toBe(['Attention', '机', '制'].join(ZWSP));
|
||||
});
|
||||
|
||||
it('纯拉丁文本不被改写', () => {
|
||||
expect(segmentCjk('attention is all you need')).toBe('attention is all you need');
|
||||
});
|
||||
|
||||
it('幂等:重复分词结果不变(迁移 0009 重跑不能累积 ZWSP)', () => {
|
||||
const once = segmentCjk('深度学习 Attention 机制');
|
||||
expect(segmentCjk(once)).toBe(once);
|
||||
});
|
||||
|
||||
it('stripZwsp 能把分词文本还原成原文', () => {
|
||||
expect(stripZwsp(segmentCjk('深度学习'))).toBe('深度学习');
|
||||
});
|
||||
|
||||
it('空串安全', () => {
|
||||
expect(segmentCjk('')).toBe('');
|
||||
expect(stripZwsp('')).toBe('');
|
||||
});
|
||||
|
||||
it('CJK 判定:汉字/假名/谚文是,拉丁与数字不是', () => {
|
||||
expect(isCjkChar('学')).toBe(true);
|
||||
expect(isCjkChar('あ')).toBe(true);
|
||||
expect(isCjkChar('한')).toBe(true);
|
||||
expect(isCjkChar('a')).toBe(false);
|
||||
expect(isCjkChar('1')).toBe(false);
|
||||
expect(hasCjk('attention 机制')).toBe(true);
|
||||
expect(hasCjk('attention only')).toBe(false);
|
||||
});
|
||||
|
||||
it('查询侧切词:汉字逐字,非 CJK 连续段整体保留', () => {
|
||||
expect(segmentQueryToken('学习')).toEqual(['学', '习']);
|
||||
expect(segmentQueryToken('机器学习')).toEqual(['机', '器', '学', '习']);
|
||||
expect(segmentQueryToken('Attention机制')).toEqual(['Attention', '机', '制']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('查询串构造(r196)', () => {
|
||||
it('中文 → FTS5 短语(每个汉字一个 token)', () => {
|
||||
expect(sanitizeFtsQuery('学习')).toBe('"学 习"');
|
||||
expect(sanitizeFtsQuery('注意力')).toBe('"注 意 力"');
|
||||
});
|
||||
|
||||
it('拉丁 → 前缀通配(旧行为不变)', () => {
|
||||
expect(sanitizeFtsQuery('transformer')).toBe('transformer*');
|
||||
});
|
||||
|
||||
it('含特殊字符的拉丁 token → 引号短语(旧行为不变)', () => {
|
||||
expect(sanitizeFtsQuery('v1.2')).toBe('"v1.2"');
|
||||
});
|
||||
|
||||
it('FTS5 保留字兜底不变', () => {
|
||||
expect(sanitizeFtsQuery('and')).toBe('a*');
|
||||
});
|
||||
|
||||
it('中英混排查询:中文短语 + 拉丁前缀', () => {
|
||||
expect(sanitizeFtsQuery('学习 transformer')).toBe('"学 习" transformer*');
|
||||
});
|
||||
|
||||
it('空查询返回空串', () => {
|
||||
expect(sanitizeFtsQuery('')).toBe('');
|
||||
expect(sanitizeFtsQuery(' ')).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('端到端:真 SQLite + 真 FTS5(r196)', () => {
|
||||
let db: Db;
|
||||
beforeEach(async () => {
|
||||
db = await openDb();
|
||||
applyMigrations(db, MIGRATIONS_DIR);
|
||||
dbRef.current = db;
|
||||
});
|
||||
afterEach(() => {
|
||||
dbRef.current = null;
|
||||
db.close();
|
||||
});
|
||||
|
||||
it('中文词内子串能搜到(旧实现 0 结果)', () => {
|
||||
insertNote('note-1', '深度学习综述', '本文介绍注意力机制的基本原理。');
|
||||
expect(searchResources('注意力').map((h) => h.resource.id)).toEqual(['note-1']);
|
||||
expect(searchResources('机制').map((h) => h.resource.id)).toEqual(['note-1']);
|
||||
expect(searchResources('学习').map((h) => h.resource.id)).toEqual(['note-1']);
|
||||
});
|
||||
|
||||
it('片段带高亮标记,且不含 ZWSP 残留', () => {
|
||||
insertNote('note-1', '深度学习综述', '本文介绍注意力机制的基本原理。');
|
||||
const hit = searchResources('注意力')[0]!;
|
||||
expect(hit.snippet).toContain('<mark>注意力</mark>');
|
||||
expect(hit.snippet).not.toContain(ZWSP);
|
||||
expect(hit.snippet).toBe('本文介绍<mark>注意力</mark>机制的基本原理。');
|
||||
});
|
||||
|
||||
it('中文标题也能搜到', () => {
|
||||
insertNote('note-1', '深度学习综述', '正文');
|
||||
expect(searchResources('综述').map((h) => h.resource.id)).toEqual(['note-1']);
|
||||
});
|
||||
|
||||
it('中英混排:中文侧与英文侧各自可搜', () => {
|
||||
insertNote('note-1', '注意力机制', 'Attention is all you need. 注意力机制的核心。');
|
||||
expect(searchResources('注意力').map((h) => h.resource.id)).toEqual(['note-1']);
|
||||
expect(searchResources('attention').map((h) => h.resource.id)).toEqual(['note-1']);
|
||||
});
|
||||
|
||||
it('拉丁前缀匹配未被改坏', () => {
|
||||
insertNote('note-1', 'Transformer', 'transformer architectures are popular.');
|
||||
expect(searchResources('transform').map((h) => h.resource.id)).toEqual(['note-1']);
|
||||
expect(searchResources('architecture').map((h) => h.resource.id)).toEqual(['note-1']);
|
||||
});
|
||||
|
||||
it('未分词的行搜不到 —— 证明分词是命中的关键(旧行为的对照)', () => {
|
||||
// 直接塞未分词的 FTS 行(模拟 r196 之前写入的索引)
|
||||
db.prepare(
|
||||
`INSERT INTO resources (id, kind, title, rel_path, abs_path, size, mtime, indexed_at, extract_status, tags)
|
||||
VALUES ('note-raw', 'note', '注意力机制', 'raw.md', '/abs/raw.md', 1, 1, 1, 'ok', '[]')`,
|
||||
).run();
|
||||
db.prepare(
|
||||
`INSERT INTO resources_fts (rowid, id, kind, title, body)
|
||||
SELECT rowid, id, kind, title, '本文介绍注意力机制。' FROM resources WHERE id = 'note-raw'`,
|
||||
).run();
|
||||
expect(searchResources('注意力')).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('迁移 0009:存量行重分词(r196)', () => {
|
||||
let db: Db;
|
||||
beforeEach(async () => {
|
||||
db = await openDb();
|
||||
// 先停在 v8(0009 之前)
|
||||
applyMigrations(db, MIGRATIONS_DIR, 8);
|
||||
dbRef.current = db;
|
||||
});
|
||||
afterEach(() => {
|
||||
dbRef.current = null;
|
||||
db.close();
|
||||
});
|
||||
|
||||
it('迁移前:未分词的存量行搜不到中文子串', () => {
|
||||
db.prepare(
|
||||
`INSERT INTO resources (id, kind, title, rel_path, abs_path, size, mtime, indexed_at, extract_status, tags)
|
||||
VALUES ('note-old', 'note', '注意力机制', 'old.md', '/abs/old.md', 1, 1, 1, 'ok', '[]')`,
|
||||
).run();
|
||||
db.prepare(
|
||||
`INSERT INTO resources_fts (rowid, id, kind, title, body)
|
||||
SELECT rowid, id, kind, title, '本文介绍注意力机制。' FROM resources WHERE id = 'note-old'`,
|
||||
).run();
|
||||
expect(searchResources('注意力')).toEqual([]);
|
||||
});
|
||||
|
||||
it('迁移后:存量行能被中文搜到,且版本号推进到 9', () => {
|
||||
db.prepare(
|
||||
`INSERT INTO resources (id, kind, title, rel_path, abs_path, size, mtime, indexed_at, extract_status, tags)
|
||||
VALUES ('note-old', 'note', '注意力机制', 'old.md', '/abs/old.md', 1, 1, 1, 'ok', '[]')`,
|
||||
).run();
|
||||
db.prepare(
|
||||
`INSERT INTO resources_fts (rowid, id, kind, title, body)
|
||||
SELECT rowid, id, kind, title, '本文介绍注意力机制。' FROM resources WHERE id = 'note-old'`,
|
||||
).run();
|
||||
|
||||
applyMigrations(db, MIGRATIONS_DIR);
|
||||
|
||||
expect(db.pragma('user_version', { simple: true })).toBe(9);
|
||||
const hits = searchResources('注意力');
|
||||
expect(hits.map((h) => h.resource.id)).toEqual(['note-old']);
|
||||
expect(hits[0]!.snippet).not.toContain(ZWSP);
|
||||
});
|
||||
|
||||
it('迁移幂等:再跑一次不累积 ZWSP', () => {
|
||||
db.prepare(
|
||||
`INSERT INTO resources (id, kind, title, rel_path, abs_path, size, mtime, indexed_at, extract_status, tags)
|
||||
VALUES ('note-old', 'note', '注意力机制', 'old.md', '/abs/old.md', 1, 1, 1, 'ok', '[]')`,
|
||||
).run();
|
||||
db.prepare(
|
||||
`INSERT INTO resources_fts (rowid, id, kind, title, body)
|
||||
SELECT rowid, id, kind, title, '本文介绍注意力机制。' FROM resources WHERE id = 'note-old'`,
|
||||
).run();
|
||||
applyMigrations(db, MIGRATIONS_DIR);
|
||||
const after = db
|
||||
.prepare(`SELECT body FROM resources_fts WHERE id = 'note-old'`)
|
||||
.get() as { body: string };
|
||||
// 强制重跑 0009:把 user_version 拨回去
|
||||
db.pragma('user_version = 8');
|
||||
applyMigrations(db, MIGRATIONS_DIR);
|
||||
const again = db
|
||||
.prepare(`SELECT body FROM resources_fts WHERE id = 'note-old'`)
|
||||
.get() as { body: string };
|
||||
expect(again.body).toBe(after.body);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user