255 lines
9.8 KiB
JavaScript
255 lines
9.8 KiB
JavaScript
'use strict';
|
||
/**
|
||
* 搜索策略 + trigram 分词器。
|
||
*
|
||
* 这里守的是一个很容易被"优化"掉的正确性问题：
|
||
* FTS5 默认的 unicode61 分词器把一整串连续中文当成一个 token，
|
||
* 而 MATCH 是整 token 匹配，所以搜"公园"在"今天去公园散步"里搜不到。
|
||
* 改用 trigram 后子串能匹配了，但 ≤2 字符的词 trigram 一条也匹配不到，
|
||
* 必须退回 LIKE —— 而两字中文词（公园/会议/复制）恰恰是最常见的搜索。
|
||
*/
|
||
const assert = require('node:assert');
|
||
const fs = require('node:fs');
|
||
const os = require('node:os');
|
||
const path = require('node:path');
|
||
const Database = require('better-sqlite3');
|
||
const { planSearch, likePatterns, escapeLike, TRIGRAM_MIN } = require('../lib/search-plan.js');
|
||
const { migrate } = require('../lib/migrate.js');
|
||
|
||
let passed = 0;
|
||
function check(name, fn) {
|
||
try { fn(); console.log(' ok -', name); passed++; }
|
||
catch (e) { console.error(' FAIL -', name, e.message); process.exitCode = 1; }
|
||
}
|
||
|
||
// ---------- planSearch：纯逻辑 ----------
|
||
|
||
check('空查询 -> 全部列出', () => {
|
||
for (const q of ['', ' ', null, undefined]) {
|
||
assert.strictEqual(planSearch(q).mode, 'all', JSON.stringify(q));
|
||
}
|
||
});
|
||
|
||
check('短查询（<3 字符）退回 LIKE', () => {
|
||
for (const q of ['公园', '会议', '复制', 'ab', 'a']) {
|
||
const p = planSearch(q);
|
||
assert.strictEqual(p.mode, 'like', `"${q}" 应走 LIKE`);
|
||
assert.deepStrictEqual(p.patterns, [`%${q}%`]);
|
||
}
|
||
});
|
||
|
||
check('长查询走 FTS', () => {
|
||
const p = planSearch('天气很好');
|
||
assert.strictEqual(p.mode, 'fts');
|
||
assert.strictEqual(p.match, '"天气很好"');
|
||
});
|
||
|
||
check('多 token 全部够长时组成 AND 查询', () => {
|
||
assert.strictEqual(planSearch('hello world').match, '"hello" "world"');
|
||
});
|
||
|
||
check('多 token 里只要有一个太短就整体退回 LIKE', () => {
|
||
// MATCH 里多个短语是 AND 关系，2 字符那个匹配不到会让整个查询返回空
|
||
const p = planSearch('天气很好 公园');
|
||
assert.strictEqual(p.mode, 'like');
|
||
// 每个 token 独立子串、AND 组合（不是把整串连空格当子串）
|
||
assert.deepStrictEqual(p.patterns, ['%天气很好%', '%公园%']);
|
||
});
|
||
|
||
// ---------- likePatterns / escapeLike：LIKE 元字符 ----------
|
||
|
||
check('likePatterns：空查询返回空数组', () => {
|
||
for (const q of ['', ' ', null, undefined]) {
|
||
assert.deepStrictEqual(likePatterns(q), [], JSON.stringify(q));
|
||
}
|
||
});
|
||
|
||
check('likePatterns：多 token 拆成独立模式', () => {
|
||
assert.deepStrictEqual(likePatterns(' hello world '), ['%hello%', '%world%']);
|
||
});
|
||
|
||
check('escapeLike：% _ \\ 三个元字符都被转义', () => {
|
||
assert.strictEqual(escapeLike('100%'), '100\\%');
|
||
assert.strictEqual(escapeLike('a_b'), 'a\\_b');
|
||
assert.strictEqual(escapeLike('C:\\path'), 'C:\\\\path');
|
||
assert.strictEqual(escapeLike('plain'), 'plain');
|
||
});
|
||
|
||
check('LIKE 元字符进了查询不再被当通配符', () => {
|
||
// 不转义时搜 "100%" 会变成 "以 100 开头"，匹配一切 100xx
|
||
assert.deepStrictEqual(likePatterns('100%'), ['%100\\%%']);
|
||
assert.deepStrictEqual(likePatterns('a_b c%d'), ['%a\\_b%', '%c\\%d%']);
|
||
});
|
||
|
||
check('FTS5 保留字符被安全转义（不会变成语法错误）', () => {
|
||
assert.strictEqual(planSearch('a"bc').match, '"a""bc"');
|
||
for (const q of ['foo-bar', 'a(b)c', 'x*yz', '^abc', 'a:b:c', 'NEAR(a b)']) {
|
||
const p = planSearch(q);
|
||
if (p.mode === 'fts') assert.ok(p.match.startsWith('"'), q);
|
||
}
|
||
});
|
||
|
||
check('按码点算长度，代理对不会被误判为够长', () => {
|
||
// 两个 emoji = 2 个码点，但 .length 是 4；不做码点处理就会错走 FTS
|
||
assert.strictEqual(planSearch('😀😀').mode, 'like');
|
||
assert.strictEqual(TRIGRAM_MIN, 3);
|
||
});
|
||
|
||
// ---------- 端到端：真的建库、真的搜 ----------
|
||
|
||
function makeDb() {
|
||
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'clip-sp-'));
|
||
const db = new Database(path.join(tmp, 'h.db'));
|
||
db.exec(`CREATE TABLE clips(
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT, type TEXT, text TEXT, image BLOB,
|
||
preview TEXT, top_at REAL, created_at REAL NOT NULL DEFAULT 0)`);
|
||
return db;
|
||
}
|
||
|
||
let fts5 = true;
|
||
{
|
||
const db = makeDb();
|
||
try { db.exec(`CREATE VIRTUAL TABLE t USING fts5(x, tokenize='trigram')`); }
|
||
catch (_) { fts5 = false; }
|
||
db.close();
|
||
}
|
||
|
||
if (!fts5) {
|
||
console.log(' SKIP - 这个 SQLite 构建不支持 trigram FTS5');
|
||
} else {
|
||
const CORPUS = [
|
||
'今天天气很好，我们去公园散步',
|
||
'会议记录：项目进度延后两周',
|
||
'hello world foo bar',
|
||
'请把这段文字复制到剪贴板',
|
||
'100% off_sale promo',
|
||
'off sale prices today',
|
||
];
|
||
|
||
function seeded() {
|
||
const db = makeDb();
|
||
const ins = db.prepare('INSERT INTO clips(type,text,created_at) VALUES(?,?,?)');
|
||
CORPUS.forEach((t, i) => ins.run('text', t, i));
|
||
assert.strictEqual(migrate(db, 1, 3), 3, '应迁移到 v3');
|
||
return db;
|
||
}
|
||
|
||
// 模拟 main.js 的 clips:list 分支（searchLikeRows：多 token AND + ESCAPE）
|
||
function search(db, query) {
|
||
const plan = planSearch(query);
|
||
if (plan.mode === 'all') return db.prepare('SELECT id FROM clips').all().length;
|
||
if (plan.mode === 'like') {
|
||
const where = plan.patterns.map(() => `text LIKE ? ESCAPE '\\'`).join(' AND ');
|
||
return db.prepare(`SELECT id FROM clips WHERE ${where}`).all(...plan.patterns).length;
|
||
}
|
||
return db.prepare(
|
||
'SELECT c.id FROM clips_fts f JOIN clips c ON c.id=f.rowid WHERE clips_fts MATCH ?'
|
||
).all(plan.match).length;
|
||
}
|
||
|
||
check('v3 迁移后 clips_fts 用的是 trigram 分词器', () => {
|
||
const db = seeded();
|
||
const sql = db.prepare(
|
||
"SELECT sql FROM sqlite_master WHERE name='clips_fts'"
|
||
).get().sql;
|
||
assert.match(sql, /tokenize\s*=\s*'trigram'/, `实际: ${sql}`);
|
||
db.close();
|
||
});
|
||
|
||
check('中文子串搜索能搜到（这正是 unicode61 做不到的）', () => {
|
||
const db = seeded();
|
||
assert.strictEqual(search(db, '天气很好'), 1, '4 字中文子串');
|
||
assert.strictEqual(search(db, '项目进度'), 1);
|
||
assert.strictEqual(search(db, '复制到'), 1, '3 字中文子串');
|
||
db.close();
|
||
});
|
||
|
||
check('两字中文词经 LIKE 兜底同样能搜到', () => {
|
||
const db = seeded();
|
||
for (const [q, n] of [['公园', 1], ['会议', 1], ['复制', 1], ['天气', 1]]) {
|
||
assert.strictEqual(search(db, q), n, `"${q}"`);
|
||
}
|
||
db.close();
|
||
});
|
||
|
||
check('英文搜索照常工作', () => {
|
||
const db = seeded();
|
||
assert.strictEqual(search(db, 'hello'), 1);
|
||
assert.strictEqual(search(db, 'world'), 1);
|
||
db.close();
|
||
});
|
||
|
||
check('多 token 搜索是 AND：都在才命中', () => {
|
||
const db = seeded();
|
||
assert.strictEqual(search(db, 'hello world'), 1, '两词都在第 3 条');
|
||
assert.strictEqual(search(db, 'hello 公园'), 0, '跨条目不应命中');
|
||
db.close();
|
||
});
|
||
|
||
check('搜 LIKE 元字符不会变通配符（端到端）', () => {
|
||
const db = seeded();
|
||
db.prepare('INSERT INTO clips(type,text,created_at) VALUES(?,?,?)').run('text', '折扣 100% off', 99);
|
||
db.prepare('INSERT INTO clips(type,text,created_at) VALUES(?,?,?)').run('text', '打折活动 10086', 100);
|
||
// '折' 是短 token，强制走 LIKE 分支；'100%' 若不转义，% 是通配符，
|
||
// '打折活动 10086'（含子串 100）也会命中 → 不转义得 2，转义后应得 1。
|
||
assert.strictEqual(search(db, '折 100%'), 1, '应只命中字面量含 100% 的条目');
|
||
db.close();
|
||
});
|
||
|
||
check('搜不到的东西返回 0（不是全部）', () => {
|
||
const db = seeded();
|
||
assert.strictEqual(search(db, '不存在的内容'), 0);
|
||
db.close();
|
||
});
|
||
|
||
check('每种查询形态的结果都与 LIKE 基准一致', () => {
|
||
const db = seeded();
|
||
const like = (q) => db.prepare('SELECT id FROM clips WHERE text LIKE ?').all(`%${q}%`).length;
|
||
for (const q of ['天气很好', '项目进度', '复制到', 'hello', 'world', '剪贴板', '散步', '公园']) {
|
||
assert.strictEqual(search(db, q), like(q), `"${q}" 与 LIKE 基准不一致`);
|
||
}
|
||
db.close();
|
||
});
|
||
|
||
check('保留字符不会让搜索抛异常', () => {
|
||
const db = seeded();
|
||
for (const q of ['foo-bar', 'a"b"c', 'x*y', '(paren)', 'a:b', 'NEAR(x y)', '^^^']) {
|
||
assert.doesNotThrow(() => search(db, q), `"${q}" 抛异常了`);
|
||
}
|
||
db.close();
|
||
});
|
||
|
||
check('v2 库能就地升到 v3（不丢数据）', () => {
|
||
const db = makeDb();
|
||
const ins = db.prepare('INSERT INTO clips(type,text,created_at) VALUES(?,?,?)');
|
||
CORPUS.forEach((t, i) => ins.run('text', t, i));
|
||
assert.strictEqual(migrate(db, 1, 2), 2); // 先停在 v2（unicode61）
|
||
assert.strictEqual(search(db, '天气很好'), 0, 'v2 下中文子串确实搜不到');
|
||
assert.strictEqual(migrate(db, 1, 3), 3); // 再升 v3
|
||
assert.strictEqual(search(db, '天气很好'), 1, 'v3 修好了');
|
||
assert.strictEqual(db.prepare('SELECT COUNT(*) n FROM clips').get().n, CORPUS.length);
|
||
db.exec(`INSERT INTO clips_fts(clips_fts) VALUES('integrity-check')`);
|
||
db.close();
|
||
});
|
||
|
||
check('升到 v3 后触发器仍然同步新增/删除', () => {
|
||
const db = seeded();
|
||
db.prepare('INSERT INTO clips(type,text,created_at) VALUES(?,?,?)').run('text', '新增的一条测试内容', 99);
|
||
assert.strictEqual(search(db, '新增的一条'), 1, '新插入的应可搜到');
|
||
db.prepare("DELETE FROM clips WHERE text='新增的一条测试内容'").run();
|
||
assert.strictEqual(search(db, '新增的一条'), 0, '删除后应搜不到');
|
||
db.exec(`INSERT INTO clips_fts(clips_fts) VALUES('integrity-check')`);
|
||
db.close();
|
||
});
|
||
|
||
check('v3 迁移可重复执行（幂等）', () => {
|
||
const db = seeded();
|
||
for (let i = 0; i < 3; i++) migrate(db, 1, 3);
|
||
assert.strictEqual(search(db, '天气很好'), 1, '不应出现重复命中');
|
||
db.exec(`INSERT INTO clips_fts(clips_fts) VALUES('integrity-check')`);
|
||
db.close();
|
||
});
|
||
}
|
||
|
||
console.log(`\n${passed} checks passed`);
|