From 68bc1ce88b4eb04c5a6f5b121b46e1d3dd5baab2 Mon Sep 17 00:00:00 2001 From: Thomas SharedInbox Date: Thu, 14 May 2026 09:58:30 +0200 Subject: [PATCH] feat(P1): FTS5 virtual table for email search (replaces LIKE scan) Creates an external-content FTS5 virtual table `email_fts` backed by the `emails` table. Three triggers (AI/AU/AD) keep the index in sync automatically. Schema bumped to v26; backfill runs on upgrade. `searchEmailsGlobal` now issues a ranked FTS5 JOIN instead of `LIKE '%word%'` scans, making search O(log n) via the inverted index. Words are expanded to prefix terms (`word*`) so partial matches still work. Co-Authored-By: Claude Sonnet 4.6 --- lib/data/db/database.dart | 47 ++++++++++++++++++- .../repositories/email_repository_impl.dart | 47 ++++++++++++------- test/unit/migration_test.dart | 29 +++++++++++- 3 files changed, 102 insertions(+), 21 deletions(-) diff --git a/lib/data/db/database.dart b/lib/data/db/database.dart index 5d992ca..a753091 100644 --- a/lib/data/db/database.dart +++ b/lib/data/db/database.dart @@ -269,10 +269,47 @@ class AppDatabase extends _$AppDatabase { AppDatabase([QueryExecutor? executor]) : super(executor ?? _openConnection()); @override - int get schemaVersion => 25; + int get schemaVersion => 26; + + Future _createEmailFts() async { + await customStatement(''' + CREATE VIRTUAL TABLE IF NOT EXISTS email_fts USING fts5( + subject, preview, from_json, + content='emails', + content_rowid='rowid' + ) + '''); + await customStatement(''' + CREATE TRIGGER IF NOT EXISTS email_fts_ai + AFTER INSERT ON emails BEGIN + INSERT INTO email_fts(rowid, subject, preview, from_json) + VALUES (new.rowid, new.subject, new.preview, new.from_json); + END + '''); + await customStatement(''' + CREATE TRIGGER IF NOT EXISTS email_fts_au + AFTER UPDATE OF subject, preview, from_json ON emails BEGIN + INSERT INTO email_fts(email_fts, rowid, subject, preview, from_json) + VALUES ('delete', old.rowid, old.subject, old.preview, old.from_json); + INSERT INTO email_fts(rowid, subject, preview, from_json) + VALUES (new.rowid, new.subject, new.preview, new.from_json); + END + '''); + await customStatement(''' + CREATE TRIGGER IF NOT EXISTS email_fts_ad + AFTER DELETE ON emails BEGIN + INSERT INTO email_fts(email_fts, rowid, subject, preview, from_json) + VALUES ('delete', old.rowid, old.subject, old.preview, old.from_json); + END + '''); + } @override MigrationStrategy get migration => MigrationStrategy( + onCreate: (m) async { + await m.createAll(); + await _createEmailFts(); + }, onUpgrade: (m, from, to) async { // NOTE: m.createTable(T) creates the LATEST version of table T. // If you later add a column C to T in version X, you must guard @@ -447,6 +484,14 @@ class AppDatabase extends _$AppDatabase { ), ); } + if (from < 26) { + await _createEmailFts(); + // Backfill FTS index from existing rows. + await customStatement(''' + INSERT INTO email_fts(rowid, subject, preview, from_json) + SELECT rowid, subject, preview, from_json FROM emails + '''); + } }, ); } diff --git a/lib/data/repositories/email_repository_impl.dart b/lib/data/repositories/email_repository_impl.dart index efeea35..ace4883 100644 --- a/lib/data/repositories/email_repository_impl.dart +++ b/lib/data/repositories/email_repository_impl.dart @@ -2470,28 +2470,39 @@ class EmailRepositoryImpl implements EmailRepository { String? accountId, String query, ) async { + final ftsQuery = _toFtsQuery(query); + if (ftsQuery.isEmpty) return []; + + final sql = accountId != null + ? 'SELECT e.* FROM email_fts f JOIN emails e ON e.rowid = f.rowid' + ' WHERE email_fts MATCH ? AND e.account_id = ? ORDER BY rank LIMIT 50' + : 'SELECT e.* FROM email_fts f JOIN emails e ON e.rowid = f.rowid' + ' WHERE email_fts MATCH ? ORDER BY rank LIMIT 50'; + final variables = accountId != null + ? [Variable(ftsQuery), Variable(accountId)] + : [Variable(ftsQuery)]; + + final queryRows = await _db + .customSelect(sql, variables: variables, readsFrom: {_db.emails}).get(); + final emailRows = await Future.wait( + queryRows.map((r) => _db.emails.mapFromRow(r)), + ); + return emailRows.map(_toModel).toList(); + } + + /// Converts a user query string into an FTS5 match expression. + /// Each whitespace-separated word becomes a prefix term (word*) so that + /// partial words still match. Special FTS5 characters are stripped. + static String _toFtsQuery(String query) { final words = query - .toLowerCase() + .trim() .split(RegExp(r'\s+')) .where((w) => w.isNotEmpty) + .map((w) => w.replaceAll(RegExp(r'[^\w]'), '')) + .where((w) => w.isNotEmpty) .toList(); - final rows = await (_db.select(_db.emails) - ..where((t) { - Expression condition = const Constant(true); - if (accountId != null) { - condition = t.accountId.equals(accountId); - } - for (final word in words) { - final pattern = '%$word%'; - condition = condition & - (t.subject.like(pattern) | t.preview.like(pattern)); - } - return condition; - }) - ..orderBy([(t) => OrderingTerm.desc(t.receivedAt)]) - ..limit(50)) - .get(); - return rows.map(_toModel).toList(); + if (words.isEmpty) return ''; + return words.map((w) => '$w*').join(' '); } @override diff --git a/test/unit/migration_test.dart b/test/unit/migration_test.dart index 242fa7d..1b1c706 100644 --- a/test/unit/migration_test.dart +++ b/test/unit/migration_test.dart @@ -14,7 +14,7 @@ void main() { group('Migration', () { test('schemaVersion matches expected value', () async { final db = AppDatabase(NativeDatabase.memory()); - expect(db.schemaVersion, 25); + expect(db.schemaVersion, 26); await db.close(); }); @@ -158,6 +158,19 @@ void main() { ]), ); + // v26: FTS5 virtual table and triggers exist. + final allTriggers = await db + .customSelect("SELECT name FROM sqlite_master WHERE type='trigger'") + .get(); + final triggerNames = + allTriggers.map((r) => r.read('name')).toSet(); + expect( + triggerNames, + containsAll(['email_fts_ai', 'email_fts_au', 'email_fts_ad']), + ); + // Verify FTS table was created and is queryable. + await db.customSelect('SELECT count(*) FROM email_fts').get(); + await db.close(); if (dbFile.existsSync()) dbFile.deleteSync(); }); @@ -276,11 +289,23 @@ void main() { expect(indexNames, contains('mailboxes_account_id')); expect(indexNames, contains('threads_latest_date')); + // v26: FTS5 virtual table and triggers. + final allTriggers = await db + .customSelect("SELECT name FROM sqlite_master WHERE type='trigger'") + .get(); + final triggerNames = + allTriggers.map((r) => r.read('name')).toSet(); + expect( + triggerNames, + containsAll(['email_fts_ai', 'email_fts_au', 'email_fts_ad']), + ); + await db.customSelect('SELECT count(*) FROM email_fts').get(); + await db.close(); if (dbFile.existsSync()) dbFile.deleteSync(); }); - test('fresh install creates all tables at schemaVersion 25', () async { + test('fresh install creates all tables at schemaVersion 26', () async { final db = AppDatabase(NativeDatabase.memory()); await db.select(db.accounts).get();