feat: apply local Sieve rules after sync (#119)

- Add LocalSieveApplied table (schema v32) keyed by (accountId, messageId)
  so each email is processed by Sieve at most once, even across restarts.
- Implement EmailRepository.applySieveRules(): loads the active local Sieve
  script, runs the interpreter against new INBOX emails, and queues pending
  move/delete/flag_seen changes for any matched rules.
- Wire applySieveRules() into both _AccountSync._sync() and
  _JmapAccountSync._sync() after the per-mailbox email sync loop.
- Make _flushPendingChangesImap() treat NONEXISTENT / not-found errors as
  silent no-ops (counts as flushed) so a second device racing on the same
  email does not accumulate retries.
- Add migration test assertions and a dedicated unit test suite covering
  rule matching, deduplication, discard, and multi-email processing.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Thomas SharedInbox
2026-05-17 10:34:21 +02:00
co-authored by Claude Sonnet 4.6
parent 8ef1c7c96f
commit 517f799b99
13 changed files with 600 additions and 4 deletions
@@ -104,6 +104,12 @@ abstract class EmailRepository {
/// IMAP UID changed (e.g. after a server-applied move assigned a new UID).
Future<Email?> findEmailByMessageId(String accountId, String messageId);
/// Applies locally stored active Sieve rules to INBOX emails that have not
/// been processed yet. Records each processed email in LocalSieveApplied so
/// the same email is never filtered twice (across restarts or re-syncs).
/// Returns the number of emails where a rule matched and an action was taken.
Future<int> applySieveRules(String accountId);
/// Emits the accountId whenever a new change is enqueued locally.
/// Used by AccountSyncManager to trigger an immediate flush.
Stream<String> get onChangesQueued;
+2
View File
@@ -360,6 +360,7 @@ class _AccountSync implements _SyncLoop {
),
);
}
await _emails.applySieveRules(account.id);
return _SyncStats(
emailsFetched: emailResult.fetched,
emailsSkipped: emailResult.skipped,
@@ -613,6 +614,7 @@ class _JmapAccountSync implements _SyncLoop {
),
);
}
await _emails.applySieveRules(account.id);
return _SyncStats(
emailsFetched: emailResult.fetched,
emailsSkipped: emailResult.skipped,
+20 -1
View File
@@ -287,6 +287,21 @@ class UndoActions extends Table {
Set<Column> get primaryKey => {id};
}
/// Records which emails have already had local Sieve rules applied.
/// Keyed by (accountId, messageId) so the same email is never processed twice,
/// even across restarts or re-syncs.
@DataClassName('LocalSieveAppliedRow')
class LocalSieveApplied extends Table {
TextColumn get accountId =>
text().references(Accounts, #id, onDelete: KeyAction.cascade)();
// RFC 2822 Message-ID header value — stable across folder moves.
TextColumn get messageId => text()();
DateTimeColumn get appliedAt => dateTime()();
@override
Set<Column> get primaryKey => {accountId, messageId};
}
// ── Database ──────────────────────────────────────────────────────────────────
@DriftDatabase(
@@ -305,6 +320,7 @@ class UndoActions extends Table {
UndoActions,
SearchHistoryEntries,
LocalSieveScripts,
LocalSieveApplied,
ShareKeys,
],
)
@@ -312,7 +328,7 @@ class AppDatabase extends _$AppDatabase {
AppDatabase([QueryExecutor? executor]) : super(executor ?? _openConnection());
@override
int get schemaVersion => 31;
int get schemaVersion => 32;
Future<void> _createEmailFts() async {
await customStatement('''
@@ -550,6 +566,9 @@ class AppDatabase extends _$AppDatabase {
if (from < 31) {
await m.createTable(shareKeys);
}
if (from < 32) {
await m.createTable(localSieveApplied);
}
},
);
}
@@ -13,6 +13,9 @@ import 'package:sharedinbox/core/models/account.dart' as account_model;
import 'package:sharedinbox/core/models/email.dart' as model;
import 'package:sharedinbox/core/repositories/account_repository.dart';
import 'package:sharedinbox/core/repositories/email_repository.dart';
import 'package:sharedinbox/core/sieve/sieve_interpreter.dart';
import 'package:sharedinbox/core/sieve/sieve_parser.dart';
import 'package:sharedinbox/core/sieve/sieve_rule.dart';
import 'package:sharedinbox/core/utils/cid_utils.dart';
import 'package:sharedinbox/core/utils/logger.dart';
import 'package:sharedinbox/data/db/database.dart';
@@ -1917,6 +1920,218 @@ class EmailRepositoryImpl implements EmailRepository {
}
}
/// Applies locally stored active Sieve rules to INBOX emails that have not
/// been processed yet. See [EmailRepository.applySieveRules] for details.
@override
Future<int> applySieveRules(String accountId) async {
final scriptRow = await (_db.select(_db.localSieveScripts)
..where(
(t) => t.accountId.equals(accountId) & t.isActive.equals(true),
)
..limit(1))
.getSingleOrNull();
if (scriptRow == null) return 0;
List<SieveRule> rules;
try {
rules = SieveParser().parse(scriptRow.content);
} catch (e) {
log('Sieve parse error for account $accountId: $e');
return 0;
}
if (rules.isEmpty) return 0;
final inboxMailbox = await (_db.select(_db.mailboxes)
..where(
(t) => t.accountId.equals(accountId) & t.role.equals('inbox'),
)
..limit(1))
.getSingleOrNull();
final inboxPath = inboxMailbox?.path ?? 'INBOX';
final alreadyApplied = await (_db.select(_db.localSieveApplied)
..where((t) => t.accountId.equals(accountId)))
.get();
final appliedIds = alreadyApplied.map((r) => r.messageId).toSet();
final inboxEmails = await (_db.select(_db.emails)
..where(
(t) =>
t.accountId.equals(accountId) &
t.mailboxPath.equals(inboxPath) &
t.messageId.isNotNull(),
))
.get();
final account = (await _accounts.getAccount(accountId))!;
final interpreter = SieveInterpreter();
var matched = 0;
for (final row in inboxEmails) {
final msgId = row.messageId!;
if (appliedIds.contains(msgId)) continue;
final emailCtx = _buildSieveContext(row);
SieveExecutionContext result;
try {
result = interpreter.execute(rules, emailCtx);
} catch (e) {
log('Sieve interpreter error for message $msgId: $e');
await _markSieveApplied(accountId, msgId);
continue;
}
await _markSieveApplied(accountId, msgId);
if (result.isCancelled) {
await _enqueueSieveDelete(account, row);
matched++;
} else if (result.targetFolders.isNotEmpty) {
final dest = result.targetFolders.first;
await _enqueueSieveMove(account, row, dest);
matched++;
} else if (result.flagsToAdd.isNotEmpty) {
await _enqueueSieveFlagSeen(account, row);
matched++;
}
}
return matched;
}
SieveEmailContext _buildSieveContext(Email row) {
String formatAddrs(String json) {
try {
final list = jsonDecode(json) as List<dynamic>;
return list.map((e) {
final m = e as Map<String, dynamic>;
final name = m['name'] as String? ?? '';
final email = m['email'] as String? ?? '';
return name.isEmpty ? email : '$name <$email>';
}).join(', ');
} catch (_) {
return '';
}
}
return SieveEmailContext(
headers: {
if (row.subject != null && row.subject!.isNotEmpty)
'subject': [row.subject!],
'from': [formatAddrs(row.fromJson)],
'to': [formatAddrs(row.toAddresses)],
'cc': [formatAddrs(row.ccJson)],
if (row.messageId != null) 'message-id': [row.messageId!],
},
);
}
Future<void> _markSieveApplied(String accountId, String messageId) async {
await _db.into(_db.localSieveApplied).insertOnConflictUpdate(
LocalSieveAppliedCompanion.insert(
accountId: accountId,
messageId: messageId,
appliedAt: DateTime.now(),
),
);
}
Future<void> _enqueueSieveMove(
account_model.Account account,
Email row,
String folder,
) async {
String destPath;
if (account.type == account_model.AccountType.jmap) {
final destMailbox = await (_db.select(_db.mailboxes)
..where(
(t) => t.accountId.equals(account.id) & t.name.equals(folder),
)
..limit(1))
.getSingleOrNull();
if (destMailbox == null) {
log('Sieve: JMAP mailbox "$folder" not found for account ${account.id}');
return;
}
destPath = destMailbox.path;
await _enqueueChange(
account.id,
row.id,
'move',
jsonEncode({'src': row.mailboxPath, 'dest': destPath}),
);
} else {
destPath = folder;
await _enqueueChange(
account.id,
row.id,
'move',
jsonEncode({
'uid': row.uid,
'mailboxPath': row.mailboxPath,
'dest': destPath,
}),
);
}
await (_db.update(_db.emails)..where((t) => t.id.equals(row.id))).write(
EmailsCompanion(mailboxPath: Value(destPath)),
);
await _updateThread(account.id, row.mailboxPath, row.threadId ?? row.id);
await _updateThread(account.id, destPath, row.threadId ?? row.id);
}
Future<void> _enqueueSieveDelete(
account_model.Account account,
Email row,
) async {
if (account.type == account_model.AccountType.jmap) {
await _enqueueChange(
account.id,
row.id,
'delete',
jsonEncode(<String, dynamic>{}),
);
} else {
await _enqueueChange(
account.id,
row.id,
'delete',
jsonEncode({'uid': row.uid, 'mailboxPath': row.mailboxPath}),
);
}
await (_db.delete(_db.emails)..where((t) => t.id.equals(row.id))).go();
await _updateThread(account.id, row.mailboxPath, row.threadId ?? row.id);
}
Future<void> _enqueueSieveFlagSeen(
account_model.Account account,
Email row,
) async {
if (account.type == account_model.AccountType.jmap) {
await _enqueueChange(
account.id,
row.id,
'flag_seen',
jsonEncode({'seen': true}),
);
} else {
await _enqueueChange(
account.id,
row.id,
'flag_seen',
jsonEncode({
'uid': row.uid,
'mailboxPath': row.mailboxPath,
'seen': true,
}),
);
}
await (_db.update(_db.emails)..where((t) => t.id.equals(row.id))).write(
const EmailsCompanion(isSeen: Value(true)),
);
await _updateThread(account.id, row.mailboxPath, row.threadId ?? row.id);
}
/// Drains pending changes for [accountId] via the appropriate protocol.
/// Called at the start of each sync cycle. Returns count of applied changes.
@override
@@ -2032,7 +2247,18 @@ class EmailRepositoryImpl implements EmailRepository {
.go();
applied++;
} catch (e) {
await _recordChangeError(row, e);
if (_isImapNotFoundError(e)) {
// Email already gone on the server — treat as success so the
// pending change doesn't accumulate or block future changes.
await (_db.delete(
_db.pendingChanges,
)..where((t) => t.id.equals(row.id)))
.go();
applied++;
log('IMAP change ${row.id} skipped: message already gone ($e)');
} else {
await _recordChangeError(row, e);
}
}
}
} finally {
@@ -2041,6 +2267,11 @@ class EmailRepositoryImpl implements EmailRepository {
return applied;
}
bool _isImapNotFoundError(Object e) {
final s = e.toString().toLowerCase();
return s.contains('nonexistent') || s.contains('not found');
}
Future<void> _applyPendingChangeImap(
imap.ImapClient client,
PendingChangeRow row,
@@ -277,6 +277,9 @@ class _FakeEmails implements EmailRepository {
@override
Future<void> clearForResync(String accountId) async {}
@override
Future<int> applySieveRules(String accountId) async => 0;
}
class _FakeLogs implements SyncLogRepository {
+3
View File
@@ -128,6 +128,9 @@ class FakeEmailRepository implements EmailRepository {
@override
Future<void> clearForResync(String accountId) async {}
@override
Future<int> applySieveRules(String accountId) async => 0;
}
class _Log {
@@ -622,6 +622,15 @@ class MockEmailRepository extends _i1.Mock implements _i9.EmailRepository {
returnValue: _i4.Future<_i2.Email?>.value(),
) as _i4.Future<_i2.Email?>);
@override
_i4.Future<int> applySieveRules(String? accountId) => (super.noSuchMethod(
Invocation.method(
#applySieveRules,
[accountId],
),
returnValue: _i4.Future<int>.value(0),
) as _i4.Future<int>);
@override
_i4.Stream<void> watchJmapPush(
String? accountId,
+303
View File
@@ -0,0 +1,303 @@
import 'dart:convert';
import 'package:drift/drift.dart' show Value;
import 'package:flutter_test/flutter_test.dart';
import 'package:sharedinbox/core/models/account.dart';
import 'package:sharedinbox/data/db/database.dart' hide Account;
import 'package:sharedinbox/data/repositories/account_repository_impl.dart';
import 'package:sharedinbox/data/repositories/email_repository_impl.dart';
import 'account_repository_impl_test.dart' show MapSecureStorage;
import 'db_test_helper.dart';
// ── Helpers ───────────────────────────────────────────────────────────────────
const _account = Account(
id: 'sieve-acc',
displayName: 'Sieve Test',
email: 'sieve@example.com',
imapHost: 'imap.example.com',
smtpHost: 'smtp.example.com',
);
Future<(AppDatabase, EmailRepositoryImpl)> _makeSetup() async {
final db = openTestDatabase();
final storage = MapSecureStorage();
final accounts = AccountRepositoryImpl(db, storage);
await accounts.addAccount(_account, 'password');
final repo = EmailRepositoryImpl(db, accounts);
return (db, repo);
}
/// Inserts a minimal email row in the INBOX. Returns the row id.
Future<String> _insertInboxEmail(
AppDatabase db, {
required String id,
required String messageId,
String subject = 'Test',
String from = 'sender@example.com',
String mailboxPath = 'INBOX',
}) async {
await db.into(db.emails).insert(
EmailsCompanion.insert(
id: id,
accountId: _account.id,
mailboxPath: mailboxPath,
uid: int.parse(id.split(':').last),
subject: Value(subject),
receivedAt: DateTime.now(),
fromJson: Value(
jsonEncode([
{'name': '', 'email': from},
]),
),
messageId: Value(messageId),
),
);
// Insert a thread row so _updateThread does not throw.
await db.into(db.threads).insertOnConflictUpdate(
ThreadsCompanion.insert(
id: id,
accountId: _account.id,
mailboxPath: mailboxPath,
latestDate: DateTime.now(),
latestEmailId: id,
),
);
return id;
}
/// Creates an active Sieve script for the test account.
Future<void> _insertSieveScript(AppDatabase db, String content) async {
await db.into(db.localSieveScripts).insert(
LocalSieveScriptsCompanion.insert(
accountId: _account.id,
name: 'test-script',
content: Value(content),
isActive: const Value(true),
),
);
}
// ── Tests ─────────────────────────────────────────────────────────────────────
void main() {
setUpAll(configureSqliteForTests);
group('applySieveRules', () {
test('returns 0 when no active script exists', () async {
final (_, repo) = await _makeSetup();
expect(await repo.applySieveRules(_account.id), 0);
});
test('returns 0 when script has no matching rules', () async {
final (db, repo) = await _makeSetup();
await _insertSieveScript(db, '''
require ["fileinto"];
if header :contains "subject" ["NEVER_MATCHES_XYZ"] {
fileinto "Archive";
}
''');
await _insertInboxEmail(
db,
id: 'sieve-acc:1',
messageId: '<msg1@test>',
subject: 'Hello world',
);
expect(await repo.applySieveRules(_account.id), 0);
});
test('applies fileinto rule and queues a move pending change', () async {
final (db, repo) = await _makeSetup();
await _insertSieveScript(db, '''
require ["fileinto"];
if header :contains "subject" ["SPAM"] {
fileinto "Archive";
}
''');
await _insertInboxEmail(
db,
id: 'sieve-acc:1',
messageId: '<msg1@test>',
subject: 'THIS IS SPAM',
);
final count = await repo.applySieveRules(_account.id);
expect(count, 1);
final pending = await db.select(db.pendingChanges).get();
expect(pending, hasLength(1));
expect(pending.first.changeType, 'move');
final payload = jsonDecode(pending.first.payload) as Map<String, dynamic>;
expect(payload['dest'], 'Archive');
});
test('applies discard rule and queues a delete pending change', () async {
final (db, repo) = await _makeSetup();
await _insertSieveScript(db, '''
if header :contains "from" ["spam@evil.com"] {
discard;
}
''');
await _insertInboxEmail(
db,
id: 'sieve-acc:1',
messageId: '<msg1@test>',
from: 'spam@evil.com',
);
final count = await repo.applySieveRules(_account.id);
expect(count, 1);
final pending = await db.select(db.pendingChanges).get();
expect(pending, hasLength(1));
expect(pending.first.changeType, 'delete');
});
test('records email in LocalSieveApplied after processing', () async {
final (db, repo) = await _makeSetup();
await _insertSieveScript(db, '''
require ["fileinto"];
if header :contains "subject" ["SPAM"] {
fileinto "Archive";
}
''');
await _insertInboxEmail(
db,
id: 'sieve-acc:1',
messageId: '<msg1@test>',
subject: 'SPAM email',
);
await repo.applySieveRules(_account.id);
final applied = await db.select(db.localSieveApplied).get();
expect(applied, hasLength(1));
expect(applied.first.messageId, '<msg1@test>');
expect(applied.first.accountId, _account.id);
});
test('does not reprocess an email already in LocalSieveApplied', () async {
final (db, repo) = await _makeSetup();
await _insertSieveScript(db, '''
require ["fileinto"];
if header :contains "subject" ["SPAM"] {
fileinto "Archive";
}
''');
await _insertInboxEmail(
db,
id: 'sieve-acc:1',
messageId: '<msg1@test>',
subject: 'SPAM email',
);
// First run applies the rule.
expect(await repo.applySieveRules(_account.id), 1);
// Restore the email to INBOX to simulate a re-sync (e.g. second device
// moved it back). The LocalSieveApplied record prevents reprocessing.
await (db.update(db.emails)..where((t) => t.id.equals('sieve-acc:1')))
.write(const EmailsCompanion(mailboxPath: Value('INBOX')));
// Second run must not produce another pending change.
expect(await repo.applySieveRules(_account.id), 0);
final pending = await db.select(db.pendingChanges).get();
// Only the original move from the first run; no duplicates.
expect(pending, hasLength(1));
});
test('skips emails with no messageId', () async {
final (db, repo) = await _makeSetup();
await _insertSieveScript(db, '''
require ["fileinto"];
if header :contains "subject" ["SPAM"] {
fileinto "Archive";
}
''');
// Insert without messageId.
await db.into(db.emails).insert(
EmailsCompanion.insert(
id: 'sieve-acc:2',
accountId: _account.id,
mailboxPath: 'INBOX',
uid: 2,
subject: const Value('SPAM without id'),
receivedAt: DateTime.now(),
),
);
await db.into(db.threads).insertOnConflictUpdate(
ThreadsCompanion.insert(
id: 'sieve-acc:2',
accountId: _account.id,
mailboxPath: 'INBOX',
latestDate: DateTime.now(),
latestEmailId: 'sieve-acc:2',
),
);
expect(await repo.applySieveRules(_account.id), 0);
expect(await db.select(db.pendingChanges).get(), isEmpty);
});
test('emails not in INBOX are ignored', () async {
final (db, repo) = await _makeSetup();
await _insertSieveScript(db, '''
require ["fileinto"];
if header :contains "subject" ["SPAM"] {
fileinto "Archive";
}
''');
await _insertInboxEmail(
db,
id: 'sieve-acc:1',
messageId: '<msg1@test>',
subject: 'SPAM in Sent',
mailboxPath: 'Sent',
);
expect(await repo.applySieveRules(_account.id), 0);
expect(await db.select(db.pendingChanges).get(), isEmpty);
});
test('processes multiple emails independently', () async {
final (db, repo) = await _makeSetup();
await _insertSieveScript(db, '''
require ["fileinto"];
if header :contains "subject" ["SPAM"] {
fileinto "Archive";
}
''');
await _insertInboxEmail(
db,
id: 'sieve-acc:1',
messageId: '<msg1@test>',
subject: 'SPAM',
);
await _insertInboxEmail(
db,
id: 'sieve-acc:2',
messageId: '<msg2@test>',
subject: 'Hello',
);
await _insertInboxEmail(
db,
id: 'sieve-acc:3',
messageId: '<msg3@test>',
subject: 'More SPAM',
);
final count = await repo.applySieveRules(_account.id);
expect(count, 2);
final applied = await db.select(db.localSieveApplied).get();
// All three emails should be in LocalSieveApplied.
expect(applied, hasLength(3));
final pending = await db.select(db.pendingChanges).get();
expect(pending, hasLength(2));
});
});
}
+6 -2
View File
@@ -14,7 +14,7 @@ void main() {
group('Migration', () {
test('schemaVersion matches expected value', () async {
final db = AppDatabase(NativeDatabase.memory());
expect(db.schemaVersion, 31);
expect(db.schemaVersion, 32);
await db.close();
});
@@ -191,6 +191,9 @@ void main() {
await _tableColumns(db, 'sync_log_mailboxes');
expect(syncLogMailboxColumns, contains('duration_ms'));
// v32: local_sieve_applied table.
await db.customSelect('SELECT count(*) FROM local_sieve_applied').get();
await db.close();
if (dbFile.existsSync()) dbFile.deleteSync();
});
@@ -382,7 +385,7 @@ void main() {
if (dbFile.existsSync()) dbFile.deleteSync();
});
test('fresh install creates all tables at schemaVersion 31', () async {
test('fresh install creates all tables at schemaVersion 32', () async {
final db = AppDatabase(NativeDatabase.memory());
await db.select(db.accounts).get();
@@ -408,6 +411,7 @@ void main() {
'search_history_entries',
'local_sieve_scripts', // v29
'share_keys', // v31
'local_sieve_applied', // v32
]),
);
@@ -150,6 +150,8 @@ class _FakeEmails implements EmailRepository {
Future<int> wakeUpEmails(String accountId) async => 0;
@override
Future<void> clearForResync(String accountId) async {}
@override
Future<int> applySieveRules(String accountId) async => 0;
}
// ---------------------------------------------------------------------------
+2
View File
@@ -159,6 +159,8 @@ class _CountingEmails implements EmailRepository {
ReliabilityResult.healthy;
@override
Future<void> clearForResync(String accountId) async {}
@override
Future<int> applySieveRules(String accountId) async => 0;
}
class _FakeSyncLog implements SyncLogRepository {
+9
View File
@@ -482,6 +482,15 @@ class MockEmailRepository extends _i1.Mock implements _i3.EmailRepository {
returnValue: _i4.Future<_i2.Email?>.value(),
) as _i4.Future<_i2.Email?>);
@override
_i4.Future<int> applySieveRules(String? accountId) => (super.noSuchMethod(
Invocation.method(
#applySieveRules,
[accountId],
),
returnValue: _i4.Future<int>.value(0),
) as _i4.Future<int>);
@override
_i4.Stream<void> watchJmapPush(
String? accountId,
+3
View File
@@ -334,6 +334,9 @@ class FakeEmailRepository implements EmailRepository {
@override
Future<void> clearForResync(String accountId) async {}
@override
Future<int> applySieveRules(String accountId) async => 0;
}
// ---------------------------------------------------------------------------