diff --git a/LATER.md b/LATER.md index 508de10..ba7e66c 100644 --- a/LATER.md +++ b/LATER.md @@ -6,6 +6,11 @@ Create a re-usable JMAP package. --- +full-sync: Imaging the sync got out-of-sync somehow. Provide a way via UI to force a sync. First +create a plan. Avoid downloading big bodies/attachments again. + +--- + mailcoach.de diff --git a/PLAN.md b/PLAN.md index b6ea0a0..b8c005e 100644 --- a/PLAN.md +++ b/PLAN.md @@ -14,22 +14,9 @@ IMAP/SMTP server UI never touches the network. The sync layer runs independently. -## Phases +## Next -| Phase | Scope | Status | -| --- | --- | --- | -| 0 — Scaffold | pubspec, Drift schema, DI, router, enough_mail from pub.dev | Done | -| 1 — Core models | `Account`, `Mailbox`, `Email`, `EmailBody`, repository interfaces | Done | -| 2 — DB layer | Drift tables, `AccountRepositoryImpl`, `MailboxRepositoryImpl`, `EmailRepositoryImpl` | Done | -| 3 — IMAP sync | `connectImap`, `MailboxRepositoryImpl.syncMailboxes`, `EmailRepositoryImpl.syncEmails` | Done | -| 4 — IMAP IDLE | `AccountSyncManager` with exponential-backoff reconnect | Done | -| 5 — SMTP send | `connectSmtp`, `EmailRepositoryImpl.sendEmail` | Done | -| 6 — UI | AccountList, AddAccount, MailboxList, EmailList, EmailDetail, Compose, Settings | Done | -| 7 — Dev tooling | Nix flake, Taskfile, Stalwart dev server, unit + integration tests, CI, pre-commit | Done | -| 8 — UI gaps | Account picker in compose, flag/unflag, move-to-folder, attachment indicators | Done | - -## Next candidates - -- Thread view (group by `References` / `In-Reply-To`) -- Attachment download + open -- Draft auto-save +- [ ] Per-mailbox sync log — current log aggregates all mailboxes; break down fetched/skipped per mailbox path so a stale checkpoint for one folder is immediately visible +- [ ] IMAP trace logging — add `logRequests`/`logResponses` to `connectImap` for debug builds; essential to verify what `UID SEARCH ALL` actually returns from the server +- [ ] Thread view (group by `References` / `In-Reply-To`) +- [ ] Attachment download + open diff --git a/lib/core/repositories/sync_log_repository.dart b/lib/core/repositories/sync_log_repository.dart index a421e64..d1c47c7 100644 --- a/lib/core/repositories/sync_log_repository.dart +++ b/lib/core/repositories/sync_log_repository.dart @@ -1,3 +1,17 @@ +class MailboxSyncStats { + const MailboxSyncStats({ + required this.mailboxPath, + required this.fetched, + required this.skipped, + required this.bytesTransferred, + }); + + final String mailboxPath; + final int fetched; + final int skipped; + final int bytesTransferred; +} + class SyncLogEntry { const SyncLogEntry({ required this.id, @@ -11,6 +25,7 @@ class SyncLogEntry { required this.bytesTransferred, required this.startedAt, required this.finishedAt, + this.mailboxStats = const [], }); final int id; @@ -24,6 +39,7 @@ class SyncLogEntry { final int bytesTransferred; final DateTime startedAt; final DateTime finishedAt; + final List mailboxStats; Duration get duration => finishedAt.difference(startedAt); bool get isOk => result == 'ok'; @@ -42,6 +58,7 @@ abstract class SyncLogRepository { required int bytesTransferred, required DateTime startedAt, required DateTime finishedAt, + List mailboxStats = const [], }); Stream> observeSyncLogs(String accountId); @@ -63,6 +80,7 @@ class NoOpSyncLogRepository implements SyncLogRepository { required int bytesTransferred, required DateTime startedAt, required DateTime finishedAt, + List mailboxStats = const [], }) async {} @override diff --git a/lib/core/sync/account_sync_manager.dart b/lib/core/sync/account_sync_manager.dart index f9b4285..f41bce0 100644 --- a/lib/core/sync/account_sync_manager.dart +++ b/lib/core/sync/account_sync_manager.dart @@ -136,6 +136,7 @@ class _AccountSync implements _SyncLoop { bytesTransferred: stats.bytesTransferred, startedAt: startedAt, finishedAt: DateTime.now(), + mailboxStats: stats.mailboxStats, ); await _idle(); _backoffSeconds = 5; @@ -175,9 +176,19 @@ class _AccountSync implements _SyncLoop { final mailboxesSynced = await _mailboxes.syncMailboxes(account.id); final mailboxes = await _mailboxes.observeMailboxes(account.id).first; var emailResult = SyncEmailsResult.zero; + final mailboxStats = []; for (final mailbox in mailboxes) { if (!_running) break; - emailResult += await _emails.syncEmails(account.id, mailbox.path); + final r = await _emails.syncEmails(account.id, mailbox.path); + emailResult += r; + mailboxStats.add( + MailboxSyncStats( + mailboxPath: mailbox.path, + fetched: r.fetched, + skipped: r.skipped, + bytesTransferred: r.bytesTransferred, + ), + ); } return _SyncStats( emailsFetched: emailResult.fetched, @@ -185,6 +196,7 @@ class _AccountSync implements _SyncLoop { mailboxesSynced: mailboxesSynced, pendingFlushed: pendingFlushed, bytesTransferred: emailResult.bytesTransferred, + mailboxStats: mailboxStats, ); } @@ -284,6 +296,7 @@ class _JmapAccountSync implements _SyncLoop { bytesTransferred: stats.bytesTransferred, startedAt: startedAt, finishedAt: DateTime.now(), + mailboxStats: stats.mailboxStats, ); _backoffSeconds = 5; await _wait(); @@ -327,9 +340,19 @@ class _JmapAccountSync implements _SyncLoop { final mailboxes = await _mailboxes.observeMailboxes(account.id).first; var emailResult = SyncEmailsResult.zero; + final mailboxStats = []; for (final mailbox in mailboxes) { if (!_running) break; - emailResult += await _emails.syncEmails(account.id, mailbox.path); + final r = await _emails.syncEmails(account.id, mailbox.path); + emailResult += r; + mailboxStats.add( + MailboxSyncStats( + mailboxPath: mailbox.path, + fetched: r.fetched, + skipped: r.skipped, + bytesTransferred: r.bytesTransferred, + ), + ); } return _SyncStats( emailsFetched: emailResult.fetched, @@ -337,6 +360,7 @@ class _JmapAccountSync implements _SyncLoop { mailboxesSynced: mailboxesSynced, pendingFlushed: pendingFlushed, bytesTransferred: emailResult.bytesTransferred, + mailboxStats: mailboxStats, ); } @@ -384,6 +408,7 @@ class _SyncStats { required this.mailboxesSynced, required this.pendingFlushed, required this.bytesTransferred, + required this.mailboxStats, }); final int emailsFetched; @@ -391,4 +416,5 @@ class _SyncStats { final int mailboxesSynced; final int pendingFlushed; final int bytesTransferred; + final List mailboxStats; } diff --git a/lib/data/db/database.dart b/lib/data/db/database.dart index 3808e95..f146615 100644 --- a/lib/data/db/database.dart +++ b/lib/data/db/database.dart @@ -139,6 +139,19 @@ class SyncLogs extends Table { DateTimeColumn get finishedAt => dateTime()(); } +/// Per-mailbox breakdown for a single sync cycle. +/// Each row is a child of one SyncLogs row. +@DataClassName('SyncLogMailboxRow') +class SyncLogMailboxes extends Table { + IntColumn get id => integer().autoIncrement()(); + IntColumn get syncLogId => + integer().references(SyncLogs, #id, onDelete: KeyAction.cascade)(); + TextColumn get mailboxPath => text()(); + IntColumn get fetched => integer().withDefault(const Constant(0))(); + IntColumn get skipped => integer().withDefault(const Constant(0))(); + IntColumn get bytesTransferred => integer().withDefault(const Constant(0))(); +} + /// Auto-saved compose drafts — persisted across app restarts. class Drafts extends Table { IntColumn get id => integer().autoIncrement()(); @@ -165,13 +178,14 @@ class Drafts extends Table { SyncStates, PendingChanges, SyncLogs, + SyncLogMailboxes, ], ) class AppDatabase extends _$AppDatabase { AppDatabase([QueryExecutor? executor]) : super(executor ?? _openConnection()); @override - int get schemaVersion => 11; + int get schemaVersion => 12; @override MigrationStrategy get migration => MigrationStrategy( @@ -210,6 +224,9 @@ class AppDatabase extends _$AppDatabase { await m.addColumn(syncLogs, syncLogs.emailsSkipped); await m.addColumn(syncLogs, syncLogs.bytesTransferred); } + if (from < 12) { + await m.createTable(syncLogMailboxes); + } }, ); } diff --git a/lib/data/repositories/sync_log_repository_impl.dart b/lib/data/repositories/sync_log_repository_impl.dart index 615816c..4c7d706 100644 --- a/lib/data/repositories/sync_log_repository_impl.dart +++ b/lib/data/repositories/sync_log_repository_impl.dart @@ -21,49 +21,79 @@ class SyncLogRepositoryImpl implements SyncLogRepository { required int bytesTransferred, required DateTime startedAt, required DateTime finishedAt, + List mailboxStats = const [], }) async { - await _db.into(_db.syncLogs).insert( - SyncLogsCompanion.insert( - accountId: accountId, - result: success ? 'ok' : 'error', - errorMessage: Value(errorMessage), - protocol: Value(protocol), - itemsSynced: Value(emailsFetched), - emailsSkipped: Value(emailsSkipped), - mailboxesSynced: Value(mailboxesSynced), - pendingFlushed: Value(pendingFlushed), - bytesTransferred: Value(bytesTransferred), - startedAt: startedAt, - finishedAt: finishedAt, - ), - ); + await _db.transaction(() async { + final logId = await _db.into(_db.syncLogs).insert( + SyncLogsCompanion.insert( + accountId: accountId, + result: success ? 'ok' : 'error', + errorMessage: Value(errorMessage), + protocol: Value(protocol), + itemsSynced: Value(emailsFetched), + emailsSkipped: Value(emailsSkipped), + mailboxesSynced: Value(mailboxesSynced), + pendingFlushed: Value(pendingFlushed), + bytesTransferred: Value(bytesTransferred), + startedAt: startedAt, + finishedAt: finishedAt, + ), + ); + for (final s in mailboxStats) { + await _db.into(_db.syncLogMailboxes).insert( + SyncLogMailboxesCompanion.insert( + syncLogId: logId, + mailboxPath: s.mailboxPath, + fetched: Value(s.fetched), + skipped: Value(s.skipped), + bytesTransferred: Value(s.bytesTransferred), + ), + ); + } + }); } @override Stream> observeSyncLogs(String accountId) { - return (_db.select(_db.syncLogs) - ..where((t) => t.accountId.equals(accountId)) - ..orderBy([(t) => OrderingTerm.desc(t.startedAt)]) - ..limit(100)) - .watch() - .map( - (rows) => rows - .map( - (r) => SyncLogEntry( - id: r.id, - result: r.result, - errorMessage: r.errorMessage, - protocol: r.protocol, - emailsFetched: r.itemsSynced, - emailsSkipped: r.emailsSkipped, - mailboxesSynced: r.mailboxesSynced, - pendingFlushed: r.pendingFlushed, - bytesTransferred: r.bytesTransferred, - startedAt: r.startedAt, - finishedAt: r.finishedAt, - ), - ) - .toList(), + final logsQuery = _db.select(_db.syncLogs) + ..where((t) => t.accountId.equals(accountId)) + ..orderBy([(t) => OrderingTerm.desc(t.startedAt)]) + ..limit(100); + + return logsQuery.watch().asyncMap((rows) async { + final entries = []; + for (final r in rows) { + final mailboxRows = await (_db.select(_db.syncLogMailboxes) + ..where((t) => t.syncLogId.equals(r.id)) + ..orderBy([(t) => OrderingTerm.asc(t.mailboxPath)])) + .get(); + entries.add( + SyncLogEntry( + id: r.id, + result: r.result, + errorMessage: r.errorMessage, + protocol: r.protocol, + emailsFetched: r.itemsSynced, + emailsSkipped: r.emailsSkipped, + mailboxesSynced: r.mailboxesSynced, + pendingFlushed: r.pendingFlushed, + bytesTransferred: r.bytesTransferred, + startedAt: r.startedAt, + finishedAt: r.finishedAt, + mailboxStats: mailboxRows + .map( + (m) => MailboxSyncStats( + mailboxPath: m.mailboxPath, + fetched: m.fetched, + skipped: m.skipped, + bytesTransferred: m.bytesTransferred, + ), + ) + .toList(), + ), ); + } + return entries; + }); } } diff --git a/lib/ui/screens/sync_log_screen.dart b/lib/ui/screens/sync_log_screen.dart index b99339a..be5b354 100644 --- a/lib/ui/screens/sync_log_screen.dart +++ b/lib/ui/screens/sync_log_screen.dart @@ -93,6 +93,20 @@ class _SyncLogTile extends StatelessWidget { _row('Mailboxes synced', '${entry.mailboxesSynced}'), _row('Pending changes flushed', '${entry.pendingFlushed}'), _row('Data transferred', _fmtBytes(entry.bytesTransferred)), + if (entry.mailboxStats.isNotEmpty) ...[ + const Padding( + padding: EdgeInsets.only(top: 6, bottom: 2), + child: Text( + 'Per mailbox', + style: TextStyle(fontSize: 12, color: Colors.grey), + ), + ), + for (final m in entry.mailboxStats) + _row( + ' ${m.mailboxPath}', + '${m.fetched} new · ${m.skipped} up-to-date', + ), + ], if (entry.errorMessage != null) Padding( padding: const EdgeInsets.only(top: 4), diff --git a/test/unit/account_sync_manager_test.dart b/test/unit/account_sync_manager_test.dart index 6a97eda..b0b8c3e 100644 --- a/test/unit/account_sync_manager_test.dart +++ b/test/unit/account_sync_manager_test.dart @@ -201,6 +201,7 @@ class _CapturingSyncLogRepository implements SyncLogRepository { required int bytesTransferred, required DateTime startedAt, required DateTime finishedAt, + List mailboxStats = const [], }) async { entries.add( SyncLogEntry(