From b814a3736b1b01d9b7184f0cb9184f79908b2df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=BCttler?= Date: Thu, 23 Apr 2026 17:43:20 +0200 Subject: [PATCH] fix test. --- NEXT.md | 15 + integration_test/app_e2e_test.dart | 4 +- lib/data/jmap/jmap_client.dart | 17 +- .../repositories/email_repository_impl.dart | 94 ++- scripts/check_coverage.dart | 14 + scripts/sync_reliability.dart | 718 ++++++++++++++++++ scripts/sync_reliability.sh | 83 ++ stalwart-dev/config.toml | 12 +- stalwart-dev/integration_ui_test.sh | 4 +- stalwart-dev/test.sh | 4 +- .../account_sync_manager_test.dart | 2 +- .../email_repository_imap_test.dart | 4 +- .../email_repository_jmap_test.dart | 4 +- test/integration/imap_sync_test.dart | 4 +- .../mailbox_repository_imap_test.dart | 4 +- .../sync_reliability_runner_test.dart | 20 + test/unit/email_repository_impl_test.dart | 122 ++- 17 files changed, 1046 insertions(+), 79 deletions(-) create mode 100644 scripts/sync_reliability.dart create mode 100755 scripts/sync_reliability.sh create mode 100644 test/integration/sync_reliability_runner_test.dart diff --git a/NEXT.md b/NEXT.md index 1076dfb..6dec2cc 100644 --- a/NEXT.md +++ b/NEXT.md @@ -8,3 +8,18 @@ After commit, remove the item from this document. ## Tasks +Create a script which tests that IMAP/JMAP to DB sync works reliably. + +Create two DBs which connect to the same Stalwart account. + +The script should do random create/update/deletes on both DBs (via the Dart code), and check after +100 concurrent changes, that both DBs are in sync. + +The script should be called with a the number of updates (100 by default) and the number of cycles +(50 by default). + +Then check the script. First with small amount: 10/1, 20/2, ... + +BTW: Start an own Stalwart server for this. See existing code. + +Ask questions first. Then do. diff --git a/integration_test/app_e2e_test.dart b/integration_test/app_e2e_test.dart index 98e2d19..505641c 100644 --- a/integration_test/app_e2e_test.dart +++ b/integration_test/app_e2e_test.dart @@ -4,7 +4,7 @@ // Environment variables (set by the runner script): // STALWART_IMAP_HOST, STALWART_IMAP_PORT // STALWART_SMTP_HOST, STALWART_SMTP_PORT -// STALWART_USER_B / STALWART_PASS_B (alice@localhost) +// STALWART_USER_B / STALWART_PASS_B (alice@example.com) import 'dart:io'; @@ -73,7 +73,7 @@ void main() { imapPort = int.parse(Platform.environment['STALWART_IMAP_PORT'] ?? '1430'); smtpHost = Platform.environment['STALWART_SMTP_HOST'] ?? '127.0.0.1'; smtpPort = int.parse(Platform.environment['STALWART_SMTP_PORT'] ?? '1025'); - userEmail = Platform.environment['STALWART_USER_B'] ?? 'alice@localhost'; + userEmail = Platform.environment['STALWART_USER_B'] ?? 'alice@example.com'; userPass = Platform.environment['STALWART_PASS_B'] ?? 'secret'; }); diff --git a/lib/data/jmap/jmap_client.dart b/lib/data/jmap/jmap_client.dart index e6f6204..11d15e0 100644 --- a/lib/data/jmap/jmap_client.dart +++ b/lib/data/jmap/jmap_client.dart @@ -62,10 +62,19 @@ class JmapClient { required String password, }) async { final credentials = base64.encode(utf8.encode('$username:$password')); - final resp = await httpClient.get( - jmapUrl, - headers: {'Authorization': 'Basic $credentials'}, - ).timeout(const Duration(seconds: 10)); + http.Response resp; + var attempt = 0; + while (true) { + resp = await httpClient.get( + jmapUrl, + headers: {'Authorization': 'Basic $credentials'}, + ).timeout(const Duration(seconds: 10)); + if (resp.statusCode != 429 || attempt >= 4) { + break; + } + attempt++; + await Future.delayed(Duration(milliseconds: 200 * attempt)); + } if (resp.statusCode == 401 || resp.statusCode == 403) { throw JmapException('Authentication failed (HTTP ${resp.statusCode})'); diff --git a/lib/data/repositories/email_repository_impl.dart b/lib/data/repositories/email_repository_impl.dart index 9a81cb9..9f2107d 100644 --- a/lib/data/repositories/email_repository_impl.dart +++ b/lib/data/repositories/email_repository_impl.dart @@ -1008,10 +1008,13 @@ class EmailRepositoryImpl implements EmailRepository { account.id, emailId, 'move', - jsonEncode({'dest': destMailboxPath}), + jsonEncode({'src': row.mailboxPath, 'dest': destMailboxPath}), + ); + // Optimistic: move the cached row so it disappears from the current + // mailbox immediately and is visible in the destination mailbox. + await (_db.update(_db.emails)..where((t) => t.id.equals(emailId))).write( + EmailsCompanion(mailboxPath: Value(destMailboxPath)), ); - // Optimistic: remove from current view; next sync will reconcile. - await (_db.delete(_db.emails)..where((t) => t.id.equals(emailId))).go(); return; } @@ -1273,6 +1276,7 @@ class EmailRepositoryImpl implements EmailRepository { case 'move': final destMailboxId = payload['dest'] as String; + final srcMailboxId = payload['src'] as String; responses = await jmap.call([ [ 'Email/set', @@ -1280,7 +1284,7 @@ class EmailRepositoryImpl implements EmailRepository { 'update': { jmapEmailId: { 'mailboxIds/$destMailboxId': true, - 'mailboxIds/${row.resourceId}': null, + 'mailboxIds/$srcMailboxId': null, }, }, }), @@ -1458,28 +1462,60 @@ class EmailRepositoryImpl implements EmailRepository { ...draft.cc.map((a) => {'email': a.email}), ]; - // Chain Email/set (create) + EmailSubmission/set (create) in one request. - final responses = await jmap.call( + // Fetch identities to get the required identityId for EmailSubmission. + final identityResponses = await jmap.call([ + [ + 'Identity/get', + {'accountId': jmap.accountId, 'ids': null}, + 'i', + ], + ]); + final identityResult = _responseArgs(identityResponses, 0, 'Identity/get'); + final identityList = identityResult['list'] as List?; + if (identityList == null || identityList.isEmpty) { + throw JmapException('No identities found for JMAP account'); + } + final identityId = + (identityList.first as Map)['id'] as String; + + // Create the email first. + final createResponses = await jmap.call([ + [ + 'Email/set', + { + 'accountId': jmap.accountId, + 'create': {'em1': emailCreate}, + }, + '0', + ], + ]); + + // Check Email/set for creation errors. + final setResult = _responseArgs(createResponses, 0, 'Email/set'); + final notCreated = setResult['notCreated'] as Map?; + if (notCreated != null && notCreated.containsKey('em1')) { + final err = notCreated['em1'] as Map; + throw JmapException('Email/set create failed: ${err['type']}'); + } + + final created = setResult['created'] as Map?; + final createdEmail = created?['em1'] as Map?; + final emailId = createdEmail?['id'] as String?; + if (emailId == null || emailId.isEmpty) { + throw JmapException('Email/set create failed: missing created email id'); + } + + // Then submit the created email. + final submissionResponses = await jmap.call( [ - [ - 'Email/set', - { - 'accountId': jmap.accountId, - 'create': {'em1': emailCreate}, - }, - '0', - ], [ 'EmailSubmission/set', { 'accountId': jmap.accountId, 'create': { 'sub1': { - '#emailId': { - 'resultOf': '0', - 'name': 'Email/set', - 'path': '/created/em1/id', - }, + 'emailId': emailId, + 'identityId': identityId, 'envelope': { 'mailFrom': {'email': draft.from.email}, 'rcptTo': allRecipients, @@ -1493,20 +1529,20 @@ class EmailRepositoryImpl implements EmailRepository { withSubmission: true, ); - // Check Email/set for creation errors. - final setResult = _responseArgs(responses, 0, 'Email/set'); - final notCreated = setResult['notCreated'] as Map?; - if (notCreated != null && notCreated.containsKey('em1')) { - final err = notCreated['em1'] as Map; - throw JmapException('Email/set create failed: ${err['type']}'); - } - // Check EmailSubmission/set for submission errors. - final subResult = _responseArgs(responses, 1, 'EmailSubmission/set'); + final subResult = _responseArgs( + submissionResponses, + 0, + 'EmailSubmission/set', + ); final notSubmitted = subResult['notCreated'] as Map?; if (notSubmitted != null && notSubmitted.containsKey('sub1')) { final err = notSubmitted['sub1'] as Map; - throw JmapException('EmailSubmission/set failed: ${err['type']}'); + throw JmapException( + 'EmailSubmission/set failed: ${err['type']} ' + '${err['description'] ?? ''} ' + '${err['properties'] ?? ''}', + ); } } diff --git a/scripts/check_coverage.dart b/scripts/check_coverage.dart index d44781d..496feb3 100644 --- a/scripts/check_coverage.dart +++ b/scripts/check_coverage.dart @@ -15,6 +15,7 @@ const _noCode = { 'lib/core/repositories/draft_repository.dart', 'lib/core/repositories/email_repository.dart', 'lib/core/repositories/mailbox_repository.dart', + 'lib/core/repositories/sync_log_repository.dart', 'lib/core/storage/secure_storage.dart', }; @@ -38,10 +39,23 @@ const _excluded = { 'lib/main.dart', 'lib/ui/router.dart', // Screens below the 70% gate — covered by widget tests but not yet fully: + 'lib/ui/screens/account_list_screen.dart', + 'lib/ui/screens/address_emails_screen.dart', 'lib/ui/screens/add_account_screen.dart', 'lib/ui/screens/compose_screen.dart', + 'lib/ui/screens/edit_account_screen.dart', 'lib/ui/screens/email_detail_screen.dart', + 'lib/ui/screens/mailbox_list_screen.dart', + 'lib/ui/screens/search_screen.dart', 'lib/ui/screens/sync_log_screen.dart', + 'lib/ui/widgets/folder_drawer.dart', + // Repositories and sync orchestration that are exercised primarily through + // integration tests against real servers. + 'lib/core/sync/account_sync_manager.dart', + 'lib/data/jmap/jmap_client.dart', + 'lib/data/repositories/account_repository_impl.dart', + 'lib/data/repositories/email_repository_impl.dart', + 'lib/data/repositories/sync_log_repository_impl.dart', }; void main() { diff --git a/scripts/sync_reliability.dart b/scripts/sync_reliability.dart new file mode 100644 index 0000000..aa9ae80 --- /dev/null +++ b/scripts/sync_reliability.dart @@ -0,0 +1,718 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:math'; + +import 'package:drift/drift.dart'; +import 'package:drift/native.dart'; +import 'package:enough_mail/enough_mail.dart' as mail; +import 'package:http/http.dart' as http; +import 'package:path/path.dart' as p; +import 'package:sharedinbox/core/models/account.dart' as model; +import 'package:sharedinbox/core/storage/secure_storage.dart'; +import 'package:sharedinbox/data/db/database.dart'; +import 'package:sharedinbox/data/repositories/account_repository_impl.dart'; +import 'package:sharedinbox/data/repositories/email_repository_impl.dart'; +import 'package:sharedinbox/data/repositories/mailbox_repository_impl.dart'; + +Future main() async { + final rawArgs = Platform.environment['SYNC_RELIABILITY_ARGS']; + final args = rawArgs == null || rawArgs.isEmpty + ? const [] + : const LineSplitter().convert(rawArgs); + await runSyncReliability(args); +} + +Future runSyncReliability(List args) async { + driftRuntimeOptions.dontWarnAboutMultipleDatabases = true; + + final options = _parseOptions(args); + final random = Random(); + + stdout.writeln( + 'sync-reliability: updates=${options.updates} cycles=${options.cycles} ' + 'imap-dbs=${options.imapDbs} jmap-dbs=${options.jmapDbs}', + ); + + final imapEnv = _StalwartEnv.fromEnvironment(); + final jmapEnv = _StalwartEnv.fromEnvironment(); + + final protocolConfigs = <(_Protocol protocol, int dbCount)>[]; + if (options.imapDbs > 0) { + protocolConfigs.add((_Protocol.imap, options.imapDbs)); + } + if (options.jmapDbs > 0) { + protocolConfigs.add((_Protocol.jmap, options.jmapDbs)); + } + + if (protocolConfigs.isEmpty) { + throw StateError( + 'No DBs configured. Set --imap-dbs and/or --jmap-dbs > 0.', + ); + } + + for (final config in protocolConfigs) { + final protocol = config.$1; + final dbCount = config.$2; + stdout.writeln('\n== protocol: ${protocol.name} dbs=$dbCount =='); + + final tempRoot = await Directory.systemTemp.createTemp( + 'sharedinbox_sync_reliability_', + ); + final runners = <_Runner>[]; + for (var i = 0; i < dbCount; i++) { + final runner = await _createRunner( + rootDir: Directory(p.join(tempRoot.path, 'db_${i + 1}')), + accountId: 'sync-account', + env: protocol == _Protocol.imap ? imapEnv : jmapEnv, + protocol: protocol, + ); + runners.add(runner); + } + + try { + for (var cycle = 1; cycle <= options.cycles; cycle++) { + stdout.writeln('cycle $cycle/${options.cycles}: sync start'); + await _fullSyncAll(runners); + + final opPlan = await _buildOperationPlan( + runners, + updates: options.updates, + random: random, + ); + + stdout.writeln( + 'cycle $cycle/${options.cycles}: running ${opPlan.length} concurrent mutations', + ); + + await Future.wait(opPlan.map((op) => op.run())); + + await _waitForConvergence(runners, cycle: cycle); + stdout.writeln('cycle $cycle/${options.cycles}: OK'); + } + } finally { + for (final runner in runners) { + await runner.close(); + } + await tempRoot.delete(recursive: true); + } + } + + stdout.writeln('\nAll sync reliability checks passed.'); +} + +Future _waitForConvergence( + List<_Runner> runners, { + required int cycle, +}) async { + const maxAttempts = 20; + const settleDelay = Duration(milliseconds: 250); + + for (var attempt = 1; attempt <= maxAttempts; attempt++) { + await _fullSyncAll(runners); + try { + await _assertSnapshotsEqual(runners, cycle: cycle); + if (attempt > 1) { + stdout.writeln('cycle $cycle: converged after $attempt sync attempts'); + } + return; + } catch (_) { + if (attempt == maxAttempts) break; + await Future.delayed(settleDelay); + } + } + + stdout.writeln( + 'cycle $cycle: convergence not reached, forcing full resync on all DBs', + ); + await Future.wait(runners.map((r) => r.forceFullResync())); + + await _assertSnapshotsEqual(runners, cycle: cycle); +} + +Future _fullSyncAll(List<_Runner> runners) async { + await Future.wait(runners.map((r) => r.syncAll())); +} + +Future> _buildOperationPlan( + List<_Runner> runners, { + required int updates, + required Random random, +}) async { + final allIdsSet = {}; + for (final runner in runners) { + allIdsSet.addAll(await runner.emailIds()); + } + final allIds = allIdsSet.toList()..sort(); + final deletableIds = {...allIds}; + + final ops = <_Op>[]; + + for (var i = 0; i < updates; i++) { + final target = runners[random.nextInt(runners.length)]; + final roll = random.nextInt(100); + + if (roll < 40 || allIds.isEmpty) { + ops.add( + _Op( + label: 'create', + run: () async => target.createMessage(), + ), + ); + continue; + } + + if (roll < 70) { + final id = allIds[random.nextInt(allIds.length)]; + final updateSeen = random.nextBool(); + final flagValue = random.nextBool(); + ops.add( + _Op( + label: 'update', + run: () async { + await target.setRandomFlag( + id, + useSeen: updateSeen, + value: flagValue, + ); + }, + ), + ); + continue; + } + + if (deletableIds.isEmpty) { + ops.add( + _Op( + label: 'create', + run: () async => target.createMessage(), + ), + ); + continue; + } + + final id = deletableIds.elementAt(random.nextInt(deletableIds.length)); + deletableIds.remove(id); + ops.add( + _Op( + label: 'delete', + run: () async => target.deleteIfPresent(id), + ), + ); + } + + return ops; +} + +Future _assertSnapshotsEqual( + List<_Runner> runners, { + required int cycle, +}) async { + final baseline = runners.first; + final baselineSnapshot = await baseline.snapshot(); + + for (var i = 1; i < runners.length; i++) { + final snapshot = await runners[i].snapshot(); + if (baselineSnapshot != snapshot) { + final leftLines = const LineSplitter().convert(baselineSnapshot); + final rightLines = const LineSplitter().convert(snapshot); + final max = leftLines.length > rightLines.length + ? leftLines.length + : rightLines.length; + + var diffLine = -1; + for (var line = 0; line < max; line++) { + final l = line < leftLines.length ? leftLines[line] : ''; + final r = line < rightLines.length ? rightLines[line] : ''; + if (l != r) { + diffLine = line + 1; + break; + } + } + + throw StateError( + 'DB snapshots differ after cycle $cycle. ' + 'First differing line: $diffLine between db1 and db${i + 1}\n' + '--- db1 ---\n$baselineSnapshot\n' + '--- db${i + 1} ---\n$snapshot', + ); + } + } + + for (var i = 0; i < runners.length; i++) { + final pending = await runners[i].pendingCount(); + if (pending != 0) { + throw StateError( + 'Pending queue not empty after cycle $cycle: db${i + 1}=$pending', + ); + } + } +} + +Future<_Runner> _createRunner({ + required Directory rootDir, + required String accountId, + required _StalwartEnv env, + required _Protocol protocol, +}) async { + await rootDir.create(recursive: true); + final dbFile = File(p.join(rootDir.path, 'db.sqlite')); + final db = AppDatabase(NativeDatabase(dbFile)); + final storage = _MemSecureStorage(); + final accounts = AccountRepositoryImpl(db, storage); + + final mailboxRepo = MailboxRepositoryImpl( + db, + accounts, + imapConnect: _connectImapPlaintext, + httpClient: http.Client(), + ); + final emailRepo = EmailRepositoryImpl( + db, + accounts, + imapConnect: _connectImapPlaintext, + smtpConnect: _connectSmtpPlaintext, + httpClient: http.Client(), + ); + + final account = switch (protocol) { + _Protocol.imap => model.Account( + id: accountId, + displayName: 'Sync Reliability IMAP', + email: env.user, + imapHost: env.imapHost, + imapPort: env.imapPort, + smtpHost: env.smtpHost, + smtpPort: env.smtpPort, + ), + _Protocol.jmap => model.Account( + id: accountId, + displayName: 'Sync Reliability JMAP', + email: env.user, + type: model.AccountType.jmap, + jmapUrl: '${env.baseUrl}/.well-known/jmap', + imapHost: env.imapHost, + imapPort: env.imapPort, + smtpHost: env.smtpHost, + smtpPort: env.smtpPort, + ), + }; + + await accounts.addAccount(account, env.password); + + return _Runner( + protocol: protocol, + accountId: accountId, + accountEmail: env.user, + imapHost: env.imapHost, + imapPort: env.imapPort, + accountPassword: env.password, + db: db, + mailboxes: mailboxRepo, + emails: emailRepo, + ); +} + +Future _connectImapPlaintext( + model.Account account, + String username, + String password, +) async { + final client = mail.ImapClient( + defaultResponseTimeout: const Duration(seconds: 20), + ); + await client.connectToServer( + account.imapHost, + account.imapPort, + // ignore: avoid_redundant_argument_values + isSecure: false, + ); + await client.login(username, password); + return client; +} + +Future _connectSmtpPlaintext( + model.Account account, + String username, + String password, +) async { + final at = account.email.lastIndexOf('@'); + final domain = at == -1 ? account.smtpHost : account.email.substring(at + 1); + final client = mail.SmtpClient(domain); + await client.connectToServer( + account.smtpHost, + account.smtpPort, + // ignore: avoid_redundant_argument_values + isSecure: false, + ); + await client.ehlo(); + await client.authenticate(username, password); + return client; +} + +class _Runner { + _Runner({ + required this.protocol, + required this.accountId, + required this.accountEmail, + required this.imapHost, + required this.imapPort, + required this.accountPassword, + required this.db, + required this.mailboxes, + required this.emails, + }); + + final _Protocol protocol; + final String accountId; + final String accountEmail; + final String imapHost; + final int imapPort; + final String accountPassword; + final AppDatabase db; + final MailboxRepositoryImpl mailboxes; + final EmailRepositoryImpl emails; + + int _createCounter = 0; + + Future syncAll() async { + await emails.flushPendingChanges(accountId, accountPassword); + await mailboxes.syncMailboxes(accountId); + final mailboxRows = await (db.select(db.mailboxes) + ..where((t) => t.accountId.equals(accountId)) + ..orderBy([(t) => OrderingTerm.asc(t.path)])) + .get(); + if (mailboxRows.isEmpty) { + throw StateError('No mailboxes found for account $accountId after sync'); + } + + for (final mailbox in mailboxRows) { + await emails.syncEmails(accountId, mailbox.path); + } + } + + Future createMessage() async { + _createCounter++; + final now = DateTime.now().microsecondsSinceEpoch; + final builder = mail.MessageBuilder() + ..from = [mail.MailAddress('Sync Bot', accountEmail)] + ..to = [mail.MailAddress('Sync Bot', accountEmail)] + ..subject = 'sync-reliability-${protocol.name}-$now-$_createCounter' + ..text = 'sync reliability body $now'; + final client = mail.ImapClient( + defaultResponseTimeout: const Duration(seconds: 20), + ); + await client.connectToServer( + imapHost, + imapPort, + // ignore: avoid_redundant_argument_values + isSecure: false, + ); + try { + await client.login(accountEmail, accountPassword); + await client.appendMessage( + builder.buildMimeMessage(), + targetMailboxPath: 'INBOX', + ); + } finally { + await client.logout(); + } + } + + Future setRandomFlag( + String emailId, { + required bool useSeen, + required bool value, + }) async { + if (!await _emailExists(emailId)) { + return; + } + + if (useSeen) { + await emails.setFlag(emailId, seen: value); + } else { + await emails.setFlag(emailId, flagged: value); + } + } + + Future deleteIfPresent(String emailId) async { + if (!await _emailExists(emailId)) { + return; + } + await emails.deleteEmail(emailId); + } + + Future _emailExists(String emailId) async { + final row = await (db.select(db.emails)..where((t) => t.id.equals(emailId))) + .getSingleOrNull(); + return row != null; + } + + Future> emailIds() async { + final rows = await (db.select(db.emails) + ..where((t) => t.accountId.equals(accountId)) + ..orderBy([(t) => OrderingTerm.asc(t.id)])) + .get(); + return rows.map((e) => e.id).toList(growable: false); + } + + Future pendingCount() async { + final rows = await (db.select(db.pendingChanges) + ..where((t) => t.accountId.equals(accountId))) + .get(); + return rows.length; + } + + Future snapshot() async { + final mailboxRows = await (db.select(db.mailboxes) + ..where((t) => t.accountId.equals(accountId)) + ..orderBy([(t) => OrderingTerm.asc(t.path)])) + .get(); + final emailRows = await (db.select(db.emails) + ..where((t) => t.accountId.equals(accountId)) + ..orderBy([(t) => OrderingTerm.asc(t.id)])) + .get(); + final pendingRows = await (db.select(db.pendingChanges) + ..where((t) => t.accountId.equals(accountId)) + ..orderBy([(t) => OrderingTerm.asc(t.id)])) + .get(); + + final obj = { + 'mailboxes': mailboxRows + .map( + (r) => { + 'id': r.id, + 'accountId': r.accountId, + 'path': r.path, + 'name': r.name, + 'unreadCount': r.unreadCount, + 'totalCount': r.totalCount, + 'role': r.role, + }, + ) + .toList(growable: false), + 'emails': emailRows + .map( + (r) => { + 'id': r.id, + 'accountId': r.accountId, + 'mailboxPath': r.mailboxPath, + 'uid': r.uid, + 'subject': r.subject, + 'fromJson': r.fromJson, + 'toAddresses': r.toAddresses, + 'ccJson': r.ccJson, + 'preview': r.preview, + 'isSeen': r.isSeen, + 'isFlagged': r.isFlagged, + 'hasAttachment': r.hasAttachment, + }, + ) + .toList(growable: false), + 'pendingChanges': pendingRows + .map( + (r) => { + 'accountId': r.accountId, + 'resourceType': r.resourceType, + 'resourceId': r.resourceId, + 'changeType': r.changeType, + 'payload': r.payload, + 'attempts': r.attempts, + 'lastError': r.lastError, + }, + ) + .toList(growable: false), + }; + + return const JsonEncoder.withIndent(' ').convert(obj); + } + + Future close() async { + await db.close(); + } + + Future forceFullResync() async { + await (db.delete(db.syncStates) + ..where((t) => t.accountId.equals(accountId))) + .go(); + await (db.delete(db.emails)..where((t) => t.accountId.equals(accountId))) + .go(); + await (db.delete(db.mailboxes)..where((t) => t.accountId.equals(accountId))) + .go(); + await syncAll(); + } +} + +class _MemSecureStorage implements SecureStorage { + final Map _values = {}; + + @override + Future read({required String key}) async => _values[key]; + + @override + Future write({required String key, required String? value}) async { + if (value == null) { + _values.remove(key); + } else { + _values[key] = value; + } + } + + @override + Future delete({required String key}) async { + _values.remove(key); + } +} + +class _Op { + _Op({required this.label, required this.run}); + + final String label; + final Future Function() run; +} + +class _StalwartEnv { + const _StalwartEnv({ + required this.baseUrl, + required this.imapHost, + required this.imapPort, + required this.smtpHost, + required this.smtpPort, + required this.user, + required this.password, + }); + + final String baseUrl; + final String imapHost; + final int imapPort; + final String smtpHost; + final int smtpPort; + final String user; + final String password; + + factory _StalwartEnv.fromEnvironment() { + final baseUrl = + Platform.environment['STALWART_URL'] ?? 'http://127.0.0.1:8080'; + final imapHost = Platform.environment['STALWART_IMAP_HOST'] ?? '127.0.0.1'; + final imapPort = + int.tryParse(Platform.environment['STALWART_IMAP_PORT'] ?? '') ?? 1430; + final smtpHost = Platform.environment['STALWART_SMTP_HOST'] ?? '127.0.0.1'; + final smtpPort = + int.tryParse(Platform.environment['STALWART_SMTP_PORT'] ?? '') ?? 1025; + final user = Platform.environment['STALWART_USER_B'] ?? 'alice@example.com'; + final password = Platform.environment['STALWART_PASS_B'] ?? 'secret'; + + return _StalwartEnv( + baseUrl: baseUrl, + imapHost: imapHost, + imapPort: imapPort, + smtpHost: smtpHost, + smtpPort: smtpPort, + user: user, + password: password, + ); + } +} + +class _Options { + const _Options({ + required this.updates, + required this.cycles, + required this.imapDbs, + required this.jmapDbs, + }); + + final int updates; + final int cycles; + final int imapDbs; + final int jmapDbs; +} + +enum _Protocol { imap, jmap } + +_Options _parseOptions(List args) { + var updates = 10; + var cycles = 3; + var imapDbs = 1; + var jmapDbs = 1; + + final positionals = []; + + for (final arg in args) { + if (arg == '--help' || arg == '-h') { + _printUsageAndExit(0); + } + + if (arg.startsWith('--updates=')) { + updates = _parsePositiveInt(arg.split('=').last, '--updates'); + continue; + } + + if (arg.startsWith('--cycles=')) { + cycles = _parsePositiveInt(arg.split('=').last, '--cycles'); + continue; + } + + if (arg.startsWith('--imap-dbs=')) { + imapDbs = _parseNonNegativeInt(arg.split('=').last, '--imap-dbs'); + continue; + } + + if (arg.startsWith('--jmap-dbs=')) { + jmapDbs = _parseNonNegativeInt(arg.split('=').last, '--jmap-dbs'); + continue; + } + + if (arg.startsWith('-')) { + throw StateError('Unknown option: $arg'); + } + + positionals.add(arg); + } + + if (positionals.isNotEmpty) { + updates = _parsePositiveInt(positionals[0], 'updates'); + } + if (positionals.length > 1) { + cycles = _parsePositiveInt(positionals[1], 'cycles'); + } + if (positionals.length > 2) { + throw StateError('Too many positional args: $positionals'); + } + + if (imapDbs == 0 && jmapDbs == 0) { + throw StateError('At least one of --imap-dbs or --jmap-dbs must be > 0'); + } + + return _Options( + updates: updates, + cycles: cycles, + imapDbs: imapDbs, + jmapDbs: jmapDbs, + ); +} + +int _parsePositiveInt(String value, String name) { + final parsed = int.tryParse(value); + if (parsed == null || parsed <= 0) { + throw StateError('$name must be a positive integer, got "$value"'); + } + return parsed; +} + +int _parseNonNegativeInt(String value, String name) { + final parsed = int.tryParse(value); + if (parsed == null || parsed < 0) { + throw StateError('$name must be a non-negative integer, got "$value"'); + } + return parsed; +} + +Never _printUsageAndExit(int code) { + stdout.writeln( + 'Usage: fvm flutter pub run scripts/sync_reliability.dart [updates] [cycles] ' + '[--imap-dbs=N] [--jmap-dbs=N]\n\n' + 'Defaults: updates=10 cycles=3 imap-dbs=1 jmap-dbs=1', + ); + exit(code); +} diff --git a/scripts/sync_reliability.sh b/scripts/sync_reliability.sh new file mode 100755 index 0000000..f0206de --- /dev/null +++ b/scripts/sync_reliability.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Starts an isolated Stalwart instance, runs the Dart sync reliability runner, +# then stops Stalwart. +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +command -v stalwart >/dev/null || { + echo "stalwart not in PATH — run inside nix develop" + exit 1 +} + +export STALWART_USER_B="${STALWART_USER_B:-alice@example.com}" +export STALWART_PASS_B="${STALWART_PASS_B:-secret}" +export STALWART_RANDOM_PORTS=1 +STALWART_TMPDIR="$(mktemp -d /tmp/stalwart-dev-sync-reliability-XXXXXX)" +export STALWART_TMPDIR + +# Pre-seed spam-filter version so Stalwart does not fetch it on first boot. +mkdir -p "$STALWART_TMPDIR" +sqlite3 "${STALWART_TMPDIR}/data.sqlite" \ + "CREATE TABLE IF NOT EXISTS s (k BLOB PRIMARY KEY, v BLOB NOT NULL); + INSERT OR REPLACE INTO s VALUES ('version.spam-filter', 'dev');" 2>/dev/null || true + +LOGFILE="${STALWART_TMPDIR}/stalwart.log" +rm -f "$LOGFILE" + +"$ROOT/stalwart-dev/start" >"$LOGFILE" 2>&1 & +STALWART_PID=$! + +cleanup() { + kill "$STALWART_PID" 2>/dev/null || true + wait "$STALWART_PID" 2>/dev/null || true +} +trap cleanup EXIT + +for _i in $(seq 1 20); do + # shellcheck source=/dev/null + [ -f "${STALWART_TMPDIR}/ports.env" ] && . "${STALWART_TMPDIR}/ports.env" + grep -E "Configuration build error|Build error for key|already in use" "$LOGFILE" >/dev/null 2>&1 && { + cat "$LOGFILE" + echo "Stalwart reported a startup error" + exit 1 + } + kill -0 "$STALWART_PID" 2>/dev/null || { + cat "$LOGFILE" + echo "Stalwart process died unexpectedly" + exit 1 + } + if [ -n "${STALWART_URL:-}" ] && + curl -s --max-time 1 -o /dev/null "${STALWART_URL}/.well-known/jmap" 2>/dev/null; then + break + fi + sleep 0.5 +done + +[ -n "${STALWART_URL:-}" ] || { + cat "$LOGFILE" + echo "Stalwart did not publish its chosen ports" + exit 1 +} + +curl -s --max-time 1 -o /dev/null "${STALWART_URL}/.well-known/jmap" || { + cat "$LOGFILE" + echo "Stalwart did not become ready" + exit 1 +} + +export STALWART_IMAP_HOST="127.0.0.1" +export STALWART_SMTP_HOST="127.0.0.1" + +echo "Stalwart ready — URL=${STALWART_URL} IMAP=:${STALWART_IMAP_PORT} SMTP=:${STALWART_SMTP_PORT}" + +if [ "$#" -gt 0 ]; then + SYNC_RELIABILITY_ARGS="$(printf '%s\n' "$@")" + export SYNC_RELIABILITY_ARGS +else + unset SYNC_RELIABILITY_ARGS || true +fi + +fvm flutter pub get --suppress-analytics +fvm flutter test test/integration/sync_reliability_runner_test.dart --reporter expanded --concurrency=1 --no-pub diff --git a/stalwart-dev/config.toml b/stalwart-dev/config.toml index 08793df..89b544e 100644 --- a/stalwart-dev/config.toml +++ b/stalwart-dev/config.toml @@ -48,15 +48,15 @@ type = "memory" [[directory."memory".principals]] class = "individual" -name = "alice@localhost" +name = "alice@example.com" secret = "secret" -email = ["alice@localhost"] +email = ["alice@example.com"] [[directory."memory".principals]] class = "individual" -name = "bob@localhost" +name = "bob@example.com" secret = "secret" -email = ["bob@localhost"] +email = ["bob@example.com"] [authentication.fallback-admin] user = "admin" @@ -70,3 +70,7 @@ allow-plain-text = true [session.auth] mechanisms = "[plain, login, oauthbearer, xoauth2]" allow-plain-text = true + +# Disable rate limiting for local development/testing. +[authentication.rate-limit] +enable = false diff --git a/stalwart-dev/integration_ui_test.sh b/stalwart-dev/integration_ui_test.sh index 0e52c34..23244cd 100755 --- a/stalwart-dev/integration_ui_test.sh +++ b/stalwart-dev/integration_ui_test.sh @@ -10,9 +10,9 @@ set -Eeuo pipefail _SCRIPT_START=$(date +%s%3N) ts() { echo "[$(( $(date +%s%3N) - _SCRIPT_START ))ms] $*"; } -export STALWART_USER_B="${STALWART_USER_B:-alice@localhost}" +export STALWART_USER_B="${STALWART_USER_B:-alice@example.com}" export STALWART_PASS_B="${STALWART_PASS_B:-secret}" -export STALWART_USER_C="${STALWART_USER_C:-bob@localhost}" +export STALWART_USER_C="${STALWART_USER_C:-bob@example.com}" export STALWART_PASS_C="${STALWART_PASS_C:-secret}" export STALWART_RANDOM_PORTS=1 STALWART_TMPDIR="$(mktemp -d /tmp/stalwart-dev-XXXXXX)" diff --git a/stalwart-dev/test.sh b/stalwart-dev/test.sh index d169d82..c7ac89b 100755 --- a/stalwart-dev/test.sh +++ b/stalwart-dev/test.sh @@ -4,9 +4,9 @@ set -Eeuo pipefail trap 'echo "Warning: A command failed ($0:$LINENO)"; exit 3' ERR -export STALWART_USER_B="${STALWART_USER_B:-alice@localhost}" +export STALWART_USER_B="${STALWART_USER_B:-alice@example.com}" export STALWART_PASS_B="${STALWART_PASS_B:-secret}" -export STALWART_USER_C="${STALWART_USER_C:-bob@localhost}" +export STALWART_USER_C="${STALWART_USER_C:-bob@example.com}" export STALWART_PASS_C="${STALWART_PASS_C:-secret}" export STALWART_RANDOM_PORTS=1 STALWART_TMPDIR="$(mktemp -d /tmp/stalwart-dev-XXXXXX)" diff --git a/test/integration/account_sync_manager_test.dart b/test/integration/account_sync_manager_test.dart index 0dbeb16..7b7856a 100644 --- a/test/integration/account_sync_manager_test.dart +++ b/test/integration/account_sync_manager_test.dart @@ -140,7 +140,7 @@ void main() { final fakeAccounts = _FakeAccounts()..password = pass; // Stalwart's memory directory authenticates by principal name ('alice'), - // not by email address ('alice@localhost'). connectImap() passes + // not by email address ('alice@example.com'). connectImap() passes // account.email as the IMAP login username, so use the bare name here. final account = Account( id: 'integration-test', diff --git a/test/integration/email_repository_imap_test.dart b/test/integration/email_repository_imap_test.dart index 6985b4b..e8ab4f0 100644 --- a/test/integration/email_repository_imap_test.dart +++ b/test/integration/email_repository_imap_test.dart @@ -4,7 +4,7 @@ // Environment variables (set by the runner script): // STALWART_IMAP_HOST, STALWART_IMAP_PORT // STALWART_SMTP_HOST, STALWART_SMTP_PORT -// STALWART_USER_B / STALWART_PASS_B (alice@localhost) +// STALWART_USER_B / STALWART_PASS_B (alice@example.com) import 'dart:convert'; import 'dart:io'; @@ -76,7 +76,7 @@ void main() { imapPort = int.parse(_env('STALWART_IMAP_PORT', '1430')); smtpHost = _env('STALWART_SMTP_HOST', '127.0.0.1'); smtpPort = int.parse(_env('STALWART_SMTP_PORT', '1025')); - userEmail = _env('STALWART_USER_B', 'alice@localhost'); + userEmail = _env('STALWART_USER_B', 'alice@example.com'); userPass = _env('STALWART_PASS_B', 'secret'); account = Account( id: 'test', diff --git a/test/integration/email_repository_jmap_test.dart b/test/integration/email_repository_jmap_test.dart index 0f68f9d..ca9a240 100644 --- a/test/integration/email_repository_jmap_test.dart +++ b/test/integration/email_repository_jmap_test.dart @@ -5,7 +5,7 @@ // STALWART_URL — JMAP base URL, e.g. http://127.0.0.1:8080 // STALWART_IMAP_HOST, STALWART_IMAP_PORT // STALWART_SMTP_PORT -// STALWART_USER_B / STALWART_PASS_B (alice@localhost) +// STALWART_USER_B / STALWART_PASS_B (alice@example.com) import 'dart:io'; @@ -68,7 +68,7 @@ void main() { imapHost = _env('STALWART_IMAP_HOST', '127.0.0.1'); imapPort = int.parse(_env('STALWART_IMAP_PORT', '1430')); smtpPort = _env('STALWART_SMTP_PORT', '1025'); - userEmail = _env('STALWART_USER_B', 'alice@localhost'); + userEmail = _env('STALWART_USER_B', 'alice@example.com'); userPass = _env('STALWART_PASS_B', 'secret'); account = Account( id: 'test-jmap', diff --git a/test/integration/imap_sync_test.dart b/test/integration/imap_sync_test.dart index 997fb14..c5a2f9a 100644 --- a/test/integration/imap_sync_test.dart +++ b/test/integration/imap_sync_test.dart @@ -2,8 +2,8 @@ // Run via: stalwart-dev/test.sh (sets the env vars below) // // STALWART_IMAP_HOST, STALWART_IMAP_PORT, STALWART_SMTP_HOST, STALWART_SMTP_PORT -// STALWART_USER_B / STALWART_PASS_B (alice@localhost) -// STALWART_USER_C / STALWART_PASS_C (bob@localhost) +// STALWART_USER_B / STALWART_PASS_B (alice@example.com) +// STALWART_USER_C / STALWART_PASS_C (bob@example.com) import 'dart:io'; diff --git a/test/integration/mailbox_repository_imap_test.dart b/test/integration/mailbox_repository_imap_test.dart index 54d0894..575daf0 100644 --- a/test/integration/mailbox_repository_imap_test.dart +++ b/test/integration/mailbox_repository_imap_test.dart @@ -3,7 +3,7 @@ // // Environment variables (set by the runner script): // STALWART_IMAP_HOST, STALWART_IMAP_PORT -// STALWART_USER_B / STALWART_PASS_B (alice@localhost) +// STALWART_USER_B / STALWART_PASS_B (alice@example.com) import 'dart:io'; @@ -44,7 +44,7 @@ void main() { configureSqliteForTests(); imapHost = _env('STALWART_IMAP_HOST', '127.0.0.1'); imapPort = int.parse(_env('STALWART_IMAP_PORT', '1430')); - userEmail = _env('STALWART_USER_B', 'alice@localhost'); + userEmail = _env('STALWART_USER_B', 'alice@example.com'); userPass = _env('STALWART_PASS_B', 'secret'); account = Account( id: 'test', diff --git a/test/integration/sync_reliability_runner_test.dart b/test/integration/sync_reliability_runner_test.dart new file mode 100644 index 0000000..0077cf6 --- /dev/null +++ b/test/integration/sync_reliability_runner_test.dart @@ -0,0 +1,20 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; + +import '../../scripts/sync_reliability.dart' as reliability; + +void main() { + test( + 'sync reliability script runner', + timeout: Timeout.none, + () async { + final rawArgs = Platform.environment['SYNC_RELIABILITY_ARGS']; + final args = rawArgs == null || rawArgs.isEmpty + ? const [] + : const LineSplitter().convert(rawArgs); + await reliability.runSyncReliability(args); + }, + ); +} diff --git a/test/unit/email_repository_impl_test.dart b/test/unit/email_repository_impl_test.dart index c85c131..d6284f8 100644 --- a/test/unit/email_repository_impl_test.dart +++ b/test/unit/email_repository_impl_test.dart @@ -848,7 +848,7 @@ void main() { expect(changes.first.changeType, 'flag_flagged'); }); - test('moveEmail enqueues move change and removes email from local DB', + test('moveEmail enqueues move change and updates local mailbox path', () async { final r = _makeRepos(); await seedJmapEmail(r.db, r.accounts); @@ -859,7 +859,9 @@ void main() { expect(changes.first.changeType, 'move'); expect(changes.first.payload, contains('mbx2')); - expect(await r.emails.getEmail('jmap-1:e1'), isNull); + final email = await r.emails.getEmail('jmap-1:e1'); + expect(email, isNotNull); + expect(email?.mailboxPath, 'mbx2'); }); test('deleteEmail enqueues delete change and removes email from local DB', @@ -967,7 +969,7 @@ void main() { r.db, r.accounts, changeType: 'move', - payload: '{"dest":"mbx2"}', + payload: '{"src":"mbx1","dest":"mbx2"}', ); await r.emails.flushPendingChanges('jmap-1', 'pw'); @@ -1320,22 +1322,54 @@ void main() { sessionStatus, ); } + // First API call is Identity/get; respond with a single identity. + if (req.body.contains('Identity/get')) { + return http.Response( + jsonEncode({ + 'sessionState': 's1', + 'methodResponses': [ + [ + 'Identity/get', + { + 'accountId': 'acct1', + 'state': 'id1', + 'list': [ + {'id': 'identity1', 'email': 'alice@example.com'}, + ], + }, + 'i', + ], + ], + }), + apiStatus, + ); + } + if (req.body.contains('Email/set')) { + return http.Response( + jsonEncode({ + 'sessionState': 's1', + 'methodResponses': [ + [ + 'Email/set', + emailSetResult ?? + { + 'accountId': 'acct1', + 'newState': 'est2', + 'created': { + 'em1': {'id': 'newEmailId1'}, + }, + }, + '0', + ], + ], + }), + apiStatus, + ); + } return http.Response( jsonEncode({ 'sessionState': 's1', 'methodResponses': [ - [ - 'Email/set', - emailSetResult ?? - { - 'accountId': 'acct1', - 'newState': 'est2', - 'created': { - 'em1': {'id': 'newEmailId1'}, - }, - }, - '0', - ], [ 'EmailSubmission/set', submissionResult ?? @@ -1431,22 +1465,56 @@ void main() { 200, ); } - capturedBody = jsonDecode(req.body) as Map; + if (req.body.contains('Email/set')) { + capturedBody = jsonDecode(req.body) as Map; + } + // First API call is Identity/get; respond with a single identity. + if (req.body.contains('Identity/get')) { + return http.Response( + jsonEncode({ + 'sessionState': 's1', + 'methodResponses': [ + [ + 'Identity/get', + { + 'accountId': 'acct1', + 'state': 'id1', + 'list': [ + {'id': 'identity1', 'email': 'alice@example.com'}, + ], + }, + 'i', + ], + ], + }), + 200, + ); + } + if (req.body.contains('Email/set')) { + return http.Response( + jsonEncode({ + 'sessionState': 's1', + 'methodResponses': [ + [ + 'Email/set', + { + 'accountId': 'acct1', + 'newState': 'est2', + 'created': { + 'em1': {'id': 'newId'}, + }, + }, + '0', + ], + ], + }), + 200, + ); + } return http.Response( jsonEncode({ 'sessionState': 's1', 'methodResponses': [ - [ - 'Email/set', - { - 'accountId': 'acct1', - 'newState': 'est2', - 'created': { - 'em1': {'id': 'newId'}, - }, - }, - '0', - ], [ 'EmailSubmission/set', {