feat: add chaos monkey backend test running daily in CI (#448)

Introduces a headless chaos monkey test that drives the email repository through 30 rounds of random IMAP/SMTP operations against a live Stalwart instance to surface crashes and data-corruption bugs. - test/backend/chaos_monkey_test.dart: random sync, send, flag, delete, and flush operations; seed logged for reproducibility (CHAOS_SEED env) - ci/main.go: ChaosMonkeyBackend Dagger function (same pattern as TestBackend) - Taskfile.yml: chaos-monkey-backend task - .forgejo/workflows/chaos-monkey.yml: daily cron at 03:00 UTC + workflow_dispatch Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-05 22:40:07 +02:00
parent 8446b05601
commit 3519be1151
4 changed files with 245 additions and 0 deletions
@@ -0,0 +1,20 @@
+name: Chaos Monkey
+
+on:
+  schedule:
+    - cron: '0 3 * * *'
+  workflow_dispatch:
+
+jobs:
+  chaos-monkey-backend:
+    name: Chaos Monkey (backend)
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Dagger Remote Engine
+        env:
+          SOPS_AGE_KEY: ${{ secrets.SOPS_AGE_KEY }}
+        run: scripts/setup_dagger_remote.sh
+      - name: Run backend chaos monkey
+        run: task chaos-monkey-backend
@@ -722,6 +722,11 @@ tasks:
    cmds:
      - fvm flutter test test/screenshot_automation_test.dart --update-goldens

+  chaos-monkey-backend:
+    desc: Chaos monkey — random IMAP/SMTP ops against Stalwart (via Dagger, headless)
+    cmds:
+      - timeout --kill-after=10 600 dagger call --progress=plain -q -m ci --source=. chaos-monkey-backend
+
  check:
    desc: Full check suite — unit tests first, then integration (merges coverage), then gate
    deps: [analyze, build-linux, test]
@@ -565,6 +565,16 @@ func (m *Ci) TestSyncReliability(ctx context.Context) (string, error) {
 		Stdout(ctx)
 }

+// ChaosMonkeyBackend runs random IMAP/SMTP operations against Stalwart to surface crashes.
+func (m *Ci) ChaosMonkeyBackend(ctx context.Context) (string, error) {
+	return m.WithStalwart(m.setup(m.backendSrc())).
+		WithExec([]string{"/bin/bash", "-c",
+			`tmp=$(mktemp); trap 'rm -f "$tmp"' EXIT; ` +
+				`flutter test test/backend/chaos_monkey_test.dart --reporter expanded --concurrency=1 --no-pub >"$tmp" 2>&1 || { cat "$tmp"; exit 1; }; ` +
+				`grep -E '^All [0-9]+ tests passed' "$tmp" || tail -1 "$tmp"`}).
+		Stdout(ctx)
+}
+
 // Check runs the full check suite.
 func (m *Ci) Check(ctx context.Context) (string, error) {
 	ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
@@ -0,0 +1,210 @@
+// Chaos monkey test — drives the email repository through random operations
+// against a live Stalwart instance to surface crashes and data-corruption bugs.
+//
+// Run via: stalwart-dev/test.sh
+//
+// Environment variables:
+//   STALWART_IMAP_HOST, STALWART_IMAP_PORT
+//   STALWART_SMTP_HOST, STALWART_SMTP_PORT
+//   STALWART_USER_B / STALWART_PASS_B  (alice@example.com)
+//   CHAOS_ROUNDS  (default: 30) — number of random operations to perform
+//   CHAOS_SEED    (default: current epoch ms) — seed for reproducibility
+
+import 'dart:io';
+import 'dart:math';
+
+import 'package:enough_mail/enough_mail.dart';
+import 'package:sharedinbox/core/models/account.dart';
+import 'package:sharedinbox/core/models/email.dart' as email_model;
+import 'package:sharedinbox/data/db/database.dart' hide Account;
+import 'package:sharedinbox/data/repositories/account_repository_impl.dart';
+import 'package:sharedinbox/data/repositories/email_repository_impl.dart';
+import 'package:test/test.dart';
+
+import '../unit/account_repository_impl_test.dart' show MapSecureStorage;
+import '../unit/db_test_helper.dart';
+
+String _env(String key, [String fallback = '']) =>
+    Platform.environment[key] ?? fallback;
+
+Future<ImapClient> _imapConnectPlain(
+  Account account,
+  String username,
+  String password,
+) async {
+  final client = ImapClient(defaultResponseTimeout: const Duration(seconds: 20));
+  await client.connectToServer(account.imapHost, account.imapPort, isSecure: false);
+  await client.login(username, password);
+  return client;
+}
+
+Future<SmtpClient> _smtpConnectPlain(
+  Account account,
+  String username,
+  String password,
+) async {
+  final atIndex = account.email.lastIndexOf('@');
+  final domain =
+      atIndex != -1 ? account.email.substring(atIndex + 1) : account.smtpHost;
+  final client = SmtpClient(domain);
+  await client.connectToServer(account.smtpHost, account.smtpPort, isSecure: false);
+  await client.ehlo();
+  await client.authenticate(username, password);
+  return client;
+}
+
+Future<void> _clearMailbox(
+  Account account,
+  String userEmail,
+  String userPass,
+  String mailboxPath,
+) async {
+  final client = await _imapConnectPlain(account, userEmail, userPass);
+  try {
+    final box = await client.selectMailboxByPath(mailboxPath);
+    if (box.messagesExists == 0) return;
+    final result = await client.uidSearchMessages(searchCriteria: 'ALL');
+    final uids = result.matchingSequence?.toList() ?? [];
+    if (uids.isEmpty) return;
+    final seq = MessageSequence.fromIds(uids, isUid: true);
+    await client.uidMarkDeleted(seq);
+    await client.uidExpunge(seq);
+  } finally {
+    await client.logout();
+  }
+}
+
+void main() {
+  late String imapHost;
+  late int imapPort;
+  late String smtpHost;
+  late int smtpPort;
+  late String userEmail;
+  late String userPass;
+  late Account account;
+  late AppDatabase db;
+  late EmailRepositoryImpl emails;
+
+  setUpAll(configureSqliteForTests);
+
+  setUp(() async {
+    imapHost = _env('STALWART_IMAP_HOST', '127.0.0.1');
+    imapPort = int.parse(_env('STALWART_IMAP_PORT', '1430'));
+    smtpHost = _env('STALWART_SMTP_HOST', '127.0.0.1');
+    smtpPort = int.parse(_env('STALWART_SMTP_PORT', '1025'));
+    userEmail = _env('STALWART_USER_B', 'alice@example.com');
+    userPass = _env('STALWART_PASS_B', 'secret');
+
+    account = Account(
+      id: 'chaos',
+      displayName: 'Chaos',
+      email: userEmail,
+      imapHost: imapHost,
+      imapPort: imapPort,
+      imapSsl: false,
+      smtpHost: smtpHost,
+      smtpPort: smtpPort,
+    );
+
+    db = openTestDatabase();
+    final secureStorage = MapSecureStorage();
+    final accounts = AccountRepositoryImpl(db, secureStorage);
+    await accounts.addAccount(account, userPass);
+    emails = EmailRepositoryImpl(
+      db,
+      accounts,
+      imapConnect: _imapConnectPlain,
+      smtpConnect: _smtpConnectPlain,
+    );
+
+    await _clearMailbox(account, userEmail, userPass, 'INBOX');
+  });
+
+  tearDown(() => db.close());
+
+  test('chaos monkey — random operations do not crash the repository', () async {
+    final seedStr = _env('CHAOS_SEED', '');
+    final seed = seedStr.isEmpty
+        ? DateTime.now().millisecondsSinceEpoch
+        : int.parse(seedStr);
+    final rounds = int.parse(_env('CHAOS_ROUNDS', '30'));
+    final rng = Random(seed);
+
+    print('chaos-monkey: seed=$seed rounds=$rounds');
+
+    // Seed INBOX with a few messages so early rounds have something to act on.
+    for (var i = 0; i < 3; i++) {
+      await emails.sendEmail(
+        account.id,
+        email_model.EmailDraft(
+          from: email_model.EmailAddress(name: 'Chaos', email: userEmail),
+          to: [email_model.EmailAddress(email: userEmail)],
+          cc: [],
+          subject: 'seed-$i',
+          body: 'Seed email $i.',
+        ),
+      );
+    }
+    await emails.syncEmails(account.id, 'INBOX');
+
+    for (var round = 0; round < rounds; round++) {
+      final action = rng.nextInt(8);
+      print('chaos-monkey: round=$round action=$action');
+
+      switch (action) {
+        case 0: // sync INBOX
+          await emails.syncEmails(account.id, 'INBOX');
+
+        case 1: // sync Sent
+          await emails.syncEmails(account.id, 'Sent');
+
+        case 2: // send email to self
+          final subject = 'chaos-$round-${rng.nextInt(9999)}';
+          await emails.sendEmail(
+            account.id,
+            email_model.EmailDraft(
+              from: email_model.EmailAddress(name: 'Chaos', email: userEmail),
+              to: [email_model.EmailAddress(email: userEmail)],
+              cc: [],
+              subject: subject,
+              body: 'Round $round. Value: ${rng.nextInt(1000000)}.',
+            ),
+          );
+
+        case 3: // mark random email seen
+          final inbox = await emails.observeEmails(account.id, 'INBOX').first;
+          if (inbox.isEmpty) break;
+          final e = inbox[rng.nextInt(inbox.length)];
+          await emails.setFlag(e.id, seen: true);
+
+        case 4: // mark random email unseen
+          final inbox = await emails.observeEmails(account.id, 'INBOX').first;
+          if (inbox.isEmpty) break;
+          final e = inbox[rng.nextInt(inbox.length)];
+          await emails.setFlag(e.id, seen: false);
+
+        case 5: // toggle flagged on random email
+          final inbox = await emails.observeEmails(account.id, 'INBOX').first;
+          if (inbox.isEmpty) break;
+          final e = inbox[rng.nextInt(inbox.length)];
+          await emails.setFlag(e.id, flagged: !e.isFlagged);
+
+        case 6: // flush pending changes to server
+          final flushed = await emails.flushPendingChanges(account.id, userPass);
+          print('chaos-monkey: flushed $flushed pending changes');
+
+        case 7: // delete random email
+          final inbox = await emails.observeEmails(account.id, 'INBOX').first;
+          if (inbox.isEmpty) break;
+          final e = inbox[rng.nextInt(inbox.length)];
+          await emails.deleteEmail(e.id);
+      }
+    }
+
+    // Final flush and sync to confirm the server is in a consistent state.
+    final flushed = await emails.flushPendingChanges(account.id, userPass);
+    print('chaos-monkey: final flush flushed=$flushed');
+    final result = await emails.syncEmails(account.id, 'INBOX');
+    print('chaos-monkey: final sync fetched=${result.fetched}');
+  });
+}