From 8cbe8c01bb891d80899e51dafcc90311893777c3 Mon Sep 17 00:00:00 2001 From: Gemini CLI Date: Sun, 17 May 2026 16:01:42 +0200 Subject: [PATCH] ci: use idiomatic Dagger service bindings for Stalwart Refactor the CI pipeline to use WithServiceBinding for the Stalwart mail server, replacing legacy shell scripts and manual port management. Introduces pre-seeded data for the Stalwart service to avoid network hits and improves headless UI testing with Xvfb. --- .daggerignore | 38 +++++++++++++++++--- Taskfile.yml | 27 ++++++++------- ci/main.go | 82 +++++++++++++++++++++++++++++++------------- stalwart-dev/start | 41 ++++++++++++---------- stalwart-dev/test.sh | 9 ++--- 5 files changed, 131 insertions(+), 66 deletions(-) diff --git a/.daggerignore b/.daggerignore index 33abc9c..21a4839 100644 --- a/.daggerignore +++ b/.daggerignore @@ -1,13 +1,23 @@ # Dagger context ignore file. -# Since we use explicit inclusion in ci/main.go (Base function), -# we only need to ignore large or sensitive directories here to -# avoid unnecessary upload overhead to the Dagger engine. +# Version control .git/ +.gitmodules + +# Build artifacts and tools build/ .dart_tool/ .fvm/ .pub-cache/ +.dart-cli-completion/ +.dartServer/ +.direnv/ +.dotnet/ +.rustup/ +.task/ +.vscode/ +.vscode-server/ +coverage/ node_modules/ ios/Pods/ macos/Pods/ @@ -15,6 +25,26 @@ linux/flutter/ephemeral/ website/public/ website/resources/ -# Sensitive files +# Sensitive or system files .env* .ssh/ +.local/ +.cache/ +.config/ +.atuin/ +.gemini/ +.bash* +.profile +.zsh* +.zcompdump +.tmux.conf +.lesshst +.metadata +.emulator_console_auth_token +snap/ +.gitconfig +.flutter-plugins-dependencies +.flutter +.forgejo +.fvmrc +.envrc diff --git a/Taskfile.yml b/Taskfile.yml index b8317bf..f5170de 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -172,22 +172,25 @@ tasks: - fvm flutter test test-backend: - desc: Backend tests against a local Stalwart mail server - deps: [_flutter-check] - sources: - - lib/**/*.dart - - test/backend/**/*.dart + desc: Backend tests against a local Stalwart mail server (via Dagger) cmds: - - stalwart-dev/test.sh + - dagger call -m ci test-backend --source . integration-ui: - desc: UI E2E tests on Linux via Xvfb — headless, no emulator needed - deps: [_preflight, _linux-deps-check, _pub-get] - sources: - - lib/**/*.dart - - integration_test/app_e2e_test.dart + desc: UI E2E tests on Linux via Xvfb — headless, no emulator needed (via Dagger) cmds: - - stalwart-dev/integration_ui_test.sh + - dagger call -m ci test-integration --source . + + sync-reliability: + desc: Run sync reliability runner (via Dagger) + cmds: + - dagger call -m ci test-sync-reliability --source . + + stalwart: + desc: Start a Stalwart instance for local development (via Dagger) + cmds: + - echo "Starting Stalwart on default ports (JMAP=8080, IMAP=1430, SMTP=1025, SIEVE=4190)" + - dagger call -m ci stalwart up --ports 8080:8080 --ports 1430:1430 --ports 1025:1025 --ports 4190:4190 integration-android: desc: UI integration tests on a connected Android emulator (Stalwart on host, emulator reaches it via 10.0.2.2) diff --git a/ci/main.go b/ci/main.go index d699d39..71c386c 100644 --- a/ci/main.go +++ b/ci/main.go @@ -41,7 +41,7 @@ func (m *Ci) Base() *dagger.Container { return dag.Container(). From("ghcr.io/cirruslabs/flutter:3.41.6"). WithExec([]string{"apt-get", "update"}). - WithExec([]string{"apt-get", "install", "-y", "clang", "cmake", "ninja-build", "pkg-config", "libgtk-3-dev", "liblzma-dev", "libsecret-1-dev", "libgcrypt20-dev", "libjsoncpp-dev", "sqlite3", "curl", "python3", "iproute2", "netcat-openbsd"}). + WithExec([]string{"apt-get", "install", "-y", "clang", "cmake", "ninja-build", "pkg-config", "libgtk-3-dev", "liblzma-dev", "libsecret-1-dev", "libgcrypt20-dev", "libjsoncpp-dev", "sqlite3", "curl", "python3", "iproute2", "netcat-openbsd", "xvfb", "libosmesa6", "libgles2-mesa", "libegl1"}). WithMountedCache("/root/.pub-cache", dag.CacheVolume("flutter-pub-cache")). WithMountedCache("/root/.gradle", dag.CacheVolume("gradle-cache")). WithEnvVariable("PUB_CACHE", "/root/.pub-cache"). @@ -75,15 +75,35 @@ func (m *Ci) Stalwart() *dagger.Service { return dag.Container(). From("stalwartlabs/stalwart:v0.14.1"). WithFile("/etc/stalwart/config.toml", config). - // Create data dir in /tmp where permissions are usually more relaxed. + // Pre-seed data directory and spam-filter version to avoid network hits on startup. WithExec([]string{"/bin/sh", "-c", "mkdir -p /tmp/stalwart && chmod 777 /tmp/stalwart"}). + WithExec([]string{"sqlite3", "/tmp/stalwart/data.sqlite", "CREATE TABLE IF NOT EXISTS s (k BLOB PRIMARY KEY, v BLOB NOT NULL); INSERT OR REPLACE INTO s VALUES ('version.spam-filter', 'dev');"}). WithExposedPort(8080). // JMAP WithExposedPort(1430). // IMAP WithExposedPort(1025). // SMTP WithExposedPort(4190). // ManageSieve + // Explicitly run the binary with the config path to avoid bootstrap mode. + WithEntrypoint([]string{"stalwart", "--config", "/etc/stalwart/config.toml"}). AsService() } +// Helper to bind Stalwart service and set up environment variables for tests +func (m *Ci) WithStalwart(container *dagger.Container) *dagger.Container { + stalwart := m.Stalwart() + return container. + WithServiceBinding("stalwart", stalwart). + WithEnvVariable("STALWART_IMAP_HOST", "stalwart"). + WithEnvVariable("STALWART_SMTP_HOST", "stalwart"). + WithEnvVariable("STALWART_URL", "http://stalwart:8080"). + WithEnvVariable("STALWART_IMAP_PORT", "1430"). + WithEnvVariable("STALWART_SMTP_PORT", "1025"). + WithEnvVariable("STALWART_SIEVE_PORT", "4190"). + WithEnvVariable("STALWART_USER_B", "alice@example.com"). + WithEnvVariable("STALWART_PASS_B", "secret"). + WithEnvVariable("STALWART_USER_C", "bob@example.com"). + WithEnvVariable("STALWART_PASS_C", "secret") +} + // Setup environment: pub get and build_runner func (m *Ci) Setup() *dagger.Container { return m.Base(). @@ -129,15 +149,37 @@ func (m *Ci) CheckMocks(ctx context.Context) (string, error) { // Run coverage check func (m *Ci) Coverage(ctx context.Context) (string, error) { return m.Setup(). - WithExec([]string{"flutter", "test", "test/unit", "--coverage"}). + WithExec([]string{"flutter", "test", "test/unit", "--coverage", "--reporter", "expanded"}). WithExec([]string{"dart", "scripts/check_coverage.dart"}). Stdout(ctx) } +// Run backend tests (IMAP/JMAP sync logic) +func (m *Ci) TestBackend(ctx context.Context) (string, error) { + return m.WithStalwart(m.Setup()). + WithExec([]string{"flutter", "test", "--concurrency=1", "--reporter", "expanded", "test/backend"}). + Stdout(ctx) +} + +// Run UI integration tests via Xvfb +func (m *Ci) TestIntegration(ctx context.Context) (string, error) { + return m.WithStalwart(m.Setup()). + // Use xvfb-run for simpler X11 management. + // LIBGL_ALWAYS_SOFTWARE=1 ensures software rendering for Flutter in headless environments. + WithEnvVariable("LIBGL_ALWAYS_SOFTWARE", "1"). + WithExec([]string{"xvfb-run", "-s", "-screen 0 1280x720x24", "flutter", "test", "integration_test/", "-d", "linux"}). + Stdout(ctx) +} + +// Run sync reliability runner +func (m *Ci) TestSyncReliability(ctx context.Context) (string, error) { + return m.WithStalwart(m.Setup()). + WithExec([]string{"flutter", "test", "test/backend/sync_reliability_test.dart", "--reporter", "expanded", "--concurrency=1"}). + Stdout(ctx) +} + // Full check suite (equivalent to task check) func (m *Ci) Check(ctx context.Context) (string, error) { - setup := m.Setup() - // Hygiene & Layers if _, err := m.CheckHygiene(ctx); err != nil { return "Hygiene check failed", err @@ -146,6 +188,8 @@ func (m *Ci) Check(ctx context.Context) (string, error) { return "Layer check failed", err } + setup := m.Setup() + // Format (Running after Setup/pub get ensures package resolution context) if _, err := setup.WithExec([]string{"dart", "format", "--output=none", "--set-exit-if-changed", "lib", "test"}).Stdout(ctx); err != nil { return "Format check failed", err @@ -163,29 +207,19 @@ func (m *Ci) Check(ctx context.Context) (string, error) { return coverage, err } - // Run backend tests (requires Stalwart Service) - stalwart := m.Stalwart() - testBackend, err := setup. - WithServiceBinding("stalwart", stalwart). - WithEnvVariable("STALWART_IMAP_HOST", "stalwart"). - WithEnvVariable("STALWART_SMTP_HOST", "stalwart"). - WithEnvVariable("STALWART_URL", "http://stalwart:8080"). - WithEnvVariable("STALWART_IMAP_PORT", "1430"). - WithEnvVariable("STALWART_SMTP_PORT", "1025"). - WithEnvVariable("STALWART_SIEVE_PORT", "4190"). - WithEnvVariable("STALWART_USER_B", "alice@example.com"). - WithEnvVariable("STALWART_PASS_B", "secret"). - WithEnvVariable("STALWART_USER_C", "bob@example.com"). - WithEnvVariable("STALWART_PASS_C", "secret"). - // Wait for Stalwart to be ready before running tests. - WithExec([]string{"/bin/bash", "-c", "for i in {1..30}; do nc -z stalwart 1430 && echo 'Stalwart is ready' && break; echo 'Waiting for Stalwart...'; sleep 1; done"}). - WithExec([]string{"flutter", "test", "--concurrency=1", "test/backend"}). - Stdout(ctx) + // Run backend tests + testBackend, err := m.TestBackend(ctx) if err != nil { return testBackend, err } - return fmt.Sprintf("All checks passed!\n\nAnalysis:\n%s\n\n%s\n\nBackend Tests:\n%s\n", analyze, coverage, testBackend), nil + // Run UI integration tests + testIntegration, err := m.TestIntegration(ctx) + if err != nil { + return testIntegration, err + } + + return fmt.Sprintf("All checks passed!\n\nAnalysis:\n%s\n\n%s\n\nBackend Tests:\n%s\n\nIntegration Tests:\n%s\n", analyze, coverage, testBackend, testIntegration), nil } // Generate build history Hugo content by scanning the remote server diff --git a/stalwart-dev/start b/stalwart-dev/start index 23bcf3c..09efcbd 100755 --- a/stalwart-dev/start +++ b/stalwart-dev/start @@ -6,16 +6,6 @@ # STALWART_TMPDIR/ports.env for other scripts to source. set -euo pipefail -command -v stalwart >/dev/null || { - echo "stalwart not in PATH — run inside nix develop" - exit 1 -} - -command -v ss >/dev/null || { - echo "ss not in PATH — cannot verify Stalwart ports" - exit 1 -} - if [ "${STALWART_RANDOM_PORTS:-0}" = "1" ] || [ "${STALWART_PORT:-0}" = "0" ]; then command -v python3 >/dev/null || { echo "python3 not in PATH — cannot choose random Stalwart ports" @@ -61,17 +51,30 @@ export STALWART_SIEVE_PORT=${STALWART_SIEVE_PORT} export STALWART_URL=${STALWART_URL} EOF +# Find a container runtime +if command -v podman >/dev/null 2>&1; then + RUNTIME="podman" +elif command -v docker >/dev/null 2>&1; then + RUNTIME="docker" +else + echo "No container runtime (podman or docker) found" >&2 + exit 1 +fi + echo "Stalwart ports: JMAP=${STALWART_PORT} IMAP=${STALWART_IMAP_PORT} SMTP=${STALWART_SMTP_PORT} SIEVE=${STALWART_SIEVE_PORT}" >&2 -echo "Stalwart is running in the foreground. Press Ctrl+C to stop." >&2 +echo "Stalwart is running in a container (${RUNTIME}). Press Ctrl+C to stop." >&2 echo "Connection info written to ${TMPDIR}/ports.env" >&2 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" -sed -e "s|0.0.0.0:8080|0.0.0.0:${STALWART_PORT}|" \ - -e "s|0.0.0.0:1430|0.0.0.0:${STALWART_IMAP_PORT}|" \ - -e "s|0.0.0.0:1025|0.0.0.0:${STALWART_SMTP_PORT}|" \ - -e "s|0.0.0.0:4190|0.0.0.0:${STALWART_SIEVE_PORT}|" \ - -e "s|/tmp/stalwart|${TMPDIR}|" \ - "${REPO_ROOT}/stalwart-dev/config.toml" >"${TMPDIR}/config.toml" - -exec stalwart --config "${TMPDIR}/config.toml" +# Run Stalwart in container, mapping the random host ports to the fixed container ports. +# We mount the config.toml and use /tmp/stalwart for data (mapped to our local TMPDIR). +exec "${RUNTIME}" run --rm -i \ + -p "${STALWART_PORT}:8080" \ + -p "${STALWART_IMAP_PORT}:1430" \ + -p "${STALWART_SMTP_PORT}:1025" \ + -p "${STALWART_SIEVE_PORT}:4190" \ + -v "${REPO_ROOT}/stalwart-dev/config.toml:/etc/stalwart/config.toml:ro" \ + -v "${TMPDIR}:/tmp/stalwart:rw" \ + docker.io/stalwartlabs/stalwart:v0.14.1 \ + stalwart --config /etc/stalwart/config.toml diff --git a/stalwart-dev/test.sh b/stalwart-dev/test.sh index 9671f63..6170974 100755 --- a/stalwart-dev/test.sh +++ b/stalwart-dev/test.sh @@ -12,11 +12,6 @@ export STALWART_RANDOM_PORTS=1 STALWART_TMPDIR="$(mktemp -d /tmp/stalwart-dev-XXXXXX)" export STALWART_TMPDIR -command -v stalwart >/dev/null || { - echo "stalwart not in PATH — run inside nix develop" - exit 1 -} - # Kill any stalwart left over from a previous run. pkill -x stalwart 2>/dev/null && sleep 0.5 || true @@ -34,8 +29,8 @@ tmp=$(mktemp) STALWART_PID=$! trap 'kill "$STALWART_PID" 2>/dev/null || true; wait "$STALWART_PID" 2>/dev/null || true; rm -f "$tmp"' EXIT -# Wait until Stalwart is accepting connections (up to 10 s). -for _i in $(seq 1 20); do +# Wait until Stalwart is accepting connections (up to 60 s). +for _i in $(seq 1 120); do # shellcheck source=/dev/null [ -f "${STALWART_TMPDIR}/ports.env" ] && . "${STALWART_TMPDIR}/ports.env" grep -E "already in use" "$LOGFILE" >/dev/null 2>&1 && {