From 8718339b4eb5aac108c77cce7d546f3e44f96567 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bot=20of=20Thomas=20G=C3=BCttler?= Date: Fri, 5 Jun 2026 11:49:30 +0200 Subject: [PATCH] ci: add timeouts to all CI/CD jobs, Dagger tasks, and runner scripts (#432) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #415 ## Summary - Adds missing `timeout-minutes` to `ci.yml` (`check` job, 60 min) and `windows-nightly.yml` (90 min, ready for when the Windows runner is registered) - Wraps `ssh-keyscan` and `ssh -f -N -L` tunnel creation in `setup_dagger_remote.sh` with `timeout 30`; emits a `::warning::` annotation when either takes more than 10 s - Adds `timeout --kill-after=10 ` to all bare `dagger call` invocations in `Taskfile.yml`: 600 s for test/query tasks, 1800 s for build/deploy tasks, 60 s for `ci-graph`; `stalwart` and `check-dagger` (already protected) left untouched - Adds `timeout --kill-after=10 2400` per attempt in `run_firebase_test.sh`; emits `::warning::` on exit 124 instead of silently retrying ## Test plan - CI passes on this PR (the `check` job now has `timeout-minutes: 60` and will self-enforce) - All `dagger call` lines in `Taskfile.yml` now have a `timeout` prefix (visible in the diff) - `setup_dagger_remote.sh` logic is unchanged — only the two network calls are wrapped Co-authored-by: Thomas SharedInbox Reviewed-on: https://codeberg.org/guettli/sharedinbox/pulls/432 --- .forgejo/workflows/ci.yml | 1 + .forgejo/workflows/windows-nightly.yml | 1 + Taskfile.yml | 22 +++++++++++----------- scripts/run_firebase_test.sh | 6 +++++- scripts/setup_dagger_remote.sh | 14 ++++++++++++-- 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 6e5cc8b..e25cbc5 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -4,6 +4,7 @@ jobs: check: name: Full Project Check runs-on: ubuntu-latest + timeout-minutes: 60 steps: - uses: actions/checkout@v4 - name: Setup Dagger Remote Engine diff --git a/.forgejo/workflows/windows-nightly.yml b/.forgejo/workflows/windows-nightly.yml index f0f29bb..3cae732 100644 --- a/.forgejo/workflows/windows-nightly.yml +++ b/.forgejo/workflows/windows-nightly.yml @@ -10,6 +10,7 @@ jobs: # Disabled until a self-hosted runner with label "windows-runner" is registered. name: Build & Deploy Windows (Nightly) runs-on: windows-runner + timeout-minutes: 90 if: false steps: diff --git a/Taskfile.yml b/Taskfile.yml index b39632d..5a901e8 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -177,17 +177,17 @@ tasks: test-backend: desc: Backend tests against a local Stalwart mail server (via Dagger) cmds: - - dagger call --progress=plain -q -m ci --source=. test-backend + - timeout --kill-after=10 600 dagger call --progress=plain -q -m ci --source=. test-backend integration-ui: desc: UI E2E tests on Linux via Xvfb — headless, no emulator needed (via Dagger) cmds: - - dagger call --progress=plain -q -m ci --source=. test-integration + - timeout --kill-after=10 600 dagger call --progress=plain -q -m ci --source=. test-integration sync-reliability: desc: Run sync reliability runner (via Dagger) cmds: - - dagger call --progress=plain -q -m ci --source=. test-sync-reliability + - timeout --kill-after=10 600 dagger call --progress=plain -q -m ci --source=. test-sync-reliability test-android-firebase: desc: Build Android debug APKs and run instrumented tests on Firebase Test Lab (via Dagger) @@ -202,7 +202,7 @@ tasks: ci-graph: desc: Print a Mermaid diagram of the CI pipeline — paste into mermaid.live or any Markdown renderer cmds: - - dagger call --progress=plain -q -m ci --source=. graph + - timeout --kill-after=10 60 dagger call --progress=plain -q -m ci --source=. graph stalwart: desc: Start a Stalwart instance for local development (via Dagger) @@ -218,13 +218,13 @@ tasks: - sh: test -n "$SSH_KNOWN_HOSTS" msg: "SSH_KNOWN_HOSTS is not set" cmds: - - HASH=$(git rev-parse --short HEAD) && scripts/silent_on_success.sh dagger call --progress=plain -q -m ci --source=. deploy-linux --ssh-key env:SSH_PRIVATE_KEY --known-hosts env:SSH_KNOWN_HOSTS --ssh-user "$SSH_USER" --ssh-host "$SSH_HOST" --commit-hash "$HASH" + - HASH=$(git rev-parse --short HEAD) && scripts/silent_on_success.sh timeout --kill-after=10 1800 dagger call --progress=plain -q -m ci --source=. deploy-linux --ssh-key env:SSH_PRIVATE_KEY --known-hosts env:SSH_KNOWN_HOSTS --ssh-user "$SSH_USER" --ssh-host "$SSH_HOST" --commit-hash "$HASH" build-android-bundle: desc: Build AAB via Dagger (cached, versionCode=1 placeholder) and export locally cmds: - mkdir -p build/app/outputs/bundle/release - - HASH=$(git rev-parse --short HEAD) && dagger call --progress=plain -q -m ci --source=. build-android-release --commit-hash "$HASH" -o build/app/outputs/bundle/release/app-release.aab + - HASH=$(git rev-parse --short HEAD) && timeout --kill-after=10 1800 dagger call --progress=plain -q -m ci --source=. build-android-release --commit-hash "$HASH" -o build/app/outputs/bundle/release/app-release.aab upload-android-bundle: desc: Upload AAB from build/ to Play Store via Dagger @@ -234,7 +234,7 @@ tasks: - sh: test -f build/app/outputs/bundle/release/app-release.aab msg: "AAB not found — run build-android-bundle first" cmds: - - dagger call --progress=plain -q -m ci --source=. upload-to-play-store --aab build/app/outputs/bundle/release/app-release.aab --play-store-config env:PLAY_STORE_CONFIG_JSON + - timeout --kill-after=10 600 dagger call --progress=plain -q -m ci --source=. upload-to-play-store --aab build/app/outputs/bundle/release/app-release.aab --play-store-config env:PLAY_STORE_CONFIG_JSON publish-android: desc: Build cached AAB, stamp versionCode, sign, and publish to Play Store via Dagger @@ -247,7 +247,7 @@ tasks: - sh: test -n "$ANDROID_KEYSTORE_PASSWORD" msg: "ANDROID_KEYSTORE_PASSWORD is not set" cmds: - - HASH=$(git rev-parse --short HEAD) && scripts/silent_on_success.sh dagger call --progress=plain -q -m ci --source=. publish-android --play-store-config env:PLAY_STORE_CONFIG_JSON --keystore-base64 env:ANDROID_KEYSTORE_BASE64 --keystore-password env:ANDROID_KEYSTORE_PASSWORD --commit-hash "$HASH" + - HASH=$(git rev-parse --short HEAD) && scripts/silent_on_success.sh timeout --kill-after=10 1800 dagger call --progress=plain -q -m ci --source=. publish-android --play-store-config env:PLAY_STORE_CONFIG_JSON --keystore-base64 env:ANDROID_KEYSTORE_BASE64 --keystore-password env:ANDROID_KEYSTORE_PASSWORD --commit-hash "$HASH" deploy-apk: desc: Build and deploy Android APK via Dagger @@ -261,7 +261,7 @@ tasks: - sh: test -n "$ANDROID_KEYSTORE_PASSWORD" msg: "ANDROID_KEYSTORE_PASSWORD is not set" cmds: - - HASH=$(git rev-parse --short HEAD) && scripts/silent_on_success.sh dagger call --progress=plain -q -m ci --source=. deploy-apk --ssh-key env:SSH_PRIVATE_KEY --known-hosts env:SSH_KNOWN_HOSTS --ssh-user "$SSH_USER" --ssh-host "$SSH_HOST" --commit-hash "$HASH" --keystore-base64 env:ANDROID_KEYSTORE_BASE64 --keystore-password env:ANDROID_KEYSTORE_PASSWORD --build-number "$(git log -1 --format=%ct HEAD)" + - HASH=$(git rev-parse --short HEAD) && scripts/silent_on_success.sh timeout --kill-after=10 1800 dagger call --progress=plain -q -m ci --source=. deploy-apk --ssh-key env:SSH_PRIVATE_KEY --known-hosts env:SSH_KNOWN_HOSTS --ssh-user "$SSH_USER" --ssh-host "$SSH_HOST" --commit-hash "$HASH" --keystore-base64 env:ANDROID_KEYSTORE_BASE64 --keystore-password env:ANDROID_KEYSTORE_PASSWORD --build-number "$(git log -1 --format=%ct HEAD)" publish-website: desc: Build and publish website via Dagger @@ -271,7 +271,7 @@ tasks: - sh: test -n "$SSH_KNOWN_HOSTS" msg: "SSH_KNOWN_HOSTS is not set" cmds: - - HASH=$(git rev-parse --short HEAD) && dagger call --progress=plain -q -m ci --source=. publish-website --ssh-key env:SSH_PRIVATE_KEY --known-hosts env:SSH_KNOWN_HOSTS --ssh-user "$SSH_USER" --ssh-host "$SSH_HOST" --commit-hash "$HASH" + - HASH=$(git rev-parse --short HEAD) && timeout --kill-after=10 600 dagger call --progress=plain -q -m ci --source=. publish-website --ssh-key env:SSH_PRIVATE_KEY --known-hosts env:SSH_KNOWN_HOSTS --ssh-user "$SSH_USER" --ssh-host "$SSH_HOST" --commit-hash "$HASH" check-dagger: desc: Run full check suite via Dagger (with OTEL timing report if python3 is available) @@ -351,7 +351,7 @@ tasks: - sh: test -n "$RENOVATE_FORGEJO_TOKEN" msg: "RENOVATE_FORGEJO_TOKEN is not set" cmds: - - dagger call --progress=plain -q -m ci --source=. renovate --renovate-token env:RENOVATE_FORGEJO_TOKEN + - timeout --kill-after=10 1800 dagger call --progress=plain -q -m ci --source=. renovate --renovate-token env:RENOVATE_FORGEJO_TOKEN integration-android: desc: UI integration tests on a connected Android emulator (Stalwart on host, emulator reaches it via 10.0.2.2) diff --git a/scripts/run_firebase_test.sh b/scripts/run_firebase_test.sh index abf8a71..aa7552f 100755 --- a/scripts/run_firebase_test.sh +++ b/scripts/run_firebase_test.sh @@ -34,7 +34,7 @@ _filter_noise() { _run() { : > "$OUT" ; : > "$RC_FILE" { - dagger call --progress=plain -q -m ci --source=. test-android-firebase \ + timeout --kill-after=10 2400 dagger call --progress=plain -q -m ci --source=. test-android-firebase \ --service-account-key env:FIREBASE_TEST_LAB_SERVICE_ACCOUNT_KEY \ --project-id "$FIREBASE_PROJECT_ID" echo $? > "$RC_FILE" @@ -44,6 +44,10 @@ _run() { for attempt in 1 2 3; do _run && break RC=$(cat "$RC_FILE" 2>/dev/null || echo 1) + if [ "$RC" -eq 124 ]; then + echo "::warning::[firebase] attempt $attempt/3 timed out after 2400s" >&2 + exit 124 + fi if [ "$attempt" -lt 3 ] && grep -qE "connection reset|context canceled|connection refused|No Dagger server responded" "$OUT"; then echo "[firebase] dagger connectivity error on attempt $attempt/3, retrying..." >&2 else diff --git a/scripts/setup_dagger_remote.sh b/scripts/setup_dagger_remote.sh index 369c0cb..7a3f41a 100755 --- a/scripts/setup_dagger_remote.sh +++ b/scripts/setup_dagger_remote.sh @@ -54,12 +54,22 @@ echo "$DAGGER_SSH_KEY" > ~/.ssh/dagger_key chmod 600 ~/.ssh/dagger_key # Add remote host to known_hosts -ssh-keyscan -H "$DAGGER_ENGINE_HOST" >> ~/.ssh/known_hosts 2>/dev/null +_t0=$SECONDS +timeout 30 ssh-keyscan -H "$DAGGER_ENGINE_HOST" >> ~/.ssh/known_hosts 2>/dev/null +_elapsed=$(( SECONDS - _t0 )) +if [ "$_elapsed" -gt 10 ]; then + echo "::warning::ssh-keyscan took ${_elapsed}s — Dagger engine host may be slow to respond" +fi # Create a background SSH tunnel to the Dagger engine. # We map local port 8080 to remote port 1774 (where our socat bridge is listening). echo "Establishing SSH tunnel to $DAGGER_ENGINE_HOST..." -ssh -i ~/.ssh/dagger_key -o StrictHostKeyChecking=no -f -N -L 8080:localhost:1774 "dagger@$DAGGER_ENGINE_HOST" +_t0=$SECONDS +timeout 30 ssh -i ~/.ssh/dagger_key -o StrictHostKeyChecking=no -f -N -L 8080:localhost:1774 "dagger@$DAGGER_ENGINE_HOST" +_elapsed=$(( SECONDS - _t0 )) +if [ "$_elapsed" -gt 10 ]; then + echo "::warning::SSH tunnel setup took ${_elapsed}s" +fi # Export _EXPERIMENTAL_DAGGER_RUNNER_HOST to use the tunnel. export _EXPERIMENTAL_DAGGER_RUNNER_HOST="tcp://localhost:8080"