ci: add timeouts to all CI/CD jobs, Dagger tasks, and runner scripts (#432)
Closes #415 ## Summary - Adds missing `timeout-minutes` to `ci.yml` (`check` job, 60 min) and `windows-nightly.yml` (90 min, ready for when the Windows runner is registered) - Wraps `ssh-keyscan` and `ssh -f -N -L` tunnel creation in `setup_dagger_remote.sh` with `timeout 30`; emits a `::warning::` annotation when either takes more than 10 s - Adds `timeout --kill-after=10 <N>` to all bare `dagger call` invocations in `Taskfile.yml`: 600 s for test/query tasks, 1800 s for build/deploy tasks, 60 s for `ci-graph`; `stalwart` and `check-dagger` (already protected) left untouched - Adds `timeout --kill-after=10 2400` per attempt in `run_firebase_test.sh`; emits `::warning::` on exit 124 instead of silently retrying ## Test plan - CI passes on this PR (the `check` job now has `timeout-minutes: 60` and will self-enforce) - All `dagger call` lines in `Taskfile.yml` now have a `timeout` prefix (visible in the diff) - `setup_dagger_remote.sh` logic is unchanged — only the two network calls are wrapped Co-authored-by: Thomas SharedInbox <sharedinbox@thomas-guettler.de> Reviewed-on: https://codeberg.org/guettli/sharedinbox/pulls/432
This commit was merged in pull request #432.
This commit is contained in:
Bot of Thomas Güttler
committed by
guettli
guettli
co-authored by
guettli
Thomas SharedInbox
guettli
Thomas SharedInbox
parent
ccefccf6a6
commit
8718339b4e
@@ -4,6 +4,7 @@ jobs:
|
||||
check:
|
||||
name: Full Project Check
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Setup Dagger Remote Engine
|
||||
|
||||
@@ -10,6 +10,7 @@ jobs:
|
||||
# Disabled until a self-hosted runner with label "windows-runner" is registered.
|
||||
name: Build & Deploy Windows (Nightly)
|
||||
runs-on: windows-runner
|
||||
timeout-minutes: 90
|
||||
if: false
|
||||
|
||||
steps:
|
||||
|
||||
+11
-11
@@ -177,17 +177,17 @@ tasks:
|
||||
test-backend:
|
||||
desc: Backend tests against a local Stalwart mail server (via Dagger)
|
||||
cmds:
|
||||
- dagger call --progress=plain -q -m ci --source=. test-backend
|
||||
- timeout --kill-after=10 600 dagger call --progress=plain -q -m ci --source=. test-backend
|
||||
|
||||
integration-ui:
|
||||
desc: UI E2E tests on Linux via Xvfb — headless, no emulator needed (via Dagger)
|
||||
cmds:
|
||||
- dagger call --progress=plain -q -m ci --source=. test-integration
|
||||
- timeout --kill-after=10 600 dagger call --progress=plain -q -m ci --source=. test-integration
|
||||
|
||||
sync-reliability:
|
||||
desc: Run sync reliability runner (via Dagger)
|
||||
cmds:
|
||||
- dagger call --progress=plain -q -m ci --source=. test-sync-reliability
|
||||
- timeout --kill-after=10 600 dagger call --progress=plain -q -m ci --source=. test-sync-reliability
|
||||
|
||||
test-android-firebase:
|
||||
desc: Build Android debug APKs and run instrumented tests on Firebase Test Lab (via Dagger)
|
||||
@@ -202,7 +202,7 @@ tasks:
|
||||
ci-graph:
|
||||
desc: Print a Mermaid diagram of the CI pipeline — paste into mermaid.live or any Markdown renderer
|
||||
cmds:
|
||||
- dagger call --progress=plain -q -m ci --source=. graph
|
||||
- timeout --kill-after=10 60 dagger call --progress=plain -q -m ci --source=. graph
|
||||
|
||||
stalwart:
|
||||
desc: Start a Stalwart instance for local development (via Dagger)
|
||||
@@ -218,13 +218,13 @@ tasks:
|
||||
- sh: test -n "$SSH_KNOWN_HOSTS"
|
||||
msg: "SSH_KNOWN_HOSTS is not set"
|
||||
cmds:
|
||||
- HASH=$(git rev-parse --short HEAD) && scripts/silent_on_success.sh dagger call --progress=plain -q -m ci --source=. deploy-linux --ssh-key env:SSH_PRIVATE_KEY --known-hosts env:SSH_KNOWN_HOSTS --ssh-user "$SSH_USER" --ssh-host "$SSH_HOST" --commit-hash "$HASH"
|
||||
- HASH=$(git rev-parse --short HEAD) && scripts/silent_on_success.sh timeout --kill-after=10 1800 dagger call --progress=plain -q -m ci --source=. deploy-linux --ssh-key env:SSH_PRIVATE_KEY --known-hosts env:SSH_KNOWN_HOSTS --ssh-user "$SSH_USER" --ssh-host "$SSH_HOST" --commit-hash "$HASH"
|
||||
|
||||
build-android-bundle:
|
||||
desc: Build AAB via Dagger (cached, versionCode=1 placeholder) and export locally
|
||||
cmds:
|
||||
- mkdir -p build/app/outputs/bundle/release
|
||||
- HASH=$(git rev-parse --short HEAD) && dagger call --progress=plain -q -m ci --source=. build-android-release --commit-hash "$HASH" -o build/app/outputs/bundle/release/app-release.aab
|
||||
- HASH=$(git rev-parse --short HEAD) && timeout --kill-after=10 1800 dagger call --progress=plain -q -m ci --source=. build-android-release --commit-hash "$HASH" -o build/app/outputs/bundle/release/app-release.aab
|
||||
|
||||
upload-android-bundle:
|
||||
desc: Upload AAB from build/ to Play Store via Dagger
|
||||
@@ -234,7 +234,7 @@ tasks:
|
||||
- sh: test -f build/app/outputs/bundle/release/app-release.aab
|
||||
msg: "AAB not found — run build-android-bundle first"
|
||||
cmds:
|
||||
- dagger call --progress=plain -q -m ci --source=. upload-to-play-store --aab build/app/outputs/bundle/release/app-release.aab --play-store-config env:PLAY_STORE_CONFIG_JSON
|
||||
- timeout --kill-after=10 600 dagger call --progress=plain -q -m ci --source=. upload-to-play-store --aab build/app/outputs/bundle/release/app-release.aab --play-store-config env:PLAY_STORE_CONFIG_JSON
|
||||
|
||||
publish-android:
|
||||
desc: Build cached AAB, stamp versionCode, sign, and publish to Play Store via Dagger
|
||||
@@ -247,7 +247,7 @@ tasks:
|
||||
- sh: test -n "$ANDROID_KEYSTORE_PASSWORD"
|
||||
msg: "ANDROID_KEYSTORE_PASSWORD is not set"
|
||||
cmds:
|
||||
- HASH=$(git rev-parse --short HEAD) && scripts/silent_on_success.sh dagger call --progress=plain -q -m ci --source=. publish-android --play-store-config env:PLAY_STORE_CONFIG_JSON --keystore-base64 env:ANDROID_KEYSTORE_BASE64 --keystore-password env:ANDROID_KEYSTORE_PASSWORD --commit-hash "$HASH"
|
||||
- HASH=$(git rev-parse --short HEAD) && scripts/silent_on_success.sh timeout --kill-after=10 1800 dagger call --progress=plain -q -m ci --source=. publish-android --play-store-config env:PLAY_STORE_CONFIG_JSON --keystore-base64 env:ANDROID_KEYSTORE_BASE64 --keystore-password env:ANDROID_KEYSTORE_PASSWORD --commit-hash "$HASH"
|
||||
|
||||
deploy-apk:
|
||||
desc: Build and deploy Android APK via Dagger
|
||||
@@ -261,7 +261,7 @@ tasks:
|
||||
- sh: test -n "$ANDROID_KEYSTORE_PASSWORD"
|
||||
msg: "ANDROID_KEYSTORE_PASSWORD is not set"
|
||||
cmds:
|
||||
- HASH=$(git rev-parse --short HEAD) && scripts/silent_on_success.sh dagger call --progress=plain -q -m ci --source=. deploy-apk --ssh-key env:SSH_PRIVATE_KEY --known-hosts env:SSH_KNOWN_HOSTS --ssh-user "$SSH_USER" --ssh-host "$SSH_HOST" --commit-hash "$HASH" --keystore-base64 env:ANDROID_KEYSTORE_BASE64 --keystore-password env:ANDROID_KEYSTORE_PASSWORD --build-number "$(git log -1 --format=%ct HEAD)"
|
||||
- HASH=$(git rev-parse --short HEAD) && scripts/silent_on_success.sh timeout --kill-after=10 1800 dagger call --progress=plain -q -m ci --source=. deploy-apk --ssh-key env:SSH_PRIVATE_KEY --known-hosts env:SSH_KNOWN_HOSTS --ssh-user "$SSH_USER" --ssh-host "$SSH_HOST" --commit-hash "$HASH" --keystore-base64 env:ANDROID_KEYSTORE_BASE64 --keystore-password env:ANDROID_KEYSTORE_PASSWORD --build-number "$(git log -1 --format=%ct HEAD)"
|
||||
|
||||
publish-website:
|
||||
desc: Build and publish website via Dagger
|
||||
@@ -271,7 +271,7 @@ tasks:
|
||||
- sh: test -n "$SSH_KNOWN_HOSTS"
|
||||
msg: "SSH_KNOWN_HOSTS is not set"
|
||||
cmds:
|
||||
- HASH=$(git rev-parse --short HEAD) && dagger call --progress=plain -q -m ci --source=. publish-website --ssh-key env:SSH_PRIVATE_KEY --known-hosts env:SSH_KNOWN_HOSTS --ssh-user "$SSH_USER" --ssh-host "$SSH_HOST" --commit-hash "$HASH"
|
||||
- HASH=$(git rev-parse --short HEAD) && timeout --kill-after=10 600 dagger call --progress=plain -q -m ci --source=. publish-website --ssh-key env:SSH_PRIVATE_KEY --known-hosts env:SSH_KNOWN_HOSTS --ssh-user "$SSH_USER" --ssh-host "$SSH_HOST" --commit-hash "$HASH"
|
||||
|
||||
check-dagger:
|
||||
desc: Run full check suite via Dagger (with OTEL timing report if python3 is available)
|
||||
@@ -351,7 +351,7 @@ tasks:
|
||||
- sh: test -n "$RENOVATE_FORGEJO_TOKEN"
|
||||
msg: "RENOVATE_FORGEJO_TOKEN is not set"
|
||||
cmds:
|
||||
- dagger call --progress=plain -q -m ci --source=. renovate --renovate-token env:RENOVATE_FORGEJO_TOKEN
|
||||
- timeout --kill-after=10 1800 dagger call --progress=plain -q -m ci --source=. renovate --renovate-token env:RENOVATE_FORGEJO_TOKEN
|
||||
|
||||
integration-android:
|
||||
desc: UI integration tests on a connected Android emulator (Stalwart on host, emulator reaches it via 10.0.2.2)
|
||||
|
||||
@@ -34,7 +34,7 @@ _filter_noise() {
|
||||
_run() {
|
||||
: > "$OUT" ; : > "$RC_FILE"
|
||||
{
|
||||
dagger call --progress=plain -q -m ci --source=. test-android-firebase \
|
||||
timeout --kill-after=10 2400 dagger call --progress=plain -q -m ci --source=. test-android-firebase \
|
||||
--service-account-key env:FIREBASE_TEST_LAB_SERVICE_ACCOUNT_KEY \
|
||||
--project-id "$FIREBASE_PROJECT_ID"
|
||||
echo $? > "$RC_FILE"
|
||||
@@ -44,6 +44,10 @@ _run() {
|
||||
for attempt in 1 2 3; do
|
||||
_run && break
|
||||
RC=$(cat "$RC_FILE" 2>/dev/null || echo 1)
|
||||
if [ "$RC" -eq 124 ]; then
|
||||
echo "::warning::[firebase] attempt $attempt/3 timed out after 2400s" >&2
|
||||
exit 124
|
||||
fi
|
||||
if [ "$attempt" -lt 3 ] && grep -qE "connection reset|context canceled|connection refused|No Dagger server responded" "$OUT"; then
|
||||
echo "[firebase] dagger connectivity error on attempt $attempt/3, retrying..." >&2
|
||||
else
|
||||
|
||||
@@ -54,12 +54,22 @@ echo "$DAGGER_SSH_KEY" > ~/.ssh/dagger_key
|
||||
chmod 600 ~/.ssh/dagger_key
|
||||
|
||||
# Add remote host to known_hosts
|
||||
ssh-keyscan -H "$DAGGER_ENGINE_HOST" >> ~/.ssh/known_hosts 2>/dev/null
|
||||
_t0=$SECONDS
|
||||
timeout 30 ssh-keyscan -H "$DAGGER_ENGINE_HOST" >> ~/.ssh/known_hosts 2>/dev/null
|
||||
_elapsed=$(( SECONDS - _t0 ))
|
||||
if [ "$_elapsed" -gt 10 ]; then
|
||||
echo "::warning::ssh-keyscan took ${_elapsed}s — Dagger engine host may be slow to respond"
|
||||
fi
|
||||
|
||||
# Create a background SSH tunnel to the Dagger engine.
|
||||
# We map local port 8080 to remote port 1774 (where our socat bridge is listening).
|
||||
echo "Establishing SSH tunnel to $DAGGER_ENGINE_HOST..."
|
||||
ssh -i ~/.ssh/dagger_key -o StrictHostKeyChecking=no -f -N -L 8080:localhost:1774 "dagger@$DAGGER_ENGINE_HOST"
|
||||
_t0=$SECONDS
|
||||
timeout 30 ssh -i ~/.ssh/dagger_key -o StrictHostKeyChecking=no -f -N -L 8080:localhost:1774 "dagger@$DAGGER_ENGINE_HOST"
|
||||
_elapsed=$(( SECONDS - _t0 ))
|
||||
if [ "$_elapsed" -gt 10 ]; then
|
||||
echo "::warning::SSH tunnel setup took ${_elapsed}s"
|
||||
fi
|
||||
|
||||
# Export _EXPERIMENTAL_DAGGER_RUNNER_HOST to use the tunnel.
|
||||
export _EXPERIMENTAL_DAGGER_RUNNER_HOST="tcp://localhost:8080"
|
||||
|
||||
Reference in New Issue
Block a user