feat: run Firebase tests once daily via dedicated workflow (#272)
Move Android Firebase instrumented tests out of deploy.yml into a new firebase-tests.yml workflow that runs once per day (3 AM UTC) and only when Firebase-relevant files changed in the last 24 hours. On failure, the workflow automatically creates a Forgejo issue labelled "Ready" with instructions to find the root cause and fix it. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Thomas SharedInbox
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
f57a8c502d
commit
720c54433a
@@ -83,44 +83,6 @@ jobs:
|
||||
&& echo "linux=true" >> "$GITHUB_OUTPUT" \
|
||||
|| echo "linux=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
test-android-firebase:
|
||||
name: Android Instrumented Tests (Firebase Test Lab)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
needs: [check-changes]
|
||||
if: needs.check-changes.outputs.android == 'true'
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Check runner tools
|
||||
run: |
|
||||
command -v dagger >/dev/null 2>&1 || { echo "ERROR: dagger is not installed in the runner image. Add it to .forgejo/Dockerfile."; exit 1; }
|
||||
command -v task >/dev/null 2>&1 || { echo "ERROR: task is not installed in the runner image. Add it to .forgejo/Dockerfile."; exit 1; }
|
||||
dpkg -s stunnel4 netcat-openbsd >/dev/null 2>&1 || { echo "ERROR: stunnel4/netcat-openbsd are not installed in the runner image. Add them to .forgejo/Dockerfile."; exit 1; }
|
||||
|
||||
- name: Setup Dagger Remote Engine (via stunnel)
|
||||
env:
|
||||
DAGGER_STUNNEL_URL: ${{ secrets.DAGGER_STUNNEL_URL }}
|
||||
DAGGER_CA_CERT: ${{ secrets.DAGGER_CA_CERT }}
|
||||
DAGGER_CLIENT_CERT: ${{ secrets.DAGGER_CLIENT_CERT }}
|
||||
DAGGER_CLIENT_KEY: ${{ secrets.DAGGER_CLIENT_KEY }}
|
||||
run: scripts/setup_dagger_remote.sh
|
||||
|
||||
- name: Run Android Tests on Firebase Test Lab
|
||||
if: ${{ secrets.FIREBASE_TEST_LAB_SERVICE_ACCOUNT_KEY != '' }}
|
||||
env:
|
||||
FIREBASE_TEST_LAB_SERVICE_ACCOUNT_KEY: ${{ secrets.FIREBASE_TEST_LAB_SERVICE_ACCOUNT_KEY }}
|
||||
FIREBASE_PROJECT_ID: ${{ vars.FIREBASE_PROJECT_ID }}
|
||||
DAGGER_NO_NAG: "1"
|
||||
run: task test-android-firebase
|
||||
|
||||
- name: Cleanup TLS credentials
|
||||
if: always()
|
||||
run: rm -rf /tmp/dagger-tls /tmp/stunnel-dagger.conf /tmp/stunnel.pid
|
||||
|
||||
deploy-playstore:
|
||||
name: Build & Deploy to Play Store
|
||||
runs-on: ubuntu-latest
|
||||
@@ -287,10 +249,9 @@ jobs:
|
||||
label-deploy-health:
|
||||
name: Update Deploy Health Label
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test-android-firebase, deploy-playstore, deploy-apk, build-linux]
|
||||
needs: [deploy-playstore, deploy-apk, build-linux]
|
||||
if: |
|
||||
always() && vars.DEPLOY_HEALTH_ISSUE != '' && (
|
||||
needs.test-android-firebase.result == 'success' || needs.test-android-firebase.result == 'failure' ||
|
||||
needs.deploy-playstore.result == 'success' || needs.deploy-playstore.result == 'failure' ||
|
||||
needs.deploy-apk.result == 'success' || needs.deploy-apk.result == 'failure' ||
|
||||
needs.build-linux.result == 'success' || needs.build-linux.result == 'failure'
|
||||
@@ -303,7 +264,7 @@ jobs:
|
||||
FORGEJO_TOKEN: ${{ github.token }}
|
||||
FORGEJO_URL: ${{ github.server_url }}
|
||||
DEPLOY_HEALTH_ISSUE: ${{ vars.DEPLOY_HEALTH_ISSUE }}
|
||||
ALL_SUCCEEDED: ${{ (needs.test-android-firebase.result == 'success' || needs.test-android-firebase.result == 'skipped') && (needs.deploy-playstore.result == 'success' || needs.deploy-playstore.result == 'skipped') && (needs.deploy-apk.result == 'success' || needs.deploy-apk.result == 'skipped') && (needs.build-linux.result == 'success' || needs.build-linux.result == 'skipped') }}
|
||||
ALL_SUCCEEDED: ${{ (needs.deploy-playstore.result == 'success' || needs.deploy-playstore.result == 'skipped') && (needs.deploy-apk.result == 'success' || needs.deploy-apk.result == 'skipped') && (needs.build-linux.result == 'success' || needs.build-linux.result == 'skipped') }}
|
||||
run: |
|
||||
python3 - << 'PYEOF'
|
||||
import os, json, urllib.request, urllib.error
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
name: Firebase Tests
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 3 * * *' # once per day at 3 AM
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
check-changes:
|
||||
name: Detect Firebase-Relevant Changes
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
has_changes: ${{ steps.diff.outputs.has_changes }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Detect Firebase-relevant changes in last 24 hours
|
||||
id: diff
|
||||
shell: bash
|
||||
run: |
|
||||
# On workflow_dispatch always run
|
||||
if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then
|
||||
echo "has_changes=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
SINCE=$(date -u -d '24 hours ago' '+%Y-%m-%dT%H:%M:%S')
|
||||
CHANGED=$(git log --since="$SINCE" --name-only --format= -- \
|
||||
'android/' 'integration_test/' 'lib/' 'pubspec.yaml' 'pubspec.lock' 'drift_schemas/' \
|
||||
| sort -u | grep -v '^$')
|
||||
|
||||
if [ -n "$CHANGED" ]; then
|
||||
echo "Firebase-relevant files changed since $SINCE:"
|
||||
echo "$CHANGED"
|
||||
echo "has_changes=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "No Firebase-relevant changes in the last 24 hours — skipping tests"
|
||||
echo "has_changes=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
test-android-firebase:
|
||||
name: Android Instrumented Tests (Firebase Test Lab)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
needs: [check-changes]
|
||||
if: needs.check-changes.outputs.has_changes == 'true'
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Check runner tools
|
||||
run: |
|
||||
command -v dagger >/dev/null 2>&1 || { echo "ERROR: dagger is not installed in the runner image. Add it to .forgejo/Dockerfile."; exit 1; }
|
||||
command -v task >/dev/null 2>&1 || { echo "ERROR: task is not installed in the runner image. Add it to .forgejo/Dockerfile."; exit 1; }
|
||||
dpkg -s stunnel4 netcat-openbsd >/dev/null 2>&1 || { echo "ERROR: stunnel4/netcat-openbsd are not installed in the runner image. Add them to .forgejo/Dockerfile."; exit 1; }
|
||||
|
||||
- name: Setup Dagger Remote Engine (via stunnel)
|
||||
env:
|
||||
DAGGER_STUNNEL_URL: ${{ secrets.DAGGER_STUNNEL_URL }}
|
||||
DAGGER_CA_CERT: ${{ secrets.DAGGER_CA_CERT }}
|
||||
DAGGER_CLIENT_CERT: ${{ secrets.DAGGER_CLIENT_CERT }}
|
||||
DAGGER_CLIENT_KEY: ${{ secrets.DAGGER_CLIENT_KEY }}
|
||||
run: scripts/setup_dagger_remote.sh
|
||||
|
||||
- name: Run Android Tests on Firebase Test Lab
|
||||
if: ${{ secrets.FIREBASE_TEST_LAB_SERVICE_ACCOUNT_KEY != '' }}
|
||||
env:
|
||||
FIREBASE_TEST_LAB_SERVICE_ACCOUNT_KEY: ${{ secrets.FIREBASE_TEST_LAB_SERVICE_ACCOUNT_KEY }}
|
||||
FIREBASE_PROJECT_ID: ${{ vars.FIREBASE_PROJECT_ID }}
|
||||
DAGGER_NO_NAG: "1"
|
||||
run: task test-android-firebase
|
||||
|
||||
- name: Cleanup TLS credentials
|
||||
if: always()
|
||||
run: rm -rf /tmp/dagger-tls /tmp/stunnel-dagger.conf /tmp/stunnel.pid
|
||||
|
||||
- name: Create issue on test failure
|
||||
if: failure()
|
||||
env:
|
||||
FORGEJO_TOKEN: ${{ github.token }}
|
||||
FORGEJO_URL: ${{ github.server_url }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
python3 - << 'PYEOF'
|
||||
import os, json, urllib.request, urllib.error
|
||||
|
||||
token = os.environ["FORGEJO_TOKEN"]
|
||||
url_base = os.environ["FORGEJO_URL"].rstrip("/")
|
||||
run_url = os.environ["RUN_URL"]
|
||||
|
||||
headers = {"Authorization": f"token {token}", "Content-Type": "application/json"}
|
||||
api = f"{url_base}/api/v1/repos/guettli/sharedinbox"
|
||||
|
||||
def api_get(path):
|
||||
req = urllib.request.Request(f"{api}{path}", headers=headers)
|
||||
with urllib.request.urlopen(req) as r:
|
||||
return json.loads(r.read())
|
||||
|
||||
def api_post(path, body):
|
||||
data = json.dumps(body).encode()
|
||||
req = urllib.request.Request(f"{api}{path}", data=data, headers=headers, method="POST")
|
||||
with urllib.request.urlopen(req) as r:
|
||||
return json.loads(r.read())
|
||||
|
||||
repo_labels = api_get("/labels")
|
||||
label_map = {l["name"]: l["id"] for l in repo_labels}
|
||||
|
||||
label_ids = [label_map["Ready"]] if "Ready" in label_map else []
|
||||
|
||||
title = "Firebase Tests failed — find root cause and fix"
|
||||
body = (
|
||||
"Firebase instrumented tests failed in the daily run.\n\n"
|
||||
f"**Failed run:** {run_url}\n\n"
|
||||
"## Steps to resolve\n\n"
|
||||
"1. **Find the root cause**: Check the test run logs linked above and identify which test(s) failed and why.\n"
|
||||
"2. **Fix if possible**: If the failure is caused by a code bug, create a fix. If it is a flaky or infrastructure issue, document the findings.\n"
|
||||
"3. Close this issue once the root cause is resolved and the tests pass.\n"
|
||||
)
|
||||
|
||||
issue = api_post("/issues", {
|
||||
"title": title,
|
||||
"body": body,
|
||||
"labels": label_ids,
|
||||
})
|
||||
print(f"Created issue #{issue['number']}: {issue['html_url']}")
|
||||
PYEOF
|
||||
Reference in New Issue
Block a user