diff --git a/ci/main.go b/ci/main.go index c536a03..e88093a 100644 --- a/ci/main.go +++ b/ci/main.go @@ -682,10 +682,16 @@ func (m *Ci) TestAndroidFirebase( WithSecretVariable("FIREBASE_SA_KEY", serviceAccountKey). WithEnvVariable("FIREBASE_PROJECT_ID", projectID). WithExec([]string{"/bin/bash", "-c", - `echo "$FIREBASE_SA_KEY" > /tmp/key.json && \ - gcloud auth activate-service-account --key-file=/tmp/key.json && \ - rm /tmp/key.json && \ - gcloud config set project "$FIREBASE_PROJECT_ID" && \ + `auth_err=$(mktemp); trap 'rm -f "$auth_err"' EXIT; \ + echo "$FIREBASE_SA_KEY" > /tmp/key.json; \ + gcloud auth activate-service-account --key-file=/tmp/key.json 2>"$auth_err" \ + || { cat "$auth_err"; exit 1; }; \ + rm -f /tmp/key.json; \ + gcloud config set project "$FIREBASE_PROJECT_ID" 2>>"$auth_err" \ + || { cat "$auth_err"; exit 1; }; \ + unknown=$(grep -vF "Activated service account credentials for:" "$auth_err" \ + | grep -vF "Updated property [core/project]." | grep -v "^$" || true); \ + [ -z "$unknown" ] || { echo "ERROR: unexpected gcloud auth output: $unknown"; exit 1; }; \ out=$(gcloud firebase test android run \ --type instrumentation \ --app /apks/app-debug.apk \ @@ -693,8 +699,12 @@ func (m *Ci) TestAndroidFirebase( --device model=oriole,version=33,locale=en,orientation=portrait \ --results-bucket=gs://sharedinbox-ftl-results 2>&1); rc=$?; echo "$out"; \ [ "$rc" -eq 0 ] || { echo "ERROR: gcloud firebase test exited with code $rc"; exit "$rc"; }; \ - echo "$out" | grep -qiE 'non-retryable error|infrastructure_failure|test execution failed' && { echo "ERROR: Firebase error detected in output"; exit 1; } || true; \ - echo "$out" | grep -qE 'Passed|passed' || { echo "ERROR: no passing test results reported — tests did not run"; exit 1; }`}). + echo "$out" | grep -qwi 'error' && { echo "ERROR: 'error' found in firebase test output"; exit 1; } || true; \ + expected_devices=1; \ + actual_devices=$(echo "$out" | grep "│" | grep -cE "(Passed|Failed|Inconclusive|Skipped)") || actual_devices=0; \ + [ "$actual_devices" -eq "$expected_devices" ] || \ + { echo "ERROR: expected $expected_devices test result(s) but found $actual_devices"; exit 1; }; \ + echo "$out" | grep -q "Passed" || { echo "ERROR: no passing test results — tests failed or did not run"; exit 1; }`}). Stdout(ctx) } diff --git a/scripts/test_firebase_check.sh b/scripts/test_firebase_check.sh new file mode 100755 index 0000000..9e0152e --- /dev/null +++ b/scripts/test_firebase_check.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Tests for Firebase CI check patterns used in ci/main.go. +# Run directly: bash scripts/test_firebase_check.sh + +PASS=0 +FAIL=0 + +_assert() { + local name="$1" expected="$2" actual="$3" + if [ "$actual" = "$expected" ]; then + PASS=$((PASS + 1)) + else + echo "FAIL: $name" + echo " expected: '$expected'" + echo " actual: '$actual'" + FAIL=$((FAIL + 1)) + fi +} + +# --- auth stderr filter --- +# Lines ignored: "Activated service account credentials for: [...]" +# "Updated property [core/project]." +_filter_auth() { + grep -vF "Activated service account credentials for:" \ + | grep -vF "Updated property [core/project]." \ + | grep -v "^$" \ + || true +} + +_assert "auth: both known messages produce empty output" "" \ + "$(printf 'Activated service account credentials for: [ci@sa.iam.gserviceaccount.com]\nUpdated property [core/project].\n' | _filter_auth)" + +_assert "auth: only credentials line produces empty output" "" \ + "$(printf 'Activated service account credentials for: [ci@sa.iam.gserviceaccount.com]\n' | _filter_auth)" + +_assert "auth: only property line produces empty output" "" \ + "$(printf 'Updated property [core/project].\n' | _filter_auth)" + +_assert "auth: empty input produces empty output" "" \ + "$(printf '' | _filter_auth)" + +_assert "auth: unexpected line passes through" "some unexpected error" \ + "$(printf 'some unexpected error\n' | _filter_auth)" + +_assert "auth: unknown line kept alongside known messages" "unexpected line" \ + "$(printf 'Activated service account credentials for: [x]\nunexpected line\nUpdated property [core/project].\n' | _filter_auth)" + +# --- "error" word detection: grep -qwi 'error' --- +# Matches "error" as a whole word (case-insensitive). +# Must NOT match "error" as part of another word (e.g. "stderr", "AssertionError"). +_has_err() { printf '%s\n' "$1" | grep -qwi 'error' && echo yes || echo no; } + +_assert "error: non-retryable error line matched" yes "$(_has_err 'A non-retryable error occurred.')" +_assert "error: uppercase ERROR matched" yes "$(_has_err 'ERROR: infrastructure_failure')" +_assert "error: mixed-case Error matched" yes "$(_has_err 'Error: something went wrong')" +_assert "error: normal pending line not matched" no "$(_has_err 'Test is Pending')" +_assert "error: timing line not matched" no "$(_has_err 'Done. Test time = 183 (secs)')" +_assert "error: completion line not matched" no "$(_has_err 'Instrumentation testing complete.')" +_assert "error: 'stderr' word not matched" no "$(_has_err 'some stderr: gcloud output')" +_assert "error: 'AssertionError' not matched" no "$(_has_err 'java.lang.AssertionError: expected true')" + +# --- device count from result table --- +# Counts data rows by looking for lines with "│" that contain an outcome word. +TABLE_PASS="┌─────────┬───────────────────────┬──────────────┐ +│ OUTCOME │ TEST_AXIS_VALUE │ TEST_DETAILS │ +├─────────┼───────────────────────┼──────────────┤ +│ Passed │ oriole-33-en-portrait │ -- │ +└─────────┴───────────────────────┴──────────────┘" + +TABLE_FAIL="┌─────────┬───────────────────────┬──────────────┐ +│ OUTCOME │ TEST_AXIS_VALUE │ TEST_DETAILS │ +├─────────┼───────────────────────┼──────────────┤ +│ Failed │ oriole-33-en-portrait │ -- │ +└─────────┴───────────────────────┴──────────────┘" + +_count() { + local n + n=$(printf '%s' "$1" | grep "│" | grep -cE "(Passed|Failed|Inconclusive|Skipped)") || n=0 + printf '%s' "$n" +} + +_assert "count: one passing device gives 1" 1 "$(_count "$TABLE_PASS")" +_assert "count: one failing device gives 1" 1 "$(_count "$TABLE_FAIL")" +_assert "count: no table gives 0" 0 "$(_count 'Test is Pending\nDone.')" +_assert "count: plain output gives 0" 0 "$(_count 'Instrumentation testing complete.')" + +echo "" +echo "Results: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ] || exit 1