ci: validate gcloud auth stderr, fail on 'error' in output, check test count (#145)

- Capture gcloud auth stderr separately and fail on unexpected output;
  ignore the two known informational lines ("Activated service account
  credentials for: [...]" and "Updated property [core/project].") while
  keeping a strict "fail if unknown stderr" check for anything else.
- Replace the narrow pattern grep (non-retryable error|infrastructure_failure|
  test execution failed) with a broad whole-word case-insensitive grep for
  'error', so any infrastructure or Firebase error in the output causes CI
  failure.
- Verify that the number of device result rows in the result table matches
  the expected device count (1), so a silent test-run failure cannot slip
  through.
- Add scripts/test_firebase_check.sh with 18 unit tests for the three new
  bash patterns (auth stderr filter, error-word detection, device count).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

ci/main.go

@@ -682,10 +682,16 @@ func (m *Ci) TestAndroidFirebase(
 		WithSecretVariable("FIREBASE_SA_KEY", serviceAccountKey).
 		WithEnvVariable("FIREBASE_PROJECT_ID", projectID).
 		WithExec([]string{"/bin/bash", "-c",
-			`echo "$FIREBASE_SA_KEY" > /tmp/key.json && \
+			`auth_err=$(mktemp); trap 'rm -f "$auth_err"' EXIT; \
-			 gcloud auth activate-service-account --key-file=/tmp/key.json && \
+			 echo "$FIREBASE_SA_KEY" > /tmp/key.json; \
-			 rm /tmp/key.json && \
+			 gcloud auth activate-service-account --key-file=/tmp/key.json 2>"$auth_err" \
-			 gcloud config set project "$FIREBASE_PROJECT_ID" && \
+			   || { cat "$auth_err"; exit 1; }; \
+			 rm -f /tmp/key.json; \
+			 gcloud config set project "$FIREBASE_PROJECT_ID" 2>>"$auth_err" \
+			   || { cat "$auth_err"; exit 1; }; \
+			 unknown=$(grep -vF "Activated service account credentials for:" "$auth_err" \
+			   | grep -vF "Updated property [core/project]." | grep -v "^$" || true); \
+			 [ -z "$unknown" ] || { echo "ERROR: unexpected gcloud auth output: $unknown"; exit 1; }; \
 			 out=$(gcloud firebase test android run \
 			   --type instrumentation \
 			   --app /apks/app-debug.apk \
@@ -693,8 +699,12 @@ func (m *Ci) TestAndroidFirebase(
 			   --device model=oriole,version=33,locale=en,orientation=portrait \
 			   --results-bucket=gs://sharedinbox-ftl-results 2>&1); rc=$?; echo "$out"; \
 			 [ "$rc" -eq 0 ] || { echo "ERROR: gcloud firebase test exited with code $rc"; exit "$rc"; }; \
-			 echo "$out" | grep -qiE 'non-retryable error|infrastructure_failure|test execution failed' && { echo "ERROR: Firebase error detected in output"; exit 1; } || true; \
+			 echo "$out" | grep -qwi 'error' && { echo "ERROR: 'error' found in firebase test output"; exit 1; } || true; \
-			 echo "$out" | grep -qE 'Passed|passed' || { echo "ERROR: no passing test results reported — tests did not run"; exit 1; }`}).
+			 expected_devices=1; \
+			 actual_devices=$(echo "$out" | grep "│" | grep -cE "(Passed|Failed|Inconclusive|Skipped)") || actual_devices=0; \
+			 [ "$actual_devices" -eq "$expected_devices" ] || \
+			   { echo "ERROR: expected $expected_devices test result(s) but found $actual_devices"; exit 1; }; \
+			 echo "$out" | grep -q "Passed" || { echo "ERROR: no passing test results — tests failed or did not run"; exit 1; }`}).
 		Stdout(ctx)
 }
 

scripts/test_firebase_check.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+# Tests for Firebase CI check patterns used in ci/main.go.
+# Run directly: bash scripts/test_firebase_check.sh
+
+PASS=0
+FAIL=0
+
+_assert() {
+    local name="$1" expected="$2" actual="$3"
+    if [ "$actual" = "$expected" ]; then
+        PASS=$((PASS + 1))
+    else
+        echo "FAIL: $name"
+        echo "  expected: '$expected'"
+        echo "  actual:   '$actual'"
+        FAIL=$((FAIL + 1))
+    fi
+}
+
+# --- auth stderr filter ---
+# Lines ignored: "Activated service account credentials for: [...]"
+#                "Updated property [core/project]."
+_filter_auth() {
+    grep -vF "Activated service account credentials for:" \
+    | grep -vF "Updated property [core/project]." \
+    | grep -v "^$" \
+    || true
+}
+
+_assert "auth: both known messages produce empty output" "" \
+    "$(printf 'Activated service account credentials for: [ci@sa.iam.gserviceaccount.com]\nUpdated property [core/project].\n' | _filter_auth)"
+
+_assert "auth: only credentials line produces empty output" "" \
+    "$(printf 'Activated service account credentials for: [ci@sa.iam.gserviceaccount.com]\n' | _filter_auth)"
+
+_assert "auth: only property line produces empty output" "" \
+    "$(printf 'Updated property [core/project].\n' | _filter_auth)"
+
+_assert "auth: empty input produces empty output" "" \
+    "$(printf '' | _filter_auth)"
+
+_assert "auth: unexpected line passes through" "some unexpected error" \
+    "$(printf 'some unexpected error\n' | _filter_auth)"
+
+_assert "auth: unknown line kept alongside known messages" "unexpected line" \
+    "$(printf 'Activated service account credentials for: [x]\nunexpected line\nUpdated property [core/project].\n' | _filter_auth)"
+
+# --- "error" word detection: grep -qwi 'error' ---
+# Matches "error" as a whole word (case-insensitive).
+# Must NOT match "error" as part of another word (e.g. "stderr", "AssertionError").
+_has_err() { printf '%s\n' "$1" | grep -qwi 'error' && echo yes || echo no; }
+
+_assert "error: non-retryable error line matched"  yes "$(_has_err 'A non-retryable error occurred.')"
+_assert "error: uppercase ERROR matched"           yes "$(_has_err 'ERROR: infrastructure_failure')"
+_assert "error: mixed-case Error matched"          yes "$(_has_err 'Error: something went wrong')"
+_assert "error: normal pending line not matched"   no  "$(_has_err 'Test is Pending')"
+_assert "error: timing line not matched"           no  "$(_has_err 'Done. Test time = 183 (secs)')"
+_assert "error: completion line not matched"       no  "$(_has_err 'Instrumentation testing complete.')"
+_assert "error: 'stderr' word not matched"         no  "$(_has_err 'some stderr: gcloud output')"
+_assert "error: 'AssertionError' not matched"      no  "$(_has_err 'java.lang.AssertionError: expected true')"
+
+# --- device count from result table ---
+# Counts data rows by looking for lines with "│" that contain an outcome word.
+TABLE_PASS="┌─────────┬───────────────────────┬──────────────┐
+│ OUTCOME │    TEST_AXIS_VALUE    │ TEST_DETAILS │
+├─────────┼───────────────────────┼──────────────┤
+│ Passed  │ oriole-33-en-portrait │ --           │
+└─────────┴───────────────────────┴──────────────┘"
+
+TABLE_FAIL="┌─────────┬───────────────────────┬──────────────┐
+│ OUTCOME │    TEST_AXIS_VALUE    │ TEST_DETAILS │
+├─────────┼───────────────────────┼──────────────┤
+│ Failed  │ oriole-33-en-portrait │ --           │
+└─────────┴───────────────────────┴──────────────┘"
+
+_count() {
+    local n
+    n=$(printf '%s' "$1" | grep "│" | grep -cE "(Passed|Failed|Inconclusive|Skipped)") || n=0
+    printf '%s' "$n"
+}
+
+_assert "count: one passing device gives 1"  1 "$(_count "$TABLE_PASS")"
+_assert "count: one failing device gives 1"  1 "$(_count "$TABLE_FAIL")"
+_assert "count: no table gives 0"            0 "$(_count 'Test is Pending\nDone.')"
+_assert "count: plain output gives 0"        0 "$(_count 'Instrumentation testing complete.')"
+
+echo ""
+echo "Results: $PASS passed, $FAIL failed"
+[ "$FAIL" -eq 0 ] || exit 1