Compare commits

...
Author SHA1 Message Date
Thomas SharedInboxandClaude Sonnet 4.6 c517f604e0 test: update deploy_playstore tests for requests-based transport
The previous tests patched google_auth_httplib2 and googleapiclient which
no longer exist in the new implementation. Rewrite to mock AuthorizedSession
and _upload_aab_resumable, covering the same scenarios: happy path, retry
on transient errors, backoff delays, and exhausted attempts.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-24 07:40:17 +02:00
Thomas SharedInboxandClaude Sonnet 4.6 7d393ec818 fix: switch Play Store upload from httplib2 to requests
httplib2 treats 308 Resume Incomplete responses (used by Google's
resumable upload API) as redirects and raises RedirectMissingLocation
when the response lacks a Location header. Switch to
google.auth.transport.requests.AuthorizedSession + direct HTTP calls
so the upload uses the requests library, which handles 308 correctly.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-24 07:32:22 +02:00
Bot of Thomas Güttler 5c38357033 fix: limit dagger-data volume growth by pruning named caches (#193) (#197) 2026-05-24 06:00:14 +02:00
Thomas SharedInboxandClaude Sonnet 4.6 7715190cbf fix: retry AAB upload on RedirectMissingLocation with exponential backoff
Adds a 3-attempt retry loop around the resumable AAB upload that catches
httplib2.error.RedirectMissingLocation (a transient network error) and
retries with exponential backoff (10s, 20s). A fresh MediaFileUpload is
created on each attempt because resumable upload objects cannot be reused
after failure. Also adds TestUploadRetry covering the retry path.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-24 05:30:24 +02:00
Thomas SharedInboxandClaude Sonnet 4.6 80cde04d87 fix: retry AAB upload on RedirectMissingLocation with exponential backoff (#186)
Wrap the resumable bundle upload in a loop of up to _MAX_UPLOAD_ATTEMPTS (3)
attempts. On httplib2.error.RedirectMissingLocation, recreate MediaFileUpload
(resumable uploads cannot reuse the same object) and wait 10 s / 20 s before
retrying. After all attempts are exhausted, raise RuntimeError chained to the
last exception. Add tests covering the retry path, backoff delays, fresh
MediaFileUpload on each attempt, and exhaustion.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-24 04:59:05 +02:00
Bot of Thomas Güttler 83060bc1bf fix: add timeout and retries to Play Store upload (#185) (#195) 2026-05-24 04:45:07 +02:00
5 changed files with 242 additions and 76 deletions
+6 -2
View File
@@ -55,7 +55,10 @@ jobs:
- name: Prune Dagger cache before check - name: Prune Dagger cache before check
env: env:
DAGGER_NO_NAG: "1" DAGGER_NO_NAG: "1"
run: dagger query '{ engine { localCache { prune } } }' 2>/dev/null || true # prune(maxUsedSpace) also reclaims named cache volumes (gradle-cache, go-build-cache, etc.)
# when total cache exceeds the limit; without args only unreferenced entries are removed.
run: |
dagger query '{ engine { localCache { prune(maxUsedSpace: "75gb", targetSpace: "50gb") } } }' || true
- name: Run Full Check Suite - name: Run Full Check Suite
env: env:
@@ -66,7 +69,8 @@ jobs:
if: always() if: always()
env: env:
DAGGER_NO_NAG: "1" DAGGER_NO_NAG: "1"
run: dagger query '{ engine { localCache { prune } } }' 2>/dev/null || true run: |
dagger query '{ engine { localCache { prune(maxUsedSpace: "75gb", targetSpace: "50gb") } } }' || true
- name: Cleanup TLS credentials - name: Cleanup TLS credentials
if: always() if: always()
+7 -1
View File
@@ -288,7 +288,7 @@ tasks:
echo "$(_ts) dagger: network error on attempt $attempt/3, retrying..." >&2 echo "$(_ts) dagger: network error on attempt $attempt/3, retrying..." >&2
elif [ "$attempt" -lt 3 ] && grep -q "No space left on device" "$DAGGER_OUT"; then elif [ "$attempt" -lt 3 ] && grep -q "No space left on device" "$DAGGER_OUT"; then
echo "$(_ts) dagger: disk space error on attempt $attempt/3, pruning Dagger cache..." >&2 echo "$(_ts) dagger: disk space error on attempt $attempt/3, pruning Dagger cache..." >&2
dagger query '{ engine { localCache { prune } } }' 2>/dev/null || true dagger query '{ engine { localCache { prune(targetSpace: "20gb") } } }' 2>/dev/null || true
echo "$(_ts) dagger: waiting 90s for freed space to settle..." >&2 echo "$(_ts) dagger: waiting 90s for freed space to settle..." >&2
sleep 90 sleep 90
else else
@@ -320,6 +320,12 @@ tasks:
wait "$RECV_PID" 2>/dev/null || true wait "$RECV_PID" 2>/dev/null || true
exit $RC exit $RC
dagger-prune:
desc: Prune the Dagger engine cache (keeps named volumes unless total exceeds 75 GB, then targets 50 GB)
cmds:
- |
dagger query '{ engine { localCache { prune(maxUsedSpace: "75gb", targetSpace: "50gb") } } }'
integration-android: integration-android:
desc: UI integration tests on a connected Android emulator (Stalwart on host, emulator reaches it via 10.0.2.2) desc: UI integration tests on a connected Android emulator (Stalwart on host, emulator reaches it via 10.0.2.2)
deps: [_preflight, _android-sdk-check, _android-avd-setup] deps: [_preflight, _android-sdk-check, _android-avd-setup]
+1 -1
View File
@@ -739,7 +739,7 @@ func (m *Ci) UploadToPlayStore(
From("python:3.12-alpine"). From("python:3.12-alpine").
WithExec([]string{"apk", "add", "--no-cache", "curl"}). WithExec([]string{"apk", "add", "--no-cache", "curl"}).
WithMountedCache("/root/.cache/pip", dag.CacheVolume("pip-cache")). WithMountedCache("/root/.cache/pip", dag.CacheVolume("pip-cache")).
WithExec([]string{"pip", "install", "google-api-python-client", "google-auth-httplib2", "httplib2"}). WithExec([]string{"pip", "install", "google-auth", "requests"}).
WithFile("/src/build/app/outputs/bundle/release/app-release.aab", aab). WithFile("/src/build/app/outputs/bundle/release/app-release.aab", aab).
WithFile("/src/scripts/deploy_playstore.py", scriptSource.File("scripts/deploy_playstore.py")). WithFile("/src/scripts/deploy_playstore.py", scriptSource.File("scripts/deploy_playstore.py")).
WithSecretVariable("PLAY_STORE_CONFIG_JSON", playStoreConfig). WithSecretVariable("PLAY_STORE_CONFIG_JSON", playStoreConfig).
+73 -25
View File
@@ -4,17 +4,51 @@
import json import json
import os import os
import sys import sys
import time
import google_auth_httplib2 from google.auth.transport.requests import AuthorizedSession
import httplib2
from google.oauth2 import service_account from google.oauth2 import service_account
from googleapiclient.discovery import build
from googleapiclient.http import MediaFileUpload
PACKAGE_NAME = "de.sharedinbox.mua" PACKAGE_NAME = "de.sharedinbox.mua"
AAB_PATH = "build/app/outputs/bundle/release/app-release.aab" AAB_PATH = "build/app/outputs/bundle/release/app-release.aab"
TRACK = "internal" TRACK = "internal"
_TIMEOUT = 300 # seconds — AAB uploads can be large _BASE = "https://androidpublisher.googleapis.com/androidpublisher/v3/applications"
_UPLOAD_BASE = "https://androidpublisher.googleapis.com/upload/androidpublisher/v3/applications"
_MAX_UPLOAD_ATTEMPTS = 3
def _upload_aab_resumable(session, package, edit_id, aab_path):
"""Upload AAB using the Google resumable upload protocol."""
file_size = os.path.getsize(aab_path)
init_url = f"{_UPLOAD_BASE}/{package}/edits/{edit_id}/bundles"
# Step 1: initiate the resumable upload session
init_resp = session.post(
init_url,
params={"uploadType": "resumable"},
headers={
"X-Upload-Content-Type": "application/octet-stream",
"X-Upload-Content-Length": str(file_size),
"Content-Length": "0",
},
timeout=60,
)
init_resp.raise_for_status()
upload_url = init_resp.headers["Location"]
# Step 2: upload the file in a single PUT to the session URI
with open(aab_path, "rb") as f:
upload_resp = session.put(
upload_url,
data=f,
headers={
"Content-Type": "application/octet-stream",
"Content-Length": str(file_size),
},
timeout=600,
)
upload_resp.raise_for_status()
return upload_resp.json()
def main(): def main():
@@ -31,33 +65,47 @@ def main():
json.loads(config_json), json.loads(config_json),
scopes=["https://www.googleapis.com/auth/androidpublisher"], scopes=["https://www.googleapis.com/auth/androidpublisher"],
) )
session = AuthorizedSession(creds)
authorized_http = google_auth_httplib2.AuthorizedHttp( edit_resp = session.post(f"{_BASE}/{PACKAGE_NAME}/edits", json={}, timeout=30)
creds, http=httplib2.Http(timeout=_TIMEOUT) edit_resp.raise_for_status()
) edit_id = edit_resp.json()["id"]
service = build("androidpublisher", "v3", http=authorized_http)
edit = service.edits().insert(body={}, packageName=PACKAGE_NAME).execute(num_retries=3) last_exc = None
edit_id = edit["id"] bundle = None
for attempt in range(_MAX_UPLOAD_ATTEMPTS):
try:
bundle = _upload_aab_resumable(session, PACKAGE_NAME, edit_id, AAB_PATH)
break
except Exception as exc:
last_exc = exc
if attempt < _MAX_UPLOAD_ATTEMPTS - 1:
delay = 10 * (2 ** attempt)
print(
f"Upload attempt {attempt + 1} failed ({type(exc).__name__}: {exc}), "
f"retrying in {delay}s…"
)
time.sleep(delay)
if bundle is None:
raise RuntimeError(
f"AAB upload failed after {_MAX_UPLOAD_ATTEMPTS} attempts"
) from last_exc
media = MediaFileUpload(AAB_PATH, mimetype="application/octet-stream", resumable=True)
bundle = (
service.edits()
.bundles()
.upload(packageName=PACKAGE_NAME, editId=edit_id, media_body=media)
.execute(num_retries=3)
)
version_code = bundle["versionCode"] version_code = bundle["versionCode"]
print(f"Uploaded AAB, version code: {version_code}") print(f"Uploaded AAB, version code: {version_code}")
service.edits().tracks().update( track_resp = session.put(
packageName=PACKAGE_NAME, f"{_BASE}/{PACKAGE_NAME}/edits/{edit_id}/tracks/{TRACK}",
editId=edit_id, json={"releases": [{"versionCodes": [version_code], "status": "completed"}]},
track=TRACK, timeout=30,
body={"releases": [{"versionCodes": [version_code], "status": "completed"}]}, )
).execute(num_retries=3) track_resp.raise_for_status()
service.edits().commit(packageName=PACKAGE_NAME, editId=edit_id).execute(num_retries=3) commit_resp = session.post(
f"{_BASE}/{PACKAGE_NAME}/edits/{edit_id}:commit",
timeout=30,
)
commit_resp.raise_for_status()
print(f"Deployed version {version_code} to {TRACK} track") print(f"Deployed version {version_code} to {TRACK} track")
+155 -47
View File
@@ -1,9 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Tests for deploy_playstore.py.""" """Tests for deploy_playstore.py."""
import io
import os import os
import sys import sys
import tempfile
import unittest import unittest
from pathlib import Path from pathlib import Path
from unittest.mock import MagicMock, call, patch from unittest.mock import MagicMock, call, patch
@@ -13,6 +11,35 @@ sys.path.insert(0, str(Path(__file__).parent))
import deploy_playstore import deploy_playstore
def _make_session(
edit_id="edit-42",
version_code=7,
upload_side_effects=None,
):
"""Return a mock AuthorizedSession with sensible defaults."""
session = MagicMock()
# POST /edits → create edit
edit_resp = MagicMock()
edit_resp.json.return_value = {"id": edit_id}
session.post.return_value = edit_resp
# POST resumable-init → Location header
init_resp = MagicMock()
init_resp.headers = {"Location": "https://upload.example.com/session"}
# PUT upload → bundle JSON
upload_resp = MagicMock()
upload_resp.json.return_value = {"versionCode": version_code}
if upload_side_effects is not None:
# Use side_effect list: first call is edit create, rest are upload inits
# We override the PUT side effects via _upload_aab_resumable mock instead
pass
return session, init_resp, upload_resp
class TestMainEnvChecks(unittest.TestCase): class TestMainEnvChecks(unittest.TestCase):
def test_missing_env_exits(self): def test_missing_env_exits(self):
with patch.dict(os.environ, {}, clear=True): with patch.dict(os.environ, {}, clear=True):
@@ -30,62 +57,143 @@ class TestMainEnvChecks(unittest.TestCase):
class TestMainHappyPath(unittest.TestCase): class TestMainHappyPath(unittest.TestCase):
def _run_main(self, fake_config): def _run_main(self, fake_config='{"type":"service_account"}'):
mock_service = MagicMock() mock_session = MagicMock()
mock_edits = mock_service.edits.return_value # POST for edit create and commit
mock_edits.insert.return_value.execute.return_value = {"id": "edit-42"} post_responses = [
mock_edits.bundles.return_value.upload.return_value.execute.return_value = { MagicMock(**{"json.return_value": {"id": "edit-42"}}), # create edit
"versionCode": 7 MagicMock(), # commit
} ]
mock_edits.tracks.return_value.update.return_value.execute.return_value = {} mock_session.post.side_effect = post_responses
mock_edits.commit.return_value.execute.return_value = {} # PUT for track update
mock_session.put.return_value = MagicMock()
with patch.dict(os.environ, {"PLAY_STORE_CONFIG_JSON": fake_config}): with patch.dict(os.environ, {"PLAY_STORE_CONFIG_JSON": fake_config}):
with patch("deploy_playstore.os.path.exists", return_value=True): with patch("deploy_playstore.os.path.exists", return_value=True):
with patch("deploy_playstore.service_account.Credentials.from_service_account_info"): with patch("deploy_playstore.service_account.Credentials.from_service_account_info"):
with patch("deploy_playstore.google_auth_httplib2.AuthorizedHttp"): with patch("deploy_playstore.AuthorizedSession", return_value=mock_session):
with patch("deploy_playstore.build", return_value=mock_service): with patch(
with patch("deploy_playstore.MediaFileUpload"): "deploy_playstore._upload_aab_resumable",
return_value={"versionCode": 7},
):
deploy_playstore.main()
return mock_session
def test_creates_edit(self):
session = self._run_main()
create_call = session.post.call_args_list[0]
self.assertIn("/edits", create_call[0][0])
def test_commits_edit(self):
session = self._run_main()
commit_call = session.post.call_args_list[1]
self.assertIn(":commit", commit_call[0][0])
def test_updates_track(self):
session = self._run_main()
track_call = session.put.call_args_list[0]
self.assertIn("/tracks/", track_call[0][0])
class TestUploadRetry(unittest.TestCase):
def _run_main(self, upload_side_effects, sleep_mock=None):
mock_session = MagicMock()
post_responses = [
MagicMock(**{"json.return_value": {"id": "edit-1"}}),
MagicMock(),
]
mock_session.post.side_effect = post_responses
mock_session.put.return_value = MagicMock()
patches = [
patch.dict(os.environ, {"PLAY_STORE_CONFIG_JSON": '{"type":"service_account"}'}),
patch("deploy_playstore.os.path.exists", return_value=True),
patch("deploy_playstore.service_account.Credentials.from_service_account_info"),
patch("deploy_playstore.AuthorizedSession", return_value=mock_session),
patch("deploy_playstore._upload_aab_resumable", side_effect=upload_side_effects),
patch("deploy_playstore.time.sleep"),
]
for p in patches:
p.start()
try:
deploy_playstore.main()
finally:
for p in patches:
p.stop()
def test_succeeds_on_first_attempt(self):
with patch("deploy_playstore._upload_aab_resumable", return_value={"versionCode": 5}) as mock_upload:
with patch.dict(os.environ, {"PLAY_STORE_CONFIG_JSON": '{"type":"service_account"}'}):
with patch("deploy_playstore.os.path.exists", return_value=True):
with patch("deploy_playstore.service_account.Credentials.from_service_account_info"):
mock_session = MagicMock()
mock_session.post.side_effect = [
MagicMock(**{"json.return_value": {"id": "e1"}}),
MagicMock(),
]
mock_session.put.return_value = MagicMock()
with patch("deploy_playstore.AuthorizedSession", return_value=mock_session):
deploy_playstore.main()
mock_upload.assert_called_once()
def test_retries_once_on_error_then_succeeds(self):
self._run_main([ValueError("transient"), {"versionCode": 9}])
def test_raises_after_all_attempts_exhausted(self):
with self.assertRaises(RuntimeError) as ctx:
self._run_main([ValueError("err"), ValueError("err"), ValueError("err")])
self.assertIn(str(deploy_playstore._MAX_UPLOAD_ATTEMPTS), str(ctx.exception))
def test_backoff_delays_are_10s_then_20s(self):
mock_session = MagicMock()
mock_session.post.side_effect = [
MagicMock(**{"json.return_value": {"id": "e1"}}),
MagicMock(),
]
mock_session.put.return_value = MagicMock()
with patch.dict(os.environ, {"PLAY_STORE_CONFIG_JSON": '{"type":"service_account"}'}):
with patch("deploy_playstore.os.path.exists", return_value=True):
with patch("deploy_playstore.service_account.Credentials.from_service_account_info"):
with patch("deploy_playstore.AuthorizedSession", return_value=mock_session):
with patch(
"deploy_playstore._upload_aab_resumable",
side_effect=[ValueError("e"), ValueError("e"), {"versionCode": 3}],
):
with patch("deploy_playstore.time.sleep") as mock_sleep:
deploy_playstore.main() deploy_playstore.main()
return mock_edits mock_sleep.assert_has_calls([call(10), call(20)])
def test_insert_called_with_num_retries(self):
edits = self._run_main('{"type":"service_account"}')
edits.insert.return_value.execute.assert_called_once_with(num_retries=3)
def test_bundle_upload_called_with_num_retries(self): class TestUploadAabResumable(unittest.TestCase):
edits = self._run_main('{"type":"service_account"}') def test_initiates_and_uploads(self):
edits.bundles.return_value.upload.return_value.execute.assert_called_once_with(num_retries=3) mock_session = MagicMock()
init_resp = MagicMock()
init_resp.headers = {"Location": "https://upload.example.com/sess"}
upload_resp = MagicMock()
upload_resp.json.return_value = {"versionCode": 42}
mock_session.post.return_value = init_resp
mock_session.put.return_value = upload_resp
def test_tracks_update_called_with_num_retries(self): import tempfile
edits = self._run_main('{"type":"service_account"}') with tempfile.NamedTemporaryFile(delete=False) as f:
edits.tracks.return_value.update.return_value.execute.assert_called_once_with(num_retries=3) f.write(b"fake-aab-content")
aab_path = f.name
def test_commit_called_with_num_retries(self): try:
edits = self._run_main('{"type":"service_account"}') result = deploy_playstore._upload_aab_resumable(
edits.commit.return_value.execute.assert_called_once_with(num_retries=3) mock_session, "com.example.app", "edit-1", aab_path
)
finally:
os.unlink(aab_path)
def test_authorized_http_uses_timeout(self): self.assertEqual(result["versionCode"], 42)
fake_config = '{"type":"service_account"}' mock_session.post.assert_called_once()
with patch.dict(os.environ, {"PLAY_STORE_CONFIG_JSON": fake_config}): mock_session.put.assert_called_once()
with patch("deploy_playstore.os.path.exists", return_value=True): put_call = mock_session.put.call_args
with patch("deploy_playstore.service_account.Credentials.from_service_account_info"): self.assertEqual(put_call[0][0], "https://upload.example.com/sess")
with patch("deploy_playstore.httplib2.Http") as mock_http_cls:
with patch("deploy_playstore.google_auth_httplib2.AuthorizedHttp") as mock_auth:
mock_service = MagicMock()
mock_edits = mock_service.edits.return_value
mock_edits.insert.return_value.execute.return_value = {"id": "e1"}
mock_edits.bundles.return_value.upload.return_value.execute.return_value = {
"versionCode": 1
}
mock_edits.tracks.return_value.update.return_value.execute.return_value = {}
mock_edits.commit.return_value.execute.return_value = {}
with patch("deploy_playstore.build", return_value=mock_service):
with patch("deploy_playstore.MediaFileUpload"):
deploy_playstore.main()
mock_http_cls.assert_called_once_with(timeout=deploy_playstore._TIMEOUT)
if __name__ == "__main__": if __name__ == "__main__":