fix: retry AAB upload on RedirectMissingLocation with exponential backoff (#186)
Wrap the resumable bundle upload in a loop of up to _MAX_UPLOAD_ATTEMPTS (3) attempts. On httplib2.error.RedirectMissingLocation, recreate MediaFileUpload (resumable uploads cannot reuse the same object) and wait 10 s / 20 s before retrying. After all attempts are exhausted, raise RuntimeError chained to the last exception. Add tests covering the retry path, backoff delays, fresh MediaFileUpload on each attempt, and exhaustion. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Thomas SharedInbox
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
fb6f2cca68
commit
80cde04d87
@@ -4,6 +4,7 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import google_auth_httplib2
|
||||
import httplib2
|
||||
@@ -15,6 +16,7 @@ PACKAGE_NAME = "de.sharedinbox.mua"
|
||||
AAB_PATH = "build/app/outputs/bundle/release/app-release.aab"
|
||||
TRACK = "internal"
|
||||
_TIMEOUT = 300 # seconds — AAB uploads can be large
|
||||
_MAX_UPLOAD_ATTEMPTS = 3
|
||||
|
||||
|
||||
def main():
|
||||
@@ -40,14 +42,38 @@ def main():
|
||||
edit = service.edits().insert(body={}, packageName=PACKAGE_NAME).execute(num_retries=3)
|
||||
edit_id = edit["id"]
|
||||
|
||||
media = MediaFileUpload(AAB_PATH, mimetype="application/octet-stream", resumable=True)
|
||||
bundle = (
|
||||
service.edits()
|
||||
.bundles()
|
||||
.upload(packageName=PACKAGE_NAME, editId=edit_id, media_body=media)
|
||||
.execute(num_retries=3)
|
||||
)
|
||||
version_code = bundle["versionCode"]
|
||||
# The resumable upload can fail with RedirectMissingLocation on transient
|
||||
# network hiccups. Retry with a fresh MediaFileUpload each time (resumable
|
||||
# uploads can't reuse the same object) using exponential backoff.
|
||||
version_code = None
|
||||
last_exc = None
|
||||
for attempt in range(_MAX_UPLOAD_ATTEMPTS):
|
||||
try:
|
||||
media = MediaFileUpload(
|
||||
AAB_PATH, mimetype="application/octet-stream", resumable=True
|
||||
)
|
||||
bundle = (
|
||||
service.edits()
|
||||
.bundles()
|
||||
.upload(packageName=PACKAGE_NAME, editId=edit_id, media_body=media)
|
||||
.execute(num_retries=3)
|
||||
)
|
||||
version_code = bundle["versionCode"]
|
||||
break
|
||||
except httplib2.error.RedirectMissingLocation as exc:
|
||||
last_exc = exc
|
||||
if attempt < _MAX_UPLOAD_ATTEMPTS - 1:
|
||||
delay = 10 * (2 ** attempt)
|
||||
print(
|
||||
f"Upload attempt {attempt + 1} failed (redirect error), "
|
||||
f"retrying in {delay}s…"
|
||||
)
|
||||
time.sleep(delay)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"AAB upload failed after {_MAX_UPLOAD_ATTEMPTS} attempts"
|
||||
) from last_exc
|
||||
|
||||
print(f"Uploaded AAB, version code: {version_code}")
|
||||
|
||||
service.edits().tracks().update(
|
||||
|
||||
@@ -88,5 +88,112 @@ class TestMainHappyPath(unittest.TestCase):
|
||||
mock_http_cls.assert_called_once_with(timeout=deploy_playstore._TIMEOUT)
|
||||
|
||||
|
||||
def _redirect_error():
|
||||
import httplib2
|
||||
return httplib2.error.RedirectMissingLocation("redirect missing", {}, b"")
|
||||
|
||||
|
||||
class TestUploadRetry(unittest.TestCase):
|
||||
def _make_mock_service(self, upload_side_effects):
|
||||
mock_service = MagicMock()
|
||||
mock_edits = mock_service.edits.return_value
|
||||
mock_edits.insert.return_value.execute.return_value = {"id": "edit-1"}
|
||||
mock_edits.bundles.return_value.upload.return_value.execute.side_effect = (
|
||||
upload_side_effects
|
||||
)
|
||||
mock_edits.tracks.return_value.update.return_value.execute.return_value = {}
|
||||
mock_edits.commit.return_value.execute.return_value = {}
|
||||
return mock_service, mock_edits
|
||||
|
||||
def _run_with_service(self, mock_service):
|
||||
fake_config = '{"type":"service_account"}'
|
||||
with patch.dict(os.environ, {"PLAY_STORE_CONFIG_JSON": fake_config}):
|
||||
with patch("deploy_playstore.os.path.exists", return_value=True):
|
||||
with patch("deploy_playstore.service_account.Credentials.from_service_account_info"):
|
||||
with patch("deploy_playstore.google_auth_httplib2.AuthorizedHttp"):
|
||||
with patch("deploy_playstore.build", return_value=mock_service):
|
||||
with patch("deploy_playstore.MediaFileUpload"):
|
||||
with patch("deploy_playstore.time.sleep"):
|
||||
deploy_playstore.main()
|
||||
|
||||
def test_succeeds_on_first_attempt(self):
|
||||
mock_service, mock_edits = self._make_mock_service([{"versionCode": 5}])
|
||||
self._run_with_service(mock_service)
|
||||
mock_edits.bundles.return_value.upload.return_value.execute.assert_called_once_with(
|
||||
num_retries=3
|
||||
)
|
||||
|
||||
def test_retries_once_on_redirect_error_then_succeeds(self):
|
||||
mock_service, mock_edits = self._make_mock_service(
|
||||
[_redirect_error(), {"versionCode": 9}]
|
||||
)
|
||||
|
||||
with patch.dict(os.environ, {"PLAY_STORE_CONFIG_JSON": '{"type":"service_account"}'}):
|
||||
with patch("deploy_playstore.os.path.exists", return_value=True):
|
||||
with patch("deploy_playstore.service_account.Credentials.from_service_account_info"):
|
||||
with patch("deploy_playstore.google_auth_httplib2.AuthorizedHttp"):
|
||||
with patch("deploy_playstore.build", return_value=mock_service):
|
||||
with patch("deploy_playstore.MediaFileUpload") as mock_media_cls:
|
||||
with patch("deploy_playstore.time.sleep") as mock_sleep:
|
||||
deploy_playstore.main()
|
||||
|
||||
self.assertEqual(
|
||||
mock_edits.bundles.return_value.upload.return_value.execute.call_count, 2
|
||||
)
|
||||
mock_sleep.assert_called_once_with(10)
|
||||
self.assertEqual(mock_media_cls.call_count, 2)
|
||||
|
||||
def test_raises_after_all_attempts_exhausted(self):
|
||||
mock_service, _ = self._make_mock_service(
|
||||
[_redirect_error(), _redirect_error(), _redirect_error()]
|
||||
)
|
||||
|
||||
with patch.dict(os.environ, {"PLAY_STORE_CONFIG_JSON": '{"type":"service_account"}'}):
|
||||
with patch("deploy_playstore.os.path.exists", return_value=True):
|
||||
with patch("deploy_playstore.service_account.Credentials.from_service_account_info"):
|
||||
with patch("deploy_playstore.google_auth_httplib2.AuthorizedHttp"):
|
||||
with patch("deploy_playstore.build", return_value=mock_service):
|
||||
with patch("deploy_playstore.MediaFileUpload"):
|
||||
with patch("deploy_playstore.time.sleep"):
|
||||
with self.assertRaises(RuntimeError) as ctx:
|
||||
deploy_playstore.main()
|
||||
|
||||
self.assertIn(
|
||||
str(deploy_playstore._MAX_UPLOAD_ATTEMPTS), str(ctx.exception)
|
||||
)
|
||||
|
||||
def test_backoff_delays_are_10s_then_20s(self):
|
||||
mock_service, _ = self._make_mock_service(
|
||||
[_redirect_error(), _redirect_error(), {"versionCode": 3}]
|
||||
)
|
||||
|
||||
with patch.dict(os.environ, {"PLAY_STORE_CONFIG_JSON": '{"type":"service_account"}'}):
|
||||
with patch("deploy_playstore.os.path.exists", return_value=True):
|
||||
with patch("deploy_playstore.service_account.Credentials.from_service_account_info"):
|
||||
with patch("deploy_playstore.google_auth_httplib2.AuthorizedHttp"):
|
||||
with patch("deploy_playstore.build", return_value=mock_service):
|
||||
with patch("deploy_playstore.MediaFileUpload"):
|
||||
with patch("deploy_playstore.time.sleep") as mock_sleep:
|
||||
deploy_playstore.main()
|
||||
|
||||
mock_sleep.assert_has_calls([call(10), call(20)])
|
||||
|
||||
def test_fresh_media_upload_created_on_each_attempt(self):
|
||||
mock_service, _ = self._make_mock_service(
|
||||
[_redirect_error(), {"versionCode": 2}]
|
||||
)
|
||||
|
||||
with patch.dict(os.environ, {"PLAY_STORE_CONFIG_JSON": '{"type":"service_account"}'}):
|
||||
with patch("deploy_playstore.os.path.exists", return_value=True):
|
||||
with patch("deploy_playstore.service_account.Credentials.from_service_account_info"):
|
||||
with patch("deploy_playstore.google_auth_httplib2.AuthorizedHttp"):
|
||||
with patch("deploy_playstore.build", return_value=mock_service):
|
||||
with patch("deploy_playstore.MediaFileUpload") as mock_media_cls:
|
||||
with patch("deploy_playstore.time.sleep"):
|
||||
deploy_playstore.main()
|
||||
|
||||
self.assertEqual(mock_media_cls.call_count, 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user