feat(builds): populate builds page with Linux and Android history (#94)

The builds page at /builds/ was empty because generate-build-history
only ran inside deploy-playstore; if that job failed early (e.g. Play
Store secrets not configured) the website was never updated, and the
build-linux job never triggered a website update at all.

Changes:
- generate_build_history.py: extend to cover Linux tarballs in addition
  to Android APKs, capped at MAX_BUILDS_PER_PLATFORM (30) each
- Taskfile: add website-publish task (generate-build-history +
  website-deploy), exclude *.tar.gz from rsync, update descriptions
- .forgejo/workflows/ci.yml: add publish-website job that waits for
  both build-linux and deploy-playstore (using always() so it runs
  even when deploy-playstore fails), then removes the duplicate
  generate/deploy steps from deploy-playstore
- .github/workflows/ci.yml: add deploy job that deploys Linux build,
  generates build history, builds Hugo site, and rsyncs to server
- .gitignore: ignore website/content/builds/_index.md (generated),
  Python __pycache__, and widget test failure screenshots
- stalwart-dev/integration_ui_test.sh: use ${USER:-$(id -un)} for
  robustness in environments where USER is unset
- scripts/test_generate_build_history.py: unit tests for parse_builds
  and render_entries covering both platforms

Generated content (builds/_index.md and per-day pages) is not tracked
in git; it is produced at CI time and rsynced to the server.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Thomas SharedInbox
2026-05-15 19:08:55 +02:00
co-authored by Claude Sonnet 4.6
parent 0620663630
commit cf277064cc
7 changed files with 360 additions and 60 deletions
+28 -9
View File
@@ -127,16 +127,35 @@ jobs:
ANDROID_KEYSTORE_PASSWORD: ${{ secrets.ANDROID_KEYSTORE_PASSWORD }}
run: nix develop --no-warn-dirty --command task deploy-apk-to-server
- name: Generate build history
continue-on-error: true
env:
SSH_USER: ${{ secrets.SSH_USER }}
SSH_HOST: ${{ secrets.SSH_HOST }}
run: nix develop --no-warn-dirty --command task generate-build-history
publish-website:
name: Publish Website Build History
runs-on: self-hosted
needs: [build-linux, deploy-playstore]
if: |
always() &&
github.ref == 'refs/heads/main' &&
(needs.build-linux.result == 'success' || needs.deploy-playstore.result == 'success')
- name: Deploy website
continue-on-error: true
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 1
- name: Enable Nix flakes
run: |
mkdir -p ~/.config/nix
echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf
- name: Set up SSH key
env:
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
run: |
mkdir -p ~/.ssh
echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
- name: Generate build history and deploy website
env:
SSH_USER: ${{ secrets.SSH_USER }}
SSH_HOST: ${{ secrets.SSH_HOST }}
run: nix develop --no-warn-dirty --command task website-deploy
run: nix develop --no-warn-dirty --command task website-publish
+96
View File
@@ -151,3 +151,99 @@ jobs:
- name: Build Linux release
run: flutter build linux --release
deploy:
name: Deploy Linux build & publish website
runs-on: ubuntu-latest
needs: build-linux
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
env:
SSH_HOST: ${{ secrets.SSH_HOST }}
SSH_USER: ${{ secrets.SSH_USER }}
steps:
- uses: actions/checkout@v4
- name: Install build & deploy dependencies
run: |
sudo apt-get update -q
sudo apt-get install -y --no-install-recommends \
libgtk-3-dev pkg-config cmake ninja-build clang \
libsecret-1-dev hugo rsync
- uses: subosito/flutter-action@v2
with:
flutter-version: "3.41.6"
channel: stable
cache: true
- name: Cache pub packages
uses: actions/cache@v4
with:
path: ~/.pub-cache
key: pub-${{ hashFiles('pubspec.lock') }}
restore-keys: pub-
- name: Install dependencies
run: flutter pub get
- name: Generate Drift code
run: flutter pub run build_runner build --delete-conflicting-outputs
- name: Generate changelog
run: |
mkdir -p assets
git log -n 50 \
--pretty=format:'* %ad [%h](https://codeberg.org/guettli/sharedinbox/commit/%H): %s' \
--date=short > assets/changelog.txt
- name: Setup SSH
run: |
mkdir -p ~/.ssh
printf '%s\n' "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
- name: Build Linux release
run: |
HASH=$(git rev-parse --short HEAD)
flutter build linux --release --no-pub --dart-define=GIT_HASH=$HASH
- name: Deploy Linux build to server
run: |
HASH=$(git rev-parse --short HEAD)
DATE_PATH=$(date -u +%Y/%m/%d)
REMOTE_DIR="public_html/builds/$DATE_PATH"
TARBALL="sharedinbox-linux-amd64-$HASH.tar.gz"
tar -czf /tmp/$TARBALL -C build/linux/x64/release bundle
ssh -o StrictHostKeyChecking=no "$SSH_USER@$SSH_HOST" "mkdir -p $REMOTE_DIR"
scp -o StrictHostKeyChecking=no /tmp/$TARBALL "$SSH_USER@$SSH_HOST:$REMOTE_DIR/$TARBALL"
DOWNLOAD_URL="https://sharedinbox.de/builds/$DATE_PATH/$TARBALL"
EXISTING=$(ssh -o StrictHostKeyChecking=no "$SSH_USER@$SSH_HOST" \
"cat public_html/latest.json 2>/dev/null || echo '{}'")
WINDOWS_URL=$(echo "$EXISTING" | \
python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('windows',''))" \
2>/dev/null || true)
if [ -n "$WINDOWS_URL" ]; then
echo "{\"version\":\"$HASH\",\"linux\":\"$DOWNLOAD_URL\",\"windows\":\"$WINDOWS_URL\"}" | \
ssh -o StrictHostKeyChecking=no "$SSH_USER@$SSH_HOST" "cat > public_html/latest.json"
else
echo "{\"version\":\"$HASH\",\"linux\":\"$DOWNLOAD_URL\"}" | \
ssh -o StrictHostKeyChecking=no "$SSH_USER@$SSH_HOST" "cat > public_html/latest.json"
fi
- name: Generate build history pages
run: python3 scripts/generate_build_history.py
- name: Build website
env:
HUGO_PARAMS_GITVERSION: ${{ github.sha }}
run: hugo --source website --minify
- name: Deploy website
run: |
rsync -avz --delete \
--exclude='*.apk' \
--exclude='*.tar.gz' \
-e "ssh -o StrictHostKeyChecking=no" \
website/public/ \
"$SSH_USER@$SSH_HOST:public_html/"
+6
View File
@@ -58,6 +58,10 @@ linux/flutter/generated_plugins.cmake
.flutter-plugins-dependencies
.metadata
# --- Python ---
__pycache__/
*.pyc
# --- Tools & Cache ---
.fvm/
fvm/
@@ -98,6 +102,7 @@ sharedinbox-runner/runner-data/
website/public/
website/resources/
website/.hugo_build.lock
website/content/builds/_index.md
website/content/builds/[0-9]*/
.copilot/
@@ -106,4 +111,5 @@ website/content/builds/[0-9]*/
.wget-hsts
tmp/
test/widget/failures/
.claude*
+13 -1
View File
@@ -459,7 +459,7 @@ tasks:
echo "Uploaded $APK_NAME to $REMOTE_DIR"
generate-build-history:
desc: Generate Hugo build-history pages from APKs on the server
desc: Generate Hugo build-history pages from Linux and Android builds on the server
preconditions:
- sh: test -n "$SSH_USER"
msg: "SSH_USER is not set"
@@ -468,6 +468,17 @@ tasks:
cmds:
- python3 scripts/generate_build_history.py
website-publish:
desc: Generate build history, build Hugo site, and rsync to server (requires SSH_USER + SSH_HOST)
preconditions:
- sh: test -n "$SSH_USER"
msg: "SSH_USER is not set"
- sh: test -n "$SSH_HOST"
msg: "SSH_HOST is not set"
cmds:
- task: generate-build-history
- task: website-deploy
website-deploy:
desc: Deploy the website via rsync to public_html
deps: [website-build]
@@ -475,6 +486,7 @@ tasks:
- |
rsync -avz --delete \
--exclude='*.apk' \
--exclude='*.tar.gz' \
-e "ssh -o StrictHostKeyChecking=no" \
website/public/ \
${SSH_USER}@${SSH_HOST}:public_html/
+98 -44
View File
@@ -1,10 +1,16 @@
#!/usr/bin/env python3
"""Generate Hugo markdown pages listing Android builds fetched from the server.
"""Generate Hugo markdown pages listing builds fetched from the server.
Reads APK files under public_html/builds/ on the deployment server via SSH,
Reads build artifacts under public_html/builds/ on the deployment server via SSH,
parses the git hash from each filename, fetches the commit title from the
Codeberg API, then writes Hugo content pages to website/content/builds/.
Covers two platforms:
- Linux: sharedinbox-linux-amd64-<hash>.tar.gz
- Android: sharedinbox-mua-<hash>.apk
At most MAX_BUILDS_PER_PLATFORM of the most-recent builds are shown per platform.
These generated pages are not tracked in git (see .gitignore).
"""
import json
@@ -20,22 +26,23 @@ REMOTE_BUILDS_DIR = "public_html/builds"
CONTENT_DIR = Path("website/content/builds")
BASE_URL = "https://sharedinbox.de"
CODEBERG_BASE = "https://codeberg.org"
MAX_BUILDS_PER_PLATFORM = 30
def list_apks(ssh_user: str, ssh_host: str) -> list[str]:
def list_remote_files(ssh_user: str, ssh_host: str, pattern: str) -> list[str]:
result = subprocess.run(
[
"ssh",
"-o",
"StrictHostKeyChecking=no",
f"{ssh_user}@{ssh_host}",
f"find {REMOTE_BUILDS_DIR} -name '*.apk' -type f | sort",
f"find {REMOTE_BUILDS_DIR} -name '{pattern}' -type f | sort",
],
capture_output=True,
text=True,
check=True,
)
return [l.strip() for l in result.stdout.splitlines() if l.strip()]
return [line.strip() for line in result.stdout.splitlines() if line.strip()]
def get_commit_info(hash_val: str) -> tuple[str, str]:
@@ -53,6 +60,41 @@ def get_commit_info(hash_val: str) -> tuple[str, str]:
return hash_val, ""
def parse_builds(
paths: list[str],
path_re: re.Pattern, # type: ignore[type-arg]
) -> dict[str, list[tuple[str, str, str, str]]]:
"""Parse build file paths into {date_key: [(hash, url, title, dt), ...]}."""
limited = paths[-MAX_BUILDS_PER_PLATFORM:] if len(paths) > MAX_BUILDS_PER_PLATFORM else paths
days: dict[str, list[tuple[str, str, str, str]]] = {}
for path in limited:
m = path_re.match(path)
if not m:
print(f" skipping unexpected path: {path}", file=sys.stderr)
continue
year, month, day, filename, hash_val = m.groups()
date_key = f"{year}/{month}/{day}"
download_url = f"{BASE_URL}/builds/{year}/{month}/{day}/{filename}"
commit_title, commit_dt = get_commit_info(hash_val)
days.setdefault(date_key, []).append((hash_val, download_url, commit_title, commit_dt))
return days
def render_entries(
entries: list[tuple[str, str, str, str]],
link_label: str,
) -> str:
lines = []
for hash_val, download_url, commit_title, commit_dt in entries:
commit_url = f"{CODEBERG_BASE}/{CODEBERG_REPO}/commit/{hash_val}"
dt_str = f" · {commit_dt}" if commit_dt else ""
lines.append(
f"- [{commit_title}]({commit_url}){dt_str} \n"
f" [{link_label}]({download_url}) (`{hash_val}`)\n"
)
return "".join(lines)
def main() -> None:
ssh_user = os.environ.get("SSH_USER", "")
ssh_host = os.environ.get("SSH_HOST", "")
@@ -60,62 +102,74 @@ def main() -> None:
print("SSH_USER and SSH_HOST must be set", file=sys.stderr)
sys.exit(1)
print(f"Listing APKs on {ssh_host}")
apk_paths = list_apks(ssh_user, ssh_host)
print(f"Found {len(apk_paths)} APK(s)")
# Group by YYYY/MM/DD
days: dict[str, list[tuple[str, str, str]]] = {}
for apk_path in apk_paths:
m = re.match(
r"public_html/builds/(\d{4})/(\d{2})/(\d{2})/(sharedinbox-mua-(.+)\.apk)$",
apk_path,
print(f"Listing Linux builds on {ssh_host}")
linux_paths = list_remote_files(ssh_user, ssh_host, "sharedinbox-linux-amd64-*.tar.gz")
print(f"Found {len(linux_paths)} Linux build(s)")
linux_re = re.compile(
r"public_html/builds/(\d{4})/(\d{2})/(\d{2})/(sharedinbox-linux-amd64-(.+)\.tar\.gz)$"
)
if not m:
print(f" skipping unexpected path: {apk_path}", file=sys.stderr)
continue
year, month, day, filename, hash_val = m.groups()
date_key = f"{year}/{month}/{day}"
download_url = f"{BASE_URL}/builds/{year}/{month}/{day}/{filename}"
commit_title, commit_dt = get_commit_info(hash_val)
days.setdefault(date_key, []).append((hash_val, download_url, commit_title, commit_dt))
linux_days = parse_builds(linux_paths, linux_re)
print(f"Listing Android APKs on {ssh_host}")
apk_paths = list_remote_files(ssh_user, ssh_host, "*.apk")
print(f"Found {len(apk_paths)} APK(s)")
apk_re = re.compile(
r"public_html/builds/(\d{4})/(\d{2})/(\d{2})/(sharedinbox-mua-(.+)\.apk)$"
)
android_days = parse_builds(apk_paths, apk_re)
CONTENT_DIR.mkdir(parents=True, exist_ok=True)
# _index.md: flat list of all builds, newest first
# _index.md: platform sections, newest-first within each
index_lines = ["---\ntitle: Builds\n---\n\n"]
for date_key in sorted(days, reverse=True):
index_lines.append(f"## Linux (last {MAX_BUILDS_PER_PLATFORM})\n\n")
if linux_days:
for date_key in sorted(linux_days, reverse=True):
year, month, day = date_key.split("/")
date_iso = f"{year}-{month}-{day}"
index_lines.append(f"## {date_iso}\n\n")
for hash_val, download_url, commit_title, commit_dt in days[date_key]:
commit_url = f"{CODEBERG_BASE}/{CODEBERG_REPO}/commit/{hash_val}"
dt_str = f" · {commit_dt}" if commit_dt else ""
index_lines.append(
f"- [{commit_title}]({commit_url}){dt_str} \n"
f" [Download APK]({download_url}) (`{hash_val}`)\n"
)
index_lines.append(f"### {year}-{month}-{day}\n\n")
index_lines.append(render_entries(linux_days[date_key], "Download"))
index_lines.append("\n")
else:
index_lines.append("_No Linux builds yet._\n\n")
index_lines.append(f"## Android (last {MAX_BUILDS_PER_PLATFORM})\n\n")
if android_days:
for date_key in sorted(android_days, reverse=True):
year, month, day = date_key.split("/")
index_lines.append(f"### {year}-{month}-{day}\n\n")
index_lines.append(render_entries(android_days[date_key], "Download APK"))
index_lines.append("\n")
else:
index_lines.append("_No Android builds yet._\n\n")
(CONTENT_DIR / "_index.md").write_text("".join(index_lines), encoding="utf-8")
for date_key in sorted(days):
# Per-day pages (combined)
all_days = set(linux_days) | set(android_days)
for date_key in sorted(all_days):
year, month, day = date_key.split("/")
date_iso = f"{year}-{month}-{day}"
day_dir = CONTENT_DIR / year / month
day_dir.mkdir(parents=True, exist_ok=True)
lines = [f"---\ntitle: 'Builds for {date_iso}'\ndate: {date_iso}T00:00:00Z\n---\n\n"]
for hash_val, download_url, commit_title, commit_dt in days[date_key]:
commit_url = f"{CODEBERG_BASE}/{CODEBERG_REPO}/commit/{hash_val}"
dt_str = f" · {commit_dt}" if commit_dt else ""
lines.append(
f"- [{commit_title}]({commit_url}){dt_str} \n"
f" [Download APK]({download_url}) (`{hash_val}`)\n"
)
if date_key in linux_days:
lines.append("## Linux\n\n")
lines.append(render_entries(linux_days[date_key], "Download"))
lines.append("\n")
if date_key in android_days:
lines.append("## Android\n\n")
lines.append(render_entries(android_days[date_key], "Download APK"))
lines.append("\n")
(day_dir / f"{day}.md").write_text("".join(lines), encoding="utf-8")
total_builds = sum(len(v) for v in days.values())
print(f"Generated {len(days)} day page(s) covering {total_builds} build(s)")
total_linux = sum(len(v) for v in linux_days.values())
total_android = sum(len(v) for v in android_days.values())
print(
f"Generated pages: {total_linux} Linux build(s), {total_android} Android build(s) "
f"across {len(all_days)} day(s)"
)
if __name__ == "__main__":
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env python3
"""Tests for pure functions in generate_build_history.py."""
import re
import unittest
from unittest.mock import patch
from generate_build_history import MAX_BUILDS_PER_PLATFORM, parse_builds, render_entries
LINUX_RE = re.compile(
r"public_html/builds/(\d{4})/(\d{2})/(\d{2})/(sharedinbox-linux-amd64-(.+)\.tar\.gz)$"
)
APK_RE = re.compile(
r"public_html/builds/(\d{4})/(\d{2})/(\d{2})/(sharedinbox-mua-(.+)\.apk)$"
)
def _fake_commit_info(hash_val: str):
return (f"feat: {hash_val}", "2025-05-10T12:00:00Z")
class TestParseBuilds(unittest.TestCase):
def setUp(self):
patcher = patch("generate_build_history.get_commit_info", side_effect=_fake_commit_info)
self.mock_commit = patcher.start()
self.addCleanup(patcher.stop)
def test_linux_path_parsed(self):
paths = ["public_html/builds/2025/05/10/sharedinbox-linux-amd64-abc1234.tar.gz"]
result = parse_builds(paths, LINUX_RE)
self.assertIn("2025/05/10", result)
entry = result["2025/05/10"][0]
self.assertEqual(entry[0], "abc1234")
self.assertIn("sharedinbox-linux-amd64-abc1234.tar.gz", entry[1])
def test_apk_path_parsed(self):
paths = ["public_html/builds/2025/05/11/sharedinbox-mua-def5678.apk"]
result = parse_builds(paths, APK_RE)
self.assertIn("2025/05/11", result)
entry = result["2025/05/11"][0]
self.assertEqual(entry[0], "def5678")
self.assertIn("sharedinbox-mua-def5678.apk", entry[1])
def test_unexpected_path_skipped(self):
paths = [
"public_html/builds/2025/05/10/sharedinbox-linux-amd64-abc1234.tar.gz",
"public_html/builds/bad-path/other.tar.gz",
]
result = parse_builds(paths, LINUX_RE)
self.assertEqual(len(result), 1)
def test_multiple_builds_same_day(self):
paths = [
"public_html/builds/2025/05/10/sharedinbox-linux-amd64-aaa0001.tar.gz",
"public_html/builds/2025/05/10/sharedinbox-linux-amd64-bbb0002.tar.gz",
]
result = parse_builds(paths, LINUX_RE)
self.assertEqual(len(result["2025/05/10"]), 2)
def test_limited_to_max_builds(self):
paths = [
f"public_html/builds/2025/05/{i:02d}/sharedinbox-linux-amd64-hash{i:03d}.tar.gz"
for i in range(1, MAX_BUILDS_PER_PLATFORM + 5)
]
result = parse_builds(paths, LINUX_RE)
total = sum(len(v) for v in result.values())
self.assertEqual(total, MAX_BUILDS_PER_PLATFORM)
def test_download_url_contains_date_and_filename(self):
paths = ["public_html/builds/2025/03/15/sharedinbox-linux-amd64-cafebabe.tar.gz"]
result = parse_builds(paths, LINUX_RE)
url = result["2025/03/15"][0][1]
self.assertIn("/2025/03/15/", url)
self.assertIn("sharedinbox-linux-amd64-cafebabe.tar.gz", url)
self.assertTrue(url.startswith("https://"))
class TestRenderEntries(unittest.TestCase):
def _make_entry(self, hash_val="abc1234", url="https://example.com/file.apk",
title="feat: something", dt="2025-05-10T12:00:00Z"):
return (hash_val, url, title, dt)
def test_output_contains_title_and_link(self):
entry = self._make_entry()
out = render_entries([entry], "Download APK")
self.assertIn("feat: something", out)
self.assertIn("Download APK", out)
self.assertIn("abc1234", out)
def test_commit_url_uses_hash(self):
entry = self._make_entry(hash_val="deadbeef")
out = render_entries([entry], "Download")
self.assertIn("deadbeef", out)
self.assertIn("codeberg.org", out)
def test_datetime_shown_when_present(self):
entry = self._make_entry(dt="2025-05-10T12:00:00Z")
out = render_entries([entry], "Download")
self.assertIn("2025-05-10T12:00:00Z", out)
def test_datetime_omitted_when_empty(self):
entry = self._make_entry(dt="")
out = render_entries([entry], "Download")
self.assertNotIn(" · ", out)
def test_multiple_entries_all_rendered(self):
entries = [self._make_entry(hash_val=f"hash{i}", title=f"commit {i}") for i in range(3)]
out = render_entries(entries, "Download")
for i in range(3):
self.assertIn(f"commit {i}", out)
if __name__ == "__main__":
unittest.main()
+4 -4
View File
@@ -46,7 +46,7 @@ command -v xvfb-run >/dev/null || {
# but the leftover Xvfb's stale /tmp/.X11-unix/X<N> socket and lock file confuse
# its cleanup, producing "kill: No such process" on exit and a non-zero status
# even when the test itself passed.
for _xvfb_pid in $(pgrep -u "$USER" -x Xvfb 2>/dev/null); do
for _xvfb_pid in $(pgrep -u "${USER:-$(id -un)}" -x Xvfb 2>/dev/null); do
_xvfb_display=$(tr '\0' ' ' < "/proc/${_xvfb_pid}/cmdline" 2>/dev/null \
| grep -oE ':[0-9]+' | head -1)
kill "$_xvfb_pid" 2>/dev/null || true
@@ -109,8 +109,8 @@ ts "flutter test start"
# Stale processes can hold onto the Xvfb display, causing the new Flutter app
# to hang indefinitely during GTK initialisation without ever connecting back
# to the flutter test runner.
pkill -u "$USER" -f "sharedinbox" 2>/dev/null || true
pkill -u "$USER" -f "flutter.*integration" 2>/dev/null || true
pkill -u "${USER:-$(id -un)}" -f "sharedinbox" 2>/dev/null || true
pkill -u "${USER:-$(id -un)}" -f "flutter.*integration" 2>/dev/null || true
sleep 1
# Find an unused display number.
@@ -145,7 +145,7 @@ for _attempt in 1 2; do
[ "$_e2e_exit" -eq 0 ] && break || true
if [ $_attempt -lt 2 ]; then
ts "E2E attempt $_attempt failed (exit $_e2e_exit), restarting Xvfb and retrying..."
pkill -u "$USER" -f "sharedinbox" 2>/dev/null || true
pkill -u "${USER:-$(id -un)}" -f "sharedinbox" 2>/dev/null || true
# Kill the old Xvfb and start a fresh one on a new display number.
kill "${XVFB_PID:-}" 2>/dev/null || true
wait "${XVFB_PID:-}" 2>/dev/null || true