From d6d405f0e6fa1687bc04d00d81d23b275a8ffc2e Mon Sep 17 00:00:00 2001 From: Hugo bessa Date: Wed, 7 Oct 2026 01:21:42 -0300 Subject: [PATCH] Stop the release waiting on unpublished apps and stale indexes release_all.py waited for every package to reach PyPI, but vintasend-api and vintasend-templates-management-api have no publish workflow and never do, so a run stalled until its timeout. A package without .github/workflows/publish.yml is now released by its tag alone: it is done once the tag is on origin, and the wave map marks it (tag-only). release_waves refuses a package pinning such a sibling. on_pypi also counted a version as live once the JSON API listed it, while pip resolves against the simple index, which can lag. In 3.2.0 that failed two Python 3.12 publish jobs. A version now counts as live only once the simple index lists it too. --- ai-tools/skills/release-package/SKILL.md | 9 +++++ scripts/_packages.py | 24 +++++++++++- scripts/_pypi.py | 50 +++++++++++++++++++++--- scripts/release_all.py | 39 +++++++++++++----- 4 files changed, 107 insertions(+), 15 deletions(-) diff --git a/ai-tools/skills/release-package/SKILL.md b/ai-tools/skills/release-package/SKILL.md index df967b6..7b45472 100644 --- a/ai-tools/skills/release-package/SKILL.md +++ b/ai-tools/skills/release-package/SKILL.md @@ -146,6 +146,15 @@ Follow that precedent rather than unilaterally switching to major bumps. But: half-published. A package already tagged on origin is reported as done, which is what makes the second and third runs safe. + `vintasend-api` and `vintasend-templates-management-api` are applications: they have no + `.github/workflows/publish.yml`, so their tag is the whole release and they never appear on + PyPI. The wave map marks them `(tag-only)`. Don't wait for them there. + + A version counts as live only once PyPI's simple index lists it, not just the JSON API. pip + reads the simple index, and in 3.2.0 two Python 3.12 publish jobs failed with "No matching + distribution" a couple of minutes after the JSON API already listed the new `vintasend`. If a + job still fails that way, re-run its failed jobs on the same tag: nothing was uploaded. + [`scripts/release_all.py`](../../../scripts/release_all.py) drives that entire loop — root tag, wait for PyPI, then lock/tag/wait per wave — in one unattended run: diff --git a/scripts/_packages.py b/scripts/_packages.py index 4622969..5238e6f 100644 --- a/scripts/_packages.py +++ b/scripts/_packages.py @@ -29,6 +29,10 @@ # skipped in silence. NON_PYTHON_SUBMODULES = frozenset({"tools/vintasend-dashboard"}) +# A package with this workflow uploads to PyPI when it is tagged. One without it is an +# application released as a tag alone -- see `Package.publishes`. +PUBLISH_WORKFLOW = Path(".github/workflows/publish.yml") + # Tables whose `name`/`version` keys describe the package itself. The repo uses # both spellings: PEP 621 `[project]` and legacy `[tool.poetry]`. VERSION_TABLES = ("project", "tool.poetry") @@ -81,6 +85,16 @@ def dir(self) -> Path: def rel(self) -> str: return str(self.path.parent.relative_to(REPO_ROOT)) or "." + @property + def publishes(self) -> bool: + """Whether tagging this package uploads it to PyPI, or the tag is the whole release. + + `vintasend-api` and `vintasend-templates-management-api` are applications: they have a + CI workflow but no publish workflow, so PyPI never hears of them. A script waiting for + one of them to appear there would wait until its timeout. + """ + return (self.dir / PUBLISH_WORKFLOW).is_file() + def read_metadata(pkg: Package) -> None: """Pull `name` and `version` out of the package's own declaring table.""" @@ -204,6 +218,14 @@ def release_waves(packages: list[Package]) -> list[list[Package]]: root. Each wave is a set that can go together; the next one waits for it. """ by_name = {p.name: p for p in packages} + for pkg in packages: + for dep in sibling_deps(pkg): + sibling = by_name.get(dep.name) + if sibling is not None and not sibling.publishes and not dep.local: + raise PackageError( + f"{pkg.name} depends on {dep.name}, which is released as a tag only and " + "never reaches PyPI, so that dependency can never resolve" + ) needs = { p.name: {d.name for d in sibling_deps(p) if d.name in by_name and d.name != p.name} for p in packages @@ -229,7 +251,7 @@ def wave_lines(waves: list[list[Package]]) -> list[str]: rendered = ["release order (each wave waits for the one before it to be live on PyPI):"] for number, wave in enumerate(waves, start=1): body = textwrap.fill( - ", ".join(p.name for p in wave), + ", ".join(p.name if p.publishes else f"{p.name} (tag-only)" for p in wave), width=84, subsequent_indent=" " * 12, break_on_hyphens=False, diff --git a/scripts/_pypi.py b/scripts/_pypi.py index b43152e..23ccc81 100644 --- a/scripts/_pypi.py +++ b/scripts/_pypi.py @@ -8,6 +8,8 @@ from __future__ import annotations +import json +import re import time import urllib.error import urllib.request @@ -16,6 +18,9 @@ TIMEOUT = 15 +# The JSON flavour of the simple index (PEP 691), which lists every version (PEP 700). +SIMPLE_INDEX_ACCEPT = "application/vnd.pypi.simple.v1+json" + _SEEN: dict[tuple[str, str], bool | None] = {} @@ -35,17 +40,52 @@ def on_pypi(name: str, version: str, refresh: bool = False) -> bool | None: if key in _SEEN and not refresh: return _SEEN[key] + found = _on_json_api(name, version) + if found is True: + found = _on_simple_index(name, version) + + _SEEN[key] = found + return found + + +def _on_json_api(name: str, version: str) -> bool | None: + """Whether PyPI's JSON API knows `name==version`. None means the question failed.""" url = f"https://pypi.org/pypi/{name}/{version}/json" try: with urllib.request.urlopen(url, timeout=TIMEOUT) as response: # noqa: S310 -- literal https - found: bool | None = response.status == 200 + return response.status == 200 except urllib.error.HTTPError as exc: - found = False if exc.code == 404 else None + return False if exc.code == 404 else None except (urllib.error.URLError, TimeoutError, OSError): - found = None + return None - _SEEN[key] = found - return found + +def _on_simple_index(name: str, version: str) -> bool | None: + """Whether the simple index -- what pip actually resolves against -- lists `version`. + + The JSON API can report an upload before the simple index does. In the 3.2.0 release, + two packages' publish jobs failed with "No matching distribution found for + vintasend==3.2.0" a couple of minutes after the JSON API had answered yes, while their + other matrix jobs, moments later, installed it fine. So a version only counts as live + once the index pip reads lists it too. + """ + normalized = re.sub(r"[-_.]+", "-", name).lower() # PEP 503 + request = urllib.request.Request( # noqa: S310 -- literal https + f"https://pypi.org/simple/{normalized}/", + headers={"Accept": SIMPLE_INDEX_ACCEPT}, + ) + try: + with urllib.request.urlopen(request, timeout=TIMEOUT) as response: # noqa: S310 + payload = json.load(response) + except urllib.error.HTTPError as exc: + return False if exc.code == 404 else None + except (urllib.error.URLError, TimeoutError, OSError, ValueError): + return None + + versions = payload.get("versions") if isinstance(payload, dict) else None + if not isinstance(versions, list): + return None + return version in versions def wait_for_pypi( diff --git a/scripts/release_all.py b/scripts/release_all.py index 0df8a9d..bea91ba 100755 --- a/scripts/release_all.py +++ b/scripts/release_all.py @@ -10,6 +10,10 @@ 5. wait until every package in the wave is on PyPI 6. back to 3 for the next wave, until nothing is left +A package with no `.github/workflows/publish.yml` -- `vintasend-api` and +`vintasend-templates-management-api`, which are applications -- is released by its +tag alone. It is done once the tag is on origin, and nothing waits for it on PyPI. + The waiting is what makes this a script rather than a list. A subpackage pins `vintasend` at the version being released, so `poetry lock` cannot resolve until the root is actually live, and `vintasend-django-templates-manager` cannot @@ -70,13 +74,20 @@ def run_script(name: str, *args: str) -> int: return subprocess.run(argv, cwd=REPO_ROOT, check=False).returncode # noqa: S603 -def published(pkg: Package, version: str) -> bool: - return on_pypi(pkg.name, version) is True +def released(pkg: Package, version: str) -> bool: + """Whether `pkg` is already out at `version`. + + For a package that publishes, that means installable from PyPI. A tag-only package (see + `Package.publishes`) never reaches PyPI, so for it the pushed tag is the release. + """ + if pkg.publishes: + return on_pypi(pkg.name, version) is True + return remote_tag_exists(pkg.dir, f"v{version}") def release_root(root: Package, version: str, args: argparse.Namespace) -> tuple[bool, str]: """Tag and publish the root package. Returns (ok, what happened).""" - if published(root, version): + if released(root, version): return True, f"{root.name} {version} is already on PyPI" if remote_tag_exists(REPO_ROOT, f"v{version}"): @@ -105,9 +116,9 @@ def release_wave( wave: list[Package], number: int, version: str, args: argparse.Namespace ) -> tuple[bool, str]: """Lock, commit, tag and publish one wave of subpackages.""" - todo = [pkg for pkg in wave if not published(pkg, version)] + todo = [pkg for pkg in wave if not released(pkg, version)] if not todo: - return True, f"wave {number} is already on PyPI" + return True, f"wave {number} is already released" only: list[str] = [] for pkg in todo: @@ -122,8 +133,14 @@ def release_wave( " publishing; re-run this script once the rest is fixed." ) + # A tag-only package is done once tag_subpackages.py has pushed its tag; only the rest + # have an upload to wait for. + uploading = [pkg.name for pkg in todo if pkg.publishes] + if not uploading: + return True, f"wave {number} tagged: {', '.join(pkg.name for pkg in todo)}" + print(f"\nwaiting for wave {number} to reach PyPI ...") - late = wait_for_pypi([pkg.name for pkg in todo], version, args.timeout * 60, args.poll) + late = wait_for_pypi(uploading, version, args.timeout * 60, args.poll) if late: return False, ( f"these did not reach PyPI within {args.timeout:.0f} minutes: {', '.join(late)}\n" @@ -203,15 +220,19 @@ def main() -> int: for line in wave_lines(waves): print(line) - print("\nalready on PyPI:") - done = [pkg.name for pkg in packages if published(pkg, version)] + print("\nalready released:") + done = [ + pkg.name if pkg.publishes else f"{pkg.name} (tagged)" + for pkg in packages + if released(pkg, version) + ] print(f" {', '.join(done) if done else '(nothing yet)'}") if args.dry_run: print("\n" + "=" * 78) print("DRY RUN -- running each script's own checks, publishing nothing") print("=" * 78) - if published(root, version): + if released(root, version): print(f"\nskipping tag_release.py: {root.name} {version} is already on PyPI") else: run_script("tag_release.py", "--dry-run")