Stop the release waiting on unpublished apps and stale indexes

ai-tools/skills/release-package/SKILL.md

@@ -146,6 +146,15 @@ Follow that precedent rather than unilaterally switching to major bumps. But:
    half-published. A package already tagged on origin is reported as done, which is what makes the
    second and third runs safe.
 
+   `vintasend-api` and `vintasend-templates-management-api` are applications: they have no
+   `.github/workflows/publish.yml`, so their tag is the whole release and they never appear on
+   PyPI. The wave map marks them `(tag-only)`. Don't wait for them there.
+
+   A version counts as live only once PyPI's simple index lists it, not just the JSON API. pip
+   reads the simple index, and in 3.2.0 two Python 3.12 publish jobs failed with "No matching
+   distribution" a couple of minutes after the JSON API already listed the new `vintasend`. If a
+   job still fails that way, re-run its failed jobs on the same tag: nothing was uploaded.
+
    [`scripts/release_all.py`](../../../scripts/release_all.py) drives that entire loop — root tag,
    wait for PyPI, then lock/tag/wait per wave — in one unattended run:
 

scripts/_packages.py

@@ -29,6 +29,10 @@
 # skipped in silence.
 NON_PYTHON_SUBMODULES = frozenset({"tools/vintasend-dashboard"})
 
+# A package with this workflow uploads to PyPI when it is tagged. One without it is an
+# application released as a tag alone -- see `Package.publishes`.
+PUBLISH_WORKFLOW = Path(".github/workflows/publish.yml")
+
 # Tables whose `name`/`version` keys describe the package itself. The repo uses
 # both spellings: PEP 621 `[project]` and legacy `[tool.poetry]`.
 VERSION_TABLES = ("project", "tool.poetry")
@@ -81,6 +85,16 @@ def dir(self) -> Path:
     def rel(self) -> str:
         return str(self.path.parent.relative_to(REPO_ROOT)) or "."
 
+    @property
+    def publishes(self) -> bool:
+        """Whether tagging this package uploads it to PyPI, or the tag is the whole release.
+
+        `vintasend-api` and `vintasend-templates-management-api` are applications: they have a
+        CI workflow but no publish workflow, so PyPI never hears of them. A script waiting for
+        one of them to appear there would wait until its timeout.
+        """
+        return (self.dir / PUBLISH_WORKFLOW).is_file()
+
 
 def read_metadata(pkg: Package) -> None:
     """Pull `name` and `version` out of the package's own declaring table."""
@@ -204,6 +218,14 @@ def release_waves(packages: list[Package]) -> list[list[Package]]:
     root. Each wave is a set that can go together; the next one waits for it.
     """
     by_name = {p.name: p for p in packages}
+    for pkg in packages:
+        for dep in sibling_deps(pkg):
+            sibling = by_name.get(dep.name)
+            if sibling is not None and not sibling.publishes and not dep.local:
+                raise PackageError(
+                    f"{pkg.name} depends on {dep.name}, which is released as a tag only and "
+                    "never reaches PyPI, so that dependency can never resolve"
+                )
     needs = {
         p.name: {d.name for d in sibling_deps(p) if d.name in by_name and d.name != p.name}
         for p in packages
@@ -229,7 +251,7 @@ def wave_lines(waves: list[list[Package]]) -> list[str]:
     rendered = ["release order (each wave waits for the one before it to be live on PyPI):"]
     for number, wave in enumerate(waves, start=1):
         body = textwrap.fill(
-            ", ".join(p.name for p in wave),
+            ", ".join(p.name if p.publishes else f"{p.name} (tag-only)" for p in wave),
             width=84,
             subsequent_indent=" " * 12,
             break_on_hyphens=False,

scripts/_pypi.py

@@ -8,6 +8,8 @@
 
 from __future__ import annotations
 
+import json
+import re
 import time
 import urllib.error
 import urllib.request
@@ -16,6 +18,9 @@
 
 TIMEOUT = 15
 
+# The JSON flavour of the simple index (PEP 691), which lists every version (PEP 700).
+SIMPLE_INDEX_ACCEPT = "application/vnd.pypi.simple.v1+json"
+
 _SEEN: dict[tuple[str, str], bool | None] = {}
 
 
@@ -35,17 +40,52 @@ def on_pypi(name: str, version: str, refresh: bool = False) -> bool | None:
     if key in _SEEN and not refresh:
         return _SEEN[key]
 
+    found = _on_json_api(name, version)
+    if found is True:
+        found = _on_simple_index(name, version)
+
+    _SEEN[key] = found
+    return found
+
+
+def _on_json_api(name: str, version: str) -> bool | None:
+    """Whether PyPI's JSON API knows `name==version`. None means the question failed."""
     url = f"https://pypi.org/pypi/{name}/{version}/json"
     try:
         with urllib.request.urlopen(url, timeout=TIMEOUT) as response:  # noqa: S310 -- literal https
-            found: bool | None = response.status == 200
+            return response.status == 200
     except urllib.error.HTTPError as exc:
-        found = False if exc.code == 404 else None
+        return False if exc.code == 404 else None
     except (urllib.error.URLError, TimeoutError, OSError):
-        found = None
+        return None
 
-    _SEEN[key] = found
-    return found
+
+def _on_simple_index(name: str, version: str) -> bool | None:
+    """Whether the simple index -- what pip actually resolves against -- lists `version`.
+
+    The JSON API can report an upload before the simple index does. In the 3.2.0 release,
+    two packages' publish jobs failed with "No matching distribution found for
+    vintasend==3.2.0" a couple of minutes after the JSON API had answered yes, while their
+    other matrix jobs, moments later, installed it fine. So a version only counts as live
+    once the index pip reads lists it too.
+    """
+    normalized = re.sub(r"[-_.]+", "-", name).lower()  # PEP 503
+    request = urllib.request.Request(  # noqa: S310 -- literal https
+        f"https://pypi.org/simple/{normalized}/",
+        headers={"Accept": SIMPLE_INDEX_ACCEPT},
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=TIMEOUT) as response:  # noqa: S310
+            payload = json.load(response)
+    except urllib.error.HTTPError as exc:
+        return False if exc.code == 404 else None
+    except (urllib.error.URLError, TimeoutError, OSError, ValueError):
+        return None
+
+    versions = payload.get("versions") if isinstance(payload, dict) else None
+    if not isinstance(versions, list):
+        return None
+    return version in versions
 
 
 def wait_for_pypi(

scripts/release_all.py

@@ -10,6 +10,10 @@
     5. wait                    until every package in the wave is on PyPI
     6. back to 3 for the next wave, until nothing is left
 
+A package with no `.github/workflows/publish.yml` -- `vintasend-api` and
+`vintasend-templates-management-api`, which are applications -- is released by its
+tag alone. It is done once the tag is on origin, and nothing waits for it on PyPI.
+
 The waiting is what makes this a script rather than a list. A subpackage pins
 `vintasend` at the version being released, so `poetry lock` cannot resolve until
 the root is actually live, and `vintasend-django-templates-manager` cannot
@@ -70,13 +74,20 @@ def run_script(name: str, *args: str) -> int:
     return subprocess.run(argv, cwd=REPO_ROOT, check=False).returncode  # noqa: S603
 
 
-def published(pkg: Package, version: str) -> bool:
-    return on_pypi(pkg.name, version) is True
+def released(pkg: Package, version: str) -> bool:
+    """Whether `pkg` is already out at `version`.
+
+    For a package that publishes, that means installable from PyPI. A tag-only package (see
+    `Package.publishes`) never reaches PyPI, so for it the pushed tag is the release.
+    """
+    if pkg.publishes:
+        return on_pypi(pkg.name, version) is True
+    return remote_tag_exists(pkg.dir, f"v{version}")
 
 
 def release_root(root: Package, version: str, args: argparse.Namespace) -> tuple[bool, str]:
     """Tag and publish the root package. Returns (ok, what happened)."""
-    if published(root, version):
+    if released(root, version):
         return True, f"{root.name} {version} is already on PyPI"
 
     if remote_tag_exists(REPO_ROOT, f"v{version}"):
@@ -105,9 +116,9 @@ def release_wave(
     wave: list[Package], number: int, version: str, args: argparse.Namespace
 ) -> tuple[bool, str]:
     """Lock, commit, tag and publish one wave of subpackages."""
-    todo = [pkg for pkg in wave if not published(pkg, version)]
+    todo = [pkg for pkg in wave if not released(pkg, version)]
     if not todo:
-        return True, f"wave {number} is already on PyPI"
+        return True, f"wave {number} is already released"
 
     only: list[str] = []
     for pkg in todo:
@@ -122,8 +133,14 @@ def release_wave(
             "  publishing; re-run this script once the rest is fixed."
         )
 
+    # A tag-only package is done once tag_subpackages.py has pushed its tag; only the rest
+    # have an upload to wait for.
+    uploading = [pkg.name for pkg in todo if pkg.publishes]
+    if not uploading:
+        return True, f"wave {number} tagged: {', '.join(pkg.name for pkg in todo)}"
+
     print(f"\nwaiting for wave {number} to reach PyPI ...")
-    late = wait_for_pypi([pkg.name for pkg in todo], version, args.timeout * 60, args.poll)
+    late = wait_for_pypi(uploading, version, args.timeout * 60, args.poll)
     if late:
         return False, (
             f"these did not reach PyPI within {args.timeout:.0f} minutes: {', '.join(late)}\n"
@@ -203,15 +220,19 @@ def main() -> int:
     for line in wave_lines(waves):
         print(line)
 
-    print("\nalready on PyPI:")
-    done = [pkg.name for pkg in packages if published(pkg, version)]
+    print("\nalready released:")
+    done = [
+        pkg.name if pkg.publishes else f"{pkg.name} (tagged)"
+        for pkg in packages
+        if released(pkg, version)
+    ]
     print(f"  {', '.join(done) if done else '(nothing yet)'}")
 
     if args.dry_run:
         print("\n" + "=" * 78)
         print("DRY RUN -- running each script's own checks, publishing nothing")
         print("=" * 78)
-        if published(root, version):
+        if released(root, version):
             print(f"\nskipping tag_release.py: {root.name} {version} is already on PyPI")
         else:
             run_script("tag_release.py", "--dry-run")