Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions static/compatibilities/astarte-operator.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
icon: https://raw.githubusercontent.com/astarte-platform/astarte-kubernetes-operator/master/mascotte.svg
git_url: https://github.com/astarte-platform/astarte-kubernetes-operator
release_url: https://github.com/astarte-platform/astarte-kubernetes-operator/releases/tag/v{vsn}
readme_url: https://docs.astarte-platform.org/astarte-kubernetes-operator/latest/001-intro_administrator.html
helm_repository_url: https://helm.astarte-platform.org
chart_name: astarte-operator
versions:
- version: 26.5.1
kube: ['1.35', '1.34', '1.33', '1.32', '1.31', '1.30', '1.29', '1.28', '1.27', '1.26',
'1.25', '1.24']
chart_version: 26.5.2
requirements: [Astarte v1.3.x]
incompatibilities: []
- version: 24.5.2
kube: ['1.35', '1.34', '1.33', '1.32', '1.31', '1.30', '1.29', '1.28', '1.27', '1.26',
'1.25', '1.24']
chart_version: 24.5.2
requirements: [Astarte v1.2.1 - v1.2.x]
incompatibilities: []
- version: 24.5.1
kube: ['1.35', '1.34', '1.33', '1.32', '1.31', '1.30', '1.29', '1.28', '1.27', '1.26',
'1.25', '1.24']
chart_version: 24.5.1
requirements: [Astarte v1.0 - v1.2.0]
incompatibilities: []
- version: 23.5.2
kube: ['1.35', '1.34', '1.33', '1.32', '1.31', '1.30', '1.29', '1.28', '1.27', '1.26',
'1.25', '1.24', '1.23', '1.22']
chart_version: 23.5.2
requirements: [Astarte v1.0 - v1.2.0]
incompatibilities: []
- version: 22.11.1
kube: ['1.35', '1.34', '1.33', '1.32', '1.31', '1.30', '1.29', '1.28', '1.27', '1.26',
'1.25', '1.24', '1.23', '1.22']
chart_version: 22.11.1
requirements: [Astarte v1.0 - v1.2.0]
incompatibilities: []
1 change: 1 addition & 0 deletions static/compatibilities/manifest.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
names:
- astarte-operator
- argo-rollouts
- aws-ebs-csi-driver
- aws-efs-csi-driver
Expand Down
195 changes: 195 additions & 0 deletions utils/compatibility/scrapers/astarte-operator.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
from __future__ import annotations

import re
from collections import OrderedDict

import yaml
from bs4 import BeautifulSoup

from utils import (
current_kube_version,
expand_kube_versions,
fetch_page,
print_error,
update_compatibility_info,
validate_semver,
)


APP_NAME = "astarte-operator"
DOCS_URL = "https://docs.astarte-platform.org/astarte-kubernetes-operator/latest/001-intro_administrator.html"
HELM_INDEX_URL = "https://helm.astarte-platform.org/index.yaml"
TARGET_FILE = f"../../static/compatibilities/{APP_NAME}.yaml"


def _decode(content):
return content.decode("utf-8", errors="replace") if isinstance(content, bytes) else content


def _fetch_index():
content = fetch_page(HELM_INDEX_URL)
if not content:
return None
try:
return yaml.safe_load(content)
except yaml.YAMLError as exc:
print_error(f"Failed to parse Astarte Operator helm index: {exc}")
return None


def _normalize_version(value: str) -> str | None:
match = re.search(r"v?(\d+\.\d+(?:\.\d+)?)", value)
return match.group(1) if match else None


def _series_key(version: str) -> str:
parts = version.split(".")
return ".".join(parts[:2])


def _series_specificity(version: str) -> int:
return len(version.split("."))


def _parse_min_kube(value: str) -> list[str]:
match = re.search(r"v?(\d+\.\d+)\+", value)
if not match:
return []
end = current_kube_version()
if not end:
print_error("KUBE_VERSION not available; cannot expand kube versions.")
return []
return expand_kube_versions(match.group(1), end)


def _latest_chart_entries(entries) -> dict[str, dict[str, str]]:
latest: dict[str, dict[str, object]] = {}

for chart in entries:
raw_app = str(chart.get("appVersion", "")).lstrip("v")
raw_chart = str(chart.get("version", "")).lstrip("v")

app_semver = validate_semver(raw_app)
if not app_semver:
continue
chart_semver = validate_semver(raw_chart)
key = _series_key(str(app_semver))
current = latest.get(key)

if not current or app_semver > current["_app_semver"]:
latest[key] = {
"version": str(app_semver),
"chart_version": str(chart_semver) if chart_semver else "",
"_app_semver": app_semver,
}

for entry in latest.values():
entry.pop("_app_semver", None)

return latest


def _find_compatibility_table(soup: BeautifulSoup):
for table in soup.find_all("table"):
headers = [th.get_text(" ", strip=True) for th in table.find_all("th")]
if (
"Astarte Operator Version" in headers
and "Astarte Version" in headers
and "Kubernetes Version" in headers
):
return table, headers
return None, []
Comment on lines +93 to +101

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 _find_compatibility_table only validates that "Astarte Operator Version" and "Kubernetes Version" appear in the headers, but _parse_compatibility_rows unconditionally calls headers.index("Astarte Version"). If the docs page ever renders a table that matches the first two columns but lacks the "Astarte Version" column (renamed, merged, or split), the scraper crashes with an uncaught ValueError instead of reporting a clean error and exiting.

Suggested change
for table in soup.find_all("table"):
headers = [th.get_text(" ", strip=True) for th in table.find_all("th")]
if "Astarte Operator Version" in headers and "Kubernetes Version" in headers:
return table, headers
return None, []
for table in soup.find_all("table"):
headers = [th.get_text(" ", strip=True) for th in table.find_all("th")]
if (
"Astarte Operator Version" in headers
and "Astarte Version" in headers
and "Kubernetes Version" in headers
):
return table, headers
return None, []



def _parse_compatibility_rows(table, headers):
operator_idx = headers.index("Astarte Operator Version")
astarte_idx = headers.index("Astarte Version")
kube_idx = headers.index("Kubernetes Version")
rows = []

for row in table.find_all("tr")[1:]:
cells = [td.get_text(" ", strip=True) for td in row.find_all("td")]
if len(cells) <= max(operator_idx, astarte_idx, kube_idx):
continue

operator_version = _normalize_version(cells[operator_idx])
kube_versions = _parse_min_kube(cells[kube_idx])
if not operator_version or not kube_versions:
continue

rows.append(
{
"operator_version": operator_version,
"series": _series_key(operator_version),
"specificity": _series_specificity(operator_version),
"kube": kube_versions,
"astarte": cells[astarte_idx].replace("–", "-"),
}
)

return rows


def _best_row_for(entry: dict[str, str], compatibility_rows):
version = entry["version"]
exact = [row for row in compatibility_rows if version == row["operator_version"]]
if exact:
return exact[0]

series = _series_key(version)
series_rows = [
row
for row in compatibility_rows
if row["series"] == series and row["specificity"] == 2
]
return series_rows[0] if series_rows else None


def scrape():
page_content = fetch_page(DOCS_URL)
if not page_content:
print_error("Failed to fetch Astarte Operator documentation.")
return

soup = BeautifulSoup(_decode(page_content), "html.parser")
table, headers = _find_compatibility_table(soup)
if not table:
print_error("Astarte Operator compatibility matrix not found.")
return

compatibility_rows = _parse_compatibility_rows(table, headers)
if not compatibility_rows:
print_error("No Astarte Operator compatibility rows parsed.")
return

index = _fetch_index()
if not index:
return

entries = index.get("entries", {}).get(APP_NAME, [])
if not entries:
print_error("No Astarte Operator chart entries found in helm index.")
return

versions = []
for entry in _latest_chart_entries(entries).values():
row = _best_row_for(entry, compatibility_rows)
if not row:
continue

version_info = OrderedDict(
[
("version", entry["version"]),
("kube", row["kube"]),
("chart_version", entry["chart_version"]),
("requirements", [f"Astarte {' '.join(row['astarte'].split())}"]),
("incompatibilities", []),
]
)
versions.append(version_info)

if not versions:
print_error("No Astarte Operator versions extracted from compatibility matrix.")
return

update_compatibility_info(TARGET_FILE, versions)