diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..634cdf4 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Default code owners +* @d-hickey @pohuigin diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..224bd3e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: "uv" + directory: "/backend/" + schedule: + interval: "monthly" + open-pull-requests-limit: 20 + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + open-pull-requests-limit: 20 diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 0000000..372b854 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,48 @@ +# File types +documentation: +- changed-files: + - any-glob-to-any-file: ['**/*.md', '**/*.mdx'] + +localization: +- changed-files: + - any-glob-to-any-file: ['**/localization/**/*'] + +test: +- changed-files: + - any-glob-to-any-file: ['**/*.test.ts', '**/*.test.tsx', '**/*.spec.ts', '**/*.spec.tsx'] + + +# Languages +python: +- changed-files: + - any-glob-to-any-file: ['**/*.py'] + +typescript: +- changed-files: + - any-glob-to-any-file: ['**/*.ts'] + +react: +- changed-files: + - any-glob-to-any-file: ['**/*.tsx'] + +c-sharp: +- changed-files: + - any-glob-to-any-file: ['**/*.cs'] + +yaml: +- changed-files: + - any-glob-to-any-file: ['**/*.yml'] + + +# Functional areas +backend: +- changed-files: + - any-glob-to-any-file: ['backend/**/*'] + +frontend: +- changed-files: + - any-glob-to-any-file: ['frontend/**/*'] + +github actions: +- changed-files: + - any-glob-to-any-file: ['.github/**/*'] diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..d337156 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,8 @@ +## What? + +... + +For more background, see ticket +#L3D-123. + +## Testing diff --git a/.github/release.yaml b/.github/release.yaml new file mode 100644 index 0000000..8cf0c0e --- /dev/null +++ b/.github/release.yaml @@ -0,0 +1,25 @@ +changelog: + categories: + - title: 🚨 Breaking Changes + labels: + - breaking change + + - title: ✨ Features + labels: + - feature + + - title: 🐛 Bugs + labels: + - bug + + - title: 📋 Tasks + labels: + - task + + - title: 📑 Change Requests + labels: + - CR + + - title: 📦 Other Changes + labels: + - "*" \ No newline at end of file diff --git a/.github/workflows/backend_validate.yml b/.github/workflows/backend_validate.yml new file mode 100644 index 0000000..5673397 --- /dev/null +++ b/.github/workflows/backend_validate.yml @@ -0,0 +1,76 @@ +name: Backend Build, Lint, and Test + +on: + workflow_call: + + # Allow manual trigger + workflow_dispatch: + +jobs: + build: + name: Build and Test + # Switching to self-hosted runner from `runs-on: ubuntu-latest` + # TEMP: runner filtering (we prob need new dedicated runners): + runs-on: [ubuntu-latest] + defaults: + run: + working-directory: ./backend + # Don't run the job if the PR is a draft PR + if: ${{ !github.event.pull_request.draft }} + + services: + postgres: + image: postgres:alpine + env: + POSTGRES_USER: ${{ env.PROCRASTINATE_DB_USER }} + POSTGRES_DB: ${{ env.PROCRASTINATE_DB_NAME }} + POSTGRES_PASSWORD: ${{ env.PROCRASTINATE_DB_PASSWORD }} + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + env: + PROCRASTINATE_DB_USER: db_admin + PROCRASTINATE_DB_NAME: shredder_procrastinate_db + PROCRASTINATE_DB_PASSWORD: db_password + # Determine whether the workflow was triggered by merge queue + MERGE_QUEUE: ${{ github.event_name == 'merge_group' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 1 + + - name: Setup Python + uses: astral-sh/setup-uv@v8.0.0 + with: + enable-cache: true + cache-dependency-glob: "./pyproject.toml" + working-directory: "./backend" + prune-cache: false + + - name: Build + run: | + cp .env.example .env + make env + make build + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any + + - name: Lint + id: run-lint + run: | + make lint + + - name: Run Tests with Coverage + run: | + make test-cov + + - name: Coverage threshold (warn only) + continue-on-error: true + run: | + COVERAGE_FILE=.pytest_coverage ~/.local/bin/uv run coverage report --fail-under=80 diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 0000000..13d5e7c --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,20 @@ +# This workflow will triage pull requests and apply a label based on the +# paths that are modified in the pull request. +# +# To use this workflow, you will need to set up a .github/labeler.yml +# file with configuration. For more information, see: +# https://github.com/actions/labeler + +name: "PR Labeler 🏷️" +on: [pull_request] + +jobs: + triage: + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + steps: + - uses: actions/labeler@v6 + with: + sync-labels: true diff --git a/.github/workflows/lint-actions.yml b/.github/workflows/lint-actions.yml new file mode 100644 index 0000000..3b92ae7 --- /dev/null +++ b/.github/workflows/lint-actions.yml @@ -0,0 +1,22 @@ +# This workflow will lint github actions workflows +# on pull requests that modify the .github directory. + +name: "Workflow linter 🧹" +on: + workflow_call: + +jobs: + lint: + name: Lint Workflows + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 1 + - uses: actions/setup-go@v5 + with: + go-version: "1.24.0" # The Go version to download (if necessary) and use. + cache: true + - run: go install github.com/rhysd/actionlint/cmd/actionlint@latest + - run: actionlint diff --git a/.github/workflows/pull_request_verify.yml b/.github/workflows/pull_request_verify.yml new file mode 100644 index 0000000..8a3583f --- /dev/null +++ b/.github/workflows/pull_request_verify.yml @@ -0,0 +1,61 @@ +name: Verify Changes + +# This workflow will run on PR open or update +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + merge_group: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + paths: + runs-on: ubuntu-latest + permissions: + pull-requests: read + actions: read + contents: read + # Set job outputs to values from filter step + outputs: + backend: ${{ steps.filter.outputs.backend }} + lint_actions: ${{ steps.filter.outputs.lint_actions }} + steps: + - name: Checkout code + if: github.event_name == 'merge_group' # Run only for merge queue + uses: actions/checkout@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: dorny/paths-filter@v4 + id: filter + with: + filters: | + backend: + - "backend/**" + lint_actions: + - ".github/**" + + # Verify Backend + backend: + needs: paths + uses: ./.github/workflows/backend_validate.yml + if: ${{ needs.paths.outputs.backend == 'true' }} + + # Verify Workflows + lint_actions: + needs: paths + if: ${{ needs.paths.outputs.lint_actions == 'true' }} + uses: ./.github/workflows/lint-actions.yml + + # Verify All Passed + verify_all_passed: + needs: [backend, lint_actions] + runs-on: ubuntu-latest + if: ${{ always() && (failure() || cancelled()) }} + steps: + - name: Fail job + uses: actions/github-script@v8.0.0 + with: + script: core.setFailed('Some checks failed or cancelled.') diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..115af5d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,72 @@ +name: Release Backend + +on: + workflow_call: + inputs: + environment: + required: true + type: string + +jobs: + build-and-push: + environment: ${{ inputs.environment }} + name: Backend Release to GitHub + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + defaults: + run: + working-directory: ./backend + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 1 + + - name: Generate .env file + run: | + cp .env.example .env + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker (app) + id: meta + uses: docker/metadata-action@v6 + with: + images: ghcr.io/propylon/${{ github.repository }}/backend + + - name: Build and push Docker image (app) + uses: docker/build-push-action@v7 + with: + context: ./backend/. + file: ./backend/Dockerfile + target: migrate_stage + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Extract metadata (tags, labels) for Docker (worker) + id: meta-worker + uses: docker/metadata-action@v6 + with: + images: ghcr.io/propylon/${{ github.repository }}/backend-worker + + - name: Build and push Docker image (worker) + uses: docker/build-push-action@v7 + with: + context: ./backend/. + file: ./backend/Dockerfile + target: worker_stage + push: true + tags: ${{ steps.meta-worker.outputs.tags }} + labels: ${{ steps.meta-worker.outputs.labels }} \ No newline at end of file diff --git a/backend/.env.example b/backend/.env.example new file mode 100644 index 0000000..4f4581e --- /dev/null +++ b/backend/.env.example @@ -0,0 +1,36 @@ +# Deploy +DEBUG=1 +ENV_TYPE=local +SERVICE_NAME=shredder_app + +# Build +PYTHON_VERSION=3.13 +PYTHONPATH=./shredder/src +ENV_DIR=.venv +UV_PROJECT_ENVIRONMENT=${ENV_DIR} + +# Docker +DOCKER_COMPOSE_PROJECT=shredder +COMPOSE_ENV_FILES=.env +DOCKER_NETWORK_NAME=backend + +# Procrastinate +PROCRASTINATE_APP=shredder.app +PROCRASTINATE_WORKER=shredder_worker +PROCRASTINATE_TASK_PATHS=shredder.tasks +PROCRASTINATE_QUEUE=shredder_queue +# Specify how many jobs are handled at once by the Procrastinate worker +# (depends on max threads and also how much downtime is expected in the job flow: I/O, network latency, etc.) +PROCRASTINATE_CONCURRENCY=4 +# Seconds. Maximum time a job is allowed to run before being force-failed +PROCRASTINATE_MAX_JOB_TTL=600 +# Postgres DB Connection +PROCRASTINATE_DB_HOST=127.0.0.1 +PROCRASTINATE_DB_PORT=5432 +PROCRASTINATE_DB_NAME=shredder_procrastinate_db +# UPDATE ME: +PROCRASTINATE_DB_USER=db_admin +PROCRASTINATE_DB_PASSWORD=db_password + +# PDF2HTMLEX settings (optional) +SHREDDER_CONVERSION_CONFIG=--embed-css,0,--css-filename,pages.css,--embed-outline,1,--process-annotation,1,--decompose-ligature,1,--bg-format,svg diff --git a/backend/.gitignore b/backend/.gitignore new file mode 100644 index 0000000..b954175 --- /dev/null +++ b/backend/.gitignore @@ -0,0 +1,139 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +/static +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.pytest_coverage +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env_python* +.envrc +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# PyCharm +.idea/ + +# Local file system storage for running the env +/documents/* \ No newline at end of file diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..5db1d3e --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,69 @@ +# TODO: Make the build-stage dynamic. +# Currently the Python version and image/binary URLs are hard-coded + +ARG base_image=pdf2htmlex/pdf2htmlex:0.18.8.rc2-master-20200820-alpine-3.12.0-x86_64 + +# ################################################################################################# +# Stage 1: Build Image - Grab a prebuilt, portable Python 3.13 (musl/Alpine-compatible) +FROM ${base_image} AS buid-image + +# Download Python +ARG PYTHON_VERSION +RUN echo "PYTHON VERSION ${PYTHON_VERSION:?PYTHON_VERSION must be set}" +ARG PYTHON_BUILD_LIST_URL=https://api.github.com/repos/astral-sh/python-build-standalone/releases/latest + +RUN apk update +RUN apk add --no-cache curl +RUN apk add --no-cache jq + +RUN DOWNLOAD_URL=$(curl -sSL "$PYTHON_BUILD_LIST_URL" | \ + jq -r '.assets[] | select(.name | contains("cpython-'"$PYTHON_VERSION"'") and contains("x86_64-unknown-linux-musl-install_only_stripped")) | .browser_download_url') \ + && curl -sSL $DOWNLOAD_URL -o /tmp/python.tar.gz + +RUN mkdir -p /opt/python \ + && tar -xzf /tmp/python.tar.gz -C /opt/python --strip-components=1 + +# ################################################################################################# +# Stage 2: Service Image +FROM ${base_image} AS service-image + +# Copy the APK package registry to avoid re-running apk update +COPY --from=buid-image /var/cache/apk/ /var/cache/apk/ + +# Copy the Python installation +ARG PYTHON_VERSION +COPY --from=buid-image /opt/python /opt/python +ENV PATH="/opt/python/bin:${PATH}" +RUN ln -sf /opt/python/bin/python$PYTHON_VERSION /usr/local/bin/python3 \ + && ln -sf /opt/python/bin/python$PYTHON_VERSION /usr/local/bin/python +# Validate python +RUN python3 --version + +# For Procrastinate PG Connector - ctypes.find_library needs libpq's linker files and binutils on musl. +RUN apk add --no-cache postgresql-dev binutils + +COPY .env /propylon/ +COPY pyproject.toml /propylon/ +COPY uv.lock /propylon/ +COPY ./etc /propylon/etc/ +COPY ./shredder/src /propylon/shredder/src/ + +RUN chmod +x /propylon/etc/run_server.sh +RUN set -o allexport && source /propylon/.env && set +o allexport + +# TODO:Recommended by PDF2HTMLEX to get better results. Add to subprocess calls +# RUN apk add ttfautohint && \ + +RUN pip install uv +RUN uv venv --allow-existing --python "$PYTHON_VERSION" /propylon/.venv + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 +ENV PATH="/propylon/.venv/bin:$PATH" + +WORKDIR /propylon/ +RUN uv sync --locked --no-dev + +# Override the original base image entrypoint +ENTRYPOINT [] +CMD ["/bin/sh", "-c", "exec /propylon/etc/run_server.sh"] diff --git a/backend/Makefile b/backend/Makefile new file mode 100644 index 0000000..574e0e5 --- /dev/null +++ b/backend/Makefile @@ -0,0 +1,91 @@ +# Import all constants from .env file. +# defaults set in the .env file will be applied to the commands below. +include .env +export + +# Allow passing extra args, e.g. make docker -- --build +%: + @: + +UV=~/.local/bin/uv +IN_ENV=$(UV) run + +# ==================== +# Build +# ==================== + +# Running make env does not overwrite the existing venv folder +env: + curl -LsSf https://astral.sh/uv/install.sh | sh + $(UV) venv --allow-existing --python $(PYTHON_VERSION) $(ENV_DIR) + @printf "Environment installed at $(ENV_DIR), run the following command to activate it: \n$(IN_ENV)\n" + +# Build the environment +build: + $(UV) sync --locked --group dev + +env-update: + $(UV) lock --upgrade + +# ==================== +# Clean +# ==================== + +clean: + - @rm -rf ./*.egg-info + - @find ./ -name '*.pyc' -delete + - @find ./ -name '__pycache__' | xargs rm -rf + - @rm -f ./.pytest_coverage.* + +clean-env: clean + - @rm -rf $(ENV_DIR) + +clean-all: clean clean-env + @echo "Removed the virtual environment, build-files, and cached-files." + +# ==================== +# Docker +# ==================== + +docker-network: + -docker network create $(DOCKER_NETWORK_NAME) + +docker: docker-network + $(IN_ENV) docker-compose -p $(DOCKER_COMPOSE_PROJECT) up -d $(filter-out $@,$(MAKECMDGOALS)) + +# ==================== +# Formatting / Linting +# ==================== + +# Runs ruff check and fix using the pyproject.toml config +# TEMP: using unsafe fixes to speed things up +fixup: + $(IN_ENV) $(UV) run ruff check --fix + +lint: + $(IN_ENV) $(UV) run ruff check + $(IN_ENV) $(UV) run ty check + +# ==================== +# Testing +# ==================== + +test-functional: + $(IN_ENV) py.test -k test_functional + +test-integration: + $(IN_ENV) py.test -k test_integration + +test: test-functional + +test-suite: test-functional test-integration + +test-cov: + COVERAGE_PROCESS_START=pyproject.toml COVERAGE_FILE=.pytest_coverage $(IN_ENV) coverage run -m pytest -k test_functional + - COVERAGE_FILE=.pytest_coverage $(IN_ENV) coverage combine + COVERAGE_FILE=.pytest_coverage $(IN_ENV) coverage report + +test-cov-html: + COVERAGE_PROCESS_START=pyproject.toml COVERAGE_FILE=.pytest_coverage $(IN_ENV) coverage run -m pytest -k test_functional + - COVERAGE_FILE=.pytest_coverage $(IN_ENV) coverage combine + COVERAGE_FILE=.pytest_coverage $(IN_ENV) coverage html diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 0000000..469daba --- /dev/null +++ b/backend/README.md @@ -0,0 +1,96 @@ +# Propylon Shredder + +Shredder is a custom Procrastinate worker service that converts PDF files to paginated HTML with +[pdf2htmlEX](https://github.com/pdf2htmlEX/pdf2htmlEX). It reads jobs from +PostgreSQL and POSTs a ZIP of the converted files to each job's callback URL. + +## Run with Docker + +You need Docker, Docker Compose, and `make`. From this directory: + +1. Copy `.env.example` to `.env` and set the PostgreSQL connection values. Replace + the example password. The database must be reachable from the worker container; + `127.0.0.1` refers to the container itself, not your host. +2. Install `uv` and create the Python environment (the Makefile's `env` target + installs `uv` if needed): + + ```sh + make env + make build + ``` + +3. Create a PostgreSQL database for Procrastinate, then apply its schema once: + + ```sh + set -a + . ./.env + set +a + uv run procrastinate --app shredder.app schema --apply + ``` + +4. Start the worker and its Docker network: + + ```sh + make docker + ``` + +The worker listens to the queue set by `PROCRASTINATE_QUEUE`. Shredder does not +start PostgreSQL; provide a database separately and configure its connection in +`.env`. + +## Queue a conversion + +With the `.env` values exported as above, enqueue a job using Procrastinate's CLI: + +```sh +uv run procrastinate --app shredder.app defer shredder.tasks.convert_pdf_to_html \ + '{"doc_path":"https://example.com/input.pdf","call_back_url":"https://example.com/shredder-result","doc_provider":"url","metadata":{"reference":"123"}}' +``` + +`doc_path` must be a URL the worker can GET. `doc_provider` currently supports +only `url`. You can optionally include `call_back_auth`, which is sent as the +callback's `Authorization` header, and `metadata`, which is returned as +additional form fields. + +## Callback + +After conversion, Shredder sends a multipart POST to `call_back_url`. It includes +the ZIP in the `file` field and form fields `job_id`, `doc_path`, and +`doc_provider`, plus any supplied metadata (with fields flattened). The ZIP contains the generated HTML +and related files. +NOTE: do not include field names in the input metadata JSON conflicting with +`file`, `call_back_url`, `job_id`, `doc_path`, `doc_provider`. + +## Configuration + +Set values in `.env` for local Docker runs. The main settings are: + +- `PROCRASTINATE_DB_HOST`, `PROCRASTINATE_DB_PORT`, `PROCRASTINATE_DB_NAME`, + `PROCRASTINATE_DB_USER`, and `PROCRASTINATE_DB_PASSWORD`: PostgreSQL connection. +- `PROCRASTINATE_QUEUE`: queue to consume (default: `shredder_queue`). +- `PROCRASTINATE_CONCURRENCY`: number of jobs processed concurrently (default: `4`). +- `PROCRASTINATE_MAX_JOB_TTL`: maximum job runtime in seconds (default: `600`). +- `SHREDDER_CONVERSION_CONFIG`: optional comma-separated pdf2htmlEX arguments. + +See `.env.example` for the full set of build and worker settings. Do not commit +credentials. + +## Development + +The project requires Python 3.13.11. Install the development dependencies with: + +```sh +make env +make build +``` + +Tests: + +- `make test` runs the functional tests. They mock HTTP and `pdf2htmlEX`, so no + database or container is needed. +- `make test-integration` starts the worker container, queues a real job, and checks + the callback. It needs the database from `.env` reachable on `127.0.0.1` (override + with `TEST_PROCRASTINATE_DB_HOST`/`TEST_PROCRASTINATE_DB_PORT`). The test starts a + mock server on port 1234 that serves `example.pdf` and receives the callback; the + worker reaches it through `host.docker.internal`. +- `make test-suite` runs both. diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml new file mode 100644 index 0000000..4f56440 --- /dev/null +++ b/backend/docker-compose.yml @@ -0,0 +1,26 @@ +name: shredder + +services: + app: + container_name: propylon_shredder + hostname: propylon_shredder + build: + context: ${PWD} + dockerfile: Dockerfile + args: + PYTHON_VERSION: ${PYTHON_VERSION} + env_file: .env + command: ["/bin/sh", "-c", "exec /propylon/etc/run_server.sh"] + volumes: + - ${PWD}/shredder/src:/propylon/shredder/src + networks: + - backend + # Lets the worker reach services on the host, e.g. the integration test mock server. + extra_hosts: + - "host.docker.internal:host-gateway" + +networks: + backend: + external: true + name: ${DOCKER_NETWORK_NAME} + driver: bridge diff --git a/backend/etc/run_server.sh b/backend/etc/run_server.sh new file mode 100644 index 0000000..8a72cdc --- /dev/null +++ b/backend/etc/run_server.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env ash + +set -e + +# TODO: TEMP: currently pulling env vars from .env file directly +# In production they should be injected into the container at run time. +# NOTE: Could simply include an empty .env file for remote deployment to avoid clobbering variables. +set -a +. /propylon/.env +set +a + +export PYTHONPATH=/propylon/shredder/src + +echo "Starting worker..." + +# Run 1 ingest workers running a max of WORKER_CONCURRENCY at a time with WORKER_QUEUES queues +WORKER_QUEUES="${PROCRASTINATE_QUEUE:?PROCRASTINATE_QUEUE is not set}" +WORKER_CONCURRENCY="${PROCRASTINATE_CONCURRENCY:?PROCRASTINATE_CONCURRENCY is not set}" +exec procrastinate --log-format "%(asctime)s %(levelname)-7s %(name)s %(message)s" worker -q "$WORKER_QUEUES" -c "$WORKER_CONCURRENCY" + +# NOTE: exec lets the worker receive Docker’s shutdown signals directly diff --git a/backend/pyproject.toml b/backend/pyproject.toml new file mode 100644 index 0000000..29db1e3 --- /dev/null +++ b/backend/pyproject.toml @@ -0,0 +1,120 @@ +[project] +name = "propylon-shredder" +version = "0.1.0" +description = "Service to run PDF2HTMLEX library using Procrastinate." +requires-python = ">=3.13.11, <3.14" +dependencies = [ + "httpx>=0.28.1", + "procrastinate>=3.8.1", + "psycopg[pool]", + "sqlalchemy>=2.0.44", +] + +[dependency-groups] +dev = [ + "pytest>=8.0.0", + "pytest-asyncio>=0.23.0", + "coverage", + "ruff==0.16.3", + "ty==0.0.72", +] + +[build-system] +build-backend = "uv_build" +requires = ["uv_build>=0.10.1,<0.11.0"] + +[tool.uv] +required-environments = [ + "sys_platform == 'linux' and platform_machine == 'x86_64'", +] + +[tool.uv.build-backend] +module-name = "shredder" +module-root = "shredder/src" + +[tool.pytest.ini_options] +asyncio_mode = "auto" +pythonpath = [".", "shredder/src"] +testpaths = ["shredder/tests"] + +# Ruff configuration +[tool.ruff] +target-version = "py313" +line-length = 100 +indent-width = 4 +src = ["shredder/src", "shredder/tests"] +extend-exclude = [] # e.g. ["*.ipynb"] + +lint.select = [ + # --- Lean baseline --- + # Core correctness & style (pycodestyle / pyflakes) + "E", "W", "F", + # Imports ordering & cleanups + "I", + # Safe modernizations (pyupgrade) + "UP", + # Potential bugs / unsafe patterns + "B", + # Comprehension clarity + "C4", + # Phased expansion candidates (uncomment when ready): + # "RUF" - ruff's own high‑value internal rules + "RUF003", + "RUF005", + "RUF013", + "RUF017", + "RUF018", + "RUF019", + "RUF021", + "RUF022", + "RUF010", + "N", # naming consistency + "A", # builtins shadowing + "ARG", # unused function args + "ERA", # eradicate dead commented code + "TID", # tidy imports (import star / cyclic) + "SIM", # simplify constructs + "TRY", # exception handling patterns + "DTZ", # timezone-aware datetimes + "RET", # consistent returns + "PLR", # refactor metrics (complexity/too many branches) + "PTH", # prefer pathlib over os.path + "G", # logging format issues + "PGH" # invalid escape sequences, highlighting +] + +# Ignore the same rules as current flake8 config +lint.ignore = [ + "PLR0917", + "PLR0913", + "TRY003", +] + +lint.fixable = ["ALL"] + +# Ruff formatter configuration - replaces black +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" + +[tool.ruff.lint.isort] +combine-as-imports = true +known-first-party = ["shredder", "tests"] + +[tool.ruff.lint.isort.sections] + +[tool.ty.src] +include = ["shredder/src", "shredder/tests"] +exclude = [] + +[tool.coverage.report] +include = [ + "shredder/src/**/*.py" +] +skip_covered = false +exclude_lines = [ + "pragma: no cover", + "if TYPE_CHECKING:", +] \ No newline at end of file diff --git a/backend/shredder/src/shredder/__init__.py b/backend/shredder/src/shredder/__init__.py new file mode 100644 index 0000000..ea589cd --- /dev/null +++ b/backend/shredder/src/shredder/__init__.py @@ -0,0 +1,29 @@ +import os + +import procrastinate +from procrastinate.manager import JobManager + +from shredder import settings, utils + +DB_CONFIG_FSTRING = "postgres://{user}:{password}@{host}:{port}/{dbname}" + +PG_CONNECTOR = procrastinate.PsycopgConnector( + conninfo=DB_CONFIG_FSTRING.format( + user=os.environ["PROCRASTINATE_DB_USER"], + password=os.environ["PROCRASTINATE_DB_PASSWORD"], + host=os.getenv("PROCRASTINATE_DB_HOST", "127.0.0.1"), + port=os.environ["PROCRASTINATE_DB_PORT"], + dbname=os.environ["PROCRASTINATE_DB_NAME"], + ) +) + +JOB_MANAGER = JobManager(connector=PG_CONNECTOR) + +task_path_list = settings.PROCRASTINATE_TASK_PATHS + +app = procrastinate.App(import_paths=task_path_list, connector=PG_CONNECTOR) + +__all__ = [ + "settings", + "utils", +] diff --git a/backend/shredder/src/shredder/settings.py b/backend/shredder/src/shredder/settings.py new file mode 100644 index 0000000..18e35ad --- /dev/null +++ b/backend/shredder/src/shredder/settings.py @@ -0,0 +1,15 @@ +import os + +PROCRASTINATE_MAX_JOB_TTL = int(os.getenv("PROCRASTINATE_MAX_JOB_TTL", 600)) + +PROCRASTINATE_QUEUE = os.getenv("PROCRASTINATE_QUEUE", "shredder_queue") +PROCRASTINATE_WORKER = os.getenv("PROCRASTINATE_WORKER", "shredder_worker") +PROCRASTINATE_TASK_PATHS = os.environ["PROCRASTINATE_TASK_PATHS"].split(",") + +PROCRASTINATE_DB_USER = os.environ["PROCRASTINATE_DB_USER"] +PROCRASTINATE_DB_PASSWORD = os.environ["PROCRASTINATE_DB_PASSWORD"] +PROCRASTINATE_DB_HOST = os.environ["PROCRASTINATE_DB_HOST"] +PROCRASTINATE_DB_PORT = os.environ["PROCRASTINATE_DB_PORT"] +PROCRASTINATE_DB_NAME = os.environ["PROCRASTINATE_DB_NAME"] + +SHREDDER_CONVERSION_CONFIG = os.getenv("SHREDDER_CONVERSION_CONFIG") diff --git a/backend/shredder/src/shredder/tasks.py b/backend/shredder/src/shredder/tasks.py new file mode 100644 index 0000000..a72c65e --- /dev/null +++ b/backend/shredder/src/shredder/tasks.py @@ -0,0 +1,168 @@ +import asyncio +import logging +import os +import tempfile +import zipfile +from pathlib import Path + +import httpx + +import shredder +from shredder import settings, utils + +logger = logging.getLogger(__name__) + + +class ConversionError(Exception): + pass + + +# TODO: need to inject the client from the local context +async def get_doc_from_url(doc_path: str, local_ctx: dict) -> bytes: + """ + Downloads the document from the given URL and returns its content. + """ + client = local_ctx["http_client"] + response = await client.get(doc_path) + return response.content + +DOC_PROVIDER = { + "url": get_doc_from_url, + # TODO: Add additional providers +} + + +def zip_directory(src_dir: str, zip_path: str) -> None: + """ + Zips every file under src_dir into zip_path. Blocking; run it via asyncio.to_thread. + """ + with zipfile.ZipFile(zip_path, "w") as zip_ff: + for root, _, files in os.walk(src_dir): + for file in files: + zip_ff.write(Path(root) / file, arcname=file) + + +async def handle_conversion( + job_id, + doc_path, + call_back_url, + call_back_auth, + doc_provider, + metadata +): + """ + Handles the conversion of a PDF document to HTML using pdf2htmlEX. + - Downloads the document from the specified provider + - Runs the conversion + - Stores the output in a temporary directory + - Zips the output HTML files + - Sends the result to the callback URL + """ + + if isinstance(settings.SHREDDER_CONVERSION_CONFIG, str): + conversion_config = settings.SHREDDER_CONVERSION_CONFIG.split(",") + else: + conversion_config = [ + "--embed-css", "0", + "--css-filename", "pages.css", + "--process-annotation", "1", + "--decompose-ligature", "1", + "--bg-format", "svg", + ] + + # Open an asynchronous http client that handles GET and Post requests for the life of the task. + async with httpx.AsyncClient() as http_client: + # Generate a temp file to store the downloaded PDF + # and a temp directory for the output HTML files + with ( + tempfile.NamedTemporaryFile(suffix=".pdf", delete=True) as tff, # The incoming PDF file + tempfile.TemporaryDirectory(delete=True) as tdir, # Directory for conversion output + tempfile.TemporaryDirectory(delete=True) as zip_dir, # Directory for the zipped output + ): + # Download the document from the specified provider and write it to the temp file + doc_provider_func = DOC_PROVIDER[doc_provider] + tff.write( + await doc_provider_func(doc_path, local_ctx=locals()) + ) + tff.seek(0) + + # Run the PDF to HTML conversion command in a subprocess + command = [ + "/usr/local/bin/pdf2htmlEX", # points to the installed location in the container. + "--dest-dir", tdir, + *conversion_config, + "--tmp-dir", tdir, + tff.name, + ] + + process = await asyncio.create_subprocess_exec( + *command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await process.communicate() + if process.returncode: + raise ConversionError(f"{process.returncode} {command} {stdout} {stderr}") + + # Zip the output HTML files + # Written outside tdir so the archive does not include itself while tdir is walked. + zip_path = f"{zip_dir}/{job_id}.zip" + await asyncio.to_thread(zip_directory, tdir, zip_path) + + # Post the zipped result to the callback URL + with Path(zip_path).open("rb") as zip_ff: + headers = {} + data = { + "job_id": job_id, + "doc_path": doc_path, + "doc_provider": doc_provider, + **(metadata or {}) + } + if call_back_auth: + headers["Authorization"] = call_back_auth + + response = await http_client.post( + call_back_url, + data=data, + files={"file": (Path(zip_path).name, zip_ff, "application/octet-stream")}, + headers=headers, + ) + zip_ff_len = zip_ff.seek(0, os.SEEK_END) + logger.info( + "Posted result data: %s with file len %d " \ + "to callback URL: %s, response code: %d", + str(data), + int(zip_ff_len), + str(call_back_url), + int(response.status_code) + ) + + +@shredder.app.task(queue=settings.PROCRASTINATE_QUEUE, pass_context=True) +async def convert_pdf_to_html( + job_context, + doc_path: str, + call_back_url: str, + call_back_auth: str | None = None, + doc_provider: str = "url", + metadata: dict | None = None +): + """ + A wrapper task for the conversion function. Allows setting a job time-out. + + Inputs: + - job_context: Automatically supplied by procrastinate when triggered. + - doc_path: The URL pointing to the file to be converted. + - call_back_url: The URL to post the conversion result (zip file) to. + - call_back_auth: Optional `Bearer TOKEN` string + for authorization header for the callback endpoint. + - doc_provider: The provider to fetch the document from. Only "url" is currently supported. + - metadata: Optional passed-through metadata dict/JSON to include in the callback. + """ + job_id = job_context.job.id + + @utils.set_timeout(settings.PROCRASTINATE_MAX_JOB_TTL) + async def _run_job(*args): + return await handle_conversion(*args) + + return await _run_job(job_id, doc_path, call_back_url, call_back_auth, doc_provider, metadata) diff --git a/backend/shredder/src/shredder/utils.py b/backend/shredder/src/shredder/utils.py new file mode 100644 index 0000000..e93fe2c --- /dev/null +++ b/backend/shredder/src/shredder/utils.py @@ -0,0 +1,18 @@ +import asyncio +from functools import wraps + + +def set_timeout(seconds): + """ + Decorator that enforces a timeout on an async function. + Raises asyncio.TimeoutError if the function exceeds the timeout. + """ + + def decorator(func): + @wraps(func) + async def wrapper(*args, **kwargs): + return await asyncio.wait_for(func(*args, **kwargs), timeout=seconds) + + return wrapper + + return decorator diff --git a/backend/shredder/tests/__init__.py b/backend/shredder/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/shredder/tests/mock_server.py b/backend/shredder/tests/mock_server.py new file mode 100644 index 0000000..6ddd45a --- /dev/null +++ b/backend/shredder/tests/mock_server.py @@ -0,0 +1,96 @@ +""" +Mock HTTP server for the integration tests. Run it in a background thread of the test +process so the tests can read the callbacks it receives directly from memory. + +Endpoints: +- GET /example.pdf Serves tests/static/example.pdf as the document to convert. +- POST /webhook Records Shredder's multipart callback, keyed by its job_id field. +""" + +import threading +from email import policy +from email.parser import BytesParser +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +STATIC_DIR = Path(__file__).parent / "static" +DEFAULT_PORT = 1234 + + +def parse_multipart(content_type: str, body: bytes) -> tuple[dict, dict]: + """Split a multipart/form-data body into (fields, files).""" + message = BytesParser(policy=policy.HTTP).parsebytes( + f"Content-Type: {content_type}\r\n\r\n".encode() + body + ) + fields, files = {}, {} + for part in message.iter_parts(): + name = part.get_param("name", header="content-disposition") + payload = bytes(part.get_payload(decode=True)) or b"" + filename = part.get_filename() + if filename is None: + fields[name] = payload.decode() + else: + files[name] = {"filename": filename, "content": payload} + return fields, files + + +class MockServerHandler(BaseHTTPRequestHandler): + server: "MockServer" + + def do_GET(self): + if self.path == "/example.pdf": + self._send(200, (STATIC_DIR / "example.pdf").read_bytes(), "application/pdf") + else: + self._send(404, b"not found", "text/plain") + + def do_POST(self): + if self.path != "/webhook": + self._send(404, b"not found", "text/plain") + return + body = self.rfile.read(int(self.headers.get("Content-Length", 0))) + fields, files = parse_multipart(self.headers.get("Content-Type", ""), body) + self.server.record_callback( + fields.get("job_id", ""), + { + "authorization": self.headers.get("Authorization"), + "fields": fields, + "files": files, + }, + ) + self._send(200, b"ok", "text/plain") + + def _send(self, status: int, body: bytes, content_type: str): + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +class MockServer(ThreadingHTTPServer): + """ + Each request is handled on its own thread, so callbacks are shared through a + lock-protected dict, with a threading.Event per job_id to wake up waiting tests. + """ + + def __init__(self, port: int = DEFAULT_PORT, host: str = "0.0.0.0"): # noqa: S104 + super().__init__((host, port), MockServerHandler) + self._lock = threading.Lock() + self._callbacks: dict[str, dict] = {} + self._events: dict[str, threading.Event] = {} + + def _event_for(self, job_id: str) -> threading.Event: + with self._lock: + return self._events.setdefault(job_id, threading.Event()) + + def record_callback(self, job_id: str, record: dict) -> None: + with self._lock: + self._callbacks[job_id] = record + self._event_for(job_id).set() + + def wait_for_callback(self, job_id: str, timeout: float) -> dict | None: + """Block until the callback for job_id arrives; None if it doesn't within timeout.""" + if not self._event_for(job_id).wait(timeout): + return None + with self._lock: + return self._callbacks[job_id] diff --git a/backend/shredder/tests/static/example.pdf b/backend/shredder/tests/static/example.pdf new file mode 100644 index 0000000..6cc662c Binary files /dev/null and b/backend/shredder/tests/static/example.pdf differ diff --git a/backend/shredder/tests/test_functional.py b/backend/shredder/tests/test_functional.py new file mode 100644 index 0000000..efb1998 --- /dev/null +++ b/backend/shredder/tests/test_functional.py @@ -0,0 +1,181 @@ +"""Functional tests for the shredder application. No database, network, or worker required.""" + +import asyncio +import io +import time +import zipfile +from contextlib import contextmanager +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +from shredder.tasks import ConversionError, convert_pdf_to_html, zip_directory +from shredder.utils import set_timeout +from tests.mock_server import parse_multipart + +TIMEOUT_SECONDS = 0.1 +MAX_EXPECTED_SECONDS = 1 +EXAMPLE_PDF = Path(__file__).parent / "static" / "example.pdf" +PDF2HTMLEX = Path("/usr/local/bin/pdf2htmlEX") +DOC_URL = "http://docs.example/example.pdf" +CALLBACK_URL = "http://callback.example/webhook" +CALLBACK_AUTH = "Bearer test-token" + + +@set_timeout(TIMEOUT_SECONDS) +async def slow_task(): + await asyncio.sleep(2) + + +@set_timeout(1) +async def fast_task(): + return "done" + + +async def test_functional_set_timeout(): + """A task that exceeds its timeout is cancelled with a TimeoutError.""" + start = time.monotonic() + with pytest.raises(TimeoutError): + await slow_task() + elapsed = time.monotonic() - start + assert TIMEOUT_SECONDS <= elapsed < MAX_EXPECTED_SECONDS + + +async def test_functional_set_timeout_returns_result(): + assert await fast_task() == "done" + + +def _job_context(job_id=42): + return SimpleNamespace(job=SimpleNamespace(id=job_id)) + + +def _fake_pdf2htmlex(returncode=0): + """Stand-in for pdf2htmlEX: checks its input and writes HTML into --dest-dir.""" + calls = [] + + async def fake_exec(*command, **_kwargs): + calls.append(command) + input_pdf = Path(command[-1]) + assert input_pdf.read_bytes() == EXAMPLE_PDF.read_bytes() + if returncode == 0: + dest_dir = Path(command[command.index("--dest-dir") + 1]) + (dest_dir / f"{input_pdf.stem}.html").write_text("converted") + (dest_dir / "pages.css").write_text("body {}") + process = MagicMock(returncode=returncode) + + async def communicate(): + return b"", b"" if returncode == 0 else b"conversion failed" + + process.communicate = communicate + return process + + return fake_exec, calls + + +@contextmanager +def mock_http(): + """Route the task's httpx client to an in-memory transport and record its requests.""" + calls = {"get": [], "post": []} + + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "GET": + calls["get"].append(str(request.url)) + return httpx.Response(200, content=EXAMPLE_PDF.read_bytes()) + fields, files = parse_multipart(request.headers["content-type"], request.content) + calls["post"].append( + { + "url": str(request.url), + "authorization": request.headers.get("authorization"), + "fields": fields, + "files": files, + } + ) + return httpx.Response(200) + + real_client = httpx.AsyncClient + + def client_factory(*args, **kwargs): + return real_client(*args, transport=httpx.MockTransport(handler), **kwargs) + + with patch("shredder.tasks.httpx.AsyncClient", side_effect=client_factory): + yield calls + + +async def _run_conversion(**overrides): + kwargs = { + "doc_path": DOC_URL, + "call_back_url": CALLBACK_URL, + "call_back_auth": CALLBACK_AUTH, + "doc_provider": "url", + "metadata": {"reference": "abc-123"}, + **overrides, + } + return await convert_pdf_to_html(_job_context(), **kwargs) + + +async def test_functional_convert_pdf_to_html(): + """The task downloads the PDF, converts it, and POSTs a zip to the callback URL.""" + fake_exec, exec_calls = _fake_pdf2htmlex() + with ( + mock_http() as http, + patch("shredder.tasks.asyncio.create_subprocess_exec", side_effect=fake_exec), + patch("shredder.tasks.asyncio.to_thread", wraps=asyncio.to_thread) as to_thread, + ): + await _run_conversion() + + assert http["get"] == [DOC_URL] + to_thread.assert_called_once() + assert to_thread.call_args.args[0] is zip_directory + assert len(exec_calls) == 1 + assert exec_calls[0][0] == str(PDF2HTMLEX) + + assert len(http["post"]) == 1 + posted = http["post"][0] + assert posted["url"] == CALLBACK_URL + assert posted["authorization"] == CALLBACK_AUTH + assert posted["fields"] == { + "job_id": "42", + "doc_path": DOC_URL, + "doc_provider": "url", + "reference": "abc-123", + } + upload = posted["files"]["file"] + assert upload["filename"] == "42.zip" + assert zipfile.is_zipfile(io.BytesIO(upload["content"])) + with zipfile.ZipFile(io.BytesIO(upload["content"])) as archive: + names = archive.namelist() + html_files = [n for n in names if n.endswith(".html")] + assert len(html_files) == 1 + assert archive.read(html_files[0]) == b"converted" + assert "pages.css" in names + assert not [n for n in names if n.endswith(".zip")], "zip must not contain itself" + + +async def test_functional_convert_pdf_to_html_without_auth(): + """No Authorization header is sent when call_back_auth is not supplied.""" + fake_exec, _ = _fake_pdf2htmlex() + with ( + mock_http() as http, + patch("shredder.tasks.asyncio.create_subprocess_exec", side_effect=fake_exec), + ): + await _run_conversion(call_back_auth=None, metadata=None) + + posted = http["post"][0] + assert posted["authorization"] is None + assert set(posted["fields"]) == {"job_id", "doc_path", "doc_provider"} + + +async def test_functional_convert_pdf_to_html_conversion_error(): + """A failing pdf2htmlEX run raises ConversionError and nothing is POSTed.""" + fake_exec, _ = _fake_pdf2htmlex(returncode=1) + with ( + mock_http() as http, + patch("shredder.tasks.asyncio.create_subprocess_exec", side_effect=fake_exec), + pytest.raises(ConversionError, match="conversion failed"), + ): + await _run_conversion() + + assert http["post"] == [] diff --git a/backend/shredder/tests/test_integration.py b/backend/shredder/tests/test_integration.py new file mode 100644 index 0000000..a13298f --- /dev/null +++ b/backend/shredder/tests/test_integration.py @@ -0,0 +1,130 @@ +""" +Integration tests for the shredder application. + +Requires the shredder worker container and its Procrastinate database to be running. +`make test-integration` starts the container. The test starts the mock server in a background +thread and reads the callbacks it receives directly from memory. + +Optional environment overrides: +- MOCK_SERVER_PORT: mock server port (default 1234). +- MOCK_SERVER_CALLBACK_HOST: hostname the worker container uses to reach the mock server + (default host.docker.internal). +- TEST_PROCRASTINATE_DB_HOST / TEST_PROCRASTINATE_DB_PORT: database address as seen from the + machine running the tests (default 127.0.0.1 / PROCRASTINATE_DB_PORT). +- MOCK_SERVER_STARTUP_DELAY: seconds to wait after starting the mock server, because Docker + Desktop takes a moment to forward a newly opened host port into containers (default 2). +- INTEGRATION_TIMEOUT: seconds to wait for the callback (default 120). +""" + +import asyncio +import io +import os +import threading +import time +import zipfile + +import procrastinate +import pytest +from procrastinate.jobs import Status +from psycopg.conninfo import make_conninfo + +from tests.mock_server import MockServer + +MOCK_SERVER_PORT = int(os.getenv("MOCK_SERVER_PORT", "1234")) +CALLBACK_HOST = os.getenv("MOCK_SERVER_CALLBACK_HOST", "host.docker.internal") +TIMEOUT = float(os.getenv("INTEGRATION_TIMEOUT", "5")) +STARTUP_DELAY = float(os.getenv("MOCK_SERVER_STARTUP_DELAY", "2")) +WORKER_URL = f"http://{CALLBACK_HOST}:{MOCK_SERVER_PORT}" +TASK_NAME = "shredder.tasks.convert_pdf_to_html" +FAILED_STATUSES = {Status.FAILED, Status.CANCELLED, Status.ABORTED} + + +@pytest.fixture(scope="module") +def mock_server(): + """Start the mock server in a background thread.""" + server = MockServer(MOCK_SERVER_PORT) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + time.sleep(STARTUP_DELAY) + try: + yield server + finally: + server.shutdown() + server.server_close() + + +@pytest.fixture +async def procrastinate_app(): + """Add an actual job to the procrastinate worker queue for testing.""" + connector = procrastinate.PsycopgConnector( + conninfo=make_conninfo( + host=os.getenv("TEST_PROCRASTINATE_DB_HOST", "127.0.0.1"), + port=os.getenv("TEST_PROCRASTINATE_DB_PORT", os.environ["PROCRASTINATE_DB_PORT"]), + user=os.environ["PROCRASTINATE_DB_USER"], + password=os.environ["PROCRASTINATE_DB_PASSWORD"], + dbname=os.environ["PROCRASTINATE_DB_NAME"], + ) + ) + app = procrastinate.App(connector=connector) + async with app.open_async(): + yield app + + +async def _wait_for_callback( + app: procrastinate.App, + server: MockServer, + job_id: int, + *, + callback_auth: str, + expected_fields: dict, +) -> None: + """Wait for the job's callback to reach the mock server, then validate it.""" + received = None + deadline = time.monotonic() + TIMEOUT + while received is None and time.monotonic() < deadline: + # Wait in 1s steps on a worker thread so the event loop stays free and a failed + # job is reported straight away. + received = await asyncio.to_thread(server.wait_for_callback, str(job_id), 1) + if received is None: + status = await app.job_manager.get_job_status_async(job_id) + if status in FAILED_STATUSES: + pytest.fail(f"Job {job_id} ended with status {status.value}; see worker logs.") + if received is None: + pytest.fail(f"No callback received for job {job_id} within {TIMEOUT}s.") + + assert received["authorization"] == callback_auth + assert received["fields"] == {"job_id": str(job_id), **expected_fields} + upload = received["files"]["file"] + assert upload["filename"] == f"{job_id}.zip" + assert zipfile.is_zipfile(io.BytesIO(upload["content"])) + with zipfile.ZipFile(io.BytesIO(upload["content"])) as archive: + html_files = [n for n in archive.namelist() if n.endswith(".html")] + assert len(html_files) == 1 + html = archive.read(html_files[0]) + assert b"pdf2htmlEX" in html + assert b"page-container" in html + + +async def test_integration_conversion(procrastinate_app, mock_server): + """Queue a real job and check the worker POSTs the converted zip back to the mock server.""" + doc_path = f"{WORKER_URL}/example.pdf" + callback_auth = "Bearer integration-test-token" + metadata = {"reference": "integration-test"} + + job_id = await procrastinate_app.configure_task( + name=TASK_NAME, queue=os.getenv("PROCRASTINATE_QUEUE", "shredder_queue") + ).defer_async( + doc_path=doc_path, + call_back_url=f"{WORKER_URL}/webhook", + call_back_auth=callback_auth, + doc_provider="url", + metadata=metadata, + ) + + await _wait_for_callback( + procrastinate_app, + mock_server, + job_id, + callback_auth=callback_auth, + expected_fields={"doc_path": doc_path, "doc_provider": "url", **metadata}, + ) diff --git a/backend/uv.lock b/backend/uv.lock new file mode 100644 index 0000000..2c2cf4b --- /dev/null +++ b/backend/uv.lock @@ -0,0 +1,401 @@ +version = 1 +revision = 3 +requires-python = ">=3.13.11, <3.14" +required-markers = [ + "platform_machine == 'x86_64' and sys_platform == 'linux'", +] + +[[package]] +name = "anyio" +version = "4.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/d2/f4d173e22df740bc37b1db102b386ba719b66e95b0f0d751f556b387e6d2/anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94", size = 276966, upload-time = "2026-09-05T10:42:39.44Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b8/4bd346e22b28902df4d651910f5242c28d84e4a5c2435ca5c3f797ed7e2e/anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101", size = 132079, upload-time = "2026-09-05T10:42:37.923Z" }, +] + +[[package]] +name = "asgiref" +version = "3.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/90/f5fce14e95fccf53b58e8e2c47cd84d40d105d64073ae1ac75166db150e0/asgiref-3.13.0.tar.gz", hash = "sha256:034cc8cbf72d00553b4be256703ac545120b4e50af13b49dec99da899aeeecb4", size = 52941, upload-time = "2026-10-23T08:13:14.715Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/b0/ec211e1f7966247cb6469158d856bac9aa4c2fe13621bd6a352e4f850453/asgiref-3.13.0-py3-none-any.whl", hash = "sha256:f9596a2434512686473d50f96ff6207fbba49ae1a785dfac614df4aeb085674d", size = 27593, upload-time = "2026-10-23T08:13:13.394Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "certifi" +version = "2026.10.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd", size = 136225, upload-time = "2026-10-23T11:36:16.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1", size = 135082, upload-time = "2026-10-23T11:36:14.763Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d2/79/e78bd8f75a9b8e32233ce92d4607f0a21ce88a9156cace2884e7175a82f5/coverage-7.17.0.tar.gz", hash = "sha256:cc010a0a7fbdd9bb8bdfd0d3f04a5151275860213cbd7e2b876b29778489c712", size = 991021, upload-time = "2026-10-22T16:06:50.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/e6/1f01afbc9f4fa8ca5e1fa6fdf537a5c58b5765d1b4167ee284cddb9acc72/coverage-7.17.0-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:84538c040f4e6214f663459d89372ce4e6b2bbdd9d397ef0dca7b3ecd56b8eb2", size = 227080, upload-time = "2026-10-22T16:04:42.325Z" }, + { url = "https://files.pythonhosted.org/packages/92/7a/bdb7e6260c4a211389aeac9b89dc31662ebe122153ad0ecfd156897912df/coverage-7.17.0-cp313-cp313-android_24_x86_64.whl", hash = "sha256:6abfb6a4cf1cb1b67bc8fccdde177e2d2b781d0d34449e7078156547025140e5", size = 227152, upload-time = "2026-10-22T16:04:43.737Z" }, + { url = "https://files.pythonhosted.org/packages/37/4e/276e7d5b3cfb124871bdb544c27b004a8b4e270e6bf8e7414149c54b5e5b/coverage-7.17.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:ef65e96f0fc1a7c8b7644c7011d47a8228555e7888ac4d6246f715a7f3d7c398", size = 225397, upload-time = "2026-10-22T16:04:45.292Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7d/dac073db12f3c131598a6693253aded581e4d7d622a89864f11bfb0ca2f5/coverage-7.17.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:55939ea4c097fda2edabf80fac5e49bef15597b950a9f29fc56292a895245feb", size = 225705, upload-time = "2026-10-22T16:04:46.603Z" }, + { url = "https://files.pythonhosted.org/packages/00/45/877b0a9248e921080ae91d115c13192d17e308f5aee86bcc7f7bc2a54ed1/coverage-7.17.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:bfb6f868af552c0ceb2d19220b8cbcc2be096ca26163ac30e682a13041433429", size = 225828, upload-time = "2026-10-22T16:04:48.065Z" }, + { url = "https://files.pythonhosted.org/packages/4f/f5/2c48d47abec0e4590b302920334be5ee3854f96bbd22524feabb8d92b220/coverage-7.17.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8d823d66afd59566030539c49c7a6acaf3083996682be2dda760c63555842094", size = 226088, upload-time = "2026-10-22T16:04:49.513Z" }, + { url = "https://files.pythonhosted.org/packages/a8/a0/ec0442f3f372c7fd742f49b3022edd392f79e37a0048e1fa333f56d01ec3/coverage-7.17.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:50678474e714c684b82c108e63fbb8f1bcf1266d8293a49980f1d08526f093b5", size = 257257, upload-time = "2026-10-22T16:04:50.982Z" }, + { url = "https://files.pythonhosted.org/packages/92/e2/eb0927ba93809f76eee69fa1e47999825fc7dbbabc3e0e1a2744f2677acb/coverage-7.17.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:045c1eeb19fb4eae949850a761102fffb3a5f638b97f977912f95843e54f6669", size = 260165, upload-time = "2026-10-22T16:04:52.409Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a2/8d4c6055efb32099b15c9cc33d007b8aabebc4832f372a3274de5ba0afc5/coverage-7.17.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfb12891a773cbc2132349c77c690a045a882fe3f98b4bda50dfabb01893f9bd", size = 261098, upload-time = "2026-10-22T16:04:53.763Z" }, + { url = "https://files.pythonhosted.org/packages/2c/1e/93412288688a7b03f152a734c8ef086e395108a0a6a746985d3967e220f2/coverage-7.17.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3067608b4429629f640c1b23d2f69b05d2fd45e00ebb6ee9e8fad220a6c11722", size = 263164, upload-time = "2026-10-22T16:04:55.386Z" }, + { url = "https://files.pythonhosted.org/packages/3c/20/7eb8d805c4f1a212245cef20181dfbf39535357985073d49730ee9cf08d4/coverage-7.17.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a1762e85cbbe3b5ff9c6dcbcc1b89d5c72320094292b7b9572bf882c85e428b7", size = 257287, upload-time = "2026-10-22T16:04:56.976Z" }, + { url = "https://files.pythonhosted.org/packages/1a/c1/1df29630b4ed17e135c76390afcc6bd273c3dc5ba2a21a76813a600fe98c/coverage-7.17.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:62bb77987d3f72a0aaa77c567ee7f9806bc2e748c8f46fafbe1fdb90dd148398", size = 259256, upload-time = "2026-10-22T16:04:58.545Z" }, + { url = "https://files.pythonhosted.org/packages/a1/79/2365d1008d28aafe468a450a24c6c63363de794b5140223d67685cd0e205/coverage-7.17.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6cc31f27240d4e080437e3b8229d92f1d809c9a4f233607385373b73974e183e", size = 257178, upload-time = "2026-10-22T16:05:00.103Z" }, + { url = "https://files.pythonhosted.org/packages/c0/9f/f002952d295d772688dffce0d991e3822795b1e3aad158073712a5a5e86e/coverage-7.17.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:61cde5222f1933d476647c5a25ed1a8678b7703098cb5a15f4a0aa39967d8c28", size = 261275, upload-time = "2026-10-22T16:05:01.462Z" }, + { url = "https://files.pythonhosted.org/packages/40/fe/f9e15fbb426be97cd679ee5d359151e046f103bcd76b5092c318c26ed596/coverage-7.17.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:11f5f7fb2d97a0fe865f5e7bad4eb0619b957637f676bc088d13844adf26ba95", size = 257029, upload-time = "2026-10-22T16:05:02.8Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4a/3c398f38694387334ff4c9f71afc8e15b4fa7db7bc0bea4d5b6032eb0df0/coverage-7.17.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6e09612b1b82c9df1d2b6c85dee88d5aad3e73d77a8ec0b16ac6a7322cbdf5d9", size = 259109, upload-time = "2026-10-22T16:05:04.241Z" }, + { url = "https://files.pythonhosted.org/packages/bd/a9/908a3cf1a2335aec1c788682a06c2f742a62a4f987033f080de1c5fd1997/coverage-7.17.0-cp313-cp313-win32.whl", hash = "sha256:bd3e9eb493631ff1ca34bf5efac25b5c66724f837bf6182c4fc771c55946423d", size = 227691, upload-time = "2026-10-22T16:05:05.636Z" }, + { url = "https://files.pythonhosted.org/packages/13/f7/8cdd7bfd8a63b3e6b133fc395ecfc7b1cb6f561926e7e2ad050de85b83e7/coverage-7.17.0-cp313-cp313-win_amd64.whl", hash = "sha256:f1fc9a7cea899c51d8a53e66c2f082a3d9e461ece32418d3ef0a0b90c32bfd3d", size = 228418, upload-time = "2026-10-22T16:05:07.057Z" }, + { url = "https://files.pythonhosted.org/packages/09/cf/0b3fc57f735cd3831ca13d0be5312a5477d5934066683c5d34018cab7541/coverage-7.17.0-cp313-cp313-win_arm64.whl", hash = "sha256:e65a1e61a68493fbfbe44d8bec71b3231d81b9917f62d3dafe0e105072147e83", size = 228142, upload-time = "2026-10-22T16:05:08.435Z" }, + { url = "https://files.pythonhosted.org/packages/c9/17/48032919343afa2906112d458aceda2043afdb24a2ee8eb4c60904d8ddf1/coverage-7.17.0-py3-none-any.whl", hash = "sha256:bac487dce0d2aff28ddf1a533a916f771416eee0e493960f611e49037b878e9d", size = 217383, upload-time = "2026-10-22T16:06:48.471Z" }, +] + +[[package]] +name = "croniter" +version = "6.2.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/57/2e2a65aee2a70483cb28e2b7e15a072d00a523207593b44400d4717bb100/croniter-6.2.4.tar.gz", hash = "sha256:fc124f751b1b04805c2a04b061898b436b45ab2320b045e1e052ea895de65189", size = 166267, upload-time = "2026-07-10T09:52:59.955Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/ba/d678e5bd329646ca51d3c92addbc77804e86d21f4b6b6a027218e6abb010/croniter-6.2.4-py3-none-any.whl", hash = "sha256:8ef3d544107a5c05a150a2d78f8bf5a8eb9c5c4d93405a736b824109574e3f4d", size = 46677, upload-time = "2026-07-10T09:52:58.425Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "procrastinate" +version = "3.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asgiref" }, + { name = "attrs" }, + { name = "croniter" }, + { name = "packaging" }, + { name = "psycopg", extra = ["pool"] }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/8a/ab3a9841b38b9349eb3cb871c47d0683ce4811669c9f0fb7b889343bea8f/procrastinate-3.10.0.tar.gz", hash = "sha256:3668da769088682abf91b8c760de467be1203218411872f1a26ac7515d4f980d", size = 91101, upload-time = "2026-09-23T21:22:57.59Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/51/c7ea18faef04260a1e67cb60c2c815ff0f2792c323c4f3a733478cbfc883/procrastinate-3.10.0-py3-none-any.whl", hash = "sha256:949ad25b4377c30e03a9d0e053d15dfa47ef02b3f12ae8a7f3254686183c152c", size = 155244, upload-time = "2026-09-23T21:22:55.873Z" }, +] + +[[package]] +name = "propylon-shredder" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "httpx" }, + { name = "procrastinate" }, + { name = "psycopg", extra = ["pool"] }, + { name = "sqlalchemy" }, +] + +[package.dev-dependencies] +dev = [ + { name = "coverage" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "ruff" }, + { name = "ty" }, +] + +[package.metadata] +requires-dist = [ + { name = "httpx", specifier = ">=0.28.1" }, + { name = "procrastinate", specifier = ">=3.8.1" }, + { name = "psycopg", extras = ["pool"] }, + { name = "sqlalchemy", specifier = ">=2.0.44" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "coverage" }, + { name = "pytest", specifier = ">=8.0.0" }, + { name = "pytest-asyncio", specifier = ">=0.23.0" }, + { name = "ruff", specifier = "==0.16.3" }, + { name = "ty", specifier = "==0.0.72" }, +] + +[[package]] +name = "psycopg" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/26/3ea4ca5eaea1c0debcdf7ee7c1613fbe721dc27a03c461c0817ffd8a0601/psycopg-3.3.6.tar.gz", hash = "sha256:c081f2250df751a943036e42db6df4571c66cd0aabe8291a7a506512b12007d2", size = 168171, upload-time = "2026-09-18T13:22:55.152Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/de/748bd7609c71cae5d737f0ba9192f19329f70180ecda8fff3cac02c5abe3/psycopg-3.3.6-py3-none-any.whl", hash = "sha256:a1db9f7148b06a28606767efaca51fa6f9398c5c0a3810519be69d7000bdb631", size = 215490, upload-time = "2026-09-18T13:15:29.374Z" }, +] + +[package.optional-dependencies] +pool = [ + { name = "psycopg-pool" }, +] + +[[package]] +name = "psycopg-pool" +version = "3.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/5e/c0664b968b102ff68b811d999c728546c48d5c1eec03e3bbaf88c0cb4472/psycopg_pool-3.3.3.tar.gz", hash = "sha256:df87b5d9d0ad7db37f6cdad4fa8ce113d250f5997f6db38e9a99192fb67f9e1d", size = 32006, upload-time = "2026-09-22T15:53:24.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/b4/452c6607a0f479465cd8a9b0d9956919fcb150050c1f83f9f11e6b8ee8dc/psycopg_pool-3.3.3-py3-none-any.whl", hash = "sha256:9b9cd6a4fcec47a410f7e82d408540e7f77b478509e91b44c1a5457a13e5ff37", size = 40304, upload-time = "2026-09-22T15:53:23.712Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/42/bb4641652e58dedeac2ec51abc1f8cae4fee4ec63899a0999016229a0f2a/pytest-9.2.0.tar.gz", hash = "sha256:97c4f3b1f3c7414aa36465e09b8220f1d03627993bd51b1282e193753c80557b", size = 1797555, upload-time = "2026-10-27T21:30:20.728Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/39/1f5d8c8877a0fa327db959c19341d4be61660de1ab6b7391f9eb805bcd67/pytest-9.2.0-py3-none-any.whl", hash = "sha256:07acdcf88ec174f907dc01af70bf52d49b014bbab03faeeade31c96b18c2e588", size = 411776, upload-time = "2026-10-27T21:30:19.131Z" }, +] + +[[package]] +name = "pytest-asyncio" +version = "1.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/9a/887535e477847f5159377ab35303923644840b13c0ca03c0a467f84d4680/pytest_asyncio-1.4.1.tar.gz", hash = "sha256:e8266eb094e5ee9ffdb2068c9836f33e209ea77a970eb846933d416b5602669b", size = 145237, upload-time = "2026-10-25T21:05:37.135Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/d7/cd9e2708afdbf059658adaf08bdb0792ca52489871051d6ce7d498b51277/pytest_asyncio-1.4.1-py3-none-any.whl", hash = "sha256:507afa78ec77c6661164ef1a2368bc7820383fd1a14b2a7f813de8b5a92949cc", size = 17061, upload-time = "2026-10-25T21:05:35.473Z" }, +] + +[[package]] +name = "python-dateutil" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/df/f15f80383f152452d8985b36d95cb8751f901a3e0ff4cd27f7e118f168e2/python_dateutil-3.0.0.tar.gz", hash = "sha256:7ebcb3c29a92f3ceafe79a7a48a5c45677e9e004bbeec5742ac0bbb00560b7a2", size = 271066, upload-time = "2026-10-21T23:41:45.075Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/17/cebee01d1c0c1441174cf0456ddc91b280d79acb65340c05e41d77866ee8/python_dateutil-3.0.0-py2.py3-none-any.whl", hash = "sha256:3c79630202e4152c6add678446984236efe4f57e84d8e892aab0637bab82e9fd", size = 94659, upload-time = "2026-10-21T23:41:43.431Z" }, +] + +[[package]] +name = "ruff" +version = "0.16.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/61/b3/3213589383f8f1b3938781bd1278713f6d18621a14992b3e81fefb8a5ef9/ruff-0.16.3.tar.gz", hash = "sha256:e76d33a347661a84b5be6d043d0347fdc745dfdcf825a8f4fed64b5e26eebdf2", size = 4891904, upload-time = "2026-08-13T15:17:13.381Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/96/493770daebd68c0a67f1549fdf519f53be51fc435186c0585bcc272fd76c/ruff-0.16.3-py3-none-linux_armv6l.whl", hash = "sha256:0c5710e247a58a4521e66e124ba9a74655b414f61ba3a2e9e3811e11098f48f7", size = 10902799, upload-time = "2026-08-13T15:16:27.382Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e6/2becf3942fddc29a29b8df47691d456fb1085391a694f74d84513251418c/ruff-0.16.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fe155130631a2471fd2e14a7a664a4dfbd7194b8229c3d7b2a40b21178639081", size = 11135539, upload-time = "2026-08-13T15:16:30.87Z" }, + { url = "https://files.pythonhosted.org/packages/3e/1e/4b8b72f0d006dbf19326aa99f9ca0ee2ff374187c4d301cf529a51aa06fe/ruff-0.16.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e2ed719e14aa64d895c2ee922594a90a43c861a93f0575a95ff8c47cdbd13eb9", size = 10475095, upload-time = "2026-08-13T15:16:33.259Z" }, + { url = "https://files.pythonhosted.org/packages/92/32/2201fa49ba1f6c101ee321e83f051ac7a4b8d07b0ef6b4d3f2772b302275/ruff-0.16.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e0b1da805eb043654645d74d5de1e5ce2edc686e40790d2b86f56d71cc06a84", size = 10668771, upload-time = "2026-08-13T15:16:35.65Z" }, + { url = "https://files.pythonhosted.org/packages/c3/66/4afc5c8363bd04d45effce1b7c8713ca037d7a6740b7451a2403a6e3a972/ruff-0.16.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a37bdea0bbe21780f590bf437d6412c8c4e1b6cd010f91a65c2c40c5e5f5f870", size = 10699568, upload-time = "2026-08-13T15:16:38.195Z" }, + { url = "https://files.pythonhosted.org/packages/53/fd/c67d246bf36bf1698551c56de39e95cd07f70e64433e0098e6267d77061b/ruff-0.16.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09571e6d1288ed9be475207a3ac04ada404f1cd898104be0f6ab8d7df438575b", size = 11499365, upload-time = "2026-08-13T15:16:40.623Z" }, + { url = "https://files.pythonhosted.org/packages/67/0b/00ecbceb99a263af7b12f6f05ac3c92bc47b905e91adc3f207a836e3bc01/ruff-0.16.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c18c5a101eb540010638cc1ff3c84944d3adb3df62b8d98ca8f22ba484d3413", size = 12311728, upload-time = "2026-08-13T15:16:43.564Z" }, + { url = "https://files.pythonhosted.org/packages/54/b2/b7b3bb54f4d3f7db504e476ad4ab8de530dceebe2c061384b2757ee419e8/ruff-0.16.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8457c44f15033c85ddbb77b15d451df9e24e4bd03b628396dd3610cedc3b8f82", size = 11699896, upload-time = "2026-08-13T15:16:46.209Z" }, + { url = "https://files.pythonhosted.org/packages/c7/30/4c468429ac195addc5ee1b717b6ab1b66632786737ca3b2ed3443fb0c26a/ruff-0.16.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:294b95c4ae0cda9388525c2047778aa758d6b8d4bb876fd4e9eaa3ebc92343eb", size = 11058736, upload-time = "2026-08-13T15:16:48.823Z" }, + { url = "https://files.pythonhosted.org/packages/43/67/7a113cdaddf24b64d7f75b1242a99d04c82fcef4f6921fdbb832beaffb5f/ruff-0.16.3-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:3d0c7c40c87c2a820509c31ba007968da6e1306468c067b2d82fbfdbcd0e8474", size = 11586911, upload-time = "2026-08-13T15:16:51.913Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c1/2e66f24c0f3ead25a5e660111778685e505e5da353c82802bf49f0cbe7b9/ruff-0.16.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:9f738c0fdfa8eed0b2ce7fb27ee7258208a92a68d7949e62aa15164bc7b389da", size = 10954265, upload-time = "2026-08-13T15:16:54.763Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ba/4cee23bf52cba9a058d3726de623624daf50ef9638868edd86f4126157f6/ruff-0.16.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fb785f0be25abe69d320415cd4f833b59e17ba7613d9ba6a958023b6bceb0a50", size = 10709886, upload-time = "2026-08-13T15:16:57.339Z" }, + { url = "https://files.pythonhosted.org/packages/82/df/7da7194fa5d9dc0a285f7e6fa5a4722e7c63faac0b45b614ded9314363a1/ruff-0.16.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c5536e3acfbf9563085aa2be7b13c629c3077e902afc5b941ac44024dbb9f506", size = 11210392, upload-time = "2026-08-13T15:17:00.171Z" }, + { url = "https://files.pythonhosted.org/packages/35/85/7795f6e817af050e7517bf3e7aa9b061cce70ef33d280aad902c956c1ecf/ruff-0.16.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a2d85c02f9b8e165d85e6779184d38c4132de12603dab59c51c28e22584f9e4d", size = 11626910, upload-time = "2026-08-13T15:17:03.299Z" }, + { url = "https://files.pythonhosted.org/packages/78/9b/475b927cf27a5cbbda3c7bafb69ed6ff77e1d7923d5d85f17c2749d7ae32/ruff-0.16.3-py3-none-win32.whl", hash = "sha256:388cdf2166642bd9b13d52b5932d3170f34f8abed7e8d9a855f1d84b83645a0a", size = 10931415, upload-time = "2026-08-13T15:17:05.726Z" }, + { url = "https://files.pythonhosted.org/packages/b2/99/e2a2bfc4fbf0a1e8a916bc9ebe6fe6c58cc34c28e0ffc6ce281d572d1c2e/ruff-0.16.3-py3-none-win_amd64.whl", hash = "sha256:e80a7d69ca2a6d1c4d352ec91458cdca6e56c83cdbcabd93e4abe1e53591d948", size = 11445993, upload-time = "2026-08-13T15:17:08.353Z" }, + { url = "https://files.pythonhosted.org/packages/69/3e/4132e539aed78c148854d4997a2685b0ed4dc4e87110b59ce528564e184e/ruff-0.16.3-py3-none-win_arm64.whl", hash = "sha256:b8ca152da82c1acc1fa8d5874b15951935f0eef46f10e6954c83859011b6178a", size = 11399302, upload-time = "2026-08-13T15:17:10.908Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "sqlalchemy" +version = "2.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/dc/9addbf490110b5727380059ee542ad8353bdcee45a37ba6a5e0976966ae0/sqlalchemy-2.1.6.tar.gz", hash = "sha256:499eae5c9051efe96cb9a73ec2fbde8ffa4186f102b9094ebab8fcc9d3b99f52", size = 10589700, upload-time = "2026-10-21T16:25:06.497Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/db/ecf1cf1f5c8cb7661081dcb0561633c174213d7af5fccc17ab48cbe9812a/sqlalchemy-2.1.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0534d3f41295a74d9fe2e0838e06ce6329a0855b2965df5f5d1c4d3f6f1406d0", size = 2468626, upload-time = "2026-10-21T18:49:33.602Z" }, + { url = "https://files.pythonhosted.org/packages/9b/98/1cccefb4fc94f4efd621604063acbbea4e3d530c27151fd7249d68adabf2/sqlalchemy-2.1.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:650f97996a4c529f4930969f3ba11bcdb57ff160ed1cb87ea0174393bf4f5f73", size = 4602545, upload-time = "2026-10-21T18:54:18.766Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b0/1817d7406ed92c8647a1d1db965a1bc2dd5a3f80d18e4d28ad62daf9bd72/sqlalchemy-2.1.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bada6bd29ae58a0581bf6977c0a43dfd67388fd09129f1475c37eca370cccd8", size = 4655491, upload-time = "2026-10-21T19:05:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/41/5c/0bdaa1676fb182b93e3c87c09cafd08a734ef29820aac6312b87a88f8537/sqlalchemy-2.1.6-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:96f5fcdfb7e89b6d895e2ddc165f67b3126a585dfdf2a95b0c1f3fcf3380d7d4", size = 4319667, upload-time = "2026-10-21T19:23:16.637Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d7/319e5a049b6f5235d1229858972abb05b8eae2fdd6639dc7bdbbc9cdb68d/sqlalchemy-2.1.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:015bd7b0687b5feaecb9988bc2f36ecb39daca83bd806b094b43b19ea503b0a9", size = 4525975, upload-time = "2026-10-21T18:54:21.553Z" }, + { url = "https://files.pythonhosted.org/packages/19/3b/6e8e5c14a3bcfae59a081a27f33e4f225df8140daab8e90fc3c42afd0d0e/sqlalchemy-2.1.6-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:bf883ae772971bc6155cb64cdfb9e31c50a2099f3b7c8b51b0401bf608a9fe8e", size = 4320900, upload-time = "2026-10-21T19:23:18.835Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a7/7ad220d5ea6308a9a32395785f4ce8beff4d00b0ae56ab9f2dafcca28743/sqlalchemy-2.1.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1ff897d13de613768123dee68474956a51fe641c42bba7af6cb7776716a8dbcb", size = 4616756, upload-time = "2026-10-21T19:05:22.265Z" }, + { url = "https://files.pythonhosted.org/packages/08/81/53a198f5a29f247adc68c0c6da69bd2c7bdd89d3ffc58f9e626ee3160f18/sqlalchemy-2.1.6-cp313-cp313-win32.whl", hash = "sha256:9db8bacb2a69b4b5388bc1439463170af28f4cc710933085b81d1658d931d2c2", size = 2382042, upload-time = "2026-10-21T17:11:33.968Z" }, + { url = "https://files.pythonhosted.org/packages/b4/87/69e86a23cd9ccef0d654c09dc69dfea6d62b1311d2f13ca3daab1bcd922e/sqlalchemy-2.1.6-cp313-cp313-win_amd64.whl", hash = "sha256:b9f0b92b4a88b630f1e67df816b43cb017bb583479917dd100541d4f490a9517", size = 2432597, upload-time = "2026-10-21T17:11:35.514Z" }, + { url = "https://files.pythonhosted.org/packages/98/90/a7f81395135cc5da724403ece4340f7e04c1ce752da15d0ad337591ec9c9/sqlalchemy-2.1.6-cp313-cp313-win_arm64.whl", hash = "sha256:bc495d00f235c3df6f376187a4ef48a56e162fdcc0a113ee34518332aa1aa2df", size = 2393224, upload-time = "2026-10-21T18:54:18.405Z" }, + { url = "https://files.pythonhosted.org/packages/c1/69/c75534e336b06b3fd3f76de73d89736510800e60c77c58d4e4ce18389018/sqlalchemy-2.1.6-py3-none-any.whl", hash = "sha256:106878cf3795cd3d357e2e858609f9e44f20f19b2d8e48a3532bef3f90b03fbe", size = 2063055, upload-time = "2026-10-21T18:42:10.234Z" }, +] + +[[package]] +name = "ty" +version = "0.0.72" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/df/656e684bafb13c1d146e7d5b5f3e7978ca177232acc84998ff36427e9462/ty-0.0.72.tar.gz", hash = "sha256:ec2b8066b618df18cab4cb8e992f8da45d360332acb23fa34df7fa29cd1b9d3a", size = 6654939, upload-time = "2026-08-14T21:35:42.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/3b/f51461239a4e66565d4b362f97a3b55fe7fdba2e944068341f87c62f6743/ty-0.0.72-py3-none-linux_armv6l.whl", hash = "sha256:fda86db153ffd85ee52000cf175d6a3f1c0223772cf7c5b6f726200bf92c7b44", size = 12621989, upload-time = "2026-08-14T21:35:01.676Z" }, + { url = "https://files.pythonhosted.org/packages/ca/fb/79ddf683affc679ca856f3510b5640ec3a88a842ba5f654f5d4bc78f1786/ty-0.0.72-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ceb944c612529b9023acfdc9cf4c0dcbb722549f9d17d46baecd1141baf01d7f", size = 12233910, upload-time = "2026-08-14T21:35:04.334Z" }, + { url = "https://files.pythonhosted.org/packages/5d/45/10562a0d84802158db8fa4ec46de54aa9fdcecdeeaabbfe3639ae7042b66/ty-0.0.72-py3-none-macosx_11_0_arm64.whl", hash = "sha256:108d76218333d6c092e5f1cebf8e9b06f25738613a0236a28e2dd47c936ee52c", size = 12084108, upload-time = "2026-08-14T21:35:06.686Z" }, + { url = "https://files.pythonhosted.org/packages/a1/dc/1fe1aef8d697e3509face271a5331700c7aa1d1e44a4b622707bdfa41d4b/ty-0.0.72-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f3943f186f741a2499a31053872169250c9264a9a49684920e48d8fcf4ef4f5", size = 12132640, upload-time = "2026-08-14T21:35:09.305Z" }, + { url = "https://files.pythonhosted.org/packages/14/46/41ceb265e96969487311a2014bd0e53abb4fbc1395efb2ebe411fcb4db62/ty-0.0.72-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf283c07dc3cc52ca48a3ad8ab100fb5aec3aebbd03ef6a12d5f910b8e596fc5", size = 12402489, upload-time = "2026-08-14T21:35:11.555Z" }, + { url = "https://files.pythonhosted.org/packages/2b/45/30bf43cb4fd505c5c2dd30fda27dde5f05208686cd21217adec77c954204/ty-0.0.72-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:95f3b6462c38f9f115d10cee21f47fedf715fcf2040daf36eef210359300bc7c", size = 13130835, upload-time = "2026-08-14T21:35:13.746Z" }, + { url = "https://files.pythonhosted.org/packages/31/2f/03bba754d2613f640df168335c41f83f41db150bb515839c60d80e3a7880/ty-0.0.72-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30caf658feb8ffb250d9e9e47107657a78f5f3425c227df1664d8df2ebe38880", size = 13590392, upload-time = "2026-08-14T21:35:16.839Z" }, + { url = "https://files.pythonhosted.org/packages/04/c7/03c67f00e63005ec41585653dc3096064570b1e6273742baae2798cd242f/ty-0.0.72-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:27bdc012ddfbeec8948e4a6036c0dc39ac7cf2c8ec7c7d48dc7d2fd56d57b399", size = 13309629, upload-time = "2026-08-14T21:35:19.169Z" }, + { url = "https://files.pythonhosted.org/packages/c1/df/102d3b264eb7f2a58dd11952f229bb5150bb5668d176a6154976a6675981/ty-0.0.72-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:802c5970a77d7739e6f499921fbb6984fb7ad8a31d95e1ff42fd46f3642e4f3b", size = 12734028, upload-time = "2026-08-14T21:35:22.099Z" }, + { url = "https://files.pythonhosted.org/packages/61/85/d0737c8c54d0ba67366ddfb9f31d88edf0b02299e65923e6945ae60ebcb5/ty-0.0.72-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:47dce65114fdc615c68ca0edb393b433df0956447e4267df0e264137a789598d", size = 13174832, upload-time = "2026-08-14T21:35:24.71Z" }, + { url = "https://files.pythonhosted.org/packages/1e/31/497f5a96c36d9b586ab6afe0574986835c6fd5b835a89773d2bec4711b49/ty-0.0.72-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:325144fa07e2675d0faa337fcc864213c272a499eb0cfe5bde2fdc62282d27bc", size = 12215005, upload-time = "2026-08-14T21:35:26.892Z" }, + { url = "https://files.pythonhosted.org/packages/df/7d/46e65b17b4966c7cd0140f134380d33d8e84fe6efccd761533ce793dc502/ty-0.0.72-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a5c9f15d0f58e43707d8848274be1821a0ef408eccb8aa7dda28a4a9eddf7640", size = 12421298, upload-time = "2026-08-14T21:35:29.301Z" }, + { url = "https://files.pythonhosted.org/packages/08/2a/12ada4ec17700b3cb1d4fd3bc3e5b1852df9e6885288429318cade87b3c1/ty-0.0.72-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8ee508d64b381871529cc22c412b41071bf5e908b7aa5d66a38f3f6b2573a806", size = 12669242, upload-time = "2026-08-14T21:35:31.444Z" }, + { url = "https://files.pythonhosted.org/packages/1c/1a/4692536880790fb550ed6d44a6096778dc71bb112f2c6d615cebb01a57e5/ty-0.0.72-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:3699e2ec7921d44da79d6b089f7bf239b2cc53c4e45a5a38430adc34ee9e9a55", size = 12988199, upload-time = "2026-08-14T21:35:33.749Z" }, + { url = "https://files.pythonhosted.org/packages/9a/0d/f5e5a50322e9c45865e7b7a428ba6cd6527387cf0f2472492ac3cf746243/ty-0.0.72-py3-none-win32.whl", hash = "sha256:f25f72a67bd36cd247707c4784e52fad0b6b4f42a1b7dd14804110fa95c486ed", size = 11939708, upload-time = "2026-08-14T21:35:36.006Z" }, + { url = "https://files.pythonhosted.org/packages/3f/4e/8af3534b2e4214e6184a5a59c34101e94a68d578f081f97b995866bab1bf/ty-0.0.72-py3-none-win_amd64.whl", hash = "sha256:cdeee869341717e1736cea2e2d7856738c6957c320f584ed2f68c8f90100d2f5", size = 12643876, upload-time = "2026-08-14T21:35:38.141Z" }, + { url = "https://files.pythonhosted.org/packages/ff/ea/a2606e654c7276bd08586391a2525b0af3f3bf60228a8c57b2d248f273f9/ty-0.0.72-py3-none-win_arm64.whl", hash = "sha256:1bd3ac3ed4424a6d6990a85dc388556aea012bd752de21349a84b685951de0d8", size = 12394857, upload-time = "2026-08-14T21:35:40.277Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7a/29/6a1391d31cd00ff9e927215a224a880e1e84e826b64ba80a499e51c074d8/tzdata-2026.6.tar.gz", hash = "sha256:85d2cd9c09f81f0bb29db27759fc83af4f1e620efb814f8c5bf8b4cc4feece6d", size = 201017, upload-time = "2026-10-28T09:57:09.137Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/66/9a06c2227fedfd4919f0eaaa8b94377d9730882a7e4f95823c9a331ab5e1/tzdata-2026.6-py2.py3-none-any.whl", hash = "sha256:0a10eec6e13ef16b627fabed9a5f2ba61a62b1c67f327305f4e7e412a70b4aec", size = 348210, upload-time = "2026-10-28T09:57:07.276Z" }, +]