diff --git a/.github/workflows/backend-QA.yml b/.github/workflows/backend-QA.yml index 98a754a4e..1e60baba3 100644 --- a/.github/workflows/backend-QA.yml +++ b/.github/workflows/backend-QA.yml @@ -26,16 +26,16 @@ jobs: working-directory: backend run: | pip install -U pip - pip install -e .[lint,scripts,test,check,api] + pip install -e .[qa,scripts,test,api] - - name: Check black formatting + - name: Check Linting working-directory: backend - run: inv lint-black + run: inv check-lint - - name: Check ruff + - name: Check Formatting working-directory: backend - run: inv lint-ruff + run: inv check-format - - name: Check pyright + - name: Check Type working-directory: backend - run: inv check-pyright + run: inv check-type diff --git a/.github/workflows/healthcheck-QA.yml b/.github/workflows/healthcheck-QA.yml index 0c88af892..3e3ccc06b 100644 --- a/.github/workflows/healthcheck-QA.yml +++ b/.github/workflows/healthcheck-QA.yml @@ -26,16 +26,16 @@ jobs: working-directory: healthcheck run: | pip install -U pip - pip install -e .[lint,scripts,test,check] + pip install -e .[qa,scripts,test] - - name: Check black formatting + - name: Check Linting working-directory: healthcheck - run: inv lint-black + run: inv check-lint - - name: Check ruff + - name: Check Formatting working-directory: healthcheck - run: inv lint-ruff + run: inv check-format - - name: Check pyright + - name: Check Type working-directory: healthcheck - run: inv check-pyright + run: inv check-type diff --git a/.github/workflows/monitor-QA.yml b/.github/workflows/monitor-QA.yml index b8414769d..93483e1c4 100644 --- a/.github/workflows/monitor-QA.yml +++ b/.github/workflows/monitor-QA.yml @@ -26,16 +26,16 @@ jobs: working-directory: monitor/parent run: | pip install -U pip - pip install black ruff pyright + pip install ruff pyright - - name: Check black formatting + - name: Check Formatting working-directory: monitor/parent - run: black --check --diff . + run: ruff format --check . - - name: Check ruff + - name: Check Linting working-directory: monitor/parent run: ruff check . - - name: Check pyright + - name: Check Type working-directory: monitor/parent run: pyright diff --git a/.github/workflows/recipesauto-QA.yml b/.github/workflows/recipesauto-QA.yml index ee9f07e9a..b7eb3b0eb 100644 --- a/.github/workflows/recipesauto-QA.yml +++ b/.github/workflows/recipesauto-QA.yml @@ -26,16 +26,16 @@ jobs: working-directory: recipesauto run: | pip install -U pip - pip install -e .[lint,scripts,test,check] + pip install -e .[qa,scripts,test] - - name: Check black formatting + - name: Check Linting working-directory: recipesauto - run: inv lint-black + run: inv check-lint - - name: Check ruff + - name: Check Formatting working-directory: recipesauto - run: inv lint-ruff + run: inv check-format - - name: Check pyright + - name: Check Type working-directory: recipesauto - run: inv check-pyright + run: inv check-type diff --git a/.github/workflows/watcher-QA.yml b/.github/workflows/watcher-QA.yml index ce82843b7..48dae3cd1 100644 --- a/.github/workflows/watcher-QA.yml +++ b/.github/workflows/watcher-QA.yml @@ -27,16 +27,16 @@ jobs: run: | pip install -U pip pip install -r requirements.txt - pip install black ruff pyright + pip install ruff pyright - - name: Check black formatting + - name: Check Formatting working-directory: watcher - run: black --check --diff . + run: ruff format --check . - - name: Check ruff + - name: Check Linting working-directory: watcher run: ruff check . - - name: Check pyright + - name: Check Type working-directory: watcher run: pyright diff --git a/.github/workflows/worker-QA.yml b/.github/workflows/worker-QA.yml index 0c3957aae..41d9dd464 100644 --- a/.github/workflows/worker-QA.yml +++ b/.github/workflows/worker-QA.yml @@ -26,16 +26,16 @@ jobs: working-directory: worker run: | pip install -U pip - pip install -e .[lint,scripts,test,check,task] + pip install -e .[qa,scripts,test,task] - - name: Check black formatting + - name: Check Linting working-directory: worker - run: inv lint-black + run: inv check-lint - - name: Check ruff + - name: Check Formatting working-directory: worker - run: inv lint-ruff + run: inv check-format - - name: Check pyright + - name: Check Type working-directory: worker - run: inv check-pyright + run: inv check-type diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 46f7dcffa..596f24e78 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,13 +9,8 @@ repos: - id: end-of-file-fixer - id: mixed-line-ending - - repo: https://github.com/psf/black - rev: "25.1.0" - hooks: - - id: black - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.10 + rev: v0.15.12 hooks: - id: ruff-check - id: ruff-format diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9f2e6967f..80519303a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,7 @@ Before contributing, make sure you mention you'll be working on it in the associ We have a very limited number of contributors so rules are quite limited: - Submit your changes over a Pull Request -- Make sure all your python code is black formatted. +- Make sure all your python code is formatted with ruff. - Make sure Codefactor reports no issue. - API code (backend) must be tested. Make sure your PR doesn't decrease code coverage. diff --git a/backend/V1_TO_V2_MIGRATION.md b/backend/V1_TO_V2_MIGRATION.md index 65afa9832..a4ef48da6 100644 --- a/backend/V1_TO_V2_MIGRATION.md +++ b/backend/V1_TO_V2_MIGRATION.md @@ -111,7 +111,6 @@ Zimfarm Backend v2 represents a significant modernization of the codebase, migra **v2 New Features:** -- Black code formatter - Ruff linter - Pyright type checker - Pre-commit hooks diff --git a/backend/pyproject.toml b/backend/pyproject.toml index ecf785732..af6c1beaa 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -17,7 +17,7 @@ classifiers = [ "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", ] dependencies = [ - "alembic == 1.15.2", + "alembic == 1.18.4", "pydantic[email] == 2.11.4", "pydantic-extra-types == 2.10.6", "SQLAlchemy == 2.0.41", @@ -44,12 +44,9 @@ api = [ scripts = [ "invoke == 2.2.0", ] -lint = [ - "black == 25.1.0", - "ruff == 0.11.10", -] -check = [ - "pyright == 1.1.400", +qa = [ + "ruff==0.15.12", + "pyright==1.1.409", "types-humanfriendly == 10.0.0" ] test = [ @@ -65,9 +62,8 @@ dev = [ "ipykernel == 6.29.5", "zimfarm_backend[api]", "zimfarm_backend[scripts]", - "zimfarm_backend[lint]", + "zimfarm_backend[qa]", "zimfarm_backend[test]", - "zimfarm_backend[check]", ] [project.urls] @@ -103,29 +99,19 @@ report-cov = "inv report-cov" coverage = "inv coverage --args '{args}'" html = "inv coverage --html --args '{args}'" -[tool.hatch.envs.lint] -template = "lint" +[tool.hatch.envs.qa] +template = "qa" skip-install = false -features = ["scripts", "lint"] - -[tool.hatch.envs.lint.scripts] -black = "inv lint-black --args '{args}'" -ruff = "inv lint-ruff --args '{args}'" -all = "inv lintall --args '{args}'" -fix-black = "inv fix-black --args '{args}'" -fix-ruff = "inv fix-ruff --args '{args}'" -fixall = "inv fixall --args '{args}'" - -[tool.hatch.envs.check] -features = ["scripts", "test", "check"] - -[tool.hatch.envs.check.scripts] -pyright = "inv check-pyright --args '{args}'" -all = "inv checkall --args '{args}'" - -[tool.black] -line-length = 88 -target-version = ['py313'] +features = ["scripts", "qa", "test"] + +[tool.hatch.envs.qa.scripts] +check-lint = "inv check-lint --args '{args}'" +check-format = "inv check-format --args '{args}'" +check-type = "inv check-type --args '{args}'" +check-all = "inv check-all --args '{args}'" +fix-format = "inv fix-format --args '{args}'" +fix-lint = "inv fix-lint --args '{args}'" +fix-all = "inv fix-all --args '{args}'" [tool.ruff] line-length = 88 diff --git a/backend/src/zimfarm_backend/alembic.ini b/backend/src/zimfarm_backend/alembic.ini index 7f0f9a68d..083b797a1 100644 --- a/backend/src/zimfarm_backend/alembic.ini +++ b/backend/src/zimfarm_backend/alembic.ini @@ -68,14 +68,10 @@ sqlalchemy.url = driver://user:pass@localhost/dbname # on newly generated revision scripts. See the documentation for further # detail and examples -# format using "black" - use the console_scripts runner, against the "black" entrypoint -hooks = black isort -black.type = console_scripts -black.entrypoint = black -black.options = REVISION_SCRIPT_FILENAME -isort.type = console_scripts -isort.entrypoint = isort -isort.options = --profile black REVISION_SCRIPT_FILENAME +hooks = ruff +ruff.type = module +ruff.module = ruff +ruff.options = format REVISION_SCRIPT_FILENAME # Logging configuration [loggers] diff --git a/backend/src/zimfarm_backend/api/routes/models.py b/backend/src/zimfarm_backend/api/routes/models.py index 2e28fdf3d..8d5e45687 100644 --- a/backend/src/zimfarm_backend/api/routes/models.py +++ b/backend/src/zimfarm_backend/api/routes/models.py @@ -1,4 +1,4 @@ -from typing import Generic, TypeVar +from typing import TypeVar from zimfarm_backend.common.schemas import BaseModel from zimfarm_backend.common.schemas.models import Paginator @@ -6,6 +6,6 @@ T = TypeVar("T") -class ListResponse(BaseModel, Generic[T]): +class ListResponse[T](BaseModel): meta: Paginator items: list[T] diff --git a/backend/src/zimfarm_backend/common/schemas/offliners/builder.py b/backend/src/zimfarm_backend/common/schemas/offliners/builder.py index 79f3db488..f402fe55b 100644 --- a/backend/src/zimfarm_backend/common/schemas/offliners/builder.py +++ b/backend/src/zimfarm_backend/common/schemas/offliners/builder.py @@ -1,4 +1,4 @@ -# ruff: noqa: UP007 +# ruff: noqa: UP045 from collections.abc import Callable from enum import Enum, StrEnum from itertools import chain diff --git a/backend/src/zimfarm_backend/db/requested_task.py b/backend/src/zimfarm_backend/db/requested_task.py index 7f52bd2aa..68c8d3329 100644 --- a/backend/src/zimfarm_backend/db/requested_task.py +++ b/backend/src/zimfarm_backend/db/requested_task.py @@ -584,8 +584,10 @@ def filter_req_task_with_no_recipe_duration(task: RequestedTask): [ create_requested_task_with_duration(session, task=task, worker=worker) for task in filter( - lambda task: filter_req_task_for_ip_issues(task) - and filter_req_task_with_no_recipe_duration(task), + lambda task: ( + filter_req_task_for_ip_issues(task) + and filter_req_task_with_no_recipe_duration(task) + ), session.scalars(query), ) ], diff --git a/backend/tasks.py b/backend/tasks.py index 87cd55296..91def6a3e 100644 --- a/backend/tasks.py +++ b/backend/tasks.py @@ -42,69 +42,65 @@ def coverage(ctx: Context, args: str = "", *, html: bool = False): report_cov(ctx, html=html) -@task(optional=["args"], help={"args": "black additional arguments"}) -def lint_black(ctx: Context, args: str = "."): +def _lint(ctx: Context, args: str = "."): args = args or "." # needed for hatch script - ctx.run("black --version", pty=use_pty) - ctx.run(f"black --check --diff {args}", pty=use_pty) + ctx.run("ruff --version", pty=use_pty) + ctx.run(f"ruff check {args}", pty=use_pty) @task(optional=["args"], help={"args": "ruff additional arguments"}) -def lint_ruff(ctx: Context, args: str = "."): +def check_lint(ctx: Context, args: str = "."): + """check linting with ruff""" args = args or "." # needed for hatch script - ctx.run("ruff --version", pty=use_pty) - ctx.run(f"ruff check {args}", pty=use_pty) + _lint(ctx, args) -@task( - optional=["args"], - help={ - "args": "linting tools (black, ruff) additional arguments, typically a path", - }, -) -def lintall(ctx: Context, args: str = "."): - """Check linting""" +@task(optional=["args"], help={"args": "ruff additional arguments"}) +def fix_lint(ctx: Context, args: str = "."): + """fix linting issues with ruff""" args = args or "." # needed for hatch script - lint_black(ctx, args) - lint_ruff(ctx, args) + _lint(ctx, f"--fix {args}") @task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) -def check_pyright(ctx: Context, args: str = ""): +def check_type(ctx: Context, args: str = ""): """check static types with pyright""" ctx.run("pyright --version") ctx.run(f"pyright {args}", pty=use_pty) -@task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) -def checkall(ctx: Context, args: str = ""): - """check static types""" - check_pyright(ctx, args) +def _format(ctx: Context, args: str = "."): + args = args or "." # needed for hatch script + ctx.run("ruff --version", pty=use_pty) + ctx.run(f"ruff format {args}", pty=use_pty) -@task(optional=["args"], help={"args": "black additional arguments"}) -def fix_black(ctx: Context, args: str = "."): - """fix black formatting""" +@task(optional=["args"], help={"args": "ruff additional arguments"}) +def check_format(ctx: Context, args: str = "."): + """check formatting with ruff""" args = args or "." # needed for hatch script - ctx.run(f"black {args}", pty=use_pty) + _format(ctx, f"--check {args}") @task(optional=["args"], help={"args": "ruff additional arguments"}) -def fix_ruff(ctx: Context, args: str = "."): - """fix all ruff rules""" +def fix_format(ctx: Context, args: str = "."): + """fix formatting with ruff""" args = args or "." # needed for hatch script - ctx.run(f"ruff check --fix {args}", pty=use_pty) + _format(ctx, args) -@task( - optional=["args"], - help={ - "args": "linting tools (black, ruff) additional arguments, typically a path", - }, -) -def fixall(ctx: Context, args: str = "."): +@task(optional=["args"], help={"args": "additional arguments"}) +def check_all(ctx: Context, args: str = ""): + """check linting, formatting and static types""" + args = args or "." # needed for hatch script + check_lint(ctx, args) + check_format(ctx, args) + check_type(ctx, args) + + +@task(optional=["args"], help={"args": "additional arguments"}) +def fix_all(ctx: Context, args: str = ""): """Fix everything automatically""" args = args or "." # needed for hatch script - fix_black(ctx, args) - fix_ruff(ctx, args) - lintall(ctx, args) + fix_lint(ctx, args) + fix_format(ctx, args) diff --git a/backend/tests/db/test_account.py b/backend/tests/db/test_account.py index ff01bde0f..d2197fe39 100644 --- a/backend/tests/db/test_account.py +++ b/backend/tests/db/test_account.py @@ -215,7 +215,7 @@ def test_update_account_partial(dbsession: OrmSession, account: Account): def test_update_account_no_display_name(dbsession: OrmSession, account: Account): - with pytest.raises(ValueError, match="Account must have a display name."): + with pytest.raises(ValueError, match=r"Account must have a display name."): update_account( dbsession, account_id=account.id, diff --git a/backend/tests/test_offliners_utils.py b/backend/tests/test_offliners_utils.py index cfa102726..4ba6cae0a 100644 --- a/backend/tests/test_offliners_utils.py +++ b/backend/tests/test_offliners_utils.py @@ -227,6 +227,7 @@ def test_command_for_is_publisher_unset( ], ) def test_command_for_std_output( + *, mwoffliner_definition: OfflinerDefinitionSchema, mwoffliner: OfflinerSchema, recipe_config: RecipeConfigSchema, @@ -268,6 +269,7 @@ def test_command_for_std_output_unset( ], ) def test_command_for_std_stats( + *, mwoffliner_definition: OfflinerDefinitionSchema, mwoffliner: OfflinerSchema, recipe_config: RecipeConfigSchema, diff --git a/backend/tests/test_token_decoder.py b/backend/tests/test_token_decoder.py index 8592b28e4..d6b2d61d1 100644 --- a/backend/tests/test_token_decoder.py +++ b/backend/tests/test_token_decoder.py @@ -831,6 +831,6 @@ def test_ssh_token_decoder_no_session( decoder = SshTokenDecoder() with pytest.raises( - ValueError, match="OrmSession is required to decode SSH bearer tokens." + ValueError, match=r"OrmSession is required to decode SSH bearer tokens." ): decoder.decode(token) diff --git a/backend/tests/test_validators.py b/backend/tests/test_validators.py index 87d93e3c1..e119e21da 100644 --- a/backend/tests/test_validators.py +++ b/backend/tests/test_validators.py @@ -234,7 +234,7 @@ def test_parse_bool(value: Any, *, expected: bool): ], ) def test_relaxed_boolean_model( - value: str | int | bool, expected: RaisesContext[Exception] + *, value: str | int | bool, expected: RaisesContext[Exception] ): with expected: SkipableBoolModel.model_validate({"value": value}) @@ -273,7 +273,7 @@ def test_relaxed_boolean_model( ], ) def test_relaxed_boolean_skip_validation( - value: str | int | bool, expected: RaisesContext[Exception] + *, value: str | int | bool, expected: RaisesContext[Exception] ): with expected: SkipableBoolModel.model_validate( diff --git a/healthcheck/pyproject.toml b/healthcheck/pyproject.toml index 344263484..5361db59f 100644 --- a/healthcheck/pyproject.toml +++ b/healthcheck/pyproject.toml @@ -31,12 +31,9 @@ dynamic = ["version"] scripts = [ "invoke == 2.2.0", ] -lint = [ - "black == 25.1.0", - "ruff == 0.11.10", -] -check = [ - "pyright == 1.1.400", +qa = [ + "ruff==0.15.12", + "pyright==1.1.409", "types-humanfriendly == 10.0.0" ] test = [ @@ -51,9 +48,8 @@ dev = [ "pre-commit == 4.2.0", "ipykernel == 6.29.5", "healthcheck[scripts]", - "healthcheck[lint]", + "healthcheck[qa]", "healthcheck[test]", - "healthcheck[check]", ] [project.urls] @@ -84,29 +80,19 @@ report-cov = "inv report-cov" coverage = "inv coverage --args '{args}'" html = "inv coverage --html --args '{args}'" -[tool.hatch.envs.lint] -template = "lint" +[tool.hatch.envs.qa] +template = "qa" skip-install = false -features = ["scripts", "lint"] - -[tool.hatch.envs.lint.scripts] -black = "inv lint-black --args '{args}'" -ruff = "inv lint-ruff --args '{args}'" -all = "inv lintall --args '{args}'" -fix-black = "inv fix-black --args '{args}'" -fix-ruff = "inv fix-ruff --args '{args}'" -fixall = "inv fixall --args '{args}'" - -[tool.hatch.envs.check] -features = ["scripts", "test", "check"] - -[tool.hatch.envs.check.scripts] -pyright = "inv check-pyright --args '{args}'" -all = "inv checkall --args '{args}'" - -[tool.black] -line-length = 88 -target-version = ['py313'] +features = ["scripts", "qa", "test"] + +[tool.hatch.envs.qa.scripts] +check-lint = "inv check-lint --args '{args}'" +check-format = "inv check-format --args '{args}'" +check-type = "inv check-type --args '{args}'" +check-all = "inv check-all --args '{args}'" +fix-format = "inv fix-format --args '{args}'" +fix-lint = "inv fix-lint --args '{args}'" +fix-all = "inv fix-all --args '{args}'" [tool.ruff] line-length = 88 diff --git a/healthcheck/src/healthcheck/status/__init__.py b/healthcheck/src/healthcheck/status/__init__.py index b8d404262..9da7be4d4 100644 --- a/healthcheck/src/healthcheck/status/__init__.py +++ b/healthcheck/src/healthcheck/status/__init__.py @@ -1,6 +1,6 @@ import logging from http import HTTPStatus -from typing import Generic, TypeVar +from typing import TypeVar from pydantic import BaseModel @@ -24,7 +24,7 @@ T = TypeVar("T") -class Result(BaseModel, Generic[T]): +class Result[T](BaseModel): status_code: HTTPStatus success: bool data: T | None = None diff --git a/healthcheck/tasks.py b/healthcheck/tasks.py index 87cd55296..91def6a3e 100644 --- a/healthcheck/tasks.py +++ b/healthcheck/tasks.py @@ -42,69 +42,65 @@ def coverage(ctx: Context, args: str = "", *, html: bool = False): report_cov(ctx, html=html) -@task(optional=["args"], help={"args": "black additional arguments"}) -def lint_black(ctx: Context, args: str = "."): +def _lint(ctx: Context, args: str = "."): args = args or "." # needed for hatch script - ctx.run("black --version", pty=use_pty) - ctx.run(f"black --check --diff {args}", pty=use_pty) + ctx.run("ruff --version", pty=use_pty) + ctx.run(f"ruff check {args}", pty=use_pty) @task(optional=["args"], help={"args": "ruff additional arguments"}) -def lint_ruff(ctx: Context, args: str = "."): +def check_lint(ctx: Context, args: str = "."): + """check linting with ruff""" args = args or "." # needed for hatch script - ctx.run("ruff --version", pty=use_pty) - ctx.run(f"ruff check {args}", pty=use_pty) + _lint(ctx, args) -@task( - optional=["args"], - help={ - "args": "linting tools (black, ruff) additional arguments, typically a path", - }, -) -def lintall(ctx: Context, args: str = "."): - """Check linting""" +@task(optional=["args"], help={"args": "ruff additional arguments"}) +def fix_lint(ctx: Context, args: str = "."): + """fix linting issues with ruff""" args = args or "." # needed for hatch script - lint_black(ctx, args) - lint_ruff(ctx, args) + _lint(ctx, f"--fix {args}") @task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) -def check_pyright(ctx: Context, args: str = ""): +def check_type(ctx: Context, args: str = ""): """check static types with pyright""" ctx.run("pyright --version") ctx.run(f"pyright {args}", pty=use_pty) -@task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) -def checkall(ctx: Context, args: str = ""): - """check static types""" - check_pyright(ctx, args) +def _format(ctx: Context, args: str = "."): + args = args or "." # needed for hatch script + ctx.run("ruff --version", pty=use_pty) + ctx.run(f"ruff format {args}", pty=use_pty) -@task(optional=["args"], help={"args": "black additional arguments"}) -def fix_black(ctx: Context, args: str = "."): - """fix black formatting""" +@task(optional=["args"], help={"args": "ruff additional arguments"}) +def check_format(ctx: Context, args: str = "."): + """check formatting with ruff""" args = args or "." # needed for hatch script - ctx.run(f"black {args}", pty=use_pty) + _format(ctx, f"--check {args}") @task(optional=["args"], help={"args": "ruff additional arguments"}) -def fix_ruff(ctx: Context, args: str = "."): - """fix all ruff rules""" +def fix_format(ctx: Context, args: str = "."): + """fix formatting with ruff""" args = args or "." # needed for hatch script - ctx.run(f"ruff check --fix {args}", pty=use_pty) + _format(ctx, args) -@task( - optional=["args"], - help={ - "args": "linting tools (black, ruff) additional arguments, typically a path", - }, -) -def fixall(ctx: Context, args: str = "."): +@task(optional=["args"], help={"args": "additional arguments"}) +def check_all(ctx: Context, args: str = ""): + """check linting, formatting and static types""" + args = args or "." # needed for hatch script + check_lint(ctx, args) + check_format(ctx, args) + check_type(ctx, args) + + +@task(optional=["args"], help={"args": "additional arguments"}) +def fix_all(ctx: Context, args: str = ""): """Fix everything automatically""" args = args or "." # needed for hatch script - fix_black(ctx, args) - fix_ruff(ctx, args) - lintall(ctx, args) + fix_lint(ctx, args) + fix_format(ctx, args) diff --git a/recipesauto/pyproject.toml b/recipesauto/pyproject.toml index 1ee57a5c5..7a8b75e4a 100644 --- a/recipesauto/pyproject.toml +++ b/recipesauto/pyproject.toml @@ -25,16 +25,17 @@ additional-keywords = ["zimfarm"] [project.optional-dependencies] scripts = ["invoke==2.2.0"] -lint = ["black==24.8.0", "ruff==0.6.4"] -check = ["pyright==1.1.380"] +qa = [ + "ruff==0.15.12", + "pyright==1.1.409", +] test = ["pytest==8.3.3", "coverage==7.6.1"] dev = [ "pre-commit==3.8.0", "debugpy==1.8.5", "recipesauto[scripts]", - "recipesauto[lint]", "recipesauto[test]", - "recipesauto[check]", + "recipesauto[qa]", "humanfriendly==10.0" ] @@ -63,31 +64,19 @@ report-cov = "inv report-cov" coverage = "inv coverage --args '{args}'" html = "inv coverage --html --args '{args}'" -[tool.hatch.envs.lint] -template = "lint" -python = "py312" +[tool.hatch.envs.qa] +template = "qa" skip-install = false -features = ["scripts", "lint"] - -[tool.hatch.envs.lint.scripts] -black = "inv lint-black --args '{args}'" -ruff = "inv lint-ruff --args '{args}'" -all = "inv lintall --args '{args}'" -fix-black = "inv fix-black --args '{args}'" -fix-ruff = "inv fix-ruff --args '{args}'" -fixall = "inv fixall --args '{args}'" - -[tool.hatch.envs.check] -features = ["scripts", "check"] - -[tool.hatch.envs.check.scripts] -pyright = "inv check-pyright --args '{args}'" -all = "inv checkall --args '{args}'" - -[tool.black] -line-length = 88 -target-version = ['py312'] -exclude = "(src/recipesauto/templates/.*|.hatch/.*)" +features = ["scripts", "qa", "test"] + +[tool.hatch.envs.qa.scripts] +check-lint = "inv check-lint --args '{args}'" +check-format = "inv check-format --args '{args}'" +check-type = "inv check-type --args '{args}'" +check-all = "inv check-all --args '{args}'" +fix-format = "inv fix-format --args '{args}'" +fix-lint = "inv fix-lint --args '{args}'" +fix-all = "inv fix-all --args '{args}'" [tool.ruff] target-version = "py312" diff --git a/recipesauto/src/recipesauto/__main__.py b/recipesauto/src/recipesauto/__main__.py index befbaf6e4..3119dabc5 100644 --- a/recipesauto/src/recipesauto/__main__.py +++ b/recipesauto/src/recipesauto/__main__.py @@ -1,3 +1,4 @@ +# ruff: noqa: PLC0415 import sys from recipesauto.constants import logger diff --git a/recipesauto/src/recipesauto/processor.py b/recipesauto/src/recipesauto/processor.py index 20f694209..33680d59e 100644 --- a/recipesauto/src/recipesauto/processor.py +++ b/recipesauto/src/recipesauto/processor.py @@ -1,3 +1,5 @@ +# ruff: noqa: PLC0415 + import copy from http import HTTPStatus from typing import Any diff --git a/recipesauto/tasks.py b/recipesauto/tasks.py index 87cd55296..91def6a3e 100644 --- a/recipesauto/tasks.py +++ b/recipesauto/tasks.py @@ -42,69 +42,65 @@ def coverage(ctx: Context, args: str = "", *, html: bool = False): report_cov(ctx, html=html) -@task(optional=["args"], help={"args": "black additional arguments"}) -def lint_black(ctx: Context, args: str = "."): +def _lint(ctx: Context, args: str = "."): args = args or "." # needed for hatch script - ctx.run("black --version", pty=use_pty) - ctx.run(f"black --check --diff {args}", pty=use_pty) + ctx.run("ruff --version", pty=use_pty) + ctx.run(f"ruff check {args}", pty=use_pty) @task(optional=["args"], help={"args": "ruff additional arguments"}) -def lint_ruff(ctx: Context, args: str = "."): +def check_lint(ctx: Context, args: str = "."): + """check linting with ruff""" args = args or "." # needed for hatch script - ctx.run("ruff --version", pty=use_pty) - ctx.run(f"ruff check {args}", pty=use_pty) + _lint(ctx, args) -@task( - optional=["args"], - help={ - "args": "linting tools (black, ruff) additional arguments, typically a path", - }, -) -def lintall(ctx: Context, args: str = "."): - """Check linting""" +@task(optional=["args"], help={"args": "ruff additional arguments"}) +def fix_lint(ctx: Context, args: str = "."): + """fix linting issues with ruff""" args = args or "." # needed for hatch script - lint_black(ctx, args) - lint_ruff(ctx, args) + _lint(ctx, f"--fix {args}") @task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) -def check_pyright(ctx: Context, args: str = ""): +def check_type(ctx: Context, args: str = ""): """check static types with pyright""" ctx.run("pyright --version") ctx.run(f"pyright {args}", pty=use_pty) -@task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) -def checkall(ctx: Context, args: str = ""): - """check static types""" - check_pyright(ctx, args) +def _format(ctx: Context, args: str = "."): + args = args or "." # needed for hatch script + ctx.run("ruff --version", pty=use_pty) + ctx.run(f"ruff format {args}", pty=use_pty) -@task(optional=["args"], help={"args": "black additional arguments"}) -def fix_black(ctx: Context, args: str = "."): - """fix black formatting""" +@task(optional=["args"], help={"args": "ruff additional arguments"}) +def check_format(ctx: Context, args: str = "."): + """check formatting with ruff""" args = args or "." # needed for hatch script - ctx.run(f"black {args}", pty=use_pty) + _format(ctx, f"--check {args}") @task(optional=["args"], help={"args": "ruff additional arguments"}) -def fix_ruff(ctx: Context, args: str = "."): - """fix all ruff rules""" +def fix_format(ctx: Context, args: str = "."): + """fix formatting with ruff""" args = args or "." # needed for hatch script - ctx.run(f"ruff check --fix {args}", pty=use_pty) + _format(ctx, args) -@task( - optional=["args"], - help={ - "args": "linting tools (black, ruff) additional arguments, typically a path", - }, -) -def fixall(ctx: Context, args: str = "."): +@task(optional=["args"], help={"args": "additional arguments"}) +def check_all(ctx: Context, args: str = ""): + """check linting, formatting and static types""" + args = args or "." # needed for hatch script + check_lint(ctx, args) + check_format(ctx, args) + check_type(ctx, args) + + +@task(optional=["args"], help={"args": "additional arguments"}) +def fix_all(ctx: Context, args: str = ""): """Fix everything automatically""" args = args or "." # needed for hatch script - fix_black(ctx, args) - fix_ruff(ctx, args) - lintall(ctx, args) + fix_lint(ctx, args) + fix_format(ctx, args) diff --git a/worker/pyproject.toml b/worker/pyproject.toml index 813287552..5124ca0e6 100644 --- a/worker/pyproject.toml +++ b/worker/pyproject.toml @@ -31,13 +31,11 @@ dynamic = ["version"] scripts = [ "invoke == 2.2.0", ] -lint = [ - "black == 25.1.0", - "ruff == 0.11.10", -] -check = [ - "pyright == 1.1.400", - "types-humanfriendly == 10.0.0" + +qa = [ + "ruff==0.15.12", + "pyright==1.1.409", + "types-humanfriendly == 10.0.0" ] test = [ @@ -55,10 +53,9 @@ dev = [ "pre-commit == 4.2.0", "ipykernel == 6.29.5", "zimfarm_worker[scripts]", - "zimfarm_worker[lint]", "zimfarm_worker[test]", - "zimfarm_worker[check]", - "zimfarm_worker[task]" + "zimfarm_worker[test]", + "zimfarm_worker[qa]" ] @@ -96,29 +93,19 @@ report-cov = "inv report-cov" coverage = "inv coverage --args '{args}'" html = "inv coverage --html --args '{args}'" -[tool.hatch.envs.lint] -template = "lint" +[tool.hatch.envs.qa] +template = "qa" skip-install = false -features = ["scripts", "lint"] - -[tool.hatch.envs.lint.scripts] -black = "inv lint-black --args '{args}'" -ruff = "inv lint-ruff --args '{args}'" -all = "inv lintall --args '{args}'" -fix-black = "inv fix-black --args '{args}'" -fix-ruff = "inv fix-ruff --args '{args}'" -fixall = "inv fixall --args '{args}'" - -[tool.hatch.envs.check] -features = ["scripts", "test", "check"] - -[tool.hatch.envs.check.scripts] -pyright = "inv check-pyright --args '{args}'" -all = "inv checkall --args '{args}'" - -[tool.black] -line-length = 88 -target-version = ['py313'] +features = ["scripts", "qa", "test"] + +[tool.hatch.envs.qa.scripts] +check-lint = "inv check-lint --args '{args}'" +check-format = "inv check-format --args '{args}'" +check-type = "inv check-type --args '{args}'" +check-all = "inv check-all --args '{args}'" +fix-format = "inv fix-format --args '{args}'" +fix-lint = "inv fix-lint --args '{args}'" +fix-all = "inv fix-all --args '{args}'" [tool.ruff] line-length = 88 diff --git a/worker/src/zimfarm_worker/common/docker.py b/worker/src/zimfarm_worker/common/docker.py index 9746a815e..5f8899259 100644 --- a/worker/src/zimfarm_worker/common/docker.py +++ b/worker/src/zimfarm_worker/common/docker.py @@ -98,9 +98,7 @@ def pull_image(client: DockerClient, repository: str, tag: str | None = None): @retry def run_container(client: DockerClient, image: Image, **kwargs: Any) -> Container: - return client.containers.run( - image, **kwargs - ) # pyright: ignore[reportGeneralTypeIssues, reportReturnType, reportUnknownVariableType] + return client.containers.run(image, **kwargs) # pyright: ignore[reportGeneralTypeIssues, reportReturnType, reportUnknownVariableType] @retry @@ -121,9 +119,7 @@ def list_containers(client: DockerClient, **kwargs: Any): @retry def remove_container(client: DockerClient, **kwargs: Any): """container="", v=False, link=False, force=False""" - return client.api.remove_container( - **kwargs - ) # pyright: ignore[reportGeneralTypeIssues, reportReturnType] + return client.api.remove_container(**kwargs) # pyright: ignore[reportGeneralTypeIssues, reportReturnType] @retry @@ -151,9 +147,7 @@ def inspect_image(client: DockerClient, image: str) -> dict[str, Any]: @retry def stop_container(client: DockerClient, container: str, **kwargs: Any): """container="", timeout=None""" - return client.api.stop( - container, **kwargs - ) # pyright: ignore[reportGeneralTypeIssues, reportReturnType] + return client.api.stop(container, **kwargs) # pyright: ignore[reportGeneralTypeIssues, reportReturnType] @retry @@ -166,9 +160,7 @@ def wait_container(client: DockerClient, **kwargs: Any): def container_logs(client: DockerClient, **kwargs: Any): """container, stdout=True, stderr=True, stream=False, timestamps=False, tail='all', since=None, follow=None, until=None""" - return client.api.logs( - **kwargs - ) # pyright: ignore[reportGeneralTypeIssues, reportReturnType, reportUnknownVariableType] + return client.api.logs(**kwargs) # pyright: ignore[reportGeneralTypeIssues, reportReturnType, reportUnknownVariableType] @retry @@ -176,13 +168,9 @@ def get_or_pull_image(client: DockerClient, name: str): """attempt to get locally or pull and return. Name is repo:tag""" if ":" not in name: # consider missing :tag info as a local image for tests - return client.images.get( - name - ) # pyright: ignore[reportGeneralTypeIssues,reportReturnType] + return client.images.get(name) # pyright: ignore[reportGeneralTypeIssues,reportReturnType] - return client.images.pull( - name - ) # pyright: ignore[reportGeneralTypeIssues,reportReturnType] + return client.images.pull(name) # pyright: ignore[reportGeneralTypeIssues,reportReturnType] @dataclass(kw_only=True) diff --git a/worker/src/zimfarm_worker/task/worker.py b/worker/src/zimfarm_worker/task/worker.py index b317f9963..a7992ec61 100644 --- a/worker/src/zimfarm_worker/task/worker.py +++ b/worker/src/zimfarm_worker/task/worker.py @@ -109,9 +109,9 @@ def __init__( self.dns: list[str] | None = None # list of DNS IPs or None self.monitor: Container | None = None # monitor container - self.zim_files_actions_status: dict[str, dict[str, str]] = ( - {} - ) # ZIM files registry + self.zim_files_actions_status: dict[ + str, dict[str, str] + ] = {} # ZIM files registry self.zim_upload_retries: defaultdict[str, int] = defaultdict( int ) # ZIM files with upload errors (registry) @@ -1102,9 +1102,11 @@ def busy_zim_files(self) -> list[tuple[str, dict[str, str]]]: including DOING as those might fail and go back to PENDING""" return list( filter( - lambda x: x[1][ZIM_UPLOAD] in (PENDING, DOING) - or x[1][ZIM_CHECK] in (PENDING, DOING) - or x[1][CHK_UPLOAD] in (PENDING, DOING), + lambda x: ( + x[1][ZIM_UPLOAD] in (PENDING, DOING) + or x[1][ZIM_CHECK] in (PENDING, DOING) + or x[1][CHK_UPLOAD] in (PENDING, DOING) + ), self.zim_files_actions_status.items(), ) ) @@ -1165,12 +1167,11 @@ def check_zims(self): ).strip() zim_info = get_zim_info( - self.task_workdir - / zim_file # pyright: ignore[reportUnknownArgumentType] + self.task_workdir / zim_file # pyright: ignore[reportUnknownArgumentType] ) try: - zimcheck_result = ujson.loads(zimcheck_log) + zimcheck_result = ujson.loads(zimcheck_log) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] except Exception as exc: # Failed to parse JSON, send raw log zimcheck_result = None diff --git a/worker/tasks.py b/worker/tasks.py index 87cd55296..91def6a3e 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -42,69 +42,65 @@ def coverage(ctx: Context, args: str = "", *, html: bool = False): report_cov(ctx, html=html) -@task(optional=["args"], help={"args": "black additional arguments"}) -def lint_black(ctx: Context, args: str = "."): +def _lint(ctx: Context, args: str = "."): args = args or "." # needed for hatch script - ctx.run("black --version", pty=use_pty) - ctx.run(f"black --check --diff {args}", pty=use_pty) + ctx.run("ruff --version", pty=use_pty) + ctx.run(f"ruff check {args}", pty=use_pty) @task(optional=["args"], help={"args": "ruff additional arguments"}) -def lint_ruff(ctx: Context, args: str = "."): +def check_lint(ctx: Context, args: str = "."): + """check linting with ruff""" args = args or "." # needed for hatch script - ctx.run("ruff --version", pty=use_pty) - ctx.run(f"ruff check {args}", pty=use_pty) + _lint(ctx, args) -@task( - optional=["args"], - help={ - "args": "linting tools (black, ruff) additional arguments, typically a path", - }, -) -def lintall(ctx: Context, args: str = "."): - """Check linting""" +@task(optional=["args"], help={"args": "ruff additional arguments"}) +def fix_lint(ctx: Context, args: str = "."): + """fix linting issues with ruff""" args = args or "." # needed for hatch script - lint_black(ctx, args) - lint_ruff(ctx, args) + _lint(ctx, f"--fix {args}") @task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) -def check_pyright(ctx: Context, args: str = ""): +def check_type(ctx: Context, args: str = ""): """check static types with pyright""" ctx.run("pyright --version") ctx.run(f"pyright {args}", pty=use_pty) -@task(optional=["args"], help={"args": "check tools (pyright) additional arguments"}) -def checkall(ctx: Context, args: str = ""): - """check static types""" - check_pyright(ctx, args) +def _format(ctx: Context, args: str = "."): + args = args or "." # needed for hatch script + ctx.run("ruff --version", pty=use_pty) + ctx.run(f"ruff format {args}", pty=use_pty) -@task(optional=["args"], help={"args": "black additional arguments"}) -def fix_black(ctx: Context, args: str = "."): - """fix black formatting""" +@task(optional=["args"], help={"args": "ruff additional arguments"}) +def check_format(ctx: Context, args: str = "."): + """check formatting with ruff""" args = args or "." # needed for hatch script - ctx.run(f"black {args}", pty=use_pty) + _format(ctx, f"--check {args}") @task(optional=["args"], help={"args": "ruff additional arguments"}) -def fix_ruff(ctx: Context, args: str = "."): - """fix all ruff rules""" +def fix_format(ctx: Context, args: str = "."): + """fix formatting with ruff""" args = args or "." # needed for hatch script - ctx.run(f"ruff check --fix {args}", pty=use_pty) + _format(ctx, args) -@task( - optional=["args"], - help={ - "args": "linting tools (black, ruff) additional arguments, typically a path", - }, -) -def fixall(ctx: Context, args: str = "."): +@task(optional=["args"], help={"args": "additional arguments"}) +def check_all(ctx: Context, args: str = ""): + """check linting, formatting and static types""" + args = args or "." # needed for hatch script + check_lint(ctx, args) + check_format(ctx, args) + check_type(ctx, args) + + +@task(optional=["args"], help={"args": "additional arguments"}) +def fix_all(ctx: Context, args: str = ""): """Fix everything automatically""" args = args or "." # needed for hatch script - fix_black(ctx, args) - fix_ruff(ctx, args) - lintall(ctx, args) + fix_lint(ctx, args) + fix_format(ctx, args)