diff --git a/.github/workflows/ibis-main.yml b/.github/workflows/ibis-main.yml index 1a901fba17b4..4759e1cc3772 100644 --- a/.github/workflows/ibis-main.yml +++ b/.github/workflows/ibis-main.yml @@ -49,8 +49,12 @@ jobs: - "3.10" - "3.14" pyarrow: - - true - - false + - none + - latest + - old + exclude: + - python-version: "3.14" + pyarrow: old steps: - name: checkout uses: actions/checkout@v6 @@ -76,9 +80,13 @@ jobs: if: matrix.os == 'windows-latest' run: choco install graphviz - - name: install numpy/pandas/pyarrow - if: matrix.pyarrow - run: pip install numpy pandas pyarrow pyarrow-hotfix + - name: install numpy/pandas/pyarrow (latest) + if: matrix.pyarrow == 'latest' + run: pip install numpy pandas pyarrow + + - name: install numpy/pandas/pyarrow (old) + if: matrix.pyarrow == 'old' + run: pip install "numpy<2" pandas pyarrow==10.0.1 pyarrow-hotfix - uses: extractions/setup-just@v3 env: diff --git a/ibis/backends/__init__.py b/ibis/backends/__init__.py index 908654622508..872f370287d8 100644 --- a/ibis/backends/__init__.py +++ b/ibis/backends/__init__.py @@ -101,7 +101,7 @@ def _import_pyarrow(): "Exporting to arrow formats requires `pyarrow` but it is not installed" ) else: - import pyarrow_hotfix # noqa: F401 + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 return pyarrow @@ -1607,9 +1607,10 @@ class PyArrowExampleLoader(ExampleLoader): temporary_example: bool = True def _load_parquet(self, *, path: str | Path, table_name: str) -> ir.Table: - import pyarrow_hotfix # noqa: F401, I001 import pyarrow.parquet as pq + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 + table = pq.read_table(path) return self.create_table( table_name, @@ -1619,10 +1620,12 @@ def _load_parquet(self, *, path: str | Path, table_name: str) -> ir.Table: ) def _load_csv(self, *, path: str | Path, table_name: str) -> ir.Table: - import pyarrow_hotfix # noqa: F401, I001 + import pyarrow as pa import pyarrow.csv + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 + # The convert options lets pyarrow treat empty strings as null for # string columns, but not quoted empty strings. table = pyarrow.csv.read_csv( diff --git a/ibis/backends/athena/__init__.py b/ibis/backends/athena/__init__.py index bd392b72f701..a8d193d2e0a3 100644 --- a/ibis/backends/athena/__init__.py +++ b/ibis/backends/athena/__init__.py @@ -11,7 +11,6 @@ from typing import TYPE_CHECKING, Any import fsspec -import pyarrow_hotfix # noqa: F401 import pyathena import sqlglot as sg import sqlglot.expressions as sge @@ -27,6 +26,7 @@ from ibis.backends import CanCreateDatabase, NoExampleLoader, UrlFromPath from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import AlterTable, RenameTable +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 if TYPE_CHECKING: from collections.abc import Callable, Mapping diff --git a/ibis/backends/clickhouse/__init__.py b/ibis/backends/clickhouse/__init__.py index 62601bf8f4cf..ce1b5d8c77ec 100644 --- a/ibis/backends/clickhouse/__init__.py +++ b/ibis/backends/clickhouse/__init__.py @@ -11,7 +11,6 @@ import clickhouse_connect as cc import pyarrow as pa -import pyarrow_hotfix # noqa: F401 import sqlglot as sg import sqlglot.expressions as sge import toolz @@ -35,6 +34,7 @@ from ibis.backends.clickhouse.converter import ClickHousePandasData from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import C +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping diff --git a/ibis/backends/databricks/__init__.py b/ibis/backends/databricks/__init__.py index 24bbf55de565..4f155387f6f9 100644 --- a/ibis/backends/databricks/__init__.py +++ b/ibis/backends/databricks/__init__.py @@ -14,7 +14,6 @@ import databricks.sql import pyarrow as pa -import pyarrow_hotfix # noqa: F401 import sqlglot as sg import sqlglot.expressions as sge @@ -30,6 +29,7 @@ from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import STAR, AlterTable, RenameTable from ibis.backends.sql.datatypes import DatabricksType +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 if TYPE_CHECKING: from collections.abc import Callable, Iterable, Mapping diff --git a/ibis/backends/datafusion/__init__.py b/ibis/backends/datafusion/__init__.py index ddb866ba6aa2..1ebd28b8d0d0 100644 --- a/ibis/backends/datafusion/__init__.py +++ b/ibis/backends/datafusion/__init__.py @@ -9,7 +9,6 @@ import datafusion as df import pyarrow as pa -import pyarrow_hotfix # noqa: F401 import sqlglot as sg import sqlglot.expressions as sge @@ -32,10 +31,16 @@ ) from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import C +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 from ibis.common.dispatch import lazy_singledispatch from ibis.expr.operations.udf import InputType from ibis.formats.pyarrow import PyArrowSchema, PyArrowType -from ibis.util import gen_name, normalize_filename, normalize_filenames, warn_deprecated +from ibis.util import ( + gen_name, + normalize_filename, + normalize_filenames, + warn_deprecated, +) try: from datafusion import ExecutionContext as SessionContext diff --git a/ibis/backends/datafusion/udfs.py b/ibis/backends/datafusion/udfs.py index 7ef1fc97c9b5..cea9eebecc8c 100644 --- a/ibis/backends/datafusion/udfs.py +++ b/ibis/backends/datafusion/udfs.py @@ -5,10 +5,10 @@ import pyarrow as pa import pyarrow.compute as pc -import pyarrow_hotfix # noqa: F401 import ibis.common.exceptions as com import ibis.expr.datatypes as dt +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 def _extract_epoch_seconds(array) -> dt.int32: diff --git a/ibis/backends/duckdb/__init__.py b/ibis/backends/duckdb/__init__.py index 7e46b9bc3b24..5b87e36acab7 100644 --- a/ibis/backends/duckdb/__init__.py +++ b/ibis/backends/duckdb/__init__.py @@ -33,6 +33,7 @@ ) from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import STAR, AlterTable, C, RenameTable +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 from ibis.common.dispatch import lazy_singledispatch from ibis.expr.operations.udf import InputType @@ -42,7 +43,6 @@ import pandas as pd import polars as pl import pyarrow as pa - import pyarrow_hotfix # noqa: F401 import torch from fsspec import AbstractFileSystem @@ -1380,7 +1380,6 @@ def to_pyarrow_batches( The number of rows to fetch per batch """ import pyarrow as pa - import pyarrow_hotfix # noqa: F401 self._run_pre_execute_hooks(expr) table = expr.as_table() @@ -1422,7 +1421,6 @@ def execute( """Execute an expression.""" import pandas as pd import pyarrow.types as pat - import pyarrow_hotfix # noqa: F401 from ibis.backends.duckdb.converter import DuckDBPandasData diff --git a/ibis/backends/flink/__init__.py b/ibis/backends/flink/__init__.py index a29e0bd39f68..ded53530b501 100644 --- a/ibis/backends/flink/__init__.py +++ b/ibis/backends/flink/__init__.py @@ -493,9 +493,9 @@ def create_table( """ import pandas as pd import pyarrow as pa - import pyarrow_hotfix # noqa: F401 import ibis.expr.types as ir + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 if obj is None and schema is None: raise exc.IbisError("`schema` or `obj` is required") @@ -937,7 +937,8 @@ def insert( """ import pandas as pd import pyarrow as pa - import pyarrow_hotfix # noqa: F401 + + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 if isinstance(obj, ir.Table): statement = InsertSelect( @@ -982,7 +983,8 @@ def to_pyarrow( **kwargs: Any, ) -> pa.Table: import pyarrow as pa - import pyarrow_hotfix # noqa: F401 + + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 pyarrow_batches = iter( self.to_pyarrow_batches(expr, params=params, limit=limit, **kwargs) @@ -1009,7 +1011,8 @@ def to_pyarrow_batches( **kwargs: Any, ): import pyarrow as pa - import pyarrow_hotfix # noqa: F401 + + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 ibis_table = expr.as_table() @@ -1059,12 +1062,12 @@ def _from_pyflink_table_to_pyarrow_batches( chunk_size: int | None = None, ): import pyarrow as pa - import pyarrow_hotfix # noqa: F401 from pyflink.java_gateway import get_gateway from pyflink.table.serializers import ArrowSerializer from pyflink.table.types import create_arrow_schema from ibis.backends.flink.datatypes import get_field_data_types + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 # Note (mehmet): Implementation of this is based on # pyflink/table/table.py: to_pandas(). diff --git a/ibis/backends/impala/__init__.py b/ibis/backends/impala/__init__.py index 1daa4a528c9e..dfd6efad8e5e 100644 --- a/ibis/backends/impala/__init__.py +++ b/ibis/backends/impala/__init__.py @@ -1334,8 +1334,8 @@ def to_pyarrow( **kwargs: Any, ) -> pa.Table: import pyarrow as pa - import pyarrow_hotfix # noqa: F401 + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 from ibis.formats.pyarrow import PyArrowData self._run_pre_execute_hooks(expr) diff --git a/ibis/backends/pyspark/__init__.py b/ibis/backends/pyspark/__init__.py index ddf7f67125af..535852dbc3e5 100644 --- a/ibis/backends/pyspark/__init__.py +++ b/ibis/backends/pyspark/__init__.py @@ -1042,8 +1042,8 @@ def to_pyarrow( "PySpark in streaming mode does not support to_pyarrow" ) import pyarrow as pa - import pyarrow_hotfix # noqa: F401 + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 from ibis.formats.pyarrow import PyArrowData table_expr = expr.as_table() diff --git a/ibis/backends/snowflake/__init__.py b/ibis/backends/snowflake/__init__.py index fbf45902a4e5..a8e07b712983 100644 --- a/ibis/backends/snowflake/__init__.py +++ b/ibis/backends/snowflake/__init__.py @@ -14,7 +14,6 @@ from urllib.request import urlcleanup, urlretrieve import pyarrow as pa -import pyarrow_hotfix # noqa: F401 import sqlglot as sg import sqlglot.expressions as sge @@ -36,6 +35,7 @@ from ibis.backends.snowflake.converter import SnowflakePandasData from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import STAR +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 if TYPE_CHECKING: from collections.abc import Generator, Iterator, Mapping diff --git a/ibis/backends/snowflake/tests/conftest.py b/ibis/backends/snowflake/tests/conftest.py index 493d75241fc3..b0935452d1b5 100644 --- a/ibis/backends/snowflake/tests/conftest.py +++ b/ibis/backends/snowflake/tests/conftest.py @@ -10,7 +10,6 @@ from urllib.request import urlretrieve import pyarrow.parquet as pq -import pyarrow_hotfix # noqa: F401 import pytest import snowflake.connector as sc import sqlglot as sg @@ -19,6 +18,7 @@ from ibis.backends.conftest import TEST_TABLES from ibis.backends.sql.datatypes import SnowflakeType from ibis.backends.tests.base import BackendTest +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 from ibis.formats.pyarrow import PyArrowSchema if TYPE_CHECKING: diff --git a/ibis/common/import_to_try_pyarrow_hotfix.py b/ibis/common/import_to_try_pyarrow_hotfix.py new file mode 100644 index 000000000000..cf97a4a5dd11 --- /dev/null +++ b/ibis/common/import_to_try_pyarrow_hotfix.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +import pyarrow as pa + +if tuple(int(x) for x in pa.__version__.split(".")[:3]) < (14, 0, 1): + try: + import pyarrow_hotfix # noqa: F401 + except ImportError: + raise ImportError("pyarrow_hotfix should be installed for pyarrow<14.0.1") diff --git a/ibis/formats/__init__.py b/ibis/formats/__init__.py index 0e76b1db735e..a4f52f2e7970 100644 --- a/ibis/formats/__init__.py +++ b/ibis/formats/__init__.py @@ -253,7 +253,8 @@ def to_polars(self, schema: Schema) -> pl.DataFrame: # pragma: no cover def to_pyarrow_bytes(self, schema: Schema) -> bytes: import pyarrow as pa - import pyarrow_hotfix # noqa: F401 + + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 data = self.to_pyarrow(schema=schema) out = pa.BufferOutputStream() diff --git a/ibis/formats/pandas.py b/ibis/formats/pandas.py index 06d79ca2dde4..8cf3d9c15f2d 100644 --- a/ibis/formats/pandas.py +++ b/ibis/formats/pandas.py @@ -433,7 +433,8 @@ def to_pyarrow(self, schema: sch.Schema) -> pa.Table: from decimal import Decimal import pyarrow as pa - import pyarrow_hotfix # noqa: F401 + + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 pyarrow_schema = PyArrowSchema.from_ibis(schema) diff --git a/ibis/formats/pyarrow.py b/ibis/formats/pyarrow.py index 5ce9205f15f2..48622bf53522 100644 --- a/ibis/formats/pyarrow.py +++ b/ibis/formats/pyarrow.py @@ -4,10 +4,10 @@ from typing import TYPE_CHECKING, Any import pyarrow as pa -import pyarrow_hotfix # noqa: F401 import ibis.common.exceptions as com import ibis.expr.datatypes as dt +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 from ibis.expr.schema import Schema from ibis.formats import DataMapper, SchemaMapper, TableProxy, TypeMapper from ibis.util import V