feat: make pyarrow hot fix optional

.github/workflows/ibis-main.yml

@@ -49,8 +49,12 @@ jobs:
           - "3.10"
           - "3.14"
         pyarrow:
-          - true
-          - false
+          - none
+          - latest
+          - old
+        exclude:
+          - python-version: "3.14"
+            pyarrow: old
     steps:
       - name: checkout
         uses: actions/checkout@v6
@@ -76,9 +80,13 @@ jobs:
         if: matrix.os == 'windows-latest'
         run: choco install graphviz
 
-      - name: install numpy/pandas/pyarrow
-        if: matrix.pyarrow
-        run: pip install numpy pandas pyarrow pyarrow-hotfix
+      - name: install numpy/pandas/pyarrow (latest)
+        if: matrix.pyarrow == 'latest'
+        run: pip install numpy pandas pyarrow
+
+      - name: install numpy/pandas/pyarrow (old)
+        if: matrix.pyarrow == 'old'
+        run: pip install "numpy<2" pandas pyarrow==10.0.1 pyarrow-hotfix
 
       - uses: extractions/setup-just@v3
         env:

ibis/backends/__init__.py

@@ -101,7 +101,9 @@ def _import_pyarrow():
                 "Exporting to arrow formats requires `pyarrow` but it is not installed"
             )
         else:
-            import pyarrow_hotfix  # noqa: F401
+            from ibis.util import apply_pyarrow_hotfix
+
+            apply_pyarrow_hotfix()
 
             return pyarrow
 
@@ -1607,7 +1609,9 @@ class PyArrowExampleLoader(ExampleLoader):
     temporary_example: bool = True
 
     def _load_parquet(self, *, path: str | Path, table_name: str) -> ir.Table:
-        import pyarrow_hotfix  # noqa: F401, I001
+        from ibis.util import apply_pyarrow_hotfix
+
+        apply_pyarrow_hotfix()
         import pyarrow.parquet as pq
 
         table = pq.read_table(path)
@@ -1619,7 +1623,9 @@ def _load_parquet(self, *, path: str | Path, table_name: str) -> ir.Table:
         )
 
     def _load_csv(self, *, path: str | Path, table_name: str) -> ir.Table:
-        import pyarrow_hotfix  # noqa: F401, I001
+        from ibis.util import apply_pyarrow_hotfix
+
+        apply_pyarrow_hotfix()
         import pyarrow as pa
         import pyarrow.csv
 

ibis/backends/athena/__init__.py

@@ -11,7 +11,6 @@
 from typing import TYPE_CHECKING, Any
 
 import fsspec
-import pyarrow_hotfix  # noqa: F401
 import pyathena
 import sqlglot as sg
 import sqlglot.expressions as sge
@@ -27,6 +26,9 @@
 from ibis.backends import CanCreateDatabase, NoExampleLoader, UrlFromPath
 from ibis.backends.sql import SQLBackend
 from ibis.backends.sql.compilers.base import AlterTable, RenameTable
+from ibis.util import apply_pyarrow_hotfix
+
+apply_pyarrow_hotfix()
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Mapping

ibis/backends/clickhouse/__init__.py

@@ -11,7 +11,6 @@
 
 import clickhouse_connect as cc
 import pyarrow as pa
-import pyarrow_hotfix  # noqa: F401
 import sqlglot as sg
 import sqlglot.expressions as sge
 import toolz
@@ -35,6 +34,9 @@
 from ibis.backends.clickhouse.converter import ClickHousePandasData
 from ibis.backends.sql import SQLBackend
 from ibis.backends.sql.compilers.base import C
+from ibis.util import apply_pyarrow_hotfix
+
+apply_pyarrow_hotfix()
 
 if TYPE_CHECKING:
     from collections.abc import Iterable, Iterator, Mapping

ibis/backends/databricks/__init__.py

@@ -14,7 +14,6 @@
 
 import databricks.sql
 import pyarrow as pa
-import pyarrow_hotfix  # noqa: F401
 import sqlglot as sg
 import sqlglot.expressions as sge
 
@@ -30,6 +29,9 @@
 from ibis.backends.sql import SQLBackend
 from ibis.backends.sql.compilers.base import STAR, AlterTable, RenameTable
 from ibis.backends.sql.datatypes import DatabricksType
+from ibis.util import apply_pyarrow_hotfix
+
+apply_pyarrow_hotfix()
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterable, Mapping

ibis/backends/datafusion/__init__.py

@@ -9,7 +9,6 @@
 
 import datafusion as df
 import pyarrow as pa
-import pyarrow_hotfix  # noqa: F401
 import sqlglot as sg
 import sqlglot.expressions as sge
 
@@ -35,7 +34,15 @@
 from ibis.common.dispatch import lazy_singledispatch
 from ibis.expr.operations.udf import InputType
 from ibis.formats.pyarrow import PyArrowSchema, PyArrowType
-from ibis.util import gen_name, normalize_filename, normalize_filenames, warn_deprecated
+from ibis.util import (
+    apply_pyarrow_hotfix,
+    gen_name,
+    normalize_filename,
+    normalize_filenames,
+    warn_deprecated,
+)
+
+apply_pyarrow_hotfix()
 
 try:
     from datafusion import ExecutionContext as SessionContext

ibis/backends/datafusion/udfs.py

@@ -5,10 +5,12 @@
 
 import pyarrow as pa
 import pyarrow.compute as pc
-import pyarrow_hotfix  # noqa: F401
 
 import ibis.common.exceptions as com
 import ibis.expr.datatypes as dt
+from ibis.util import apply_pyarrow_hotfix
+
+apply_pyarrow_hotfix()
 
 
 def _extract_epoch_seconds(array) -> dt.int32:

ibis/backends/duckdb/__init__.py

@@ -35,14 +35,16 @@
 from ibis.backends.sql.compilers.base import STAR, AlterTable, C, RenameTable
 from ibis.common.dispatch import lazy_singledispatch
 from ibis.expr.operations.udf import InputType
+from ibis.util import apply_pyarrow_hotfix
+
+apply_pyarrow_hotfix()
 
 if TYPE_CHECKING:
     from collections.abc import Iterable, Mapping, MutableMapping, Sequence
 
     import pandas as pd
     import polars as pl
     import pyarrow as pa
-    import pyarrow_hotfix  # noqa: F401
     import torch
     from fsspec import AbstractFileSystem
 
@@ -1380,7 +1382,10 @@ def to_pyarrow_batches(
             The number of rows to fetch per batch
         """
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
+
+        from ibis.util import apply_pyarrow_hotfix
+
+        apply_pyarrow_hotfix()
 
         self._run_pre_execute_hooks(expr)
         table = expr.as_table()
@@ -1422,7 +1427,10 @@ def execute(
         """Execute an expression."""
         import pandas as pd
         import pyarrow.types as pat
-        import pyarrow_hotfix  # noqa: F401
+
+        from ibis.util import apply_pyarrow_hotfix
+
+        apply_pyarrow_hotfix()
 
         from ibis.backends.duckdb.converter import DuckDBPandasData
 

ibis/backends/flink/__init__.py

@@ -493,7 +493,10 @@ def create_table(
         """
         import pandas as pd
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
+
+        from ibis.util import apply_pyarrow_hotfix
+
+        apply_pyarrow_hotfix()
 
         import ibis.expr.types as ir
 
@@ -937,7 +940,10 @@ def insert(
         """
         import pandas as pd
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
+
+        from ibis.util import apply_pyarrow_hotfix
+
+        apply_pyarrow_hotfix()
 
         if isinstance(obj, ir.Table):
             statement = InsertSelect(
@@ -982,7 +988,10 @@ def to_pyarrow(
         **kwargs: Any,
     ) -> pa.Table:
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
+
+        from ibis.util import apply_pyarrow_hotfix
+
+        apply_pyarrow_hotfix()
 
         pyarrow_batches = iter(
             self.to_pyarrow_batches(expr, params=params, limit=limit, **kwargs)
@@ -1009,7 +1018,10 @@ def to_pyarrow_batches(
         **kwargs: Any,
     ):
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
+
+        from ibis.util import apply_pyarrow_hotfix
+
+        apply_pyarrow_hotfix()
 
         ibis_table = expr.as_table()
 
@@ -1059,7 +1071,10 @@ def _from_pyflink_table_to_pyarrow_batches(
         chunk_size: int | None = None,
     ):
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
+
+        from ibis.util import apply_pyarrow_hotfix
+
+        apply_pyarrow_hotfix()
         from pyflink.java_gateway import get_gateway
         from pyflink.table.serializers import ArrowSerializer
         from pyflink.table.types import create_arrow_schema

ibis/backends/impala/__init__.py

@@ -1334,7 +1334,10 @@ def to_pyarrow(
         **kwargs: Any,
     ) -> pa.Table:
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
+
+        from ibis.util import apply_pyarrow_hotfix
+
+        apply_pyarrow_hotfix()
 
         from ibis.formats.pyarrow import PyArrowData
 

ibis/backends/pyspark/__init__.py

@@ -1042,7 +1042,10 @@ def to_pyarrow(
                 "PySpark in streaming mode does not support to_pyarrow"
             )
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
+
+        from ibis.util import apply_pyarrow_hotfix
+
+        apply_pyarrow_hotfix()
 
         from ibis.formats.pyarrow import PyArrowData
 

ibis/backends/snowflake/__init__.py

@@ -14,7 +14,6 @@
 from urllib.request import urlcleanup, urlretrieve
 
 import pyarrow as pa
-import pyarrow_hotfix  # noqa: F401
 import sqlglot as sg
 import sqlglot.expressions as sge
 
@@ -36,6 +35,9 @@
 from ibis.backends.snowflake.converter import SnowflakePandasData
 from ibis.backends.sql import SQLBackend
 from ibis.backends.sql.compilers.base import STAR
+from ibis.util import apply_pyarrow_hotfix
+
+apply_pyarrow_hotfix()
 
 if TYPE_CHECKING:
     from collections.abc import Generator, Iterator, Mapping

ibis/backends/snowflake/tests/conftest.py

@@ -10,7 +10,6 @@
 from urllib.request import urlretrieve
 
 import pyarrow.parquet as pq
-import pyarrow_hotfix  # noqa: F401
 import pytest
 import snowflake.connector as sc
 import sqlglot as sg

ibis/formats/__init__.py

@@ -253,7 +253,10 @@ def to_polars(self, schema: Schema) -> pl.DataFrame:  # pragma: no cover
 
     def to_pyarrow_bytes(self, schema: Schema) -> bytes:
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
+
+        from ibis.util import apply_pyarrow_hotfix
+
+        apply_pyarrow_hotfix()
 
         data = self.to_pyarrow(schema=schema)
         out = pa.BufferOutputStream()

ibis/formats/pandas.py

@@ -433,7 +433,10 @@ def to_pyarrow(self, schema: sch.Schema) -> pa.Table:
         from decimal import Decimal
 
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
+
+        from ibis.util import apply_pyarrow_hotfix
+
+        apply_pyarrow_hotfix()
 
         pyarrow_schema = PyArrowSchema.from_ibis(schema)
 

ibis/formats/pyarrow.py

@@ -4,13 +4,14 @@
 from typing import TYPE_CHECKING, Any
 
 import pyarrow as pa
-import pyarrow_hotfix  # noqa: F401
 
 import ibis.common.exceptions as com
 import ibis.expr.datatypes as dt
 from ibis.expr.schema import Schema
 from ibis.formats import DataMapper, SchemaMapper, TableProxy, TypeMapper
-from ibis.util import V
+from ibis.util import V, apply_pyarrow_hotfix
+
+apply_pyarrow_hotfix()
 
 if TYPE_CHECKING:
     from collections.abc import Sequence

ibis/util.py

@@ -42,6 +42,14 @@
 HORIZONTAL_ELLIPSIS = "…"
 
 
+def apply_pyarrow_hotfix():
+    """Apply pyarrow hotfix for CVE-2023-47248 on pyarrow < 14.0.1."""
+    import pyarrow as pa
+
+    if tuple(int(x) for x in pa.__version__.split(".")[:3]) < (14, 0, 1):
+        import pyarrow_hotfix  # noqa: F401
+
+
 def guid() -> str:
     """Return a uuid4 hexadecimal value."""
     return uuid4().hex