8587 test erros on pytorch release 2508 on series 50

monai/apps/auto3dseg/bundle_gen.py

@@ -264,8 +264,7 @@ def _run_cmd(self, cmd: str, devices_info: str = "") -> subprocess.CompletedProc
                 look_up_option(self.device_setting["MN_START_METHOD"], ["bcprun"])
             except ValueError as err:
                 raise NotImplementedError(
-                    f"{self.device_setting['MN_START_METHOD']} is not supported yet."
-                    "Try modify BundleAlgo._run_cmd for your cluster."
+                    f"{self.device_setting['MN_START_METHOD']} is not supported yet. Try modify BundleAlgo._run_cmd for your cluster."
                 ) from err
 
             return _run_cmd_bcprun(cmd, n=self.device_setting["NUM_NODES"], p=self.device_setting["n_devices"])

monai/apps/detection/networks/retinanet_detector.py

@@ -342,8 +342,7 @@ def set_regular_matcher(
         """
         if fg_iou_thresh < bg_iou_thresh:
             raise ValueError(
-                "Require fg_iou_thresh >= bg_iou_thresh. "
-                f"Got fg_iou_thresh={fg_iou_thresh}, bg_iou_thresh={bg_iou_thresh}."
+                f"Require fg_iou_thresh >= bg_iou_thresh. Got fg_iou_thresh={fg_iou_thresh}, bg_iou_thresh={bg_iou_thresh}."
             )
         self.proposal_matcher = Matcher(
             fg_iou_thresh, bg_iou_thresh, allow_low_quality_matches=allow_low_quality_matches
@@ -519,7 +518,7 @@ def forward(
         else:
             if self.inferer is None:
                 raise ValueError(
-                    "`self.inferer` is not defined." "Please refer to function self.set_sliding_window_inferer(*)."
+                    "`self.inferer` is not defined. Please refer to function self.set_sliding_window_inferer(*)."
                 )
             head_outputs = predict_with_inferer(
                 images, self.network, keys=[self.cls_key, self.box_reg_key], inferer=self.inferer

monai/auto3dseg/analyzer.py

@@ -965,8 +965,7 @@ def __call__(self, data: dict) -> dict:
             self.hist_range = nr_channels * self.hist_range
         if len(self.hist_range) != nr_channels:
             raise ValueError(
-                f"There is a mismatch between the number of channels ({nr_channels}) "
-                f"and histogram ranges ({len(self.hist_range)})."
+                f"There is a mismatch between the number of channels ({nr_channels}) and histogram ranges ({len(self.hist_range)})."
             )
 
         # perform calculation

monai/data/dataset.py

@@ -609,7 +609,7 @@ def __init__(
         # the cache is created without multi-threading
         self._read_env: Any | None = None
         # this runs on the primary thread/process
-        self._fill_cache_start_reader(show_progress=self.progress)
+        self._read_env = self._fill_cache_start_reader(show_progress=self.progress)
         print(f"Accessing lmdb file: {self.db_file.absolute()}.")
 
     def set_data(self, data: Sequence):
@@ -637,6 +637,12 @@ def _fill_cache_start_reader(self, show_progress=True):
         Args:
             show_progress: whether to show the progress bar if possible.
         """
+        # Close any open read environment before attempting write-mode access
+        # to prevent "environment already open" errors when multiple LMDBDataset
+        # instances target the same db file
+        if self._read_env is not None:
+            self._read_env.close()
+            self._read_env = None
         # create cache
         self.lmdb_kwargs["readonly"] = False
         env = lmdb.open(path=f"{self.db_file}", subdir=False, **self.lmdb_kwargs)
@@ -664,7 +670,7 @@ def _fill_cache_start_reader(self, show_progress=True):
                         size = env.info()["map_size"]
                         new_size = size * 2
                         warnings.warn(
-                            f"Resizing the cache database from {int(size) >> 20}MB" f" to {int(new_size) >> 20}MB."
+                            f"Resizing the cache database from {int(size) >> 20}MB to {int(new_size) >> 20}MB."
                         )
                         env.set_mapsize(new_size)
                     except lmdb.MapResizedError:

monai/data/wsi_reader.py

@@ -416,8 +416,7 @@ def get_data(
             # Check if there are three color channels for RGB
             elif mode in "RGB" and patch.shape[self.channel_dim] != 3:
                 raise ValueError(
-                    f"The image is expected to have three color channels in '{mode}' mode but has "
-                    f"{patch.shape[self.channel_dim]}. "
+                    f"The image is expected to have three color channels in '{mode}' mode but has {patch.shape[self.channel_dim]}. "
                 )
             # Get patch-related metadata
             metadata: dict = self._get_metadata(wsi=each_wsi, patch=patch, location=location, size=size, level=level)

monai/networks/blocks/warp.py

@@ -20,6 +20,7 @@
 from monai.config.deviceconfig import USE_COMPILED
 from monai.networks.layers.spatial_transforms import grid_pull
 from monai.networks.utils import meshgrid_ij
+from monai.transforms.spatial.functional import _compiled_unsupported
 from monai.utils import GridSampleMode, GridSamplePadMode, optional_import
 
 _C, _ = optional_import("monai._C")
@@ -63,6 +64,18 @@ def __init__(self, mode=GridSampleMode.BILINEAR.value, padding_mode=GridSamplePa
         super().__init__()
         # resolves _interp_mode for different methods
 
+        # Native string modes are always stored for the PyTorch fallback path.
+        # When USE_COMPILED=True but the device is unsupported at runtime (e.g. Blackwell GPU),
+        # forward() falls back to F.grid_sample which requires string, not integer, modes.
+        self._interp_mode_native = (
+            GridSampleMode(mode).value if mode in (m.value for m in GridSampleMode) else GridSampleMode.BILINEAR.value
+        )
+        self._padding_mode_native = (
+            GridSamplePadMode(padding_mode).value
+            if padding_mode in (p.value for p in GridSamplePadMode)
+            else GridSamplePadMode.BORDER.value
+        )
+
         if USE_COMPILED:
             if mode in (inter.value for inter in GridSampleMode):
                 mode = GridSampleMode(mode)
@@ -77,7 +90,7 @@ def __init__(self, mode=GridSampleMode.BILINEAR.value, padding_mode=GridSamplePa
             self._interp_mode = mode
         else:
             warnings.warn("monai.networks.blocks.Warp: Using PyTorch native grid_sample.")
-            self._interp_mode = GridSampleMode(mode).value
+            self._interp_mode = self._interp_mode_native
 
         # resolves _padding_mode for different methods
         if USE_COMPILED:
@@ -93,7 +106,7 @@ def __init__(self, mode=GridSampleMode.BILINEAR.value, padding_mode=GridSamplePa
                     padding_mode = 0  # default to nearest
             self._padding_mode = padding_mode
         else:
-            self._padding_mode = GridSamplePadMode(padding_mode).value
+            self._padding_mode = self._padding_mode_native
 
         self.ref_grid = None
         self.jitter = jitter
@@ -138,13 +151,15 @@ def forward(self, image: torch.Tensor, ddf: torch.Tensor):
         grid = self.get_reference_grid(ddf, jitter=self.jitter) + ddf
         grid = grid.permute([0] + list(range(2, 2 + spatial_dims)) + [1])  # (batch, ..., spatial_dims)
 
-        if not USE_COMPILED:  # pytorch native grid_sample
+        _use_compiled = USE_COMPILED and not _compiled_unsupported(image.device)
+
+        if not _use_compiled:  # pytorch native grid_sample
             for i, dim in enumerate(grid.shape[1:-1]):
                 grid[..., i] = grid[..., i] * 2 / (dim - 1) - 1
             index_ordering: list[int] = list(range(spatial_dims - 1, -1, -1))
             grid = grid[..., index_ordering]  # z, y, x -> x, y, z
             return F.grid_sample(
-                image, grid, mode=self._interp_mode, padding_mode=f"{self._padding_mode}", align_corners=True
+                image, grid, mode=self._interp_mode_native, padding_mode=self._padding_mode_native, align_corners=True
             )
 
         # using csrc resampling

monai/transforms/regularization/array.py

@@ -24,7 +24,6 @@
 
 
 class Mixer(RandomizableTransform):
-
     def __init__(self, batch_size: int, alpha: float = 1.0) -> None:
         """
         Mixer is a base class providing the basic logic for the mixup-class of

monai/transforms/spatial/array.py

@@ -34,6 +34,7 @@
 from monai.transforms.croppad.array import CenterSpatialCrop, ResizeWithPadOrCrop
 from monai.transforms.inverse import InvertibleTransform
 from monai.transforms.spatial.functional import (
+    _compiled_unsupported,
     affine_func,
     convert_box_to_points,
     convert_points_to_box,
@@ -2104,14 +2105,15 @@ def __call__(
         _align_corners = self.align_corners if align_corners is None else align_corners
         img_t, *_ = convert_data_type(img, torch.Tensor, dtype=_dtype, device=_device)
         sr = min(len(img_t.peek_pending_shape() if isinstance(img_t, MetaTensor) else img_t.shape[1:]), 3)
+        _use_compiled = USE_COMPILED and not _compiled_unsupported(img_t.device)
         backend, _interp_mode, _padding_mode, _ = resolves_modes(
             self.mode if mode is None else mode,
             self.padding_mode if padding_mode is None else padding_mode,
             backend=None,
-            use_compiled=USE_COMPILED,
+            use_compiled=_use_compiled,
         )
 
-        if USE_COMPILED or backend == TransformBackends.NUMPY:
+        if _use_compiled or backend == TransformBackends.NUMPY:
             grid_t, *_ = convert_to_dst_type(grid[:sr], img_t, dtype=grid.dtype, wrap_sequence=True)
             if isinstance(grid, torch.Tensor) and grid_t.data_ptr() == grid.data_ptr():
                 grid_t = grid_t.clone(memory_format=torch.contiguous_format)
@@ -2122,7 +2124,7 @@ def __call__(
                     grid_t[i] = ((_dim - 1) / _dim) * grid_t[i] + t if _align_corners else grid_t[i] + t
                 elif _align_corners:
                     grid_t[i] = ((_dim - 1) / _dim) * (grid_t[i] + 0.5)
-            if USE_COMPILED and backend == TransformBackends.TORCH:  # compiled is using torch backend param name
+            if _use_compiled and backend == TransformBackends.TORCH:  # compiled is using torch backend param name
                 grid_t = moveaxis(grid_t, 0, -1)  # type: ignore
                 out = grid_pull(
                     img_t.unsqueeze(0),
@@ -2140,6 +2142,20 @@ def __call__(
                     [_map_coord(c, grid_np, order=_interp_mode, mode=_padding_mode) for c in img_np]
                 )
                 out = convert_to_dst_type(out, img_t)[0]
+            else:
+                # Fallback to PyTorch grid_sample when compiled extension is unsupported.
+                # Convert grid coordinates from compiled convention [0, size-1] to PyTorch [-1, 1]
+                for i, dim in enumerate(img_t.shape[1 : 1 + sr]):
+                    _dim = max(2, dim)
+                    grid_t[i] = (grid_t[i] * 2.0 / _dim) - 1.0
+                grid_t = moveaxis(grid_t, 0, -1)  # type: ignore
+                out = torch.nn.functional.grid_sample(
+                    img_t.unsqueeze(0),
+                    grid_t.unsqueeze(0),
+                    mode=_interp_mode,
+                    padding_mode=_padding_mode,
+                    align_corners=None if _align_corners == TraceKeys.NONE else _align_corners,  # type: ignore
+                )[0]
         else:
             grid_t = moveaxis(grid[list(range(sr - 1, -1, -1))], 0, -1)  # type: ignore
             grid_t = convert_to_dst_type(grid_t, img_t, wrap_sequence=True)[0].unsqueeze(0)

monai/transforms/spatial/functional.py

@@ -54,6 +54,35 @@
 __all__ = ["spatial_resample", "orientation", "flip", "resize", "rotate", "zoom", "rotate90", "affine_func"]
 
 
+def _compiled_unsupported(device: torch.device) -> bool:
+    """
+    Return True if ``monai._C`` (the compiled C extension providing ``grid_pull``) is not
+    compiled with support for the given CUDA device's compute capability.
+
+    Args:
+        device: The torch device to check for compiled extension support.
+
+    Returns:
+        True if the device is CUDA with compute capability major >= 12 (Blackwell+),
+        False otherwise. Always returns False for CPU devices.
+
+    Note:
+        ``monai._C`` is built at install time against a fixed set of CUDA architectures.
+        NVIDIA Blackwell GPUs (sm_120, compute capability 12.x) and newer were not included in
+        the default ``TORCH_CUDA_ARCH_LIST`` when the MONAI slim image was originally built,
+        so executing ``grid_pull`` on those devices produces incorrect results.  Falling back to
+        the PyTorch-native ``affine_grid`` + ``grid_sample`` path (``USE_COMPILED=False``) gives
+        correct output on all architectures.
+
+        The threshold (``major >= 12``) matches the first architecture family (Blackwell, sm_120)
+        that shipped after the highest sm supported in the current default build list (sm_90,
+        Hopper).  Adjust this constant when ``monai._C`` is rebuilt with sm_120+ support.
+    """
+    if device.type != "cuda":
+        return False
+    return torch.cuda.get_device_properties(device).major >= 12
+
+
 def _maybe_new_metatensor(img, dtype=None, device=None):
     """create a metatensor with fresh metadata if track_meta is True otherwise convert img into a torch tensor"""
     return convert_to_tensor(
@@ -158,7 +187,8 @@ def spatial_resample(
         xform_shape = [-1] + in_sp_size
         img = img.reshape(xform_shape)
     img = img.to(dtype_pt)
-    if isinstance(mode, int) or USE_COMPILED:
+    _use_compiled = USE_COMPILED and not _compiled_unsupported(img.device)
+    if isinstance(mode, int) or _use_compiled:
         dst_xform = create_translate(spatial_rank, [float(d - 1) / 2 for d in spatial_size])
         xform = xform @ convert_to_dst_type(dst_xform, xform)[0]
         affine_xform = monai.transforms.Affine(

monai/visualize/img2tensorboard.py

@@ -65,7 +65,7 @@ def _image3_animated_gif(
     img_str = b""
     for b_data in PIL.GifImagePlugin.getheader(ims[0])[0]:
         img_str += b_data
-    img_str += b"\x21\xff\x0b\x4e\x45\x54\x53\x43\x41\x50" b"\x45\x32\x2e\x30\x03\x01\x00\x00\x00"
+    img_str += b"\x21\xff\x0b\x4e\x45\x54\x53\x43\x41\x50\x45\x32\x2e\x30\x03\x01\x00\x00\x00"
     for i in ims:
         for b_data in PIL.GifImagePlugin.getdata(i):
             img_str += b_data
@@ -204,8 +204,17 @@ def plot_2d_or_3d_image(
         if d_chans == 3 and max_channels == 3 and has_tensorboardx and isinstance(writer, SummaryWriterX):  # RGB
             # move the expected frame dim to the end as `T` dim for video
             d = np.moveaxis(d, frame_dim, -1)
-            writer.add_video(tag, d[None], step, fps=max_frames, dataformats="NCHWT")
-            return
+            try:
+                writer.add_video(tag, d[None], step, fps=max_frames, dataformats="NCHWT")
+                return
+            except TypeError as e:
+                # Fallback for numpy 2.x incompatibility in tensorboardX
+                # (https://github.com/lanpa/tensorboardX/issues/XXX)
+                # tensorboardX uses np.reshape(..., newshape=...) which is not supported in numpy 2.0+
+                if "newshape" in str(e):
+                    pass  # Fall through to animated GIF approach
+                else:
+                    raise
         # scale data to 0 - 255 for visualization
         max_channels = min(max_channels, d_chans)
         d = np.stack([rescale_array(i, 0, 255) for i in d[:max_channels]], axis=0)

runtests.sh

@@ -53,6 +53,7 @@ doMypyFormat=false
 doCleanup=false
 doDistTests=false
 doPrecommit=false
+testTimeout=0
 
 NUM_PARALLEL=1
 
@@ -109,6 +110,8 @@ function print_usage {
     echo "    -v, --version     : show MONAI and system version information and exit"
     echo "    -p, --path        : specify the path used for formatting, default is the current dir if unspecified"
     echo "    --formatfix       : format code using \"isort\" and \"black\" for user specified directories"
+    echo "    --timeout [secs]  : per-test timeout in seconds; tests exceeding this are marked as errors and skipped"
+    echo "                        (default: 180s when flag is given without a value; 0 = disabled)"
     echo ""
     echo "${separator}For bug reports and feature requests, please file an issue at:"
     echo "    https://github.com/Project-MONAI/MONAI/issues/new/choose"
@@ -344,6 +347,15 @@ do
             testdir=$2
             shift
         ;;
+        --timeout)
+            # Accept an optional numeric value; default to 180s if none given.
+            if [[ -n "$2" ]] && [[ "$2" =~ ^[0-9]+$ ]]; then
+                testTimeout=$2
+                shift
+            else
+                testTimeout=180
+            fi
+        ;;
         *)
             print_error_msg "Incorrect commandline provided, invalid key: $key"
             print_usage
@@ -695,7 +707,11 @@ if [ $doUnitTests = true ]
 then
     echo "${separator}${blue}unittests${noColor}"
     torch_validate
-    ${cmdPrefix}${cmd} ./tests/runner.py -p "^(?!test_integration|test_perceptual_loss|test_auto3dseg_ensemble).*(?<!_dist)$"  # excluding integration/dist/perceptual_loss tests
+    timeoutArg=""
+    if [ "$testTimeout" -gt 0 ] 2>/dev/null; then
+        timeoutArg="--timeout $testTimeout"
+    fi
+    ${cmdPrefix}${cmd} ./tests/runner.py -p "^(?!test_integration|test_perceptual_loss|test_auto3dseg_ensemble).*(?<!_dist)$" $timeoutArg  # excluding integration/dist/perceptual_loss tests
 fi
 
 # distributed test only