diff --git a/src/executorlib/standalone/hdf.py b/src/executorlib/standalone/hdf.py index dffc5d4b..2f1a0d71 100644 --- a/src/executorlib/standalone/hdf.py +++ b/src/executorlib/standalone/hdf.py @@ -36,7 +36,10 @@ def dump(file_name: Optional[str], data_dict: dict) -> None: if data_key in group_dict: fname.create_dataset( name="/" + group_dict[data_key], - data=np.void(cloudpickle.dumps(data_value)), + data=np.frombuffer( + cloudpickle.dumps(data_value), dtype=np.uint8 + ), + compression="gzip", ) @@ -53,26 +56,26 @@ def load(file_name: str) -> dict: with h5py.File(file_name, "r") as hdf: data_dict = {} if "function" in hdf: - data_dict["fn"] = cloudpickle.loads(np.void(hdf["/function"])) + data_dict["fn"] = cloudpickle.loads(hdf["/function"][()].tobytes()) else: raise TypeError("Function not found in HDF5 file.") if "input_args" in hdf: - data_dict["args"] = cloudpickle.loads(np.void(hdf["/input_args"])) + data_dict["args"] = cloudpickle.loads(hdf["/input_args"][()].tobytes()) else: data_dict["args"] = () if "input_kwargs" in hdf: - data_dict["kwargs"] = cloudpickle.loads(np.void(hdf["/input_kwargs"])) + data_dict["kwargs"] = cloudpickle.loads(hdf["/input_kwargs"][()].tobytes()) else: data_dict["kwargs"] = {} if "resource_dict" in hdf: data_dict["resource_dict"] = cloudpickle.loads( - np.void(hdf["/resource_dict"]) + hdf["/resource_dict"][()].tobytes() ) else: data_dict["resource_dict"] = {} if "error_log_file" in hdf: data_dict["error_log_file"] = cloudpickle.loads( - np.void(hdf["/error_log_file"]) + hdf["/error_log_file"][()].tobytes() ) return data_dict @@ -91,9 +94,9 @@ def get_output(file_name: str) -> tuple[bool, bool, Any]: def get_output_helper(file_name: str) -> tuple[bool, bool, Any]: with h5py.File(file_name, "r") as hdf: if "output" in hdf: - return True, True, cloudpickle.loads(np.void(hdf["/output"])) + return True, True, cloudpickle.loads(hdf["/output"][()].tobytes()) elif "error" in hdf: - return True, False, cloudpickle.loads(np.void(hdf["/error"])) + return True, False, cloudpickle.loads(hdf["/error"][()].tobytes()) else: return False, False, None @@ -120,7 +123,7 @@ def get_runtime(file_name: str) -> float: """ with h5py.File(file_name, "r") as hdf: if "runtime" in hdf: - return cloudpickle.loads(np.void(hdf["/runtime"])) + return cloudpickle.loads(hdf["/runtime"][()].tobytes()) else: return 0.0 @@ -138,7 +141,7 @@ def get_queue_id(file_name: Optional[str]) -> Optional[int]: if file_name is not None and os.path.exists(file_name): with h5py.File(file_name, "r") as hdf: if "queue_id" in hdf: - return cloudpickle.loads(np.void(hdf["/queue_id"])) + return cloudpickle.loads(hdf["/queue_id"][()].tobytes()) return None @@ -224,7 +227,7 @@ def _get_content_of_file(file_name: str) -> dict: """ with h5py.File(file_name, "r") as hdf: return { - key: cloudpickle.loads(np.void(hdf["/" + key])) + key: cloudpickle.loads(hdf["/" + key][()].tobytes()) for key in group_dict.values() if key in hdf }