Skip to content

[Feature] Switch from numpy void() to frombuffer() - requires major release #984

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
jan-janssen wants to merge 3 commits into
base: main
Choose a base branch
from
+14 −11
Draft

[Feature] Switch from numpy void() to frombuffer() - requires major release #984

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
072779d
Switch from numpy void() to frombuffer()
jan-janssen May 8, 2026
bcac159
fix reader
jan-janssen May 8, 2026
c65a65b
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] May 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 14 additions & 11 deletions src/executorlib/standalone/hdf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,10 @@ def dump(file_name: Optional[str], data_dict: dict) -> None:
if data_key in group_dict:
fname.create_dataset(
name="/" + group_dict[data_key],
data=np.void(cloudpickle.dumps(data_value)),
data=np.frombuffer(
cloudpickle.dumps(data_value), dtype=np.uint8
),
compression="gzip",
)


Expand All @@ -53,26 +56,26 @@ def load(file_name: str) -> dict:
with h5py.File(file_name, "r") as hdf:
data_dict = {}
if "function" in hdf:
data_dict["fn"] = cloudpickle.loads(np.void(hdf["/function"]))
data_dict["fn"] = cloudpickle.loads(hdf["/function"][()].tobytes())
else:
raise TypeError("Function not found in HDF5 file.")
if "input_args" in hdf:
data_dict["args"] = cloudpickle.loads(np.void(hdf["/input_args"]))
data_dict["args"] = cloudpickle.loads(hdf["/input_args"][()].tobytes())
else:
data_dict["args"] = ()
if "input_kwargs" in hdf:
data_dict["kwargs"] = cloudpickle.loads(np.void(hdf["/input_kwargs"]))
data_dict["kwargs"] = cloudpickle.loads(hdf["/input_kwargs"][()].tobytes())
else:
data_dict["kwargs"] = {}
if "resource_dict" in hdf:
data_dict["resource_dict"] = cloudpickle.loads(
np.void(hdf["/resource_dict"])
hdf["/resource_dict"][()].tobytes()
)
else:
data_dict["resource_dict"] = {}
if "error_log_file" in hdf:
data_dict["error_log_file"] = cloudpickle.loads(
np.void(hdf["/error_log_file"])
hdf["/error_log_file"][()].tobytes()
)
return data_dict

Expand All @@ -91,9 +94,9 @@ def get_output(file_name: str) -> tuple[bool, bool, Any]:
def get_output_helper(file_name: str) -> tuple[bool, bool, Any]:
with h5py.File(file_name, "r") as hdf:
if "output" in hdf:
return True, True, cloudpickle.loads(np.void(hdf["/output"]))
return True, True, cloudpickle.loads(hdf["/output"][()].tobytes())
elif "error" in hdf:
return True, False, cloudpickle.loads(np.void(hdf["/error"]))
return True, False, cloudpickle.loads(hdf["/error"][()].tobytes())
else:
return False, False, None

Expand All @@ -120,7 +123,7 @@ def get_runtime(file_name: str) -> float:
"""
with h5py.File(file_name, "r") as hdf:
if "runtime" in hdf:
return cloudpickle.loads(np.void(hdf["/runtime"]))
return cloudpickle.loads(hdf["/runtime"][()].tobytes())
else:
return 0.0

Expand All @@ -138,7 +141,7 @@ def get_queue_id(file_name: Optional[str]) -> Optional[int]:
if file_name is not None and os.path.exists(file_name):
with h5py.File(file_name, "r") as hdf:
if "queue_id" in hdf:
return cloudpickle.loads(np.void(hdf["/queue_id"]))
return cloudpickle.loads(hdf["/queue_id"][()].tobytes())
return None


Expand Down Expand Up @@ -224,7 +227,7 @@ def _get_content_of_file(file_name: str) -> dict:
"""
with h5py.File(file_name, "r") as hdf:
return {
key: cloudpickle.loads(np.void(hdf["/" + key]))
key: cloudpickle.loads(hdf["/" + key][()].tobytes())
for key in group_dict.values()
if key in hdf
}
Loading