Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/con_duct/_constants.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""Constants used throughout con-duct."""

__schema_version__ = "0.2.2"
__schema_version__ = "0.2.3"

ENV_PREFIXES = ("PBS_", "SLURM_", "OSG")
SUFFIXES = {
Expand Down
14 changes: 14 additions & 0 deletions src/con_duct/_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,20 @@ class SystemInfo:
hostname: str | None
uid: int
user: str | None
os_name: str
os_release: str
os_version: str
arch: str
processor: str
distro_id: str
distro_id_like: str
distro_name: str
distro_version: str
distro_version_id: str
distro_codename: str
distro_variant_id: str
distro_pretty_name: str
distro_build_id: str


@dataclass
Expand Down
20 changes: 20 additions & 0 deletions src/con_duct/_tracker.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import logging
import math
import os
import platform
import shutil
import socket
import subprocess
Expand Down Expand Up @@ -89,12 +90,31 @@ def collect_environment(self) -> None:

def get_system_info(self) -> None:
"""Gathers system information related to CPU, GPU, memory, and environment variables."""
uname = platform.uname()
try:
osr = platform.freedesktop_os_release()
except OSError:
osr = {}
self.system_info = SystemInfo(
cpu_total=os.sysconf("SC_NPROCESSORS_CONF"),
memory_total=os.sysconf("SC_PAGESIZE") * os.sysconf("SC_PHYS_PAGES"),
hostname=socket.gethostname(),
uid=os.getuid(),
user=os.environ.get("USER"),
os_name=uname.system,
os_release=uname.release,
os_version=uname.version,
arch=uname.machine,
processor=uname.processor,
distro_id=osr.get("ID", ""),
distro_id_like=osr.get("ID_LIKE", ""),
distro_name=osr.get("NAME", ""),
distro_version=osr.get("VERSION", ""),
distro_version_id=osr.get("VERSION_ID", ""),
distro_codename=osr.get("VERSION_CODENAME", ""),
distro_variant_id=osr.get("VARIANT_ID", ""),
distro_pretty_name=osr.get("PRETTY_NAME", ""),
distro_build_id=osr.get("BUILD_ID", ""),
)
# GPU information
if shutil.which("nvidia-smi") is not None:
Expand Down
33 changes: 33 additions & 0 deletions src/con_duct/ls.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,18 @@
}

NON_TRANSFORMED_FIELDS: List[str] = [
"arch",
"command",
"cpu_total",
"distro_build_id",
"distro_codename",
"distro_id",
"distro_id_like",
"distro_name",
"distro_pretty_name",
"distro_variant_id",
"distro_version",
"distro_version_id",
"duct_version",
"gpu",
"hostname",
Expand All @@ -52,7 +62,11 @@
"message",
"num_samples",
"num_reports",
"os_name",
"os_release",
"os_version",
"prefix",
"processor",
"schema_version",
"stderr",
"stdout",
Expand Down Expand Up @@ -118,6 +132,25 @@ def ensure_compliant_schema(info_dict: dict) -> None:
# message field added in 0.2.2
if parse_version(info_dict["schema_version"]) < parse_version("0.2.2"):
info_dict["message"] = ""
# OS and distro provenance fields added to system block in 0.2.3
if parse_version(info_dict["schema_version"]) < parse_version("0.2.3"):
for field in (
"os_name",
"os_release",
"os_version",
"arch",
"processor",
"distro_id",
"distro_id_like",
"distro_name",
"distro_version",
"distro_version_id",
"distro_codename",
"distro_variant_id",
"distro_pretty_name",
"distro_build_id",
):
info_dict["system"][field] = ""


def process_run_data(
Expand Down
21 changes: 21 additions & 0 deletions test/duct_main/test_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,27 @@ def test_system_info_sanity(mock_log_paths: mock.MagicMock) -> None:
assert report.system_info.memory_total > 10
assert report.system_info.uid == os.getuid()
assert report.system_info.user == os.environ.get("USER")
# uname-derived fields are populated on any POSIX host (which is the only
# platform get_system_info supports — sysconf calls above are POSIX-only).
assert report.system_info.os_name
assert report.system_info.os_release
assert report.system_info.os_version
assert report.system_info.arch
# processor may be empty on linux; just assert it is a string
assert isinstance(report.system_info.processor, str)
# distro_* fields come from /etc/os-release; absent on macOS, so allow empty.
for field in (
"distro_id",
"distro_id_like",
"distro_name",
"distro_version",
"distro_version_id",
"distro_codename",
"distro_variant_id",
"distro_pretty_name",
"distro_build_id",
):
assert isinstance(getattr(report.system_info, field), str)


@mock.patch("con_duct._tracker.shutil.which")
Expand Down
43 changes: 40 additions & 3 deletions test/test_ls.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,12 @@

def test_load_duct_runs_sanity() -> None:
mock_json = json.dumps(
{"schema_version": "0.2.1", "prefix": "/test/path_", "command": "echo hello"}
{
"schema_version": "0.2.1",
"prefix": "/test/path_",
"command": "echo hello",
"system": {},
}
)
with patch("builtins.open", mock_open(read_data=mock_json)):
result = load_duct_runs(["/test/path_info.json"])
Expand All @@ -47,6 +52,7 @@ def test_load_duct_runs_uses_filenames_not_stored_prefix() -> None:
"schema_version": "0.2.1",
"prefix": "/test/not_anymore_",
"command": "echo hello",
"system": {},
}
)
with patch("builtins.open", mock_open(read_data=mock_json)):
Expand Down Expand Up @@ -96,10 +102,31 @@ def test_ensure_compliant_schema_noop_for_current_version() -> None:


def test_ensure_compliant_schema_adds_field_for_old_version() -> None:
info: Dict[str, Any] = {"schema_version": "0.2.0", "execution_summary": {}}
info: Dict[str, Any] = {
"schema_version": "0.2.0",
"execution_summary": {},
"system": {},
}
ensure_compliant_schema(info)
assert info["execution_summary"]["working_directory"] == ""
assert info["message"] == ""
for field in (
"os_name",
"os_release",
"os_version",
"arch",
"processor",
"distro_id",
"distro_id_like",
"distro_name",
"distro_version",
"distro_version_id",
"distro_codename",
"distro_variant_id",
"distro_pretty_name",
"distro_build_id",
):
assert info["system"][field] == ""


def test_ensure_compliant_schema_ignores_unexpected_future_version() -> None:
Expand Down Expand Up @@ -142,7 +169,12 @@ def test_load_duct_runs_mixed_empty_and_valid_files(
) -> None:
"""Test behavior with mix of empty and valid JSON files."""
valid_json = json.dumps(
{"schema_version": "0.2.1", "prefix": "/test/path_", "command": "echo hello"}
{
"schema_version": "0.2.1",
"prefix": "/test/path_",
"command": "echo hello",
"system": {},
}
)

def side_effect(filename: str) -> Any:
Expand Down Expand Up @@ -174,28 +206,33 @@ def setUp(self) -> None:
"file1_info.json": {
"schema_version": MINIMUM_SCHEMA_VERSION,
"execution_summary": {},
"system": {},
"prefix": "test1",
"filter_this": "yes",
},
"file2_info.json": {
"schema_version": MINIMUM_SCHEMA_VERSION,
"execution_summary": {},
"system": {},
"prefix": "test2",
"filter_this": "no",
},
"file3_info.json": {
"schema_version": "0.1.0",
"execution_summary": {},
"system": {},
"prefix": "old_version",
},
"not_matching.json": {
"schema_version": MINIMUM_SCHEMA_VERSION,
"execution_summary": {},
"system": {},
"prefix": "no_match",
},
".duct/logs/default_logpath_info.json": {
"schema_version": MINIMUM_SCHEMA_VERSION,
"execution_summary": {},
"system": {},
"prefix": "default_file1",
},
}
Expand Down
Loading