From c654e1ae83fc535b10d7001cebde2758afef7a63 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 22 Apr 2026 18:01:25 -0700 Subject: [PATCH 1/7] chore: adds refactor dev docs --- doc/dev/refactor/README.md | 96 +++++++ doc/dev/refactor/feature_single_entrypoint.md | 137 ++++++++++ doc/dev/refactor/feature_srcroot.md | 121 +++++++++ doc/dev/refactor/implementation.md | 235 ++++++++++++++++++ doc/dev/refactor/plan.md | 170 +++++++++++++ 5 files changed, 759 insertions(+) create mode 100644 doc/dev/refactor/README.md create mode 100644 doc/dev/refactor/feature_single_entrypoint.md create mode 100644 doc/dev/refactor/feature_srcroot.md create mode 100644 doc/dev/refactor/implementation.md create mode 100644 doc/dev/refactor/plan.md diff --git a/doc/dev/refactor/README.md b/doc/dev/refactor/README.md new file mode 100644 index 00000000000..b986a35d8e3 --- /dev/null +++ b/doc/dev/refactor/README.md @@ -0,0 +1,96 @@ +# CIME Refactor Documentation + +Comprehensive documentation for the CIME refactoring effort. + +## Quick Start + +- **New to refactor?** Read `plan.md` +- **Implementing a slice?** See `implementation.md` +- **Working on SRCROOT feature?** See `feature_srcroot.md` +- **Working on single entrypoint?** See `feature_single_entrypoint.md` + +## Documents + +### [plan.md](plan.md) +Architecture, principles, and migration strategy. +- Compatibility-first policy +- Package structure and DI patterns +- 5 implementation slices +- Success criteria + +### [implementation.md](implementation.md) +Detailed tasks, code patterns, and timeline. +- Week-by-week breakdown +- Code examples for each slice +- Testing strategy +- Code review checklist + +### [feature_srcroot.md](feature_srcroot.md) +SRCROOT standardization feature (Slice 3A). +- Removes config_files.xml indirection +- DI-compliant implementation +- Migration plan +- Testing with mocks + +### [feature_single_entrypoint.md](feature_single_entrypoint.md) +Single CIME entrypoint feature. +- Replaces ~20 symlinks with one +- Shell wrappers for tool compatibility +- Simplifies CIME version switching +- Can run parallel with other slices + +## Overview + +**Goal**: Improve CIME's dependency injection, resiliency, and testability while maintaining compatibility with E3SM, CESM, and NorESM. + +**Timeline**: 22-24 weeks (5 slices) + +**Approach**: Incremental refactor with compatibility preserved at each step. + +## Implementation Slices + +1. **Foundation** (3 weeks): Core abstractions (FileSystem, ProcessRunner, etc.) +2. **Batch** (4 weeks): Extract batch/submit logic +3. **SRCROOT** (4 weeks): Standardize config loading (NEW FEATURE) +4. **Build** (5 weeks): Extract build logic +5. **Case** (6 weeks): Extract Case internals to focused components + +## Key Principles + +- **Compatibility first**: External models must continue working +- **Constructor injection**: All dependencies injected, no globals +- **Incremental**: Each slice independently testable +- **Test-driven**: 80%+ coverage requirement + +## For Developers + +**Before coding**: +1. Read relevant slice in `implementation.md` +2. Understand DI patterns in `plan.md` +3. Review code examples + +**During development**: +- Use constructor injection +- Write tests with mocks +- Maintain backward compatibility +- Update documentation + +**Code review checklist**: +- [ ] Constructor injection used +- [ ] No direct filesystem/process/env access in core classes +- [ ] Factory functions provide defaults +- [ ] 80%+ test coverage +- [ ] All tests pass +- [ ] Docs updated + +## For Downstream Models + +**E3SM, CESM, NorESM integrators**: +- Each slice maintains compatibility +- SRCROOT feature (Slice 3A) has migration guide +- Test during opt-in periods +- Provide feedback early + +## Questions? + +File issues in CIME repository or discuss in development meetings. diff --git a/doc/dev/refactor/feature_single_entrypoint.md b/doc/dev/refactor/feature_single_entrypoint.md new file mode 100644 index 00000000000..28a2c54eaf3 --- /dev/null +++ b/doc/dev/refactor/feature_single_entrypoint.md @@ -0,0 +1,137 @@ +# Feature: Single CIME Entrypoint + +Collapse all symlinked case scripts into one `cime` entrypoint with shell wrappers. + +## Changes + +### Current: ~20 Symlinks +``` +case_dir/ +├── case.setup -> /path/to/CIME/scripts/Tools/case.setup +├── case.build -> /path/to/CIME/scripts/Tools/case.build +├── xmlchange -> /path/to/CIME/scripts/Tools/xmlchange +└── ... (~17 more symlinks) +``` + +**Problem**: Changing CIME version requires updating all symlinks. + +### New: 1 Symlink + Shell Wrappers +``` +case_dir/ +├── cime -> /path/to/CIME/scripts/cime # Only symlink +├── case.setup # Wrapper script +├── case.build # Wrapper script +├── xmlchange # Wrapper script +└── ... +``` + +**Benefit**: Change CIME version by updating one symlink. + +## Implementation + +### Single Entrypoint: `CIME/scripts/cime` + +```python +#!/usr/bin/env python3 +"""Single entrypoint for all CIME case operations.""" + +COMMANDS = { + 'case.setup': 'CIME.Tools.case_setup', + 'case.build': 'CIME.Tools.case_build', + 'xmlchange': 'CIME.Tools.xmlchange', + # ... all tools +} + +def main(): + command = sys.argv[1] + module = __import__(COMMANDS[command], fromlist=['main']) + sys.argv = [f"cime {command}"] + sys.argv[2:] + return module.main() +``` + +### Shell Wrapper Template + +```bash +#!/usr/bin/env bash +# Wrapper for: cime case.setup +exec "$(dirname "$0")/cime" case.setup "$@" +``` + +**Key**: +- `exec` replaces shell (clean exit codes) +- `"$@"` preserves all arguments +- Finds `cime` relative to wrapper + +### Case Setup + +```python +def create_case_scripts(case_root: Path, cimeroot: Path): + # Create single cime symlink + (case_root / "cime").symlink_to(cimeroot / "scripts" / "cime") + + # Create shell wrappers + wrapper = '#!/usr/bin/env bash\nexec "$(dirname "$0")/cime" {cmd} "$@"\n' + + for tool in ["case.setup", "case.build", "xmlchange", ...]: + path = case_root / tool + path.write_text(wrapper.format(cmd=tool)) + path.chmod(0o755) +``` + +## Migration + +### New Cases +Immediately use new pattern (single symlink + wrappers). + +### Existing Cases +**Option 1**: Leave unchanged - both patterns work +**Option 2**: Provide upgrade utility (optional) + +**No breaking changes** - old and new patterns coexist. + +## Benefits + +1. **Simple CIME updates**: Change one symlink instead of ~20 +2. **Cleaner case dirs**: One symlink vs. many +3. **Clear CIME version**: Easy to see which CIME in use +4. **Better portability**: Fewer symlinks to manage +5. **CLI foundation**: Natural path to full CLI layer + +## Testing + +- Command routing and argument passing +- All major case operations +- Old and new case patterns +- Exit codes and output + +## Integration with Refactor + +- **Uses**: Slice 1 bootstrap (`ensure_cime_on_path`) +- **Enables**: Future CLI layer (optional) +- **Timeline**: 2-3 weeks, can run parallel with other slices + +## Future Enhancements + +### Subcommands (optional) +```bash +cime case setup # Alternative syntax +cime xml change VAR=val +``` + +### Global Options (optional) +```bash +cime --case /path case.build +cime --verbose case.setup +cime --help +``` + +## Decision Points + +1. **Existing cases**: Auto-upgrade or leave unchanged? + - Recommend: Leave unchanged (safer) + +2. **Wrapper language**: Shell or Python? + - Recommend: Shell (simpler, minimal overhead) + +3. **Command syntax**: Keep dots (`case.setup`)? + - Recommend: Keep dots (backward compatible) diff --git a/doc/dev/refactor/feature_srcroot.md b/doc/dev/refactor/feature_srcroot.md new file mode 100644 index 00000000000..a4e085d6bba --- /dev/null +++ b/doc/dev/refactor/feature_srcroot.md @@ -0,0 +1,121 @@ +# Feature: SRCROOT Standardization + +Remove `config_files.xml` indirection and standardize SRCROOT resolution for submodule usage. + +## Changes + +### SRCROOT Resolution Priority +1. Environment variable `SRCROOT` +2. User config `~/.cime/config` +3. Command-line flag `--srcroot` +4. Step up one directory from CIMEROOT +5. CIMEROOT itself (standalone mode) + +### What's Removed +- `CIME/data/config/{e3sm,cesm,ufs}/config_files.xml` +- `MODEL_CONFIG_FILES` indirection +- Model-specific `get_config_path()` logic + +### What's Added +- `CIME/core/config/srcroot.py` - SRCROOT resolver with DI +- `CIME/core/config/loader.py` - Direct config file loader +- `--srcroot` flag to CLI tools +- Standalone mode for unit testing + +### Expected Directory Structure +``` +$SRCROOT/ +├── cime_config/ # Model config (validated) +│ ├── config_grids.xml +│ ├── config_compsets.xml +│ ├── config_machines.xml +│ └── config_tests.xml +└── cime/ # CIME submodule +``` + +## Implementation (DI-Compliant) + +### SRCROOTResolver +```python +class SRCROOTResolver: + """Resolves SRCROOT with dependency injection.""" + + def __init__(self, cimeroot: Path, filesystem: FileSystem, + environment: EnvironmentProvider, user_config: UserConfigProvider): + # All dependencies injected + + def resolve(self, flag_value: Optional[str] = None) -> SRCROOTResolution: + # Priority: env > user_config > flag > implicit + # Returns: SRCROOTResolution(path, source, standalone_mode) +``` + +**DI Pattern**: Constructor injection, no globals, fully testable with mocks. + +### ConfigFileLoader +```python +class ConfigFileLoader: + """Loads config files from $SRCROOT/cime_config with DI.""" + + def __init__(self, srcroot: Path, filesystem: FileSystem): + # Filesystem injected for testability + + def validate_and_load(self, require_full: bool = True) -> ConfigFiles: + # Validates cime_config exists + # Returns: ConfigFiles(paths to all config XMLs) +``` + +**DI Pattern**: Uses injected FileSystem, no direct path.is_dir() calls. + +### Factory Functions +```python +def create_srcroot_resolver(...) -> SRCROOTResolver: + # Provides defaults for production, allows injection for tests + +def create_config_loader(...) -> ConfigFileLoader: + # Provides defaults for production, allows injection for tests +``` + +## Testing + +**Mock-based testing** (no global state, no real filesystem): + +```python +def test_srcroot_from_environment(): + mock_env = MockEnvironment({"SRCROOT": "/test"}) + mock_fs = MockFileSystem({Path("/test/cime_config")}) + + resolver = SRCROOTResolver(cimeroot, mock_fs, mock_env, mock_config) + resolution = resolver.resolve() + + assert resolution.path == Path("/test") + assert resolution.source == "environment" +``` + +## Migration + +**4-Stage Rollout:** +1. **Opt-in** (Weeks 1-2): Feature flag `CIME_USE_NEW_CONFIG_LOADER=true` +2. **Opt-out** (Weeks 3-4): New system default, old available via flag +3. **Deprecation**: Warnings on old code paths +4. **Removal**: Delete old config_files.xml system + +## Integration with Refactor + +- **Uses Slice 1**: FileSystem, EnvironmentProvider abstractions +- **New Slice 3A**: SRCROOT standardization (3-4 weeks) +- **Before Slice 3B**: Must complete before build system refactor + +## Validation + +- E3SM case creation workflow +- CESM case creation workflow +- NorESM case creation workflow +- Standalone mode for unit tests + +## Benefits + +- Simpler config loading (no indirection) +- Better error messages (fail early) +- Fully testable (DI throughout) +- Explicit standalone mode +- Deterministic SRCROOT resolution diff --git a/doc/dev/refactor/implementation.md b/doc/dev/refactor/implementation.md new file mode 100644 index 00000000000..5f4fee81f3b --- /dev/null +++ b/doc/dev/refactor/implementation.md @@ -0,0 +1,235 @@ +# Implementation Plan + +Detailed tasks for the CIME refactor. See `plan.md` for architecture and principles. + +## Timeline: 22-24 weeks (5 slices) + +| Slice | Weeks | Focus | +|-------|-------|-------| +| 1. Foundation | 1-3 | Core abstractions | +| 2. Batch | 4-7 | Scheduler/submit | +| 3A. SRCROOT | 8-11 | Config loading (NEW) | +| 3B. Build | 12-16 | Build system | +| 4. Case | 17-22 | Case refactor | + +--- + +## Slice 1: Foundation (Weeks 1-3) + +**Goal**: Core abstractions with DI, no behavior changes. + +### Tasks +- `CIME/core/config/bootstrap.py` - Centralized sys.path management +- `CIME/core/exceptions.py` - Typed exception hierarchy +- `CIME/core/filesystem.py` - FileSystem protocol + RealFileSystem +- `CIME/core/process.py` - ProcessRunner protocol + RealProcessRunner +- `CIME/core/environment.py` - EnvironmentProvider protocol +- `CIME/core/clock.py` - Clock protocol + +### Patterns +**All use constructor injection**: +```python +class BatchSubmitter: + def __init__(self, filesystem: FileSystem, process: ProcessRunner): + self._fs = filesystem # Injected, not imported + self._proc = process + +def create_batch_submitter() -> BatchSubmitter: + return BatchSubmitter(get_filesystem(), get_process_runner()) +``` + +### Success +- 80%+ test coverage +- All existing tests pass +- No external API changes + +--- + +## Slice 2: Batch System (Weeks 4-7) + +**Goal**: Extract batch logic to `CIME/core/batch/`. + +### Tasks +- `CIME/core/batch/scheduler.py` - Scheduler protocol (PBS, Slurm, LSF) +- `CIME/core/batch/submitter.py` - Job submitter with DI +- Keep existing batch entrypoints as wrappers + +### Pattern +```python +class JobSubmitter: + def __init__(self, scheduler: Scheduler, filesystem: FileSystem): + self._scheduler = scheduler + self._fs = filesystem +``` + +### Success +- All batch tests pass +- External models submit jobs unchanged + +--- + +## Slice 3A: SRCROOT Standardization (Weeks 8-11) **NEW** + +**Goal**: Remove config_files.xml, standardize SRCROOT. + +See `feature_srcroot.md` for details. + +### Tasks +- `CIME/core/config/srcroot.py` - SRCROOTResolver with DI +- `CIME/core/config/loader.py` - ConfigFileLoader with DI +- Add `--srcroot` flag to CLI tools +- Refactor `CIME/XML/files.py` with feature flag +- 4-stage migration (opt-in → opt-out → deprecate → remove) + +### Success +- Works with E3SM, CESM, NorESM +- Standalone mode for unit tests +- config_files.xml deprecated + +--- + +## Slice 3B: Build System (Weeks 12-16) + +**Goal**: Extract build logic to `CIME/core/build/`. + +### Tasks +- `CIME/core/build/planner.py` - Build planning +- `CIME/core/build/orchestrator.py` - Build execution +- Keep `CIME/build_scripts/` as stable wrappers + +### Pattern +```python +class BuildOrchestrator: + def __init__(self, filesystem: FileSystem, process: ProcessRunner): + self._fs = filesystem + self._proc = process +``` + +### Success +- All build tests pass +- `build_scripts` compatibility maintained +- External models build unchanged + +--- + +## Slice 4: Case Refactoring (Weeks 17-22) + +**Goal**: Extract Case internals to focused components. + +### Tasks +- `CIME/core/status/` - Status tracking +- `CIME/core/xml/` - XML storage +- `CIME/core/locking/` - Lock management +- Case class becomes facade + +### Pattern +```python +class Case: + def __init__(self): + self._status = create_status_tracker() + self._xml_store = create_xml_store() + + def set_value(self, vid, value): + # Delegate to xml store + self._xml_store.set_value(self._caseroot, vid, value) +``` + +### Success +- Case API unchanged +- Internal complexity reduced +- Better testability + +--- + +## DI Guidelines + +### Constructor Injection +```python +# Good +class StatusTracker: + def __init__(self, filesystem: FileSystem): + self._fs = filesystem + +# Bad +class StatusTracker: + def do_thing(self): + import os + os.path.exists(...) # Direct global access +``` + +### Factory Functions +```python +def create_status_tracker(filesystem: Optional[FileSystem] = None) -> StatusTracker: + if filesystem is None: + filesystem = get_filesystem() # Default for production + return StatusTracker(filesystem) # Tests inject mocks +``` + +### Protocol Interfaces +```python +class FileSystem(Protocol): + def exists(self, path: Path) -> bool: ... +``` + +--- + +## Testing Strategy + +### Unit Tests (80%+ coverage) +- Mock all injected dependencies +- No global state manipulation +- Fast, isolated tests + +```python +def test_status_tracker(): + mock_fs = MockFileSystem(existing_files={...}) + tracker = StatusTracker(mock_fs) + result = tracker.get_status() + assert result == expected +``` + +### Integration Tests +- Key workflows end-to-end +- Real dependencies where needed + +### Compatibility Tests +- E3SM, CESM, NorESM workflows +- Case creation, build, submit +- Validate each slice + +--- + +## Success Metrics + +- **Compatibility**: 100% external tests pass +- **Coverage**: 80%+ for new code +- **Performance**: No regression (within 5%) +- **Maintainability**: Reduced coupling, clear boundaries + +--- + +## Risk Management + +**High-Risk Areas**: +1. SRCROOT standardization (breaking change) +2. config_files.xml removal (migration required) +3. Case refactoring (high external usage) + +**Mitigation**: +- Feature flags for transitions +- Multi-stage rollout +- Comprehensive external model testing +- Rollback plans per slice + +--- + +## Code Review Checklist + +For each PR, verify: +- [ ] Constructor injection (no direct imports in business logic) +- [ ] Factory functions provide defaults +- [ ] Tests use mocked dependencies +- [ ] No global state in core classes +- [ ] 80%+ test coverage +- [ ] All existing tests pass +- [ ] Documentation updated diff --git a/doc/dev/refactor/plan.md b/doc/dev/refactor/plan.md new file mode 100644 index 00000000000..514b81eb1bb --- /dev/null +++ b/doc/dev/refactor/plan.md @@ -0,0 +1,170 @@ +# CIME Refactor Plan + +Incremental refactor to improve dependency injection, resiliency, modularity, and testability while preserving compatibility with external models (E3SM, CESM, NorESM). + +See also: `implementation.md` for detailed tasks, `feature_srcroot.md` for config loading changes. + +--- + +## Compatibility-First Policy + +**Preserve current usage patterns unless change is absolutely required.** + +Breaking changes must be: +- Explicitly justified +- Accompanied by migration plan +- Validated with external models + +--- + +## Package Structure + +``` +CIME/ +├── api/ # Stable user-facing facades (Case, etc.) +├── cli/ # Optional thin CLI layer +├── core/ # Core implementation (DI-based) +│ ├── build/ +│ ├── batch/ +│ ├── config/ # Bootstrap, SRCROOT, config loading +│ ├── xml/ +│ ├── status/ +│ ├── locking/ +│ └── ... +├── data/ +├── build_scripts/ # Compatibility wrapper (stable) +├── SystemTests/ +└── tests/ +``` + +**Principle**: Internal evolution behind stable surfaces. + +--- + +## Architecture Patterns + +### Dependency Injection + +**Strategy**: Lightweight DI via constructor injection, protocols, factory functions. + +**Injectable boundaries**: +- Filesystem operations → `FileSystem` protocol +- Process execution → `ProcessRunner` protocol +- Environment variables → `EnvironmentProvider` protocol +- Time operations → `Clock` protocol +- XML/config loading → Protocol-based + +**Example**: +```python +class BuildOrchestrator: + def __init__(self, filesystem: FileSystem, process: ProcessRunner): + self._fs = filesystem # Injected + self._proc = process + +def create_build_orchestrator() -> BuildOrchestrator: + return BuildOrchestrator(get_filesystem(), get_process_runner()) +``` + +### Resiliency + +**Typed exceptions** replace broad fatal errors: +```python +class CIMEError(Exception): pass +class ConfigurationError(CIMEError): pass +class RetryableExternalCommandError(CIMEError): pass +``` + +**Explicit workflow states**: READY → VALIDATING → RUNNING → SUCCEEDED/FAILED + +**Idempotency**: Lock/unlock, status updates, regeneration steps + +### Error Handling + +- Fail early with clear messages +- Retry policies for transient failures +- Graceful degradation where appropriate + +--- + +## Key Constraints + +### Non-Installed Package +CIME is not installed via pip. Cases create symlinked tools that modify `sys.path`. This must continue to work. + +**Solution**: Centralized bootstrap in `CIME/core/config/bootstrap.py` + +### External Model Compatibility +E3SM, CESM, NorESM rely on CIME. Changes must not break their workflows. + +**Validation**: Test representative workflows from each model after major changes. + +### Stable Compatibility Surfaces +- `CIME/api/` - User-facing classes +- `CIME/build_scripts/` - External script entrypoints +- Case-created symlinked tools +- Import paths (until coordinated migration) + +--- + +## Migration Slices + +### Slice 1: Foundation (Weeks 1-3) +Core abstractions: FileSystem, ProcessRunner, Environment, Clock, Exceptions, Bootstrap + +### Slice 2: Batch (Weeks 4-7) +Extract batch/submit to `CIME/core/batch/`, scheduler abstractions + +### Slice 3A: SRCROOT Standardization (Weeks 8-11) **NEW FEATURE** +Remove config_files.xml, standardize SRCROOT resolution, direct config loading + +### Slice 3B: Build (Weeks 12-16) +Extract build to `CIME/core/build/`, keep `build_scripts` as wrapper + +### Slice 4: Case (Weeks 17-22) +Extract Case internals (status, XML, locking) to focused components + +--- + +## Success Criteria + +1. **Compatibility**: External models work without modification (or with documented migration) +2. **Testability**: 80%+ coverage, isolated unit tests +3. **Resiliency**: Better error handling and recovery +4. **Maintainability**: Clear boundaries, reduced coupling +5. **Documentation**: Complete docs for new patterns + +--- + +## Validation Requirements + +Each slice must pass: +- All existing CIME tests +- E3SM representative workflows +- CESM representative workflows +- NorESM representative workflows +- Performance benchmarks (no regression) + +--- + +## Non-Goals + +This refactor does NOT require: +- Immediate installed-package conversion +- Immediate CLI layer +- Removal of symlinked tools +- Wholesale rewrite in one step +- Breaking changes to Case API or build_scripts + +--- + +## Documentation + +- **This file (plan.md)**: Architecture and principles +- **implementation.md**: Detailed tasks and timeline +- **feature_srcroot.md**: SRCROOT standardization feature spec + +--- + +## Questions? + +File issues in CIME repository or discuss in development meetings. From 032959ca600e2ed08521dffb3428aa9f5fab1851 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Sat, 25 Apr 2026 02:25:54 -0700 Subject: [PATCH 2/7] docs: update refactor plan with codebase audit findings and slice details Rewrite plan.md, implementation.md, and README.md to reflect the reorganize-don't-rewrite philosophy, utils.py distribution strategy, cross-cutting issues inventory, and Case two-phase refactoring approach. --- doc/dev/refactor/README.md | 56 ++-- doc/dev/refactor/implementation.md | 428 +++++++++++++++++++++-------- doc/dev/refactor/plan.md | 305 +++++++++++++++----- 3 files changed, 581 insertions(+), 208 deletions(-) diff --git a/doc/dev/refactor/README.md b/doc/dev/refactor/README.md index b986a35d8e3..d3cf0f99997 100644 --- a/doc/dev/refactor/README.md +++ b/doc/dev/refactor/README.md @@ -1,6 +1,6 @@ # CIME Refactor Documentation -Comprehensive documentation for the CIME refactoring effort. +Documentation for the CIME refactoring effort. ## Quick Start @@ -14,71 +14,76 @@ Comprehensive documentation for the CIME refactoring effort. ### [plan.md](plan.md) Architecture, principles, and migration strategy. - Compatibility-first policy -- Package structure and DI patterns +- Reorganize-don't-rewrite approach - 5 implementation slices - Success criteria ### [implementation.md](implementation.md) -Detailed tasks, code patterns, and timeline. -- Week-by-week breakdown -- Code examples for each slice -- Testing strategy +Detailed tasks, timeline, and testing approach. +- Slice-by-slice breakdown +- Standard mocking strategy (no custom DI) - Code review checklist ### [feature_srcroot.md](feature_srcroot.md) SRCROOT standardization feature (Slice 3A). - Removes config_files.xml indirection -- DI-compliant implementation - Migration plan -- Testing with mocks ### [feature_single_entrypoint.md](feature_single_entrypoint.md) Single CIME entrypoint feature. - Replaces ~20 symlinks with one - Shell wrappers for tool compatibility -- Simplifies CIME version switching - Can run parallel with other slices ## Overview -**Goal**: Improve CIME's dependency injection, resiliency, and testability while maintaining compatibility with E3SM, CESM, and NorESM. +**Goal**: Improve CIME's modularity, error handling, and testability while +maintaining compatibility with E3SM, CESM, and NorESM. **Timeline**: 22-24 weeks (5 slices) -**Approach**: Incremental refactor with compatibility preserved at each step. +**Approach**: Move existing code into focused modules. Consolidate scattered +functions back into the classes that own them. Don't rewrite working code or +wrap stdlib behind abstraction layers. ## Implementation Slices -1. **Foundation** (3 weeks): Core abstractions (FileSystem, ProcessRunner, etc.) -2. **Batch** (4 weeks): Extract batch/submit logic +1. **Foundation** (3 weeks): Typed exceptions, centralized bootstrap +2. **Batch** (4 weeks): Move batch/submit logic to `CIME/core/batch/` 3. **SRCROOT** (4 weeks): Standardize config loading (NEW FEATURE) -4. **Build** (5 weeks): Extract build logic -5. **Case** (6 weeks): Extract Case internals to focused components +4. **Build** (5 weeks): Move build logic to `CIME/core/build/` +5. **Case** (6 weeks): Consolidate scattered Case functions, then decompose + into focused subsystems (status, locking, XML storage) ## Key Principles - **Compatibility first**: External models must continue working -- **Constructor injection**: All dependencies injected, no globals +- **Reorganize, don't rewrite**: Move code to better homes +- **DI where it earns its keep**: Protocols and constructor injection for real + CIME polymorphism (scheduler backends, config loaders); `mock.patch` for stdlib - **Incremental**: Each slice independently testable -- **Test-driven**: 80%+ coverage requirement +- **Test-driven**: 80%+ coverage for reorganized code ## For Developers **Before coding**: 1. Read relevant slice in `implementation.md` -2. Understand DI patterns in `plan.md` -3. Review code examples +2. Understand the reorganize-don't-rewrite principle in `plan.md` **During development**: -- Use constructor injection -- Write tests with mocks +- Move existing functions, don't rewrite them +- Add re-exports from old import paths +- Write tests with `unittest.mock`, `monkeypatch`, `tmp_path` +- Use DI/protocols for real CIME interfaces, not stdlib wrapping - Maintain backward compatibility -- Update documentation **Code review checklist**: -- [ ] Constructor injection used -- [ ] No direct filesystem/process/env access in core classes -- [ ] Factory functions provide defaults +- [ ] Code moved, not rewritten (unless simplifying complexity) +- [ ] Free functions that take an object consolidated as methods where appropriate +- [ ] Internal CIME imports updated to `CIME/core/` +- [ ] Old import paths still work via re-exports (for downstream models) +- [ ] Standard mocking for stdlib; DI/protocols for CIME interfaces +- [ ] No unnecessary abstraction layers - [ ] 80%+ test coverage - [ ] All tests pass - [ ] Docs updated @@ -87,6 +92,7 @@ Single CIME entrypoint feature. **E3SM, CESM, NorESM integrators**: - Each slice maintains compatibility +- Old import paths continue to work via re-exports - SRCROOT feature (Slice 3A) has migration guide - Test during opt-in periods - Provide feedback early diff --git a/doc/dev/refactor/implementation.md b/doc/dev/refactor/implementation.md index 5f4fee81f3b..f1ee4248a31 100644 --- a/doc/dev/refactor/implementation.md +++ b/doc/dev/refactor/implementation.md @@ -2,11 +2,22 @@ Detailed tasks for the CIME refactor. See `plan.md` for architecture and principles. +**Key rule: move existing code, don't rewrite it.** Extract functions into +focused modules. Only refactor when a module is too large or too tangled to +move as-is. + +**Target: Python 3.9+.** Do not use 3.10+ syntax (`X | Y` unions, +`match`/`case`). Use `typing.Union`, `typing.Optional`, etc. + +**Migration pattern**: Move code to `CIME/core/`, then make the original +module import from `CIME/core/` and re-export. See `plan.md` "Migration +Pattern" for details and examples. + ## Timeline: 22-24 weeks (5 slices) | Slice | Weeks | Focus | |-------|-------|-------| -| 1. Foundation | 1-3 | Core abstractions | +| 1. Foundation | 1-3 | Exceptions, bootstrap | | 2. Batch | 4-7 | Scheduler/submit | | 3A. SRCROOT | 8-11 | Config loading (NEW) | | 3B. Build | 12-16 | Build system | @@ -14,57 +25,132 @@ Detailed tasks for the CIME refactor. See `plan.md` for architecture and princip --- +## Handling `utils.py` (2700 lines, 104 functions) + +`utils.py` is the largest module and doesn't belong to any single slice. +It contains functions across every category: + +| Category | Count | Examples | +|----------|-------|---------| +| General utilities | 57 | `run_cmd`, `safe_copy`, logging, time conversion | +| SRCROOT/config | 19 | `get_cime_root`, `get_src_root`, `get_model` | +| Foundation | 9 | `CIMEError`, `expect`, `fixup_sys_path` | +| Case | 9 | `parse_test_name`, `wait_for_unlocked` | +| Batch | 7 | `batch_jobid`, `get_project`, `transform_vars` | +| Build | 3 | `analyze_build_log`, `copy_local_macros_to_dir` | + +**Strategy**: Each slice moves its own functions **plus the general utilities +those functions depend on**. As general utils get pulled into `core/`, they +land in focused modules: + +- `CIME/core/shell.py` — `run_cmd`, `run_cmd_no_fail` (pulled by Slice 2/Batch) +- `CIME/core/logging.py` — logging setup, formatters (pulled by Slice 1) +- `CIME/core/time.py` — time conversion helpers (pulled by Slice 4/Case) +- `CIME/core/fileops.py` — `safe_copy`, `symlink_force`, `find_files` (pulled as needed) +- `CIME/core/convert.py` — type conversion helpers (pulled as needed) + +By the end of Slice 4, `utils.py` should be a thin re-export file with no +implementation code of its own. + +--- + ## Slice 1: Foundation (Weeks 1-3) -**Goal**: Core abstractions with DI, no behavior changes. +**Goal**: Typed exceptions, centralized bootstrap, and eliminate the worst +cross-cutting code smells. No behavior changes. ### Tasks -- `CIME/core/config/bootstrap.py` - Centralized sys.path management -- `CIME/core/exceptions.py` - Typed exception hierarchy -- `CIME/core/filesystem.py` - FileSystem protocol + RealFileSystem -- `CIME/core/process.py` - ProcessRunner protocol + RealProcessRunner -- `CIME/core/environment.py` - EnvironmentProvider protocol -- `CIME/core/clock.py` - Clock protocol - -### Patterns -**All use constructor injection**: -```python -class BatchSubmitter: - def __init__(self, filesystem: FileSystem, process: ProcessRunner): - self._fs = filesystem # Injected, not imported - self._proc = process - -def create_batch_submitter() -> BatchSubmitter: - return BatchSubmitter(get_filesystem(), get_process_runner()) -``` +- `CIME/core/exceptions.py` — Typed exception hierarchy extending existing `CIMEError`. + **Important**: `CIMEError` currently lives in `utils.py:155` — move it to + `core/exceptions.py` and re-export from `utils.py`. The typed subclasses + (`ConfigurationError`, `BuildError`, etc.) won't have callers yet but + establish the hierarchy for later slices. +- `CIME/core/config/bootstrap.py` — Centralized `sys.path` management (consolidates + the `sys.path.insert` calls currently scattered across 11+ files, plus the + duplicate logic in `standard_script_setup.py` and `standard_module_setup.py`) +- Move from `utils.py`: `CIMEError`, `expect`, `deprecate_action`, + `fixup_sys_path`, `import_from_file`, `import_and_run_sub_or_cmd`, + `run_sub_or_cmd`, and their helpers +- Move logging infrastructure from `utils.py` → `CIME/core/logging.py` + (`IndentFormatter`, `set_logger_indent`, `configure_logging`, + `setup_standard_logging_options`, etc.) +- Migrate `sys.path` call-sites incrementally to use bootstrap module +- Update internal CIME imports; leave re-exports in `utils.py` +- **Eliminate star imports from `standard_module_setup`** — this single + `from CIME.XML.standard_module_setup import *` appears in ~60 files, + polluting every module's namespace with `os`, `sys`, `re`, `expect`, + `run_cmd`, etc. Replace with explicit imports in each file. This is the + highest-impact code quality fix in the entire refactor. +- **Eliminate star imports from `test_status`** — `from CIME.test_status import *` + appears in ~10 files, exporting ~30 constants. Replace with explicit imports. +- **Remove `GLOBAL = {}` mutable state** from `utils.py:21` — this dict is used + to pass `SRCROOT` between `case.py`, `create_test.py`, and `utils.get_src_root()` + via implicit global mutation. Replace with explicit parameter passing. +- **Fix `Servers/__init__.py`** — runs `shutil.which()` at import time for + `globus-url-copy`, `svn`, `wget`, making package import non-deterministic. + Make discovery lazy. +- Standardize error handling: audit `sys.exit()` calls (6 sites) and bare + `raise RuntimeError` (~30 sites) and migrate to `CIMEError` / `expect()` + where appropriate +- **Fix `conftest.py`** — currently requires host model config and machine XML + to run any tests. Split so unit tests (`test_unit_*.py`) can run standalone + without machine initialization; keep machine setup for system tests only. +- **Fix circular imports** in Case — `case.py:84-102` injects methods from + 10 sibling modules at class body level. The move to `core/` modules should + start breaking these cycles. + +### What we're NOT doing in Slice 1 +- No Protocol classes wrapping stdlib (`os.path`, `subprocess`, `time`, `os.environ`) +- No factory functions or service locators +- No new abstraction layers ### Success -- 80%+ test coverage - All existing tests pass -- No external API changes +- Zero star imports from `standard_module_setup` and `test_status` +- `GLOBAL = {}` eliminated; SRCROOT passed explicitly +- `sys.path` manipulation consolidated +- Consistent error handling (no bare `sys.exit()` in library code) +- `Servers/__init__.py` no longer runs executables at import time +- Unit tests (`test_unit_*.py`) run without host model / machine config +- Exception hierarchy in place for later slices to use --- ## Slice 2: Batch System (Weeks 4-7) -**Goal**: Extract batch logic to `CIME/core/batch/`. +**Goal**: Move batch logic from scattered locations into `CIME/core/batch/`. ### Tasks -- `CIME/core/batch/scheduler.py` - Scheduler protocol (PBS, Slurm, LSF) -- `CIME/core/batch/submitter.py` - Job submitter with DI -- Keep existing batch entrypoints as wrappers - -### Pattern -```python -class JobSubmitter: - def __init__(self, scheduler: Scheduler, filesystem: FileSystem): - self._scheduler = scheduler - self._fs = filesystem -``` +- Move scheduler-specific logic from XML classes into `CIME/core/batch/` +- Move submission logic from `case_submit.py` internals +- Move from `utils.py`: `batch_jobid`, `get_batch_script_for_job`, + `get_project`, `get_charge_account`, `add_mail_type_args`, + `resolve_mail_type_args`, `transform_vars` +- Move `run_cmd`, `run_cmd_no_fail` from `utils.py` → `CIME/core/shell.py` + (general utility, pulled here because batch is the first heavy consumer) +- A `Scheduler` protocol is appropriate here — PBS, Slurm, and LSF are genuine + polymorphic backends worth abstracting behind a common interface +- Keep existing entrypoints as thin wrappers calling into new location +- Update internal CIME imports; leave re-exports in old locations + +### Consolidation +- `transform_vars` (utils.py:2118) takes `case` and calls `case.get_value()` + repeatedly — consolidate as a `Case` method, with the core template logic + in `CIME/core/batch/` +- `batch_jobid` (utils.py:2360) calls `case.get_job_id()` — consolidate into + the batch subsystem +- **`EnvBatch`** (XML/env_batch.py, 1590 lines, 43 methods) — this is the + second-largest class in CIME. It handles batch config parsing, job submission, + queue management, and directive generation all in one class. Decompose into: + - `CIME/core/batch/config.py` — batch config/queue parsing (from EnvBatch) + - `CIME/core/batch/submit.py` — job submission logic (from EnvBatch + case_submit.py) + - `CIME/core/batch/directives.py` — directive generation (from EnvBatch) + - `EnvBatch` becomes a thinner wrapper that delegates to these modules ### Success - All batch tests pass - External models submit jobs unchanged +- Batch logic in one place instead of spread across XML/ and case/ --- @@ -75,11 +161,18 @@ class JobSubmitter: See `feature_srcroot.md` for details. ### Tasks -- `CIME/core/config/srcroot.py` - SRCROOTResolver with DI -- `CIME/core/config/loader.py` - ConfigFileLoader with DI +- `CIME/core/config/srcroot.py` — SRCROOT resolution logic +- `CIME/core/config/loader.py` — Config file loading +- Move from `utils.py`: `get_cime_root`, `get_src_root`, `get_model`, + `set_model`, `get_cime_config`, `get_config_path`, `get_schema_path`, + `get_template_path`, `get_tools_path`, `get_cime_default_driver`, + `get_all_cime_models`, `get_model_config_location_within_cime`, + `get_scripts_root`, `get_model_config_root`, `get_htmlroot`, `get_urlroot`, + and their helpers - Add `--srcroot` flag to CLI tools - Refactor `CIME/XML/files.py` with feature flag - 4-stage migration (opt-in → opt-out → deprecate → remove) +- Update internal CIME imports; leave re-exports in `utils.py` ### Success - Works with E3SM, CESM, NorESM @@ -90,20 +183,32 @@ See `feature_srcroot.md` for details. ## Slice 3B: Build System (Weeks 12-16) -**Goal**: Extract build logic to `CIME/core/build/`. +**Goal**: Move build logic from `build.py` (1350 lines) into focused modules. ### Tasks -- `CIME/core/build/planner.py` - Build planning -- `CIME/core/build/orchestrator.py` - Build execution +- Extract build planning logic into `CIME/core/build/` +- Extract build execution/orchestration +- Move from `utils.py`: `analyze_build_log`, `run_bld_cmd_ensure_logging`, + `copy_local_macros_to_dir` +- Move file operation helpers from `utils.py` → `CIME/core/fileops.py` + (`safe_copy`, `safe_recursive_copy`, `symlink_force`, `copy_globs`, + `copy_over_file`, `copyifnewer` — pulled here because build is a heavy consumer) - Keep `CIME/build_scripts/` as stable wrappers - -### Pattern -```python -class BuildOrchestrator: - def __init__(self, filesystem: FileSystem, process: ProcessRunner): - self._fs = filesystem - self._proc = process -``` +- `build.py` becomes thin re-export layer +- Update internal CIME imports; leave re-exports in old locations + +### Consolidation +- `case_build()`, `clean()`, `_clean_impl()`, `post_build()` (build.py) + all take `case`, access `case._gitinterface` (private!), call + `case.get_value()` / `case.set_value()` / `case.flush()` — these should + become `Case` methods that delegate to `CIME/core/build/` +- `get_standard_cmake_args()`, `get_standard_makefile_args()`, `uses_kokkos()` + (build.py) extract ~15 values from Case — consolidate as Case methods or a + `BuildConfig` data class populated from Case +- `generate_makefile_macro()` (build.py) calls `case.get_value()` — same pattern +- Deduplicate the four copy functions in `utils.py` (`safe_copy`, + `copy_over_file`, `copyifnewer`, `copy_globs`) into a coherent + `CIME/core/fileops.py` API ### Success - All build tests pass @@ -114,87 +219,175 @@ class BuildOrchestrator: ## Slice 4: Case Refactoring (Weeks 17-22) -**Goal**: Extract Case internals to focused components. +**Goal**: Decompose the `Case` class into focused subsystems while +consolidating scattered Case-related functions. + +### Why Case needs refactoring + +`Case` (`case.py`, 2600 lines) is a god object. It directly handles: +- XML value get/set across multiple env files +- Status tracking (CaseStatus file) +- Lock management (LockedFiles directory) +- Build orchestration +- Job submission +- Namelist generation +- Clone/setup/test workflows + +On top of that, many Case operations live *outside* the class as free +functions in `utils.py`, `build.py`, `status.py`, `locked_files.py`, and +`case_run.py` — functions that take `case` as a parameter and reach into +its internals (including private attributes like `case._gitinterface`). + +The result: Case responsibility is smeared across ~10 files with no clear +boundaries, and the class itself is too large to reason about or test in +isolation. + +### Two-phase approach + +**Phase 1 — Consolidate into Case**: Bring the scattered free functions +back into Case (or into subsystem classes that Case owns). This *temporarily* +makes Case bigger, but it gathers all the responsibility in one place so we +can see the seams clearly. + +**Phase 2 — Extract subsystems out of Case**: Once the responsibility is +consolidated, decompose Case into focused subsystem modules that Case +delegates to. Case becomes a thinner coordinator/facade. + +The end result is a Case that's smaller than today, with clear delegation +to subsystems that are independently testable. + +### Phase 1 tasks — Consolidate + +Bring free functions into Case (or subsystem classes Case will delegate to): + +- **Status** (from `status.py`): `append_case_status()`, + `run_and_log_case_status()` — every caller extracts `caseroot` and + `case._gitinterface` just to pass them in +- **Locking** (from `locked_files.py`): `check_lockedfiles()`, + `check_lockedfile()`, `diff_lockedfile()`, `check_diff()` — pure Case + integrity checks deeply coupled to Case env XML subsystem +- **Case queries** (from `utils.py`): `is_comp_standalone()`, + `find_system_test()`, `get_lids()`, `new_lid()` — predicates and queries + on Case state +- **Case run helpers** (from `case/case_run.py`): `_pre_run_check()`, + `_run_model_impl()`, `_post_run_check()`, `_resubmit_check()` — free + functions that take `case`; make them methods +- **TestScheduler** (from `test_scheduler.py`): `_get_time_est()`, + `_order_tests_by_runtime()`, `_translate_test_names_for_new_pecount()` — + module-level helpers with `_TIME_CACHE` that should be instance state + +### Phase 2 tasks — Extract subsystems + +Decompose Case into focused modules it delegates to: + +- `CIME/core/status/` — status tracking (CaseStatus file, `append_status`, + `run_and_log_status`) +- `CIME/core/locking/` — lock management (LockedFiles, `check_locked`, + `diff_locked`) +- `CIME/core/xml/` — XML value storage (env file get/set/flush, the + `_env_entryid_files` and `_env_generic_files` arrays) + +Case keeps its public API (`get_value`, `set_value`, `build`, `submit`, etc.) +but each method delegates to the appropriate subsystem: -### Tasks -- `CIME/core/status/` - Status tracking -- `CIME/core/xml/` - XML storage -- `CIME/core/locking/` - Lock management -- Case class becomes facade - -### Pattern ```python +# After refactoring — Case delegates, doesn't implement class Case: - def __init__(self): - self._status = create_status_tracker() - self._xml_store = create_xml_store() - - def set_value(self, vid, value): - # Delegate to xml store - self._xml_store.set_value(self._caseroot, vid, value) + def __init__(self, ...): + self._status = StatusTracker(self._caseroot, self._gitinterface) + self._locks = LockManager(self._caseroot) + self._xml = XmlStore(self._env_files) + + def append_status(self, msg, ...): + self._status.append(msg, ...) + + def check_lockedfiles(self): + self._locks.check(self._xml) ``` +### Other Slice 4 tasks +- Move from `utils.py`: `wait_for_unlocked`, `normalize_case_id`, + `parse_test_name`, `get_full_test_name`, `compute_total_time` +- Move remaining general utils from `utils.py`: + - `CIME/core/time.py` — time conversion (`convert_to_seconds`, + `convert_to_babylonian_time`, `get_time_in_seconds`, `format_time`) + - `CIME/core/convert.py` — type conversion (`convert_to_type`, + `convert_to_unknown_type`, `convert_to_string`, `stringify_bool`) +- `utils.py` should be a thin re-export file by end of Slice 4 +- Update all internal CIME imports + +### Other large modules to decompose in Slice 4 + +- **`case/case_st_archive.py`** (1395 lines, ~20 free functions taking `case`) + — largest procedural module; consolidate functions as Case methods or + extract into `CIME/core/archive/` +- **`case/check_input_data.py`** (693 lines, ~12 free functions taking `case`) + — consolidate into Case or a focused input-data module +- **`baselines/performance.py`** (612 lines, ~12 free functions taking `case`) + — consolidate into a baseline comparison subsystem +- **`hist_utils.py`** (836 lines, ~7 free functions taking `case`) + — history file handling, consolidate into appropriate subsystem +- **`TestScheduler`** (test_scheduler.py, 1340 lines, 28 methods) — mixes + test creation, phase management, job submission, and result processing; + decompose into focused components + ### Success -- Case API unchanged -- Internal complexity reduced -- Better testability +- Case API unchanged from external perspective +- Internal modules are independently testable +- No single module over ~500 lines --- -## DI Guidelines - -### Constructor Injection -```python -# Good -class StatusTracker: - def __init__(self, filesystem: FileSystem): - self._fs = filesystem - -# Bad -class StatusTracker: - def do_thing(self): - import os - os.path.exists(...) # Direct global access -``` +## Testing Strategy -### Factory Functions +### Standard mocking for stdlib ```python -def create_status_tracker(filesystem: Optional[FileSystem] = None) -> StatusTracker: - if filesystem is None: - filesystem = get_filesystem() # Default for production - return StatusTracker(filesystem) # Tests inject mocks +# Good: use unittest.mock for stdlib calls +from unittest.mock import patch + +def test_build_checks_file(): + with patch("os.path.exists", return_value=True): + result = check_build_prereqs("/some/path") + assert result is True + +# Good: use tmp_path for filesystem tests +def test_writes_status(tmp_path): + tracker = StatusTracker(str(tmp_path)) + tracker.set_status("BUILT") + assert (tmp_path / "CaseStatus").read_text() == "BUILT" + +# Good: use monkeypatch for env vars +def test_reads_cimeroot(monkeypatch): + monkeypatch.setenv("CIMEROOT", "/my/cime") + assert find_cimeroot() == "/my/cime" ``` -### Protocol Interfaces +### DI / Protocols where they earn their keep ```python -class FileSystem(Protocol): - def exists(self, path: Path) -> bool: ... +# Good: Scheduler is genuinely polymorphic — PBS, Slurm, LSF are +# different backends behind a common interface +class Scheduler(Protocol): + def submit(self, job: Job) -> str: ... + def poll(self, job_id: str) -> JobStatus: ... + +def test_submission_retries(): + fake = FakeScheduler(fail_first=2) + submitter = JobSubmitter(scheduler=fake) + result = submitter.submit_with_retry(job, max_retries=3) + assert result.success ``` ---- - -## Testing Strategy +**Rule of thumb**: if the variation is in *CIME's own code* (scheduler types, +config loaders, model components), a protocol may be warranted. If you're just +calling `os.path` or `subprocess`, use `mock.patch`. -### Unit Tests (80%+ coverage) -- Mock all injected dependencies -- No global state manipulation -- Fast, isolated tests +### Coverage +- 80%+ for reorganized code +- Existing tests must continue to pass -```python -def test_status_tracker(): - mock_fs = MockFileSystem(existing_files={...}) - tracker = StatusTracker(mock_fs) - result = tracker.get_status() - assert result == expected -``` - -### Integration Tests +### Integration/Compatibility Tests - Key workflows end-to-end -- Real dependencies where needed - -### Compatibility Tests -- E3SM, CESM, NorESM workflows -- Case creation, build, submit +- E3SM, CESM, NorESM representative workflows - Validate each slice --- @@ -202,9 +395,9 @@ def test_status_tracker(): ## Success Metrics - **Compatibility**: 100% external tests pass -- **Coverage**: 80%+ for new code +- **Coverage**: 80%+ for reorganized code - **Performance**: No regression (within 5%) -- **Maintainability**: Reduced coupling, clear boundaries +- **Maintainability**: No module over ~500 lines, clear boundaries --- @@ -226,10 +419,13 @@ def test_status_tracker(): ## Code Review Checklist For each PR, verify: -- [ ] Constructor injection (no direct imports in business logic) -- [ ] Factory functions provide defaults -- [ ] Tests use mocked dependencies -- [ ] No global state in core classes -- [ ] 80%+ test coverage +- [ ] Existing code moved, not rewritten (unless refactoring complexity) +- [ ] Free functions that take an object consolidated as methods where appropriate +- [ ] Internal CIME imports updated to point to new `CIME/core/` location +- [ ] Old import paths still work via re-exports (for external consumers) +- [ ] Tests use standard mocking (`unittest.mock`, `monkeypatch`, `tmp_path`) +- [ ] DI/protocols used only for real CIME polymorphism, not stdlib wrapping +- [ ] No unnecessary abstraction layers +- [ ] 80%+ test coverage for moved/new code - [ ] All existing tests pass - [ ] Documentation updated diff --git a/doc/dev/refactor/plan.md b/doc/dev/refactor/plan.md index 514b81eb1bb..7d320fe6882 100644 --- a/doc/dev/refactor/plan.md +++ b/doc/dev/refactor/plan.md @@ -1,8 +1,15 @@ # CIME Refactor Plan -Incremental refactor to improve dependency injection, resiliency, modularity, and testability while preserving compatibility with external models (E3SM, CESM, NorESM). +Incremental refactor to improve modularity, error handling, and testability +while preserving compatibility with external models (E3SM, CESM, NorESM). -See also: `implementation.md` for detailed tasks, `feature_srcroot.md` for config loading changes. +**Core principle: reorganize existing code, don't rewrite it.** Move functions +to better homes, break apart oversized modules, improve error handling. Don't +wrap standard library APIs in abstraction layers — `os.path`, `subprocess`, +`os.environ`, and `time` are already easily mockable with `unittest.mock.patch`. + +See also: `implementation.md` for detailed tasks, `feature_srcroot.md` for +config loading changes. --- @@ -21,117 +28,278 @@ Breaking changes must be: ``` CIME/ -├── api/ # Stable user-facing facades (Case, etc.) -├── cli/ # Optional thin CLI layer -├── core/ # Core implementation (DI-based) -│ ├── build/ -│ ├── batch/ +├── core/ # Reorganized internals (all implementation lives here) │ ├── config/ # Bootstrap, SRCROOT, config loading -│ ├── xml/ -│ ├── status/ -│ ├── locking/ -│ └── ... -├── data/ -├── build_scripts/ # Compatibility wrapper (stable) -├── SystemTests/ -└── tests/ +│ ├── batch/ # Batch/scheduler logic (from existing code) +│ ├── build/ # Build logic (from existing code) +│ ├── status/ # Status tracking (from status.py, case) +│ ├── locking/ # Lock management (from locked_files.py, case) +│ ├── exceptions.py # Typed exception hierarchy +│ ├── shell.py # run_cmd, run_cmd_no_fail (from utils.py) +│ ├── logging.py # Logging setup (from utils.py) +│ ├── fileops.py # File helpers: safe_copy, symlink_force (from utils.py) +│ ├── time.py # Time conversion helpers (from utils.py) +│ └── convert.py # Type conversion helpers (from utils.py) +├── utils.py # Thin re-exports only (was 2700 lines) +├── build.py # Thin re-exports only +├── case/ # Case modules (thinned, delegates to core/) +├── XML/ # XML parsers (existing) +├── data/ # Config files (existing) +├── build_scripts/ # Compatibility wrapper (stable, unchanged) +├── SystemTests/ # System tests (existing) +├── Tools/ # CLI tools (existing) +└── tests/ # Tests ``` -**Principle**: Internal evolution behind stable surfaces. +**Principle**: Move existing code into focused modules. Existing import paths +continue to work via re-exports until downstream models migrate. --- -## Architecture Patterns - -### Dependency Injection +## Migration Pattern -**Strategy**: Lightweight DI via constructor injection, protocols, factory functions. +All implementation code moves to `CIME/core/`. Existing modules become thin +wrappers that import from `core/` and re-export, preserving current usage. -**Injectable boundaries**: -- Filesystem operations → `FileSystem` protocol -- Process execution → `ProcessRunner` protocol -- Environment variables → `EnvironmentProvider` protocol -- Time operations → `Clock` protocol -- XML/config loading → Protocol-based +**Workflow for each piece of code**: +1. Move the function/class from its current location into the appropriate + `CIME/core/` module +2. In the original file, add a thin re-export (so **external** callers like + E3SM/CESM/NorESM are unaffected) +3. Update all **internal** CIME imports to point to the new `CIME/core/` location +4. Add/update tests against the new location -**Example**: +**Example** — moving `run_cmd` from `utils.py`: ```python -class BuildOrchestrator: - def __init__(self, filesystem: FileSystem, process: ProcessRunner): - self._fs = filesystem # Injected - self._proc = process - -def create_build_orchestrator() -> BuildOrchestrator: - return BuildOrchestrator(get_filesystem(), get_process_runner()) +# CIME/core/batch/submitter.py (new home — real code lives here) +def run_cmd(cmd, ...): + # actual implementation moved here + ... + +# CIME/utils.py (thin re-export for external consumers only) +from CIME.core.batch.submitter import run_cmd # noqa: F401 + +# CIME/case/case_submit.py (internal — updated to import from core/) +# Before: +# from CIME.utils import run_cmd +# After: +from CIME.core.batch.submitter import run_cmd ``` -### Resiliency - -**Typed exceptions** replace broad fatal errors: -```python -class CIMEError(Exception): pass -class ConfigurationError(CIMEError): pass -class RetryableExternalCommandError(CIMEError): pass -``` +This means: +- `CIME/core/` holds all implementation code +- **Internal** CIME code imports from `CIME/core/` directly +- `CIME/utils.py`, `CIME/build.py`, `CIME/case/*.py`, etc. keep thin + re-exports **only** for external downstream consumers +- `CIME/build_scripts/`, `CIME/Tools/`, and case symlinked scripts remain + as-is — they're entrypoints, not implementation +- Downstream models (E3SM, CESM, NorESM) continue working with zero changes +- Over time, downstream models can migrate to importing from `CIME/core/` + directly, and the old re-exports can eventually be deprecated -**Explicit workflow states**: READY → VALIDATING → RUNNING → SUCCEEDED/FAILED +--- -**Idempotency**: Lock/unlock, status updates, regeneration steps +## Approach: Reorganize, Don't Rewrite + +### What we DO +- **Move** functions from bloated modules (`utils.py` at 2700 lines) into + focused modules under `CIME/core/` +- **Consolidate** scattered functionality back into the classes that own it — + free functions that take `case` as a parameter and mutate Case state should + become methods on `Case` or its subsystem classes +- **Extract** coherent subsystems from `Case` (status tracking, locking, + XML manipulation) into their own modules +- **Consolidate** scattered patterns (e.g. `sys.path` manipulation in 11 files) +- **Deduplicate** overlapping functionality (e.g. four copy functions in utils.py) +- **Improve** error handling with typed exceptions +- **Eliminate** star imports, global mutable state, and import-time side effects +- **Add** tests for code that currently lacks coverage +- **Fix** test infrastructure so unit tests can run without host model config + +### Consolidation principle +If a free function takes an object and operates on its state, it should be a +method on that object. This is especially prevalent with `case`: -### Error Handling +```python +# Before: free function reaching into Case +def case_build(case, ...): + case._gitinterface # accessing private state! + case.get_value("EXEROOT") + case.set_value("BUILD_COMPLETE", "TRUE") + case.flush() + +# After: method on Case (or delegated subsystem) +class Case: + def build(self, ...): + ... +``` -- Fail early with clear messages -- Retry policies for transient failures -- Graceful degradation where appropriate +**Key consolidation targets identified**: + +| Current location | Functions | Problem | +|-----------------|-----------|---------| +| `build.py` | `case_build`, `clean`, `_clean_impl`, `post_build` | Access `case._gitinterface`, mutate Case state | +| `locked_files.py` | `check_lockedfiles`, `check_lockedfile`, `diff_lockedfile` | Pure Case integrity checks | +| `status.py` | `append_case_status`, `run_and_log_case_status` | Every caller extracts `caseroot` and `_gitinterface` from Case | +| `utils.py` | `transform_vars`, `is_comp_standalone`, `find_system_test` | Case queries living in a utility grab-bag | +| `test_scheduler.py` | `_get_time_est`, `_order_tests_by_runtime` | Module-level cache should be instance state on `TestScheduler` | +| `utils.py` | `safe_copy`, `copy_over_file`, `copyifnewer`, `copy_globs` | Overlapping copy semantics | + +### Large classes and modules requiring decomposition + +Beyond `Case`, several other classes and modules have grown too large and +exhibit the same god-object or scattered-function patterns: + +**Large classes (>20 methods):** + +| Class | File | Methods | Lines | Problem | +|-------|------|--------:|------:|---------| +| `Case` | `case/case.py` | 63 | 2600 | God object; functions scattered across ~10 files | +| `EnvBatch` | `XML/env_batch.py` | 43 | 1590 | Batch config, submission, queues, directives all in one | +| `GenericXML` | `XML/generic_xml.py` | 41 | 700 | Very wide interface; base for all XML classes | +| `NamelistGenerator` | `nmlgen.py` | 36 | 870 | Mixes definition lookup, resolution, streams, file I/O | +| `EnvMachSpecific` | `XML/env_mach_specific.py` | 35 | 770 | Module loading + env vars + mpirun + GPU config | +| `SystemTestsCommon` | `SystemTests/system_tests_common.py` | 33 | 1020 | Large but intentional base class | +| `TestScheduler` | `test_scheduler.py` | 28 | 1340 | Test creation + phase mgmt + submission + results | + +**Procedural modules (free functions taking `case`, no class):** + +| File | Lines | Funcs w/ `case` | Problem | +|------|------:|----------------:|---------| +| `case/case_st_archive.py` | 1395 | ~20 | Largest procedural module, all Case operations | +| `case/check_input_data.py` | 693 | ~12 | Scattered Case queries | +| `baselines/performance.py` | 612 | ~12 | Baseline comparison, all Case-dependent | +| `hist_utils.py` | 836 | ~7 | History file handling, all Case-dependent | + +These are addressed primarily in **Slice 2** (EnvBatch → `core/batch/`), +**Slice 3B** (build decomposition), and **Slice 4** (Case, TestScheduler, +and procedural case/ modules). + +### Cross-cutting issues (addressed primarily in Slice 1) + +These issues cut across all modules and should be fixed early: + +**Star imports** — `from CIME.XML.standard_module_setup import *` appears in +~60 source files, injecting `os`, `sys`, `re`, `expect`, `run_cmd`, etc. into +every namespace. `from CIME.test_status import *` appears in ~10 files, +exporting ~30 constants. These must be replaced with explicit imports to enable +static analysis and make dependencies visible. + +**Global mutable state** — `GLOBAL = {}` in `utils.py:21` is used to pass +`SRCROOT` between `case.py`, `create_test.py`, and `utils.get_src_root()` via +implicit mutation. Other globals include `_CIMECONFIG` (singleton config cache), +`_TIME_CACHE` (test scheduler cache), and `_ALL_TESTS` (test registry cache). +Replace with explicit parameter passing or instance state. + +**Import-time side effects** — `Servers/__init__.py` runs `shutil.which()` for +external tools at import time. `standard_script_setup.py` modifies `sys.path` +and `os.environ` at import time. `standard_module_setup.py` also modifies +`sys.path` at import time. These make imports non-deterministic and prevent +clean test isolation. + +**Inconsistent error handling** — 5 different patterns in use: `expect()`, +`raise CIMEError`, `sys.exit()`, `raise RuntimeError`, and +`logger.fatal()` + `sys.exit()`. Standardize on `expect()` / `CIMEError` +for library code; `sys.exit()` only in CLI entrypoints. + +**Circular imports** — The `Case` class injects methods from 10 sibling modules +at class body level (case.py:84-102) to work around circular dependencies. Other +circular imports are handled by deferred imports inside function bodies +(utils.py:456, env_batch.py:198). The `core/` reorganization should break +these cycles. + +**Test infrastructure** — `conftest.py` requires a valid `srcroot` with +`cime_config/` directory and calls `Machines()` which requires machine config +XML. This means unit tests cannot run standalone without host model integration. +Fix by making conftest machine-aware only for system tests, not unit tests. + +### Where DI / Protocols make sense +Use DI, protocols, and factory functions where CIME has **real polymorphism** +or **complex internal interfaces** worth decoupling: +- **Scheduler backends** (PBS, Slurm, LSF) — a `Scheduler` protocol with + concrete implementations is natural here +- **XML config loading** — swappable loaders for testing without real config files +- **Components that cross subsystem boundaries** — e.g. build orchestration + depending on a batch submitter interface + +### What we DON'T do +- Don't wrap stdlib behind Protocol classes — `os.path.exists()` doesn't need + a `FileSystem` abstraction; use `unittest.mock.patch` for testing +- Don't introduce DI frameworks or service locators +- Don't rewrite working code just to match a pattern +- Don't create abstractions without a concrete need (real polymorphism or + testability problem that `mock.patch` can't solve cleanly) + +### Testing +- Use `unittest.mock.patch` for mocking stdlib calls +- Use `tmp_path` / `tempfile` for filesystem tests +- Use `monkeypatch` for environment variables +- Protocols and constructor injection where the code genuinely benefits + (e.g. testing submission logic without a real scheduler) --- ## Key Constraints +### Python 3.9+ +All code must target Python 3.9 as the minimum supported version. Use +`typing.Protocol` (available since 3.8), `typing.Union`, `typing.Optional`, etc. +Do not use syntax that requires 3.10+ (e.g. `X | Y` union syntax, +`match`/`case` statements). + ### Non-Installed Package -CIME is not installed via pip. Cases create symlinked tools that modify `sys.path`. This must continue to work. +CIME is not installed via pip. Cases create symlinked tools that modify +`sys.path`. This must continue to work. **Solution**: Centralized bootstrap in `CIME/core/config/bootstrap.py` +### Missing Package Structure +`CIME/Tools/xmlconvertors/` and `CIME/ParamGen/` lack `__init__.py` files +and use try/except import fallbacks. Fix during Slice 1. + ### External Model Compatibility E3SM, CESM, NorESM rely on CIME. Changes must not break their workflows. **Validation**: Test representative workflows from each model after major changes. ### Stable Compatibility Surfaces -- `CIME/api/` - User-facing classes -- `CIME/build_scripts/` - External script entrypoints +- `CIME/build_scripts/` — External script entrypoints - Case-created symlinked tools -- Import paths (until coordinated migration) +- Import paths (re-export from old locations during transition) --- ## Migration Slices ### Slice 1: Foundation (Weeks 1-3) -Core abstractions: FileSystem, ProcessRunner, Environment, Clock, Exceptions, Bootstrap +Typed exceptions, centralized bootstrap. Groundwork only. ### Slice 2: Batch (Weeks 4-7) -Extract batch/submit to `CIME/core/batch/`, scheduler abstractions +Move batch/submit logic from `case_submit.py` and XML classes into +`CIME/core/batch/`. Existing entrypoints become thin wrappers. ### Slice 3A: SRCROOT Standardization (Weeks 8-11) **NEW FEATURE** -Remove config_files.xml, standardize SRCROOT resolution, direct config loading +Remove config_files.xml, standardize SRCROOT resolution, direct config loading. ### Slice 3B: Build (Weeks 12-16) -Extract build to `CIME/core/build/`, keep `build_scripts` as wrapper +Move build logic from `build.py` into `CIME/core/build/`. Keep +`build_scripts/` as stable wrappers. ### Slice 4: Case (Weeks 17-22) -Extract Case internals (status, XML, locking) to focused components +Decompose the `Case` god object (2600 lines). Phase 1: consolidate scattered +free functions (from `utils.py`, `build.py`, `status.py`, `locked_files.py`, +`case_run.py`) back into Case. Phase 2: extract subsystems (status, locking, +XML storage) into focused modules that Case delegates to. Case ends up smaller +than today with clear internal boundaries. --- ## Success Criteria 1. **Compatibility**: External models work without modification (or with documented migration) -2. **Testability**: 80%+ coverage, isolated unit tests -3. **Resiliency**: Better error handling and recovery -4. **Maintainability**: Clear boundaries, reduced coupling -5. **Documentation**: Complete docs for new patterns +2. **Testability**: 80%+ coverage for reorganized code +3. **Maintainability**: No module over ~500 lines; clear boundaries +4. **Documentation**: Updated docs for reorganized structure --- @@ -148,12 +316,15 @@ Each slice must pass: ## Non-Goals -This refactor does NOT require: -- Immediate installed-package conversion -- Immediate CLI layer -- Removal of symlinked tools -- Wholesale rewrite in one step -- Breaking changes to Case API or build_scripts +This refactor does NOT: +- Wrap stdlib in abstraction layers (`os.path`, `subprocess`, etc.) +- Introduce heavyweight DI frameworks or service locators +- Convert CIME to an installed package (yet) +- Rewrite working code for aesthetics +- Break Case API or build_scripts + +DI, protocols, and factory functions ARE used where CIME has genuine +polymorphism (scheduler backends, config loaders, etc.). --- From 097031e4d56e478f7d3b730639641f9e3c804a70 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Sat, 25 Apr 2026 02:26:02 -0700 Subject: [PATCH 3/7] feat: add CIME/core/ package with typed exception hierarchy and bootstrap module Add CIME.core.exceptions with CIMEError and typed subclasses (ConfigurationError, BuildError, SubmitError, etc.). CIMEError preserves dual SystemExit+Exception inheritance for traceback suppression. Add CIME.core.config.bootstrap with centralized sys.path management (bootstrap_cime, find_cimeroot, check_minimum_python_version). Include 40 unit tests with 98% coverage. --- CIME/core/__init__.py | 7 ++ CIME/core/config/__init__.py | 1 + CIME/core/config/bootstrap.py | 152 ++++++++++++++++++++++++ CIME/core/exceptions.py | 55 +++++++++ CIME/tests/test_unit_core_bootstrap.py | 137 +++++++++++++++++++++ CIME/tests/test_unit_core_exceptions.py | 97 +++++++++++++++ 6 files changed, 449 insertions(+) create mode 100644 CIME/core/__init__.py create mode 100644 CIME/core/config/__init__.py create mode 100644 CIME/core/config/bootstrap.py create mode 100644 CIME/core/exceptions.py create mode 100644 CIME/tests/test_unit_core_bootstrap.py create mode 100644 CIME/tests/test_unit_core_exceptions.py diff --git a/CIME/core/__init__.py b/CIME/core/__init__.py new file mode 100644 index 00000000000..14d792d5027 --- /dev/null +++ b/CIME/core/__init__.py @@ -0,0 +1,7 @@ +""" +CIME core package — DI-based abstractions for filesystem, process, environment, +clock, and configuration bootstrap. + +These protocols are designed for constructor injection so that business logic +can be tested with lightweight mocks instead of hitting real OS resources. +""" diff --git a/CIME/core/config/__init__.py b/CIME/core/config/__init__.py new file mode 100644 index 00000000000..ddadce17d5b --- /dev/null +++ b/CIME/core/config/__init__.py @@ -0,0 +1 @@ +"""Configuration bootstrap and loading utilities.""" diff --git a/CIME/core/config/bootstrap.py b/CIME/core/config/bootstrap.py new file mode 100644 index 00000000000..5b852791a8a --- /dev/null +++ b/CIME/core/config/bootstrap.py @@ -0,0 +1,152 @@ +""" +Centralized sys.path management for CIME. + +CIME is not pip-installed; it relies on sys.path manipulation so that +scripts executed via symlinks (e.g. case.setup, xmlchange) can find +the CIME package. This module consolidates the various sys.path.insert +patterns scattered across the codebase into one place. + +**This module is standalone for Slice 1** — existing call-sites are NOT +modified yet. Migration will happen incrementally in later slices. + +Typical usage (future, after wiring):: + + from CIME.core.config.bootstrap import bootstrap_cime + bootstrap_cime() # auto-detect CIMEROOT + bootstrap_cime(cimeroot="/explicit") # explicit root +""" + +import os +import sys +from typing import List, Optional, Sequence + + +def find_cimeroot(starting_dir: Optional[str] = None) -> str: + """Locate the CIME root directory. + + Resolution order: + 1. ``starting_dir`` argument (if provided) + 2. ``CIMEROOT`` environment variable + 3. Walk upward from this file to find the directory containing ``CIME/`` + + Returns: + Absolute path to the CIME root directory. + + Raises: + RuntimeError: If CIMEROOT cannot be determined. + """ + if starting_dir is not None: + resolved = os.path.abspath(starting_dir) + if _is_cimeroot(resolved): + return resolved + raise RuntimeError( + f"Specified directory is not a valid CIMEROOT: {resolved}" + ) + + env_root = os.environ.get("CIMEROOT") + if env_root and _is_cimeroot(env_root): + return os.path.abspath(env_root) + + # Walk up from this file: CIME/core/config/bootstrap.py -> CIME/core/config -> CIME/core -> CIME -> root + candidate = os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "..") + ) + if _is_cimeroot(candidate): + return candidate + + raise RuntimeError( + "Cannot determine CIMEROOT. Set the CIMEROOT environment variable " + "or pass cimeroot explicitly." + ) + + +def _is_cimeroot(path: str) -> bool: + """Check whether a directory looks like a valid CIMEROOT.""" + return os.path.isdir(os.path.join(path, "CIME")) + + +def bootstrap_cime( + cimeroot: Optional[str] = None, + extra_paths: Optional[Sequence[str]] = None, + set_env: bool = True, +) -> str: + """Set up sys.path for CIME and return the resolved CIMEROOT. + + This is the single function that should be called to prepare the + Python environment for CIME imports. It: + + 1. Resolves CIMEROOT + 2. Inserts CIMEROOT at the front of sys.path (if not already present) + 3. Inserts CIME/Tools at position 1 (if not already present) + 4. Inserts any extra_paths + 5. Optionally sets the CIMEROOT environment variable + + Args: + cimeroot: Explicit CIMEROOT path. If None, auto-detected. + extra_paths: Additional paths to insert after CIMEROOT and Tools. + set_env: Whether to set ``os.environ["CIMEROOT"]``. + + Returns: + The resolved CIMEROOT path. + """ + root = find_cimeroot(cimeroot) + tools_path = os.path.join(root, "CIME", "Tools") + + # Build the list of paths to ensure are in sys.path + paths_to_add: List[str] = [root, tools_path] + if extra_paths: + paths_to_add.extend(str(p) for p in extra_paths) + + _ensure_paths(paths_to_add) + + if set_env: + os.environ["CIMEROOT"] = root + + return root + + +def _ensure_paths(paths: Sequence[str]) -> None: + """Ensure each path is in sys.path, inserted at the front in order. + + Paths already present are moved to the correct position rather than + duplicated. + """ + for i, p in enumerate(paths): + absp = os.path.abspath(p) + # Remove existing entry if present (to re-position) + if absp in sys.path: + sys.path.remove(absp) + sys.path.insert(i, absp) + + +def get_tools_path(cimeroot: Optional[str] = None) -> str: + """Return the CIME/Tools directory path.""" + root = cimeroot or find_cimeroot() + return os.path.join(root, "CIME", "Tools") + + +def check_minimum_python_version( + major: int = 3, minor: int = 9, warn_only: bool = False +) -> None: + """Check that the running Python meets minimum version requirements. + + Consolidated from CIME/Tools/standard_script_setup.py. + + Args: + major: Required major version. + minor: Required minimum minor version. + warn_only: If True, print warning instead of raising. + + Raises: + RuntimeError: If version is insufficient and warn_only is False. + """ + if sys.version_info >= (major, minor): + return + msg = ( + f"Python {major}.{minor} is {'recommended' if warn_only else 'required'} " + f"to run CIME. You have {sys.version_info[0]}.{sys.version_info[1]}" + ) + if warn_only: + print(msg + ".", file=sys.stderr) + return + raise RuntimeError(msg + " - please use a newer version of Python.") diff --git a/CIME/core/exceptions.py b/CIME/core/exceptions.py new file mode 100644 index 00000000000..725b735f7cf --- /dev/null +++ b/CIME/core/exceptions.py @@ -0,0 +1,55 @@ +""" +Typed exception hierarchy for CIME. + +CIMEError inherits from both SystemExit and Exception so that: +- Tracebacks are suppressed in normal usage (SystemExit behavior) +- Exceptions are still catchable (Exception behavior) +- Users can run with --debug to see full tracebacks + +All CIME-specific exceptions should inherit from CIMEError. +""" + + +class CIMEError(SystemExit, Exception): + """Base exception for all CIME errors. + + Inherits from SystemExit to suppress tracebacks in normal usage, + and from Exception to remain catchable. + """ + + +class ConfigurationError(CIMEError): + """Invalid or missing configuration (XML files, env vars, etc.).""" + + +class BuildError(CIMEError): + """Failure during model build.""" + + +class SubmitError(CIMEError): + """Failure during job submission.""" + + +class SystemTestError(CIMEError): + """Failure in a CIME system test.""" + + +class LockingError(CIMEError): + """Case locking or unlocking failure.""" + + +class ExternalCommandError(CIMEError): + """An external command (subprocess) failed.""" + + def __init__(self, message: str, returncode: int = 1, command: str = ""): + super().__init__(message) + self.returncode = returncode + self.command = command + + +class RetryableExternalCommandError(ExternalCommandError): + """An external command failed but may succeed on retry (transient error).""" + + +class InputError(CIMEError): + """Invalid user input (bad arguments, missing required values).""" diff --git a/CIME/tests/test_unit_core_bootstrap.py b/CIME/tests/test_unit_core_bootstrap.py new file mode 100644 index 00000000000..fdbc2d3b1ce --- /dev/null +++ b/CIME/tests/test_unit_core_bootstrap.py @@ -0,0 +1,137 @@ +"""Unit tests for CIME.core.config.bootstrap.""" + +import os +import sys + +import pytest + +from CIME.core.config.bootstrap import ( + _ensure_paths, + _is_cimeroot, + bootstrap_cime, + check_minimum_python_version, + find_cimeroot, + get_tools_path, +) + + +class TestFindCimeroot: + def test_explicit_dir(self, tmp_path): + """find_cimeroot accepts an explicit directory with a CIME/ subdir.""" + (tmp_path / "CIME").mkdir() + root = find_cimeroot(str(tmp_path)) + assert root == str(tmp_path.resolve()) + + def test_explicit_dir_invalid(self, tmp_path): + with pytest.raises(RuntimeError, match="not a valid CIMEROOT"): + find_cimeroot(str(tmp_path)) + + def test_env_var(self, tmp_path, monkeypatch): + (tmp_path / "CIME").mkdir() + monkeypatch.setenv("CIMEROOT", str(tmp_path)) + root = find_cimeroot() + assert root == str(tmp_path.resolve()) + + def test_walks_up_from_file(self): + """The default detection should find the real CIMEROOT from this repo.""" + root = find_cimeroot() + assert os.path.isdir(os.path.join(root, "CIME")) + + +class TestIsCimeroot: + def test_valid(self, tmp_path): + (tmp_path / "CIME").mkdir() + assert _is_cimeroot(str(tmp_path)) is True + + def test_invalid(self, tmp_path): + assert _is_cimeroot(str(tmp_path)) is False + + +class TestBootstrapCime: + def setup_method(self): + self._orig_path = sys.path[:] + self._orig_env = os.environ.get("CIMEROOT") + + def teardown_method(self): + sys.path[:] = self._orig_path + if self._orig_env is not None: + os.environ["CIMEROOT"] = self._orig_env + else: + os.environ.pop("CIMEROOT", None) + + def test_bootstrap_sets_sys_path(self, tmp_path): + (tmp_path / "CIME" / "Tools").mkdir(parents=True) + root = bootstrap_cime(cimeroot=str(tmp_path)) + assert root == str(tmp_path.resolve()) + assert str(tmp_path.resolve()) in sys.path + tools = os.path.join(str(tmp_path.resolve()), "CIME", "Tools") + assert tools in sys.path + + def test_bootstrap_sets_env(self, tmp_path): + (tmp_path / "CIME" / "Tools").mkdir(parents=True) + bootstrap_cime(cimeroot=str(tmp_path)) + assert os.environ["CIMEROOT"] == str(tmp_path.resolve()) + + def test_bootstrap_no_env(self, tmp_path): + (tmp_path / "CIME" / "Tools").mkdir(parents=True) + os.environ.pop("CIMEROOT", None) + bootstrap_cime(cimeroot=str(tmp_path), set_env=False) + assert "CIMEROOT" not in os.environ + + def test_extra_paths(self, tmp_path): + (tmp_path / "CIME" / "Tools").mkdir(parents=True) + extra = tmp_path / "extras" + extra.mkdir() + bootstrap_cime(cimeroot=str(tmp_path), extra_paths=[str(extra)]) + assert str(extra.resolve()) in sys.path + + def test_no_duplicate_paths(self, tmp_path): + (tmp_path / "CIME" / "Tools").mkdir(parents=True) + bootstrap_cime(cimeroot=str(tmp_path)) + bootstrap_cime(cimeroot=str(tmp_path)) + root_str = str(tmp_path.resolve()) + assert sys.path.count(root_str) == 1 + + +class TestEnsurePaths: + def setup_method(self): + self._orig_path = sys.path[:] + + def teardown_method(self): + sys.path[:] = self._orig_path + + def test_inserts_in_order(self, tmp_path): + a = str(tmp_path / "a") + b = str(tmp_path / "b") + _ensure_paths([a, b]) + assert sys.path[0] == a + assert sys.path[1] == b + + def test_moves_existing_entry(self, tmp_path): + p = str(tmp_path / "x") + sys.path.append(p) + _ensure_paths([p]) + assert sys.path[0] == p + assert sys.path.count(p) == 1 + + +class TestGetToolsPath: + def test_returns_tools_dir(self, tmp_path): + (tmp_path / "CIME" / "Tools").mkdir(parents=True) + tools = get_tools_path(cimeroot=str(tmp_path)) + assert tools == os.path.join(str(tmp_path), "CIME", "Tools") + + +class TestCheckMinimumPythonVersion: + def test_passes_current_version(self): + # Current Python is >= 3.9 (required by CIME) + check_minimum_python_version(3, 9) + + def test_fails_future_version(self): + with pytest.raises(RuntimeError, match="required"): + check_minimum_python_version(99, 0) + + def test_warn_only(self, capsys): + check_minimum_python_version(99, 0, warn_only=True) + captured = capsys.readouterr() + assert "recommended" in captured.err diff --git a/CIME/tests/test_unit_core_exceptions.py b/CIME/tests/test_unit_core_exceptions.py new file mode 100644 index 00000000000..be2554f6d42 --- /dev/null +++ b/CIME/tests/test_unit_core_exceptions.py @@ -0,0 +1,97 @@ +"""Unit tests for CIME.core.exceptions.""" + +import pytest + +from CIME.core.exceptions import ( + BuildError, + CIMEError, + ConfigurationError, + ExternalCommandError, + InputError, + LockingError, + RetryableExternalCommandError, + SubmitError, + SystemTestError, +) + + +class TestCIMEError: + """Tests for the base CIMEError exception.""" + + def test_inherits_system_exit(self): + assert issubclass(CIMEError, SystemExit) + + def test_inherits_exception(self): + assert issubclass(CIMEError, Exception) + + def test_catchable_as_exception(self): + with pytest.raises(Exception): + raise CIMEError("test") + + def test_catchable_as_system_exit(self): + with pytest.raises(SystemExit): + raise CIMEError("test") + + def test_message_preserved(self): + try: + raise CIMEError("my message") + except CIMEError as exc: + assert str(exc) == "my message" + + +class TestSubclasses: + """All typed exceptions inherit from CIMEError.""" + + @pytest.mark.parametrize( + "exc_class", + [ + ConfigurationError, + BuildError, + SubmitError, + SystemTestError, + LockingError, + ExternalCommandError, + RetryableExternalCommandError, + InputError, + ], + ) + def test_is_cime_error(self, exc_class): + assert issubclass(exc_class, CIMEError) + + @pytest.mark.parametrize( + "exc_class", + [ + ConfigurationError, + BuildError, + SubmitError, + SystemTestError, + LockingError, + InputError, + ], + ) + def test_simple_subclass_message(self, exc_class): + with pytest.raises(exc_class, match="oops"): + raise exc_class("oops") + + +class TestExternalCommandError: + """Tests for ExternalCommandError and its retryable variant.""" + + def test_stores_returncode_and_command(self): + exc = ExternalCommandError("failed", returncode=42, command="make") + assert exc.returncode == 42 + assert exc.command == "make" + assert str(exc) == "failed" + + def test_defaults(self): + exc = ExternalCommandError("failed") + assert exc.returncode == 1 + assert exc.command == "" + + def test_retryable_is_external_command_error(self): + assert issubclass(RetryableExternalCommandError, ExternalCommandError) + + def test_retryable_stores_fields(self): + exc = RetryableExternalCommandError("timeout", returncode=124, command="srun") + assert exc.returncode == 124 + assert exc.command == "srun" From 45c19b765c0d388a54d25161f384e2df7dfe70cd Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Sat, 25 Apr 2026 02:26:09 -0700 Subject: [PATCH 4/7] refactor: migrate CIMEError imports to CIME.core.exceptions Replace the CIMEError class definition in utils.py with a re-export from CIME.core.exceptions for backward compatibility. Update 19 internal callers to import CIMEError directly from CIME.core.exceptions. External consumers using 'from CIME.utils import CIMEError' or 'utils.CIMEError' continue to work unchanged via the re-export. --- CIME/SystemTests/test_mods.py | 2 +- CIME/XML/compsets.py | 2 +- CIME/XML/machines.py | 3 ++- CIME/XML/namelist_definition.py | 4 +--- CIME/XML/tests.py | 4 +++- CIME/case/case_run.py | 3 ++- CIME/case/case_submit.py | 16 +++++++--------- CIME/compare_namelists.py | 4 +++- CIME/hist_utils.py | 3 ++- CIME/tests/test_unit_bless_test_results.py | 2 +- CIME/tests/test_unit_case_run.py | 2 +- CIME/tests/test_unit_expected_fails_file.py | 2 +- CIME/tests/test_unit_fake_case.py | 3 ++- CIME/tests/test_unit_grids.py | 8 +++----- CIME/tests/test_unit_locked_files.py | 2 +- CIME/tests/test_unit_user_mod_support.py | 2 +- CIME/tests/test_unit_xml_env_batch.py | 15 ++++++--------- CIME/tests/test_unit_xml_grids.py | 3 +-- CIME/utils.py | 6 ++++-- CIME/wait_for_tests.py | 4 +++- 20 files changed, 46 insertions(+), 44 deletions(-) diff --git a/CIME/SystemTests/test_mods.py b/CIME/SystemTests/test_mods.py index 2db22c5833a..d7f010f1d86 100644 --- a/CIME/SystemTests/test_mods.py +++ b/CIME/SystemTests/test_mods.py @@ -1,7 +1,7 @@ import logging import os -from CIME.utils import CIMEError +from CIME.core.exceptions import CIMEError from CIME.XML.files import Files logger = logging.getLogger(__name__) diff --git a/CIME/XML/compsets.py b/CIME/XML/compsets.py index eef9253da0f..70241ec2e98 100644 --- a/CIME/XML/compsets.py +++ b/CIME/XML/compsets.py @@ -6,7 +6,7 @@ from CIME.XML.generic_xml import GenericXML from CIME.XML.entry_id import EntryID from CIME.XML.files import Files -from CIME.utils import CIMEError +from CIME.core.exceptions import CIMEError logger = logging.getLogger(__name__) diff --git a/CIME/XML/machines.py b/CIME/XML/machines.py index c40e73ba3b6..eabd6a8f9ef 100644 --- a/CIME/XML/machines.py +++ b/CIME/XML/machines.py @@ -5,7 +5,8 @@ from CIME.XML.standard_module_setup import * from CIME.XML.generic_xml import GenericXML from CIME.XML.files import Files -from CIME.utils import CIMEError, expect, convert_to_unknown_type, get_cime_config +from CIME.core.exceptions import CIMEError +from CIME.utils import expect, convert_to_unknown_type, get_cime_config import re import logging diff --git a/CIME/XML/namelist_definition.py b/CIME/XML/namelist_definition.py index 25f7c0e0432..38a184d1019 100644 --- a/CIME/XML/namelist_definition.py +++ b/CIME/XML/namelist_definition.py @@ -21,7 +21,7 @@ Namelist, get_fortran_name_only, ) -from CIME.utils import CIMEError +from CIME.core.exceptions import CIMEError from CIME.XML.standard_module_setup import * from CIME.XML.entry_id import EntryID from CIME.XML.files import Files @@ -32,7 +32,6 @@ class CaseInsensitiveDict(dict): - """Basic case insensitive dict with strings only keys. From https://stackoverflow.com/a/27890005""" @@ -65,7 +64,6 @@ def __setitem__(self, k, v): class NamelistDefinition(EntryID): - """Class representing variable definitions for a namelist. This class inherits from `EntryID`, and supports most inherited methods; however, `set_value` is unsupported. diff --git a/CIME/XML/tests.py b/CIME/XML/tests.py index e760d86604f..92feaca91c8 100644 --- a/CIME/XML/tests.py +++ b/CIME/XML/tests.py @@ -1,11 +1,13 @@ """ Interface to the config_tests.xml file. This class inherits from GenericEntry """ + from CIME.XML.standard_module_setup import * from CIME.XML.generic_xml import GenericXML from CIME.XML.files import Files -from CIME.utils import find_system_test, CIMEError +from CIME.core.exceptions import CIMEError +from CIME.utils import find_system_test from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.SystemTests.system_tests_compare_n import SystemTestsCompareN diff --git a/CIME/case/case_run.py b/CIME/case/case_run.py index b9912589e5f..5c44c693b14 100644 --- a/CIME/case/case_run.py +++ b/CIME/case/case_run.py @@ -5,7 +5,8 @@ from CIME.XML.standard_module_setup import * from CIME.config import Config from CIME.utils import gzip_existing_file, new_lid -from CIME.utils import run_sub_or_cmd, safe_copy, model_log, CIMEError +from CIME.core.exceptions import CIMEError +from CIME.utils import run_sub_or_cmd, safe_copy, model_log from CIME.utils import batch_jobid, is_comp_standalone from CIME.status import append_status, run_and_log_case_status from CIME.get_timing import get_timing diff --git a/CIME/case/case_submit.py b/CIME/case/case_submit.py index 9251e4ab092..8b02328dfd6 100644 --- a/CIME/case/case_submit.py +++ b/CIME/case/case_submit.py @@ -6,9 +6,11 @@ jobs. submit, check_case and check_da_settings are members of class Case in file case.py """ + import configparser from CIME.XML.standard_module_setup import * -from CIME.utils import expect, CIMEError, get_time_in_seconds +from CIME.core.exceptions import CIMEError +from CIME.utils import expect, get_time_in_seconds from CIME.status import run_and_log_case_status from CIME.locked_files import ( unlock_file, @@ -117,12 +119,10 @@ def _submit( if batch_system != "none" and env_batch_has_changed and not external_workflow: # May need to regen batch files if user made batch setting changes (e.g. walltime, queue, etc) - logger.warning( - """ + logger.warning(""" env_batch.xml appears to have changed, regenerating batch scripts manual edits to these file will be lost! -""" - ) +""") env_batch.make_all_batch_files(case) case.flush() @@ -159,12 +159,10 @@ def _submit( if env_batch.get_batch_system_type() != "none" and env_batch_has_changed: # May need to regen batch files if user made batch setting changes (e.g. walltime, queue, etc) - logger.warning( - """ + logger.warning(""" env_batch.xml appears to have changed, regenerating batch scripts manual edits to these file will be lost! -""" - ) +""") env_batch.make_all_batch_files(case) unlock_file(os.path.basename(env_batch.filename), caseroot) diff --git a/CIME/compare_namelists.py b/CIME/compare_namelists.py index 9fdc005e3cc..6e98a7df9a5 100644 --- a/CIME/compare_namelists.py +++ b/CIME/compare_namelists.py @@ -1,10 +1,12 @@ import os, re, logging -from CIME.utils import expect, CIMEError +from CIME.core.exceptions import CIMEError +from CIME.utils import expect logger = logging.getLogger(__name__) # pragma pylint: disable=unsubscriptable-object + ############################################################################### def _normalize_lists(value_str): ############################################################################### diff --git a/CIME/hist_utils.py b/CIME/hist_utils.py index fa48f942c56..ab8b387e853 100644 --- a/CIME/hist_utils.py +++ b/CIME/hist_utils.py @@ -1,6 +1,7 @@ """ Functions for actions pertaining to history files. """ + import logging import os import re @@ -17,7 +18,7 @@ SharedArea, parse_test_name, ) -from CIME.utils import CIMEError +from CIME.core.exceptions import CIMEError logger = logging.getLogger(__name__) diff --git a/CIME/tests/test_unit_bless_test_results.py b/CIME/tests/test_unit_bless_test_results.py index 3138dd75c77..058c10caab8 100644 --- a/CIME/tests/test_unit_bless_test_results.py +++ b/CIME/tests/test_unit_bless_test_results.py @@ -15,7 +15,7 @@ ) from CIME.test_status import ALL_PHASES, GENERATE_PHASE from CIME.tests import utils as test_utils -from CIME.utils import CIMEError +from CIME.core.exceptions import CIMEError class TestUnitBlessTestResults(unittest.TestCase): diff --git a/CIME/tests/test_unit_case_run.py b/CIME/tests/test_unit_case_run.py index 88c746d0d8c..ce74f675366 100644 --- a/CIME/tests/test_unit_case_run.py +++ b/CIME/tests/test_unit_case_run.py @@ -1,7 +1,7 @@ import unittest from unittest import mock -from CIME.utils import CIMEError +from CIME.core.exceptions import CIMEError from CIME.case.case_run import TERMINATION_TEXT from CIME.case.case_run import _post_run_check diff --git a/CIME/tests/test_unit_expected_fails_file.py b/CIME/tests/test_unit_expected_fails_file.py index 1e0e5878e2b..30c9dd3a6e8 100755 --- a/CIME/tests/test_unit_expected_fails_file.py +++ b/CIME/tests/test_unit_expected_fails_file.py @@ -5,7 +5,7 @@ import shutil import tempfile from CIME.XML.expected_fails_file import ExpectedFailsFile -from CIME.utils import CIMEError +from CIME.core.exceptions import CIMEError from CIME.expected_fails import ExpectedFails diff --git a/CIME/tests/test_unit_fake_case.py b/CIME/tests/test_unit_fake_case.py index c4cebebcd2c..b7b20ac6abc 100755 --- a/CIME/tests/test_unit_fake_case.py +++ b/CIME/tests/test_unit_fake_case.py @@ -7,7 +7,8 @@ import unittest import os -from CIME.utils import get_model, CIMEError, GLOBAL, get_src_root +from CIME.core.exceptions import CIMEError +from CIME.utils import get_model, GLOBAL, get_src_root from CIME.BuildTools.configure import FakeCase diff --git a/CIME/tests/test_unit_grids.py b/CIME/tests/test_unit_grids.py index 8417177e83e..01afaa4f73f 100755 --- a/CIME/tests/test_unit_grids.py +++ b/CIME/tests/test_unit_grids.py @@ -19,7 +19,7 @@ import string import tempfile from CIME.XML.grids import Grids, _ComponentGrids, _add_grid_info, _strip_grid_from_name -from CIME.utils import CIMEError +from CIME.core.exceptions import CIMEError class TestGrids(unittest.TestCase): @@ -28,8 +28,7 @@ class TestGrids(unittest.TestCase): Note that much of the functionality of CIME.XML.grids is NOT covered here """ - _CONFIG_GRIDS_TEMPLATE = string.Template( - """ + _CONFIG_GRIDS_TEMPLATE = string.Template(""" @@ -70,8 +69,7 @@ class TestGrids(unittest.TestCase): $GRIDMAP_ENTRIES -""" - ) +""") _MODEL_GRID_F09_G17 = """ diff --git a/CIME/tests/test_unit_locked_files.py b/CIME/tests/test_unit_locked_files.py index fc871e3b768..2732f16f3f0 100644 --- a/CIME/tests/test_unit_locked_files.py +++ b/CIME/tests/test_unit_locked_files.py @@ -4,7 +4,7 @@ from pathlib import Path from CIME import locked_files -from CIME.utils import CIMEError +from CIME.core.exceptions import CIMEError from CIME.XML.entry_id import EntryID from CIME.XML.env_batch import EnvBatch from CIME.XML.files import Files diff --git a/CIME/tests/test_unit_user_mod_support.py b/CIME/tests/test_unit_user_mod_support.py index 51ffb4778ca..c2d0fc37e13 100755 --- a/CIME/tests/test_unit_user_mod_support.py +++ b/CIME/tests/test_unit_user_mod_support.py @@ -5,7 +5,7 @@ import tempfile import os from CIME.user_mod_support import apply_user_mods -from CIME.utils import CIMEError +from CIME.core.exceptions import CIMEError # ======================================================================== # Define some parameters diff --git a/CIME/tests/test_unit_xml_env_batch.py b/CIME/tests/test_unit_xml_env_batch.py index 3599cfe1dfc..1f3ebc15a8c 100755 --- a/CIME/tests/test_unit_xml_env_batch.py +++ b/CIME/tests/test_unit_xml_env_batch.py @@ -6,7 +6,8 @@ from contextlib import ExitStack from unittest import mock -from CIME.utils import CIMEError, expect +from CIME.core.exceptions import CIMEError +from CIME.utils import expect from CIME.XML.env_batch import EnvBatch, get_job_deps from CIME.XML.env_workflow import EnvWorkflow from CIME.BuildTools.configure import FakeCase @@ -513,8 +514,7 @@ def test_get_job_deps(self): def test_get_submit_args_job_queue(self): with tempfile.NamedTemporaryFile() as tfile: - tfile.write( - b""" + tfile.write(b"""
These variables may be changed anytime during a run, they @@ -547,8 +547,7 @@ def test_get_submit_args_job_queue(self): -""" - ) +""") tfile.seek(0) @@ -570,8 +569,7 @@ def test_get_submit_args_job_queue(self): @mock.patch.dict(os.environ, {"TEST": "GOOD"}) def test_get_submit_args(self): with tempfile.NamedTemporaryFile() as tfile: - tfile.write( - b""" + tfile.write(b"""
These variables may be changed anytime during a run, they @@ -628,8 +626,7 @@ def test_get_submit_args(self): -""" - ) +""") tfile.seek(0) diff --git a/CIME/tests/test_unit_xml_grids.py b/CIME/tests/test_unit_xml_grids.py index 17c01e355de..cceae5b419d 100644 --- a/CIME/tests/test_unit_xml_grids.py +++ b/CIME/tests/test_unit_xml_grids.py @@ -6,10 +6,9 @@ from pathlib import Path from unittest import mock -from CIME.utils import CIMEError +from CIME.core.exceptions import CIMEError from CIME.XML.grids import Grids - TEST_CONFIG = """ diff --git a/CIME/utils.py b/CIME/utils.py index e4a49c4457e..948632b4068 100644 --- a/CIME/utils.py +++ b/CIME/utils.py @@ -152,8 +152,10 @@ def __exit__(self, *args): # hate seeing. It's a subclass of Exception because we want it to be # "catchable". If you are debugging CIME and want to see the stacktrace, # run your CIME command with the --debug flag. -class CIMEError(SystemExit, Exception): - pass +# +# Canonical definition lives in CIME.core.exceptions; re-exported here +# for backward compatibility. +from CIME.core.exceptions import CIMEError # noqa: F401 def expect(condition, error_msg, exc_type=CIMEError, error_prefix="ERROR:"): diff --git a/CIME/wait_for_tests.py b/CIME/wait_for_tests.py index 1f6940138a9..a32278ed052 100644 --- a/CIME/wait_for_tests.py +++ b/CIME/wait_for_tests.py @@ -7,7 +7,8 @@ import xml.etree.ElementTree as xmlet import CIME.utils -from CIME.utils import expect, Timeout, run_cmd, run_cmd_no_fail, safe_copy, CIMEError +from CIME.core.exceptions import CIMEError +from CIME.utils import expect, Timeout, run_cmd, run_cmd_no_fail, safe_copy from CIME.XML.machines import Machines from CIME.test_status import * from CIME.provenance import save_test_success @@ -19,6 +20,7 @@ SLEEP_INTERVAL_SEC = 0.1 ENV_VAR_KEEP_CDASH = "CIME_TEST_CDASH_WFT" + ############################################################################### def signal_handler(*_): ############################################################################### From 32864782913d2700f5dcf612ac4af3f8aa6571fa Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 26 May 2026 14:12:52 -0700 Subject: [PATCH 5/7] fix: update exception and add tests --- CIME/case/case_submit.py | 12 +++++--- CIME/core/config/bootstrap.py | 4 +-- CIME/core/exceptions.py | 32 ++++++++++++++++++-- CIME/tests/test_unit_core_exceptions.py | 39 +++++++++++++++++++++---- CIME/tests/test_unit_grids.py | 6 ++-- CIME/tests/test_unit_xml_env_batch.py | 12 +++++--- 6 files changed, 83 insertions(+), 22 deletions(-) diff --git a/CIME/case/case_submit.py b/CIME/case/case_submit.py index 8b02328dfd6..6e2e7bc6278 100644 --- a/CIME/case/case_submit.py +++ b/CIME/case/case_submit.py @@ -119,10 +119,12 @@ def _submit( if batch_system != "none" and env_batch_has_changed and not external_workflow: # May need to regen batch files if user made batch setting changes (e.g. walltime, queue, etc) - logger.warning(""" + logger.warning( + """ env_batch.xml appears to have changed, regenerating batch scripts manual edits to these file will be lost! -""") +""" + ) env_batch.make_all_batch_files(case) case.flush() @@ -159,10 +161,12 @@ def _submit( if env_batch.get_batch_system_type() != "none" and env_batch_has_changed: # May need to regen batch files if user made batch setting changes (e.g. walltime, queue, etc) - logger.warning(""" + logger.warning( + """ env_batch.xml appears to have changed, regenerating batch scripts manual edits to these file will be lost! -""") +""" + ) env_batch.make_all_batch_files(case) unlock_file(os.path.basename(env_batch.filename), caseroot) diff --git a/CIME/core/config/bootstrap.py b/CIME/core/config/bootstrap.py index 5b852791a8a..e8d2efbaeb8 100644 --- a/CIME/core/config/bootstrap.py +++ b/CIME/core/config/bootstrap.py @@ -39,9 +39,7 @@ def find_cimeroot(starting_dir: Optional[str] = None) -> str: resolved = os.path.abspath(starting_dir) if _is_cimeroot(resolved): return resolved - raise RuntimeError( - f"Specified directory is not a valid CIMEROOT: {resolved}" - ) + raise RuntimeError(f"Specified directory is not a valid CIMEROOT: {resolved}") env_root = os.environ.get("CIMEROOT") if env_root and _is_cimeroot(env_root): diff --git a/CIME/core/exceptions.py b/CIME/core/exceptions.py index 725b735f7cf..c8f69c59e4f 100644 --- a/CIME/core/exceptions.py +++ b/CIME/core/exceptions.py @@ -39,12 +39,38 @@ class LockingError(CIMEError): class ExternalCommandError(CIMEError): - """An external command (subprocess) failed.""" + """An external command (subprocess) failed. - def __init__(self, message: str, returncode: int = 1, command: str = ""): + Modeled after subprocess.CalledProcessError for consistency. + + Attributes: + returncode: Exit code of the command. + cmd: The command that was run (string or list). + output: Standard output captured from the command, if any. + stderr: Standard error captured from the command, if any. + """ + + def __init__( + self, + message: str, + returncode: int = 1, + cmd: str = "", + output: str = "", + stderr: str = "", + # Deprecated: use cmd instead + command: str = "", + ): super().__init__(message) self.returncode = returncode - self.command = command + # Support both 'cmd' (preferred) and 'command' (deprecated) for compatibility + self.cmd = cmd or command + self.output = output + self.stderr = stderr + + @property + def command(self) -> str: + """Deprecated: Use cmd instead.""" + return self.cmd class RetryableExternalCommandError(ExternalCommandError): diff --git a/CIME/tests/test_unit_core_exceptions.py b/CIME/tests/test_unit_core_exceptions.py index be2554f6d42..79c5907d812 100644 --- a/CIME/tests/test_unit_core_exceptions.py +++ b/CIME/tests/test_unit_core_exceptions.py @@ -77,21 +77,48 @@ def test_simple_subclass_message(self, exc_class): class TestExternalCommandError: """Tests for ExternalCommandError and its retryable variant.""" - def test_stores_returncode_and_command(self): - exc = ExternalCommandError("failed", returncode=42, command="make") + def test_stores_returncode_and_cmd(self): + exc = ExternalCommandError("failed", returncode=42, cmd="make all") assert exc.returncode == 42 - assert exc.command == "make" + assert exc.cmd == "make all" assert str(exc) == "failed" + def test_stores_output_and_stderr(self): + exc = ExternalCommandError( + "failed", + returncode=1, + cmd="make", + output="Building...", + stderr="error: missing file", + ) + assert exc.output == "Building..." + assert exc.stderr == "error: missing file" + def test_defaults(self): exc = ExternalCommandError("failed") assert exc.returncode == 1 - assert exc.command == "" + assert exc.cmd == "" + assert exc.output == "" + assert exc.stderr == "" + + def test_command_property_deprecated(self): + """The command property is deprecated but still works.""" + exc = ExternalCommandError("failed", cmd="make") + assert exc.command == "make" # Deprecated property + assert exc.cmd == "make" # Preferred attribute + + def test_command_kwarg_deprecated(self): + """The command kwarg is deprecated but still works.""" + exc = ExternalCommandError("failed", command="make") + assert exc.cmd == "make" + assert exc.command == "make" def test_retryable_is_external_command_error(self): assert issubclass(RetryableExternalCommandError, ExternalCommandError) def test_retryable_stores_fields(self): - exc = RetryableExternalCommandError("timeout", returncode=124, command="srun") + exc = RetryableExternalCommandError( + "timeout", returncode=124, cmd="srun ./model" + ) assert exc.returncode == 124 - assert exc.command == "srun" + assert exc.cmd == "srun ./model" diff --git a/CIME/tests/test_unit_grids.py b/CIME/tests/test_unit_grids.py index 01afaa4f73f..e33349d52b5 100755 --- a/CIME/tests/test_unit_grids.py +++ b/CIME/tests/test_unit_grids.py @@ -28,7 +28,8 @@ class TestGrids(unittest.TestCase): Note that much of the functionality of CIME.XML.grids is NOT covered here """ - _CONFIG_GRIDS_TEMPLATE = string.Template(""" + _CONFIG_GRIDS_TEMPLATE = string.Template( + """ @@ -69,7 +70,8 @@ class TestGrids(unittest.TestCase): $GRIDMAP_ENTRIES -""") +""" + ) _MODEL_GRID_F09_G17 = """ diff --git a/CIME/tests/test_unit_xml_env_batch.py b/CIME/tests/test_unit_xml_env_batch.py index 1f3ebc15a8c..5b1295c4a68 100755 --- a/CIME/tests/test_unit_xml_env_batch.py +++ b/CIME/tests/test_unit_xml_env_batch.py @@ -514,7 +514,8 @@ def test_get_job_deps(self): def test_get_submit_args_job_queue(self): with tempfile.NamedTemporaryFile() as tfile: - tfile.write(b""" + tfile.write( + b"""
These variables may be changed anytime during a run, they @@ -547,7 +548,8 @@ def test_get_submit_args_job_queue(self): -""") +""" + ) tfile.seek(0) @@ -569,7 +571,8 @@ def test_get_submit_args_job_queue(self): @mock.patch.dict(os.environ, {"TEST": "GOOD"}) def test_get_submit_args(self): with tempfile.NamedTemporaryFile() as tfile: - tfile.write(b""" + tfile.write( + b"""
These variables may be changed anytime during a run, they @@ -626,7 +629,8 @@ def test_get_submit_args(self): -""") +""" + ) tfile.seek(0) From d665092bc40526ac624f17c10ba247e50b20f8e3 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 26 May 2026 14:54:16 -0700 Subject: [PATCH 6/7] fix: removes star import from standard_module_setup --- CIME/BuildTools/configure.py | 14 +-- CIME/Servers/ftp.py | 13 ++- CIME/Servers/generic_server.py | 3 +- CIME/Servers/gftp.py | 5 +- CIME/Servers/svn.py | 6 +- CIME/Servers/wget.py | 6 +- CIME/SystemTests/dae.py | 5 +- CIME/SystemTests/eri.py | 11 ++- CIME/SystemTests/erio.py | 5 +- CIME/SystemTests/erp.py | 3 +- CIME/SystemTests/err.py | 8 +- CIME/SystemTests/erri.py | 9 +- CIME/SystemTests/ers.py | 8 +- CIME/SystemTests/ert.py | 3 +- CIME/SystemTests/funit.py | 9 +- CIME/SystemTests/hommebaseclass.py | 18 ++-- CIME/SystemTests/icp.py | 2 +- CIME/SystemTests/irt.py | 6 +- CIME/SystemTests/ldsta.py | 11 +-- CIME/SystemTests/mcc.py | 4 +- CIME/SystemTests/mvk.py | 19 ++-- CIME/SystemTests/nck.py | 3 +- CIME/SystemTests/ncr.py | 4 +- CIME/SystemTests/nodefail.py | 5 +- CIME/SystemTests/pea.py | 4 +- CIME/SystemTests/pem.py | 3 +- CIME/SystemTests/pet.py | 3 +- CIME/SystemTests/pfs.py | 3 +- CIME/SystemTests/pgn.py | 17 ++-- CIME/SystemTests/pre.py | 7 +- CIME/SystemTests/restart_tests.py | 4 +- CIME/SystemTests/reuseinitfiles.py | 5 +- CIME/SystemTests/seq.py | 4 +- CIME/SystemTests/sms.py | 3 +- CIME/SystemTests/system_tests_common.py | 72 ++++++++------- CIME/SystemTests/system_tests_compare_n.py | 11 ++- CIME/SystemTests/system_tests_compare_two.py | 11 ++- CIME/SystemTests/test_utils/user_nl_utils.py | 2 +- CIME/SystemTests/tsc.py | 13 ++- CIME/Tools/archive_metadata | 9 +- CIME/Tools/bld_diff | 7 +- CIME/Tools/bless_test_results | 12 ++- CIME/Tools/case.build | 3 +- CIME/Tools/case.cmpgen_namelists | 4 +- CIME/Tools/case.qstatus | 1 + CIME/Tools/case.setup | 1 + CIME/Tools/case.submit | 3 + CIME/Tools/case_diff | 6 +- CIME/Tools/check_case | 6 +- CIME/Tools/check_input_data | 4 +- CIME/Tools/check_lockedfiles | 1 + CIME/Tools/cime_bisect | 8 +- CIME/Tools/code_checker | 12 +-- CIME/Tools/compare_namelists | 6 +- CIME/Tools/compare_test_results | 8 +- CIME/Tools/component_compare_baseline | 1 + CIME/Tools/component_compare_copy | 1 + CIME/Tools/component_compare_test | 1 + CIME/Tools/component_generate_baseline | 1 + CIME/Tools/cs.status | 12 ++- CIME/Tools/e3sm_check_env | 7 +- CIME/Tools/generate_cylc_workflow.py | 8 +- CIME/Tools/getTiming | 6 +- CIME/Tools/get_case_env | 14 +-- CIME/Tools/get_standard_makefile_args | 1 + CIME/Tools/jenkins_generic_job | 3 +- CIME/Tools/list_e3sm_tests | 10 +-- CIME/Tools/mvsource | 6 +- CIME/Tools/normalize_cases | 7 +- CIME/Tools/pelayout | 7 +- CIME/Tools/preview_namelists | 3 +- CIME/Tools/save_provenance | 6 +- CIME/Tools/simple_compare | 6 +- CIME/Tools/standard_script_setup.py | 15 +++- CIME/Tools/testreporter.py | 9 +- CIME/Tools/wait_for_tests | 6 +- CIME/Tools/xmlchange | 14 +-- .../xmlconvertors/config_pes_converter.py | 17 ++-- .../Tools/xmlconvertors/convert-grid-v1-to-v2 | 9 +- .../Tools/xmlconvertors/grid_xml_converter.py | 11 ++- CIME/Tools/xmlquery | 8 +- CIME/XML/archive.py | 6 +- CIME/XML/archive_base.py | 8 +- CIME/XML/batch.py | 9 +- CIME/XML/component.py | 6 +- CIME/XML/compsets.py | 8 +- CIME/XML/entry_id.py | 8 +- CIME/XML/env_archive.py | 7 +- CIME/XML/env_base.py | 7 +- CIME/XML/env_batch.py | 29 +++--- CIME/XML/env_build.py | 4 +- CIME/XML/env_case.py | 4 +- CIME/XML/env_mach_pes.py | 8 +- CIME/XML/env_mach_specific.py | 16 ++-- CIME/XML/env_postprocessing.py | 5 +- CIME/XML/env_run.py | 5 +- CIME/XML/env_test.py | 5 +- CIME/XML/env_workflow.py | 8 +- CIME/XML/expected_fails_file.py | 5 +- CIME/XML/files.py | 10 +-- CIME/XML/generic_xml.py | 33 ++++--- CIME/XML/grids.py | 6 +- CIME/XML/headers.py | 5 +- CIME/XML/inputdata.py | 7 +- CIME/XML/machines.py | 14 +-- CIME/XML/namelist_definition.py | 14 +-- CIME/XML/pes.py | 8 +- CIME/XML/pio.py | 7 +- CIME/XML/postprocessing.py | 6 +- CIME/XML/standard_module_setup.py | 16 +++- CIME/XML/stream.py | 9 +- CIME/XML/test_reporter.py | 7 +- CIME/XML/testlist.py | 6 +- CIME/XML/tests.py | 11 +-- CIME/XML/testspec.py | 5 +- CIME/XML/workflow.py | 8 +- CIME/aprun.py | 4 +- CIME/baselines/performance.py | 7 +- CIME/bless_test_results.py | 37 ++++---- CIME/build.py | 33 ++++--- CIME/build_scripts/buildlib.cprnc | 11 ++- CIME/build_scripts/buildlib.gptl | 11 ++- .../buildlib.internal_components | 5 +- CIME/build_scripts/buildlib.kokkos | 11 ++- CIME/build_scripts/buildlib.mct | 16 ++-- CIME/build_scripts/buildlib.mpi-serial | 13 +-- CIME/build_scripts/buildlib.pio | 13 ++- .../buildlib_cmake.internal_components | 5 +- CIME/buildlib.py | 15 ++-- CIME/buildnml.py | 11 ++- CIME/case/case.py | 90 +++++++++++-------- CIME/case/case_clone.py | 12 ++- CIME/case/case_cmpgen_namelists.py | 24 +++-- CIME/case/case_run.py | 27 ++++-- CIME/case/case_setup.py | 39 ++++---- CIME/case/case_st_archive.py | 21 +++-- CIME/case/case_submit.py | 12 +-- CIME/case/case_test.py | 12 +-- CIME/case/check_input_data.py | 14 +-- CIME/case/preview_namelists.py | 9 +- CIME/code_checker.py | 13 ++- CIME/compare_namelists.py | 5 +- CIME/compare_test_results.py | 14 +-- CIME/config.py | 10 +-- CIME/cs_status.py | 8 +- CIME/cs_status_creator.py | 4 +- CIME/date.py | 4 +- CIME/expected_fails.py | 2 +- CIME/get_tests.py | 7 +- CIME/get_timing.py | 10 ++- CIME/gitinterface.py | 7 +- CIME/hist_utils.py | 11 +-- CIME/jenkins_generic_job.py | 14 ++- CIME/locked_files.py | 9 +- CIME/namelist.py | 14 ++- CIME/nmlgen.py | 16 ++-- CIME/provenance.py | 8 +- CIME/scripts/configure | 2 +- CIME/scripts/create_clone.py | 16 ++-- CIME/scripts/create_newcase.py | 16 ++-- CIME/scripts/create_test.py | 27 +++--- CIME/scripts/query_config.py | 9 +- CIME/scripts/query_testlists.py | 5 +- CIME/simple_compare.py | 4 +- CIME/status.py | 8 +- CIME/test_scheduler.py | 56 +++++++----- CIME/test_status.py | 7 +- CIME/test_utils.py | 12 ++- CIME/tests/base.py | 7 +- CIME/tests/custom_assertions_test_status.py | 6 +- CIME/tests/scripts_regression_tests.py | 41 +++------ CIME/tests/test_sys_bless_tests_results.py | 2 +- CIME/tests/test_sys_cime_case.py | 4 +- CIME/tests/test_sys_create_newcase.py | 4 +- CIME/tests/test_sys_full_system.py | 5 +- CIME/tests/test_sys_jenkins_generic_job.py | 3 +- CIME/tests/test_sys_manage_and_query.py | 1 - CIME/tests/test_sys_save_timings.py | 5 +- CIME/tests/test_sys_single_submit.py | 1 - CIME/tests/test_sys_test_scheduler.py | 7 +- CIME/tests/test_sys_unittest.py | 2 +- CIME/tests/test_sys_wait_for_tests.py | 17 ++-- CIME/tests/test_unit_baselines_performance.py | 20 ++--- CIME/tests/test_unit_bless_test_results.py | 9 +- CIME/tests/test_unit_build.py | 1 - CIME/tests/test_unit_case.py | 5 +- CIME/tests/test_unit_case_fake.py | 4 +- CIME/tests/test_unit_case_run.py | 3 +- CIME/tests/test_unit_case_setup.py | 4 +- CIME/tests/test_unit_case_st_archive.py | 2 +- CIME/tests/test_unit_compare_test_results.py | 7 +- CIME/tests/test_unit_compare_two.py | 6 +- CIME/tests/test_unit_config.py | 4 +- CIME/tests/test_unit_configure.py | 7 +- CIME/tests/test_unit_cs_status.py | 9 +- ...test_unit_custom_assertions_test_status.py | 1 + CIME/tests/test_unit_doctest.py | 10 --- CIME/tests/test_unit_expected_fails_file.py | 5 +- CIME/tests/test_unit_fake_case.py | 6 +- CIME/tests/test_unit_grids.py | 5 +- CIME/tests/test_unit_hist_utils.py | 2 - CIME/tests/test_unit_locked_files.py | 4 +- CIME/tests/test_unit_nmlgen.py | 3 +- CIME/tests/test_unit_paramgen.py | 3 +- CIME/tests/test_unit_system_tests.py | 9 +- CIME/tests/test_unit_system_tests_mvk.py | 9 +- CIME/tests/test_unit_test_status.py | 6 +- .../test_unit_two_link_to_case2_output.py | 3 +- CIME/tests/test_unit_user_mod_support.py | 7 +- CIME/tests/test_unit_user_nl_utils.py | 3 +- CIME/tests/test_unit_utils.py | 9 +- CIME/tests/test_unit_xml_archive_base.py | 4 +- CIME/tests/test_unit_xml_env_batch.py | 4 +- CIME/tests/test_unit_xml_env_mach_specific.py | 2 +- CIME/tests/test_unit_xml_grids.py | 6 +- CIME/tests/test_unit_xml_tests.py | 2 +- CIME/tests/utils.py | 13 +-- CIME/user_mod_support.py | 6 +- CIME/utils.py | 21 ++++- CIME/wait_for_tests.py | 20 +++-- 220 files changed, 1232 insertions(+), 829 deletions(-) diff --git a/CIME/BuildTools/configure.py b/CIME/BuildTools/configure.py index 6e77a1a15c5..2e833735472 100755 --- a/CIME/BuildTools/configure.py +++ b/CIME/BuildTools/configure.py @@ -15,20 +15,22 @@ COMPILER, MPILIB, and DEBUG, respectively. """ -from CIME.XML.standard_module_setup import * +import glob +import logging +import os +import shutil + +from CIME.build import CmakeTmpBuildDir from CIME.utils import ( + copy_local_macros_to_dir, expect, - safe_copy, get_model, get_src_root, + safe_copy, stringify_bool, - copy_local_macros_to_dir, ) from CIME.XML.env_mach_specific import EnvMachSpecific from CIME.XML.files import Files -from CIME.build import CmakeTmpBuildDir - -import shutil, glob logger = logging.getLogger(__name__) diff --git a/CIME/Servers/ftp.py b/CIME/Servers/ftp.py index 209525a2bac..12c50e18767 100644 --- a/CIME/Servers/ftp.py +++ b/CIME/Servers/ftp.py @@ -1,15 +1,20 @@ """ FTP Server class. Interact with a server using FTP protocol """ + # pylint: disable=super-init-not-called -from CIME.XML.standard_module_setup import * -from CIME.Servers.generic_server import GenericServer -from CIME.utils import Timeout +import logging +import os +import socket from ftplib import FTP as FTPpy from ftplib import all_errors as all_ftp_errors -import socket + +from CIME.Servers.generic_server import GenericServer +from CIME.utils import Timeout, expect logger = logging.getLogger(__name__) + + # I think that multiple inheritence would be useful here, but I couldnt make it work # in a py2/3 compatible way. class FTP(GenericServer): diff --git a/CIME/Servers/generic_server.py b/CIME/Servers/generic_server.py index 537df181324..61fcef21b8f 100644 --- a/CIME/Servers/generic_server.py +++ b/CIME/Servers/generic_server.py @@ -3,9 +3,10 @@ to make sure that specific server classes maintain a consistant argument list and functionality so that they are interchangable objects """ + # pylint: disable=unused-argument -from CIME.XML.standard_module_setup import * +import logging from socket import _GLOBAL_DEFAULT_TIMEOUT logger = logging.getLogger(__name__) diff --git a/CIME/Servers/gftp.py b/CIME/Servers/gftp.py index f23943b583a..fea2a8a0915 100644 --- a/CIME/Servers/gftp.py +++ b/CIME/Servers/gftp.py @@ -1,8 +1,11 @@ """ GridFTP Server class. Interact with a server using GridFTP protocol """ + # pylint: disable=super-init-not-called -from CIME.XML.standard_module_setup import * +import logging +import os + from CIME.Servers.generic_server import GenericServer from CIME.utils import run_cmd diff --git a/CIME/Servers/svn.py b/CIME/Servers/svn.py index 7e06ab310d4..b407683ae07 100644 --- a/CIME/Servers/svn.py +++ b/CIME/Servers/svn.py @@ -1,9 +1,13 @@ """ SVN Server class. Interact with a server using SVN protocol """ + # pylint: disable=super-init-not-called -from CIME.XML.standard_module_setup import * +import logging +import os + from CIME.Servers.generic_server import GenericServer +from CIME.utils import run_cmd logger = logging.getLogger(__name__) diff --git a/CIME/Servers/wget.py b/CIME/Servers/wget.py index fde411460e8..583e0972bc4 100644 --- a/CIME/Servers/wget.py +++ b/CIME/Servers/wget.py @@ -1,9 +1,13 @@ """ WGET Server class. Interact with a server using WGET protocol """ + # pylint: disable=super-init-not-called -from CIME.XML.standard_module_setup import * +import logging +import os + from CIME.Servers.generic_server import GenericServer +from CIME.utils import run_cmd logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/dae.py b/CIME/SystemTests/dae.py index 175254d2d1b..e9472678de2 100644 --- a/CIME/SystemTests/dae.py +++ b/CIME/SystemTests/dae.py @@ -6,14 +6,15 @@ """ -import os.path -import logging import glob import gzip +import logging +import os.path from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.utils import expect + ############################################################################### class DAE(SystemTestsCompareTwo): ############################################################################### diff --git a/CIME/SystemTests/eri.py b/CIME/SystemTests/eri.py index 597cfe4c709..151f07d55ac 100644 --- a/CIME/SystemTests/eri.py +++ b/CIME/SystemTests/eri.py @@ -2,11 +2,14 @@ CIME ERI test This class inherits from SystemTestsCommon """ -from CIME.XML.standard_module_setup import * -from CIME.utils import safe_copy -from CIME.SystemTests.system_tests_common import SystemTestsCommon +import glob +import logging +import os +import shutil from stat import S_ISDIR, ST_CTIME, ST_MODE -import shutil, glob, os + +from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.utils import expect, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/erio.py b/CIME/SystemTests/erio.py index a1e7b041cc6..0ded2f0d273 100644 --- a/CIME/SystemTests/erio.py +++ b/CIME/SystemTests/erio.py @@ -3,8 +3,11 @@ This class inherits from SystemTestsCommon """ -from CIME.XML.standard_module_setup import * + +import logging + from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/erp.py b/CIME/SystemTests/erp.py index aeab1b0020b..b0fa12696c0 100644 --- a/CIME/SystemTests/erp.py +++ b/CIME/SystemTests/erp.py @@ -8,7 +8,8 @@ (2) Do a restart test with half the number of tasks and threads (suffix rest) """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.restart_tests import RestartTest logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/err.py b/CIME/SystemTests/err.py index ff5a2e4a4f7..755381e552c 100644 --- a/CIME/SystemTests/err.py +++ b/CIME/SystemTests/err.py @@ -3,10 +3,12 @@ ERR tests short term archiving and restart capabilities """ -import glob, os -from CIME.XML.standard_module_setup import * +import glob +import logging +import os + from CIME.SystemTests.restart_tests import RestartTest -from CIME.utils import safe_copy, ls_sorted_by_fname +from CIME.utils import expect, ls_sorted_by_fname, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/erri.py b/CIME/SystemTests/erri.py index 7851bd4bb66..e3453aeaccb 100644 --- a/CIME/SystemTests/erri.py +++ b/CIME/SystemTests/erri.py @@ -3,10 +3,13 @@ ERRI tests short term archiving and restart capabilities with "incomplete" (unzipped) log files """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.err import ERR +import glob +import gzip +import logging +import os +import shutil -import shutil, glob, gzip +from CIME.SystemTests.err import ERR logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/ers.py b/CIME/SystemTests/ers.py index 0e93d8afa1c..b39bf71aef1 100644 --- a/CIME/SystemTests/ers.py +++ b/CIME/SystemTests/ers.py @@ -1,9 +1,13 @@ """ CIME restart test This class inherits from SystemTestsCommon """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon + import glob +import logging +import os + +from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/ert.py b/CIME/SystemTests/ert.py index b912f7248b7..5a298336e0f 100644 --- a/CIME/SystemTests/ert.py +++ b/CIME/SystemTests/ert.py @@ -3,7 +3,8 @@ Exact restart from startup, default 2 month + 1 month """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_common import SystemTestsCommon logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/funit.py b/CIME/SystemTests/funit.py index 7681a6200d6..5ccf8324bcd 100644 --- a/CIME/SystemTests/funit.py +++ b/CIME/SystemTests/funit.py @@ -2,12 +2,15 @@ CIME FUNIT test. This class inherits from SystemTestsCommon. It runs the fortran unit tests; grid and compset are ignored. """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon + +import logging +import os + from CIME.build import post_build from CIME.status import append_testlog -from CIME.utils import get_cime_root +from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.test_status import * +from CIME.utils import expect, get_cime_root, run_cmd logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/hommebaseclass.py b/CIME/SystemTests/hommebaseclass.py index d275553a545..e811ce90747 100644 --- a/CIME/SystemTests/hommebaseclass.py +++ b/CIME/SystemTests/hommebaseclass.py @@ -2,14 +2,22 @@ CIME HOMME test. This class inherits from SystemTestsCommon """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon +import logging +import os +import shutil + from CIME.build import post_build from CIME.status import append_testlog -from CIME.utils import SharedArea, new_lid, gzip_existing_file +from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.test_status import * - -import shutil +from CIME.utils import ( + SharedArea, + expect, + gzip_existing_file, + new_lid, + run_cmd, + run_cmd_no_fail, +) logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/icp.py b/CIME/SystemTests/icp.py index 8d8c5e0ea59..c5357784671 100644 --- a/CIME/SystemTests/icp.py +++ b/CIME/SystemTests/icp.py @@ -1,7 +1,7 @@ """ CIME ICP test This class inherits from SystemTestsCommon """ -from CIME.XML.standard_module_setup import * + from CIME.SystemTests.system_tests_common import SystemTestsCommon diff --git a/CIME/SystemTests/irt.py b/CIME/SystemTests/irt.py index 633893adcd0..419dfd8164a 100644 --- a/CIME/SystemTests/irt.py +++ b/CIME/SystemTests/irt.py @@ -11,9 +11,11 @@ """ +import logging +import os + from CIME.SystemTests.restart_tests import RestartTest -from CIME.XML.standard_module_setup import * -from CIME.utils import ls_sorted_by_fname +from CIME.utils import expect, ls_sorted_by_fname logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/ldsta.py b/CIME/SystemTests/ldsta.py index a5f7c9196d5..4dd96fb8bae 100644 --- a/CIME/SystemTests/ldsta.py +++ b/CIME/SystemTests/ldsta.py @@ -4,19 +4,20 @@ The test verifies the archive directory contains the expected files """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon -from CIME.utils import expect -from CIME.date import get_file_date - import datetime import glob +import logging import os import random import shutil +from CIME.date import get_file_date +from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.utils import expect + logger = logging.getLogger(__name__) + # datetime objects can't be used anywhere else def _date_to_datetime(date_obj): return datetime.datetime( diff --git a/CIME/SystemTests/mcc.py b/CIME/SystemTests/mcc.py index a4b839cf1e9..f46030cdffd 100644 --- a/CIME/SystemTests/mcc.py +++ b/CIME/SystemTests/mcc.py @@ -4,7 +4,9 @@ This does two runs: In the first we run a three member ensemble using the MULTI_DRIVER capability, then we run a second single instance case and compare """ -from CIME.XML.standard_module_setup import * + +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 60f863a2670..852a460d248 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -6,23 +6,22 @@ This class inherits from SystemTestsCommon. """ -import os import json import logging +import os from shutil import copytree -from CIME import test_status -from CIME import utils -from CIME.status import append_testlog -from CIME.SystemTests.system_tests_common import SystemTestsCommon +import evv4esm # pylint: disable=import-error +from evv4esm.__main__ import main as evv # pylint: disable=import-error + +from CIME import test_status, utils from CIME.case.case_setup import case_setup -from CIME.XML.machines import Machines from CIME.config import ConfigBase -from CIME.SystemTests import test_mods from CIME.namelist import Namelist - -import evv4esm # pylint: disable=import-error -from evv4esm.__main__ import main as evv # pylint: disable=import-error +from CIME.status import append_testlog +from CIME.SystemTests import test_mods +from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.XML.machines import Machines version = evv4esm.__version_info__ diff --git a/CIME/SystemTests/nck.py b/CIME/SystemTests/nck.py index f75a2914215..9f17b28c09d 100644 --- a/CIME/SystemTests/nck.py +++ b/CIME/SystemTests/nck.py @@ -8,7 +8,8 @@ Lay all of the components out sequentially """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/ncr.py b/CIME/SystemTests/ncr.py index f0de168ac13..39b16f25ea2 100644 --- a/CIME/SystemTests/ncr.py +++ b/CIME/SystemTests/ncr.py @@ -8,7 +8,9 @@ NOTE: This is currently untested, and may not be working properly """ -from CIME.XML.standard_module_setup import * + +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/nodefail.py b/CIME/SystemTests/nodefail.py index d975cfc5bfd..559f49c690b 100644 --- a/CIME/SystemTests/nodefail.py +++ b/CIME/SystemTests/nodefail.py @@ -1,7 +1,10 @@ """ CIME restart upon failed node test. """ -from CIME.XML.standard_module_setup import * + +import logging +import os + from CIME.SystemTests.ers import ERS from CIME.utils import get_model diff --git a/CIME/SystemTests/pea.py b/CIME/SystemTests/pea.py index 4fb3a4569ca..c7534dd8340 100644 --- a/CIME/SystemTests/pea.py +++ b/CIME/SystemTests/pea.py @@ -6,8 +6,10 @@ (2) do a run with mpi-serial (suffix mpi-serial) """ +import logging +import os + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo -from CIME.XML.standard_module_setup import * from CIME.XML.machines import Machines logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/pem.py b/CIME/SystemTests/pem.py index d98f9e4a2c7..d26108d8197 100644 --- a/CIME/SystemTests/pem.py +++ b/CIME/SystemTests/pem.py @@ -8,7 +8,8 @@ (2) Run with half the number of tasks (suffix modpes) """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/pet.py b/CIME/SystemTests/pet.py index 161dac4d536..35e905f2399 100644 --- a/CIME/SystemTests/pet.py +++ b/CIME/SystemTests/pet.py @@ -6,7 +6,8 @@ (2) do another initial run with nthrds=1 for all components (suffix: single_thread) """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/pfs.py b/CIME/SystemTests/pfs.py index ed61d204e8a..ffd1ed46d3b 100644 --- a/CIME/SystemTests/pfs.py +++ b/CIME/SystemTests/pfs.py @@ -4,7 +4,8 @@ 20 days performance test, no restart files written """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_common import SystemTestsCommon logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/pgn.py b/CIME/SystemTests/pgn.py index 07ac1f4cb08..ed27bc6d641 100644 --- a/CIME/SystemTests/pgn.py +++ b/CIME/SystemTests/pgn.py @@ -9,30 +9,27 @@ from __future__ import division +import json +import logging import os import re -import json import shutil -import logging - from collections import OrderedDict from shutil import copytree -import pandas as pd +import evv4esm # pylint: disable=import-error import numpy as np - +import pandas as pd +from evv4esm.__main__ import main as evv # pylint: disable=import-error +from evv4esm.extensions import pg # pylint: disable=import-error import CIME.test_status import CIME.utils +from CIME.case.case_setup import case_setup from CIME.status import append_testlog from CIME.SystemTests.system_tests_common import SystemTestsCommon -from CIME.case.case_setup import case_setup from CIME.XML.machines import Machines -import evv4esm # pylint: disable=import-error -from evv4esm.extensions import pg # pylint: disable=import-error -from evv4esm.__main__ import main as evv # pylint: disable=import-error - evv_lib_dir = os.path.abspath(os.path.dirname(evv4esm.__file__)) logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/pre.py b/CIME/SystemTests/pre.py index 23547d46430..c5245364eb1 100644 --- a/CIME/SystemTests/pre.py +++ b/CIME/SystemTests/pre.py @@ -6,13 +6,14 @@ Test requires DESP component to function correctly. """ -import os.path -import logging import glob +import logging +import os.path +from CIME.hist_utils import cprnc from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.utils import expect -from CIME.hist_utils import cprnc + ############################################################################### class PRE(SystemTestsCompareTwo): diff --git a/CIME/SystemTests/restart_tests.py b/CIME/SystemTests/restart_tests.py index ffc50270ce2..6883e95baa0 100644 --- a/CIME/SystemTests/restart_tests.py +++ b/CIME/SystemTests/restart_tests.py @@ -2,8 +2,10 @@ Abstract class for restart tests """ +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo -from CIME.XML.standard_module_setup import * +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/reuseinitfiles.py b/CIME/SystemTests/reuseinitfiles.py index 76d8bb0522e..98cf8216380 100644 --- a/CIME/SystemTests/reuseinitfiles.py +++ b/CIME/SystemTests/reuseinitfiles.py @@ -14,9 +14,10 @@ import os import shutil -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo + from CIME.SystemTests.system_tests_common import INIT_GENERATED_FILES_DIRNAME +from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo +from CIME.utils import expect class REUSEINITFILES(SystemTestsCompareTwo): diff --git a/CIME/SystemTests/seq.py b/CIME/SystemTests/seq.py index 7413f900899..d0a6ca96495 100644 --- a/CIME/SystemTests/seq.py +++ b/CIME/SystemTests/seq.py @@ -1,7 +1,9 @@ """ sequencing bfb test (10 day seq,conc tests) """ -from CIME.XML.standard_module_setup import * + +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/sms.py b/CIME/SystemTests/sms.py index 17672b47052..18a42fc529a 100644 --- a/CIME/SystemTests/sms.py +++ b/CIME/SystemTests/sms.py @@ -3,7 +3,8 @@ It does a startup run with restarts off and optionally compares to or generates baselines """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_common import SystemTestsCommon logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 67ed8416671..aa605b32d1c 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -2,45 +2,53 @@ Base class for CIME system tests """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_run import EnvRun -from CIME.XML.env_test import EnvTest -from CIME.status import append_testlog -from CIME.utils import ( - get_model, - safe_copy, - get_timestamp, - CIMEError, - expect, - get_current_commit, - SharedArea, - is_comp_standalone, +import calendar +import glob +import gzip +import logging +import math +import os +import re +import time +import traceback +from contextlib import ExitStack +from datetime import datetime, timedelta + +import CIME.build as build +from CIME.baselines.performance import ( + get_latest_cpl_logs, + load_coupler_customization, + perf_compare_memory_baseline, + perf_compare_throughput_baseline, + perf_get_memory_list, + perf_write_baseline, ) -from CIME.test_status import * +from CIME.config import Config from CIME.hist_utils import ( - copy_histfiles, + compare_baseline, compare_test, + copy_histfiles, + generate_baseline, generate_teststatus, - compare_baseline, get_ts_synopsis, - generate_baseline, ) -from CIME.config import Config -from CIME.provenance import save_test_time, get_test_success -from CIME.locked_files import LOCKED_DIR, lock_file, is_locked -from CIME.baselines.performance import ( - get_latest_cpl_logs, - perf_get_memory_list, - perf_compare_memory_baseline, - perf_compare_throughput_baseline, - perf_write_baseline, - load_coupler_customization, +from CIME.locked_files import LOCKED_DIR, is_locked, lock_file +from CIME.provenance import get_test_success, save_test_time +from CIME.status import append_testlog +from CIME.test_status import * +from CIME.utils import ( + CIMEError, + SharedArea, + expect, + get_current_commit, + get_model, + get_timestamp, + is_comp_standalone, + run_cmd, + safe_copy, ) -import CIME.build as build -from datetime import datetime, timedelta -import glob, gzip, time, traceback, os, math, calendar - -from contextlib import ExitStack +from CIME.XML.env_run import EnvRun +from CIME.XML.env_test import EnvTest logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/system_tests_compare_n.py b/CIME/SystemTests/system_tests_compare_n.py index 93724eeef3e..4ce15286d94 100644 --- a/CIME/SystemTests/system_tests_compare_n.py +++ b/CIME/SystemTests/system_tests_compare_n.py @@ -39,13 +39,16 @@ Use this to do arbitrary actions immediately after running case one """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon, fix_single_exe_case +import glob +import logging +import os +import shutil + from CIME.case import Case from CIME.config import Config +from CIME.SystemTests.system_tests_common import SystemTestsCommon, fix_single_exe_case from CIME.test_status import * - -import shutil, os, glob +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/system_tests_compare_two.py b/CIME/SystemTests/system_tests_compare_two.py index 61232b34117..0866761e10b 100644 --- a/CIME/SystemTests/system_tests_compare_two.py +++ b/CIME/SystemTests/system_tests_compare_two.py @@ -47,13 +47,16 @@ Use this to do arbitrary actions immediately after running case two """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon, fix_single_exe_case +import glob +import logging +import os +import shutil + from CIME.case import Case from CIME.config import Config +from CIME.SystemTests.system_tests_common import SystemTestsCommon, fix_single_exe_case from CIME.test_status import * - -import shutil, os, glob +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/test_utils/user_nl_utils.py b/CIME/SystemTests/test_utils/user_nl_utils.py index 930d683666b..00cfdad37d7 100644 --- a/CIME/SystemTests/test_utils/user_nl_utils.py +++ b/CIME/SystemTests/test_utils/user_nl_utils.py @@ -2,8 +2,8 @@ This module contains functions for working with user_nl files in system tests. """ -import os import glob +import os def append_to_user_nl_files(caseroot, component, contents): diff --git a/CIME/SystemTests/tsc.py b/CIME/SystemTests/tsc.py index be695b7f482..5c16a068752 100644 --- a/CIME/SystemTests/tsc.py +++ b/CIME/SystemTests/tsc.py @@ -7,23 +7,22 @@ This class inherits from SystemTestsCommon. """ -import os import json import logging - +import os from shutil import copytree +import evv4esm # pylint: disable=import-error +from evv4esm.__main__ import main as evv # pylint: disable=import-error + import CIME.test_status import CIME.utils -from CIME.status import append_testlog -from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case.case_setup import case_setup from CIME.hist_utils import rename_all_hist_files +from CIME.status import append_testlog +from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.XML.machines import Machines -import evv4esm # pylint: disable=import-error -from evv4esm.__main__ import main as evv # pylint: disable=import-error - evv_lib_dir = os.path.abspath(os.path.dirname(evv4esm.__file__)) logger = logging.getLogger(__name__) diff --git a/CIME/Tools/archive_metadata b/CIME/Tools/archive_metadata index 191050254f9..4054db8612d 100755 --- a/CIME/Tools/archive_metadata +++ b/CIME/Tools/archive_metadata @@ -6,25 +6,26 @@ via a web post and SVN check-in Author: CSEG """ import argparse +import configparser import datetime import filecmp import getpass import glob import gzip -import json import io +import json import os -from os.path import expanduser import re import shutil import ssl import subprocess import sys -from string import Template -import configparser import urllib +from os.path import expanduser +from string import Template from standard_script_setup import * + from CIME.case import Case from CIME.utils import is_last_process_complete diff --git a/CIME/Tools/bld_diff b/CIME/Tools/bld_diff index 27193d61560..ea614562886 100755 --- a/CIME/Tools/bld_diff +++ b/CIME/Tools/bld_diff @@ -5,10 +5,15 @@ Try to calculate and succinctly present the differences between two bld logs for the same component """ +import argparse +import gzip +import os +import sys + from standard_script_setup import * + from CIME.utils import run_cmd_no_fail -import argparse, sys, os, gzip ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/bless_test_results b/CIME/Tools/bless_test_results index 093ae0d79b8..335f1d6a0d4 100755 --- a/CIME/Tools/bless_test_results +++ b/CIME/Tools/bless_test_results @@ -8,16 +8,14 @@ blessing of diffs. You may need to load modules for cprnc to work. """ +import argparse +import os +import sys + from standard_script_setup import * -from CIME.utils import expect -from CIME.XML.machines import Machines from CIME.bless_test_results import bless_test_results - -import argparse -import sys -import os -import logging +from CIME.XML.machines import Machines _MACHINE = Machines() diff --git a/CIME/Tools/case.build b/CIME/Tools/case.build index 131ca94564d..0f638876bcf 100755 --- a/CIME/Tools/case.build +++ b/CIME/Tools/case.build @@ -46,8 +46,9 @@ from standard_script_setup import * import CIME.build as build from CIME.case import Case -from CIME.utils import find_system_test, get_model from CIME.test_status import * +from CIME.utils import find_system_test, get_model + ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/case.cmpgen_namelists b/CIME/Tools/case.cmpgen_namelists index c9930e71795..e97936f336b 100755 --- a/CIME/Tools/case.cmpgen_namelists +++ b/CIME/Tools/case.cmpgen_namelists @@ -5,10 +5,12 @@ case.cmpgen_namelists - perform namelist baseline operations (compare, generate, or both) for this case. """ +from argparse import RawTextHelpFormatter + from standard_script_setup import * from CIME.case import Case -from argparse import RawTextHelpFormatter + ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/case.qstatus b/CIME/Tools/case.qstatus index 4902dc511d5..1ce2705b482 100755 --- a/CIME/Tools/case.qstatus +++ b/CIME/Tools/case.qstatus @@ -12,6 +12,7 @@ from standard_script_setup import * from CIME.case import Case from CIME.test_status import * + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/case.setup b/CIME/Tools/case.setup index 75eeadd05b0..ac4ca3d41e1 100755 --- a/CIME/Tools/case.setup +++ b/CIME/Tools/case.setup @@ -22,6 +22,7 @@ from CIME.case import Case from CIME.test_status import * from CIME.utils import find_system_test + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/case.submit b/CIME/Tools/case.submit index 5de69368100..1f4dee5a81c 100755 --- a/CIME/Tools/case.submit +++ b/CIME/Tools/case.submit @@ -16,10 +16,13 @@ Other examples: """ import configparser + from standard_script_setup import * + from CIME.case import Case from CIME.utils import expect + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/case_diff b/CIME/Tools/case_diff index e642dd8e7ac..167845ed91d 100755 --- a/CIME/Tools/case_diff +++ b/CIME/Tools/case_diff @@ -5,10 +5,14 @@ Try to calculate and succinctly present the differences between two large directory trees. """ +import argparse +import os +import sys + from standard_script_setup import * + from CIME.utils import run_cmd, run_cmd_no_fail -import argparse, sys, os ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/check_case b/CIME/Tools/check_case index 9b061046643..30bf1361019 100755 --- a/CIME/Tools/check_case +++ b/CIME/Tools/check_case @@ -17,13 +17,13 @@ automatically when running case.submit. However, you can run this if you want to perform these checks without actually submitting the case. """ +import argparse + from standard_script_setup import * -from CIME.utils import expect from CIME.case import Case from CIME.locked_files import check_lockedfiles - -import argparse +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/Tools/check_input_data b/CIME/Tools/check_input_data index 179af50c4fd..df11a8b539d 100755 --- a/CIME/Tools/check_input_data +++ b/CIME/Tools/check_input_data @@ -18,10 +18,12 @@ To obtain missing datasets from the input data server(s) use: This script is automatically called by the case control system, when the case is built and submitted. So manual usage of this script is optional. """ +import argparse + from standard_script_setup import * + from CIME.case import Case -import argparse ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/check_lockedfiles b/CIME/Tools/check_lockedfiles index a959b953570..068f24793f5 100755 --- a/CIME/Tools/check_lockedfiles +++ b/CIME/Tools/check_lockedfiles @@ -4,6 +4,7 @@ This script compares xml files """ from standard_script_setup import * + from CIME.case import Case from CIME.locked_files import check_lockedfiles diff --git a/CIME/Tools/cime_bisect b/CIME/Tools/cime_bisect index e427679f692..0a4969599fd 100755 --- a/CIME/Tools/cime_bisect +++ b/CIME/Tools/cime_bisect @@ -9,10 +9,14 @@ NOTE: this tool will only work for models that use git and, for bisecting CIME, bring in CIME via submodule or clone. """ +import argparse +import os +import re +import sys + from standard_script_setup import * -from CIME.utils import expect, run_cmd_no_fail, run_cmd -import argparse, sys, os, re +from CIME.utils import expect, run_cmd, run_cmd_no_fail logger = logging.getLogger(__name__) diff --git a/CIME/Tools/code_checker b/CIME/Tools/code_checker index 6f7510337ab..b84c5278c22 100755 --- a/CIME/Tools/code_checker +++ b/CIME/Tools/code_checker @@ -5,15 +5,17 @@ Ensure that all CIME python files are free of errors and follow the PEP8 standard. """ -from standard_script_setup import * - -from CIME.code_checker import check_code, expect - -import argparse, sys, os +import argparse +import os +import sys # pylint: disable=import-error from shutil import which +from standard_script_setup import * + +from CIME.code_checker import check_code, expect + logger = logging.getLogger(__name__) ############################################################################### diff --git a/CIME/Tools/compare_namelists b/CIME/Tools/compare_namelists index 5aba49edbc0..9e71be71397 100755 --- a/CIME/Tools/compare_namelists +++ b/CIME/Tools/compare_namelists @@ -5,11 +5,15 @@ Compare namelists. Should be called by an ACME test. Designed to not be sensitive to order or whitespace. """ +import argparse +import os +import sys + from standard_script_setup import * + import CIME.compare_namelists from CIME.utils import expect -import argparse, sys, os ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/compare_test_results b/CIME/Tools/compare_test_results index 3b59ccd007f..ce7be67c038 100755 --- a/CIME/Tools/compare_test_results +++ b/CIME/Tools/compare_test_results @@ -21,12 +21,14 @@ You may need to load modules for cprnc to work. """ +import argparse +import os +import sys + from standard_script_setup import * -from CIME.XML.machines import Machines from CIME.compare_test_results import compare_test_results - -import argparse, sys, os +from CIME.XML.machines import Machines _MACHINE = Machines() diff --git a/CIME/Tools/component_compare_baseline b/CIME/Tools/component_compare_baseline index 2adad3b94ad..8a15a3af8cf 100755 --- a/CIME/Tools/component_compare_baseline +++ b/CIME/Tools/component_compare_baseline @@ -9,6 +9,7 @@ from standard_script_setup import * from CIME.case import Case from CIME.hist_utils import compare_baseline + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/component_compare_copy b/CIME/Tools/component_compare_copy index 28005cd6674..3ed0dbecac0 100755 --- a/CIME/Tools/component_compare_copy +++ b/CIME/Tools/component_compare_copy @@ -10,6 +10,7 @@ from standard_script_setup import * from CIME.case import Case from CIME.hist_utils import copy_histfiles + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/component_compare_test b/CIME/Tools/component_compare_test index ece602e1af4..d4600a4db76 100755 --- a/CIME/Tools/component_compare_test +++ b/CIME/Tools/component_compare_test @@ -9,6 +9,7 @@ from standard_script_setup import * from CIME.case import Case from CIME.hist_utils import compare_test + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/component_generate_baseline b/CIME/Tools/component_generate_baseline index ff8f39170f0..565ee8c5771 100755 --- a/CIME/Tools/component_generate_baseline +++ b/CIME/Tools/component_generate_baseline @@ -9,6 +9,7 @@ from standard_script_setup import * from CIME.case import Case from CIME.hist_utils import generate_baseline + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/cs.status b/CIME/Tools/cs.status index 4ab7b7e8ecd..e8e7ccd8cd9 100755 --- a/CIME/Tools/cs.status +++ b/CIME/Tools/cs.status @@ -8,11 +8,15 @@ Typical usage: Returns True if no errors occured (not based on test statuses). """ -from standard_script_setup import * -import argparse, sys, os, logging, glob -from CIME.utils import expect -from CIME.cs_status import cs_status +import argparse +import glob +import os +import sys + + from CIME import test_status +from CIME.cs_status import cs_status +from CIME.utils import expect _PERFORMANCE_PHASES = [test_status.THROUGHPUT_PHASE, test_status.MEMCOMP_PHASE] diff --git a/CIME/Tools/e3sm_check_env b/CIME/Tools/e3sm_check_env index b1756c9cfd3..79f593a7b26 100755 --- a/CIME/Tools/e3sm_check_env +++ b/CIME/Tools/e3sm_check_env @@ -6,10 +6,13 @@ A script to verify that the environment is compliant with E3SM's software requir Be sure to source your env_mach_specific file before running this check. """ +import argparse +import os +import sys + from standard_script_setup import * -from CIME.utils import run_cmd -import sys, os, argparse +from CIME.utils import run_cmd # Here's where we keep the various reports and instructions. LOG = [] diff --git a/CIME/Tools/generate_cylc_workflow.py b/CIME/Tools/generate_cylc_workflow.py index cac503b342b..b7465359228 100755 --- a/CIME/Tools/generate_cylc_workflow.py +++ b/CIME/Tools/generate_cylc_workflow.py @@ -3,20 +3,22 @@ """ Generates a cylc workflow file for the case. See https://cylc.github.io for details about cylc """ + import os import sys sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) -from CIME.Tools.standard_script_setup import * +import argparse +import re from CIME.case import Case +from CIME.Tools.standard_script_setup import * from CIME.utils import expect, transform_vars -import argparse, re - logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/getTiming b/CIME/Tools/getTiming index 3f16847385a..7a1352d714e 100755 --- a/CIME/Tools/getTiming +++ b/CIME/Tools/getTiming @@ -3,8 +3,12 @@ Get timing information from run """ +import argparse +import os +import sys + from standard_script_setup import * -import argparse, sys, os + from CIME.case import Case from CIME.get_timing import get_timing diff --git a/CIME/Tools/get_case_env b/CIME/Tools/get_case_env index 7e7db4b393a..9ddd2fe462a 100755 --- a/CIME/Tools/get_case_env +++ b/CIME/Tools/get_case_env @@ -5,14 +5,18 @@ Dump what the CIME environment would be for a case. Only supports E3SM for now. """ +import argparse +import shutil +import tempfile + from standard_script_setup import * -from CIME.XML.machines import Machines -from CIME.test_scheduler import TestScheduler -from CIME.utils import parse_test_name, expect, get_src_root -from CIME.config import Config + import CIME.get_tests +from CIME.config import Config +from CIME.test_scheduler import TestScheduler +from CIME.utils import expect, get_src_root +from CIME.XML.machines import Machines -import argparse, tempfile, shutil ############################################################################### def parse_command_line(raw_args, description): diff --git a/CIME/Tools/get_standard_makefile_args b/CIME/Tools/get_standard_makefile_args index 5357c09cc1a..d361d36d571 100755 --- a/CIME/Tools/get_standard_makefile_args +++ b/CIME/Tools/get_standard_makefile_args @@ -11,6 +11,7 @@ from CIME.build import get_standard_makefile_args from CIME.case import Case from CIME.test_status import * + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/jenkins_generic_job b/CIME/Tools/jenkins_generic_job index b02a5b69199..b4228524260 100755 --- a/CIME/Tools/jenkins_generic_job +++ b/CIME/Tools/jenkins_generic_job @@ -10,9 +10,10 @@ ensures that the batch system is left in a clean state. from standard_script_setup import * import CIME.wait_for_tests +from CIME.jenkins_generic_job import jenkins_generic_job from CIME.utils import expect from CIME.XML.machines import Machines -from CIME.jenkins_generic_job import jenkins_generic_job + ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/list_e3sm_tests b/CIME/Tools/list_e3sm_tests index 58bb436da76..8bf69d94cc0 100755 --- a/CIME/Tools/list_e3sm_tests +++ b/CIME/Tools/list_e3sm_tests @@ -5,16 +5,14 @@ List e3sm test suites. Can be used to show what's being tested. Can just list tested grids, compsets, etc. """ -from standard_script_setup import * -from CIME import utils -from CIME import get_tests -from CIME.XML.files import Files -from CIME.XML.compsets import Compsets - import argparse import logging +from CIME import get_tests, utils +from CIME.XML.compsets import Compsets +from CIME.XML.files import Files + logger = logging.getLogger(__name__) diff --git a/CIME/Tools/mvsource b/CIME/Tools/mvsource index e02f20c91bf..0b3d042af3e 100755 --- a/CIME/Tools/mvsource +++ b/CIME/Tools/mvsource @@ -2,7 +2,8 @@ # This tool expects two arguments, the caseroot and the new cimeroot # It is intended to fix links and update your case source if the caseroot or src is moved. # -import os, sys +import os +import sys caseroot = sys.argv[1] cimeroot = sys.argv[2] @@ -12,7 +13,8 @@ sys.path.append(_LIBDIR) _LIBDIR = os.path.join(cimeroot, "scripts", "lib") sys.path.append(_LIBDIR) try: - from standard_script_setup import * + pass + from CIME.case import Case from CIME.utils import symlink_force except: diff --git a/CIME/Tools/normalize_cases b/CIME/Tools/normalize_cases index 6fa0e8111dc..2d7d35e6915 100755 --- a/CIME/Tools/normalize_cases +++ b/CIME/Tools/normalize_cases @@ -8,10 +8,15 @@ This is for debugging purposes and meant to assist the user when they want to run case_diff. """ +import argparse +import glob +import os +import sys + from standard_script_setup import * + from CIME.utils import expect, run_cmd_no_fail -import argparse, sys, os, glob ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/pelayout b/CIME/Tools/pelayout index d4c3575359b..c109435c96c 100755 --- a/CIME/Tools/pelayout +++ b/CIME/Tools/pelayout @@ -38,12 +38,13 @@ If you encounter problems with this tool or find it is missing any feature that you need, please open an issue on https://github.com/ESMCI/cime """ +import re +import sys + from standard_script_setup import * from CIME.case import Case -from CIME.utils import expect, convert_to_string -import sys -import re +from CIME.utils import convert_to_string, expect logger = logging.getLogger(__name__) diff --git a/CIME/Tools/preview_namelists b/CIME/Tools/preview_namelists index 69b89eb5663..0cc3dc988ae 100755 --- a/CIME/Tools/preview_namelists +++ b/CIME/Tools/preview_namelists @@ -15,12 +15,13 @@ Typical usage is simply: ./preview_namelists """ +import argparse + from standard_script_setup import * from CIME.case import Case from CIME.utils import expect -import argparse ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/save_provenance b/CIME/Tools/save_provenance index 7e3506ff50c..ce4498e0e62 100755 --- a/CIME/Tools/save_provenance +++ b/CIME/Tools/save_provenance @@ -4,15 +4,15 @@ This tool provide command-line access to provenance-saving functionality """ +import logging + from standard_script_setup import * from CIME.case import Case from CIME.config import Config +from CIME.get_timing import get_timing from CIME.provenance import * from CIME.utils import get_lids -from CIME.get_timing import get_timing - -import logging logger = logging.getLogger(__name__) diff --git a/CIME/Tools/simple_compare b/CIME/Tools/simple_compare index 59bc328341c..034f3862337 100755 --- a/CIME/Tools/simple_compare +++ b/CIME/Tools/simple_compare @@ -5,11 +5,15 @@ Compare files in a normalized way. Used by create_test for diffing non-namelist files. """ +import argparse +import os +import sys + from standard_script_setup import * + import CIME.simple_compare from CIME.utils import expect -import argparse, sys, os ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/standard_script_setup.py b/CIME/Tools/standard_script_setup.py index 2c38a5c4035..f82dcd65ed1 100644 --- a/CIME/Tools/standard_script_setup.py +++ b/CIME/Tools/standard_script_setup.py @@ -1,11 +1,19 @@ """ Encapsulate the importing of python utils and logging setup, things that every script should do. + +This module is DEPRECATED for star imports. New scripts should use explicit +imports instead of `from CIME.Tools.standard_script_setup import *`. + +The exports are maintained for backward compatibility. """ + # pylint: disable=unused-import -import sys, os -import __main__ as main +import argparse # noqa: F401 +import logging # noqa: F401 +import os +import sys def check_minimum_python_version(major, minor, warn_only=False): @@ -48,7 +56,6 @@ def check_minimum_python_version(major, minor, warn_only=False): # Important: Allows external tools to link up with CIME os.environ["CIMEROOT"] = cimeroot -import CIME.utils +import CIME.utils # noqa: E402 CIME.utils.stop_buffering_output() -import argparse, logging diff --git a/CIME/Tools/testreporter.py b/CIME/Tools/testreporter.py index a76c113ac09..7397d1f72c8 100755 --- a/CIME/Tools/testreporter.py +++ b/CIME/Tools/testreporter.py @@ -3,21 +3,22 @@ """ Simple script to populate CESM test database with test results. """ + import os import sys sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) -from CIME.Tools.standard_script_setup import * +import glob +from CIME.Tools.standard_script_setup import * +from CIME.utils import expect from CIME.XML.env_build import EnvBuild from CIME.XML.env_case import EnvCase from CIME.XML.env_test import EnvTest -from CIME.XML.test_reporter import TestReporter -from CIME.utils import expect from CIME.XML.generic_xml import GenericXML +from CIME.XML.test_reporter import TestReporter -import glob ############################################################################### def parse_command_line(args): diff --git a/CIME/Tools/wait_for_tests b/CIME/Tools/wait_for_tests index 484b393b6f4..19e298d861e 100755 --- a/CIME/Tools/wait_for_tests +++ b/CIME/Tools/wait_for_tests @@ -8,11 +8,13 @@ for the RUN phase specifically and will not terminate if the RUN phase didn't happen. """ -from standard_script_setup import * +import argparse +import os +import sys + import CIME.wait_for_tests -import argparse, sys, os ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/xmlchange b/CIME/Tools/xmlchange index 4101f9b5dba..23b76f95e53 100755 --- a/CIME/Tools/xmlchange +++ b/CIME/Tools/xmlchange @@ -50,18 +50,18 @@ Examples: ./xmlchange JOB_WALLCLOCK_TIME=0:30 """ +import re + from standard_script_setup import * +from CIME.case import Case +from CIME.locked_files import check_lockedfiles +from CIME.status import append_case_status from CIME.utils import ( - expect, - convert_to_type, Timeout, + convert_to_type, + expect, ) -from CIME.status import append_case_status -from CIME.case import Case -from CIME.locked_files import check_lockedfiles - -import re # Set logger logger = logging.getLogger("xmlchange") diff --git a/CIME/Tools/xmlconvertors/config_pes_converter.py b/CIME/Tools/xmlconvertors/config_pes_converter.py index cf57c21fb9b..2c36090199d 100755 --- a/CIME/Tools/xmlconvertors/config_pes_converter.py +++ b/CIME/Tools/xmlconvertors/config_pes_converter.py @@ -8,21 +8,26 @@ CIME2: cime/machines-acme/config_pes.xml CIME5: config/acme/allactive/config_pesall.xml """ -import os, sys + +import os +import sys sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) ) +import xml.etree.ElementTree as ET +from shutil import which + +import grid_xml_converter + from CIME import utils from CIME.Tools.standard_script_setup import * from CIME.utils import run_cmd -from shutil import which -import xml.etree.ElementTree as ET -import grid_xml_converter LOGGER = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args): ############################################################################### @@ -200,7 +205,9 @@ def set_data(self, xmlnode): class PesTree(grid_xml_converter.DataTree): def __init__(self, xmlfilename): # original xml file has bad comments - import re, StringIO + import re + + import StringIO if os.access(xmlfilename, os.R_OK): with open(xmlfilename, "r") as xmlfile: diff --git a/CIME/Tools/xmlconvertors/convert-grid-v1-to-v2 b/CIME/Tools/xmlconvertors/convert-grid-v1-to-v2 index 53b22d0e4a6..42d7a497bff 100755 --- a/CIME/Tools/xmlconvertors/convert-grid-v1-to-v2 +++ b/CIME/Tools/xmlconvertors/convert-grid-v1-to-v2 @@ -4,16 +4,19 @@ Convert a grid file from v1 to v2. """ -import argparse, sys, os +import argparse +import os +import sys sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +from collections import OrderedDict + from standard_script_setup import * + from CIME.utils import expect from CIME.XML.generic_xml import GenericXML -import xml.etree.ElementTree as ET -from collections import OrderedDict ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/xmlconvertors/grid_xml_converter.py b/CIME/Tools/xmlconvertors/grid_xml_converter.py index 01de9ddcf9f..acd19b94850 100755 --- a/CIME/Tools/xmlconvertors/grid_xml_converter.py +++ b/CIME/Tools/xmlconvertors/grid_xml_converter.py @@ -16,21 +16,24 @@ # CIME5: config/acme/config_grids.xml # -import os, sys +import os +import sys sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) ) +import operator +import xml.etree.ElementTree as ET +from shutil import which + from CIME import utils from CIME.Tools.standard_script_setup import * from CIME.utils import run_cmd_no_fail -from shutil import which -import xml.etree.ElementTree as ET -import operator logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args): ############################################################################### diff --git a/CIME/Tools/xmlquery b/CIME/Tools/xmlquery index 25ca0a2ea2f..56b6cb4a3d4 100755 --- a/CIME/Tools/xmlquery +++ b/CIME/Tools/xmlquery @@ -97,12 +97,14 @@ There are two usage modes: - The env_mach_specific.xml and env_archive.xml files are not supported by this tool. """ +import re +import sys +import textwrap + from standard_script_setup import * from CIME.case import Case -from CIME.utils import expect, convert_to_string - -import textwrap, sys, re +from CIME.utils import convert_to_string, expect logger = logging.getLogger("xmlquery") unsupported_files = ["env_mach_specific.xml", "env_archive.xml"] diff --git a/CIME/XML/archive.py b/CIME/XML/archive.py index e9d13eae686..8c34780acbb 100644 --- a/CIME/XML/archive.py +++ b/CIME/XML/archive.py @@ -2,11 +2,13 @@ Interface to the archive.xml file. This class inherits from GenericXML.py """ -from CIME.XML.standard_module_setup import * +import logging +import os +from copy import deepcopy + from CIME.config import Config from CIME.XML.archive_base import ArchiveBase from CIME.XML.files import Files -from copy import deepcopy logger = logging.getLogger(__name__) diff --git a/CIME/XML/archive_base.py b/CIME/XML/archive_base.py index 360424b734d..306cd2b7f71 100644 --- a/CIME/XML/archive_base.py +++ b/CIME/XML/archive_base.py @@ -1,9 +1,13 @@ """ Base class for archive files. This class inherits from generic_xml.py """ -from CIME.XML.standard_module_setup import * + +import logging +import os +import re + +from CIME.utils import convert_to_type, expect from CIME.XML.generic_xml import GenericXML -from CIME.utils import convert_to_type logger = logging.getLogger(__name__) diff --git a/CIME/XML/batch.py b/CIME/XML/batch.py index 75d9b1cfb94..fa2213f7c0d 100644 --- a/CIME/XML/batch.py +++ b/CIME/XML/batch.py @@ -4,10 +4,13 @@ The batch_system type="foo" blocks define most things. Machine-specific overrides can be defined by providing a batch_system MACH="mach" block. """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files + +import logging +import os + from CIME.utils import expect +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/component.py b/CIME/XML/component.py index abd09c86fd0..6120769ece8 100644 --- a/CIME/XML/component.py +++ b/CIME/XML/component.py @@ -1,11 +1,13 @@ """ Interface to the config_component.xml files. This class inherits from EntryID.py """ -from CIME.XML.standard_module_setup import * +import logging +import re + +from CIME.utils import expect from CIME.XML.entry_id import EntryID from CIME.XML.files import Files -from CIME.utils import get_cime_root logger = logging.getLogger(__name__) diff --git a/CIME/XML/compsets.py b/CIME/XML/compsets.py index 70241ec2e98..415b16a14a2 100644 --- a/CIME/XML/compsets.py +++ b/CIME/XML/compsets.py @@ -2,11 +2,13 @@ Common interface to XML files which follow the compsets format, """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML +import logging + +from CIME.core.exceptions import CIMEError +from CIME.utils import expect from CIME.XML.entry_id import EntryID from CIME.XML.files import Files -from CIME.core.exceptions import CIMEError +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/entry_id.py b/CIME/XML/entry_id.py index 8a7acd92a4b..48927be1df9 100644 --- a/CIME/XML/entry_id.py +++ b/CIME/XML/entry_id.py @@ -3,8 +3,12 @@ this is an abstract class and is expected to be used by other XML interface modules and not directly. """ -from CIME.XML.standard_module_setup import * -from CIME.utils import expect, convert_to_string, convert_to_type + +import logging +import os +import re + +from CIME.utils import convert_to_string, convert_to_type, expect from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_archive.py b/CIME/XML/env_archive.py index e3752017aa2..26357452d04 100644 --- a/CIME/XML/env_archive.py +++ b/CIME/XML/env_archive.py @@ -1,12 +1,17 @@ """ Interface to the env_archive.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * + +import logging +import os + from CIME import utils from CIME.XML.archive_base import ArchiveBase from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) + + # pylint: disable=super-init-not-called class EnvArchive(ArchiveBase, EnvBase): def __init__(self, case_root=None, infile="env_archive.xml", read_only=False): diff --git a/CIME/XML/env_base.py b/CIME/XML/env_base.py index b30d33da988..f0789761333 100644 --- a/CIME/XML/env_base.py +++ b/CIME/XML/env_base.py @@ -1,10 +1,13 @@ """ Base class for env files. This class inherits from EntryID.py """ -from CIME.XML.standard_module_setup import * + +import logging +import os + +from CIME.utils import convert_to_type, expect from CIME.XML.entry_id import EntryID from CIME.XML.headers import Headers -from CIME.utils import convert_to_type logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_batch.py b/CIME/XML/env_batch.py index fbcc112c56f..f2e5e32088f 100644 --- a/CIME/XML/env_batch.py +++ b/CIME/XML/env_batch.py @@ -2,25 +2,30 @@ Interface to the env_batch.xml file. This class inherits from EnvBase """ +import logging +import math import os -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase +import pathlib +import re +import stat +from collections import OrderedDict +from itertools import zip_longest + from CIME import utils from CIME.utils import ( - transform_vars, - get_cime_root, - convert_to_seconds, + add_flag_to_cmd, convert_to_babylonian_time, - get_cime_config, + convert_to_seconds, + expect, + format_time, get_batch_script_for_job, + get_cime_config, get_logging_options, - format_time, - add_flag_to_cmd, + run_cmd, + run_cmd_no_fail, + transform_vars, ) -from collections import OrderedDict -import stat, re, math -import pathlib -from itertools import zip_longest +from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_build.py b/CIME/XML/env_build.py index fe863e414ef..bc72258e1c6 100644 --- a/CIME/XML/env_build.py +++ b/CIME/XML/env_build.py @@ -1,7 +1,9 @@ """ Interface to the env_build.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * + +import logging +import os from CIME import utils from CIME.XML.env_base import EnvBase diff --git a/CIME/XML/env_case.py b/CIME/XML/env_case.py index 1b4c85d6f88..e0af1f951d6 100644 --- a/CIME/XML/env_case.py +++ b/CIME/XML/env_case.py @@ -1,7 +1,9 @@ """ Interface to the env_case.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * + +import logging +import os from CIME import utils from CIME.XML.env_base import EnvBase diff --git a/CIME/XML/env_mach_pes.py b/CIME/XML/env_mach_pes.py index 86a9cb7d40c..8051ab91dcf 100644 --- a/CIME/XML/env_mach_pes.py +++ b/CIME/XML/env_mach_pes.py @@ -1,10 +1,14 @@ """ Interface to the env_mach_pes.xml file. This class inherits from EntryID """ -from CIME.XML.standard_module_setup import * + +import logging +import math +import os + from CIME import utils +from CIME.utils import expect from CIME.XML.env_base import EnvBase -import math logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_mach_specific.py b/CIME/XML/env_mach_specific.py index dcd282d614f..d382ca46792 100644 --- a/CIME/XML/env_mach_specific.py +++ b/CIME/XML/env_mach_specific.py @@ -1,14 +1,19 @@ """ Interface to the env_mach_specific.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase -from CIME import utils -from CIME.utils import transform_vars, get_cime_root -import string, resource, platform +import logging +import os +import platform +import re +import resource +import string from collections import OrderedDict +from CIME import utils +from CIME.utils import expect, run_cmd, run_cmd_no_fail, transform_vars +from CIME.XML.env_base import EnvBase + logger = logging.getLogger(__name__) # Helpful map for converting resource limits to shell commands @@ -32,6 +37,7 @@ "RLIMIT_STACK": ("stacksize", "-s"), } + # Is not of type EntryID but can use functions from EntryID (e.g # get_type) otherwise need to implement own functions and make GenericXML parent class class EnvMachSpecific(EnvBase): diff --git a/CIME/XML/env_postprocessing.py b/CIME/XML/env_postprocessing.py index 90f56f24d64..7346c1c8b7d 100644 --- a/CIME/XML/env_postprocessing.py +++ b/CIME/XML/env_postprocessing.py @@ -1,11 +1,12 @@ """ Interface to the env_postprocessing.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase +import logging +import os from CIME import utils +from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_run.py b/CIME/XML/env_run.py index 2e34c86c8f7..453c101eae7 100644 --- a/CIME/XML/env_run.py +++ b/CIME/XML/env_run.py @@ -1,12 +1,13 @@ """ Interface to the env_run.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase +import logging +import os from CIME import utils from CIME.utils import convert_to_type +from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_test.py b/CIME/XML/env_test.py index 9dbe3615eff..cbf331fe76d 100644 --- a/CIME/XML/env_test.py +++ b/CIME/XML/env_test.py @@ -1,10 +1,11 @@ """ Interface to the env_test.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase +import logging + from CIME.utils import convert_to_type +from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_workflow.py b/CIME/XML/env_workflow.py index 6de05f34e93..fc5d2e4311c 100644 --- a/CIME/XML/env_workflow.py +++ b/CIME/XML/env_workflow.py @@ -2,11 +2,11 @@ Interface to the env_workflow.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase -from CIME.utils import get_cime_root +import logging +import math -import re, math +from CIME.utils import expect +from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) diff --git a/CIME/XML/expected_fails_file.py b/CIME/XML/expected_fails_file.py index 82e6ef0a298..f0ba93d0991 100644 --- a/CIME/XML/expected_fails_file.py +++ b/CIME/XML/expected_fails_file.py @@ -42,11 +42,12 @@ """ -from CIME.XML.standard_module_setup import * +import logging +import os from CIME import utils -from CIME.XML.generic_xml import GenericXML from CIME.expected_fails import ExpectedFails +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/files.py b/CIME/XML/files.py index b2612cc8937..8a29283dd4b 100644 --- a/CIME/XML/files.py +++ b/CIME/XML/files.py @@ -2,19 +2,19 @@ Interface to the config_files.xml file. This class inherits from EntryID.py """ -import re +import logging import os -from CIME.XML.standard_module_setup import * +import re -from CIME.XML.entry_id import EntryID from CIME.utils import ( expect, + get_cime_default_driver, get_cime_root, get_config_path, - get_schema_path, get_model, - get_cime_default_driver, + get_schema_path, ) +from CIME.XML.entry_id import EntryID logger = logging.getLogger(__name__) diff --git a/CIME/XML/generic_xml.py b/CIME/XML/generic_xml.py index 6c7ff812a88..e350f108304 100644 --- a/CIME/XML/generic_xml.py +++ b/CIME/XML/generic_xml.py @@ -2,16 +2,19 @@ Common interface to XML files, this is an abstract class and is expected to be used by other XML interface modules and not directly. """ -from CIME.XML.standard_module_setup import * -from CIME.utils import safe_copy, get_src_root +import getpass +import logging +import os +import re import xml.etree.ElementTree as ET +from collections import namedtuple +from copy import deepcopy # pylint: disable=import-error from shutil import which -import getpass -from copy import deepcopy -from collections import namedtuple + +from CIME.utils import expect, get_cime_root, get_src_root, run_cmd_no_fail, safe_copy logger = logging.getLogger(__name__) @@ -398,10 +401,12 @@ def get_child(self, name=None, attributes=None, root=None, err_msg=None): ) expect( child, - err_msg - if err_msg - else "Expected one child, found None with name '{}' and attribs '{}' in file {}".format( - name, attributes, self.filename + ( + err_msg + if err_msg + else "Expected one child, found None with name '{}' and attribs '{}' in file {}".format( + name, attributes, self.filename + ) ), ) return child @@ -418,10 +423,12 @@ def get_optional_child(self, name=None, attributes=None, root=None, err_msg=None expect( len(children) <= 1, - err_msg - if err_msg - else "Multiple matches for name '{}' and attribs '{}' in file {}".format( - name, attributes, self.filename + ( + err_msg + if err_msg + else "Multiple matches for name '{}' and attribs '{}' in file {}".format( + name, attributes, self.filename + ) ), ) return children[0] if children else None diff --git a/CIME/XML/grids.py b/CIME/XML/grids.py index 60bc1ff3d02..fbde52f6dc2 100644 --- a/CIME/XML/grids.py +++ b/CIME/XML/grids.py @@ -3,8 +3,12 @@ This is not an abstract class - but inherits from the abstact class GenericXML """ +import logging +import os +import re from collections import OrderedDict -from CIME.XML.standard_module_setup import * + +from CIME.utils import expect from CIME.XML.files import Files from CIME.XML.generic_xml import GenericXML diff --git a/CIME/XML/headers.py b/CIME/XML/headers.py index 5937a1d03cb..164ba7bccb2 100644 --- a/CIME/XML/headers.py +++ b/CIME/XML/headers.py @@ -1,10 +1,11 @@ """ Interface to the config_headers.xml file. This class inherits from EntryID.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML +import logging + from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/inputdata.py b/CIME/XML/inputdata.py index 18b71dca4dd..46459ccfff6 100644 --- a/CIME/XML/inputdata.py +++ b/CIME/XML/inputdata.py @@ -1,10 +1,11 @@ """ Interface to the config_inputdata.xml file. This class inherits from GenericXML.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML + +import logging + from CIME.XML.files import Files -from CIME.utils import expect +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/machines.py b/CIME/XML/machines.py index eabd6a8f9ef..467c83ead91 100644 --- a/CIME/XML/machines.py +++ b/CIME/XML/machines.py @@ -2,18 +2,18 @@ Interface to the config_machines.xml file. This class inherits from GenericXML.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files -from CIME.core.exceptions import CIMEError -from CIME.utils import expect, convert_to_unknown_type, get_cime_config - -import re import logging +import os +import re import socket from functools import partial from pathlib import Path +from CIME.core.exceptions import CIMEError +from CIME.utils import convert_to_unknown_type, expect, get_cime_config +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML + logger = logging.getLogger(__name__) diff --git a/CIME/XML/namelist_definition.py b/CIME/XML/namelist_definition.py index 38a184d1019..ebb35c09582 100644 --- a/CIME/XML/namelist_definition.py +++ b/CIME/XML/namelist_definition.py @@ -10,19 +10,21 @@ # Disable warnings due to using `standard_module_setup` # pylint:disable=wildcard-import,unused-wildcard-import -import re import collections +import logging +import os +import re +from CIME.core.exceptions import CIMEError from CIME.namelist import ( - fortran_namelist_base_value, - is_valid_fortran_namelist_literal, + Namelist, character_literal_to_string, expand_literal_list, - Namelist, + fortran_namelist_base_value, get_fortran_name_only, + is_valid_fortran_namelist_literal, ) -from CIME.core.exceptions import CIMEError -from CIME.XML.standard_module_setup import * +from CIME.utils import expect from CIME.XML.entry_id import EntryID from CIME.XML.files import Files diff --git a/CIME/XML/pes.py b/CIME/XML/pes.py index 666382c0a38..e45af99c939 100644 --- a/CIME/XML/pes.py +++ b/CIME/XML/pes.py @@ -2,10 +2,12 @@ Interface to the config_pes.xml file. This class inherits from GenericXML.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files +import logging +import re + from CIME.utils import expect +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/pio.py b/CIME/XML/pio.py index 54af6112bf0..42953263a01 100644 --- a/CIME/XML/pio.py +++ b/CIME/XML/pio.py @@ -1,12 +1,13 @@ """ Class for config_pio files . This class inherits from EntryID.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.entry_id import EntryID -from CIME.XML.files import Files +import logging from collections import OrderedDict +from CIME.XML.entry_id import EntryID +from CIME.XML.files import Files + logger = logging.getLogger(__name__) diff --git a/CIME/XML/postprocessing.py b/CIME/XML/postprocessing.py index 3287d145142..22144455aaa 100644 --- a/CIME/XML/postprocessing.py +++ b/CIME/XML/postprocessing.py @@ -2,10 +2,12 @@ Interface to the config_postprocessing.xml file. This class inherits from EntryID """ -from CIME.XML.standard_module_setup import * +import logging +import os + +from CIME.utils import expect from CIME.XML.entry_id import EntryID from CIME.XML.files import Files -from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/XML/standard_module_setup.py b/CIME/XML/standard_module_setup.py index 1c934407da5..7957ee191db 100644 --- a/CIME/XML/standard_module_setup.py +++ b/CIME/XML/standard_module_setup.py @@ -1,8 +1,20 @@ # pragma pylint: disable=unused-import +""" +Standard imports for CIME XML modules. -import logging, os, sys, re +This module is DEPRECATED. New code should use explicit imports instead of +`from CIME.XML.standard_module_setup import *`. + +The exports are maintained for backward compatibility with external code +(E3SM, CESM, NorESM cime_config directories). +""" + +import logging # noqa: F401 +import os # noqa: F401 +import re # noqa: F401 +import sys # noqa: F401 LIB_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(LIB_DIR) -from CIME.utils import expect, run_cmd, run_cmd_no_fail, get_cime_root +from CIME.utils import expect, get_cime_root, run_cmd, run_cmd_no_fail # noqa: F401 diff --git a/CIME/XML/stream.py b/CIME/XML/stream.py index 95dfa30ef7d..1418a09da09 100644 --- a/CIME/XML/stream.py +++ b/CIME/XML/stream.py @@ -3,10 +3,13 @@ stream files predate cime and so do not conform to entry id format """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files + +import logging +import os + from CIME.utils import expect +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/test_reporter.py b/CIME/XML/test_reporter.py index 93f117a8a46..17003715d3f 100644 --- a/CIME/XML/test_reporter.py +++ b/CIME/XML/test_reporter.py @@ -2,12 +2,15 @@ Interface to the testreporter xml. This class inherits from GenericXML.py """ + +import os +import ssl + # pylint: disable=import-error import urllib.parse import urllib.request -from CIME.XML.standard_module_setup import * + from CIME.XML.generic_xml import GenericXML -import ssl # pylint: disable=protected-access ssl._create_default_https_context = ssl._create_unverified_context diff --git a/CIME/XML/testlist.py b/CIME/XML/testlist.py index 6dbe79b8f98..dbe75db6332 100644 --- a/CIME/XML/testlist.py +++ b/CIME/XML/testlist.py @@ -35,10 +35,12 @@ - workflow: adds a workflow to the test """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML +import logging + +from CIME.utils import expect from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/tests.py b/CIME/XML/tests.py index 92feaca91c8..bcef473d5e5 100644 --- a/CIME/XML/tests.py +++ b/CIME/XML/tests.py @@ -2,14 +2,15 @@ Interface to the config_tests.xml file. This class inherits from GenericEntry """ -from CIME.XML.standard_module_setup import * +import logging +import os -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files from CIME.core.exceptions import CIMEError -from CIME.utils import find_system_test -from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.SystemTests.system_tests_compare_n import SystemTestsCompareN +from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo +from CIME.utils import find_system_test +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/testspec.py b/CIME/XML/testspec.py index 9b4e7c37724..5686a800f37 100644 --- a/CIME/XML/testspec.py +++ b/CIME/XML/testspec.py @@ -1,8 +1,11 @@ """ Interface to the testspec.xml file. This class inherits from generic_xml.py """ -from CIME.XML.standard_module_setup import * +import logging +import os + +from CIME.utils import expect from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/workflow.py b/CIME/XML/workflow.py index cdb2d27b403..113ab7b4965 100644 --- a/CIME/XML/workflow.py +++ b/CIME/XML/workflow.py @@ -2,10 +2,12 @@ Interface to the config_workflow.xml file. This class inherits from GenericXML.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files +import logging +import os + from CIME.utils import expect +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/aprun.py b/CIME/aprun.py index ca767c88db8..65b391dd071 100755 --- a/CIME/aprun.py +++ b/CIME/aprun.py @@ -3,12 +3,12 @@ code to compute and assemble aprun commands. """ -from CIME.XML.standard_module_setup import * - +import logging import math logger = logging.getLogger(__name__) + ############################################################################### def _get_aprun_cmd_for_case_impl( ntasks, diff --git a/CIME/baselines/performance.py b/CIME/baselines/performance.py index 67c19dbd43f..a46d4304265 100644 --- a/CIME/baselines/performance.py +++ b/CIME/baselines/performance.py @@ -1,10 +1,11 @@ -import os import glob -import re import gzip import logging +import os +import re + from CIME.config import Config -from CIME.utils import expect, get_src_root, get_current_commit, get_timestamp +from CIME.utils import expect, get_current_commit, get_src_root, get_timestamp logger = logging.getLogger(__name__) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 3028cfa4390..7080a13987c 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -1,24 +1,29 @@ -import CIME.compare_namelists, CIME.simple_compare -from CIME.test_scheduler import NAMELIST_PHASE -from CIME.utils import ( - run_cmd, - get_scripts_root, - EnvironmentContext, - parse_test_name, - match_any, - CIMEError, +import logging +import os +import re +import time + +import CIME.compare_namelists +import CIME.simple_compare +from CIME.baselines.performance import ( + perf_compare_memory_baseline, + perf_compare_throughput_baseline, + perf_write_baseline, ) +from CIME.case import Case from CIME.config import Config +from CIME.hist_utils import NO_ORIGINAL, compare_baseline, generate_baseline +from CIME.test_scheduler import NAMELIST_PHASE from CIME.test_status import * -from CIME.hist_utils import generate_baseline, compare_baseline, NO_ORIGINAL -from CIME.case import Case from CIME.test_utils import get_test_status_files -from CIME.baselines.performance import ( - perf_compare_throughput_baseline, - perf_compare_memory_baseline, - perf_write_baseline, +from CIME.utils import ( + CIMEError, + EnvironmentContext, + get_scripts_root, + match_any, + parse_test_name, + run_cmd, ) -import os, time logger = logging.getLogger(__name__) diff --git a/CIME/build.py b/CIME/build.py index af1709bb59c..66208d7af22 100644 --- a/CIME/build.py +++ b/CIME/build.py @@ -2,26 +2,35 @@ functions for building CIME models """ -import glob, shutil, time, threading, subprocess +import glob +import logging +import os +import re +import shutil +import subprocess +import threading +import time from pathlib import Path -from CIME.XML.standard_module_setup import * + +from CIME.config import Config +from CIME.locked_files import check_lockedfiles, lock_file, unlock_file from CIME.status import run_and_log_case_status from CIME.utils import ( - get_model, analyze_build_log, - stringify_bool, - get_timestamp, - run_sub_or_cmd, - run_cmd, + expect, get_batch_script_for_job, - gzip_existing_file, - safe_copy, - is_python_executable, get_logging_options, + get_model, + get_timestamp, + gzip_existing_file, import_from_file, + is_python_executable, + run_cmd, + run_cmd_no_fail, + run_sub_or_cmd, + safe_copy, + stringify_bool, ) -from CIME.config import Config -from CIME.locked_files import lock_file, unlock_file, check_lockedfiles from CIME.XML.files import Files logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.cprnc b/CIME/build_scripts/buildlib.cprnc index 51426de27c6..187fb5c53f1 100755 --- a/CIME/build_scripts/buildlib.cprnc +++ b/CIME/build_scripts/buildlib.cprnc @@ -1,14 +1,17 @@ #!/usr/bin/env python3 -import sys, os, logging, argparse +import argparse +import logging +import os +import sys _CIMEROOT = os.getenv("CIMEROOT") sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) -from standard_script_setup import * + from CIME import utils -from CIME.utils import run_bld_cmd_ensure_logging, CIMEError -from CIME.case import Case from CIME.build import get_standard_cmake_args +from CIME.case import Case +from CIME.utils import CIMEError, run_bld_cmd_ensure_logging logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.gptl b/CIME/build_scripts/buildlib.gptl index 774e5659396..786943dbc53 100755 --- a/CIME/build_scripts/buildlib.gptl +++ b/CIME/build_scripts/buildlib.gptl @@ -1,14 +1,17 @@ #!/usr/bin/env python3 -import sys, os, logging, argparse +import argparse +import logging +import os +import sys cimeroot = os.getenv("CIMEROOT") sys.path.append(os.path.join(cimeroot, "CIME", "Tools")) -from standard_script_setup import * + from CIME import utils -from CIME.utils import run_bld_cmd_ensure_logging -from CIME.case import Case from CIME.build import get_standard_makefile_args +from CIME.case import Case +from CIME.utils import run_bld_cmd_ensure_logging logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.internal_components b/CIME/build_scripts/buildlib.internal_components index f7f597d776e..5d79e5234f5 100755 --- a/CIME/build_scripts/buildlib.internal_components +++ b/CIME/build_scripts/buildlib.internal_components @@ -5,14 +5,15 @@ build cime component model library. This buildlib script is used by all cime i components. """ -import sys, os +import os +import sys _CIMEROOT = os.environ.get("CIMEROOT") if _CIMEROOT == None: raise ValueError("ERROR: CIMEROOT not defined in buildlib.internal_components.") sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) -from standard_script_setup import * + from CIME.buildlib import build_cime_component_lib, parse_input from CIME.case import Case diff --git a/CIME/build_scripts/buildlib.kokkos b/CIME/build_scripts/buildlib.kokkos index 6ecceb6ae48..5ed918d6332 100755 --- a/CIME/build_scripts/buildlib.kokkos +++ b/CIME/build_scripts/buildlib.kokkos @@ -1,12 +1,15 @@ #!/usr/bin/env python3 -import os, sys, argparse, logging +import argparse +import logging +import os +import sys + -from standard_script_setup import * from CIME import utils -from CIME.utils import expect, run_bld_cmd_ensure_logging, run_cmd_no_fail, run_cmd -from CIME.case import Case from CIME.build import get_standard_makefile_args +from CIME.case import Case +from CIME.utils import expect, run_bld_cmd_ensure_logging, run_cmd, run_cmd_no_fail logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.mct b/CIME/build_scripts/buildlib.mct index 446020839e4..884572f7a51 100755 --- a/CIME/build_scripts/buildlib.mct +++ b/CIME/build_scripts/buildlib.mct @@ -1,16 +1,20 @@ #!/usr/bin/env python3 -import sys, os, logging, argparse +import argparse +import logging +import os +import sys _CIMEROOT = os.getenv("CIMEROOT") sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) -from standard_script_setup import * -from CIME.config import Config +import glob + + from CIME import utils -from CIME.utils import copyifnewer, run_bld_cmd_ensure_logging, expect -from CIME.case import Case from CIME.build import get_standard_makefile_args -import glob +from CIME.case import Case +from CIME.config import Config +from CIME.utils import copyifnewer, expect, run_bld_cmd_ensure_logging logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.mpi-serial b/CIME/build_scripts/buildlib.mpi-serial index f13d949ae45..b851a1139dc 100755 --- a/CIME/build_scripts/buildlib.mpi-serial +++ b/CIME/build_scripts/buildlib.mpi-serial @@ -1,13 +1,16 @@ #!/usr/bin/env python3 -import os, sys, logging, argparse +import argparse +import glob +import logging +import os +import sys + -from standard_script_setup import * from CIME import utils +from CIME.build import get_standard_makefile_args +from CIME.case import Case from CIME.config import Config from CIME.utils import copyifnewer, run_bld_cmd_ensure_logging -from CIME.case import Case -from CIME.build import get_standard_makefile_args -import glob logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.pio b/CIME/build_scripts/buildlib.pio index b04367b7b19..2e7a2935446 100755 --- a/CIME/build_scripts/buildlib.pio +++ b/CIME/build_scripts/buildlib.pio @@ -1,15 +1,20 @@ #!/usr/bin/env python3 -import sys, os, logging, argparse +import argparse +import logging +import os +import sys cimeroot = os.getenv("CIMEROOT") sys.path.append(os.path.join(cimeroot, "CIME", "Tools")) -import glob, re -from standard_script_setup import * +import glob +import re + + from CIME import utils -from CIME.utils import expect, run_bld_cmd_ensure_logging, safe_copy from CIME.build import get_standard_makefile_args from CIME.case import Case +from CIME.utils import expect, run_bld_cmd_ensure_logging, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib_cmake.internal_components b/CIME/build_scripts/buildlib_cmake.internal_components index 4c939041e62..b8a56583973 100755 --- a/CIME/build_scripts/buildlib_cmake.internal_components +++ b/CIME/build_scripts/buildlib_cmake.internal_components @@ -5,7 +5,8 @@ build cime component model library. This buildlib script is used by all cime i components. """ -import sys, os +import os +import sys _CIMEROOT = os.environ.get("CIMEROOT") if _CIMEROOT == None: @@ -14,7 +15,7 @@ if _CIMEROOT == None: ) sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) -from standard_script_setup import * + from CIME.buildlib import build_cime_component_lib, parse_input from CIME.case import Case diff --git a/CIME/buildlib.py b/CIME/buildlib.py index b1152f8923a..a487734423c 100644 --- a/CIME/buildlib.py +++ b/CIME/buildlib.py @@ -2,21 +2,24 @@ common utilities for buildlib """ -from CIME.XML.standard_module_setup import * +import argparse +import logging +import os + +from CIME.build import get_standard_makefile_args from CIME.case import Case +from CIME.config import Config from CIME.utils import ( parse_args_and_handle_standard_logging_options, - setup_standard_logging_options, + run_cmd, safe_copy, + setup_standard_logging_options, ) -from CIME.config import Config -from CIME.build import get_standard_makefile_args from CIME.XML.files import Files -import sys, os, argparse - logger = logging.getLogger(__name__) + ############################################################################### def parse_input(argv): ############################################################################### diff --git a/CIME/buildnml.py b/CIME/buildnml.py index 69cbc5f7dca..d16e05f1ab6 100644 --- a/CIME/buildnml.py +++ b/CIME/buildnml.py @@ -4,17 +4,22 @@ These are used by components///cime_config/buildnml """ -from CIME.XML.standard_module_setup import * +import argparse +import glob +import logging +import os +import re + from CIME.utils import ( expect, parse_args_and_handle_standard_logging_options, + safe_copy, setup_standard_logging_options, ) -from CIME.utils import safe_copy -import sys, os, argparse, glob logger = logging.getLogger(__name__) + ############################################################################### def parse_input(argv): ############################################################################### diff --git a/CIME/case/case.py b/CIME/case/case.py index bd6db1e503c..b7f549b7ac9 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -5,46 +5,64 @@ All interaction with and between the module files in XML/ takes place through the Case module. """ -from copy import deepcopy + +import getpass +import glob +import hashlib +import logging +import math +import os +import re +import shutil +import socket import sys -import glob, os, shutil, math, time, hashlib, socket, getpass -from CIME.XML.standard_module_setup import * +import time +from copy import deepcopy # pylint: disable=import-error,redefined-builtin from CIME import utils +from CIME.aprun import get_aprun_cmd_for_case from CIME.config import Config -from CIME.status import append_status -from CIME.utils import expect, get_cime_root -from CIME.utils import convert_to_type, get_model, set_model -from CIME.utils import get_project, get_charge_account, check_name -from CIME.utils import get_current_commit, safe_copy, get_cime_default_driver from CIME.gitinterface import GitInterface from CIME.locked_files import LOCKED_DIR, lock_file -from CIME.XML.machines import Machines -from CIME.XML.pes import Pes -from CIME.XML.files import Files -from CIME.XML.testlist import Testlist +from CIME.status import append_status +from CIME.user_mod_support import apply_user_mods +from CIME.utils import ( + check_name, + convert_to_type, + expect, + get_charge_account, + get_cime_default_driver, + get_cime_root, + get_current_commit, + get_model, + get_project, + safe_copy, + set_model, +) +from CIME.XML.archive import Archive +from CIME.XML.batch import Batch from CIME.XML.component import Component from CIME.XML.compsets import Compsets -from CIME.XML.grids import Grids -from CIME.XML.batch import Batch -from CIME.XML.workflow import Workflow -from CIME.XML.postprocessing import Postprocessing -from CIME.XML.pio import PIO -from CIME.XML.archive import Archive -from CIME.XML.env_test import EnvTest -from CIME.XML.env_mach_specific import EnvMachSpecific +from CIME.XML.env_archive import EnvArchive +from CIME.XML.env_batch import EnvBatch +from CIME.XML.env_build import EnvBuild from CIME.XML.env_case import EnvCase from CIME.XML.env_mach_pes import EnvMachPes -from CIME.XML.env_build import EnvBuild +from CIME.XML.env_mach_specific import EnvMachSpecific +from CIME.XML.env_postprocessing import EnvPostprocessing from CIME.XML.env_run import EnvRun -from CIME.XML.env_archive import EnvArchive -from CIME.XML.env_batch import EnvBatch +from CIME.XML.env_test import EnvTest from CIME.XML.env_workflow import EnvWorkflow -from CIME.XML.env_postprocessing import EnvPostprocessing +from CIME.XML.files import Files from CIME.XML.generic_xml import GenericXML -from CIME.user_mod_support import apply_user_mods -from CIME.aprun import get_aprun_cmd_for_case +from CIME.XML.grids import Grids +from CIME.XML.machines import Machines +from CIME.XML.pes import Pes +from CIME.XML.pio import PIO +from CIME.XML.postprocessing import Postprocessing +from CIME.XML.testlist import Testlist +from CIME.XML.workflow import Workflow logger = logging.getLogger(__name__) @@ -81,25 +99,25 @@ class Case(object): """ - from CIME.case.case_setup import case_setup, _create_case_repo - from CIME.case.case_clone import create_clone, _copy_user_modified_to_clone - from CIME.case.case_test import case_test - from CIME.case.case_submit import check_DA_settings, check_case, submit + from CIME.case.case_clone import _copy_user_modified_to_clone, create_clone + from CIME.case.case_cmpgen_namelists import case_cmpgen_namelists + from CIME.case.case_run import case_run + from CIME.case.case_setup import _create_case_repo, case_setup from CIME.case.case_st_archive import ( + archive_last_restarts, case_st_archive, restore_from_archive, - archive_last_restarts, - test_st_archive, test_env_archive, + test_st_archive, ) - from CIME.case.case_run import case_run - from CIME.case.case_cmpgen_namelists import case_cmpgen_namelists - from CIME.case.preview_namelists import create_dirs, create_namelists + from CIME.case.case_submit import check_case, check_DA_settings, submit + from CIME.case.case_test import case_test from CIME.case.check_input_data import ( check_all_input_data, - stage_refcase, check_input_data, + stage_refcase, ) + from CIME.case.preview_namelists import create_dirs, create_namelists def __init__(self, case_root=None, read_only=True, record=False, non_local=False): if case_root is None: diff --git a/CIME/case/case_clone.py b/CIME/case/case_clone.py index 8ed6ae89948..6f0b070076a 100644 --- a/CIME/case/case_clone.py +++ b/CIME/case/case_clone.py @@ -1,12 +1,16 @@ """ create_clone is a member of the Case class from file case.py """ -import os, glob, shutil -from CIME.XML.standard_module_setup import * -from CIME.utils import expect, check_name, safe_copy, get_model -from CIME.simple_compare import compare_files + +import glob +import logging +import os +import shutil + from CIME.locked_files import lock_file +from CIME.simple_compare import compare_files from CIME.user_mod_support import apply_user_mods +from CIME.utils import check_name, expect, get_cime_root, get_model, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/case/case_cmpgen_namelists.py b/CIME/case/case_cmpgen_namelists.py index 6a2579ada35..78334b7de7f 100644 --- a/CIME/case/case_cmpgen_namelists.py +++ b/CIME/case/case_cmpgen_namelists.py @@ -3,15 +3,19 @@ case_cmpgen_namelists is a member of Class case from file case.py """ -from CIME.XML.standard_module_setup import * - -from CIME.compare_namelists import is_namelist_file, compare_namelist_files +import glob +import logging +import os +import shutil +import stat +import sys +import traceback + +from CIME.compare_namelists import compare_namelist_files, is_namelist_file from CIME.simple_compare import compare_files, compare_runconfigfiles -from CIME.utils import safe_copy, SharedArea from CIME.status import append_status from CIME.test_status import * - -import os, shutil, traceback, stat, glob +from CIME.utils import SharedArea, expect, safe_copy logger = logging.getLogger(__name__) @@ -41,9 +45,11 @@ def _do_full_nl_comp(case, test, compare_name, baseline_root=None): comments = "NLCOMP\n" for item in all_items_to_compare: baseline_counterpart = os.path.join( - baseline_casedocs - if os.path.dirname(item).endswith("CaseDocs") - else baseline_dir, + ( + baseline_casedocs + if os.path.dirname(item).endswith("CaseDocs") + else baseline_dir + ), os.path.basename(item), ) if not os.path.exists(baseline_counterpart): diff --git a/CIME/case/case_run.py b/CIME/case/case_run.py index 5c44c693b14..01b69c6ce29 100644 --- a/CIME/case/case_run.py +++ b/CIME/case/case_run.py @@ -2,17 +2,30 @@ case_run is a member of Class Case '""" -from CIME.XML.standard_module_setup import * +import glob +import logging +import os +import re +import shutil +import time + from CIME.config import Config -from CIME.utils import gzip_existing_file, new_lid from CIME.core.exceptions import CIMEError -from CIME.utils import run_sub_or_cmd, safe_copy, model_log -from CIME.utils import batch_jobid, is_comp_standalone -from CIME.status import append_status, run_and_log_case_status from CIME.get_timing import get_timing from CIME.locked_files import check_lockedfiles - -import shutil, time, sys, os, glob +from CIME.status import append_status, run_and_log_case_status +from CIME.utils import ( + batch_jobid, + expect, + gzip_existing_file, + is_comp_standalone, + model_log, + new_lid, + run_cmd, + run_cmd_no_fail, + run_sub_or_cmd, + safe_copy, +) TERMINATION_TEXT = ("HAS ENDED", "END OF MODEL RUN", "SUCCESSFUL TERMINATION") diff --git a/CIME/case/case_setup.py b/CIME/case/case_setup.py index 38d6a4b0336..6a4f658f6c2 100644 --- a/CIME/case/case_setup.py +++ b/CIME/case/case_setup.py @@ -3,31 +3,32 @@ case_setup is a member of class Case from file case.py """ +import errno +import logging import os +import shutil -from CIME.XML.standard_module_setup import * -from CIME.config import Config -from CIME.XML.machines import Machines from CIME.BuildTools.configure import ( - generate_env_mach_specific, copy_depends_files, + generate_env_mach_specific, ) +from CIME.config import Config +from CIME.gitinterface import GitInterface +from CIME.locked_files import check_lockedfiles, lock_file, unlock_file +from CIME.status import append_case_status, run_and_log_case_status +from CIME.test_status import * from CIME.utils import ( - get_batch_script_for_job, - safe_copy, + CIMEError, + batch_jobid, + copy_local_macros_to_dir, + expect, file_contains_python_function, + get_batch_script_for_job, import_from_file, - copy_local_macros_to_dir, - batch_jobid, run_cmd_no_fail, - CIMEError, + safe_copy, ) -from CIME.status import run_and_log_case_status, append_case_status -from CIME.test_status import * -from CIME.locked_files import unlock_file, lock_file, check_lockedfiles -from CIME.gitinterface import GitInterface - -import errno, shutil +from CIME.XML.machines import Machines logger = logging.getLogger(__name__) @@ -432,9 +433,11 @@ def _case_setup_impl( if ngpus_per_node >= 0: case.set_value( "NGPUS_PER_NODE", - max(1, ngpus_per_node) - if ngpus_per_node <= max_gpus_per_node - else max_gpus_per_node, + ( + max(1, ngpus_per_node) + if ngpus_per_node <= max_gpus_per_node + else max_gpus_per_node + ), ) elif gpu_offload: raise CIMEError( diff --git a/CIME/case/case_st_archive.py b/CIME/case/case_st_archive.py index 932987d1a9a..e8c12692481 100644 --- a/CIME/case/case_st_archive.py +++ b/CIME/case/case_st_archive.py @@ -4,21 +4,26 @@ are members of class Case from file case.py """ -import shutil, glob, re, os +import glob +import logging +import os +import re +import shutil +from os.path import isdir, join -from CIME.XML.standard_module_setup import * +from CIME.date import get_file_date +from CIME.status import run_and_log_case_status from CIME.utils import ( + batch_jobid, + expect, + find_files, ls_sorted_by_mtime, - symlink_force, + run_cmd, safe_copy, - find_files, - batch_jobid, + symlink_force, ) -from CIME.status import run_and_log_case_status -from CIME.date import get_file_date from CIME.XML.archive import Archive from CIME.XML.files import Files -from os.path import isdir, join logger = logging.getLogger(__name__) diff --git a/CIME/case/case_submit.py b/CIME/case/case_submit.py index 6e2e7bc6278..0c4dfc13761 100644 --- a/CIME/case/case_submit.py +++ b/CIME/case/case_submit.py @@ -8,17 +8,19 @@ """ import configparser -from CIME.XML.standard_module_setup import * +import logging +import os + from CIME.core.exceptions import CIMEError -from CIME.utils import expect, get_time_in_seconds -from CIME.status import run_and_log_case_status from CIME.locked_files import ( - unlock_file, - lock_file, check_lockedfile, check_lockedfiles, + lock_file, + unlock_file, ) +from CIME.status import run_and_log_case_status from CIME.test_status import * +from CIME.utils import expect, get_time_in_seconds logger = logging.getLogger(__name__) diff --git a/CIME/case/case_test.py b/CIME/case/case_test.py index c607dd93786..554526171dc 100644 --- a/CIME/case/case_test.py +++ b/CIME/case/case_test.py @@ -3,12 +3,14 @@ case_test is a member of class Case from case.py """ -from CIME.XML.standard_module_setup import * -from CIME.utils import expect, find_system_test, find_proc_id -from CIME.SystemTests.system_tests_common import * -from CIME.status import append_testlog +import logging +import os +import signal +import sys -import sys, signal +from CIME.status import append_testlog +from CIME.SystemTests.system_tests_common import * +from CIME.utils import expect, find_proc_id, find_system_test def _iter_signal_names(): diff --git a/CIME/case/check_input_data.py b/CIME/case/check_input_data.py index cc67e1c6aca..f85f9cb64da 100644 --- a/CIME/case/check_input_data.py +++ b/CIME/case/check_input_data.py @@ -1,12 +1,16 @@ """ API for checking input for testcase """ -from CIME.XML.standard_module_setup import * -from CIME.utils import SharedArea, find_files, safe_copy, expect -from CIME.XML.inputdata import Inputdata -import CIME.Servers -import glob, hashlib, shutil +import glob +import hashlib +import logging +import os +import shutil + +import CIME.Servers +from CIME.utils import SharedArea, expect, find_files, safe_copy +from CIME.XML.inputdata import Inputdata logger = logging.getLogger(__name__) # The inputdata_checksum.dat file will be read into this hash if it's available diff --git a/CIME/case/preview_namelists.py b/CIME/case/preview_namelists.py index 95d276e039a..e9f5848b5a6 100644 --- a/CIME/case/preview_namelists.py +++ b/CIME/case/preview_namelists.py @@ -3,9 +3,12 @@ create_dirs and create_namelists are members of Class case from file case.py """ -from CIME.XML.standard_module_setup import * -from CIME.utils import import_and_run_sub_or_cmd, safe_copy -import time, glob +import glob +import logging +import os +import time + +from CIME.utils import expect, import_and_run_sub_or_cmd, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/code_checker.py b/CIME/code_checker.py index 08a5a021b0f..52fbdd55886 100644 --- a/CIME/code_checker.py +++ b/CIME/code_checker.py @@ -2,24 +2,21 @@ Libraries for checking python code with pylint """ -import os import json +import logging +import os from shutil import which -from CIME.XML.standard_module_setup import * - from CIME.utils import ( - run_cmd, - run_cmd_no_fail, expect, + get_cime_default_driver, get_cime_root, get_src_root, is_python_executable, - get_cime_default_driver, + run_cmd, + run_cmd_no_fail, ) -from multiprocessing.dummy import Pool as ThreadPool - logger = logging.getLogger(__name__) diff --git a/CIME/compare_namelists.py b/CIME/compare_namelists.py index 6e98a7df9a5..7a830ccc18e 100644 --- a/CIME/compare_namelists.py +++ b/CIME/compare_namelists.py @@ -1,4 +1,7 @@ -import os, re, logging +import logging +import os +import re + from CIME.core.exceptions import CIMEError from CIME.utils import expect diff --git a/CIME/compare_test_results.py b/CIME/compare_test_results.py index 35274cc74ba..cef36c1da78 100644 --- a/CIME/compare_test_results.py +++ b/CIME/compare_test_results.py @@ -1,12 +1,14 @@ -import CIME.compare_namelists, CIME.simple_compare +import logging +import os + +import CIME.compare_namelists +import CIME.simple_compare +from CIME.case import Case +from CIME.hist_utils import compare_baseline, get_ts_synopsis from CIME.status import append_status -from CIME.utils import EnvironmentContext, parse_test_name from CIME.test_status import * -from CIME.hist_utils import compare_baseline, get_ts_synopsis -from CIME.case import Case from CIME.test_utils import get_test_status_files - -import os, logging +from CIME.utils import EnvironmentContext, parse_test_name def append_status_cprnc_log(msg, logfile_name, test_dir): diff --git a/CIME/config.py b/CIME/config.py index f63a4bea78d..de34232fc4d 100644 --- a/CIME/config.py +++ b/CIME/config.py @@ -1,10 +1,10 @@ -import os -import re -import sys -import logging import importlib.machinery import importlib.util import inspect +import logging +import os +import re +import sys from pathlib import Path from CIME import utils @@ -185,7 +185,7 @@ def print_method_rst(self): if x[1].__class__ != Config and x[0] not in ignore ] - for (name, sig, doc) in child_methods: + for name, sig, doc in child_methods: if doc is None: continue print(".. code-block::\n") diff --git a/CIME/cs_status.py b/CIME/cs_status.py index 6a65ca4da71..2ab3de40979 100644 --- a/CIME/cs_status.py +++ b/CIME/cs_status.py @@ -4,13 +4,15 @@ """ from __future__ import print_function -from CIME.XML.standard_module_setup import * -from CIME.XML.expected_fails_file import ExpectedFailsFile -from CIME.test_status import TestStatus, SHAREDLIB_BUILD_PHASE, TEST_PEND_STATUS + import os import sys from collections import defaultdict +from CIME.test_status import SHAREDLIB_BUILD_PHASE, TEST_PEND_STATUS, TestStatus +from CIME.utils import expect +from CIME.XML.expected_fails_file import ExpectedFailsFile + def cs_status( test_paths, diff --git a/CIME/cs_status_creator.py b/CIME/cs_status_creator.py index 7a0fd40361a..49b456cb788 100644 --- a/CIME/cs_status_creator.py +++ b/CIME/cs_status_creator.py @@ -2,11 +2,11 @@ Creates a test suite-specific cs.status file from a template """ -from CIME.XML.standard_module_setup import * -import CIME.utils import os import stat +import CIME.utils + def create_cs_status(test_root, test_id, extra_args="", filename=None): """Create a test suite-specific cs.status file from the template diff --git a/CIME/date.py b/CIME/date.py index 0d12b2ceede..0e2179b9007 100644 --- a/CIME/date.py +++ b/CIME/date.py @@ -1,7 +1,9 @@ +import logging import re -from CIME.XML.standard_module_setup import * logger = logging.getLogger(__name__) + + ############################################################################### def get_file_date(filename): ############################################################################### diff --git a/CIME/expected_fails.py b/CIME/expected_fails.py index f8e5339702b..5bc457fcba8 100644 --- a/CIME/expected_fails.py +++ b/CIME/expected_fails.py @@ -2,7 +2,7 @@ Contains the definition of a class to hold information on expected failures for a single test """ -from CIME.XML.standard_module_setup import * +from CIME.utils import expect EXPECTED_FAILURE_COMMENT = "(EXPECTED FAILURE)" UNEXPECTED_FAILURE_COMMENT_START = "(UNEXPECTED" # There will be some additional text after this, before the end parentheses diff --git a/CIME/get_tests.py b/CIME/get_tests.py index a4d4abb3410..72c673e850c 100644 --- a/CIME/get_tests.py +++ b/CIME/get_tests.py @@ -1,7 +1,9 @@ +import os +import sys + import CIME.utils -from CIME.utils import expect, convert_to_seconds, parse_test_name, get_cime_root +from CIME.utils import convert_to_seconds, expect, get_cime_root, parse_test_name from CIME.XML.machines import Machines -import sys, os # Expect that, if a model wants to use python-based test lists, they will have a file # $model/cime_config/tests.py , containing a test dictionary called _TESTS. Currently, @@ -143,6 +145,7 @@ _ALL_TESTS.update(_CIME_TESTS) + ############################################################################### def _get_key_data(raw_dict, key, the_type): ############################################################################### diff --git a/CIME/get_timing.py b/CIME/get_timing.py index 92ac32509de..ab1127e4518 100644 --- a/CIME/get_timing.py +++ b/CIME/get_timing.py @@ -5,11 +5,13 @@ information from a run. """ -from CIME.XML.standard_module_setup import * -from CIME.utils import safe_copy -from CIME.status import append_case_status +import datetime +import logging +import os +import re -import datetime, re +from CIME.status import append_case_status +from CIME.utils import expect, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/gitinterface.py b/CIME/gitinterface.py index 790f1fc70f2..5c5b4d2423a 100644 --- a/CIME/gitinterface.py +++ b/CIME/gitinterface.py @@ -1,7 +1,10 @@ -import sys, shutil, re -from CIME.utils import run_cmd_no_fail +import re +import shutil +import sys from pathlib import Path +from CIME.utils import run_cmd_no_fail + class GitInterface: def __init__(self, repo_path, logger, branch=None): diff --git a/CIME/hist_utils.py b/CIME/hist_utils.py index ab8b387e853..699c7aab969 100644 --- a/CIME/hist_utils.py +++ b/CIME/hist_utils.py @@ -2,23 +2,24 @@ Functions for actions pertaining to history files. """ +import filecmp import logging import os import re -import filecmp import shutil -from CIME.XML.standard_module_setup import * from CIME.config import Config +from CIME.core.exceptions import CIMEError from CIME.test_status import TEST_NO_BASELINES_COMMENT, TEST_STATUS_FILENAME from CIME.utils import ( + SharedArea, + expect, get_current_commit, get_timestamp, - safe_copy, - SharedArea, parse_test_name, + run_cmd, + safe_copy, ) -from CIME.core.exceptions import CIMEError logger = logging.getLogger(__name__) diff --git a/CIME/jenkins_generic_job.py b/CIME/jenkins_generic_job.py index e35a5d04e67..c140d08a9ea 100644 --- a/CIME/jenkins_generic_job.py +++ b/CIME/jenkins_generic_job.py @@ -1,8 +1,18 @@ +import glob +import logging +import os +import re +import shutil +import signal +import sys +import tarfile +import threading +import time + import CIME.wait_for_tests -from CIME.utils import expect, run_cmd_no_fail from CIME.case import Case +from CIME.utils import expect, run_cmd_no_fail -import os, shutil, glob, signal, logging, threading, sys, re, tarfile, time ############################################################################## def cleanup_queue(test_root, test_id): diff --git a/CIME/locked_files.py b/CIME/locked_files.py index 3be6f9c3573..91c2206543d 100644 --- a/CIME/locked_files.py +++ b/CIME/locked_files.py @@ -1,11 +1,12 @@ +import logging +import os from pathlib import Path -from CIME.utils import safe_copy -from CIME.XML.standard_module_setup import * +from CIME.utils import expect, safe_copy +from CIME.XML.env_batch import EnvBatch from CIME.XML.env_build import EnvBuild -from CIME.XML.env_mach_pes import EnvMachPes from CIME.XML.env_case import EnvCase -from CIME.XML.env_batch import EnvBatch +from CIME.XML.env_mach_pes import EnvMachPes from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/namelist.py b/CIME/namelist.py index 486955bca69..e2a7cbdc93c 100644 --- a/CIME/namelist.py +++ b/CIME/namelist.py @@ -102,14 +102,15 @@ # (rather specific) pylint naming conventions. # pylint: disable=line-too-long,too-many-lines,invalid-name +import logging import re from contextlib import contextmanager +from CIME.utils import expect, string_in_list + # Disable these because this is our standard setup # pylint: disable=wildcard-import,unused-wildcard-import -from CIME.XML.standard_module_setup import * -from CIME.utils import expect, string_in_list logger = logging.getLogger(__name__) @@ -250,7 +251,7 @@ def get_fortran_variable_indices(varname, varlen=1, allow_any_len=False): (1, -1, 1) """ m = FORTRAN_NAME_REGEX.search(varname) - (minindex, maxindex, step) = (1, varlen, 1) + minindex, maxindex, step = (1, varlen, 1) if m.group(4) is not None: minindex = int(m.group(4)) @@ -893,7 +894,6 @@ def parse(in_file=None, text=None, groupless=False, convert_tab_to_space=True): class Namelist(object): - """Class representing a Fortran namelist. Public methods: @@ -1184,7 +1184,6 @@ def get_group_variables(self, group_name): def write( self, out_file, groups=None, append=False, format_="nml", sorted_groups=True ): - """Write a the output data (normally fortran namelist) to the out_file As with `parse`, the `out_file` argument can be either a file name, or a @@ -1359,7 +1358,6 @@ def _write_nuopc(self, out_file, groups, sorted_groups): class _NamelistEOF(Exception): - """Exception thrown for an unexpected end-of-file in a namelist. This is an internal helper class, and should never be raised in a context @@ -1381,7 +1379,6 @@ def __str__(self): class _NamelistParseError(Exception): - """Exception thrown when namelist input has a syntax error. This is an internal helper class, and should never be raised in a context @@ -1403,7 +1400,6 @@ def __str__(self): class _NamelistParser(object): # pylint:disable=too-few-public-methods - """Class to validate and read from Fortran namelist input. This is intended to be an internal helper class and should not be used @@ -2222,7 +2218,7 @@ def _parse_name_and_values(self, allow_eof_end=False): break # and if it really is a literal, add it. values.append(literal) - (minindex, maxindex, step) = get_fortran_variable_indices( + minindex, maxindex, step = get_fortran_variable_indices( name, allow_any_len=True ) if (minindex > 1 or maxindex > minindex or step > 1) and maxindex > 0: diff --git a/CIME/nmlgen.py b/CIME/nmlgen.py index abfb5650c07..003ddc024e0 100644 --- a/CIME/nmlgen.py +++ b/CIME/nmlgen.py @@ -7,23 +7,24 @@ # pylint: disable=wildcard-import,unused-wildcard-import import datetime -import re import hashlib +import logging +import os +import re -from CIME.XML.standard_module_setup import * from CIME.namelist import ( Namelist, - parse, character_literal_to_string, - string_to_character_literal, - expand_literal_list, compress_literal_list, + expand_literal_list, merge_literal_lists, + parse, + string_to_character_literal, ) -from CIME.XML.namelist_definition import NamelistDefinition from CIME.utils import expect, safe_copy -from CIME.XML.stream import Stream from CIME.XML.grids import GRID_SEP +from CIME.XML.namelist_definition import NamelistDefinition +from CIME.XML.stream import Stream logger = logging.getLogger(__name__) @@ -66,7 +67,6 @@ class NamelistGenerator(object): - """Utility class for generating namelists for a given component.""" _streams_variables = [] diff --git a/CIME/provenance.py b/CIME/provenance.py index 849bb2757d1..23789033f0e 100644 --- a/CIME/provenance.py +++ b/CIME/provenance.py @@ -4,16 +4,18 @@ Library for saving build/run provenance. """ -from CIME.XML.standard_module_setup import * +import logging +import os +import sys + from CIME.utils import ( SharedArea, convert_to_babylonian_time, + expect, get_current_commit, run_cmd, ) -import sys - logger = logging.getLogger(__name__) diff --git a/CIME/scripts/configure b/CIME/scripts/configure index 73a326da2bf..5bb1d0685e4 100755 --- a/CIME/scripts/configure +++ b/CIME/scripts/configure @@ -24,9 +24,9 @@ real_file_dir = os.path.dirname(os.path.realpath(__file__)) cimeroot = os.path.abspath(os.path.join(real_file_dir, "..", "..")) sys.path.insert(0, cimeroot) +from CIME.BuildTools.configure import configure from CIME.Tools.standard_script_setup import * from CIME.utils import expect, get_model -from CIME.BuildTools.configure import configure from CIME.XML.machines import Machines logger = logging.getLogger(__name__) diff --git a/CIME/scripts/create_clone.py b/CIME/scripts/create_clone.py index 536ae73d232..23ba07b4402 100755 --- a/CIME/scripts/create_clone.py +++ b/CIME/scripts/create_clone.py @@ -1,13 +1,15 @@ #!/usr/bin/env python3 +import re +from argparse import RawTextHelpFormatter + +from CIME.case import Case from CIME.Tools.standard_script_setup import * from CIME.utils import expect -from CIME.case import Case -from argparse import RawTextHelpFormatter -import re logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args): ############################################################################### @@ -144,9 +146,11 @@ def _main_func(): if user_mods_dirs is not None: user_mods_dirs = [ - os.path.abspath(one_user_mods_dir) - if os.path.isdir(one_user_mods_dir) - else one_user_mods_dir + ( + os.path.abspath(one_user_mods_dir) + if os.path.isdir(one_user_mods_dir) + else one_user_mods_dir + ) for one_user_mods_dir in user_mods_dirs ] nint = len(startval) diff --git a/CIME/scripts/create_newcase.py b/CIME/scripts/create_newcase.py index 879a5d167c8..1b219137da6 100755 --- a/CIME/scripts/create_newcase.py +++ b/CIME/scripts/create_newcase.py @@ -6,6 +6,10 @@ Script to create a new CIME Case Control System (CSS) experimental case. """ +from argparse import RawTextHelpFormatter + +from CIME.case import Case +from CIME.config import Config from CIME.Tools.standard_script_setup import * from CIME.utils import ( expect, @@ -13,12 +17,10 @@ get_cime_default_driver, get_src_root, ) -from CIME.config import Config -from CIME.case import Case -from argparse import RawTextHelpFormatter logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args, cimeroot, description): ############################################################################### @@ -384,9 +386,11 @@ def _main_func(description=None): if user_mods_dirs is not None: user_mods_dirs = [ - os.path.abspath(one_user_mods_dir) - if os.path.isdir(one_user_mods_dir) - else one_user_mods_dir + ( + os.path.abspath(one_user_mods_dir) + if os.path.isdir(one_user_mods_dir) + else one_user_mods_dir + ) for one_user_mods_dir in user_mods_dirs ] diff --git a/CIME/scripts/create_test.py b/CIME/scripts/create_test.py index 051c7430592..f2abdf1b19f 100755 --- a/CIME/scripts/create_test.py +++ b/CIME/scripts/create_test.py @@ -26,28 +26,31 @@ If this tool is missing any feature that you need, please add an issue on https://github.com/ESMCI/cime """ + +import argparse +import glob +import math +from argparse import RawTextHelpFormatter + +from CIME import get_tests, utils +from CIME.case import Case +from CIME.config import Config +from CIME.test_scheduler import RUN_PHASE, TestScheduler +from CIME.test_utils import get_tests_from_xml from CIME.Tools.standard_script_setup import * -from CIME import get_tests -from CIME.test_scheduler import TestScheduler, RUN_PHASE -from CIME import utils from CIME.utils import ( - expect, - convert_to_seconds, compute_total_time, convert_to_babylonian_time, - run_cmd_no_fail, + convert_to_seconds, + expect, get_cime_config, + run_cmd_no_fail, ) -from CIME.config import Config from CIME.XML.machines import Machines -from CIME.case import Case -from CIME.test_utils import get_tests_from_xml -from argparse import RawTextHelpFormatter - -import argparse, math, glob logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/scripts/query_config.py b/CIME/scripts/query_config.py index 41b8d804578..79f201ef41d 100755 --- a/CIME/scripts/query_config.py +++ b/CIME/scripts/query_config.py @@ -6,19 +6,18 @@ information will be listed for each. """ +import argparse +import logging import os import sys -import logging -import argparse -from CIME.Tools.standard_script_setup import * from CIME import utils -from CIME.XML.files import Files +from CIME.config import Config from CIME.XML.component import Component from CIME.XML.compsets import Compsets +from CIME.XML.files import Files from CIME.XML.grids import Grids from CIME.XML.machines import Machines -from CIME.config import Config logger = logging.getLogger(__name__) diff --git a/CIME/scripts/query_testlists.py b/CIME/scripts/query_testlists.py index ad00733cd61..02e9386543a 100755 --- a/CIME/scripts/query_testlists.py +++ b/CIME/scripts/query_testlists.py @@ -14,13 +14,14 @@ All of the above support the various --xml-* arguments for subsetting which tests are included. """ -from CIME.Tools.standard_script_setup import * from CIME.test_utils import get_tests_from_xml, test_to_string -from CIME.XML.tests import Tests +from CIME.Tools.standard_script_setup import * from CIME.utils import expect +from CIME.XML.tests import Tests logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/simple_compare.py b/CIME/simple_compare.py index f1292778157..f3c59cf6109 100644 --- a/CIME/simple_compare.py +++ b/CIME/simple_compare.py @@ -1,7 +1,9 @@ -import os, re +import os +import re from CIME.utils import expect + ############################################################################### def _normalize_string_value(value, case): ############################################################################### diff --git a/CIME/status.py b/CIME/status.py index 08618c7219d..e7093e4b2f7 100644 --- a/CIME/status.py +++ b/CIME/status.py @@ -1,6 +1,10 @@ # These routines were moved from utils.py to avoid circular dependancies -import time, os, sys, logging -from CIME.utils import Timeout, CASE_SUCCESS, CASE_FAILURE +import logging +import os +import sys +import time + +from CIME.utils import CASE_FAILURE, CASE_SUCCESS, Timeout logger = logging.getLogger(__name__) diff --git a/CIME/test_scheduler.py b/CIME/test_scheduler.py index d94a99e08fa..c20fc3b3e4c 100644 --- a/CIME/test_scheduler.py +++ b/CIME/test_scheduler.py @@ -8,45 +8,53 @@ they can be run outside the context of TestScheduler. """ +import glob +import logging import os -import traceback, stat, threading, time, glob +import re +import stat +import sys +import threading +import time +import traceback from collections import OrderedDict -from CIME.XML.standard_module_setup import * -from CIME.get_tests import get_recommended_test_time, get_build_groups, is_perf_test +from CIME.build import post_build +from CIME.case import Case +from CIME.config import Config +from CIME.cs_status_creator import create_cs_status +from CIME.get_tests import get_build_groups, get_recommended_test_time, is_perf_test +from CIME.hist_utils import generate_teststatus +from CIME.locked_files import lock_file +from CIME.provenance import get_recommended_test_time_based_on_past from CIME.status import append_status, append_testlog +from CIME.SystemTests.test_mods import find_test_mods +from CIME.test_status import * from CIME.utils import ( TESTS_FAILED_ERR_CODE, - parse_test_name, - get_full_test_name, - get_model, + CIMEError, convert_to_seconds, + expect, + get_cime_default_driver, get_cime_root, + get_full_test_name, + get_model, + get_project, get_src_root, - get_tools_path, get_template_path, - get_project, get_timestamp, - get_cime_default_driver, - CIMEError, + get_tools_path, + parse_test_name, + run_cmd, ) -from CIME.config import Config -from CIME.test_status import * -from CIME.XML.machines import Machines -from CIME.XML.generic_xml import GenericXML -from CIME.XML.env_test import EnvTest +from CIME.wait_for_tests import wait_for_tests +from CIME.XML.component import Component from CIME.XML.env_mach_pes import EnvMachPes +from CIME.XML.env_test import EnvTest from CIME.XML.files import Files -from CIME.XML.component import Component +from CIME.XML.generic_xml import GenericXML +from CIME.XML.machines import Machines from CIME.XML.tests import Tests -from CIME.case import Case -from CIME.wait_for_tests import wait_for_tests -from CIME.provenance import get_recommended_test_time_based_on_past -from CIME.locked_files import lock_file -from CIME.cs_status_creator import create_cs_status -from CIME.hist_utils import generate_teststatus -from CIME.build import post_build -from CIME.SystemTests.test_mods import find_test_mods logger = logging.getLogger(__name__) diff --git a/CIME/test_status.py b/CIME/test_status.py index 24ce5bc6e74..d3f4580403f 100644 --- a/CIME/test_status.py +++ b/CIME/test_status.py @@ -26,9 +26,12 @@ """ -from CIME.XML.standard_module_setup import * -import os, itertools +import itertools +import logging +import os + from CIME import expected_fails +from CIME.utils import expect TEST_STATUS_FILENAME = "TestStatus" diff --git a/CIME/test_utils.py b/CIME/test_utils.py index 6dea4e3bddc..c3a3c964da0 100644 --- a/CIME/test_utils.py +++ b/CIME/test_utils.py @@ -2,12 +2,16 @@ Utility functions used in test_scheduler.py, and by other utilities that need to get test lists. """ + import glob -from CIME.XML.standard_module_setup import * -from CIME.XML.testlist import Testlist -from CIME.XML.files import Files -from CIME.test_status import TEST_STATUS_FILENAME +import logging +import os + import CIME.utils +from CIME.test_status import TEST_STATUS_FILENAME +from CIME.utils import expect +from CIME.XML.files import Files +from CIME.XML.testlist import Testlist logger = logging.getLogger(__name__) diff --git a/CIME/tests/base.py b/CIME/tests/base.py index 8873924b60a..b2c4fea8bd9 100644 --- a/CIME/tests/base.py +++ b/CIME/tests/base.py @@ -2,17 +2,16 @@ import glob import os -import tempfile -import time -import signal import shutil +import signal import stat import sys +import tempfile +import time import unittest from CIME import utils from CIME.config import Config -from CIME.XML.machines import Machines def typed_os_environ(key, default_value, expected_type=None): diff --git a/CIME/tests/custom_assertions_test_status.py b/CIME/tests/custom_assertions_test_status.py index 16dc44e5cea..365b0f9d34b 100644 --- a/CIME/tests/custom_assertions_test_status.py +++ b/CIME/tests/custom_assertions_test_status.py @@ -3,11 +3,11 @@ can be used when testing TestStatus. """ -from CIME.XML.standard_module_setup import * - -import unittest import re +import unittest + from CIME import test_status +from CIME.utils import expect class CustomAssertionsTestStatus(unittest.TestCase): diff --git a/CIME/tests/scripts_regression_tests.py b/CIME/tests/scripts_regression_tests.py index 66f4c015298..7590e53e3cf 100755 --- a/CIME/tests/scripts_regression_tests.py +++ b/CIME/tests/scripts_regression_tests.py @@ -5,46 +5,31 @@ to confirm overall CIME correctness. """ -import glob, os, re, shutil, signal, sys, tempfile, threading, time, logging, unittest, getpass, filecmp, time, atexit, functools +import atexit +import functools +import logging +import os +import sys +import unittest CIMEROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) sys.path.insert(0, CIMEROOT) -from xml.etree.ElementTree import ParseError - -import subprocess, argparse +import argparse +import subprocess subprocess.call('/bin/rm -f $(find . -name "*.pyc")', shell=True, cwd=CIMEROOT) -import stat as osstat - -import collections +import CIME.test_scheduler +import CIME.wait_for_tests +from CIME import utils +from CIME.config import Config +from CIME.tests.base import BaseTestCase from CIME.utils import ( - run_cmd, - run_cmd_no_fail, - get_lids, get_current_commit, - safe_copy, - CIMEError, - get_cime_root, - get_src_root, - Timeout, - import_from_file, get_model, ) -import CIME.test_scheduler, CIME.wait_for_tests -from CIME import get_tests -from CIME.test_scheduler import TestScheduler -from CIME.XML.env_run import EnvRun from CIME.XML.machines import Machines -from CIME.XML.files import Files -from CIME.case import Case -from CIME.code_checker import check_code, get_all_checkable_files -from CIME.test_status import * -from CIME.provenance import get_test_success, save_test_success -from CIME import utils -from CIME.tests.base import BaseTestCase -from CIME.config import Config os.environ["CIME_GLOBAL_WALLTIME"] = "0:05:00" diff --git a/CIME/tests/test_sys_bless_tests_results.py b/CIME/tests/test_sys_bless_tests_results.py index 2f2beeb461d..9d8b061b466 100644 --- a/CIME/tests/test_sys_bless_tests_results.py +++ b/CIME/tests/test_sys_bless_tests_results.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import glob -import re import os +import re import stat from CIME import utils diff --git a/CIME/tests/test_sys_cime_case.py b/CIME/tests/test_sys_cime_case.py index c4095972d93..3b142118e53 100644 --- a/CIME/tests/test_sys_cime_case.py +++ b/CIME/tests/test_sys_cime_case.py @@ -8,10 +8,10 @@ import time from CIME import utils -from CIME.tests import base from CIME.case.case import Case -from CIME.XML.env_run import EnvRun +from CIME.tests import base from CIME.utils import find_system_test +from CIME.XML.env_run import EnvRun try: collectionsAbc = collections.abc diff --git a/CIME/tests/test_sys_create_newcase.py b/CIME/tests/test_sys_create_newcase.py index 1be636aff36..fce9743de09 100644 --- a/CIME/tests/test_sys_create_newcase.py +++ b/CIME/tests/test_sys_create_newcase.py @@ -7,9 +7,9 @@ import sys from CIME import utils -from CIME.tests import base -from CIME.case.case import Case from CIME.build import CmakeTmpBuildDir +from CIME.case.case import Case +from CIME.tests import base class TestCreateNewcase(base.BaseTestCase): diff --git a/CIME/tests/test_sys_full_system.py b/CIME/tests/test_sys_full_system.py index ad4485d2b81..d6e334839a7 100644 --- a/CIME/tests/test_sys_full_system.py +++ b/CIME/tests/test_sys_full_system.py @@ -2,10 +2,7 @@ import os -from CIME import get_tests -from CIME import test_status -from CIME import utils -from CIME import wait_for_tests +from CIME import get_tests, test_status, utils, wait_for_tests from CIME.tests import base diff --git a/CIME/tests/test_sys_jenkins_generic_job.py b/CIME/tests/test_sys_jenkins_generic_job.py index 30b31c5c8d6..19572717418 100644 --- a/CIME/tests/test_sys_jenkins_generic_job.py +++ b/CIME/tests/test_sys_jenkins_generic_job.py @@ -7,8 +7,7 @@ import threading import time -from CIME import get_tests -from CIME import utils +from CIME import get_tests, utils from CIME.tests import base diff --git a/CIME/tests/test_sys_manage_and_query.py b/CIME/tests/test_sys_manage_and_query.py index 31ae3392bdb..f99b47aefb9 100644 --- a/CIME/tests/test_sys_manage_and_query.py +++ b/CIME/tests/test_sys_manage_and_query.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -from CIME import utils from CIME.tests import base from CIME.XML.files import Files diff --git a/CIME/tests/test_sys_save_timings.py b/CIME/tests/test_sys_save_timings.py index ee4a964c4b3..c5579489330 100644 --- a/CIME/tests/test_sys_save_timings.py +++ b/CIME/tests/test_sys_save_timings.py @@ -4,10 +4,9 @@ import glob import os -from CIME import provenance -from CIME import utils -from CIME.tests import base +from CIME import provenance, utils from CIME.case.case import Case +from CIME.tests import base class TestSaveTimings(base.BaseTestCase): diff --git a/CIME/tests/test_sys_single_submit.py b/CIME/tests/test_sys_single_submit.py index fed850263df..beeb80207b8 100644 --- a/CIME/tests/test_sys_single_submit.py +++ b/CIME/tests/test_sys_single_submit.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -from CIME import utils from CIME.tests import base diff --git a/CIME/tests/test_sys_test_scheduler.py b/CIME/tests/test_sys_test_scheduler.py index 1b72ed78d0b..df42ad14180 100755 --- a/CIME/tests/test_sys_test_scheduler.py +++ b/CIME/tests/test_sys_test_scheduler.py @@ -1,16 +1,13 @@ #!/usr/bin/env python3 -import re import glob import logging import os +import re import unittest from unittest import mock -from CIME import get_tests -from CIME import utils -from CIME import test_status -from CIME import test_scheduler +from CIME import get_tests, test_scheduler, test_status, utils from CIME.tests import base diff --git a/CIME/tests/test_sys_unittest.py b/CIME/tests/test_sys_unittest.py index 8bfd21eb16f..c22ff7d750c 100755 --- a/CIME/tests/test_sys_unittest.py +++ b/CIME/tests/test_sys_unittest.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 import os +import re import shutil import sys -import re from CIME import utils from CIME.tests import base diff --git a/CIME/tests/test_sys_wait_for_tests.py b/CIME/tests/test_sys_wait_for_tests.py index 3484c9af7a5..f0afd6d712f 100644 --- a/CIME/tests/test_sys_wait_for_tests.py +++ b/CIME/tests/test_sys_wait_for_tests.py @@ -1,17 +1,16 @@ #!/usr/bin/env python3 import os -import signal import shutil +import signal import sys -import time import threading +import time -from CIME import utils -from CIME import test_status -from CIME.wait_for_tests import ENV_VAR_KEEP_CDASH +from CIME import test_status, utils from CIME.tests import base from CIME.tests import utils as test_utils +from CIME.wait_for_tests import ENV_VAR_KEEP_CDASH class TestWaitForTests(base.BaseTestCase): @@ -338,9 +337,11 @@ def live_test_impl(self, testdir, expected_results, last_phase, last_status): ) as ts: ts.set_status( core_phase, - last_status - if core_phase == last_phase - else test_status.TEST_PASS_STATUS, + ( + last_status + if core_phase == last_phase + else test_status.TEST_PASS_STATUS + ), ) time.sleep(5) diff --git a/CIME/tests/test_unit_baselines_performance.py b/CIME/tests/test_unit_baselines_performance.py index 0ef5b387fa4..8f8e2e574d0 100644 --- a/CIME/tests/test_unit_baselines_performance.py +++ b/CIME/tests/test_unit_baselines_performance.py @@ -3,8 +3,8 @@ import gzip import tempfile import unittest -from unittest import mock from pathlib import Path +from unittest import mock from CIME.baselines import performance from CIME.tests.test_unit_system_tests import CPLLOG @@ -300,7 +300,7 @@ def test_perf_compare_throughput_baseline_no_baseline( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_throughput_baseline( + below_tolerance, comment = performance.perf_compare_throughput_baseline( case ) @@ -332,7 +332,7 @@ def test_perf_compare_throughput_baseline_no_tolerance( None, ) - (below_tolerance, comment) = performance.perf_compare_throughput_baseline( + below_tolerance, comment = performance.perf_compare_throughput_baseline( case ) @@ -364,7 +364,7 @@ def test_perf_compare_throughput_baseline_above_threshold( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_throughput_baseline( + below_tolerance, comment = performance.perf_compare_throughput_baseline( case ) @@ -396,7 +396,7 @@ def test_perf_compare_throughput_baseline( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_throughput_baseline( + below_tolerance, comment = performance.perf_compare_throughput_baseline( case ) @@ -433,7 +433,7 @@ def test_perf_compare_memory_baseline_no_baseline( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) + below_tolerance, comment = performance.perf_compare_memory_baseline(case) assert below_tolerance assert ( @@ -464,7 +464,7 @@ def test_perf_compare_memory_baseline_not_enough_samples( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) + below_tolerance, comment = performance.perf_compare_memory_baseline(case) assert below_tolerance is None assert comment == "Found 2 memory usage samples, need atleast 4" @@ -524,7 +524,7 @@ def test_perf_compare_memory_baseline_no_tolerance( None, ) - (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) + below_tolerance, comment = performance.perf_compare_memory_baseline(case) assert below_tolerance assert ( @@ -559,7 +559,7 @@ def test_perf_compare_memory_baseline_above_threshold( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) + below_tolerance, comment = performance.perf_compare_memory_baseline(case) assert not below_tolerance assert ( @@ -594,7 +594,7 @@ def test_perf_compare_memory_baseline( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) + below_tolerance, comment = performance.perf_compare_memory_baseline(case) assert below_tolerance assert ( diff --git a/CIME/tests/test_unit_bless_test_results.py b/CIME/tests/test_unit_bless_test_results.py index 058c10caab8..63af7ecc140 100644 --- a/CIME/tests/test_unit_bless_test_results.py +++ b/CIME/tests/test_unit_bless_test_results.py @@ -1,21 +1,20 @@ import re -import unittest import tempfile +import unittest from contextlib import ExitStack -from unittest import mock from pathlib import Path +from unittest import mock from CIME.bless_test_results import ( - bless_test_results, - _bless_throughput, _bless_memory, + _bless_throughput, bless_history, bless_namelists, + bless_test_results, is_hist_bless_needed, ) from CIME.test_status import ALL_PHASES, GENERATE_PHASE from CIME.tests import utils as test_utils -from CIME.core.exceptions import CIMEError class TestUnitBlessTestResults(unittest.TestCase): diff --git a/CIME/tests/test_unit_build.py b/CIME/tests/test_unit_build.py index cb9ea6d3f81..0afce20bb0b 100644 --- a/CIME/tests/test_unit_build.py +++ b/CIME/tests/test_unit_build.py @@ -3,7 +3,6 @@ import os import unittest from unittest import mock -from pathlib import Path from CIME import build from CIME.tests.utils import mock_case diff --git a/CIME/tests/test_unit_case.py b/CIME/tests/test_unit_case.py index 75fcbeed490..5cb73e198d4 100755 --- a/CIME/tests/test_unit_case.py +++ b/CIME/tests/test_unit_case.py @@ -1,13 +1,12 @@ #!/usr/bin/env python3 import os +import tempfile import unittest from unittest import mock -import tempfile -from CIME.case import case_submit -from CIME.case import Case from CIME import utils +from CIME.case import Case, case_submit from CIME.tests.utils import mock_case diff --git a/CIME/tests/test_unit_case_fake.py b/CIME/tests/test_unit_case_fake.py index 448931ecc7c..adc3252aa85 100755 --- a/CIME/tests/test_unit_case_fake.py +++ b/CIME/tests/test_unit_case_fake.py @@ -4,10 +4,10 @@ This module contains unit tests of CaseFake """ -import unittest -import tempfile import os import shutil +import tempfile +import unittest from CIME.tests.case_fake import CaseFake diff --git a/CIME/tests/test_unit_case_run.py b/CIME/tests/test_unit_case_run.py index ce74f675366..3b869d6e49d 100644 --- a/CIME/tests/test_unit_case_run.py +++ b/CIME/tests/test_unit_case_run.py @@ -1,9 +1,8 @@ import unittest from unittest import mock +from CIME.case.case_run import TERMINATION_TEXT, _post_run_check from CIME.core.exceptions import CIMEError -from CIME.case.case_run import TERMINATION_TEXT -from CIME.case.case_run import _post_run_check def _case_post_run_check(): diff --git a/CIME/tests/test_unit_case_setup.py b/CIME/tests/test_unit_case_setup.py index a00dcf1b413..80ad93d8b46 100644 --- a/CIME/tests/test_unit_case_setup.py +++ b/CIME/tests/test_unit_case_setup.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 +import contextlib import os -import unittest import tempfile -import contextlib +import unittest from pathlib import Path from unittest import mock diff --git a/CIME/tests/test_unit_case_st_archive.py b/CIME/tests/test_unit_case_st_archive.py index bf27ab1c201..5b72f8f8765 100644 --- a/CIME/tests/test_unit_case_st_archive.py +++ b/CIME/tests/test_unit_case_st_archive.py @@ -7,8 +7,8 @@ from CIME import date from CIME.case import case_st_archive -from CIME.XML import env_archive as _env_archive from CIME.tests import utils +from CIME.XML import env_archive as _env_archive UNSET = r""" diff --git a/CIME/tests/test_unit_compare_test_results.py b/CIME/tests/test_unit_compare_test_results.py index 4844a96c1a6..14e79d36345 100755 --- a/CIME/tests/test_unit_compare_test_results.py +++ b/CIME/tests/test_unit_compare_test_results.py @@ -4,13 +4,12 @@ This module contains unit tests for compare_test_results """ -import unittest -import tempfile import os import shutil +import tempfile +import unittest -from CIME import utils -from CIME import compare_test_results +from CIME import compare_test_results, utils from CIME.test_status import * from CIME.tests.case_fake import CaseFake diff --git a/CIME/tests/test_unit_compare_two.py b/CIME/tests/test_unit_compare_two.py index e5f96001bc0..4cece00b660 100755 --- a/CIME/tests/test_unit_compare_two.py +++ b/CIME/tests/test_unit_compare_two.py @@ -9,16 +9,16 @@ # # pylint:disable=protected-access -import unittest -from collections import namedtuple import functools import os import shutil import tempfile +import unittest +from collections import namedtuple from unittest import mock -from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo import CIME.test_status as test_status +from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.tests.case_fake import CaseFake # ======================================================================== diff --git a/CIME/tests/test_unit_config.py b/CIME/tests/test_unit_config.py index 93da359b046..f928b8a7479 100644 --- a/CIME/tests/test_unit_config.py +++ b/CIME/tests/test_unit_config.py @@ -1,8 +1,8 @@ import os -import unittest import tempfile -from unittest import mock +import unittest from pathlib import Path +from unittest import mock from CIME.config import Config diff --git a/CIME/tests/test_unit_configure.py b/CIME/tests/test_unit_configure.py index 693abccdfbd..7dc8971e2a3 100644 --- a/CIME/tests/test_unit_configure.py +++ b/CIME/tests/test_unit_configure.py @@ -1,11 +1,10 @@ import os -import unittest -import tempfile import shutil - -from CIME.utils import expect +import tempfile +import unittest from CIME.BuildTools.configure import generate_env_mach_specific +from CIME.utils import expect from CIME.XML.machines import Machines diff --git a/CIME/tests/test_unit_cs_status.py b/CIME/tests/test_unit_cs_status.py index 70efd47935f..1e44ce996e0 100755 --- a/CIME/tests/test_unit_cs_status.py +++ b/CIME/tests/test_unit_cs_status.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 import io -import unittest -import shutil import os -import tempfile import re -from CIME.cs_status import cs_status +import shutil +import tempfile +import unittest + from CIME import test_status +from CIME.cs_status import cs_status from CIME.tests.custom_assertions_test_status import CustomAssertionsTestStatus diff --git a/CIME/tests/test_unit_custom_assertions_test_status.py b/CIME/tests/test_unit_custom_assertions_test_status.py index 99e6cb05d04..4715377e5c8 100755 --- a/CIME/tests/test_unit_custom_assertions_test_status.py +++ b/CIME/tests/test_unit_custom_assertions_test_status.py @@ -5,6 +5,7 @@ """ import unittest + from CIME import test_status from CIME.tests.custom_assertions_test_status import CustomAssertionsTestStatus diff --git a/CIME/tests/test_unit_doctest.py b/CIME/tests/test_unit_doctest.py index 571b9af41db..1b972c0a6c9 100644 --- a/CIME/tests/test_unit_doctest.py +++ b/CIME/tests/test_unit_doctest.py @@ -1,17 +1,7 @@ #!/usr/bin/env python3 -import glob -import re import os -import stat -import doctest -import sys -import pkgutil -import unittest -import functools -import CIME -from CIME import utils from CIME.tests import base diff --git a/CIME/tests/test_unit_expected_fails_file.py b/CIME/tests/test_unit_expected_fails_file.py index 30c9dd3a6e8..d2898a5022e 100755 --- a/CIME/tests/test_unit_expected_fails_file.py +++ b/CIME/tests/test_unit_expected_fails_file.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -import unittest import os import shutil import tempfile -from CIME.XML.expected_fails_file import ExpectedFailsFile +import unittest + from CIME.core.exceptions import CIMEError from CIME.expected_fails import ExpectedFails +from CIME.XML.expected_fails_file import ExpectedFailsFile class TestExpectedFailsFile(unittest.TestCase): diff --git a/CIME/tests/test_unit_fake_case.py b/CIME/tests/test_unit_fake_case.py index b7b20ac6abc..47e6fe88c5f 100755 --- a/CIME/tests/test_unit_fake_case.py +++ b/CIME/tests/test_unit_fake_case.py @@ -5,12 +5,12 @@ This is seperate from FakeCase that's under CIME/tests """ -import unittest import os -from CIME.core.exceptions import CIMEError -from CIME.utils import get_model, GLOBAL, get_src_root +import unittest from CIME.BuildTools.configure import FakeCase +from CIME.core.exceptions import CIMEError +from CIME.utils import get_model, get_src_root class TestFakeCase(unittest.TestCase): diff --git a/CIME/tests/test_unit_grids.py b/CIME/tests/test_unit_grids.py index e33349d52b5..2e2da3878b5 100755 --- a/CIME/tests/test_unit_grids.py +++ b/CIME/tests/test_unit_grids.py @@ -13,13 +13,14 @@ # # pylint:disable=line-too-long -import unittest import os import shutil import string import tempfile -from CIME.XML.grids import Grids, _ComponentGrids, _add_grid_info, _strip_grid_from_name +import unittest + from CIME.core.exceptions import CIMEError +from CIME.XML.grids import Grids, _add_grid_info, _ComponentGrids, _strip_grid_from_name class TestGrids(unittest.TestCase): diff --git a/CIME/tests/test_unit_hist_utils.py b/CIME/tests/test_unit_hist_utils.py index fe6d4866c34..86fcc5a9c95 100644 --- a/CIME/tests/test_unit_hist_utils.py +++ b/CIME/tests/test_unit_hist_utils.py @@ -1,9 +1,7 @@ -import io import unittest from unittest import mock from CIME.hist_utils import copy_histfiles -from CIME.XML.archive import Archive class TestHistUtils(unittest.TestCase): diff --git a/CIME/tests/test_unit_locked_files.py b/CIME/tests/test_unit_locked_files.py index 2732f16f3f0..abd858433b2 100644 --- a/CIME/tests/test_unit_locked_files.py +++ b/CIME/tests/test_unit_locked_files.py @@ -1,13 +1,11 @@ import tempfile import unittest -from unittest import mock from pathlib import Path +from unittest import mock from CIME import locked_files from CIME.core.exceptions import CIMEError -from CIME.XML.entry_id import EntryID from CIME.XML.env_batch import EnvBatch -from CIME.XML.files import Files def create_batch_system(env_batch, batch_submit_value=None): diff --git a/CIME/tests/test_unit_nmlgen.py b/CIME/tests/test_unit_nmlgen.py index 52e04d28856..76c274871be 100644 --- a/CIME/tests/test_unit_nmlgen.py +++ b/CIME/tests/test_unit_nmlgen.py @@ -1,10 +1,11 @@ -from collections import OrderedDict import tempfile import unittest +from collections import OrderedDict from unittest import mock from CIME.nmlgen import NamelistGenerator + # pylint: disable=protected-access class TestNamelistGenerator(unittest.TestCase): def test_init_defaults(self): diff --git a/CIME/tests/test_unit_paramgen.py b/CIME/tests/test_unit_paramgen.py index 26b2be4ac6e..698a77093a1 100755 --- a/CIME/tests/test_unit_paramgen.py +++ b/CIME/tests/test_unit_paramgen.py @@ -13,8 +13,9 @@ # # pylint:disable=line-too-long -import unittest import tempfile +import unittest + from CIME.ParamGen.paramgen import ParamGen ############### diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index c95c7fd2d9a..f2bbff1924f 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -1,18 +1,17 @@ #!/usr/bin/env python3 -import os -import tempfile import gzip +import os import re -from re import A +import tempfile import unittest -from unittest import mock from pathlib import Path +from unittest import mock from CIME.config import Config from CIME.SystemTests.system_tests_common import SystemTestsCommon -from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.SystemTests.system_tests_compare_n import SystemTestsCompareN +from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo CPLLOG = """ tStamp_write: model date = 00010102 0 wall clock = 2023-09-19 19:39:42 avg dt = 0.33 dt = 0.33 diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py index 0bb33f8d605..3bcf4a1713d 100644 --- a/CIME/tests/test_unit_system_tests_mvk.py +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 -import os -import json -import unittest -import tempfile import contextlib +import json +import os import sysconfig +import tempfile +import unittest from pathlib import Path from unittest import mock + from CIME.tests.utils import chdir evv4esm = False diff --git a/CIME/tests/test_unit_test_status.py b/CIME/tests/test_unit_test_status.py index 9b3036801fc..061e2cafe40 100755 --- a/CIME/tests/test_unit_test_status.py +++ b/CIME/tests/test_unit_test_status.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -import unittest import os -from CIME import test_status -from CIME import expected_fails +import unittest + +from CIME import expected_fails, test_status from CIME.tests.custom_assertions_test_status import CustomAssertionsTestStatus diff --git a/CIME/tests/test_unit_two_link_to_case2_output.py b/CIME/tests/test_unit_two_link_to_case2_output.py index 2860db08ac1..5d59a6ca298 100755 --- a/CIME/tests/test_unit_two_link_to_case2_output.py +++ b/CIME/tests/test_unit_two_link_to_case2_output.py @@ -10,10 +10,11 @@ # # pylint:disable=protected-access -import unittest import os import shutil import tempfile +import unittest + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.tests.case_fake import CaseFake diff --git a/CIME/tests/test_unit_user_mod_support.py b/CIME/tests/test_unit_user_mod_support.py index c2d0fc37e13..ca7289f1596 100755 --- a/CIME/tests/test_unit_user_mod_support.py +++ b/CIME/tests/test_unit_user_mod_support.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 -import unittest +import os import shutil import tempfile -import os -from CIME.user_mod_support import apply_user_mods +import unittest + from CIME.core.exceptions import CIMEError +from CIME.user_mod_support import apply_user_mods # ======================================================================== # Define some parameters diff --git a/CIME/tests/test_unit_user_nl_utils.py b/CIME/tests/test_unit_user_nl_utils.py index 9220182eeeb..2ec199874b8 100755 --- a/CIME/tests/test_unit_user_nl_utils.py +++ b/CIME/tests/test_unit_user_nl_utils.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -import unittest import os import shutil import tempfile +import unittest + from CIME.SystemTests.test_utils import user_nl_utils diff --git a/CIME/tests/test_unit_utils.py b/CIME/tests/test_unit_utils.py index 8abde4d8fef..00f8fd1a047 100755 --- a/CIME/tests/test_unit_utils.py +++ b/CIME/tests/test_unit_utils.py @@ -1,21 +1,20 @@ #!/usr/bin/env python3 import os -import stat import shutil import sys import tempfile - import unittest from unittest import mock + from CIME.status import run_and_log_case_status from CIME.utils import ( - indent_string, - import_from_file, _line_defines_python_function, - file_contains_python_function, copy_globs, + file_contains_python_function, import_and_run_sub_or_cmd, + import_from_file, + indent_string, ) diff --git a/CIME/tests/test_unit_xml_archive_base.py b/CIME/tests/test_unit_xml_archive_base.py index 98f58055c9c..42789333dc0 100644 --- a/CIME/tests/test_unit_xml_archive_base.py +++ b/CIME/tests/test_unit_xml_archive_base.py @@ -1,12 +1,10 @@ #!/usr/bin/env python3 -import os import io -import unittest import tempfile +import unittest from contextlib import contextmanager from pathlib import Path -from unittest import mock from CIME.XML.archive_base import ArchiveBase diff --git a/CIME/tests/test_unit_xml_env_batch.py b/CIME/tests/test_unit_xml_env_batch.py index 5b1295c4a68..b3777031d27 100755 --- a/CIME/tests/test_unit_xml_env_batch.py +++ b/CIME/tests/test_unit_xml_env_batch.py @@ -1,16 +1,16 @@ #!/usr/bin/env python3 import os -import unittest import tempfile +import unittest from contextlib import ExitStack from unittest import mock +from CIME.BuildTools.configure import FakeCase from CIME.core.exceptions import CIMEError from CIME.utils import expect from CIME.XML.env_batch import EnvBatch, get_job_deps from CIME.XML.env_workflow import EnvWorkflow -from CIME.BuildTools.configure import FakeCase # pylint: disable=unused-argument diff --git a/CIME/tests/test_unit_xml_env_mach_specific.py b/CIME/tests/test_unit_xml_env_mach_specific.py index a900034fad8..ef72a061ac5 100644 --- a/CIME/tests/test_unit_xml_env_mach_specific.py +++ b/CIME/tests/test_unit_xml_env_mach_specific.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -import unittest import tempfile +import unittest from unittest import mock from CIME import utils diff --git a/CIME/tests/test_unit_xml_grids.py b/CIME/tests/test_unit_xml_grids.py index cceae5b419d..292012033bc 100644 --- a/CIME/tests/test_unit_xml_grids.py +++ b/CIME/tests/test_unit_xml_grids.py @@ -1,10 +1,6 @@ import os -import io -import unittest import tempfile -from contextlib import contextmanager -from pathlib import Path -from unittest import mock +import unittest from CIME.core.exceptions import CIMEError from CIME.XML.grids import Grids diff --git a/CIME/tests/test_unit_xml_tests.py b/CIME/tests/test_unit_xml_tests.py index 898e00bb0b8..23d5d8fac4c 100644 --- a/CIME/tests/test_unit_xml_tests.py +++ b/CIME/tests/test_unit_xml_tests.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import re -import unittest import tempfile +import unittest from pathlib import Path from unittest import mock diff --git a/CIME/tests/utils.py b/CIME/tests/utils.py index 5987f0fd18e..d74d0798f44 100644 --- a/CIME/tests/utils.py +++ b/CIME/tests/utils.py @@ -1,19 +1,16 @@ +import contextlib import io import os -import tempfile -import signal import shutil import sys -import time -import contextlib +import tempfile +import xml.etree.ElementTree as ET from collections.abc import Iterable from unittest import mock -import xml.etree.ElementTree as ET -from CIME import utils from CIME import test_status -from CIME.utils import expect from CIME.case import Case +from CIME.utils import expect from CIME.XML.entry_id import EntryID MACRO_PRESERVE_ENV = [ @@ -321,7 +318,6 @@ def get_default_compiler(self): class MakefileTester(object): - """Helper class for checking Makefile output. Public methods: @@ -421,7 +417,6 @@ def assert_variable_matches(self, var_name, regex, env=None, var=None): class CMakeTester(object): - """Helper class for checking CMake output. Public methods: diff --git a/CIME/user_mod_support.py b/CIME/user_mod_support.py index 2c182a19dbd..b4eeec21c1a 100644 --- a/CIME/user_mod_support.py +++ b/CIME/user_mod_support.py @@ -2,9 +2,11 @@ user_mod_support.py """ -from CIME.XML.standard_module_setup import * -from CIME.utils import expect, run_cmd_no_fail, safe_copy import glob +import logging +import os + +from CIME.utils import expect, run_cmd_no_fail, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/utils.py b/CIME/utils.py index 948632b4068..52d38c76ed8 100644 --- a/CIME/utils.py +++ b/CIME/utils.py @@ -3,12 +3,27 @@ Warning: you cannot use CIME Classes in this module as it causes circular dependencies """ -import shlex import configparser -import io, logging, gzip, sys, os, time, re, shutil, glob, string, random, importlib, fnmatch +import errno +import filecmp +import fnmatch +import glob +import gzip +import importlib import importlib.util -import errno, signal, warnings, filecmp +import io +import logging +import os +import random +import re +import shlex +import shutil +import signal import stat as statlib +import string +import sys +import time +import warnings from argparse import Action from contextlib import contextmanager diff --git a/CIME/wait_for_tests.py b/CIME/wait_for_tests.py index a32278ed052..3f4dc348745 100644 --- a/CIME/wait_for_tests.py +++ b/CIME/wait_for_tests.py @@ -1,18 +1,26 @@ # pylint: disable=import-error -import queue, os, time, threading, socket, signal, shutil, glob, tempfile -from pathlib import Path +import glob # pylint: disable=import-error import logging +import os +import queue +import shutil +import signal +import socket +import tempfile +import threading +import time import xml.etree.ElementTree as xmlet +from pathlib import Path import CIME.utils +from CIME.case.case import Case from CIME.core.exceptions import CIMEError -from CIME.utils import expect, Timeout, run_cmd, run_cmd_no_fail, safe_copy -from CIME.XML.machines import Machines -from CIME.test_status import * from CIME.provenance import save_test_success -from CIME.case.case import Case +from CIME.test_status import * +from CIME.utils import Timeout, expect, run_cmd, run_cmd_no_fail, safe_copy +from CIME.XML.machines import Machines SIGNAL_RECEIVED = False E3SM_MAIN_CDASH = "E3SM" From cafef9e3935005cdd990e9004f2326ef6e85a7f2 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 26 May 2026 15:19:37 -0700 Subject: [PATCH 7/7] fix: removes test_status star imports --- CIME/SystemTests/funit.py | 2 +- CIME/SystemTests/hommebaseclass.py | 2 +- CIME/SystemTests/system_tests_common.py | 19 ++++++++++++++++++- CIME/SystemTests/system_tests_compare_n.py | 2 +- CIME/SystemTests/system_tests_compare_two.py | 2 +- CIME/Tools/case.build | 7 ++++++- CIME/Tools/case.qstatus | 2 -- CIME/Tools/case.setup | 2 +- CIME/Tools/get_standard_makefile_args | 2 -- CIME/bless_test_results.py | 15 +++++++++++++-- CIME/case/case_cmpgen_namelists.py | 8 +++++++- CIME/case/case_setup.py | 8 +++++++- CIME/case/case_submit.py | 7 ++++++- CIME/compare_test_results.py | 10 +++++++++- CIME/test_scheduler.py | 16 +++++++++++++++- CIME/tests/test_unit_compare_test_results.py | 7 ++++++- CIME/wait_for_tests.py | 12 +++++++++++- 17 files changed, 103 insertions(+), 20 deletions(-) diff --git a/CIME/SystemTests/funit.py b/CIME/SystemTests/funit.py index 5ccf8324bcd..2a366e6b3d6 100644 --- a/CIME/SystemTests/funit.py +++ b/CIME/SystemTests/funit.py @@ -9,7 +9,7 @@ from CIME.build import post_build from CIME.status import append_testlog from CIME.SystemTests.system_tests_common import SystemTestsCommon -from CIME.test_status import * +from CIME.test_status import BASELINE_PHASE, GENERATE_PHASE, TEST_PASS_STATUS from CIME.utils import expect, get_cime_root, run_cmd logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/hommebaseclass.py b/CIME/SystemTests/hommebaseclass.py index e811ce90747..dd3183c5ad6 100644 --- a/CIME/SystemTests/hommebaseclass.py +++ b/CIME/SystemTests/hommebaseclass.py @@ -9,7 +9,7 @@ from CIME.build import post_build from CIME.status import append_testlog from CIME.SystemTests.system_tests_common import SystemTestsCommon -from CIME.test_status import * +from CIME.test_status import BASELINE_PHASE, GENERATE_PHASE, TEST_PASS_STATUS from CIME.utils import ( SharedArea, expect, diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index aa605b32d1c..5eeb3e5aed7 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -35,7 +35,24 @@ from CIME.locked_files import LOCKED_DIR, is_locked, lock_file from CIME.provenance import get_test_success, save_test_time from CIME.status import append_testlog -from CIME.test_status import * +from CIME.test_status import ( + BASELINE_PHASE, + COMPARE_PHASE, + GENERATE_PHASE, + MEMCOMP_PHASE, + MEMLEAK_PHASE, + MODEL_BUILD_PHASE, + RUN_PHASE, + SHAREDLIB_BUILD_PHASE, + STARCHIVE_PHASE, + SUBMIT_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TEST_PEND_STATUS, + TEST_RERUN_COMMENT, + THROUGHPUT_PHASE, + TestStatus, +) from CIME.utils import ( CIMEError, SharedArea, diff --git a/CIME/SystemTests/system_tests_compare_n.py b/CIME/SystemTests/system_tests_compare_n.py index 4ce15286d94..b9d49edfbb6 100644 --- a/CIME/SystemTests/system_tests_compare_n.py +++ b/CIME/SystemTests/system_tests_compare_n.py @@ -47,7 +47,7 @@ from CIME.case import Case from CIME.config import Config from CIME.SystemTests.system_tests_common import SystemTestsCommon, fix_single_exe_case -from CIME.test_status import * +from CIME.test_status import COMPARE_PHASE, RUN_PHASE, TEST_PEND_STATUS from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/system_tests_compare_two.py b/CIME/SystemTests/system_tests_compare_two.py index 0866761e10b..9652a21a6fa 100644 --- a/CIME/SystemTests/system_tests_compare_two.py +++ b/CIME/SystemTests/system_tests_compare_two.py @@ -55,7 +55,7 @@ from CIME.case import Case from CIME.config import Config from CIME.SystemTests.system_tests_common import SystemTestsCommon, fix_single_exe_case -from CIME.test_status import * +from CIME.test_status import COMPARE_PHASE, RUN_PHASE, TEST_PEND_STATUS from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/Tools/case.build b/CIME/Tools/case.build index 0f638876bcf..d69384de9f7 100755 --- a/CIME/Tools/case.build +++ b/CIME/Tools/case.build @@ -46,7 +46,12 @@ from standard_script_setup import * import CIME.build as build from CIME.case import Case -from CIME.test_status import * +from CIME.test_status import ( + MODEL_BUILD_PHASE, + SHAREDLIB_BUILD_PHASE, + TEST_FAIL_STATUS, + TestStatus, +) from CIME.utils import find_system_test, get_model diff --git a/CIME/Tools/case.qstatus b/CIME/Tools/case.qstatus index 1ce2705b482..305f7c78c69 100755 --- a/CIME/Tools/case.qstatus +++ b/CIME/Tools/case.qstatus @@ -10,8 +10,6 @@ Typical usage is simply: from standard_script_setup import * from CIME.case import Case -from CIME.test_status import * - ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/case.setup b/CIME/Tools/case.setup index ac4ca3d41e1..0a7f4d6937b 100755 --- a/CIME/Tools/case.setup +++ b/CIME/Tools/case.setup @@ -19,7 +19,7 @@ To rerun after making changes to env_mach_pes.xml or env_mach_specific.xml, run: from standard_script_setup import * from CIME.case import Case -from CIME.test_status import * +from CIME.test_status import SETUP_PHASE, TEST_FAIL_STATUS, TestStatus from CIME.utils import find_system_test diff --git a/CIME/Tools/get_standard_makefile_args b/CIME/Tools/get_standard_makefile_args index d361d36d571..2c002717125 100755 --- a/CIME/Tools/get_standard_makefile_args +++ b/CIME/Tools/get_standard_makefile_args @@ -9,8 +9,6 @@ from standard_script_setup import * from CIME.build import get_standard_makefile_args from CIME.case import Case -from CIME.test_status import * - ############################################################################### def parse_command_line(args, description): diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 7080a13987c..f4066969e84 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -13,12 +13,23 @@ from CIME.case import Case from CIME.config import Config from CIME.hist_utils import NO_ORIGINAL, compare_baseline, generate_baseline -from CIME.test_scheduler import NAMELIST_PHASE -from CIME.test_status import * +from CIME.test_status import ( + ALL_PHASES, + BASELINE_PHASE, + GENERATE_PHASE, + MEMCOMP_PHASE, + NAMELIST_PHASE, + RUN_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + THROUGHPUT_PHASE, + TestStatus, +) from CIME.test_utils import get_test_status_files from CIME.utils import ( CIMEError, EnvironmentContext, + expect, get_scripts_root, match_any, parse_test_name, diff --git a/CIME/case/case_cmpgen_namelists.py b/CIME/case/case_cmpgen_namelists.py index 78334b7de7f..a4b016c3998 100644 --- a/CIME/case/case_cmpgen_namelists.py +++ b/CIME/case/case_cmpgen_namelists.py @@ -14,7 +14,13 @@ from CIME.compare_namelists import compare_namelist_files, is_namelist_file from CIME.simple_compare import compare_files, compare_runconfigfiles from CIME.status import append_status -from CIME.test_status import * +from CIME.test_status import ( + NAMELIST_PHASE, + RUN_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TestStatus, +) from CIME.utils import SharedArea, expect, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/case/case_setup.py b/CIME/case/case_setup.py index 6a4f658f6c2..ece6e60466f 100644 --- a/CIME/case/case_setup.py +++ b/CIME/case/case_setup.py @@ -16,7 +16,13 @@ from CIME.gitinterface import GitInterface from CIME.locked_files import check_lockedfiles, lock_file, unlock_file from CIME.status import append_case_status, run_and_log_case_status -from CIME.test_status import * +from CIME.test_status import ( + SETUP_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TEST_PEND_STATUS, + TestStatus, +) from CIME.utils import ( CIMEError, batch_jobid, diff --git a/CIME/case/case_submit.py b/CIME/case/case_submit.py index 0c4dfc13761..2be69e4141b 100644 --- a/CIME/case/case_submit.py +++ b/CIME/case/case_submit.py @@ -19,7 +19,12 @@ unlock_file, ) from CIME.status import run_and_log_case_status -from CIME.test_status import * +from CIME.test_status import ( + SUBMIT_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TestStatus, +) from CIME.utils import expect, get_time_in_seconds logger = logging.getLogger(__name__) diff --git a/CIME/compare_test_results.py b/CIME/compare_test_results.py index cef36c1da78..dce0267d177 100644 --- a/CIME/compare_test_results.py +++ b/CIME/compare_test_results.py @@ -6,7 +6,15 @@ from CIME.case import Case from CIME.hist_utils import compare_baseline, get_ts_synopsis from CIME.status import append_status -from CIME.test_status import * +from CIME.test_status import ( + BASELINE_PHASE, + NAMELIST_PHASE, + RUN_PHASE, + SETUP_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TestStatus, +) from CIME.test_utils import get_test_status_files from CIME.utils import EnvironmentContext, parse_test_name diff --git a/CIME/test_scheduler.py b/CIME/test_scheduler.py index c20fc3b3e4c..7b1152e1713 100644 --- a/CIME/test_scheduler.py +++ b/CIME/test_scheduler.py @@ -29,7 +29,21 @@ from CIME.provenance import get_recommended_test_time_based_on_past from CIME.status import append_status, append_testlog from CIME.SystemTests.test_mods import find_test_mods -from CIME.test_status import * +from CIME.test_status import ( + CORE_PHASES, + CREATE_NEWCASE_PHASE, + MODEL_BUILD_PHASE, + RUN_PHASE, + SETUP_PHASE, + SHAREDLIB_BUILD_PHASE, + SUBMIT_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TEST_PEND_STATUS, + TEST_RERUN_COMMENT, + TestStatus, + XML_PHASE, +) from CIME.utils import ( TESTS_FAILED_ERR_CODE, CIMEError, diff --git a/CIME/tests/test_unit_compare_test_results.py b/CIME/tests/test_unit_compare_test_results.py index 14e79d36345..fe984390072 100755 --- a/CIME/tests/test_unit_compare_test_results.py +++ b/CIME/tests/test_unit_compare_test_results.py @@ -10,7 +10,12 @@ import unittest from CIME import compare_test_results, utils -from CIME.test_status import * +from CIME.test_status import ( + CREATE_NEWCASE_PHASE, + RUN_PHASE, + SETUP_PHASE, + TestStatus, +) from CIME.tests.case_fake import CaseFake diff --git a/CIME/wait_for_tests.py b/CIME/wait_for_tests.py index 3f4dc348745..51b53ce7f63 100644 --- a/CIME/wait_for_tests.py +++ b/CIME/wait_for_tests.py @@ -18,7 +18,17 @@ from CIME.case.case import Case from CIME.core.exceptions import CIMEError from CIME.provenance import save_test_success -from CIME.test_status import * +from CIME.test_status import ( + CREATE_NEWCASE_PHASE, + NAMELIST_FAIL_STATUS, + NAMELIST_PHASE, + RUN_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TEST_PEND_STATUS, + TEST_STATUS_FILENAME, + TestStatus, +) from CIME.utils import Timeout, expect, run_cmd, run_cmd_no_fail, safe_copy from CIME.XML.machines import Machines