diff --git a/CIME/BuildTools/configure.py b/CIME/BuildTools/configure.py index 6e77a1a15c5..2e833735472 100755 --- a/CIME/BuildTools/configure.py +++ b/CIME/BuildTools/configure.py @@ -15,20 +15,22 @@ COMPILER, MPILIB, and DEBUG, respectively. """ -from CIME.XML.standard_module_setup import * +import glob +import logging +import os +import shutil + +from CIME.build import CmakeTmpBuildDir from CIME.utils import ( + copy_local_macros_to_dir, expect, - safe_copy, get_model, get_src_root, + safe_copy, stringify_bool, - copy_local_macros_to_dir, ) from CIME.XML.env_mach_specific import EnvMachSpecific from CIME.XML.files import Files -from CIME.build import CmakeTmpBuildDir - -import shutil, glob logger = logging.getLogger(__name__) diff --git a/CIME/Servers/ftp.py b/CIME/Servers/ftp.py index 209525a2bac..12c50e18767 100644 --- a/CIME/Servers/ftp.py +++ b/CIME/Servers/ftp.py @@ -1,15 +1,20 @@ """ FTP Server class. Interact with a server using FTP protocol """ + # pylint: disable=super-init-not-called -from CIME.XML.standard_module_setup import * -from CIME.Servers.generic_server import GenericServer -from CIME.utils import Timeout +import logging +import os +import socket from ftplib import FTP as FTPpy from ftplib import all_errors as all_ftp_errors -import socket + +from CIME.Servers.generic_server import GenericServer +from CIME.utils import Timeout, expect logger = logging.getLogger(__name__) + + # I think that multiple inheritence would be useful here, but I couldnt make it work # in a py2/3 compatible way. class FTP(GenericServer): diff --git a/CIME/Servers/generic_server.py b/CIME/Servers/generic_server.py index 537df181324..61fcef21b8f 100644 --- a/CIME/Servers/generic_server.py +++ b/CIME/Servers/generic_server.py @@ -3,9 +3,10 @@ to make sure that specific server classes maintain a consistant argument list and functionality so that they are interchangable objects """ + # pylint: disable=unused-argument -from CIME.XML.standard_module_setup import * +import logging from socket import _GLOBAL_DEFAULT_TIMEOUT logger = logging.getLogger(__name__) diff --git a/CIME/Servers/gftp.py b/CIME/Servers/gftp.py index f23943b583a..fea2a8a0915 100644 --- a/CIME/Servers/gftp.py +++ b/CIME/Servers/gftp.py @@ -1,8 +1,11 @@ """ GridFTP Server class. Interact with a server using GridFTP protocol """ + # pylint: disable=super-init-not-called -from CIME.XML.standard_module_setup import * +import logging +import os + from CIME.Servers.generic_server import GenericServer from CIME.utils import run_cmd diff --git a/CIME/Servers/svn.py b/CIME/Servers/svn.py index 7e06ab310d4..b407683ae07 100644 --- a/CIME/Servers/svn.py +++ b/CIME/Servers/svn.py @@ -1,9 +1,13 @@ """ SVN Server class. Interact with a server using SVN protocol """ + # pylint: disable=super-init-not-called -from CIME.XML.standard_module_setup import * +import logging +import os + from CIME.Servers.generic_server import GenericServer +from CIME.utils import run_cmd logger = logging.getLogger(__name__) diff --git a/CIME/Servers/wget.py b/CIME/Servers/wget.py index fde411460e8..583e0972bc4 100644 --- a/CIME/Servers/wget.py +++ b/CIME/Servers/wget.py @@ -1,9 +1,13 @@ """ WGET Server class. Interact with a server using WGET protocol """ + # pylint: disable=super-init-not-called -from CIME.XML.standard_module_setup import * +import logging +import os + from CIME.Servers.generic_server import GenericServer +from CIME.utils import run_cmd logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/dae.py b/CIME/SystemTests/dae.py index 175254d2d1b..e9472678de2 100644 --- a/CIME/SystemTests/dae.py +++ b/CIME/SystemTests/dae.py @@ -6,14 +6,15 @@ """ -import os.path -import logging import glob import gzip +import logging +import os.path from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.utils import expect + ############################################################################### class DAE(SystemTestsCompareTwo): ############################################################################### diff --git a/CIME/SystemTests/eri.py b/CIME/SystemTests/eri.py index 597cfe4c709..151f07d55ac 100644 --- a/CIME/SystemTests/eri.py +++ b/CIME/SystemTests/eri.py @@ -2,11 +2,14 @@ CIME ERI test This class inherits from SystemTestsCommon """ -from CIME.XML.standard_module_setup import * -from CIME.utils import safe_copy -from CIME.SystemTests.system_tests_common import SystemTestsCommon +import glob +import logging +import os +import shutil from stat import S_ISDIR, ST_CTIME, ST_MODE -import shutil, glob, os + +from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.utils import expect, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/erio.py b/CIME/SystemTests/erio.py index a1e7b041cc6..0ded2f0d273 100644 --- a/CIME/SystemTests/erio.py +++ b/CIME/SystemTests/erio.py @@ -3,8 +3,11 @@ This class inherits from SystemTestsCommon """ -from CIME.XML.standard_module_setup import * + +import logging + from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/erp.py b/CIME/SystemTests/erp.py index aeab1b0020b..b0fa12696c0 100644 --- a/CIME/SystemTests/erp.py +++ b/CIME/SystemTests/erp.py @@ -8,7 +8,8 @@ (2) Do a restart test with half the number of tasks and threads (suffix rest) """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.restart_tests import RestartTest logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/err.py b/CIME/SystemTests/err.py index ff5a2e4a4f7..755381e552c 100644 --- a/CIME/SystemTests/err.py +++ b/CIME/SystemTests/err.py @@ -3,10 +3,12 @@ ERR tests short term archiving and restart capabilities """ -import glob, os -from CIME.XML.standard_module_setup import * +import glob +import logging +import os + from CIME.SystemTests.restart_tests import RestartTest -from CIME.utils import safe_copy, ls_sorted_by_fname +from CIME.utils import expect, ls_sorted_by_fname, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/erri.py b/CIME/SystemTests/erri.py index 7851bd4bb66..e3453aeaccb 100644 --- a/CIME/SystemTests/erri.py +++ b/CIME/SystemTests/erri.py @@ -3,10 +3,13 @@ ERRI tests short term archiving and restart capabilities with "incomplete" (unzipped) log files """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.err import ERR +import glob +import gzip +import logging +import os +import shutil -import shutil, glob, gzip +from CIME.SystemTests.err import ERR logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/ers.py b/CIME/SystemTests/ers.py index 0e93d8afa1c..b39bf71aef1 100644 --- a/CIME/SystemTests/ers.py +++ b/CIME/SystemTests/ers.py @@ -1,9 +1,13 @@ """ CIME restart test This class inherits from SystemTestsCommon """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon + import glob +import logging +import os + +from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/ert.py b/CIME/SystemTests/ert.py index b912f7248b7..5a298336e0f 100644 --- a/CIME/SystemTests/ert.py +++ b/CIME/SystemTests/ert.py @@ -3,7 +3,8 @@ Exact restart from startup, default 2 month + 1 month """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_common import SystemTestsCommon logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/funit.py b/CIME/SystemTests/funit.py index 7681a6200d6..2a366e6b3d6 100644 --- a/CIME/SystemTests/funit.py +++ b/CIME/SystemTests/funit.py @@ -2,12 +2,15 @@ CIME FUNIT test. This class inherits from SystemTestsCommon. It runs the fortran unit tests; grid and compset are ignored. """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon + +import logging +import os + from CIME.build import post_build from CIME.status import append_testlog -from CIME.utils import get_cime_root -from CIME.test_status import * +from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.test_status import BASELINE_PHASE, GENERATE_PHASE, TEST_PASS_STATUS +from CIME.utils import expect, get_cime_root, run_cmd logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/hommebaseclass.py b/CIME/SystemTests/hommebaseclass.py index d275553a545..dd3183c5ad6 100644 --- a/CIME/SystemTests/hommebaseclass.py +++ b/CIME/SystemTests/hommebaseclass.py @@ -2,14 +2,22 @@ CIME HOMME test. This class inherits from SystemTestsCommon """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon +import logging +import os +import shutil + from CIME.build import post_build from CIME.status import append_testlog -from CIME.utils import SharedArea, new_lid, gzip_existing_file -from CIME.test_status import * - -import shutil +from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.test_status import BASELINE_PHASE, GENERATE_PHASE, TEST_PASS_STATUS +from CIME.utils import ( + SharedArea, + expect, + gzip_existing_file, + new_lid, + run_cmd, + run_cmd_no_fail, +) logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/icp.py b/CIME/SystemTests/icp.py index 8d8c5e0ea59..c5357784671 100644 --- a/CIME/SystemTests/icp.py +++ b/CIME/SystemTests/icp.py @@ -1,7 +1,7 @@ """ CIME ICP test This class inherits from SystemTestsCommon """ -from CIME.XML.standard_module_setup import * + from CIME.SystemTests.system_tests_common import SystemTestsCommon diff --git a/CIME/SystemTests/irt.py b/CIME/SystemTests/irt.py index 633893adcd0..419dfd8164a 100644 --- a/CIME/SystemTests/irt.py +++ b/CIME/SystemTests/irt.py @@ -11,9 +11,11 @@ """ +import logging +import os + from CIME.SystemTests.restart_tests import RestartTest -from CIME.XML.standard_module_setup import * -from CIME.utils import ls_sorted_by_fname +from CIME.utils import expect, ls_sorted_by_fname logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/ldsta.py b/CIME/SystemTests/ldsta.py index a5f7c9196d5..4dd96fb8bae 100644 --- a/CIME/SystemTests/ldsta.py +++ b/CIME/SystemTests/ldsta.py @@ -4,19 +4,20 @@ The test verifies the archive directory contains the expected files """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon -from CIME.utils import expect -from CIME.date import get_file_date - import datetime import glob +import logging import os import random import shutil +from CIME.date import get_file_date +from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.utils import expect + logger = logging.getLogger(__name__) + # datetime objects can't be used anywhere else def _date_to_datetime(date_obj): return datetime.datetime( diff --git a/CIME/SystemTests/mcc.py b/CIME/SystemTests/mcc.py index a4b839cf1e9..f46030cdffd 100644 --- a/CIME/SystemTests/mcc.py +++ b/CIME/SystemTests/mcc.py @@ -4,7 +4,9 @@ This does two runs: In the first we run a three member ensemble using the MULTI_DRIVER capability, then we run a second single instance case and compare """ -from CIME.XML.standard_module_setup import * + +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 60f863a2670..852a460d248 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -6,23 +6,22 @@ This class inherits from SystemTestsCommon. """ -import os import json import logging +import os from shutil import copytree -from CIME import test_status -from CIME import utils -from CIME.status import append_testlog -from CIME.SystemTests.system_tests_common import SystemTestsCommon +import evv4esm # pylint: disable=import-error +from evv4esm.__main__ import main as evv # pylint: disable=import-error + +from CIME import test_status, utils from CIME.case.case_setup import case_setup -from CIME.XML.machines import Machines from CIME.config import ConfigBase -from CIME.SystemTests import test_mods from CIME.namelist import Namelist - -import evv4esm # pylint: disable=import-error -from evv4esm.__main__ import main as evv # pylint: disable=import-error +from CIME.status import append_testlog +from CIME.SystemTests import test_mods +from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.XML.machines import Machines version = evv4esm.__version_info__ diff --git a/CIME/SystemTests/nck.py b/CIME/SystemTests/nck.py index f75a2914215..9f17b28c09d 100644 --- a/CIME/SystemTests/nck.py +++ b/CIME/SystemTests/nck.py @@ -8,7 +8,8 @@ Lay all of the components out sequentially """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/ncr.py b/CIME/SystemTests/ncr.py index f0de168ac13..39b16f25ea2 100644 --- a/CIME/SystemTests/ncr.py +++ b/CIME/SystemTests/ncr.py @@ -8,7 +8,9 @@ NOTE: This is currently untested, and may not be working properly """ -from CIME.XML.standard_module_setup import * + +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/nodefail.py b/CIME/SystemTests/nodefail.py index d975cfc5bfd..559f49c690b 100644 --- a/CIME/SystemTests/nodefail.py +++ b/CIME/SystemTests/nodefail.py @@ -1,7 +1,10 @@ """ CIME restart upon failed node test. """ -from CIME.XML.standard_module_setup import * + +import logging +import os + from CIME.SystemTests.ers import ERS from CIME.utils import get_model diff --git a/CIME/SystemTests/pea.py b/CIME/SystemTests/pea.py index 4fb3a4569ca..c7534dd8340 100644 --- a/CIME/SystemTests/pea.py +++ b/CIME/SystemTests/pea.py @@ -6,8 +6,10 @@ (2) do a run with mpi-serial (suffix mpi-serial) """ +import logging +import os + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo -from CIME.XML.standard_module_setup import * from CIME.XML.machines import Machines logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/pem.py b/CIME/SystemTests/pem.py index d98f9e4a2c7..d26108d8197 100644 --- a/CIME/SystemTests/pem.py +++ b/CIME/SystemTests/pem.py @@ -8,7 +8,8 @@ (2) Run with half the number of tasks (suffix modpes) """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/pet.py b/CIME/SystemTests/pet.py index 161dac4d536..35e905f2399 100644 --- a/CIME/SystemTests/pet.py +++ b/CIME/SystemTests/pet.py @@ -6,7 +6,8 @@ (2) do another initial run with nthrds=1 for all components (suffix: single_thread) """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/pfs.py b/CIME/SystemTests/pfs.py index ed61d204e8a..ffd1ed46d3b 100644 --- a/CIME/SystemTests/pfs.py +++ b/CIME/SystemTests/pfs.py @@ -4,7 +4,8 @@ 20 days performance test, no restart files written """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_common import SystemTestsCommon logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/pgn.py b/CIME/SystemTests/pgn.py index 07ac1f4cb08..ed27bc6d641 100644 --- a/CIME/SystemTests/pgn.py +++ b/CIME/SystemTests/pgn.py @@ -9,30 +9,27 @@ from __future__ import division +import json +import logging import os import re -import json import shutil -import logging - from collections import OrderedDict from shutil import copytree -import pandas as pd +import evv4esm # pylint: disable=import-error import numpy as np - +import pandas as pd +from evv4esm.__main__ import main as evv # pylint: disable=import-error +from evv4esm.extensions import pg # pylint: disable=import-error import CIME.test_status import CIME.utils +from CIME.case.case_setup import case_setup from CIME.status import append_testlog from CIME.SystemTests.system_tests_common import SystemTestsCommon -from CIME.case.case_setup import case_setup from CIME.XML.machines import Machines -import evv4esm # pylint: disable=import-error -from evv4esm.extensions import pg # pylint: disable=import-error -from evv4esm.__main__ import main as evv # pylint: disable=import-error - evv_lib_dir = os.path.abspath(os.path.dirname(evv4esm.__file__)) logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/pre.py b/CIME/SystemTests/pre.py index 23547d46430..c5245364eb1 100644 --- a/CIME/SystemTests/pre.py +++ b/CIME/SystemTests/pre.py @@ -6,13 +6,14 @@ Test requires DESP component to function correctly. """ -import os.path -import logging import glob +import logging +import os.path +from CIME.hist_utils import cprnc from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.utils import expect -from CIME.hist_utils import cprnc + ############################################################################### class PRE(SystemTestsCompareTwo): diff --git a/CIME/SystemTests/restart_tests.py b/CIME/SystemTests/restart_tests.py index ffc50270ce2..6883e95baa0 100644 --- a/CIME/SystemTests/restart_tests.py +++ b/CIME/SystemTests/restart_tests.py @@ -2,8 +2,10 @@ Abstract class for restart tests """ +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo -from CIME.XML.standard_module_setup import * +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/reuseinitfiles.py b/CIME/SystemTests/reuseinitfiles.py index 76d8bb0522e..98cf8216380 100644 --- a/CIME/SystemTests/reuseinitfiles.py +++ b/CIME/SystemTests/reuseinitfiles.py @@ -14,9 +14,10 @@ import os import shutil -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo + from CIME.SystemTests.system_tests_common import INIT_GENERATED_FILES_DIRNAME +from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo +from CIME.utils import expect class REUSEINITFILES(SystemTestsCompareTwo): diff --git a/CIME/SystemTests/seq.py b/CIME/SystemTests/seq.py index 7413f900899..d0a6ca96495 100644 --- a/CIME/SystemTests/seq.py +++ b/CIME/SystemTests/seq.py @@ -1,7 +1,9 @@ """ sequencing bfb test (10 day seq,conc tests) """ -from CIME.XML.standard_module_setup import * + +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/sms.py b/CIME/SystemTests/sms.py index 17672b47052..18a42fc529a 100644 --- a/CIME/SystemTests/sms.py +++ b/CIME/SystemTests/sms.py @@ -3,7 +3,8 @@ It does a startup run with restarts off and optionally compares to or generates baselines """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_common import SystemTestsCommon logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 67ed8416671..5eeb3e5aed7 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -2,45 +2,70 @@ Base class for CIME system tests """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_run import EnvRun -from CIME.XML.env_test import EnvTest -from CIME.status import append_testlog -from CIME.utils import ( - get_model, - safe_copy, - get_timestamp, - CIMEError, - expect, - get_current_commit, - SharedArea, - is_comp_standalone, +import calendar +import glob +import gzip +import logging +import math +import os +import re +import time +import traceback +from contextlib import ExitStack +from datetime import datetime, timedelta + +import CIME.build as build +from CIME.baselines.performance import ( + get_latest_cpl_logs, + load_coupler_customization, + perf_compare_memory_baseline, + perf_compare_throughput_baseline, + perf_get_memory_list, + perf_write_baseline, ) -from CIME.test_status import * +from CIME.config import Config from CIME.hist_utils import ( - copy_histfiles, + compare_baseline, compare_test, + copy_histfiles, + generate_baseline, generate_teststatus, - compare_baseline, get_ts_synopsis, - generate_baseline, ) -from CIME.config import Config -from CIME.provenance import save_test_time, get_test_success -from CIME.locked_files import LOCKED_DIR, lock_file, is_locked -from CIME.baselines.performance import ( - get_latest_cpl_logs, - perf_get_memory_list, - perf_compare_memory_baseline, - perf_compare_throughput_baseline, - perf_write_baseline, - load_coupler_customization, +from CIME.locked_files import LOCKED_DIR, is_locked, lock_file +from CIME.provenance import get_test_success, save_test_time +from CIME.status import append_testlog +from CIME.test_status import ( + BASELINE_PHASE, + COMPARE_PHASE, + GENERATE_PHASE, + MEMCOMP_PHASE, + MEMLEAK_PHASE, + MODEL_BUILD_PHASE, + RUN_PHASE, + SHAREDLIB_BUILD_PHASE, + STARCHIVE_PHASE, + SUBMIT_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TEST_PEND_STATUS, + TEST_RERUN_COMMENT, + THROUGHPUT_PHASE, + TestStatus, ) -import CIME.build as build -from datetime import datetime, timedelta -import glob, gzip, time, traceback, os, math, calendar - -from contextlib import ExitStack +from CIME.utils import ( + CIMEError, + SharedArea, + expect, + get_current_commit, + get_model, + get_timestamp, + is_comp_standalone, + run_cmd, + safe_copy, +) +from CIME.XML.env_run import EnvRun +from CIME.XML.env_test import EnvTest logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/system_tests_compare_n.py b/CIME/SystemTests/system_tests_compare_n.py index 93724eeef3e..b9d49edfbb6 100644 --- a/CIME/SystemTests/system_tests_compare_n.py +++ b/CIME/SystemTests/system_tests_compare_n.py @@ -39,13 +39,16 @@ Use this to do arbitrary actions immediately after running case one """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon, fix_single_exe_case +import glob +import logging +import os +import shutil + from CIME.case import Case from CIME.config import Config -from CIME.test_status import * - -import shutil, os, glob +from CIME.SystemTests.system_tests_common import SystemTestsCommon, fix_single_exe_case +from CIME.test_status import COMPARE_PHASE, RUN_PHASE, TEST_PEND_STATUS +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/system_tests_compare_two.py b/CIME/SystemTests/system_tests_compare_two.py index 61232b34117..9652a21a6fa 100644 --- a/CIME/SystemTests/system_tests_compare_two.py +++ b/CIME/SystemTests/system_tests_compare_two.py @@ -47,13 +47,16 @@ Use this to do arbitrary actions immediately after running case two """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon, fix_single_exe_case +import glob +import logging +import os +import shutil + from CIME.case import Case from CIME.config import Config -from CIME.test_status import * - -import shutil, os, glob +from CIME.SystemTests.system_tests_common import SystemTestsCommon, fix_single_exe_case +from CIME.test_status import COMPARE_PHASE, RUN_PHASE, TEST_PEND_STATUS +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/test_mods.py b/CIME/SystemTests/test_mods.py index 2db22c5833a..d7f010f1d86 100644 --- a/CIME/SystemTests/test_mods.py +++ b/CIME/SystemTests/test_mods.py @@ -1,7 +1,7 @@ import logging import os -from CIME.utils import CIMEError +from CIME.core.exceptions import CIMEError from CIME.XML.files import Files logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/test_utils/user_nl_utils.py b/CIME/SystemTests/test_utils/user_nl_utils.py index 930d683666b..00cfdad37d7 100644 --- a/CIME/SystemTests/test_utils/user_nl_utils.py +++ b/CIME/SystemTests/test_utils/user_nl_utils.py @@ -2,8 +2,8 @@ This module contains functions for working with user_nl files in system tests. """ -import os import glob +import os def append_to_user_nl_files(caseroot, component, contents): diff --git a/CIME/SystemTests/tsc.py b/CIME/SystemTests/tsc.py index be695b7f482..5c16a068752 100644 --- a/CIME/SystemTests/tsc.py +++ b/CIME/SystemTests/tsc.py @@ -7,23 +7,22 @@ This class inherits from SystemTestsCommon. """ -import os import json import logging - +import os from shutil import copytree +import evv4esm # pylint: disable=import-error +from evv4esm.__main__ import main as evv # pylint: disable=import-error + import CIME.test_status import CIME.utils -from CIME.status import append_testlog -from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case.case_setup import case_setup from CIME.hist_utils import rename_all_hist_files +from CIME.status import append_testlog +from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.XML.machines import Machines -import evv4esm # pylint: disable=import-error -from evv4esm.__main__ import main as evv # pylint: disable=import-error - evv_lib_dir = os.path.abspath(os.path.dirname(evv4esm.__file__)) logger = logging.getLogger(__name__) diff --git a/CIME/Tools/archive_metadata b/CIME/Tools/archive_metadata index 191050254f9..4054db8612d 100755 --- a/CIME/Tools/archive_metadata +++ b/CIME/Tools/archive_metadata @@ -6,25 +6,26 @@ via a web post and SVN check-in Author: CSEG """ import argparse +import configparser import datetime import filecmp import getpass import glob import gzip -import json import io +import json import os -from os.path import expanduser import re import shutil import ssl import subprocess import sys -from string import Template -import configparser import urllib +from os.path import expanduser +from string import Template from standard_script_setup import * + from CIME.case import Case from CIME.utils import is_last_process_complete diff --git a/CIME/Tools/bld_diff b/CIME/Tools/bld_diff index 27193d61560..ea614562886 100755 --- a/CIME/Tools/bld_diff +++ b/CIME/Tools/bld_diff @@ -5,10 +5,15 @@ Try to calculate and succinctly present the differences between two bld logs for the same component """ +import argparse +import gzip +import os +import sys + from standard_script_setup import * + from CIME.utils import run_cmd_no_fail -import argparse, sys, os, gzip ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/bless_test_results b/CIME/Tools/bless_test_results index 093ae0d79b8..335f1d6a0d4 100755 --- a/CIME/Tools/bless_test_results +++ b/CIME/Tools/bless_test_results @@ -8,16 +8,14 @@ blessing of diffs. You may need to load modules for cprnc to work. """ +import argparse +import os +import sys + from standard_script_setup import * -from CIME.utils import expect -from CIME.XML.machines import Machines from CIME.bless_test_results import bless_test_results - -import argparse -import sys -import os -import logging +from CIME.XML.machines import Machines _MACHINE = Machines() diff --git a/CIME/Tools/case.build b/CIME/Tools/case.build index 131ca94564d..d69384de9f7 100755 --- a/CIME/Tools/case.build +++ b/CIME/Tools/case.build @@ -46,8 +46,14 @@ from standard_script_setup import * import CIME.build as build from CIME.case import Case +from CIME.test_status import ( + MODEL_BUILD_PHASE, + SHAREDLIB_BUILD_PHASE, + TEST_FAIL_STATUS, + TestStatus, +) from CIME.utils import find_system_test, get_model -from CIME.test_status import * + ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/case.cmpgen_namelists b/CIME/Tools/case.cmpgen_namelists index c9930e71795..e97936f336b 100755 --- a/CIME/Tools/case.cmpgen_namelists +++ b/CIME/Tools/case.cmpgen_namelists @@ -5,10 +5,12 @@ case.cmpgen_namelists - perform namelist baseline operations (compare, generate, or both) for this case. """ +from argparse import RawTextHelpFormatter + from standard_script_setup import * from CIME.case import Case -from argparse import RawTextHelpFormatter + ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/case.qstatus b/CIME/Tools/case.qstatus index 4902dc511d5..305f7c78c69 100755 --- a/CIME/Tools/case.qstatus +++ b/CIME/Tools/case.qstatus @@ -10,7 +10,6 @@ Typical usage is simply: from standard_script_setup import * from CIME.case import Case -from CIME.test_status import * ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/case.setup b/CIME/Tools/case.setup index 75eeadd05b0..0a7f4d6937b 100755 --- a/CIME/Tools/case.setup +++ b/CIME/Tools/case.setup @@ -19,9 +19,10 @@ To rerun after making changes to env_mach_pes.xml or env_mach_specific.xml, run: from standard_script_setup import * from CIME.case import Case -from CIME.test_status import * +from CIME.test_status import SETUP_PHASE, TEST_FAIL_STATUS, TestStatus from CIME.utils import find_system_test + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/case.submit b/CIME/Tools/case.submit index 5de69368100..1f4dee5a81c 100755 --- a/CIME/Tools/case.submit +++ b/CIME/Tools/case.submit @@ -16,10 +16,13 @@ Other examples: """ import configparser + from standard_script_setup import * + from CIME.case import Case from CIME.utils import expect + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/case_diff b/CIME/Tools/case_diff index e642dd8e7ac..167845ed91d 100755 --- a/CIME/Tools/case_diff +++ b/CIME/Tools/case_diff @@ -5,10 +5,14 @@ Try to calculate and succinctly present the differences between two large directory trees. """ +import argparse +import os +import sys + from standard_script_setup import * + from CIME.utils import run_cmd, run_cmd_no_fail -import argparse, sys, os ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/check_case b/CIME/Tools/check_case index 9b061046643..30bf1361019 100755 --- a/CIME/Tools/check_case +++ b/CIME/Tools/check_case @@ -17,13 +17,13 @@ automatically when running case.submit. However, you can run this if you want to perform these checks without actually submitting the case. """ +import argparse + from standard_script_setup import * -from CIME.utils import expect from CIME.case import Case from CIME.locked_files import check_lockedfiles - -import argparse +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/Tools/check_input_data b/CIME/Tools/check_input_data index 179af50c4fd..df11a8b539d 100755 --- a/CIME/Tools/check_input_data +++ b/CIME/Tools/check_input_data @@ -18,10 +18,12 @@ To obtain missing datasets from the input data server(s) use: This script is automatically called by the case control system, when the case is built and submitted. So manual usage of this script is optional. """ +import argparse + from standard_script_setup import * + from CIME.case import Case -import argparse ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/check_lockedfiles b/CIME/Tools/check_lockedfiles index a959b953570..068f24793f5 100755 --- a/CIME/Tools/check_lockedfiles +++ b/CIME/Tools/check_lockedfiles @@ -4,6 +4,7 @@ This script compares xml files """ from standard_script_setup import * + from CIME.case import Case from CIME.locked_files import check_lockedfiles diff --git a/CIME/Tools/cime_bisect b/CIME/Tools/cime_bisect index e427679f692..0a4969599fd 100755 --- a/CIME/Tools/cime_bisect +++ b/CIME/Tools/cime_bisect @@ -9,10 +9,14 @@ NOTE: this tool will only work for models that use git and, for bisecting CIME, bring in CIME via submodule or clone. """ +import argparse +import os +import re +import sys + from standard_script_setup import * -from CIME.utils import expect, run_cmd_no_fail, run_cmd -import argparse, sys, os, re +from CIME.utils import expect, run_cmd, run_cmd_no_fail logger = logging.getLogger(__name__) diff --git a/CIME/Tools/code_checker b/CIME/Tools/code_checker index 6f7510337ab..b84c5278c22 100755 --- a/CIME/Tools/code_checker +++ b/CIME/Tools/code_checker @@ -5,15 +5,17 @@ Ensure that all CIME python files are free of errors and follow the PEP8 standard. """ -from standard_script_setup import * - -from CIME.code_checker import check_code, expect - -import argparse, sys, os +import argparse +import os +import sys # pylint: disable=import-error from shutil import which +from standard_script_setup import * + +from CIME.code_checker import check_code, expect + logger = logging.getLogger(__name__) ############################################################################### diff --git a/CIME/Tools/compare_namelists b/CIME/Tools/compare_namelists index 5aba49edbc0..9e71be71397 100755 --- a/CIME/Tools/compare_namelists +++ b/CIME/Tools/compare_namelists @@ -5,11 +5,15 @@ Compare namelists. Should be called by an ACME test. Designed to not be sensitive to order or whitespace. """ +import argparse +import os +import sys + from standard_script_setup import * + import CIME.compare_namelists from CIME.utils import expect -import argparse, sys, os ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/compare_test_results b/CIME/Tools/compare_test_results index 3b59ccd007f..ce7be67c038 100755 --- a/CIME/Tools/compare_test_results +++ b/CIME/Tools/compare_test_results @@ -21,12 +21,14 @@ You may need to load modules for cprnc to work. """ +import argparse +import os +import sys + from standard_script_setup import * -from CIME.XML.machines import Machines from CIME.compare_test_results import compare_test_results - -import argparse, sys, os +from CIME.XML.machines import Machines _MACHINE = Machines() diff --git a/CIME/Tools/component_compare_baseline b/CIME/Tools/component_compare_baseline index 2adad3b94ad..8a15a3af8cf 100755 --- a/CIME/Tools/component_compare_baseline +++ b/CIME/Tools/component_compare_baseline @@ -9,6 +9,7 @@ from standard_script_setup import * from CIME.case import Case from CIME.hist_utils import compare_baseline + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/component_compare_copy b/CIME/Tools/component_compare_copy index 28005cd6674..3ed0dbecac0 100755 --- a/CIME/Tools/component_compare_copy +++ b/CIME/Tools/component_compare_copy @@ -10,6 +10,7 @@ from standard_script_setup import * from CIME.case import Case from CIME.hist_utils import copy_histfiles + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/component_compare_test b/CIME/Tools/component_compare_test index ece602e1af4..d4600a4db76 100755 --- a/CIME/Tools/component_compare_test +++ b/CIME/Tools/component_compare_test @@ -9,6 +9,7 @@ from standard_script_setup import * from CIME.case import Case from CIME.hist_utils import compare_test + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/component_generate_baseline b/CIME/Tools/component_generate_baseline index ff8f39170f0..565ee8c5771 100755 --- a/CIME/Tools/component_generate_baseline +++ b/CIME/Tools/component_generate_baseline @@ -9,6 +9,7 @@ from standard_script_setup import * from CIME.case import Case from CIME.hist_utils import generate_baseline + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/cs.status b/CIME/Tools/cs.status index 4ab7b7e8ecd..e8e7ccd8cd9 100755 --- a/CIME/Tools/cs.status +++ b/CIME/Tools/cs.status @@ -8,11 +8,15 @@ Typical usage: Returns True if no errors occured (not based on test statuses). """ -from standard_script_setup import * -import argparse, sys, os, logging, glob -from CIME.utils import expect -from CIME.cs_status import cs_status +import argparse +import glob +import os +import sys + + from CIME import test_status +from CIME.cs_status import cs_status +from CIME.utils import expect _PERFORMANCE_PHASES = [test_status.THROUGHPUT_PHASE, test_status.MEMCOMP_PHASE] diff --git a/CIME/Tools/e3sm_check_env b/CIME/Tools/e3sm_check_env index b1756c9cfd3..79f593a7b26 100755 --- a/CIME/Tools/e3sm_check_env +++ b/CIME/Tools/e3sm_check_env @@ -6,10 +6,13 @@ A script to verify that the environment is compliant with E3SM's software requir Be sure to source your env_mach_specific file before running this check. """ +import argparse +import os +import sys + from standard_script_setup import * -from CIME.utils import run_cmd -import sys, os, argparse +from CIME.utils import run_cmd # Here's where we keep the various reports and instructions. LOG = [] diff --git a/CIME/Tools/generate_cylc_workflow.py b/CIME/Tools/generate_cylc_workflow.py index cac503b342b..b7465359228 100755 --- a/CIME/Tools/generate_cylc_workflow.py +++ b/CIME/Tools/generate_cylc_workflow.py @@ -3,20 +3,22 @@ """ Generates a cylc workflow file for the case. See https://cylc.github.io for details about cylc """ + import os import sys sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) -from CIME.Tools.standard_script_setup import * +import argparse +import re from CIME.case import Case +from CIME.Tools.standard_script_setup import * from CIME.utils import expect, transform_vars -import argparse, re - logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/getTiming b/CIME/Tools/getTiming index 3f16847385a..7a1352d714e 100755 --- a/CIME/Tools/getTiming +++ b/CIME/Tools/getTiming @@ -3,8 +3,12 @@ Get timing information from run """ +import argparse +import os +import sys + from standard_script_setup import * -import argparse, sys, os + from CIME.case import Case from CIME.get_timing import get_timing diff --git a/CIME/Tools/get_case_env b/CIME/Tools/get_case_env index 7e7db4b393a..9ddd2fe462a 100755 --- a/CIME/Tools/get_case_env +++ b/CIME/Tools/get_case_env @@ -5,14 +5,18 @@ Dump what the CIME environment would be for a case. Only supports E3SM for now. """ +import argparse +import shutil +import tempfile + from standard_script_setup import * -from CIME.XML.machines import Machines -from CIME.test_scheduler import TestScheduler -from CIME.utils import parse_test_name, expect, get_src_root -from CIME.config import Config + import CIME.get_tests +from CIME.config import Config +from CIME.test_scheduler import TestScheduler +from CIME.utils import expect, get_src_root +from CIME.XML.machines import Machines -import argparse, tempfile, shutil ############################################################################### def parse_command_line(raw_args, description): diff --git a/CIME/Tools/get_standard_makefile_args b/CIME/Tools/get_standard_makefile_args index 5357c09cc1a..2c002717125 100755 --- a/CIME/Tools/get_standard_makefile_args +++ b/CIME/Tools/get_standard_makefile_args @@ -9,7 +9,6 @@ from standard_script_setup import * from CIME.build import get_standard_makefile_args from CIME.case import Case -from CIME.test_status import * ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/jenkins_generic_job b/CIME/Tools/jenkins_generic_job index b02a5b69199..b4228524260 100755 --- a/CIME/Tools/jenkins_generic_job +++ b/CIME/Tools/jenkins_generic_job @@ -10,9 +10,10 @@ ensures that the batch system is left in a clean state. from standard_script_setup import * import CIME.wait_for_tests +from CIME.jenkins_generic_job import jenkins_generic_job from CIME.utils import expect from CIME.XML.machines import Machines -from CIME.jenkins_generic_job import jenkins_generic_job + ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/list_e3sm_tests b/CIME/Tools/list_e3sm_tests index 58bb436da76..8bf69d94cc0 100755 --- a/CIME/Tools/list_e3sm_tests +++ b/CIME/Tools/list_e3sm_tests @@ -5,16 +5,14 @@ List e3sm test suites. Can be used to show what's being tested. Can just list tested grids, compsets, etc. """ -from standard_script_setup import * -from CIME import utils -from CIME import get_tests -from CIME.XML.files import Files -from CIME.XML.compsets import Compsets - import argparse import logging +from CIME import get_tests, utils +from CIME.XML.compsets import Compsets +from CIME.XML.files import Files + logger = logging.getLogger(__name__) diff --git a/CIME/Tools/mvsource b/CIME/Tools/mvsource index e02f20c91bf..0b3d042af3e 100755 --- a/CIME/Tools/mvsource +++ b/CIME/Tools/mvsource @@ -2,7 +2,8 @@ # This tool expects two arguments, the caseroot and the new cimeroot # It is intended to fix links and update your case source if the caseroot or src is moved. # -import os, sys +import os +import sys caseroot = sys.argv[1] cimeroot = sys.argv[2] @@ -12,7 +13,8 @@ sys.path.append(_LIBDIR) _LIBDIR = os.path.join(cimeroot, "scripts", "lib") sys.path.append(_LIBDIR) try: - from standard_script_setup import * + pass + from CIME.case import Case from CIME.utils import symlink_force except: diff --git a/CIME/Tools/normalize_cases b/CIME/Tools/normalize_cases index 6fa0e8111dc..2d7d35e6915 100755 --- a/CIME/Tools/normalize_cases +++ b/CIME/Tools/normalize_cases @@ -8,10 +8,15 @@ This is for debugging purposes and meant to assist the user when they want to run case_diff. """ +import argparse +import glob +import os +import sys + from standard_script_setup import * + from CIME.utils import expect, run_cmd_no_fail -import argparse, sys, os, glob ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/pelayout b/CIME/Tools/pelayout index d4c3575359b..c109435c96c 100755 --- a/CIME/Tools/pelayout +++ b/CIME/Tools/pelayout @@ -38,12 +38,13 @@ If you encounter problems with this tool or find it is missing any feature that you need, please open an issue on https://github.com/ESMCI/cime """ +import re +import sys + from standard_script_setup import * from CIME.case import Case -from CIME.utils import expect, convert_to_string -import sys -import re +from CIME.utils import convert_to_string, expect logger = logging.getLogger(__name__) diff --git a/CIME/Tools/preview_namelists b/CIME/Tools/preview_namelists index 69b89eb5663..0cc3dc988ae 100755 --- a/CIME/Tools/preview_namelists +++ b/CIME/Tools/preview_namelists @@ -15,12 +15,13 @@ Typical usage is simply: ./preview_namelists """ +import argparse + from standard_script_setup import * from CIME.case import Case from CIME.utils import expect -import argparse ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/save_provenance b/CIME/Tools/save_provenance index 7e3506ff50c..ce4498e0e62 100755 --- a/CIME/Tools/save_provenance +++ b/CIME/Tools/save_provenance @@ -4,15 +4,15 @@ This tool provide command-line access to provenance-saving functionality """ +import logging + from standard_script_setup import * from CIME.case import Case from CIME.config import Config +from CIME.get_timing import get_timing from CIME.provenance import * from CIME.utils import get_lids -from CIME.get_timing import get_timing - -import logging logger = logging.getLogger(__name__) diff --git a/CIME/Tools/simple_compare b/CIME/Tools/simple_compare index 59bc328341c..034f3862337 100755 --- a/CIME/Tools/simple_compare +++ b/CIME/Tools/simple_compare @@ -5,11 +5,15 @@ Compare files in a normalized way. Used by create_test for diffing non-namelist files. """ +import argparse +import os +import sys + from standard_script_setup import * + import CIME.simple_compare from CIME.utils import expect -import argparse, sys, os ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/standard_script_setup.py b/CIME/Tools/standard_script_setup.py index 2c38a5c4035..f82dcd65ed1 100644 --- a/CIME/Tools/standard_script_setup.py +++ b/CIME/Tools/standard_script_setup.py @@ -1,11 +1,19 @@ """ Encapsulate the importing of python utils and logging setup, things that every script should do. + +This module is DEPRECATED for star imports. New scripts should use explicit +imports instead of `from CIME.Tools.standard_script_setup import *`. + +The exports are maintained for backward compatibility. """ + # pylint: disable=unused-import -import sys, os -import __main__ as main +import argparse # noqa: F401 +import logging # noqa: F401 +import os +import sys def check_minimum_python_version(major, minor, warn_only=False): @@ -48,7 +56,6 @@ def check_minimum_python_version(major, minor, warn_only=False): # Important: Allows external tools to link up with CIME os.environ["CIMEROOT"] = cimeroot -import CIME.utils +import CIME.utils # noqa: E402 CIME.utils.stop_buffering_output() -import argparse, logging diff --git a/CIME/Tools/testreporter.py b/CIME/Tools/testreporter.py index a76c113ac09..7397d1f72c8 100755 --- a/CIME/Tools/testreporter.py +++ b/CIME/Tools/testreporter.py @@ -3,21 +3,22 @@ """ Simple script to populate CESM test database with test results. """ + import os import sys sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) -from CIME.Tools.standard_script_setup import * +import glob +from CIME.Tools.standard_script_setup import * +from CIME.utils import expect from CIME.XML.env_build import EnvBuild from CIME.XML.env_case import EnvCase from CIME.XML.env_test import EnvTest -from CIME.XML.test_reporter import TestReporter -from CIME.utils import expect from CIME.XML.generic_xml import GenericXML +from CIME.XML.test_reporter import TestReporter -import glob ############################################################################### def parse_command_line(args): diff --git a/CIME/Tools/wait_for_tests b/CIME/Tools/wait_for_tests index 484b393b6f4..19e298d861e 100755 --- a/CIME/Tools/wait_for_tests +++ b/CIME/Tools/wait_for_tests @@ -8,11 +8,13 @@ for the RUN phase specifically and will not terminate if the RUN phase didn't happen. """ -from standard_script_setup import * +import argparse +import os +import sys + import CIME.wait_for_tests -import argparse, sys, os ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/xmlchange b/CIME/Tools/xmlchange index 4101f9b5dba..23b76f95e53 100755 --- a/CIME/Tools/xmlchange +++ b/CIME/Tools/xmlchange @@ -50,18 +50,18 @@ Examples: ./xmlchange JOB_WALLCLOCK_TIME=0:30 """ +import re + from standard_script_setup import * +from CIME.case import Case +from CIME.locked_files import check_lockedfiles +from CIME.status import append_case_status from CIME.utils import ( - expect, - convert_to_type, Timeout, + convert_to_type, + expect, ) -from CIME.status import append_case_status -from CIME.case import Case -from CIME.locked_files import check_lockedfiles - -import re # Set logger logger = logging.getLogger("xmlchange") diff --git a/CIME/Tools/xmlconvertors/config_pes_converter.py b/CIME/Tools/xmlconvertors/config_pes_converter.py index cf57c21fb9b..2c36090199d 100755 --- a/CIME/Tools/xmlconvertors/config_pes_converter.py +++ b/CIME/Tools/xmlconvertors/config_pes_converter.py @@ -8,21 +8,26 @@ CIME2: cime/machines-acme/config_pes.xml CIME5: config/acme/allactive/config_pesall.xml """ -import os, sys + +import os +import sys sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) ) +import xml.etree.ElementTree as ET +from shutil import which + +import grid_xml_converter + from CIME import utils from CIME.Tools.standard_script_setup import * from CIME.utils import run_cmd -from shutil import which -import xml.etree.ElementTree as ET -import grid_xml_converter LOGGER = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args): ############################################################################### @@ -200,7 +205,9 @@ def set_data(self, xmlnode): class PesTree(grid_xml_converter.DataTree): def __init__(self, xmlfilename): # original xml file has bad comments - import re, StringIO + import re + + import StringIO if os.access(xmlfilename, os.R_OK): with open(xmlfilename, "r") as xmlfile: diff --git a/CIME/Tools/xmlconvertors/convert-grid-v1-to-v2 b/CIME/Tools/xmlconvertors/convert-grid-v1-to-v2 index 53b22d0e4a6..42d7a497bff 100755 --- a/CIME/Tools/xmlconvertors/convert-grid-v1-to-v2 +++ b/CIME/Tools/xmlconvertors/convert-grid-v1-to-v2 @@ -4,16 +4,19 @@ Convert a grid file from v1 to v2. """ -import argparse, sys, os +import argparse +import os +import sys sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +from collections import OrderedDict + from standard_script_setup import * + from CIME.utils import expect from CIME.XML.generic_xml import GenericXML -import xml.etree.ElementTree as ET -from collections import OrderedDict ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/xmlconvertors/grid_xml_converter.py b/CIME/Tools/xmlconvertors/grid_xml_converter.py index 01de9ddcf9f..acd19b94850 100755 --- a/CIME/Tools/xmlconvertors/grid_xml_converter.py +++ b/CIME/Tools/xmlconvertors/grid_xml_converter.py @@ -16,21 +16,24 @@ # CIME5: config/acme/config_grids.xml # -import os, sys +import os +import sys sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) ) +import operator +import xml.etree.ElementTree as ET +from shutil import which + from CIME import utils from CIME.Tools.standard_script_setup import * from CIME.utils import run_cmd_no_fail -from shutil import which -import xml.etree.ElementTree as ET -import operator logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args): ############################################################################### diff --git a/CIME/Tools/xmlquery b/CIME/Tools/xmlquery index 25ca0a2ea2f..56b6cb4a3d4 100755 --- a/CIME/Tools/xmlquery +++ b/CIME/Tools/xmlquery @@ -97,12 +97,14 @@ There are two usage modes: - The env_mach_specific.xml and env_archive.xml files are not supported by this tool. """ +import re +import sys +import textwrap + from standard_script_setup import * from CIME.case import Case -from CIME.utils import expect, convert_to_string - -import textwrap, sys, re +from CIME.utils import convert_to_string, expect logger = logging.getLogger("xmlquery") unsupported_files = ["env_mach_specific.xml", "env_archive.xml"] diff --git a/CIME/XML/archive.py b/CIME/XML/archive.py index e9d13eae686..8c34780acbb 100644 --- a/CIME/XML/archive.py +++ b/CIME/XML/archive.py @@ -2,11 +2,13 @@ Interface to the archive.xml file. This class inherits from GenericXML.py """ -from CIME.XML.standard_module_setup import * +import logging +import os +from copy import deepcopy + from CIME.config import Config from CIME.XML.archive_base import ArchiveBase from CIME.XML.files import Files -from copy import deepcopy logger = logging.getLogger(__name__) diff --git a/CIME/XML/archive_base.py b/CIME/XML/archive_base.py index 360424b734d..306cd2b7f71 100644 --- a/CIME/XML/archive_base.py +++ b/CIME/XML/archive_base.py @@ -1,9 +1,13 @@ """ Base class for archive files. This class inherits from generic_xml.py """ -from CIME.XML.standard_module_setup import * + +import logging +import os +import re + +from CIME.utils import convert_to_type, expect from CIME.XML.generic_xml import GenericXML -from CIME.utils import convert_to_type logger = logging.getLogger(__name__) diff --git a/CIME/XML/batch.py b/CIME/XML/batch.py index 75d9b1cfb94..fa2213f7c0d 100644 --- a/CIME/XML/batch.py +++ b/CIME/XML/batch.py @@ -4,10 +4,13 @@ The batch_system type="foo" blocks define most things. Machine-specific overrides can be defined by providing a batch_system MACH="mach" block. """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files + +import logging +import os + from CIME.utils import expect +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/component.py b/CIME/XML/component.py index abd09c86fd0..6120769ece8 100644 --- a/CIME/XML/component.py +++ b/CIME/XML/component.py @@ -1,11 +1,13 @@ """ Interface to the config_component.xml files. This class inherits from EntryID.py """ -from CIME.XML.standard_module_setup import * +import logging +import re + +from CIME.utils import expect from CIME.XML.entry_id import EntryID from CIME.XML.files import Files -from CIME.utils import get_cime_root logger = logging.getLogger(__name__) diff --git a/CIME/XML/compsets.py b/CIME/XML/compsets.py index eef9253da0f..415b16a14a2 100644 --- a/CIME/XML/compsets.py +++ b/CIME/XML/compsets.py @@ -2,11 +2,13 @@ Common interface to XML files which follow the compsets format, """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML +import logging + +from CIME.core.exceptions import CIMEError +from CIME.utils import expect from CIME.XML.entry_id import EntryID from CIME.XML.files import Files -from CIME.utils import CIMEError +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/entry_id.py b/CIME/XML/entry_id.py index 8a7acd92a4b..48927be1df9 100644 --- a/CIME/XML/entry_id.py +++ b/CIME/XML/entry_id.py @@ -3,8 +3,12 @@ this is an abstract class and is expected to be used by other XML interface modules and not directly. """ -from CIME.XML.standard_module_setup import * -from CIME.utils import expect, convert_to_string, convert_to_type + +import logging +import os +import re + +from CIME.utils import convert_to_string, convert_to_type, expect from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_archive.py b/CIME/XML/env_archive.py index e3752017aa2..26357452d04 100644 --- a/CIME/XML/env_archive.py +++ b/CIME/XML/env_archive.py @@ -1,12 +1,17 @@ """ Interface to the env_archive.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * + +import logging +import os + from CIME import utils from CIME.XML.archive_base import ArchiveBase from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) + + # pylint: disable=super-init-not-called class EnvArchive(ArchiveBase, EnvBase): def __init__(self, case_root=None, infile="env_archive.xml", read_only=False): diff --git a/CIME/XML/env_base.py b/CIME/XML/env_base.py index b30d33da988..f0789761333 100644 --- a/CIME/XML/env_base.py +++ b/CIME/XML/env_base.py @@ -1,10 +1,13 @@ """ Base class for env files. This class inherits from EntryID.py """ -from CIME.XML.standard_module_setup import * + +import logging +import os + +from CIME.utils import convert_to_type, expect from CIME.XML.entry_id import EntryID from CIME.XML.headers import Headers -from CIME.utils import convert_to_type logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_batch.py b/CIME/XML/env_batch.py index fbcc112c56f..f2e5e32088f 100644 --- a/CIME/XML/env_batch.py +++ b/CIME/XML/env_batch.py @@ -2,25 +2,30 @@ Interface to the env_batch.xml file. This class inherits from EnvBase """ +import logging +import math import os -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase +import pathlib +import re +import stat +from collections import OrderedDict +from itertools import zip_longest + from CIME import utils from CIME.utils import ( - transform_vars, - get_cime_root, - convert_to_seconds, + add_flag_to_cmd, convert_to_babylonian_time, - get_cime_config, + convert_to_seconds, + expect, + format_time, get_batch_script_for_job, + get_cime_config, get_logging_options, - format_time, - add_flag_to_cmd, + run_cmd, + run_cmd_no_fail, + transform_vars, ) -from collections import OrderedDict -import stat, re, math -import pathlib -from itertools import zip_longest +from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_build.py b/CIME/XML/env_build.py index fe863e414ef..bc72258e1c6 100644 --- a/CIME/XML/env_build.py +++ b/CIME/XML/env_build.py @@ -1,7 +1,9 @@ """ Interface to the env_build.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * + +import logging +import os from CIME import utils from CIME.XML.env_base import EnvBase diff --git a/CIME/XML/env_case.py b/CIME/XML/env_case.py index 1b4c85d6f88..e0af1f951d6 100644 --- a/CIME/XML/env_case.py +++ b/CIME/XML/env_case.py @@ -1,7 +1,9 @@ """ Interface to the env_case.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * + +import logging +import os from CIME import utils from CIME.XML.env_base import EnvBase diff --git a/CIME/XML/env_mach_pes.py b/CIME/XML/env_mach_pes.py index 86a9cb7d40c..8051ab91dcf 100644 --- a/CIME/XML/env_mach_pes.py +++ b/CIME/XML/env_mach_pes.py @@ -1,10 +1,14 @@ """ Interface to the env_mach_pes.xml file. This class inherits from EntryID """ -from CIME.XML.standard_module_setup import * + +import logging +import math +import os + from CIME import utils +from CIME.utils import expect from CIME.XML.env_base import EnvBase -import math logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_mach_specific.py b/CIME/XML/env_mach_specific.py index dcd282d614f..d382ca46792 100644 --- a/CIME/XML/env_mach_specific.py +++ b/CIME/XML/env_mach_specific.py @@ -1,14 +1,19 @@ """ Interface to the env_mach_specific.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase -from CIME import utils -from CIME.utils import transform_vars, get_cime_root -import string, resource, platform +import logging +import os +import platform +import re +import resource +import string from collections import OrderedDict +from CIME import utils +from CIME.utils import expect, run_cmd, run_cmd_no_fail, transform_vars +from CIME.XML.env_base import EnvBase + logger = logging.getLogger(__name__) # Helpful map for converting resource limits to shell commands @@ -32,6 +37,7 @@ "RLIMIT_STACK": ("stacksize", "-s"), } + # Is not of type EntryID but can use functions from EntryID (e.g # get_type) otherwise need to implement own functions and make GenericXML parent class class EnvMachSpecific(EnvBase): diff --git a/CIME/XML/env_postprocessing.py b/CIME/XML/env_postprocessing.py index 90f56f24d64..7346c1c8b7d 100644 --- a/CIME/XML/env_postprocessing.py +++ b/CIME/XML/env_postprocessing.py @@ -1,11 +1,12 @@ """ Interface to the env_postprocessing.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase +import logging +import os from CIME import utils +from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_run.py b/CIME/XML/env_run.py index 2e34c86c8f7..453c101eae7 100644 --- a/CIME/XML/env_run.py +++ b/CIME/XML/env_run.py @@ -1,12 +1,13 @@ """ Interface to the env_run.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase +import logging +import os from CIME import utils from CIME.utils import convert_to_type +from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_test.py b/CIME/XML/env_test.py index 9dbe3615eff..cbf331fe76d 100644 --- a/CIME/XML/env_test.py +++ b/CIME/XML/env_test.py @@ -1,10 +1,11 @@ """ Interface to the env_test.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase +import logging + from CIME.utils import convert_to_type +from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_workflow.py b/CIME/XML/env_workflow.py index 6de05f34e93..fc5d2e4311c 100644 --- a/CIME/XML/env_workflow.py +++ b/CIME/XML/env_workflow.py @@ -2,11 +2,11 @@ Interface to the env_workflow.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase -from CIME.utils import get_cime_root +import logging +import math -import re, math +from CIME.utils import expect +from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) diff --git a/CIME/XML/expected_fails_file.py b/CIME/XML/expected_fails_file.py index 82e6ef0a298..f0ba93d0991 100644 --- a/CIME/XML/expected_fails_file.py +++ b/CIME/XML/expected_fails_file.py @@ -42,11 +42,12 @@ """ -from CIME.XML.standard_module_setup import * +import logging +import os from CIME import utils -from CIME.XML.generic_xml import GenericXML from CIME.expected_fails import ExpectedFails +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/files.py b/CIME/XML/files.py index b2612cc8937..8a29283dd4b 100644 --- a/CIME/XML/files.py +++ b/CIME/XML/files.py @@ -2,19 +2,19 @@ Interface to the config_files.xml file. This class inherits from EntryID.py """ -import re +import logging import os -from CIME.XML.standard_module_setup import * +import re -from CIME.XML.entry_id import EntryID from CIME.utils import ( expect, + get_cime_default_driver, get_cime_root, get_config_path, - get_schema_path, get_model, - get_cime_default_driver, + get_schema_path, ) +from CIME.XML.entry_id import EntryID logger = logging.getLogger(__name__) diff --git a/CIME/XML/generic_xml.py b/CIME/XML/generic_xml.py index 6c7ff812a88..e350f108304 100644 --- a/CIME/XML/generic_xml.py +++ b/CIME/XML/generic_xml.py @@ -2,16 +2,19 @@ Common interface to XML files, this is an abstract class and is expected to be used by other XML interface modules and not directly. """ -from CIME.XML.standard_module_setup import * -from CIME.utils import safe_copy, get_src_root +import getpass +import logging +import os +import re import xml.etree.ElementTree as ET +from collections import namedtuple +from copy import deepcopy # pylint: disable=import-error from shutil import which -import getpass -from copy import deepcopy -from collections import namedtuple + +from CIME.utils import expect, get_cime_root, get_src_root, run_cmd_no_fail, safe_copy logger = logging.getLogger(__name__) @@ -398,10 +401,12 @@ def get_child(self, name=None, attributes=None, root=None, err_msg=None): ) expect( child, - err_msg - if err_msg - else "Expected one child, found None with name '{}' and attribs '{}' in file {}".format( - name, attributes, self.filename + ( + err_msg + if err_msg + else "Expected one child, found None with name '{}' and attribs '{}' in file {}".format( + name, attributes, self.filename + ) ), ) return child @@ -418,10 +423,12 @@ def get_optional_child(self, name=None, attributes=None, root=None, err_msg=None expect( len(children) <= 1, - err_msg - if err_msg - else "Multiple matches for name '{}' and attribs '{}' in file {}".format( - name, attributes, self.filename + ( + err_msg + if err_msg + else "Multiple matches for name '{}' and attribs '{}' in file {}".format( + name, attributes, self.filename + ) ), ) return children[0] if children else None diff --git a/CIME/XML/grids.py b/CIME/XML/grids.py index 60bc1ff3d02..fbde52f6dc2 100644 --- a/CIME/XML/grids.py +++ b/CIME/XML/grids.py @@ -3,8 +3,12 @@ This is not an abstract class - but inherits from the abstact class GenericXML """ +import logging +import os +import re from collections import OrderedDict -from CIME.XML.standard_module_setup import * + +from CIME.utils import expect from CIME.XML.files import Files from CIME.XML.generic_xml import GenericXML diff --git a/CIME/XML/headers.py b/CIME/XML/headers.py index 5937a1d03cb..164ba7bccb2 100644 --- a/CIME/XML/headers.py +++ b/CIME/XML/headers.py @@ -1,10 +1,11 @@ """ Interface to the config_headers.xml file. This class inherits from EntryID.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML +import logging + from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/inputdata.py b/CIME/XML/inputdata.py index 18b71dca4dd..46459ccfff6 100644 --- a/CIME/XML/inputdata.py +++ b/CIME/XML/inputdata.py @@ -1,10 +1,11 @@ """ Interface to the config_inputdata.xml file. This class inherits from GenericXML.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML + +import logging + from CIME.XML.files import Files -from CIME.utils import expect +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/machines.py b/CIME/XML/machines.py index c40e73ba3b6..467c83ead91 100644 --- a/CIME/XML/machines.py +++ b/CIME/XML/machines.py @@ -2,17 +2,18 @@ Interface to the config_machines.xml file. This class inherits from GenericXML.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files -from CIME.utils import CIMEError, expect, convert_to_unknown_type, get_cime_config - -import re import logging +import os +import re import socket from functools import partial from pathlib import Path +from CIME.core.exceptions import CIMEError +from CIME.utils import convert_to_unknown_type, expect, get_cime_config +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML + logger = logging.getLogger(__name__) diff --git a/CIME/XML/namelist_definition.py b/CIME/XML/namelist_definition.py index 25f7c0e0432..ebb35c09582 100644 --- a/CIME/XML/namelist_definition.py +++ b/CIME/XML/namelist_definition.py @@ -10,19 +10,21 @@ # Disable warnings due to using `standard_module_setup` # pylint:disable=wildcard-import,unused-wildcard-import -import re import collections +import logging +import os +import re +from CIME.core.exceptions import CIMEError from CIME.namelist import ( - fortran_namelist_base_value, - is_valid_fortran_namelist_literal, + Namelist, character_literal_to_string, expand_literal_list, - Namelist, + fortran_namelist_base_value, get_fortran_name_only, + is_valid_fortran_namelist_literal, ) -from CIME.utils import CIMEError -from CIME.XML.standard_module_setup import * +from CIME.utils import expect from CIME.XML.entry_id import EntryID from CIME.XML.files import Files @@ -32,7 +34,6 @@ class CaseInsensitiveDict(dict): - """Basic case insensitive dict with strings only keys. From https://stackoverflow.com/a/27890005""" @@ -65,7 +66,6 @@ def __setitem__(self, k, v): class NamelistDefinition(EntryID): - """Class representing variable definitions for a namelist. This class inherits from `EntryID`, and supports most inherited methods; however, `set_value` is unsupported. diff --git a/CIME/XML/pes.py b/CIME/XML/pes.py index 666382c0a38..e45af99c939 100644 --- a/CIME/XML/pes.py +++ b/CIME/XML/pes.py @@ -2,10 +2,12 @@ Interface to the config_pes.xml file. This class inherits from GenericXML.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files +import logging +import re + from CIME.utils import expect +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/pio.py b/CIME/XML/pio.py index 54af6112bf0..42953263a01 100644 --- a/CIME/XML/pio.py +++ b/CIME/XML/pio.py @@ -1,12 +1,13 @@ """ Class for config_pio files . This class inherits from EntryID.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.entry_id import EntryID -from CIME.XML.files import Files +import logging from collections import OrderedDict +from CIME.XML.entry_id import EntryID +from CIME.XML.files import Files + logger = logging.getLogger(__name__) diff --git a/CIME/XML/postprocessing.py b/CIME/XML/postprocessing.py index 3287d145142..22144455aaa 100644 --- a/CIME/XML/postprocessing.py +++ b/CIME/XML/postprocessing.py @@ -2,10 +2,12 @@ Interface to the config_postprocessing.xml file. This class inherits from EntryID """ -from CIME.XML.standard_module_setup import * +import logging +import os + +from CIME.utils import expect from CIME.XML.entry_id import EntryID from CIME.XML.files import Files -from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/XML/standard_module_setup.py b/CIME/XML/standard_module_setup.py index 1c934407da5..7957ee191db 100644 --- a/CIME/XML/standard_module_setup.py +++ b/CIME/XML/standard_module_setup.py @@ -1,8 +1,20 @@ # pragma pylint: disable=unused-import +""" +Standard imports for CIME XML modules. -import logging, os, sys, re +This module is DEPRECATED. New code should use explicit imports instead of +`from CIME.XML.standard_module_setup import *`. + +The exports are maintained for backward compatibility with external code +(E3SM, CESM, NorESM cime_config directories). +""" + +import logging # noqa: F401 +import os # noqa: F401 +import re # noqa: F401 +import sys # noqa: F401 LIB_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(LIB_DIR) -from CIME.utils import expect, run_cmd, run_cmd_no_fail, get_cime_root +from CIME.utils import expect, get_cime_root, run_cmd, run_cmd_no_fail # noqa: F401 diff --git a/CIME/XML/stream.py b/CIME/XML/stream.py index 95dfa30ef7d..1418a09da09 100644 --- a/CIME/XML/stream.py +++ b/CIME/XML/stream.py @@ -3,10 +3,13 @@ stream files predate cime and so do not conform to entry id format """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files + +import logging +import os + from CIME.utils import expect +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/test_reporter.py b/CIME/XML/test_reporter.py index 93f117a8a46..17003715d3f 100644 --- a/CIME/XML/test_reporter.py +++ b/CIME/XML/test_reporter.py @@ -2,12 +2,15 @@ Interface to the testreporter xml. This class inherits from GenericXML.py """ + +import os +import ssl + # pylint: disable=import-error import urllib.parse import urllib.request -from CIME.XML.standard_module_setup import * + from CIME.XML.generic_xml import GenericXML -import ssl # pylint: disable=protected-access ssl._create_default_https_context = ssl._create_unverified_context diff --git a/CIME/XML/testlist.py b/CIME/XML/testlist.py index 6dbe79b8f98..dbe75db6332 100644 --- a/CIME/XML/testlist.py +++ b/CIME/XML/testlist.py @@ -35,10 +35,12 @@ - workflow: adds a workflow to the test """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML +import logging + +from CIME.utils import expect from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/tests.py b/CIME/XML/tests.py index e760d86604f..bcef473d5e5 100644 --- a/CIME/XML/tests.py +++ b/CIME/XML/tests.py @@ -1,13 +1,16 @@ """ Interface to the config_tests.xml file. This class inherits from GenericEntry """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files -from CIME.utils import find_system_test, CIMEError -from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo +import logging +import os + +from CIME.core.exceptions import CIMEError from CIME.SystemTests.system_tests_compare_n import SystemTestsCompareN +from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo +from CIME.utils import find_system_test +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/testspec.py b/CIME/XML/testspec.py index 9b4e7c37724..5686a800f37 100644 --- a/CIME/XML/testspec.py +++ b/CIME/XML/testspec.py @@ -1,8 +1,11 @@ """ Interface to the testspec.xml file. This class inherits from generic_xml.py """ -from CIME.XML.standard_module_setup import * +import logging +import os + +from CIME.utils import expect from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/workflow.py b/CIME/XML/workflow.py index cdb2d27b403..113ab7b4965 100644 --- a/CIME/XML/workflow.py +++ b/CIME/XML/workflow.py @@ -2,10 +2,12 @@ Interface to the config_workflow.xml file. This class inherits from GenericXML.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files +import logging +import os + from CIME.utils import expect +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/aprun.py b/CIME/aprun.py index ca767c88db8..65b391dd071 100755 --- a/CIME/aprun.py +++ b/CIME/aprun.py @@ -3,12 +3,12 @@ code to compute and assemble aprun commands. """ -from CIME.XML.standard_module_setup import * - +import logging import math logger = logging.getLogger(__name__) + ############################################################################### def _get_aprun_cmd_for_case_impl( ntasks, diff --git a/CIME/baselines/performance.py b/CIME/baselines/performance.py index 67c19dbd43f..a46d4304265 100644 --- a/CIME/baselines/performance.py +++ b/CIME/baselines/performance.py @@ -1,10 +1,11 @@ -import os import glob -import re import gzip import logging +import os +import re + from CIME.config import Config -from CIME.utils import expect, get_src_root, get_current_commit, get_timestamp +from CIME.utils import expect, get_current_commit, get_src_root, get_timestamp logger = logging.getLogger(__name__) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 3028cfa4390..f4066969e84 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -1,24 +1,40 @@ -import CIME.compare_namelists, CIME.simple_compare -from CIME.test_scheduler import NAMELIST_PHASE -from CIME.utils import ( - run_cmd, - get_scripts_root, - EnvironmentContext, - parse_test_name, - match_any, - CIMEError, -) -from CIME.config import Config -from CIME.test_status import * -from CIME.hist_utils import generate_baseline, compare_baseline, NO_ORIGINAL -from CIME.case import Case -from CIME.test_utils import get_test_status_files +import logging +import os +import re +import time + +import CIME.compare_namelists +import CIME.simple_compare from CIME.baselines.performance import ( - perf_compare_throughput_baseline, perf_compare_memory_baseline, + perf_compare_throughput_baseline, perf_write_baseline, ) -import os, time +from CIME.case import Case +from CIME.config import Config +from CIME.hist_utils import NO_ORIGINAL, compare_baseline, generate_baseline +from CIME.test_status import ( + ALL_PHASES, + BASELINE_PHASE, + GENERATE_PHASE, + MEMCOMP_PHASE, + NAMELIST_PHASE, + RUN_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + THROUGHPUT_PHASE, + TestStatus, +) +from CIME.test_utils import get_test_status_files +from CIME.utils import ( + CIMEError, + EnvironmentContext, + expect, + get_scripts_root, + match_any, + parse_test_name, + run_cmd, +) logger = logging.getLogger(__name__) diff --git a/CIME/build.py b/CIME/build.py index af1709bb59c..66208d7af22 100644 --- a/CIME/build.py +++ b/CIME/build.py @@ -2,26 +2,35 @@ functions for building CIME models """ -import glob, shutil, time, threading, subprocess +import glob +import logging +import os +import re +import shutil +import subprocess +import threading +import time from pathlib import Path -from CIME.XML.standard_module_setup import * + +from CIME.config import Config +from CIME.locked_files import check_lockedfiles, lock_file, unlock_file from CIME.status import run_and_log_case_status from CIME.utils import ( - get_model, analyze_build_log, - stringify_bool, - get_timestamp, - run_sub_or_cmd, - run_cmd, + expect, get_batch_script_for_job, - gzip_existing_file, - safe_copy, - is_python_executable, get_logging_options, + get_model, + get_timestamp, + gzip_existing_file, import_from_file, + is_python_executable, + run_cmd, + run_cmd_no_fail, + run_sub_or_cmd, + safe_copy, + stringify_bool, ) -from CIME.config import Config -from CIME.locked_files import lock_file, unlock_file, check_lockedfiles from CIME.XML.files import Files logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.cprnc b/CIME/build_scripts/buildlib.cprnc index 51426de27c6..187fb5c53f1 100755 --- a/CIME/build_scripts/buildlib.cprnc +++ b/CIME/build_scripts/buildlib.cprnc @@ -1,14 +1,17 @@ #!/usr/bin/env python3 -import sys, os, logging, argparse +import argparse +import logging +import os +import sys _CIMEROOT = os.getenv("CIMEROOT") sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) -from standard_script_setup import * + from CIME import utils -from CIME.utils import run_bld_cmd_ensure_logging, CIMEError -from CIME.case import Case from CIME.build import get_standard_cmake_args +from CIME.case import Case +from CIME.utils import CIMEError, run_bld_cmd_ensure_logging logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.gptl b/CIME/build_scripts/buildlib.gptl index 774e5659396..786943dbc53 100755 --- a/CIME/build_scripts/buildlib.gptl +++ b/CIME/build_scripts/buildlib.gptl @@ -1,14 +1,17 @@ #!/usr/bin/env python3 -import sys, os, logging, argparse +import argparse +import logging +import os +import sys cimeroot = os.getenv("CIMEROOT") sys.path.append(os.path.join(cimeroot, "CIME", "Tools")) -from standard_script_setup import * + from CIME import utils -from CIME.utils import run_bld_cmd_ensure_logging -from CIME.case import Case from CIME.build import get_standard_makefile_args +from CIME.case import Case +from CIME.utils import run_bld_cmd_ensure_logging logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.internal_components b/CIME/build_scripts/buildlib.internal_components index f7f597d776e..5d79e5234f5 100755 --- a/CIME/build_scripts/buildlib.internal_components +++ b/CIME/build_scripts/buildlib.internal_components @@ -5,14 +5,15 @@ build cime component model library. This buildlib script is used by all cime i components. """ -import sys, os +import os +import sys _CIMEROOT = os.environ.get("CIMEROOT") if _CIMEROOT == None: raise ValueError("ERROR: CIMEROOT not defined in buildlib.internal_components.") sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) -from standard_script_setup import * + from CIME.buildlib import build_cime_component_lib, parse_input from CIME.case import Case diff --git a/CIME/build_scripts/buildlib.kokkos b/CIME/build_scripts/buildlib.kokkos index 6ecceb6ae48..5ed918d6332 100755 --- a/CIME/build_scripts/buildlib.kokkos +++ b/CIME/build_scripts/buildlib.kokkos @@ -1,12 +1,15 @@ #!/usr/bin/env python3 -import os, sys, argparse, logging +import argparse +import logging +import os +import sys + -from standard_script_setup import * from CIME import utils -from CIME.utils import expect, run_bld_cmd_ensure_logging, run_cmd_no_fail, run_cmd -from CIME.case import Case from CIME.build import get_standard_makefile_args +from CIME.case import Case +from CIME.utils import expect, run_bld_cmd_ensure_logging, run_cmd, run_cmd_no_fail logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.mct b/CIME/build_scripts/buildlib.mct index 446020839e4..884572f7a51 100755 --- a/CIME/build_scripts/buildlib.mct +++ b/CIME/build_scripts/buildlib.mct @@ -1,16 +1,20 @@ #!/usr/bin/env python3 -import sys, os, logging, argparse +import argparse +import logging +import os +import sys _CIMEROOT = os.getenv("CIMEROOT") sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) -from standard_script_setup import * -from CIME.config import Config +import glob + + from CIME import utils -from CIME.utils import copyifnewer, run_bld_cmd_ensure_logging, expect -from CIME.case import Case from CIME.build import get_standard_makefile_args -import glob +from CIME.case import Case +from CIME.config import Config +from CIME.utils import copyifnewer, expect, run_bld_cmd_ensure_logging logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.mpi-serial b/CIME/build_scripts/buildlib.mpi-serial index f13d949ae45..b851a1139dc 100755 --- a/CIME/build_scripts/buildlib.mpi-serial +++ b/CIME/build_scripts/buildlib.mpi-serial @@ -1,13 +1,16 @@ #!/usr/bin/env python3 -import os, sys, logging, argparse +import argparse +import glob +import logging +import os +import sys + -from standard_script_setup import * from CIME import utils +from CIME.build import get_standard_makefile_args +from CIME.case import Case from CIME.config import Config from CIME.utils import copyifnewer, run_bld_cmd_ensure_logging -from CIME.case import Case -from CIME.build import get_standard_makefile_args -import glob logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.pio b/CIME/build_scripts/buildlib.pio index b04367b7b19..2e7a2935446 100755 --- a/CIME/build_scripts/buildlib.pio +++ b/CIME/build_scripts/buildlib.pio @@ -1,15 +1,20 @@ #!/usr/bin/env python3 -import sys, os, logging, argparse +import argparse +import logging +import os +import sys cimeroot = os.getenv("CIMEROOT") sys.path.append(os.path.join(cimeroot, "CIME", "Tools")) -import glob, re -from standard_script_setup import * +import glob +import re + + from CIME import utils -from CIME.utils import expect, run_bld_cmd_ensure_logging, safe_copy from CIME.build import get_standard_makefile_args from CIME.case import Case +from CIME.utils import expect, run_bld_cmd_ensure_logging, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib_cmake.internal_components b/CIME/build_scripts/buildlib_cmake.internal_components index 4c939041e62..b8a56583973 100755 --- a/CIME/build_scripts/buildlib_cmake.internal_components +++ b/CIME/build_scripts/buildlib_cmake.internal_components @@ -5,7 +5,8 @@ build cime component model library. This buildlib script is used by all cime i components. """ -import sys, os +import os +import sys _CIMEROOT = os.environ.get("CIMEROOT") if _CIMEROOT == None: @@ -14,7 +15,7 @@ if _CIMEROOT == None: ) sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) -from standard_script_setup import * + from CIME.buildlib import build_cime_component_lib, parse_input from CIME.case import Case diff --git a/CIME/buildlib.py b/CIME/buildlib.py index b1152f8923a..a487734423c 100644 --- a/CIME/buildlib.py +++ b/CIME/buildlib.py @@ -2,21 +2,24 @@ common utilities for buildlib """ -from CIME.XML.standard_module_setup import * +import argparse +import logging +import os + +from CIME.build import get_standard_makefile_args from CIME.case import Case +from CIME.config import Config from CIME.utils import ( parse_args_and_handle_standard_logging_options, - setup_standard_logging_options, + run_cmd, safe_copy, + setup_standard_logging_options, ) -from CIME.config import Config -from CIME.build import get_standard_makefile_args from CIME.XML.files import Files -import sys, os, argparse - logger = logging.getLogger(__name__) + ############################################################################### def parse_input(argv): ############################################################################### diff --git a/CIME/buildnml.py b/CIME/buildnml.py index 69cbc5f7dca..d16e05f1ab6 100644 --- a/CIME/buildnml.py +++ b/CIME/buildnml.py @@ -4,17 +4,22 @@ These are used by components///cime_config/buildnml """ -from CIME.XML.standard_module_setup import * +import argparse +import glob +import logging +import os +import re + from CIME.utils import ( expect, parse_args_and_handle_standard_logging_options, + safe_copy, setup_standard_logging_options, ) -from CIME.utils import safe_copy -import sys, os, argparse, glob logger = logging.getLogger(__name__) + ############################################################################### def parse_input(argv): ############################################################################### diff --git a/CIME/case/case.py b/CIME/case/case.py index bd6db1e503c..b7f549b7ac9 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -5,46 +5,64 @@ All interaction with and between the module files in XML/ takes place through the Case module. """ -from copy import deepcopy + +import getpass +import glob +import hashlib +import logging +import math +import os +import re +import shutil +import socket import sys -import glob, os, shutil, math, time, hashlib, socket, getpass -from CIME.XML.standard_module_setup import * +import time +from copy import deepcopy # pylint: disable=import-error,redefined-builtin from CIME import utils +from CIME.aprun import get_aprun_cmd_for_case from CIME.config import Config -from CIME.status import append_status -from CIME.utils import expect, get_cime_root -from CIME.utils import convert_to_type, get_model, set_model -from CIME.utils import get_project, get_charge_account, check_name -from CIME.utils import get_current_commit, safe_copy, get_cime_default_driver from CIME.gitinterface import GitInterface from CIME.locked_files import LOCKED_DIR, lock_file -from CIME.XML.machines import Machines -from CIME.XML.pes import Pes -from CIME.XML.files import Files -from CIME.XML.testlist import Testlist +from CIME.status import append_status +from CIME.user_mod_support import apply_user_mods +from CIME.utils import ( + check_name, + convert_to_type, + expect, + get_charge_account, + get_cime_default_driver, + get_cime_root, + get_current_commit, + get_model, + get_project, + safe_copy, + set_model, +) +from CIME.XML.archive import Archive +from CIME.XML.batch import Batch from CIME.XML.component import Component from CIME.XML.compsets import Compsets -from CIME.XML.grids import Grids -from CIME.XML.batch import Batch -from CIME.XML.workflow import Workflow -from CIME.XML.postprocessing import Postprocessing -from CIME.XML.pio import PIO -from CIME.XML.archive import Archive -from CIME.XML.env_test import EnvTest -from CIME.XML.env_mach_specific import EnvMachSpecific +from CIME.XML.env_archive import EnvArchive +from CIME.XML.env_batch import EnvBatch +from CIME.XML.env_build import EnvBuild from CIME.XML.env_case import EnvCase from CIME.XML.env_mach_pes import EnvMachPes -from CIME.XML.env_build import EnvBuild +from CIME.XML.env_mach_specific import EnvMachSpecific +from CIME.XML.env_postprocessing import EnvPostprocessing from CIME.XML.env_run import EnvRun -from CIME.XML.env_archive import EnvArchive -from CIME.XML.env_batch import EnvBatch +from CIME.XML.env_test import EnvTest from CIME.XML.env_workflow import EnvWorkflow -from CIME.XML.env_postprocessing import EnvPostprocessing +from CIME.XML.files import Files from CIME.XML.generic_xml import GenericXML -from CIME.user_mod_support import apply_user_mods -from CIME.aprun import get_aprun_cmd_for_case +from CIME.XML.grids import Grids +from CIME.XML.machines import Machines +from CIME.XML.pes import Pes +from CIME.XML.pio import PIO +from CIME.XML.postprocessing import Postprocessing +from CIME.XML.testlist import Testlist +from CIME.XML.workflow import Workflow logger = logging.getLogger(__name__) @@ -81,25 +99,25 @@ class Case(object): """ - from CIME.case.case_setup import case_setup, _create_case_repo - from CIME.case.case_clone import create_clone, _copy_user_modified_to_clone - from CIME.case.case_test import case_test - from CIME.case.case_submit import check_DA_settings, check_case, submit + from CIME.case.case_clone import _copy_user_modified_to_clone, create_clone + from CIME.case.case_cmpgen_namelists import case_cmpgen_namelists + from CIME.case.case_run import case_run + from CIME.case.case_setup import _create_case_repo, case_setup from CIME.case.case_st_archive import ( + archive_last_restarts, case_st_archive, restore_from_archive, - archive_last_restarts, - test_st_archive, test_env_archive, + test_st_archive, ) - from CIME.case.case_run import case_run - from CIME.case.case_cmpgen_namelists import case_cmpgen_namelists - from CIME.case.preview_namelists import create_dirs, create_namelists + from CIME.case.case_submit import check_case, check_DA_settings, submit + from CIME.case.case_test import case_test from CIME.case.check_input_data import ( check_all_input_data, - stage_refcase, check_input_data, + stage_refcase, ) + from CIME.case.preview_namelists import create_dirs, create_namelists def __init__(self, case_root=None, read_only=True, record=False, non_local=False): if case_root is None: diff --git a/CIME/case/case_clone.py b/CIME/case/case_clone.py index 8ed6ae89948..6f0b070076a 100644 --- a/CIME/case/case_clone.py +++ b/CIME/case/case_clone.py @@ -1,12 +1,16 @@ """ create_clone is a member of the Case class from file case.py """ -import os, glob, shutil -from CIME.XML.standard_module_setup import * -from CIME.utils import expect, check_name, safe_copy, get_model -from CIME.simple_compare import compare_files + +import glob +import logging +import os +import shutil + from CIME.locked_files import lock_file +from CIME.simple_compare import compare_files from CIME.user_mod_support import apply_user_mods +from CIME.utils import check_name, expect, get_cime_root, get_model, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/case/case_cmpgen_namelists.py b/CIME/case/case_cmpgen_namelists.py index 6a2579ada35..a4b016c3998 100644 --- a/CIME/case/case_cmpgen_namelists.py +++ b/CIME/case/case_cmpgen_namelists.py @@ -3,15 +3,25 @@ case_cmpgen_namelists is a member of Class case from file case.py """ -from CIME.XML.standard_module_setup import * - -from CIME.compare_namelists import is_namelist_file, compare_namelist_files +import glob +import logging +import os +import shutil +import stat +import sys +import traceback + +from CIME.compare_namelists import compare_namelist_files, is_namelist_file from CIME.simple_compare import compare_files, compare_runconfigfiles -from CIME.utils import safe_copy, SharedArea from CIME.status import append_status -from CIME.test_status import * - -import os, shutil, traceback, stat, glob +from CIME.test_status import ( + NAMELIST_PHASE, + RUN_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TestStatus, +) +from CIME.utils import SharedArea, expect, safe_copy logger = logging.getLogger(__name__) @@ -41,9 +51,11 @@ def _do_full_nl_comp(case, test, compare_name, baseline_root=None): comments = "NLCOMP\n" for item in all_items_to_compare: baseline_counterpart = os.path.join( - baseline_casedocs - if os.path.dirname(item).endswith("CaseDocs") - else baseline_dir, + ( + baseline_casedocs + if os.path.dirname(item).endswith("CaseDocs") + else baseline_dir + ), os.path.basename(item), ) if not os.path.exists(baseline_counterpart): diff --git a/CIME/case/case_run.py b/CIME/case/case_run.py index b9912589e5f..01b69c6ce29 100644 --- a/CIME/case/case_run.py +++ b/CIME/case/case_run.py @@ -2,16 +2,30 @@ case_run is a member of Class Case '""" -from CIME.XML.standard_module_setup import * +import glob +import logging +import os +import re +import shutil +import time + from CIME.config import Config -from CIME.utils import gzip_existing_file, new_lid -from CIME.utils import run_sub_or_cmd, safe_copy, model_log, CIMEError -from CIME.utils import batch_jobid, is_comp_standalone -from CIME.status import append_status, run_and_log_case_status +from CIME.core.exceptions import CIMEError from CIME.get_timing import get_timing from CIME.locked_files import check_lockedfiles - -import shutil, time, sys, os, glob +from CIME.status import append_status, run_and_log_case_status +from CIME.utils import ( + batch_jobid, + expect, + gzip_existing_file, + is_comp_standalone, + model_log, + new_lid, + run_cmd, + run_cmd_no_fail, + run_sub_or_cmd, + safe_copy, +) TERMINATION_TEXT = ("HAS ENDED", "END OF MODEL RUN", "SUCCESSFUL TERMINATION") diff --git a/CIME/case/case_setup.py b/CIME/case/case_setup.py index 38d6a4b0336..ece6e60466f 100644 --- a/CIME/case/case_setup.py +++ b/CIME/case/case_setup.py @@ -3,31 +3,38 @@ case_setup is a member of class Case from file case.py """ +import errno +import logging import os +import shutil -from CIME.XML.standard_module_setup import * -from CIME.config import Config -from CIME.XML.machines import Machines from CIME.BuildTools.configure import ( - generate_env_mach_specific, copy_depends_files, + generate_env_mach_specific, +) +from CIME.config import Config +from CIME.gitinterface import GitInterface +from CIME.locked_files import check_lockedfiles, lock_file, unlock_file +from CIME.status import append_case_status, run_and_log_case_status +from CIME.test_status import ( + SETUP_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TEST_PEND_STATUS, + TestStatus, ) from CIME.utils import ( - get_batch_script_for_job, - safe_copy, + CIMEError, + batch_jobid, + copy_local_macros_to_dir, + expect, file_contains_python_function, + get_batch_script_for_job, import_from_file, - copy_local_macros_to_dir, - batch_jobid, run_cmd_no_fail, - CIMEError, + safe_copy, ) -from CIME.status import run_and_log_case_status, append_case_status -from CIME.test_status import * -from CIME.locked_files import unlock_file, lock_file, check_lockedfiles -from CIME.gitinterface import GitInterface - -import errno, shutil +from CIME.XML.machines import Machines logger = logging.getLogger(__name__) @@ -432,9 +439,11 @@ def _case_setup_impl( if ngpus_per_node >= 0: case.set_value( "NGPUS_PER_NODE", - max(1, ngpus_per_node) - if ngpus_per_node <= max_gpus_per_node - else max_gpus_per_node, + ( + max(1, ngpus_per_node) + if ngpus_per_node <= max_gpus_per_node + else max_gpus_per_node + ), ) elif gpu_offload: raise CIMEError( diff --git a/CIME/case/case_st_archive.py b/CIME/case/case_st_archive.py index 932987d1a9a..e8c12692481 100644 --- a/CIME/case/case_st_archive.py +++ b/CIME/case/case_st_archive.py @@ -4,21 +4,26 @@ are members of class Case from file case.py """ -import shutil, glob, re, os +import glob +import logging +import os +import re +import shutil +from os.path import isdir, join -from CIME.XML.standard_module_setup import * +from CIME.date import get_file_date +from CIME.status import run_and_log_case_status from CIME.utils import ( + batch_jobid, + expect, + find_files, ls_sorted_by_mtime, - symlink_force, + run_cmd, safe_copy, - find_files, - batch_jobid, + symlink_force, ) -from CIME.status import run_and_log_case_status -from CIME.date import get_file_date from CIME.XML.archive import Archive from CIME.XML.files import Files -from os.path import isdir, join logger = logging.getLogger(__name__) diff --git a/CIME/case/case_submit.py b/CIME/case/case_submit.py index 9251e4ab092..2be69e4141b 100644 --- a/CIME/case/case_submit.py +++ b/CIME/case/case_submit.py @@ -6,17 +6,26 @@ jobs. submit, check_case and check_da_settings are members of class Case in file case.py """ + import configparser -from CIME.XML.standard_module_setup import * -from CIME.utils import expect, CIMEError, get_time_in_seconds -from CIME.status import run_and_log_case_status +import logging +import os + +from CIME.core.exceptions import CIMEError from CIME.locked_files import ( - unlock_file, - lock_file, check_lockedfile, check_lockedfiles, + lock_file, + unlock_file, +) +from CIME.status import run_and_log_case_status +from CIME.test_status import ( + SUBMIT_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TestStatus, ) -from CIME.test_status import * +from CIME.utils import expect, get_time_in_seconds logger = logging.getLogger(__name__) diff --git a/CIME/case/case_test.py b/CIME/case/case_test.py index c607dd93786..554526171dc 100644 --- a/CIME/case/case_test.py +++ b/CIME/case/case_test.py @@ -3,12 +3,14 @@ case_test is a member of class Case from case.py """ -from CIME.XML.standard_module_setup import * -from CIME.utils import expect, find_system_test, find_proc_id -from CIME.SystemTests.system_tests_common import * -from CIME.status import append_testlog +import logging +import os +import signal +import sys -import sys, signal +from CIME.status import append_testlog +from CIME.SystemTests.system_tests_common import * +from CIME.utils import expect, find_proc_id, find_system_test def _iter_signal_names(): diff --git a/CIME/case/check_input_data.py b/CIME/case/check_input_data.py index cc67e1c6aca..f85f9cb64da 100644 --- a/CIME/case/check_input_data.py +++ b/CIME/case/check_input_data.py @@ -1,12 +1,16 @@ """ API for checking input for testcase """ -from CIME.XML.standard_module_setup import * -from CIME.utils import SharedArea, find_files, safe_copy, expect -from CIME.XML.inputdata import Inputdata -import CIME.Servers -import glob, hashlib, shutil +import glob +import hashlib +import logging +import os +import shutil + +import CIME.Servers +from CIME.utils import SharedArea, expect, find_files, safe_copy +from CIME.XML.inputdata import Inputdata logger = logging.getLogger(__name__) # The inputdata_checksum.dat file will be read into this hash if it's available diff --git a/CIME/case/preview_namelists.py b/CIME/case/preview_namelists.py index 95d276e039a..e9f5848b5a6 100644 --- a/CIME/case/preview_namelists.py +++ b/CIME/case/preview_namelists.py @@ -3,9 +3,12 @@ create_dirs and create_namelists are members of Class case from file case.py """ -from CIME.XML.standard_module_setup import * -from CIME.utils import import_and_run_sub_or_cmd, safe_copy -import time, glob +import glob +import logging +import os +import time + +from CIME.utils import expect, import_and_run_sub_or_cmd, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/code_checker.py b/CIME/code_checker.py index 08a5a021b0f..52fbdd55886 100644 --- a/CIME/code_checker.py +++ b/CIME/code_checker.py @@ -2,24 +2,21 @@ Libraries for checking python code with pylint """ -import os import json +import logging +import os from shutil import which -from CIME.XML.standard_module_setup import * - from CIME.utils import ( - run_cmd, - run_cmd_no_fail, expect, + get_cime_default_driver, get_cime_root, get_src_root, is_python_executable, - get_cime_default_driver, + run_cmd, + run_cmd_no_fail, ) -from multiprocessing.dummy import Pool as ThreadPool - logger = logging.getLogger(__name__) diff --git a/CIME/compare_namelists.py b/CIME/compare_namelists.py index 9fdc005e3cc..7a830ccc18e 100644 --- a/CIME/compare_namelists.py +++ b/CIME/compare_namelists.py @@ -1,10 +1,15 @@ -import os, re, logging -from CIME.utils import expect, CIMEError +import logging +import os +import re + +from CIME.core.exceptions import CIMEError +from CIME.utils import expect logger = logging.getLogger(__name__) # pragma pylint: disable=unsubscriptable-object + ############################################################################### def _normalize_lists(value_str): ############################################################################### diff --git a/CIME/compare_test_results.py b/CIME/compare_test_results.py index 35274cc74ba..dce0267d177 100644 --- a/CIME/compare_test_results.py +++ b/CIME/compare_test_results.py @@ -1,12 +1,22 @@ -import CIME.compare_namelists, CIME.simple_compare -from CIME.status import append_status -from CIME.utils import EnvironmentContext, parse_test_name -from CIME.test_status import * -from CIME.hist_utils import compare_baseline, get_ts_synopsis +import logging +import os + +import CIME.compare_namelists +import CIME.simple_compare from CIME.case import Case +from CIME.hist_utils import compare_baseline, get_ts_synopsis +from CIME.status import append_status +from CIME.test_status import ( + BASELINE_PHASE, + NAMELIST_PHASE, + RUN_PHASE, + SETUP_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TestStatus, +) from CIME.test_utils import get_test_status_files - -import os, logging +from CIME.utils import EnvironmentContext, parse_test_name def append_status_cprnc_log(msg, logfile_name, test_dir): diff --git a/CIME/config.py b/CIME/config.py index f63a4bea78d..de34232fc4d 100644 --- a/CIME/config.py +++ b/CIME/config.py @@ -1,10 +1,10 @@ -import os -import re -import sys -import logging import importlib.machinery import importlib.util import inspect +import logging +import os +import re +import sys from pathlib import Path from CIME import utils @@ -185,7 +185,7 @@ def print_method_rst(self): if x[1].__class__ != Config and x[0] not in ignore ] - for (name, sig, doc) in child_methods: + for name, sig, doc in child_methods: if doc is None: continue print(".. code-block::\n") diff --git a/CIME/core/__init__.py b/CIME/core/__init__.py new file mode 100644 index 00000000000..14d792d5027 --- /dev/null +++ b/CIME/core/__init__.py @@ -0,0 +1,7 @@ +""" +CIME core package — DI-based abstractions for filesystem, process, environment, +clock, and configuration bootstrap. + +These protocols are designed for constructor injection so that business logic +can be tested with lightweight mocks instead of hitting real OS resources. +""" diff --git a/CIME/core/config/__init__.py b/CIME/core/config/__init__.py new file mode 100644 index 00000000000..ddadce17d5b --- /dev/null +++ b/CIME/core/config/__init__.py @@ -0,0 +1 @@ +"""Configuration bootstrap and loading utilities.""" diff --git a/CIME/core/config/bootstrap.py b/CIME/core/config/bootstrap.py new file mode 100644 index 00000000000..e8d2efbaeb8 --- /dev/null +++ b/CIME/core/config/bootstrap.py @@ -0,0 +1,150 @@ +""" +Centralized sys.path management for CIME. + +CIME is not pip-installed; it relies on sys.path manipulation so that +scripts executed via symlinks (e.g. case.setup, xmlchange) can find +the CIME package. This module consolidates the various sys.path.insert +patterns scattered across the codebase into one place. + +**This module is standalone for Slice 1** — existing call-sites are NOT +modified yet. Migration will happen incrementally in later slices. + +Typical usage (future, after wiring):: + + from CIME.core.config.bootstrap import bootstrap_cime + bootstrap_cime() # auto-detect CIMEROOT + bootstrap_cime(cimeroot="/explicit") # explicit root +""" + +import os +import sys +from typing import List, Optional, Sequence + + +def find_cimeroot(starting_dir: Optional[str] = None) -> str: + """Locate the CIME root directory. + + Resolution order: + 1. ``starting_dir`` argument (if provided) + 2. ``CIMEROOT`` environment variable + 3. Walk upward from this file to find the directory containing ``CIME/`` + + Returns: + Absolute path to the CIME root directory. + + Raises: + RuntimeError: If CIMEROOT cannot be determined. + """ + if starting_dir is not None: + resolved = os.path.abspath(starting_dir) + if _is_cimeroot(resolved): + return resolved + raise RuntimeError(f"Specified directory is not a valid CIMEROOT: {resolved}") + + env_root = os.environ.get("CIMEROOT") + if env_root and _is_cimeroot(env_root): + return os.path.abspath(env_root) + + # Walk up from this file: CIME/core/config/bootstrap.py -> CIME/core/config -> CIME/core -> CIME -> root + candidate = os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "..") + ) + if _is_cimeroot(candidate): + return candidate + + raise RuntimeError( + "Cannot determine CIMEROOT. Set the CIMEROOT environment variable " + "or pass cimeroot explicitly." + ) + + +def _is_cimeroot(path: str) -> bool: + """Check whether a directory looks like a valid CIMEROOT.""" + return os.path.isdir(os.path.join(path, "CIME")) + + +def bootstrap_cime( + cimeroot: Optional[str] = None, + extra_paths: Optional[Sequence[str]] = None, + set_env: bool = True, +) -> str: + """Set up sys.path for CIME and return the resolved CIMEROOT. + + This is the single function that should be called to prepare the + Python environment for CIME imports. It: + + 1. Resolves CIMEROOT + 2. Inserts CIMEROOT at the front of sys.path (if not already present) + 3. Inserts CIME/Tools at position 1 (if not already present) + 4. Inserts any extra_paths + 5. Optionally sets the CIMEROOT environment variable + + Args: + cimeroot: Explicit CIMEROOT path. If None, auto-detected. + extra_paths: Additional paths to insert after CIMEROOT and Tools. + set_env: Whether to set ``os.environ["CIMEROOT"]``. + + Returns: + The resolved CIMEROOT path. + """ + root = find_cimeroot(cimeroot) + tools_path = os.path.join(root, "CIME", "Tools") + + # Build the list of paths to ensure are in sys.path + paths_to_add: List[str] = [root, tools_path] + if extra_paths: + paths_to_add.extend(str(p) for p in extra_paths) + + _ensure_paths(paths_to_add) + + if set_env: + os.environ["CIMEROOT"] = root + + return root + + +def _ensure_paths(paths: Sequence[str]) -> None: + """Ensure each path is in sys.path, inserted at the front in order. + + Paths already present are moved to the correct position rather than + duplicated. + """ + for i, p in enumerate(paths): + absp = os.path.abspath(p) + # Remove existing entry if present (to re-position) + if absp in sys.path: + sys.path.remove(absp) + sys.path.insert(i, absp) + + +def get_tools_path(cimeroot: Optional[str] = None) -> str: + """Return the CIME/Tools directory path.""" + root = cimeroot or find_cimeroot() + return os.path.join(root, "CIME", "Tools") + + +def check_minimum_python_version( + major: int = 3, minor: int = 9, warn_only: bool = False +) -> None: + """Check that the running Python meets minimum version requirements. + + Consolidated from CIME/Tools/standard_script_setup.py. + + Args: + major: Required major version. + minor: Required minimum minor version. + warn_only: If True, print warning instead of raising. + + Raises: + RuntimeError: If version is insufficient and warn_only is False. + """ + if sys.version_info >= (major, minor): + return + msg = ( + f"Python {major}.{minor} is {'recommended' if warn_only else 'required'} " + f"to run CIME. You have {sys.version_info[0]}.{sys.version_info[1]}" + ) + if warn_only: + print(msg + ".", file=sys.stderr) + return + raise RuntimeError(msg + " - please use a newer version of Python.") diff --git a/CIME/core/exceptions.py b/CIME/core/exceptions.py new file mode 100644 index 00000000000..c8f69c59e4f --- /dev/null +++ b/CIME/core/exceptions.py @@ -0,0 +1,81 @@ +""" +Typed exception hierarchy for CIME. + +CIMEError inherits from both SystemExit and Exception so that: +- Tracebacks are suppressed in normal usage (SystemExit behavior) +- Exceptions are still catchable (Exception behavior) +- Users can run with --debug to see full tracebacks + +All CIME-specific exceptions should inherit from CIMEError. +""" + + +class CIMEError(SystemExit, Exception): + """Base exception for all CIME errors. + + Inherits from SystemExit to suppress tracebacks in normal usage, + and from Exception to remain catchable. + """ + + +class ConfigurationError(CIMEError): + """Invalid or missing configuration (XML files, env vars, etc.).""" + + +class BuildError(CIMEError): + """Failure during model build.""" + + +class SubmitError(CIMEError): + """Failure during job submission.""" + + +class SystemTestError(CIMEError): + """Failure in a CIME system test.""" + + +class LockingError(CIMEError): + """Case locking or unlocking failure.""" + + +class ExternalCommandError(CIMEError): + """An external command (subprocess) failed. + + Modeled after subprocess.CalledProcessError for consistency. + + Attributes: + returncode: Exit code of the command. + cmd: The command that was run (string or list). + output: Standard output captured from the command, if any. + stderr: Standard error captured from the command, if any. + """ + + def __init__( + self, + message: str, + returncode: int = 1, + cmd: str = "", + output: str = "", + stderr: str = "", + # Deprecated: use cmd instead + command: str = "", + ): + super().__init__(message) + self.returncode = returncode + # Support both 'cmd' (preferred) and 'command' (deprecated) for compatibility + self.cmd = cmd or command + self.output = output + self.stderr = stderr + + @property + def command(self) -> str: + """Deprecated: Use cmd instead.""" + return self.cmd + + +class RetryableExternalCommandError(ExternalCommandError): + """An external command failed but may succeed on retry (transient error).""" + + +class InputError(CIMEError): + """Invalid user input (bad arguments, missing required values).""" diff --git a/CIME/cs_status.py b/CIME/cs_status.py index 6a65ca4da71..2ab3de40979 100644 --- a/CIME/cs_status.py +++ b/CIME/cs_status.py @@ -4,13 +4,15 @@ """ from __future__ import print_function -from CIME.XML.standard_module_setup import * -from CIME.XML.expected_fails_file import ExpectedFailsFile -from CIME.test_status import TestStatus, SHAREDLIB_BUILD_PHASE, TEST_PEND_STATUS + import os import sys from collections import defaultdict +from CIME.test_status import SHAREDLIB_BUILD_PHASE, TEST_PEND_STATUS, TestStatus +from CIME.utils import expect +from CIME.XML.expected_fails_file import ExpectedFailsFile + def cs_status( test_paths, diff --git a/CIME/cs_status_creator.py b/CIME/cs_status_creator.py index 7a0fd40361a..49b456cb788 100644 --- a/CIME/cs_status_creator.py +++ b/CIME/cs_status_creator.py @@ -2,11 +2,11 @@ Creates a test suite-specific cs.status file from a template """ -from CIME.XML.standard_module_setup import * -import CIME.utils import os import stat +import CIME.utils + def create_cs_status(test_root, test_id, extra_args="", filename=None): """Create a test suite-specific cs.status file from the template diff --git a/CIME/date.py b/CIME/date.py index 0d12b2ceede..0e2179b9007 100644 --- a/CIME/date.py +++ b/CIME/date.py @@ -1,7 +1,9 @@ +import logging import re -from CIME.XML.standard_module_setup import * logger = logging.getLogger(__name__) + + ############################################################################### def get_file_date(filename): ############################################################################### diff --git a/CIME/expected_fails.py b/CIME/expected_fails.py index f8e5339702b..5bc457fcba8 100644 --- a/CIME/expected_fails.py +++ b/CIME/expected_fails.py @@ -2,7 +2,7 @@ Contains the definition of a class to hold information on expected failures for a single test """ -from CIME.XML.standard_module_setup import * +from CIME.utils import expect EXPECTED_FAILURE_COMMENT = "(EXPECTED FAILURE)" UNEXPECTED_FAILURE_COMMENT_START = "(UNEXPECTED" # There will be some additional text after this, before the end parentheses diff --git a/CIME/get_tests.py b/CIME/get_tests.py index a4d4abb3410..72c673e850c 100644 --- a/CIME/get_tests.py +++ b/CIME/get_tests.py @@ -1,7 +1,9 @@ +import os +import sys + import CIME.utils -from CIME.utils import expect, convert_to_seconds, parse_test_name, get_cime_root +from CIME.utils import convert_to_seconds, expect, get_cime_root, parse_test_name from CIME.XML.machines import Machines -import sys, os # Expect that, if a model wants to use python-based test lists, they will have a file # $model/cime_config/tests.py , containing a test dictionary called _TESTS. Currently, @@ -143,6 +145,7 @@ _ALL_TESTS.update(_CIME_TESTS) + ############################################################################### def _get_key_data(raw_dict, key, the_type): ############################################################################### diff --git a/CIME/get_timing.py b/CIME/get_timing.py index 92ac32509de..ab1127e4518 100644 --- a/CIME/get_timing.py +++ b/CIME/get_timing.py @@ -5,11 +5,13 @@ information from a run. """ -from CIME.XML.standard_module_setup import * -from CIME.utils import safe_copy -from CIME.status import append_case_status +import datetime +import logging +import os +import re -import datetime, re +from CIME.status import append_case_status +from CIME.utils import expect, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/gitinterface.py b/CIME/gitinterface.py index 790f1fc70f2..5c5b4d2423a 100644 --- a/CIME/gitinterface.py +++ b/CIME/gitinterface.py @@ -1,7 +1,10 @@ -import sys, shutil, re -from CIME.utils import run_cmd_no_fail +import re +import shutil +import sys from pathlib import Path +from CIME.utils import run_cmd_no_fail + class GitInterface: def __init__(self, repo_path, logger, branch=None): diff --git a/CIME/hist_utils.py b/CIME/hist_utils.py index fa48f942c56..699c7aab969 100644 --- a/CIME/hist_utils.py +++ b/CIME/hist_utils.py @@ -1,23 +1,25 @@ """ Functions for actions pertaining to history files. """ + +import filecmp import logging import os import re -import filecmp import shutil -from CIME.XML.standard_module_setup import * from CIME.config import Config +from CIME.core.exceptions import CIMEError from CIME.test_status import TEST_NO_BASELINES_COMMENT, TEST_STATUS_FILENAME from CIME.utils import ( + SharedArea, + expect, get_current_commit, get_timestamp, - safe_copy, - SharedArea, parse_test_name, + run_cmd, + safe_copy, ) -from CIME.utils import CIMEError logger = logging.getLogger(__name__) diff --git a/CIME/jenkins_generic_job.py b/CIME/jenkins_generic_job.py index e35a5d04e67..c140d08a9ea 100644 --- a/CIME/jenkins_generic_job.py +++ b/CIME/jenkins_generic_job.py @@ -1,8 +1,18 @@ +import glob +import logging +import os +import re +import shutil +import signal +import sys +import tarfile +import threading +import time + import CIME.wait_for_tests -from CIME.utils import expect, run_cmd_no_fail from CIME.case import Case +from CIME.utils import expect, run_cmd_no_fail -import os, shutil, glob, signal, logging, threading, sys, re, tarfile, time ############################################################################## def cleanup_queue(test_root, test_id): diff --git a/CIME/locked_files.py b/CIME/locked_files.py index 3be6f9c3573..91c2206543d 100644 --- a/CIME/locked_files.py +++ b/CIME/locked_files.py @@ -1,11 +1,12 @@ +import logging +import os from pathlib import Path -from CIME.utils import safe_copy -from CIME.XML.standard_module_setup import * +from CIME.utils import expect, safe_copy +from CIME.XML.env_batch import EnvBatch from CIME.XML.env_build import EnvBuild -from CIME.XML.env_mach_pes import EnvMachPes from CIME.XML.env_case import EnvCase -from CIME.XML.env_batch import EnvBatch +from CIME.XML.env_mach_pes import EnvMachPes from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/namelist.py b/CIME/namelist.py index 486955bca69..e2a7cbdc93c 100644 --- a/CIME/namelist.py +++ b/CIME/namelist.py @@ -102,14 +102,15 @@ # (rather specific) pylint naming conventions. # pylint: disable=line-too-long,too-many-lines,invalid-name +import logging import re from contextlib import contextmanager +from CIME.utils import expect, string_in_list + # Disable these because this is our standard setup # pylint: disable=wildcard-import,unused-wildcard-import -from CIME.XML.standard_module_setup import * -from CIME.utils import expect, string_in_list logger = logging.getLogger(__name__) @@ -250,7 +251,7 @@ def get_fortran_variable_indices(varname, varlen=1, allow_any_len=False): (1, -1, 1) """ m = FORTRAN_NAME_REGEX.search(varname) - (minindex, maxindex, step) = (1, varlen, 1) + minindex, maxindex, step = (1, varlen, 1) if m.group(4) is not None: minindex = int(m.group(4)) @@ -893,7 +894,6 @@ def parse(in_file=None, text=None, groupless=False, convert_tab_to_space=True): class Namelist(object): - """Class representing a Fortran namelist. Public methods: @@ -1184,7 +1184,6 @@ def get_group_variables(self, group_name): def write( self, out_file, groups=None, append=False, format_="nml", sorted_groups=True ): - """Write a the output data (normally fortran namelist) to the out_file As with `parse`, the `out_file` argument can be either a file name, or a @@ -1359,7 +1358,6 @@ def _write_nuopc(self, out_file, groups, sorted_groups): class _NamelistEOF(Exception): - """Exception thrown for an unexpected end-of-file in a namelist. This is an internal helper class, and should never be raised in a context @@ -1381,7 +1379,6 @@ def __str__(self): class _NamelistParseError(Exception): - """Exception thrown when namelist input has a syntax error. This is an internal helper class, and should never be raised in a context @@ -1403,7 +1400,6 @@ def __str__(self): class _NamelistParser(object): # pylint:disable=too-few-public-methods - """Class to validate and read from Fortran namelist input. This is intended to be an internal helper class and should not be used @@ -2222,7 +2218,7 @@ def _parse_name_and_values(self, allow_eof_end=False): break # and if it really is a literal, add it. values.append(literal) - (minindex, maxindex, step) = get_fortran_variable_indices( + minindex, maxindex, step = get_fortran_variable_indices( name, allow_any_len=True ) if (minindex > 1 or maxindex > minindex or step > 1) and maxindex > 0: diff --git a/CIME/nmlgen.py b/CIME/nmlgen.py index abfb5650c07..003ddc024e0 100644 --- a/CIME/nmlgen.py +++ b/CIME/nmlgen.py @@ -7,23 +7,24 @@ # pylint: disable=wildcard-import,unused-wildcard-import import datetime -import re import hashlib +import logging +import os +import re -from CIME.XML.standard_module_setup import * from CIME.namelist import ( Namelist, - parse, character_literal_to_string, - string_to_character_literal, - expand_literal_list, compress_literal_list, + expand_literal_list, merge_literal_lists, + parse, + string_to_character_literal, ) -from CIME.XML.namelist_definition import NamelistDefinition from CIME.utils import expect, safe_copy -from CIME.XML.stream import Stream from CIME.XML.grids import GRID_SEP +from CIME.XML.namelist_definition import NamelistDefinition +from CIME.XML.stream import Stream logger = logging.getLogger(__name__) @@ -66,7 +67,6 @@ class NamelistGenerator(object): - """Utility class for generating namelists for a given component.""" _streams_variables = [] diff --git a/CIME/provenance.py b/CIME/provenance.py index 849bb2757d1..23789033f0e 100644 --- a/CIME/provenance.py +++ b/CIME/provenance.py @@ -4,16 +4,18 @@ Library for saving build/run provenance. """ -from CIME.XML.standard_module_setup import * +import logging +import os +import sys + from CIME.utils import ( SharedArea, convert_to_babylonian_time, + expect, get_current_commit, run_cmd, ) -import sys - logger = logging.getLogger(__name__) diff --git a/CIME/scripts/configure b/CIME/scripts/configure index 73a326da2bf..5bb1d0685e4 100755 --- a/CIME/scripts/configure +++ b/CIME/scripts/configure @@ -24,9 +24,9 @@ real_file_dir = os.path.dirname(os.path.realpath(__file__)) cimeroot = os.path.abspath(os.path.join(real_file_dir, "..", "..")) sys.path.insert(0, cimeroot) +from CIME.BuildTools.configure import configure from CIME.Tools.standard_script_setup import * from CIME.utils import expect, get_model -from CIME.BuildTools.configure import configure from CIME.XML.machines import Machines logger = logging.getLogger(__name__) diff --git a/CIME/scripts/create_clone.py b/CIME/scripts/create_clone.py index 536ae73d232..23ba07b4402 100755 --- a/CIME/scripts/create_clone.py +++ b/CIME/scripts/create_clone.py @@ -1,13 +1,15 @@ #!/usr/bin/env python3 +import re +from argparse import RawTextHelpFormatter + +from CIME.case import Case from CIME.Tools.standard_script_setup import * from CIME.utils import expect -from CIME.case import Case -from argparse import RawTextHelpFormatter -import re logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args): ############################################################################### @@ -144,9 +146,11 @@ def _main_func(): if user_mods_dirs is not None: user_mods_dirs = [ - os.path.abspath(one_user_mods_dir) - if os.path.isdir(one_user_mods_dir) - else one_user_mods_dir + ( + os.path.abspath(one_user_mods_dir) + if os.path.isdir(one_user_mods_dir) + else one_user_mods_dir + ) for one_user_mods_dir in user_mods_dirs ] nint = len(startval) diff --git a/CIME/scripts/create_newcase.py b/CIME/scripts/create_newcase.py index 879a5d167c8..1b219137da6 100755 --- a/CIME/scripts/create_newcase.py +++ b/CIME/scripts/create_newcase.py @@ -6,6 +6,10 @@ Script to create a new CIME Case Control System (CSS) experimental case. """ +from argparse import RawTextHelpFormatter + +from CIME.case import Case +from CIME.config import Config from CIME.Tools.standard_script_setup import * from CIME.utils import ( expect, @@ -13,12 +17,10 @@ get_cime_default_driver, get_src_root, ) -from CIME.config import Config -from CIME.case import Case -from argparse import RawTextHelpFormatter logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args, cimeroot, description): ############################################################################### @@ -384,9 +386,11 @@ def _main_func(description=None): if user_mods_dirs is not None: user_mods_dirs = [ - os.path.abspath(one_user_mods_dir) - if os.path.isdir(one_user_mods_dir) - else one_user_mods_dir + ( + os.path.abspath(one_user_mods_dir) + if os.path.isdir(one_user_mods_dir) + else one_user_mods_dir + ) for one_user_mods_dir in user_mods_dirs ] diff --git a/CIME/scripts/create_test.py b/CIME/scripts/create_test.py index 051c7430592..f2abdf1b19f 100755 --- a/CIME/scripts/create_test.py +++ b/CIME/scripts/create_test.py @@ -26,28 +26,31 @@ If this tool is missing any feature that you need, please add an issue on https://github.com/ESMCI/cime """ + +import argparse +import glob +import math +from argparse import RawTextHelpFormatter + +from CIME import get_tests, utils +from CIME.case import Case +from CIME.config import Config +from CIME.test_scheduler import RUN_PHASE, TestScheduler +from CIME.test_utils import get_tests_from_xml from CIME.Tools.standard_script_setup import * -from CIME import get_tests -from CIME.test_scheduler import TestScheduler, RUN_PHASE -from CIME import utils from CIME.utils import ( - expect, - convert_to_seconds, compute_total_time, convert_to_babylonian_time, - run_cmd_no_fail, + convert_to_seconds, + expect, get_cime_config, + run_cmd_no_fail, ) -from CIME.config import Config from CIME.XML.machines import Machines -from CIME.case import Case -from CIME.test_utils import get_tests_from_xml -from argparse import RawTextHelpFormatter - -import argparse, math, glob logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/scripts/query_config.py b/CIME/scripts/query_config.py index 41b8d804578..79f201ef41d 100755 --- a/CIME/scripts/query_config.py +++ b/CIME/scripts/query_config.py @@ -6,19 +6,18 @@ information will be listed for each. """ +import argparse +import logging import os import sys -import logging -import argparse -from CIME.Tools.standard_script_setup import * from CIME import utils -from CIME.XML.files import Files +from CIME.config import Config from CIME.XML.component import Component from CIME.XML.compsets import Compsets +from CIME.XML.files import Files from CIME.XML.grids import Grids from CIME.XML.machines import Machines -from CIME.config import Config logger = logging.getLogger(__name__) diff --git a/CIME/scripts/query_testlists.py b/CIME/scripts/query_testlists.py index ad00733cd61..02e9386543a 100755 --- a/CIME/scripts/query_testlists.py +++ b/CIME/scripts/query_testlists.py @@ -14,13 +14,14 @@ All of the above support the various --xml-* arguments for subsetting which tests are included. """ -from CIME.Tools.standard_script_setup import * from CIME.test_utils import get_tests_from_xml, test_to_string -from CIME.XML.tests import Tests +from CIME.Tools.standard_script_setup import * from CIME.utils import expect +from CIME.XML.tests import Tests logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/simple_compare.py b/CIME/simple_compare.py index f1292778157..f3c59cf6109 100644 --- a/CIME/simple_compare.py +++ b/CIME/simple_compare.py @@ -1,7 +1,9 @@ -import os, re +import os +import re from CIME.utils import expect + ############################################################################### def _normalize_string_value(value, case): ############################################################################### diff --git a/CIME/status.py b/CIME/status.py index 08618c7219d..e7093e4b2f7 100644 --- a/CIME/status.py +++ b/CIME/status.py @@ -1,6 +1,10 @@ # These routines were moved from utils.py to avoid circular dependancies -import time, os, sys, logging -from CIME.utils import Timeout, CASE_SUCCESS, CASE_FAILURE +import logging +import os +import sys +import time + +from CIME.utils import CASE_FAILURE, CASE_SUCCESS, Timeout logger = logging.getLogger(__name__) diff --git a/CIME/test_scheduler.py b/CIME/test_scheduler.py index d94a99e08fa..7b1152e1713 100644 --- a/CIME/test_scheduler.py +++ b/CIME/test_scheduler.py @@ -8,45 +8,67 @@ they can be run outside the context of TestScheduler. """ +import glob +import logging import os -import traceback, stat, threading, time, glob +import re +import stat +import sys +import threading +import time +import traceback from collections import OrderedDict -from CIME.XML.standard_module_setup import * -from CIME.get_tests import get_recommended_test_time, get_build_groups, is_perf_test +from CIME.build import post_build +from CIME.case import Case +from CIME.config import Config +from CIME.cs_status_creator import create_cs_status +from CIME.get_tests import get_build_groups, get_recommended_test_time, is_perf_test +from CIME.hist_utils import generate_teststatus +from CIME.locked_files import lock_file +from CIME.provenance import get_recommended_test_time_based_on_past from CIME.status import append_status, append_testlog +from CIME.SystemTests.test_mods import find_test_mods +from CIME.test_status import ( + CORE_PHASES, + CREATE_NEWCASE_PHASE, + MODEL_BUILD_PHASE, + RUN_PHASE, + SETUP_PHASE, + SHAREDLIB_BUILD_PHASE, + SUBMIT_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TEST_PEND_STATUS, + TEST_RERUN_COMMENT, + TestStatus, + XML_PHASE, +) from CIME.utils import ( TESTS_FAILED_ERR_CODE, - parse_test_name, - get_full_test_name, - get_model, + CIMEError, convert_to_seconds, + expect, + get_cime_default_driver, get_cime_root, + get_full_test_name, + get_model, + get_project, get_src_root, - get_tools_path, get_template_path, - get_project, get_timestamp, - get_cime_default_driver, - CIMEError, + get_tools_path, + parse_test_name, + run_cmd, ) -from CIME.config import Config -from CIME.test_status import * -from CIME.XML.machines import Machines -from CIME.XML.generic_xml import GenericXML -from CIME.XML.env_test import EnvTest +from CIME.wait_for_tests import wait_for_tests +from CIME.XML.component import Component from CIME.XML.env_mach_pes import EnvMachPes +from CIME.XML.env_test import EnvTest from CIME.XML.files import Files -from CIME.XML.component import Component +from CIME.XML.generic_xml import GenericXML +from CIME.XML.machines import Machines from CIME.XML.tests import Tests -from CIME.case import Case -from CIME.wait_for_tests import wait_for_tests -from CIME.provenance import get_recommended_test_time_based_on_past -from CIME.locked_files import lock_file -from CIME.cs_status_creator import create_cs_status -from CIME.hist_utils import generate_teststatus -from CIME.build import post_build -from CIME.SystemTests.test_mods import find_test_mods logger = logging.getLogger(__name__) diff --git a/CIME/test_status.py b/CIME/test_status.py index 24ce5bc6e74..d3f4580403f 100644 --- a/CIME/test_status.py +++ b/CIME/test_status.py @@ -26,9 +26,12 @@ """ -from CIME.XML.standard_module_setup import * -import os, itertools +import itertools +import logging +import os + from CIME import expected_fails +from CIME.utils import expect TEST_STATUS_FILENAME = "TestStatus" diff --git a/CIME/test_utils.py b/CIME/test_utils.py index 6dea4e3bddc..c3a3c964da0 100644 --- a/CIME/test_utils.py +++ b/CIME/test_utils.py @@ -2,12 +2,16 @@ Utility functions used in test_scheduler.py, and by other utilities that need to get test lists. """ + import glob -from CIME.XML.standard_module_setup import * -from CIME.XML.testlist import Testlist -from CIME.XML.files import Files -from CIME.test_status import TEST_STATUS_FILENAME +import logging +import os + import CIME.utils +from CIME.test_status import TEST_STATUS_FILENAME +from CIME.utils import expect +from CIME.XML.files import Files +from CIME.XML.testlist import Testlist logger = logging.getLogger(__name__) diff --git a/CIME/tests/base.py b/CIME/tests/base.py index 8873924b60a..b2c4fea8bd9 100644 --- a/CIME/tests/base.py +++ b/CIME/tests/base.py @@ -2,17 +2,16 @@ import glob import os -import tempfile -import time -import signal import shutil +import signal import stat import sys +import tempfile +import time import unittest from CIME import utils from CIME.config import Config -from CIME.XML.machines import Machines def typed_os_environ(key, default_value, expected_type=None): diff --git a/CIME/tests/custom_assertions_test_status.py b/CIME/tests/custom_assertions_test_status.py index 16dc44e5cea..365b0f9d34b 100644 --- a/CIME/tests/custom_assertions_test_status.py +++ b/CIME/tests/custom_assertions_test_status.py @@ -3,11 +3,11 @@ can be used when testing TestStatus. """ -from CIME.XML.standard_module_setup import * - -import unittest import re +import unittest + from CIME import test_status +from CIME.utils import expect class CustomAssertionsTestStatus(unittest.TestCase): diff --git a/CIME/tests/scripts_regression_tests.py b/CIME/tests/scripts_regression_tests.py index 66f4c015298..7590e53e3cf 100755 --- a/CIME/tests/scripts_regression_tests.py +++ b/CIME/tests/scripts_regression_tests.py @@ -5,46 +5,31 @@ to confirm overall CIME correctness. """ -import glob, os, re, shutil, signal, sys, tempfile, threading, time, logging, unittest, getpass, filecmp, time, atexit, functools +import atexit +import functools +import logging +import os +import sys +import unittest CIMEROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) sys.path.insert(0, CIMEROOT) -from xml.etree.ElementTree import ParseError - -import subprocess, argparse +import argparse +import subprocess subprocess.call('/bin/rm -f $(find . -name "*.pyc")', shell=True, cwd=CIMEROOT) -import stat as osstat - -import collections +import CIME.test_scheduler +import CIME.wait_for_tests +from CIME import utils +from CIME.config import Config +from CIME.tests.base import BaseTestCase from CIME.utils import ( - run_cmd, - run_cmd_no_fail, - get_lids, get_current_commit, - safe_copy, - CIMEError, - get_cime_root, - get_src_root, - Timeout, - import_from_file, get_model, ) -import CIME.test_scheduler, CIME.wait_for_tests -from CIME import get_tests -from CIME.test_scheduler import TestScheduler -from CIME.XML.env_run import EnvRun from CIME.XML.machines import Machines -from CIME.XML.files import Files -from CIME.case import Case -from CIME.code_checker import check_code, get_all_checkable_files -from CIME.test_status import * -from CIME.provenance import get_test_success, save_test_success -from CIME import utils -from CIME.tests.base import BaseTestCase -from CIME.config import Config os.environ["CIME_GLOBAL_WALLTIME"] = "0:05:00" diff --git a/CIME/tests/test_sys_bless_tests_results.py b/CIME/tests/test_sys_bless_tests_results.py index 2f2beeb461d..9d8b061b466 100644 --- a/CIME/tests/test_sys_bless_tests_results.py +++ b/CIME/tests/test_sys_bless_tests_results.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import glob -import re import os +import re import stat from CIME import utils diff --git a/CIME/tests/test_sys_cime_case.py b/CIME/tests/test_sys_cime_case.py index c4095972d93..3b142118e53 100644 --- a/CIME/tests/test_sys_cime_case.py +++ b/CIME/tests/test_sys_cime_case.py @@ -8,10 +8,10 @@ import time from CIME import utils -from CIME.tests import base from CIME.case.case import Case -from CIME.XML.env_run import EnvRun +from CIME.tests import base from CIME.utils import find_system_test +from CIME.XML.env_run import EnvRun try: collectionsAbc = collections.abc diff --git a/CIME/tests/test_sys_create_newcase.py b/CIME/tests/test_sys_create_newcase.py index 1be636aff36..fce9743de09 100644 --- a/CIME/tests/test_sys_create_newcase.py +++ b/CIME/tests/test_sys_create_newcase.py @@ -7,9 +7,9 @@ import sys from CIME import utils -from CIME.tests import base -from CIME.case.case import Case from CIME.build import CmakeTmpBuildDir +from CIME.case.case import Case +from CIME.tests import base class TestCreateNewcase(base.BaseTestCase): diff --git a/CIME/tests/test_sys_full_system.py b/CIME/tests/test_sys_full_system.py index ad4485d2b81..d6e334839a7 100644 --- a/CIME/tests/test_sys_full_system.py +++ b/CIME/tests/test_sys_full_system.py @@ -2,10 +2,7 @@ import os -from CIME import get_tests -from CIME import test_status -from CIME import utils -from CIME import wait_for_tests +from CIME import get_tests, test_status, utils, wait_for_tests from CIME.tests import base diff --git a/CIME/tests/test_sys_jenkins_generic_job.py b/CIME/tests/test_sys_jenkins_generic_job.py index 30b31c5c8d6..19572717418 100644 --- a/CIME/tests/test_sys_jenkins_generic_job.py +++ b/CIME/tests/test_sys_jenkins_generic_job.py @@ -7,8 +7,7 @@ import threading import time -from CIME import get_tests -from CIME import utils +from CIME import get_tests, utils from CIME.tests import base diff --git a/CIME/tests/test_sys_manage_and_query.py b/CIME/tests/test_sys_manage_and_query.py index 31ae3392bdb..f99b47aefb9 100644 --- a/CIME/tests/test_sys_manage_and_query.py +++ b/CIME/tests/test_sys_manage_and_query.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -from CIME import utils from CIME.tests import base from CIME.XML.files import Files diff --git a/CIME/tests/test_sys_save_timings.py b/CIME/tests/test_sys_save_timings.py index ee4a964c4b3..c5579489330 100644 --- a/CIME/tests/test_sys_save_timings.py +++ b/CIME/tests/test_sys_save_timings.py @@ -4,10 +4,9 @@ import glob import os -from CIME import provenance -from CIME import utils -from CIME.tests import base +from CIME import provenance, utils from CIME.case.case import Case +from CIME.tests import base class TestSaveTimings(base.BaseTestCase): diff --git a/CIME/tests/test_sys_single_submit.py b/CIME/tests/test_sys_single_submit.py index fed850263df..beeb80207b8 100644 --- a/CIME/tests/test_sys_single_submit.py +++ b/CIME/tests/test_sys_single_submit.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -from CIME import utils from CIME.tests import base diff --git a/CIME/tests/test_sys_test_scheduler.py b/CIME/tests/test_sys_test_scheduler.py index 1b72ed78d0b..df42ad14180 100755 --- a/CIME/tests/test_sys_test_scheduler.py +++ b/CIME/tests/test_sys_test_scheduler.py @@ -1,16 +1,13 @@ #!/usr/bin/env python3 -import re import glob import logging import os +import re import unittest from unittest import mock -from CIME import get_tests -from CIME import utils -from CIME import test_status -from CIME import test_scheduler +from CIME import get_tests, test_scheduler, test_status, utils from CIME.tests import base diff --git a/CIME/tests/test_sys_unittest.py b/CIME/tests/test_sys_unittest.py index 8bfd21eb16f..c22ff7d750c 100755 --- a/CIME/tests/test_sys_unittest.py +++ b/CIME/tests/test_sys_unittest.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 import os +import re import shutil import sys -import re from CIME import utils from CIME.tests import base diff --git a/CIME/tests/test_sys_wait_for_tests.py b/CIME/tests/test_sys_wait_for_tests.py index 3484c9af7a5..f0afd6d712f 100644 --- a/CIME/tests/test_sys_wait_for_tests.py +++ b/CIME/tests/test_sys_wait_for_tests.py @@ -1,17 +1,16 @@ #!/usr/bin/env python3 import os -import signal import shutil +import signal import sys -import time import threading +import time -from CIME import utils -from CIME import test_status -from CIME.wait_for_tests import ENV_VAR_KEEP_CDASH +from CIME import test_status, utils from CIME.tests import base from CIME.tests import utils as test_utils +from CIME.wait_for_tests import ENV_VAR_KEEP_CDASH class TestWaitForTests(base.BaseTestCase): @@ -338,9 +337,11 @@ def live_test_impl(self, testdir, expected_results, last_phase, last_status): ) as ts: ts.set_status( core_phase, - last_status - if core_phase == last_phase - else test_status.TEST_PASS_STATUS, + ( + last_status + if core_phase == last_phase + else test_status.TEST_PASS_STATUS + ), ) time.sleep(5) diff --git a/CIME/tests/test_unit_baselines_performance.py b/CIME/tests/test_unit_baselines_performance.py index 0ef5b387fa4..8f8e2e574d0 100644 --- a/CIME/tests/test_unit_baselines_performance.py +++ b/CIME/tests/test_unit_baselines_performance.py @@ -3,8 +3,8 @@ import gzip import tempfile import unittest -from unittest import mock from pathlib import Path +from unittest import mock from CIME.baselines import performance from CIME.tests.test_unit_system_tests import CPLLOG @@ -300,7 +300,7 @@ def test_perf_compare_throughput_baseline_no_baseline( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_throughput_baseline( + below_tolerance, comment = performance.perf_compare_throughput_baseline( case ) @@ -332,7 +332,7 @@ def test_perf_compare_throughput_baseline_no_tolerance( None, ) - (below_tolerance, comment) = performance.perf_compare_throughput_baseline( + below_tolerance, comment = performance.perf_compare_throughput_baseline( case ) @@ -364,7 +364,7 @@ def test_perf_compare_throughput_baseline_above_threshold( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_throughput_baseline( + below_tolerance, comment = performance.perf_compare_throughput_baseline( case ) @@ -396,7 +396,7 @@ def test_perf_compare_throughput_baseline( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_throughput_baseline( + below_tolerance, comment = performance.perf_compare_throughput_baseline( case ) @@ -433,7 +433,7 @@ def test_perf_compare_memory_baseline_no_baseline( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) + below_tolerance, comment = performance.perf_compare_memory_baseline(case) assert below_tolerance assert ( @@ -464,7 +464,7 @@ def test_perf_compare_memory_baseline_not_enough_samples( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) + below_tolerance, comment = performance.perf_compare_memory_baseline(case) assert below_tolerance is None assert comment == "Found 2 memory usage samples, need atleast 4" @@ -524,7 +524,7 @@ def test_perf_compare_memory_baseline_no_tolerance( None, ) - (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) + below_tolerance, comment = performance.perf_compare_memory_baseline(case) assert below_tolerance assert ( @@ -559,7 +559,7 @@ def test_perf_compare_memory_baseline_above_threshold( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) + below_tolerance, comment = performance.perf_compare_memory_baseline(case) assert not below_tolerance assert ( @@ -594,7 +594,7 @@ def test_perf_compare_memory_baseline( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) + below_tolerance, comment = performance.perf_compare_memory_baseline(case) assert below_tolerance assert ( diff --git a/CIME/tests/test_unit_bless_test_results.py b/CIME/tests/test_unit_bless_test_results.py index 3138dd75c77..63af7ecc140 100644 --- a/CIME/tests/test_unit_bless_test_results.py +++ b/CIME/tests/test_unit_bless_test_results.py @@ -1,21 +1,20 @@ import re -import unittest import tempfile +import unittest from contextlib import ExitStack -from unittest import mock from pathlib import Path +from unittest import mock from CIME.bless_test_results import ( - bless_test_results, - _bless_throughput, _bless_memory, + _bless_throughput, bless_history, bless_namelists, + bless_test_results, is_hist_bless_needed, ) from CIME.test_status import ALL_PHASES, GENERATE_PHASE from CIME.tests import utils as test_utils -from CIME.utils import CIMEError class TestUnitBlessTestResults(unittest.TestCase): diff --git a/CIME/tests/test_unit_build.py b/CIME/tests/test_unit_build.py index cb9ea6d3f81..0afce20bb0b 100644 --- a/CIME/tests/test_unit_build.py +++ b/CIME/tests/test_unit_build.py @@ -3,7 +3,6 @@ import os import unittest from unittest import mock -from pathlib import Path from CIME import build from CIME.tests.utils import mock_case diff --git a/CIME/tests/test_unit_case.py b/CIME/tests/test_unit_case.py index 75fcbeed490..5cb73e198d4 100755 --- a/CIME/tests/test_unit_case.py +++ b/CIME/tests/test_unit_case.py @@ -1,13 +1,12 @@ #!/usr/bin/env python3 import os +import tempfile import unittest from unittest import mock -import tempfile -from CIME.case import case_submit -from CIME.case import Case from CIME import utils +from CIME.case import Case, case_submit from CIME.tests.utils import mock_case diff --git a/CIME/tests/test_unit_case_fake.py b/CIME/tests/test_unit_case_fake.py index 448931ecc7c..adc3252aa85 100755 --- a/CIME/tests/test_unit_case_fake.py +++ b/CIME/tests/test_unit_case_fake.py @@ -4,10 +4,10 @@ This module contains unit tests of CaseFake """ -import unittest -import tempfile import os import shutil +import tempfile +import unittest from CIME.tests.case_fake import CaseFake diff --git a/CIME/tests/test_unit_case_run.py b/CIME/tests/test_unit_case_run.py index 88c746d0d8c..3b869d6e49d 100644 --- a/CIME/tests/test_unit_case_run.py +++ b/CIME/tests/test_unit_case_run.py @@ -1,9 +1,8 @@ import unittest from unittest import mock -from CIME.utils import CIMEError -from CIME.case.case_run import TERMINATION_TEXT -from CIME.case.case_run import _post_run_check +from CIME.case.case_run import TERMINATION_TEXT, _post_run_check +from CIME.core.exceptions import CIMEError def _case_post_run_check(): diff --git a/CIME/tests/test_unit_case_setup.py b/CIME/tests/test_unit_case_setup.py index a00dcf1b413..80ad93d8b46 100644 --- a/CIME/tests/test_unit_case_setup.py +++ b/CIME/tests/test_unit_case_setup.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 +import contextlib import os -import unittest import tempfile -import contextlib +import unittest from pathlib import Path from unittest import mock diff --git a/CIME/tests/test_unit_case_st_archive.py b/CIME/tests/test_unit_case_st_archive.py index bf27ab1c201..5b72f8f8765 100644 --- a/CIME/tests/test_unit_case_st_archive.py +++ b/CIME/tests/test_unit_case_st_archive.py @@ -7,8 +7,8 @@ from CIME import date from CIME.case import case_st_archive -from CIME.XML import env_archive as _env_archive from CIME.tests import utils +from CIME.XML import env_archive as _env_archive UNSET = r""" diff --git a/CIME/tests/test_unit_compare_test_results.py b/CIME/tests/test_unit_compare_test_results.py index 4844a96c1a6..fe984390072 100755 --- a/CIME/tests/test_unit_compare_test_results.py +++ b/CIME/tests/test_unit_compare_test_results.py @@ -4,14 +4,18 @@ This module contains unit tests for compare_test_results """ -import unittest -import tempfile import os import shutil +import tempfile +import unittest -from CIME import utils -from CIME import compare_test_results -from CIME.test_status import * +from CIME import compare_test_results, utils +from CIME.test_status import ( + CREATE_NEWCASE_PHASE, + RUN_PHASE, + SETUP_PHASE, + TestStatus, +) from CIME.tests.case_fake import CaseFake diff --git a/CIME/tests/test_unit_compare_two.py b/CIME/tests/test_unit_compare_two.py index e5f96001bc0..4cece00b660 100755 --- a/CIME/tests/test_unit_compare_two.py +++ b/CIME/tests/test_unit_compare_two.py @@ -9,16 +9,16 @@ # # pylint:disable=protected-access -import unittest -from collections import namedtuple import functools import os import shutil import tempfile +import unittest +from collections import namedtuple from unittest import mock -from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo import CIME.test_status as test_status +from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.tests.case_fake import CaseFake # ======================================================================== diff --git a/CIME/tests/test_unit_config.py b/CIME/tests/test_unit_config.py index 93da359b046..f928b8a7479 100644 --- a/CIME/tests/test_unit_config.py +++ b/CIME/tests/test_unit_config.py @@ -1,8 +1,8 @@ import os -import unittest import tempfile -from unittest import mock +import unittest from pathlib import Path +from unittest import mock from CIME.config import Config diff --git a/CIME/tests/test_unit_configure.py b/CIME/tests/test_unit_configure.py index 693abccdfbd..7dc8971e2a3 100644 --- a/CIME/tests/test_unit_configure.py +++ b/CIME/tests/test_unit_configure.py @@ -1,11 +1,10 @@ import os -import unittest -import tempfile import shutil - -from CIME.utils import expect +import tempfile +import unittest from CIME.BuildTools.configure import generate_env_mach_specific +from CIME.utils import expect from CIME.XML.machines import Machines diff --git a/CIME/tests/test_unit_core_bootstrap.py b/CIME/tests/test_unit_core_bootstrap.py new file mode 100644 index 00000000000..fdbc2d3b1ce --- /dev/null +++ b/CIME/tests/test_unit_core_bootstrap.py @@ -0,0 +1,137 @@ +"""Unit tests for CIME.core.config.bootstrap.""" + +import os +import sys + +import pytest + +from CIME.core.config.bootstrap import ( + _ensure_paths, + _is_cimeroot, + bootstrap_cime, + check_minimum_python_version, + find_cimeroot, + get_tools_path, +) + + +class TestFindCimeroot: + def test_explicit_dir(self, tmp_path): + """find_cimeroot accepts an explicit directory with a CIME/ subdir.""" + (tmp_path / "CIME").mkdir() + root = find_cimeroot(str(tmp_path)) + assert root == str(tmp_path.resolve()) + + def test_explicit_dir_invalid(self, tmp_path): + with pytest.raises(RuntimeError, match="not a valid CIMEROOT"): + find_cimeroot(str(tmp_path)) + + def test_env_var(self, tmp_path, monkeypatch): + (tmp_path / "CIME").mkdir() + monkeypatch.setenv("CIMEROOT", str(tmp_path)) + root = find_cimeroot() + assert root == str(tmp_path.resolve()) + + def test_walks_up_from_file(self): + """The default detection should find the real CIMEROOT from this repo.""" + root = find_cimeroot() + assert os.path.isdir(os.path.join(root, "CIME")) + + +class TestIsCimeroot: + def test_valid(self, tmp_path): + (tmp_path / "CIME").mkdir() + assert _is_cimeroot(str(tmp_path)) is True + + def test_invalid(self, tmp_path): + assert _is_cimeroot(str(tmp_path)) is False + + +class TestBootstrapCime: + def setup_method(self): + self._orig_path = sys.path[:] + self._orig_env = os.environ.get("CIMEROOT") + + def teardown_method(self): + sys.path[:] = self._orig_path + if self._orig_env is not None: + os.environ["CIMEROOT"] = self._orig_env + else: + os.environ.pop("CIMEROOT", None) + + def test_bootstrap_sets_sys_path(self, tmp_path): + (tmp_path / "CIME" / "Tools").mkdir(parents=True) + root = bootstrap_cime(cimeroot=str(tmp_path)) + assert root == str(tmp_path.resolve()) + assert str(tmp_path.resolve()) in sys.path + tools = os.path.join(str(tmp_path.resolve()), "CIME", "Tools") + assert tools in sys.path + + def test_bootstrap_sets_env(self, tmp_path): + (tmp_path / "CIME" / "Tools").mkdir(parents=True) + bootstrap_cime(cimeroot=str(tmp_path)) + assert os.environ["CIMEROOT"] == str(tmp_path.resolve()) + + def test_bootstrap_no_env(self, tmp_path): + (tmp_path / "CIME" / "Tools").mkdir(parents=True) + os.environ.pop("CIMEROOT", None) + bootstrap_cime(cimeroot=str(tmp_path), set_env=False) + assert "CIMEROOT" not in os.environ + + def test_extra_paths(self, tmp_path): + (tmp_path / "CIME" / "Tools").mkdir(parents=True) + extra = tmp_path / "extras" + extra.mkdir() + bootstrap_cime(cimeroot=str(tmp_path), extra_paths=[str(extra)]) + assert str(extra.resolve()) in sys.path + + def test_no_duplicate_paths(self, tmp_path): + (tmp_path / "CIME" / "Tools").mkdir(parents=True) + bootstrap_cime(cimeroot=str(tmp_path)) + bootstrap_cime(cimeroot=str(tmp_path)) + root_str = str(tmp_path.resolve()) + assert sys.path.count(root_str) == 1 + + +class TestEnsurePaths: + def setup_method(self): + self._orig_path = sys.path[:] + + def teardown_method(self): + sys.path[:] = self._orig_path + + def test_inserts_in_order(self, tmp_path): + a = str(tmp_path / "a") + b = str(tmp_path / "b") + _ensure_paths([a, b]) + assert sys.path[0] == a + assert sys.path[1] == b + + def test_moves_existing_entry(self, tmp_path): + p = str(tmp_path / "x") + sys.path.append(p) + _ensure_paths([p]) + assert sys.path[0] == p + assert sys.path.count(p) == 1 + + +class TestGetToolsPath: + def test_returns_tools_dir(self, tmp_path): + (tmp_path / "CIME" / "Tools").mkdir(parents=True) + tools = get_tools_path(cimeroot=str(tmp_path)) + assert tools == os.path.join(str(tmp_path), "CIME", "Tools") + + +class TestCheckMinimumPythonVersion: + def test_passes_current_version(self): + # Current Python is >= 3.9 (required by CIME) + check_minimum_python_version(3, 9) + + def test_fails_future_version(self): + with pytest.raises(RuntimeError, match="required"): + check_minimum_python_version(99, 0) + + def test_warn_only(self, capsys): + check_minimum_python_version(99, 0, warn_only=True) + captured = capsys.readouterr() + assert "recommended" in captured.err diff --git a/CIME/tests/test_unit_core_exceptions.py b/CIME/tests/test_unit_core_exceptions.py new file mode 100644 index 00000000000..79c5907d812 --- /dev/null +++ b/CIME/tests/test_unit_core_exceptions.py @@ -0,0 +1,124 @@ +"""Unit tests for CIME.core.exceptions.""" + +import pytest + +from CIME.core.exceptions import ( + BuildError, + CIMEError, + ConfigurationError, + ExternalCommandError, + InputError, + LockingError, + RetryableExternalCommandError, + SubmitError, + SystemTestError, +) + + +class TestCIMEError: + """Tests for the base CIMEError exception.""" + + def test_inherits_system_exit(self): + assert issubclass(CIMEError, SystemExit) + + def test_inherits_exception(self): + assert issubclass(CIMEError, Exception) + + def test_catchable_as_exception(self): + with pytest.raises(Exception): + raise CIMEError("test") + + def test_catchable_as_system_exit(self): + with pytest.raises(SystemExit): + raise CIMEError("test") + + def test_message_preserved(self): + try: + raise CIMEError("my message") + except CIMEError as exc: + assert str(exc) == "my message" + + +class TestSubclasses: + """All typed exceptions inherit from CIMEError.""" + + @pytest.mark.parametrize( + "exc_class", + [ + ConfigurationError, + BuildError, + SubmitError, + SystemTestError, + LockingError, + ExternalCommandError, + RetryableExternalCommandError, + InputError, + ], + ) + def test_is_cime_error(self, exc_class): + assert issubclass(exc_class, CIMEError) + + @pytest.mark.parametrize( + "exc_class", + [ + ConfigurationError, + BuildError, + SubmitError, + SystemTestError, + LockingError, + InputError, + ], + ) + def test_simple_subclass_message(self, exc_class): + with pytest.raises(exc_class, match="oops"): + raise exc_class("oops") + + +class TestExternalCommandError: + """Tests for ExternalCommandError and its retryable variant.""" + + def test_stores_returncode_and_cmd(self): + exc = ExternalCommandError("failed", returncode=42, cmd="make all") + assert exc.returncode == 42 + assert exc.cmd == "make all" + assert str(exc) == "failed" + + def test_stores_output_and_stderr(self): + exc = ExternalCommandError( + "failed", + returncode=1, + cmd="make", + output="Building...", + stderr="error: missing file", + ) + assert exc.output == "Building..." + assert exc.stderr == "error: missing file" + + def test_defaults(self): + exc = ExternalCommandError("failed") + assert exc.returncode == 1 + assert exc.cmd == "" + assert exc.output == "" + assert exc.stderr == "" + + def test_command_property_deprecated(self): + """The command property is deprecated but still works.""" + exc = ExternalCommandError("failed", cmd="make") + assert exc.command == "make" # Deprecated property + assert exc.cmd == "make" # Preferred attribute + + def test_command_kwarg_deprecated(self): + """The command kwarg is deprecated but still works.""" + exc = ExternalCommandError("failed", command="make") + assert exc.cmd == "make" + assert exc.command == "make" + + def test_retryable_is_external_command_error(self): + assert issubclass(RetryableExternalCommandError, ExternalCommandError) + + def test_retryable_stores_fields(self): + exc = RetryableExternalCommandError( + "timeout", returncode=124, cmd="srun ./model" + ) + assert exc.returncode == 124 + assert exc.cmd == "srun ./model" diff --git a/CIME/tests/test_unit_cs_status.py b/CIME/tests/test_unit_cs_status.py index 70efd47935f..1e44ce996e0 100755 --- a/CIME/tests/test_unit_cs_status.py +++ b/CIME/tests/test_unit_cs_status.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 import io -import unittest -import shutil import os -import tempfile import re -from CIME.cs_status import cs_status +import shutil +import tempfile +import unittest + from CIME import test_status +from CIME.cs_status import cs_status from CIME.tests.custom_assertions_test_status import CustomAssertionsTestStatus diff --git a/CIME/tests/test_unit_custom_assertions_test_status.py b/CIME/tests/test_unit_custom_assertions_test_status.py index 99e6cb05d04..4715377e5c8 100755 --- a/CIME/tests/test_unit_custom_assertions_test_status.py +++ b/CIME/tests/test_unit_custom_assertions_test_status.py @@ -5,6 +5,7 @@ """ import unittest + from CIME import test_status from CIME.tests.custom_assertions_test_status import CustomAssertionsTestStatus diff --git a/CIME/tests/test_unit_doctest.py b/CIME/tests/test_unit_doctest.py index 571b9af41db..1b972c0a6c9 100644 --- a/CIME/tests/test_unit_doctest.py +++ b/CIME/tests/test_unit_doctest.py @@ -1,17 +1,7 @@ #!/usr/bin/env python3 -import glob -import re import os -import stat -import doctest -import sys -import pkgutil -import unittest -import functools -import CIME -from CIME import utils from CIME.tests import base diff --git a/CIME/tests/test_unit_expected_fails_file.py b/CIME/tests/test_unit_expected_fails_file.py index 1e0e5878e2b..d2898a5022e 100755 --- a/CIME/tests/test_unit_expected_fails_file.py +++ b/CIME/tests/test_unit_expected_fails_file.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -import unittest import os import shutil import tempfile -from CIME.XML.expected_fails_file import ExpectedFailsFile -from CIME.utils import CIMEError +import unittest + +from CIME.core.exceptions import CIMEError from CIME.expected_fails import ExpectedFails +from CIME.XML.expected_fails_file import ExpectedFailsFile class TestExpectedFailsFile(unittest.TestCase): diff --git a/CIME/tests/test_unit_fake_case.py b/CIME/tests/test_unit_fake_case.py index c4cebebcd2c..47e6fe88c5f 100755 --- a/CIME/tests/test_unit_fake_case.py +++ b/CIME/tests/test_unit_fake_case.py @@ -5,11 +5,12 @@ This is seperate from FakeCase that's under CIME/tests """ -import unittest import os -from CIME.utils import get_model, CIMEError, GLOBAL, get_src_root +import unittest from CIME.BuildTools.configure import FakeCase +from CIME.core.exceptions import CIMEError +from CIME.utils import get_model, get_src_root class TestFakeCase(unittest.TestCase): diff --git a/CIME/tests/test_unit_grids.py b/CIME/tests/test_unit_grids.py index 8417177e83e..2e2da3878b5 100755 --- a/CIME/tests/test_unit_grids.py +++ b/CIME/tests/test_unit_grids.py @@ -13,13 +13,14 @@ # # pylint:disable=line-too-long -import unittest import os import shutil import string import tempfile -from CIME.XML.grids import Grids, _ComponentGrids, _add_grid_info, _strip_grid_from_name -from CIME.utils import CIMEError +import unittest + +from CIME.core.exceptions import CIMEError +from CIME.XML.grids import Grids, _add_grid_info, _ComponentGrids, _strip_grid_from_name class TestGrids(unittest.TestCase): diff --git a/CIME/tests/test_unit_hist_utils.py b/CIME/tests/test_unit_hist_utils.py index fe6d4866c34..86fcc5a9c95 100644 --- a/CIME/tests/test_unit_hist_utils.py +++ b/CIME/tests/test_unit_hist_utils.py @@ -1,9 +1,7 @@ -import io import unittest from unittest import mock from CIME.hist_utils import copy_histfiles -from CIME.XML.archive import Archive class TestHistUtils(unittest.TestCase): diff --git a/CIME/tests/test_unit_locked_files.py b/CIME/tests/test_unit_locked_files.py index fc871e3b768..abd858433b2 100644 --- a/CIME/tests/test_unit_locked_files.py +++ b/CIME/tests/test_unit_locked_files.py @@ -1,13 +1,11 @@ import tempfile import unittest -from unittest import mock from pathlib import Path +from unittest import mock from CIME import locked_files -from CIME.utils import CIMEError -from CIME.XML.entry_id import EntryID +from CIME.core.exceptions import CIMEError from CIME.XML.env_batch import EnvBatch -from CIME.XML.files import Files def create_batch_system(env_batch, batch_submit_value=None): diff --git a/CIME/tests/test_unit_nmlgen.py b/CIME/tests/test_unit_nmlgen.py index 52e04d28856..76c274871be 100644 --- a/CIME/tests/test_unit_nmlgen.py +++ b/CIME/tests/test_unit_nmlgen.py @@ -1,10 +1,11 @@ -from collections import OrderedDict import tempfile import unittest +from collections import OrderedDict from unittest import mock from CIME.nmlgen import NamelistGenerator + # pylint: disable=protected-access class TestNamelistGenerator(unittest.TestCase): def test_init_defaults(self): diff --git a/CIME/tests/test_unit_paramgen.py b/CIME/tests/test_unit_paramgen.py index 26b2be4ac6e..698a77093a1 100755 --- a/CIME/tests/test_unit_paramgen.py +++ b/CIME/tests/test_unit_paramgen.py @@ -13,8 +13,9 @@ # # pylint:disable=line-too-long -import unittest import tempfile +import unittest + from CIME.ParamGen.paramgen import ParamGen ############### diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index c95c7fd2d9a..f2bbff1924f 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -1,18 +1,17 @@ #!/usr/bin/env python3 -import os -import tempfile import gzip +import os import re -from re import A +import tempfile import unittest -from unittest import mock from pathlib import Path +from unittest import mock from CIME.config import Config from CIME.SystemTests.system_tests_common import SystemTestsCommon -from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.SystemTests.system_tests_compare_n import SystemTestsCompareN +from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo CPLLOG = """ tStamp_write: model date = 00010102 0 wall clock = 2023-09-19 19:39:42 avg dt = 0.33 dt = 0.33 diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py index 0bb33f8d605..3bcf4a1713d 100644 --- a/CIME/tests/test_unit_system_tests_mvk.py +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 -import os -import json -import unittest -import tempfile import contextlib +import json +import os import sysconfig +import tempfile +import unittest from pathlib import Path from unittest import mock + from CIME.tests.utils import chdir evv4esm = False diff --git a/CIME/tests/test_unit_test_status.py b/CIME/tests/test_unit_test_status.py index 9b3036801fc..061e2cafe40 100755 --- a/CIME/tests/test_unit_test_status.py +++ b/CIME/tests/test_unit_test_status.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -import unittest import os -from CIME import test_status -from CIME import expected_fails +import unittest + +from CIME import expected_fails, test_status from CIME.tests.custom_assertions_test_status import CustomAssertionsTestStatus diff --git a/CIME/tests/test_unit_two_link_to_case2_output.py b/CIME/tests/test_unit_two_link_to_case2_output.py index 2860db08ac1..5d59a6ca298 100755 --- a/CIME/tests/test_unit_two_link_to_case2_output.py +++ b/CIME/tests/test_unit_two_link_to_case2_output.py @@ -10,10 +10,11 @@ # # pylint:disable=protected-access -import unittest import os import shutil import tempfile +import unittest + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.tests.case_fake import CaseFake diff --git a/CIME/tests/test_unit_user_mod_support.py b/CIME/tests/test_unit_user_mod_support.py index 51ffb4778ca..ca7289f1596 100755 --- a/CIME/tests/test_unit_user_mod_support.py +++ b/CIME/tests/test_unit_user_mod_support.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 -import unittest +import os import shutil import tempfile -import os +import unittest + +from CIME.core.exceptions import CIMEError from CIME.user_mod_support import apply_user_mods -from CIME.utils import CIMEError # ======================================================================== # Define some parameters diff --git a/CIME/tests/test_unit_user_nl_utils.py b/CIME/tests/test_unit_user_nl_utils.py index 9220182eeeb..2ec199874b8 100755 --- a/CIME/tests/test_unit_user_nl_utils.py +++ b/CIME/tests/test_unit_user_nl_utils.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -import unittest import os import shutil import tempfile +import unittest + from CIME.SystemTests.test_utils import user_nl_utils diff --git a/CIME/tests/test_unit_utils.py b/CIME/tests/test_unit_utils.py index 8abde4d8fef..00f8fd1a047 100755 --- a/CIME/tests/test_unit_utils.py +++ b/CIME/tests/test_unit_utils.py @@ -1,21 +1,20 @@ #!/usr/bin/env python3 import os -import stat import shutil import sys import tempfile - import unittest from unittest import mock + from CIME.status import run_and_log_case_status from CIME.utils import ( - indent_string, - import_from_file, _line_defines_python_function, - file_contains_python_function, copy_globs, + file_contains_python_function, import_and_run_sub_or_cmd, + import_from_file, + indent_string, ) diff --git a/CIME/tests/test_unit_xml_archive_base.py b/CIME/tests/test_unit_xml_archive_base.py index 98f58055c9c..42789333dc0 100644 --- a/CIME/tests/test_unit_xml_archive_base.py +++ b/CIME/tests/test_unit_xml_archive_base.py @@ -1,12 +1,10 @@ #!/usr/bin/env python3 -import os import io -import unittest import tempfile +import unittest from contextlib import contextmanager from pathlib import Path -from unittest import mock from CIME.XML.archive_base import ArchiveBase diff --git a/CIME/tests/test_unit_xml_env_batch.py b/CIME/tests/test_unit_xml_env_batch.py index 3599cfe1dfc..b3777031d27 100755 --- a/CIME/tests/test_unit_xml_env_batch.py +++ b/CIME/tests/test_unit_xml_env_batch.py @@ -1,15 +1,16 @@ #!/usr/bin/env python3 import os -import unittest import tempfile +import unittest from contextlib import ExitStack from unittest import mock -from CIME.utils import CIMEError, expect +from CIME.BuildTools.configure import FakeCase +from CIME.core.exceptions import CIMEError +from CIME.utils import expect from CIME.XML.env_batch import EnvBatch, get_job_deps from CIME.XML.env_workflow import EnvWorkflow -from CIME.BuildTools.configure import FakeCase # pylint: disable=unused-argument diff --git a/CIME/tests/test_unit_xml_env_mach_specific.py b/CIME/tests/test_unit_xml_env_mach_specific.py index a900034fad8..ef72a061ac5 100644 --- a/CIME/tests/test_unit_xml_env_mach_specific.py +++ b/CIME/tests/test_unit_xml_env_mach_specific.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -import unittest import tempfile +import unittest from unittest import mock from CIME import utils diff --git a/CIME/tests/test_unit_xml_grids.py b/CIME/tests/test_unit_xml_grids.py index 17c01e355de..292012033bc 100644 --- a/CIME/tests/test_unit_xml_grids.py +++ b/CIME/tests/test_unit_xml_grids.py @@ -1,15 +1,10 @@ import os -import io -import unittest import tempfile -from contextlib import contextmanager -from pathlib import Path -from unittest import mock +import unittest -from CIME.utils import CIMEError +from CIME.core.exceptions import CIMEError from CIME.XML.grids import Grids - TEST_CONFIG = """ diff --git a/CIME/tests/test_unit_xml_tests.py b/CIME/tests/test_unit_xml_tests.py index 898e00bb0b8..23d5d8fac4c 100644 --- a/CIME/tests/test_unit_xml_tests.py +++ b/CIME/tests/test_unit_xml_tests.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import re -import unittest import tempfile +import unittest from pathlib import Path from unittest import mock diff --git a/CIME/tests/utils.py b/CIME/tests/utils.py index 5987f0fd18e..d74d0798f44 100644 --- a/CIME/tests/utils.py +++ b/CIME/tests/utils.py @@ -1,19 +1,16 @@ +import contextlib import io import os -import tempfile -import signal import shutil import sys -import time -import contextlib +import tempfile +import xml.etree.ElementTree as ET from collections.abc import Iterable from unittest import mock -import xml.etree.ElementTree as ET -from CIME import utils from CIME import test_status -from CIME.utils import expect from CIME.case import Case +from CIME.utils import expect from CIME.XML.entry_id import EntryID MACRO_PRESERVE_ENV = [ @@ -321,7 +318,6 @@ def get_default_compiler(self): class MakefileTester(object): - """Helper class for checking Makefile output. Public methods: @@ -421,7 +417,6 @@ def assert_variable_matches(self, var_name, regex, env=None, var=None): class CMakeTester(object): - """Helper class for checking CMake output. Public methods: diff --git a/CIME/user_mod_support.py b/CIME/user_mod_support.py index 2c182a19dbd..b4eeec21c1a 100644 --- a/CIME/user_mod_support.py +++ b/CIME/user_mod_support.py @@ -2,9 +2,11 @@ user_mod_support.py """ -from CIME.XML.standard_module_setup import * -from CIME.utils import expect, run_cmd_no_fail, safe_copy import glob +import logging +import os + +from CIME.utils import expect, run_cmd_no_fail, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/utils.py b/CIME/utils.py index e4a49c4457e..52d38c76ed8 100644 --- a/CIME/utils.py +++ b/CIME/utils.py @@ -3,12 +3,27 @@ Warning: you cannot use CIME Classes in this module as it causes circular dependencies """ -import shlex import configparser -import io, logging, gzip, sys, os, time, re, shutil, glob, string, random, importlib, fnmatch +import errno +import filecmp +import fnmatch +import glob +import gzip +import importlib import importlib.util -import errno, signal, warnings, filecmp +import io +import logging +import os +import random +import re +import shlex +import shutil +import signal import stat as statlib +import string +import sys +import time +import warnings from argparse import Action from contextlib import contextmanager @@ -152,8 +167,10 @@ def __exit__(self, *args): # hate seeing. It's a subclass of Exception because we want it to be # "catchable". If you are debugging CIME and want to see the stacktrace, # run your CIME command with the --debug flag. -class CIMEError(SystemExit, Exception): - pass +# +# Canonical definition lives in CIME.core.exceptions; re-exported here +# for backward compatibility. +from CIME.core.exceptions import CIMEError # noqa: F401 def expect(condition, error_msg, exc_type=CIMEError, error_prefix="ERROR:"): diff --git a/CIME/wait_for_tests.py b/CIME/wait_for_tests.py index 1f6940138a9..51b53ce7f63 100644 --- a/CIME/wait_for_tests.py +++ b/CIME/wait_for_tests.py @@ -1,17 +1,36 @@ # pylint: disable=import-error -import queue, os, time, threading, socket, signal, shutil, glob, tempfile -from pathlib import Path +import glob # pylint: disable=import-error import logging +import os +import queue +import shutil +import signal +import socket +import tempfile +import threading +import time import xml.etree.ElementTree as xmlet +from pathlib import Path import CIME.utils -from CIME.utils import expect, Timeout, run_cmd, run_cmd_no_fail, safe_copy, CIMEError -from CIME.XML.machines import Machines -from CIME.test_status import * -from CIME.provenance import save_test_success from CIME.case.case import Case +from CIME.core.exceptions import CIMEError +from CIME.provenance import save_test_success +from CIME.test_status import ( + CREATE_NEWCASE_PHASE, + NAMELIST_FAIL_STATUS, + NAMELIST_PHASE, + RUN_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TEST_PEND_STATUS, + TEST_STATUS_FILENAME, + TestStatus, +) +from CIME.utils import Timeout, expect, run_cmd, run_cmd_no_fail, safe_copy +from CIME.XML.machines import Machines SIGNAL_RECEIVED = False E3SM_MAIN_CDASH = "E3SM" @@ -19,6 +38,7 @@ SLEEP_INTERVAL_SEC = 0.1 ENV_VAR_KEEP_CDASH = "CIME_TEST_CDASH_WFT" + ############################################################################### def signal_handler(*_): ############################################################################### diff --git a/doc/dev/refactor/README.md b/doc/dev/refactor/README.md new file mode 100644 index 00000000000..d3cf0f99997 --- /dev/null +++ b/doc/dev/refactor/README.md @@ -0,0 +1,102 @@ +# CIME Refactor Documentation + +Documentation for the CIME refactoring effort. + +## Quick Start + +- **New to refactor?** Read `plan.md` +- **Implementing a slice?** See `implementation.md` +- **Working on SRCROOT feature?** See `feature_srcroot.md` +- **Working on single entrypoint?** See `feature_single_entrypoint.md` + +## Documents + +### [plan.md](plan.md) +Architecture, principles, and migration strategy. +- Compatibility-first policy +- Reorganize-don't-rewrite approach +- 5 implementation slices +- Success criteria + +### [implementation.md](implementation.md) +Detailed tasks, timeline, and testing approach. +- Slice-by-slice breakdown +- Standard mocking strategy (no custom DI) +- Code review checklist + +### [feature_srcroot.md](feature_srcroot.md) +SRCROOT standardization feature (Slice 3A). +- Removes config_files.xml indirection +- Migration plan + +### [feature_single_entrypoint.md](feature_single_entrypoint.md) +Single CIME entrypoint feature. +- Replaces ~20 symlinks with one +- Shell wrappers for tool compatibility +- Can run parallel with other slices + +## Overview + +**Goal**: Improve CIME's modularity, error handling, and testability while +maintaining compatibility with E3SM, CESM, and NorESM. + +**Timeline**: 22-24 weeks (5 slices) + +**Approach**: Move existing code into focused modules. Consolidate scattered +functions back into the classes that own them. Don't rewrite working code or +wrap stdlib behind abstraction layers. + +## Implementation Slices + +1. **Foundation** (3 weeks): Typed exceptions, centralized bootstrap +2. **Batch** (4 weeks): Move batch/submit logic to `CIME/core/batch/` +3. **SRCROOT** (4 weeks): Standardize config loading (NEW FEATURE) +4. **Build** (5 weeks): Move build logic to `CIME/core/build/` +5. **Case** (6 weeks): Consolidate scattered Case functions, then decompose + into focused subsystems (status, locking, XML storage) + +## Key Principles + +- **Compatibility first**: External models must continue working +- **Reorganize, don't rewrite**: Move code to better homes +- **DI where it earns its keep**: Protocols and constructor injection for real + CIME polymorphism (scheduler backends, config loaders); `mock.patch` for stdlib +- **Incremental**: Each slice independently testable +- **Test-driven**: 80%+ coverage for reorganized code + +## For Developers + +**Before coding**: +1. Read relevant slice in `implementation.md` +2. Understand the reorganize-don't-rewrite principle in `plan.md` + +**During development**: +- Move existing functions, don't rewrite them +- Add re-exports from old import paths +- Write tests with `unittest.mock`, `monkeypatch`, `tmp_path` +- Use DI/protocols for real CIME interfaces, not stdlib wrapping +- Maintain backward compatibility + +**Code review checklist**: +- [ ] Code moved, not rewritten (unless simplifying complexity) +- [ ] Free functions that take an object consolidated as methods where appropriate +- [ ] Internal CIME imports updated to `CIME/core/` +- [ ] Old import paths still work via re-exports (for downstream models) +- [ ] Standard mocking for stdlib; DI/protocols for CIME interfaces +- [ ] No unnecessary abstraction layers +- [ ] 80%+ test coverage +- [ ] All tests pass +- [ ] Docs updated + +## For Downstream Models + +**E3SM, CESM, NorESM integrators**: +- Each slice maintains compatibility +- Old import paths continue to work via re-exports +- SRCROOT feature (Slice 3A) has migration guide +- Test during opt-in periods +- Provide feedback early + +## Questions? + +File issues in CIME repository or discuss in development meetings. diff --git a/doc/dev/refactor/feature_single_entrypoint.md b/doc/dev/refactor/feature_single_entrypoint.md new file mode 100644 index 00000000000..28a2c54eaf3 --- /dev/null +++ b/doc/dev/refactor/feature_single_entrypoint.md @@ -0,0 +1,137 @@ +# Feature: Single CIME Entrypoint + +Collapse all symlinked case scripts into one `cime` entrypoint with shell wrappers. + +## Changes + +### Current: ~20 Symlinks +``` +case_dir/ +├── case.setup -> /path/to/CIME/scripts/Tools/case.setup +├── case.build -> /path/to/CIME/scripts/Tools/case.build +├── xmlchange -> /path/to/CIME/scripts/Tools/xmlchange +└── ... (~17 more symlinks) +``` + +**Problem**: Changing CIME version requires updating all symlinks. + +### New: 1 Symlink + Shell Wrappers +``` +case_dir/ +├── cime -> /path/to/CIME/scripts/cime # Only symlink +├── case.setup # Wrapper script +├── case.build # Wrapper script +├── xmlchange # Wrapper script +└── ... +``` + +**Benefit**: Change CIME version by updating one symlink. + +## Implementation + +### Single Entrypoint: `CIME/scripts/cime` + +```python +#!/usr/bin/env python3 +"""Single entrypoint for all CIME case operations.""" + +COMMANDS = { + 'case.setup': 'CIME.Tools.case_setup', + 'case.build': 'CIME.Tools.case_build', + 'xmlchange': 'CIME.Tools.xmlchange', + # ... all tools +} + +def main(): + command = sys.argv[1] + module = __import__(COMMANDS[command], fromlist=['main']) + sys.argv = [f"cime {command}"] + sys.argv[2:] + return module.main() +``` + +### Shell Wrapper Template + +```bash +#!/usr/bin/env bash +# Wrapper for: cime case.setup +exec "$(dirname "$0")/cime" case.setup "$@" +``` + +**Key**: +- `exec` replaces shell (clean exit codes) +- `"$@"` preserves all arguments +- Finds `cime` relative to wrapper + +### Case Setup + +```python +def create_case_scripts(case_root: Path, cimeroot: Path): + # Create single cime symlink + (case_root / "cime").symlink_to(cimeroot / "scripts" / "cime") + + # Create shell wrappers + wrapper = '#!/usr/bin/env bash\nexec "$(dirname "$0")/cime" {cmd} "$@"\n' + + for tool in ["case.setup", "case.build", "xmlchange", ...]: + path = case_root / tool + path.write_text(wrapper.format(cmd=tool)) + path.chmod(0o755) +``` + +## Migration + +### New Cases +Immediately use new pattern (single symlink + wrappers). + +### Existing Cases +**Option 1**: Leave unchanged - both patterns work +**Option 2**: Provide upgrade utility (optional) + +**No breaking changes** - old and new patterns coexist. + +## Benefits + +1. **Simple CIME updates**: Change one symlink instead of ~20 +2. **Cleaner case dirs**: One symlink vs. many +3. **Clear CIME version**: Easy to see which CIME in use +4. **Better portability**: Fewer symlinks to manage +5. **CLI foundation**: Natural path to full CLI layer + +## Testing + +- Command routing and argument passing +- All major case operations +- Old and new case patterns +- Exit codes and output + +## Integration with Refactor + +- **Uses**: Slice 1 bootstrap (`ensure_cime_on_path`) +- **Enables**: Future CLI layer (optional) +- **Timeline**: 2-3 weeks, can run parallel with other slices + +## Future Enhancements + +### Subcommands (optional) +```bash +cime case setup # Alternative syntax +cime xml change VAR=val +``` + +### Global Options (optional) +```bash +cime --case /path case.build +cime --verbose case.setup +cime --help +``` + +## Decision Points + +1. **Existing cases**: Auto-upgrade or leave unchanged? + - Recommend: Leave unchanged (safer) + +2. **Wrapper language**: Shell or Python? + - Recommend: Shell (simpler, minimal overhead) + +3. **Command syntax**: Keep dots (`case.setup`)? + - Recommend: Keep dots (backward compatible) diff --git a/doc/dev/refactor/feature_srcroot.md b/doc/dev/refactor/feature_srcroot.md new file mode 100644 index 00000000000..a4e085d6bba --- /dev/null +++ b/doc/dev/refactor/feature_srcroot.md @@ -0,0 +1,121 @@ +# Feature: SRCROOT Standardization + +Remove `config_files.xml` indirection and standardize SRCROOT resolution for submodule usage. + +## Changes + +### SRCROOT Resolution Priority +1. Environment variable `SRCROOT` +2. User config `~/.cime/config` +3. Command-line flag `--srcroot` +4. Step up one directory from CIMEROOT +5. CIMEROOT itself (standalone mode) + +### What's Removed +- `CIME/data/config/{e3sm,cesm,ufs}/config_files.xml` +- `MODEL_CONFIG_FILES` indirection +- Model-specific `get_config_path()` logic + +### What's Added +- `CIME/core/config/srcroot.py` - SRCROOT resolver with DI +- `CIME/core/config/loader.py` - Direct config file loader +- `--srcroot` flag to CLI tools +- Standalone mode for unit testing + +### Expected Directory Structure +``` +$SRCROOT/ +├── cime_config/ # Model config (validated) +│ ├── config_grids.xml +│ ├── config_compsets.xml +│ ├── config_machines.xml +│ └── config_tests.xml +└── cime/ # CIME submodule +``` + +## Implementation (DI-Compliant) + +### SRCROOTResolver +```python +class SRCROOTResolver: + """Resolves SRCROOT with dependency injection.""" + + def __init__(self, cimeroot: Path, filesystem: FileSystem, + environment: EnvironmentProvider, user_config: UserConfigProvider): + # All dependencies injected + + def resolve(self, flag_value: Optional[str] = None) -> SRCROOTResolution: + # Priority: env > user_config > flag > implicit + # Returns: SRCROOTResolution(path, source, standalone_mode) +``` + +**DI Pattern**: Constructor injection, no globals, fully testable with mocks. + +### ConfigFileLoader +```python +class ConfigFileLoader: + """Loads config files from $SRCROOT/cime_config with DI.""" + + def __init__(self, srcroot: Path, filesystem: FileSystem): + # Filesystem injected for testability + + def validate_and_load(self, require_full: bool = True) -> ConfigFiles: + # Validates cime_config exists + # Returns: ConfigFiles(paths to all config XMLs) +``` + +**DI Pattern**: Uses injected FileSystem, no direct path.is_dir() calls. + +### Factory Functions +```python +def create_srcroot_resolver(...) -> SRCROOTResolver: + # Provides defaults for production, allows injection for tests + +def create_config_loader(...) -> ConfigFileLoader: + # Provides defaults for production, allows injection for tests +``` + +## Testing + +**Mock-based testing** (no global state, no real filesystem): + +```python +def test_srcroot_from_environment(): + mock_env = MockEnvironment({"SRCROOT": "/test"}) + mock_fs = MockFileSystem({Path("/test/cime_config")}) + + resolver = SRCROOTResolver(cimeroot, mock_fs, mock_env, mock_config) + resolution = resolver.resolve() + + assert resolution.path == Path("/test") + assert resolution.source == "environment" +``` + +## Migration + +**4-Stage Rollout:** +1. **Opt-in** (Weeks 1-2): Feature flag `CIME_USE_NEW_CONFIG_LOADER=true` +2. **Opt-out** (Weeks 3-4): New system default, old available via flag +3. **Deprecation**: Warnings on old code paths +4. **Removal**: Delete old config_files.xml system + +## Integration with Refactor + +- **Uses Slice 1**: FileSystem, EnvironmentProvider abstractions +- **New Slice 3A**: SRCROOT standardization (3-4 weeks) +- **Before Slice 3B**: Must complete before build system refactor + +## Validation + +- E3SM case creation workflow +- CESM case creation workflow +- NorESM case creation workflow +- Standalone mode for unit tests + +## Benefits + +- Simpler config loading (no indirection) +- Better error messages (fail early) +- Fully testable (DI throughout) +- Explicit standalone mode +- Deterministic SRCROOT resolution diff --git a/doc/dev/refactor/implementation.md b/doc/dev/refactor/implementation.md new file mode 100644 index 00000000000..f1ee4248a31 --- /dev/null +++ b/doc/dev/refactor/implementation.md @@ -0,0 +1,431 @@ +# Implementation Plan + +Detailed tasks for the CIME refactor. See `plan.md` for architecture and principles. + +**Key rule: move existing code, don't rewrite it.** Extract functions into +focused modules. Only refactor when a module is too large or too tangled to +move as-is. + +**Target: Python 3.9+.** Do not use 3.10+ syntax (`X | Y` unions, +`match`/`case`). Use `typing.Union`, `typing.Optional`, etc. + +**Migration pattern**: Move code to `CIME/core/`, then make the original +module import from `CIME/core/` and re-export. See `plan.md` "Migration +Pattern" for details and examples. + +## Timeline: 22-24 weeks (5 slices) + +| Slice | Weeks | Focus | +|-------|-------|-------| +| 1. Foundation | 1-3 | Exceptions, bootstrap | +| 2. Batch | 4-7 | Scheduler/submit | +| 3A. SRCROOT | 8-11 | Config loading (NEW) | +| 3B. Build | 12-16 | Build system | +| 4. Case | 17-22 | Case refactor | + +--- + +## Handling `utils.py` (2700 lines, 104 functions) + +`utils.py` is the largest module and doesn't belong to any single slice. +It contains functions across every category: + +| Category | Count | Examples | +|----------|-------|---------| +| General utilities | 57 | `run_cmd`, `safe_copy`, logging, time conversion | +| SRCROOT/config | 19 | `get_cime_root`, `get_src_root`, `get_model` | +| Foundation | 9 | `CIMEError`, `expect`, `fixup_sys_path` | +| Case | 9 | `parse_test_name`, `wait_for_unlocked` | +| Batch | 7 | `batch_jobid`, `get_project`, `transform_vars` | +| Build | 3 | `analyze_build_log`, `copy_local_macros_to_dir` | + +**Strategy**: Each slice moves its own functions **plus the general utilities +those functions depend on**. As general utils get pulled into `core/`, they +land in focused modules: + +- `CIME/core/shell.py` — `run_cmd`, `run_cmd_no_fail` (pulled by Slice 2/Batch) +- `CIME/core/logging.py` — logging setup, formatters (pulled by Slice 1) +- `CIME/core/time.py` — time conversion helpers (pulled by Slice 4/Case) +- `CIME/core/fileops.py` — `safe_copy`, `symlink_force`, `find_files` (pulled as needed) +- `CIME/core/convert.py` — type conversion helpers (pulled as needed) + +By the end of Slice 4, `utils.py` should be a thin re-export file with no +implementation code of its own. + +--- + +## Slice 1: Foundation (Weeks 1-3) + +**Goal**: Typed exceptions, centralized bootstrap, and eliminate the worst +cross-cutting code smells. No behavior changes. + +### Tasks +- `CIME/core/exceptions.py` — Typed exception hierarchy extending existing `CIMEError`. + **Important**: `CIMEError` currently lives in `utils.py:155` — move it to + `core/exceptions.py` and re-export from `utils.py`. The typed subclasses + (`ConfigurationError`, `BuildError`, etc.) won't have callers yet but + establish the hierarchy for later slices. +- `CIME/core/config/bootstrap.py` — Centralized `sys.path` management (consolidates + the `sys.path.insert` calls currently scattered across 11+ files, plus the + duplicate logic in `standard_script_setup.py` and `standard_module_setup.py`) +- Move from `utils.py`: `CIMEError`, `expect`, `deprecate_action`, + `fixup_sys_path`, `import_from_file`, `import_and_run_sub_or_cmd`, + `run_sub_or_cmd`, and their helpers +- Move logging infrastructure from `utils.py` → `CIME/core/logging.py` + (`IndentFormatter`, `set_logger_indent`, `configure_logging`, + `setup_standard_logging_options`, etc.) +- Migrate `sys.path` call-sites incrementally to use bootstrap module +- Update internal CIME imports; leave re-exports in `utils.py` +- **Eliminate star imports from `standard_module_setup`** — this single + `from CIME.XML.standard_module_setup import *` appears in ~60 files, + polluting every module's namespace with `os`, `sys`, `re`, `expect`, + `run_cmd`, etc. Replace with explicit imports in each file. This is the + highest-impact code quality fix in the entire refactor. +- **Eliminate star imports from `test_status`** — `from CIME.test_status import *` + appears in ~10 files, exporting ~30 constants. Replace with explicit imports. +- **Remove `GLOBAL = {}` mutable state** from `utils.py:21` — this dict is used + to pass `SRCROOT` between `case.py`, `create_test.py`, and `utils.get_src_root()` + via implicit global mutation. Replace with explicit parameter passing. +- **Fix `Servers/__init__.py`** — runs `shutil.which()` at import time for + `globus-url-copy`, `svn`, `wget`, making package import non-deterministic. + Make discovery lazy. +- Standardize error handling: audit `sys.exit()` calls (6 sites) and bare + `raise RuntimeError` (~30 sites) and migrate to `CIMEError` / `expect()` + where appropriate +- **Fix `conftest.py`** — currently requires host model config and machine XML + to run any tests. Split so unit tests (`test_unit_*.py`) can run standalone + without machine initialization; keep machine setup for system tests only. +- **Fix circular imports** in Case — `case.py:84-102` injects methods from + 10 sibling modules at class body level. The move to `core/` modules should + start breaking these cycles. + +### What we're NOT doing in Slice 1 +- No Protocol classes wrapping stdlib (`os.path`, `subprocess`, `time`, `os.environ`) +- No factory functions or service locators +- No new abstraction layers + +### Success +- All existing tests pass +- Zero star imports from `standard_module_setup` and `test_status` +- `GLOBAL = {}` eliminated; SRCROOT passed explicitly +- `sys.path` manipulation consolidated +- Consistent error handling (no bare `sys.exit()` in library code) +- `Servers/__init__.py` no longer runs executables at import time +- Unit tests (`test_unit_*.py`) run without host model / machine config +- Exception hierarchy in place for later slices to use + +--- + +## Slice 2: Batch System (Weeks 4-7) + +**Goal**: Move batch logic from scattered locations into `CIME/core/batch/`. + +### Tasks +- Move scheduler-specific logic from XML classes into `CIME/core/batch/` +- Move submission logic from `case_submit.py` internals +- Move from `utils.py`: `batch_jobid`, `get_batch_script_for_job`, + `get_project`, `get_charge_account`, `add_mail_type_args`, + `resolve_mail_type_args`, `transform_vars` +- Move `run_cmd`, `run_cmd_no_fail` from `utils.py` → `CIME/core/shell.py` + (general utility, pulled here because batch is the first heavy consumer) +- A `Scheduler` protocol is appropriate here — PBS, Slurm, and LSF are genuine + polymorphic backends worth abstracting behind a common interface +- Keep existing entrypoints as thin wrappers calling into new location +- Update internal CIME imports; leave re-exports in old locations + +### Consolidation +- `transform_vars` (utils.py:2118) takes `case` and calls `case.get_value()` + repeatedly — consolidate as a `Case` method, with the core template logic + in `CIME/core/batch/` +- `batch_jobid` (utils.py:2360) calls `case.get_job_id()` — consolidate into + the batch subsystem +- **`EnvBatch`** (XML/env_batch.py, 1590 lines, 43 methods) — this is the + second-largest class in CIME. It handles batch config parsing, job submission, + queue management, and directive generation all in one class. Decompose into: + - `CIME/core/batch/config.py` — batch config/queue parsing (from EnvBatch) + - `CIME/core/batch/submit.py` — job submission logic (from EnvBatch + case_submit.py) + - `CIME/core/batch/directives.py` — directive generation (from EnvBatch) + - `EnvBatch` becomes a thinner wrapper that delegates to these modules + +### Success +- All batch tests pass +- External models submit jobs unchanged +- Batch logic in one place instead of spread across XML/ and case/ + +--- + +## Slice 3A: SRCROOT Standardization (Weeks 8-11) **NEW** + +**Goal**: Remove config_files.xml, standardize SRCROOT. + +See `feature_srcroot.md` for details. + +### Tasks +- `CIME/core/config/srcroot.py` — SRCROOT resolution logic +- `CIME/core/config/loader.py` — Config file loading +- Move from `utils.py`: `get_cime_root`, `get_src_root`, `get_model`, + `set_model`, `get_cime_config`, `get_config_path`, `get_schema_path`, + `get_template_path`, `get_tools_path`, `get_cime_default_driver`, + `get_all_cime_models`, `get_model_config_location_within_cime`, + `get_scripts_root`, `get_model_config_root`, `get_htmlroot`, `get_urlroot`, + and their helpers +- Add `--srcroot` flag to CLI tools +- Refactor `CIME/XML/files.py` with feature flag +- 4-stage migration (opt-in → opt-out → deprecate → remove) +- Update internal CIME imports; leave re-exports in `utils.py` + +### Success +- Works with E3SM, CESM, NorESM +- Standalone mode for unit tests +- config_files.xml deprecated + +--- + +## Slice 3B: Build System (Weeks 12-16) + +**Goal**: Move build logic from `build.py` (1350 lines) into focused modules. + +### Tasks +- Extract build planning logic into `CIME/core/build/` +- Extract build execution/orchestration +- Move from `utils.py`: `analyze_build_log`, `run_bld_cmd_ensure_logging`, + `copy_local_macros_to_dir` +- Move file operation helpers from `utils.py` → `CIME/core/fileops.py` + (`safe_copy`, `safe_recursive_copy`, `symlink_force`, `copy_globs`, + `copy_over_file`, `copyifnewer` — pulled here because build is a heavy consumer) +- Keep `CIME/build_scripts/` as stable wrappers +- `build.py` becomes thin re-export layer +- Update internal CIME imports; leave re-exports in old locations + +### Consolidation +- `case_build()`, `clean()`, `_clean_impl()`, `post_build()` (build.py) + all take `case`, access `case._gitinterface` (private!), call + `case.get_value()` / `case.set_value()` / `case.flush()` — these should + become `Case` methods that delegate to `CIME/core/build/` +- `get_standard_cmake_args()`, `get_standard_makefile_args()`, `uses_kokkos()` + (build.py) extract ~15 values from Case — consolidate as Case methods or a + `BuildConfig` data class populated from Case +- `generate_makefile_macro()` (build.py) calls `case.get_value()` — same pattern +- Deduplicate the four copy functions in `utils.py` (`safe_copy`, + `copy_over_file`, `copyifnewer`, `copy_globs`) into a coherent + `CIME/core/fileops.py` API + +### Success +- All build tests pass +- `build_scripts` compatibility maintained +- External models build unchanged + +--- + +## Slice 4: Case Refactoring (Weeks 17-22) + +**Goal**: Decompose the `Case` class into focused subsystems while +consolidating scattered Case-related functions. + +### Why Case needs refactoring + +`Case` (`case.py`, 2600 lines) is a god object. It directly handles: +- XML value get/set across multiple env files +- Status tracking (CaseStatus file) +- Lock management (LockedFiles directory) +- Build orchestration +- Job submission +- Namelist generation +- Clone/setup/test workflows + +On top of that, many Case operations live *outside* the class as free +functions in `utils.py`, `build.py`, `status.py`, `locked_files.py`, and +`case_run.py` — functions that take `case` as a parameter and reach into +its internals (including private attributes like `case._gitinterface`). + +The result: Case responsibility is smeared across ~10 files with no clear +boundaries, and the class itself is too large to reason about or test in +isolation. + +### Two-phase approach + +**Phase 1 — Consolidate into Case**: Bring the scattered free functions +back into Case (or into subsystem classes that Case owns). This *temporarily* +makes Case bigger, but it gathers all the responsibility in one place so we +can see the seams clearly. + +**Phase 2 — Extract subsystems out of Case**: Once the responsibility is +consolidated, decompose Case into focused subsystem modules that Case +delegates to. Case becomes a thinner coordinator/facade. + +The end result is a Case that's smaller than today, with clear delegation +to subsystems that are independently testable. + +### Phase 1 tasks — Consolidate + +Bring free functions into Case (or subsystem classes Case will delegate to): + +- **Status** (from `status.py`): `append_case_status()`, + `run_and_log_case_status()` — every caller extracts `caseroot` and + `case._gitinterface` just to pass them in +- **Locking** (from `locked_files.py`): `check_lockedfiles()`, + `check_lockedfile()`, `diff_lockedfile()`, `check_diff()` — pure Case + integrity checks deeply coupled to Case env XML subsystem +- **Case queries** (from `utils.py`): `is_comp_standalone()`, + `find_system_test()`, `get_lids()`, `new_lid()` — predicates and queries + on Case state +- **Case run helpers** (from `case/case_run.py`): `_pre_run_check()`, + `_run_model_impl()`, `_post_run_check()`, `_resubmit_check()` — free + functions that take `case`; make them methods +- **TestScheduler** (from `test_scheduler.py`): `_get_time_est()`, + `_order_tests_by_runtime()`, `_translate_test_names_for_new_pecount()` — + module-level helpers with `_TIME_CACHE` that should be instance state + +### Phase 2 tasks — Extract subsystems + +Decompose Case into focused modules it delegates to: + +- `CIME/core/status/` — status tracking (CaseStatus file, `append_status`, + `run_and_log_status`) +- `CIME/core/locking/` — lock management (LockedFiles, `check_locked`, + `diff_locked`) +- `CIME/core/xml/` — XML value storage (env file get/set/flush, the + `_env_entryid_files` and `_env_generic_files` arrays) + +Case keeps its public API (`get_value`, `set_value`, `build`, `submit`, etc.) +but each method delegates to the appropriate subsystem: + +```python +# After refactoring — Case delegates, doesn't implement +class Case: + def __init__(self, ...): + self._status = StatusTracker(self._caseroot, self._gitinterface) + self._locks = LockManager(self._caseroot) + self._xml = XmlStore(self._env_files) + + def append_status(self, msg, ...): + self._status.append(msg, ...) + + def check_lockedfiles(self): + self._locks.check(self._xml) +``` + +### Other Slice 4 tasks +- Move from `utils.py`: `wait_for_unlocked`, `normalize_case_id`, + `parse_test_name`, `get_full_test_name`, `compute_total_time` +- Move remaining general utils from `utils.py`: + - `CIME/core/time.py` — time conversion (`convert_to_seconds`, + `convert_to_babylonian_time`, `get_time_in_seconds`, `format_time`) + - `CIME/core/convert.py` — type conversion (`convert_to_type`, + `convert_to_unknown_type`, `convert_to_string`, `stringify_bool`) +- `utils.py` should be a thin re-export file by end of Slice 4 +- Update all internal CIME imports + +### Other large modules to decompose in Slice 4 + +- **`case/case_st_archive.py`** (1395 lines, ~20 free functions taking `case`) + — largest procedural module; consolidate functions as Case methods or + extract into `CIME/core/archive/` +- **`case/check_input_data.py`** (693 lines, ~12 free functions taking `case`) + — consolidate into Case or a focused input-data module +- **`baselines/performance.py`** (612 lines, ~12 free functions taking `case`) + — consolidate into a baseline comparison subsystem +- **`hist_utils.py`** (836 lines, ~7 free functions taking `case`) + — history file handling, consolidate into appropriate subsystem +- **`TestScheduler`** (test_scheduler.py, 1340 lines, 28 methods) — mixes + test creation, phase management, job submission, and result processing; + decompose into focused components + +### Success +- Case API unchanged from external perspective +- Internal modules are independently testable +- No single module over ~500 lines + +--- + +## Testing Strategy + +### Standard mocking for stdlib +```python +# Good: use unittest.mock for stdlib calls +from unittest.mock import patch + +def test_build_checks_file(): + with patch("os.path.exists", return_value=True): + result = check_build_prereqs("/some/path") + assert result is True + +# Good: use tmp_path for filesystem tests +def test_writes_status(tmp_path): + tracker = StatusTracker(str(tmp_path)) + tracker.set_status("BUILT") + assert (tmp_path / "CaseStatus").read_text() == "BUILT" + +# Good: use monkeypatch for env vars +def test_reads_cimeroot(monkeypatch): + monkeypatch.setenv("CIMEROOT", "/my/cime") + assert find_cimeroot() == "/my/cime" +``` + +### DI / Protocols where they earn their keep +```python +# Good: Scheduler is genuinely polymorphic — PBS, Slurm, LSF are +# different backends behind a common interface +class Scheduler(Protocol): + def submit(self, job: Job) -> str: ... + def poll(self, job_id: str) -> JobStatus: ... + +def test_submission_retries(): + fake = FakeScheduler(fail_first=2) + submitter = JobSubmitter(scheduler=fake) + result = submitter.submit_with_retry(job, max_retries=3) + assert result.success +``` + +**Rule of thumb**: if the variation is in *CIME's own code* (scheduler types, +config loaders, model components), a protocol may be warranted. If you're just +calling `os.path` or `subprocess`, use `mock.patch`. + +### Coverage +- 80%+ for reorganized code +- Existing tests must continue to pass + +### Integration/Compatibility Tests +- Key workflows end-to-end +- E3SM, CESM, NorESM representative workflows +- Validate each slice + +--- + +## Success Metrics + +- **Compatibility**: 100% external tests pass +- **Coverage**: 80%+ for reorganized code +- **Performance**: No regression (within 5%) +- **Maintainability**: No module over ~500 lines, clear boundaries + +--- + +## Risk Management + +**High-Risk Areas**: +1. SRCROOT standardization (breaking change) +2. config_files.xml removal (migration required) +3. Case refactoring (high external usage) + +**Mitigation**: +- Feature flags for transitions +- Multi-stage rollout +- Comprehensive external model testing +- Rollback plans per slice + +--- + +## Code Review Checklist + +For each PR, verify: +- [ ] Existing code moved, not rewritten (unless refactoring complexity) +- [ ] Free functions that take an object consolidated as methods where appropriate +- [ ] Internal CIME imports updated to point to new `CIME/core/` location +- [ ] Old import paths still work via re-exports (for external consumers) +- [ ] Tests use standard mocking (`unittest.mock`, `monkeypatch`, `tmp_path`) +- [ ] DI/protocols used only for real CIME polymorphism, not stdlib wrapping +- [ ] No unnecessary abstraction layers +- [ ] 80%+ test coverage for moved/new code +- [ ] All existing tests pass +- [ ] Documentation updated diff --git a/doc/dev/refactor/plan.md b/doc/dev/refactor/plan.md new file mode 100644 index 00000000000..7d320fe6882 --- /dev/null +++ b/doc/dev/refactor/plan.md @@ -0,0 +1,341 @@ +# CIME Refactor Plan + +Incremental refactor to improve modularity, error handling, and testability +while preserving compatibility with external models (E3SM, CESM, NorESM). + +**Core principle: reorganize existing code, don't rewrite it.** Move functions +to better homes, break apart oversized modules, improve error handling. Don't +wrap standard library APIs in abstraction layers — `os.path`, `subprocess`, +`os.environ`, and `time` are already easily mockable with `unittest.mock.patch`. + +See also: `implementation.md` for detailed tasks, `feature_srcroot.md` for +config loading changes. + +--- + +## Compatibility-First Policy + +**Preserve current usage patterns unless change is absolutely required.** + +Breaking changes must be: +- Explicitly justified +- Accompanied by migration plan +- Validated with external models + +--- + +## Package Structure + +``` +CIME/ +├── core/ # Reorganized internals (all implementation lives here) +│ ├── config/ # Bootstrap, SRCROOT, config loading +│ ├── batch/ # Batch/scheduler logic (from existing code) +│ ├── build/ # Build logic (from existing code) +│ ├── status/ # Status tracking (from status.py, case) +│ ├── locking/ # Lock management (from locked_files.py, case) +│ ├── exceptions.py # Typed exception hierarchy +│ ├── shell.py # run_cmd, run_cmd_no_fail (from utils.py) +│ ├── logging.py # Logging setup (from utils.py) +│ ├── fileops.py # File helpers: safe_copy, symlink_force (from utils.py) +│ ├── time.py # Time conversion helpers (from utils.py) +│ └── convert.py # Type conversion helpers (from utils.py) +├── utils.py # Thin re-exports only (was 2700 lines) +├── build.py # Thin re-exports only +├── case/ # Case modules (thinned, delegates to core/) +├── XML/ # XML parsers (existing) +├── data/ # Config files (existing) +├── build_scripts/ # Compatibility wrapper (stable, unchanged) +├── SystemTests/ # System tests (existing) +├── Tools/ # CLI tools (existing) +└── tests/ # Tests +``` + +**Principle**: Move existing code into focused modules. Existing import paths +continue to work via re-exports until downstream models migrate. + +--- + +## Migration Pattern + +All implementation code moves to `CIME/core/`. Existing modules become thin +wrappers that import from `core/` and re-export, preserving current usage. + +**Workflow for each piece of code**: +1. Move the function/class from its current location into the appropriate + `CIME/core/` module +2. In the original file, add a thin re-export (so **external** callers like + E3SM/CESM/NorESM are unaffected) +3. Update all **internal** CIME imports to point to the new `CIME/core/` location +4. Add/update tests against the new location + +**Example** — moving `run_cmd` from `utils.py`: +```python +# CIME/core/batch/submitter.py (new home — real code lives here) +def run_cmd(cmd, ...): + # actual implementation moved here + ... + +# CIME/utils.py (thin re-export for external consumers only) +from CIME.core.batch.submitter import run_cmd # noqa: F401 + +# CIME/case/case_submit.py (internal — updated to import from core/) +# Before: +# from CIME.utils import run_cmd +# After: +from CIME.core.batch.submitter import run_cmd +``` + +This means: +- `CIME/core/` holds all implementation code +- **Internal** CIME code imports from `CIME/core/` directly +- `CIME/utils.py`, `CIME/build.py`, `CIME/case/*.py`, etc. keep thin + re-exports **only** for external downstream consumers +- `CIME/build_scripts/`, `CIME/Tools/`, and case symlinked scripts remain + as-is — they're entrypoints, not implementation +- Downstream models (E3SM, CESM, NorESM) continue working with zero changes +- Over time, downstream models can migrate to importing from `CIME/core/` + directly, and the old re-exports can eventually be deprecated + +--- + +## Approach: Reorganize, Don't Rewrite + +### What we DO +- **Move** functions from bloated modules (`utils.py` at 2700 lines) into + focused modules under `CIME/core/` +- **Consolidate** scattered functionality back into the classes that own it — + free functions that take `case` as a parameter and mutate Case state should + become methods on `Case` or its subsystem classes +- **Extract** coherent subsystems from `Case` (status tracking, locking, + XML manipulation) into their own modules +- **Consolidate** scattered patterns (e.g. `sys.path` manipulation in 11 files) +- **Deduplicate** overlapping functionality (e.g. four copy functions in utils.py) +- **Improve** error handling with typed exceptions +- **Eliminate** star imports, global mutable state, and import-time side effects +- **Add** tests for code that currently lacks coverage +- **Fix** test infrastructure so unit tests can run without host model config + +### Consolidation principle +If a free function takes an object and operates on its state, it should be a +method on that object. This is especially prevalent with `case`: + +```python +# Before: free function reaching into Case +def case_build(case, ...): + case._gitinterface # accessing private state! + case.get_value("EXEROOT") + case.set_value("BUILD_COMPLETE", "TRUE") + case.flush() + +# After: method on Case (or delegated subsystem) +class Case: + def build(self, ...): + ... +``` + +**Key consolidation targets identified**: + +| Current location | Functions | Problem | +|-----------------|-----------|---------| +| `build.py` | `case_build`, `clean`, `_clean_impl`, `post_build` | Access `case._gitinterface`, mutate Case state | +| `locked_files.py` | `check_lockedfiles`, `check_lockedfile`, `diff_lockedfile` | Pure Case integrity checks | +| `status.py` | `append_case_status`, `run_and_log_case_status` | Every caller extracts `caseroot` and `_gitinterface` from Case | +| `utils.py` | `transform_vars`, `is_comp_standalone`, `find_system_test` | Case queries living in a utility grab-bag | +| `test_scheduler.py` | `_get_time_est`, `_order_tests_by_runtime` | Module-level cache should be instance state on `TestScheduler` | +| `utils.py` | `safe_copy`, `copy_over_file`, `copyifnewer`, `copy_globs` | Overlapping copy semantics | + +### Large classes and modules requiring decomposition + +Beyond `Case`, several other classes and modules have grown too large and +exhibit the same god-object or scattered-function patterns: + +**Large classes (>20 methods):** + +| Class | File | Methods | Lines | Problem | +|-------|------|--------:|------:|---------| +| `Case` | `case/case.py` | 63 | 2600 | God object; functions scattered across ~10 files | +| `EnvBatch` | `XML/env_batch.py` | 43 | 1590 | Batch config, submission, queues, directives all in one | +| `GenericXML` | `XML/generic_xml.py` | 41 | 700 | Very wide interface; base for all XML classes | +| `NamelistGenerator` | `nmlgen.py` | 36 | 870 | Mixes definition lookup, resolution, streams, file I/O | +| `EnvMachSpecific` | `XML/env_mach_specific.py` | 35 | 770 | Module loading + env vars + mpirun + GPU config | +| `SystemTestsCommon` | `SystemTests/system_tests_common.py` | 33 | 1020 | Large but intentional base class | +| `TestScheduler` | `test_scheduler.py` | 28 | 1340 | Test creation + phase mgmt + submission + results | + +**Procedural modules (free functions taking `case`, no class):** + +| File | Lines | Funcs w/ `case` | Problem | +|------|------:|----------------:|---------| +| `case/case_st_archive.py` | 1395 | ~20 | Largest procedural module, all Case operations | +| `case/check_input_data.py` | 693 | ~12 | Scattered Case queries | +| `baselines/performance.py` | 612 | ~12 | Baseline comparison, all Case-dependent | +| `hist_utils.py` | 836 | ~7 | History file handling, all Case-dependent | + +These are addressed primarily in **Slice 2** (EnvBatch → `core/batch/`), +**Slice 3B** (build decomposition), and **Slice 4** (Case, TestScheduler, +and procedural case/ modules). + +### Cross-cutting issues (addressed primarily in Slice 1) + +These issues cut across all modules and should be fixed early: + +**Star imports** — `from CIME.XML.standard_module_setup import *` appears in +~60 source files, injecting `os`, `sys`, `re`, `expect`, `run_cmd`, etc. into +every namespace. `from CIME.test_status import *` appears in ~10 files, +exporting ~30 constants. These must be replaced with explicit imports to enable +static analysis and make dependencies visible. + +**Global mutable state** — `GLOBAL = {}` in `utils.py:21` is used to pass +`SRCROOT` between `case.py`, `create_test.py`, and `utils.get_src_root()` via +implicit mutation. Other globals include `_CIMECONFIG` (singleton config cache), +`_TIME_CACHE` (test scheduler cache), and `_ALL_TESTS` (test registry cache). +Replace with explicit parameter passing or instance state. + +**Import-time side effects** — `Servers/__init__.py` runs `shutil.which()` for +external tools at import time. `standard_script_setup.py` modifies `sys.path` +and `os.environ` at import time. `standard_module_setup.py` also modifies +`sys.path` at import time. These make imports non-deterministic and prevent +clean test isolation. + +**Inconsistent error handling** — 5 different patterns in use: `expect()`, +`raise CIMEError`, `sys.exit()`, `raise RuntimeError`, and +`logger.fatal()` + `sys.exit()`. Standardize on `expect()` / `CIMEError` +for library code; `sys.exit()` only in CLI entrypoints. + +**Circular imports** — The `Case` class injects methods from 10 sibling modules +at class body level (case.py:84-102) to work around circular dependencies. Other +circular imports are handled by deferred imports inside function bodies +(utils.py:456, env_batch.py:198). The `core/` reorganization should break +these cycles. + +**Test infrastructure** — `conftest.py` requires a valid `srcroot` with +`cime_config/` directory and calls `Machines()` which requires machine config +XML. This means unit tests cannot run standalone without host model integration. +Fix by making conftest machine-aware only for system tests, not unit tests. + +### Where DI / Protocols make sense +Use DI, protocols, and factory functions where CIME has **real polymorphism** +or **complex internal interfaces** worth decoupling: +- **Scheduler backends** (PBS, Slurm, LSF) — a `Scheduler` protocol with + concrete implementations is natural here +- **XML config loading** — swappable loaders for testing without real config files +- **Components that cross subsystem boundaries** — e.g. build orchestration + depending on a batch submitter interface + +### What we DON'T do +- Don't wrap stdlib behind Protocol classes — `os.path.exists()` doesn't need + a `FileSystem` abstraction; use `unittest.mock.patch` for testing +- Don't introduce DI frameworks or service locators +- Don't rewrite working code just to match a pattern +- Don't create abstractions without a concrete need (real polymorphism or + testability problem that `mock.patch` can't solve cleanly) + +### Testing +- Use `unittest.mock.patch` for mocking stdlib calls +- Use `tmp_path` / `tempfile` for filesystem tests +- Use `monkeypatch` for environment variables +- Protocols and constructor injection where the code genuinely benefits + (e.g. testing submission logic without a real scheduler) + +--- + +## Key Constraints + +### Python 3.9+ +All code must target Python 3.9 as the minimum supported version. Use +`typing.Protocol` (available since 3.8), `typing.Union`, `typing.Optional`, etc. +Do not use syntax that requires 3.10+ (e.g. `X | Y` union syntax, +`match`/`case` statements). + +### Non-Installed Package +CIME is not installed via pip. Cases create symlinked tools that modify +`sys.path`. This must continue to work. + +**Solution**: Centralized bootstrap in `CIME/core/config/bootstrap.py` + +### Missing Package Structure +`CIME/Tools/xmlconvertors/` and `CIME/ParamGen/` lack `__init__.py` files +and use try/except import fallbacks. Fix during Slice 1. + +### External Model Compatibility +E3SM, CESM, NorESM rely on CIME. Changes must not break their workflows. + +**Validation**: Test representative workflows from each model after major changes. + +### Stable Compatibility Surfaces +- `CIME/build_scripts/` — External script entrypoints +- Case-created symlinked tools +- Import paths (re-export from old locations during transition) + +--- + +## Migration Slices + +### Slice 1: Foundation (Weeks 1-3) +Typed exceptions, centralized bootstrap. Groundwork only. + +### Slice 2: Batch (Weeks 4-7) +Move batch/submit logic from `case_submit.py` and XML classes into +`CIME/core/batch/`. Existing entrypoints become thin wrappers. + +### Slice 3A: SRCROOT Standardization (Weeks 8-11) **NEW FEATURE** +Remove config_files.xml, standardize SRCROOT resolution, direct config loading. + +### Slice 3B: Build (Weeks 12-16) +Move build logic from `build.py` into `CIME/core/build/`. Keep +`build_scripts/` as stable wrappers. + +### Slice 4: Case (Weeks 17-22) +Decompose the `Case` god object (2600 lines). Phase 1: consolidate scattered +free functions (from `utils.py`, `build.py`, `status.py`, `locked_files.py`, +`case_run.py`) back into Case. Phase 2: extract subsystems (status, locking, +XML storage) into focused modules that Case delegates to. Case ends up smaller +than today with clear internal boundaries. + +--- + +## Success Criteria + +1. **Compatibility**: External models work without modification (or with documented migration) +2. **Testability**: 80%+ coverage for reorganized code +3. **Maintainability**: No module over ~500 lines; clear boundaries +4. **Documentation**: Updated docs for reorganized structure + +--- + +## Validation Requirements + +Each slice must pass: +- All existing CIME tests +- E3SM representative workflows +- CESM representative workflows +- NorESM representative workflows +- Performance benchmarks (no regression) + +--- + +## Non-Goals + +This refactor does NOT: +- Wrap stdlib in abstraction layers (`os.path`, `subprocess`, etc.) +- Introduce heavyweight DI frameworks or service locators +- Convert CIME to an installed package (yet) +- Rewrite working code for aesthetics +- Break Case API or build_scripts + +DI, protocols, and factory functions ARE used where CIME has genuine +polymorphism (scheduler backends, config loaders, etc.). + +--- + +## Documentation + +- **This file (plan.md)**: Architecture and principles +- **implementation.md**: Detailed tasks and timeline +- **feature_srcroot.md**: SRCROOT standardization feature spec + +--- + +## Questions? + +File issues in CIME repository or discuss in development meetings.