From ebec62ea01d52ce45ba47aaf196a982e04cc11d7 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Tue, 23 Nov 2021 18:33:07 -0700 Subject: [PATCH 1/4] First try importing buildnml in preview_namelists Proof of concept commit: see update_cime_config branch in https://github.com/mnlevy1981/POP2-CESM.git for companion component. If preview_namelist can import {compname}_cime_py, then the script will call buildnml() from this package instead of using run_sub_or_cmd(). To help find this package, the script will add each component's cime_config/ directory to the system path (and the POP branch above has a directory cime_config/pop_cime_py/ with a simple __init__.py file in it). Lots of improvements still needed: 1. Support putting buildnml.py in SourceMods/ and figure out a way to import it 2. Is there a cleaner way to ensure buildnml() is coming from where we expect it to be? 3. Does it make more sense to push some of this logic into run_sub_or_cmd()? --- scripts/lib/CIME/case/preview_namelists.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/scripts/lib/CIME/case/preview_namelists.py b/scripts/lib/CIME/case/preview_namelists.py index 2c588293ff6..25b7cdcff2d 100644 --- a/scripts/lib/CIME/case/preview_namelists.py +++ b/scripts/lib/CIME/case/preview_namelists.py @@ -6,6 +6,7 @@ from CIME.XML.standard_module_setup import * from CIME.utils import run_sub_or_cmd, safe_copy import time, glob +import importlib logger = logging.getLogger(__name__) def create_dirs(self): @@ -65,6 +66,7 @@ def create_namelists(self, component=None): # Create namelists - must have cpl last in the list below # Note - cpl must be last in the loop below so that in generating its namelist, # it can use xml vars potentially set by other component's buildnml scripts + comp_cime_py = dict() models = self.get_values("COMP_CLASSES") models += [models.pop(0)] for model in models: @@ -84,10 +86,22 @@ def create_namelists(self, component=None): else: # otherwise look in the component config_dir cmd = os.path.join(config_dir, "buildnml") - expect(os.path.isfile(cmd), "Could not find buildnml file for component {}".format(compname)) - logger.info("Create namelist for component {}".format(compname)) - run_sub_or_cmd(cmd, (caseroot), "buildnml", - (self, caseroot, compname), case=self) + # For now, only try to import if not using SourceMods + try: + # can we import buildnml? + sys.path.append(config_dir) + comp_cime_py[compname] = importlib.import_module(f"{compname}_cime_py") + except: + pass + # If successfully imported {compname}_cime_py, call buildnml(), + # otherwise just run buildnml via run_sub_or_cmd() + if compname in comp_cime_py: + comp_cime_py[compname].buildnml(self, caseroot, compname) + else: + expect(os.path.isfile(cmd), "Could not find buildnml file for component {}".format(compname)) + logger.info("Create namelist for component {}".format(compname)) + run_sub_or_cmd(cmd, (caseroot), "buildnml", + (self, caseroot, compname), case=self) logger.debug("Finished creating component namelists, component {} models = {}".format(component, models)) From bf6852dba563ce1b09afa832bf3e3ebc5601be7f Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Thu, 2 Dec 2021 14:58:57 -0700 Subject: [PATCH 2/4] More generalized implementation The previous commit added logic to preview_namelists.py to specifically allow the import of buildnml if a component was set up correctly; this commit reverts preview_namelist to what is on master, but adds a wrapper to run_sub_or_cmd() in util.py that attempts to import and run the command before falling back to the old run_sub_or_cmd() function. preview_namelists.py has been updated to use this new import_and_run_sub_or_cmd() function (which perhaps could use a better name) --- scripts/lib/CIME/case/preview_namelists.py | 23 ++++--------------- scripts/lib/CIME/utils.py | 26 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/scripts/lib/CIME/case/preview_namelists.py b/scripts/lib/CIME/case/preview_namelists.py index 25b7cdcff2d..3ab10251471 100644 --- a/scripts/lib/CIME/case/preview_namelists.py +++ b/scripts/lib/CIME/case/preview_namelists.py @@ -4,9 +4,8 @@ """ from CIME.XML.standard_module_setup import * -from CIME.utils import run_sub_or_cmd, safe_copy +from CIME.utils import import_and_run_sub_or_cmd, safe_copy import time, glob -import importlib logger = logging.getLogger(__name__) def create_dirs(self): @@ -66,7 +65,6 @@ def create_namelists(self, component=None): # Create namelists - must have cpl last in the list below # Note - cpl must be last in the loop below so that in generating its namelist, # it can use xml vars potentially set by other component's buildnml scripts - comp_cime_py = dict() models = self.get_values("COMP_CLASSES") models += [models.pop(0)] for model in models: @@ -86,22 +84,9 @@ def create_namelists(self, component=None): else: # otherwise look in the component config_dir cmd = os.path.join(config_dir, "buildnml") - # For now, only try to import if not using SourceMods - try: - # can we import buildnml? - sys.path.append(config_dir) - comp_cime_py[compname] = importlib.import_module(f"{compname}_cime_py") - except: - pass - # If successfully imported {compname}_cime_py, call buildnml(), - # otherwise just run buildnml via run_sub_or_cmd() - if compname in comp_cime_py: - comp_cime_py[compname].buildnml(self, caseroot, compname) - else: - expect(os.path.isfile(cmd), "Could not find buildnml file for component {}".format(compname)) - logger.info("Create namelist for component {}".format(compname)) - run_sub_or_cmd(cmd, (caseroot), "buildnml", - (self, caseroot, compname), case=self) + logger.info("Create namelist for component {}".format(compname)) + import_and_run_sub_or_cmd(cmd, (caseroot), "buildnml", + (self, caseroot, compname), config_dir, compname, case=self) logger.debug("Finished creating component namelists, component {} models = {}".format(component, models)) diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index 398f19776a1..648bf8a0e9b 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -422,6 +422,28 @@ def file_contains_python_function(filepath, funcname): return has_function +def import_and_run_sub_or_cmd(cmd, cmdargs, subname, subargs, config_dir, compname, + logfile=None, case=None, from_dir=None, timeout=None): + sys_path_old = sys.path + sys.path.insert(1, config_dir) + try: + mod = importlib.import_module(f"{compname}_cime_py") + getattr(mod, subname)(*subargs) + except (ModuleNotFoundError, AttributeError) as _: + # * ModuleNotFoundError if importlib can not find module, + # * AttributeError if importlib finds the module but + # {subname} is not defined in the module + expect(os.path.isfile(cmd), f"Could not find {subname} file for component {compname}") + run_sub_or_cmd(cmd, cmdargs, subname, subargs, logfile, case, from_dir, timeout) + except Exception: + if logfile: + with open(logfile, "a") as log_fd: + log_fd.write(str(sys.exc_info()[1])) + expect(False, "{} FAILED, cat {}".format(cmd, logfile)) + else: + raise + sys.path = sys_path_old + def run_sub_or_cmd(cmd, cmdargs, subname, subargs, logfile=None, case=None, from_dir=None, timeout=None): """ @@ -430,6 +452,10 @@ def run_sub_or_cmd(cmd, cmdargs, subname, subargs, logfile=None, case=None, Raises exception on failure. """ +# Potential strategy: +# 1. Can we run this function by importing package in cime_config/? +# 2. If not, fall back to old behavior (either import function from file +# or just run the script) if file_contains_python_function(cmd, subname): do_run_cmd = False else: From daf06db950c6262df9c62f7a1090ccbdd806a06d Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Tue, 14 Dec 2021 15:21:02 -0700 Subject: [PATCH 3/4] Remove support for buildnml in SourceMods --- scripts/lib/CIME/case/preview_namelists.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/scripts/lib/CIME/case/preview_namelists.py b/scripts/lib/CIME/case/preview_namelists.py index 3ab10251471..90addfe06ed 100644 --- a/scripts/lib/CIME/case/preview_namelists.py +++ b/scripts/lib/CIME/case/preview_namelists.py @@ -77,13 +77,7 @@ def create_namelists(self, component=None): else: compname = self.get_value("COMP_{}".format(model_str.upper())) if component is None or component == model_str or compname=="ufsatm": - # first look in the case SourceMods directory - cmd = os.path.join(caseroot, "SourceMods", "src."+compname, "buildnml") - if os.path.isfile(cmd): - logger.warning("\nWARNING: Using local buildnml file {}\n".format(cmd)) - else: - # otherwise look in the component config_dir - cmd = os.path.join(config_dir, "buildnml") + cmd = os.path.join(config_dir, "buildnml") logger.info("Create namelist for component {}".format(compname)) import_and_run_sub_or_cmd(cmd, (caseroot), "buildnml", (self, caseroot, compname), config_dir, compname, case=self) From 04d76ee75c0b9e269c0032640d27b2fbeea9007f Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Wed, 15 Dec 2021 10:56:46 -0700 Subject: [PATCH 4/4] Remove some temporary comments I had added comments reminding myself of a couple of different approaches to try for implementing the features on this branch. Now that at path has been chosen, those notes are not necessary. --- scripts/lib/CIME/utils.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index 648bf8a0e9b..4bc320b6426 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -452,10 +452,6 @@ def run_sub_or_cmd(cmd, cmdargs, subname, subargs, logfile=None, case=None, Raises exception on failure. """ -# Potential strategy: -# 1. Can we run this function by importing package in cime_config/? -# 2. If not, fall back to old behavior (either import function from file -# or just run the script) if file_contains_python_function(cmd, subname): do_run_cmd = False else: