From f1618bfa2631cf6eda75da89e71891ba300e97ac Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 17 Feb 2026 16:46:01 -0700 Subject: [PATCH 01/55] added spack source mirrors capability, needs testing --- stackinator/builder.py | 2 ++ stackinator/schema/config.json | 7 +++++++ stackinator/templates/Makefile | 10 ++++++++++ 3 files changed, 19 insertions(+) diff --git a/stackinator/builder.py b/stackinator/builder.py index 2c05e3c1..c3de44db 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -232,6 +232,8 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, + # pass source_mirrors to Makefile render + source_mirrors=recipe.config.get("source_mirrors", {}), exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index d6fec3a0..4b91011d 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -64,6 +64,13 @@ } } }, + "source_mirrors" : { + "type" : "object", + "additionalProperties": { + "type" : "string" + }, + "default": {} + }, "modules" : { "type": "boolean" }, diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 10a0ea58..3da2e00e 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -39,6 +39,16 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} {% endif %} {% endif %} + {% if source_mirrors %} + echo "Replacing all instances of mirror.spack.io... Just in case" + grep -rl "https://mirror.spack.io" . | xargs sed -i 's/https:\/\/mirror.spack.io/https:\/\/pe-serve.lanl.gov\/spack-mirror/g' + echo "Adding mirrors" + {% for name, url in source_mirrors.items() %} + $(SANDBOX) $(SPACK) mirror add {{ name }} {{ url }} + {% endfor %} + echo "Current mirror list:" + spack mirror list + {% endif %} touch mirror-setup compilers: mirror-setup From 954a6901a6544ef6e190bea4cdfc8ada4b6acffe Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 18 Feb 2026 14:34:05 -0700 Subject: [PATCH 02/55] removed lanl stuff --- stackinator/templates/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 3da2e00e..fcf6d68a 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -40,13 +40,13 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% endif %} {% endif %} {% if source_mirrors %} - echo "Replacing all instances of mirror.spack.io... Just in case" - grep -rl "https://mirror.spack.io" . | xargs sed -i 's/https:\/\/mirror.spack.io/https:\/\/pe-serve.lanl.gov\/spack-mirror/g' + echo "Removing all instances of mirror.spack.io... Just in case" + grep -rl "https://mirror.spack.io" . | xargs sed -i 's|https://mirror.spack.io||g' echo "Adding mirrors" {% for name, url in source_mirrors.items() %} $(SANDBOX) $(SPACK) mirror add {{ name }} {{ url }} {% endfor %} - echo "Current mirror list:" + echo "Spack mirrors for this recipe:" spack mirror list {% endif %} touch mirror-setup From e2c646267a6e61a41ebe64b0b538609897ac4423 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 24 Feb 2026 10:43:33 -0700 Subject: [PATCH 03/55] add source mirrors via config.yaml and retain spack default mirror --- stackinator/templates/Makefile | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index fcf6d68a..1d94c15a 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -34,22 +34,20 @@ pre-install: spack-setup mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% if cache %} - $(SANDBOX) $(SPACK) buildcache keys --install --trust - {% if cache.key %} - $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} - {% endif %} - {% endif %} - {% if source_mirrors %} - echo "Removing all instances of mirror.spack.io... Just in case" - grep -rl "https://mirror.spack.io" . | xargs sed -i 's|https://mirror.spack.io||g' - echo "Adding mirrors" - {% for name, url in source_mirrors.items() %} - $(SANDBOX) $(SPACK) mirror add {{ name }} {{ url }} - {% endfor %} - echo "Spack mirrors for this recipe:" - spack mirror list - {% endif %} - touch mirror-setup + $(SANDBOX) $(SPACK) buildcache keys --install --trust + {% if cache.key %} + $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} + {% endif %} + {% endif %} + {% if source_mirrors %} + @echo "Adding mirrors" + {% for name, url in source_mirrors.items() | reverse %} + $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} + {% endfor %} + @echo "Current mirror list:" + $(SANDBOX) $(SPACK) mirror list + {% endif %} + touch mirror-setup compilers: mirror-setup $(SANDBOX) $(MAKE) -C $@ From 69f9bb69df6fc700cc23d1976ef729059c1f74fd Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 24 Feb 2026 16:04:05 -0700 Subject: [PATCH 04/55] fixed spaces/tabs typo --- stackinator/templates/Makefile | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 1d94c15a..ca8fb311 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -32,22 +32,22 @@ pre-install: spack-setup $(SANDBOX) $(STORE)/pre-install-hook mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - + {% if cache %} - $(SANDBOX) $(SPACK) buildcache keys --install --trust - {% if cache.key %} - $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} - {% endif %} - {% endif %} - {% if source_mirrors %} - @echo "Adding mirrors" - {% for name, url in source_mirrors.items() | reverse %} - $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} - {% endfor %} - @echo "Current mirror list:" - $(SANDBOX) $(SPACK) mirror list - {% endif %} - touch mirror-setup + $(SANDBOX) $(SPACK) buildcache keys --install --trust + {% if cache.key %} + $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} + {% endif %} + {% endif %} + {% if source_mirrors %} + @echo "Adding mirrors" + {% for name, url in source_mirrors.items() | reverse %} + $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} + {% endfor %} + @echo "Current mirror list:" + $(SANDBOX) $(SPACK) mirror list + {% endif %} + touch mirror-setup compilers: mirror-setup $(SANDBOX) $(MAKE) -C $@ From 8b392c258eb75249eefbb89d5858f84a34d4416f Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 09:16:51 -0700 Subject: [PATCH 05/55] Added mirror configuration json schema. --- stackinator/schema/config.json | 7 ------- stackinator/schema/mirror.json | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 stackinator/schema/mirror.json diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index 4b91011d..d6fec3a0 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -64,13 +64,6 @@ } } }, - "source_mirrors" : { - "type" : "object", - "additionalProperties": { - "type" : "string" - }, - "default": {} - }, "modules" : { "type": "boolean" }, diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json new file mode 100644 index 00000000..b32a3cd1 --- /dev/null +++ b/stackinator/schema/mirror.json @@ -0,0 +1,37 @@ +# This config handles source mirrors, binary caches, and bootstrap mirrors (of both forms) +{ + # Order matters, so we need an array. + "type" : "array", + "items": { + "type": "object", + "required": ["name", "url"] + "properties": { + "name": { + "type": "string", + "description": "The name of this mirror. Should be follow standard variable naming syntax.", + }, + "url": { + "type": "string", + "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI).", + }, + "enabled": { + "type": "boolean", + "default": true, + "description": "Whether this mirror is enabled.", + }, + "bootstrap": { + "type": "boolean", + "default": false, + "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror.", + } + "public_key": { + "type": "string", + "description": "Public PGP key for validating binary cache packages.", + }, + "description": { + "type": "string", + "description": "What this mirror is for." + } + } + } +} From dba9698cd781953a4fe1d938f763ec303c588986 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 09:48:06 -0700 Subject: [PATCH 06/55] Incorporating Makefile changes. --- stackinator/templates/Makefile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index ca8fb311..d1a06dd4 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -32,21 +32,25 @@ pre-install: spack-setup $(SANDBOX) $(STORE)/pre-install-hook mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - {% if cache %} + # The old way of managing mirrors $(SANDBOX) $(SPACK) buildcache keys --install --trust {% if cache.key %} $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} {% endif %} {% endif %} - {% if source_mirrors %} - @echo "Adding mirrors" - {% for name, url in source_mirrors.items() | reverse %} - $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} + {% if mirrors %} + @echo "Adding mirrors and gpg keys." + {% for mirror_info in mirrors | reverse %} + $(SANDBOX) $(SPACK) mirror add --scope=site {{ mirror_info.name }} {{ mirror_info.url }} + $(SANDBOX) $(SPACK) gpg trust {{ mirror_info.key_path }} {% endfor %} @echo "Current mirror list:" $(SANDBOX) $(SPACK) mirror list {% endif %} + {% for mirror_info in filter(lambda m: m['bootstrap'], mirrors) | filter() %} + $(SANDBOX) $(SPACK) bootstrap add --scope=site {{ mirror_info.name }} bootstrap/{{ mirror_info.name }} + {% endfor %} touch mirror-setup compilers: mirror-setup From b0a8071eb668af9712ad21c7b1e11b2f32306fd2 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 10:05:24 -0700 Subject: [PATCH 07/55] mirrors --- stackinator/mirror.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 stackinator/mirror.py diff --git a/stackinator/mirror.py b/stackinator/mirror.py new file mode 100644 index 00000000..e69de29b From 6a044ae4ffaee02c903675edfb4f469d16ff371a Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 10:26:59 -0700 Subject: [PATCH 08/55] mirrors --- stackinator/mirror.py | 58 +++++++++++++++++++++++++++++++++++++++++++ stackinator/recipe.py | 32 ++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e69de29b..83989b83 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -0,0 +1,58 @@ +import os +import pathlib + +import yaml + +from . import schema + + +def configuration_from_file(file, mount): + with file.open() as fid: + # load the raw yaml input + raw = yaml.load(fid, Loader=yaml.Loader) + + # validate the yaml + schema.CacheValidator.validate(raw) + + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(raw["root"])) + if not path.is_absolute(): + raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + if not path.is_dir(): + raise FileNotFoundError(f"The build cache path '{path}' does not exist") + + raw["root"] = path + + # Put the build cache in a sub-directory named after the mount point. + # This avoids relocation issues. + raw["path"] = pathlib.Path(path.as_posix() + mount.as_posix()) + + # verify that the key file exists if it was specified + key = raw["key"] + if key is not None: + key = pathlib.Path(os.path.expandvars(key)) + if not key.is_absolute(): + raise FileNotFoundError(f"The build cache key '{key}' is not absolute") + if not key.is_file(): + raise FileNotFoundError(f"The build cache key '{key}' does not exist") + raw["key"] = key + + return raw + + +def generate_mirrors_yaml(config): + path = config["path"].as_posix() + mirrors = { + "mirrors": { + "alpscache": { + "fetch": { + "url": f"file://{path}", + }, + "push": { + "url": f"file://{path}", + }, + } + } + } + + return yaml.dump(mirrors, default_flow_style=False) \ No newline at end of file diff --git a/stackinator/recipe.py b/stackinator/recipe.py index ca8d2b3d..d0ec018f 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -170,14 +170,20 @@ def __init__(self, args): self.generate_environment_specs(raw) # optional mirror configurtion + if mirrors_path.is_file():zx mirrors_path = self.path / "mirrors.yaml" - if mirrors_path.is_file(): self._logger.warning( "mirrors.yaml have been removed from recipes, use the --cache option on stack-config instead." ) raise RuntimeError("Unsupported mirrors.yaml file in recipe.") - self.mirror = (args.cache, self.mount) + # self.mirror = (args.cache, self.mount) + + # load the optional mirrors.yaml from system config: + mirrors_path = self.system_config_path / "mirrors.yaml" + if mirrors_path.is_file(): + self.mirrors = (mirrors_path, self.mount) + # update mirror setter and cache.configuration_from_file() # optional post install hook if self.post_install_hook is not None: @@ -262,6 +268,28 @@ def mirror(self, configuration): self._mirror = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) + @property + def mirrors(self): + return self._mirrors + + # old: self.mirror = (args.cache, self.mount) + # new: self.mirror = (mirrors_yaml_path, self.mount) + + @mirrors.setter + def (self, configuration): + self._logger.debug(f"configuring mirrors with {configuration}") + self._mirrors = None + + file, mount = configuration + + if file is not None: + mirror_config_path = pathlib.Path(file) + if not mirror_config_path.is_file(): + raise FileNotFoundError(f"The mirror configuration '{file}' is not a file") + + self._mirrors = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) + + @property def config(self): return self._config From 0ad5022265a1b95081202aea6204d6166649c7c6 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 10:40:10 -0700 Subject: [PATCH 09/55] validate mirror config --- stackinator/mirror.py | 47 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 83989b83..be784864 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,5 +1,6 @@ import os import pathlib +import urllib.request import yaml @@ -14,30 +15,28 @@ def configuration_from_file(file, mount): # validate the yaml schema.CacheValidator.validate(raw) - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(raw["root"])) - if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") - - raw["root"] = path - - # Put the build cache in a sub-directory named after the mount point. - # This avoids relocation issues. - raw["path"] = pathlib.Path(path.as_posix() + mount.as_posix()) - - # verify that the key file exists if it was specified - key = raw["key"] - if key is not None: - key = pathlib.Path(os.path.expandvars(key)) - if not key.is_absolute(): - raise FileNotFoundError(f"The build cache key '{key}' is not absolute") - if not key.is_file(): - raise FileNotFoundError(f"The build cache key '{key}' does not exist") - raw["key"] = key - - return raw + mirrors = [mirror for mirror in raw if mirror["enabled"]] + + for mirror in mirrors: + url = mirror["url"] + if url.beginswith("file://"): + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(url)) + if not path.is_absolute(): + raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + if not path.is_dir(): + raise FileNotFoundError(f"The build cache path '{path}' does not exist") + + mirror["url"] = path + + else: + try: + request = urllib.request.Request(url, method='HEAD') + response = urllib.request.urlopen(request) + except urllib.error.URLError as e: + print(f'Error: {e.reason}') + + return mirrors def generate_mirrors_yaml(config): From 3036013f07303966ac536b59b510aa7f0f7ec64c Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 11:01:41 -0700 Subject: [PATCH 10/55] Updating recipe to handle new mirrors format. --- stackinator/mirror.py | 4 +-- stackinator/recipe.py | 72 ++++++++++--------------------------------- 2 files changed, 19 insertions(+), 57 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index be784864..08c303bb 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -7,7 +7,7 @@ from . import schema -def configuration_from_file(file, mount): +def configuration_from_file(file): with file.open() as fid: # load the raw yaml input raw = yaml.load(fid, Loader=yaml.Loader) @@ -54,4 +54,4 @@ def generate_mirrors_yaml(config): } } - return yaml.dump(mirrors, default_flow_style=False) \ No newline at end of file + return yaml.dump(mirrors, default_flow_style=False) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index d0ec018f..281ab8b2 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -4,8 +4,9 @@ import jinja2 import yaml +from typing import Optional -from . import cache, root_logger, schema, spack_util +from . import cache, root_logger, schema, spack_util, mirror from .etc import envvars @@ -169,21 +170,11 @@ def __init__(self, args): schema.EnvironmentsValidator.validate(raw) self.generate_environment_specs(raw) - # optional mirror configurtion - if mirrors_path.is_file():zx - mirrors_path = self.path / "mirrors.yaml" - self._logger.warning( - "mirrors.yaml have been removed from recipes, use the --cache option on stack-config instead." - ) - raise RuntimeError("Unsupported mirrors.yaml file in recipe.") - - # self.mirror = (args.cache, self.mount) + mirrors_path = self.system_config_path/'mirrors.yaml' + self._logger.debug(f"opening {mirrors_path}") # load the optional mirrors.yaml from system config: - mirrors_path = self.system_config_path / "mirrors.yaml" - if mirrors_path.is_file(): - self.mirrors = (mirrors_path, self.mount) - # update mirror setter and cache.configuration_from_file() + self.mirrors = self.system_config_path / "mirrors.yaml" # optional post install hook if self.post_install_hook is not None: @@ -242,54 +233,25 @@ def pre_install_hook(self): return hook_path return None - # Returns a dictionary with the following fields - # - # root: /path/to/cache - # path: /path/to/cache/user-environment - # key: /path/to/private-pgp-key @property - def mirror(self): - return self._mirror + def mirrors(self): + return self._mirrors # configuration is a tuple with two fields: # - a Path of the yaml file containing the cache configuration # - the mount point of the image - @mirror.setter - def mirror(self, configuration): - self._logger.debug(f"configuring build cache mirror with {configuration}") - self._mirror = None - - file, mount = configuration - - if file is not None: - mirror_config_path = pathlib.Path(file) - if not mirror_config_path.is_file(): - raise FileNotFoundError(f"The cache configuration '{file}' is not a file") - - self._mirror = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) - - @property - def mirrors(self): - return self._mirrors - - # old: self.mirror = (args.cache, self.mount) - # new: self.mirror = (mirrors_yaml_path, self.mount) - @mirrors.setter - def (self, configuration): - self._logger.debug(f"configuring mirrors with {configuration}") + def mirrors(self, path: Optional[pathlib.Path]): + """Initialize the mirrors property from config.""" self._mirrors = None + if path is not None: + if not path.is_file(): + raise FileNotFoundError("The system config 'mirrors.yaml' file exists, but isn't a " + "readable file.") - file, mount = configuration - - if file is not None: - mirror_config_path = pathlib.Path(file) - if not mirror_config_path.is_file(): - raise FileNotFoundError(f"The mirror configuration '{file}' is not a file") - - self._mirrors = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) - - + self._logger.debug(f"configuring mirrors from {path}") + self._mirrors = mirror.configuration_from_file(path) + @property def config(self): return self._config @@ -569,7 +531,7 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.mirror is not None + push_to_cache = self.mirrors files["makefile"] = makefile_template.render( compilers=self.compilers, push_to_cache=push_to_cache, From a1c486d7c9d6801533f544f0b6780e474392c3aa Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 11:35:42 -0700 Subject: [PATCH 11/55] Updating mirror configuration more. --- stackinator/main.py | 16 ++++--- stackinator/mirror.py | 79 +++++++++++++++++++++++----------- stackinator/recipe.py | 27 +++--------- stackinator/schema/mirror.json | 7 ++- 4 files changed, 78 insertions(+), 51 deletions(-) diff --git a/stackinator/main.py b/stackinator/main.py index 44406215..6f30ddfc 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -81,13 +81,19 @@ def log_header(args): def make_argparser(): parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from a recipe.")) parser.add_argument("--version", action="version", version=f"stackinator version {VERSION}") - parser.add_argument("-b", "--build", required=True, type=str) + parser.add_argument("-b", "--build", required=True, type=str, + help="Where to set up the stackinator build directory. " + "('/tmp' is not allowed, use '/var/tmp'") parser.add_argument("--no-bwrap", action="store_true", required=False) - parser.add_argument("-r", "--recipe", required=True, type=str) - parser.add_argument("-s", "--system", required=True, type=str) + parser.add_argument("-r", "--recipe", required=True, type=str, + help="Name of (and/or path to) the Stackinator recipe.") + parser.add_argument("-s", "--system", required=True, type=str, + help="Name of (and/or path to) the Stackinator system configuration.") parser.add_argument("-d", "--debug", action="store_true") - parser.add_argument("-m", "--mount", required=False, type=str) - parser.add_argument("-c", "--cache", required=False, type=str) + parser.add_argument("-m", "--mount", required=False, type=str, + help="The mount point where the environment will be located.") + parser.add_argument("-c", "--cache", required=False, type=str, + help="Buildcache location or name (from system config's mirrors.yaml).") parser.add_argument("--develop", action="store_true", required=False) return parser diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 08c303bb..40487f48 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,42 +1,73 @@ import os import pathlib import urllib.request +from typing import Optional import yaml from . import schema -def configuration_from_file(file): - with file.open() as fid: - # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) +def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = None): + """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + + if path.exists(): + with path.open() as fid: + # load the raw yaml input + raw = yaml.load(fid, Loader=yaml.Loader) + + print(f"Configuring mirrors and buildcache from '{path}'") # validate the yaml schema.CacheValidator.validate(raw) mirrors = [mirror for mirror in raw if mirror["enabled"]] + else: + mirrors = [] + + buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) + if buildcache_dest_count > 1: + raise RuntimeError("Mirror config has more than one mirror specified as the build cache destination " + "in the system config's 'mirrors.yaml'.") + elif buildcache_dest_count == 1 and cmdline_cache: + raise RuntimeError("Build cache destination specified on the command line and in the system config's " + "'mirrors.yaml'. It can be one or the other, but not both.") + + # Add or set the cache given on the command line as the buildcache destination + if cmdline_cache is not None: + existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] + # If the mirror name given on the command line isn't in the config, assume it + # is the URL to a build cache. + if not existing_mirror: + mirrors.append( + { + 'name': 'cmdline_cache', + 'url': cmdline_cache, + 'buildcache': True, + 'bootstrap': False, + } + ) + + for mirror in mirrors: + url = mirror["url"] + if url.beginswith("file://"): + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(url)) + if not path.is_absolute(): + raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + if not path.is_dir(): + raise FileNotFoundError(f"The build cache path '{path}' does not exist") + + mirror["url"] = path + + else: + try: + request = urllib.request.Request(url, method='HEAD') + response = urllib.request.urlopen(request) + except urllib.error.URLError as e: + print(f'Error: {e.reason}') - for mirror in mirrors: - url = mirror["url"] - if url.beginswith("file://"): - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(url)) - if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") - - mirror["url"] = path - - else: - try: - request = urllib.request.Request(url, method='HEAD') - response = urllib.request.urlopen(request) - except urllib.error.URLError as e: - print(f'Error: {e.reason}') - - return mirrors + return mirrors def generate_mirrors_yaml(config): diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 281ab8b2..4b7c038c 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -170,11 +170,11 @@ def __init__(self, args): schema.EnvironmentsValidator.validate(raw) self.generate_environment_specs(raw) - mirrors_path = self.system_config_path/'mirrors.yaml' - self._logger.debug(f"opening {mirrors_path}") - - # load the optional mirrors.yaml from system config: - self.mirrors = self.system_config_path / "mirrors.yaml" + # load the optional mirrors.yaml from system config, and add any additional + # mirrors specified on the command line. + self._mirrors = None + self._logger.debug("Configuring mirrors.") + self._mirrors = mirror.configuration_from_file(self.system_config_path/"mirrors.yaml", args.cache) # optional post install hook if self.post_install_hook is not None: @@ -236,22 +236,7 @@ def pre_install_hook(self): @property def mirrors(self): return self._mirrors - - # configuration is a tuple with two fields: - # - a Path of the yaml file containing the cache configuration - # - the mount point of the image - @mirrors.setter - def mirrors(self, path: Optional[pathlib.Path]): - """Initialize the mirrors property from config.""" - self._mirrors = None - if path is not None: - if not path.is_file(): - raise FileNotFoundError("The system config 'mirrors.yaml' file exists, but isn't a " - "readable file.") - - self._logger.debug(f"configuring mirrors from {path}") - self._mirrors = mirror.configuration_from_file(path) - + @property def config(self): return self._config diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index b32a3cd1..a53cc34e 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -23,7 +23,12 @@ "type": "boolean", "default": false, "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror.", - } + }, + "buildcache": { + "type": "boolean", + "default": false, + "description": "Use this mirror as the buildcache push destination. Can only be enabled on a single mirror." + }, "public_key": { "type": "string", "description": "Public PGP key for validating binary cache packages.", From 2b606830c4daf0305b19fe7587766cd4c034353b Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 11:25:37 -0700 Subject: [PATCH 12/55] mirror yaml generator --- stackinator/mirror.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 40487f48..315d2e8e 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -70,19 +70,16 @@ def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = N return mirrors -def generate_mirrors_yaml(config): - path = config["path"].as_posix() - mirrors = { - "mirrors": { - "alpscache": { - "fetch": { - "url": f"file://{path}", - }, - "push": { - "url": f"file://{path}", - }, - } +def generate_mirrors_yaml(mirrors): + yaml = {"mirrors": {}} + + for m in mirrors: + name = m["name"] + url = m["url"] + + yaml["mirrors"][name] = { + "fetch": {"url": url}, + "push": {"url": url}, } - } - return yaml.dump(mirrors, default_flow_style=False) + return yaml.dump(yaml, default_flow_style=False) \ No newline at end of file From 095862c37402394424991963f83ed7dbe6bca896 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 11:37:34 -0700 Subject: [PATCH 13/55] update mirrors --- stackinator/builder.py | 10 ++++------ stackinator/mirror.py | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index c3de44db..5b223ffe 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -226,14 +226,12 @@ def generate(self, recipe): with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( - cache=recipe.mirror, modules=recipe.with_modules, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - # pass source_mirrors to Makefile render - source_mirrors=recipe.config.get("source_mirrors", {}), + mirrors=recipe.mirrors exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) @@ -314,11 +312,11 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured - if recipe.mirror: + if recipe.mirrors: dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the build cache mirror: {dst}") + self._logger.debug(f"generate the spack mirrors.yaml: {dst}") with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirror)) + fid.write(cache.generate_mirrors_yaml(recipe.mirrors)) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 315d2e8e..e802018a 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -67,7 +67,20 @@ def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = N except urllib.error.URLError as e: print(f'Error: {e.reason}') - return mirrors + if mirror["key"]: + #if path, check if exists + path = pathlib.Path(os.path.expandvars(mirror["key"])) + if path.exists(): + if not path.is_file(): + raise FileNotFoundError(f"The key path '{path}' is not a file") + else + #if key, save to file, change to path + + if mirror["bootstrap"]: + #make bootstrap dirs + #bootstrap//metadata.yaml + + return mirrors def generate_mirrors_yaml(mirrors): From 31a16f1d6550a545c625d779afdacc138cfa1e68 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Sat, 7 Mar 2026 00:54:59 -0700 Subject: [PATCH 14/55] validate keys in mirror config and fixed yaml generator --- stackinator/mirror.py | 74 ++++++++++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e802018a..afbacafb 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -2,15 +2,21 @@ import pathlib import urllib.request from typing import Optional +import magic import yaml from . import schema +class MirrorConfigError(RuntimeError): + """Exception class for errors thrown by mirror configuration problems.""" -def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = None): + + +def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + path = system_config_root/"mirrors.yaml" if path.exists(): with path.open() as fid: # load the raw yaml input @@ -54,36 +60,33 @@ def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = N # verify that the root path exists path = pathlib.Path(os.path.expandvars(url)) if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + raise FileNotFoundError(f"The mirror path '{path}' is not absolute") if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") + raise FileNotFoundError(f"The mirror path '{path}' does not exist") mirror["url"] = path - else: + elif url.beginswith("https://"): try: request = urllib.request.Request(url, method='HEAD') response = urllib.request.urlopen(request) except urllib.error.URLError as e: - print(f'Error: {e.reason}') - - if mirror["key"]: - #if path, check if exists - path = pathlib.Path(os.path.expandvars(mirror["key"])) - if path.exists(): - if not path.is_file(): - raise FileNotFoundError(f"The key path '{path}' is not a file") - else - #if key, save to file, change to path + raise MirrorConfigError( + f"Could not reach the mirror url '{url}'. " + f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - if mirror["bootstrap"]: - #make bootstrap dirs - #bootstrap//metadata.yaml + if mirror["bootstrap"]: + #make bootstrap dirs + #bootstrap//metadata.yaml return mirrors -def generate_mirrors_yaml(mirrors): +def setup(mirrors, config_path): + dst = config_path / "mirrors.yaml" + self._logger.debug(f"generate the spack mirrors.yaml: {dst}") + with dst.open("w") as fid: + fid.write() yaml = {"mirrors": {}} for m in mirrors: @@ -95,4 +98,37 @@ def generate_mirrors_yaml(mirrors): "push": {"url": url}, } - return yaml.dump(yaml, default_flow_style=False) \ No newline at end of file + return yaml.dump(yaml, default_flow_style=False) + +#called from builder +def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: pathlib.Path): + for mirror in mirrors: + if mirror["key"]: + key = mirror["key"] + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if path.exists(): + if not path.is_absolute(): + #try prepending system config path + path = system_config_path + path + if not.path.is_file() + raise FileNotFoundError( + f"The key path '{path}' is not a file. " + f"Check the key listed in mirrors.yaml in system config.") + file_type = magic.from_file(path) + if not file_type.startswith("OpenPGP Public Key"): + raise MirrorConfigError( + f"'{key}' is not a valid GPG key. " + f"Check the key listed in mirrors.yaml in system config.") + # copy file to key store + with file open: + data = key.read + dest = mkdir(new_key_file) + dest.write(data) + # mirror["key"] = new_path + + else: + # if PGP key, convert to binary, ???, convert back + # if key, save to file, change to path + + \ No newline at end of file From 9ef113dca30631628a59da1f3e814e230579de3e Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Sat, 7 Mar 2026 00:57:53 -0700 Subject: [PATCH 15/55] validate keys in mirror config and fixed yaml generator --- stackinator/mirror.py | 44 +++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index afbacafb..e521b597 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -12,7 +12,6 @@ class MirrorConfigError(RuntimeError): """Exception class for errors thrown by mirror configuration problems.""" - def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" @@ -62,7 +61,7 @@ def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Opt if not path.is_absolute(): raise FileNotFoundError(f"The mirror path '{path}' is not absolute") if not path.is_dir(): - raise FileNotFoundError(f"The mirror path '{path}' does not exist") + raise FileNotFoundError(f"The mirror path '{path}' is not a directory") mirror["url"] = path @@ -82,11 +81,12 @@ def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Opt return mirrors -def setup(mirrors, config_path): +def yaml_setup(mirrors, config_path): + """Generate the mirrors.yaml for spack""" + dst = config_path / "mirrors.yaml" + self._logger.debug(f"generate the spack mirrors.yaml: {dst}") - with dst.open("w") as fid: - fid.write() yaml = {"mirrors": {}} for m in mirrors: @@ -98,13 +98,22 @@ def setup(mirrors, config_path): "push": {"url": url}, } - return yaml.dump(yaml, default_flow_style=False) + with dst.open("w") as file: + yaml.dump(yaml, default_flow_style=False) + + # return dst + -#called from builder def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: pathlib.Path): + """Validate mirror keys, relocate to key_store, and update mirror config with new key paths""" + for mirror in mirrors: if mirror["key"]: key = mirror["key"] + + # key will be saved under key_store/mirror_name.gpg + dst = (key_store / f"'{mirror["name"]}'.gpg").resolve() + # if path, check if abs path, if not, append sys config path in front and check again path = pathlib.Path(os.path.expandvars(key)) if path.exists(): @@ -115,20 +124,23 @@ def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: raise FileNotFoundError( f"The key path '{path}' is not a file. " f"Check the key listed in mirrors.yaml in system config.") + file_type = magic.from_file(path) + if not file_type.startswith("OpenPGP Public Key"): raise MirrorConfigError( - f"'{key}' is not a valid GPG key. " + f"'{path}' is not a valid GPG key. " f"Check the key listed in mirrors.yaml in system config.") - # copy file to key store - with file open: - data = key.read - dest = mkdir(new_key_file) - dest.write(data) - # mirror["key"] = new_path + + # copy key to new destination in key store + with open(path, 'r') as reader, open(dst, 'w') as writer: + data = reader.read() + writer.write(data) else: # if PGP key, convert to binary, ???, convert back - # if key, save to file, change to path + with open(dst, "w") as file: + file.write(key) - \ No newline at end of file + # update mirror with new path + mirror["key"] = dst \ No newline at end of file From b9fe48e4af23f682a3360ce50300a8ac1cc8f4ea Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Mon, 9 Mar 2026 14:33:15 -0600 Subject: [PATCH 16/55] connecting mirrors to builder.py --- stackinator/builder.py | 9 +++++---- stackinator/mirror.py | 9 ++++----- stackinator/recipe.py | 2 +- stackinator/schema/mirror.json | 16 +++++++--------- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 5b223ffe..7918cb24 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -11,7 +11,7 @@ import jinja2 import yaml -from . import VERSION, cache, root_logger, spack_util +from . import VERSION, cache, root_logger, spack_util, mirror def install(src, dst, *, ignore=None, symlinks=False): @@ -231,7 +231,7 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - mirrors=recipe.mirrors + mirrors=recipe.mirrors, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) @@ -312,11 +312,12 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured + key_store = self.path / ".gnupg" if recipe.mirrors: + mirror.key_setup(recipe.mirrors, config_path, key_store) dst = config_path / "mirrors.yaml" self._logger.debug(f"generate the spack mirrors.yaml: {dst}") - with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirrors)) + mirror.spack_yaml_setup(recipe.mirrors, dst) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e521b597..0e5ee5d1 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,7 +1,7 @@ import os import pathlib import urllib.request -from typing import Optional +from typing import Optional, List, Dict import magic import yaml @@ -74,19 +74,18 @@ def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Opt f"Could not reach the mirror url '{url}'. " f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - if mirror["bootstrap"]: + #if mirror["bootstrap"]: #make bootstrap dirs #bootstrap//metadata.yaml return mirrors -def yaml_setup(mirrors, config_path): +def spack_yaml_setup(mirrors, config_path): """Generate the mirrors.yaml for spack""" dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the spack mirrors.yaml: {dst}") yaml = {"mirrors": {}} for m in mirrors: @@ -120,7 +119,7 @@ def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: if not path.is_absolute(): #try prepending system config path path = system_config_path + path - if not.path.is_file() + if not path.is_file(): raise FileNotFoundError( f"The key path '{path}' is not a file. " f"Check the key listed in mirrors.yaml in system config.") diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 4b7c038c..c59790b2 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -174,7 +174,7 @@ def __init__(self, args): # mirrors specified on the command line. self._mirrors = None self._logger.debug("Configuring mirrors.") - self._mirrors = mirror.configuration_from_file(self.system_config_path/"mirrors.yaml", args.cache) + self._mirrors = mirror.configuration_from_file(self.system_config_path, args.cache) # optional post install hook if self.post_install_hook is not None: diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index a53cc34e..a8be6ab3 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -1,28 +1,26 @@ -# This config handles source mirrors, binary caches, and bootstrap mirrors (of both forms) { - # Order matters, so we need an array. "type" : "array", "items": { "type": "object", - "required": ["name", "url"] + "required": ["name", "url"], "properties": { "name": { "type": "string", - "description": "The name of this mirror. Should be follow standard variable naming syntax.", + "description": "The name of this mirror. Should be follow standard variable naming syntax." }, "url": { "type": "string", - "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI).", + "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI)." }, "enabled": { "type": "boolean", "default": true, - "description": "Whether this mirror is enabled.", + "description": "Whether this mirror is enabled." }, "bootstrap": { "type": "boolean", "default": false, - "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror.", + "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror." }, "buildcache": { "type": "boolean", @@ -31,7 +29,7 @@ }, "public_key": { "type": "string", - "description": "Public PGP key for validating binary cache packages.", + "description": "Public PGP key for validating binary cache packages." }, "description": { "type": "string", @@ -39,4 +37,4 @@ } } } -} +} \ No newline at end of file From e2ee9ba4a12cebeddda5193a7b1978a74b8513c5 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 13:42:30 -0600 Subject: [PATCH 17/55] Put the mirror manipulation code in a class. --- stackinator/builder.py | 14 ++- stackinator/mirror.py | 239 +++++++++++++++++++++-------------------- stackinator/recipe.py | 3 +- 3 files changed, 134 insertions(+), 122 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 7918cb24..b35545b0 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -164,6 +164,7 @@ def environment_meta(self, recipe): self._environment_meta = meta def generate(self, recipe): + """Setup the recipe build environment.""" # make the paths, in case bwrap is not used, directly write to recipe.mount store_path = self.path / "store" if not recipe.no_bwrap else pathlib.Path(recipe.mount) tmp_path = self.path / "tmp" @@ -313,11 +314,14 @@ def generate(self, recipe): # generate a mirrors.yaml file if build caches have been configured key_store = self.path / ".gnupg" - if recipe.mirrors: - mirror.key_setup(recipe.mirrors, config_path, key_store) - dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the spack mirrors.yaml: {dst}") - mirror.spack_yaml_setup(recipe.mirrors, dst) + mirrors = recipe.mirrors + if mirrors: + mirrors.key_setup(recipe.mirrors, config_path, key_store) + dest = config_path / "mirrors.yaml" + self._logger.debug(f"generate the spack mirrors.yaml: {dest}") + mirrors.create_spack_mirrors_yaml(dest) + + # Setup bootstrap mirror configs. # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 0e5ee5d1..bccd5422 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,6 +1,7 @@ import os import pathlib import urllib.request +import urllib.error from typing import Optional, List, Dict import magic @@ -8,138 +9,146 @@ from . import schema -class MirrorConfigError(RuntimeError): +class MirrorError(RuntimeError): """Exception class for errors thrown by mirror configuration problems.""" - -def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): - """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" - - path = system_config_root/"mirrors.yaml" - if path.exists(): - with path.open() as fid: - # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) - - print(f"Configuring mirrors and buildcache from '{path}'") - - # validate the yaml - schema.CacheValidator.validate(raw) - - mirrors = [mirror for mirror in raw if mirror["enabled"]] - else: - mirrors = [] - - buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) - if buildcache_dest_count > 1: - raise RuntimeError("Mirror config has more than one mirror specified as the build cache destination " - "in the system config's 'mirrors.yaml'.") - elif buildcache_dest_count == 1 and cmdline_cache: - raise RuntimeError("Build cache destination specified on the command line and in the system config's " - "'mirrors.yaml'. It can be one or the other, but not both.") - - # Add or set the cache given on the command line as the buildcache destination - if cmdline_cache is not None: - existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] - # If the mirror name given on the command line isn't in the config, assume it - # is the URL to a build cache. - if not existing_mirror: - mirrors.append( - { - 'name': 'cmdline_cache', - 'url': cmdline_cache, - 'buildcache': True, - 'bootstrap': False, - } - ) - - for mirror in mirrors: - url = mirror["url"] - if url.beginswith("file://"): - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(url)) - if not path.is_absolute(): - raise FileNotFoundError(f"The mirror path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The mirror path '{path}' is not a directory") - - mirror["url"] = path - - elif url.beginswith("https://"): - try: - request = urllib.request.Request(url, method='HEAD') - response = urllib.request.urlopen(request) - except urllib.error.URLError as e: - raise MirrorConfigError( - f"Could not reach the mirror url '{url}'. " - f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - - #if mirror["bootstrap"]: - #make bootstrap dirs - #bootstrap//metadata.yaml +class Mirrors: + """Manage the definition of mirrors in a recipe.""" + + def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): + """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + + self._system_config_root = system_config_root + + self.mirrors = self._load_mirrors(cmdline_cache) + self._check_mirrors() + + self.build_cache_mirrors = [mirror for mirror in self.mirrors if mirror.get('buildcache', False)] + self.keys = [mirror['key'] for mirror in self.mirrors if mirror.get('key') is not None] + + def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: + """Load the mirrors file, if one exists.""" + path = self._system_config_root/"mirrors.yaml" + if path.exists(): + with path.open() as fid: + # load the raw yaml input + raw = yaml.load(fid, Loader=yaml.Loader) + + # validate the yaml + schema.CacheValidator.validate(raw) + + mirrors = [mirror for mirror in raw if mirror["enabled"]] + else: + mirrors = [] + + buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) + if buildcache_dest_count > 1: + raise MirrorError("Mirror config has more than one mirror specified as the build cache destination " + "in the system config's 'mirrors.yaml'.") + elif buildcache_dest_count == 1 and cmdline_cache: + raise MirrorError("Build cache destination specified on the command line and in the system config's " + "'mirrors.yaml'. It can be one or the other, but not both.") + + # Add or set the cache given on the command line as the buildcache destination + if cmdline_cache is not None: + existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] + # If the mirror name given on the command line isn't in the config, assume it + # is the URL to a build cache. + if not existing_mirror: + mirrors.append( + { + 'name': 'cmdline_cache', + 'url': cmdline_cache, + 'buildcache': True, + 'bootstrap': False, + } + ) return mirrors + def _check_mirrors(self): + """Validate the mirror config entries.""" -def spack_yaml_setup(mirrors, config_path): - """Generate the mirrors.yaml for spack""" + for mirror in self.mirrors: + url = mirror["url"] + if url.beginswith("file://"): + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(url)) + if not path.is_absolute(): + raise MirrorError(f"The mirror path '{path}' is not absolute") + if not path.is_dir(): + raise MirrorError(f"The mirror path '{path}' is not a directory") - dst = config_path / "mirrors.yaml" + mirror["url"] = path - yaml = {"mirrors": {}} + elif url.beginswith("https://"): + try: + request = urllib.request.Request(url, method='HEAD') + urllib.request.urlopen(request) + except urllib.error.URLError as e: + raise MirrorError( + f"Could not reach the mirror url '{url}'. " + f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - for m in mirrors: - name = m["name"] - url = m["url"] + def create_spack_mirrors_yaml(self, dest: pathlib.Path): + """Generate the mirrors.yaml for our build directory.""" - yaml["mirrors"][name] = { - "fetch": {"url": url}, - "push": {"url": url}, - } + raw = {"mirrors": {}} - with dst.open("w") as file: - yaml.dump(yaml, default_flow_style=False) + for m in self.mirrors: + name = m["name"] + url = m["url"] - # return dst + raw["mirrors"][name] = { + "fetch": {"url": url}, + "push": {"url": url}, + } + with dest.open("w") as file: + yaml.dump(raw, file, default_flow_style=False) -def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: pathlib.Path): - """Validate mirror keys, relocate to key_store, and update mirror config with new key paths""" + def bootstrap_setup(self, config_root: pathlib.Path): + """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" - for mirror in mirrors: - if mirror["key"]: - key = mirror["key"] - # key will be saved under key_store/mirror_name.gpg - dst = (key_store / f"'{mirror["name"]}'.gpg").resolve() - # if path, check if abs path, if not, append sys config path in front and check again - path = pathlib.Path(os.path.expandvars(key)) - if path.exists(): - if not path.is_absolute(): - #try prepending system config path - path = system_config_path + path - if not path.is_file(): - raise FileNotFoundError( - f"The key path '{path}' is not a file. " - f"Check the key listed in mirrors.yaml in system config.") + def key_setup(self, key_store: pathlib.Path): + """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" - file_type = magic.from_file(path) + for mirror in self.mirrors: + if mirror["key"]: + key = mirror["key"] - if not file_type.startswith("OpenPGP Public Key"): - raise MirrorConfigError( - f"'{path}' is not a valid GPG key. " - f"Check the key listed in mirrors.yaml in system config.") - - # copy key to new destination in key store - with open(path, 'r') as reader, open(dst, 'w') as writer: - data = reader.read() - writer.write(data) + # key will be saved under key_store/mirror_name.gpg + dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() + + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if path.exists(): + if not path.is_absolute(): + #try prepending system config path + path = self._system_config_root/path + if not path.is_file(): + raise MirrorError( + f"The key path '{path}' is not a file. " + f"Check the key listed in mirrors.yaml in system config.") + + file_type = magic.from_file(path) + + if not file_type.startswith("OpenPGP Public Key"): + raise MirrorError( + f"'{path}' is not a valid GPG key. " + f"Check the key listed in mirrors.yaml in system config.") + + # copy key to new destination in key store + with open(path, 'r') as reader, open(dest, 'w') as writer: + data = reader.read() + writer.write(data) + + else: + # if PGP key, convert to binary, ???, convert back + with open(dest, "w") as file: + file.write(key) - else: - # if PGP key, convert to binary, ???, convert back - with open(dst, "w") as file: - file.write(key) - - # update mirror with new path - mirror["key"] = dst \ No newline at end of file + # update mirror with new path + mirror["key"] = dest diff --git a/stackinator/recipe.py b/stackinator/recipe.py index c59790b2..b03d7510 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -172,9 +172,8 @@ def __init__(self, args): # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. - self._mirrors = None self._logger.debug("Configuring mirrors.") - self._mirrors = mirror.configuration_from_file(self.system_config_path, args.cache) + self._mirrors = mirror.Mirrors(self.system_config_path, args.cache) # optional post install hook if self.post_install_hook is not None: From 798a21651c6baa826dcc4f939aac890a20502b1a Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Thu, 12 Mar 2026 14:04:10 -0600 Subject: [PATCH 18/55] preserve cache for makefile --- stackinator/builder.py | 1 + stackinator/recipe.py | 9 +++++++-- stackinator/schema.py | 1 + stackinator/templates/Makefile | 21 ++++++++------------- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index b35545b0..2a2ceefb 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -227,6 +227,7 @@ def generate(self, recipe): with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( + cache = recipe.cache, modules=recipe.with_modules, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, diff --git a/stackinator/recipe.py b/stackinator/recipe.py index b03d7510..9915bf91 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -174,6 +174,7 @@ def __init__(self, args): # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") self._mirrors = mirror.Mirrors(self.system_config_path, args.cache) + self._cache = [mirror for mirror in self.mirrors if mirror["buildcache"]] # optional post install hook if self.post_install_hook is not None: @@ -235,6 +236,10 @@ def pre_install_hook(self): @property def mirrors(self): return self._mirrors + + @property + def cache(self): + return self._cache @property def config(self): @@ -515,7 +520,7 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.mirrors + push_to_cache = self.cache files["makefile"] = makefile_template.render( compilers=self.compilers, push_to_cache=push_to_cache, @@ -546,7 +551,7 @@ def environment_files(self): jenv.filters["py2yaml"] = schema.py2yaml makefile_template = jenv.get_template("Makefile.environments") - push_to_cache = self.mirror is not None + push_to_cache = self.cache is not None files["makefile"] = makefile_template.render( environments=self.environments, push_to_cache=push_to_cache, diff --git a/stackinator/schema.py b/stackinator/schema.py index 3a2a9842..d461ff0e 100644 --- a/stackinator/schema.py +++ b/stackinator/schema.py @@ -121,3 +121,4 @@ def check_module_paths(instance): EnvironmentsValidator = SchemaValidator(prefix / "schema/environments.json") CacheValidator = SchemaValidator(prefix / "schema/cache.json") ModulesValidator = SchemaValidator(prefix / "schema/modules.json", check_module_paths) +MirrorsValidator = SchemaValidator(prefix / "schema/mirror.json") diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index d1a06dd4..9484e3c7 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -32,25 +32,20 @@ pre-install: spack-setup $(SANDBOX) $(STORE)/pre-install-hook mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} + {% if cache %} - # The old way of managing mirrors $(SANDBOX) $(SPACK) buildcache keys --install --trust - {% if cache.key %} - $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} - {% endif %} {% endif %} {% if mirrors %} - @echo "Adding mirrors and gpg keys." - {% for mirror_info in mirrors | reverse %} - $(SANDBOX) $(SPACK) mirror add --scope=site {{ mirror_info.name }} {{ mirror_info.url }} - $(SANDBOX) $(SPACK) gpg trust {{ mirror_info.key_path }} + @echo "Adding mirror gpg keys." + {% for mirror in mirrors | reverse %} + {% if mirror.public_key %} + $(SANDBOX) $(SPACK) gpg trust {{ mirror.public_key }} + {% endif %} {% endfor %} @echo "Current mirror list:" $(SANDBOX) $(SPACK) mirror list {% endif %} - {% for mirror_info in filter(lambda m: m['bootstrap'], mirrors) | filter() %} - $(SANDBOX) $(SPACK) bootstrap add --scope=site {{ mirror_info.name }} bootstrap/{{ mirror_info.name }} - {% endfor %} touch mirror-setup compilers: mirror-setup @@ -89,14 +84,14 @@ store.squashfs: post-install # Force push all built packages to the build cache cache-force: mirror-setup -{% if cache.key %} +{% if cache %} $(warning ================================================================================) $(warning Generate the config in order to force push partially built compiler environments) $(warning if this step is performed with partially built compiler envs, you will) $(warning likely have to start a fresh build (but that's okay, because build caches FTW)) $(warning ================================================================================) $(SANDBOX) $(MAKE) -C generate-config - $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package alpscache \ + $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package cache.name \ $$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ From 64ff5d2a547b61561d5728ed82444617795bbb55 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:09:46 -0600 Subject: [PATCH 19/55] Adding bootstrap mirror configs. --- stackinator/builder.py | 17 ++++++++-------- stackinator/mirror.py | 45 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 2a2ceefb..acc2d91b 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -314,15 +314,14 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured - key_store = self.path / ".gnupg" - mirrors = recipe.mirrors - if mirrors: - mirrors.key_setup(recipe.mirrors, config_path, key_store) - dest = config_path / "mirrors.yaml" - self._logger.debug(f"generate the spack mirrors.yaml: {dest}") - mirrors.create_spack_mirrors_yaml(dest) - - # Setup bootstrap mirror configs. + if recipe.mirrors: + recipe.mirrors.key_setup(config_path) + + self._logger.debug(f"Generating the spack mirrors.yaml in '{config_path}'") + recipe.mirrors.create_spack_mirrors_yaml(config_path/'mirrors.yaml') + + # Setup bootstrap mirror configs. + recipe.mirrors.create_bootstrap_configs(config_path) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index bccd5422..9a7dc797 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -2,7 +2,7 @@ import pathlib import urllib.request import urllib.error -from typing import Optional, List, Dict +from typing import ByteString, Optional, List, Dict import magic import yaml @@ -22,8 +22,10 @@ def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str self.mirrors = self._load_mirrors(cmdline_cache) self._check_mirrors() - - self.build_cache_mirrors = [mirror for mirror in self.mirrors if mirror.get('buildcache', False)] + + self.build_cache_mirror = ([mirror for mirror in self.mirrors if mirror.get('buildcache', False)] + + [None]).pop(0) + self.bootstrap_mirrors = [mirror for mirror in self.mirrors if mirror.get('bootstrap', False)] self.keys = [mirror['key'] for mirror in self.mirrors if mirror.get('key') is not None] def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: @@ -107,12 +109,43 @@ def create_spack_mirrors_yaml(self, dest: pathlib.Path): with dest.open("w") as file: yaml.dump(raw, file, default_flow_style=False) - def bootstrap_setup(self, config_root: pathlib.Path): + def create_bootstrap_configs(self, config_root: pathlib.Path): """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" + if not self.bootstrap_mirrors: + return + + bootstrap_yaml = { + 'sources': [], + 'trusted': {}, + } + + for mirror in self.bootstrap_mirrors: + name = mirror['name'] + bs_mirror_path = config_root/f'bootstrap/{name}' + # Tell spack where to find the metadata for each bootstrap mirror. + bootstrap_yaml['sources'].append( + { + 'name': name, + 'metadata': bs_mirror_path, + } + ) + # And trust each one + bootstrap_yaml['trusted'][name] = True + + # Create the metadata dir and metadata.yaml + bs_mirror_path.mkdir(parents=True) + bs_mirror_yaml = { + 'type': 'install', + 'info': mirror['url'], + } + with (bs_mirror_path/'metadata.yaml').open('w') as file: + yaml.dump(bs_mirror_yaml, file, default_flow_style=False) + + with (config_root/'bootstrap.yaml').open('w') as file: + yaml.dump(bootstrap_yaml, file, default_flow_style=False) - - def key_setup(self, key_store: pathlib.Path): + def key_setup(self, config_root: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: From d55c1525c0869fb5bd9168d45d8470db64cae73e Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:16:21 -0600 Subject: [PATCH 20/55] Reverted to defining the key store path in builder. --- stackinator/builder.py | 2 +- stackinator/mirror.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index acc2d91b..80a3977b 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -315,7 +315,7 @@ def generate(self, recipe): # generate a mirrors.yaml file if build caches have been configured if recipe.mirrors: - recipe.mirrors.key_setup(config_path) + recipe.mirrors.key_setup(config_path/'key_store') self._logger.debug(f"Generating the spack mirrors.yaml in '{config_path}'") recipe.mirrors.create_spack_mirrors_yaml(config_path/'mirrors.yaml') diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 9a7dc797..9f162768 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -145,7 +145,7 @@ def create_bootstrap_configs(self, config_root: pathlib.Path): with (config_root/'bootstrap.yaml').open('w') as file: yaml.dump(bootstrap_yaml, file, default_flow_style=False) - def key_setup(self, config_root: pathlib.Path): + def key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: From 21c507ca74bf027d2f6c893e88034ca6d721e5e7 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:22:31 -0600 Subject: [PATCH 21/55] Compressed mirror config setup into a single interface. --- stackinator/builder.py | 5 +---- stackinator/mirror.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 80a3977b..f593fb05 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -315,12 +315,9 @@ def generate(self, recipe): # generate a mirrors.yaml file if build caches have been configured if recipe.mirrors: + self._logger.debug(f"Generating the spack mirror configs in '{config_path}'") recipe.mirrors.key_setup(config_path/'key_store') - - self._logger.debug(f"Generating the spack mirrors.yaml in '{config_path}'") recipe.mirrors.create_spack_mirrors_yaml(config_path/'mirrors.yaml') - - # Setup bootstrap mirror configs. recipe.mirrors.create_bootstrap_configs(config_path) # Add custom spack package recipes, configured via Spack repos. diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 9f162768..6de59937 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -15,6 +15,9 @@ class MirrorError(RuntimeError): class Mirrors: """Manage the definition of mirrors in a recipe.""" + KEY_STORE_DIR = 'key_store' + MIRRORS_YAML = 'mirrors.yaml' + def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" @@ -92,7 +95,14 @@ def _check_mirrors(self): f"Could not reach the mirror url '{url}'. " f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - def create_spack_mirrors_yaml(self, dest: pathlib.Path): + def setup_configs(self, config_root: pathlib.Path): + """Setup all mirror configs in the given config_root.""" + + self._key_setup(config_root/self.KEY_STORE_DIR) + self._create_spack_mirrors_yaml(config_root/self.MIRRORS_YAML) + self._create_bootstrap_configs(config_root) + + def _create_spack_mirrors_yaml(self, dest: pathlib.Path): """Generate the mirrors.yaml for our build directory.""" raw = {"mirrors": {}} @@ -109,7 +119,7 @@ def create_spack_mirrors_yaml(self, dest: pathlib.Path): with dest.open("w") as file: yaml.dump(raw, file, default_flow_style=False) - def create_bootstrap_configs(self, config_root: pathlib.Path): + def _create_bootstrap_configs(self, config_root: pathlib.Path): """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" if not self.bootstrap_mirrors: @@ -145,7 +155,7 @@ def create_bootstrap_configs(self, config_root: pathlib.Path): with (config_root/'bootstrap.yaml').open('w') as file: yaml.dump(bootstrap_yaml, file, default_flow_style=False) - def key_setup(self, key_store: pathlib.Path): + def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: From d33de01b8d059b08aaaa196b544c07ebf4dfd7b6 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:26:49 -0600 Subject: [PATCH 22/55] Catching builder exceptions. --- stackinator/builder.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index f593fb05..5bab0eae 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -314,11 +314,12 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured - if recipe.mirrors: - self._logger.debug(f"Generating the spack mirror configs in '{config_path}'") - recipe.mirrors.key_setup(config_path/'key_store') - recipe.mirrors.create_spack_mirrors_yaml(config_path/'mirrors.yaml') - recipe.mirrors.create_bootstrap_configs(config_path) + self._logger.debug(f"Generating the spack mirror configs in '{config_path}'") + try: + recipe.mirrors.setup_configs(config_path) + except mirror.MirrorError as err: + self._logger.error(f"Could not set up mirrors.\n{err}") + return 1 # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to From 3eac13d83b584c0e87625e2f1f30e20972524844 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Thu, 12 Mar 2026 14:33:08 -0600 Subject: [PATCH 23/55] fixing key setup --- stackinator/mirror.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 6de59937..c3b2c404 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -159,8 +159,8 @@ def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: - if mirror["key"]: - key = mirror["key"] + if mirror["public_key"]: + key = mirror["public_key"] # key will be saved under key_store/mirror_name.gpg dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() From d31422c02db8497105b14bb487d3b70eebb284ab Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:46:21 -0600 Subject: [PATCH 24/55] In progress. --- stackinator/mirror.py | 78 ++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index c3b2c404..8f5e4400 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,3 +1,4 @@ +import base64 import os import pathlib import urllib.request @@ -29,7 +30,8 @@ def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str self.build_cache_mirror = ([mirror for mirror in self.mirrors if mirror.get('buildcache', False)] + [None]).pop(0) self.bootstrap_mirrors = [mirror for mirror in self.mirrors if mirror.get('bootstrap', False)] - self.keys = [mirror['key'] for mirror in self.mirrors if mirror.get('key') is not None] + # Will hold a list of all the keys + self.keys = None def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: """Load the mirrors file, if one exists.""" @@ -159,39 +161,47 @@ def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: - if mirror["public_key"]: - key = mirror["public_key"] - - # key will be saved under key_store/mirror_name.gpg - dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() - - # if path, check if abs path, if not, append sys config path in front and check again - path = pathlib.Path(os.path.expandvars(key)) - if path.exists(): - if not path.is_absolute(): - #try prepending system config path - path = self._system_config_root/path - if not path.is_file(): - raise MirrorError( - f"The key path '{path}' is not a file. " - f"Check the key listed in mirrors.yaml in system config.") - - file_type = magic.from_file(path) - - if not file_type.startswith("OpenPGP Public Key"): + if not mirror["public_key"]: + continue + + key = mirror["public_key"] + + # key will be saved under key_store/mirror_name.gpg + dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() + + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if path.exists(): + if not path.is_absolute(): + #try prepending system config path + path = self._system_config_root/path + if not path.is_file(): raise MirrorError( - f"'{path}' is not a valid GPG key. " + f"The key path '{path}' is not a file. " f"Check the key listed in mirrors.yaml in system config.") - - # copy key to new destination in key store - with open(path, 'r') as reader, open(dest, 'w') as writer: - data = reader.read() - writer.write(data) - - else: - # if PGP key, convert to binary, ???, convert back - with open(dest, "w") as file: - file.write(key) + + file_type = magic.from_file(path) + + if not file_type.startswith("OpenPGP Public Key"): + raise MirrorError( + f"'{path}' is not a valid GPG key. " + f"Check the key listed in mirrors.yaml in system config.") - # update mirror with new path - mirror["key"] = dest + # copy key to new destination in key store + with open(path, 'r') as reader, open(dest, 'w') as writer: + data = reader.read() + writer.write(data) + + else: + try: + key = base64.b64decode(key) + except ValueError as err: + pass + magic.from_buffer(key) + + # if PGP key, convert to binary, ???, convert back + with open(dest, "wb") as file: + file.write(key) + + # update mirror with new path + mirror["key"] = dest From d1c1dc1e939362394791e4d18f489ada06c96b99 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 09:43:02 -0600 Subject: [PATCH 25/55] added GPG key verification --- stackinator/builder.py | 2 +- stackinator/mirror.py | 73 ++++++++++++++++++++++-------------------- stackinator/recipe.py | 12 ++----- 3 files changed, 41 insertions(+), 46 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 5bab0eae..f8083997 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -233,7 +233,7 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - mirrors=recipe.mirrors, + mirrors=recipe.mirrors.mirrors, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 8f5e4400..48d30660 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -5,6 +5,7 @@ import urllib.error from typing import ByteString, Optional, List, Dict import magic +import base64 import yaml @@ -42,7 +43,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: raw = yaml.load(fid, Loader=yaml.Loader) # validate the yaml - schema.CacheValidator.validate(raw) + #schema.CacheValidator.validate(raw) mirrors = [mirror for mirror in raw if mirror["enabled"]] else: @@ -78,7 +79,7 @@ def _check_mirrors(self): for mirror in self.mirrors: url = mirror["url"] - if url.beginswith("file://"): + if url.startswith("file://"): # verify that the root path exists path = pathlib.Path(os.path.expandvars(url)) if not path.is_absolute(): @@ -88,7 +89,7 @@ def _check_mirrors(self): mirror["url"] = path - elif url.beginswith("https://"): + elif url.startswith("https://"): try: request = urllib.request.Request(url, method='HEAD') urllib.request.urlopen(request) @@ -159,49 +160,51 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" + + key_store.mkdir(exist_ok=True) for mirror in self.mirrors: - if not mirror["public_key"]: - continue + if mirror.get("public_key"): + key = mirror["public_key"] - key = mirror["public_key"] + # key will be saved under key_store/mirror_name.gpg - # key will be saved under key_store/mirror_name.gpg - dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() + dest = pathlib.Path(key_store / f"{mirror["name"]}.gpg") - # if path, check if abs path, if not, append sys config path in front and check again - path = pathlib.Path(os.path.expandvars(key)) - if path.exists(): + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) if not path.is_absolute(): #try prepending system config path path = self._system_config_root/path + + if path.exists(): if not path.is_file(): raise MirrorError( - f"The key path '{path}' is not a file. " + f"The key path '{path}' is not a file. \n" f"Check the key listed in mirrors.yaml in system config.") - - file_type = magic.from_file(path) - - if not file_type.startswith("OpenPGP Public Key"): + + with open(path, 'rb') as reader: + binary_key = reader.read() + + # convert base64 key to binary + else: + try: + binary_key = base64.b64decode(key) + except ValueError: + raise MirrorError( + f"Key for mirror {mirror["name"]} is not valid. \n" + f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" + f"Check the key listed in mirrors.yaml in system config.") + + file_type = magic.from_buffer(binary_key, mime=True) + print("magic type:" , file_type) + if file_type != "application/x-gnupg-keyring": raise MirrorError( - f"'{path}' is not a valid GPG key. " + f"Key for mirror {mirror["name"]} is not a valid GPG key. \n" f"Check the key listed in mirrors.yaml in system config.") - + # copy key to new destination in key store - with open(path, 'r') as reader, open(dest, 'w') as writer: - data = reader.read() - writer.write(data) - - else: - try: - key = base64.b64decode(key) - except ValueError as err: - pass - magic.from_buffer(key) - - # if PGP key, convert to binary, ???, convert back - with open(dest, "wb") as file: - file.write(key) - - # update mirror with new path - mirror["key"] = dest + with open(dest, 'wb') as writer: + writer.write(binary_key) + # update mirror with new path + mirror["public_key"] = dest diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 9915bf91..a15569b6 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -173,8 +173,8 @@ def __init__(self, args): # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") - self._mirrors = mirror.Mirrors(self.system_config_path, args.cache) - self._cache = [mirror for mirror in self.mirrors if mirror["buildcache"]] + self.mirrors = mirror.Mirrors(self.system_config_path, args.cache) + self.cache = self.mirrors.build_cache_mirror # optional post install hook if self.post_install_hook is not None: @@ -232,14 +232,6 @@ def pre_install_hook(self): if hook_path.exists() and hook_path.is_file(): return hook_path return None - - @property - def mirrors(self): - return self._mirrors - - @property - def cache(self): - return self._cache @property def config(self): From 14e77fc5140cd8b2b88023e1f7725cbd1a4b5a03 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 14:29:56 -0600 Subject: [PATCH 26/55] unit tests for mirrors --- stackinator/mirror.py | 4 +- unittests/__init__.py | 0 .../data/systems/mirror-bad-key/bad_key.gpg | 1 + .../data/systems/mirror-bad-key/mirrors.yaml | 3 ++ .../systems/mirror-bad-keypath/mirrors.yaml | 3 ++ .../data/systems/mirror-bad-url/mirrors.yaml | 2 + unittests/data/systems/mirror-ok/mirrors.yaml | 11 +++++ unittests/test_mirrors.py | 42 +++++++++++++++++++ 8 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 unittests/__init__.py create mode 100644 unittests/data/systems/mirror-bad-key/bad_key.gpg create mode 100644 unittests/data/systems/mirror-bad-key/mirrors.yaml create mode 100644 unittests/data/systems/mirror-bad-keypath/mirrors.yaml create mode 100644 unittests/data/systems/mirror-bad-url/mirrors.yaml create mode 100644 unittests/data/systems/mirror-ok/mirrors.yaml create mode 100644 unittests/test_mirrors.py diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 48d30660..d021aee9 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -43,7 +43,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: raw = yaml.load(fid, Loader=yaml.Loader) # validate the yaml - #schema.CacheValidator.validate(raw) + schema.CacheValidator.validate(raw) mirrors = [mirror for mirror in raw if mirror["enabled"]] else: @@ -197,7 +197,7 @@ def _key_setup(self, key_store: pathlib.Path): f"Check the key listed in mirrors.yaml in system config.") file_type = magic.from_buffer(binary_key, mime=True) - print("magic type:" , file_type) + if file_type != "application/x-gnupg-keyring": raise MirrorError( f"Key for mirror {mirror["name"]} is not a valid GPG key. \n" diff --git a/unittests/__init__.py b/unittests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/unittests/data/systems/mirror-bad-key/bad_key.gpg b/unittests/data/systems/mirror-bad-key/bad_key.gpg new file mode 100644 index 00000000..d7980bbf --- /dev/null +++ b/unittests/data/systems/mirror-bad-key/bad_key.gpg @@ -0,0 +1 @@ +This is a bad key \ No newline at end of file diff --git a/unittests/data/systems/mirror-bad-key/mirrors.yaml b/unittests/data/systems/mirror-bad-key/mirrors.yaml new file mode 100644 index 00000000..ed27df7a --- /dev/null +++ b/unittests/data/systems/mirror-bad-key/mirrors.yaml @@ -0,0 +1,3 @@ +- name: bad-key + url: https://mirror.spack.io + public_key: /bad_key.gpg \ No newline at end of file diff --git a/unittests/data/systems/mirror-bad-keypath/mirrors.yaml b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml new file mode 100644 index 00000000..e671c45a --- /dev/null +++ b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml @@ -0,0 +1,3 @@ +- name: bad-key-path + url: https://mirror.spack.io + public_key: /path/doesnt/exist \ No newline at end of file diff --git a/unittests/data/systems/mirror-bad-url/mirrors.yaml b/unittests/data/systems/mirror-bad-url/mirrors.yaml new file mode 100644 index 00000000..522c2326 --- /dev/null +++ b/unittests/data/systems/mirror-bad-url/mirrors.yaml @@ -0,0 +1,2 @@ +- name: bad-url + url: google.com \ No newline at end of file diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml new file mode 100644 index 00000000..9edf8d49 --- /dev/null +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -0,0 +1,11 @@ +- name: fake-mirror + url: https://google.com +- name: disabled-mirror + url: https://google.com + enabled: false +- name: buildcache-mirror + url: https://cache.spack.io/ + buildcache: true +- name: bootstrap-mirror + url: https://mirror.spack.io + bootstrap: true \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py new file mode 100644 index 00000000..6283c1a3 --- /dev/null +++ b/unittests/test_mirrors.py @@ -0,0 +1,42 @@ +import pytest +import pathlib +import stackinator.mirror as mirror +import yaml + +@pytest.fixture +def test_path(): + return pathlib.Path(__file__).parent.resolve() + +@pytest.fixture +def systems_path(test_path): + return test_path / "data" / "systems" + +@pytest.fixture +def valid_mirrors(systems_path): + mirrors = {} + mirrors["fake-mirror"] = {'url': 'https://google.com'} + mirrors["buildcache-mirror"] = {'url': 'https://cache.spack.io/', 'buildcache': True} + mirrors["bootstrap-mirror"] = {'url': 'https://mirror.spack.io', 'bootstrap': True} + return mirrors + +def test_mirror_init(systems_path, valid_mirrors): + path = systems_path / "mirror_ok" + mirrors = mirror.Mirrors(path) + print(valid_mirrors) + print(mirrors) + assert mirrors == valid_mirrors + assert mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors if mirror["bootstrap"]] + assert mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors if mirror['buildcache']] + # assert disabled mirror not in mirrors + for mir in mirrors: + assert mir["enabled"] + # test that cmdline_cache gets added to mirrors? + +def test_create_spack_mirrors_yaml(systems_path): + pass + +def test_create_bootstrap_configs(): + pass + +def test_key_setup(): + pass \ No newline at end of file From ff1a35c85e304e305b6114fd0e19f9b0d4ef500b Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 14:45:07 -0600 Subject: [PATCH 27/55] mirrors.yaml is now a name:{} mapping --- stackinator/mirror.py | 221 +++++++++++++++++++++------------ stackinator/schema/mirror.json | 30 +++-- 2 files changed, 161 insertions(+), 90 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index d021aee9..80e1791e 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,15 +1,15 @@ + +from typing import Optional, List, Dict import base64 +import io +import magic import os import pathlib -import urllib.request import urllib.error -from typing import ByteString, Optional, List, Dict -import magic -import base64 - +import urllib.request import yaml -from . import schema +from . import schema, root_logger class MirrorError(RuntimeError): """Exception class for errors thrown by mirror configuration problems.""" @@ -20,72 +20,124 @@ class Mirrors: KEY_STORE_DIR = 'key_store' MIRRORS_YAML = 'mirrors.yaml' - def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): + def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None, + mount_point: Optional[pathlib.Path] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" self._system_config_root = system_config_root + self._mount_point = mount_point + + self._logger = root_logger self.mirrors = self._load_mirrors(cmdline_cache) self._check_mirrors() - self.build_cache_mirror = ([mirror for mirror in self.mirrors if mirror.get('buildcache', False)] - + [None]).pop(0) - self.bootstrap_mirrors = [mirror for mirror in self.mirrors if mirror.get('bootstrap', False)] - # Will hold a list of all the keys - self.keys = None + self.build_cache_mirror : Optional[str] = \ + ([name for name, mirror in self.mirrors.items() if mirror.get('cache', False)] + + [None]).pop(0) + self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() + if mirror.get('bootstrap', False)] + + # Will hold a list of all the gpg keys (public and private) + self._keys: Optional[List[pathlib.Path]] = [] - def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: + def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" path = self._system_config_root/"mirrors.yaml" if path.exists(): with path.open() as fid: # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid, Loader=yaml.SafeLoader) # validate the yaml - schema.CacheValidator.validate(raw) + schema.MirrorsValidator.validate(raw) - mirrors = [mirror for mirror in raw if mirror["enabled"]] + mirrors = {name: mirror for name, mirror in raw.items() if mirror["enabled"]} else: - mirrors = [] - - buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) - if buildcache_dest_count > 1: - raise MirrorError("Mirror config has more than one mirror specified as the build cache destination " - "in the system config's 'mirrors.yaml'.") - elif buildcache_dest_count == 1 and cmdline_cache: - raise MirrorError("Build cache destination specified on the command line and in the system config's " - "'mirrors.yaml'. It can be one or the other, but not both.") + mirrors = {} # Add or set the cache given on the command line as the buildcache destination if cmdline_cache is not None: - existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] + existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache] # If the mirror name given on the command line isn't in the config, assume it # is the URL to a build cache. if not existing_mirror: - mirrors.append( - { - 'name': 'cmdline_cache', + mirrors['cmdline_cache'] = { 'url': cmdline_cache, - 'buildcache': True, + 'description': "Cache configured via command line.", + 'enabled': True, + 'cache': True, 'bootstrap': False, + 'mount_specific': True, } - ) + + # Load the cache as defined by the deprecated 'cache.yaml' file. + mirrors['legacy_cache_cfg'] = self._load_legacy_cache() + + caches = [mirror for mirror in mirrors.values() if mirror['cache']] + if len(caches) > 1: + raise MirrorError( + "Mirror config has more than one mirror specified as the build cache destination.\n" + "Some of these may have come from a legacy 'cache.yaml' or the '--cache' option.\n" + f"{self._pp_yaml(caches)}") return mirrors + @staticmethod + def _pp_yaml(object): + """Pretty print the given object as yaml.""" + + example_yaml_stream = io.StringIO() + yaml.dump(object, example_yaml_stream, default_flow_style=False) + return example_yaml_stream.getvalue() + + def _load_legacy_cache(self): + """Load the mirror definition from the legacy cache.yaml file.""" + + cache_config_path = self._system_config_root/'cache.yaml' + + if cache_config_path.is_file(): + + with cache_config_path.open('r') as file: + try: + raw = yaml.load(file, Loader=yaml.SafeLoader) + except ValueError as err: + raise MirrorError( + f"Error loading yaml from cache config at '{cache_config_path}'\n{err}") + + try: + schema.CacheValidator.validate(raw) + except ValueError as err: + raise MirrorError( + f"Error validating contents of cache config at '{cache_config_path}'.\n{err}") + + mirror_cfg = { + 'url': f'file://{raw['root']}', + 'description': "Buildcache dest loaded from legacy cache.yaml", + 'buildcache_push': True, + 'mount_specific': True, + 'enabled': True, + 'private_key': raw['key'], + } + + self._logger.warning("Configuring the buildcache from the system cache.yaml file.\n" + "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" + f"The equivalent 'mirrors.yaml' would look like: \n{self._pp_yaml([mirror_cfg])}") + + return mirror_cfg + def _check_mirrors(self): """Validate the mirror config entries.""" - for mirror in self.mirrors: + for name, mirror in self.mirrors.items(): url = mirror["url"] if url.startswith("file://"): # verify that the root path exists path = pathlib.Path(os.path.expandvars(url)) if not path.is_absolute(): - raise MirrorError(f"The mirror path '{path}' is not absolute") + raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not absolute") if not path.is_dir(): - raise MirrorError(f"The mirror path '{path}' is not a directory") + raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not a directory") mirror["url"] = path @@ -98,6 +150,16 @@ def _check_mirrors(self): f"Could not reach the mirror url '{url}'. " f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") + @property + def keys(self): + """Return the list of public and private key file paths.""" + + if self._keys is None: + raise RuntimeError("The mirror.keys method was accessed before setup_configs() was called.") + + return self._keys + + def setup_configs(self, config_root: pathlib.Path): """Setup all mirror configs in the given config_root.""" @@ -110,9 +172,9 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): raw = {"mirrors": {}} - for m in self.mirrors: - name = m["name"] - url = m["url"] + for name, mirror in self.mirrors.items(): + name = mirror["name"] + url = mirror["url"] raw["mirrors"][name] = { "fetch": {"url": url}, @@ -133,9 +195,9 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): 'trusted': {}, } - for mirror in self.bootstrap_mirrors: - name = mirror['name'] + for name in self.bootstrap_mirrors: bs_mirror_path = config_root/f'bootstrap/{name}' + mirror = self.mirrors[name] # Tell spack where to find the metadata for each bootstrap mirror. bootstrap_yaml['sources'].append( { @@ -161,50 +223,53 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" + self._keys = [] key_store.mkdir(exist_ok=True) - for mirror in self.mirrors: - if mirror.get("public_key"): - key = mirror["public_key"] + for name, mirror in self.mirrors.items(): + if mirror.get("public_key") is None: + continue - # key will be saved under key_store/mirror_name.gpg + key = mirror["public_key"] - dest = pathlib.Path(key_store / f"{mirror["name"]}.gpg") + # key will be saved under key_store/mirror_name.gpg - # if path, check if abs path, if not, append sys config path in front and check again - path = pathlib.Path(os.path.expandvars(key)) - if not path.is_absolute(): - #try prepending system config path - path = self._system_config_root/path - - if path.exists(): - if not path.is_file(): - raise MirrorError( - f"The key path '{path}' is not a file. \n" - f"Check the key listed in mirrors.yaml in system config.") - - with open(path, 'rb') as reader: - binary_key = reader.read() - - # convert base64 key to binary - else: - try: - binary_key = base64.b64decode(key) - except ValueError: - raise MirrorError( - f"Key for mirror {mirror["name"]} is not valid. \n" - f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" - f"Check the key listed in mirrors.yaml in system config.") - - file_type = magic.from_buffer(binary_key, mime=True) + dest = pathlib.Path(key_store / f"{name}.gpg") + + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if not path.is_absolute(): + #try prepending system config path + path = self._system_config_root/path - if file_type != "application/x-gnupg-keyring": + if path.exists(): + if not path.is_file(): raise MirrorError( - f"Key for mirror {mirror["name"]} is not a valid GPG key. \n" + f"The key path '{path}' is not a file. \n" f"Check the key listed in mirrors.yaml in system config.") - - # copy key to new destination in key store - with open(dest, 'wb') as writer: - writer.write(binary_key) - # update mirror with new path - mirror["public_key"] = dest + + with open(path, 'rb') as reader: + binary_key = reader.read() + + # convert base64 key to binary + else: + try: + binary_key = base64.b64decode(key) + except ValueError: + raise MirrorError( + f"Key for mirror '{name}' is not valid. \n" + f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" + f"Check the key listed in mirrors.yaml in system config.") + + file_type = magic.from_buffer(binary_key, mime=True) + print("magic type:" , file_type) + if file_type != "application/x-gnupg-keyring": + raise MirrorError( + f"Key for mirror {name} is not a valid GPG key. \n" + f"Check the key listed in mirrors.yaml in system config.") + + # copy key to new destination in key store + with open(dest, 'wb') as writer: + writer.write(binary_key) + + self._keys.append(dest) diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index a8be6ab3..5efb7253 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -1,17 +1,18 @@ { - "type" : "array", - "items": { + "type" : "object", + "additionalProperties": { "type": "object", - "required": ["name", "url"], + "required": ["url"], + "additionalProperties": false, "properties": { - "name": { - "type": "string", - "description": "The name of this mirror. Should be follow standard variable naming syntax." - }, "url": { "type": "string", "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI)." }, + "description": { + "type": "string", + "description": "What this mirror is for." + } "enabled": { "type": "boolean", "default": true, @@ -22,19 +23,24 @@ "default": false, "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror." }, - "buildcache": { + "cache": { "type": "boolean", "default": false, "description": "Use this mirror as the buildcache push destination. Can only be enabled on a single mirror." }, "public_key": { "type": "string", - "description": "Public PGP key for validating binary cache packages." + "description": "Public PGP key for validating binary cache packages. A path or base64 encoded key." }, - "description": { + "private_key": { "type": "string", - "description": "What this mirror is for." + "description": "Private PGP key for signing binary cache packages. (Path only)", + }, + "mount_specific": { + "type": "boolean", + "default": false, + "description": "Use a mount specific buildcache path (specified path + recipe mount point).", } } } -} \ No newline at end of file +} From a19a6d3eb4f0a100f5803c29cf8c07c3a53e77b7 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 15:09:44 -0600 Subject: [PATCH 28/55] Now add mount specific paths to certain mirrors. --- stackinator/builder.py | 4 ++-- stackinator/mirror.py | 5 ++++- stackinator/templates/Makefile | 11 +++-------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index f8083997..b6192bb2 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -227,13 +227,13 @@ def generate(self, recipe): with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( - cache = recipe.cache, modules=recipe.with_modules, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - mirrors=recipe.mirrors.mirrors, + gpg_keys=recipe.mirrors.keys, + cache=recipe.mirrors.buildcache, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 80e1791e..cad746c0 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -173,9 +173,12 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): raw = {"mirrors": {}} for name, mirror in self.mirrors.items(): - name = mirror["name"] url = mirror["url"] + # Make the mirror path specific to the mount point + if mirror['mount_specific'] and self._mount_point is not None: + url = url.rstrip('/') + '/' + self._mount_point.as_posix().lstrip('/') + raw["mirrors"][name] = { "fetch": {"url": url}, "push": {"url": url}, diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 9484e3c7..f0b90ebe 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -33,19 +33,14 @@ pre-install: spack-setup mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - {% if cache %} + @echo "Pulling and trusting keys from configured buildcaches." $(SANDBOX) $(SPACK) buildcache keys --install --trust - {% endif %} - {% if mirrors %} @echo "Adding mirror gpg keys." - {% for mirror in mirrors | reverse %} - {% if mirror.public_key %} - $(SANDBOX) $(SPACK) gpg trust {{ mirror.public_key }} - {% endif %} + {% for key_path in gpg_keys %} + $(SANDBOX) $(SPACK) gpg trust {{ key_path }} {% endfor %} @echo "Current mirror list:" $(SANDBOX) $(SPACK) mirror list - {% endif %} touch mirror-setup compilers: mirror-setup From b5c1e7e46ef465bf95b4df38e5b8f3022a13ed49 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 15:22:24 -0600 Subject: [PATCH 29/55] updated test mirror format --- stackinator/schema/mirror.json | 6 +++--- unittests/data/systems/mirror-ok/mirrors.yaml | 14 +++++--------- unittests/test_mirrors.py | 18 ++++++++---------- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 5efb7253..89770831 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -12,7 +12,7 @@ "description": { "type": "string", "description": "What this mirror is for." - } + }, "enabled": { "type": "boolean", "default": true, @@ -34,12 +34,12 @@ }, "private_key": { "type": "string", - "description": "Private PGP key for signing binary cache packages. (Path only)", + "description": "Private PGP key for signing binary cache packages. (Path only)" }, "mount_specific": { "type": "boolean", "default": false, - "description": "Use a mount specific buildcache path (specified path + recipe mount point).", + "description": "Use a mount specific buildcache path (specified path + recipe mount point)." } } } diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index 9edf8d49..e84fc3c3 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -1,11 +1,7 @@ -- name: fake-mirror - url: https://google.com -- name: disabled-mirror - url: https://google.com +- url: https://google.com +- url: https://google.com enabled: false -- name: buildcache-mirror - url: https://cache.spack.io/ - buildcache: true -- name: bootstrap-mirror - url: https://mirror.spack.io +- url: https://cache.spack.io/ + cache: true +- url: https://mirror.spack.io bootstrap: true \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 6283c1a3..bd96dce1 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -14,21 +14,19 @@ def systems_path(test_path): @pytest.fixture def valid_mirrors(systems_path): mirrors = {} - mirrors["fake-mirror"] = {'url': 'https://google.com'} - mirrors["buildcache-mirror"] = {'url': 'https://cache.spack.io/', 'buildcache': True} - mirrors["bootstrap-mirror"] = {'url': 'https://mirror.spack.io', 'bootstrap': True} + mirrors["fake-mirror"] = {'url': 'https://google.com', 'enabled': True, 'bootstrap': False, 'cache': False, 'mount_specific': False} + mirrors["buildcache-mirror"] = {'url': 'https://cache.spack.io/', 'enabled': True, 'bootstrap': False, 'cache': True, 'mount_specific': False} + mirrors["bootstrap-mirror"] = {'url': 'https://mirror.spack.io', 'enabled': True, 'bootstrap': True, 'cache': False, 'mount_specific': False} return mirrors def test_mirror_init(systems_path, valid_mirrors): path = systems_path / "mirror_ok" - mirrors = mirror.Mirrors(path) - print(valid_mirrors) - print(mirrors) - assert mirrors == valid_mirrors - assert mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors if mirror["bootstrap"]] - assert mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors if mirror['buildcache']] + mirrors_obj = mirror.Mirrors(path) + assert mirrors_obj.mirrors == valid_mirrors + assert mirrors_obj.mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors.values() if mirror.get('bootstrap')] + assert mirrors_obj.mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors.values() if mirror.get('buildcache')] # assert disabled mirror not in mirrors - for mir in mirrors: + for mir in mirrors_obj.mirrors: assert mir["enabled"] # test that cmdline_cache gets added to mirrors? From 8f4fa369d527aeee378c7d01de4654403ffa28ff Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 15:23:31 -0600 Subject: [PATCH 30/55] Fixed build cache enabling via cmdline. --- stackinator/mirror.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index cad746c0..8a91162a 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -47,29 +47,30 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: if path.exists(): with path.open() as fid: # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.SafeLoader) + mirrors = yaml.load(fid, Loader=yaml.SafeLoader) # validate the yaml - schema.MirrorsValidator.validate(raw) - - mirrors = {name: mirror for name, mirror in raw.items() if mirror["enabled"]} + schema.MirrorsValidator.validate(mirrors) else: mirrors = {} # Add or set the cache given on the command line as the buildcache destination if cmdline_cache is not None: - existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache] # If the mirror name given on the command line isn't in the config, assume it # is the URL to a build cache. - if not existing_mirror: + if cmdline_cache in mirrors: mirrors['cmdline_cache'] = { 'url': cmdline_cache, 'description': "Cache configured via command line.", - 'enabled': True, 'cache': True, 'bootstrap': False, 'mount_specific': True, } + else: + # Enable the specified mirror and set it as the build cache dest + mirror = mirrors[cmdline_cache] + mirror['enabled'] = True + mirror['cache'] = True # Load the cache as defined by the deprecated 'cache.yaml' file. mirrors['legacy_cache_cfg'] = self._load_legacy_cache() @@ -81,7 +82,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: "Some of these may have come from a legacy 'cache.yaml' or the '--cache' option.\n" f"{self._pp_yaml(caches)}") - return mirrors + return {name: mirror for name, mirror in raw.items() if mirror["enabled"]} @staticmethod def _pp_yaml(object): @@ -116,7 +117,6 @@ def _load_legacy_cache(self): 'description': "Buildcache dest loaded from legacy cache.yaml", 'buildcache_push': True, 'mount_specific': True, - 'enabled': True, 'private_key': raw['key'], } From 4c995ff0274e191b57791470032ed9a7b6c5b227 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 15:26:57 -0600 Subject: [PATCH 31/55] Error handling --- stackinator/mirror.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 8a91162a..5643f2fc 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -45,12 +45,12 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" path = self._system_config_root/"mirrors.yaml" if path.exists(): - with path.open() as fid: - # load the raw yaml input - mirrors = yaml.load(fid, Loader=yaml.SafeLoader) - - # validate the yaml - schema.MirrorsValidator.validate(mirrors) + try: + with path.open() as fid: + # load the raw yaml input + mirrors = yaml.load(fid, Loader=yaml.SafeLoader) + except (OSError, PermissionError) as err: + raise MirrorError("Could not open/read mirrors.yaml file.\n{err}") else: mirrors = {} @@ -62,6 +62,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: mirrors['cmdline_cache'] = { 'url': cmdline_cache, 'description': "Cache configured via command line.", + 'enabled': True, 'cache': True, 'bootstrap': False, 'mount_specific': True, @@ -75,6 +76,15 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: # Load the cache as defined by the deprecated 'cache.yaml' file. mirrors['legacy_cache_cfg'] = self._load_legacy_cache() + + try: + # validate the yaml, including anything we added + schema.MirrorsValidator.validate(mirrors) + except ValueError as err: + raise MirrorError( + "Mirror config does not comply with schema.\n{err}" + ) + caches = [mirror for mirror in mirrors.values() if mirror['cache']] if len(caches) > 1: raise MirrorError( @@ -115,7 +125,8 @@ def _load_legacy_cache(self): mirror_cfg = { 'url': f'file://{raw['root']}', 'description': "Buildcache dest loaded from legacy cache.yaml", - 'buildcache_push': True, + 'cache': True, + 'enabled': True, 'mount_specific': True, 'private_key': raw['key'], } From 98f6407f3f76d1ce47d25507104880f2c5350489 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 15:46:38 -0600 Subject: [PATCH 32/55] Adding a unittest. --- stackinator/mirror.py | 21 ++++++++++----------- unittests/test_mirrors.py | 10 +++++++++- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 5643f2fc..f1dbb17b 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -54,11 +54,16 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: else: mirrors = {} + try: + schema.MirrorsValidator.validate(mirrors) + except ValueError as err: + raise MirrorError("Mirror config does not comply with schema.\n{err}") + # Add or set the cache given on the command line as the buildcache destination if cmdline_cache is not None: # If the mirror name given on the command line isn't in the config, assume it # is the URL to a build cache. - if cmdline_cache in mirrors: + if cmdline_cache not in mirrors: mirrors['cmdline_cache'] = { 'url': cmdline_cache, 'description': "Cache configured via command line.", @@ -74,16 +79,10 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: mirror['cache'] = True # Load the cache as defined by the deprecated 'cache.yaml' file. - mirrors['legacy_cache_cfg'] = self._load_legacy_cache() - + legacy_cache = self._load_legacy_cache() + if legacy_cache is not None: + mirrors['legacy_cache_cfg'] = legacy_cache - try: - # validate the yaml, including anything we added - schema.MirrorsValidator.validate(mirrors) - except ValueError as err: - raise MirrorError( - "Mirror config does not comply with schema.\n{err}" - ) caches = [mirror for mirror in mirrors.values() if mirror['cache']] if len(caches) > 1: @@ -92,7 +91,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: "Some of these may have come from a legacy 'cache.yaml' or the '--cache' option.\n" f"{self._pp_yaml(caches)}") - return {name: mirror for name, mirror in raw.items() if mirror["enabled"]} + return {name: mirror for name, mirror in mirrors.items() if mirror["enabled"]} @staticmethod def _pp_yaml(object): diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index bd96dce1..6579bb11 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -30,6 +30,14 @@ def test_mirror_init(systems_path, valid_mirrors): assert mir["enabled"] # test that cmdline_cache gets added to mirrors? +def test_command_line_cache(systems_path): + """Check that adding a cache from the command line works.""" + + mirrors = mirror.Mirrors(systems_path/'mirror-ok', cmdline_cache=systems_path.as_posix()) + + assert len(mirrors.mirrors) == 4 + + def test_create_spack_mirrors_yaml(systems_path): pass @@ -37,4 +45,4 @@ def test_create_bootstrap_configs(): pass def test_key_setup(): - pass \ No newline at end of file + pass From 780c43f86ab7607ce9ab58f9f386b4727de5201c Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 15:45:58 -0600 Subject: [PATCH 33/55] updated yaml formatting --- unittests/data/systems/mirror-ok/mirrors.yaml | 12 ++++++++---- unittests/test_mirrors.py | 4 +++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index e84fc3c3..f0030a45 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -1,7 +1,11 @@ -- url: https://google.com -- url: https://google.com +fake-mirror: + url: https://google.com +disabled-mirror: + url: https://google.com enabled: false -- url: https://cache.spack.io/ +buildcache-mirror: + url: https://cache.spack.io/ cache: true -- url: https://mirror.spack.io +bootstrap-mirror: + url: https://mirror.spack.io bootstrap: true \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 6579bb11..c1d46600 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -20,8 +20,10 @@ def valid_mirrors(systems_path): return mirrors def test_mirror_init(systems_path, valid_mirrors): - path = systems_path / "mirror_ok" + path = systems_path / "mirror-ok" + print(path) mirrors_obj = mirror.Mirrors(path) + print(mirrors_obj.mirrors.items()) assert mirrors_obj.mirrors == valid_mirrors assert mirrors_obj.mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors.values() if mirror.get('bootstrap')] assert mirrors_obj.mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors.values() if mirror.get('buildcache')] From da5781af59a3e71f0ccea7a96359c838aab982f7 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 15:56:15 -0600 Subject: [PATCH 34/55] Fixed error message. --- stackinator/mirror.py | 2 +- unittests/data/systems/mirror-basic/mirrors.yaml | 8 ++++++++ unittests/test_mirrors.py | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 unittests/data/systems/mirror-basic/mirrors.yaml diff --git a/stackinator/mirror.py b/stackinator/mirror.py index f1dbb17b..d9e1e803 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -84,7 +84,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: mirrors['legacy_cache_cfg'] = legacy_cache - caches = [mirror for mirror in mirrors.values() if mirror['cache']] + caches = {name: mirror for name, mirror in mirrors.items() if mirror['cache']] if len(caches) > 1: raise MirrorError( "Mirror config has more than one mirror specified as the build cache destination.\n" diff --git a/unittests/data/systems/mirror-basic/mirrors.yaml b/unittests/data/systems/mirror-basic/mirrors.yaml new file mode 100644 index 00000000..ee93f2c7 --- /dev/null +++ b/unittests/data/systems/mirror-basic/mirrors.yaml @@ -0,0 +1,8 @@ +fake-mirror: + url: https://google.com +disabled-mirror: + url: https://google.com + enabled: false +bootstrap-mirror: + url: https://mirror.spack.io + bootstrap: true diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index c1d46600..acd360f1 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -35,9 +35,9 @@ def test_mirror_init(systems_path, valid_mirrors): def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" - mirrors = mirror.Mirrors(systems_path/'mirror-ok', cmdline_cache=systems_path.as_posix()) + mirrors = mirror.Mirrors(systems_path/'mirror-basic', cmdline_cache=systems_path.as_posix()) - assert len(mirrors.mirrors) == 4 + assert len(mirrors.mirrors) == 3 def test_create_spack_mirrors_yaml(systems_path): From 84666d63db33cee93840578a4f2de9cb692645e8 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 15:57:31 -0600 Subject: [PATCH 35/55] debugging --- unittests/test_mirrors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index acd360f1..b374aca4 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -21,9 +21,9 @@ def valid_mirrors(systems_path): def test_mirror_init(systems_path, valid_mirrors): path = systems_path / "mirror-ok" - print(path) + #print(path) mirrors_obj = mirror.Mirrors(path) - print(mirrors_obj.mirrors.items()) + #print(mirrors_obj.mirrors.items()) assert mirrors_obj.mirrors == valid_mirrors assert mirrors_obj.mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors.values() if mirror.get('bootstrap')] assert mirrors_obj.mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors.values() if mirror.get('buildcache')] From 602244cc47b1e0754aa068b4541c84c10be4b66a Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 16:02:18 -0600 Subject: [PATCH 36/55] Minor fix. --- stackinator/mirror.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index d9e1e803..88168476 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -50,14 +50,14 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: # load the raw yaml input mirrors = yaml.load(fid, Loader=yaml.SafeLoader) except (OSError, PermissionError) as err: - raise MirrorError("Could not open/read mirrors.yaml file.\n{err}") + raise MirrorError(f"Could not open/read mirrors.yaml file.\n{err}") else: mirrors = {} try: schema.MirrorsValidator.validate(mirrors) except ValueError as err: - raise MirrorError("Mirror config does not comply with schema.\n{err}") + raise MirrorError(f"Mirror config does not comply with schema.\n{err}") # Add or set the cache given on the command line as the buildcache destination if cmdline_cache is not None: @@ -84,7 +84,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: mirrors['legacy_cache_cfg'] = legacy_cache - caches = {name: mirror for name, mirror in mirrors.items() if mirror['cache']] + caches = {name: mirror for name, mirror in mirrors.items() if mirror['cache']} if len(caches) > 1: raise MirrorError( "Mirror config has more than one mirror specified as the build cache destination.\n" From eba1f1373db74037b395516389d330db3a056c8a Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 16:06:49 -0600 Subject: [PATCH 37/55] Fixing urls. --- unittests/data/systems/mirror-basic/mirrors.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unittests/data/systems/mirror-basic/mirrors.yaml b/unittests/data/systems/mirror-basic/mirrors.yaml index ee93f2c7..2667cbc3 100644 --- a/unittests/data/systems/mirror-basic/mirrors.yaml +++ b/unittests/data/systems/mirror-basic/mirrors.yaml @@ -1,8 +1,8 @@ fake-mirror: - url: https://google.com + url: https://github.com disabled-mirror: - url: https://google.com + url: /tmp/ enabled: false bootstrap-mirror: - url: https://mirror.spack.io + url: https://github.com bootstrap: true From 3eadc82aec5c718d13b419d3746564f1b95e0ee2 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 16:07:37 -0600 Subject: [PATCH 38/55] fixed url for testing --- unittests/data/systems/mirror-ok/mirrors.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index f0030a45..06f2e329 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -1,10 +1,10 @@ fake-mirror: - url: https://google.com + url: https://github.com disabled-mirror: - url: https://google.com + url: https://github.com enabled: false buildcache-mirror: - url: https://cache.spack.io/ + url: https://mirror.spack.io cache: true bootstrap-mirror: url: https://mirror.spack.io From 2fee464768fb65f75cd5440c1c82c6a31487a60f Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 16:34:52 -0600 Subject: [PATCH 39/55] validated mirror tests --- unittests/test_mirrors.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index b374aca4..fdc4e7b3 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -14,23 +14,21 @@ def systems_path(test_path): @pytest.fixture def valid_mirrors(systems_path): mirrors = {} - mirrors["fake-mirror"] = {'url': 'https://google.com', 'enabled': True, 'bootstrap': False, 'cache': False, 'mount_specific': False} - mirrors["buildcache-mirror"] = {'url': 'https://cache.spack.io/', 'enabled': True, 'bootstrap': False, 'cache': True, 'mount_specific': False} + mirrors["fake-mirror"] = {'url': 'https://github.com', 'enabled': True, 'bootstrap': False, 'cache': False, 'mount_specific': False} + mirrors["buildcache-mirror"] = {'url': 'https://mirror.spack.io', 'enabled': True, 'bootstrap': False, 'cache': True, 'mount_specific': False} mirrors["bootstrap-mirror"] = {'url': 'https://mirror.spack.io', 'enabled': True, 'bootstrap': True, 'cache': False, 'mount_specific': False} return mirrors def test_mirror_init(systems_path, valid_mirrors): path = systems_path / "mirror-ok" - #print(path) mirrors_obj = mirror.Mirrors(path) - #print(mirrors_obj.mirrors.items()) + assert mirrors_obj.mirrors == valid_mirrors - assert mirrors_obj.mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors.values() if mirror.get('bootstrap')] - assert mirrors_obj.mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors.values() if mirror.get('buildcache')] - # assert disabled mirror not in mirrors + assert mirrors_obj.bootstrap_mirrors == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('bootstrap')] + assert mirrors_obj.build_cache_mirror == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('cache')].pop(0) + for mir in mirrors_obj.mirrors: - assert mir["enabled"] - # test that cmdline_cache gets added to mirrors? + assert mirrors_obj.mirrors[mir].get('enabled') def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" From efd5628f23a9a7d48f762eb8d5dbe3bad950e9d9 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 16:37:02 -0600 Subject: [PATCH 40/55] added test description --- unittests/test_mirrors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index fdc4e7b3..ecc79082 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -20,6 +20,7 @@ def valid_mirrors(systems_path): return mirrors def test_mirror_init(systems_path, valid_mirrors): + """Check that Mirror objects are initialized correctly.""" path = systems_path / "mirror-ok" mirrors_obj = mirror.Mirrors(path) From 5c5faed6216f1a90acd1c77579b6b9f209a4aa9d Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 10:20:00 -0600 Subject: [PATCH 41/55] Adding unit tests. --- stackinator/mirror.py | 105 ++++++++---------- .../data/systems/mirror-basic/cache.yaml | 2 + unittests/data/test-gpg-priv.asc | Bin 0 -> 4924 bytes unittests/data/test-gpg-pub.asc | Bin 0 -> 2250 bytes unittests/test_mirrors.py | 17 ++- 5 files changed, 63 insertions(+), 61 deletions(-) create mode 100644 unittests/data/systems/mirror-basic/cache.yaml create mode 100644 unittests/data/test-gpg-priv.asc create mode 100644 unittests/data/test-gpg-pub.asc diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 88168476..a5191865 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -19,8 +19,9 @@ class Mirrors: KEY_STORE_DIR = 'key_store' MIRRORS_YAML = 'mirrors.yaml' + CMDLINE_CACHE = 'cmdline_cache' - def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None, + def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[pathlib.Path] = None, mount_point: Optional[pathlib.Path] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" @@ -32,16 +33,22 @@ def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str self.mirrors = self._load_mirrors(cmdline_cache) self._check_mirrors() - self.build_cache_mirror : Optional[str] = \ - ([name for name, mirror in self.mirrors.items() if mirror.get('cache', False)] - + [None]).pop(0) + # Always use the cache given on the command line + if self.CMDLINE_CACHE in self.mirrors: + self.build_cache_mirror = self.CMDLINE_CACHE + else: + # Otherwise, grab the configured cache (or None) + self.build_cache_mirror : Optional[str] = \ + ([name for name, mirror in self.mirrors.items() if mirror.get('cache', False)] + + [None]).pop(0) + self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() if mirror.get('bootstrap', False)] # Will hold a list of all the gpg keys (public and private) self._keys: Optional[List[pathlib.Path]] = [] - def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: + def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" path = self._system_config_root/"mirrors.yaml" if path.exists(): @@ -59,38 +66,16 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: except ValueError as err: raise MirrorError(f"Mirror config does not comply with schema.\n{err}") - # Add or set the cache given on the command line as the buildcache destination - if cmdline_cache is not None: - # If the mirror name given on the command line isn't in the config, assume it - # is the URL to a build cache. - if cmdline_cache not in mirrors: - mirrors['cmdline_cache'] = { - 'url': cmdline_cache, - 'description': "Cache configured via command line.", - 'enabled': True, - 'cache': True, - 'bootstrap': False, - 'mount_specific': True, - } - else: - # Enable the specified mirror and set it as the build cache dest - mirror = mirrors[cmdline_cache] - mirror['enabled'] = True - mirror['cache'] = True - - # Load the cache as defined by the deprecated 'cache.yaml' file. - legacy_cache = self._load_legacy_cache() - if legacy_cache is not None: - mirrors['legacy_cache_cfg'] = legacy_cache - - caches = {name: mirror for name, mirror in mirrors.items() if mirror['cache']} if len(caches) > 1: raise MirrorError( "Mirror config has more than one mirror specified as the build cache destination.\n" - "Some of these may have come from a legacy 'cache.yaml' or the '--cache' option.\n" f"{self._pp_yaml(caches)}") + # Load the cache as defined by the deprecated 'cache.yaml' file. + if cmdline_cache is not None: + mirrors[self.CMDLINE_CACHE] = self._load_cmdline_cache(cmdline_cache) + return {name: mirror for name, mirror in mirrors.items() if mirror["enabled"]} @staticmethod @@ -101,40 +86,42 @@ def _pp_yaml(object): yaml.dump(object, example_yaml_stream, default_flow_style=False) return example_yaml_stream.getvalue() - def _load_legacy_cache(self): - """Load the mirror definition from the legacy cache.yaml file.""" - - cache_config_path = self._system_config_root/'cache.yaml' - - if cache_config_path.is_file(): - - with cache_config_path.open('r') as file: - try: - raw = yaml.load(file, Loader=yaml.SafeLoader) - except ValueError as err: - raise MirrorError( - f"Error loading yaml from cache config at '{cache_config_path}'\n{err}") + def _load_cmdline_cache(self, cache_config_path: pathlib.Path) -> Dict: + """Load the mirror definition from the legacy 'cache.yaml' file.""" + if not cache_config_path.is_file(): + raise MirrorError( + f"Binary cache configuration path given on the command line '{cache_config_path}' " + f"does not exist.") + + with cache_config_path.open('r') as file: try: - schema.CacheValidator.validate(raw) + raw = yaml.load(file, Loader=yaml.SafeLoader) except ValueError as err: raise MirrorError( - f"Error validating contents of cache config at '{cache_config_path}'.\n{err}") - - mirror_cfg = { - 'url': f'file://{raw['root']}', - 'description': "Buildcache dest loaded from legacy cache.yaml", - 'cache': True, - 'enabled': True, - 'mount_specific': True, - 'private_key': raw['key'], - } + f"Error loading yaml from cache config at '{cache_config_path}'\n{err}") + + try: + schema.CacheValidator.validate(raw) + except ValueError as err: + raise MirrorError( + f"Error validating contents of cache config at '{cache_config_path}'.\n{err}") + + mirror_cfg = { + 'url': raw['root'], + 'description': "Buildcache dest loaded from legacy cache.yaml", + 'cache': True, + 'enabled': True, + 'bootstrap': False, + 'mount_specific': True, + 'private_key': raw['key'], + } - self._logger.warning("Configuring the buildcache from the system cache.yaml file.\n" - "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" - f"The equivalent 'mirrors.yaml' would look like: \n{self._pp_yaml([mirror_cfg])}") + self._logger.warning("Configuring the buildcache from the system cache.yaml file.\n" + "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" + f"The equivalent 'mirrors.yaml' would look like: \n{self._pp_yaml([mirror_cfg])}") - return mirror_cfg + return mirror_cfg def _check_mirrors(self): """Validate the mirror config entries.""" diff --git a/unittests/data/systems/mirror-basic/cache.yaml b/unittests/data/systems/mirror-basic/cache.yaml new file mode 100644 index 00000000..ad7de37c --- /dev/null +++ b/unittests/data/systems/mirror-basic/cache.yaml @@ -0,0 +1,2 @@ +root: /tmp/foo +key: ../../test-gpg-priv.asc diff --git a/unittests/data/test-gpg-priv.asc b/unittests/data/test-gpg-priv.asc new file mode 100644 index 0000000000000000000000000000000000000000..eaa2dc19eb7ecaff70e1c1cf53e66825a6996425 GIT binary patch literal 4924 zcmajhWgr{?pull=>g37OoSW`;YI?f6CZ=O@;>4L|Om}R0dYHDc>0#<{dYb8&Ht)R; zFTTC+|L^}_ChiL?mv-8F00~C=s0g;qrC~&pac&Qi3xB%d4BI$n%GzhUu9pzu3NJ9U z(wLyd`_8QFj#lg=2y|QSH@51OU|Bq>B#$00$+%r zsLiq6J>_j9-lyM&p)&t0kE?4>7U}w1E*wsLi39&OcFVam*(SCy^ybA$*Q`h{w^0TC z!rsU^&HfAC5pS0EWko`NRF7*w^Bq*O8_SplGc$}6j|P zQl|HmHq+0TRjw!4wqqv(IM60Z=_RVF3n>okAA5WFOLuO#nB*Op6gAjt0&uk=Joi_wxL?hDEhOUHN!?AnR3oy*7vLa|wdQ6Scs(lh@=Vd&RRqtU6n~O0yVEaOzXjbe^!#e?_RFZ<cQ=Rt3jfU6%{Z1%t{^H`G^<^osKj}Nb96uAA&pU%gI~2y z$1WI_C|NtaqF|&dcjZKW2G8b))P>vO3p_Za%JvfxCC}$H_&%jEH(IyeNGOO>X*H}* zFp<4kQ*-{|>n2RLh1s$JdXb4MuS;qvI3q9(!T%d|yKcZ!O^dxT5vgS{3UFrf&-LE# z5>C4X<-Yfy2r+5GCDVIDYi3D@#@LEf7x_g&(c@=@$(y3CM<(=Cz;KMT`xtR3r$XrM zB!c+h=;p^d2v^Q}RbxM;2C(A3U8@8|TV9H|M$@ZT`zK_t*iIJgl`b3ZEAGrn2FKZnR|88d<@%6_gK+`$* z$l+TjqN&}|HF9bl8F|83W$XKEfm8_z7uFT_+FpNkQqCI zcqfSw!@pOO@IeCXXgJ=_gTH1MpQSz2i{la(A5W9b)nPQ3r=&azj7d0E95wK}33;44 zC(@buEB6ZvfrKPqj_jJce!foh=)`Mi;i^Dnel)Sgk{kNKs!bNOswyT%V>4X)vb89$ zmfgCnl$e16H3lPzmkkG`E zFLZ_uzJ5>*hajl*Yfmo+x!0Zn&JI2fe6Kt`p2q|Au*d=U7*Y^stRj#OJcf|#{Lh}n zAzXp*^`sP495(R39t@3^YtInHX?_x5qO8muaA6_ogq z-1H3ORrJ^W(LI_HVWspRNYR>3!x?+A&e0bvpr^c%RL%efeT}f`%`d3~?L@DQ*Vt#) zSY%ferQeu)XIfKNs@}}#kO_$Jl#nExRiE#sZ+3^uuaYE8S0i<%THWv;#lkUB%(ubW zaYXB#WN&SGLiy|-N*)~K2JP7%A%}=fvBdcf6(k@F1k8V@UusKsW6`~6?CQQZw8FEC z_Xex5oZv(A(vr(|pG&a>y8z$qXH4d8)q<5_6My@(Jvv!ZTcv&M z1`b+XRKm@y*-`t`!D(b%8xhlZyC6y%8QJwS^7=HL!FoZKOQ6M?juy&R_`h0`!L zVo8plb?(Tz_3sGYpDEB`w~I)&N#k@6N6Wri&5WroFM2!BJ4ix_9tseQAbO3o8p|Uwrm2xJLj;&&F6m^X{P{yh)KXs3wwiykQsL z87StZ{N8ihIgj9*sa_z-RF6IkkEwQID@#PHKLUjhKjGuzuNV@1xLAU>v1Oe|@*+Eu z(z}OYEhRTQOMGeT)B+~`WC4LDoPNB}XuHP?MMgDcw&R_>ZRM%ml8i5U+qr_)C8Yi| z2f$1SO0wv|n>c3-IU=r@QF||geHAtJeZicK%GRnNwYd}5&CP$`?%e94Qn)i|4`o%$dlxmL(VCSd8GZ8jgvyw6jh zAQ!O(zwDwBmi0gtrksyd+6U`w|32=VC9@rAlA0sUWWl2ckEko1J;JNV#Qod6XW_~W zdcmt*MAXXz=0Zn|QJ$iO^5X^;uMZWDqeK(;&0rDnWb=|9IDJc8ach-#G;Cp!)G``Q zTSCmJj5O&FJ`*a9_f3G#-{ZO!9(|^p?g7vpGe;pN#rk~KycaR6W%Me)E>=I`I)%A4 zr7dL{yQO|pAE<2o=Ttc2wkL)?;kVuq?U;ly8aq6GGO@OqX0^BWKXMcE8he|`Qc(I7 zwG<4EjFC%pk?D1!x8h< z&S5>fS5RHWf+8WAVc217uv6VK#oRpf{RL@#%uP6bj2_0X=;%@V^m+0f<0-hU`z$rr z5;*vGgachu-Ko>?l^`){;J((-{ZC^tdos6tF-nSx86g0ZnAjA@69ceHxPxRb*6YAVfxDd`4l-`fb`}B4o5m>r_@}=t65x(XKmoS5u#@^jF+e z)>n1m83 z(MMTi-7WDZW+bvQx)HkayO9IuNJZ3ccuUTDMSf=syJrtpgyphx=f1!D!0gK(^s{`Z zo4pcWkgoqqK=G2{zQiZO1eMmJDsX{1d8qxWSCY~hX=HvJh4<+Pn`o*=?Rrj$u~7N? z8s*k|2Rnc&cHx(Y$j^2xy(F-}4V5g-Zyub^-53c8lq`PjF8xdJSIcjxcX4%bLuA)z zrWdr>Yk#pRNjzdG@jewWvyQBijb(d9jw4&QP7JJMzh`SH7@itP&8rzXkl6eNZw;#(}9hlQVk%X2VaqPW1Q^+^L+1xKYoefzi`*pKD0>USw z8Kt13`w8$fwAQJZV$%qKZeax^U*oH_eALymt={X>*Y9=fB{f3UF?_lEB|z3^XhmiS z9gS%p=*c0t+#h@!vAf|w>5>0>*>hergIi?=ebRB%Jv)9UPm$-Op%G zbCv#2yv})}v3=b(zT$ORmK<)gxNfVV-Qo}lPw1rZP_}`;d2FX*`qO4}Yq0WBjj8a* znKD?YsK~mEU*>kaZrIrga&{8BK~iHwzeJ0QI{r-flXg6aDZxTSxP~ci198cgyWE+@spgp zcIL70HcdIu60_IfVC&3?yO+(xZ5B5kD27G-pE!g5KR6@&zXCI10zX*_*1P1GlMxc` z%)?R=S07x@X1-m-&$Ls}mDr_yHYTn^x0Au)C^OITfQf-9;BZL%?G@?lct-tV@IbLTNfTbCO+!d8^X)gQ+>Q z#Lu~@3184=2Ujjy5t#O2!)*1EH6$jV@0#)!=HB*j({^71ls&())~&Qy7~rT18YWCp zs#=itG`?Ojr(te6_%{=a%B)o>B+5VaSlS{dK5l9l_}Vna(t0uKw1 z_(B$aekP4YwSd{abSfNJ;;;n~PMSY3^QHYs91id)+1}=S<~kWXprhLrK}E7~V#fIg z58^xVG2h@x6p~;-3$z|+0K;$vqq&`(8tCX84 zopfONbdlh~doqjgsF%flDZQ`ZFG@Y)tr^&cbh_5SOC5~r901&0XgdQG&nen4h#)LQ mZ>7vztr91~Oq1#)>Ed{1^SU?%yx#0qXiGx|!qiao5&sW-Z9mQc literal 0 HcmV?d00001 diff --git a/unittests/data/test-gpg-pub.asc b/unittests/data/test-gpg-pub.asc new file mode 100644 index 0000000000000000000000000000000000000000..aa72b0281f39bd4ad4d0281742b9133f19e84c6a GIT binary patch literal 2250 zcmV;*2sQVa0u2OdxElKb5CD_QGzC5MRDz9GnZJK33evCf1y+X9UvlNGzzlk7IGj{n zOV1h!!68L#*h9k$cx&9mR`YBM1Y`iIP7^GpgRmCGb0J4Jm4p-S4@~@{L_B=$sif8l zp%P+j!xq+yV=FoV1Nqpa0oxriYr{=<+#z>ny;aTf4|~0-Zz8Hnu?{m>wKqL3j??fl z<%FL7OsTwod^ACtFi#WMhm%0?XoZjydMC(cgMUkUK(&wsqSzcG4>V<$UvpIpSbqPT z;xFrpnx-(?c+JH<-0XElO&weapA%^0u&yeLenM?y9Ws+8e~R^JVXI4tVvkyg+HPVT`pMz?z6WtKB@@w-%_Sm-P?kH?ur*C4V5os?AFvn`K9rv7{j}m6TSD?Zrjb`lrJ3IY3SN6hh<=3)XA< z%oP87l$5JNg$gvLxG)?62YWmXry(I_mpt=?NVlFtG^u4D)gGwa6j-WRU~#vpTvhsr zyD{m@c&p#C4yRNi^aw=l2hSZj>H7p%!Dc^AM$FNuiB-AK#eeQ#sH8N1QTF3pKc9)m zC_potxh89e74ZNO0RREC8&qX;bRbJ*c_2J)Z*XNmZf|#JWpZUMV{dIfi2_js69EbU zI~E}Xo(NAkh72n5>gZeSI6@3=&^s511p;Zf8v6np0|g5S2nPZJA_4{#3JC}c0t6NU z0|5da2Lcy>0162ZI6@3=&^s517TOQ~Fzm@df0V8}8rvUt>$u+J@YC^+3ooI-cT{RQ z102Z9UjpyK4q|Ra*hD@qnCvS~ZPk$V?axJ8 ztf8|Lll35Ym}p6t`(%juOC?8ZrWFc+VNktuf_QcHf)C@q7aJls`BTP zwrdLgHh=@vA^Cfkhz;So5`keXeJ*1BqWon*#$+b_7VLWCHjLM~ zM}Gm92m+gnQlnuK_*=Z%R%&eM$K5RF33UXWr3;_4+tF;uVhwF^f?F@o!gD={h1(Qr zCd_uB*sgyb46eGl%MlUNy911@)MbHs#6KzvkU)p8@eM&fr@;=X%?Gli#6B8)TVk6dPBa-Cs09bef$DloNQ8yX-cxdIIYX}B8u0T2Mw!yyGkB#(*JXt|d8cDZ}$)0|HyVuFug zk|}I}*gs?5m_Jx?uSXeS=gmkgAA*Fv(G#sd)Q{M=qwcXal67A_20}uX`(MLCYJ9yEq}RjIS))HzE7-iTnb=B{}%K(-d*l>yuSVV5o>*K+@OF)9vQFF)E<1YgCe13+>+lAm9f zcEc-}+gav35UcG>k1_51ZG`qu=nVFQjOqEBBY;I6QG4FI4H(?TS~Be(gl{#SLCsTJ zZtOztgf)!mSt5do64yFy2E*Gbh~KCB876*%7(Gj8gZeSI6@3= z&^s511p;Zf8v6np3;+rV5I8~%ZqPdyh*uvE|6DPyX_#6P3=HPiB9M&U{A%&jzVnIG zxD*?rVv#jON;)90f_FqOb9eHW^ne8DNQSMgF?Lni39?HvFhVm?qh_&)hXdSLsu_YO zYDlSPq1fYWHQBHh56tFl8K(Rt+s1Y>l27>7I=h{k-aKT+k(kvEJ}VduMjDrUr`lZ$ zSi=P(VNs7uInX}mSS_8I`M)a~yz~J@Z@ne0-L+d&21qefiqREFTM@spZrxfLA+_uN z(}&cRtVf*V%#EHaqgX6R{A9prF3BhAfm~5?49i{eA;r#jm(6#4<@v_ zLhD=xCV33dTKpj{lIo1ecXFcT*1vGtGISxuva>l520ckLGgv#2gy^}{Y9rh-V$@n z`0UYwH<4mp;FwQ8)cZ;rih#Do$OI1i@+f9@O5QWvKsJ8Lx?Jb0zjo-x^u5?g?T zUZHf5z-~B3qiu;_9VW+5yWbDnxgJU=0SB2NxE&w~XN6RG3x1GM-K23zrv`#qOc(Es Y^X}8*_o0vv;j?R Date: Mon, 16 Mar 2026 10:44:23 -0600 Subject: [PATCH 42/55] Unit test tweaking. --- stackinator/mirror.py | 6 +++++- unittests/data/systems/mirror-ok/cache.yaml | 2 ++ unittests/data/systems/mirror-ok/mirrors.yaml | 3 ++- unittests/test_mirrors.py | 13 ++++--------- 4 files changed, 13 insertions(+), 11 deletions(-) create mode 100644 unittests/data/systems/mirror-ok/cache.yaml diff --git a/stackinator/mirror.py b/stackinator/mirror.py index a5191865..a423833c 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -66,11 +66,15 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict except ValueError as err: raise MirrorError(f"Mirror config does not comply with schema.\n{err}") - caches = {name: mirror for name, mirror in mirrors.items() if mirror['cache']} + caches = [name for name, mirror in mirrors.items() if mirror['cache']] if len(caches) > 1: raise MirrorError( "Mirror config has more than one mirror specified as the build cache destination.\n" f"{self._pp_yaml(caches)}") + elif caches: + cache = mirrors[caches[0]] + if not cache.get('private_key'): + raise MirrorError(f"Mirror build cache config '{caches[0]}' missing a required 'private_key' path.") # Load the cache as defined by the deprecated 'cache.yaml' file. if cmdline_cache is not None: diff --git a/unittests/data/systems/mirror-ok/cache.yaml b/unittests/data/systems/mirror-ok/cache.yaml new file mode 100644 index 00000000..ad7de37c --- /dev/null +++ b/unittests/data/systems/mirror-ok/cache.yaml @@ -0,0 +1,2 @@ +root: /tmp/foo +key: ../../test-gpg-priv.asc diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index 06f2e329..21e1e3b4 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -5,7 +5,8 @@ disabled-mirror: enabled: false buildcache-mirror: url: https://mirror.spack.io + private_key: '../test-gpg-priv.asc' cache: true bootstrap-mirror: url: https://mirror.spack.io - bootstrap: true \ No newline at end of file + bootstrap: true diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 981fe744..c5bede3e 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -34,10 +34,11 @@ def test_mirror_init(systems_path, valid_mirrors): def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" - mirrors = mirror.Mirrors(systems_path/'mirror-basic', - cmdline_cache=systems_path/'mirror-basic/cache.yaml') + mirrors = mirror.Mirrors(systems_path/'mirror-ok', + cmdline_cache=systems_path/'mirror-ok/cache.yaml') - assert len(mirrors.mirrors) == 3 + assert len(mirrors.mirrors) == 4 + # This should always be the build cache even though one is already defined. assert mirrors.build_cache_mirror == 'cmdline_cache' cache_mirror = mirrors.mirrors['cmdline_cache'] assert cache_mirror['url'] == '/tmp/foo' @@ -46,12 +47,6 @@ def test_command_line_cache(systems_path): assert not cache_mirror['bootstrap'] assert cache_mirror['mount_specific'] -def test_multi_buildcache(systems_path): - """Make sure we throw appropriate errors when there's more than one build cache defined.""" - - - - def test_create_spack_mirrors_yaml(systems_path): pass From cb2be1136c77f8ee504dfb0758fc5f582cace978 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 11:04:25 -0600 Subject: [PATCH 43/55] Added unittests for key setup --- stackinator/mirror.py | 2 +- unittests/data/systems/mirror-ok/mirrors.yaml | 43 +++++++++++++++++++ unittests/test_mirrors.py | 17 +++++++- 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index a423833c..7296389e 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -267,7 +267,7 @@ def _key_setup(self, key_store: pathlib.Path): file_type = magic.from_buffer(binary_key, mime=True) print("magic type:" , file_type) - if file_type != "application/x-gnupg-keyring": + if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): raise MirrorError( f"Key for mirror {name} is not a valid GPG key. \n" f"Check the key listed in mirrors.yaml in system config.") diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index 21e1e3b4..96fc72f5 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -1,11 +1,54 @@ fake-mirror: url: https://github.com + public_key: ../../test-gpg-pub.asc disabled-mirror: url: https://github.com enabled: false buildcache-mirror: url: https://mirror.spack.io private_key: '../test-gpg-priv.asc' + public_key: | + mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML + eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB + 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF + kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK + IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb + WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh + jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA + TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3 + qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH + KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe + CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK + CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+ + LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i + qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr + HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1 + WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i + /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x + glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE + PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW + lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw + kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe + mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm + HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH + Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0 + UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF + zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx + Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y + 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H + sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr + 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q + OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR + NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw + Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z + +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx + 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7 + xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6 + lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd + eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8 + eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG + gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3 + cache: true bootstrap-mirror: url: https://mirror.spack.io diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index c5bede3e..3b068c90 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -53,5 +53,18 @@ def test_create_spack_mirrors_yaml(systems_path): def test_create_bootstrap_configs(): pass -def test_key_setup(): - pass +def test_key_setup(systems_path, tmp_path): + """Check that public keys are set up properly.""" + + mirrors = mirror.Mirrors(systems_path/'mirror-ok') + + mirrors._key_setup(tmp_path) + + key_files = list(tmp_path.iterdir()) + assert {key_file.name for key_file in key_files} == {'buildcache-mirror.gpg', 'fake-mirror.gpg'} + # The two files should be identical in content + key_file_data = [] + for key_file in key_files: + with key_file.open('rb') as file: + key_file_data.append(file.read()) + assert key_file_data[0] == key_file_data[1] From e386498cb7b669d6e0a202c4f06a43fba5927e5e Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 12:16:45 -0600 Subject: [PATCH 44/55] Added test for bad keys. --- stackinator/mirror.py | 11 +++-------- unittests/data/systems/mirror-bad-key/mirrors.yaml | 4 ++-- .../data/systems/mirror-bad-keypath/mirrors.yaml | 4 ++-- unittests/data/systems/mirror-basic/cache.yaml | 2 -- unittests/data/systems/mirror-basic/mirrors.yaml | 8 -------- unittests/test_mirrors.py | 14 ++++++++++++++ 6 files changed, 21 insertions(+), 22 deletions(-) delete mode 100644 unittests/data/systems/mirror-basic/cache.yaml delete mode 100644 unittests/data/systems/mirror-basic/mirrors.yaml diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 7296389e..317deead 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -246,12 +246,7 @@ def _key_setup(self, key_store: pathlib.Path): #try prepending system config path path = self._system_config_root/path - if path.exists(): - if not path.is_file(): - raise MirrorError( - f"The key path '{path}' is not a file. \n" - f"Check the key listed in mirrors.yaml in system config.") - + if path.is_file(): with open(path, 'rb') as reader: binary_key = reader.read() @@ -261,15 +256,15 @@ def _key_setup(self, key_store: pathlib.Path): binary_key = base64.b64decode(key) except ValueError: raise MirrorError( - f"Key for mirror '{name}' is not valid. \n" + f"Key for mirror '{name}' is not valid: '{path}'. \n" f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" f"Check the key listed in mirrors.yaml in system config.") file_type = magic.from_buffer(binary_key, mime=True) - print("magic type:" , file_type) if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): raise MirrorError( f"Key for mirror {name} is not a valid GPG key. \n" + f"The file (or base64) was readable, but the data itself was not a PGP key.\n" f"Check the key listed in mirrors.yaml in system config.") # copy key to new destination in key store diff --git a/unittests/data/systems/mirror-bad-key/mirrors.yaml b/unittests/data/systems/mirror-bad-key/mirrors.yaml index ed27df7a..d5154dd3 100644 --- a/unittests/data/systems/mirror-bad-key/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-key/mirrors.yaml @@ -1,3 +1,3 @@ -- name: bad-key +bad-key: url: https://mirror.spack.io - public_key: /bad_key.gpg \ No newline at end of file + public_key: bad_key.gpg diff --git a/unittests/data/systems/mirror-bad-keypath/mirrors.yaml b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml index e671c45a..3433e04a 100644 --- a/unittests/data/systems/mirror-bad-keypath/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml @@ -1,3 +1,3 @@ -- name: bad-key-path +bad-key-path: url: https://mirror.spack.io - public_key: /path/doesnt/exist \ No newline at end of file + public_key: /path/doesnt/exist diff --git a/unittests/data/systems/mirror-basic/cache.yaml b/unittests/data/systems/mirror-basic/cache.yaml deleted file mode 100644 index ad7de37c..00000000 --- a/unittests/data/systems/mirror-basic/cache.yaml +++ /dev/null @@ -1,2 +0,0 @@ -root: /tmp/foo -key: ../../test-gpg-priv.asc diff --git a/unittests/data/systems/mirror-basic/mirrors.yaml b/unittests/data/systems/mirror-basic/mirrors.yaml deleted file mode 100644 index 2667cbc3..00000000 --- a/unittests/data/systems/mirror-basic/mirrors.yaml +++ /dev/null @@ -1,8 +0,0 @@ -fake-mirror: - url: https://github.com -disabled-mirror: - url: /tmp/ - enabled: false -bootstrap-mirror: - url: https://github.com - bootstrap: true diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 3b068c90..5b4a0a89 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -68,3 +68,17 @@ def test_key_setup(systems_path, tmp_path): with key_file.open('rb') as file: key_file_data.append(file.read()) assert key_file_data[0] == key_file_data[1] + +@pytest.mark.parametrize("system_name", [ + 'mirror-bad-key', + 'mirror-bad-keypath', +]) +def test_key_setup_bad_key(tmp_path, systems_path, system_name): + """asdfasdf""" + + mirrors = mirror.Mirrors(systems_path/system_name) + with pytest.raises(mirror.MirrorError): + mirrors._key_setup(tmp_path) + + + From f3759c72b60c016aa5709c40a78dc9934f21525a Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Mon, 16 Mar 2026 11:17:52 -0600 Subject: [PATCH 45/55] added more mirror tests --- stackinator/mirror.py | 4 +-- unittests/test_mirrors.py | 61 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 317deead..502943ae 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -206,14 +206,14 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): bootstrap_yaml['sources'].append( { 'name': name, - 'metadata': bs_mirror_path, + 'metadata': str(bs_mirror_path), } ) # And trust each one bootstrap_yaml['trusted'][name] = True # Create the metadata dir and metadata.yaml - bs_mirror_path.mkdir(parents=True) + bs_mirror_path.mkdir(parents=True, exist_ok=True) bs_mirror_yaml = { 'type': 'install', 'info': mirror['url'], diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 5b4a0a89..8e189228 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -48,10 +48,65 @@ def test_command_line_cache(systems_path): assert cache_mirror['mount_specific'] def test_create_spack_mirrors_yaml(systems_path): - pass + """Check that the mirrors.yaml passed to spack is correct""" + + valid_spack_yaml = { + "mirrors": { + "fake-mirror": { + "fetch": {"url": "https://github.com"}, + "push": {"url": "https://github.com"}, + }, + "buildcache-mirror": { + "fetch": {"url": "https://mirror.spack.io"}, + "push": {"url": "https://mirror.spack.io"}, + }, + "bootstrap-mirror": { + "fetch": {"url": "https://mirror.spack.io"}, + "push": {"url": "https://mirror.spack.io"}, + } + } + } + + dest = systems_path / "mirror-ok" / "test_output.yaml" + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok") + mirrors_obj._create_spack_mirrors_yaml(dest) + + with dest.open() as f: + data = yaml.safe_load(f) + + assert data == valid_spack_yaml + +def test_create_bootstrap_configs(systems_path): + """Check that spack bootstrap configs are generated correctly""" + + valid_yaml = { + "sources": [ + { + "name": "bootstrap-mirror", + "metadata": str(systems_path / "mirror-ok" / "bootstrap" / "bootstrap-mirror"), + } + ], + "trusted": { + "bootstrap-mirror": True + }, + } + valid_metadata = { + "type": "install", + "info": "https://mirror.spack.io", + } + + path = systems_path / "mirror-ok" + bs_mirror_path = path / "bootstrap/bootstrap-mirror" + mirrors_obj = mirror.Mirrors(path) + mirrors_obj._create_bootstrap_configs(path) + + with (path/'bootstrap.yaml').open() as f: + bs_data = yaml.safe_load(f) + assert bs_data == valid_yaml -def test_create_bootstrap_configs(): - pass + with (bs_mirror_path/'metadata.yaml').open() as f: + metadata = yaml.safe_load(f) + assert metadata == valid_metadata def test_key_setup(systems_path, tmp_path): """Check that public keys are set up properly.""" From 4f4b1271e98c008feaabdb15147f746989cf4827 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Mon, 16 Mar 2026 12:39:16 -0600 Subject: [PATCH 46/55] added test for bad urls --- unittests/data/systems/mirror-bad-url/mirrors.yaml | 4 ++-- unittests/test_mirrors.py | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/unittests/data/systems/mirror-bad-url/mirrors.yaml b/unittests/data/systems/mirror-bad-url/mirrors.yaml index 522c2326..8ffce331 100644 --- a/unittests/data/systems/mirror-bad-url/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-url/mirrors.yaml @@ -1,2 +1,2 @@ -- name: bad-url - url: google.com \ No newline at end of file +bad-url: + url: https://www.testsite.io/services \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 8e189228..191e7088 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -31,6 +31,14 @@ def test_mirror_init(systems_path, valid_mirrors): for mir in mirrors_obj.mirrors: assert mirrors_obj.mirrors[mir].get('enabled') +def test_mirror_init_bad_url(systems_path): + """Check that MirrorError is raised for a bad url.""" + + path = systems_path / "mirror-bad-url" + + with pytest.raises(mirror.MirrorError): + mirrors_obj = mirror.Mirrors(path) + def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" From 1285fab535d2d66d43d3035cda1a195a123b37b4 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Mon, 16 Mar 2026 12:54:48 -0600 Subject: [PATCH 47/55] added requirements.txt --- requirements.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..d0de0ac4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +attrs==25.4.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +packaging==26.0 +pluggy==1.6.0 +pygments==2.19.2 +pytest==9.0.2 +python-magic==0.4.27 +pyyaml==6.0.3 +referencing==0.37.0 +rpds-py==0.30.0 From 48621c9d3843cff56bd71c363fd33c1e7ba020d5 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 13:02:02 -0600 Subject: [PATCH 48/55] Fixed unittest. --- unittests/data/systems/mirror-ok/mirrors.yaml | 82 +++++++++---------- unittests/test_mirrors.py | 60 +++++++++----- 2 files changed, 81 insertions(+), 61 deletions(-) diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index 96fc72f5..0a1b8434 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -7,47 +7,47 @@ disabled-mirror: buildcache-mirror: url: https://mirror.spack.io private_key: '../test-gpg-priv.asc' - public_key: | - mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML - eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB - 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF - kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK - IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb - WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh - jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA - TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3 - qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH - KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe - CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK - CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+ - LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i - qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr - HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1 - WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i - /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x - glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE - PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW - lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw - kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe - mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm - HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH - Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0 - UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF - zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx - Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y - 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H - sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr - 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q - OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR - NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw - Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z - +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx - 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7 - xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6 - lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd - eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8 - eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG - gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3 + public_key: "\ + mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML\ + eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB\ + 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF\ + kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK\ + IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb\ + WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh\ + jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA\ + TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3\ + qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH\ + KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe\ + CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK\ + CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+\ + LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i\ + qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr\ + HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1\ + WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i\ + /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x\ + glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE\ + PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW\ + lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw\ + kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe\ + mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm\ + HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH\ + Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0\ + UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF\ + zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx\ + Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y\ + 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H\ + sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr\ + 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q\ + OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR\ + NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw\ + Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z\ + +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx\ + 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7\ + xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6\ + lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd\ + eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8\ + eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG\ + gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3" cache: true bootstrap-mirror: diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 191e7088..dbc2a97d 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -1,5 +1,6 @@ -import pytest +import base64 import pathlib +import pytest import stackinator.mirror as mirror import yaml @@ -11,19 +12,38 @@ def test_path(): def systems_path(test_path): return test_path / "data" / "systems" -@pytest.fixture -def valid_mirrors(systems_path): - mirrors = {} - mirrors["fake-mirror"] = {'url': 'https://github.com', 'enabled': True, 'bootstrap': False, 'cache': False, 'mount_specific': False} - mirrors["buildcache-mirror"] = {'url': 'https://mirror.spack.io', 'enabled': True, 'bootstrap': False, 'cache': True, 'mount_specific': False} - mirrors["bootstrap-mirror"] = {'url': 'https://mirror.spack.io', 'enabled': True, 'bootstrap': True, 'cache': False, 'mount_specific': False} - return mirrors - -def test_mirror_init(systems_path, valid_mirrors): +def test_mirror_init(systems_path): """Check that Mirror objects are initialized correctly.""" path = systems_path / "mirror-ok" mirrors_obj = mirror.Mirrors(path) + valid_mirrors = { + "fake-mirror": { + 'url': 'https://github.com', + 'enabled': True, + 'bootstrap': False, + 'cache': False, + 'public_key': '../../test-gpg-pub.asc', + 'mount_specific': False}, + "buildcache-mirror": { + 'url': 'https://mirror.spack.io', + 'enabled': True, + 'bootstrap': False, + 'cache': True, + 'private_key': '../test-gpg-priv.asc', + 'mount_specific': False}, + "bootstrap-mirror": { + 'url': 'https://mirror.spack.io', + 'enabled': True, + 'bootstrap': True, + 'cache': False, + 'mount_specific': False} + } + + with (systems_path/'../test-gpg-pub.asc').open('rb') as pub_key_file: + key = base64.b64encode(pub_key_file.read()).decode() + valid_mirrors['buildcache-mirror']['public_key'] = key + assert mirrors_obj.mirrors == valid_mirrors assert mirrors_obj.bootstrap_mirrors == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('bootstrap')] assert mirrors_obj.build_cache_mirror == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('cache')].pop(0) @@ -55,7 +75,7 @@ def test_command_line_cache(systems_path): assert not cache_mirror['bootstrap'] assert cache_mirror['mount_specific'] -def test_create_spack_mirrors_yaml(systems_path): +def test_create_spack_mirrors_yaml(tmp_path, systems_path): """Check that the mirrors.yaml passed to spack is correct""" valid_spack_yaml = { @@ -75,7 +95,7 @@ def test_create_spack_mirrors_yaml(systems_path): } } - dest = systems_path / "mirror-ok" / "test_output.yaml" + dest = tmp_path / "test_output.yaml" mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok") mirrors_obj._create_spack_mirrors_yaml(dest) @@ -84,14 +104,14 @@ def test_create_spack_mirrors_yaml(systems_path): assert data == valid_spack_yaml -def test_create_bootstrap_configs(systems_path): +def test_create_bootstrap_configs(tmp_path, systems_path): """Check that spack bootstrap configs are generated correctly""" valid_yaml = { "sources": [ { "name": "bootstrap-mirror", - "metadata": str(systems_path / "mirror-ok" / "bootstrap" / "bootstrap-mirror"), + "metadata": str(tmp_path / "bootstrap/bootstrap-mirror"), } ], "trusted": { @@ -103,16 +123,16 @@ def test_create_bootstrap_configs(systems_path): "info": "https://mirror.spack.io", } - path = systems_path / "mirror-ok" - bs_mirror_path = path / "bootstrap/bootstrap-mirror" - mirrors_obj = mirror.Mirrors(path) - mirrors_obj._create_bootstrap_configs(path) + mirrors_obj = mirror.Mirrors(systems_path/'mirror-ok') + mirrors_obj._create_bootstrap_configs(tmp_path) - with (path/'bootstrap.yaml').open() as f: + with (tmp_path/'bootstrap.yaml').open() as f: bs_data = yaml.safe_load(f) + print(bs_data) + print(valid_yaml) assert bs_data == valid_yaml - with (bs_mirror_path/'metadata.yaml').open() as f: + with (tmp_path/'bootstrap/bootstrap-mirror/metadata.yaml').open() as f: metadata = yaml.safe_load(f) assert metadata == valid_metadata From d3dc3fa18b43fadf08e0cd8beec3ffb6411e0bae Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 13:06:51 -0600 Subject: [PATCH 49/55] Added one more unittest. --- unittests/test_mirrors.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index dbc2a97d..72f2190c 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -57,7 +57,17 @@ def test_mirror_init_bad_url(systems_path): path = systems_path / "mirror-bad-url" with pytest.raises(mirror.MirrorError): - mirrors_obj = mirror.Mirrors(path) + mirror.Mirrors(path) + +def test_setup_configs(tmp_path, systems_path): + """Test general config setup.""" + + mir = mirror.Mirrors(systems_path/'mirror-ok') + mir.setup_configs(tmp_path) + + assert (tmp_path/'mirrors.yaml').is_file() + assert (tmp_path/'bootstrap').is_dir() + assert (tmp_path/'key_store').is_dir() def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" From 9eaf7a99cc471aab8d8b9fb8d504a5655c427139 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 13:13:05 -0600 Subject: [PATCH 50/55] Linting --- stackinator/builder.py | 2 +- stackinator/main.py | 34 +++++++--- stackinator/mirror.py | 137 ++++++++++++++++++++------------------ stackinator/recipe.py | 7 +- unittests/test_mirrors.py | 124 +++++++++++++++++++--------------- 5 files changed, 168 insertions(+), 136 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index b6192bb2..ca0f8753 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -11,7 +11,7 @@ import jinja2 import yaml -from . import VERSION, cache, root_logger, spack_util, mirror +from . import VERSION, root_logger, spack_util, mirror def install(src, dst, *, ignore=None, symlinks=False): diff --git a/stackinator/main.py b/stackinator/main.py index 6f30ddfc..0d0b9bd8 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -81,19 +81,31 @@ def log_header(args): def make_argparser(): parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from a recipe.")) parser.add_argument("--version", action="version", version=f"stackinator version {VERSION}") - parser.add_argument("-b", "--build", required=True, type=str, - help="Where to set up the stackinator build directory. " - "('/tmp' is not allowed, use '/var/tmp'") + parser.add_argument( + "-b", + "--build", + required=True, + type=str, + help="Where to set up the stackinator build directory. ('/tmp' is not allowed, use '/var/tmp'", + ) parser.add_argument("--no-bwrap", action="store_true", required=False) - parser.add_argument("-r", "--recipe", required=True, type=str, - help="Name of (and/or path to) the Stackinator recipe.") - parser.add_argument("-s", "--system", required=True, type=str, - help="Name of (and/or path to) the Stackinator system configuration.") + parser.add_argument( + "-r", "--recipe", required=True, type=str, help="Name of (and/or path to) the Stackinator recipe." + ) + parser.add_argument( + "-s", "--system", required=True, type=str, help="Name of (and/or path to) the Stackinator system configuration." + ) parser.add_argument("-d", "--debug", action="store_true") - parser.add_argument("-m", "--mount", required=False, type=str, - help="The mount point where the environment will be located.") - parser.add_argument("-c", "--cache", required=False, type=str, - help="Buildcache location or name (from system config's mirrors.yaml).") + parser.add_argument( + "-m", "--mount", required=False, type=str, help="The mount point where the environment will be located." + ) + parser.add_argument( + "-c", + "--cache", + required=False, + type=str, + help="Buildcache location or name (from system config's mirrors.yaml).", + ) parser.add_argument("--develop", action="store_true", required=False) return parser diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 502943ae..e6ba268b 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,4 +1,3 @@ - from typing import Optional, List, Dict import base64 import io @@ -11,18 +10,24 @@ from . import schema, root_logger + class MirrorError(RuntimeError): """Exception class for errors thrown by mirror configuration problems.""" + class Mirrors: """Manage the definition of mirrors in a recipe.""" - KEY_STORE_DIR = 'key_store' - MIRRORS_YAML = 'mirrors.yaml' - CMDLINE_CACHE = 'cmdline_cache' + KEY_STORE_DIR = "key_store" + MIRRORS_YAML = "mirrors.yaml" + CMDLINE_CACHE = "cmdline_cache" - def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[pathlib.Path] = None, - mount_point: Optional[pathlib.Path] = None): + def __init__( + self, + system_config_root: pathlib.Path, + cmdline_cache: Optional[pathlib.Path] = None, + mount_point: Optional[pathlib.Path] = None, + ): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" self._system_config_root = system_config_root @@ -32,25 +37,24 @@ def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[pat self.mirrors = self._load_mirrors(cmdline_cache) self._check_mirrors() - + # Always use the cache given on the command line if self.CMDLINE_CACHE in self.mirrors: self.build_cache_mirror = self.CMDLINE_CACHE else: # Otherwise, grab the configured cache (or None) - self.build_cache_mirror : Optional[str] = \ - ([name for name, mirror in self.mirrors.items() if mirror.get('cache', False)] - + [None]).pop(0) + self.build_cache_mirror: Optional[str] = ( + [name for name, mirror in self.mirrors.items() if mirror.get("cache", False)] + [None] + ).pop(0) - self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() - if mirror.get('bootstrap', False)] + self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() if mirror.get("bootstrap", False)] # Will hold a list of all the gpg keys (public and private) - self._keys: Optional[List[pathlib.Path]] = [] + self._keys: Optional[List[pathlib.Path]] = [] def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" - path = self._system_config_root/"mirrors.yaml" + path = self._system_config_root / "mirrors.yaml" if path.exists(): try: with path.open() as fid: @@ -66,14 +70,15 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict except ValueError as err: raise MirrorError(f"Mirror config does not comply with schema.\n{err}") - caches = [name for name, mirror in mirrors.items() if mirror['cache']] + caches = [name for name, mirror in mirrors.items() if mirror["cache"]] if len(caches) > 1: raise MirrorError( "Mirror config has more than one mirror specified as the build cache destination.\n" - f"{self._pp_yaml(caches)}") + f"{self._pp_yaml(caches)}" + ) elif caches: cache = mirrors[caches[0]] - if not cache.get('private_key'): + if not cache.get("private_key"): raise MirrorError(f"Mirror build cache config '{caches[0]}' missing a required 'private_key' path.") # Load the cache as defined by the deprecated 'cache.yaml' file. @@ -82,7 +87,7 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict return {name: mirror for name, mirror in mirrors.items() if mirror["enabled"]} - @staticmethod + @staticmethod def _pp_yaml(object): """Pretty print the given object as yaml.""" @@ -95,35 +100,35 @@ def _load_cmdline_cache(self, cache_config_path: pathlib.Path) -> Dict: if not cache_config_path.is_file(): raise MirrorError( - f"Binary cache configuration path given on the command line '{cache_config_path}' " - f"does not exist.") - - with cache_config_path.open('r') as file: + f"Binary cache configuration path given on the command line '{cache_config_path}' does not exist." + ) + + with cache_config_path.open("r") as file: try: raw = yaml.load(file, Loader=yaml.SafeLoader) except ValueError as err: - raise MirrorError( - f"Error loading yaml from cache config at '{cache_config_path}'\n{err}") + raise MirrorError(f"Error loading yaml from cache config at '{cache_config_path}'\n{err}") try: schema.CacheValidator.validate(raw) except ValueError as err: - raise MirrorError( - f"Error validating contents of cache config at '{cache_config_path}'.\n{err}") + raise MirrorError(f"Error validating contents of cache config at '{cache_config_path}'.\n{err}") mirror_cfg = { - 'url': raw['root'], - 'description': "Buildcache dest loaded from legacy cache.yaml", - 'cache': True, - 'enabled': True, - 'bootstrap': False, - 'mount_specific': True, - 'private_key': raw['key'], + "url": raw["root"], + "description": "Buildcache dest loaded from legacy cache.yaml", + "cache": True, + "enabled": True, + "bootstrap": False, + "mount_specific": True, + "private_key": raw["key"], } - self._logger.warning("Configuring the buildcache from the system cache.yaml file.\n" + self._logger.warning( + "Configuring the buildcache from the system cache.yaml file.\n" "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" - f"The equivalent 'mirrors.yaml' would look like: \n{self._pp_yaml([mirror_cfg])}") + f"The equivalent 'mirrors.yaml' would look like: \n{self._pp_yaml([mirror_cfg])}" + ) return mirror_cfg @@ -144,12 +149,13 @@ def _check_mirrors(self): elif url.startswith("https://"): try: - request = urllib.request.Request(url, method='HEAD') + request = urllib.request.Request(url, method="HEAD") urllib.request.urlopen(request) except urllib.error.URLError as e: raise MirrorError( - f"Could not reach the mirror url '{url}'. " - f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") + f"Could not reach the mirror url '{url}'. " + f"Check the url listed in mirrors.yaml in system config. \n{e.reason}" + ) @property def keys(self): @@ -160,12 +166,11 @@ def keys(self): return self._keys - def setup_configs(self, config_root: pathlib.Path): """Setup all mirror configs in the given config_root.""" - self._key_setup(config_root/self.KEY_STORE_DIR) - self._create_spack_mirrors_yaml(config_root/self.MIRRORS_YAML) + self._key_setup(config_root / self.KEY_STORE_DIR) + self._create_spack_mirrors_yaml(config_root / self.MIRRORS_YAML) self._create_bootstrap_configs(config_root) def _create_spack_mirrors_yaml(self, dest: pathlib.Path): @@ -177,8 +182,8 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): url = mirror["url"] # Make the mirror path specific to the mount point - if mirror['mount_specific'] and self._mount_point is not None: - url = url.rstrip('/') + '/' + self._mount_point.as_posix().lstrip('/') + if mirror["mount_specific"] and self._mount_point is not None: + url = url.rstrip("/") + "/" + self._mount_point.as_posix().lstrip("/") raw["mirrors"][name] = { "fetch": {"url": url}, @@ -193,40 +198,40 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): if not self.bootstrap_mirrors: return - + bootstrap_yaml = { - 'sources': [], - 'trusted': {}, + "sources": [], + "trusted": {}, } for name in self.bootstrap_mirrors: - bs_mirror_path = config_root/f'bootstrap/{name}' + bs_mirror_path = config_root / f"bootstrap/{name}" mirror = self.mirrors[name] # Tell spack where to find the metadata for each bootstrap mirror. - bootstrap_yaml['sources'].append( + bootstrap_yaml["sources"].append( { - 'name': name, - 'metadata': str(bs_mirror_path), + "name": name, + "metadata": str(bs_mirror_path), } ) # And trust each one - bootstrap_yaml['trusted'][name] = True + bootstrap_yaml["trusted"][name] = True # Create the metadata dir and metadata.yaml bs_mirror_path.mkdir(parents=True, exist_ok=True) bs_mirror_yaml = { - 'type': 'install', - 'info': mirror['url'], + "type": "install", + "info": mirror["url"], } - with (bs_mirror_path/'metadata.yaml').open('w') as file: + with (bs_mirror_path / "metadata.yaml").open("w") as file: yaml.dump(bs_mirror_yaml, file, default_flow_style=False) - - with (config_root/'bootstrap.yaml').open('w') as file: + + with (config_root / "bootstrap.yaml").open("w") as file: yaml.dump(bootstrap_yaml, file, default_flow_style=False) def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" - + self._keys = [] key_store.mkdir(exist_ok=True) @@ -243,13 +248,13 @@ def _key_setup(self, key_store: pathlib.Path): # if path, check if abs path, if not, append sys config path in front and check again path = pathlib.Path(os.path.expandvars(key)) if not path.is_absolute(): - #try prepending system config path - path = self._system_config_root/path + # try prepending system config path + path = self._system_config_root / path if path.is_file(): - with open(path, 'rb') as reader: + with open(path, "rb") as reader: binary_key = reader.read() - + # convert base64 key to binary else: try: @@ -258,17 +263,19 @@ def _key_setup(self, key_store: pathlib.Path): raise MirrorError( f"Key for mirror '{name}' is not valid: '{path}'. \n" f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" - f"Check the key listed in mirrors.yaml in system config.") - + f"Check the key listed in mirrors.yaml in system config." + ) + file_type = magic.from_buffer(binary_key, mime=True) if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): raise MirrorError( f"Key for mirror {name} is not a valid GPG key. \n" f"The file (or base64) was readable, but the data itself was not a PGP key.\n" - f"Check the key listed in mirrors.yaml in system config.") + f"Check the key listed in mirrors.yaml in system config." + ) # copy key to new destination in key store - with open(dest, 'wb') as writer: + with open(dest, "wb") as writer: writer.write(binary_key) self._keys.append(dest) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index a15569b6..76cba826 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -4,9 +4,8 @@ import jinja2 import yaml -from typing import Optional -from . import cache, root_logger, schema, spack_util, mirror +from . import root_logger, schema, spack_util, mirror from .etc import envvars @@ -170,7 +169,7 @@ def __init__(self, args): schema.EnvironmentsValidator.validate(raw) self.generate_environment_specs(raw) - # load the optional mirrors.yaml from system config, and add any additional + # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") self.mirrors = mirror.Mirrors(self.system_config_path, args.cache) @@ -232,7 +231,7 @@ def pre_install_hook(self): if hook_path.exists() and hook_path.is_file(): return hook_path return None - + @property def config(self): return self._config diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 72f2190c..d262cd1b 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -4,14 +4,17 @@ import stackinator.mirror as mirror import yaml + @pytest.fixture def test_path(): return pathlib.Path(__file__).parent.resolve() + @pytest.fixture def systems_path(test_path): return test_path / "data" / "systems" + def test_mirror_init(systems_path): """Check that Mirror objects are initialized correctly.""" path = systems_path / "mirror-ok" @@ -19,37 +22,45 @@ def test_mirror_init(systems_path): valid_mirrors = { "fake-mirror": { - 'url': 'https://github.com', - 'enabled': True, - 'bootstrap': False, - 'cache': False, - 'public_key': '../../test-gpg-pub.asc', - 'mount_specific': False}, + "url": "https://github.com", + "enabled": True, + "bootstrap": False, + "cache": False, + "public_key": "../../test-gpg-pub.asc", + "mount_specific": False, + }, "buildcache-mirror": { - 'url': 'https://mirror.spack.io', - 'enabled': True, - 'bootstrap': False, - 'cache': True, - 'private_key': '../test-gpg-priv.asc', - 'mount_specific': False}, + "url": "https://mirror.spack.io", + "enabled": True, + "bootstrap": False, + "cache": True, + "private_key": "../test-gpg-priv.asc", + "mount_specific": False, + }, "bootstrap-mirror": { - 'url': 'https://mirror.spack.io', - 'enabled': True, - 'bootstrap': True, - 'cache': False, - 'mount_specific': False} + "url": "https://mirror.spack.io", + "enabled": True, + "bootstrap": True, + "cache": False, + "mount_specific": False, + }, } - with (systems_path/'../test-gpg-pub.asc').open('rb') as pub_key_file: + with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: key = base64.b64encode(pub_key_file.read()).decode() - valid_mirrors['buildcache-mirror']['public_key'] = key + valid_mirrors["buildcache-mirror"]["public_key"] = key assert mirrors_obj.mirrors == valid_mirrors - assert mirrors_obj.bootstrap_mirrors == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('bootstrap')] - assert mirrors_obj.build_cache_mirror == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('cache')].pop(0) - + assert mirrors_obj.bootstrap_mirrors == [ + name for name in valid_mirrors.keys() if valid_mirrors[name].get("bootstrap") + ] + assert mirrors_obj.build_cache_mirror == [ + name for name in valid_mirrors.keys() if valid_mirrors[name].get("cache") + ].pop(0) + for mir in mirrors_obj.mirrors: - assert mirrors_obj.mirrors[mir].get('enabled') + assert mirrors_obj.mirrors[mir].get("enabled") + def test_mirror_init_bad_url(systems_path): """Check that MirrorError is raised for a bad url.""" @@ -59,31 +70,33 @@ def test_mirror_init_bad_url(systems_path): with pytest.raises(mirror.MirrorError): mirror.Mirrors(path) + def test_setup_configs(tmp_path, systems_path): """Test general config setup.""" - mir = mirror.Mirrors(systems_path/'mirror-ok') + mir = mirror.Mirrors(systems_path / "mirror-ok") mir.setup_configs(tmp_path) - assert (tmp_path/'mirrors.yaml').is_file() - assert (tmp_path/'bootstrap').is_dir() - assert (tmp_path/'key_store').is_dir() + assert (tmp_path / "mirrors.yaml").is_file() + assert (tmp_path / "bootstrap").is_dir() + assert (tmp_path / "key_store").is_dir() + def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" - mirrors = mirror.Mirrors(systems_path/'mirror-ok', - cmdline_cache=systems_path/'mirror-ok/cache.yaml') + mirrors = mirror.Mirrors(systems_path / "mirror-ok", cmdline_cache=systems_path / "mirror-ok/cache.yaml") assert len(mirrors.mirrors) == 4 # This should always be the build cache even though one is already defined. - assert mirrors.build_cache_mirror == 'cmdline_cache' - cache_mirror = mirrors.mirrors['cmdline_cache'] - assert cache_mirror['url'] == '/tmp/foo' - assert cache_mirror['enabled'] - assert cache_mirror['cache'] - assert not cache_mirror['bootstrap'] - assert cache_mirror['mount_specific'] + assert mirrors.build_cache_mirror == "cmdline_cache" + cache_mirror = mirrors.mirrors["cmdline_cache"] + assert cache_mirror["url"] == "/tmp/foo" + assert cache_mirror["enabled"] + assert cache_mirror["cache"] + assert not cache_mirror["bootstrap"] + assert cache_mirror["mount_specific"] + def test_create_spack_mirrors_yaml(tmp_path, systems_path): """Check that the mirrors.yaml passed to spack is correct""" @@ -101,7 +114,7 @@ def test_create_spack_mirrors_yaml(tmp_path, systems_path): "bootstrap-mirror": { "fetch": {"url": "https://mirror.spack.io"}, "push": {"url": "https://mirror.spack.io"}, - } + }, } } @@ -114,9 +127,10 @@ def test_create_spack_mirrors_yaml(tmp_path, systems_path): assert data == valid_spack_yaml + def test_create_bootstrap_configs(tmp_path, systems_path): """Check that spack bootstrap configs are generated correctly""" - + valid_yaml = { "sources": [ { @@ -124,54 +138,54 @@ def test_create_bootstrap_configs(tmp_path, systems_path): "metadata": str(tmp_path / "bootstrap/bootstrap-mirror"), } ], - "trusted": { - "bootstrap-mirror": True - }, + "trusted": {"bootstrap-mirror": True}, } valid_metadata = { "type": "install", "info": "https://mirror.spack.io", } - mirrors_obj = mirror.Mirrors(systems_path/'mirror-ok') + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok") mirrors_obj._create_bootstrap_configs(tmp_path) - with (tmp_path/'bootstrap.yaml').open() as f: + with (tmp_path / "bootstrap.yaml").open() as f: bs_data = yaml.safe_load(f) print(bs_data) print(valid_yaml) assert bs_data == valid_yaml - with (tmp_path/'bootstrap/bootstrap-mirror/metadata.yaml').open() as f: + with (tmp_path / "bootstrap/bootstrap-mirror/metadata.yaml").open() as f: metadata = yaml.safe_load(f) assert metadata == valid_metadata + def test_key_setup(systems_path, tmp_path): """Check that public keys are set up properly.""" - mirrors = mirror.Mirrors(systems_path/'mirror-ok') + mirrors = mirror.Mirrors(systems_path / "mirror-ok") mirrors._key_setup(tmp_path) key_files = list(tmp_path.iterdir()) - assert {key_file.name for key_file in key_files} == {'buildcache-mirror.gpg', 'fake-mirror.gpg'} + assert {key_file.name for key_file in key_files} == {"buildcache-mirror.gpg", "fake-mirror.gpg"} # The two files should be identical in content key_file_data = [] for key_file in key_files: - with key_file.open('rb') as file: + with key_file.open("rb") as file: key_file_data.append(file.read()) assert key_file_data[0] == key_file_data[1] -@pytest.mark.parametrize("system_name", [ - 'mirror-bad-key', - 'mirror-bad-keypath', -]) + +@pytest.mark.parametrize( + "system_name", + [ + "mirror-bad-key", + "mirror-bad-keypath", + ], +) def test_key_setup_bad_key(tmp_path, systems_path, system_name): """asdfasdf""" - mirrors = mirror.Mirrors(systems_path/system_name) + mirrors = mirror.Mirrors(systems_path / system_name) with pytest.raises(mirror.MirrorError): mirrors._key_setup(tmp_path) - - - From da43eb1cbb31b54d763ea73dfcffdc076edf44e6 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 13:14:56 -0600 Subject: [PATCH 51/55] Got rid of cache.py --- stackinator/cache.py | 58 -------------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 stackinator/cache.py diff --git a/stackinator/cache.py b/stackinator/cache.py deleted file mode 100644 index 24177e33..00000000 --- a/stackinator/cache.py +++ /dev/null @@ -1,58 +0,0 @@ -import os -import pathlib - -import yaml - -from . import schema - - -def configuration_from_file(file, mount): - with file.open() as fid: - # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) - - # validate the yaml - schema.CacheValidator.validate(raw) - - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(raw["root"])) - if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") - - raw["root"] = path - - # Put the build cache in a sub-directory named after the mount point. - # This avoids relocation issues. - raw["path"] = pathlib.Path(path.as_posix() + mount.as_posix()) - - # verify that the key file exists if it was specified - key = raw["key"] - if key is not None: - key = pathlib.Path(os.path.expandvars(key)) - if not key.is_absolute(): - raise FileNotFoundError(f"The build cache key '{key}' is not absolute") - if not key.is_file(): - raise FileNotFoundError(f"The build cache key '{key}' does not exist") - raw["key"] = key - - return raw - - -def generate_mirrors_yaml(config): - path = config["path"].as_posix() - mirrors = { - "mirrors": { - "alpscache": { - "fetch": { - "url": f"file://{path}", - }, - "push": { - "url": f"file://{path}", - }, - } - } - } - - return yaml.dump(mirrors, default_flow_style=False) From 7b5d8b135253392a85a1b7b91ba70578ef53fff1 Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 17 Mar 2026 12:41:31 +0100 Subject: [PATCH 52/55] use pyproject for dependencies --- bin/stack-config | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/bin/stack-config b/bin/stack-config index 66496991..02f48700 100755 --- a/bin/stack-config +++ b/bin/stack-config @@ -2,6 +2,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ +# "python-magic", # "jinja2", # "jsonschema", # "pyYAML", diff --git a/pyproject.toml b/pyproject.toml index 583f5d46..25fd1ddf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ license-files = ["LICENSE"] dynamic = ["version"] requires-python = ">=3.12" dependencies = [ + "python-magic", "Jinja2", "jsonschema", "PyYAML", From 42f9d8d682c808797cd73228a0b1e3b67a0f6b5a Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 17 Mar 2026 12:43:53 +0100 Subject: [PATCH 53/55] add unit test and lint hints to readme --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 5c7e0408..265b0ac3 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,17 @@ A tool for building a scientific software stack from a recipe for vClusters on C Read the [documentation](https://eth-cscs.github.io/stackinator/) to get started. Create a ticket in our [GitHub issues](https://github.com/eth-cscs/stackinator/issues) if you find a bug, have a feature request or have a question. + +## running tests: + +Use uv to run the tests, which will in turn ensure that the correct dependencies from `pyproject.toml` are used: + +``` +uv run pytest +``` + +Before pushing, apply the linting rules (this calls uv under the hood): + +``` +./lint +``` From fc2fef3015b7a54fa69fc60f651f38cc71843ffc Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 17 Mar 2026 12:44:19 +0100 Subject: [PATCH 54/55] tweak presentation of build cache by recipe --- stackinator/builder.py | 2 +- stackinator/main.py | 2 +- stackinator/recipe.py | 14 ++++++++++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index ca0f8753..47a73b05 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -233,7 +233,7 @@ def generate(self, recipe): spack_version=spack_version, spack_meta=spack_meta, gpg_keys=recipe.mirrors.keys, - cache=recipe.mirrors.buildcache, + cache=recipe.build_cache_mirror, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) diff --git a/stackinator/main.py b/stackinator/main.py index 0d0b9bd8..ec384561 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -86,7 +86,7 @@ def make_argparser(): "--build", required=True, type=str, - help="Where to set up the stackinator build directory. ('/tmp' is not allowed, use '/var/tmp'", + help="Where to set up the stackinator build directory. ('/tmp' is not allowed, use '/var/tmp')", ) parser.add_argument("--no-bwrap", action="store_true", required=False) parser.add_argument( diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 76cba826..ff3d8e27 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -172,8 +172,7 @@ def __init__(self, args): # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") - self.mirrors = mirror.Mirrors(self.system_config_path, args.cache) - self.cache = self.mirrors.build_cache_mirror + self.mirrors = mirror.Mirrors(self.system_config_path, pathlib.Path(args.cache)) # optional post install hook if self.post_install_hook is not None: @@ -202,6 +201,13 @@ def spack_repo(self): return repo_path return None + # Returns: + # Path: if the recipe specified a build cache mirror + # None: if no build cache mirror is used + @property + def build_cache_mirror(self): + return self.mirrors.build_cache_mirror + # Returns: # Path: of the recipe extra path if it exists # None: if there is no user-provided extra path in the recipe @@ -511,7 +517,7 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.cache + push_to_cache = self.build_cache_mirror is not None files["makefile"] = makefile_template.render( compilers=self.compilers, push_to_cache=push_to_cache, @@ -542,7 +548,7 @@ def environment_files(self): jenv.filters["py2yaml"] = schema.py2yaml makefile_template = jenv.get_template("Makefile.environments") - push_to_cache = self.cache is not None + push_to_cache = self.build_cache_mirror is not None files["makefile"] = makefile_template.render( environments=self.environments, push_to_cache=push_to_cache, From 62a5c5936f0296874b4db46d88183b7a242bf4c7 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 20 Mar 2026 14:14:10 -0600 Subject: [PATCH 55/55] modified bootstrap yaml setup and removed hardcoded alpscache --- stackinator/mirror.py | 18 +++++++++++------- stackinator/recipe.py | 9 ++++----- stackinator/templates/Makefile | 2 +- stackinator/templates/Makefile.compilers | 4 ++-- stackinator/templates/Makefile.environments | 4 ++-- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e6ba268b..13e54e00 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -47,7 +47,7 @@ def __init__( [name for name, mirror in self.mirrors.items() if mirror.get("cache", False)] + [None] ).pop(0) - self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() if mirror.get("bootstrap", False)] + self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() if mirror.get("bootstrap", True)] # Will hold a list of all the gpg keys (public and private) self._keys: Optional[List[pathlib.Path]] = [] @@ -200,29 +200,33 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): return bootstrap_yaml = { - "sources": [], - "trusted": {}, + "bootstrap": { + "sources": [], + "trusted": {}, + } } for name in self.bootstrap_mirrors: bs_mirror_path = config_root / f"bootstrap/{name}" mirror = self.mirrors[name] # Tell spack where to find the metadata for each bootstrap mirror. - bootstrap_yaml["sources"].append( + bootstrap_yaml["bootstrap"]["sources"].append( { "name": name, "metadata": str(bs_mirror_path), } ) # And trust each one - bootstrap_yaml["trusted"][name] = True + bootstrap_yaml["bootstrap"]["trusted"][name] = True # Create the metadata dir and metadata.yaml bs_mirror_path.mkdir(parents=True, exist_ok=True) bs_mirror_yaml = { "type": "install", - "info": mirror["url"], - } + "info": { + "url": mirror["url"], + } + } with (bs_mirror_path / "metadata.yaml").open("w") as file: yaml.dump(bs_mirror_yaml, file, default_flow_style=False) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index ff3d8e27..520a826f 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -172,7 +172,8 @@ def __init__(self, args): # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") - self.mirrors = mirror.Mirrors(self.system_config_path, pathlib.Path(args.cache)) + self.mirrors = mirror.Mirrors(self.system_config_path, + pathlib.Path(args.cache) if args.cache else None) # optional post install hook if self.post_install_hook is not None: @@ -517,11 +518,10 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.build_cache_mirror is not None files["makefile"] = makefile_template.render( compilers=self.compilers, - push_to_cache=push_to_cache, spack_version=self.spack_version, + cache = self.build_cache_mirror, ) files["config"] = {} @@ -548,11 +548,10 @@ def environment_files(self): jenv.filters["py2yaml"] = schema.py2yaml makefile_template = jenv.get_template("Makefile.environments") - push_to_cache = self.build_cache_mirror is not None files["makefile"] = makefile_template.render( environments=self.environments, - push_to_cache=push_to_cache, spack_version=self.spack_version, + cache=self.build_cache_mirror, ) files["config"] = {} diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index f0b90ebe..7a931777 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -86,7 +86,7 @@ cache-force: mirror-setup $(warning likely have to start a fresh build (but that's okay, because build caches FTW)) $(warning ================================================================================) $(SANDBOX) $(MAKE) -C generate-config - $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package cache.name \ + $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package cache \ $$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ diff --git a/stackinator/templates/Makefile.compilers b/stackinator/templates/Makefile.compilers index f9520bd6..43a78edc 100644 --- a/stackinator/templates/Makefile.compilers +++ b/stackinator/templates/Makefile.compilers @@ -18,8 +18,8 @@ all:{% for compiler in compilers %} {{ compiler }}/generated/build_cache{% endfo {% for compiler, config in compilers.items() %} {{ compiler }}/generated/build_cache: {{ compiler }}/generated/env -{% if push_to_cache %} - $(SPACK) -e ./{{ compiler }} buildcache create --rebuild-index --only=package alpscache \ +{% if cache %} + $(SPACK) -e ./{{ compiler }} buildcache create --rebuild-index --only=package cache \ $$($(SPACK_HELPER) -e ./{{ compiler }} find --format '{name};{/hash}' \ | grep -v -E '^({% for p in config.exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | cut -d ';' -f2) diff --git a/stackinator/templates/Makefile.environments b/stackinator/templates/Makefile.environments index 5a530232..906eb38e 100644 --- a/stackinator/templates/Makefile.environments +++ b/stackinator/templates/Makefile.environments @@ -17,8 +17,8 @@ all:{% for env in environments %} {{ env }}/generated/build_cache{% endfor %} # Push built packages to a binary cache if a key has been provided {% for env, config in environments.items() %} {{ env }}/generated/build_cache: {{ env }}/generated/view_config -{% if push_to_cache %} - $(SPACK) --color=never -e ./{{ env }} buildcache create --rebuild-index --only=package alpscache \ +{% if cache %} + $(SPACK) --color=never -e ./{{ env }} buildcache create --rebuild-index --only=package cache \ $$($(SPACK_HELPER) -e ./{{ env }} find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in config.exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\