From f1618bfa2631cf6eda75da89e71891ba300e97ac Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 17 Feb 2026 16:46:01 -0700 Subject: [PATCH 01/69] added spack source mirrors capability, needs testing --- stackinator/builder.py | 2 ++ stackinator/schema/config.json | 7 +++++++ stackinator/templates/Makefile | 10 ++++++++++ 3 files changed, 19 insertions(+) diff --git a/stackinator/builder.py b/stackinator/builder.py index 2c05e3c..c3de44d 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -232,6 +232,8 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, + # pass source_mirrors to Makefile render + source_mirrors=recipe.config.get("source_mirrors", {}), exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index d6fec3a..4b91011 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -64,6 +64,13 @@ } } }, + "source_mirrors" : { + "type" : "object", + "additionalProperties": { + "type" : "string" + }, + "default": {} + }, "modules" : { "type": "boolean" }, diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 10a0ea5..3da2e00 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -39,6 +39,16 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} {% endif %} {% endif %} + {% if source_mirrors %} + echo "Replacing all instances of mirror.spack.io... Just in case" + grep -rl "https://mirror.spack.io" . | xargs sed -i 's/https:\/\/mirror.spack.io/https:\/\/pe-serve.lanl.gov\/spack-mirror/g' + echo "Adding mirrors" + {% for name, url in source_mirrors.items() %} + $(SANDBOX) $(SPACK) mirror add {{ name }} {{ url }} + {% endfor %} + echo "Current mirror list:" + spack mirror list + {% endif %} touch mirror-setup compilers: mirror-setup From 954a6901a6544ef6e190bea4cdfc8ada4b6acffe Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 18 Feb 2026 14:34:05 -0700 Subject: [PATCH 02/69] removed lanl stuff --- stackinator/templates/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 3da2e00..fcf6d68 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -40,13 +40,13 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% endif %} {% endif %} {% if source_mirrors %} - echo "Replacing all instances of mirror.spack.io... Just in case" - grep -rl "https://mirror.spack.io" . | xargs sed -i 's/https:\/\/mirror.spack.io/https:\/\/pe-serve.lanl.gov\/spack-mirror/g' + echo "Removing all instances of mirror.spack.io... Just in case" + grep -rl "https://mirror.spack.io" . | xargs sed -i 's|https://mirror.spack.io||g' echo "Adding mirrors" {% for name, url in source_mirrors.items() %} $(SANDBOX) $(SPACK) mirror add {{ name }} {{ url }} {% endfor %} - echo "Current mirror list:" + echo "Spack mirrors for this recipe:" spack mirror list {% endif %} touch mirror-setup From 97d43a0f88e8455cf1ec13b5a18d342b1d0d5698 Mon Sep 17 00:00:00 2001 From: Claire Ann Winogrodzki Date: Tue, 24 Feb 2026 09:38:20 -0700 Subject: [PATCH 03/69] add source mirrors via config.yaml --- stackinator/templates/Makefile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index fcf6d68..d7abb1e 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -40,14 +40,12 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% endif %} {% endif %} {% if source_mirrors %} - echo "Removing all instances of mirror.spack.io... Just in case" - grep -rl "https://mirror.spack.io" . | xargs sed -i 's|https://mirror.spack.io||g' - echo "Adding mirrors" - {% for name, url in source_mirrors.items() %} - $(SANDBOX) $(SPACK) mirror add {{ name }} {{ url }} + @echo "Adding mirrors" + {% for name, url in source_mirrors.items() | reverse %} + $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} {% endfor %} - echo "Spack mirrors for this recipe:" - spack mirror list + @echo "Current mirror list:" + $(SANDBOX) $(SPACK) mirror list {% endif %} touch mirror-setup From 15e97f1ede71cc5198f07908236056b5cf49be5a Mon Sep 17 00:00:00 2001 From: Claire Ann Winogrodzki Date: Tue, 24 Feb 2026 09:42:30 -0700 Subject: [PATCH 04/69] add source mirrors via config.yaml --- stackinator/templates/Makefile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index fcf6d68..d7abb1e 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -40,14 +40,12 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% endif %} {% endif %} {% if source_mirrors %} - echo "Removing all instances of mirror.spack.io... Just in case" - grep -rl "https://mirror.spack.io" . | xargs sed -i 's|https://mirror.spack.io||g' - echo "Adding mirrors" - {% for name, url in source_mirrors.items() %} - $(SANDBOX) $(SPACK) mirror add {{ name }} {{ url }} + @echo "Adding mirrors" + {% for name, url in source_mirrors.items() | reverse %} + $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} {% endfor %} - echo "Spack mirrors for this recipe:" - spack mirror list + @echo "Current mirror list:" + $(SANDBOX) $(SPACK) mirror list {% endif %} touch mirror-setup From f91fbad96675e6645c9ad356c2efc5bad481971f Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 24 Feb 2026 10:43:33 -0700 Subject: [PATCH 05/69] add source mirrors via config.yaml and retain spack default mirror --- nohup.out | 21 +++++++++++++++++++++ output.log | 2 ++ 2 files changed, 23 insertions(+) create mode 100644 nohup.out create mode 100644 output.log diff --git a/nohup.out b/nohup.out new file mode 100644 index 0000000..47d83f4 --- /dev/null +++ b/nohup.out @@ -0,0 +1,21 @@ +Stackinator + recipe path: /usr/projects/hpctools/grodzki/uenv/my-recipes/recipes/prgenv-gnu-openmpi + build path : /dev/shm/grodzki/build + system : /usr/projects/hpctools/grodzki/uenv/my-recipes/cluster-config/venadito + mount : default + build cache: None + develop : False +spack: https://github.com/spack/spack.git already cloned to /dev/shm/grodzki/build/spack +spack: fetching releases/v1.0 +spack: checking out releases/v1.0 +spack: commit hash is f36409b78591f8b02e8332eb4ad78da62c02571e +spack-packages: https://github.com/spack/spack-packages.git already cloned to /dev/shm/grodzki/build/spack-packages +spack-packages: fetching develop +spack-packages: checking out develop +spack-packages: commit hash is fc656595ff4c7e9c7e5045625dfdd3e92e97b10e + +Configuration finished, run the following to build the environment: + +cd /dev/shm/grodzki/build +env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin HOME=$HOME make store.squashfs -j32 +see logfile for more information /tmp/grodzki/log_stackinator_20260428-112546 diff --git a/output.log b/output.log new file mode 100644 index 0000000..d0fea53 --- /dev/null +++ b/output.log @@ -0,0 +1,2 @@ +nohup: ignoring input +/usr/bin/env: ‘uv’: No such file or directory From 2a6571e9f6c23ceb14cdabe69316f016b07475aa Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 24 Feb 2026 16:04:05 -0700 Subject: [PATCH 06/69] fixed spaces/tabs typo --- stackinator/templates/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index d7abb1e..ca8fb31 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -32,7 +32,7 @@ pre-install: spack-setup $(SANDBOX) $(STORE)/pre-install-hook mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - + {% if cache %} $(SANDBOX) $(SPACK) buildcache keys --install --trust {% if cache.key %} From 06f9dacbcab8a3dfcb25d620854effd54e79cf83 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 09:16:51 -0700 Subject: [PATCH 07/69] Added mirror configuration json schema. --- stackinator/schema/config.json | 7 ------- stackinator/schema/mirror.json | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 stackinator/schema/mirror.json diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index 4b91011..d6fec3a 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -64,13 +64,6 @@ } } }, - "source_mirrors" : { - "type" : "object", - "additionalProperties": { - "type" : "string" - }, - "default": {} - }, "modules" : { "type": "boolean" }, diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json new file mode 100644 index 0000000..b32a3cd --- /dev/null +++ b/stackinator/schema/mirror.json @@ -0,0 +1,37 @@ +# This config handles source mirrors, binary caches, and bootstrap mirrors (of both forms) +{ + # Order matters, so we need an array. + "type" : "array", + "items": { + "type": "object", + "required": ["name", "url"] + "properties": { + "name": { + "type": "string", + "description": "The name of this mirror. Should be follow standard variable naming syntax.", + }, + "url": { + "type": "string", + "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI).", + }, + "enabled": { + "type": "boolean", + "default": true, + "description": "Whether this mirror is enabled.", + }, + "bootstrap": { + "type": "boolean", + "default": false, + "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror.", + } + "public_key": { + "type": "string", + "description": "Public PGP key for validating binary cache packages.", + }, + "description": { + "type": "string", + "description": "What this mirror is for." + } + } + } +} From 7692677f5e29f8975c5ea03c89794addf84415ea Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 09:48:06 -0700 Subject: [PATCH 08/69] Incorporating Makefile changes. --- stackinator/templates/Makefile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index ca8fb31..d1a06dd 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -32,21 +32,25 @@ pre-install: spack-setup $(SANDBOX) $(STORE)/pre-install-hook mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - {% if cache %} + # The old way of managing mirrors $(SANDBOX) $(SPACK) buildcache keys --install --trust {% if cache.key %} $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} {% endif %} {% endif %} - {% if source_mirrors %} - @echo "Adding mirrors" - {% for name, url in source_mirrors.items() | reverse %} - $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} + {% if mirrors %} + @echo "Adding mirrors and gpg keys." + {% for mirror_info in mirrors | reverse %} + $(SANDBOX) $(SPACK) mirror add --scope=site {{ mirror_info.name }} {{ mirror_info.url }} + $(SANDBOX) $(SPACK) gpg trust {{ mirror_info.key_path }} {% endfor %} @echo "Current mirror list:" $(SANDBOX) $(SPACK) mirror list {% endif %} + {% for mirror_info in filter(lambda m: m['bootstrap'], mirrors) | filter() %} + $(SANDBOX) $(SPACK) bootstrap add --scope=site {{ mirror_info.name }} bootstrap/{{ mirror_info.name }} + {% endfor %} touch mirror-setup compilers: mirror-setup From 940886b14534faf8d293382e8f966f6286910af0 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 10:05:24 -0700 Subject: [PATCH 09/69] mirrors --- stackinator/mirror.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 stackinator/mirror.py diff --git a/stackinator/mirror.py b/stackinator/mirror.py new file mode 100644 index 0000000..e69de29 From ffb60834f94c91f018d2b57cf3d4ed1778033008 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 10:26:59 -0700 Subject: [PATCH 10/69] mirrors --- stackinator/mirror.py | 58 +++++++++++++++++++++++++++++++++++++++++++ stackinator/recipe.py | 32 ++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e69de29..83989b8 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -0,0 +1,58 @@ +import os +import pathlib + +import yaml + +from . import schema + + +def configuration_from_file(file, mount): + with file.open() as fid: + # load the raw yaml input + raw = yaml.load(fid, Loader=yaml.Loader) + + # validate the yaml + schema.CacheValidator.validate(raw) + + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(raw["root"])) + if not path.is_absolute(): + raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + if not path.is_dir(): + raise FileNotFoundError(f"The build cache path '{path}' does not exist") + + raw["root"] = path + + # Put the build cache in a sub-directory named after the mount point. + # This avoids relocation issues. + raw["path"] = pathlib.Path(path.as_posix() + mount.as_posix()) + + # verify that the key file exists if it was specified + key = raw["key"] + if key is not None: + key = pathlib.Path(os.path.expandvars(key)) + if not key.is_absolute(): + raise FileNotFoundError(f"The build cache key '{key}' is not absolute") + if not key.is_file(): + raise FileNotFoundError(f"The build cache key '{key}' does not exist") + raw["key"] = key + + return raw + + +def generate_mirrors_yaml(config): + path = config["path"].as_posix() + mirrors = { + "mirrors": { + "alpscache": { + "fetch": { + "url": f"file://{path}", + }, + "push": { + "url": f"file://{path}", + }, + } + } + } + + return yaml.dump(mirrors, default_flow_style=False) \ No newline at end of file diff --git a/stackinator/recipe.py b/stackinator/recipe.py index ca8d2b3..d0ec018 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -170,14 +170,20 @@ def __init__(self, args): self.generate_environment_specs(raw) # optional mirror configurtion + if mirrors_path.is_file():zx mirrors_path = self.path / "mirrors.yaml" - if mirrors_path.is_file(): self._logger.warning( "mirrors.yaml have been removed from recipes, use the --cache option on stack-config instead." ) raise RuntimeError("Unsupported mirrors.yaml file in recipe.") - self.mirror = (args.cache, self.mount) + # self.mirror = (args.cache, self.mount) + + # load the optional mirrors.yaml from system config: + mirrors_path = self.system_config_path / "mirrors.yaml" + if mirrors_path.is_file(): + self.mirrors = (mirrors_path, self.mount) + # update mirror setter and cache.configuration_from_file() # optional post install hook if self.post_install_hook is not None: @@ -262,6 +268,28 @@ def mirror(self, configuration): self._mirror = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) + @property + def mirrors(self): + return self._mirrors + + # old: self.mirror = (args.cache, self.mount) + # new: self.mirror = (mirrors_yaml_path, self.mount) + + @mirrors.setter + def (self, configuration): + self._logger.debug(f"configuring mirrors with {configuration}") + self._mirrors = None + + file, mount = configuration + + if file is not None: + mirror_config_path = pathlib.Path(file) + if not mirror_config_path.is_file(): + raise FileNotFoundError(f"The mirror configuration '{file}' is not a file") + + self._mirrors = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) + + @property def config(self): return self._config From 04f99084f4877657c32fdd251dcadd4f1e93f074 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 10:40:10 -0700 Subject: [PATCH 11/69] validate mirror config --- stackinator/mirror.py | 47 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 83989b8..be78486 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,5 +1,6 @@ import os import pathlib +import urllib.request import yaml @@ -14,30 +15,28 @@ def configuration_from_file(file, mount): # validate the yaml schema.CacheValidator.validate(raw) - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(raw["root"])) - if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") - - raw["root"] = path - - # Put the build cache in a sub-directory named after the mount point. - # This avoids relocation issues. - raw["path"] = pathlib.Path(path.as_posix() + mount.as_posix()) - - # verify that the key file exists if it was specified - key = raw["key"] - if key is not None: - key = pathlib.Path(os.path.expandvars(key)) - if not key.is_absolute(): - raise FileNotFoundError(f"The build cache key '{key}' is not absolute") - if not key.is_file(): - raise FileNotFoundError(f"The build cache key '{key}' does not exist") - raw["key"] = key - - return raw + mirrors = [mirror for mirror in raw if mirror["enabled"]] + + for mirror in mirrors: + url = mirror["url"] + if url.beginswith("file://"): + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(url)) + if not path.is_absolute(): + raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + if not path.is_dir(): + raise FileNotFoundError(f"The build cache path '{path}' does not exist") + + mirror["url"] = path + + else: + try: + request = urllib.request.Request(url, method='HEAD') + response = urllib.request.urlopen(request) + except urllib.error.URLError as e: + print(f'Error: {e.reason}') + + return mirrors def generate_mirrors_yaml(config): From b45039d71a99af60cbd657c42439fd1e60bd0c5d Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 11:01:41 -0700 Subject: [PATCH 12/69] Updating recipe to handle new mirrors format. --- stackinator/mirror.py | 4 +-- stackinator/recipe.py | 72 ++++++++++--------------------------------- 2 files changed, 19 insertions(+), 57 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index be78486..08c303b 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -7,7 +7,7 @@ from . import schema -def configuration_from_file(file, mount): +def configuration_from_file(file): with file.open() as fid: # load the raw yaml input raw = yaml.load(fid, Loader=yaml.Loader) @@ -54,4 +54,4 @@ def generate_mirrors_yaml(config): } } - return yaml.dump(mirrors, default_flow_style=False) \ No newline at end of file + return yaml.dump(mirrors, default_flow_style=False) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index d0ec018..281ab8b 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -4,8 +4,9 @@ import jinja2 import yaml +from typing import Optional -from . import cache, root_logger, schema, spack_util +from . import cache, root_logger, schema, spack_util, mirror from .etc import envvars @@ -169,21 +170,11 @@ def __init__(self, args): schema.EnvironmentsValidator.validate(raw) self.generate_environment_specs(raw) - # optional mirror configurtion - if mirrors_path.is_file():zx - mirrors_path = self.path / "mirrors.yaml" - self._logger.warning( - "mirrors.yaml have been removed from recipes, use the --cache option on stack-config instead." - ) - raise RuntimeError("Unsupported mirrors.yaml file in recipe.") - - # self.mirror = (args.cache, self.mount) + mirrors_path = self.system_config_path/'mirrors.yaml' + self._logger.debug(f"opening {mirrors_path}") # load the optional mirrors.yaml from system config: - mirrors_path = self.system_config_path / "mirrors.yaml" - if mirrors_path.is_file(): - self.mirrors = (mirrors_path, self.mount) - # update mirror setter and cache.configuration_from_file() + self.mirrors = self.system_config_path / "mirrors.yaml" # optional post install hook if self.post_install_hook is not None: @@ -242,54 +233,25 @@ def pre_install_hook(self): return hook_path return None - # Returns a dictionary with the following fields - # - # root: /path/to/cache - # path: /path/to/cache/user-environment - # key: /path/to/private-pgp-key @property - def mirror(self): - return self._mirror + def mirrors(self): + return self._mirrors # configuration is a tuple with two fields: # - a Path of the yaml file containing the cache configuration # - the mount point of the image - @mirror.setter - def mirror(self, configuration): - self._logger.debug(f"configuring build cache mirror with {configuration}") - self._mirror = None - - file, mount = configuration - - if file is not None: - mirror_config_path = pathlib.Path(file) - if not mirror_config_path.is_file(): - raise FileNotFoundError(f"The cache configuration '{file}' is not a file") - - self._mirror = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) - - @property - def mirrors(self): - return self._mirrors - - # old: self.mirror = (args.cache, self.mount) - # new: self.mirror = (mirrors_yaml_path, self.mount) - @mirrors.setter - def (self, configuration): - self._logger.debug(f"configuring mirrors with {configuration}") + def mirrors(self, path: Optional[pathlib.Path]): + """Initialize the mirrors property from config.""" self._mirrors = None + if path is not None: + if not path.is_file(): + raise FileNotFoundError("The system config 'mirrors.yaml' file exists, but isn't a " + "readable file.") - file, mount = configuration - - if file is not None: - mirror_config_path = pathlib.Path(file) - if not mirror_config_path.is_file(): - raise FileNotFoundError(f"The mirror configuration '{file}' is not a file") - - self._mirrors = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) - - + self._logger.debug(f"configuring mirrors from {path}") + self._mirrors = mirror.configuration_from_file(path) + @property def config(self): return self._config @@ -569,7 +531,7 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.mirror is not None + push_to_cache = self.mirrors files["makefile"] = makefile_template.render( compilers=self.compilers, push_to_cache=push_to_cache, From d3ee8e48507231b9c935d1bbdbb4f1f58d4741d9 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 11:35:42 -0700 Subject: [PATCH 13/69] Updating mirror configuration more. --- stackinator/main.py | 16 ++++--- stackinator/mirror.py | 79 +++++++++++++++++++++++----------- stackinator/recipe.py | 27 +++--------- stackinator/schema/mirror.json | 7 ++- 4 files changed, 78 insertions(+), 51 deletions(-) diff --git a/stackinator/main.py b/stackinator/main.py index 4440621..6f30ddf 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -81,13 +81,19 @@ def log_header(args): def make_argparser(): parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from a recipe.")) parser.add_argument("--version", action="version", version=f"stackinator version {VERSION}") - parser.add_argument("-b", "--build", required=True, type=str) + parser.add_argument("-b", "--build", required=True, type=str, + help="Where to set up the stackinator build directory. " + "('/tmp' is not allowed, use '/var/tmp'") parser.add_argument("--no-bwrap", action="store_true", required=False) - parser.add_argument("-r", "--recipe", required=True, type=str) - parser.add_argument("-s", "--system", required=True, type=str) + parser.add_argument("-r", "--recipe", required=True, type=str, + help="Name of (and/or path to) the Stackinator recipe.") + parser.add_argument("-s", "--system", required=True, type=str, + help="Name of (and/or path to) the Stackinator system configuration.") parser.add_argument("-d", "--debug", action="store_true") - parser.add_argument("-m", "--mount", required=False, type=str) - parser.add_argument("-c", "--cache", required=False, type=str) + parser.add_argument("-m", "--mount", required=False, type=str, + help="The mount point where the environment will be located.") + parser.add_argument("-c", "--cache", required=False, type=str, + help="Buildcache location or name (from system config's mirrors.yaml).") parser.add_argument("--develop", action="store_true", required=False) return parser diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 08c303b..40487f4 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,42 +1,73 @@ import os import pathlib import urllib.request +from typing import Optional import yaml from . import schema -def configuration_from_file(file): - with file.open() as fid: - # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) +def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = None): + """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + + if path.exists(): + with path.open() as fid: + # load the raw yaml input + raw = yaml.load(fid, Loader=yaml.Loader) + + print(f"Configuring mirrors and buildcache from '{path}'") # validate the yaml schema.CacheValidator.validate(raw) mirrors = [mirror for mirror in raw if mirror["enabled"]] + else: + mirrors = [] + + buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) + if buildcache_dest_count > 1: + raise RuntimeError("Mirror config has more than one mirror specified as the build cache destination " + "in the system config's 'mirrors.yaml'.") + elif buildcache_dest_count == 1 and cmdline_cache: + raise RuntimeError("Build cache destination specified on the command line and in the system config's " + "'mirrors.yaml'. It can be one or the other, but not both.") + + # Add or set the cache given on the command line as the buildcache destination + if cmdline_cache is not None: + existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] + # If the mirror name given on the command line isn't in the config, assume it + # is the URL to a build cache. + if not existing_mirror: + mirrors.append( + { + 'name': 'cmdline_cache', + 'url': cmdline_cache, + 'buildcache': True, + 'bootstrap': False, + } + ) + + for mirror in mirrors: + url = mirror["url"] + if url.beginswith("file://"): + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(url)) + if not path.is_absolute(): + raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + if not path.is_dir(): + raise FileNotFoundError(f"The build cache path '{path}' does not exist") + + mirror["url"] = path + + else: + try: + request = urllib.request.Request(url, method='HEAD') + response = urllib.request.urlopen(request) + except urllib.error.URLError as e: + print(f'Error: {e.reason}') - for mirror in mirrors: - url = mirror["url"] - if url.beginswith("file://"): - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(url)) - if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") - - mirror["url"] = path - - else: - try: - request = urllib.request.Request(url, method='HEAD') - response = urllib.request.urlopen(request) - except urllib.error.URLError as e: - print(f'Error: {e.reason}') - - return mirrors + return mirrors def generate_mirrors_yaml(config): diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 281ab8b..4b7c038 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -170,11 +170,11 @@ def __init__(self, args): schema.EnvironmentsValidator.validate(raw) self.generate_environment_specs(raw) - mirrors_path = self.system_config_path/'mirrors.yaml' - self._logger.debug(f"opening {mirrors_path}") - - # load the optional mirrors.yaml from system config: - self.mirrors = self.system_config_path / "mirrors.yaml" + # load the optional mirrors.yaml from system config, and add any additional + # mirrors specified on the command line. + self._mirrors = None + self._logger.debug("Configuring mirrors.") + self._mirrors = mirror.configuration_from_file(self.system_config_path/"mirrors.yaml", args.cache) # optional post install hook if self.post_install_hook is not None: @@ -236,22 +236,7 @@ def pre_install_hook(self): @property def mirrors(self): return self._mirrors - - # configuration is a tuple with two fields: - # - a Path of the yaml file containing the cache configuration - # - the mount point of the image - @mirrors.setter - def mirrors(self, path: Optional[pathlib.Path]): - """Initialize the mirrors property from config.""" - self._mirrors = None - if path is not None: - if not path.is_file(): - raise FileNotFoundError("The system config 'mirrors.yaml' file exists, but isn't a " - "readable file.") - - self._logger.debug(f"configuring mirrors from {path}") - self._mirrors = mirror.configuration_from_file(path) - + @property def config(self): return self._config diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index b32a3cd..a53cc34 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -23,7 +23,12 @@ "type": "boolean", "default": false, "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror.", - } + }, + "buildcache": { + "type": "boolean", + "default": false, + "description": "Use this mirror as the buildcache push destination. Can only be enabled on a single mirror." + }, "public_key": { "type": "string", "description": "Public PGP key for validating binary cache packages.", From 0ed23d8f6f487100d522875afcbb6d20955e9aa4 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 11:25:37 -0700 Subject: [PATCH 14/69] mirror yaml generator --- stackinator/mirror.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 40487f4..315d2e8 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -70,19 +70,16 @@ def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = N return mirrors -def generate_mirrors_yaml(config): - path = config["path"].as_posix() - mirrors = { - "mirrors": { - "alpscache": { - "fetch": { - "url": f"file://{path}", - }, - "push": { - "url": f"file://{path}", - }, - } +def generate_mirrors_yaml(mirrors): + yaml = {"mirrors": {}} + + for m in mirrors: + name = m["name"] + url = m["url"] + + yaml["mirrors"][name] = { + "fetch": {"url": url}, + "push": {"url": url}, } - } - return yaml.dump(mirrors, default_flow_style=False) + return yaml.dump(yaml, default_flow_style=False) \ No newline at end of file From e2a67ab1d3a6a08e1f8f5222a46bab715b77d96c Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 11:37:34 -0700 Subject: [PATCH 15/69] update mirrors --- stackinator/builder.py | 10 ++++------ stackinator/mirror.py | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index c3de44d..5b223ff 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -226,14 +226,12 @@ def generate(self, recipe): with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( - cache=recipe.mirror, modules=recipe.with_modules, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - # pass source_mirrors to Makefile render - source_mirrors=recipe.config.get("source_mirrors", {}), + mirrors=recipe.mirrors exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) @@ -314,11 +312,11 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured - if recipe.mirror: + if recipe.mirrors: dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the build cache mirror: {dst}") + self._logger.debug(f"generate the spack mirrors.yaml: {dst}") with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirror)) + fid.write(cache.generate_mirrors_yaml(recipe.mirrors)) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 315d2e8..e802018 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -67,7 +67,20 @@ def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = N except urllib.error.URLError as e: print(f'Error: {e.reason}') - return mirrors + if mirror["key"]: + #if path, check if exists + path = pathlib.Path(os.path.expandvars(mirror["key"])) + if path.exists(): + if not path.is_file(): + raise FileNotFoundError(f"The key path '{path}' is not a file") + else + #if key, save to file, change to path + + if mirror["bootstrap"]: + #make bootstrap dirs + #bootstrap//metadata.yaml + + return mirrors def generate_mirrors_yaml(mirrors): From 73463861a35602c9a0fc77dfb594ff58cb748c10 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Sat, 7 Mar 2026 00:54:59 -0700 Subject: [PATCH 16/69] validate keys in mirror config and fixed yaml generator --- stackinator/mirror.py | 74 ++++++++++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e802018..afbacaf 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -2,15 +2,21 @@ import pathlib import urllib.request from typing import Optional +import magic import yaml from . import schema +class MirrorConfigError(RuntimeError): + """Exception class for errors thrown by mirror configuration problems.""" -def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = None): + + +def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + path = system_config_root/"mirrors.yaml" if path.exists(): with path.open() as fid: # load the raw yaml input @@ -54,36 +60,33 @@ def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = N # verify that the root path exists path = pathlib.Path(os.path.expandvars(url)) if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + raise FileNotFoundError(f"The mirror path '{path}' is not absolute") if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") + raise FileNotFoundError(f"The mirror path '{path}' does not exist") mirror["url"] = path - else: + elif url.beginswith("https://"): try: request = urllib.request.Request(url, method='HEAD') response = urllib.request.urlopen(request) except urllib.error.URLError as e: - print(f'Error: {e.reason}') - - if mirror["key"]: - #if path, check if exists - path = pathlib.Path(os.path.expandvars(mirror["key"])) - if path.exists(): - if not path.is_file(): - raise FileNotFoundError(f"The key path '{path}' is not a file") - else - #if key, save to file, change to path + raise MirrorConfigError( + f"Could not reach the mirror url '{url}'. " + f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - if mirror["bootstrap"]: - #make bootstrap dirs - #bootstrap//metadata.yaml + if mirror["bootstrap"]: + #make bootstrap dirs + #bootstrap//metadata.yaml return mirrors -def generate_mirrors_yaml(mirrors): +def setup(mirrors, config_path): + dst = config_path / "mirrors.yaml" + self._logger.debug(f"generate the spack mirrors.yaml: {dst}") + with dst.open("w") as fid: + fid.write() yaml = {"mirrors": {}} for m in mirrors: @@ -95,4 +98,37 @@ def generate_mirrors_yaml(mirrors): "push": {"url": url}, } - return yaml.dump(yaml, default_flow_style=False) \ No newline at end of file + return yaml.dump(yaml, default_flow_style=False) + +#called from builder +def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: pathlib.Path): + for mirror in mirrors: + if mirror["key"]: + key = mirror["key"] + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if path.exists(): + if not path.is_absolute(): + #try prepending system config path + path = system_config_path + path + if not.path.is_file() + raise FileNotFoundError( + f"The key path '{path}' is not a file. " + f"Check the key listed in mirrors.yaml in system config.") + file_type = magic.from_file(path) + if not file_type.startswith("OpenPGP Public Key"): + raise MirrorConfigError( + f"'{key}' is not a valid GPG key. " + f"Check the key listed in mirrors.yaml in system config.") + # copy file to key store + with file open: + data = key.read + dest = mkdir(new_key_file) + dest.write(data) + # mirror["key"] = new_path + + else: + # if PGP key, convert to binary, ???, convert back + # if key, save to file, change to path + + \ No newline at end of file From e722f498c8315ce0fc3cd7471824a5648b557da2 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Sat, 7 Mar 2026 00:57:53 -0700 Subject: [PATCH 17/69] validate keys in mirror config and fixed yaml generator --- stackinator/mirror.py | 44 +++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index afbacaf..e521b59 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -12,7 +12,6 @@ class MirrorConfigError(RuntimeError): """Exception class for errors thrown by mirror configuration problems.""" - def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" @@ -62,7 +61,7 @@ def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Opt if not path.is_absolute(): raise FileNotFoundError(f"The mirror path '{path}' is not absolute") if not path.is_dir(): - raise FileNotFoundError(f"The mirror path '{path}' does not exist") + raise FileNotFoundError(f"The mirror path '{path}' is not a directory") mirror["url"] = path @@ -82,11 +81,12 @@ def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Opt return mirrors -def setup(mirrors, config_path): +def yaml_setup(mirrors, config_path): + """Generate the mirrors.yaml for spack""" + dst = config_path / "mirrors.yaml" + self._logger.debug(f"generate the spack mirrors.yaml: {dst}") - with dst.open("w") as fid: - fid.write() yaml = {"mirrors": {}} for m in mirrors: @@ -98,13 +98,22 @@ def setup(mirrors, config_path): "push": {"url": url}, } - return yaml.dump(yaml, default_flow_style=False) + with dst.open("w") as file: + yaml.dump(yaml, default_flow_style=False) + + # return dst + -#called from builder def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: pathlib.Path): + """Validate mirror keys, relocate to key_store, and update mirror config with new key paths""" + for mirror in mirrors: if mirror["key"]: key = mirror["key"] + + # key will be saved under key_store/mirror_name.gpg + dst = (key_store / f"'{mirror["name"]}'.gpg").resolve() + # if path, check if abs path, if not, append sys config path in front and check again path = pathlib.Path(os.path.expandvars(key)) if path.exists(): @@ -115,20 +124,23 @@ def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: raise FileNotFoundError( f"The key path '{path}' is not a file. " f"Check the key listed in mirrors.yaml in system config.") + file_type = magic.from_file(path) + if not file_type.startswith("OpenPGP Public Key"): raise MirrorConfigError( - f"'{key}' is not a valid GPG key. " + f"'{path}' is not a valid GPG key. " f"Check the key listed in mirrors.yaml in system config.") - # copy file to key store - with file open: - data = key.read - dest = mkdir(new_key_file) - dest.write(data) - # mirror["key"] = new_path + + # copy key to new destination in key store + with open(path, 'r') as reader, open(dst, 'w') as writer: + data = reader.read() + writer.write(data) else: # if PGP key, convert to binary, ???, convert back - # if key, save to file, change to path + with open(dst, "w") as file: + file.write(key) - \ No newline at end of file + # update mirror with new path + mirror["key"] = dst \ No newline at end of file From eb2685f020534c0c993fa2779589ebf08ad0f226 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Mon, 9 Mar 2026 14:33:15 -0600 Subject: [PATCH 18/69] connecting mirrors to builder.py --- stackinator/builder.py | 9 +++++---- stackinator/mirror.py | 9 ++++----- stackinator/recipe.py | 2 +- stackinator/schema/mirror.json | 16 +++++++--------- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 5b223ff..7918cb2 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -11,7 +11,7 @@ import jinja2 import yaml -from . import VERSION, cache, root_logger, spack_util +from . import VERSION, cache, root_logger, spack_util, mirror def install(src, dst, *, ignore=None, symlinks=False): @@ -231,7 +231,7 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - mirrors=recipe.mirrors + mirrors=recipe.mirrors, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) @@ -312,11 +312,12 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured + key_store = self.path / ".gnupg" if recipe.mirrors: + mirror.key_setup(recipe.mirrors, config_path, key_store) dst = config_path / "mirrors.yaml" self._logger.debug(f"generate the spack mirrors.yaml: {dst}") - with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirrors)) + mirror.spack_yaml_setup(recipe.mirrors, dst) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e521b59..0e5ee5d 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,7 +1,7 @@ import os import pathlib import urllib.request -from typing import Optional +from typing import Optional, List, Dict import magic import yaml @@ -74,19 +74,18 @@ def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Opt f"Could not reach the mirror url '{url}'. " f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - if mirror["bootstrap"]: + #if mirror["bootstrap"]: #make bootstrap dirs #bootstrap//metadata.yaml return mirrors -def yaml_setup(mirrors, config_path): +def spack_yaml_setup(mirrors, config_path): """Generate the mirrors.yaml for spack""" dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the spack mirrors.yaml: {dst}") yaml = {"mirrors": {}} for m in mirrors: @@ -120,7 +119,7 @@ def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: if not path.is_absolute(): #try prepending system config path path = system_config_path + path - if not.path.is_file() + if not path.is_file(): raise FileNotFoundError( f"The key path '{path}' is not a file. " f"Check the key listed in mirrors.yaml in system config.") diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 4b7c038..c59790b 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -174,7 +174,7 @@ def __init__(self, args): # mirrors specified on the command line. self._mirrors = None self._logger.debug("Configuring mirrors.") - self._mirrors = mirror.configuration_from_file(self.system_config_path/"mirrors.yaml", args.cache) + self._mirrors = mirror.configuration_from_file(self.system_config_path, args.cache) # optional post install hook if self.post_install_hook is not None: diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index a53cc34..a8be6ab 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -1,28 +1,26 @@ -# This config handles source mirrors, binary caches, and bootstrap mirrors (of both forms) { - # Order matters, so we need an array. "type" : "array", "items": { "type": "object", - "required": ["name", "url"] + "required": ["name", "url"], "properties": { "name": { "type": "string", - "description": "The name of this mirror. Should be follow standard variable naming syntax.", + "description": "The name of this mirror. Should be follow standard variable naming syntax." }, "url": { "type": "string", - "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI).", + "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI)." }, "enabled": { "type": "boolean", "default": true, - "description": "Whether this mirror is enabled.", + "description": "Whether this mirror is enabled." }, "bootstrap": { "type": "boolean", "default": false, - "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror.", + "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror." }, "buildcache": { "type": "boolean", @@ -31,7 +29,7 @@ }, "public_key": { "type": "string", - "description": "Public PGP key for validating binary cache packages.", + "description": "Public PGP key for validating binary cache packages." }, "description": { "type": "string", @@ -39,4 +37,4 @@ } } } -} +} \ No newline at end of file From 9d40a836ae0e380a282c208e8498bba741de1fe6 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 13:42:30 -0600 Subject: [PATCH 19/69] Put the mirror manipulation code in a class. --- stackinator/builder.py | 14 ++- stackinator/mirror.py | 239 +++++++++++++++++++++-------------------- stackinator/recipe.py | 3 +- 3 files changed, 134 insertions(+), 122 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 7918cb2..b35545b 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -164,6 +164,7 @@ def environment_meta(self, recipe): self._environment_meta = meta def generate(self, recipe): + """Setup the recipe build environment.""" # make the paths, in case bwrap is not used, directly write to recipe.mount store_path = self.path / "store" if not recipe.no_bwrap else pathlib.Path(recipe.mount) tmp_path = self.path / "tmp" @@ -313,11 +314,14 @@ def generate(self, recipe): # generate a mirrors.yaml file if build caches have been configured key_store = self.path / ".gnupg" - if recipe.mirrors: - mirror.key_setup(recipe.mirrors, config_path, key_store) - dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the spack mirrors.yaml: {dst}") - mirror.spack_yaml_setup(recipe.mirrors, dst) + mirrors = recipe.mirrors + if mirrors: + mirrors.key_setup(recipe.mirrors, config_path, key_store) + dest = config_path / "mirrors.yaml" + self._logger.debug(f"generate the spack mirrors.yaml: {dest}") + mirrors.create_spack_mirrors_yaml(dest) + + # Setup bootstrap mirror configs. # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 0e5ee5d..bccd542 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,6 +1,7 @@ import os import pathlib import urllib.request +import urllib.error from typing import Optional, List, Dict import magic @@ -8,138 +9,146 @@ from . import schema -class MirrorConfigError(RuntimeError): +class MirrorError(RuntimeError): """Exception class for errors thrown by mirror configuration problems.""" - -def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): - """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" - - path = system_config_root/"mirrors.yaml" - if path.exists(): - with path.open() as fid: - # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) - - print(f"Configuring mirrors and buildcache from '{path}'") - - # validate the yaml - schema.CacheValidator.validate(raw) - - mirrors = [mirror for mirror in raw if mirror["enabled"]] - else: - mirrors = [] - - buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) - if buildcache_dest_count > 1: - raise RuntimeError("Mirror config has more than one mirror specified as the build cache destination " - "in the system config's 'mirrors.yaml'.") - elif buildcache_dest_count == 1 and cmdline_cache: - raise RuntimeError("Build cache destination specified on the command line and in the system config's " - "'mirrors.yaml'. It can be one or the other, but not both.") - - # Add or set the cache given on the command line as the buildcache destination - if cmdline_cache is not None: - existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] - # If the mirror name given on the command line isn't in the config, assume it - # is the URL to a build cache. - if not existing_mirror: - mirrors.append( - { - 'name': 'cmdline_cache', - 'url': cmdline_cache, - 'buildcache': True, - 'bootstrap': False, - } - ) - - for mirror in mirrors: - url = mirror["url"] - if url.beginswith("file://"): - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(url)) - if not path.is_absolute(): - raise FileNotFoundError(f"The mirror path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The mirror path '{path}' is not a directory") - - mirror["url"] = path - - elif url.beginswith("https://"): - try: - request = urllib.request.Request(url, method='HEAD') - response = urllib.request.urlopen(request) - except urllib.error.URLError as e: - raise MirrorConfigError( - f"Could not reach the mirror url '{url}'. " - f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - - #if mirror["bootstrap"]: - #make bootstrap dirs - #bootstrap//metadata.yaml +class Mirrors: + """Manage the definition of mirrors in a recipe.""" + + def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): + """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + + self._system_config_root = system_config_root + + self.mirrors = self._load_mirrors(cmdline_cache) + self._check_mirrors() + + self.build_cache_mirrors = [mirror for mirror in self.mirrors if mirror.get('buildcache', False)] + self.keys = [mirror['key'] for mirror in self.mirrors if mirror.get('key') is not None] + + def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: + """Load the mirrors file, if one exists.""" + path = self._system_config_root/"mirrors.yaml" + if path.exists(): + with path.open() as fid: + # load the raw yaml input + raw = yaml.load(fid, Loader=yaml.Loader) + + # validate the yaml + schema.CacheValidator.validate(raw) + + mirrors = [mirror for mirror in raw if mirror["enabled"]] + else: + mirrors = [] + + buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) + if buildcache_dest_count > 1: + raise MirrorError("Mirror config has more than one mirror specified as the build cache destination " + "in the system config's 'mirrors.yaml'.") + elif buildcache_dest_count == 1 and cmdline_cache: + raise MirrorError("Build cache destination specified on the command line and in the system config's " + "'mirrors.yaml'. It can be one or the other, but not both.") + + # Add or set the cache given on the command line as the buildcache destination + if cmdline_cache is not None: + existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] + # If the mirror name given on the command line isn't in the config, assume it + # is the URL to a build cache. + if not existing_mirror: + mirrors.append( + { + 'name': 'cmdline_cache', + 'url': cmdline_cache, + 'buildcache': True, + 'bootstrap': False, + } + ) return mirrors + def _check_mirrors(self): + """Validate the mirror config entries.""" -def spack_yaml_setup(mirrors, config_path): - """Generate the mirrors.yaml for spack""" + for mirror in self.mirrors: + url = mirror["url"] + if url.beginswith("file://"): + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(url)) + if not path.is_absolute(): + raise MirrorError(f"The mirror path '{path}' is not absolute") + if not path.is_dir(): + raise MirrorError(f"The mirror path '{path}' is not a directory") - dst = config_path / "mirrors.yaml" + mirror["url"] = path - yaml = {"mirrors": {}} + elif url.beginswith("https://"): + try: + request = urllib.request.Request(url, method='HEAD') + urllib.request.urlopen(request) + except urllib.error.URLError as e: + raise MirrorError( + f"Could not reach the mirror url '{url}'. " + f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - for m in mirrors: - name = m["name"] - url = m["url"] + def create_spack_mirrors_yaml(self, dest: pathlib.Path): + """Generate the mirrors.yaml for our build directory.""" - yaml["mirrors"][name] = { - "fetch": {"url": url}, - "push": {"url": url}, - } + raw = {"mirrors": {}} - with dst.open("w") as file: - yaml.dump(yaml, default_flow_style=False) + for m in self.mirrors: + name = m["name"] + url = m["url"] - # return dst + raw["mirrors"][name] = { + "fetch": {"url": url}, + "push": {"url": url}, + } + with dest.open("w") as file: + yaml.dump(raw, file, default_flow_style=False) -def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: pathlib.Path): - """Validate mirror keys, relocate to key_store, and update mirror config with new key paths""" + def bootstrap_setup(self, config_root: pathlib.Path): + """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" - for mirror in mirrors: - if mirror["key"]: - key = mirror["key"] - # key will be saved under key_store/mirror_name.gpg - dst = (key_store / f"'{mirror["name"]}'.gpg").resolve() - # if path, check if abs path, if not, append sys config path in front and check again - path = pathlib.Path(os.path.expandvars(key)) - if path.exists(): - if not path.is_absolute(): - #try prepending system config path - path = system_config_path + path - if not path.is_file(): - raise FileNotFoundError( - f"The key path '{path}' is not a file. " - f"Check the key listed in mirrors.yaml in system config.") + def key_setup(self, key_store: pathlib.Path): + """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" - file_type = magic.from_file(path) + for mirror in self.mirrors: + if mirror["key"]: + key = mirror["key"] - if not file_type.startswith("OpenPGP Public Key"): - raise MirrorConfigError( - f"'{path}' is not a valid GPG key. " - f"Check the key listed in mirrors.yaml in system config.") - - # copy key to new destination in key store - with open(path, 'r') as reader, open(dst, 'w') as writer: - data = reader.read() - writer.write(data) + # key will be saved under key_store/mirror_name.gpg + dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() + + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if path.exists(): + if not path.is_absolute(): + #try prepending system config path + path = self._system_config_root/path + if not path.is_file(): + raise MirrorError( + f"The key path '{path}' is not a file. " + f"Check the key listed in mirrors.yaml in system config.") + + file_type = magic.from_file(path) + + if not file_type.startswith("OpenPGP Public Key"): + raise MirrorError( + f"'{path}' is not a valid GPG key. " + f"Check the key listed in mirrors.yaml in system config.") + + # copy key to new destination in key store + with open(path, 'r') as reader, open(dest, 'w') as writer: + data = reader.read() + writer.write(data) + + else: + # if PGP key, convert to binary, ???, convert back + with open(dest, "w") as file: + file.write(key) - else: - # if PGP key, convert to binary, ???, convert back - with open(dst, "w") as file: - file.write(key) - - # update mirror with new path - mirror["key"] = dst \ No newline at end of file + # update mirror with new path + mirror["key"] = dest diff --git a/stackinator/recipe.py b/stackinator/recipe.py index c59790b..b03d751 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -172,9 +172,8 @@ def __init__(self, args): # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. - self._mirrors = None self._logger.debug("Configuring mirrors.") - self._mirrors = mirror.configuration_from_file(self.system_config_path, args.cache) + self._mirrors = mirror.Mirrors(self.system_config_path, args.cache) # optional post install hook if self.post_install_hook is not None: From 2a7bd1dcacf433389d726f8b497db330da78c749 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Thu, 12 Mar 2026 14:04:10 -0600 Subject: [PATCH 20/69] preserve cache for makefile --- stackinator/builder.py | 1 + stackinator/recipe.py | 9 +++++++-- stackinator/schema.py | 1 + stackinator/templates/Makefile | 21 ++++++++------------- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index b35545b..2a2ceef 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -227,6 +227,7 @@ def generate(self, recipe): with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( + cache = recipe.cache, modules=recipe.with_modules, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, diff --git a/stackinator/recipe.py b/stackinator/recipe.py index b03d751..9915bf9 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -174,6 +174,7 @@ def __init__(self, args): # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") self._mirrors = mirror.Mirrors(self.system_config_path, args.cache) + self._cache = [mirror for mirror in self.mirrors if mirror["buildcache"]] # optional post install hook if self.post_install_hook is not None: @@ -235,6 +236,10 @@ def pre_install_hook(self): @property def mirrors(self): return self._mirrors + + @property + def cache(self): + return self._cache @property def config(self): @@ -515,7 +520,7 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.mirrors + push_to_cache = self.cache files["makefile"] = makefile_template.render( compilers=self.compilers, push_to_cache=push_to_cache, @@ -546,7 +551,7 @@ def environment_files(self): jenv.filters["py2yaml"] = schema.py2yaml makefile_template = jenv.get_template("Makefile.environments") - push_to_cache = self.mirror is not None + push_to_cache = self.cache is not None files["makefile"] = makefile_template.render( environments=self.environments, push_to_cache=push_to_cache, diff --git a/stackinator/schema.py b/stackinator/schema.py index 3a2a984..d461ff0 100644 --- a/stackinator/schema.py +++ b/stackinator/schema.py @@ -121,3 +121,4 @@ def check_module_paths(instance): EnvironmentsValidator = SchemaValidator(prefix / "schema/environments.json") CacheValidator = SchemaValidator(prefix / "schema/cache.json") ModulesValidator = SchemaValidator(prefix / "schema/modules.json", check_module_paths) +MirrorsValidator = SchemaValidator(prefix / "schema/mirror.json") diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index d1a06dd..9484e3c 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -32,25 +32,20 @@ pre-install: spack-setup $(SANDBOX) $(STORE)/pre-install-hook mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} + {% if cache %} - # The old way of managing mirrors $(SANDBOX) $(SPACK) buildcache keys --install --trust - {% if cache.key %} - $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} - {% endif %} {% endif %} {% if mirrors %} - @echo "Adding mirrors and gpg keys." - {% for mirror_info in mirrors | reverse %} - $(SANDBOX) $(SPACK) mirror add --scope=site {{ mirror_info.name }} {{ mirror_info.url }} - $(SANDBOX) $(SPACK) gpg trust {{ mirror_info.key_path }} + @echo "Adding mirror gpg keys." + {% for mirror in mirrors | reverse %} + {% if mirror.public_key %} + $(SANDBOX) $(SPACK) gpg trust {{ mirror.public_key }} + {% endif %} {% endfor %} @echo "Current mirror list:" $(SANDBOX) $(SPACK) mirror list {% endif %} - {% for mirror_info in filter(lambda m: m['bootstrap'], mirrors) | filter() %} - $(SANDBOX) $(SPACK) bootstrap add --scope=site {{ mirror_info.name }} bootstrap/{{ mirror_info.name }} - {% endfor %} touch mirror-setup compilers: mirror-setup @@ -89,14 +84,14 @@ store.squashfs: post-install # Force push all built packages to the build cache cache-force: mirror-setup -{% if cache.key %} +{% if cache %} $(warning ================================================================================) $(warning Generate the config in order to force push partially built compiler environments) $(warning if this step is performed with partially built compiler envs, you will) $(warning likely have to start a fresh build (but that's okay, because build caches FTW)) $(warning ================================================================================) $(SANDBOX) $(MAKE) -C generate-config - $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package alpscache \ + $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package cache.name \ $$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ From 8589cff12c92fca787c71e5f9bae2adc77fdf2d9 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:09:46 -0600 Subject: [PATCH 21/69] Adding bootstrap mirror configs. --- stackinator/builder.py | 17 ++++++++-------- stackinator/mirror.py | 45 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 2a2ceef..acc2d91 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -314,15 +314,14 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured - key_store = self.path / ".gnupg" - mirrors = recipe.mirrors - if mirrors: - mirrors.key_setup(recipe.mirrors, config_path, key_store) - dest = config_path / "mirrors.yaml" - self._logger.debug(f"generate the spack mirrors.yaml: {dest}") - mirrors.create_spack_mirrors_yaml(dest) - - # Setup bootstrap mirror configs. + if recipe.mirrors: + recipe.mirrors.key_setup(config_path) + + self._logger.debug(f"Generating the spack mirrors.yaml in '{config_path}'") + recipe.mirrors.create_spack_mirrors_yaml(config_path/'mirrors.yaml') + + # Setup bootstrap mirror configs. + recipe.mirrors.create_bootstrap_configs(config_path) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index bccd542..9a7dc79 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -2,7 +2,7 @@ import pathlib import urllib.request import urllib.error -from typing import Optional, List, Dict +from typing import ByteString, Optional, List, Dict import magic import yaml @@ -22,8 +22,10 @@ def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str self.mirrors = self._load_mirrors(cmdline_cache) self._check_mirrors() - - self.build_cache_mirrors = [mirror for mirror in self.mirrors if mirror.get('buildcache', False)] + + self.build_cache_mirror = ([mirror for mirror in self.mirrors if mirror.get('buildcache', False)] + + [None]).pop(0) + self.bootstrap_mirrors = [mirror for mirror in self.mirrors if mirror.get('bootstrap', False)] self.keys = [mirror['key'] for mirror in self.mirrors if mirror.get('key') is not None] def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: @@ -107,12 +109,43 @@ def create_spack_mirrors_yaml(self, dest: pathlib.Path): with dest.open("w") as file: yaml.dump(raw, file, default_flow_style=False) - def bootstrap_setup(self, config_root: pathlib.Path): + def create_bootstrap_configs(self, config_root: pathlib.Path): """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" + if not self.bootstrap_mirrors: + return + + bootstrap_yaml = { + 'sources': [], + 'trusted': {}, + } + + for mirror in self.bootstrap_mirrors: + name = mirror['name'] + bs_mirror_path = config_root/f'bootstrap/{name}' + # Tell spack where to find the metadata for each bootstrap mirror. + bootstrap_yaml['sources'].append( + { + 'name': name, + 'metadata': bs_mirror_path, + } + ) + # And trust each one + bootstrap_yaml['trusted'][name] = True + + # Create the metadata dir and metadata.yaml + bs_mirror_path.mkdir(parents=True) + bs_mirror_yaml = { + 'type': 'install', + 'info': mirror['url'], + } + with (bs_mirror_path/'metadata.yaml').open('w') as file: + yaml.dump(bs_mirror_yaml, file, default_flow_style=False) + + with (config_root/'bootstrap.yaml').open('w') as file: + yaml.dump(bootstrap_yaml, file, default_flow_style=False) - - def key_setup(self, key_store: pathlib.Path): + def key_setup(self, config_root: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: From d6a3df73d17fb60086187cba44427c7b454fc847 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:16:21 -0600 Subject: [PATCH 22/69] Reverted to defining the key store path in builder. --- stackinator/builder.py | 2 +- stackinator/mirror.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index acc2d91..80a3977 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -315,7 +315,7 @@ def generate(self, recipe): # generate a mirrors.yaml file if build caches have been configured if recipe.mirrors: - recipe.mirrors.key_setup(config_path) + recipe.mirrors.key_setup(config_path/'key_store') self._logger.debug(f"Generating the spack mirrors.yaml in '{config_path}'") recipe.mirrors.create_spack_mirrors_yaml(config_path/'mirrors.yaml') diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 9a7dc79..9f16276 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -145,7 +145,7 @@ def create_bootstrap_configs(self, config_root: pathlib.Path): with (config_root/'bootstrap.yaml').open('w') as file: yaml.dump(bootstrap_yaml, file, default_flow_style=False) - def key_setup(self, config_root: pathlib.Path): + def key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: From ee0d45fe39bf61cc3d0ac8b131afe359ff91314c Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:22:31 -0600 Subject: [PATCH 23/69] Compressed mirror config setup into a single interface. --- stackinator/builder.py | 5 +---- stackinator/mirror.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 80a3977..f593fb0 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -315,12 +315,9 @@ def generate(self, recipe): # generate a mirrors.yaml file if build caches have been configured if recipe.mirrors: + self._logger.debug(f"Generating the spack mirror configs in '{config_path}'") recipe.mirrors.key_setup(config_path/'key_store') - - self._logger.debug(f"Generating the spack mirrors.yaml in '{config_path}'") recipe.mirrors.create_spack_mirrors_yaml(config_path/'mirrors.yaml') - - # Setup bootstrap mirror configs. recipe.mirrors.create_bootstrap_configs(config_path) # Add custom spack package recipes, configured via Spack repos. diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 9f16276..6de5993 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -15,6 +15,9 @@ class MirrorError(RuntimeError): class Mirrors: """Manage the definition of mirrors in a recipe.""" + KEY_STORE_DIR = 'key_store' + MIRRORS_YAML = 'mirrors.yaml' + def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" @@ -92,7 +95,14 @@ def _check_mirrors(self): f"Could not reach the mirror url '{url}'. " f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - def create_spack_mirrors_yaml(self, dest: pathlib.Path): + def setup_configs(self, config_root: pathlib.Path): + """Setup all mirror configs in the given config_root.""" + + self._key_setup(config_root/self.KEY_STORE_DIR) + self._create_spack_mirrors_yaml(config_root/self.MIRRORS_YAML) + self._create_bootstrap_configs(config_root) + + def _create_spack_mirrors_yaml(self, dest: pathlib.Path): """Generate the mirrors.yaml for our build directory.""" raw = {"mirrors": {}} @@ -109,7 +119,7 @@ def create_spack_mirrors_yaml(self, dest: pathlib.Path): with dest.open("w") as file: yaml.dump(raw, file, default_flow_style=False) - def create_bootstrap_configs(self, config_root: pathlib.Path): + def _create_bootstrap_configs(self, config_root: pathlib.Path): """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" if not self.bootstrap_mirrors: @@ -145,7 +155,7 @@ def create_bootstrap_configs(self, config_root: pathlib.Path): with (config_root/'bootstrap.yaml').open('w') as file: yaml.dump(bootstrap_yaml, file, default_flow_style=False) - def key_setup(self, key_store: pathlib.Path): + def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: From 90d7733cba58581938fd6809505f248827556e8a Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:26:49 -0600 Subject: [PATCH 24/69] Catching builder exceptions. --- stackinator/builder.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index f593fb0..5bab0ea 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -314,11 +314,12 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured - if recipe.mirrors: - self._logger.debug(f"Generating the spack mirror configs in '{config_path}'") - recipe.mirrors.key_setup(config_path/'key_store') - recipe.mirrors.create_spack_mirrors_yaml(config_path/'mirrors.yaml') - recipe.mirrors.create_bootstrap_configs(config_path) + self._logger.debug(f"Generating the spack mirror configs in '{config_path}'") + try: + recipe.mirrors.setup_configs(config_path) + except mirror.MirrorError as err: + self._logger.error(f"Could not set up mirrors.\n{err}") + return 1 # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to From 7362cbcc17072c518a46793ebca0228c98d2c6c7 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Thu, 12 Mar 2026 14:33:08 -0600 Subject: [PATCH 25/69] fixing key setup --- stackinator/mirror.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 6de5993..c3b2c40 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -159,8 +159,8 @@ def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: - if mirror["key"]: - key = mirror["key"] + if mirror["public_key"]: + key = mirror["public_key"] # key will be saved under key_store/mirror_name.gpg dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() From 2a4632d900cb6023d113fcbb21d7f2ff49690d1a Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Thu, 12 Mar 2026 14:46:21 -0600 Subject: [PATCH 26/69] In progress. --- stackinator/mirror.py | 78 ++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index c3b2c40..8f5e440 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,3 +1,4 @@ +import base64 import os import pathlib import urllib.request @@ -29,7 +30,8 @@ def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str self.build_cache_mirror = ([mirror for mirror in self.mirrors if mirror.get('buildcache', False)] + [None]).pop(0) self.bootstrap_mirrors = [mirror for mirror in self.mirrors if mirror.get('bootstrap', False)] - self.keys = [mirror['key'] for mirror in self.mirrors if mirror.get('key') is not None] + # Will hold a list of all the keys + self.keys = None def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: """Load the mirrors file, if one exists.""" @@ -159,39 +161,47 @@ def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" for mirror in self.mirrors: - if mirror["public_key"]: - key = mirror["public_key"] - - # key will be saved under key_store/mirror_name.gpg - dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() - - # if path, check if abs path, if not, append sys config path in front and check again - path = pathlib.Path(os.path.expandvars(key)) - if path.exists(): - if not path.is_absolute(): - #try prepending system config path - path = self._system_config_root/path - if not path.is_file(): - raise MirrorError( - f"The key path '{path}' is not a file. " - f"Check the key listed in mirrors.yaml in system config.") - - file_type = magic.from_file(path) - - if not file_type.startswith("OpenPGP Public Key"): + if not mirror["public_key"]: + continue + + key = mirror["public_key"] + + # key will be saved under key_store/mirror_name.gpg + dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() + + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if path.exists(): + if not path.is_absolute(): + #try prepending system config path + path = self._system_config_root/path + if not path.is_file(): raise MirrorError( - f"'{path}' is not a valid GPG key. " + f"The key path '{path}' is not a file. " f"Check the key listed in mirrors.yaml in system config.") - - # copy key to new destination in key store - with open(path, 'r') as reader, open(dest, 'w') as writer: - data = reader.read() - writer.write(data) - - else: - # if PGP key, convert to binary, ???, convert back - with open(dest, "w") as file: - file.write(key) + + file_type = magic.from_file(path) + + if not file_type.startswith("OpenPGP Public Key"): + raise MirrorError( + f"'{path}' is not a valid GPG key. " + f"Check the key listed in mirrors.yaml in system config.") - # update mirror with new path - mirror["key"] = dest + # copy key to new destination in key store + with open(path, 'r') as reader, open(dest, 'w') as writer: + data = reader.read() + writer.write(data) + + else: + try: + key = base64.b64decode(key) + except ValueError as err: + pass + magic.from_buffer(key) + + # if PGP key, convert to binary, ???, convert back + with open(dest, "wb") as file: + file.write(key) + + # update mirror with new path + mirror["key"] = dest From 39150366f7ab3bd15f24c1e4ecfe039f8dff7d34 Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi <9337627+albestro@users.noreply.github.com> Date: Thu, 26 Feb 2026 08:50:31 +0100 Subject: [PATCH 27/69] Enforce gcc~builtins (#284) With https://github.com/spack/spack-packages/pull/3106 a default value change for `gcc` variant `binutils` has been introduced. This triggers a problem (see [CI error](https://gitlab.com/cscs-ci/ci-testing/webhook-ci/mirrors/551234120955960/1440398897047560/-/jobs/13261281018#L447)) when building `binutils`, which by default is built `libs=static,shared` but we explicitly prefer `zstd libs=static`. https://github.com/eth-cscs/stackinator/blob/c7db13cd3c12d595cf6a7793db88603db9495889/stackinator/templates/compilers.gcc.spack.yaml#L22-L23 The proposed change does not fix the root problem, but it simply prefers building `gcc~builtins` as we've always did, completely workarounding the underlying problem. I'm going to address the root issue in spack, but for what concerns stackinator, at the moment IMHO this is the most reasonable and simple solution. --- stackinator/templates/compilers.gcc.spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackinator/templates/compilers.gcc.spack.yaml b/stackinator/templates/compilers.gcc.spack.yaml index 0c07d06..601d951 100644 --- a/stackinator/templates/compilers.gcc.spack.yaml +++ b/stackinator/templates/compilers.gcc.spack.yaml @@ -12,7 +12,7 @@ spack: reuse: false packages: gcc: - variants: [build_type=Release +bootstrap +profiled +strip] + variants: [build_type=Release +bootstrap +profiled +strip ~binutils] mpc: variants: [libs=static] gmp: From e11685cbc4a3067e0f2915d84911e5bfab1613aa Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 09:43:02 -0600 Subject: [PATCH 28/69] added GPG key verification --- stackinator/builder.py | 2 +- stackinator/mirror.py | 73 ++++++++++++++++++++++-------------------- stackinator/recipe.py | 12 ++----- 3 files changed, 41 insertions(+), 46 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 5bab0ea..f808399 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -233,7 +233,7 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - mirrors=recipe.mirrors, + mirrors=recipe.mirrors.mirrors, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 8f5e440..48d3066 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -5,6 +5,7 @@ import urllib.error from typing import ByteString, Optional, List, Dict import magic +import base64 import yaml @@ -42,7 +43,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: raw = yaml.load(fid, Loader=yaml.Loader) # validate the yaml - schema.CacheValidator.validate(raw) + #schema.CacheValidator.validate(raw) mirrors = [mirror for mirror in raw if mirror["enabled"]] else: @@ -78,7 +79,7 @@ def _check_mirrors(self): for mirror in self.mirrors: url = mirror["url"] - if url.beginswith("file://"): + if url.startswith("file://"): # verify that the root path exists path = pathlib.Path(os.path.expandvars(url)) if not path.is_absolute(): @@ -88,7 +89,7 @@ def _check_mirrors(self): mirror["url"] = path - elif url.beginswith("https://"): + elif url.startswith("https://"): try: request = urllib.request.Request(url, method='HEAD') urllib.request.urlopen(request) @@ -159,49 +160,51 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" + + key_store.mkdir(exist_ok=True) for mirror in self.mirrors: - if not mirror["public_key"]: - continue + if mirror.get("public_key"): + key = mirror["public_key"] - key = mirror["public_key"] + # key will be saved under key_store/mirror_name.gpg - # key will be saved under key_store/mirror_name.gpg - dest = (key_store / f"'{mirror["name"]}'.gpg").resolve() + dest = pathlib.Path(key_store / f"{mirror["name"]}.gpg") - # if path, check if abs path, if not, append sys config path in front and check again - path = pathlib.Path(os.path.expandvars(key)) - if path.exists(): + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) if not path.is_absolute(): #try prepending system config path path = self._system_config_root/path + + if path.exists(): if not path.is_file(): raise MirrorError( - f"The key path '{path}' is not a file. " + f"The key path '{path}' is not a file. \n" f"Check the key listed in mirrors.yaml in system config.") - - file_type = magic.from_file(path) - - if not file_type.startswith("OpenPGP Public Key"): + + with open(path, 'rb') as reader: + binary_key = reader.read() + + # convert base64 key to binary + else: + try: + binary_key = base64.b64decode(key) + except ValueError: + raise MirrorError( + f"Key for mirror {mirror["name"]} is not valid. \n" + f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" + f"Check the key listed in mirrors.yaml in system config.") + + file_type = magic.from_buffer(binary_key, mime=True) + print("magic type:" , file_type) + if file_type != "application/x-gnupg-keyring": raise MirrorError( - f"'{path}' is not a valid GPG key. " + f"Key for mirror {mirror["name"]} is not a valid GPG key. \n" f"Check the key listed in mirrors.yaml in system config.") - + # copy key to new destination in key store - with open(path, 'r') as reader, open(dest, 'w') as writer: - data = reader.read() - writer.write(data) - - else: - try: - key = base64.b64decode(key) - except ValueError as err: - pass - magic.from_buffer(key) - - # if PGP key, convert to binary, ???, convert back - with open(dest, "wb") as file: - file.write(key) - - # update mirror with new path - mirror["key"] = dest + with open(dest, 'wb') as writer: + writer.write(binary_key) + # update mirror with new path + mirror["public_key"] = dest diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 9915bf9..a15569b 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -173,8 +173,8 @@ def __init__(self, args): # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") - self._mirrors = mirror.Mirrors(self.system_config_path, args.cache) - self._cache = [mirror for mirror in self.mirrors if mirror["buildcache"]] + self.mirrors = mirror.Mirrors(self.system_config_path, args.cache) + self.cache = self.mirrors.build_cache_mirror # optional post install hook if self.post_install_hook is not None: @@ -232,14 +232,6 @@ def pre_install_hook(self): if hook_path.exists() and hook_path.is_file(): return hook_path return None - - @property - def mirrors(self): - return self._mirrors - - @property - def cache(self): - return self._cache @property def config(self): From 593159fcde470503a79063fdfd524818de670a3b Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 14:29:56 -0600 Subject: [PATCH 29/69] unit tests for mirrors --- stackinator/mirror.py | 4 +- unittests/__init__.py | 0 .../data/systems/mirror-bad-key/bad_key.gpg | 1 + .../data/systems/mirror-bad-key/mirrors.yaml | 3 ++ .../systems/mirror-bad-keypath/mirrors.yaml | 3 ++ .../data/systems/mirror-bad-url/mirrors.yaml | 2 + unittests/data/systems/mirror-ok/mirrors.yaml | 11 +++++ unittests/test_mirrors.py | 42 +++++++++++++++++++ 8 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 unittests/__init__.py create mode 100644 unittests/data/systems/mirror-bad-key/bad_key.gpg create mode 100644 unittests/data/systems/mirror-bad-key/mirrors.yaml create mode 100644 unittests/data/systems/mirror-bad-keypath/mirrors.yaml create mode 100644 unittests/data/systems/mirror-bad-url/mirrors.yaml create mode 100644 unittests/data/systems/mirror-ok/mirrors.yaml create mode 100644 unittests/test_mirrors.py diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 48d3066..d021aee 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -43,7 +43,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: raw = yaml.load(fid, Loader=yaml.Loader) # validate the yaml - #schema.CacheValidator.validate(raw) + schema.CacheValidator.validate(raw) mirrors = [mirror for mirror in raw if mirror["enabled"]] else: @@ -197,7 +197,7 @@ def _key_setup(self, key_store: pathlib.Path): f"Check the key listed in mirrors.yaml in system config.") file_type = magic.from_buffer(binary_key, mime=True) - print("magic type:" , file_type) + if file_type != "application/x-gnupg-keyring": raise MirrorError( f"Key for mirror {mirror["name"]} is not a valid GPG key. \n" diff --git a/unittests/__init__.py b/unittests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/unittests/data/systems/mirror-bad-key/bad_key.gpg b/unittests/data/systems/mirror-bad-key/bad_key.gpg new file mode 100644 index 0000000..d7980bb --- /dev/null +++ b/unittests/data/systems/mirror-bad-key/bad_key.gpg @@ -0,0 +1 @@ +This is a bad key \ No newline at end of file diff --git a/unittests/data/systems/mirror-bad-key/mirrors.yaml b/unittests/data/systems/mirror-bad-key/mirrors.yaml new file mode 100644 index 0000000..ed27df7 --- /dev/null +++ b/unittests/data/systems/mirror-bad-key/mirrors.yaml @@ -0,0 +1,3 @@ +- name: bad-key + url: https://mirror.spack.io + public_key: /bad_key.gpg \ No newline at end of file diff --git a/unittests/data/systems/mirror-bad-keypath/mirrors.yaml b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml new file mode 100644 index 0000000..e671c45 --- /dev/null +++ b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml @@ -0,0 +1,3 @@ +- name: bad-key-path + url: https://mirror.spack.io + public_key: /path/doesnt/exist \ No newline at end of file diff --git a/unittests/data/systems/mirror-bad-url/mirrors.yaml b/unittests/data/systems/mirror-bad-url/mirrors.yaml new file mode 100644 index 0000000..522c232 --- /dev/null +++ b/unittests/data/systems/mirror-bad-url/mirrors.yaml @@ -0,0 +1,2 @@ +- name: bad-url + url: google.com \ No newline at end of file diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml new file mode 100644 index 0000000..9edf8d4 --- /dev/null +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -0,0 +1,11 @@ +- name: fake-mirror + url: https://google.com +- name: disabled-mirror + url: https://google.com + enabled: false +- name: buildcache-mirror + url: https://cache.spack.io/ + buildcache: true +- name: bootstrap-mirror + url: https://mirror.spack.io + bootstrap: true \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py new file mode 100644 index 0000000..6283c1a --- /dev/null +++ b/unittests/test_mirrors.py @@ -0,0 +1,42 @@ +import pytest +import pathlib +import stackinator.mirror as mirror +import yaml + +@pytest.fixture +def test_path(): + return pathlib.Path(__file__).parent.resolve() + +@pytest.fixture +def systems_path(test_path): + return test_path / "data" / "systems" + +@pytest.fixture +def valid_mirrors(systems_path): + mirrors = {} + mirrors["fake-mirror"] = {'url': 'https://google.com'} + mirrors["buildcache-mirror"] = {'url': 'https://cache.spack.io/', 'buildcache': True} + mirrors["bootstrap-mirror"] = {'url': 'https://mirror.spack.io', 'bootstrap': True} + return mirrors + +def test_mirror_init(systems_path, valid_mirrors): + path = systems_path / "mirror_ok" + mirrors = mirror.Mirrors(path) + print(valid_mirrors) + print(mirrors) + assert mirrors == valid_mirrors + assert mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors if mirror["bootstrap"]] + assert mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors if mirror['buildcache']] + # assert disabled mirror not in mirrors + for mir in mirrors: + assert mir["enabled"] + # test that cmdline_cache gets added to mirrors? + +def test_create_spack_mirrors_yaml(systems_path): + pass + +def test_create_bootstrap_configs(): + pass + +def test_key_setup(): + pass \ No newline at end of file From e1a40fd0239e784b1c7d4276db2558a2f3c2ba9f Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 14:45:07 -0600 Subject: [PATCH 30/69] mirrors.yaml is now a name:{} mapping --- stackinator/mirror.py | 221 +++++++++++++++++++++------------ stackinator/schema/mirror.json | 30 +++-- 2 files changed, 161 insertions(+), 90 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index d021aee..80e1791 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,15 +1,15 @@ + +from typing import Optional, List, Dict import base64 +import io +import magic import os import pathlib -import urllib.request import urllib.error -from typing import ByteString, Optional, List, Dict -import magic -import base64 - +import urllib.request import yaml -from . import schema +from . import schema, root_logger class MirrorError(RuntimeError): """Exception class for errors thrown by mirror configuration problems.""" @@ -20,72 +20,124 @@ class Mirrors: KEY_STORE_DIR = 'key_store' MIRRORS_YAML = 'mirrors.yaml' - def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): + def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None, + mount_point: Optional[pathlib.Path] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" self._system_config_root = system_config_root + self._mount_point = mount_point + + self._logger = root_logger self.mirrors = self._load_mirrors(cmdline_cache) self._check_mirrors() - self.build_cache_mirror = ([mirror for mirror in self.mirrors if mirror.get('buildcache', False)] - + [None]).pop(0) - self.bootstrap_mirrors = [mirror for mirror in self.mirrors if mirror.get('bootstrap', False)] - # Will hold a list of all the keys - self.keys = None + self.build_cache_mirror : Optional[str] = \ + ([name for name, mirror in self.mirrors.items() if mirror.get('cache', False)] + + [None]).pop(0) + self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() + if mirror.get('bootstrap', False)] + + # Will hold a list of all the gpg keys (public and private) + self._keys: Optional[List[pathlib.Path]] = [] - def _load_mirrors(self, cmdline_cache: Optional[str]) -> List[Dict]: + def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" path = self._system_config_root/"mirrors.yaml" if path.exists(): with path.open() as fid: # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid, Loader=yaml.SafeLoader) # validate the yaml - schema.CacheValidator.validate(raw) + schema.MirrorsValidator.validate(raw) - mirrors = [mirror for mirror in raw if mirror["enabled"]] + mirrors = {name: mirror for name, mirror in raw.items() if mirror["enabled"]} else: - mirrors = [] - - buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) - if buildcache_dest_count > 1: - raise MirrorError("Mirror config has more than one mirror specified as the build cache destination " - "in the system config's 'mirrors.yaml'.") - elif buildcache_dest_count == 1 and cmdline_cache: - raise MirrorError("Build cache destination specified on the command line and in the system config's " - "'mirrors.yaml'. It can be one or the other, but not both.") + mirrors = {} # Add or set the cache given on the command line as the buildcache destination if cmdline_cache is not None: - existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] + existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache] # If the mirror name given on the command line isn't in the config, assume it # is the URL to a build cache. if not existing_mirror: - mirrors.append( - { - 'name': 'cmdline_cache', + mirrors['cmdline_cache'] = { 'url': cmdline_cache, - 'buildcache': True, + 'description': "Cache configured via command line.", + 'enabled': True, + 'cache': True, 'bootstrap': False, + 'mount_specific': True, } - ) + + # Load the cache as defined by the deprecated 'cache.yaml' file. + mirrors['legacy_cache_cfg'] = self._load_legacy_cache() + + caches = [mirror for mirror in mirrors.values() if mirror['cache']] + if len(caches) > 1: + raise MirrorError( + "Mirror config has more than one mirror specified as the build cache destination.\n" + "Some of these may have come from a legacy 'cache.yaml' or the '--cache' option.\n" + f"{self._pp_yaml(caches)}") return mirrors + @staticmethod + def _pp_yaml(object): + """Pretty print the given object as yaml.""" + + example_yaml_stream = io.StringIO() + yaml.dump(object, example_yaml_stream, default_flow_style=False) + return example_yaml_stream.getvalue() + + def _load_legacy_cache(self): + """Load the mirror definition from the legacy cache.yaml file.""" + + cache_config_path = self._system_config_root/'cache.yaml' + + if cache_config_path.is_file(): + + with cache_config_path.open('r') as file: + try: + raw = yaml.load(file, Loader=yaml.SafeLoader) + except ValueError as err: + raise MirrorError( + f"Error loading yaml from cache config at '{cache_config_path}'\n{err}") + + try: + schema.CacheValidator.validate(raw) + except ValueError as err: + raise MirrorError( + f"Error validating contents of cache config at '{cache_config_path}'.\n{err}") + + mirror_cfg = { + 'url': f'file://{raw['root']}', + 'description': "Buildcache dest loaded from legacy cache.yaml", + 'buildcache_push': True, + 'mount_specific': True, + 'enabled': True, + 'private_key': raw['key'], + } + + self._logger.warning("Configuring the buildcache from the system cache.yaml file.\n" + "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" + f"The equivalent 'mirrors.yaml' would look like: \n{self._pp_yaml([mirror_cfg])}") + + return mirror_cfg + def _check_mirrors(self): """Validate the mirror config entries.""" - for mirror in self.mirrors: + for name, mirror in self.mirrors.items(): url = mirror["url"] if url.startswith("file://"): # verify that the root path exists path = pathlib.Path(os.path.expandvars(url)) if not path.is_absolute(): - raise MirrorError(f"The mirror path '{path}' is not absolute") + raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not absolute") if not path.is_dir(): - raise MirrorError(f"The mirror path '{path}' is not a directory") + raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not a directory") mirror["url"] = path @@ -98,6 +150,16 @@ def _check_mirrors(self): f"Could not reach the mirror url '{url}'. " f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") + @property + def keys(self): + """Return the list of public and private key file paths.""" + + if self._keys is None: + raise RuntimeError("The mirror.keys method was accessed before setup_configs() was called.") + + return self._keys + + def setup_configs(self, config_root: pathlib.Path): """Setup all mirror configs in the given config_root.""" @@ -110,9 +172,9 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): raw = {"mirrors": {}} - for m in self.mirrors: - name = m["name"] - url = m["url"] + for name, mirror in self.mirrors.items(): + name = mirror["name"] + url = mirror["url"] raw["mirrors"][name] = { "fetch": {"url": url}, @@ -133,9 +195,9 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): 'trusted': {}, } - for mirror in self.bootstrap_mirrors: - name = mirror['name'] + for name in self.bootstrap_mirrors: bs_mirror_path = config_root/f'bootstrap/{name}' + mirror = self.mirrors[name] # Tell spack where to find the metadata for each bootstrap mirror. bootstrap_yaml['sources'].append( { @@ -161,50 +223,53 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" + self._keys = [] key_store.mkdir(exist_ok=True) - for mirror in self.mirrors: - if mirror.get("public_key"): - key = mirror["public_key"] + for name, mirror in self.mirrors.items(): + if mirror.get("public_key") is None: + continue - # key will be saved under key_store/mirror_name.gpg + key = mirror["public_key"] - dest = pathlib.Path(key_store / f"{mirror["name"]}.gpg") + # key will be saved under key_store/mirror_name.gpg - # if path, check if abs path, if not, append sys config path in front and check again - path = pathlib.Path(os.path.expandvars(key)) - if not path.is_absolute(): - #try prepending system config path - path = self._system_config_root/path - - if path.exists(): - if not path.is_file(): - raise MirrorError( - f"The key path '{path}' is not a file. \n" - f"Check the key listed in mirrors.yaml in system config.") - - with open(path, 'rb') as reader: - binary_key = reader.read() - - # convert base64 key to binary - else: - try: - binary_key = base64.b64decode(key) - except ValueError: - raise MirrorError( - f"Key for mirror {mirror["name"]} is not valid. \n" - f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" - f"Check the key listed in mirrors.yaml in system config.") - - file_type = magic.from_buffer(binary_key, mime=True) + dest = pathlib.Path(key_store / f"{name}.gpg") + + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if not path.is_absolute(): + #try prepending system config path + path = self._system_config_root/path - if file_type != "application/x-gnupg-keyring": + if path.exists(): + if not path.is_file(): raise MirrorError( - f"Key for mirror {mirror["name"]} is not a valid GPG key. \n" + f"The key path '{path}' is not a file. \n" f"Check the key listed in mirrors.yaml in system config.") - - # copy key to new destination in key store - with open(dest, 'wb') as writer: - writer.write(binary_key) - # update mirror with new path - mirror["public_key"] = dest + + with open(path, 'rb') as reader: + binary_key = reader.read() + + # convert base64 key to binary + else: + try: + binary_key = base64.b64decode(key) + except ValueError: + raise MirrorError( + f"Key for mirror '{name}' is not valid. \n" + f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" + f"Check the key listed in mirrors.yaml in system config.") + + file_type = magic.from_buffer(binary_key, mime=True) + print("magic type:" , file_type) + if file_type != "application/x-gnupg-keyring": + raise MirrorError( + f"Key for mirror {name} is not a valid GPG key. \n" + f"Check the key listed in mirrors.yaml in system config.") + + # copy key to new destination in key store + with open(dest, 'wb') as writer: + writer.write(binary_key) + + self._keys.append(dest) diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index a8be6ab..5efb725 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -1,17 +1,18 @@ { - "type" : "array", - "items": { + "type" : "object", + "additionalProperties": { "type": "object", - "required": ["name", "url"], + "required": ["url"], + "additionalProperties": false, "properties": { - "name": { - "type": "string", - "description": "The name of this mirror. Should be follow standard variable naming syntax." - }, "url": { "type": "string", "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI)." }, + "description": { + "type": "string", + "description": "What this mirror is for." + } "enabled": { "type": "boolean", "default": true, @@ -22,19 +23,24 @@ "default": false, "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror." }, - "buildcache": { + "cache": { "type": "boolean", "default": false, "description": "Use this mirror as the buildcache push destination. Can only be enabled on a single mirror." }, "public_key": { "type": "string", - "description": "Public PGP key for validating binary cache packages." + "description": "Public PGP key for validating binary cache packages. A path or base64 encoded key." }, - "description": { + "private_key": { "type": "string", - "description": "What this mirror is for." + "description": "Private PGP key for signing binary cache packages. (Path only)", + }, + "mount_specific": { + "type": "boolean", + "default": false, + "description": "Use a mount specific buildcache path (specified path + recipe mount point).", } } } -} \ No newline at end of file +} From fb3289d314ecbe2c00aacf0b75f7ea33ba6240a6 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 15:09:44 -0600 Subject: [PATCH 31/69] Now add mount specific paths to certain mirrors. --- stackinator/builder.py | 4 ++-- stackinator/mirror.py | 5 ++++- stackinator/templates/Makefile | 11 +++-------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index f808399..b6192bb 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -227,13 +227,13 @@ def generate(self, recipe): with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( - cache = recipe.cache, modules=recipe.with_modules, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - mirrors=recipe.mirrors.mirrors, + gpg_keys=recipe.mirrors.keys, + cache=recipe.mirrors.buildcache, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 80e1791..cad746c 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -173,9 +173,12 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): raw = {"mirrors": {}} for name, mirror in self.mirrors.items(): - name = mirror["name"] url = mirror["url"] + # Make the mirror path specific to the mount point + if mirror['mount_specific'] and self._mount_point is not None: + url = url.rstrip('/') + '/' + self._mount_point.as_posix().lstrip('/') + raw["mirrors"][name] = { "fetch": {"url": url}, "push": {"url": url}, diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 9484e3c..f0b90eb 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -33,19 +33,14 @@ pre-install: spack-setup mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - {% if cache %} + @echo "Pulling and trusting keys from configured buildcaches." $(SANDBOX) $(SPACK) buildcache keys --install --trust - {% endif %} - {% if mirrors %} @echo "Adding mirror gpg keys." - {% for mirror in mirrors | reverse %} - {% if mirror.public_key %} - $(SANDBOX) $(SPACK) gpg trust {{ mirror.public_key }} - {% endif %} + {% for key_path in gpg_keys %} + $(SANDBOX) $(SPACK) gpg trust {{ key_path }} {% endfor %} @echo "Current mirror list:" $(SANDBOX) $(SPACK) mirror list - {% endif %} touch mirror-setup compilers: mirror-setup From d47e2f3c6aad37b090d2b91b8b18d4a114b0ce39 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 15:22:24 -0600 Subject: [PATCH 32/69] updated test mirror format --- stackinator/schema/mirror.json | 6 +++--- unittests/data/systems/mirror-ok/mirrors.yaml | 14 +++++--------- unittests/test_mirrors.py | 18 ++++++++---------- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 5efb725..8977083 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -12,7 +12,7 @@ "description": { "type": "string", "description": "What this mirror is for." - } + }, "enabled": { "type": "boolean", "default": true, @@ -34,12 +34,12 @@ }, "private_key": { "type": "string", - "description": "Private PGP key for signing binary cache packages. (Path only)", + "description": "Private PGP key for signing binary cache packages. (Path only)" }, "mount_specific": { "type": "boolean", "default": false, - "description": "Use a mount specific buildcache path (specified path + recipe mount point).", + "description": "Use a mount specific buildcache path (specified path + recipe mount point)." } } } diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index 9edf8d4..e84fc3c 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -1,11 +1,7 @@ -- name: fake-mirror - url: https://google.com -- name: disabled-mirror - url: https://google.com +- url: https://google.com +- url: https://google.com enabled: false -- name: buildcache-mirror - url: https://cache.spack.io/ - buildcache: true -- name: bootstrap-mirror - url: https://mirror.spack.io +- url: https://cache.spack.io/ + cache: true +- url: https://mirror.spack.io bootstrap: true \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 6283c1a..bd96dce 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -14,21 +14,19 @@ def systems_path(test_path): @pytest.fixture def valid_mirrors(systems_path): mirrors = {} - mirrors["fake-mirror"] = {'url': 'https://google.com'} - mirrors["buildcache-mirror"] = {'url': 'https://cache.spack.io/', 'buildcache': True} - mirrors["bootstrap-mirror"] = {'url': 'https://mirror.spack.io', 'bootstrap': True} + mirrors["fake-mirror"] = {'url': 'https://google.com', 'enabled': True, 'bootstrap': False, 'cache': False, 'mount_specific': False} + mirrors["buildcache-mirror"] = {'url': 'https://cache.spack.io/', 'enabled': True, 'bootstrap': False, 'cache': True, 'mount_specific': False} + mirrors["bootstrap-mirror"] = {'url': 'https://mirror.spack.io', 'enabled': True, 'bootstrap': True, 'cache': False, 'mount_specific': False} return mirrors def test_mirror_init(systems_path, valid_mirrors): path = systems_path / "mirror_ok" - mirrors = mirror.Mirrors(path) - print(valid_mirrors) - print(mirrors) - assert mirrors == valid_mirrors - assert mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors if mirror["bootstrap"]] - assert mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors if mirror['buildcache']] + mirrors_obj = mirror.Mirrors(path) + assert mirrors_obj.mirrors == valid_mirrors + assert mirrors_obj.mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors.values() if mirror.get('bootstrap')] + assert mirrors_obj.mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors.values() if mirror.get('buildcache')] # assert disabled mirror not in mirrors - for mir in mirrors: + for mir in mirrors_obj.mirrors: assert mir["enabled"] # test that cmdline_cache gets added to mirrors? From c8fb6f959cebc1ec12d6a82b5b6f240a1a5cbe59 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 15:23:31 -0600 Subject: [PATCH 33/69] Fixed build cache enabling via cmdline. --- stackinator/mirror.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index cad746c..8a91162 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -47,29 +47,30 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: if path.exists(): with path.open() as fid: # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.SafeLoader) + mirrors = yaml.load(fid, Loader=yaml.SafeLoader) # validate the yaml - schema.MirrorsValidator.validate(raw) - - mirrors = {name: mirror for name, mirror in raw.items() if mirror["enabled"]} + schema.MirrorsValidator.validate(mirrors) else: mirrors = {} # Add or set the cache given on the command line as the buildcache destination if cmdline_cache is not None: - existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache] # If the mirror name given on the command line isn't in the config, assume it # is the URL to a build cache. - if not existing_mirror: + if cmdline_cache in mirrors: mirrors['cmdline_cache'] = { 'url': cmdline_cache, 'description': "Cache configured via command line.", - 'enabled': True, 'cache': True, 'bootstrap': False, 'mount_specific': True, } + else: + # Enable the specified mirror and set it as the build cache dest + mirror = mirrors[cmdline_cache] + mirror['enabled'] = True + mirror['cache'] = True # Load the cache as defined by the deprecated 'cache.yaml' file. mirrors['legacy_cache_cfg'] = self._load_legacy_cache() @@ -81,7 +82,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: "Some of these may have come from a legacy 'cache.yaml' or the '--cache' option.\n" f"{self._pp_yaml(caches)}") - return mirrors + return {name: mirror for name, mirror in raw.items() if mirror["enabled"]} @staticmethod def _pp_yaml(object): @@ -116,7 +117,6 @@ def _load_legacy_cache(self): 'description': "Buildcache dest loaded from legacy cache.yaml", 'buildcache_push': True, 'mount_specific': True, - 'enabled': True, 'private_key': raw['key'], } From 951aeb4294ca78954695ca13b554a87d08097ab8 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 15:26:57 -0600 Subject: [PATCH 34/69] Error handling --- stackinator/mirror.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 8a91162..5643f2f 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -45,12 +45,12 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" path = self._system_config_root/"mirrors.yaml" if path.exists(): - with path.open() as fid: - # load the raw yaml input - mirrors = yaml.load(fid, Loader=yaml.SafeLoader) - - # validate the yaml - schema.MirrorsValidator.validate(mirrors) + try: + with path.open() as fid: + # load the raw yaml input + mirrors = yaml.load(fid, Loader=yaml.SafeLoader) + except (OSError, PermissionError) as err: + raise MirrorError("Could not open/read mirrors.yaml file.\n{err}") else: mirrors = {} @@ -62,6 +62,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: mirrors['cmdline_cache'] = { 'url': cmdline_cache, 'description': "Cache configured via command line.", + 'enabled': True, 'cache': True, 'bootstrap': False, 'mount_specific': True, @@ -75,6 +76,15 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: # Load the cache as defined by the deprecated 'cache.yaml' file. mirrors['legacy_cache_cfg'] = self._load_legacy_cache() + + try: + # validate the yaml, including anything we added + schema.MirrorsValidator.validate(mirrors) + except ValueError as err: + raise MirrorError( + "Mirror config does not comply with schema.\n{err}" + ) + caches = [mirror for mirror in mirrors.values() if mirror['cache']] if len(caches) > 1: raise MirrorError( @@ -115,7 +125,8 @@ def _load_legacy_cache(self): mirror_cfg = { 'url': f'file://{raw['root']}', 'description': "Buildcache dest loaded from legacy cache.yaml", - 'buildcache_push': True, + 'cache': True, + 'enabled': True, 'mount_specific': True, 'private_key': raw['key'], } From 2ea49e9facf291ca5e71153a23424c8c92938d6d Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 15:46:38 -0600 Subject: [PATCH 35/69] Adding a unittest. --- stackinator/mirror.py | 21 ++++++++++----------- unittests/test_mirrors.py | 10 +++++++++- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 5643f2f..f1dbb17 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -54,11 +54,16 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: else: mirrors = {} + try: + schema.MirrorsValidator.validate(mirrors) + except ValueError as err: + raise MirrorError("Mirror config does not comply with schema.\n{err}") + # Add or set the cache given on the command line as the buildcache destination if cmdline_cache is not None: # If the mirror name given on the command line isn't in the config, assume it # is the URL to a build cache. - if cmdline_cache in mirrors: + if cmdline_cache not in mirrors: mirrors['cmdline_cache'] = { 'url': cmdline_cache, 'description': "Cache configured via command line.", @@ -74,16 +79,10 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: mirror['cache'] = True # Load the cache as defined by the deprecated 'cache.yaml' file. - mirrors['legacy_cache_cfg'] = self._load_legacy_cache() - + legacy_cache = self._load_legacy_cache() + if legacy_cache is not None: + mirrors['legacy_cache_cfg'] = legacy_cache - try: - # validate the yaml, including anything we added - schema.MirrorsValidator.validate(mirrors) - except ValueError as err: - raise MirrorError( - "Mirror config does not comply with schema.\n{err}" - ) caches = [mirror for mirror in mirrors.values() if mirror['cache']] if len(caches) > 1: @@ -92,7 +91,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: "Some of these may have come from a legacy 'cache.yaml' or the '--cache' option.\n" f"{self._pp_yaml(caches)}") - return {name: mirror for name, mirror in raw.items() if mirror["enabled"]} + return {name: mirror for name, mirror in mirrors.items() if mirror["enabled"]} @staticmethod def _pp_yaml(object): diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index bd96dce..6579bb1 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -30,6 +30,14 @@ def test_mirror_init(systems_path, valid_mirrors): assert mir["enabled"] # test that cmdline_cache gets added to mirrors? +def test_command_line_cache(systems_path): + """Check that adding a cache from the command line works.""" + + mirrors = mirror.Mirrors(systems_path/'mirror-ok', cmdline_cache=systems_path.as_posix()) + + assert len(mirrors.mirrors) == 4 + + def test_create_spack_mirrors_yaml(systems_path): pass @@ -37,4 +45,4 @@ def test_create_bootstrap_configs(): pass def test_key_setup(): - pass \ No newline at end of file + pass From c668b5b375a8c40a2f4f9dad7774fa24bc551910 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 15:45:58 -0600 Subject: [PATCH 36/69] updated yaml formatting --- unittests/data/systems/mirror-ok/mirrors.yaml | 12 ++++++++---- unittests/test_mirrors.py | 4 +++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index e84fc3c..f0030a4 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -1,7 +1,11 @@ -- url: https://google.com -- url: https://google.com +fake-mirror: + url: https://google.com +disabled-mirror: + url: https://google.com enabled: false -- url: https://cache.spack.io/ +buildcache-mirror: + url: https://cache.spack.io/ cache: true -- url: https://mirror.spack.io +bootstrap-mirror: + url: https://mirror.spack.io bootstrap: true \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 6579bb1..c1d4660 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -20,8 +20,10 @@ def valid_mirrors(systems_path): return mirrors def test_mirror_init(systems_path, valid_mirrors): - path = systems_path / "mirror_ok" + path = systems_path / "mirror-ok" + print(path) mirrors_obj = mirror.Mirrors(path) + print(mirrors_obj.mirrors.items()) assert mirrors_obj.mirrors == valid_mirrors assert mirrors_obj.mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors.values() if mirror.get('bootstrap')] assert mirrors_obj.mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors.values() if mirror.get('buildcache')] From 45b1331396978c871d4b47744b713029ad939644 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 15:56:15 -0600 Subject: [PATCH 37/69] Fixed error message. --- stackinator/mirror.py | 2 +- unittests/data/systems/mirror-basic/mirrors.yaml | 8 ++++++++ unittests/test_mirrors.py | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 unittests/data/systems/mirror-basic/mirrors.yaml diff --git a/stackinator/mirror.py b/stackinator/mirror.py index f1dbb17..d9e1e80 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -84,7 +84,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: mirrors['legacy_cache_cfg'] = legacy_cache - caches = [mirror for mirror in mirrors.values() if mirror['cache']] + caches = {name: mirror for name, mirror in mirrors.items() if mirror['cache']] if len(caches) > 1: raise MirrorError( "Mirror config has more than one mirror specified as the build cache destination.\n" diff --git a/unittests/data/systems/mirror-basic/mirrors.yaml b/unittests/data/systems/mirror-basic/mirrors.yaml new file mode 100644 index 0000000..ee93f2c --- /dev/null +++ b/unittests/data/systems/mirror-basic/mirrors.yaml @@ -0,0 +1,8 @@ +fake-mirror: + url: https://google.com +disabled-mirror: + url: https://google.com + enabled: false +bootstrap-mirror: + url: https://mirror.spack.io + bootstrap: true diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index c1d4660..acd360f 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -35,9 +35,9 @@ def test_mirror_init(systems_path, valid_mirrors): def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" - mirrors = mirror.Mirrors(systems_path/'mirror-ok', cmdline_cache=systems_path.as_posix()) + mirrors = mirror.Mirrors(systems_path/'mirror-basic', cmdline_cache=systems_path.as_posix()) - assert len(mirrors.mirrors) == 4 + assert len(mirrors.mirrors) == 3 def test_create_spack_mirrors_yaml(systems_path): From 533a6061d7b26f29f5a7f106d5debf57e4cfeaaa Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 15:57:31 -0600 Subject: [PATCH 38/69] debugging --- unittests/test_mirrors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index acd360f..b374aca 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -21,9 +21,9 @@ def valid_mirrors(systems_path): def test_mirror_init(systems_path, valid_mirrors): path = systems_path / "mirror-ok" - print(path) + #print(path) mirrors_obj = mirror.Mirrors(path) - print(mirrors_obj.mirrors.items()) + #print(mirrors_obj.mirrors.items()) assert mirrors_obj.mirrors == valid_mirrors assert mirrors_obj.mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors.values() if mirror.get('bootstrap')] assert mirrors_obj.mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors.values() if mirror.get('buildcache')] From 34d0c6574cad38a1265e91077a406b86b5fc9ca9 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 16:02:18 -0600 Subject: [PATCH 39/69] Minor fix. --- stackinator/mirror.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index d9e1e80..8816847 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -50,14 +50,14 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: # load the raw yaml input mirrors = yaml.load(fid, Loader=yaml.SafeLoader) except (OSError, PermissionError) as err: - raise MirrorError("Could not open/read mirrors.yaml file.\n{err}") + raise MirrorError(f"Could not open/read mirrors.yaml file.\n{err}") else: mirrors = {} try: schema.MirrorsValidator.validate(mirrors) except ValueError as err: - raise MirrorError("Mirror config does not comply with schema.\n{err}") + raise MirrorError(f"Mirror config does not comply with schema.\n{err}") # Add or set the cache given on the command line as the buildcache destination if cmdline_cache is not None: @@ -84,7 +84,7 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: mirrors['legacy_cache_cfg'] = legacy_cache - caches = {name: mirror for name, mirror in mirrors.items() if mirror['cache']] + caches = {name: mirror for name, mirror in mirrors.items() if mirror['cache']} if len(caches) > 1: raise MirrorError( "Mirror config has more than one mirror specified as the build cache destination.\n" From f989631e0675ea8b4aeacc8d50b5945f5d45ab96 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 13 Mar 2026 16:06:49 -0600 Subject: [PATCH 40/69] Fixing urls. --- unittests/data/systems/mirror-basic/mirrors.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unittests/data/systems/mirror-basic/mirrors.yaml b/unittests/data/systems/mirror-basic/mirrors.yaml index ee93f2c..2667cbc 100644 --- a/unittests/data/systems/mirror-basic/mirrors.yaml +++ b/unittests/data/systems/mirror-basic/mirrors.yaml @@ -1,8 +1,8 @@ fake-mirror: - url: https://google.com + url: https://github.com disabled-mirror: - url: https://google.com + url: /tmp/ enabled: false bootstrap-mirror: - url: https://mirror.spack.io + url: https://github.com bootstrap: true From 7822fb764859b93a6ace31245a2e92f00a270d1d Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 16:07:37 -0600 Subject: [PATCH 41/69] fixed url for testing --- unittests/data/systems/mirror-ok/mirrors.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index f0030a4..06f2e32 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -1,10 +1,10 @@ fake-mirror: - url: https://google.com + url: https://github.com disabled-mirror: - url: https://google.com + url: https://github.com enabled: false buildcache-mirror: - url: https://cache.spack.io/ + url: https://mirror.spack.io cache: true bootstrap-mirror: url: https://mirror.spack.io From 8c814aabbd374e1a5680bb76791d6abd595917a9 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 16:34:52 -0600 Subject: [PATCH 42/69] validated mirror tests --- unittests/test_mirrors.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index b374aca..fdc4e7b 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -14,23 +14,21 @@ def systems_path(test_path): @pytest.fixture def valid_mirrors(systems_path): mirrors = {} - mirrors["fake-mirror"] = {'url': 'https://google.com', 'enabled': True, 'bootstrap': False, 'cache': False, 'mount_specific': False} - mirrors["buildcache-mirror"] = {'url': 'https://cache.spack.io/', 'enabled': True, 'bootstrap': False, 'cache': True, 'mount_specific': False} + mirrors["fake-mirror"] = {'url': 'https://github.com', 'enabled': True, 'bootstrap': False, 'cache': False, 'mount_specific': False} + mirrors["buildcache-mirror"] = {'url': 'https://mirror.spack.io', 'enabled': True, 'bootstrap': False, 'cache': True, 'mount_specific': False} mirrors["bootstrap-mirror"] = {'url': 'https://mirror.spack.io', 'enabled': True, 'bootstrap': True, 'cache': False, 'mount_specific': False} return mirrors def test_mirror_init(systems_path, valid_mirrors): path = systems_path / "mirror-ok" - #print(path) mirrors_obj = mirror.Mirrors(path) - #print(mirrors_obj.mirrors.items()) + assert mirrors_obj.mirrors == valid_mirrors - assert mirrors_obj.mirrors.bootstrap_mirrors == [mirror for mirror in valid_mirrors.values() if mirror.get('bootstrap')] - assert mirrors_obj.mirrors.build_cache_mirror == [mirror for mirror in valid_mirrors.values() if mirror.get('buildcache')] - # assert disabled mirror not in mirrors + assert mirrors_obj.bootstrap_mirrors == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('bootstrap')] + assert mirrors_obj.build_cache_mirror == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('cache')].pop(0) + for mir in mirrors_obj.mirrors: - assert mir["enabled"] - # test that cmdline_cache gets added to mirrors? + assert mirrors_obj.mirrors[mir].get('enabled') def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" From 5ab3279581809771782904b8fbc6ab4884bb91d0 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 13 Mar 2026 16:37:02 -0600 Subject: [PATCH 43/69] added test description --- unittests/test_mirrors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index fdc4e7b..ecc7908 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -20,6 +20,7 @@ def valid_mirrors(systems_path): return mirrors def test_mirror_init(systems_path, valid_mirrors): + """Check that Mirror objects are initialized correctly.""" path = systems_path / "mirror-ok" mirrors_obj = mirror.Mirrors(path) From 5999ba4a4b6fb95f99e90941c47424fa1b10e858 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 10:20:00 -0600 Subject: [PATCH 44/69] Adding unit tests. --- stackinator/mirror.py | 105 ++++++++---------- .../data/systems/mirror-basic/cache.yaml | 2 + unittests/data/test-gpg-priv.asc | Bin 0 -> 4924 bytes unittests/data/test-gpg-pub.asc | Bin 0 -> 2250 bytes unittests/test_mirrors.py | 17 ++- 5 files changed, 63 insertions(+), 61 deletions(-) create mode 100644 unittests/data/systems/mirror-basic/cache.yaml create mode 100644 unittests/data/test-gpg-priv.asc create mode 100644 unittests/data/test-gpg-pub.asc diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 8816847..a519186 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -19,8 +19,9 @@ class Mirrors: KEY_STORE_DIR = 'key_store' MIRRORS_YAML = 'mirrors.yaml' + CMDLINE_CACHE = 'cmdline_cache' - def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None, + def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[pathlib.Path] = None, mount_point: Optional[pathlib.Path] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" @@ -32,16 +33,22 @@ def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[str self.mirrors = self._load_mirrors(cmdline_cache) self._check_mirrors() - self.build_cache_mirror : Optional[str] = \ - ([name for name, mirror in self.mirrors.items() if mirror.get('cache', False)] - + [None]).pop(0) + # Always use the cache given on the command line + if self.CMDLINE_CACHE in self.mirrors: + self.build_cache_mirror = self.CMDLINE_CACHE + else: + # Otherwise, grab the configured cache (or None) + self.build_cache_mirror : Optional[str] = \ + ([name for name, mirror in self.mirrors.items() if mirror.get('cache', False)] + + [None]).pop(0) + self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() if mirror.get('bootstrap', False)] # Will hold a list of all the gpg keys (public and private) self._keys: Optional[List[pathlib.Path]] = [] - def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: + def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" path = self._system_config_root/"mirrors.yaml" if path.exists(): @@ -59,38 +66,16 @@ def _load_mirrors(self, cmdline_cache: Optional[str]) -> Dict[str, Dict]: except ValueError as err: raise MirrorError(f"Mirror config does not comply with schema.\n{err}") - # Add or set the cache given on the command line as the buildcache destination - if cmdline_cache is not None: - # If the mirror name given on the command line isn't in the config, assume it - # is the URL to a build cache. - if cmdline_cache not in mirrors: - mirrors['cmdline_cache'] = { - 'url': cmdline_cache, - 'description': "Cache configured via command line.", - 'enabled': True, - 'cache': True, - 'bootstrap': False, - 'mount_specific': True, - } - else: - # Enable the specified mirror and set it as the build cache dest - mirror = mirrors[cmdline_cache] - mirror['enabled'] = True - mirror['cache'] = True - - # Load the cache as defined by the deprecated 'cache.yaml' file. - legacy_cache = self._load_legacy_cache() - if legacy_cache is not None: - mirrors['legacy_cache_cfg'] = legacy_cache - - caches = {name: mirror for name, mirror in mirrors.items() if mirror['cache']} if len(caches) > 1: raise MirrorError( "Mirror config has more than one mirror specified as the build cache destination.\n" - "Some of these may have come from a legacy 'cache.yaml' or the '--cache' option.\n" f"{self._pp_yaml(caches)}") + # Load the cache as defined by the deprecated 'cache.yaml' file. + if cmdline_cache is not None: + mirrors[self.CMDLINE_CACHE] = self._load_cmdline_cache(cmdline_cache) + return {name: mirror for name, mirror in mirrors.items() if mirror["enabled"]} @staticmethod @@ -101,40 +86,42 @@ def _pp_yaml(object): yaml.dump(object, example_yaml_stream, default_flow_style=False) return example_yaml_stream.getvalue() - def _load_legacy_cache(self): - """Load the mirror definition from the legacy cache.yaml file.""" - - cache_config_path = self._system_config_root/'cache.yaml' - - if cache_config_path.is_file(): - - with cache_config_path.open('r') as file: - try: - raw = yaml.load(file, Loader=yaml.SafeLoader) - except ValueError as err: - raise MirrorError( - f"Error loading yaml from cache config at '{cache_config_path}'\n{err}") + def _load_cmdline_cache(self, cache_config_path: pathlib.Path) -> Dict: + """Load the mirror definition from the legacy 'cache.yaml' file.""" + if not cache_config_path.is_file(): + raise MirrorError( + f"Binary cache configuration path given on the command line '{cache_config_path}' " + f"does not exist.") + + with cache_config_path.open('r') as file: try: - schema.CacheValidator.validate(raw) + raw = yaml.load(file, Loader=yaml.SafeLoader) except ValueError as err: raise MirrorError( - f"Error validating contents of cache config at '{cache_config_path}'.\n{err}") - - mirror_cfg = { - 'url': f'file://{raw['root']}', - 'description': "Buildcache dest loaded from legacy cache.yaml", - 'cache': True, - 'enabled': True, - 'mount_specific': True, - 'private_key': raw['key'], - } + f"Error loading yaml from cache config at '{cache_config_path}'\n{err}") + + try: + schema.CacheValidator.validate(raw) + except ValueError as err: + raise MirrorError( + f"Error validating contents of cache config at '{cache_config_path}'.\n{err}") + + mirror_cfg = { + 'url': raw['root'], + 'description': "Buildcache dest loaded from legacy cache.yaml", + 'cache': True, + 'enabled': True, + 'bootstrap': False, + 'mount_specific': True, + 'private_key': raw['key'], + } - self._logger.warning("Configuring the buildcache from the system cache.yaml file.\n" - "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" - f"The equivalent 'mirrors.yaml' would look like: \n{self._pp_yaml([mirror_cfg])}") + self._logger.warning("Configuring the buildcache from the system cache.yaml file.\n" + "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" + f"The equivalent 'mirrors.yaml' would look like: \n{self._pp_yaml([mirror_cfg])}") - return mirror_cfg + return mirror_cfg def _check_mirrors(self): """Validate the mirror config entries.""" diff --git a/unittests/data/systems/mirror-basic/cache.yaml b/unittests/data/systems/mirror-basic/cache.yaml new file mode 100644 index 0000000..ad7de37 --- /dev/null +++ b/unittests/data/systems/mirror-basic/cache.yaml @@ -0,0 +1,2 @@ +root: /tmp/foo +key: ../../test-gpg-priv.asc diff --git a/unittests/data/test-gpg-priv.asc b/unittests/data/test-gpg-priv.asc new file mode 100644 index 0000000000000000000000000000000000000000..eaa2dc19eb7ecaff70e1c1cf53e66825a6996425 GIT binary patch literal 4924 zcmajhWgr{?pull=>g37OoSW`;YI?f6CZ=O@;>4L|Om}R0dYHDc>0#<{dYb8&Ht)R; zFTTC+|L^}_ChiL?mv-8F00~C=s0g;qrC~&pac&Qi3xB%d4BI$n%GzhUu9pzu3NJ9U z(wLyd`_8QFj#lg=2y|QSH@51OU|Bq>B#$00$+%r zsLiq6J>_j9-lyM&p)&t0kE?4>7U}w1E*wsLi39&OcFVam*(SCy^ybA$*Q`h{w^0TC z!rsU^&HfAC5pS0EWko`NRF7*w^Bq*O8_SplGc$}6j|P zQl|HmHq+0TRjw!4wqqv(IM60Z=_RVF3n>okAA5WFOLuO#nB*Op6gAjt0&uk=Joi_wxL?hDEhOUHN!?AnR3oy*7vLa|wdQ6Scs(lh@=Vd&RRqtU6n~O0yVEaOzXjbe^!#e?_RFZ<cQ=Rt3jfU6%{Z1%t{^H`G^<^osKj}Nb96uAA&pU%gI~2y z$1WI_C|NtaqF|&dcjZKW2G8b))P>vO3p_Za%JvfxCC}$H_&%jEH(IyeNGOO>X*H}* zFp<4kQ*-{|>n2RLh1s$JdXb4MuS;qvI3q9(!T%d|yKcZ!O^dxT5vgS{3UFrf&-LE# z5>C4X<-Yfy2r+5GCDVIDYi3D@#@LEf7x_g&(c@=@$(y3CM<(=Cz;KMT`xtR3r$XrM zB!c+h=;p^d2v^Q}RbxM;2C(A3U8@8|TV9H|M$@ZT`zK_t*iIJgl`b3ZEAGrn2FKZnR|88d<@%6_gK+`$* z$l+TjqN&}|HF9bl8F|83W$XKEfm8_z7uFT_+FpNkQqCI zcqfSw!@pOO@IeCXXgJ=_gTH1MpQSz2i{la(A5W9b)nPQ3r=&azj7d0E95wK}33;44 zC(@buEB6ZvfrKPqj_jJce!foh=)`Mi;i^Dnel)Sgk{kNKs!bNOswyT%V>4X)vb89$ zmfgCnl$e16H3lPzmkkG`E zFLZ_uzJ5>*hajl*Yfmo+x!0Zn&JI2fe6Kt`p2q|Au*d=U7*Y^stRj#OJcf|#{Lh}n zAzXp*^`sP495(R39t@3^YtInHX?_x5qO8muaA6_ogq z-1H3ORrJ^W(LI_HVWspRNYR>3!x?+A&e0bvpr^c%RL%efeT}f`%`d3~?L@DQ*Vt#) zSY%ferQeu)XIfKNs@}}#kO_$Jl#nExRiE#sZ+3^uuaYE8S0i<%THWv;#lkUB%(ubW zaYXB#WN&SGLiy|-N*)~K2JP7%A%}=fvBdcf6(k@F1k8V@UusKsW6`~6?CQQZw8FEC z_Xex5oZv(A(vr(|pG&a>y8z$qXH4d8)q<5_6My@(Jvv!ZTcv&M z1`b+XRKm@y*-`t`!D(b%8xhlZyC6y%8QJwS^7=HL!FoZKOQ6M?juy&R_`h0`!L zVo8plb?(Tz_3sGYpDEB`w~I)&N#k@6N6Wri&5WroFM2!BJ4ix_9tseQAbO3o8p|Uwrm2xJLj;&&F6m^X{P{yh)KXs3wwiykQsL z87StZ{N8ihIgj9*sa_z-RF6IkkEwQID@#PHKLUjhKjGuzuNV@1xLAU>v1Oe|@*+Eu z(z}OYEhRTQOMGeT)B+~`WC4LDoPNB}XuHP?MMgDcw&R_>ZRM%ml8i5U+qr_)C8Yi| z2f$1SO0wv|n>c3-IU=r@QF||geHAtJeZicK%GRnNwYd}5&CP$`?%e94Qn)i|4`o%$dlxmL(VCSd8GZ8jgvyw6jh zAQ!O(zwDwBmi0gtrksyd+6U`w|32=VC9@rAlA0sUWWl2ckEko1J;JNV#Qod6XW_~W zdcmt*MAXXz=0Zn|QJ$iO^5X^;uMZWDqeK(;&0rDnWb=|9IDJc8ach-#G;Cp!)G``Q zTSCmJj5O&FJ`*a9_f3G#-{ZO!9(|^p?g7vpGe;pN#rk~KycaR6W%Me)E>=I`I)%A4 zr7dL{yQO|pAE<2o=Ttc2wkL)?;kVuq?U;ly8aq6GGO@OqX0^BWKXMcE8he|`Qc(I7 zwG<4EjFC%pk?D1!x8h< z&S5>fS5RHWf+8WAVc217uv6VK#oRpf{RL@#%uP6bj2_0X=;%@V^m+0f<0-hU`z$rr z5;*vGgachu-Ko>?l^`){;J((-{ZC^tdos6tF-nSx86g0ZnAjA@69ceHxPxRb*6YAVfxDd`4l-`fb`}B4o5m>r_@}=t65x(XKmoS5u#@^jF+e z)>n1m83 z(MMTi-7WDZW+bvQx)HkayO9IuNJZ3ccuUTDMSf=syJrtpgyphx=f1!D!0gK(^s{`Z zo4pcWkgoqqK=G2{zQiZO1eMmJDsX{1d8qxWSCY~hX=HvJh4<+Pn`o*=?Rrj$u~7N? z8s*k|2Rnc&cHx(Y$j^2xy(F-}4V5g-Zyub^-53c8lq`PjF8xdJSIcjxcX4%bLuA)z zrWdr>Yk#pRNjzdG@jewWvyQBijb(d9jw4&QP7JJMzh`SH7@itP&8rzXkl6eNZw;#(}9hlQVk%X2VaqPW1Q^+^L+1xKYoefzi`*pKD0>USw z8Kt13`w8$fwAQJZV$%qKZeax^U*oH_eALymt={X>*Y9=fB{f3UF?_lEB|z3^XhmiS z9gS%p=*c0t+#h@!vAf|w>5>0>*>hergIi?=ebRB%Jv)9UPm$-Op%G zbCv#2yv})}v3=b(zT$ORmK<)gxNfVV-Qo}lPw1rZP_}`;d2FX*`qO4}Yq0WBjj8a* znKD?YsK~mEU*>kaZrIrga&{8BK~iHwzeJ0QI{r-flXg6aDZxTSxP~ci198cgyWE+@spgp zcIL70HcdIu60_IfVC&3?yO+(xZ5B5kD27G-pE!g5KR6@&zXCI10zX*_*1P1GlMxc` z%)?R=S07x@X1-m-&$Ls}mDr_yHYTn^x0Au)C^OITfQf-9;BZL%?G@?lct-tV@IbLTNfTbCO+!d8^X)gQ+>Q z#Lu~@3184=2Ujjy5t#O2!)*1EH6$jV@0#)!=HB*j({^71ls&())~&Qy7~rT18YWCp zs#=itG`?Ojr(te6_%{=a%B)o>B+5VaSlS{dK5l9l_}Vna(t0uKw1 z_(B$aekP4YwSd{abSfNJ;;;n~PMSY3^QHYs91id)+1}=S<~kWXprhLrK}E7~V#fIg z58^xVG2h@x6p~;-3$z|+0K;$vqq&`(8tCX84 zopfONbdlh~doqjgsF%flDZQ`ZFG@Y)tr^&cbh_5SOC5~r901&0XgdQG&nen4h#)LQ mZ>7vztr91~Oq1#)>Ed{1^SU?%yx#0qXiGx|!qiao5&sW-Z9mQc literal 0 HcmV?d00001 diff --git a/unittests/data/test-gpg-pub.asc b/unittests/data/test-gpg-pub.asc new file mode 100644 index 0000000000000000000000000000000000000000..aa72b0281f39bd4ad4d0281742b9133f19e84c6a GIT binary patch literal 2250 zcmV;*2sQVa0u2OdxElKb5CD_QGzC5MRDz9GnZJK33evCf1y+X9UvlNGzzlk7IGj{n zOV1h!!68L#*h9k$cx&9mR`YBM1Y`iIP7^GpgRmCGb0J4Jm4p-S4@~@{L_B=$sif8l zp%P+j!xq+yV=FoV1Nqpa0oxriYr{=<+#z>ny;aTf4|~0-Zz8Hnu?{m>wKqL3j??fl z<%FL7OsTwod^ACtFi#WMhm%0?XoZjydMC(cgMUkUK(&wsqSzcG4>V<$UvpIpSbqPT z;xFrpnx-(?c+JH<-0XElO&weapA%^0u&yeLenM?y9Ws+8e~R^JVXI4tVvkyg+HPVT`pMz?z6WtKB@@w-%_Sm-P?kH?ur*C4V5os?AFvn`K9rv7{j}m6TSD?Zrjb`lrJ3IY3SN6hh<=3)XA< z%oP87l$5JNg$gvLxG)?62YWmXry(I_mpt=?NVlFtG^u4D)gGwa6j-WRU~#vpTvhsr zyD{m@c&p#C4yRNi^aw=l2hSZj>H7p%!Dc^AM$FNuiB-AK#eeQ#sH8N1QTF3pKc9)m zC_potxh89e74ZNO0RREC8&qX;bRbJ*c_2J)Z*XNmZf|#JWpZUMV{dIfi2_js69EbU zI~E}Xo(NAkh72n5>gZeSI6@3=&^s511p;Zf8v6np0|g5S2nPZJA_4{#3JC}c0t6NU z0|5da2Lcy>0162ZI6@3=&^s517TOQ~Fzm@df0V8}8rvUt>$u+J@YC^+3ooI-cT{RQ z102Z9UjpyK4q|Ra*hD@qnCvS~ZPk$V?axJ8 ztf8|Lll35Ym}p6t`(%juOC?8ZrWFc+VNktuf_QcHf)C@q7aJls`BTP zwrdLgHh=@vA^Cfkhz;So5`keXeJ*1BqWon*#$+b_7VLWCHjLM~ zM}Gm92m+gnQlnuK_*=Z%R%&eM$K5RF33UXWr3;_4+tF;uVhwF^f?F@o!gD={h1(Qr zCd_uB*sgyb46eGl%MlUNy911@)MbHs#6KzvkU)p8@eM&fr@;=X%?Gli#6B8)TVk6dPBa-Cs09bef$DloNQ8yX-cxdIIYX}B8u0T2Mw!yyGkB#(*JXt|d8cDZ}$)0|HyVuFug zk|}I}*gs?5m_Jx?uSXeS=gmkgAA*Fv(G#sd)Q{M=qwcXal67A_20}uX`(MLCYJ9yEq}RjIS))HzE7-iTnb=B{}%K(-d*l>yuSVV5o>*K+@OF)9vQFF)E<1YgCe13+>+lAm9f zcEc-}+gav35UcG>k1_51ZG`qu=nVFQjOqEBBY;I6QG4FI4H(?TS~Be(gl{#SLCsTJ zZtOztgf)!mSt5do64yFy2E*Gbh~KCB876*%7(Gj8gZeSI6@3= z&^s511p;Zf8v6np3;+rV5I8~%ZqPdyh*uvE|6DPyX_#6P3=HPiB9M&U{A%&jzVnIG zxD*?rVv#jON;)90f_FqOb9eHW^ne8DNQSMgF?Lni39?HvFhVm?qh_&)hXdSLsu_YO zYDlSPq1fYWHQBHh56tFl8K(Rt+s1Y>l27>7I=h{k-aKT+k(kvEJ}VduMjDrUr`lZ$ zSi=P(VNs7uInX}mSS_8I`M)a~yz~J@Z@ne0-L+d&21qefiqREFTM@spZrxfLA+_uN z(}&cRtVf*V%#EHaqgX6R{A9prF3BhAfm~5?49i{eA;r#jm(6#4<@v_ zLhD=xCV33dTKpj{lIo1ecXFcT*1vGtGISxuva>l520ckLGgv#2gy^}{Y9rh-V$@n z`0UYwH<4mp;FwQ8)cZ;rih#Do$OI1i@+f9@O5QWvKsJ8Lx?Jb0zjo-x^u5?g?T zUZHf5z-~B3qiu;_9VW+5yWbDnxgJU=0SB2NxE&w~XN6RG3x1GM-K23zrv`#qOc(Es Y^X}8*_o0vv;j?R Date: Mon, 16 Mar 2026 10:44:23 -0600 Subject: [PATCH 45/69] Unit test tweaking. --- stackinator/mirror.py | 6 +++++- unittests/data/systems/mirror-ok/cache.yaml | 2 ++ unittests/data/systems/mirror-ok/mirrors.yaml | 3 ++- unittests/test_mirrors.py | 13 ++++--------- 4 files changed, 13 insertions(+), 11 deletions(-) create mode 100644 unittests/data/systems/mirror-ok/cache.yaml diff --git a/stackinator/mirror.py b/stackinator/mirror.py index a519186..a423833 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -66,11 +66,15 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict except ValueError as err: raise MirrorError(f"Mirror config does not comply with schema.\n{err}") - caches = {name: mirror for name, mirror in mirrors.items() if mirror['cache']} + caches = [name for name, mirror in mirrors.items() if mirror['cache']] if len(caches) > 1: raise MirrorError( "Mirror config has more than one mirror specified as the build cache destination.\n" f"{self._pp_yaml(caches)}") + elif caches: + cache = mirrors[caches[0]] + if not cache.get('private_key'): + raise MirrorError(f"Mirror build cache config '{caches[0]}' missing a required 'private_key' path.") # Load the cache as defined by the deprecated 'cache.yaml' file. if cmdline_cache is not None: diff --git a/unittests/data/systems/mirror-ok/cache.yaml b/unittests/data/systems/mirror-ok/cache.yaml new file mode 100644 index 0000000..ad7de37 --- /dev/null +++ b/unittests/data/systems/mirror-ok/cache.yaml @@ -0,0 +1,2 @@ +root: /tmp/foo +key: ../../test-gpg-priv.asc diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index 06f2e32..21e1e3b 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -5,7 +5,8 @@ disabled-mirror: enabled: false buildcache-mirror: url: https://mirror.spack.io + private_key: '../test-gpg-priv.asc' cache: true bootstrap-mirror: url: https://mirror.spack.io - bootstrap: true \ No newline at end of file + bootstrap: true diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 981fe74..c5bede3 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -34,10 +34,11 @@ def test_mirror_init(systems_path, valid_mirrors): def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" - mirrors = mirror.Mirrors(systems_path/'mirror-basic', - cmdline_cache=systems_path/'mirror-basic/cache.yaml') + mirrors = mirror.Mirrors(systems_path/'mirror-ok', + cmdline_cache=systems_path/'mirror-ok/cache.yaml') - assert len(mirrors.mirrors) == 3 + assert len(mirrors.mirrors) == 4 + # This should always be the build cache even though one is already defined. assert mirrors.build_cache_mirror == 'cmdline_cache' cache_mirror = mirrors.mirrors['cmdline_cache'] assert cache_mirror['url'] == '/tmp/foo' @@ -46,12 +47,6 @@ def test_command_line_cache(systems_path): assert not cache_mirror['bootstrap'] assert cache_mirror['mount_specific'] -def test_multi_buildcache(systems_path): - """Make sure we throw appropriate errors when there's more than one build cache defined.""" - - - - def test_create_spack_mirrors_yaml(systems_path): pass From e19b868cb710f0d838ea5f62746b177438b367ca Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 11:04:25 -0600 Subject: [PATCH 46/69] Added unittests for key setup --- stackinator/mirror.py | 2 +- unittests/data/systems/mirror-ok/mirrors.yaml | 43 +++++++++++++++++++ unittests/test_mirrors.py | 17 +++++++- 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index a423833..7296389 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -267,7 +267,7 @@ def _key_setup(self, key_store: pathlib.Path): file_type = magic.from_buffer(binary_key, mime=True) print("magic type:" , file_type) - if file_type != "application/x-gnupg-keyring": + if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): raise MirrorError( f"Key for mirror {name} is not a valid GPG key. \n" f"Check the key listed in mirrors.yaml in system config.") diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index 21e1e3b..96fc72f 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -1,11 +1,54 @@ fake-mirror: url: https://github.com + public_key: ../../test-gpg-pub.asc disabled-mirror: url: https://github.com enabled: false buildcache-mirror: url: https://mirror.spack.io private_key: '../test-gpg-priv.asc' + public_key: | + mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML + eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB + 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF + kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK + IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb + WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh + jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA + TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3 + qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH + KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe + CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK + CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+ + LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i + qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr + HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1 + WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i + /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x + glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE + PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW + lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw + kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe + mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm + HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH + Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0 + UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF + zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx + Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y + 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H + sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr + 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q + OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR + NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw + Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z + +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx + 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7 + xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6 + lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd + eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8 + eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG + gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3 + cache: true bootstrap-mirror: url: https://mirror.spack.io diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index c5bede3..3b068c9 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -53,5 +53,18 @@ def test_create_spack_mirrors_yaml(systems_path): def test_create_bootstrap_configs(): pass -def test_key_setup(): - pass +def test_key_setup(systems_path, tmp_path): + """Check that public keys are set up properly.""" + + mirrors = mirror.Mirrors(systems_path/'mirror-ok') + + mirrors._key_setup(tmp_path) + + key_files = list(tmp_path.iterdir()) + assert {key_file.name for key_file in key_files} == {'buildcache-mirror.gpg', 'fake-mirror.gpg'} + # The two files should be identical in content + key_file_data = [] + for key_file in key_files: + with key_file.open('rb') as file: + key_file_data.append(file.read()) + assert key_file_data[0] == key_file_data[1] From 86047caf3d41e09b5db4902b69f2320e1d7c675e Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 12:16:45 -0600 Subject: [PATCH 47/69] Added test for bad keys. --- stackinator/mirror.py | 11 +++-------- unittests/data/systems/mirror-bad-key/mirrors.yaml | 4 ++-- .../data/systems/mirror-bad-keypath/mirrors.yaml | 4 ++-- unittests/data/systems/mirror-basic/cache.yaml | 2 -- unittests/data/systems/mirror-basic/mirrors.yaml | 8 -------- unittests/test_mirrors.py | 14 ++++++++++++++ 6 files changed, 21 insertions(+), 22 deletions(-) delete mode 100644 unittests/data/systems/mirror-basic/cache.yaml delete mode 100644 unittests/data/systems/mirror-basic/mirrors.yaml diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 7296389..317deea 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -246,12 +246,7 @@ def _key_setup(self, key_store: pathlib.Path): #try prepending system config path path = self._system_config_root/path - if path.exists(): - if not path.is_file(): - raise MirrorError( - f"The key path '{path}' is not a file. \n" - f"Check the key listed in mirrors.yaml in system config.") - + if path.is_file(): with open(path, 'rb') as reader: binary_key = reader.read() @@ -261,15 +256,15 @@ def _key_setup(self, key_store: pathlib.Path): binary_key = base64.b64decode(key) except ValueError: raise MirrorError( - f"Key for mirror '{name}' is not valid. \n" + f"Key for mirror '{name}' is not valid: '{path}'. \n" f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" f"Check the key listed in mirrors.yaml in system config.") file_type = magic.from_buffer(binary_key, mime=True) - print("magic type:" , file_type) if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): raise MirrorError( f"Key for mirror {name} is not a valid GPG key. \n" + f"The file (or base64) was readable, but the data itself was not a PGP key.\n" f"Check the key listed in mirrors.yaml in system config.") # copy key to new destination in key store diff --git a/unittests/data/systems/mirror-bad-key/mirrors.yaml b/unittests/data/systems/mirror-bad-key/mirrors.yaml index ed27df7..d5154dd 100644 --- a/unittests/data/systems/mirror-bad-key/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-key/mirrors.yaml @@ -1,3 +1,3 @@ -- name: bad-key +bad-key: url: https://mirror.spack.io - public_key: /bad_key.gpg \ No newline at end of file + public_key: bad_key.gpg diff --git a/unittests/data/systems/mirror-bad-keypath/mirrors.yaml b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml index e671c45..3433e04 100644 --- a/unittests/data/systems/mirror-bad-keypath/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml @@ -1,3 +1,3 @@ -- name: bad-key-path +bad-key-path: url: https://mirror.spack.io - public_key: /path/doesnt/exist \ No newline at end of file + public_key: /path/doesnt/exist diff --git a/unittests/data/systems/mirror-basic/cache.yaml b/unittests/data/systems/mirror-basic/cache.yaml deleted file mode 100644 index ad7de37..0000000 --- a/unittests/data/systems/mirror-basic/cache.yaml +++ /dev/null @@ -1,2 +0,0 @@ -root: /tmp/foo -key: ../../test-gpg-priv.asc diff --git a/unittests/data/systems/mirror-basic/mirrors.yaml b/unittests/data/systems/mirror-basic/mirrors.yaml deleted file mode 100644 index 2667cbc..0000000 --- a/unittests/data/systems/mirror-basic/mirrors.yaml +++ /dev/null @@ -1,8 +0,0 @@ -fake-mirror: - url: https://github.com -disabled-mirror: - url: /tmp/ - enabled: false -bootstrap-mirror: - url: https://github.com - bootstrap: true diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 3b068c9..5b4a0a8 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -68,3 +68,17 @@ def test_key_setup(systems_path, tmp_path): with key_file.open('rb') as file: key_file_data.append(file.read()) assert key_file_data[0] == key_file_data[1] + +@pytest.mark.parametrize("system_name", [ + 'mirror-bad-key', + 'mirror-bad-keypath', +]) +def test_key_setup_bad_key(tmp_path, systems_path, system_name): + """asdfasdf""" + + mirrors = mirror.Mirrors(systems_path/system_name) + with pytest.raises(mirror.MirrorError): + mirrors._key_setup(tmp_path) + + + From 3d840d05c9665c6d02d9a424a35a3940c605aeaf Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Mon, 16 Mar 2026 11:17:52 -0600 Subject: [PATCH 48/69] added more mirror tests --- stackinator/mirror.py | 4 +-- unittests/test_mirrors.py | 61 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 317deea..502943a 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -206,14 +206,14 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): bootstrap_yaml['sources'].append( { 'name': name, - 'metadata': bs_mirror_path, + 'metadata': str(bs_mirror_path), } ) # And trust each one bootstrap_yaml['trusted'][name] = True # Create the metadata dir and metadata.yaml - bs_mirror_path.mkdir(parents=True) + bs_mirror_path.mkdir(parents=True, exist_ok=True) bs_mirror_yaml = { 'type': 'install', 'info': mirror['url'], diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 5b4a0a8..8e18922 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -48,10 +48,65 @@ def test_command_line_cache(systems_path): assert cache_mirror['mount_specific'] def test_create_spack_mirrors_yaml(systems_path): - pass + """Check that the mirrors.yaml passed to spack is correct""" + + valid_spack_yaml = { + "mirrors": { + "fake-mirror": { + "fetch": {"url": "https://github.com"}, + "push": {"url": "https://github.com"}, + }, + "buildcache-mirror": { + "fetch": {"url": "https://mirror.spack.io"}, + "push": {"url": "https://mirror.spack.io"}, + }, + "bootstrap-mirror": { + "fetch": {"url": "https://mirror.spack.io"}, + "push": {"url": "https://mirror.spack.io"}, + } + } + } + + dest = systems_path / "mirror-ok" / "test_output.yaml" + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok") + mirrors_obj._create_spack_mirrors_yaml(dest) + + with dest.open() as f: + data = yaml.safe_load(f) + + assert data == valid_spack_yaml + +def test_create_bootstrap_configs(systems_path): + """Check that spack bootstrap configs are generated correctly""" + + valid_yaml = { + "sources": [ + { + "name": "bootstrap-mirror", + "metadata": str(systems_path / "mirror-ok" / "bootstrap" / "bootstrap-mirror"), + } + ], + "trusted": { + "bootstrap-mirror": True + }, + } + valid_metadata = { + "type": "install", + "info": "https://mirror.spack.io", + } + + path = systems_path / "mirror-ok" + bs_mirror_path = path / "bootstrap/bootstrap-mirror" + mirrors_obj = mirror.Mirrors(path) + mirrors_obj._create_bootstrap_configs(path) + + with (path/'bootstrap.yaml').open() as f: + bs_data = yaml.safe_load(f) + assert bs_data == valid_yaml -def test_create_bootstrap_configs(): - pass + with (bs_mirror_path/'metadata.yaml').open() as f: + metadata = yaml.safe_load(f) + assert metadata == valid_metadata def test_key_setup(systems_path, tmp_path): """Check that public keys are set up properly.""" From 0a25459a7801411ad79eed74fe6e9e6120d180c7 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Mon, 16 Mar 2026 12:39:16 -0600 Subject: [PATCH 49/69] added test for bad urls --- unittests/data/systems/mirror-bad-url/mirrors.yaml | 4 ++-- unittests/test_mirrors.py | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/unittests/data/systems/mirror-bad-url/mirrors.yaml b/unittests/data/systems/mirror-bad-url/mirrors.yaml index 522c232..8ffce33 100644 --- a/unittests/data/systems/mirror-bad-url/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-url/mirrors.yaml @@ -1,2 +1,2 @@ -- name: bad-url - url: google.com \ No newline at end of file +bad-url: + url: https://www.testsite.io/services \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 8e18922..191e708 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -31,6 +31,14 @@ def test_mirror_init(systems_path, valid_mirrors): for mir in mirrors_obj.mirrors: assert mirrors_obj.mirrors[mir].get('enabled') +def test_mirror_init_bad_url(systems_path): + """Check that MirrorError is raised for a bad url.""" + + path = systems_path / "mirror-bad-url" + + with pytest.raises(mirror.MirrorError): + mirrors_obj = mirror.Mirrors(path) + def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" From 35591bc5cffbad659d9b59217953db74e3f11ab5 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Mon, 16 Mar 2026 12:54:48 -0600 Subject: [PATCH 50/69] added requirements.txt --- requirements.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d0de0ac --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +attrs==25.4.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +packaging==26.0 +pluggy==1.6.0 +pygments==2.19.2 +pytest==9.0.2 +python-magic==0.4.27 +pyyaml==6.0.3 +referencing==0.37.0 +rpds-py==0.30.0 From 431cc9d6d4f5f440d4c140919af963b87952af2e Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 13:02:02 -0600 Subject: [PATCH 51/69] Fixed unittest. --- unittests/data/systems/mirror-ok/mirrors.yaml | 82 +++++++++---------- unittests/test_mirrors.py | 60 +++++++++----- 2 files changed, 81 insertions(+), 61 deletions(-) diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index 96fc72f..0a1b843 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -7,47 +7,47 @@ disabled-mirror: buildcache-mirror: url: https://mirror.spack.io private_key: '../test-gpg-priv.asc' - public_key: | - mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML - eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB - 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF - kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK - IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb - WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh - jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA - TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3 - qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH - KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe - CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK - CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+ - LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i - qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr - HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1 - WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i - /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x - glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE - PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW - lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw - kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe - mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm - HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH - Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0 - UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF - zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx - Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y - 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H - sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr - 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q - OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR - NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw - Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z - +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx - 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7 - xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6 - lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd - eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8 - eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG - gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3 + public_key: "\ + mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML\ + eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB\ + 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF\ + kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK\ + IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb\ + WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh\ + jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA\ + TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3\ + qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH\ + KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe\ + CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK\ + CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+\ + LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i\ + qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr\ + HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1\ + WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i\ + /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x\ + glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE\ + PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW\ + lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw\ + kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe\ + mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm\ + HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH\ + Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0\ + UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF\ + zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx\ + Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y\ + 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H\ + sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr\ + 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q\ + OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR\ + NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw\ + Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z\ + +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx\ + 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7\ + xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6\ + lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd\ + eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8\ + eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG\ + gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3" cache: true bootstrap-mirror: diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 191e708..dbc2a97 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -1,5 +1,6 @@ -import pytest +import base64 import pathlib +import pytest import stackinator.mirror as mirror import yaml @@ -11,19 +12,38 @@ def test_path(): def systems_path(test_path): return test_path / "data" / "systems" -@pytest.fixture -def valid_mirrors(systems_path): - mirrors = {} - mirrors["fake-mirror"] = {'url': 'https://github.com', 'enabled': True, 'bootstrap': False, 'cache': False, 'mount_specific': False} - mirrors["buildcache-mirror"] = {'url': 'https://mirror.spack.io', 'enabled': True, 'bootstrap': False, 'cache': True, 'mount_specific': False} - mirrors["bootstrap-mirror"] = {'url': 'https://mirror.spack.io', 'enabled': True, 'bootstrap': True, 'cache': False, 'mount_specific': False} - return mirrors - -def test_mirror_init(systems_path, valid_mirrors): +def test_mirror_init(systems_path): """Check that Mirror objects are initialized correctly.""" path = systems_path / "mirror-ok" mirrors_obj = mirror.Mirrors(path) + valid_mirrors = { + "fake-mirror": { + 'url': 'https://github.com', + 'enabled': True, + 'bootstrap': False, + 'cache': False, + 'public_key': '../../test-gpg-pub.asc', + 'mount_specific': False}, + "buildcache-mirror": { + 'url': 'https://mirror.spack.io', + 'enabled': True, + 'bootstrap': False, + 'cache': True, + 'private_key': '../test-gpg-priv.asc', + 'mount_specific': False}, + "bootstrap-mirror": { + 'url': 'https://mirror.spack.io', + 'enabled': True, + 'bootstrap': True, + 'cache': False, + 'mount_specific': False} + } + + with (systems_path/'../test-gpg-pub.asc').open('rb') as pub_key_file: + key = base64.b64encode(pub_key_file.read()).decode() + valid_mirrors['buildcache-mirror']['public_key'] = key + assert mirrors_obj.mirrors == valid_mirrors assert mirrors_obj.bootstrap_mirrors == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('bootstrap')] assert mirrors_obj.build_cache_mirror == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('cache')].pop(0) @@ -55,7 +75,7 @@ def test_command_line_cache(systems_path): assert not cache_mirror['bootstrap'] assert cache_mirror['mount_specific'] -def test_create_spack_mirrors_yaml(systems_path): +def test_create_spack_mirrors_yaml(tmp_path, systems_path): """Check that the mirrors.yaml passed to spack is correct""" valid_spack_yaml = { @@ -75,7 +95,7 @@ def test_create_spack_mirrors_yaml(systems_path): } } - dest = systems_path / "mirror-ok" / "test_output.yaml" + dest = tmp_path / "test_output.yaml" mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok") mirrors_obj._create_spack_mirrors_yaml(dest) @@ -84,14 +104,14 @@ def test_create_spack_mirrors_yaml(systems_path): assert data == valid_spack_yaml -def test_create_bootstrap_configs(systems_path): +def test_create_bootstrap_configs(tmp_path, systems_path): """Check that spack bootstrap configs are generated correctly""" valid_yaml = { "sources": [ { "name": "bootstrap-mirror", - "metadata": str(systems_path / "mirror-ok" / "bootstrap" / "bootstrap-mirror"), + "metadata": str(tmp_path / "bootstrap/bootstrap-mirror"), } ], "trusted": { @@ -103,16 +123,16 @@ def test_create_bootstrap_configs(systems_path): "info": "https://mirror.spack.io", } - path = systems_path / "mirror-ok" - bs_mirror_path = path / "bootstrap/bootstrap-mirror" - mirrors_obj = mirror.Mirrors(path) - mirrors_obj._create_bootstrap_configs(path) + mirrors_obj = mirror.Mirrors(systems_path/'mirror-ok') + mirrors_obj._create_bootstrap_configs(tmp_path) - with (path/'bootstrap.yaml').open() as f: + with (tmp_path/'bootstrap.yaml').open() as f: bs_data = yaml.safe_load(f) + print(bs_data) + print(valid_yaml) assert bs_data == valid_yaml - with (bs_mirror_path/'metadata.yaml').open() as f: + with (tmp_path/'bootstrap/bootstrap-mirror/metadata.yaml').open() as f: metadata = yaml.safe_load(f) assert metadata == valid_metadata From 0c59ee948344baa94f62c57ff226b319998f62e3 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 13:06:51 -0600 Subject: [PATCH 52/69] Added one more unittest. --- unittests/test_mirrors.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index dbc2a97..72f2190 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -57,7 +57,17 @@ def test_mirror_init_bad_url(systems_path): path = systems_path / "mirror-bad-url" with pytest.raises(mirror.MirrorError): - mirrors_obj = mirror.Mirrors(path) + mirror.Mirrors(path) + +def test_setup_configs(tmp_path, systems_path): + """Test general config setup.""" + + mir = mirror.Mirrors(systems_path/'mirror-ok') + mir.setup_configs(tmp_path) + + assert (tmp_path/'mirrors.yaml').is_file() + assert (tmp_path/'bootstrap').is_dir() + assert (tmp_path/'key_store').is_dir() def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" From d9cdfc4da5451a25d35450e37134e6ce7f18778b Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 13:13:05 -0600 Subject: [PATCH 53/69] Linting --- stackinator/builder.py | 2 +- stackinator/main.py | 34 +++++++--- stackinator/mirror.py | 137 ++++++++++++++++++++------------------ stackinator/recipe.py | 7 +- unittests/test_mirrors.py | 124 +++++++++++++++++++--------------- 5 files changed, 168 insertions(+), 136 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index b6192bb..ca0f875 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -11,7 +11,7 @@ import jinja2 import yaml -from . import VERSION, cache, root_logger, spack_util, mirror +from . import VERSION, root_logger, spack_util, mirror def install(src, dst, *, ignore=None, symlinks=False): diff --git a/stackinator/main.py b/stackinator/main.py index 6f30ddf..0d0b9bd 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -81,19 +81,31 @@ def log_header(args): def make_argparser(): parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from a recipe.")) parser.add_argument("--version", action="version", version=f"stackinator version {VERSION}") - parser.add_argument("-b", "--build", required=True, type=str, - help="Where to set up the stackinator build directory. " - "('/tmp' is not allowed, use '/var/tmp'") + parser.add_argument( + "-b", + "--build", + required=True, + type=str, + help="Where to set up the stackinator build directory. ('/tmp' is not allowed, use '/var/tmp'", + ) parser.add_argument("--no-bwrap", action="store_true", required=False) - parser.add_argument("-r", "--recipe", required=True, type=str, - help="Name of (and/or path to) the Stackinator recipe.") - parser.add_argument("-s", "--system", required=True, type=str, - help="Name of (and/or path to) the Stackinator system configuration.") + parser.add_argument( + "-r", "--recipe", required=True, type=str, help="Name of (and/or path to) the Stackinator recipe." + ) + parser.add_argument( + "-s", "--system", required=True, type=str, help="Name of (and/or path to) the Stackinator system configuration." + ) parser.add_argument("-d", "--debug", action="store_true") - parser.add_argument("-m", "--mount", required=False, type=str, - help="The mount point where the environment will be located.") - parser.add_argument("-c", "--cache", required=False, type=str, - help="Buildcache location or name (from system config's mirrors.yaml).") + parser.add_argument( + "-m", "--mount", required=False, type=str, help="The mount point where the environment will be located." + ) + parser.add_argument( + "-c", + "--cache", + required=False, + type=str, + help="Buildcache location or name (from system config's mirrors.yaml).", + ) parser.add_argument("--develop", action="store_true", required=False) return parser diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 502943a..e6ba268 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,4 +1,3 @@ - from typing import Optional, List, Dict import base64 import io @@ -11,18 +10,24 @@ from . import schema, root_logger + class MirrorError(RuntimeError): """Exception class for errors thrown by mirror configuration problems.""" + class Mirrors: """Manage the definition of mirrors in a recipe.""" - KEY_STORE_DIR = 'key_store' - MIRRORS_YAML = 'mirrors.yaml' - CMDLINE_CACHE = 'cmdline_cache' + KEY_STORE_DIR = "key_store" + MIRRORS_YAML = "mirrors.yaml" + CMDLINE_CACHE = "cmdline_cache" - def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[pathlib.Path] = None, - mount_point: Optional[pathlib.Path] = None): + def __init__( + self, + system_config_root: pathlib.Path, + cmdline_cache: Optional[pathlib.Path] = None, + mount_point: Optional[pathlib.Path] = None, + ): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" self._system_config_root = system_config_root @@ -32,25 +37,24 @@ def __init__(self, system_config_root: pathlib.Path, cmdline_cache: Optional[pat self.mirrors = self._load_mirrors(cmdline_cache) self._check_mirrors() - + # Always use the cache given on the command line if self.CMDLINE_CACHE in self.mirrors: self.build_cache_mirror = self.CMDLINE_CACHE else: # Otherwise, grab the configured cache (or None) - self.build_cache_mirror : Optional[str] = \ - ([name for name, mirror in self.mirrors.items() if mirror.get('cache', False)] - + [None]).pop(0) + self.build_cache_mirror: Optional[str] = ( + [name for name, mirror in self.mirrors.items() if mirror.get("cache", False)] + [None] + ).pop(0) - self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() - if mirror.get('bootstrap', False)] + self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() if mirror.get("bootstrap", False)] # Will hold a list of all the gpg keys (public and private) - self._keys: Optional[List[pathlib.Path]] = [] + self._keys: Optional[List[pathlib.Path]] = [] def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" - path = self._system_config_root/"mirrors.yaml" + path = self._system_config_root / "mirrors.yaml" if path.exists(): try: with path.open() as fid: @@ -66,14 +70,15 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict except ValueError as err: raise MirrorError(f"Mirror config does not comply with schema.\n{err}") - caches = [name for name, mirror in mirrors.items() if mirror['cache']] + caches = [name for name, mirror in mirrors.items() if mirror["cache"]] if len(caches) > 1: raise MirrorError( "Mirror config has more than one mirror specified as the build cache destination.\n" - f"{self._pp_yaml(caches)}") + f"{self._pp_yaml(caches)}" + ) elif caches: cache = mirrors[caches[0]] - if not cache.get('private_key'): + if not cache.get("private_key"): raise MirrorError(f"Mirror build cache config '{caches[0]}' missing a required 'private_key' path.") # Load the cache as defined by the deprecated 'cache.yaml' file. @@ -82,7 +87,7 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict return {name: mirror for name, mirror in mirrors.items() if mirror["enabled"]} - @staticmethod + @staticmethod def _pp_yaml(object): """Pretty print the given object as yaml.""" @@ -95,35 +100,35 @@ def _load_cmdline_cache(self, cache_config_path: pathlib.Path) -> Dict: if not cache_config_path.is_file(): raise MirrorError( - f"Binary cache configuration path given on the command line '{cache_config_path}' " - f"does not exist.") - - with cache_config_path.open('r') as file: + f"Binary cache configuration path given on the command line '{cache_config_path}' does not exist." + ) + + with cache_config_path.open("r") as file: try: raw = yaml.load(file, Loader=yaml.SafeLoader) except ValueError as err: - raise MirrorError( - f"Error loading yaml from cache config at '{cache_config_path}'\n{err}") + raise MirrorError(f"Error loading yaml from cache config at '{cache_config_path}'\n{err}") try: schema.CacheValidator.validate(raw) except ValueError as err: - raise MirrorError( - f"Error validating contents of cache config at '{cache_config_path}'.\n{err}") + raise MirrorError(f"Error validating contents of cache config at '{cache_config_path}'.\n{err}") mirror_cfg = { - 'url': raw['root'], - 'description': "Buildcache dest loaded from legacy cache.yaml", - 'cache': True, - 'enabled': True, - 'bootstrap': False, - 'mount_specific': True, - 'private_key': raw['key'], + "url": raw["root"], + "description": "Buildcache dest loaded from legacy cache.yaml", + "cache": True, + "enabled": True, + "bootstrap": False, + "mount_specific": True, + "private_key": raw["key"], } - self._logger.warning("Configuring the buildcache from the system cache.yaml file.\n" + self._logger.warning( + "Configuring the buildcache from the system cache.yaml file.\n" "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" - f"The equivalent 'mirrors.yaml' would look like: \n{self._pp_yaml([mirror_cfg])}") + f"The equivalent 'mirrors.yaml' would look like: \n{self._pp_yaml([mirror_cfg])}" + ) return mirror_cfg @@ -144,12 +149,13 @@ def _check_mirrors(self): elif url.startswith("https://"): try: - request = urllib.request.Request(url, method='HEAD') + request = urllib.request.Request(url, method="HEAD") urllib.request.urlopen(request) except urllib.error.URLError as e: raise MirrorError( - f"Could not reach the mirror url '{url}'. " - f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") + f"Could not reach the mirror url '{url}'. " + f"Check the url listed in mirrors.yaml in system config. \n{e.reason}" + ) @property def keys(self): @@ -160,12 +166,11 @@ def keys(self): return self._keys - def setup_configs(self, config_root: pathlib.Path): """Setup all mirror configs in the given config_root.""" - self._key_setup(config_root/self.KEY_STORE_DIR) - self._create_spack_mirrors_yaml(config_root/self.MIRRORS_YAML) + self._key_setup(config_root / self.KEY_STORE_DIR) + self._create_spack_mirrors_yaml(config_root / self.MIRRORS_YAML) self._create_bootstrap_configs(config_root) def _create_spack_mirrors_yaml(self, dest: pathlib.Path): @@ -177,8 +182,8 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): url = mirror["url"] # Make the mirror path specific to the mount point - if mirror['mount_specific'] and self._mount_point is not None: - url = url.rstrip('/') + '/' + self._mount_point.as_posix().lstrip('/') + if mirror["mount_specific"] and self._mount_point is not None: + url = url.rstrip("/") + "/" + self._mount_point.as_posix().lstrip("/") raw["mirrors"][name] = { "fetch": {"url": url}, @@ -193,40 +198,40 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): if not self.bootstrap_mirrors: return - + bootstrap_yaml = { - 'sources': [], - 'trusted': {}, + "sources": [], + "trusted": {}, } for name in self.bootstrap_mirrors: - bs_mirror_path = config_root/f'bootstrap/{name}' + bs_mirror_path = config_root / f"bootstrap/{name}" mirror = self.mirrors[name] # Tell spack where to find the metadata for each bootstrap mirror. - bootstrap_yaml['sources'].append( + bootstrap_yaml["sources"].append( { - 'name': name, - 'metadata': str(bs_mirror_path), + "name": name, + "metadata": str(bs_mirror_path), } ) # And trust each one - bootstrap_yaml['trusted'][name] = True + bootstrap_yaml["trusted"][name] = True # Create the metadata dir and metadata.yaml bs_mirror_path.mkdir(parents=True, exist_ok=True) bs_mirror_yaml = { - 'type': 'install', - 'info': mirror['url'], + "type": "install", + "info": mirror["url"], } - with (bs_mirror_path/'metadata.yaml').open('w') as file: + with (bs_mirror_path / "metadata.yaml").open("w") as file: yaml.dump(bs_mirror_yaml, file, default_flow_style=False) - - with (config_root/'bootstrap.yaml').open('w') as file: + + with (config_root / "bootstrap.yaml").open("w") as file: yaml.dump(bootstrap_yaml, file, default_flow_style=False) def _key_setup(self, key_store: pathlib.Path): """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" - + self._keys = [] key_store.mkdir(exist_ok=True) @@ -243,13 +248,13 @@ def _key_setup(self, key_store: pathlib.Path): # if path, check if abs path, if not, append sys config path in front and check again path = pathlib.Path(os.path.expandvars(key)) if not path.is_absolute(): - #try prepending system config path - path = self._system_config_root/path + # try prepending system config path + path = self._system_config_root / path if path.is_file(): - with open(path, 'rb') as reader: + with open(path, "rb") as reader: binary_key = reader.read() - + # convert base64 key to binary else: try: @@ -258,17 +263,19 @@ def _key_setup(self, key_store: pathlib.Path): raise MirrorError( f"Key for mirror '{name}' is not valid: '{path}'. \n" f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" - f"Check the key listed in mirrors.yaml in system config.") - + f"Check the key listed in mirrors.yaml in system config." + ) + file_type = magic.from_buffer(binary_key, mime=True) if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): raise MirrorError( f"Key for mirror {name} is not a valid GPG key. \n" f"The file (or base64) was readable, but the data itself was not a PGP key.\n" - f"Check the key listed in mirrors.yaml in system config.") + f"Check the key listed in mirrors.yaml in system config." + ) # copy key to new destination in key store - with open(dest, 'wb') as writer: + with open(dest, "wb") as writer: writer.write(binary_key) self._keys.append(dest) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index a15569b..76cba82 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -4,9 +4,8 @@ import jinja2 import yaml -from typing import Optional -from . import cache, root_logger, schema, spack_util, mirror +from . import root_logger, schema, spack_util, mirror from .etc import envvars @@ -170,7 +169,7 @@ def __init__(self, args): schema.EnvironmentsValidator.validate(raw) self.generate_environment_specs(raw) - # load the optional mirrors.yaml from system config, and add any additional + # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") self.mirrors = mirror.Mirrors(self.system_config_path, args.cache) @@ -232,7 +231,7 @@ def pre_install_hook(self): if hook_path.exists() and hook_path.is_file(): return hook_path return None - + @property def config(self): return self._config diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 72f2190..d262cd1 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -4,14 +4,17 @@ import stackinator.mirror as mirror import yaml + @pytest.fixture def test_path(): return pathlib.Path(__file__).parent.resolve() + @pytest.fixture def systems_path(test_path): return test_path / "data" / "systems" + def test_mirror_init(systems_path): """Check that Mirror objects are initialized correctly.""" path = systems_path / "mirror-ok" @@ -19,37 +22,45 @@ def test_mirror_init(systems_path): valid_mirrors = { "fake-mirror": { - 'url': 'https://github.com', - 'enabled': True, - 'bootstrap': False, - 'cache': False, - 'public_key': '../../test-gpg-pub.asc', - 'mount_specific': False}, + "url": "https://github.com", + "enabled": True, + "bootstrap": False, + "cache": False, + "public_key": "../../test-gpg-pub.asc", + "mount_specific": False, + }, "buildcache-mirror": { - 'url': 'https://mirror.spack.io', - 'enabled': True, - 'bootstrap': False, - 'cache': True, - 'private_key': '../test-gpg-priv.asc', - 'mount_specific': False}, + "url": "https://mirror.spack.io", + "enabled": True, + "bootstrap": False, + "cache": True, + "private_key": "../test-gpg-priv.asc", + "mount_specific": False, + }, "bootstrap-mirror": { - 'url': 'https://mirror.spack.io', - 'enabled': True, - 'bootstrap': True, - 'cache': False, - 'mount_specific': False} + "url": "https://mirror.spack.io", + "enabled": True, + "bootstrap": True, + "cache": False, + "mount_specific": False, + }, } - with (systems_path/'../test-gpg-pub.asc').open('rb') as pub_key_file: + with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: key = base64.b64encode(pub_key_file.read()).decode() - valid_mirrors['buildcache-mirror']['public_key'] = key + valid_mirrors["buildcache-mirror"]["public_key"] = key assert mirrors_obj.mirrors == valid_mirrors - assert mirrors_obj.bootstrap_mirrors == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('bootstrap')] - assert mirrors_obj.build_cache_mirror == [name for name in valid_mirrors.keys() if valid_mirrors[name].get('cache')].pop(0) - + assert mirrors_obj.bootstrap_mirrors == [ + name for name in valid_mirrors.keys() if valid_mirrors[name].get("bootstrap") + ] + assert mirrors_obj.build_cache_mirror == [ + name for name in valid_mirrors.keys() if valid_mirrors[name].get("cache") + ].pop(0) + for mir in mirrors_obj.mirrors: - assert mirrors_obj.mirrors[mir].get('enabled') + assert mirrors_obj.mirrors[mir].get("enabled") + def test_mirror_init_bad_url(systems_path): """Check that MirrorError is raised for a bad url.""" @@ -59,31 +70,33 @@ def test_mirror_init_bad_url(systems_path): with pytest.raises(mirror.MirrorError): mirror.Mirrors(path) + def test_setup_configs(tmp_path, systems_path): """Test general config setup.""" - mir = mirror.Mirrors(systems_path/'mirror-ok') + mir = mirror.Mirrors(systems_path / "mirror-ok") mir.setup_configs(tmp_path) - assert (tmp_path/'mirrors.yaml').is_file() - assert (tmp_path/'bootstrap').is_dir() - assert (tmp_path/'key_store').is_dir() + assert (tmp_path / "mirrors.yaml").is_file() + assert (tmp_path / "bootstrap").is_dir() + assert (tmp_path / "key_store").is_dir() + def test_command_line_cache(systems_path): """Check that adding a cache from the command line works.""" - mirrors = mirror.Mirrors(systems_path/'mirror-ok', - cmdline_cache=systems_path/'mirror-ok/cache.yaml') + mirrors = mirror.Mirrors(systems_path / "mirror-ok", cmdline_cache=systems_path / "mirror-ok/cache.yaml") assert len(mirrors.mirrors) == 4 # This should always be the build cache even though one is already defined. - assert mirrors.build_cache_mirror == 'cmdline_cache' - cache_mirror = mirrors.mirrors['cmdline_cache'] - assert cache_mirror['url'] == '/tmp/foo' - assert cache_mirror['enabled'] - assert cache_mirror['cache'] - assert not cache_mirror['bootstrap'] - assert cache_mirror['mount_specific'] + assert mirrors.build_cache_mirror == "cmdline_cache" + cache_mirror = mirrors.mirrors["cmdline_cache"] + assert cache_mirror["url"] == "/tmp/foo" + assert cache_mirror["enabled"] + assert cache_mirror["cache"] + assert not cache_mirror["bootstrap"] + assert cache_mirror["mount_specific"] + def test_create_spack_mirrors_yaml(tmp_path, systems_path): """Check that the mirrors.yaml passed to spack is correct""" @@ -101,7 +114,7 @@ def test_create_spack_mirrors_yaml(tmp_path, systems_path): "bootstrap-mirror": { "fetch": {"url": "https://mirror.spack.io"}, "push": {"url": "https://mirror.spack.io"}, - } + }, } } @@ -114,9 +127,10 @@ def test_create_spack_mirrors_yaml(tmp_path, systems_path): assert data == valid_spack_yaml + def test_create_bootstrap_configs(tmp_path, systems_path): """Check that spack bootstrap configs are generated correctly""" - + valid_yaml = { "sources": [ { @@ -124,54 +138,54 @@ def test_create_bootstrap_configs(tmp_path, systems_path): "metadata": str(tmp_path / "bootstrap/bootstrap-mirror"), } ], - "trusted": { - "bootstrap-mirror": True - }, + "trusted": {"bootstrap-mirror": True}, } valid_metadata = { "type": "install", "info": "https://mirror.spack.io", } - mirrors_obj = mirror.Mirrors(systems_path/'mirror-ok') + mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok") mirrors_obj._create_bootstrap_configs(tmp_path) - with (tmp_path/'bootstrap.yaml').open() as f: + with (tmp_path / "bootstrap.yaml").open() as f: bs_data = yaml.safe_load(f) print(bs_data) print(valid_yaml) assert bs_data == valid_yaml - with (tmp_path/'bootstrap/bootstrap-mirror/metadata.yaml').open() as f: + with (tmp_path / "bootstrap/bootstrap-mirror/metadata.yaml").open() as f: metadata = yaml.safe_load(f) assert metadata == valid_metadata + def test_key_setup(systems_path, tmp_path): """Check that public keys are set up properly.""" - mirrors = mirror.Mirrors(systems_path/'mirror-ok') + mirrors = mirror.Mirrors(systems_path / "mirror-ok") mirrors._key_setup(tmp_path) key_files = list(tmp_path.iterdir()) - assert {key_file.name for key_file in key_files} == {'buildcache-mirror.gpg', 'fake-mirror.gpg'} + assert {key_file.name for key_file in key_files} == {"buildcache-mirror.gpg", "fake-mirror.gpg"} # The two files should be identical in content key_file_data = [] for key_file in key_files: - with key_file.open('rb') as file: + with key_file.open("rb") as file: key_file_data.append(file.read()) assert key_file_data[0] == key_file_data[1] -@pytest.mark.parametrize("system_name", [ - 'mirror-bad-key', - 'mirror-bad-keypath', -]) + +@pytest.mark.parametrize( + "system_name", + [ + "mirror-bad-key", + "mirror-bad-keypath", + ], +) def test_key_setup_bad_key(tmp_path, systems_path, system_name): """asdfasdf""" - mirrors = mirror.Mirrors(systems_path/system_name) + mirrors = mirror.Mirrors(systems_path / system_name) with pytest.raises(mirror.MirrorError): mirrors._key_setup(tmp_path) - - - From 30521d3ca4d747d0480a75b4eb87e5565120b317 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Mon, 16 Mar 2026 13:14:56 -0600 Subject: [PATCH 54/69] Got rid of cache.py --- stackinator/cache.py | 58 -------------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 stackinator/cache.py diff --git a/stackinator/cache.py b/stackinator/cache.py deleted file mode 100644 index 24177e3..0000000 --- a/stackinator/cache.py +++ /dev/null @@ -1,58 +0,0 @@ -import os -import pathlib - -import yaml - -from . import schema - - -def configuration_from_file(file, mount): - with file.open() as fid: - # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) - - # validate the yaml - schema.CacheValidator.validate(raw) - - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(raw["root"])) - if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") - - raw["root"] = path - - # Put the build cache in a sub-directory named after the mount point. - # This avoids relocation issues. - raw["path"] = pathlib.Path(path.as_posix() + mount.as_posix()) - - # verify that the key file exists if it was specified - key = raw["key"] - if key is not None: - key = pathlib.Path(os.path.expandvars(key)) - if not key.is_absolute(): - raise FileNotFoundError(f"The build cache key '{key}' is not absolute") - if not key.is_file(): - raise FileNotFoundError(f"The build cache key '{key}' does not exist") - raw["key"] = key - - return raw - - -def generate_mirrors_yaml(config): - path = config["path"].as_posix() - mirrors = { - "mirrors": { - "alpscache": { - "fetch": { - "url": f"file://{path}", - }, - "push": { - "url": f"file://{path}", - }, - } - } - } - - return yaml.dump(mirrors, default_flow_style=False) From 191af6fe56ed2f7e7ffb4461e33e14868a9f10fc Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 17 Mar 2026 12:41:31 +0100 Subject: [PATCH 55/69] use pyproject for dependencies --- bin/stack-config | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/bin/stack-config b/bin/stack-config index 6649699..02f4870 100755 --- a/bin/stack-config +++ b/bin/stack-config @@ -2,6 +2,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ +# "python-magic", # "jinja2", # "jsonschema", # "pyYAML", diff --git a/pyproject.toml b/pyproject.toml index 583f5d4..25fd1dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ license-files = ["LICENSE"] dynamic = ["version"] requires-python = ">=3.12" dependencies = [ + "python-magic", "Jinja2", "jsonschema", "PyYAML", From 2fc29d6f19810a82eb86a32c50563969a97375ec Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 17 Mar 2026 12:43:53 +0100 Subject: [PATCH 56/69] add unit test and lint hints to readme --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 5c7e040..265b0ac 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,17 @@ A tool for building a scientific software stack from a recipe for vClusters on C Read the [documentation](https://eth-cscs.github.io/stackinator/) to get started. Create a ticket in our [GitHub issues](https://github.com/eth-cscs/stackinator/issues) if you find a bug, have a feature request or have a question. + +## running tests: + +Use uv to run the tests, which will in turn ensure that the correct dependencies from `pyproject.toml` are used: + +``` +uv run pytest +``` + +Before pushing, apply the linting rules (this calls uv under the hood): + +``` +./lint +``` From d8594a3881eabdecd619401e798637163930ac07 Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 17 Mar 2026 12:44:19 +0100 Subject: [PATCH 57/69] tweak presentation of build cache by recipe --- stackinator/builder.py | 2 +- stackinator/main.py | 2 +- stackinator/recipe.py | 14 ++++++++++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index ca0f875..47a73b0 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -233,7 +233,7 @@ def generate(self, recipe): spack_version=spack_version, spack_meta=spack_meta, gpg_keys=recipe.mirrors.keys, - cache=recipe.mirrors.buildcache, + cache=recipe.build_cache_mirror, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) diff --git a/stackinator/main.py b/stackinator/main.py index 0d0b9bd..ec38456 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -86,7 +86,7 @@ def make_argparser(): "--build", required=True, type=str, - help="Where to set up the stackinator build directory. ('/tmp' is not allowed, use '/var/tmp'", + help="Where to set up the stackinator build directory. ('/tmp' is not allowed, use '/var/tmp')", ) parser.add_argument("--no-bwrap", action="store_true", required=False) parser.add_argument( diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 76cba82..ff3d8e2 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -172,8 +172,7 @@ def __init__(self, args): # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") - self.mirrors = mirror.Mirrors(self.system_config_path, args.cache) - self.cache = self.mirrors.build_cache_mirror + self.mirrors = mirror.Mirrors(self.system_config_path, pathlib.Path(args.cache)) # optional post install hook if self.post_install_hook is not None: @@ -202,6 +201,13 @@ def spack_repo(self): return repo_path return None + # Returns: + # Path: if the recipe specified a build cache mirror + # None: if no build cache mirror is used + @property + def build_cache_mirror(self): + return self.mirrors.build_cache_mirror + # Returns: # Path: of the recipe extra path if it exists # None: if there is no user-provided extra path in the recipe @@ -511,7 +517,7 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.cache + push_to_cache = self.build_cache_mirror is not None files["makefile"] = makefile_template.render( compilers=self.compilers, push_to_cache=push_to_cache, @@ -542,7 +548,7 @@ def environment_files(self): jenv.filters["py2yaml"] = schema.py2yaml makefile_template = jenv.get_template("Makefile.environments") - push_to_cache = self.cache is not None + push_to_cache = self.build_cache_mirror is not None files["makefile"] = makefile_template.render( environments=self.environments, push_to_cache=push_to_cache, From ef9d7e88fe3f6d7da1f7dcaa729d35318741369e Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 20 Mar 2026 15:35:01 +0100 Subject: [PATCH 58/69] fix build cache configuration --- stackinator/builder.py | 6 +- stackinator/mirror.py | 80 ++++++++++++--------- stackinator/recipe.py | 6 +- stackinator/templates/Makefile.compilers | 4 +- stackinator/templates/Makefile.environments | 4 +- 5 files changed, 55 insertions(+), 45 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 47a73b0..2dc6923 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -168,10 +168,12 @@ def generate(self, recipe): # make the paths, in case bwrap is not used, directly write to recipe.mount store_path = self.path / "store" if not recipe.no_bwrap else pathlib.Path(recipe.mount) tmp_path = self.path / "tmp" + config_path = self.path / "config" self.path.mkdir(exist_ok=True, parents=True) store_path.mkdir(exist_ok=True) tmp_path.mkdir(exist_ok=True) + config_path.mkdir(exist_ok=True) # check out the version of spack spack_version = recipe.spack_version @@ -232,7 +234,7 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - gpg_keys=recipe.mirrors.keys, + gpg_keys=recipe.mirrors.key_files(config_path), cache=recipe.build_cache_mirror, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, @@ -301,8 +303,6 @@ def generate(self, recipe): # Generate the system configuration: the compilers, environments, etc. # that are defined for the target cluster. - config_path = self.path / "config" - config_path.mkdir(exist_ok=True) packages_path = config_path / "packages.yaml" # the packages.yaml configuration that will be used when building all environments diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e6ba268..66fa1d1 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,13 +1,15 @@ -from typing import Optional, List, Dict +from typing import Optional, Dict import base64 import io -import magic import os import pathlib +import shutil import urllib.error import urllib.request import yaml +import magic + from . import schema, root_logger @@ -50,7 +52,8 @@ def __init__( self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() if mirror.get("bootstrap", False)] # Will hold a list of all the gpg keys (public and private) - self._keys: Optional[List[pathlib.Path]] = [] + # self._keys: Optional[List[pathlib.Path]] = [] + self._keys = self._key_init() def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" @@ -157,14 +160,12 @@ def _check_mirrors(self): f"Check the url listed in mirrors.yaml in system config. \n{e.reason}" ) - @property - def keys(self): + def key_files(self, config_root: pathlib.Path): """Return the list of public and private key file paths.""" - if self._keys is None: raise RuntimeError("The mirror.keys method was accessed before setup_configs() was called.") - - return self._keys + key_dir = config_root / self.KEY_STORE_DIR + return [key_dir / info["path"] for info in self._keys] def setup_configs(self, config_root: pathlib.Path): """Setup all mirror configs in the given config_root.""" @@ -229,34 +230,27 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): with (config_root / "bootstrap.yaml").open("w") as file: yaml.dump(bootstrap_yaml, file, default_flow_style=False) - def _key_setup(self, key_store: pathlib.Path): - """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" - - self._keys = [] - key_store.mkdir(exist_ok=True) + def _key_init(self): + key_info = [] for name, mirror in self.mirrors.items(): - if mirror.get("public_key") is None: + if mirror.get("private_key") is None: continue - key = mirror["public_key"] - - # key will be saved under key_store/mirror_name.gpg - - dest = pathlib.Path(key_store / f"{name}.gpg") + key = mirror["private_key"] # if path, check if abs path, if not, append sys config path in front and check again path = pathlib.Path(os.path.expandvars(key)) + if not path.is_absolute(): # try prepending system config path path = self._system_config_root / path if path.is_file(): - with open(path, "rb") as reader: - binary_key = reader.read() - - # convert base64 key to binary + # use the user-provided file + key_info.append({"path": pathlib.Path(f"{name}.pgp"), "source": path}) else: + # convert base64 key to binary try: binary_key = base64.b64decode(key) except ValueError: @@ -266,16 +260,34 @@ def _key_setup(self, key_store: pathlib.Path): f"Check the key listed in mirrors.yaml in system config." ) - file_type = magic.from_buffer(binary_key, mime=True) - if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): - raise MirrorError( - f"Key for mirror {name} is not a valid GPG key. \n" - f"The file (or base64) was readable, but the data itself was not a PGP key.\n" - f"Check the key listed in mirrors.yaml in system config." - ) + file_type = magic.from_buffer(binary_key, mime=True) + if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): + raise MirrorError( + f"Key for mirror {name} is not a valid GPG key. \n" + f"The file (or base64) was readable, but the data itself was not a PGP key.\n" + f"Check the key listed in mirrors.yaml in system config." + ) + + key_info.append({"path": pathlib.Path(f"{name}.pgp"), "source": binary_key}) - # copy key to new destination in key store - with open(dest, "wb") as writer: - writer.write(binary_key) + return key_info + + def _key_setup(self, key_store: pathlib.Path): + """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" + + key_store.mkdir(exist_ok=True) - self._keys.append(dest) + for key_info in self._keys: + path = key_store / key_info["path"] + source = key_info["source"] + + match source: + case pathlib.Path(): + # copy source -> path + shutil.copy2(source, path) + case bytes(): + # open path and copy in bytes + with open(path, "wb") as writer: + writer.write(source) + case _: + raise TypeError(f"Expected Path or bytes, got {type(source).__name__}") diff --git a/stackinator/recipe.py b/stackinator/recipe.py index ff3d8e2..0688d0d 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -517,10 +517,9 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.build_cache_mirror is not None files["makefile"] = makefile_template.render( compilers=self.compilers, - push_to_cache=push_to_cache, + buildcache=self.build_cache_mirror, spack_version=self.spack_version, ) @@ -548,10 +547,9 @@ def environment_files(self): jenv.filters["py2yaml"] = schema.py2yaml makefile_template = jenv.get_template("Makefile.environments") - push_to_cache = self.build_cache_mirror is not None files["makefile"] = makefile_template.render( environments=self.environments, - push_to_cache=push_to_cache, + buildcache=self.build_cache_mirror, spack_version=self.spack_version, ) diff --git a/stackinator/templates/Makefile.compilers b/stackinator/templates/Makefile.compilers index f9520bd..d534b31 100644 --- a/stackinator/templates/Makefile.compilers +++ b/stackinator/templates/Makefile.compilers @@ -18,8 +18,8 @@ all:{% for compiler in compilers %} {{ compiler }}/generated/build_cache{% endfo {% for compiler, config in compilers.items() %} {{ compiler }}/generated/build_cache: {{ compiler }}/generated/env -{% if push_to_cache %} - $(SPACK) -e ./{{ compiler }} buildcache create --rebuild-index --only=package alpscache \ +{% if buildcache %} + $(SPACK) -e ./{{ compiler }} buildcache create --rebuild-index --only=package {{ buildcache }} \ $$($(SPACK_HELPER) -e ./{{ compiler }} find --format '{name};{/hash}' \ | grep -v -E '^({% for p in config.exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | cut -d ';' -f2) diff --git a/stackinator/templates/Makefile.environments b/stackinator/templates/Makefile.environments index 5a53023..929e40b 100644 --- a/stackinator/templates/Makefile.environments +++ b/stackinator/templates/Makefile.environments @@ -17,8 +17,8 @@ all:{% for env in environments %} {{ env }}/generated/build_cache{% endfor %} # Push built packages to a binary cache if a key has been provided {% for env, config in environments.items() %} {{ env }}/generated/build_cache: {{ env }}/generated/view_config -{% if push_to_cache %} - $(SPACK) --color=never -e ./{{ env }} buildcache create --rebuild-index --only=package alpscache \ +{% if buildcache %} + $(SPACK) --color=never -e ./{{ env }} buildcache create --rebuild-index --only=package {{ buildcache }} \ $$($(SPACK_HELPER) -e ./{{ env }} find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in config.exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ From 8b1e0afceb4043a86fd8787b56c0b7a915dd9945 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 20 Mar 2026 14:14:10 -0600 Subject: [PATCH 59/69] modified bootstrap yaml setup and removed hardcoded alpscache --- stackinator/mirror.py | 18 +++++++++++------- stackinator/recipe.py | 5 ++++- stackinator/templates/Makefile | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 66fa1d1..2c72e99 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -49,7 +49,7 @@ def __init__( [name for name, mirror in self.mirrors.items() if mirror.get("cache", False)] + [None] ).pop(0) - self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() if mirror.get("bootstrap", False)] + self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() if mirror.get("bootstrap", True)] # Will hold a list of all the gpg keys (public and private) # self._keys: Optional[List[pathlib.Path]] = [] @@ -201,29 +201,33 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): return bootstrap_yaml = { - "sources": [], - "trusted": {}, + "bootstrap": { + "sources": [], + "trusted": {}, + } } for name in self.bootstrap_mirrors: bs_mirror_path = config_root / f"bootstrap/{name}" mirror = self.mirrors[name] # Tell spack where to find the metadata for each bootstrap mirror. - bootstrap_yaml["sources"].append( + bootstrap_yaml["bootstrap"]["sources"].append( { "name": name, "metadata": str(bs_mirror_path), } ) # And trust each one - bootstrap_yaml["trusted"][name] = True + bootstrap_yaml["bootstrap"]["trusted"][name] = True # Create the metadata dir and metadata.yaml bs_mirror_path.mkdir(parents=True, exist_ok=True) bs_mirror_yaml = { "type": "install", - "info": mirror["url"], - } + "info": { + "url": mirror["url"], + } + } with (bs_mirror_path / "metadata.yaml").open("w") as file: yaml.dump(bs_mirror_yaml, file, default_flow_style=False) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 0688d0d..8f4c82f 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -172,7 +172,8 @@ def __init__(self, args): # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") - self.mirrors = mirror.Mirrors(self.system_config_path, pathlib.Path(args.cache)) + self.mirrors = mirror.Mirrors(self.system_config_path, + pathlib.Path(args.cache) if args.cache else None) # optional post install hook if self.post_install_hook is not None: @@ -521,6 +522,7 @@ def compiler_files(self): compilers=self.compilers, buildcache=self.build_cache_mirror, spack_version=self.spack_version, + cache = self.build_cache_mirror, ) files["config"] = {} @@ -551,6 +553,7 @@ def environment_files(self): environments=self.environments, buildcache=self.build_cache_mirror, spack_version=self.spack_version, + cache=self.build_cache_mirror, ) files["config"] = {} diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index f0b90eb..7a93177 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -86,7 +86,7 @@ cache-force: mirror-setup $(warning likely have to start a fresh build (but that's okay, because build caches FTW)) $(warning ================================================================================) $(SANDBOX) $(MAKE) -C generate-config - $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package cache.name \ + $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package cache \ $$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ From 3c21789af1b8d006d612312ecb72b9c6704fbdd4 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 25 Mar 2026 22:54:35 -0600 Subject: [PATCH 60/69] mirrors documentation --- docs/cluster-config.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/cluster-config.md b/docs/cluster-config.md index 2ce6201..5a7d4d5 100644 --- a/docs/cluster-config.md +++ b/docs/cluster-config.md @@ -93,6 +93,27 @@ packages: version: ["git.59b6de6a91d9637809677c50cc48b607a91a9acb=main"] ``` +### Configuring Spack mirrors: `mirrors.yaml` + +On air-gapped systems, Spack is unable to reach its default mirror to fetch packages. The `mirrors.yaml` configuration can be used to connect Spack to local mirrors for fetching and building packages. + +`mirrors.yaml` treats source mirrors, buildcaches, and bootstrap mirrors the same, and they may all be included in this file. Spack will search the topmost mirror first and the bottom-most mirror last, and will append the default Spack mirror to the bottom of the list when the Spack mirror config is generated. + +If using a buildcache, public and private keys must be provided for signing and verifying packages. + +```yaml title="mirrors.yaml" +local_filesystem: + url: file:///home/username/spack-mirror-2014-06-24 +site_server: + url: https://example.com/some/web-hosted/directory +buildcache-mirror: + url: https://example.com/some/buildcache/mirror + public_key: ../buildcache-key.public.gpg + private_key: /user-home/.gnupg/private-keys-v1.d/my-private-key.asc + cache: true + bootstrap: true +``` + ## Site and System Configurations The `repo.yaml` configuration can be used to provide a list of additional Spack package repositories to use on the target system. From 596f50ea7a4e895f750515b457a260a9148901b9 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Wed, 1 Apr 2026 09:31:33 -0600 Subject: [PATCH 61/69] Switched to the requests libs from urlopen, as urlopen doesn't handle ssl very well. --- requirements.txt | 1 + stackinator/mirror.py | 11 ++++------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index d0de0ac..82d9720 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,5 @@ pytest==9.0.2 python-magic==0.4.27 pyyaml==6.0.3 referencing==0.37.0 +requests==2.32.5 rpds-py==0.30.0 diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 2c72e99..61ba461 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -3,9 +3,7 @@ import io import os import pathlib -import shutil -import urllib.error -import urllib.request +import requests import yaml import magic @@ -152,12 +150,11 @@ def _check_mirrors(self): elif url.startswith("https://"): try: - request = urllib.request.Request(url, method="HEAD") - urllib.request.urlopen(request) - except urllib.error.URLError as e: + requests.request(url=url, method="HEAD") + except requests.exceptions.RequestException as err: raise MirrorError( f"Could not reach the mirror url '{url}'. " - f"Check the url listed in mirrors.yaml in system config. \n{e.reason}" + f"Check the url listed in mirrors.yaml in system config. \n{err}" ) def key_files(self, config_root: pathlib.Path): From ff42460cbdb531bf96db4ef5235ffab3ca076104 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 8 Apr 2026 14:14:04 -0600 Subject: [PATCH 62/69] restrict number of buildcaches and bootstraps in mirror schema --- stackinator/schema/mirror.json | 86 +++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 39 deletions(-) diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 8977083..042ac63 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -1,46 +1,54 @@ { "type" : "object", - "additionalProperties": { - "type": "object", - "required": ["url"], - "additionalProperties": false, - "properties": { - "url": { - "type": "string", - "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI)." + "additionalProperties": false, + "properties": { + "bootstrap": { + "type": "object", + "properties": { + "description": {"type": "string"}, + "url": {"type": "string"}, + "enabled": { + "type": "boolean", + "default": true + } }, - "description": { - "type": "string", - "description": "What this mirror is for." + "additionalProperties": false, + "required": ["url"] + }, + "buildcache": { + "type": "object", + "properties": { + "description": {"type": "string"}, + "url": {"type": "string"}, + "enabled": { + "type": "boolean", + "default": true + }, + "public_key": {"type": "string"}, + "private_key": {"type": "string"}, + "mount_specific": { + "type": "boolean", + "default": false + } }, - "enabled": { - "type": "boolean", - "default": true, - "description": "Whether this mirror is enabled." - }, - "bootstrap": { - "type": "boolean", - "default": false, - "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror." - }, - "cache": { - "type": "boolean", - "default": false, - "description": "Use this mirror as the buildcache push destination. Can only be enabled on a single mirror." - }, - "public_key": { - "type": "string", - "description": "Public PGP key for validating binary cache packages. A path or base64 encoded key." - }, - "private_key": { - "type": "string", - "description": "Private PGP key for signing binary cache packages. (Path only)" - }, - "mount_specific": { - "type": "boolean", - "default": false, - "description": "Use a mount specific buildcache path (specified path + recipe mount point)." + "additionalProperties": false, + "required": ["url"] + }, + "sourcecache": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "description": {"type": "string"}, + "url": {"type": "string"}, + "enabled": { + "type": "boolean", + "default": true + } + }, + "additionalProperties": false, + "required": ["url"] } } } -} +} \ No newline at end of file From e10f0c9b4bd9894687306a234051699b81501c1b Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 8 Apr 2026 14:32:51 -0600 Subject: [PATCH 63/69] fixed mistake --- stackinator/recipe.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 8f4c82f..e63165a 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -522,7 +522,6 @@ def compiler_files(self): compilers=self.compilers, buildcache=self.build_cache_mirror, spack_version=self.spack_version, - cache = self.build_cache_mirror, ) files["config"] = {} @@ -553,7 +552,6 @@ def environment_files(self): environments=self.environments, buildcache=self.build_cache_mirror, spack_version=self.spack_version, - cache=self.build_cache_mirror, ) files["config"] = {} From 6a203c39b1bff74c4caf12ae75464ce8edd0522a Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 14 Apr 2026 13:44:19 -0600 Subject: [PATCH 64/69] refactored mirrors.py to match new schema --- stackinator/mirror.py | 127 +++++++++++++++++++-------------- stackinator/schema/mirror.json | 4 ++ 2 files changed, 78 insertions(+), 53 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 61ba461..bcc3e5c 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -38,17 +38,6 @@ def __init__( self.mirrors = self._load_mirrors(cmdline_cache) self._check_mirrors() - # Always use the cache given on the command line - if self.CMDLINE_CACHE in self.mirrors: - self.build_cache_mirror = self.CMDLINE_CACHE - else: - # Otherwise, grab the configured cache (or None) - self.build_cache_mirror: Optional[str] = ( - [name for name, mirror in self.mirrors.items() if mirror.get("cache", False)] + [None] - ).pop(0) - - self.bootstrap_mirrors = [name for name, mirror in self.mirrors.items() if mirror.get("bootstrap", True)] - # Will hold a list of all the gpg keys (public and private) # self._keys: Optional[List[pathlib.Path]] = [] self._keys = self._key_init() @@ -71,22 +60,37 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict except ValueError as err: raise MirrorError(f"Mirror config does not comply with schema.\n{err}") - caches = [name for name, mirror in mirrors.items() if mirror["cache"]] - if len(caches) > 1: - raise MirrorError( - "Mirror config has more than one mirror specified as the build cache destination.\n" - f"{self._pp_yaml(caches)}" - ) - elif caches: - cache = mirrors[caches[0]] - if not cache.get("private_key"): - raise MirrorError(f"Mirror build cache config '{caches[0]}' missing a required 'private_key' path.") + enabled_mirrors: Dict[str, Dict] = {} + + buildcache = mirrors.get("buildcache") + if buildcache and buildcache.get("enabled", True): + if not buildcache.get("private_key"): + raise MirrorError( + "Mirror build cache config is missing a required 'private_key' path." + ) + self.build_cache_mirror = "buildcache" + enabled_mirrors["buildcache"] = buildcache + else: + self.build_cache_mirror = None # Load the cache as defined by the deprecated 'cache.yaml' file. if cmdline_cache is not None: - mirrors[self.CMDLINE_CACHE] = self._load_cmdline_cache(cmdline_cache) + enabled_mirrors["buildcache"] = self._load_cmdline_cache(cmdline_cache) + self.build_cache_mirror = self.CMDLINE_CACHE + + + bootstrap = mirrors.get("bootstrap") + if bootstrap and bootstrap.get("enabled", True): + self.bootstrap_mirror = bootstrap + enabled_mirrors["bootstrap"] = bootstrap + else: + self.bootstrap_mirror = None - return {name: mirror for name, mirror in mirrors.items() if mirror["enabled"]} + for name, mirror in mirrors.get("sourcecache", {}).items(): + if mirror.get("enabled", True): + enabled_mirrors[name] = mirror + + return enabled_mirrors @staticmethod def _pp_yaml(object): @@ -118,11 +122,10 @@ def _load_cmdline_cache(self, cache_config_path: pathlib.Path) -> Dict: mirror_cfg = { "url": raw["root"], "description": "Buildcache dest loaded from legacy cache.yaml", - "cache": True, "enabled": True, - "bootstrap": False, "mount_specific": True, "private_key": raw["key"], + "cmdline": True } self._logger.warning( @@ -176,11 +179,11 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): raw = {"mirrors": {}} - for name, mirror in self.mirrors.items(): - url = mirror["url"] + if self.build_cache_mirror: + url = self.mirrors.get("buildcache")["url"] + name = self.build_cache_mirror - # Make the mirror path specific to the mount point - if mirror["mount_specific"] and self._mount_point is not None: + if self.build_cache_mirror.get("mount_specific", True) and self._mount_point is not None: url = url.rstrip("/") + "/" + self._mount_point.as_posix().lstrip("/") raw["mirrors"][name] = { @@ -188,13 +191,32 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): "push": {"url": url}, } + elif self.mirrors.get("bootstrap"): + url = self.mirrors.get("bootstrap")["url"] + + raw["mirrors"]["bootstrap"] = { + "fetch": {"url": url}, + "push": {"url": url}, + } + + elif self.mirrors.get("sourcecache"): + source_mirrors = self.mirrors.get("sourcecache") + + for name, mirror in source_mirrors.items(): + url = mirror["url"] + + raw["mirrors"][name] = { + "fetch": {"url": url}, + "push": {"url": url}, + } + with dest.open("w") as file: yaml.dump(raw, file, default_flow_style=False) def _create_bootstrap_configs(self, config_root: pathlib.Path): """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" - if not self.bootstrap_mirrors: + if not self.bootstrap_mirror: return bootstrap_yaml = { @@ -204,29 +226,28 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): } } - for name in self.bootstrap_mirrors: - bs_mirror_path = config_root / f"bootstrap/{name}" - mirror = self.mirrors[name] - # Tell spack where to find the metadata for each bootstrap mirror. - bootstrap_yaml["bootstrap"]["sources"].append( - { - "name": name, - "metadata": str(bs_mirror_path), - } - ) - # And trust each one - bootstrap_yaml["bootstrap"]["trusted"][name] = True - - # Create the metadata dir and metadata.yaml - bs_mirror_path.mkdir(parents=True, exist_ok=True) - bs_mirror_yaml = { - "type": "install", - "info": { - "url": mirror["url"], - } - } - with (bs_mirror_path / "metadata.yaml").open("w") as file: - yaml.dump(bs_mirror_yaml, file, default_flow_style=False) + bs_mirror_path = config_root / f"bootstrap/{self.bootstrap_mirror}" + mirror = self.mirrors.get("bootstrap") + # Tell spack where to find the metadata for each bootstrap mirror. + bootstrap_yaml["bootstrap"]["sources"].append( + { + "name": "bootstrap_mirror", + "metadata": str(bs_mirror_path), + } + ) + # And trust each one + bootstrap_yaml["bootstrap"]["trusted"][bootstrap_mirror] = True + + # Create the metadata dir and metadata.yaml + bs_mirror_path.mkdir(parents=True, exist_ok=True) + bs_mirror_yaml = { + "type": "install", + "info": { + "url": mirror["url"], + } + } + with (bs_mirror_path / "metadata.yaml").open("w") as file: + yaml.dump(bs_mirror_yaml, file, default_flow_style=False) with (config_root / "bootstrap.yaml").open("w") as file: yaml.dump(bootstrap_yaml, file, default_flow_style=False) diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 042ac63..4965c81 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -29,6 +29,10 @@ "mount_specific": { "type": "boolean", "default": false + }, + "cmdline": { + "type": "boolean", + "default": true } }, "additionalProperties": false, From 099878ab22f7e4b6a0c3850ea28931fa9fd991fc Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 21 Apr 2026 14:45:51 -0600 Subject: [PATCH 65/69] fixing unit tests, _key_init() still broken --- stackinator/mirror.py | 145 ++++++++---------- stackinator/schema/mirror.json | 2 +- .../data/systems/mirror-bad-key/mirrors.yaml | 4 +- .../systems/mirror-bad-keypath/mirrors.yaml | 4 +- .../data/systems/mirror-bad-url/mirrors.yaml | 6 +- .../systems/mirror-ok-raw-key/mirrors.yaml | 59 +++++++ unittests/data/systems/mirror-ok/mirrors.yaml | 70 ++------- unittests/test_mirrors.py | 112 +++++++------- 8 files changed, 207 insertions(+), 195 deletions(-) create mode 100644 unittests/data/systems/mirror-ok-raw-key/mirrors.yaml diff --git a/stackinator/mirror.py b/stackinator/mirror.py index bcc3e5c..5219af0 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -68,8 +68,8 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict raise MirrorError( "Mirror build cache config is missing a required 'private_key' path." ) - self.build_cache_mirror = "buildcache" - enabled_mirrors["buildcache"] = buildcache + self.build_cache_mirror = "buildcache" + enabled_mirrors["buildcache"] = buildcache else: self.build_cache_mirror = None @@ -78,17 +78,13 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict enabled_mirrors["buildcache"] = self._load_cmdline_cache(cmdline_cache) self.build_cache_mirror = self.CMDLINE_CACHE - bootstrap = mirrors.get("bootstrap") if bootstrap and bootstrap.get("enabled", True): - self.bootstrap_mirror = bootstrap enabled_mirrors["bootstrap"] = bootstrap - else: - self.bootstrap_mirror = None for name, mirror in mirrors.get("sourcecache", {}).items(): - if mirror.get("enabled", True): - enabled_mirrors[name] = mirror + if mirror.get("enabled", True): + enabled_mirrors[name] = mirror return enabled_mirrors @@ -179,44 +175,26 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): raw = {"mirrors": {}} - if self.build_cache_mirror: - url = self.mirrors.get("buildcache")["url"] - name = self.build_cache_mirror + for name, mirror in self.mirrors.items(): + url = mirror["url"] - if self.build_cache_mirror.get("mount_specific", True) and self._mount_point is not None: - url = url.rstrip("/") + "/" + self._mount_point.as_posix().lstrip("/") + # Make the mirror path specific to the mount point + if(name=="buildcache"): + if mirror["mount_specific"] and self._mount_point is not None: + url = url.rstrip("/") + "/" + self._mount_point.as_posix().lstrip("/") raw["mirrors"][name] = { "fetch": {"url": url}, "push": {"url": url}, } - elif self.mirrors.get("bootstrap"): - url = self.mirrors.get("bootstrap")["url"] - - raw["mirrors"]["bootstrap"] = { - "fetch": {"url": url}, - "push": {"url": url}, - } - - elif self.mirrors.get("sourcecache"): - source_mirrors = self.mirrors.get("sourcecache") - - for name, mirror in source_mirrors.items(): - url = mirror["url"] - - raw["mirrors"][name] = { - "fetch": {"url": url}, - "push": {"url": url}, - } - with dest.open("w") as file: yaml.dump(raw, file, default_flow_style=False) def _create_bootstrap_configs(self, config_root: pathlib.Path): """Create the bootstrap.yaml and bootstrap metadata dirs in our build dir.""" - if not self.bootstrap_mirror: + if not self.mirrors.get("bootstrap"): return bootstrap_yaml = { @@ -226,17 +204,17 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): } } - bs_mirror_path = config_root / f"bootstrap/{self.bootstrap_mirror}" + bs_mirror_path = config_root / "bootstrap" / "bootstrap-mirror" mirror = self.mirrors.get("bootstrap") # Tell spack where to find the metadata for each bootstrap mirror. bootstrap_yaml["bootstrap"]["sources"].append( { - "name": "bootstrap_mirror", + "name": "bootstrap-mirror", "metadata": str(bs_mirror_path), } ) # And trust each one - bootstrap_yaml["bootstrap"]["trusted"][bootstrap_mirror] = True + bootstrap_yaml["bootstrap"]["trusted"]["bootstrap-mirror"] = True # Create the metadata dir and metadata.yaml bs_mirror_path.mkdir(parents=True, exist_ok=True) @@ -253,44 +231,44 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): yaml.dump(bootstrap_yaml, file, default_flow_style=False) def _key_init(self): - key_info = [] - - for name, mirror in self.mirrors.items(): - if mirror.get("private_key") is None: - continue + key_info = {} + + key = self.mirrors["buildcache"].get("private_key") - key = mirror["private_key"] + if key is None: + return - # if path, check if abs path, if not, append sys config path in front and check again - path = pathlib.Path(os.path.expandvars(key)) + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) - if not path.is_absolute(): - # try prepending system config path - path = self._system_config_root / path + if not path.is_absolute(): + # try prepending system config path + path = self._system_config_root / path - if path.is_file(): - # use the user-provided file - key_info.append({"path": pathlib.Path(f"{name}.pgp"), "source": path}) - else: - # convert base64 key to binary - try: - binary_key = base64.b64decode(key) - except ValueError: - raise MirrorError( - f"Key for mirror '{name}' is not valid: '{path}'. \n" - f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" - f"Check the key listed in mirrors.yaml in system config." - ) + if path.is_file(): + # use the user-provided file + key_info = {"path": pathlib.Path(f"buildcache.pgp"), "source": path} + else: + # convert base64 key to binary + try: + binary_key = base64.b64decode(key, validate=True) + print(binary_key) + except ValueError: + raise MirrorError( + f"Key for mirror 'buildcache' is not valid. \n" + f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" + f"Check the key listed in mirrors.yaml in system config." + ) - file_type = magic.from_buffer(binary_key, mime=True) - if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): - raise MirrorError( - f"Key for mirror {name} is not a valid GPG key. \n" - f"The file (or base64) was readable, but the data itself was not a PGP key.\n" - f"Check the key listed in mirrors.yaml in system config." - ) + file_type = magic.from_buffer(binary_key, mime=True) + if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): + raise MirrorError( + f"Key for mirror 'buildcache' is not a valid GPG key. \n" + f"The file (or base64) was readable, but the data itself was not a PGP key.\n" + f"Check the key listed in mirrors.yaml in system config." + ) - key_info.append({"path": pathlib.Path(f"{name}.pgp"), "source": binary_key}) + key_info = {"path": pathlib.Path("buildcache.pgp"), "source": binary_key} return key_info @@ -299,17 +277,20 @@ def _key_setup(self, key_store: pathlib.Path): key_store.mkdir(exist_ok=True) - for key_info in self._keys: - path = key_store / key_info["path"] - source = key_info["source"] - - match source: - case pathlib.Path(): - # copy source -> path - shutil.copy2(source, path) - case bytes(): - # open path and copy in bytes - with open(path, "wb") as writer: - writer.write(source) - case _: - raise TypeError(f"Expected Path or bytes, got {type(source).__name__}") + #for key_info in self._keys: + + #path = key_store / key_info["path"] + path = key_store / self._keys["path"] + #source = key_info["source"] + source = self._keys["source"] + + match source: + case pathlib.Path(): + # copy source -> path + shutil.copy2(source, path) + case bytes(): + # open path and copy in bytes + with open(path, "wb") as writer: + writer.write(source) + case _: + raise TypeError(f"Expected Path or bytes, got {type(source).__name__}") diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 4965c81..03013b5 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -32,7 +32,7 @@ }, "cmdline": { "type": "boolean", - "default": true + "default": false } }, "additionalProperties": false, diff --git a/unittests/data/systems/mirror-bad-key/mirrors.yaml b/unittests/data/systems/mirror-bad-key/mirrors.yaml index d5154dd..a3d0128 100644 --- a/unittests/data/systems/mirror-bad-key/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-key/mirrors.yaml @@ -1,3 +1,3 @@ -bad-key: +buildcache: url: https://mirror.spack.io - public_key: bad_key.gpg + private_key: bad_key.gpg diff --git a/unittests/data/systems/mirror-bad-keypath/mirrors.yaml b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml index 3433e04..5ac6af9 100644 --- a/unittests/data/systems/mirror-bad-keypath/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml @@ -1,3 +1,3 @@ -bad-key-path: +buildcache: url: https://mirror.spack.io - public_key: /path/doesnt/exist + private_key: /path/doesnt/exist diff --git a/unittests/data/systems/mirror-bad-url/mirrors.yaml b/unittests/data/systems/mirror-bad-url/mirrors.yaml index 8ffce33..c961d37 100644 --- a/unittests/data/systems/mirror-bad-url/mirrors.yaml +++ b/unittests/data/systems/mirror-bad-url/mirrors.yaml @@ -1,2 +1,4 @@ -bad-url: - url: https://www.testsite.io/services \ No newline at end of file +sourcecache: + bad-url: + url: https://www.testsite.io/services + enabled: true \ No newline at end of file diff --git a/unittests/data/systems/mirror-ok-raw-key/mirrors.yaml b/unittests/data/systems/mirror-ok-raw-key/mirrors.yaml new file mode 100644 index 0000000..e91cc7b --- /dev/null +++ b/unittests/data/systems/mirror-ok-raw-key/mirrors.yaml @@ -0,0 +1,59 @@ +bootstrap: + url: https://mirror.spack.io + enabled: true +buildcache: + url: https://mirror.spack.io + enabled: true + private_key: "\ + mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML\ + eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB\ + 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF\ + kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK\ + IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb\ + WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh\ + jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA\ + TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3\ + qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH\ + KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe\ + CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK\ + CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+\ + LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i\ + qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr\ + HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1\ + WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i\ + /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x\ + glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE\ + PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW\ + lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw\ + kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe\ + mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm\ + HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH\ + Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0\ + UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF\ + zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx\ + Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y\ + 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H\ + sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr\ + 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q\ + OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR\ + NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw\ + Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z\ + +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx\ + 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7\ + xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6\ + lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd\ + eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8\ + eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG\ + gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3" + mount_specific: false + cmdline: false +sourcecache: + mirror1: + url: https://github.com + enabled: true + mirror2: + url: https://github.com/spack + enabled: true + disabled-mirror: + url: https://github.com + enabled: false \ No newline at end of file diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index 0a1b843..f5bfa7f 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -1,55 +1,19 @@ -fake-mirror: - url: https://github.com - public_key: ../../test-gpg-pub.asc -disabled-mirror: - url: https://github.com - enabled: false -buildcache-mirror: +bootstrap: url: https://mirror.spack.io - private_key: '../test-gpg-priv.asc' - public_key: "\ - mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML\ - eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB\ - 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF\ - kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK\ - IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb\ - WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh\ - jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA\ - TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3\ - qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH\ - KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe\ - CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK\ - CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+\ - LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i\ - qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr\ - HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1\ - WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i\ - /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x\ - glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE\ - PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW\ - lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw\ - kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe\ - mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm\ - HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH\ - Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0\ - UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF\ - zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx\ - Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y\ - 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H\ - sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr\ - 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q\ - OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR\ - NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw\ - Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z\ - +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx\ - 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7\ - xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6\ - lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd\ - eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8\ - eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG\ - gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3" - - cache: true -bootstrap-mirror: + enabled: true +buildcache: url: https://mirror.spack.io - bootstrap: true + enabled: true + private_key: ../../test-gpg-priv.asc + mount_specific: false + cmdline: false +sourcecache: + mirror1: + url: https://github.com + enabled: true + mirror2: + url: https://github.com/spack + enabled: true + disabled-mirror: + url: https://github.com + enabled: false \ No newline at end of file diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index d262cd1..23ed4ca 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -21,42 +21,34 @@ def test_mirror_init(systems_path): mirrors_obj = mirror.Mirrors(path) valid_mirrors = { - "fake-mirror": { - "url": "https://github.com", + "bootstrap": { + "url": "https://mirror.spack.io", "enabled": True, - "bootstrap": False, - "cache": False, - "public_key": "../../test-gpg-pub.asc", - "mount_specific": False, }, - "buildcache-mirror": { + "buildcache": { "url": "https://mirror.spack.io", "enabled": True, - "bootstrap": False, - "cache": True, - "private_key": "../test-gpg-priv.asc", + "private_key": "../../test-gpg-priv.asc", "mount_specific": False, + "cmdline": False, }, - "bootstrap-mirror": { - "url": "https://mirror.spack.io", + "mirror1": { + "url": "https://github.com", "enabled": True, - "bootstrap": True, - "cache": False, - "mount_specific": False, }, + "mirror2": { + "url": "https://github.com/spack", + "enabled": True, + } } - with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: - key = base64.b64encode(pub_key_file.read()).decode() - valid_mirrors["buildcache-mirror"]["public_key"] = key + # with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: + # key = base64.b64encode(pub_key_file.read()).decode() + # valid_mirrors["buildcache"]["public_key"] = key assert mirrors_obj.mirrors == valid_mirrors - assert mirrors_obj.bootstrap_mirrors == [ - name for name in valid_mirrors.keys() if valid_mirrors[name].get("bootstrap") - ] - assert mirrors_obj.build_cache_mirror == [ - name for name in valid_mirrors.keys() if valid_mirrors[name].get("cache") - ].pop(0) + + assert mirrors_obj.build_cache_mirror == "buildcache" for mir in mirrors_obj.mirrors: assert mirrors_obj.mirrors[mir].get("enabled") @@ -90,11 +82,10 @@ def test_command_line_cache(systems_path): assert len(mirrors.mirrors) == 4 # This should always be the build cache even though one is already defined. assert mirrors.build_cache_mirror == "cmdline_cache" - cache_mirror = mirrors.mirrors["cmdline_cache"] + cache_mirror = mirrors.mirrors["buildcache"] assert cache_mirror["url"] == "/tmp/foo" assert cache_mirror["enabled"] - assert cache_mirror["cache"] - assert not cache_mirror["bootstrap"] + assert cache_mirror["cmdline"] assert cache_mirror["mount_specific"] @@ -103,18 +94,22 @@ def test_create_spack_mirrors_yaml(tmp_path, systems_path): valid_spack_yaml = { "mirrors": { - "fake-mirror": { - "fetch": {"url": "https://github.com"}, - "push": {"url": "https://github.com"}, - }, - "buildcache-mirror": { + "bootstrap": { "fetch": {"url": "https://mirror.spack.io"}, "push": {"url": "https://mirror.spack.io"}, }, - "bootstrap-mirror": { + "buildcache": { "fetch": {"url": "https://mirror.spack.io"}, "push": {"url": "https://mirror.spack.io"}, }, + "mirror1": { + "fetch": {"url": "https://github.com"}, + "push": {"url": "https://github.com"}, + }, + "mirror2": { + "fetch": {"url": "https://github.com/spack"}, + "push": {"url": "https://github.com/spack"}, + }, } } @@ -132,17 +127,21 @@ def test_create_bootstrap_configs(tmp_path, systems_path): """Check that spack bootstrap configs are generated correctly""" valid_yaml = { - "sources": [ - { - "name": "bootstrap-mirror", - "metadata": str(tmp_path / "bootstrap/bootstrap-mirror"), - } - ], - "trusted": {"bootstrap-mirror": True}, + "bootstrap": { + "sources": [ + { + "name": "bootstrap-mirror", + "metadata": str(tmp_path / "bootstrap/bootstrap-mirror"), + } + ], + "trusted": {"bootstrap-mirror": True}, + } } valid_metadata = { "type": "install", - "info": "https://mirror.spack.io", + "info": { + "url": "https://mirror.spack.io", + } } mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok") @@ -162,18 +161,26 @@ def test_create_bootstrap_configs(tmp_path, systems_path): def test_key_setup(systems_path, tmp_path): """Check that public keys are set up properly.""" - mirrors = mirror.Mirrors(systems_path / "mirror-ok") + mirrors_key_file = mirror.Mirrors(systems_path / "mirror-ok") + key_dir = tmp_path / "key_dir" + mirrors_raw_key = mirror.Mirrors(systems_path / "mirror-ok-raw-key") + raw_dir = tmp_path / "raw_dir" + + mirrors_key_file._key_setup(key_dir) + mirrors_raw_key._key_setup(raw_dir) + + key_file, = (p for p in key_dir.iterdir() if p.is_file()) + assert key_file.name == "buildcache.pgp" - mirrors._key_setup(tmp_path) + raw_key_file, = (p for p in key_dir.iterdir() if p.is_file()) + assert raw_key_file.name == "buildcache.pgp" - key_files = list(tmp_path.iterdir()) - assert {key_file.name for key_file in key_files} == {"buildcache-mirror.gpg", "fake-mirror.gpg"} # The two files should be identical in content - key_file_data = [] - for key_file in key_files: - with key_file.open("rb") as file: - key_file_data.append(file.read()) - assert key_file_data[0] == key_file_data[1] + with key_file.open("rb") as file: + key_file_data = file.read() + with raw_key_file.open("rb") as file: + raw_key_file_data = file.read() + assert key_file_data == raw_key_file_data @pytest.mark.parametrize( @@ -184,8 +191,7 @@ def test_key_setup(systems_path, tmp_path): ], ) def test_key_setup_bad_key(tmp_path, systems_path, system_name): - """asdfasdf""" + """Check that MirrorError is raised for bad keys""" - mirrors = mirror.Mirrors(systems_path / system_name) with pytest.raises(mirror.MirrorError): - mirrors._key_setup(tmp_path) + mirrors = mirror.Mirrors(systems_path / system_name) From 17f4bc6a24c942be5126b065feb9719737a88bd3 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 22 Apr 2026 23:03:36 -0600 Subject: [PATCH 66/69] reverted to old key_setup, add handling for private keys, refactored tests --- stackinator/mirror.py | 90 +++++++++---------- stackinator/schema/mirror.json | 6 +- .../systems/mirror-ok-raw-key/mirrors.yaml | 59 ------------ unittests/data/systems/mirror-ok/mirrors.yaml | 42 +++++++++ unittests/test_mirrors.py | 38 ++++---- 5 files changed, 106 insertions(+), 129 deletions(-) delete mode 100644 unittests/data/systems/mirror-ok-raw-key/mirrors.yaml diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 5219af0..7ff7bb8 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -39,8 +39,7 @@ def __init__( self._check_mirrors() # Will hold a list of all the gpg keys (public and private) - # self._keys: Optional[List[pathlib.Path]] = [] - self._keys = self._key_init() + self._keys: Optional[List[pathlib.Path]] = [] def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict]: """Load the mirrors file, if one exists.""" @@ -156,12 +155,14 @@ def _check_mirrors(self): f"Check the url listed in mirrors.yaml in system config. \n{err}" ) - def key_files(self, config_root: pathlib.Path): + @property + def keys(self): """Return the list of public and private key file paths.""" + if self._keys is None: raise RuntimeError("The mirror.keys method was accessed before setup_configs() was called.") - key_dir = config_root / self.KEY_STORE_DIR - return [key_dir / info["path"] for info in self._keys] + + return self._keys def setup_configs(self, config_root: pathlib.Path): """Setup all mirror configs in the given config_root.""" @@ -230,67 +231,64 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): with (config_root / "bootstrap.yaml").open("w") as file: yaml.dump(bootstrap_yaml, file, default_flow_style=False) - def _key_init(self): - key_info = {} - - key = self.mirrors["buildcache"].get("private_key") + def _load_key(self, key: str, dest: pathlib.Path, name: str): + """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" - if key is None: - return + # key will be saved under key_store/mirror_name.[pub/priv].gpg # if path, check if abs path, if not, append sys config path in front and check again path = pathlib.Path(os.path.expandvars(key)) - if not path.is_absolute(): # try prepending system config path path = self._system_config_root / path if path.is_file(): - # use the user-provided file - key_info = {"path": pathlib.Path(f"buildcache.pgp"), "source": path} + with open(path, "rb") as reader: + binary_key = reader.read() + + # convert base64 key to binary else: - # convert base64 key to binary try: - binary_key = base64.b64decode(key, validate=True) - print(binary_key) + binary_key = base64.b64decode(key) except ValueError: raise MirrorError( - f"Key for mirror 'buildcache' is not valid. \n" + f"Key for mirror '{name}' is not valid: '{path}'. \n" f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" f"Check the key listed in mirrors.yaml in system config." ) - file_type = magic.from_buffer(binary_key, mime=True) - if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys"): - raise MirrorError( - f"Key for mirror 'buildcache' is not a valid GPG key. \n" - f"The file (or base64) was readable, but the data itself was not a PGP key.\n" - f"Check the key listed in mirrors.yaml in system config." - ) - - key_info = {"path": pathlib.Path("buildcache.pgp"), "source": binary_key} + # private keys will evaluate as "application/octet-stream" + file_type = magic.from_buffer(binary_key, mime=True) + if file_type not in ("application/x-gnupg-keyring", "application/pgp-keys", "application/octet-stream"): + raise MirrorError( + f"Key for mirror {name} is not a valid GPG key. \n" + f"The file (or base64) was readable, but the data itself was not a PGP key.\n" + f"Check the key listed in mirrors.yaml in system config." + ) - return key_info + # copy key to new destination in key store + with open(dest, "wb") as writer: + writer.write(binary_key) + self._keys.append(dest) + + def _key_setup(self, key_store: pathlib.Path): - """Validate mirror keys, relocate to key_store, and update mirror config with new key paths.""" + """Iterate through mirror keys and load + relocate each one to key_store""" + self._keys = [] key_store.mkdir(exist_ok=True) - #for key_info in self._keys: - - #path = key_store / key_info["path"] - path = key_store / self._keys["path"] - #source = key_info["source"] - source = self._keys["source"] - - match source: - case pathlib.Path(): - # copy source -> path - shutil.copy2(source, path) - case bytes(): - # open path and copy in bytes - with open(path, "wb") as writer: - writer.write(source) - case _: - raise TypeError(f"Expected Path or bytes, got {type(source).__name__}") + for name, mirror in self.mirrors.items(): + if name == "buildcache": + if mirror.get("private_key"): + key = mirror["private_key"] + dest = pathlib.Path(key_store / f"{name}.priv.gpg") + self._load_key(key, dest, name) + + if mirror.get("public_key") is None: + continue + + key = mirror["public_key"] + dest = pathlib.Path(key_store / f"{name}.pub.gpg") + self._load_key(key, dest, name) diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index 03013b5..608dfe1 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -10,7 +10,8 @@ "enabled": { "type": "boolean", "default": true - } + }, + "public_key": {"type": "string"} }, "additionalProperties": false, "required": ["url"] @@ -48,7 +49,8 @@ "enabled": { "type": "boolean", "default": true - } + }, + "public_key": {"type": "string"} }, "additionalProperties": false, "required": ["url"] diff --git a/unittests/data/systems/mirror-ok-raw-key/mirrors.yaml b/unittests/data/systems/mirror-ok-raw-key/mirrors.yaml deleted file mode 100644 index e91cc7b..0000000 --- a/unittests/data/systems/mirror-ok-raw-key/mirrors.yaml +++ /dev/null @@ -1,59 +0,0 @@ -bootstrap: - url: https://mirror.spack.io - enabled: true -buildcache: - url: https://mirror.spack.io - enabled: true - private_key: "\ - mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML\ - eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB\ - 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF\ - kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK\ - IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb\ - WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh\ - jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA\ - TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3\ - qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH\ - KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe\ - CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK\ - CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+\ - LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i\ - qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr\ - HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1\ - WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i\ - /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x\ - glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE\ - PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW\ - lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw\ - kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe\ - mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm\ - HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH\ - Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0\ - UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF\ - zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx\ - Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y\ - 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H\ - sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr\ - 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q\ - OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR\ - NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw\ - Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z\ - +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx\ - 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7\ - xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6\ - lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd\ - eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8\ - eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG\ - gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3" - mount_specific: false - cmdline: false -sourcecache: - mirror1: - url: https://github.com - enabled: true - mirror2: - url: https://github.com/spack - enabled: true - disabled-mirror: - url: https://github.com - enabled: false \ No newline at end of file diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml index f5bfa7f..d4bc355 100644 --- a/unittests/data/systems/mirror-ok/mirrors.yaml +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -4,6 +4,47 @@ bootstrap: buildcache: url: https://mirror.spack.io enabled: true + public_key: "\ + mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML\ + eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB\ + 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF\ + kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK\ + IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb\ + WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh\ + jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA\ + TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3\ + qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH\ + KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe\ + CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK\ + CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+\ + LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i\ + qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr\ + HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1\ + WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i\ + /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x\ + glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE\ + PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW\ + lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw\ + kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe\ + mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm\ + HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH\ + Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0\ + UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF\ + zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx\ + Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y\ + 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H\ + sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr\ + 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q\ + OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR\ + NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw\ + Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z\ + +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx\ + 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7\ + xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6\ + lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd\ + eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8\ + eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG\ + gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3" private_key: ../../test-gpg-priv.asc mount_specific: false cmdline: false @@ -11,6 +52,7 @@ sourcecache: mirror1: url: https://github.com enabled: true + public_key: ../../test-gpg-pub.asc mirror2: url: https://github.com/spack enabled: true diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index 23ed4ca..e0ebb4a 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -35,6 +35,7 @@ def test_mirror_init(systems_path): "mirror1": { "url": "https://github.com", "enabled": True, + "public_key": "../../test-gpg-pub.asc" }, "mirror2": { "url": "https://github.com/spack", @@ -42,9 +43,9 @@ def test_mirror_init(systems_path): } } - # with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: - # key = base64.b64encode(pub_key_file.read()).decode() - # valid_mirrors["buildcache"]["public_key"] = key + with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: + key = base64.b64encode(pub_key_file.read()).decode() + valid_mirrors["buildcache"]["public_key"] = key assert mirrors_obj.mirrors == valid_mirrors @@ -149,8 +150,7 @@ def test_create_bootstrap_configs(tmp_path, systems_path): with (tmp_path / "bootstrap.yaml").open() as f: bs_data = yaml.safe_load(f) - print(bs_data) - print(valid_yaml) + assert bs_data == valid_yaml with (tmp_path / "bootstrap/bootstrap-mirror/metadata.yaml").open() as f: @@ -161,26 +161,19 @@ def test_create_bootstrap_configs(tmp_path, systems_path): def test_key_setup(systems_path, tmp_path): """Check that public keys are set up properly.""" - mirrors_key_file = mirror.Mirrors(systems_path / "mirror-ok") - key_dir = tmp_path / "key_dir" - mirrors_raw_key = mirror.Mirrors(systems_path / "mirror-ok-raw-key") - raw_dir = tmp_path / "raw_dir" - - mirrors_key_file._key_setup(key_dir) - mirrors_raw_key._key_setup(raw_dir) + mirrors = mirror.Mirrors(systems_path / "mirror-ok") - key_file, = (p for p in key_dir.iterdir() if p.is_file()) - assert key_file.name == "buildcache.pgp" + mirrors._key_setup(tmp_path) - raw_key_file, = (p for p in key_dir.iterdir() if p.is_file()) - assert raw_key_file.name == "buildcache.pgp" + pub_files = sorted(f for f in tmp_path.iterdir() if f.name.endswith(".pub.gpg")) + assert {pub_file.name for pub_file in pub_files} == {"buildcache.pub.gpg", "mirror1.pub.gpg"} # The two files should be identical in content - with key_file.open("rb") as file: - key_file_data = file.read() - with raw_key_file.open("rb") as file: - raw_key_file_data = file.read() - assert key_file_data == raw_key_file_data + pub_file_data = [] + for pub_file in pub_files: + with pub_file.open("rb") as file: + pub_file_data.append(file.read()) + assert pub_file_data[0] == pub_file_data[1] @pytest.mark.parametrize( @@ -193,5 +186,6 @@ def test_key_setup(systems_path, tmp_path): def test_key_setup_bad_key(tmp_path, systems_path, system_name): """Check that MirrorError is raised for bad keys""" + mirrors = mirror.Mirrors(systems_path / system_name) with pytest.raises(mirror.MirrorError): - mirrors = mirror.Mirrors(systems_path / system_name) + mirrors._key_setup(tmp_path) From 4206e68103388abf35171d9e4a3456391c6f9223 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 22 Apr 2026 23:11:42 -0600 Subject: [PATCH 67/69] linting --- stackinator/mirror.py | 17 +++++++---------- stackinator/recipe.py | 3 +-- unittests/test_mirrors.py | 10 +++------- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 7ff7bb8..d00cd82 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,4 +1,4 @@ -from typing import Optional, Dict +from typing import Optional, List, Dict import base64 import io import os @@ -64,9 +64,7 @@ def _load_mirrors(self, cmdline_cache: Optional[pathlib.Path]) -> Dict[str, Dict buildcache = mirrors.get("buildcache") if buildcache and buildcache.get("enabled", True): if not buildcache.get("private_key"): - raise MirrorError( - "Mirror build cache config is missing a required 'private_key' path." - ) + raise MirrorError("Mirror build cache config is missing a required 'private_key' path.") self.build_cache_mirror = "buildcache" enabled_mirrors["buildcache"] = buildcache else: @@ -120,7 +118,7 @@ def _load_cmdline_cache(self, cache_config_path: pathlib.Path) -> Dict: "enabled": True, "mount_specific": True, "private_key": raw["key"], - "cmdline": True + "cmdline": True, } self._logger.warning( @@ -180,7 +178,7 @@ def _create_spack_mirrors_yaml(self, dest: pathlib.Path): url = mirror["url"] # Make the mirror path specific to the mount point - if(name=="buildcache"): + if name == "buildcache": if mirror["mount_specific"] and self._mount_point is not None: url = url.rstrip("/") + "/" + self._mount_point.as_posix().lstrip("/") @@ -223,8 +221,8 @@ def _create_bootstrap_configs(self, config_root: pathlib.Path): "type": "install", "info": { "url": mirror["url"], - } - } + }, + } with (bs_mirror_path / "metadata.yaml").open("w") as file: yaml.dump(bs_mirror_yaml, file, default_flow_style=False) @@ -271,8 +269,7 @@ def _load_key(self, key: str, dest: pathlib.Path, name: str): writer.write(binary_key) self._keys.append(dest) - - + def _key_setup(self, key_store: pathlib.Path): """Iterate through mirror keys and load + relocate each one to key_store""" diff --git a/stackinator/recipe.py b/stackinator/recipe.py index e63165a..0d64059 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -172,8 +172,7 @@ def __init__(self, args): # load the optional mirrors.yaml from system config, and add any additional # mirrors specified on the command line. self._logger.debug("Configuring mirrors.") - self.mirrors = mirror.Mirrors(self.system_config_path, - pathlib.Path(args.cache) if args.cache else None) + self.mirrors = mirror.Mirrors(self.system_config_path, pathlib.Path(args.cache) if args.cache else None) # optional post install hook if self.post_install_hook is not None: diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py index e0ebb4a..7fb216d 100644 --- a/unittests/test_mirrors.py +++ b/unittests/test_mirrors.py @@ -32,15 +32,11 @@ def test_mirror_init(systems_path): "mount_specific": False, "cmdline": False, }, - "mirror1": { - "url": "https://github.com", - "enabled": True, - "public_key": "../../test-gpg-pub.asc" - }, + "mirror1": {"url": "https://github.com", "enabled": True, "public_key": "../../test-gpg-pub.asc"}, "mirror2": { "url": "https://github.com/spack", "enabled": True, - } + }, } with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: @@ -142,7 +138,7 @@ def test_create_bootstrap_configs(tmp_path, systems_path): "type": "install", "info": { "url": "https://mirror.spack.io", - } + }, } mirrors_obj = mirror.Mirrors(systems_path / "mirror-ok") From fa7ef2f266163fc2f5be9b6f467f2254e44dfdbc Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Thu, 23 Apr 2026 11:44:32 -0600 Subject: [PATCH 68/69] added 'requests' to pyproject.toml --- bin/stack-config | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/bin/stack-config b/bin/stack-config index 02f4870..08e90d7 100755 --- a/bin/stack-config +++ b/bin/stack-config @@ -2,6 +2,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ +# "requests", # "python-magic", # "jinja2", # "jsonschema", diff --git a/pyproject.toml b/pyproject.toml index 25fd1dd..4c36ae7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ license-files = ["LICENSE"] dynamic = ["version"] requires-python = ">=3.12" dependencies = [ + "requests", "python-magic", "Jinja2", "jsonschema", From 41e7630aa5753ff3db73ec7a8e9a34f87414e9f7 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Mon, 27 Apr 2026 12:43:00 -0600 Subject: [PATCH 69/69] removed old reference --- stackinator/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 2dc6923..b076694 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -234,7 +234,7 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - gpg_keys=recipe.mirrors.key_files(config_path), + gpg_keys=recipe.mirrors.keys, cache=recipe.build_cache_mirror, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False,