# Copyright Spack Project Developers. See COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""This module implements Spack's configuration file handling.
This implements Spack's configuration system, which handles merging
multiple scopes with different levels of precedence. See the
documentation on :ref:`configuration-scopes` for details on how Spack's
configuration system behaves. The scopes set up here are:
#. ``spack`` in ``$spack/etc/spack`` - controls all built-in spack scopes,
except default
#. ``defaults`` in ``$spack/etc/spack/defaults`` - defaults that Spack
needs to function
Important functions in this module are:
* :func:`~spack.config.Configuration.get_config`
* :func:`~spack.config.Configuration.update_config`
``get_config`` reads in YAML data for a particular scope and returns
it. Callers can then modify the data and write it back with
``update_config``.
When read in, Spack validates configurations with jsonschemas. The
schemas are in submodules of :py:mod:`spack.schema`.
"""
import contextlib
import copy
import functools
import os
import os.path
import pathlib
import re
import sys
import tempfile
from collections import defaultdict
from itertools import chain
from typing import Any, Callable, Dict, Generator, List, Optional, Set, Tuple, Union, cast
from spack.vendor import jsonschema
import spack.error
import spack.paths
import spack.schema
import spack.schema.bootstrap
import spack.schema.cdash
import spack.schema.ci
import spack.schema.compilers
import spack.schema.concretizer
import spack.schema.config
import spack.schema.definitions
import spack.schema.develop
import spack.schema.env
import spack.schema.env_vars
import spack.schema.include
import spack.schema.merged
import spack.schema.mirrors
import spack.schema.modules
import spack.schema.packages
import spack.schema.repos
import spack.schema.toolchains
import spack.schema.upstreams
import spack.schema.view
import spack.util.executable
import spack.util.git
import spack.util.hash
import spack.util.remote_file_cache as rfc_util
import spack.util.spack_json as sjson
import spack.util.spack_yaml as syaml
from spack.llnl.util import filesystem, lang, tty
from spack.util.cpus import cpus_available
from spack.util.spack_yaml import get_mark_from_yaml_data
from .enums import ConfigScopePriority
#: Dict from section names -> schema for that section
SECTION_SCHEMAS: Dict[str, Any] = {
"compilers": spack.schema.compilers.schema,
"concretizer": spack.schema.concretizer.schema,
"definitions": spack.schema.definitions.schema,
"env_vars": spack.schema.env_vars.schema,
"include": spack.schema.include.schema,
"view": spack.schema.view.schema,
"develop": spack.schema.develop.schema,
"mirrors": spack.schema.mirrors.schema,
"repos": spack.schema.repos.schema,
"packages": spack.schema.packages.schema,
"modules": spack.schema.modules.schema,
"config": spack.schema.config.schema,
"upstreams": spack.schema.upstreams.schema,
"bootstrap": spack.schema.bootstrap.schema,
"ci": spack.schema.ci.schema,
"cdash": spack.schema.cdash.schema,
"toolchains": spack.schema.toolchains.schema,
}
# Same as above, but including keys for environments
# this allows us to unify config reading between configs and environments
_ALL_SCHEMAS: Dict[str, Any] = {
**SECTION_SCHEMAS,
spack.schema.env.TOP_LEVEL_KEY: spack.schema.env.schema,
}
#: Path to the main configuration scope
CONFIGURATION_DEFAULTS_PATH = ("defaults", os.path.join(spack.paths.etc_path, "defaults"))
#: Hard-coded default values for some key configuration options.
#: This ensures that Spack will still work even if config.yaml in
#: the defaults scope is removed.
CONFIG_DEFAULTS = {
"config": {
"debug": False,
"connect_timeout": 10,
"verify_ssl": True,
"checksum": True,
"dirty": False,
"build_jobs": min(16, cpus_available()),
"build_stage": "$tempdir/spack-stage",
"license_dir": spack.paths.default_license_dir,
},
"concretizer": {"externals": {"completion": "default_variants"}},
}
#: metavar to use for commands that accept scopes
#: this is shorter and more readable than listing all choices
SCOPES_METAVAR = "{defaults,system,site,user,command_line} or env:ENVIRONMENT"
#: Base name for the (internal) overrides scope.
_OVERRIDES_BASE_NAME = "overrides-"
#: Type used for raw YAML configuration
YamlConfigDict = Dict[str, Any]
#: safeguard for recursive includes -- maximum include depth
MAX_RECURSIVE_INCLUDES = 100
[docs]
class ConfigScope:
def __init__(self, name: str, included: bool = False) -> None:
self.name = name
self.writable = False
self.sections = syaml.syaml_dict()
self.prefer_modify = False
self.included = included
#: included configuration scopes
self._included_scopes: Optional[List["ConfigScope"]] = None
@property
def included_scopes(self) -> List["ConfigScope"]:
"""Memoized list of included scopes, in the order they appear in this scope."""
if self._included_scopes is None:
self._included_scopes = []
includes = self.get_section("include")
if includes:
include_paths = [included_path(data) for data in includes["include"]]
included_scopes = chain(*[include.scopes(self) for include in include_paths])
# Do not include duplicate scopes
for included_scope in included_scopes:
if any([included_scope.name == scope.name for scope in self._included_scopes]):
tty.warn(f"Ignoring duplicate included scope: {included_scope.name}")
continue
if included_scope not in self._included_scopes:
self._included_scopes.append(included_scope)
return self._included_scopes
@property
def exists(self) -> bool:
"""Whether the config object indicated by the scope can be read"""
return True
[docs]
def override_include(self):
"""Whether the ``include::`` section of this scope should override lower scopes."""
include = self.sections.get("include")
if not include:
return False
# override if this has an include section and there is an override attribute on
# the include key in the dict and it is set to True.
return getattr(next(iter(include.keys()), None), "override", False)
[docs]
def transitive_includes(self, _names: Optional[Set[str]] = None) -> Set[str]:
"""Get name of this scope and names of its transitively included scopes."""
if _names is None:
_names = _set()
_names.add(self.name)
for scope in self.included_scopes:
_names |= scope.transitive_includes(_names=_names)
return _names
[docs]
def get_section_filename(self, section: str) -> str:
raise NotImplementedError
[docs]
def get_section(self, section: str) -> Optional[YamlConfigDict]:
raise NotImplementedError
def _write_section(self, section: str) -> None:
raise NotImplementedError
[docs]
def clear(self) -> None:
"""Empty cached config information."""
self.sections = syaml.syaml_dict()
def __repr__(self) -> str:
return f"<ConfigScope: {self.name}>"
[docs]
class DirectoryConfigScope(ConfigScope):
"""Config scope backed by a directory containing one file per section."""
def __init__(
self,
name: str,
path: str,
*,
writable: bool = True,
prefer_modify: bool = True,
included: bool = False,
) -> None:
super().__init__(name, included)
self.path = path
self.writable = writable
self.prefer_modify = prefer_modify
@property
def exists(self) -> bool:
return os.path.exists(self.path)
[docs]
def get_section_filename(self, section: str) -> str:
"""Returns the filename associated with a given section"""
_validate_section_name(section)
return os.path.join(self.path, f"{section}.yaml")
[docs]
def get_section(self, section: str) -> Optional[YamlConfigDict]:
"""Returns the data associated with a given section if the scope exists"""
if not self.exists:
tty.debug(f"Attempting to read from missing scope: {self} at {self.path}")
return {}
return self._get_section(section)
def _get_section(self, section: str) -> Optional[YamlConfigDict]:
"""get_section but without the existence check"""
if section not in self.sections:
path = self.get_section_filename(section)
schema = SECTION_SCHEMAS[section]
data = read_config_file(path, schema)
self.sections[section] = data
return self.sections[section]
def _write_section(self, section: str) -> None:
if not self.writable:
raise spack.error.ConfigError(f"Cannot write to immutable scope {self}")
filename = self.get_section_filename(section)
data = self._get_section(section)
if data is None:
return
validate(data, SECTION_SCHEMAS[section])
try:
filesystem.mkdirp(self.path)
fd, tmp = tempfile.mkstemp(dir=self.path, suffix=".tmp")
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
syaml.dump_config(data, stream=f, default_flow_style=False)
filesystem.rename(tmp, filename)
except Exception:
os.unlink(tmp)
raise
except (syaml.SpackYAMLError, OSError) as e:
raise ConfigFileError(f"cannot write to '{filename}'") from e
[docs]
class SingleFileScope(ConfigScope):
"""This class represents a configuration scope in a single YAML file."""
def __init__(
self,
name: str,
path: str,
schema: YamlConfigDict,
*,
yaml_path: Optional[List[str]] = None,
writable: bool = True,
prefer_modify: bool = True,
included: bool = False,
) -> None:
"""Similar to ``ConfigScope`` but can be embedded in another schema.
Arguments:
schema (dict): jsonschema for the file to read
yaml_path (list): path in the schema where config data can be
found.
If the schema accepts the following yaml data, the yaml_path
would be ['outer', 'inner']
.. code-block:: yaml
outer:
inner:
config:
install_tree: $spack/opt/spack
"""
super().__init__(name, included)
self._raw_data: Optional[YamlConfigDict] = None
self.schema = schema
self.path = path
self.writable = writable
self.prefer_modify = prefer_modify
self.yaml_path = yaml_path or []
@property
def exists(self) -> bool:
return os.path.exists(self.path)
[docs]
def get_section_filename(self, section) -> str:
return self.path
[docs]
def get_section(self, section: str) -> Optional[YamlConfigDict]:
# read raw data from the file, which looks like:
# {
# 'config': {
# ... data ...
# },
# 'packages': {
# ... data ...
# },
# }
#
# To preserve overrides up to the section level (e.g. to override
# the "packages" section with the "::" syntax), data in self.sections
# looks like this:
# {
# 'config': {
# 'config': {
# ... data ...
# }
# },
# 'packages': {
# 'packages': {
# ... data ...
# }
# }
# }
if not self.exists:
tty.debug(f"Attempting to read from missing scope: {self} at {self.path}")
return {}
# This bit ensures we have read the file and have
# the raw data in memory
if self._raw_data is None:
self._raw_data = read_config_file(self.path, self.schema)
if self._raw_data is None:
return None
# Here we know we have the raw data and ensure we
# populate the sections dictionary, which may be
# cleared by the clear() method
if not self.sections:
section_data = self._raw_data
for key in self.yaml_path:
if section_data is None:
return None
section_data = section_data[key]
for section_key, data in section_data.items():
self.sections[section_key] = {section_key: data}
return self.sections.get(section, None)
def _write_section(self, section: str) -> None:
if not self.writable:
raise spack.error.ConfigError(f"Cannot write to immutable scope {self}")
data_to_write: Optional[YamlConfigDict] = self._raw_data
# If there is no existing data, this section SingleFileScope has never
# been written to disk. We need to construct the portion of the data
# from the root of self._raw_data to the level at which the config
# sections are defined. That requires creating keys for every entry in
# self.yaml_path
if not data_to_write:
data_to_write = {}
# reverse because we construct it from the inside out
for key in reversed(self.yaml_path):
data_to_write = {key: data_to_write}
# data_update_pointer is a pointer to the part of data_to_write
# that we are currently updating.
# We start by traversing into the data to the point at which the
# config sections are defined. This means popping the keys from
# self.yaml_path
data_update_pointer = data_to_write
for key in self.yaml_path:
data_update_pointer = data_update_pointer[key]
# For each section, update the data at the level of our pointer
# with the data from the section
for key, data in self.sections.items():
data_update_pointer[key] = data[key]
validate(data_to_write, self.schema)
try:
parent = os.path.dirname(self.path)
filesystem.mkdirp(parent)
fd, tmp = tempfile.mkstemp(dir=parent, suffix=".tmp")
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
syaml.dump_config(data_to_write, stream=f, default_flow_style=False)
filesystem.rename(tmp, self.path)
except Exception:
os.unlink(tmp)
raise
except (syaml.SpackYAMLError, OSError) as e:
raise ConfigFileError(f"cannot write to config file {str(e)}") from e
def __repr__(self) -> str:
return f"<SingleFileScope: {self.name}: {self.path}>"
[docs]
class InternalConfigScope(ConfigScope):
"""An internal configuration scope that is not persisted to a file.
This is for spack internal use so that command-line options and
config file settings are accessed the same way, and Spack can easily
override settings from files.
"""
def __init__(self, name: str, data: Optional[YamlConfigDict] = None) -> None:
super().__init__(name)
self.sections = syaml.syaml_dict()
if data is not None:
data = InternalConfigScope._process_dict_keyname_overrides(data)
for section in data:
dsec = data[section]
validate({section: dsec}, SECTION_SCHEMAS[section])
self.sections[section] = _mark_internal(syaml.syaml_dict({section: dsec}), name)
[docs]
def get_section(self, section: str) -> Optional[YamlConfigDict]:
"""Just reads from an internal dictionary."""
if section not in self.sections:
self.sections[section] = None
return self.sections[section]
def _write_section(self, section: str) -> None:
"""This only validates, as the data is already in memory."""
data = self.get_section(section)
if data is not None:
validate(data, SECTION_SCHEMAS[section])
self.sections[section] = _mark_internal(data, self.name)
def __repr__(self) -> str:
return f"<InternalConfigScope: {self.name}>"
[docs]
def clear(self) -> None:
# no cache to clear here.
pass
@staticmethod
def _process_dict_keyname_overrides(data: YamlConfigDict) -> YamlConfigDict:
"""Turn a trailing `:' in a key name into an override attribute."""
# Below we have a lot of type directives, since we hack on types and monkey-patch them
# by adding attributes that otherwise they won't have.
result: YamlConfigDict = {}
for sk, sv in data.items():
if sk.endswith(":"):
key = syaml.syaml_str(sk[:-1])
key.override = True # type: ignore[attr-defined]
elif sk.endswith("+"):
key = syaml.syaml_str(sk[:-1])
key.prepend = True # type: ignore[attr-defined]
elif sk.endswith("-"):
key = syaml.syaml_str(sk[:-1])
key.append = True # type: ignore[attr-defined]
else:
key = sk # type: ignore[assignment]
if isinstance(sv, dict):
result[key] = InternalConfigScope._process_dict_keyname_overrides(sv)
else:
result[key] = copy.copy(sv)
return result
def _config_mutator(method):
"""Decorator to mark all the methods in the Configuration class
that mutate the underlying configuration. Used to clear the
memoization cache.
"""
@functools.wraps(method)
def _method(self, *args, **kwargs):
self._get_config_memoized.cache_clear()
return method(self, *args, **kwargs)
return _method
ScopeWithOptionalPriority = Union[ConfigScope, Tuple[int, ConfigScope]]
ScopeWithPriority = Tuple[int, ConfigScope]
[docs]
class Configuration:
"""A hierarchical configuration, merging a number of scopes at different priorities."""
# convert to typing.OrderedDict when we drop 3.6, or OrderedDict when we reach 3.9
scopes: lang.PriorityOrderedMapping[str, ConfigScope]
def __init__(self) -> None:
self.scopes = lang.PriorityOrderedMapping()
self.updated_scopes_by_section: Dict[str, List[ConfigScope]] = defaultdict(list)
[docs]
def ensure_unwrapped(self) -> "Configuration":
"""Ensure we unwrap this object from any dynamic wrapper (like Singleton)"""
return self
[docs]
def highest(self) -> ConfigScope:
"""Scope with the highest precedence"""
return next(self.scopes.reversed_values()) # type: ignore
[docs]
@_config_mutator
def push_scope_incremental(
self, scope: ConfigScope, priority: Optional[int] = None, _depth: int = 0
) -> Generator["Configuration", None, None]:
"""Adds a scope to the Configuration, at a given priority.
``push_scope_incremental`` yields included scopes incrementally, so that their
data can be used by higher priority scopes during config initialization. If you
push a scope that includes other, low-priority scopes, they will be pushed on
first, before the scope that included them.
If a priority is not given, it is assumed to be the current highest priority.
Args:
scope: scope to be added
priority: priority of the scope
"""
# TODO: As a follow on to #48784, change this to create a graph of the
# TODO: includes AND ensure properly sorted such that the order included
# TODO: at the highest level is reflected in the value of an option that
# TODO: is set in multiple included files.
# before pushing the scope itself, push included scopes recursively, at the same priority
for included_scope in reversed(scope.included_scopes):
if _depth + 1 > MAX_RECURSIVE_INCLUDES: # make sure we're not recursing endlessly
mark = ""
if hasattr(included_scope, "path") and syaml.marked(included_scope.path):
mark = included_scope.path._start_mark # type: ignore
raise RecursiveIncludeError(
f"Maximum include recursion exceeded in {included_scope.name}", str(mark)
)
# record this inclusion so that remove_scope() can use it
self.push_scope(included_scope, priority=priority, _depth=_depth + 1)
yield self
tty.debug(f"[CONFIGURATION: PUSH SCOPE]: {str(scope)}, priority={priority}", level=2)
self.scopes.add(scope.name, value=scope, priority=priority)
yield self
[docs]
@_config_mutator
def push_scope(
self, scope: ConfigScope, priority: Optional[int] = None, _depth: int = 0
) -> None:
"""Add a scope to the Configuration, at a given priority.
If a priority is not given, it is assumed to be the current highest priority.
Args:
scope: scope to be added
priority: priority of the scope
"""
# Use push_scope_incremental to do the real work. It returns a generator, which needs
# to be consumed to get each of the yielded scopes added to the scope stack.
# It will usually yield one scope, but if there are includes it will yield those first,
# before the scope we're actually pushing.
for _ in self.push_scope_incremental(scope=scope, priority=priority, _depth=_depth):
pass
[docs]
@_config_mutator
def remove_scope(self, scope_name: str) -> Optional[ConfigScope]:
"""Removes a scope by name, and returns it. If the scope does not exist, returns None."""
try:
scope = self.scopes.remove(scope_name)
tty.debug(f"[CONFIGURATION: REMOVE SCOPE]: {str(scope)}", level=2)
except KeyError as e:
tty.debug(f"[CONFIGURATION: REMOVE SCOPE]: {e}", level=2)
return None
# transitively remove included scopes
for included_scope in scope.included_scopes:
assert included_scope.name in self.scopes, (
f"Included scope '{included_scope.name}' was never added to configuration!"
)
self.remove_scope(included_scope.name)
return scope
@property
def writable_scopes(self) -> Generator[ConfigScope, None, None]:
"""Generator of writable scopes with an associated file."""
return (s for s in self.scopes.values() if s.writable)
@property
def existing_scopes(self) -> Generator[ConfigScope, None, None]:
"""Generator of existing scopes. These are self.scopes where the
scope has a representation on the filesystem or is internal"""
return (s for s in self.scopes.values() if s.exists)
[docs]
def highest_precedence_scope(self) -> ConfigScope:
"""Writable scope with the highest precedence."""
scope = next(s for s in self.scopes.reversed_values() if s.writable)
# if a scope prefers that we edit another, respect that.
while scope:
preferred = scope
scope = next(
(s for s in scope.included_scopes if s.writable and s.prefer_modify), None
)
return preferred
[docs]
def matching_scopes(self, reg_expr) -> List[ConfigScope]:
"""
List of all scopes whose names match the provided regular expression.
For example, ``matching_scopes(r'^command')`` will return all scopes
whose names begin with ``command``.
"""
return [s for s in self.scopes.values() if re.search(reg_expr, s.name)]
def _validate_scope(self, scope: Optional[str]) -> ConfigScope:
"""Ensure that scope is valid in this configuration.
This should be used by routines in ``config.py`` to validate
scope name arguments, and to determine a default scope where no
scope is specified.
Raises:
ValueError: if ``scope`` is not valid
Returns:
ConfigScope: a valid ConfigScope if ``scope`` is ``None`` or valid
"""
if scope is None:
# default to the scope with highest precedence.
return self.highest_precedence_scope()
elif scope in self.scopes:
return self.scopes[scope]
else:
raise ValueError(
f"Invalid config scope: '{scope}'. Must be one of "
f"{[k for k in self.scopes.keys()]}"
)
[docs]
def get_config_filename(self, scope: str, section: str) -> str:
"""For some scope and section, get the name of the configuration file."""
scope = self._validate_scope(scope)
return scope.get_section_filename(section)
[docs]
@_config_mutator
def clear_caches(self) -> None:
"""Clears the caches for configuration files,
This will cause files to be re-read upon the next request."""
for scope in self.scopes.values():
scope.clear()
[docs]
@_config_mutator
def update_config(
self, section: str, update_data: Dict, scope: Optional[str] = None, force: bool = False
) -> None:
"""Update the configuration file for a particular scope.
Overwrites contents of a section in a scope with update_data,
then writes out the config file.
update_data should have the top-level section name stripped off
(it will be re-added). Data itself can be a list, dict, or any
other yaml-ish structure.
Configuration scopes that are still written in an old schema
format will fail to update unless ``force`` is True.
Args:
section: section of the configuration to be updated
update_data: data to be used for the update
scope: scope to be updated
force: force the update
"""
if self.updated_scopes_by_section.get(section) and not force:
msg = (
'The "{0}" section of the configuration needs to be written'
" to disk, but is currently using a deprecated format. "
"Please update it using:\n\n"
"\tspack config [--scope=<scope>] update {0}\n\n"
"Note that previous versions of Spack will not be able to "
"use the updated configuration."
)
msg = msg.format(section)
raise RuntimeError(msg)
_validate_section_name(section) # validate section name
scope = self._validate_scope(scope) # get ConfigScope object
# manually preserve comments
need_comment_copy = section in scope.sections and scope.sections[section]
if need_comment_copy:
comments = syaml.extract_comments(scope.sections[section][section])
# read only the requested section's data.
scope.sections[section] = syaml.syaml_dict({section: update_data})
if need_comment_copy and comments:
syaml.set_comments(scope.sections[section][section], data_comments=comments)
scope._write_section(section)
[docs]
def get_config(
self, section: str, scope: Optional[str] = None, _merged_scope: Optional[str] = None
) -> YamlConfigDict:
"""Get configuration settings for a section.
If ``scope`` is ``None`` or not provided, return the merged contents
of all of Spack's configuration scopes. If ``scope`` is provided,
return only the configuration as specified in that scope.
This off the top-level name from the YAML section. That is, for a
YAML config file that looks like this::
config:
install_tree:
root: $spack/opt/spack
build_stage:
- $tmpdir/$user/spack-stage
``get_config('config')`` will return::
{ 'install_tree': {
'root': '$spack/opt/spack',
}
'build_stage': ['$tmpdir/$user/spack-stage']
}
"""
return self._get_config_memoized(section, scope=scope, _merged_scope=_merged_scope)
[docs]
def deepcopy_as_builtin(
self, section: str, scope: Optional[str] = None, *, line_info: bool = False
) -> Dict[str, Any]:
"""Get a deep copy of a section with native Python types, excluding YAML metadata."""
return syaml.deepcopy_as_builtin(
self.get_config(section, scope=scope), line_info=line_info
)
def _filter_overridden(self, scopes: List[ConfigScope], includes: bool = False):
"""Filter out overridden scopes.
NOTE: this does not yet handle diamonds or nested `include::` in lists. It is
sufficient for include::[] in an env, which allows isolation.
The ``includes`` option controls whether to return all active scopes (``includes=False``)
or all scopes whose includes have not been overridden (``includes=True``).
"""
# find last override in scopes
i = next((i for i, s in reversed(list(enumerate(scopes))) if s.override_include()), -1)
if i < 0:
return scopes # no overrides
keep = _set(s.name for s in scopes[i:])
keep |= _set(s.name for s in self.scopes.priority_values(ConfigScopePriority.DEFAULTS))
if not includes:
# For all sections except for the include section:
# non-included scopes are still active, as are scopes included
# from the overriding scope
# Transitive scopes from the overriding scope are not included
keep |= _set([s.name for s in scopes[i].included_scopes])
keep |= _set([s.name for s in scopes if not s.included])
# return scopes to keep, with order preserved
return [s for s in scopes if s.name in keep]
@property
def active_include_section_scopes(self) -> List[ConfigScope]:
"""Return a list of all scopes whose includes have not been overridden by include::.
This is different from the active scopes because the ``spack`` scope can be active
while its includes are overwritten, as can the transitive includes from the overriding
scope."""
return self._filter_overridden([s for s in self.scopes.values()], includes=True)
@property
def active_scopes(self) -> List[ConfigScope]:
"""Return a list of scopes that have not been overridden by include::."""
return self._filter_overridden([s for s in self.scopes.values()])
@lang.memoized
def _get_config_memoized(
self, section: str, scope: Optional[str], _merged_scope: Optional[str]
) -> YamlConfigDict:
"""Memoized helper for ``get_config()``.
Note that the memoization cache for this function is cleared whenever
any function decorated with ``@_config_mutator`` is called.
"""
_validate_section_name(section)
if scope is not None and _merged_scope is not None:
raise ValueError("Cannot specify both scope and _merged_scope")
elif scope is not None:
scopes = [self._validate_scope(scope)]
elif _merged_scope is not None:
scope_stack = list(self.scopes.values())
merge_idx = next(i for i, s in enumerate(scope_stack) if s.name == _merged_scope)
scopes = scope_stack[: merge_idx + 1]
else:
scopes = list(self.scopes.values())
# filter any scopes overridden by `include::`
scopes = self._filter_overridden(scopes)
merged_section: Dict[str, Any] = syaml.syaml_dict()
updated_scopes = []
for config_scope in scopes:
if section == "include" and config_scope not in self.active_include_section_scopes:
continue
# read potentially cached data from the scope.
data = config_scope.get_section(section)
if data and section == "include":
# Include overrides are handled by `_filter_overridden` above. Any remaining
# includes at this point are *not* actually overridden -- they're scopes with
# ConfigScopePriority.DEFAULT, which we currently do *not* remove with
# `include::`, because these scopes are needed for Spack to function correctly.
# So, we ignore :: here.
data = data.copy()
data["include"] = data.pop("include") # strip override
# Skip empty configs
if not isinstance(data, dict) or section not in data:
continue
# If configuration is in an old format, transform it and keep track of the scope that
# may need to be written out to disk.
if _update_in_memory(data, section):
updated_scopes.append(config_scope)
merged_section = spack.schema.merge_yaml(merged_section, data)
self.updated_scopes_by_section[section] = updated_scopes
# no config files -- empty config.
if section not in merged_section:
return syaml.syaml_dict()
# take the top key off before returning.
ret = merged_section[section]
if isinstance(ret, dict):
ret = syaml.syaml_dict(ret)
return ret
[docs]
def get(self, path: str, default: Optional[Any] = None, scope: Optional[str] = None) -> Any:
"""Get a config section or a single value from one.
Accepts a path syntax that allows us to grab nested config map
entries. Getting the ``config`` section would look like::
spack.config.get("config")
and the ``dirty`` section in the ``config`` scope would be::
spack.config.get("config:dirty")
We use ``:`` as the separator, like YAML objects.
"""
parts = process_config_path(path)
section = parts.pop(0)
value = self.get_config(section, scope=scope)
while parts:
key = parts.pop(0)
# cannot use value.get(key, default) in case there is another part
# and default is not a dict
if key not in value:
return default
value = value[key]
return value
[docs]
@_config_mutator
def set(self, path: str, value: Any, scope: Optional[str] = None) -> None:
"""Convenience function for setting single values in config files.
Accepts the path syntax described in ``get()``.
"""
if ":" not in path:
# handle bare section name as path
self.update_config(path, value, scope=scope)
return
parts = process_config_path(path)
section = parts.pop(0)
section_data = self.get_config(section, scope=scope)
data = section_data
while len(parts) > 1:
key = parts.pop(0)
if spack.schema.override(key):
new = type(data[key])()
del data[key]
else:
new = data[key]
if isinstance(new, dict):
# Make it an ordered dict
new = syaml.syaml_dict(new)
# reattach to parent object
data[key] = new
data = new
if spack.schema.override(parts[0]):
data.pop(parts[0], None)
# update new value
data[parts[0]] = value
self.update_config(section, section_data, scope=scope)
def __iter__(self):
"""Iterate over scopes in this configuration."""
yield from self.scopes.values()
[docs]
def print_section(
self, section: str, yaml: bool = True, blame: bool = False, *, scope: Optional[str] = None
) -> None:
"""Print a configuration to stdout.
Arguments:
section: The configuration section to print.
yaml: If True, output in YAML format, otherwise JSON (ignored when blame is True).
blame: Whether to include source locations for each entry.
scope: The configuration scope to use.
"""
try:
data = syaml.syaml_dict()
data[section] = self.get_config(section, scope=scope)
if yaml or blame:
syaml.dump_config(data, stream=sys.stdout, default_flow_style=False, blame=blame)
else:
sjson.dump(data, sys.stdout)
sys.stdout.write("\n")
except (syaml.SpackYAMLError, OSError) as e:
raise spack.error.ConfigError(f"cannot read '{section}' configuration") from e
[docs]
@contextlib.contextmanager
def override(
path_or_scope: Union[ConfigScope, str], value: Optional[Any] = None
) -> Generator[Configuration, None, None]:
"""Simple way to override config settings within a context.
Arguments:
path_or_scope (ConfigScope or str): scope or single option to override
value (object or None): value for the single option
Temporarily push a scope on the current configuration, then remove it
after the context completes. If a single option is provided, create
an internal config scope for it and push/pop that scope.
"""
if isinstance(path_or_scope, ConfigScope):
overrides = path_or_scope
CONFIG.push_scope(path_or_scope, priority=None)
else:
base_name = _OVERRIDES_BASE_NAME
# Ensure the new override gets a unique scope name
current_overrides = [s.name for s in CONFIG.matching_scopes(rf"^{base_name}")]
num_overrides = len(current_overrides)
while True:
scope_name = f"{base_name}{num_overrides}"
if scope_name in current_overrides:
num_overrides += 1
else:
break
overrides = InternalConfigScope(scope_name)
CONFIG.push_scope(overrides, priority=None)
CONFIG.set(path_or_scope, value, scope=scope_name)
try:
yield CONFIG
finally:
scope = CONFIG.remove_scope(overrides.name)
assert scope is overrides
#: Class for the relevance of an optional path conditioned on a limited
#: python code that evaluates to a boolean and or explicit specification
#: as optional.
[docs]
class OptionalInclude:
"""Base properties for all includes."""
name: str
when: str
optional: bool
prefer_modify: bool
remote: bool
_scopes: List[ConfigScope]
def __init__(self, entry: dict):
self.name = entry.get("name", "")
self.when = entry.get("when", "")
self.optional = entry.get("optional", False)
self.prefer_modify = entry.get("prefer_modify", False)
self.remote = False
self._scopes = []
@staticmethod
def _parent_scope_directory(parent_scope: Optional[ConfigScope]) -> Optional[str]:
"""Return the directory of the parent scope, or ``None`` if unavailable.
Normalizes ``SingleFileScope`` to its containing directory.
"""
path = getattr(parent_scope, "path", "") if parent_scope else ""
if not path:
return None
return os.path.dirname(path) if os.path.isfile(path) else path
[docs]
def base_directory(
self, path_or_url: str, parent_scope: Optional[ConfigScope] = None
) -> Optional[str]:
"""Return the local directory to use for this include.
For remote includes this is the cache destination directory.
For local relative includes this is the working directory from which to resolve the path.
Args:
path_or_url: path or URL of the include
parent_scope: including scope
Returns: ``None`` for a local include without an enclosing parent scope;
an appropriate subdirectory of the enclosing (parent) scope's writable
directory (when available); otherwise a stable temporary directory.
"""
scope_dir = self._parent_scope_directory(parent_scope)
if not self.remote:
return scope_dir
def _subdir():
# Prefer the provided include name over the git repository name.
# If neither, use a hash of the url or path for uniqueness.
if self.name:
return self.name
match = re.search(r"/([^/]+?)(\.git)?$", path_or_url)
if match:
if not os.path.splitext(match.group(1))[1]:
return match.group(1)
return spack.util.hash.b32_hash(path_or_url)[-7:]
# For remote includes, prefer a writable subdirectory of the parent scope.
if scope_dir and filesystem.can_write_to_dir(scope_dir):
assert parent_scope is not None
subdir = os.path.join("includes", _subdir())
if parent_scope.name.startswith("env:"):
subdir = os.path.join(".spack-env", subdir)
return os.path.join(scope_dir, subdir)
# Fall back to a stable, unique, temporary directory, logging the reason.
tmpdir = tempfile.gettempdir()
if path_or_url:
pre = self.name or getattr(parent_scope, "name", "")
subdir = f"{pre}:{path_or_url}" if pre else path_or_url
tmpdir = os.path.join(tmpdir, spack.util.hash.b32_hash(subdir)[-7:])
if not scope_dir:
tty.debug(f"No parent scope directory for include ({self}). Using {tmpdir}.")
else:
assert parent_scope is not None
tty.debug(
f"Parent scope {parent_scope.name}'s directory ({scope_dir}) is not writable. "
f"Using {tmpdir}."
)
return tmpdir
def _scope(
self, path: str, config_path: str, parent_scope: ConfigScope
) -> Optional[ConfigScope]:
"""Instantiate a configuration scope for the configuration path.
Args:
path: raw include path
config_path: configuration path
parent_scope: including scope
Returns: configuration scopes
Raises:
ValueError: the required configuration path does not exist
"""
# circular dependencies
import spack.util.path
# Ignore included concrete environment files (i.e., ``spack.lock``)
# since they are not normal configuration (scope) files and their
# processing is handled when the environment is processed.
if path and os.path.basename(path) == "spack.lock":
tty.debug(
f"Ignoring inclusion of '{path}' since environment lock files "
"are processed elsewhere"
)
return None
# Ensure the parent scope is valid
self._validate_parent_scope(parent_scope)
# Determine the configuration scope name
config_name = self.name or parent_scope.name
# But ensure that name is unique if there are multiple paths.
if not self.name or len(getattr(self, "paths", [])) > 1:
parent_path = pathlib.Path(getattr(parent_scope, "path", ""))
real_path = pathlib.Path(spack.util.path.substitute_path_variables(path))
try:
included_name = real_path.relative_to(parent_path)
except ValueError:
included_name = real_path
if sys.platform == "win32":
# Clean windows path for use in config name that looks nicer
# ie. The path: C:\\some\\path\\to\\a\\file
# becomes C/some/path/to/a/file
included_name = included_name.as_posix().replace(":", "")
config_name = f"{config_name}:{included_name}"
# Type | Extension | RESULT
# -------- | --------- | ---------
# missing | none | Directory
# missing | yaml | File
# missing | other | No scope
# directory | none/any | Directory
# file | yaml | File
# file | other | Error
exists = os.path.exists(config_path)
if not exists and not self.optional:
dest = f" at ({config_path})" if config_path != os.path.normpath(path) else ""
raise ValueError(f"Required path ({path}) does not exist{dest}")
_, ext = os.path.splitext(config_path)
if os.path.isdir(config_path) or not ext:
# directories are treated as regular ConfigScopes
tty.debug(f"Creating DirectoryConfigScope {config_name} for '{config_path}'")
return DirectoryConfigScope(
config_name, config_path, prefer_modify=self.prefer_modify, included=True
)
elif ext == ".yaml" or ext == ".yml":
tty.debug(f"Creating SingleFileScope {config_name} for '{config_path}'")
return SingleFileScope(
config_name,
config_path,
spack.schema.merged.schema,
prefer_modify=self.prefer_modify,
included=True,
)
elif exists:
raise ValueError(
f"Unsupported file-based scope: path ({path}) should have "
"a .yaml/.yml extension for file scopes, "
"or no extension for directory scopes"
)
# Nonexistent files without yaml extension are ignored
tty.debug(f"Ignoring missing config path ({path})")
return None
def _validate_parent_scope(self, parent_scope: ConfigScope):
"""Validates that a parent scope is a valid configuration object"""
# enforced by type checking but those can always be # type: ignore'd
assert isinstance(parent_scope, ConfigScope), (
f"Includes must be within a configuration scope (ConfigScope), not {type(parent_scope)}" # noqa: E501
)
assert parent_scope.name.strip(), "Parent scope of an include must have a name"
[docs]
def evaluate_condition(self) -> bool:
"""Evaluate the include condition:
Returns: ``True`` if the include condition is satisfied; else ``False``.
"""
# circular dependencies
import spack.spec
return (not self.when) or spack.spec.eval_conditional(self.when)
[docs]
def scopes(self, parent_scope: ConfigScope) -> List[ConfigScope]:
"""Instantiate configuration scopes.
Args:
parent_scope: including scope
Returns: configuration scopes for configuration files IF the when
condition is satisfied; otherwise, an empty list.
Raises:
ValueError: the required configuration path does not exist
"""
raise NotImplementedError("must be implemented in derived classes")
@property
def paths(self) -> List[str]:
"""Path(s) associated with the include."""
raise NotImplementedError("must be implemented in derived classes")
[docs]
class IncludePath(OptionalInclude):
path: str
sha256: str
destination: Optional[str]
def __init__(self, entry: dict):
# circular dependencies
import spack.util.path
super().__init__(entry)
path_override_env_var = entry.get("path_override_env_var", "")
if path_override_env_var and path_override_env_var in os.environ:
path = os.environ[path_override_env_var]
else:
path = entry.get("path", "")
self.path = spack.util.path.substitute_path_variables(path)
self.sha256 = entry.get("sha256", "")
self.remote = "sha256" in entry
self.destination = None
def __repr__(self):
return (
f"IncludePath({self.path}, sha256={self.sha256}, "
f"when='{self.when}', optional={self.optional})"
)
[docs]
def scopes(self, parent_scope: ConfigScope) -> List[ConfigScope]:
"""Instantiate a configuration scope for the included path.
Args:
parent_scope: including scope
Returns: configuration scopes IF the when condition is satisfied;
otherwise, an empty list.
Raises:
ConfigFileError: unable to access remote configuration file
ValueError: included path has an unsupported URL scheme, is required
but does not exist; configuration stage directory argument is missing
"""
if not self.evaluate_condition():
tty.debug(f"Include condition is not satisfied in {self}")
return []
if self._scopes:
tty.debug(f"Using existing scopes: {[s.name for s in self._scopes]}")
return self._scopes
# An absolute path does not need a local base directory.
if os.path.isabs(self.path):
tty.debug(f"The included path ({self}) is absolute so needs no base directory")
base = None
else:
base = self.base_directory(self.path, parent_scope)
# Make sure to use a proper working directory when obtaining the local
# path for a local (or remote) file.
tty.debug(f"Local base directory for {self.path} is {base}")
config_path = rfc_util.local_path(self.path, self.sha256, base)
assert config_path
self.destination = config_path
scope = self._scope(self.path, self.destination, parent_scope)
if scope is not None:
self._scopes = [scope]
return self._scopes
@property
def paths(self) -> List[str]:
"""Path(s) associated with the include."""
return [self.path]
[docs]
class GitIncludePaths(OptionalInclude):
git: str
branch: str
commit: str
tag: str
_paths: List[str]
destination: Optional[str]
def __init__(self, entry: dict):
# circular dependencies
import spack.util.path
super().__init__(entry)
self.git = spack.util.path.substitute_path_variables(entry.get("git", ""))
self.branch = entry.get("branch", "")
self.commit = entry.get("commit", "")
self.tag = entry.get("tag", "")
self._paths = [
spack.util.path.substitute_path_variables(path) for path in entry.get("paths", [])
]
self.destination = None
self.remote = True
if not self.branch and not self.commit and not self.tag:
raise spack.error.ConfigError(
"Git include paths ({self}) must specify one or more of: branch, commit, tag"
)
if not self._paths:
raise spack.error.ConfigError(
"Git include paths ({self}) must include one or more relative paths"
)
def __repr__(self):
if self.branch:
identifier = f"branch={self.branch}"
else:
identifier = f"commit={self.commit}, tag={self.tag}"
return (
f"GitIncludePaths('{self.name}', {self.git}, paths={self._paths}, "
f"{identifier}, when='{self.when}', optional={self.optional})"
)
def _clone(self, parent_scope: ConfigScope) -> Optional[str]:
"""Clone the repository.
Args:
parent_scope: enclosing scope
Returns: destination path if cloned or ``None``
"""
if self.fetched():
tty.debug(f"Repository ({self.git}) already cloned to {self.destination}")
return self.destination
# environment includes should be located under the environment
destination = self.base_directory(self.git, parent_scope)
assert destination, f"{self} requires a local cache directory"
tty.debug(f"Cloning {self.git} into {destination}")
with filesystem.working_dir(destination, create=True):
if not os.path.exists(".git"):
try:
tty.debug("Initializing the git repository")
spack.util.git.init_git_repo(self.git)
except spack.util.executable.ProcessError as e:
raise spack.error.ConfigError(
f"Unable to initialize repository ({self.git}) under {destination}: {e}"
)
try:
if self.commit:
tty.debug(f"Pulling commit {self.commit}")
spack.util.git.pull_checkout_commit(self.commit)
elif self.tag:
tty.debug(f"Pulling tag {self.tag}")
spack.util.git.pull_checkout_tag(self.tag)
elif self.branch:
# if the branch already exists we should use the
# previously configured remote
tty.debug(f"Pulling branch {self.branch}")
try:
git = spack.util.git.git(required=True)
output = git("config", f"branch.{self.branch}.remote", output=str)
remote = output.strip()
except spack.util.executable.ProcessError:
remote = "origin"
spack.util.git.pull_checkout_branch(self.branch, remote=remote)
else:
raise spack.error.ConfigError(f"Missing or unsupported options in {self}")
except spack.util.executable.ProcessError as e:
raise spack.error.ConfigError(
f"Unable to check out repository ({self}) in {destination}: {e}"
)
# only set the destination on successful clone/checkout
self.destination = destination
return self.destination
[docs]
def fetched(self) -> bool:
return bool(self.destination) and os.path.exists(
os.path.join(self.destination, ".git") # type: ignore[arg-type]
)
[docs]
def scopes(self, parent_scope: ConfigScope) -> List[ConfigScope]:
"""Instantiate configuration scopes for the included paths.
Args:
parent_scope: including scope
Returns: configuration scopes IF the when condition is satisfied;
otherwise, an empty list.
Raises:
ConfigFileError: unable to access remote configuration file(s)
ValueError: included path has an unsupported URL scheme, is required
but does not exist; configuration stage directory argument is missing
"""
if not self.evaluate_condition():
tty.debug(f"Include condition is not satisfied in {self}")
return []
if self._scopes:
tty.debug(f"Using existing scopes: {[s.name for s in self._scopes]}")
return self._scopes
destination = self._clone(parent_scope)
if not destination:
raise spack.error.ConfigError(f"Unable to cache the include: {self}")
scopes: List[ConfigScope] = []
for path in self.paths:
config_path = str(pathlib.Path(destination) / path)
scope = self._scope(path, config_path, parent_scope)
if scope is not None:
scopes.append(scope)
# cache the scopes if successfully able to process all of them
if scopes:
self._scopes = scopes
return self._scopes
@property
def paths(self) -> List[str]:
"""Path(s) associated with the include."""
return self._paths
[docs]
def included_path(entry: Union[str, pathlib.Path, dict]) -> Union[IncludePath, GitIncludePaths]:
"""Convert the included paths entry into the appropriate optional include.
Args:
entry: include configuration entry
Returns: converted entry, where an empty ``when`` means the path is not conditionally included
"""
if isinstance(entry, (str, pathlib.Path)):
return IncludePath({"path": str(entry)})
if entry.get("path", ""):
return IncludePath(entry)
return GitIncludePaths(entry)
[docs]
def paths_from_includes(includes: List[Union[str, dict]]) -> List[str]:
"""The path(s) from the configured includes.
Args:
includes: include configuration information
Returns: list of path or an empty list if there are none
"""
paths = []
for entry in includes:
include = included_path(entry)
paths.extend(include.paths)
return paths
[docs]
def config_paths_from_entry_points() -> List[Tuple[str, str]]:
"""Load configuration paths from entry points
A python package can register entry point metadata so that Spack can find
its configuration by adding the following to the project's pyproject.toml:
.. code-block:: toml
[project.entry-points."spack.config"]
baz = "baz:get_spack_config_path"
The function ``get_spack_config_path`` returns the path to the package's
spack configuration scope
"""
config_paths: List[Tuple[str, str]] = []
for entry_point in lang.get_entry_points(group="spack.config"):
hook = entry_point.load()
if callable(hook):
config_path = hook()
if config_path and os.path.exists(config_path):
config_paths.append(("plugin-%s" % entry_point.name, str(config_path)))
return config_paths
[docs]
def create_incremental() -> Generator[Configuration, None, None]:
"""Singleton Configuration instance.
This constructs one instance associated with this module and returns
it. It is bundled inside a function so that configuration can be
initialized lazily.
"""
# Default scopes are builtins and the default scope within the Spack instance.
# These are versioned with Spack and can be overridden by systems, sites or user scopes.
cfg = create_from(
(ConfigScopePriority.DEFAULTS, InternalConfigScope("_builtin", CONFIG_DEFAULTS)),
(ConfigScopePriority.DEFAULTS, DirectoryConfigScope(*CONFIGURATION_DEFAULTS_PATH)),
)
yield cfg
# Initial topmost scope is spack (the config scope in the spack instance).
# It includes the user, site, and system scopes. Environments and command
# line scopes go above this.
configuration_paths = [("spack", os.path.join(spack.paths.etc_path))]
# Python packages can register configuration scopes via entry_points
configuration_paths.extend(config_paths_from_entry_points())
# add each scope
for name, path in configuration_paths:
# yield the config incrementally so that each config level's init code can get
# data from the one below. This can be tricky, but it enables us to have a
# single unified config system.
#
# TODO: think about whether we want to restrict what types of config can be used
# at each level. e.g., we may want to just more forcibly disallow remote
# config (which uses ssl and other config options) for some of the scopes,
# to make the bootstrap issues more explicit, even if allowing config scope
# init to reference lower scopes is more flexible.
yield from cfg.push_scope_incremental(
DirectoryConfigScope(name, path), priority=ConfigScopePriority.CONFIG_FILES
)
[docs]
def create() -> Configuration:
"""Create a configuration using create_incremental(), return the last yielded result."""
return list(create_incremental())[-1]
#: This is the singleton configuration instance for Spack.
CONFIG = cast(Configuration, lang.Singleton(create_incremental))
[docs]
def add_from_file(filename: str, scope: Optional[str] = None) -> None:
"""Add updates to a config from a filename"""
# Extract internal attributes, if we are dealing with an environment
data = read_config_file(filename)
if data is None:
return
if spack.schema.env.TOP_LEVEL_KEY in data:
data = data[spack.schema.env.TOP_LEVEL_KEY]
msg = (
"unexpected 'None' value when retrieving configuration. "
"Please submit a bug-report at https://github.com/spack/spack/issues"
)
assert data is not None, msg
# update all sections from config dict
# We have to iterate on keys to keep overrides from the file
for section in data.keys():
if section in SECTION_SCHEMAS.keys():
# Special handling for compiler scope difference
# Has to be handled after we choose a section
if scope is None:
scope = default_modify_scope(section)
value = data[section]
existing = get(section, scope=scope)
new = spack.schema.merge_yaml(existing, value)
# We cannot call config.set directly (set is a type)
CONFIG.set(section, new, scope)
[docs]
def add(fullpath: str, scope: Optional[str] = None) -> None:
"""Add the given configuration to the specified config scope.
Add accepts a path. If you want to add from a filename, use add_from_file"""
components = process_config_path(fullpath)
has_existing_value = True
path = ""
override = False
value = components[-1]
if not isinstance(value, syaml.syaml_str):
value = syaml.load_config(value)
for idx, name in enumerate(components[:-1]):
# First handle double colons in constructing path
colon = "::" if override else ":" if path else ""
path += colon + name
if getattr(name, "override", False):
override = True
else:
override = False
# Test whether there is an existing value at this level
existing = get(path, scope=scope)
if existing is None:
has_existing_value = False
# We've nested further than existing config, so we need the
# type information for validation to know how to handle bare
# values appended to lists.
existing = get_valid_type(path)
# construct value from this point down
for component in reversed(components[idx + 1 : -1]):
value: Dict[str, str] = {component: value} # type: ignore[no-redef]
break
if override:
path += "::"
if has_existing_value:
existing = get(path, scope=scope)
# append values to lists
if isinstance(existing, list) and not isinstance(value, list):
value: List[str] = [value] # type: ignore[no-redef]
# merge value into existing
new = spack.schema.merge_yaml(existing, value)
CONFIG.set(path, new, scope)
[docs]
def get(path: str, default: Optional[Any] = None, scope: Optional[str] = None) -> Any:
"""Module-level wrapper for ``Configuration.get()``."""
return CONFIG.get(path, default, scope)
_set = set #: save this before defining set -- maybe config.set was ill-advised :)
[docs]
def set(path: str, value: Any, scope: Optional[str] = None) -> None:
"""Convenience function for setting single values in config files.
Accepts the path syntax described in ``get()``.
"""
result = CONFIG.set(path, value, scope)
return result
[docs]
def scopes() -> lang.PriorityOrderedMapping[str, ConfigScope]:
"""Convenience function to get list of configuration scopes."""
return CONFIG.scopes
[docs]
def writable_scopes() -> List[ConfigScope]:
"""Return list of writable scopes. Higher-priority scopes come first in the list."""
scopes = [x for x in CONFIG.scopes.values() if x.writable]
scopes.reverse()
return scopes
[docs]
def existing_scopes() -> List[ConfigScope]:
"""Return list of existing scopes. Scopes where Spack is
aware of said scope, and the scope has a representation
on the filesystem or are internal scopes.
Higher-priority scopes come first in the list."""
scopes = [x for x in CONFIG.scopes.values() if x.exists]
scopes.reverse()
return scopes
[docs]
def writable_scope_names() -> List[str]:
return list(x.name for x in writable_scopes())
[docs]
def existing_scope_names() -> List[str]:
return list(x.name for x in existing_scopes())
[docs]
def matched_config(cfg_path: str) -> List[Tuple[str, Any]]:
return [(scope, get(cfg_path, scope=scope)) for scope in writable_scope_names()]
[docs]
def change_or_add(
section_name: str, find_fn: Callable[[str], bool], update_fn: Callable[[str], None]
) -> None:
"""Change or add a subsection of config, with additional logic to
select a reasonable scope where the change is applied.
Search through config scopes starting with the highest priority:
the first matching a criteria (determined by ``find_fn``) is updated;
if no such config exists, find the first config scope that defines
any config for the named section; if no scopes define any related
config, then update the highest-priority config scope.
"""
configs_by_section = matched_config(section_name)
found = False
for scope, section in configs_by_section:
found = find_fn(section)
if found:
break
if found:
update_fn(section)
CONFIG.set(section_name, section, scope=scope)
return
# If no scope meets the criteria specified by ``find_fn``,
# then look for a scope that has any content (for the specified
# section name)
for scope, section in configs_by_section:
if section:
update_fn(section)
found = True
break
if found:
CONFIG.set(section_name, section, scope=scope)
return
# If no scopes define any config for the named section, then
# modify the highest-priority scope.
scope, section = configs_by_section[0]
update_fn(section)
CONFIG.set(section_name, section, scope=scope)
[docs]
def update_all(section_name: str, change_fn: Callable[[str], bool]) -> None:
"""Change a config section, which may have details duplicated
across multiple scopes.
"""
configs_by_section = matched_config("develop")
for scope, section in configs_by_section:
modified = change_fn(section)
if modified:
CONFIG.set(section_name, section, scope=scope)
def _validate_section_name(section: str) -> None:
"""Exit if the section is not a valid section."""
if section not in SECTION_SCHEMAS:
raise ConfigSectionError(
f"Invalid config section: '{section}'. Options are: {' '.join(SECTION_SCHEMAS.keys())}"
)
[docs]
def validate(
data: YamlConfigDict, schema: YamlConfigDict, filename: Optional[str] = None
) -> YamlConfigDict:
"""Validate data read in from a Spack YAML file.
Arguments:
data: data read from a Spack YAML file
schema: jsonschema to validate data
This leverages the line information (start_mark, end_mark) stored
on Spack YAML structures.
"""
try:
spack.schema.Validator(schema).validate(data)
except jsonschema.ValidationError as e:
if hasattr(e.instance, "lc"):
line_number = e.instance.lc.line + 1
else:
line_number = None
raise ConfigFormatError(e, data, filename, line_number) from e
# return the validated data so that we can access the raw data
# mostly relevant for environments
return data
[docs]
def read_config_file(
path: str, schema: Optional[YamlConfigDict] = None
) -> Optional[YamlConfigDict]:
"""Read a YAML configuration file.
User can provide a schema for validation. If no schema is provided,
we will infer the schema from the top-level key."""
# Dev: Inferring schema and allowing it to be provided directly allows us
# to preserve flexibility in calling convention (don't need to provide
# schema when it's not necessary) while allowing us to validate against a
# known schema when the top-level key could be incorrect.
try:
with open(path, encoding="utf-8") as f:
tty.debug(f"Reading config from file {path}")
data = syaml.load_config(f)
if data:
if schema is None:
key = next(iter(data))
schema = _ALL_SCHEMAS[key]
validate(data, schema)
return data
except FileNotFoundError:
# Ignore nonexistent files.
tty.debug(f"Skipping nonexistent config path {path}", level=3)
return None
except OSError as e:
raise ConfigFileError(f"Path is not a file or is not readable: {path}: {str(e)}") from e
except StopIteration as e:
raise ConfigFileError(f"Config file is empty or is not a valid YAML dict: {path}") from e
except syaml.SpackYAMLError as e:
raise ConfigFileError(str(e)) from e
def _mark_internal(data, name):
"""Add a simple name mark to raw YAML/JSON data.
This is used by `spack config blame` to show where config lines came from.
"""
if isinstance(data, dict):
d = syaml.syaml_dict(
(_mark_internal(k, name), _mark_internal(v, name)) for k, v in data.items()
)
elif isinstance(data, list):
d = syaml.syaml_list(_mark_internal(e, name) for e in data)
else:
d = syaml.syaml_type(data)
if syaml.markable(d):
d._start_mark = syaml.name_mark(name)
d._end_mark = syaml.name_mark(name)
return d
[docs]
def get_valid_type(path):
"""Returns an instance of a type that will pass validation for path.
The instance is created by calling the constructor with no arguments.
If multiple types will satisfy validation for data at the configuration
path given, the priority order is ``list``, ``dict``, ``str``, ``bool``,
``int``, ``float``.
"""
types = {
"array": list,
"object": syaml.syaml_dict,
"string": str,
"boolean": bool,
"integer": int,
"number": float,
}
components = process_config_path(path)
section = components[0]
# Use None to construct the test data
test_data = None
for component in reversed(components):
test_data = {component: test_data}
try:
validate(test_data, SECTION_SCHEMAS[section])
except (ConfigFormatError, AttributeError) as e:
jsonschema_error = e.validation_error
if jsonschema_error.validator == "type":
return types[jsonschema_error.validator_value]()
elif jsonschema_error.validator in ("anyOf", "oneOf"):
for subschema in jsonschema_error.validator_value:
schema_type = subschema.get("type")
if schema_type is not None:
return types[schema_type]()
else:
return type(None)
raise spack.error.ConfigError(f"Cannot determine valid type for path '{path}'.")
[docs]
def remove_yaml(dest, source):
"""UnMerges source from dest; entries in source take precedence over dest.
This routine may modify dest and should be assigned to dest, in
case dest was None to begin with, e.g.::
dest = remove_yaml(dest, source)
In the result, elements from lists from ``source`` will not appear
as elements of lists from ``dest``. Likewise, when iterating over keys
or items in merged ``OrderedDict`` objects, keys from ``source`` will not
appear as keys in ``dest``.
Config file authors can optionally end any attribute in a dict
with ``::`` instead of ``:``, and the key will remove the entire section
from ``dest``
"""
def they_are(t):
return isinstance(dest, t) and isinstance(source, t)
# If source is None, overwrite with source.
if source is None:
return dest
# Source list is prepended (for precedence)
if they_are(list):
# Make sure to copy ruamel comments
dest[:] = [x for x in dest if x not in source]
return dest
# Source dict is merged into dest.
elif they_are(dict):
for sk, sv in source.items():
# always remove the dest items. Python dicts do not overwrite
# keys on insert, so this ensures that source keys are copied
# into dest along with mark provenance (i.e., file/line info).
unmerge = sk in dest
old_dest_value = dest.pop(sk, None)
if unmerge and not spack.schema.override(sk):
dest[sk] = remove_yaml(old_dest_value, sv)
return dest
# If we reach here source and dest are either different types or are
# not both lists or dicts: replace with source.
return dest
[docs]
class ConfigPath:
quoted_string = "(?:\"[^\"]+\")|(?:'[^']+')"
unquoted_string = "[^:'\"]+"
element = rf"(?:(?:{quoted_string})|(?:{unquoted_string}))"
next_key_pattern = rf"({element}[+-]?)(?:\:|$)"
@staticmethod
def _split_front(string, extract):
m = re.match(extract, string)
if not m:
return None, None
token = m.group(1)
return token, string[len(token) :]
@staticmethod
def _validate(path):
"""Example valid config paths:
x:y:z
x:"y":z
x:y+:z
x:y::z
x:y+::z
x:y:
x:y::
"""
first_key, path = ConfigPath._split_front(path, ConfigPath.next_key_pattern)
if not first_key:
raise ValueError(f"Config path does not start with a parse-able key: {path}")
path_elements = [first_key]
path_index = 1
while path:
separator, path = ConfigPath._split_front(path, r"(\:+)")
if not separator:
raise ValueError(f"Expected separator for {path}")
path_elements[path_index - 1] += separator
if not path:
break
element, remainder = ConfigPath._split_front(path, ConfigPath.next_key_pattern)
if not element:
# If we can't parse something as a key, then it must be a
# value (if it's valid).
try:
syaml.load_config(path)
except syaml.SpackYAMLError as e:
raise ValueError(
"Remainder of path is not a valid key"
f" and does not parse as a value {path}"
) from e
element = path
path = None # The rest of the path was consumed into the value
else:
path = remainder
path_elements.append(element)
path_index += 1
return path_elements
[docs]
@staticmethod
def process(path):
result = []
quote = "['\"]"
seen_override_in_path = False
path_elements = ConfigPath._validate(path)
last_element_idx = len(path_elements) - 1
for i, element in enumerate(path_elements):
override = False
append = False
prepend = False
quoted = False
if element.endswith("::") or (element.endswith(":") and i == last_element_idx):
if seen_override_in_path:
raise syaml.SpackYAMLError(
"Meaningless second override indicator `::' in path `{0}'".format(path), ""
)
override = True
seen_override_in_path = True
element = element.rstrip(":")
if element.endswith("+"):
prepend = True
elif element.endswith("-"):
append = True
element = element.rstrip("+-")
if re.match(f"^{quote}", element):
quoted = True
element = element.strip("'\"")
if append or prepend or override or quoted:
element = syaml.syaml_str(element)
if append:
element.append = True
if prepend:
element.prepend = True
if override:
element.override = True
result.append(element)
return result
[docs]
def process_config_path(path: str) -> List[str]:
"""Process a path argument to config.set() that may contain overrides (``::`` or
trailing ``:``)
Colons will be treated as static strings if inside of quotes,
e.g. ``this:is:a:path:'value:with:colon'`` will yield:
.. code-block:: text
[this, is, a, path, value:with:colon]
The path may consist only of keys (e.g. for a ``get``) or may end in a value.
Keys are always strings: if a user encloses a key in quotes, the quotes
should be removed. Values with quotes should be treated as strings,
but without quotes, may be parsed as a different yaml object (e.g.
``'{}'`` is a dict, but ``'"{}"'`` is a string).
This function does not know whether the final element of the path is a
key or value, so:
* It must strip the quotes, in case it is a key (so we look for ``key`` and
not ``"key"``)
* It must indicate somehow that the quotes were stripped, in case it is a
value (so that we don't process ``"{}"`` as a YAML dict)
Therefore, all elements with quotes are stripped, and then also converted
to ``syaml_str`` (if treating the final element as a value, the caller
should not parse it in this case).
"""
return ConfigPath.process(path)
#
# Settings for commands that modify configuration
#
[docs]
def default_modify_scope(section: str = "config") -> str:
"""Return the config scope that commands should modify by default.
Commands that modify configuration by default modify the *highest*
priority scope.
Arguments:
section (bool): Section for which to get the default scope.
"""
return CONFIG.highest_precedence_scope().name
def _update_in_memory(data: YamlConfigDict, section: str) -> bool:
"""Update the format of the configuration data in memory.
This function assumes the section is valid (i.e. validation
is responsibility of the caller)
Args:
data: configuration data
section: section of the configuration to update
Returns:
True if the data was changed, False otherwise
"""
return ensure_latest_format_fn(section)(data)
[docs]
@contextlib.contextmanager
def use_configuration(
*scopes_or_paths: Union[ScopeWithOptionalPriority, str],
) -> Generator[Configuration, None, None]:
"""Use the configuration scopes passed as arguments within the context manager.
This function invalidates caches, and is therefore very slow.
Args:
*scopes_or_paths: scope objects or paths to be used
Returns:
Configuration object associated with the scopes passed as arguments
"""
global CONFIG
# Normalize input and construct a Configuration object
configuration = create_from(*scopes_or_paths)
CONFIG.clear_caches(), configuration.clear_caches()
saved_config, CONFIG = CONFIG, configuration
try:
yield configuration
finally:
CONFIG = saved_config
def _normalize_input(entry: Union[ScopeWithOptionalPriority, str]) -> ScopeWithPriority:
if isinstance(entry, tuple):
return entry
default_priority = ConfigScopePriority.CONFIG_FILES
if isinstance(entry, ConfigScope):
return default_priority, entry
# Otherwise we need to construct it
path = os.path.normpath(entry)
assert os.path.isdir(path), f'"{path}" must be a directory'
name = os.path.basename(path)
return default_priority, DirectoryConfigScope(name, path)
[docs]
@lang.memoized
def create_from(*scopes_or_paths: Union[ScopeWithOptionalPriority, str]) -> Configuration:
"""Creates a configuration object from the scopes passed in input.
Args:
*scopes_or_paths: either a tuple of (priority, ConfigScope), or a ConfigScope, or a string
If priority is not given, it is assumed to be ConfigScopePriority.CONFIG_FILES. If a
string is given, a DirectoryConfigScope is created from it.
Examples:
>>> builtin_scope = InternalConfigScope("_builtin", {"config": {"build_jobs": 1}})
>>> cl_scope = InternalConfigScope("command_line", {"config": {"build_jobs": 10}})
>>> cfg = create_from(
... (ConfigScopePriority.COMMAND_LINE, cl_scope),
... (ConfigScopePriority.BUILTIN, builtin_scope)
... )
"""
scopes_with_priority = [_normalize_input(x) for x in scopes_or_paths]
result = Configuration()
for priority, scope in scopes_with_priority:
result.push_scope(scope, priority=priority)
return result
[docs]
def determine_number_of_jobs(
*,
parallel: bool = False,
max_cpus: int = cpus_available(),
config: Optional[Configuration] = None,
) -> int:
"""
Packages that require sequential builds need 1 job. Otherwise we use the
number of jobs set on the command line. If not set, then we use the config
defaults (which is usually set through the builtin config scope), but we
cap to the number of CPUs available to avoid oversubscription.
Parameters:
parallel: true when package supports parallel builds
max_cpus: maximum number of CPUs to use (defaults to cpus_available())
config: configuration object (defaults to global config)
"""
if not parallel:
return 1
cfg = config or CONFIG
# Command line overrides all
try:
command_line = cfg.get("config:build_jobs", default=None, scope="command_line")
if command_line is not None:
return command_line
except ValueError:
pass
return min(max_cpus, cfg.get("config:build_jobs", 16))
[docs]
class ConfigSectionError(spack.error.ConfigError):
"""Error for referring to a bad config section name in a configuration."""
[docs]
class ConfigFileError(spack.error.ConfigError):
"""Issue reading or accessing a configuration file."""
[docs]
class RecursiveIncludeError(spack.error.SpackError):
"""Too many levels of recursive includes."""