# Copyright Spack Project Developers. See COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import enum
import functools
import typing
import warnings
from typing import Any, Callable, List, Mapping, Optional
import spack.binary_distribution
import spack.config
import spack.deprecation
import spack.repo
import spack.spec
import spack.traverse
import spack.util.path
from spack.active_environment import active_environment
from spack.enums import InstallRecordStatus
from spack.externals import ExternalSpecsParser
from spack.externals_config import (
all_libcs,
create_external_parser,
external_config_with_implicit_externals,
)
from spack.spec_filter import SpecFilter
if typing.TYPE_CHECKING:
import spack.context
import spack.environment
[docs]
def spec_filter_from_store(store, *, is_reusable, include=None, exclude=None) -> SpecFilter:
"""Constructs a filter that takes the specs from the store passed as argument."""
factory = functools.partial(_specs_from_store, store=store)
return SpecFilter(factory=factory, is_usable=is_reusable, include=include, exclude=exclude)
[docs]
def spec_filter_from_buildcache(
*, context: "spack.context.SpackContext", is_reusable, include=None, exclude=None
) -> SpecFilter:
"""Constructs a filter that takes the specs from the configured buildcaches."""
factory = functools.partial(
_specs_from_mirror, binary_index=context.binary_index, config=context.config
)
return SpecFilter(factory=factory, is_usable=is_reusable, include=include, exclude=exclude)
[docs]
def spec_filter_from_environment(*, is_reusable, env, include=None, exclude=None) -> SpecFilter:
factory = functools.partial(_specs_from_environment, env=env)
return SpecFilter(factory=factory, is_usable=is_reusable, include=include, exclude=exclude)
[docs]
def spec_filter_from_packages_yaml(
*, external_parser: ExternalSpecsParser, is_reusable, include=None, exclude=None
) -> SpecFilter:
return SpecFilter(
external_parser.all_specs, is_usable=is_reusable, include=include, exclude=exclude
)
def _has_runtime_dependencies(spec: spack.spec.Spec) -> bool:
# Spack v1.0 specs and later
return spec.original_spec_format() >= 5
def _external_db_hashes(store: "spack.store.Store") -> typing.FrozenSet[str]:
"""Dag hashes of the specs a cray manifest imported into ``store``."""
return frozenset(
s.dag_hash()
for s in store.db.query(origin="external-db", installed=InstallRecordStatus.ANY)
)
def _is_reusable(
spec: spack.spec.Spec,
packages_with_externals,
local: bool,
*,
repo: spack.repo.RepoPath,
external_db_hashes: typing.FrozenSet[str] = frozenset(),
) -> bool:
"""A spec is reusable if it's not a dev spec, it's imported from the cray manifest, it's not
external, or it's external with matching packages.yaml entry. The latter prevents two issues:
1. Externals in build caches: avoid installing an external on the build machine not
available on the target machine
2. Local externals: avoid reusing an external if the local config changes. This helps in
particular when a user removes an external from packages.yaml, and expects that that
takes effect immediately.
Arguments:
spec: the spec to check
packages_with_externals: the pre-processed packages configuration
local: whether the spec comes from a local source (the store or an environment)
repo: repository used to look up the virtuals a package provides
external_db_hashes: dag hashes of locally installed specs imported from a cray manifest
"""
if "dev_path" in spec.variants:
return False
if spec.name == "compiler-wrapper":
return False
if not spec.external:
return _has_runtime_dependencies(spec)
# Cray external manifest externals are always reusable
if local and spec.dag_hash() in external_db_hashes:
return True
try:
provided = repo.get(spec).provided_virtual_names()
except spack.repo.RepoError:
provided = []
for name in {spec.name, *provided}:
for entry in packages_with_externals.get(name, {}).get("externals", []):
expected_prefix = entry.get("prefix")
if expected_prefix is not None:
expected_prefix = spack.util.path.path_to_os_path(expected_prefix)[0]
if (
spec.satisfies(entry["spec"])
and spec.external_path == expected_prefix
and spec.external_modules == entry.get("modules")
):
return True
return False
[docs]
def reusable_external_specs(context: "spack.context.SpackContext") -> List[spack.spec.Spec]:
"""Return the reusable external specs declared in a context's ``packages.yaml``."""
packages_with_externals = external_config_with_implicit_externals(context)
spec_filter = spec_filter_from_packages_yaml(
external_parser=create_external_parser(packages_with_externals, context=context),
is_reusable=functools.partial(
_is_reusable,
packages_with_externals=packages_with_externals,
local=True,
repo=context.repo,
),
)
return spec_filter.selected_specs()
def _specs_from_store(store):
with store.db.read_transaction():
# The order of reused specs does not matter to the solver, so skip sorting.
return store.db.query(installed=True, sort=False)
def _specs_from_mirror(binary_index, config: spack.config.Configuration):
try:
specs = spack.binary_distribution.update_cache_and_get_specs(binary_index, config=config)
except (spack.binary_distribution.FetchCacheError, IndexError):
# this is raised when no mirrors had indices.
# TODO: update mirror configuration so it can indicate that the
# TODO: source cache (or any mirror really) doesn't have binaries.
return []
for url in sorted(binary_index.mirrors_without_index):
warnings.warn(f"the mirror at {url} cannot be used in concretization (no index found)")
return specs
def _specs_from_environment(env):
"""Return all concrete specs from the environment. This includes all included concrete"""
if env:
return list(spack.traverse.traverse_nodes([s for _, s in env.concretized_specs()]))
else:
return []
[docs]
class ReuseStrategy(enum.Enum):
ROOTS = enum.auto()
DEPENDENCIES = enum.auto()
NONE = enum.auto()
SpecFiltersFactory = Callable[
[Callable[[spack.spec.Spec], bool], spack.config.Configuration], List[SpecFilter]
]
[docs]
class ReusableSpecsSelector:
"""Selects specs that can be reused during concretization."""
def __init__(
self,
*,
context: "spack.context.SpackContext",
packages_with_externals: Any,
factory: Optional[SpecFiltersFactory] = None,
) -> None:
# Local import to break circular dependencies
import spack.environment
configuration, store, repo = context.config, context.store, context.repo
external_parser = create_external_parser(packages_with_externals, context=context)
# Membership in this set replaces a per-spec query_by_spec_hash on the store
external_db_hashes = _external_db_hashes(store)
# _is_reusable only varies by local vs. build cache, so bind the two variants once
local_is_reusable = functools.partial(
_is_reusable,
packages_with_externals=packages_with_externals,
local=True,
repo=repo,
external_db_hashes=external_db_hashes,
)
mirror_is_reusable = functools.partial(
_is_reusable, packages_with_externals=packages_with_externals, local=False, repo=repo
)
self.reuse_strategy = ReuseStrategy.ROOTS
reuse_yaml = configuration.get("concretizer:reuse", False)
self.reuse_sources = []
if factory is not None:
self.reuse_sources.extend(factory(local_is_reusable, configuration))
if not isinstance(reuse_yaml, Mapping):
self.reuse_sources.append(
spec_filter_from_packages_yaml(
external_parser=external_parser, is_reusable=local_is_reusable
)
)
if reuse_yaml is False:
self.reuse_strategy = ReuseStrategy.NONE
return
if reuse_yaml == "dependencies":
self.reuse_strategy = ReuseStrategy.DEPENDENCIES
self.reuse_sources.extend(
[
spec_filter_from_store(store, is_reusable=local_is_reusable),
spec_filter_from_buildcache(context=context, is_reusable=mirror_is_reusable),
]
)
else:
has_external_source = False
roots = reuse_yaml.get("roots", True)
if roots is True:
self.reuse_strategy = ReuseStrategy.ROOTS
else:
self.reuse_strategy = ReuseStrategy.DEPENDENCIES
default_include = reuse_yaml.get("include", [])
default_exclude = reuse_yaml.get("exclude", [])
default_sources = [{"type": "local"}, {"type": "buildcache"}]
for source in reuse_yaml.get("from", default_sources):
include = source.get("include", default_include)
exclude = source.get("exclude", default_exclude)
if source["type"] == "environment" and "path" in source:
env_dir = spack.environment.as_env_dir(source["path"])
active_env = active_environment()
if not active_env or env_dir not in active_env.included_concrete_env_root_dirs:
# If the environment is not included as a concrete environment, use the
# current specs from its lockfile.
self.reuse_sources.append(
spec_filter_from_environment(
is_reusable=local_is_reusable,
include=include,
exclude=exclude,
env=spack.environment.environment_from_name_or_dir(env_dir),
)
)
elif source["type"] == "local":
self.reuse_sources.append(
spec_filter_from_store(
store, is_reusable=local_is_reusable, include=include, exclude=exclude
)
)
elif source["type"] == "buildcache":
self.reuse_sources.append(
spec_filter_from_buildcache(
context=context,
is_reusable=mirror_is_reusable,
include=include,
exclude=exclude,
)
)
elif source["type"] == "external":
has_external_source = True
if include:
# Since libcs are implicit externals, we need to implicitly include them
include = include + sorted(all_libcs(context))
self.reuse_sources.append(
spec_filter_from_packages_yaml(
external_parser=external_parser,
is_reusable=local_is_reusable,
include=include,
exclude=exclude,
)
)
# If "external" is not specified, we assume that all externals have to be included
if not has_external_source:
self.reuse_sources.append(
spec_filter_from_packages_yaml(
external_parser=external_parser, is_reusable=local_is_reusable
)
)
[docs]
def reusable_specs(
self, specs: List[spack.spec.Spec], *, policy: Optional[spack.deprecation.Policy] = None
) -> List[spack.spec.Spec]:
result = []
for reuse_source in self.reuse_sources:
result.extend(reuse_source.selected_specs())
# If we only want to reuse dependencies, remove the root specs
if self.reuse_strategy == ReuseStrategy.DEPENDENCIES:
result = [spec for spec in result if not any(root in spec for root in specs)]
# Never hand the solver an artifact the deprecation policy would refuse to install
return spack.deprecation.reusable(result, policy=policy)