Source code for spack.concretize

# Copyright Spack Project Developers. See COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""High-level functions to concretize list of specs"""

import contextlib
import importlib
import sys
import time
from collections import Counter
from typing import (
    TYPE_CHECKING,
    Any,
    Dict,
    Iterable,
    Iterator,
    List,
    Optional,
    Sequence,
    Tuple,
    Union,
)

import spack.compilers
import spack.compilers.config
import spack.config
import spack.context
import spack.error
import spack.hash_lookup
import spack.repo
import spack.solver.core
import spack.traverse
import spack.util.parallel
from spack.concretize_ui import (
    DEFAULT_USER_SPEC_GROUP,
    ConcretizerUI,
    HeadlessUI,
    SolveKind,
    concretization_span,
    group_span,
)
from spack.spec import Spec
from spack.util import tty

SpecPairInput = Tuple[Spec, Optional[Spec]]
SpecPair = Tuple[Spec, Spec]
TestsType = Union[bool, Iterable[str]]

if TYPE_CHECKING:
    from spack.solver.asp import Solver
    from spack.solver.reuse import SpecFiltersFactory


def _needs_solving(abstract: Spec, concrete: Optional[Spec]) -> bool:
    """Return whether a spec pair has to go through a solve. A pair that does not is passed
    through unchanged, and is not reported to a frontend.
    """
    return concrete is None and not abstract.concrete


[docs] def ensure_compilers_in_configuration() -> None: """Write the compilers found on the system to packages.yaml, if none are configured. A solve sees compilers as externals declared in the configuration, so detection has to run before it, and in the parent process of a parallel concretization: the workers would otherwise write the configuration file at the same time. """ _ = spack.compilers.config.all_compilers(spack.config.CONFIG, repo=spack.repo.PATH)
def _solver(*, factory: Optional["SpecFiltersFactory"] = None) -> "Solver": """Return a solver to concretize with, with the compilers already in the configuration.""" from spack.solver.asp import Solver ensure_compilers_in_configuration() return Solver(context=spack.context.default(), specs_factory=factory) def _concretize_specs_together( abstract_specs: Sequence[Spec], *, tests: TestsType = False, factory: Optional["SpecFiltersFactory"] = None, ) -> List[Spec]: """Given a number of specs as input, tries to concretize them together. Args: abstract_specs: abstract specs to be concretized tests: list of package names for which to consider tests dependencies. If True, all nodes will have test dependencies. If False, test dependencies will be disregarded. factory: optional factory to produce a list of specs to be reused """ result = _solver(factory=factory).solve(abstract_specs, tests=tests) return [s.copy() for s in result.specs] def _concretize_together( spec_list: Sequence[SpecPairInput], *, tests: TestsType = False, factory: Optional["SpecFiltersFactory"] = None, ui: ConcretizerUI, ) -> List[SpecPair]: """Given a number of specs as input, tries to concretize them together. Args: spec_list: list of tuples to concretize. First entry is abstract spec, second entry is already concrete spec or None if not yet concretized tests: list of package names for which to consider tests dependencies. If True, all nodes will have test dependencies. If False, test dependencies will be disregarded. factory: optional factory to produce a list of specs to be reused ui: frontend to report progress to. The group these specs belong to is assumed to be opened by the caller. """ to_concretize = [concrete if concrete else abstract for abstract, concrete in spec_list] start = time.monotonic() concrete_specs = _concretize_specs_together(to_concretize, tests=tests, factory=factory) duration = time.monotonic() - start # A single solve produced all the specs, so they all report the duration of that solve result, count = [], 0 for (abstract, previous), concrete in zip(spec_list, concrete_specs): result.append((abstract, concrete)) if not _needs_solving(abstract, previous): continue count += 1 ui.on_spec_concretized(abstract, concrete=concrete, count=count, duration=duration) return result def _concretize_together_when_possible( spec_list: Sequence[SpecPairInput], *, tests: TestsType = False, factory: Optional["SpecFiltersFactory"] = None, ui: ConcretizerUI, ) -> List[SpecPair]: """Given a number of specs as input, tries to concretize them together to the extent possible. See documentation for ``unify: when_possible`` concretization for the precise definition of "to the extent possible". Args: spec_list: list of tuples to concretize. First entry is abstract spec, second entry is already concrete spec or None if not yet concretized tests: list of package names for which to consider tests dependencies. If True, all nodes will have test dependencies. If False, test dependencies will be disregarded. factory: optional factory to produce a list of specs to be reused ui: frontend to report progress to. """ to_concretize = [concrete if concrete else abstract for abstract, concrete in spec_list] old_concrete_to_abstract = { concrete: abstract for (abstract, concrete) in spec_list if concrete } to_report = { abstract for abstract, concrete in spec_list if _needs_solving(abstract, concrete) } result_by_user_spec: Dict[Spec, Spec] = {} j = 0 start = time.monotonic() for result in _solver(factory=factory).solve_in_rounds(to_concretize, tests=tests): now = time.monotonic() duration = now - start for abstract, concrete in result.specs_by_input.items(): if abstract not in to_report: continue j += 1 ui.on_spec_concretized(abstract, concrete=concrete, count=j, duration=duration) result_by_user_spec.update(result.specs_by_input) start = now # If the "abstract" spec is a concrete spec from the previous concretization # translate it back to an abstract spec. Otherwise, keep the abstract spec return [ (old_concrete_to_abstract.get(abstract, abstract), concrete) for abstract, concrete in sorted(result_by_user_spec.items()) ] def _concretize_separately( spec_list: Sequence[SpecPairInput], *, tests: TestsType = False, factory: Optional["SpecFiltersFactory"] = None, ui: ConcretizerUI, processes: int, ) -> List[SpecPair]: """Concretizes the input specs separately from each other. Args: spec_list: list of tuples to concretize. First entry is abstract spec, second entry is already concrete spec or None if not yet concretized tests: list of package names for which to consider tests dependencies. If True, all nodes will have test dependencies. If False, test dependencies will be disregarded. factory: optional factory to produce a list of specs to be reused ui: frontend to report progress to. processes: size of the process pool """ from spack.bootstrap import ( ensure_bootstrap_configuration, ensure_clingo_importable_or_raise, ensure_winsdk_external_or_raise, ) to_concretize = [abstract for abstract, concrete in spec_list if not concrete] args = [ (i, str(abstract), tests, factory) for i, abstract in enumerate(to_concretize) if not abstract.concrete ] ret = [(i, abstract) for i, abstract in enumerate(to_concretize) if abstract.concrete] try: # Ensure we don't try to bootstrap clingo in parallel importlib.import_module("clingo") except ImportError: with ensure_bootstrap_configuration(): ensure_clingo_importable_or_raise() # ensure we don't try to detect winsdk in parallel if sys.platform == "win32": ensure_winsdk_external_or_raise() # Ensure all the indexes have been built or updated, since # otherwise the processes in the pool may timeout on waiting # for a write lock. We do this indirectly by retrieving the # provider index, which should in turn trigger the update of # all the indexes if there's any need for that. _ = spack.repo.PATH.provider_index ensure_compilers_in_configuration() # Solve the environment in parallel on Linux. imap_unordered falls back to a serial map when # parallelism is disabled (e.g. Windows), and when there is at most one spec to solve for j, (i, concrete, duration) in enumerate( spack.util.parallel.imap_unordered( _concretize_task, args, processes=processes, debug=tty.is_debug(), maxtaskperchild=1, serialize_env=True, ), start=1, ): ret.append((i, concrete)) ui.on_spec_concretized(to_concretize[i], concrete=concrete, count=j, duration=duration) # Add specs in original order, then combine the ones passed in as abstract with the ones # passed in as pairs ret.sort(key=lambda x: x[0]) return [(abstract, concrete) for abstract, (_, concrete) in zip(to_concretize, ret)] + [ (abstract, concrete) for abstract, concrete in spec_list if concrete ] def _concretize_task( packed_arguments: Tuple[int, str, TestsType, Optional["SpecFiltersFactory"]], ) -> Tuple[int, Spec, float]: index, spec_str, tests, factory = packed_arguments with tty.SuppressOutput(msg_enabled=False): start = time.time() spec = concretize_one(Spec(spec_str), tests=tests, factory=factory) return index, spec, time.time() - start
[docs] def concretize_one( spec: Union[str, Spec], *, tests: TestsType = False, factory: Optional["SpecFiltersFactory"] = None, ui: Optional[ConcretizerUI] = None, ) -> Spec: """Return a concretized copy of the given spec. Args: tests: if False disregard test dependencies, if a list of names activate them for the packages in the list, if True activate test dependencies for all packages. factory: optional factory to produce a list of specs to be reused ui: frontend to report the solve to. Defaults to a headless frontend. """ ui = ui or HeadlessUI() with concretization_span(ui): return _concretize_one(spec, tests=tests, factory=factory, ui=ui)
def _concretize_one( spec: Union[str, Spec], *, tests: TestsType = False, factory: Optional["SpecFiltersFactory"] = None, ui: ConcretizerUI, ) -> Spec: """Concretize a single spec, as a group of one, inside a concretization that started.""" if isinstance(spec, str): spec = Spec(spec) spec = spack.hash_lookup.lookup_hash(spec) # A single spec takes a single solve, whatever "concretizer:unify" prescribes with group_span( ui, group=DEFAULT_USER_SPEC_GROUP, kind=SolveKind.TOGETHER, total=0 if spec.concrete else 1, processes=1, ): if spec.concrete: return spec.copy() start = time.monotonic() concrete = _solve_one(spec, tests=tests, factory=factory) ui.on_spec_concretized(spec, concrete=concrete, count=1, duration=time.monotonic() - start) return concrete def _solve_one(spec: Spec, *, tests: TestsType, factory: Optional["SpecFiltersFactory"]) -> Spec: """Run the single solve that concretizes ``spec``, and pick its answer.""" for node in spec.traverse(): if not node.name: raise spack.error.SpecError( f"Spec {node} has no name; cannot concretize an anonymous spec" ) result = _solver(factory=factory).solve([spec], tests=tests) # take the best answer opt, i, answer = min(result.answers) name = spec.name # TODO: Consolidate this code with similar code in solve.py if spack.repo.PATH.is_virtual(spec.name): providers = [s.name for s in answer.values() if s.package.provides(name)] name = providers[0] node = spack.solver.core.min_dupe_node(pkg=name) assert node in answer, ( f"cannot find {name} in the list of specs {','.join([n.pkg for n in answer.keys()])}" ) concretized = answer[node] return concretized
[docs] def solve_kind(unify: Any) -> SolveKind: """Return the kind of solve that a ``concretizer:unify`` value prescribes. Raises: spack.error.ConfigError: if ``unify`` is not a value the schema allows """ if unify == "when_possible": return SolveKind.WHEN_POSSIBLE if unify not in (True, False): raise spack.error.ConfigError(f"concretization strategy not implemented [{unify}]") return SolveKind.TOGETHER if unify else SolveKind.SEPARATELY
def _reported_total(spec_list: Sequence[SpecPairInput]) -> int: """Return how many specs a group concretizing ``spec_list`` reports as concretized, which is the ``total`` a frontend counts against. Only the specs that go through a solve are reported, whatever the strategy. """ return sum(1 for abstract, concrete in spec_list if _needs_solving(abstract, concrete)) def _processes_for(kind: SolveKind, total: int) -> int: """Return the size of the process pool that concretizing ``total`` specs as ``kind`` prescribes uses. Only solving separately runs more than one process. """ if kind is not SolveKind.SEPARATELY or not total: return 1 if not spack.util.parallel.ENABLE_PARALLELISM: return 1 return min(total, spack.config.determine_number_of_jobs(parallel=True))
[docs] @contextlib.contextmanager def solve_group( ui: ConcretizerUI, *, group: str, kind: SolveKind, spec_list: Sequence[SpecPairInput] ) -> Iterator[int]: """Open the group that concretizes ``spec_list`` as ``kind`` prescribes, and yield the size of the process pool it announced, so that the pool that runs is the one a frontend was told about. A group whose specs need no solve announces a total of zero, and a frontend sees it open and close with nothing in between. """ total = _reported_total(spec_list) processes = _processes_for(kind, total) with group_span(ui, group=group, kind=kind, total=total, processes=processes): yield processes
[docs] def concretize_spec_pairs( to_concretize: List[SpecPairInput], *, tests: TestsType = False, ui: Optional[ConcretizerUI] = None, ) -> List[Spec]: """Concretize the abstract specs of a list of (abstract, concrete) pairs. Any abstract spec with a concrete spec associated with it concretizes to that spec. Any abstract spec with ``None`` for its concrete spec is newly concretized. Respects the unification rules from configuration. Args: to_concretize: list of tuples to concretize. First entry is abstract spec, second entry is an already concrete spec, or None if not yet concretized tests: list of package names for which to consider tests dependencies. If True, all nodes will have test dependencies. If False, test dependencies will be disregarded. ui: frontend to report progress to. Defaults to a headless frontend. """ ui = ui or HeadlessUI() with concretization_span(ui): return _dispatch_concretization(to_concretize, tests=tests, ui=ui)
def _dispatch_concretization( to_concretize: List[SpecPairInput], *, tests: TestsType, ui: ConcretizerUI ) -> List[Spec]: kind = solve_kind(spack.config.CONFIG.get("concretizer:unify", False)) # Special case for concretizing a single spec if len(to_concretize) == 1: abstract, concrete = to_concretize[0] if concrete is None: return [_concretize_one(abstract, tests=tests, ui=ui)] # Nothing to solve, so the group reports a total of zero with group_span(ui, group=DEFAULT_USER_SPEC_GROUP, kind=kind, total=0, processes=1): return [concrete] # Special case if every spec is either concrete or has an abstract hash if all( concrete or abstract.concrete or abstract.abstract_hash for abstract, concrete in to_concretize ): # No spec is solved, so the group reports a total of zero with group_span(ui, group=DEFAULT_USER_SPEC_GROUP, kind=kind, total=0, processes=1): # Get all the concrete specs ret = [ concrete or (abstract if abstract.concrete else spack.hash_lookup.lookup_hash(abstract)) for abstract, concrete in to_concretize ] # If unify: true, check that specs don't conflict # Since all concrete, "when_possible" is not relevant if kind is SolveKind.TOGETHER: runtimes = spack.repo.PATH.packages_with_tags("runtime") specs_per_name = Counter( spec.name for spec in spack.traverse.traverse_nodes( ret, deptype=("link", "run"), key=spack.traverse.by_dag_hash ) if spec.name not in runtimes # runtimes are allowed multiple times ) conflicts = sorted(name for name, count in specs_per_name.items() if count > 1) if conflicts: raise spack.error.SpecError( "Specs conflict and `concretizer:unify` is configured true.", f" specs depend on multiple versions of {', '.join(conflicts)}", ) return ret # Standard case with solve_group( ui, group=DEFAULT_USER_SPEC_GROUP, kind=kind, spec_list=to_concretize ) as processes: if kind is SolveKind.TOGETHER: concretized = _concretize_together(to_concretize, tests=tests, ui=ui) elif kind is SolveKind.WHEN_POSSIBLE: concretized = _concretize_together_when_possible(to_concretize, tests=tests, ui=ui) else: concretized = _concretize_separately( to_concretize, tests=tests, ui=ui, processes=processes ) return [concrete for _, concrete in concretized]