# Copyright Spack Project Developers. See COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import argparse
import os
import shlex
import sys
import tempfile
from typing import Any, Dict, Generator, List, Optional, Tuple, Union
import spack
import spack.caches
import spack.ci
import spack.config
import spack.package_base
import spack.repo
import spack.spec
import spack.util.executable
import spack.util.filesystem as fs
import spack.util.git
import spack.util.spack_json as sjson
import spack.util.spack_yaml
from spack.cmd.common import arguments
from spack.error import SpackError
from spack.util import tty
from spack.util.tty import color
from spack.version import StandardVersion
from . import doc_dedented, doc_first_line
description = "manage package source repositories"
section = "config"
level = "long"
[docs]
def setup_parser(subparser: argparse.ArgumentParser):
sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="repo_command")
# Create
create_parser = sp.add_parser(
"create", description=doc_dedented(repo_create), help=doc_first_line(repo_create)
)
create_parser.add_argument("directory", help="directory to create the repo in")
create_parser.add_argument(
"namespace", help="name or namespace to identify packages in the repository"
)
create_parser.add_argument(
"-d",
"--subdirectory",
action="store",
dest="subdir",
default=spack.repo.packages_dir_name,
help="subdirectory to store packages in the repository\n\n"
"default 'packages'. use an empty string for no subdirectory",
)
# List
list_parser = sp.add_parser(
"list", aliases=["ls"], description=doc_dedented(repo_list), help=doc_first_line(repo_list)
)
list_parser.add_argument(
"--scope",
action=arguments.ConfigScope,
type=arguments.config_scope_readable_validator,
help="configuration scope to read from",
)
output_group = list_parser.add_mutually_exclusive_group()
output_group.add_argument("--names", action="store_true", help="show configuration names only")
output_group.add_argument(
"--namespaces", action="store_true", help="show repository namespaces only"
)
output_group.add_argument(
"--json", action="store_true", help="output repositories as machine-readable json records"
)
# Add
add_parser = sp.add_parser(
"add", description=doc_dedented(repo_add), help=doc_first_line(repo_add)
)
add_parser.add_argument(
"path_or_repo", help="path or git repository of a Spack package repository"
)
# optional positional argument for destination name in case of git repository
add_parser.add_argument(
"destination",
nargs="?",
default=None,
help="destination to clone git repository into (defaults to cache directory)",
)
add_parser.add_argument(
"--name",
action="store",
help="config name for the package repository, defaults to the namespace of the repository",
)
add_parser.add_argument(
"--path",
help="relative path to the Spack package repository inside a git repository. Can be "
"repeated to add multiple package repositories in case of a monorepo",
action="append",
default=[],
)
add_parser.add_argument(
"--scope",
action=arguments.ConfigScope,
default=lambda: spack.config.CONFIG.default_modify_scope(),
help="configuration scope to modify",
)
# Set (modify existing repository configuration)
set_parser = sp.add_parser(
"set", description=doc_dedented(repo_set), help=doc_first_line(repo_set)
)
set_parser.add_argument("namespace", help="namespace of a Spack package repository")
set_parser.add_argument(
"--destination", help="destination to clone git repository into", action="store"
)
set_parser.add_argument(
"--path",
help="relative path to the Spack package repository inside a git repository. Can be "
"repeated to add multiple package repositories in case of a monorepo",
action="append",
default=[],
)
set_parser.add_argument(
"--scope",
action=arguments.ConfigScope,
default=lambda: spack.config.CONFIG.default_modify_scope(),
help="configuration scope to modify",
)
# Remove
remove_parser = sp.add_parser(
"remove",
description=doc_dedented(repo_remove),
help=doc_first_line(repo_remove),
aliases=["rm"],
)
remove_parser.add_argument(
"namespace_or_path", help="namespace or path of a Spack package repository"
)
remove_parser.add_argument(
"--scope", action=arguments.ConfigScope, default=None, help="configuration scope to modify"
)
remove_parser.add_argument(
"--all-scopes",
action="store_true",
default=False,
help="remove from all config scopes (default: highest scope with matching repo)",
)
# Migrate
migrate_parser = sp.add_parser(
"migrate", description=doc_dedented(repo_migrate), help=doc_first_line(repo_migrate)
)
migrate_parser.add_argument(
"namespace_or_path", help="path to a Spack package repository directory"
)
patch_or_fix = migrate_parser.add_mutually_exclusive_group(required=True)
patch_or_fix.add_argument(
"--dry-run",
action="store_true",
help="do not modify the repository, but dump a patch file",
)
patch_or_fix.add_argument(
"--fix",
action="store_true",
help="automatically migrate the repository to the latest Package API",
)
# Update
update_parser = sp.add_parser(
"update", description=doc_dedented(repo_update), help=doc_first_line(repo_update)
)
update_parser.add_argument("names", nargs="*", default=[], help="repositories to update")
update_parser.add_argument(
"--remote",
"-r",
default="origin",
nargs="?",
help="name of remote to check for branches, tags, or commits",
)
update_parser.add_argument(
"--scope",
action=arguments.ConfigScope,
default=lambda: spack.config.CONFIG.default_modify_scope(),
help="configuration scope to modify",
)
update_parser.add_argument(
"--branch", "-b", nargs="?", default=None, help="name of a branch to change to"
)
refspec = update_parser.add_mutually_exclusive_group(required=False)
refspec.add_argument("--tag", "-t", nargs="?", default=None, help="name of a tag to change to")
refspec.add_argument(
"--commit", "-c", nargs="?", default=None, help="name of a commit to change to"
)
# Show updates
show_version_updates_parser = sp.add_parser(
"show-version-updates", help=repo_show_version_updates.__doc__
)
show_version_updates_parser.add_argument(
"--no-manual-packages", action="store_true", help="exclude manual packages"
)
show_version_updates_parser.add_argument(
"--no-git-versions", action="store_true", help="exclude versions from git"
)
show_version_updates_parser.add_argument(
"--only-redistributable", action="store_true", help="exclude non-redistributable packages"
)
show_version_updates_parser.add_argument(
"--no-deprecated", action="store_true", help="exclude deprecated versions"
)
show_version_updates_parser.add_argument(
"repository", help="name or path of the repository to analyze"
)
show_version_updates_parser.add_argument(
"from_ref", help="git ref from which to start looking at changes"
)
show_version_updates_parser.add_argument("to_ref", help="git ref to end looking at changes")
[docs]
def repo_create(args):
"""create a new package repository"""
full_path, namespace = spack.repo.create_repo(args.directory, args.namespace, args.subdir)
tty.msg("Created repo with namespace '%s'." % namespace)
tty.msg("To register it with spack, run this command:", "spack repo add %s" % full_path)
def _add_repo(
path_or_repo: str,
name: Optional[str],
scope: Optional[str],
paths: List[str],
destination: Optional[str],
config: Optional[spack.config.Configuration] = None,
) -> str:
config = config or spack.config.CONFIG
existing: Dict[str, Any] = config.get("repos", default={}, scope=scope)
if name and name in existing:
raise SpackError(f"A repository with the name '{name}' already exists.")
# Interpret as a git URL when it contains a colon at index 2 or more, not preceded by a
# forward slash. That allows C:/ windows paths, while following git's convention to distinguish
# between local paths on the one hand and URLs and SCP like syntax on the other.
entry: Union[str, Dict[str, Any]]
colon_idx = path_or_repo.find(":")
if colon_idx > 1 and "/" not in path_or_repo[:colon_idx]: # git URL
entry = {"git": path_or_repo}
if len(paths) >= 1:
entry["paths"] = paths
if destination:
entry["destination"] = destination
else: # local path
if destination:
raise SpackError("The 'destination' argument is only valid for git repositories")
elif paths:
raise SpackError("The --paths flag is only valid for git repositories")
entry = spack.config.canonicalize_path(path_or_repo)
descriptor = spack.repo.parse_config_descriptor(
name or "<unnamed>", entry, lock=spack.repo.package_repository_lock(spack.config.CONFIG)
)
descriptor.initialize(git=spack.util.executable.which("git"))
packages_repos = descriptor.construct(cache=spack.caches.MISC_CACHE)
usable_repos: Dict[str, spack.repo.Repo] = {}
for _path, _repo_or_err in packages_repos.items():
if isinstance(_repo_or_err, Exception):
tty.warn(f"Skipping package repository '{_path}' due to: {_repo_or_err}")
else:
usable_repos[_path] = _repo_or_err
if not usable_repos:
raise SpackError(f"No package repository could be constructed from {path_or_repo}")
# For the config key, default to --name, then to the namespace if there's only one repo.
# Otherwise, the name is unclear and we require the user to specify it.
if name:
key = name
elif len(usable_repos) == 1:
key = next(iter(usable_repos.values())).namespace
else:
raise SpackError("Multiple package repositories found, please specify a name with --name.")
if key in existing:
raise SpackError(f"A repository with the name '{key}' already exists.")
# Prepend the new repository
config.set("repos", spack.util.spack_yaml.syaml_dict({key: entry, **existing}), scope)
return key
[docs]
def repo_add(args):
"""add package repositories to Spack's configuration"""
name = _add_repo(
path_or_repo=args.path_or_repo,
name=args.name,
scope=args.scope,
paths=args.path,
destination=args.destination,
)
tty.msg(f"Added repo to config with name '{name}'.")
[docs]
def repo_remove(args):
"""remove a repository from Spack's configuration"""
scopes = [args.scope] if args.scope else reversed(list(spack.config.CONFIG.scopes.keys()))
found_and_removed = False
for scope in scopes:
found_and_removed |= _remove_repo(args.namespace_or_path, scope)
if found_and_removed and not args.all_scopes:
return
if not found_and_removed:
tty.die(f"No repository with path or namespace: {args.namespace_or_path}")
def _remove_repo(namespace_or_path, scope):
repos: Dict[str, str] = spack.config.CONFIG.get("repos", scope=scope)
if namespace_or_path in repos:
# delete by name (from config)
key = namespace_or_path
else:
# delete by namespace or path (requires constructing the repo)
canon_path = spack.config.canonicalize_path(namespace_or_path)
descriptors = spack.repo.RepoDescriptors.from_config(spack.config.CONFIG, scope=scope)
for name, descriptor in descriptors.items():
descriptor.initialize(fetch=False)
# For now you cannot delete monorepos with multiple package repositories from config,
# hence "all" and not "any". We can improve this later if needed.
if all(
r.namespace == namespace_or_path or r.root == canon_path
for r in descriptor.construct(cache=spack.caches.MISC_CACHE).values()
if isinstance(r, spack.repo.Repo)
):
key = name
break
else:
return False
del repos[key]
spack.config.CONFIG.set("repos", repos, scope)
tty.msg(f"Removed repository '{namespace_or_path}' from scope '{scope}'.")
return True
[docs]
def repo_list(args):
"""show registered repositories and their namespaces
List all package repositories known to Spack. Repositories
can be local directories or remote git repositories.
"""
descriptors = spack.repo.RepoDescriptors.from_config(spack.config.CONFIG, scope=args.scope)
# --names: just print config names
if args.names:
for name in descriptors:
print(name)
return
# --namespaces: print all repo namespaces
if args.namespaces:
for name, path, maybe_repo in _iter_repos_from_descriptors(descriptors):
if isinstance(maybe_repo, spack.repo.Repo):
print(maybe_repo.namespace)
return
# Collect all repository information
repo_info = []
for name, path, maybe_repo in _iter_repos_from_descriptors(descriptors):
if isinstance(maybe_repo, spack.repo.Repo):
status = "installed"
namespace = maybe_repo.namespace
api = maybe_repo.package_api_str
repo_path = maybe_repo.root
elif maybe_repo is None: # Uninitialized Git-based repo case
status = "uninitialized"
namespace = name
api = ""
repo_path = path
else: # Exception/error case
status = "error"
namespace = name
api = ""
repo_path = path
# Add the repo info to our list
repo_info.append(
{
"name": name,
"namespace": namespace,
"path": repo_path,
"api_version": api,
"status": status,
"error": str(maybe_repo) if isinstance(maybe_repo, Exception) else None,
}
)
# Output in JSON format if requested
if args.json:
sjson.dump(repo_info, sys.stdout)
return
# Default table format with aligned output
formatted_repo_info = []
for repo in repo_info:
if repo["status"] == "installed":
status = "@g{[+]}"
elif repo["status"] == "uninitialized":
status = "@K{ - }"
else: # error
status = "@r{[-]}"
formatted_repo_info.append((status, repo["namespace"], repo["api_version"], repo["path"]))
if formatted_repo_info:
max_namespace_width = max(len(namespace) for _, namespace, _, _ in formatted_repo_info) + 3
max_api_width = max(len(api) for _, _, api, _ in formatted_repo_info) + 3
# Print aligned output
for status, namespace, api, path in formatted_repo_info:
cpath = color.cescape(path)
color.cprint(
f"{status} {namespace:<{max_namespace_width}} {api:<{max_api_width}} {cpath}"
)
def _get_repo(name_or_path: str) -> Optional[spack.repo.Repo]:
"""get a repo by path or namespace"""
try:
return spack.repo.from_path(name_or_path)
except spack.repo.RepoError:
pass
descriptors = spack.repo.RepoDescriptors.from_config(spack.config.CONFIG)
repo_path, _ = descriptors.construct(cache=spack.caches.MISC_CACHE, fetch=False)
for repo in repo_path.repos:
if repo.namespace == name_or_path:
return repo
return None
[docs]
def repo_migrate(args: Any) -> int:
"""migrate a package repository to the latest Package API"""
from spack.repo_migrate import migrate_v1_to_v2, migrate_v2_imports
repo = _get_repo(args.namespace_or_path)
if repo is None:
tty.die(f"No such repository: {args.namespace_or_path}")
if args.dry_run:
fd, patch_file_path = tempfile.mkstemp(
suffix=".patch", prefix="repo-migrate-", dir=os.getcwd()
)
patch_file = os.fdopen(fd, "bw")
tty.msg(f"Patch file will be written to {patch_file_path}")
else:
patch_file_path = None
patch_file = None
try:
if (1, 0) <= repo.package_api < (2, 0):
success, repo_v2 = migrate_v1_to_v2(repo, patch_file=patch_file)
exit_code = 0 if success else 1
elif (2, 0) <= repo.package_api < (3, 0):
repo_v2 = None
exit_code = (
0
if migrate_v2_imports(repo.packages_path, repo.root, patch_file=patch_file)
else 1
)
else:
repo_v2 = None
exit_code = 0
finally:
if patch_file is not None:
patch_file.flush()
patch_file.close()
if patch_file_path:
tty.warn(
f"No changes were made to the '{repo.namespace}' repository with. Review "
f"the changes written to {patch_file_path}. Run \n\n"
f" spack repo migrate --fix {args.namespace_or_path}\n\n"
"to upgrade the repo."
)
elif exit_code == 1:
tty.error(
f"Repository '{repo.namespace}' could not be migrated to the latest Package API. "
"Please check the error messages above."
)
elif isinstance(repo_v2, spack.repo.Repo):
tty.info(
f"Repository '{repo_v2.namespace}' was successfully migrated from "
f"package API {repo.package_api_str} to {repo_v2.package_api_str}."
)
tty.warn(
"Remove the old repository from Spack's configuration and add the new one using:\n"
f" spack repo remove {shlex.quote(repo.root)}\n"
f" spack repo add {shlex.quote(repo_v2.root)}"
)
else:
tty.info(f"Repository '{repo.namespace}' was successfully migrated")
return exit_code
[docs]
def repo_set(args):
"""modify an existing repository configuration"""
namespace = args.namespace
# First, check if the repository exists across all scopes for validation
all_repos: Dict[str, Any] = spack.config.CONFIG.get("repos", default={})
if namespace not in all_repos:
raise SpackError(f"No repository with namespace '{namespace}' found in configuration.")
# Validate that it's a git repository
if not isinstance(all_repos[namespace], dict):
raise SpackError(
f"Repository '{namespace}' is not a git repository. "
"The 'set' command only works with git repositories."
)
# Now get the repos for the specific scope we're modifying
scope_repos: Dict[str, Any] = spack.config.CONFIG.get("repos", default={}, scope=args.scope)
updated_entry = scope_repos[namespace] if namespace in scope_repos else {}
if args.destination:
updated_entry["destination"] = args.destination
if args.path:
updated_entry["paths"] = args.path
scope_repos[namespace] = updated_entry
spack.config.CONFIG.set("repos", scope_repos, args.scope)
tty.msg(f"Updated repo '{namespace}'")
def _iter_repos_from_descriptors(
descriptors: spack.repo.RepoDescriptors,
) -> Generator[Tuple[str, str, Union[spack.repo.Repo, Exception, None]], None, None]:
"""Iterate through repository descriptors and yield (name, path, maybe_repo) tuples.
Yields:
Tuple of (config_name, path, maybe_repo) where maybe_repo is a Repo instance if it could
be instantiated, an Exception if it could not be instantiated, or None if it was not
initialized yet.
"""
for name, descriptor in descriptors.items():
descriptor.initialize(fetch=False)
repos_for_descriptor = descriptor.construct(cache=spack.caches.MISC_CACHE)
for path, maybe_repo in repos_for_descriptor.items():
yield name, path, maybe_repo
# If there are no repos, it means it's not yet cloned; yield descriptor info
if not repos_for_descriptor and isinstance(descriptor, spack.repo.RemoteRepoDescriptor):
yield name, descriptor.repository, None # None indicates remote descriptor
[docs]
def repo_update(args):
"""update one or more package repositories"""
descriptors = spack.repo.RepoDescriptors.from_config(spack.config.CONFIG)
git_flags = ["commit", "tag", "branch"]
active_flag = next((attr for attr in git_flags if getattr(args, attr)), None)
if active_flag and len(args.names) != 1:
raise SpackError(
f"Unable to set --{active_flag} because more than one namespace was given."
if len(args.names) > 1
else f"Unable to apply --{active_flag} without a namespace"
)
for name in args.names:
if name not in descriptors:
raise SpackError(f"{name} is not a known repository name.")
# filter descriptors when namespaces are provided as arguments
descriptors = spack.repo.RepoDescriptors(
{name: descriptor for name, descriptor in descriptors.items() if name in args.names}
)
# Get the repos for the specific scope we're modifying
scope_repos: Dict[str, Any] = spack.config.CONFIG.get("repos", default={}, scope=args.scope)
for name, descriptor in descriptors.items():
if not isinstance(descriptor, spack.repo.RemoteRepoDescriptor):
continue
if active_flag:
# update the git commit, tag, or branch of the descriptor
setattr(descriptor, active_flag, getattr(args, active_flag))
updated_entry = scope_repos[name] if name in scope_repos else {}
# prune previous values of git fields
for entry in {"commit", "tag"} - {active_flag}:
setattr(descriptor, entry, None)
updated_entry.pop(entry, None)
updated_entry[active_flag] = args.commit or args.tag or args.branch
scope_repos[name] = updated_entry
git = spack.util.git.git(required=True)
previous_commit = descriptor.get_commit(git=git)
descriptor.update(git=git, remote=args.remote)
new_commit = descriptor.get_commit(git=git)
if previous_commit == new_commit:
tty.msg(f"{name}: Already up to date.")
else:
fails = [
r
for r in descriptor.construct(cache=spack.caches.MISC_CACHE).values()
if type(r) is spack.repo.BadRepoVersionError
]
if fails:
min_ver = ".".join(str(n) for n in spack.min_package_api_version)
max_ver = ".".join(str(n) for n in spack.package_api_version)
tty.error(
f"{name}: repo is too new for this version of Spack. ",
f" Spack supports API v{min_ver} to v{max_ver}, but repo is {fails[0].api}",
" Please upgrade Spack or revert with:\n",
f" spack repo update --commit {previous_commit}\n",
)
else:
tty.msg(f"{name}: Updated successfully.")
if active_flag:
spack.config.CONFIG.set("repos", scope_repos, args.scope)
[docs]
def repo_show_version_updates(args):
"""show version specs that were added between two commits"""
# Get the repository by name or path
repo = _get_repo(args.repository)
if repo is None:
tty.die(f"No such repository: {args.repository}")
# Get packages that were changed or added between the refs
pkgs = spack.repo.get_all_package_diffs("AC", repo, args.from_ref, args.to_ref)
# Filter out manual packages if requested
if args.no_manual_packages:
pkgs = {pkg_name for pkg_name in pkgs if not repo.get_pkg_class(pkg_name).manual_download}
if not pkgs:
tty.info("No packages were added or changed between the specified refs", stream=sys.stderr)
return 0
# Collect version specs that were added
specs_to_output = []
for pkg_name in pkgs:
pkg_cls = repo.get_pkg_class(pkg_name)
path = repo.filename_for_package_name(pkg_name)
# Get all versions with checksums or commits
version_to_checksum: Dict[StandardVersion, str] = {}
for version in pkg_cls.versions:
version_dict = pkg_cls.versions[version]
if "sha256" in version_dict:
version_to_checksum[version] = version_dict["sha256"]
elif "commit" in version_dict:
version_to_checksum[version] = version_dict["commit"]
# Find versions added between the refs
with fs.working_dir(os.path.dirname(path)):
added_checksums = spack.ci.filter_added_checksums(
version_to_checksum.values(), path, from_ref=args.from_ref, to_ref=args.to_ref
)
new_versions = [v for v, c in version_to_checksum.items() if c in added_checksums]
# Create specs for new versions
for version in new_versions:
version_spec = spack.spec.Spec(pkg_name)
version_spec.constrain(f"@={version}")
specs_to_output.append(version_spec)
# Filter out git versions if requested
if args.no_git_versions:
specs_to_output = [
spec
for spec in specs_to_output
if "commit" not in repo.get_pkg_class(spec.name).versions[spec.version]
]
# Filter out non-redistributable packages if requested
if args.only_redistributable:
specs_to_output = [
spec
for spec in specs_to_output
if repo.get_pkg_class(spec.name).redistribute_source(spec)
]
# Filter out deprecated versions if requested
if args.no_deprecated:
specs_to_output = [
spec
for spec in specs_to_output
if not spack.package_base.deprecated_version(
repo.get_pkg_class(spec.name), spec.version
)
]
if not specs_to_output:
tty.info("No new package versions found between the specified refs", stream=sys.stderr)
return 0
# Output specs one per line
for spec in specs_to_output:
print(spec)
[docs]
def repo(parser, args):
return {
"create": repo_create,
"list": repo_list,
"ls": repo_list,
"add": repo_add,
"set": repo_set,
"remove": repo_remove,
"rm": repo_remove,
"migrate": repo_migrate,
"update": repo_update,
"show-version-updates": repo_show_version_updates,
}[args.repo_command](args)