# Copyright Spack Project Developers. See COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import functools
import os
import posixpath
import urllib.error
import urllib.parse
import urllib.request
import urllib.response
from io import BufferedReader, BytesIO, IOBase
from typing import Any, Dict, List, Optional, Tuple
from spack.vendor.typing_extensions import Literal
import spack.config
import spack.error
from spack.util import tty
#: Map (mirror name, method) tuples to s3 client instances.
s3_client_cache: Dict[Tuple[Optional[str], str], Any] = dict()
#: Allowed HTTP request methods
S3OpenMethod = Literal["get", "head", "GET", "HEAD"]
#: Allowed mirror direction selection names for S3 Mirrors
MirrorDirection = Literal["push", "fetch"]
def _get_s3_session(url, method: Literal[S3OpenMethod, MirrorDirection] = "fetch"):
# import boto and friends as late as possible. We don't want to require boto as a
# dependency unless the user actually wants to access S3 mirrors.
from boto3 import Session
from botocore import UNSIGNED
from botocore.client import Config
from botocore.exceptions import ClientError
# translate method to fetch/push
method = method.lower()
if method not in ("fetch", "push"):
method = "fetch" if method in ("get", "head") else "push"
# Circular dependency
from spack.mirrors.mirror import MirrorCollection
global s3_client_cache
# Parse the URL if not already done.
if not isinstance(url, urllib.parse.ParseResult):
url = urllib.parse.urlparse(url)
url_str = url.geturl()
def get_mirror_url(mirror):
return mirror.fetch_url if method == "fetch" else mirror.push_url
# Get all configured mirrors that could match.
all_mirrors = MirrorCollection()
mirrors = [
(name, mirror)
for name, mirror in all_mirrors.items()
if url_str.startswith(get_mirror_url(mirror))
]
name: Optional[str] = None
mirror: Any = {}
if mirrors:
# In case we have more than one mirror, we pick the longest matching url.
# The heuristic being that it's more specific, and you can have different
# credentials for a sub-bucket (if that is a thing).
name, mirror = max(
mirrors, key=lambda name_and_mirror: len(get_mirror_url(name_and_mirror[1]))
)
key = (name, method)
# Did we already create a client for this? Then return it.
if key in s3_client_cache:
return s3_client_cache[key], url
# Otherwise, create it.
s3_connection, s3_client_args = get_mirror_s3_connection_info(mirror, method)
session = Session(**s3_connection)
# if no access credentials provided above, then access anonymously
if not session.get_credentials():
s3_client_args["config"] = Config(signature_version=UNSIGNED)
client = session.client("s3", **s3_client_args)
client.ClientError = ClientError
# Cache the client.
s3_client_cache[key] = client
return client, url
def _parse_s3_endpoint_url(endpoint_url):
if not urllib.parse.urlparse(endpoint_url, scheme="").scheme:
endpoint_url = f"https://{endpoint_url}"
return endpoint_url
[docs]
def get_mirror_s3_connection_info(mirror, method):
"""Create s3 config for session/client from a Mirror instance (or just set defaults
when no mirror is given.)"""
from spack.mirrors.mirror import Mirror
s3_connection = {}
s3_client_args = {"use_ssl": spack.config.CONFIG.get("config:verify_ssl")}
# access token
if isinstance(mirror, Mirror):
credentials = mirror.get_credentials(method)
if credentials:
if "access_token" in credentials:
s3_connection["aws_session_token"] = credentials["access_token"]
if "access_pair" in credentials:
s3_connection["aws_access_key_id"] = credentials["access_pair"][0]
s3_connection["aws_secret_access_key"] = credentials["access_pair"][1]
if "profile" in credentials:
s3_connection["profile_name"] = credentials["profile"]
# endpoint url
endpoint_url = mirror.get_endpoint_url(method) or os.environ.get("S3_ENDPOINT_URL")
else:
endpoint_url = os.environ.get("S3_ENDPOINT_URL")
if endpoint_url:
s3_client_args["endpoint_url"] = _parse_s3_endpoint_url(endpoint_url)
return s3_connection, s3_client_args
# NOTE(opadron): Workaround issue in boto where its StreamingBody
# implementation is missing several APIs expected from IOBase. These missing
# APIs prevent the streams returned by boto from being passed as-are along to
# urllib.
#
# https://github.com/boto/botocore/issues/879
# https://github.com/python/cpython/pull/3249
[docs]
class WrapStream(BufferedReader):
def __init__(self, raw):
# In botocore >=1.23.47, StreamingBody inherits from IOBase, so we
# only add missing attributes in older versions.
# https://github.com/boto/botocore/commit/a624815eabac50442ed7404f3c4f2664cd0aa784
if not isinstance(raw, IOBase):
raw.readable = lambda: True
raw.writable = lambda: False
raw.seekable = lambda: False
raw.closed = False
raw.flush = lambda: None
super().__init__(raw)
[docs]
def detach(self):
self.raw = None
[docs]
def read(self, *args, **kwargs):
return self.raw.read(*args, **kwargs)
def __getattr__(self, key):
return getattr(self.raw, key)
def _s3_open(url, method: S3OpenMethod = "GET"):
s3, parsed = _get_s3_session(url, method=method)
bucket = parsed.netloc
key = parsed.path
if key.startswith("/"):
key = key[1:]
if method not in ("GET", "HEAD"):
raise urllib.error.URLError(
"Only GET and HEAD verbs are currently supported for the s3:// scheme"
)
stream: IOBase
try:
if method == "GET":
obj = s3.get_object(Bucket=bucket, Key=key)
# NOTE(opadron): Apply workaround here (see above)
stream = WrapStream(obj["Body"])
elif method == "HEAD":
obj = s3.head_object(Bucket=bucket, Key=key)
stream = BytesIO()
except s3.ClientError as e:
raise urllib.error.URLError(e) from e
headers = obj["ResponseMetadata"]["HTTPHeaders"]
return url, headers, stream
[docs]
def s3_command(method: MirrorDirection):
"""Bind the correct S3 session and capture errors from Boto3."""
def _s3_decorate_command(command):
@functools.wraps(command)
def _s3_command_wrapped(url, *args, **kwargs):
s3, url = _get_s3_session(url, method=method)
try:
return command(s3, url, *args, **kwargs)
except s3.ClientError as e:
raise urllib.error.URLError(e) from e
return _s3_command_wrapped
return _s3_decorate_command
[docs]
class UrllibS3Handler(urllib.request.BaseHandler):
[docs]
def s3_open(self, req):
orig_url = req.get_full_url()
url, headers, stream = _s3_open(orig_url, method=req.get_method())
return urllib.response.addinfourl(stream, headers, url)
def _relative_key(key, prefix):
if not key.startswith("/"):
key = "/" + key
if not prefix.startswith("/"):
prefix = "/" + prefix
# S3 keys are always POSIX-style, regardless of the host OS.
key = posixpath.relpath(key, prefix)
if key == ".":
return None
return key
def _iter_s3_prefix(s3, url, relative: bool = False, num_entries: int = 1024):
bucket = url.netloc
stripped_path = url.path.strip("/")
prefix = f"{stripped_path}/" if stripped_path else ""
paginator = s3.get_paginator("list_objects_v2")
pages = paginator.paginate(Bucket=bucket, Prefix=prefix, MaxKeys=num_entries)
for item in pages.search("Contents"):
if not item:
continue
if relative:
key = _relative_key(item["Key"], prefix)
else:
key = item["Key"]
if key is not None:
yield key
[docs]
@s3_command("fetch")
def list_objects(s3, url: urllib.parse.ParseResult, recursive: bool = False):
"""List contents under a url.
Args:
url: S3 URL (ie. s3://bucket/some/prefix/)
recursive: List prefix recursively.
Returns:
List of keys under the bucket/prefix.
"""
if recursive:
return list(_iter_s3_prefix(s3, url, relative=True))
return list({key.split("/", 1)[0] for key in _iter_s3_prefix(s3, url, relative=True)})
def _debug_print_delete_results(result):
if "Deleted" in result:
for d in result["Deleted"]:
tty.debug("Deleted {0}".format(d["Key"]))
if "Errors" in result:
for e in result["Errors"]:
tty.debug("Failed to delete {0} ({1})".format(e["Key"], e["Message"]))
[docs]
@s3_command("push")
def delete_objects(s3, url: urllib.parse.ParseResult, recursive: bool = False):
# Try to find a mirror for potential connection information
bucket = url.netloc
if not recursive:
s3.delete_object(Bucket=bucket, Key=url.path.lstrip("/"))
return
# Because list_objects_v2 can only return up to 1000 items
# at a time, we have to paginate to make sure we get it all
delete_request: Dict[str, List[Dict[str, str]]] = {"Objects": []}
for key in _iter_s3_prefix(s3, url, relative=False):
delete_request["Objects"].append({"Key": key})
# Make sure we do not try to hit S3 with a list of more
# than 1000 items
if len(delete_request["Objects"]) >= 1000:
r = s3.delete_objects(Bucket=bucket, Delete=delete_request)
_debug_print_delete_results(r)
delete_request = {"Objects": []}
# Delete any items that remain
if len(delete_request["Objects"]):
r = s3.delete_objects(Bucket=bucket, Delete=delete_request)
_debug_print_delete_results(r)
[docs]
@s3_command("fetch")
def stat_object(s3, url: urllib.parse.ParseResult) -> Optional[Tuple[int, float]]:
"""Get stat result for a URL.
Args:
url: URL to get stat result for
Returns:
A tuple of (size, mtime) if the URL exists, None otherwise.
"""
s3_bucket = url.netloc
s3_key = url.path.lstrip("/")
try:
head_request = s3.head_object(Bucket=s3_bucket, Key=s3_key)
except s3.ClientError as e:
if e.response["Error"]["Code"] == "404":
return None
raise e
mtime = head_request["LastModified"].timestamp()
size = head_request["ContentLength"]
return size, mtime
[docs]
@s3_command("push")
def push_object(s3, url: urllib.parse.ParseResult, local_file_path, extra_args):
if extra_args is None:
extra_args = {}
remote_path = url.path
while remote_path.startswith("/"):
remote_path = remote_path[1:]
if extra_args.get("IfMatch") is not None:
# IfMatch is only supported by put_object which has additional limitations
if os.stat(local_file_path).st_size >= 5e9:
raise spack.error.SpackError(f"File too large (max. 5GB): {local_file_path}")
with open(local_file_path, "rb") as fd:
s3.put_object(Bucket=url.netloc, Key=remote_path, Body=fd, **extra_args)
else:
s3.upload_file(local_file_path, url.netloc, remote_path, ExtraArgs=extra_args)