diff --git a/.github/workflows/run-crt-test.yml b/.github/workflows/run-crt-test.yml new file mode 100644 index 0000000000..8132b44873 --- /dev/null +++ b/.github/workflows/run-crt-test.yml @@ -0,0 +1,31 @@ +name: Run CRT tests + +on: + push: + pull_request: + branches-ignore: [master] + +permissions: + contents: read + +jobs: + build: + runs-on: '${{ matrix.os }}' + strategy: + fail-fast: false + matrix: + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + os: [ubuntu-latest, macOS-latest, windows-latest] + + steps: + - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 + - name: 'Set up Python ${{ matrix.python-version }}' + uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 + with: + python-version: '${{ matrix.python-version }}' + - name: Install dependencies and CRT + run: | + python scripts/ci/install --extras crt + - name: Run tests + run: | + python scripts/ci/run-crt-tests --with-cov --with-xdist diff --git a/boto3/crt.py b/boto3/crt.py new file mode 100644 index 0000000000..4b8df3140e --- /dev/null +++ b/boto3/crt.py @@ -0,0 +1,167 @@ +# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# https://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +""" +This file contains private functionality for interacting with the AWS +Common Runtime library (awscrt) in boto3. + +All code contained within this file is for internal usage within this +project and is not intended for external consumption. All interfaces +contained within are subject to abrupt breaking changes. +""" + +import threading + +import botocore.exceptions +from botocore.session import Session +from s3transfer.crt import ( + BotocoreCRTCredentialsWrapper, + BotocoreCRTRequestSerializer, + CRTTransferManager, + acquire_crt_s3_process_lock, + create_s3_crt_client, +) + +# Singletons for CRT-backed transfers +CRT_S3_CLIENT = None +BOTOCORE_CRT_SERIALIZER = None + +CLIENT_CREATION_LOCK = threading.Lock() +PROCESS_LOCK_NAME = 'boto3' + + +def _create_crt_client(session, config, region_name, cred_provider): + """Create a CRT S3 Client for file transfer. + + Instantiating many of these may lead to degraded performance or + system resource exhaustion. + """ + create_crt_client_kwargs = { + 'region': region_name, + 'use_ssl': True, + 'crt_credentials_provider': cred_provider, + } + return create_s3_crt_client(**create_crt_client_kwargs) + + +def _create_crt_request_serializer(session, region_name): + return BotocoreCRTRequestSerializer( + session, {'region_name': region_name, 'endpoint_url': None} + ) + + +def _create_crt_s3_client( + session, config, region_name, credentials, lock, **kwargs +): + """Create boto3 wrapper class to manage crt lock reference and S3 client.""" + cred_wrapper = BotocoreCRTCredentialsWrapper(credentials) + cred_provider = cred_wrapper.to_crt_credentials_provider() + return CRTS3Client( + _create_crt_client(session, config, region_name, cred_provider), + lock, + region_name, + cred_wrapper, + ) + + +def _initialize_crt_transfer_primatives(client, config): + lock = acquire_crt_s3_process_lock(PROCESS_LOCK_NAME) + if lock is None: + # If we're unable to acquire the lock, we cannot + # use the CRT in this process and should default to + # the classic s3transfer manager. + return None, None + + session = Session() + region_name = client.meta.region_name + credentials = client._get_credentials() + + serializer = _create_crt_request_serializer(session, region_name) + s3_client = _create_crt_s3_client( + session, config, region_name, credentials, lock + ) + return serializer, s3_client + + +def get_crt_s3_client(client, config): + global CRT_S3_CLIENT + global BOTOCORE_CRT_SERIALIZER + + with CLIENT_CREATION_LOCK: + if CRT_S3_CLIENT is None: + serializer, s3_client = _initialize_crt_transfer_primatives( + client, config + ) + BOTOCORE_CRT_SERIALIZER = serializer + CRT_S3_CLIENT = s3_client + + return CRT_S3_CLIENT + + +class CRTS3Client: + """ + This wrapper keeps track of our underlying CRT client, the lock used to + acquire it and the region we've used to instantiate the client. + + Due to limitations in the existing CRT interfaces, we can only make calls + in a single region and does not support redirects. We track the region to + ensure we don't use the CRT client when a successful request cannot be made. + """ + + def __init__(self, crt_client, process_lock, region, cred_provider): + self.crt_client = crt_client + self.process_lock = process_lock + self.region = region + self.cred_provider = cred_provider + + +def is_crt_compatible_request(client, crt_s3_client): + """ + Boto3 client must use same signing region and credentials + as the CRT_S3_CLIENT singleton. Otherwise fallback to classic. + """ + if crt_s3_client is None: + return False + + boto3_creds = client._get_credentials() + if boto3_creds is None: + return False + + is_same_identity = compare_identity( + boto3_creds.get_frozen_credentials(), crt_s3_client.cred_provider + ) + is_same_region = client.meta.region_name == crt_s3_client.region + return is_same_region and is_same_identity + + +def compare_identity(boto3_creds, crt_s3_creds): + try: + crt_creds = crt_s3_creds() + except botocore.exceptions.NoCredentialsError: + return False + + is_matching_identity = ( + boto3_creds.access_key == crt_creds.access_key_id + and boto3_creds.secret_key == crt_creds.secret_access_key + and boto3_creds.token == crt_creds.session_token + ) + return is_matching_identity + + +def create_crt_transfer_manager(client, config): + """Create a CRTTransferManager for optimized data transfer.""" + crt_s3_client = get_crt_s3_client(client, config) + if is_crt_compatible_request(client, crt_s3_client): + return CRTTransferManager( + crt_s3_client.crt_client, BOTOCORE_CRT_SERIALIZER + ) + return None diff --git a/boto3/s3/constants.py b/boto3/s3/constants.py new file mode 100644 index 0000000000..c7f691fc21 --- /dev/null +++ b/boto3/s3/constants.py @@ -0,0 +1,17 @@ +# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# https://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + + +# TransferConfig preferred_transfer_client settings +CLASSIC_TRANSFER_CLIENT = "classic" +AUTO_RESOLVE_TRANSFER_CLIENT = "auto" diff --git a/boto3/s3/inject.py b/boto3/s3/inject.py index c62dc3ce22..440be5a8be 100644 --- a/boto3/s3/inject.py +++ b/boto3/s3/inject.py @@ -10,6 +10,8 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +import copy as python_copy + from botocore.exceptions import ClientError from boto3 import utils @@ -432,7 +434,11 @@ def copy( if config is None: config = TransferConfig() - with create_transfer_manager(self, config) as manager: + # copy is not supported in the CRT + new_config = python_copy.copy(config) + new_config.preferred_transfer_client = "classic" + + with create_transfer_manager(self, new_config) as manager: future = manager.copy( copy_source=CopySource, bucket=Bucket, diff --git a/boto3/s3/transfer.py b/boto3/s3/transfer.py index cce5155d43..27e5471474 100644 --- a/boto3/s3/transfer.py +++ b/boto3/s3/transfer.py @@ -122,8 +122,11 @@ def __call__(self, bytes_amount): """ -from os import PathLike, fspath +import logging +import threading +from os import PathLike, fspath, getpid +from botocore.compat import HAS_CRT from botocore.exceptions import ClientError from s3transfer.exceptions import ( RetriesExceededError as S3TransferRetriesExceededError, @@ -134,11 +137,19 @@ def __call__(self, bytes_amount): from s3transfer.subscribers import BaseSubscriber from s3transfer.utils import OSUtils +import boto3.s3.constants as constants from boto3.exceptions import RetriesExceededError, S3UploadFailedError +if HAS_CRT: + import awscrt.s3 + + from boto3.crt import create_crt_transfer_manager + KB = 1024 MB = KB * KB +logger = logging.getLogger(__name__) + def create_transfer_manager(client, config, osutil=None): """Creates a transfer manager based on configuration @@ -155,6 +166,50 @@ def create_transfer_manager(client, config, osutil=None): :rtype: s3transfer.manager.TransferManager :returns: A transfer manager based on parameters provided """ + if _should_use_crt(config): + crt_transfer_manager = create_crt_transfer_manager(client, config) + if crt_transfer_manager is not None: + logger.debug( + f"Using CRT client. pid: {getpid()}, thread: {threading.get_ident()}" + ) + return crt_transfer_manager + + # If we don't resolve something above, fallback to the default. + logger.debug( + f"Using default client. pid: {getpid()}, thread: {threading.get_ident()}" + ) + return _create_default_transfer_manager(client, config, osutil) + + +def _should_use_crt(config): + if HAS_CRT: + is_optimized_instance = awscrt.s3.is_optimized_for_system() + else: + is_optimized_instance = False + pref_transfer_client = config.preferred_transfer_client.lower() + + if ( + is_optimized_instance + and pref_transfer_client == constants.AUTO_RESOLVE_TRANSFER_CLIENT + ): + logger.debug( + "Attempting to use CRTTransferManager. Config settings may be ignored." + ) + return True + + logger.debug( + "Opting out of CRT Transfer Manager. Preferred client: " + "{pref_transfer_client}, CRT available: {HAS_CRT}, " + "Instance Optimized: {is_optimized_instance}.", + pref_transfer_client, + HAS_CRT, + is_optimized_instance, + ) + return False + + +def _create_default_transfer_manager(client, config, osutil): + """Create the default TransferManager implementation for s3transfer.""" executor_cls = None if not config.use_threads: executor_cls = NonThreadedExecutor @@ -177,6 +232,7 @@ def __init__( io_chunksize=256 * KB, use_threads=True, max_bandwidth=None, + preferred_transfer_client=constants.AUTO_RESOLVE_TRANSFER_CLIENT, ): """Configuration object for managed S3 transfers @@ -217,6 +273,15 @@ def __init__( :param max_bandwidth: The maximum bandwidth that will be consumed in uploading and downloading file content. The value is an integer in terms of bytes per second. + + :param preferred_transfer_client: String specifying preferred transfer + client for transfer operations. + + Current supported settings are: + * auto (default) - Use the CRTTransferManager when calls + are made with supported environment and settings. + * classic - Only use the origin S3TransferManager with + requests. Disables possible CRT upgrade on requests. """ super().__init__( multipart_threshold=multipart_threshold, @@ -233,6 +298,7 @@ def __init__( for alias in self.ALIAS: setattr(self, alias, getattr(self, self.ALIAS[alias])) self.use_threads = use_threads + self.preferred_transfer_client = preferred_transfer_client def __setattr__(self, name, value): # If the alias name is used, make sure we set the name that it points diff --git a/scripts/ci/install b/scripts/ci/install index 9309459fe4..5b26e7e96b 100755 --- a/scripts/ci/install +++ b/scripts/ci/install @@ -1,4 +1,5 @@ #!/usr/bin/env python +import argparse import os import shutil from contextlib import contextmanager @@ -25,6 +26,14 @@ def run(command): if __name__ == "__main__": + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group() + group.add_argument( + '-e', + '--extras', + help='Install extras_require along with normal install', + ) + args = parser.parse_args() with cd(REPO_ROOT): run("pip install -r requirements.txt") run("python scripts/ci/install-dev-deps") @@ -32,4 +41,7 @@ if __name__ == "__main__": shutil.rmtree("dist") run("python setup.py bdist_wheel") wheel_dist = os.listdir("dist")[0] - run("pip install %s" % (os.path.join("dist", wheel_dist))) + package = os.path.join('dist', wheel_dist) + if args.extras: + package = f"\"{package}[{args.extras}]\"" + run(f"pip install {package}") diff --git a/scripts/ci/run-crt-tests b/scripts/ci/run-crt-tests index ceb607160f..b9e6ea06f3 100755 --- a/scripts/ci/run-crt-tests +++ b/scripts/ci/run-crt-tests @@ -36,4 +36,5 @@ except ImportError: if __name__ == "__main__": with cd(os.path.join(REPO_ROOT, "tests")): - run(f"{REPO_ROOT}/scripts/ci/run-tests unit/ functional/") + test_script = os.sep.join([REPO_ROOT, 'scripts', 'ci', 'run-tests']) + run(f"python {test_script} unit functional") diff --git a/tests/__init__.py b/tests/__init__.py index 7a084853c3..ff37162651 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -16,6 +16,8 @@ import unittest from unittest import mock +from botocore.compat import HAS_CRT + def unique_id(name): """ @@ -50,3 +52,13 @@ def setUp(self): def tearDown(self): self.bc_session_patch.stop() + + +def requires_crt(reason=None): + if reason is None: + reason = "Test requires awscrt to be installed" + + def decorator(func): + return unittest.skipIf(not HAS_CRT, reason)(func) + + return decorator diff --git a/tests/functional/test_crt.py b/tests/functional/test_crt.py new file mode 100644 index 0000000000..47de6da7f7 --- /dev/null +++ b/tests/functional/test_crt.py @@ -0,0 +1,64 @@ +# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# https://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +from contextlib import ContextDecorator + +from botocore.compat import HAS_CRT +from botocore.credentials import Credentials + +from boto3.s3.transfer import TransferConfig, create_transfer_manager +from tests import mock, requires_crt + +if HAS_CRT: + from s3transfer.crt import CRTTransferManager + + +class MockOptimizedInstance(ContextDecorator): + """Helper class to simulate a CRT optimized EC2 instance.""" + + DEFAULT_LOCK_MOCK = mock.Mock() + + def __init__(self, lock=DEFAULT_LOCK_MOCK, optimized=True): + self.acquire_process_lock = mock.patch( + 'boto3.crt.acquire_crt_s3_process_lock' + ) + self.acquire_process_lock.return_value = lock + self.is_optimized = mock.patch('awscrt.s3.is_optimized_for_system') + self.is_optimized.return_value = optimized + + def __enter__(self, *args, **kwargs): + self.acquire_process_lock.start() + self.is_optimized.start() + + def __exit__(self, *args, **kwargs): + self.acquire_process_lock.stop() + self.is_optimized.stop() + + +def create_mock_client(region_name='us-west-2'): + client = mock.Mock() + client.meta.region_name = region_name + client._get_credentials.return_value = Credentials( + 'access', 'secret', 'token' + ) + return client + + +class TestS3TransferWithCRT: + @requires_crt() + @MockOptimizedInstance() + def test_create_transfer_manager_on_optimized_instance(self): + client = create_mock_client() + config = TransferConfig() + transfer_manager = create_transfer_manager(client, config) + assert isinstance(transfer_manager, CRTTransferManager) diff --git a/tests/unit/s3/test_transfer.py b/tests/unit/s3/test_transfer.py index 1c86259677..dfc9d38159 100644 --- a/tests/unit/s3/test_transfer.py +++ b/tests/unit/s3/test_transfer.py @@ -10,10 +10,12 @@ # distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +import copy import pathlib from tempfile import NamedTemporaryFile import pytest +from botocore.credentials import Credentials from s3transfer.futures import NonThreadedExecutor from s3transfer.manager import TransferManager @@ -32,18 +34,27 @@ from tests import mock, unittest +def create_mock_client(region_name='us-west-2'): + client = mock.Mock() + client.meta.region_name = region_name + client._get_credentials.return_value = Credentials( + 'access', 'secret', 'token' + ) + return client + + class TestCreateTransferManager(unittest.TestCase): def test_create_transfer_manager(self): - client = object() - config = TransferConfig() + client = create_mock_client() + config = TransferConfig(preferred_transfer_client="classic") osutil = OSUtils() with mock.patch('boto3.s3.transfer.TransferManager') as manager: create_transfer_manager(client, config, osutil) assert manager.call_args == mock.call(client, config, osutil, None) def test_create_transfer_manager_with_no_threads(self): - client = object() - config = TransferConfig() + client = create_mock_client() + config = TransferConfig(preferred_transfer_client="classic") config.use_threads = False with mock.patch('boto3.s3.transfer.TransferManager') as manager: create_transfer_manager(client, config) @@ -51,6 +62,20 @@ def test_create_transfer_manager_with_no_threads(self): client, config, None, NonThreadedExecutor ) + def test_create_transfer_manager_with_default_config(self): + """Ensure we still default to classic transfer manager when CRT + is disabled. + """ + with mock.patch('boto3.s3.transfer.HAS_CRT', False): + client = create_mock_client() + config = TransferConfig() + assert config.preferred_transfer_client == "auto" + with mock.patch('boto3.s3.transfer.TransferManager') as manager: + create_transfer_manager(client, config) + assert manager.call_args == mock.call( + client, config, None, None + ) + class TestTransferConfig(unittest.TestCase): def assert_value_of_actual_and_alias( @@ -104,6 +129,7 @@ def test_transferconfig_parameters(self): io_chunksize=256 * KB, use_threads=True, max_bandwidth=1024 * KB, + preferred_transfer_client="classic", ) assert config.multipart_threshold == 8 * MB assert config.multipart_chunksize == 8 * MB @@ -113,6 +139,40 @@ def test_transferconfig_parameters(self): assert config.io_chunksize == 256 * KB assert config.use_threads is True assert config.max_bandwidth == 1024 * KB + assert config.preferred_transfer_client == "classic" + + def test_transferconfig_copy(self): + config = TransferConfig( + multipart_threshold=8 * MB, + max_concurrency=10, + multipart_chunksize=8 * MB, + num_download_attempts=5, + max_io_queue=100, + io_chunksize=256 * KB, + use_threads=True, + max_bandwidth=1024 * KB, + preferred_transfer_client="classic", + ) + copied_config = copy.copy(config) + + assert config is not copied_config + assert config.multipart_threshold == copied_config.multipart_threshold + assert config.multipart_chunksize == copied_config.multipart_chunksize + assert ( + config.max_request_concurrency + == copied_config.max_request_concurrency + ) + assert ( + config.num_download_attempts == copied_config.num_download_attempts + ) + assert config.max_io_queue_size == copied_config.max_io_queue_size + assert config.io_chunksize == copied_config.io_chunksize + assert config.use_threads == copied_config.use_threads + assert config.max_bandwidth == copied_config.max_bandwidth + assert ( + config.preferred_transfer_client + == copied_config.preferred_transfer_client + ) class TestProgressCallbackInvoker(unittest.TestCase): @@ -125,7 +185,7 @@ def test_on_progress(self): class TestS3Transfer(unittest.TestCase): def setUp(self): - self.client = mock.Mock() + self.client = create_mock_client() self.manager = mock.Mock(TransferManager(self.client)) self.transfer = S3Transfer(manager=self.manager) self.callback = mock.Mock() @@ -242,12 +302,14 @@ def test_propogation_s3_upload_failed_error(self): self.transfer.upload_file('smallfile', 'bucket', 'key') def test_can_create_with_just_client(self): - transfer = S3Transfer(client=mock.Mock()) + transfer = S3Transfer(client=create_mock_client()) assert isinstance(transfer, S3Transfer) def test_can_create_with_extra_configurations(self): transfer = S3Transfer( - client=mock.Mock(), config=TransferConfig(), osutil=OSUtils() + client=create_mock_client(), + config=TransferConfig(), + osutil=OSUtils(), ) assert isinstance(transfer, S3Transfer) @@ -268,12 +330,12 @@ def test_osutil_and_manager_are_mutually_exclusive(self): S3Transfer(osutil=mock.Mock(), manager=self.manager) def test_upload_requires_string_filename(self): - transfer = S3Transfer(client=mock.Mock()) + transfer = S3Transfer(client=create_mock_client()) with pytest.raises(ValueError): transfer.upload_file(filename=object(), bucket='foo', key='bar') def test_download_requires_string_filename(self): - transfer = S3Transfer(client=mock.Mock()) + transfer = S3Transfer(client=create_mock_client()) with pytest.raises(ValueError): transfer.download_file(bucket='foo', key='bar', filename=object()) diff --git a/tests/unit/test_crt.py b/tests/unit/test_crt.py new file mode 100644 index 0000000000..c88eb36fc2 --- /dev/null +++ b/tests/unit/test_crt.py @@ -0,0 +1,215 @@ +# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# https://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import botocore.exceptions +import pytest +import s3transfer +from botocore.compat import HAS_CRT +from botocore.credentials import Credentials + +import boto3 +from boto3.s3.transfer import TransferConfig +from tests import mock, requires_crt + +if HAS_CRT: + from awscrt.s3 import CrossProcessLock as CrossProcessLockClass + from s3transfer.crt import BotocoreCRTCredentialsWrapper + + import boto3.crt + + +@pytest.fixture +def mock_crt_process_lock(monkeypatch): + # The process lock is cached at the module layer whenever the + # cross process lock is successfully acquired. This patch ensures that + # test cases will start off with no previously cached process lock and + # if a cross process is instantiated/acquired it will be the mock that + # can be used for controlling lock behavior. + if HAS_CRT: + monkeypatch.setattr('s3transfer.crt.CRT_S3_PROCESS_LOCK', None) + with mock.patch('awscrt.s3.CrossProcessLock', spec=True) as mock_lock: + yield mock_lock + else: + # We cannot mock or use the lock without CRT support. + yield None + + +@pytest.fixture +def mock_crt_client_singleton(monkeypatch): + # Clear CRT state for each test + if HAS_CRT: + monkeypatch.setattr('boto3.crt.CRT_S3_CLIENT', None) + yield None + + +@pytest.fixture +def mock_serializer_singleton(monkeypatch): + # Clear CRT state for each test + if HAS_CRT: + monkeypatch.setattr('boto3.crt.BOTOCORE_CRT_SERIALIZER', None) + yield None + + +def create_test_client(service_name='s3', region_name="us-east-1"): + return boto3.client( + service_name, + region_name=region_name, + aws_access_key_id="access", + aws_secret_access_key="secret", + aws_session_token="token", + ) + + +USW2_S3_CLIENT = create_test_client(region_name="us-west-2") +USE1_S3_CLIENT = create_test_client(region_name="us-east-1") + + +class TestCRTTransferManager: + @requires_crt() + def test_create_crt_transfer_manager_with_lock_in_use( + self, + mock_crt_process_lock, + mock_crt_client_singleton, + mock_serializer_singleton, + ): + mock_crt_process_lock.return_value.acquire.side_effect = RuntimeError + + # Verify we can't create a second CRT client + tm = boto3.crt.create_crt_transfer_manager(USW2_S3_CLIENT, None) + assert tm is None + + @requires_crt() + def test_create_crt_transfer_manager( + self, + mock_crt_process_lock, + mock_crt_client_singleton, + mock_serializer_singleton, + ): + tm = boto3.crt.create_crt_transfer_manager(USW2_S3_CLIENT, None) + assert isinstance(tm, s3transfer.crt.CRTTransferManager) + + @requires_crt() + def test_crt_singleton_is_returned_every_call( + self, + mock_crt_process_lock, + mock_crt_client_singleton, + mock_serializer_singleton, + ): + first_s3_client = boto3.crt.get_crt_s3_client(USW2_S3_CLIENT, None) + second_s3_client = boto3.crt.get_crt_s3_client(USW2_S3_CLIENT, None) + + assert isinstance(first_s3_client, boto3.crt.CRTS3Client) + assert first_s3_client is second_s3_client + assert first_s3_client.crt_client is second_s3_client.crt_client + + @requires_crt() + def test_create_crt_transfer_manager_w_client_in_wrong_region( + self, + mock_crt_process_lock, + mock_crt_client_singleton, + mock_serializer_singleton, + ): + """Ensure we don't return the crt transfer manager if client is in + different region. The CRT isn't able to handle region redirects and + will consistently fail. + + We can remove this test once we have this fixed on the CRT side. + """ + usw2_s3_client = boto3.crt.create_crt_transfer_manager( + USW2_S3_CLIENT, None + ) + assert isinstance(usw2_s3_client, boto3.crt.CRTTransferManager) + + use1_s3_client = boto3.crt.create_crt_transfer_manager( + USE1_S3_CLIENT, None + ) + assert use1_s3_client is None + + @pytest.mark.parametrize( + "boto3_tuple,crt_tuple,matching", + ( + ( + ("access", "secret", "token"), + ("access", "secret", "token"), + True, + ), + ( + ("access", "secret", "token"), + ("noaccess", "secret", "token"), + False, + ), + ( + ("access", "secret", "token"), + ("access", "nosecret", "token"), + False, + ), + ( + ("access", "secret", "token"), + ("access", "secret", "notoken"), + False, + ), + ), + ) + @requires_crt() + def test_compare_identities(self, boto3_tuple, crt_tuple, matching): + boto3_creds = Credentials(*boto3_tuple) + crt_creds = Credentials(*crt_tuple) + crt_creds_wrapper = BotocoreCRTCredentialsWrapper(crt_creds) + assert ( + boto3.crt.compare_identity(boto3_creds, crt_creds_wrapper) + is matching + ) + + @requires_crt() + def test_compare_idenities_no_credentials(self): + def no_credentials(): + raise botocore.exceptions.NoCredentialsError() + + boto3_creds = Credentials("access", "secret", "token") + crt_creds_wrapper = no_credentials + assert ( + boto3.crt.compare_identity(boto3_creds, crt_creds_wrapper) is False + ) + + @requires_crt() + def test_get_crt_s3_client( + self, + mock_crt_process_lock, + mock_crt_client_singleton, + mock_serializer_singleton, + ): + config = TransferConfig() + crt_s3_client = boto3.crt.get_crt_s3_client(USW2_S3_CLIENT, config) + assert isinstance(crt_s3_client, boto3.crt.CRTS3Client) + assert isinstance(crt_s3_client.process_lock, CrossProcessLockClass) + assert crt_s3_client.region == "us-west-2" + assert isinstance( + crt_s3_client.cred_provider, BotocoreCRTCredentialsWrapper + ) + + @requires_crt() + def test_get_crt_s3_client_w_wrong_region( + self, + mock_crt_process_lock, + mock_crt_client_singleton, + mock_serializer_singleton, + ): + config = TransferConfig() + crt_s3_client = boto3.crt.get_crt_s3_client(USW2_S3_CLIENT, config) + assert isinstance(crt_s3_client, boto3.crt.CRTS3Client) + + # Ensure we don't create additional CRT clients + use1_crt_s3_client = boto3.crt.get_crt_s3_client( + USE1_S3_CLIENT, config + ) + assert use1_crt_s3_client is crt_s3_client + assert use1_crt_s3_client.region == "us-west-2"