bump botocore (#844)

Author: thehesiod

.gitignore

@@ -36,7 +36,7 @@ pip-delete-this-directory.txt
 # Unit test / coverage reports
 htmlcov/
 .tox/
-.coverage
+.coverage*
 .cache
 nosetests.xml
 coverage.xml
@@ -61,3 +61,5 @@ target/
 *.iml
 # rope
 .ropeproject
+
+Pipfile.lock

CHANGES.rst

@@ -1,5 +1,10 @@
 Changes
 -------
+1.2.0 (2021-01-11)
+^^^^^^^^^^^^^^^^^^
+* bump botocore to 1.19.52
+* use passed in http_session_cls param to create_client (#797)
+
 1.1.2 (2020-10-07)
 ^^^^^^^^^^^^^^^^^^
 * fix AioPageIterator search method #831 (thanks @joseph-jones)

Makefile

@@ -22,21 +22,25 @@ cov cover coverage: flake
 mototest:
 	docker pull alpine
 	docker pull lambci/lambda:python3.8
-	BOTO_CONFIG=/dev/null pipenv run python3 -Wd -X tracemalloc=5 -X faulthandler -m pytest -vv -m moto -n auto --cov-report term --cov-report html --cov --log-cli-level=DEBUG aiobotocore tests
+	BOTO_CONFIG=/dev/null pipenv run python3 -Wd -X tracemalloc=5 -X faulthandler -m pytest -vv -m moto -n auto --cov-report term --cov-report html --cov=aiobotocore --cov=tests --log-cli-level=DEBUG aiobotocore tests
 	@echo "open file://`pwd`/htmlcov/index.html"
 
 
 clean:
 	rm -rf `find . -name __pycache__`
+	rm -rf `find . -name .pytest_cache`
+	rm -rf `find . -name *.egg-info`
 	rm -f `find . -type f -name '*.py[co]' `
 	rm -f `find . -type f -name '*~' `
 	rm -f `find . -type f -name '.*~' `
 	rm -f `find . -type f -name '@*' `
 	rm -f `find . -type f -name '#*#' `
 	rm -f `find . -type f -name '*.orig' `
 	rm -f `find . -type f -name '*.rej' `
-	rm -f .coverage
+	rm -f .coverage*
 	rm -rf coverage
+	rm -rf coverage.xml
+	rm -rf htmlcov
 	rm -rf build
 	rm -rf cover
 	rm -rf dist

aiobotocore/__init__.py

@@ -1,4 +1,4 @@
 from .session import get_session, AioSession
 
 __all__ = ['get_session', 'AioSession']
-__version__ = '1.1.2'
+__version__ = '1.2.0'

aiobotocore/client.py

@@ -34,6 +34,9 @@ async def create_client(self, service_name, region_name, is_secure=True,
             verify, credentials, scoped_config, client_config, endpoint_bridge)
         service_client = cls(**client_args)
         self._register_retries(service_client)
+        self._register_s3_events(
+            service_client, endpoint_bridge, endpoint_url, client_config,
+            scoped_config)
         self._register_s3_events(
             service_client, endpoint_bridge, endpoint_url, client_config,
             scoped_config)

aiobotocore/credentials.py

@@ -46,6 +46,12 @@ def create_credential_resolver(session, cache=None, region_name=None):
     num_attempts = session.get_config_variable('metadata_service_num_attempts')
     disable_env_vars = session.instance_variables().get('profile') is not None
 
+    imds_config = {
+        'ec2_metadata_service_endpoint': session.get_config_variable(
+            'ec2_metadata_service_endpoint'),
+        'imds_use_ipv6': session.get_config_variable('imds_use_ipv6')
+    }
+
     if cache is None:
         cache = {}
 
@@ -55,7 +61,8 @@ def create_credential_resolver(session, cache=None, region_name=None):
         iam_role_fetcher=AioInstanceMetadataFetcher(
             timeout=metadata_timeout,
             num_attempts=num_attempts,
-            user_agent=session.user_agent())
+            user_agent=session.user_agent(),
+            config=imds_config)
     )
 
     profile_provider_builder = AioProfileProviderBuilder(
@@ -817,6 +824,8 @@ async def load_credentials(self):
 
 
 class AioSSOCredentialFetcher(AioCachedCredentialFetcher):
+    _UTC_DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+
     def __init__(self, start_url, sso_region, role_name, account_id,
                  client_creator, token_loader=None, cache=None,
                  expiry_window_seconds=None):
@@ -826,7 +835,6 @@ def __init__(self, start_url, sso_region, role_name, account_id,
         self._account_id = account_id
         self._start_url = start_url
         self._token_loader = token_loader
-
         super(AioSSOCredentialFetcher, self).__init__(
             cache, expiry_window_seconds
         )
@@ -846,7 +854,7 @@ def _parse_timestamp(self, timestamp_ms):
         # fromtimestamp expects seconds so: milliseconds / 1000 = seconds
         timestamp_seconds = timestamp_ms / 1000.0
         timestamp = datetime.datetime.fromtimestamp(timestamp_seconds, tzutc())
-        return _serialize_if_needed(timestamp)
+        return timestamp.strftime(self._UTC_DATE_FORMAT)
 
     async def _get_credentials(self):
         """Get credentials by calling SSO get role credentials."""

aiobotocore/endpoint.py

@@ -304,7 +304,7 @@ def create_endpoint(self, service_model, region_name, endpoint_url,
             ssl_context=ssl_context,
             **connector_args)
 
-        aio_session = aiohttp.ClientSession(
+        aio_session = http_session_cls(
             connector=connector,
             timeout=timeout,
             skip_auto_headers={'CONTENT-TYPE'},

aiobotocore/response.py

@@ -75,7 +75,7 @@ async def __anext__(self):
 
     anext = __anext__
 
-    async def iter_lines(self, chunk_size=1024):
+    async def iter_lines(self, chunk_size=1024, keepends=False):
         """Return an iterator to yield lines from the raw stream.
 
         This is achieved by reading chunk of bytes (of size chunk_size) at a
@@ -85,10 +85,10 @@ async def iter_lines(self, chunk_size=1024):
         async for chunk in self.iter_chunks(chunk_size):
             lines = (pending + chunk).splitlines(True)
             for line in lines[:-1]:
-                yield line.splitlines()[0]
+                yield line.splitlines(keepends)[0]
             pending = lines[-1]
         if pending:
-            yield pending.splitlines()[0]
+            yield pending.splitlines(keepends)[0]
 
     async def iter_chunks(self, chunk_size=_DEFAULT_CHUNK_SIZE):
         """Return an iterator to yield chunks of chunk_size bytes from the raw

aiobotocore/signers.py

@@ -46,9 +46,11 @@ async def sign(self, operation_name, request, region_name=None,
             }
             if expires_in is not None:
                 kwargs['expires'] = expires_in
-            if not explicit_region_name and request.context.get(
-                    'signing', {}).get('region'):
-                kwargs['region_name'] = request.context['signing']['region']
+            signing_context = request.context.get('signing', {})
+            if not explicit_region_name and signing_context.get('region'):
+                kwargs['region_name'] = signing_context['region']
+            if signing_context.get('signing_name'):
+                kwargs['signing_name'] = signing_context['signing_name']
             try:
                 auth = await self.get_auth_instance(**kwargs)
             except UnknownSignatureVersionError as e:

aiobotocore/utils.py

@@ -7,7 +7,9 @@
 from botocore.utils import ContainerMetadataFetcher, InstanceMetadataFetcher, \
     IMDSFetcher, get_environ_proxies, BadIMDSRequestError, S3RegionRedirector, \
     ClientError
-from botocore.exceptions import MetadataRetrievalError
+from botocore.exceptions import (
+    InvalidIMDSEndpointError, MetadataRetrievalError,
+)
 import botocore.awsrequest
 
 
@@ -58,6 +60,13 @@ async def _fetch_metadata_token(self):
                     logger.debug(
                         "Caught retryable HTTP exception while making metadata "
                         "service request to %s: %s", url, e, exc_info=True)
+                except aiohttp.client_exceptions.ClientConnectorError as e:
+                    if getattr(e, 'errno', None) == 8 or \
+                            str(getattr(e, 'os_error', None)) == \
+                            'Domain name not found':  # threaded vs async resolver
+                        raise InvalidIMDSEndpointError(endpoint=url, error=e)
+                    else:
+                        raise
 
         return None
 

aiobotocore/waiter.py

@@ -25,13 +25,15 @@ async def wait(self, **kwargs):
         config = kwargs.pop('WaiterConfig', {})
         sleep_amount = config.get('Delay', self.config.delay)
         max_attempts = config.get('MaxAttempts', self.config.max_attempts)
+        last_matched_acceptor = None
         num_attempts = 0
 
         while True:
             response = await self._operation_method(**kwargs)
             num_attempts += 1
             for acceptor in acceptors:
                 if acceptor.matcher_func(response):
+                    last_matched_acceptor = acceptor
                     current_state = acceptor.state
                     break
             else:
@@ -43,24 +45,36 @@ async def wait(self, **kwargs):
                     # can just handle here by raising an exception.
                     raise WaiterError(
                         name=self.name,
-                        reason=response['Error'].get('Message', 'Unknown'),
-                        last_response=response
+                        reason='An error occurred (%s): %s' % (
+                            response['Error'].get('Code', 'Unknown'),
+                            response['Error'].get('Message', 'Unknown'),
+                        ),
+                        last_response=response,
                     )
             if current_state == 'success':
                 logger.debug("Waiting complete, waiter matched the "
                              "success state.")
                 return
             if current_state == 'failure':
+                reason = 'Waiter encountered a terminal failure state: %s' % (
+                        acceptor.explanation
+                        )
                 raise WaiterError(
                     name=self.name,
-                    reason='Waiter encountered a terminal failure state',
+                    reason=reason,
                     last_response=response,
                 )
             if num_attempts >= max_attempts:
+                if last_matched_acceptor is None:
+                    reason = 'Max attempts exceeded'
+                else:
+                    reason = 'Max attempts exceeded. Previously accepted state: %s' % (
+                        acceptor.explanation
+                    )
                 raise WaiterError(
                     name=self.name,
-                    reason='Max attempts exceeded',
-                    last_response=response
+                    reason=reason,
+                    last_response=response,
                 )
             await asyncio.sleep(sleep_amount)
 

setup.py

@@ -7,7 +7,7 @@
 # NOTE: When updating botocore make sure to update awscli/boto3 versions below
 install_requires = [
     # pegged to also match items in `extras_require`
-    'botocore>=1.17.44,<1.17.45',
+    'botocore>=1.19.52,<1.19.53',
     'aiohttp>=3.3.1',
     'wrapt>=1.10.10',
     'aioitertools>=0.5.1',
@@ -19,8 +19,8 @@ def read(f):
 
 
 extras_require = {
-    'awscli': ['awscli==1.18.121'],
-    'boto3': ['boto3==1.14.44'],
+    'awscli': ['awscli==1.18.212'],
+    'boto3': ['boto3==1.16.52'],
 }
 
 

tests/botocore/test_credentials.py

@@ -1084,6 +1084,7 @@ def _f(config_loader: Optional[ConfigValueStore] = None) -> AioSession:
             'config_file': 'c',
             'metadata_service_timeout': 1,
             'metadata_service_num_attempts': 1,
+            'imds_use_ipv6': 'false',
         }
 
         def fake_get_component(self, key):
@@ -1228,15 +1229,15 @@ async def test_sso_credential_fetcher_can_fetch_credentials(
     self.assertEqual(credentials['access_key'], 'foo')
     self.assertEqual(credentials['secret_key'], 'bar')
     self.assertEqual(credentials['token'], 'baz')
-    self.assertEqual(credentials['expiry_time'], '2008-09-23T12:43:20UTC')
+    self.assertEqual(credentials['expiry_time'], '2008-09-23T12:43:20Z')
     cache_key = '048db75bbe50955c16af7aba6ff9c41a3131bb7e'
     expected_cached_credentials = {
         'ProviderType': 'sso',
         'Credentials': {
             'AccessKeyId': 'foo',
             'SecretAccessKey': 'bar',
             'SessionToken': 'baz',
-            'Expiration': '2008-09-23T12:43:20UTC',
+            'Expiration': '2008-09-23T12:43:20Z',
         }
     }
     self.assertEqual(self.cache[cache_key], expected_cached_credentials)

tests/test_patches.py

@@ -76,7 +76,7 @@
     ClientArgsCreator.get_client_args: {'e3a44e6f50159e8e31c3d76f5e8a1110dda495fa'},
 
     # client.py
-    ClientCreator.create_client: {'ee63a3d60b5917879cb644c1b0aa3fe34538b915'},
+    ClientCreator.create_client: {'281fbf7afc4e6282e5c881c7a03717c9e5e4e176'},
     ClientCreator._create_client_class: {'5e493d069eedbf314e40e12a7886bbdbcf194335'},
     ClientCreator._get_client_args: {'555e1e41f93df7558c8305a60466681e3a267ef3'},
     ClientCreator._register_s3_events: {'da3fc62a131d63964c8daa0f52124b092fd8f1b4'},
@@ -116,7 +116,7 @@
     RefreshableCredentials.get_frozen_credentials:
         {'f661c84a8b759786e011f0b1e8a468a0c6294e36'},
     SSOCredentialFetcher:
-        {'e092b115155a06760af6f3c72ccef120979b1201'},
+        {'d68353a0d2c291d5742f134d28ae1e1419faa4c6'},
     SSOProvider.load:
         {'f43d79e1520b2a7b7ef85cd537f41e19d4bce806'},
     CachedCredentialFetcher._get_credentials:
@@ -185,7 +185,7 @@
         {'c028b9776383cc566be10999745b6082f458d902'},
     BotoProvider.load: {'9351b8565c2c969937963fc1d3fbc8b3b6d8ccc1'},
     OriginalEC2Provider.load: {'bde9af019f01acf3848a6eda125338b2c588c1ab'},
-    create_credential_resolver: {'5ff7fe49d7636b795a50202ff5c089611f4e27c1'},
+    create_credential_resolver: {'c6a08bfc59a4e8f59c8b7a846dbb74db649101fd'},
     get_credentials: {'ff0c735a388ac8dd7fe300a32c1e36cdf33c0f56'},
 
     # endpoint.py
@@ -226,7 +226,7 @@
     RestJSONParser._create_event_stream: {'0564ba55383a71cc1ba3e5be7110549d7e9992f5'},
 
     # response.py
-    StreamingBody: {'bb4d872649b0c118c9a3d5e44961e1bea92eb79c'},
+    StreamingBody: {'b77bd0903f9013bc47c01f91c6d9bfb8a504d106'},
 
     # session.py
     Session.__init__: {'ccf156a76beda3425fb54363f3b2718dc0445f6d'},
@@ -242,7 +242,7 @@
 
     # signers.py
     RequestSigner.handler: {'371909df136a0964ef7469a63d25149176c2b442'},
-    RequestSigner.sign: {'7df841d3df3f4015763523c1932652aef754287a'},
+    RequestSigner.sign: {'a07e4caab222bf9375036b1fafaf021ccb5b2bf3'},
     RequestSigner.get_auth: {'4f8099bef30f9a72fa3bcaa1bd3d22c4fbd224a8'},
     RequestSigner.get_auth_instance: {'4f8099bef30f9a72fa3bcaa1bd3d22c4fbd224a8'},
     RequestSigner._choose_signer: {'d1e0e3196ada449d3ae0ec09b8ae9b5868c50d4e'},
@@ -268,9 +268,9 @@
     ContainerMetadataFetcher._get_response:
         {'7e5acdd2cf0167a047e3d5ee1439565a2f79f6a6'},
     # Overrided session and dealing with proxy support
-    IMDSFetcher.__init__: {'690e37140ccdcd67c7a85ce5d36331491a79954e'},
+    IMDSFetcher.__init__: {'a0766a5ba7dde9c26f3c51eb38d73f8e6087d492'},
     IMDSFetcher._get_request: {'96a0e580cab5a21deb4d2cd7e904aa17d5e1e504'},
-    IMDSFetcher._fetch_metadata_token: {'4fdad673b4997b1268c6d9dff09a4b99c1cb5e0d'},
+    IMDSFetcher._fetch_metadata_token: {'dcb147f6c7a425ba9e30be8ad4818be4b781305c'},
 
     InstanceMetadataFetcher.retrieve_iam_role_credentials:
         {'76737f6add82a1b9a0dc590cf10bfac0c7026a2e'},
@@ -285,7 +285,7 @@
 
     # waiter.py
     NormalizedOperationMethod.__call__: {'79723632d023739aa19c8a899bc2b814b8ab12ff'},
-    Waiter.wait: {'5502a89ed740fb5d6238a6f72a3a08efc1a9f43b'},
+    Waiter.wait: {'3a4ff0fdfc78b7ec42bfd41f3e1ba3b741f2d2b9'},
     create_waiter_with_client: {'c3d12c9a4293105cc8c2ecfc7e69a2152ad564de'},
 }