Release 3.11.3 (#188)

3.11.3 (2024-12-13) ------------------- **Fixed** - Static type checker getting confused around ``AsyncSession`` and attached overloads (``AsyncResponse`` or ``Response``). (#185) **Changed** - Default keepalive (HTTP/2, and HTTP/3) changed to 1 hour. In conformance with urllib3-future. **Removed** - Automatic resolution of pending lazy responses if there are too many of them. Previously, we hardcoded a limit of 128 * NUM_CONN_POOL maximum inflight (aka. unresolved/lazy) response. This was unrealistic due to a number of factors like (but not limited to): A) remote peers can choose at will the max streams. B) we can have multiple pool with multiple (varying) max capacities. C) retrieving max streams per pool per conn is a very costly procedure (in terms of performance). We will revisit this later on. You still can set ``max_in_flight_multiplexed`` in your ``HTTPAdapter`` to restore this broken behavior.
jawah · Dec 13, 2024 · 6bf7791 · 6bf7791
2 parents 7036130 + 1fbedfa
commit 6bf7791
Show file tree

Hide file tree

Showing 17 changed files with 84 additions and 52 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -8,3 +8,20 @@ source =
     src/niquests
     */niquests
     *\niquests
+
+[report]
+omit =
+    src/niquests/help.py
+
+exclude_lines =
+    except ModuleNotFoundError:
+    except ImportError:
+    pass
+    import
+    raise NotImplementedError
+    .* # Platform-specific.*
+    .*:.* # Python \d.*
+    .* # Abstract
+    .* # Defensive:
+    if (?:typing.)?TYPE_CHECKING:
+    ^\s*?\.\.\.\s*$
diff --git a/HISTORY.md b/HISTORY.md
@@ -1,6 +1,25 @@
 Release History
 ===============
 
+3.11.3 (2024-12-13)
+-------------------
+
+**Fixed**
+- Static type checker getting confused around ``AsyncSession`` and attached overloads (``AsyncResponse`` or ``Response``). (#185)
+
+**Changed**
+- Default keepalive (HTTP/2, and HTTP/3) changed to 1 hour. In conformance with urllib3-future.
+
+**Removed**
+- Automatic resolution of pending lazy responses if there are too many of them.
+  Previously, we hardcoded a limit of 128 * NUM_CONN_POOL maximum inflight (aka. unresolved/lazy) response.
+  This was unrealistic due to a number of factors like (but not limited to):
+  A) remote peers can choose at will the max streams.
+  B) we can have multiple pool with multiple (varying) max capacities.
+  C) retrieving max streams per pool per conn is a very costly procedure (in terms of performance).
+  We will revisit this later on. You still can set ``max_in_flight_multiplexed`` in your ``HTTPAdapter`` to
+  restore this broken behavior.
+
 3.11.2 (2024-11-29)
 -------------------
 

diff --git a/noxfile.py b/noxfile.py
@@ -43,7 +43,10 @@ def tests_impl(
         "--strict-config",
         "--strict-markers",
         *(session.posargs or ("tests/",)),
-        env={"PYTHONWARNINGS": "always::DeprecationWarning"},
+        env={
+            "PYTHONWARNINGS": "always::DeprecationWarning",
+            "NIQUESTS_STRICT_OCSP": "1",
+        },
     )
 
 

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -1,6 +1,6 @@
 -e .[socks]
 pytest>=2.8.0,<=7.4.4
-pytest-cov
+coverage>=7.2.7,<7.7
 pytest-httpbin>=2,<3
 pytest-asyncio>=0.21.1,<1.0
 httpbin==0.10.2

diff --git a/src/niquests/__version__.py b/src/niquests/__version__.py
@@ -9,9 +9,9 @@
 __url__: str = "https://niquests.readthedocs.io"
 
 __version__: str
-__version__ = "3.11.2"
+__version__ = "3.11.3"
 
-__build__: int = 0x031102
+__build__: int = 0x031103
 __author__: str = "Kenneth Reitz"
 __author_email__: str = "[email protected]"
 __license__: str = "Apache-2.0"

diff --git a/src/niquests/_async.py b/src/niquests/_async.py
@@ -20,7 +20,7 @@
     from urllib3 import ConnectionInfo
     from urllib3.contrib.resolver._async import AsyncBaseResolver
     from urllib3.contrib.webextensions._async import load_extension
-else:
+else:  # Defensive: tested in separate/isolated CI
     from urllib3_future import ConnectionInfo  # type: ignore[assignment]
     from urllib3_future.contrib.resolver._async import AsyncBaseResolver  # type: ignore[assignment]
     from urllib3_future.contrib.webextensions._async import load_extension  # type: ignore[assignment]
@@ -130,7 +130,7 @@ def __init__(
         pool_connections: int = DEFAULT_POOLSIZE,
         pool_maxsize: int = DEFAULT_POOLSIZE,
         happy_eyeballs: bool | int = False,
-        keepalive_delay: float | int | None = 300.0,
+        keepalive_delay: float | int | None = 3600.0,
         keepalive_idle_window: float | int | None = 60.0,
         base_url: str | None = None,
     ):
@@ -290,10 +290,10 @@ def __enter__(self) -> typing.NoReturn:
             'You probably meant "async with". Did you forget to prepend the "async" keyword?'
         )
 
-    async def __aenter__(self):
+    async def __aenter__(self) -> AsyncSession:
         return self
 
-    async def __aexit__(self, exc, value, tb):
+    async def __aexit__(self, exc, value, tb) -> None:
         await self.close()
 
     def mount(self, prefix: str, adapter: AsyncBaseAdapter) -> None:  # type: ignore[override]
@@ -770,7 +770,7 @@ async def request(
         allow_redirects: bool = ...,
         proxies: ProxyType | None = ...,
         hooks: HookType[PreparedRequest | Response] | None = ...,
-        stream: Literal[False] = ...,
+        stream: Literal[False] | None = ...,
         verify: TLSVerifyType | None = ...,
         cert: TLSClientCertType | None = ...,
         json: typing.Any | None = ...,
@@ -791,8 +791,7 @@ async def request(
         allow_redirects: bool = ...,
         proxies: ProxyType | None = ...,
         hooks: HookType[PreparedRequest | Response] | None = ...,
-        *,
-        stream: Literal[True],
+        stream: Literal[True] = ...,
         verify: TLSVerifyType | None = ...,
         cert: TLSClientCertType | None = ...,
         json: typing.Any | None = ...,
@@ -812,7 +811,7 @@ async def request(  # type: ignore[override]
         allow_redirects: bool = True,
         proxies: ProxyType | None = None,
         hooks: HookType[PreparedRequest | Response] | None = None,
-        stream: bool = False,
+        stream: bool | None = None,
         verify: TLSVerifyType | None = None,
         cert: TLSClientCertType | None = None,
         json: typing.Any | None = None,
@@ -880,7 +879,7 @@ async def get(
         proxies: ProxyType | None = ...,
         hooks: HookType[PreparedRequest | Response] | None = ...,
         verify: TLSVerifyType = ...,
-        stream: Literal[False] | Literal[None] = ...,
+        stream: Literal[False] | None = ...,
         cert: TLSClientCertType | None = ...,
         **kwargs: typing.Any,
     ) -> Response: ...
@@ -899,7 +898,7 @@ async def get(
         proxies: ProxyType | None = ...,
         hooks: HookType[PreparedRequest | Response] | None = ...,
         verify: TLSVerifyType = ...,
-        stream: Literal[True],
+        stream: Literal[True] = ...,
         cert: TLSClientCertType | None = ...,
         **kwargs: typing.Any,
     ) -> AsyncResponse: ...
@@ -933,7 +932,7 @@ async def get(  # type: ignore[override]
             proxies=proxies,
             hooks=hooks,
             verify=verify,
-            stream=stream,
+            stream=stream,  # type: ignore[arg-type]
             cert=cert,
             **kwargs,
         )
@@ -1005,7 +1004,7 @@ async def options(  # type: ignore[override]
             proxies=proxies,
             hooks=hooks,
             verify=verify,
-            stream=stream,
+            stream=stream,  # type: ignore[arg-type]
             cert=cert,
             **kwargs,
         )
@@ -1077,7 +1076,7 @@ async def head(  # type: ignore[override]
             proxies=proxies,
             hooks=hooks,
             verify=verify,
-            stream=stream,
+            stream=stream,  # type: ignore[arg-type]
             cert=cert,
             **kwargs,
         )
@@ -1158,7 +1157,7 @@ async def post(  # type: ignore[override]
             proxies=proxies,
             hooks=hooks,
             verify=verify,
-            stream=stream,
+            stream=stream,  # type: ignore[arg-type]
             cert=cert,
         )
 
@@ -1238,7 +1237,7 @@ async def put(  # type: ignore[override]
             proxies=proxies,
             hooks=hooks,
             verify=verify,
-            stream=stream,
+            stream=stream,  # type: ignore[arg-type]
             cert=cert,
         )
 
@@ -1318,7 +1317,7 @@ async def patch(  # type: ignore[override]
             proxies=proxies,
             hooks=hooks,
             verify=verify,
-            stream=stream,
+            stream=stream,  # type: ignore[arg-type]
             cert=cert,
         )
 
@@ -1389,7 +1388,7 @@ async def delete(  # type: ignore[override]
             proxies=proxies,
             hooks=hooks,
             verify=verify,
-            stream=stream,
+            stream=stream,  # type: ignore[arg-type]
             cert=cert,
             **kwargs,
         )

diff --git a/src/niquests/_typing.py b/src/niquests/_typing.py
@@ -12,7 +12,7 @@
     from urllib3.contrib.resolver import BaseResolver
     from urllib3.contrib.resolver._async import AsyncBaseResolver
     from urllib3.fields import RequestField
-else:
+else:  # Defensive: tested in separate/isolated CI
     from urllib3_future import (  # type: ignore[assignment]
         Retry,
         Timeout,

diff --git a/src/niquests/adapters.py b/src/niquests/adapters.py
@@ -73,7 +73,7 @@
     from urllib3.contrib.webextensions._async import (
         load_extension as async_load_extension,
     )
-else:
+else:  # Defensive: tested in separate/isolated CI
     from urllib3_future import (  # type: ignore[assignment]
         ConnectionInfo,
         HTTPConnectionPool,
@@ -354,7 +354,7 @@ def __init__(
         disable_ipv4: bool = False,
         disable_ipv6: bool = False,
         happy_eyeballs: bool | int = False,
-        keepalive_delay: float | int | None = 300.0,
+        keepalive_delay: float | int | None = 3600.0,
         keepalive_idle_window: float | int | None = 60.0,
     ):
         if isinstance(max_retries, bool):
@@ -393,11 +393,7 @@ def __init__(
         #: we keep a list of pending (lazy) response
         self._promises: dict[str, Response] = {}
         self._orphaned: list[BaseHTTPResponse] = []
-        self._max_in_flight_multiplexed = (
-            max_in_flight_multiplexed
-            if max_in_flight_multiplexed is not None
-            else self._pool_connections * 124
-        )
+        self._max_in_flight_multiplexed = max_in_flight_multiplexed
         self._promise_lock = RLock()
 
         disabled_svn = set()
@@ -871,7 +867,7 @@ def send(
         ), "Tried to send a non-initialized PreparedRequest"
 
         # We enforce a limit to avoid burning out our connection pool.
-        if multiplexed:
+        if multiplexed and self._max_in_flight_multiplexed is not None:
             with self._promise_lock:
                 if len(self._promises) >= self._max_in_flight_multiplexed:
                     self.gather()
@@ -1447,7 +1443,7 @@ def __init__(
         disable_ipv4: bool = False,
         disable_ipv6: bool = False,
         happy_eyeballs: bool | int = False,
-        keepalive_delay: float | int | None = 300.0,
+        keepalive_delay: float | int | None = 3600.0,
         keepalive_idle_window: float | int | None = 60.0,
     ):
         if isinstance(max_retries, bool):
@@ -1487,11 +1483,7 @@ def __init__(
         #: we keep a list of pending (lazy) response
         self._promises: dict[str, Response | AsyncResponse] = {}
         self._orphaned: list[BaseAsyncHTTPResponse] = []
-        self._max_in_flight_multiplexed = (
-            max_in_flight_multiplexed
-            if max_in_flight_multiplexed is not None
-            else self._pool_connections * 250
-        )
+        self._max_in_flight_multiplexed = max_in_flight_multiplexed
 
         disabled_svn = set()
 
@@ -1960,7 +1952,7 @@ async def send(
         ), "Tried to send a non-initialized PreparedRequest"
 
         # We enforce a limit to avoid burning out our connection pool.
-        if multiplexed:
+        if multiplexed and self._max_in_flight_multiplexed is not None:
             if len(self._promises) >= self._max_in_flight_multiplexed:
                 await self.gather()
 

diff --git a/src/niquests/cookies.py b/src/niquests/cookies.py
@@ -25,7 +25,7 @@
 
 if HAS_LEGACY_URLLIB3 is False:
     from urllib3 import BaseHTTPResponse
-else:
+else:  # Defensive: tested in separate/isolated CI
     from urllib3_future import BaseHTTPResponse  # type: ignore[assignment]
 
 if typing.TYPE_CHECKING:

diff --git a/src/niquests/exceptions.py b/src/niquests/exceptions.py
@@ -14,7 +14,7 @@
 
 if HAS_LEGACY_URLLIB3 is False:
     from urllib3.exceptions import HTTPError as BaseHTTPError
-else:
+else:  # Defensive: tested in separate/isolated CI
     from urllib3_future.exceptions import HTTPError as BaseHTTPError  # type: ignore
 
 if typing.TYPE_CHECKING:

diff --git a/src/niquests/extensions/_async_ocsp.py b/src/niquests/extensions/_async_ocsp.py
@@ -27,7 +27,7 @@
     from urllib3.util.url import parse_url
     from urllib3.contrib.resolver._async import AsyncBaseResolver
     from urllib3.contrib.ssa import AsyncSocket
-else:
+else:  # Defensive: tested in separate/isolated CI
     from urllib3_future import ConnectionInfo  # type: ignore[assignment]
     from urllib3_future.exceptions import SecurityWarning  # type: ignore[assignment]
     from urllib3_future.util.url import parse_url  # type: ignore[assignment]
@@ -383,7 +383,8 @@ async def verify(
             resolver=resolver, happy_eyeballs=happy_eyeballs
         ) as session:
             session.trust_env = False
-            session.proxies = proxies
+            if proxies:
+                session.proxies = proxies
 
             # When using Python native capabilities, you won't have the issuerCA DER by default (Python 3.7 to 3.9).
             # Unfortunately! But no worries, we can circumvent it! (Python 3.10+ is not concerned anymore)

diff --git a/src/niquests/extensions/_ocsp.py b/src/niquests/extensions/_ocsp.py
@@ -26,7 +26,7 @@
     from urllib3.exceptions import SecurityWarning
     from urllib3.util.url import parse_url
     from urllib3.contrib.resolver import BaseResolver
-else:
+else:  # Defensive: tested in separate/isolated CI
     from urllib3_future import ConnectionInfo  # type: ignore[assignment]
     from urllib3_future.exceptions import SecurityWarning  # type: ignore[assignment]
     from urllib3_future.util.url import parse_url  # type: ignore[assignment]
@@ -376,7 +376,8 @@ def verify(
 
     with Session(resolver=resolver, happy_eyeballs=happy_eyeballs) as session:
         session.trust_env = False
-        session.proxies = proxies
+        if proxies:
+            session.proxies = proxies
 
         # When using Python native capabilities, you won't have the issuerCA DER by default.
         # Unfortunately! But no worries, we can circumvent it!

diff --git a/src/niquests/models.py b/src/niquests/models.py
@@ -64,7 +64,7 @@
         RawExtensionFromHTTP,
         ServerSideEventExtensionFromHTTP,
     )
-else:
+else:  # Defensive: tested in separate/isolated CI
     from urllib3_future import (  # type: ignore[assignment]
         BaseHTTPResponse,
         AsyncHTTPResponse as BaseAsyncHTTPResponse,

diff --git a/src/niquests/sessions.py b/src/niquests/sessions.py
@@ -25,7 +25,7 @@
 if HAS_LEGACY_URLLIB3 is False:
     from urllib3 import ConnectionInfo
     from urllib3.contrib.webextensions import load_extension
-else:
+else:  # Defensive: tested in separate/isolated CI
     from urllib3_future import ConnectionInfo  # type: ignore[assignment]
     from urllib3_future.contrib.webextensions import load_extension  # type: ignore[assignment]
 
@@ -251,7 +251,7 @@ def __init__(
         pool_connections: int = DEFAULT_POOLSIZE,
         pool_maxsize: int = DEFAULT_POOLSIZE,
         happy_eyeballs: bool | int = False,
-        keepalive_delay: float | int | None = 300.0,
+        keepalive_delay: float | int | None = 3600.0,
         keepalive_idle_window: float | int | None = 60.0,
         base_url: str | None = None,
     ):
@@ -425,10 +425,10 @@ def __init__(
             ),
         )
 
-    def __enter__(self):
+    def __enter__(self) -> Session:
         return self
 
-    def __exit__(self, *args):
+    def __exit__(self, *args) -> None:
         self.close()
 
     def prepare_request(self, request: Request) -> PreparedRequest: