| from __future__ import absolute_import |
| |
| import errno |
| import logging |
| import re |
| import socket |
| import sys |
| import warnings |
| from socket import error as SocketError |
| from socket import timeout as SocketTimeout |
| |
| from .connection import ( |
| BaseSSLError, |
| BrokenPipeError, |
| DummyConnection, |
| HTTPConnection, |
| HTTPException, |
| HTTPSConnection, |
| VerifiedHTTPSConnection, |
| port_by_scheme, |
| ) |
| from .exceptions import ( |
| ClosedPoolError, |
| EmptyPoolError, |
| HeaderParsingError, |
| HostChangedError, |
| InsecureRequestWarning, |
| LocationValueError, |
| MaxRetryError, |
| NewConnectionError, |
| ProtocolError, |
| ProxyError, |
| ReadTimeoutError, |
| SSLError, |
| TimeoutError, |
| ) |
| from .packages import six |
| from .packages.six.moves import queue |
| from .request import RequestMethods |
| from .response import HTTPResponse |
| from .util.connection import is_connection_dropped |
| from .util.proxy import connection_requires_http_tunnel |
| from .util.queue import LifoQueue |
| from .util.request import set_file_position |
| from .util.response import assert_header_parsing |
| from .util.retry import Retry |
| from .util.ssl_match_hostname import CertificateError |
| from .util.timeout import Timeout |
| from .util.url import Url, _encode_target |
| from .util.url import _normalize_host as normalize_host |
| from .util.url import get_host, parse_url |
| |
| xrange = six.moves.xrange |
| |
| log = logging.getLogger(__name__) |
| |
| _Default = object() |
| |
| |
| # Pool objects |
| class ConnectionPool(object): |
| """ |
| Base class for all connection pools, such as |
| :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`. |
| |
| .. note:: |
| ConnectionPool.urlopen() does not normalize or percent-encode target URIs |
| which is useful if your target server doesn't support percent-encoded |
| target URIs. |
| """ |
| |
| scheme = None |
| QueueCls = LifoQueue |
| |
| def __init__(self, host, port=None): |
| if not host: |
| raise LocationValueError("No host specified.") |
| |
| self.host = _normalize_host(host, scheme=self.scheme) |
| self._proxy_host = host.lower() |
| self.port = port |
| |
| def __str__(self): |
| return "%s(host=%r, port=%r)" % (type(self).__name__, self.host, self.port) |
| |
| def __enter__(self): |
| return self |
| |
| def __exit__(self, exc_type, exc_val, exc_tb): |
| self.close() |
| # Return False to re-raise any potential exceptions |
| return False |
| |
| def close(self): |
| """ |
| Close all pooled connections and disable the pool. |
| """ |
| pass |
| |
| |
| # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252 |
| _blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK} |
| |
| |
| class HTTPConnectionPool(ConnectionPool, RequestMethods): |
| """ |
| Thread-safe connection pool for one host. |
| |
| :param host: |
| Host used for this HTTP Connection (e.g. "localhost"), passed into |
| :class:`http.client.HTTPConnection`. |
| |
| :param port: |
| Port used for this HTTP Connection (None is equivalent to 80), passed |
| into :class:`http.client.HTTPConnection`. |
| |
| :param strict: |
| Causes BadStatusLine to be raised if the status line can't be parsed |
| as a valid HTTP/1.0 or 1.1 status line, passed into |
| :class:`http.client.HTTPConnection`. |
| |
| .. note:: |
| Only works in Python 2. This parameter is ignored in Python 3. |
| |
| :param timeout: |
| Socket timeout in seconds for each individual connection. This can |
| be a float or integer, which sets the timeout for the HTTP request, |
| or an instance of :class:`urllib3.util.Timeout` which gives you more |
| fine-grained control over request timeouts. After the constructor has |
| been parsed, this is always a `urllib3.util.Timeout` object. |
| |
| :param maxsize: |
| Number of connections to save that can be reused. More than 1 is useful |
| in multithreaded situations. If ``block`` is set to False, more |
| connections will be created but they will not be saved once they've |
| been used. |
| |
| :param block: |
| If set to True, no more than ``maxsize`` connections will be used at |
| a time. When no free connections are available, the call will block |
| until a connection has been released. This is a useful side effect for |
| particular multithreaded situations where one does not want to use more |
| than maxsize connections per host to prevent flooding. |
| |
| :param headers: |
| Headers to include with all requests, unless other headers are given |
| explicitly. |
| |
| :param retries: |
| Retry configuration to use by default with requests in this pool. |
| |
| :param _proxy: |
| Parsed proxy URL, should not be used directly, instead, see |
| :class:`urllib3.ProxyManager` |
| |
| :param _proxy_headers: |
| A dictionary with proxy headers, should not be used directly, |
| instead, see :class:`urllib3.ProxyManager` |
| |
| :param \\**conn_kw: |
| Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`, |
| :class:`urllib3.connection.HTTPSConnection` instances. |
| """ |
| |
| scheme = "http" |
| ConnectionCls = HTTPConnection |
| ResponseCls = HTTPResponse |
| |
| def __init__( |
| self, |
| host, |
| port=None, |
| strict=False, |
| timeout=Timeout.DEFAULT_TIMEOUT, |
| maxsize=1, |
| block=False, |
| headers=None, |
| retries=None, |
| _proxy=None, |
| _proxy_headers=None, |
| _proxy_config=None, |
| **conn_kw |
| ): |
| ConnectionPool.__init__(self, host, port) |
| RequestMethods.__init__(self, headers) |
| |
| self.strict = strict |
| |
| if not isinstance(timeout, Timeout): |
| timeout = Timeout.from_float(timeout) |
| |
| if retries is None: |
| retries = Retry.DEFAULT |
| |
| self.timeout = timeout |
| self.retries = retries |
| |
| self.pool = self.QueueCls(maxsize) |
| self.block = block |
| |
| self.proxy = _proxy |
| self.proxy_headers = _proxy_headers or {} |
| self.proxy_config = _proxy_config |
| |
| # Fill the queue up so that doing get() on it will block properly |
| for _ in xrange(maxsize): |
| self.pool.put(None) |
| |
| # These are mostly for testing and debugging purposes. |
| self.num_connections = 0 |
| self.num_requests = 0 |
| self.conn_kw = conn_kw |
| |
| if self.proxy: |
| # Enable Nagle's algorithm for proxies, to avoid packet fragmentation. |
| # We cannot know if the user has added default socket options, so we cannot replace the |
| # list. |
| self.conn_kw.setdefault("socket_options", []) |
| |
| self.conn_kw["proxy"] = self.proxy |
| self.conn_kw["proxy_config"] = self.proxy_config |
| |
| def _new_conn(self): |
| """ |
| Return a fresh :class:`HTTPConnection`. |
| """ |
| self.num_connections += 1 |
| log.debug( |
| "Starting new HTTP connection (%d): %s:%s", |
| self.num_connections, |
| self.host, |
| self.port or "80", |
| ) |
| |
| conn = self.ConnectionCls( |
| host=self.host, |
| port=self.port, |
| timeout=self.timeout.connect_timeout, |
| strict=self.strict, |
| **self.conn_kw |
| ) |
| return conn |
| |
| def _get_conn(self, timeout=None): |
| """ |
| Get a connection. Will return a pooled connection if one is available. |
| |
| If no connections are available and :prop:`.block` is ``False``, then a |
| fresh connection is returned. |
| |
| :param timeout: |
| Seconds to wait before giving up and raising |
| :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and |
| :prop:`.block` is ``True``. |
| """ |
| conn = None |
| try: |
| conn = self.pool.get(block=self.block, timeout=timeout) |
| |
| except AttributeError: # self.pool is None |
| raise ClosedPoolError(self, "Pool is closed.") |
| |
| except queue.Empty: |
| if self.block: |
| raise EmptyPoolError( |
| self, |
| "Pool reached maximum size and no more connections are allowed.", |
| ) |
| pass # Oh well, we'll create a new connection then |
| |
| # If this is a persistent connection, check if it got disconnected |
| if conn and is_connection_dropped(conn): |
| log.debug("Resetting dropped connection: %s", self.host) |
| conn.close() |
| if getattr(conn, "auto_open", 1) == 0: |
| # This is a proxied connection that has been mutated by |
| # http.client._tunnel() and cannot be reused (since it would |
| # attempt to bypass the proxy) |
| conn = None |
| |
| return conn or self._new_conn() |
| |
| def _put_conn(self, conn): |
| """ |
| Put a connection back into the pool. |
| |
| :param conn: |
| Connection object for the current host and port as returned by |
| :meth:`._new_conn` or :meth:`._get_conn`. |
| |
| If the pool is already full, the connection is closed and discarded |
| because we exceeded maxsize. If connections are discarded frequently, |
| then maxsize should be increased. |
| |
| If the pool is closed, then the connection will be closed and discarded. |
| """ |
| try: |
| self.pool.put(conn, block=False) |
| return # Everything is dandy, done. |
| except AttributeError: |
| # self.pool is None. |
| pass |
| except queue.Full: |
| # This should never happen if self.block == True |
| log.warning( |
| "Connection pool is full, discarding connection: %s. Connection pool size: %s", |
| self.host, |
| self.pool.qsize(), |
| ) |
| # Connection never got put back into the pool, close it. |
| if conn: |
| conn.close() |
| |
| def _validate_conn(self, conn): |
| """ |
| Called right before a request is made, after the socket is created. |
| """ |
| pass |
| |
| def _prepare_proxy(self, conn): |
| # Nothing to do for HTTP connections. |
| pass |
| |
| def _get_timeout(self, timeout): |
| """Helper that always returns a :class:`urllib3.util.Timeout`""" |
| if timeout is _Default: |
| return self.timeout.clone() |
| |
| if isinstance(timeout, Timeout): |
| return timeout.clone() |
| else: |
| # User passed us an int/float. This is for backwards compatibility, |
| # can be removed later |
| return Timeout.from_float(timeout) |
| |
| def _raise_timeout(self, err, url, timeout_value): |
| """Is the error actually a timeout? Will raise a ReadTimeout or pass""" |
| |
| if isinstance(err, SocketTimeout): |
| raise ReadTimeoutError( |
| self, url, "Read timed out. (read timeout=%s)" % timeout_value |
| ) |
| |
| # See the above comment about EAGAIN in Python 3. In Python 2 we have |
| # to specifically catch it and throw the timeout error |
| if hasattr(err, "errno") and err.errno in _blocking_errnos: |
| raise ReadTimeoutError( |
| self, url, "Read timed out. (read timeout=%s)" % timeout_value |
| ) |
| |
| # Catch possible read timeouts thrown as SSL errors. If not the |
| # case, rethrow the original. We need to do this because of: |
| # http://bugs.python.org/issue10272 |
| if "timed out" in str(err) or "did not complete (read)" in str( |
| err |
| ): # Python < 2.7.4 |
| raise ReadTimeoutError( |
| self, url, "Read timed out. (read timeout=%s)" % timeout_value |
| ) |
| |
| def _make_request( |
| self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw |
| ): |
| """ |
| Perform a request on a given urllib connection object taken from our |
| pool. |
| |
| :param conn: |
| a connection from one of our connection pools |
| |
| :param timeout: |
| Socket timeout in seconds for the request. This can be a |
| float or integer, which will set the same timeout value for |
| the socket connect and the socket read, or an instance of |
| :class:`urllib3.util.Timeout`, which gives you more fine-grained |
| control over your timeouts. |
| """ |
| self.num_requests += 1 |
| |
| timeout_obj = self._get_timeout(timeout) |
| timeout_obj.start_connect() |
| conn.timeout = timeout_obj.connect_timeout |
| |
| # Trigger any extra validation we need to do. |
| try: |
| self._validate_conn(conn) |
| except (SocketTimeout, BaseSSLError) as e: |
| # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout. |
| self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) |
| raise |
| |
| # conn.request() calls http.client.*.request, not the method in |
| # urllib3.request. It also calls makefile (recv) on the socket. |
| try: |
| if chunked: |
| conn.request_chunked(method, url, **httplib_request_kw) |
| else: |
| conn.request(method, url, **httplib_request_kw) |
| |
| # We are swallowing BrokenPipeError (errno.EPIPE) since the server is |
| # legitimately able to close the connection after sending a valid response. |
| # With this behaviour, the received response is still readable. |
| except BrokenPipeError: |
| # Python 3 |
| pass |
| except IOError as e: |
| # Python 2 and macOS/Linux |
| # EPIPE and ESHUTDOWN are BrokenPipeError on Python 2, and EPROTOTYPE is needed on macOS |
| # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/ |
| if e.errno not in { |
| errno.EPIPE, |
| errno.ESHUTDOWN, |
| errno.EPROTOTYPE, |
| }: |
| raise |
| |
| # Reset the timeout for the recv() on the socket |
| read_timeout = timeout_obj.read_timeout |
| |
| # App Engine doesn't have a sock attr |
| if getattr(conn, "sock", None): |
| # In Python 3 socket.py will catch EAGAIN and return None when you |
| # try and read into the file pointer created by http.client, which |
| # instead raises a BadStatusLine exception. Instead of catching |
| # the exception and assuming all BadStatusLine exceptions are read |
| # timeouts, check for a zero timeout before making the request. |
| if read_timeout == 0: |
| raise ReadTimeoutError( |
| self, url, "Read timed out. (read timeout=%s)" % read_timeout |
| ) |
| if read_timeout is Timeout.DEFAULT_TIMEOUT: |
| conn.sock.settimeout(socket.getdefaulttimeout()) |
| else: # None or a value |
| conn.sock.settimeout(read_timeout) |
| |
| # Receive the response from the server |
| try: |
| try: |
| # Python 2.7, use buffering of HTTP responses |
| httplib_response = conn.getresponse(buffering=True) |
| except TypeError: |
| # Python 3 |
| try: |
| httplib_response = conn.getresponse() |
| except BaseException as e: |
| # Remove the TypeError from the exception chain in |
| # Python 3 (including for exceptions like SystemExit). |
| # Otherwise it looks like a bug in the code. |
| six.raise_from(e, None) |
| except (SocketTimeout, BaseSSLError, SocketError) as e: |
| self._raise_timeout(err=e, url=url, timeout_value=read_timeout) |
| raise |
| |
| # AppEngine doesn't have a version attr. |
| http_version = getattr(conn, "_http_vsn_str", "HTTP/?") |
| log.debug( |
| '%s://%s:%s "%s %s %s" %s %s', |
| self.scheme, |
| self.host, |
| self.port, |
| method, |
| url, |
| http_version, |
| httplib_response.status, |
| httplib_response.length, |
| ) |
| |
| try: |
| assert_header_parsing(httplib_response.msg) |
| except (HeaderParsingError, TypeError) as hpe: # Platform-specific: Python 3 |
| log.warning( |
| "Failed to parse headers (url=%s): %s", |
| self._absolute_url(url), |
| hpe, |
| exc_info=True, |
| ) |
| |
| return httplib_response |
| |
| def _absolute_url(self, path): |
| return Url(scheme=self.scheme, host=self.host, port=self.port, path=path).url |
| |
| def close(self): |
| """ |
| Close all pooled connections and disable the pool. |
| """ |
| if self.pool is None: |
| return |
| # Disable access to the pool |
| old_pool, self.pool = self.pool, None |
| |
| try: |
| while True: |
| conn = old_pool.get(block=False) |
| if conn: |
| conn.close() |
| |
| except queue.Empty: |
| pass # Done. |
| |
| def is_same_host(self, url): |
| """ |
| Check if the given ``url`` is a member of the same host as this |
| connection pool. |
| """ |
| if url.startswith("/"): |
| return True |
| |
| # TODO: Add optional support for socket.gethostbyname checking. |
| scheme, host, port = get_host(url) |
| if host is not None: |
| host = _normalize_host(host, scheme=scheme) |
| |
| # Use explicit default port for comparison when none is given |
| if self.port and not port: |
| port = port_by_scheme.get(scheme) |
| elif not self.port and port == port_by_scheme.get(scheme): |
| port = None |
| |
| return (scheme, host, port) == (self.scheme, self.host, self.port) |
| |
| def urlopen( |
| self, |
| method, |
| url, |
| body=None, |
| headers=None, |
| retries=None, |
| redirect=True, |
| assert_same_host=True, |
| timeout=_Default, |
| pool_timeout=None, |
| release_conn=None, |
| chunked=False, |
| body_pos=None, |
| **response_kw |
| ): |
| """ |
| Get a connection from the pool and perform an HTTP request. This is the |
| lowest level call for making a request, so you'll need to specify all |
| the raw details. |
| |
| .. note:: |
| |
| More commonly, it's appropriate to use a convenience method provided |
| by :class:`.RequestMethods`, such as :meth:`request`. |
| |
| .. note:: |
| |
| `release_conn` will only behave as expected if |
| `preload_content=False` because we want to make |
| `preload_content=False` the default behaviour someday soon without |
| breaking backwards compatibility. |
| |
| :param method: |
| HTTP request method (such as GET, POST, PUT, etc.) |
| |
| :param url: |
| The URL to perform the request on. |
| |
| :param body: |
| Data to send in the request body, either :class:`str`, :class:`bytes`, |
| an iterable of :class:`str`/:class:`bytes`, or a file-like object. |
| |
| :param headers: |
| Dictionary of custom headers to send, such as User-Agent, |
| If-None-Match, etc. If None, pool headers are used. If provided, |
| these headers completely replace any pool-specific headers. |
| |
| :param retries: |
| Configure the number of retries to allow before raising a |
| :class:`~urllib3.exceptions.MaxRetryError` exception. |
| |
| Pass ``None`` to retry until you receive a response. Pass a |
| :class:`~urllib3.util.retry.Retry` object for fine-grained control |
| over different types of retries. |
| Pass an integer number to retry connection errors that many times, |
| but no other types of errors. Pass zero to never retry. |
| |
| If ``False``, then retries are disabled and any exception is raised |
| immediately. Also, instead of raising a MaxRetryError on redirects, |
| the redirect response will be returned. |
| |
| :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. |
| |
| :param redirect: |
| If True, automatically handle redirects (status codes 301, 302, |
| 303, 307, 308). Each redirect counts as a retry. Disabling retries |
| will disable redirect, too. |
| |
| :param assert_same_host: |
| If ``True``, will make sure that the host of the pool requests is |
| consistent else will raise HostChangedError. When ``False``, you can |
| use the pool on an HTTP proxy and request foreign hosts. |
| |
| :param timeout: |
| If specified, overrides the default timeout for this one |
| request. It may be a float (in seconds) or an instance of |
| :class:`urllib3.util.Timeout`. |
| |
| :param pool_timeout: |
| If set and the pool is set to block=True, then this method will |
| block for ``pool_timeout`` seconds and raise EmptyPoolError if no |
| connection is available within the time period. |
| |
| :param release_conn: |
| If False, then the urlopen call will not release the connection |
| back into the pool once a response is received (but will release if |
| you read the entire contents of the response such as when |
| `preload_content=True`). This is useful if you're not preloading |
| the response's content immediately. You will need to call |
| ``r.release_conn()`` on the response ``r`` to return the connection |
| back into the pool. If None, it takes the value of |
| ``response_kw.get('preload_content', True)``. |
| |
| :param chunked: |
| If True, urllib3 will send the body using chunked transfer |
| encoding. Otherwise, urllib3 will send the body using the standard |
| content-length form. Defaults to False. |
| |
| :param int body_pos: |
| Position to seek to in file-like body in the event of a retry or |
| redirect. Typically this won't need to be set because urllib3 will |
| auto-populate the value when needed. |
| |
| :param \\**response_kw: |
| Additional parameters are passed to |
| :meth:`urllib3.response.HTTPResponse.from_httplib` |
| """ |
| |
| parsed_url = parse_url(url) |
| destination_scheme = parsed_url.scheme |
| |
| if headers is None: |
| headers = self.headers |
| |
| if not isinstance(retries, Retry): |
| retries = Retry.from_int(retries, redirect=redirect, default=self.retries) |
| |
| if release_conn is None: |
| release_conn = response_kw.get("preload_content", True) |
| |
| # Check host |
| if assert_same_host and not self.is_same_host(url): |
| raise HostChangedError(self, url, retries) |
| |
| # Ensure that the URL we're connecting to is properly encoded |
| if url.startswith("/"): |
| url = six.ensure_str(_encode_target(url)) |
| else: |
| url = six.ensure_str(parsed_url.url) |
| |
| conn = None |
| |
| # Track whether `conn` needs to be released before |
| # returning/raising/recursing. Update this variable if necessary, and |
| # leave `release_conn` constant throughout the function. That way, if |
| # the function recurses, the original value of `release_conn` will be |
| # passed down into the recursive call, and its value will be respected. |
| # |
| # See issue #651 [1] for details. |
| # |
| # [1] <https://github.com/urllib3/urllib3/issues/651> |
| release_this_conn = release_conn |
| |
| http_tunnel_required = connection_requires_http_tunnel( |
| self.proxy, self.proxy_config, destination_scheme |
| ) |
| |
| # Merge the proxy headers. Only done when not using HTTP CONNECT. We |
| # have to copy the headers dict so we can safely change it without those |
| # changes being reflected in anyone else's copy. |
| if not http_tunnel_required: |
| headers = headers.copy() |
| headers.update(self.proxy_headers) |
| |
| # Must keep the exception bound to a separate variable or else Python 3 |
| # complains about UnboundLocalError. |
| err = None |
| |
| # Keep track of whether we cleanly exited the except block. This |
| # ensures we do proper cleanup in finally. |
| clean_exit = False |
| |
| # Rewind body position, if needed. Record current position |
| # for future rewinds in the event of a redirect/retry. |
| body_pos = set_file_position(body, body_pos) |
| |
| try: |
| # Request a connection from the queue. |
| timeout_obj = self._get_timeout(timeout) |
| conn = self._get_conn(timeout=pool_timeout) |
| |
| conn.timeout = timeout_obj.connect_timeout |
| |
| is_new_proxy_conn = self.proxy is not None and not getattr( |
| conn, "sock", None |
| ) |
| if is_new_proxy_conn and http_tunnel_required: |
| self._prepare_proxy(conn) |
| |
| # Make the request on the httplib connection object. |
| httplib_response = self._make_request( |
| conn, |
| method, |
| url, |
| timeout=timeout_obj, |
| body=body, |
| headers=headers, |
| chunked=chunked, |
| ) |
| |
| # If we're going to release the connection in ``finally:``, then |
| # the response doesn't need to know about the connection. Otherwise |
| # it will also try to release it and we'll have a double-release |
| # mess. |
| response_conn = conn if not release_conn else None |
| |
| # Pass method to Response for length checking |
| response_kw["request_method"] = method |
| |
| # Import httplib's response into our own wrapper object |
| response = self.ResponseCls.from_httplib( |
| httplib_response, |
| pool=self, |
| connection=response_conn, |
| retries=retries, |
| **response_kw |
| ) |
| |
| # Everything went great! |
| clean_exit = True |
| |
| except EmptyPoolError: |
| # Didn't get a connection from the pool, no need to clean up |
| clean_exit = True |
| release_this_conn = False |
| raise |
| |
| except ( |
| TimeoutError, |
| HTTPException, |
| SocketError, |
| ProtocolError, |
| BaseSSLError, |
| SSLError, |
| CertificateError, |
| ) as e: |
| # Discard the connection for these exceptions. It will be |
| # replaced during the next _get_conn() call. |
| clean_exit = False |
| |
| def _is_ssl_error_message_from_http_proxy(ssl_error): |
| # We're trying to detect the message 'WRONG_VERSION_NUMBER' but |
| # SSLErrors are kinda all over the place when it comes to the message, |
| # so we try to cover our bases here! |
| message = " ".join(re.split("[^a-z]", str(ssl_error).lower())) |
| return ( |
| "wrong version number" in message or "unknown protocol" in message |
| ) |
| |
| # Try to detect a common user error with proxies which is to |
| # set an HTTP proxy to be HTTPS when it should be 'http://' |
| # (ie {'http': 'http://proxy', 'https': 'https://proxy'}) |
| # Instead we add a nice error message and point to a URL. |
| if ( |
| isinstance(e, BaseSSLError) |
| and self.proxy |
| and _is_ssl_error_message_from_http_proxy(e) |
| and conn.proxy |
| and conn.proxy.scheme == "https" |
| ): |
| e = ProxyError( |
| "Your proxy appears to only use HTTP and not HTTPS, " |
| "try changing your proxy URL to be HTTP. See: " |
| "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" |
| "#https-proxy-error-http-proxy", |
| SSLError(e), |
| ) |
| elif isinstance(e, (BaseSSLError, CertificateError)): |
| e = SSLError(e) |
| elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy: |
| e = ProxyError("Cannot connect to proxy.", e) |
| elif isinstance(e, (SocketError, HTTPException)): |
| e = ProtocolError("Connection aborted.", e) |
| |
| retries = retries.increment( |
| method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2] |
| ) |
| retries.sleep() |
| |
| # Keep track of the error for the retry warning. |
| err = e |
| |
| finally: |
| if not clean_exit: |
| # We hit some kind of exception, handled or otherwise. We need |
| # to throw the connection away unless explicitly told not to. |
| # Close the connection, set the variable to None, and make sure |
| # we put the None back in the pool to avoid leaking it. |
| conn = conn and conn.close() |
| release_this_conn = True |
| |
| if release_this_conn: |
| # Put the connection back to be reused. If the connection is |
| # expired then it will be None, which will get replaced with a |
| # fresh connection during _get_conn. |
| self._put_conn(conn) |
| |
| if not conn: |
| # Try again |
| log.warning( |
| "Retrying (%r) after connection broken by '%r': %s", retries, err, url |
| ) |
| return self.urlopen( |
| method, |
| url, |
| body, |
| headers, |
| retries, |
| redirect, |
| assert_same_host, |
| timeout=timeout, |
| pool_timeout=pool_timeout, |
| release_conn=release_conn, |
| chunked=chunked, |
| body_pos=body_pos, |
| **response_kw |
| ) |
| |
| # Handle redirect? |
| redirect_location = redirect and response.get_redirect_location() |
| if redirect_location: |
| if response.status == 303: |
| method = "GET" |
| |
| try: |
| retries = retries.increment(method, url, response=response, _pool=self) |
| except MaxRetryError: |
| if retries.raise_on_redirect: |
| response.drain_conn() |
| raise |
| return response |
| |
| response.drain_conn() |
| retries.sleep_for_retry(response) |
| log.debug("Redirecting %s -> %s", url, redirect_location) |
| return self.urlopen( |
| method, |
| redirect_location, |
| body, |
| headers, |
| retries=retries, |
| redirect=redirect, |
| assert_same_host=assert_same_host, |
| timeout=timeout, |
| pool_timeout=pool_timeout, |
| release_conn=release_conn, |
| chunked=chunked, |
| body_pos=body_pos, |
| **response_kw |
| ) |
| |
| # Check if we should retry the HTTP response. |
| has_retry_after = bool(response.getheader("Retry-After")) |
| if retries.is_retry(method, response.status, has_retry_after): |
| try: |
| retries = retries.increment(method, url, response=response, _pool=self) |
| except MaxRetryError: |
| if retries.raise_on_status: |
| response.drain_conn() |
| raise |
| return response |
| |
| response.drain_conn() |
| retries.sleep(response) |
| log.debug("Retry: %s", url) |
| return self.urlopen( |
| method, |
| url, |
| body, |
| headers, |
| retries=retries, |
| redirect=redirect, |
| assert_same_host=assert_same_host, |
| timeout=timeout, |
| pool_timeout=pool_timeout, |
| release_conn=release_conn, |
| chunked=chunked, |
| body_pos=body_pos, |
| **response_kw |
| ) |
| |
| return response |
| |
| |
| class HTTPSConnectionPool(HTTPConnectionPool): |
| """ |
| Same as :class:`.HTTPConnectionPool`, but HTTPS. |
| |
| :class:`.HTTPSConnection` uses one of ``assert_fingerprint``, |
| ``assert_hostname`` and ``host`` in this order to verify connections. |
| If ``assert_hostname`` is False, no verification is done. |
| |
| The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``, |
| ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl` |
| is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade |
| the connection socket into an SSL socket. |
| """ |
| |
| scheme = "https" |
| ConnectionCls = HTTPSConnection |
| |
| def __init__( |
| self, |
| host, |
| port=None, |
| strict=False, |
| timeout=Timeout.DEFAULT_TIMEOUT, |
| maxsize=1, |
| block=False, |
| headers=None, |
| retries=None, |
| _proxy=None, |
| _proxy_headers=None, |
| key_file=None, |
| cert_file=None, |
| cert_reqs=None, |
| key_password=None, |
| ca_certs=None, |
| ssl_version=None, |
| assert_hostname=None, |
| assert_fingerprint=None, |
| ca_cert_dir=None, |
| **conn_kw |
| ): |
| |
| HTTPConnectionPool.__init__( |
| self, |
| host, |
| port, |
| strict, |
| timeout, |
| maxsize, |
| block, |
| headers, |
| retries, |
| _proxy, |
| _proxy_headers, |
| **conn_kw |
| ) |
| |
| self.key_file = key_file |
| self.cert_file = cert_file |
| self.cert_reqs = cert_reqs |
| self.key_password = key_password |
| self.ca_certs = ca_certs |
| self.ca_cert_dir = ca_cert_dir |
| self.ssl_version = ssl_version |
| self.assert_hostname = assert_hostname |
| self.assert_fingerprint = assert_fingerprint |
| |
| def _prepare_conn(self, conn): |
| """ |
| Prepare the ``connection`` for :meth:`urllib3.util.ssl_wrap_socket` |
| and establish the tunnel if proxy is used. |
| """ |
| |
| if isinstance(conn, VerifiedHTTPSConnection): |
| conn.set_cert( |
| key_file=self.key_file, |
| key_password=self.key_password, |
| cert_file=self.cert_file, |
| cert_reqs=self.cert_reqs, |
| ca_certs=self.ca_certs, |
| ca_cert_dir=self.ca_cert_dir, |
| assert_hostname=self.assert_hostname, |
| assert_fingerprint=self.assert_fingerprint, |
| ) |
| conn.ssl_version = self.ssl_version |
| return conn |
| |
| def _prepare_proxy(self, conn): |
| """ |
| Establishes a tunnel connection through HTTP CONNECT. |
| |
| Tunnel connection is established early because otherwise httplib would |
| improperly set Host: header to proxy's IP:port. |
| """ |
| |
| conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers) |
| |
| if self.proxy.scheme == "https": |
| conn.tls_in_tls_required = True |
| |
| conn.connect() |
| |
| def _new_conn(self): |
| """ |
| Return a fresh :class:`http.client.HTTPSConnection`. |
| """ |
| self.num_connections += 1 |
| log.debug( |
| "Starting new HTTPS connection (%d): %s:%s", |
| self.num_connections, |
| self.host, |
| self.port or "443", |
| ) |
| |
| if not self.ConnectionCls or self.ConnectionCls is DummyConnection: |
| raise SSLError( |
| "Can't connect to HTTPS URL because the SSL module is not available." |
| ) |
| |
| actual_host = self.host |
| actual_port = self.port |
| if self.proxy is not None: |
| actual_host = self.proxy.host |
| actual_port = self.proxy.port |
| |
| conn = self.ConnectionCls( |
| host=actual_host, |
| port=actual_port, |
| timeout=self.timeout.connect_timeout, |
| strict=self.strict, |
| cert_file=self.cert_file, |
| key_file=self.key_file, |
| key_password=self.key_password, |
| **self.conn_kw |
| ) |
| |
| return self._prepare_conn(conn) |
| |
| def _validate_conn(self, conn): |
| """ |
| Called right before a request is made, after the socket is created. |
| """ |
| super(HTTPSConnectionPool, self)._validate_conn(conn) |
| |
| # Force connect early to allow us to validate the connection. |
| if not getattr(conn, "sock", None): # AppEngine might not have `.sock` |
| conn.connect() |
| |
| if not conn.is_verified: |
| warnings.warn( |
| ( |
| "Unverified HTTPS request is being made to host '%s'. " |
| "Adding certificate verification is strongly advised. See: " |
| "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" |
| "#ssl-warnings" % conn.host |
| ), |
| InsecureRequestWarning, |
| ) |
| |
| if getattr(conn, "proxy_is_verified", None) is False: |
| warnings.warn( |
| ( |
| "Unverified HTTPS connection done to an HTTPS proxy. " |
| "Adding certificate verification is strongly advised. See: " |
| "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" |
| "#ssl-warnings" |
| ), |
| InsecureRequestWarning, |
| ) |
| |
| |
| def connection_from_url(url, **kw): |
| """ |
| Given a url, return an :class:`.ConnectionPool` instance of its host. |
| |
| This is a shortcut for not having to parse out the scheme, host, and port |
| of the url before creating an :class:`.ConnectionPool` instance. |
| |
| :param url: |
| Absolute URL string that must include the scheme. Port is optional. |
| |
| :param \\**kw: |
| Passes additional parameters to the constructor of the appropriate |
| :class:`.ConnectionPool`. Useful for specifying things like |
| timeout, maxsize, headers, etc. |
| |
| Example:: |
| |
| >>> conn = connection_from_url('http://google.com/') |
| >>> r = conn.request('GET', '/') |
| """ |
| scheme, host, port = get_host(url) |
| port = port or port_by_scheme.get(scheme, 80) |
| if scheme == "https": |
| return HTTPSConnectionPool(host, port=port, **kw) |
| else: |
| return HTTPConnectionPool(host, port=port, **kw) |
| |
| |
| def _normalize_host(host, scheme): |
| """ |
| Normalize hosts for comparisons and use with sockets. |
| """ |
| |
| host = normalize_host(host, scheme) |
| |
| # httplib doesn't like it when we include brackets in IPv6 addresses |
| # Specifically, if we include brackets but also pass the port then |
| # httplib crazily doubles up the square brackets on the Host header. |
| # Instead, we need to make sure we never pass ``None`` as the port. |
| # However, for backward compatibility reasons we can't actually |
| # *assert* that. See http://bugs.python.org/issue28539 |
| if host.startswith("[") and host.endswith("]"): |
| host = host[1:-1] |
| return host |
