1# -*- coding: utf-8 -*-
2
3"""
4requests.adapters
5~~~~~~~~~~~~~~~~~
6
7This module contains the transport adapters that Requests uses to define
8and maintain connections.
9"""
10
11import os.path
12import socket
13
14from .models import Response
15from .packages.urllib3.poolmanager import PoolManager, proxy_from_url
16from .packages.urllib3.response import HTTPResponse
17from .packages.urllib3.util import Timeout as TimeoutSauce
18from .packages.urllib3.util.retry import Retry
19from .compat import urlparse, basestring
20from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers,
21                    prepend_scheme_if_needed, get_auth_from_url, urldefragauth,
22                    select_proxy)
23from .structures import CaseInsensitiveDict
24from .packages.urllib3.exceptions import ClosedPoolError
25from .packages.urllib3.exceptions import ConnectTimeoutError
26from .packages.urllib3.exceptions import HTTPError as _HTTPError
27from .packages.urllib3.exceptions import MaxRetryError
28from .packages.urllib3.exceptions import NewConnectionError
29from .packages.urllib3.exceptions import ProxyError as _ProxyError
30from .packages.urllib3.exceptions import ProtocolError
31from .packages.urllib3.exceptions import ReadTimeoutError
32from .packages.urllib3.exceptions import SSLError as _SSLError
33from .packages.urllib3.exceptions import ResponseError
34from .cookies import extract_cookies_to_jar
35from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError,
36                         ProxyError, RetryError)
37from .auth import _basic_auth_str
38
39DEFAULT_POOLBLOCK = False
40DEFAULT_POOLSIZE = 10
41DEFAULT_RETRIES = 0
42DEFAULT_POOL_TIMEOUT = None
43
44
45class BaseAdapter(object):
46    """The Base Transport Adapter"""
47
48    def __init__(self):
49        super(BaseAdapter, self).__init__()
50
51    def send(self):
52        raise NotImplementedError
53
54    def close(self):
55        raise NotImplementedError
56
57
58class HTTPAdapter(BaseAdapter):
59    """The built-in HTTP Adapter for urllib3.
60
61    Provides a general-case interface for Requests sessions to contact HTTP and
62    HTTPS urls by implementing the Transport Adapter interface. This class will
63    usually be created by the :class:`Session <Session>` class under the
64    covers.
65
66    :param pool_connections: The number of urllib3 connection pools to cache.
67    :param pool_maxsize: The maximum number of connections to save in the pool.
68    :param int max_retries: The maximum number of retries each connection
69        should attempt. Note, this applies only to failed DNS lookups, socket
70        connections and connection timeouts, never to requests where data has
71        made it to the server. By default, Requests does not retry failed
72        connections. If you need granular control over the conditions under
73        which we retry a request, import urllib3's ``Retry`` class and pass
74        that instead.
75    :param pool_block: Whether the connection pool should block for connections.
76
77    Usage::
78
79      >>> import requests
80      >>> s = requests.Session()
81      >>> a = requests.adapters.HTTPAdapter(max_retries=3)
82      >>> s.mount('http://', a)
83    """
84    __attrs__ = ['max_retries', 'config', '_pool_connections', '_pool_maxsize',
85                 '_pool_block']
86
87    def __init__(self, pool_connections=DEFAULT_POOLSIZE,
88                 pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES,
89                 pool_block=DEFAULT_POOLBLOCK):
90        if max_retries == DEFAULT_RETRIES:
91            self.max_retries = Retry(0, read=False)
92        else:
93            self.max_retries = Retry.from_int(max_retries)
94        self.config = {}
95        self.proxy_manager = {}
96
97        super(HTTPAdapter, self).__init__()
98
99        self._pool_connections = pool_connections
100        self._pool_maxsize = pool_maxsize
101        self._pool_block = pool_block
102
103        self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)
104
105    def __getstate__(self):
106        return dict((attr, getattr(self, attr, None)) for attr in
107                    self.__attrs__)
108
109    def __setstate__(self, state):
110        # Can't handle by adding 'proxy_manager' to self.__attrs__ because
111        # self.poolmanager uses a lambda function, which isn't pickleable.
112        self.proxy_manager = {}
113        self.config = {}
114
115        for attr, value in state.items():
116            setattr(self, attr, value)
117
118        self.init_poolmanager(self._pool_connections, self._pool_maxsize,
119                              block=self._pool_block)
120
121    def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs):
122        """Initializes a urllib3 PoolManager.
123
124        This method should not be called from user code, and is only
125        exposed for use when subclassing the
126        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
127
128        :param connections: The number of urllib3 connection pools to cache.
129        :param maxsize: The maximum number of connections to save in the pool.
130        :param block: Block when no free connections are available.
131        :param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager.
132        """
133        # save these values for pickling
134        self._pool_connections = connections
135        self._pool_maxsize = maxsize
136        self._pool_block = block
137
138        self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize,
139                                       block=block, strict=True, **pool_kwargs)
140
141    def proxy_manager_for(self, proxy, **proxy_kwargs):
142        """Return urllib3 ProxyManager for the given proxy.
143
144        This method should not be called from user code, and is only
145        exposed for use when subclassing the
146        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
147
148        :param proxy: The proxy to return a urllib3 ProxyManager for.
149        :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager.
150        :returns: ProxyManager
151        """
152        if not proxy in self.proxy_manager:
153            proxy_headers = self.proxy_headers(proxy)
154            self.proxy_manager[proxy] = proxy_from_url(
155                proxy,
156                proxy_headers=proxy_headers,
157                num_pools=self._pool_connections,
158                maxsize=self._pool_maxsize,
159                block=self._pool_block,
160                **proxy_kwargs)
161
162        return self.proxy_manager[proxy]
163
164    def cert_verify(self, conn, url, verify, cert):
165        """Verify a SSL certificate. This method should not be called from user
166        code, and is only exposed for use when subclassing the
167        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
168
169        :param conn: The urllib3 connection object associated with the cert.
170        :param url: The requested URL.
171        :param verify: Whether we should actually verify the certificate.
172        :param cert: The SSL certificate to verify.
173        """
174        if url.lower().startswith('https') and verify:
175
176            cert_loc = None
177
178            # Allow self-specified cert location.
179            if verify is not True:
180                cert_loc = verify
181
182            if not cert_loc:
183                cert_loc = DEFAULT_CA_BUNDLE_PATH
184
185            if not cert_loc:
186                raise Exception("Could not find a suitable SSL CA certificate bundle.")
187
188            conn.cert_reqs = 'CERT_REQUIRED'
189
190            if not os.path.isdir(cert_loc):
191                conn.ca_certs = cert_loc
192            else:
193                conn.ca_cert_dir = cert_loc
194        else:
195            conn.cert_reqs = 'CERT_NONE'
196            conn.ca_certs = None
197            conn.ca_cert_dir = None
198
199        if cert:
200            if not isinstance(cert, basestring):
201                conn.cert_file = cert[0]
202                conn.key_file = cert[1]
203            else:
204                conn.cert_file = cert
205
206    def build_response(self, req, resp):
207        """Builds a :class:`Response <requests.Response>` object from a urllib3
208        response. This should not be called from user code, and is only exposed
209        for use when subclassing the
210        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
211
212        :param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
213        :param resp: The urllib3 response object.
214        """
215        response = Response()
216
217        # Fallback to None if there's no status_code, for whatever reason.
218        response.status_code = getattr(resp, 'status', None)
219
220        # Make headers case-insensitive.
221        response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
222
223        # Set encoding.
224        response.encoding = get_encoding_from_headers(response.headers)
225        response.raw = resp
226        response.reason = response.raw.reason
227
228        if isinstance(req.url, bytes):
229            response.url = req.url.decode('utf-8')
230        else:
231            response.url = req.url
232
233        # Add new cookies from the server.
234        extract_cookies_to_jar(response.cookies, req, resp)
235
236        # Give the Response some context.
237        response.request = req
238        response.connection = self
239
240        return response
241
242    def get_connection(self, url, proxies=None):
243        """Returns a urllib3 connection for the given URL. This should not be
244        called from user code, and is only exposed for use when subclassing the
245        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
246
247        :param url: The URL to connect to.
248        :param proxies: (optional) A Requests-style dictionary of proxies used on this request.
249        """
250        proxy = select_proxy(url, proxies)
251
252        if proxy:
253            proxy = prepend_scheme_if_needed(proxy, 'http')
254            proxy_manager = self.proxy_manager_for(proxy)
255            conn = proxy_manager.connection_from_url(url)
256        else:
257            # Only scheme should be lower case
258            parsed = urlparse(url)
259            url = parsed.geturl()
260            conn = self.poolmanager.connection_from_url(url)
261
262        return conn
263
264    def close(self):
265        """Disposes of any internal state.
266
267        Currently, this just closes the PoolManager, which closes pooled
268        connections.
269        """
270        self.poolmanager.clear()
271
272    def request_url(self, request, proxies):
273        """Obtain the url to use when making the final request.
274
275        If the message is being sent through a HTTP proxy, the full URL has to
276        be used. Otherwise, we should only use the path portion of the URL.
277
278        This should not be called from user code, and is only exposed for use
279        when subclassing the
280        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
281
282        :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
283        :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs.
284        """
285        proxy = select_proxy(request.url, proxies)
286        scheme = urlparse(request.url).scheme
287        if proxy and scheme != 'https':
288            url = urldefragauth(request.url)
289        else:
290            url = request.path_url
291
292        return url
293
294    def add_headers(self, request, **kwargs):
295        """Add any headers needed by the connection. As of v2.0 this does
296        nothing by default, but is left for overriding by users that subclass
297        the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
298
299        This should not be called from user code, and is only exposed for use
300        when subclassing the
301        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
302
303        :param request: The :class:`PreparedRequest <PreparedRequest>` to add headers to.
304        :param kwargs: The keyword arguments from the call to send().
305        """
306        pass
307
308    def proxy_headers(self, proxy):
309        """Returns a dictionary of the headers to add to any request sent
310        through a proxy. This works with urllib3 magic to ensure that they are
311        correctly sent to the proxy, rather than in a tunnelled request if
312        CONNECT is being used.
313
314        This should not be called from user code, and is only exposed for use
315        when subclassing the
316        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
317
318        :param proxies: The url of the proxy being used for this request.
319        """
320        headers = {}
321        username, password = get_auth_from_url(proxy)
322
323        if username and password:
324            headers['Proxy-Authorization'] = _basic_auth_str(username,
325                                                             password)
326
327        return headers
328
329    def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None):
330        """Sends PreparedRequest object. Returns Response object.
331
332        :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
333        :param stream: (optional) Whether to stream the request content.
334        :param timeout: (optional) How long to wait for the server to send
335            data before giving up, as a float, or a :ref:`(connect timeout,
336            read timeout) <timeouts>` tuple.
337        :type timeout: float or tuple
338        :param verify: (optional) Whether to verify SSL certificates.
339        :param cert: (optional) Any user-provided SSL certificate to be trusted.
340        :param proxies: (optional) The proxies dictionary to apply to the request.
341        """
342
343        conn = self.get_connection(request.url, proxies)
344
345        self.cert_verify(conn, request.url, verify, cert)
346        url = self.request_url(request, proxies)
347        self.add_headers(request)
348
349        chunked = not (request.body is None or 'Content-Length' in request.headers)
350
351        if isinstance(timeout, tuple):
352            try:
353                connect, read = timeout
354                timeout = TimeoutSauce(connect=connect, read=read)
355            except ValueError as e:
356                # this may raise a string formatting error.
357                err = ("Invalid timeout {0}. Pass a (connect, read) "
358                       "timeout tuple, or a single float to set "
359                       "both timeouts to the same value".format(timeout))
360                raise ValueError(err)
361        else:
362            timeout = TimeoutSauce(connect=timeout, read=timeout)
363
364        try:
365            if not chunked:
366                resp = conn.urlopen(
367                    method=request.method,
368                    url=url,
369                    body=request.body,
370                    headers=request.headers,
371                    redirect=False,
372                    assert_same_host=False,
373                    preload_content=False,
374                    decode_content=False,
375                    retries=self.max_retries,
376                    timeout=timeout
377                )
378
379            # Send the request.
380            else:
381                if hasattr(conn, 'proxy_pool'):
382                    conn = conn.proxy_pool
383
384                low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT)
385
386                try:
387                    low_conn.putrequest(request.method,
388                                        url,
389                                        skip_accept_encoding=True)
390
391                    for header, value in request.headers.items():
392                        low_conn.putheader(header, value)
393
394                    low_conn.endheaders()
395
396                    for i in request.body:
397                        low_conn.send(hex(len(i))[2:].encode('utf-8'))
398                        low_conn.send(b'\r\n')
399                        low_conn.send(i)
400                        low_conn.send(b'\r\n')
401                    low_conn.send(b'0\r\n\r\n')
402
403                    # Receive the response from the server
404                    try:
405                        # For Python 2.7+ versions, use buffering of HTTP
406                        # responses
407                        r = low_conn.getresponse(buffering=True)
408                    except TypeError:
409                        # For compatibility with Python 2.6 versions and back
410                        r = low_conn.getresponse()
411
412                    resp = HTTPResponse.from_httplib(
413                        r,
414                        pool=conn,
415                        connection=low_conn,
416                        preload_content=False,
417                        decode_content=False
418                    )
419                except:
420                    # If we hit any problems here, clean up the connection.
421                    # Then, reraise so that we can handle the actual exception.
422                    low_conn.close()
423                    raise
424
425        except (ProtocolError, socket.error) as err:
426            raise ConnectionError(err, request=request)
427
428        except MaxRetryError as e:
429            if isinstance(e.reason, ConnectTimeoutError):
430                # TODO: Remove this in 3.0.0: see #2811
431                if not isinstance(e.reason, NewConnectionError):
432                    raise ConnectTimeout(e, request=request)
433
434            if isinstance(e.reason, ResponseError):
435                raise RetryError(e, request=request)
436
437            raise ConnectionError(e, request=request)
438
439        except ClosedPoolError as e:
440            raise ConnectionError(e, request=request)
441
442        except _ProxyError as e:
443            raise ProxyError(e)
444
445        except (_SSLError, _HTTPError) as e:
446            if isinstance(e, _SSLError):
447                raise SSLError(e, request=request)
448            elif isinstance(e, ReadTimeoutError):
449                raise ReadTimeout(e, request=request)
450            else:
451                raise
452
453        return self.build_response(request, resp)
454