david / django-roa (http://welldev.org/)

Turn your models into remote resources that you can access through Django's ORM. ROA stands for Resource Oriented Architecture.

Clone this repository (size: 560.7 KB): HTTPS / SSH
$ hg clone http://code.welldev.org/django-roa
commit 134: abc332ad5029
parent 133: 580369e844a4
branch: default
Update restkit to the latest tip (5afc7b3bed0a), using the new way to access the body of the response
David Larlet / david
8 months ago

Changed (Δ31.7 KB):

raw changeset »

django_roa/db/models.py (2 lines added, 1 lines removed)

django_roa/db/query.py (3 lines added, 3 lines removed)

examples/django_roa_client/tests.py (1 lines added, 1 lines removed)

restkit/__init__.py (2 lines added, 2 lines removed)

restkit/bin/rest_cli.py (10 lines added, 14 lines removed)

restkit/errors.py (63 lines added, 4 lines removed)

restkit/ext/eventlet_pool.py (94 lines added, 90 lines removed)

restkit/ext/webob_helper.py (61 lines added, 0 lines removed)

restkit/forms.py (150 lines added, 0 lines removed)

restkit/httpc.py (219 lines added, 177 lines removed)

restkit/pool.py (126 lines added, 162 lines removed)

restkit/rest.py (212 lines added, 678 lines removed)

restkit/utils.py (43 lines added, 0 lines removed)

Up to file-list django_roa/db/models.py:

@@ -353,10 +353,11 @@ class ROAModel(models.Model):
353
353
                except RequestFailed, e:
354
354
                    raise ROAException(e)
355
355
            
356
            response = force_unicode(response.body).encode(settings.DEFAULT_CHARSET)
357
            
356
358
            for local_name, remote_name in ROA_MODEL_NAME_MAPPING:
357
359
                response = response.replace(remote_name, local_name)
358
360
            
359
            response = force_unicode(response).encode(settings.DEFAULT_CHARSET)
360
361
            deserializer = serializers.get_deserializer(ROA_FORMAT)
361
362
            if hasattr(deserializer, 'deserialize_object'):
362
363
                result = deserializer(response).deserialize_object(response)

Up to file-list django_roa/db/query.py:

@@ -176,7 +176,7 @@ class RemoteQuerySet(query.QuerySet):
176
176
        except Exception, e:
177
177
            raise ROAException(e)
178
178
179
        response = force_unicode(response).encode(settings.DEFAULT_CHARSET)
179
        response = force_unicode(response.body).encode(settings.DEFAULT_CHARSET)
180
180
        for local_name, remote_name in ROA_MODEL_NAME_MAPPING:
181
181
            response = response.replace(remote_name, local_name)
182
182
        
@@ -211,7 +211,7 @@ class RemoteQuerySet(query.QuerySet):
211
211
        except Exception, e:
212
212
            raise ROAException(e)
213
213
        
214
        return int(response)
214
        return int(response.body)
215
215
216
216
    def _get_from_id_or_pk(self, id=None, pk=None):
217
217
        """
@@ -242,7 +242,7 @@ class RemoteQuerySet(query.QuerySet):
242
242
        except Exception, e:
243
243
            raise ROAException(e)
244
244
        
245
        response = force_unicode(response).encode(settings.DEFAULT_CHARSET)
245
        response = force_unicode(response.body).encode(settings.DEFAULT_CHARSET)
246
246
        for local_name, remote_name in ROA_MODEL_NAME_MAPPING:
247
247
            response = response.replace(remote_name, local_name)
248
248
        

Up to file-list examples/django_roa_client/tests.py:

@@ -638,7 +638,7 @@ class ROASettingsTests(ROATestCase):
638
638
        self.assertEqual(repr(page), '<RemotePage: A custom serialized page (1)>')
639
639
        rc = RestClient()
640
640
        response = rc.get('http://127.0.0.1:8081/django_roa_server/remotepage/?format=custom')
641
        self.assertEqual(repr(response), 'u\'<?xml version="1.0" encoding="utf-8"?>\\n<django-test version="1.0">\\n <object pk="1" model="django_roa_server.remotepage">\\n  <field type="CharField" name="title">A custom serialized page</field>\\n </object>\\n</django-test>\'')
641
        self.assertEqual(repr(response.body), '\'<?xml version="1.0" encoding="utf-8"?>\\n<django-test version="1.0">\\n <object pk="1" model="django_roa_server.remotepage">\\n  <field type="CharField" name="title">A custom serialized page</field>\\n </object>\\n</django-test>\'')
642
642
        self.assertEqual(len(RemotePage.objects.all()), 1)
643
643
        page = RemotePage.objects.get(id=page.id)
644
644
        self.assertEqual(repr(page), '<RemotePage: A custom serialized page (1)>')

Up to file-list restkit/__init__.py:

@@ -24,8 +24,8 @@ USER_AGENT = "restkit/%s" % __version__
24
24
debuglevel = 0
25
25
26
26
from restkit.errors import *
27
from restkit.httpc import HttpClient
28
from restkit.rest import *
27
from restkit.httpc import HttpClient, HTTPResponse, ResponseStream
28
from restkit.rest import Resource, RestClient
29
29
30
30
31
31

Up to file-list restkit/bin/rest_cli.py:

16
16
#
17
17
import os
18
18
import sys
19
from optparse import OptionParser, OptionGroup
19
from optparse import OptionParser
20
20
import urlparse
21
21
import urllib
22
22
23
# python 2.6 and above compatibility
24
try:
25
    from urlparse import parse_qs as _parse_qs
26
except ImportError:
27
    from cgi import parse_qs as _parse_qs
23
from urlparse import parse_qs as parse_qs
28
24
29
25
import restkit
30
26
from restkit import httpc
@@ -62,7 +58,7 @@ class Url(object):
62
58
            self.path = ''
63
59
64
60
        if parts[3]:
65
            self.query = _parse_qs(parts[3])
61
            self.query = parse_qs(parts[3])
66
62
        else:
67
63
            self.query = {}
68
64
@@ -79,9 +75,8 @@ def make_query(string, method='GET', fna
79
75
        return 
80
76
81
77
    if uri.username:
82
        transport = ProxiedHttpClient()
83
        transport.add_authorization(httpc.BasicAuth((uri.username, uri.password)))
84
        res = restkit.Resource(uri.uri, transport=transport)
78
        res = restkit.Resource(uri.uri)
79
        res.add_authorization(httpc.BasicAuth((uri.username, uri.password)))
85
80
    else:
86
81
        res = restkit.Resource(uri.uri)
87
82
@@ -103,15 +98,16 @@ def make_query(string, method='GET', fna
103
98
            headers['Content-Length'] = os.path.getsize(fname)
104
99
            payload = open(fname, 'r')
105
100
106
    data = res.request(method, path=uri.path, payload=payload, headers=headers, **uri.query)
101
    result = res.request(method, path=uri.path, payload=payload, headers=headers, **uri.query)
107
102
108
103
    output = output or ''
109
104
    if not output or output == '-':
110
        return data
105
        return result.body
111
106
    else:
112
107
        try:
113
108
            f = open(output, 'wb')
114
            f.write(data)
109
            for block in result.body_file:
110
                f.write(block)
115
111
            f.close()
116
112
        except:
117
113
            print >>sys.stderr, "Can't save result in %s" % output
@@ -126,7 +122,7 @@ def main():
126
122
    parser.add_option('-i', '--input', action='store', dest='input', metavar='FILE',
127
123
                      help='the name of the file to read from')
128
124
    parser.add_option('-o', '--output', action='store', dest='output',
129
                      help='the name of the file to read from')
125
                      help='the name of the file to write to')
130
126
131
127
    parser.add_option('--proxy', action='store', dest='proxy',
132
128
            help='Full uri of proxy, ex:\n'+

Up to file-list restkit/errors.py:

18
18
"""
19
19
exception classes.
20
20
"""
21
import restkit
22
import warnings
21
23
24
class deprecated_property(object):
25
    """
26
    Wraps a decorator, with a deprecation warning or error
27
    """
28
    def __init__(self, decorator, attr, message, warning=True):
29
        self.decorator = decorator
30
        self.attr = attr
31
        self.message = message
32
        self.warning = warning
33
34
    def __get__(self, obj, type=None):
35
        if obj is None:
36
            return self
37
        self.warn()
38
        return self.decorator.__get__(obj, type)
39
40
    def __set__(self, obj, value):
41
        self.warn()
42
        self.decorator.__set__(obj, value)
43
44
    def __delete__(self, obj):
45
        self.warn()
46
        self.decorator.__delete__(obj)
47
48
    def __repr__(self):
49
        return '<Deprecated attribute %s: %r>' % (
50
            self.attr,
51
            self.decorator)
52
53
    def warn(self):
54
        if not self.warning:
55
            raise DeprecationWarning(
56
                'The attribute %s is deprecated: %s' % (self.attr, self.message))
57
        else:
58
            warnings.warn(
59
                'The attribute %s is deprecated: %s' % (self.attr, self.message),
60
                DeprecationWarning,
61
                stacklevel=3)
62
                
63
                    
22
64
class ResourceError(Exception):
23
65
    """ default error class """
24
66
    def __init__(self, msg=None, http_code=None, response=None):
25
67
        self.msg = msg or ''
26
        self.status_code = http_code
68
        self.status = http_code
27
69
        self.response = response
28
70
        Exception.__init__(self)
29
71
        
72
    def _status_int__get(self):
73
        """
74
        The status as an integer
75
        """
76
        return self.status
77
    def _status_int__set(self, http_code):
78
        self.status = http_code
79
    status_int = property(_status_int__get, _status_int__set, doc=_status_int__get.__doc__)
80
        
81
    status_code = deprecated_property(
82
        status_int, 'status_code', 'use .status_int instead',
83
        warning=False)
84
        
30
85
    def _get_message(self):
31
86
        return self.msg
32
87
    def _set_message(self, msg):
@@ -37,11 +92,12 @@ class ResourceError(Exception):
37
92
        if self.msg:
38
93
            return self.msg
39
94
        try:
40
            return self._fmt % self.__dict__
95
            return str(self.__dict__)
41
96
        except (NameError, ValueError, KeyError), e:
42
97
            return 'Unprintable exception %s: %s' \
43
98
                % (self.__class__.__name__, str(e))
44
        
99
                
100
45
101
class ResourceNotFound(ResourceError):
46
102
    """Exception raised when no resource was found at the given url. 
47
103
    """
@@ -83,3 +139,6 @@ class TransportError(Exception):
83
139
class ResponseError(Exception):
84
140
    """ Error raised while getting response or decompressing response stream"""
85
141
    
142
143
class ProxyError(Exception):
144
    """ raised when proxy error happend"""

Up to file-list restkit/ext/eventlet_pool.py:

15
15
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16
16
17
17
import os
18
import time
19
import urlparse
18
20
19
21
from eventlet.green import socket
20
from eventlet.green import httplib
22
from eventlet.green import httplib as ehttplib
21
23
from eventlet.pools import Pool
22
24
from eventlet.util import wrap_socket_with_coroutine_socket
23
25
26
import restkit
27
from restkit import errors
28
from restkit.pool import get_proxy_auth, PoolInterface
29
30
url_parser = urlparse.urlparse
31
32
24
33
wrap_socket_with_coroutine_socket()
25
34
26
def make_proxy_connection(uri):
27
    headers = headers or {}
28
    proxy = None
29
    if uri.scheme == 'https':
30
        proxy = os.environ.get('https_proxy')
31
    elif uri.scheme == 'http':
32
        proxy = os.environ.get('http_proxy')
35
eventlet_httplib = False
36
def wrap_eventlet_ehttplib():
37
    if eventlet_httplib: return
38
    import httplib
39
    ehttplib.BadStatusLine = httplib.BadStatusLine
40
    
41
wrap_eventlet_ehttplib()
33
42
34
    if not proxy:
35
        return make_connection(uri, use_proxy=False)
36
  
37
    if uri.scheme == 'https':
38
        proxy_auth = _get_proxy_auth()
39
        if proxy_auth:
40
            proxy_auth = 'Proxy-authorization: %s' % proxy_auth
41
        port = uri.port
42
        if not port:
43
            port = 443
44
        proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % (uri.hostname, port)
45
        user_agent = 'User-Agent: %s\r\n' % restkit.USER_AGENT
46
        proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth, user_agent)
47
        proxy_uri = url_parser(proxy)
48
        if not proxy_uri.port:
49
            proxy_uri.port = '80'
50
        # Connect to the proxy server, very simple recv and error checking
51
        p_sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
52
        p_sock.connect((proxy_uri.host, int(proxy_uri.port)))
53
        p_sock.sendall(proxy_pieces)
54
        response = ''
55
        # Wait for the full response.
56
        while response.find("\r\n\r\n") == -1:
57
            response += p_sock.recv(8192)
58
        p_status = response.split()[1]
59
        if p_status != str(200):
60
            raise ProxyError('Error status=%s' % str(p_status))
61
        # Trivial setup for ssl socket.
62
        ssl = socket.ssl(p_sock, None, None)
63
        fake_sock = httplib.FakeSocket(p_sock, ssl)
64
        # Initalize httplib and replace with the proxy socket.
65
        connection = httplib.HTTPConnection(proxy_uri.host)
66
        connection.sock=fake_sock
67
        return connection
68
    else:
69
        proxy_uri = url_parser(proxy)
70
        if not proxy_uri.port:
71
            proxy_uri.port = '80'
72
        return httplib.HTTPConnection(proxy_uri.hostname, proxy_uri.port)
73
    return None
74
    
75
def make_connection(uri, use_proxy=True):
76
    if use_proxy:
77
        return make_proxy_connection(uri)
78
    
79
    if uri.scheme == 'https':
80
        if not uri.port:
81
            connection = httplib.HTTPSConnection(uri.hostname)
82
        else:
83
            connection = httplib.HTTPSConnection(uri.hostname, uri.port)
84
    else:
85
        if not uri.port:
86
            connection = httplib.HTTPConnection(uri.hostname)
87
        else:
88
            connection = httplib.HTTPConnection(uri.hostname, uri.port)
89
    return connection
90
91
92
class ConnectionPool(Pool):
93
    def __init__(self, uri, use_proxy=False, min_size=0, max_size=4):
43
class ConnectionPool(Pool, PoolInterface):
44
    def __init__(self, uri, use_proxy=False, key_file=None,
45
            cert_file=None, min_size=0, max_size=4, **kwargs):
46
        Pool.__init__(self, min_size, max_size)
94
47
        self.uri = uri
95
48
        self.use_proxy = use_proxy
96
        Pool.__init__(self, min_size, max_size)
49
        self.key_file = key_file
50
        self.cert_file = cert_file
51
        
52
53
    def _make_proxy_connection(self, proxy):
54
        if self.uri.scheme == 'https':
55
            proxy_auth = get_proxy_auth()
56
            if proxy_auth:
57
                proxy_auth = 'Proxy-authorization: %s' % proxy_auth
58
            port = self.uri.port
59
            if not port:
60
                port = 443
61
            proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % (self.uri.hostname, port)
62
            user_agent = 'User-Agent: %s\r\n' % restkit.USER_AGENT
63
            proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth, user_agent)
64
            proxy_uri = url_parser(proxy)
65
            if not proxy_uri.port:
66
                proxy_uri.port = '80'
67
            # Connect to the proxy server, very simple recv and error checking
68
            p_sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
69
            p_sock.connect((proxy_uri.host, int(proxy_uri.port)))
70
            p_sock.sendall(proxy_pieces)
71
            response = ''
72
            # Wait for the full response.
73
            while response.find("\r\n\r\n") == -1:
74
                response += p_sock.recv(8192)
75
            p_status = response.split()[1]
76
            if p_status != str(200):
77
                raise errors.ProxyError('Error status=%s' % str(p_status))
78
            # Trivial setup for ssl socket.
79
            ssl = socket.ssl(p_sock, None, None)
80
            fake_sock = ehttplib.FakeSocket(p_sock, ssl)
81
            # Initalize ehttplib and replace with the proxy socket.
82
            connection = ehttplib.HTTPConnection(proxy_uri.host)
83
            connection.sock=fake_sock
84
            return connection
85
        else:
86
            proxy_uri = url_parser(proxy)
87
            if not proxy_uri.port:
88
                proxy_uri.port = '80'
89
            return ehttplib.HTTPConnection(proxy_uri.hostname, proxy_uri.port)
90
        return None
91
92
    def make_connection(self):
93
        if self.use_proxy:
94
            proxy = ''
95
            if self.uri.scheme == 'https':
96
                proxy = os.environ.get('https_proxy')
97
            elif self.uri.scheme == 'http':
98
                proxy = os.environ.get('http_proxy')
99
100
            if proxy:
101
                return self._make_proxy_connection(proxy)
102
103
        kwargs = {}
104
        if hasattr(ehttplib.HTTPConnection, 'timeout'):
105
            kwargs['timeout'] = self.timeout
106
107
        if self.uri.port:
108
            kwargs['port'] = self.uri.port
109
110
        if self.uri.scheme == "https":
111
            kwargs.update(dict(key_file=self.key_file, cert_file=self.cert_file))
112
            connection = ehttplib.HTTPSConnection(self.uri.hostname, **kwargs)
113
        else:
114
            connection = ehttplib.HTTPConnection(self.uri.hostname, **kwargs)
115
116
        setattr(connection, "started", time.time())
117
        return connection
97
118
    
98
119
    def create(self):
99
        return make_connection(self.uri, self.use_proxy)
100
           
101
    def put(self, connection):
102
        if self.current_size > self.max_size:
103
            self.current_size -= 1
104
            # close the connection if needed
105
            if connection.sock is not None:
106
                connection.close()
107
            return
108
        
109
        try:
110
            response = connection.getresponse()
111
            response.read()
112
        except httplib.ResponseNotReady:
113
            pass
114
        except:
115
            connection.close()
116
            
117
        if connection.sock is None:
118
            connection = self.create()
119
        Pool.put(self, connection)
120
        return self.make_connection()
121
                
122
    def clear(self):
123
        self.free()

Up to file-list restkit/ext/webob_helper.py:

1
# -*- coding: utf-8 -
2
#
3
# Copyright (c) 2008, 2009 Benoit Chesneau <benoitc@e-engura.com> 
4
#
5
# Permission to use, copy, modify, and distribute this software for any
6
# purpose with or without fee is hereby granted, provided that the above
7
# copyright notice and this permission notice appear in all copies.
8
#
9
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16
17
import restkit.errors
18
import webob.exc
19
20
class WebobResourceError(webob.exc.WSGIHTTPException):
21
22
    def __init__(self, msg=None, http_code=None, response=None):
23
        webob.exc.WSGIHTTPException.__init__(self)
24
        
25
        http_code = http_code or 500
26
        klass = webob.exc.status_map[http_code]
27
        self.code = http_code
28
        self.title = klass.title
29
        self.status = '%s %s' % (self.code, self.title)
30
        self.explanation = msg
31
        self.response = response
32
        # default params
33
        self.msg = msg
34
35
    def _status_int__get(self):
36
        """
37
        The status as an integer
38
        """
39
        return int(self.status.split()[0])
40
    def _status_int__set(self, value):
41
        self.status = value
42
    status_int = property(_status_int__get, _status_int__set, doc=_status_int__get.__doc__)
43
    
44
    status_code = restkit.errors.deprecated_property(
45
        status_int, 'status_code', 'use .status_int instead',
46
        warning=False)
47
48
    def _get_message(self):
49
        return self.explanation
50
    def _set_message(self, msg):
51
        self.explanation = msg or ''
52
    message = property(_get_message, _set_message)
53
54
webob_exceptions = False
55
def wrap_exceptions():
56
    """ wrap restkit exception to return WebBob exceptions"""
57
    global webob_exceptions
58
    if webob_exceptions: return
59
    restkit.errors.ResourceError = WebobResourceError
60
    webob_exceptions = True
61
    

Up to file-list restkit/forms.py:

1
# -*- coding: utf-8 -
2
#
3
# Copyright (c) 2008, 2009 Benoit Chesneau <benoitc@e-engura.com> 
4
#
5
# Permission to use, copy, modify, and distribute this software for any
6
# purpose with or without fee is hereby granted, provided that the above
7
# copyright notice and this permission notice appear in all copies.
8
#
9
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16
#
17
18
import mimetypes
19
import os
20
import re
21
import urllib
22
23
24
from restkit.utils import to_bytestring, url_quote
25
26
MIME_BOUNDARY = 'END_OF_PART'
27
28
def form_encode(obj, charser="utf8"):
29
    tmp = []
30
    for key, value in obj.items():
31
        tmp.append("%s=%s" % (url_quote(key), 
32
                url_quote(value)))
33
    return to_bytestring("&".join(tmp))
34
35
36
class BoundaryItem(object):
37
    def __init__(self, name, value, fname=None, filetype=None, filesize=None):
38
        self.name = url_quote(name)
39
        if value is not None and not hasattr(value, 'read'):
40
            value = url_quote(value)
41
            self.size = len(value)
42
        self.value = value
43
        if fname is not None:
44
            if isinstance(fname, unicode):
45
                fname = fname.encode("utf-8").encode("string_escape").replace('"', '\\"')
46
            else:
47
                fname = fname.encode("string_escape").replace('"', '\\"')
48
        self.fname = fname
49
        if filetype is not None:
50
            filetype = to_bytestring(filetype)
51
        self.filetype = filetype
52
        
53
        if isinstance(value, file) and filesize is None:
54
            try:
55
                value.flush()
56
            except IOError:
57
                pass
58
            self.size = int(os.fstat(value.fileno())[6])
59
            
60
    def encode_hdr(self, boundary):
61
        """Returns the header of the encoding of this parameter"""
62
        boundary = url_quote(boundary)
63
        headers = ["--%s" % boundary]
64
        if self.fname:
65
            disposition = 'form-data; name="%s"; filename="%s"' % (self.name,
66
                    self.fname)
67
        else:
68
            disposition = 'form-data; name="%s"' % self.name
69
        headers.append("Content-Disposition: %s" % disposition)
70
        if self.filetype:
71
            filetype = self.filetype
72
        else:
73
            filetype = "text/plain; charset=utf-8"
74
        headers.append("Content-Type: %s" % filetype)
75
        headers.append("Content-Length: %i" % self.size)
76
        headers.append("")
77
        headers.append("")
78
        return "\r\n".join(headers)
79
80
    def encode(self, boundary):
81
        """Returns the string encoding of this parameter"""
82
        value = self.value
83
        if re.search("^--%s$" % re.escape(boundary), value, re.M):
84
            raise ValueError("boundary found in encoded string")
85
86
        return "%s%s\r\n" % (self.encode_hdr(boundary), value)
87
        
88
    def iter_encode(self, boundary, blocksize=16384):
89
        if not hasattr(self.value, "read"):
90
            yield self.encode(boundary)
91
        else:
92
            yield self.encode_hdr(boundary)
93
            while True:
94
                block = self.value.read(blocksize)
95
                if not block:
96
                    yield "\r\n"
97
                    break
98
                yield block
99
                
100
                
101
class MultipartForm(object):
102
    
103
    def __init__(self, params, boundary, headers):
104
        self.boundary = boundary
105
        self.boundaries = []
106
        self.size = 0
107
        
108
        self.content_length = headers.get('Content-Length')
109
        
110
        if hasattr(params, 'items'):
111
            params = params.items()
112
            
113
        for param in params:
114
            name, value = param
115
            if hasattr(value, "read"):
116
                fname = getattr(value, 'name')
117
                if fname is not None:
118
                    filetype = ';'.join(filter(None, mimetypes.guess_type(fname)))
119
                else:
120
                    filetype = None
121
                if not isinstance(value, file) and self.content_length is None:
122
                    value = value.read()
123
                    
124
                boundary = BoundaryItem(name, value, fname, filetype)
125
            else:
126
                 boundary = BoundaryItem(name, value)
127
            self.boundaries.append(boundary)
128
129
    def get_size(self):
130
        if self.content_length is not None:
131
            return int(self.content_length)
132
        size = 0
133
        for boundary in self.boundaries:
134
            size = size + boundary.size
135
        return size
136
        
137
    def __iter__(self):
138
        for boundary in self.boundaries:
139
            for block in boundary.iter_encode(self.boundary):
140
                yield block
141
        yield "--%s--\r\n" % self.boundary
142
                    
143
144
def multipart_form_encode(params, headers, boundary):
145
    headers = headers or {}
146
    boundary = urllib.quote_plus(boundary)
147
    body = MultipartForm(params, boundary, headers)
148
    headers['Content-Type'] = "multipart/form-data; boundary=%s" % boundary
149
    headers['Content-Length'] = str(body.get_size())
150
    return body, headers

Up to file-list restkit/httpc.py:

29
29
# limitations under the License.
30
30
31
31
import base64
32
import copy
32
import gzip
33
33
import httplib
34
import os
35
34
import re
36
35
import socket
37
import StringIO
36
from StringIO import StringIO
38
37
import types
39
import urllib
40
38
import urlparse
39
import zlib
41
40
42
41
import restkit
43
42
from restkit import errors
44
from restkit.pool import ConnectionPool, get_proxy_auth
43
44
from restkit import pool
45
45
from restkit.utils import to_bytestring
46
46
47
47
48
MAX_CHUNK_SIZE = 16384
48
49
url_parser = urlparse.urlparse
49
50
50
51
NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+')
@@ -63,9 +64,7 @@ def _relative_uri(uri):
63
64
64
65
class Auth(object):
65
66
    """ Interface for Auth classes """
66
    def __init__(self, credentials, **kwargs):
67
        self.credentials = credentials
68
        
67
    
69
68
    def depth(self, uri):
70
69
        return uri.path.count("/")
71
70
        
@@ -75,73 +74,127 @@ class Auth(object):
75
74
        return True
76
75
        
77
76
    def request(self, uri, method, body, headers):
77
        """ path auth info to the request """
78
78
        pass
79
79
        
80
    def response(self, response, content):
81
        """ allow us to store new auth info from the response."""
80
    def response(self, response):
81
        """ allow us to store new auth info from the response.
82
        if something is wrong, should return True to redo 
83
        the request. Else return False.
84
        """
82
85
        return False
83
        
84
    def add_credentials(self, *args, **kwargs):
85
        raise NotImplementedError
86
86
87
87
class BasicAuth(Auth):
88
88
    """ basic authentification """
89
    
90
    def __init__(self, username, password):
91
        self.credentials = (username, password)
92
    
89
93
    def request(self, uri, method, body, headers):
90
        headers['authorization'] = 'Basic ' + base64.b64encode("%s:%s" % self.credentials).strip()
94
        headers['authorization'] = 'Basic ' + base64.encodestring("%s:%s" % self.credentials)[:-1]
95
96
class HTTPResponse(object):
97
    """ Object containing response."""
98
    
99
    charset = "utf8"
100
    unicode_errors = 'strict'
101
    
102
    def __init__(self, response, release_callback):
103
        self.resp = response
104
        self.release_callback = release_callback
105
        self.headerslist = response.getheaders()
106
        self.status = "%s %s" % (response.status, response.reason)
107
        self.status_int = response.status
108
        self.version = response.version
91
109
        
92
    def add_credentials(self, username, password=None):
93
        password = password or ""
94
        self.credentials = (username, password)
110
        headers = {}
111
        for key, value in self.headerslist:
112
            headers[key.lower()] = value
113
        self.headers = headers
114
        self.closed = False
115
        self._body = ""
116
            
117
    def get_body(self, stream=False):
118
        if self._body:
119
            return self._body
120
        body = _decompress_content(self, stream=stream)
121
        if not stream:
122
            self._body = body
123
        return body
124
125
    @property
126
    def unicode_body(self):
127
        if not self.charset:
128
            raise AttributeError(
129
                "You cannot access HTTPResponse.unicode_body unless charset is set")
130
        body = self.get_body()
131
        return body.decode(self.charset, self.unicode_errors)
132
133
    @property
134
    def body(self):
135
        """ get body in one bytestring """
136
        return self.get_body()
137
    
138
    @property
139
    def body_file(self):
140
        """ get body as a file object """
141
        return self.get_body(stream=True)
142
        
143
    def close(self):
144
        """ close the response"""
145
        if not self.closed:
146
            self.closed = True
147
            self.release_callback()
148
            if not self.resp.isclosed():
149
                self.resp.close()
150
95
151
96
152
#TODO : manage authentification detection
97
153
class HttpClient(object):
98
    MAX_REDIRECTIONS = 5
99
    
100
    def __init__(self, follow_redirect=True, force_follow_redirect=False,
101
            use_proxy=False, min_size=0, max_size=4, pool_class=None):
154
    max_redirections = 5
155
    pool_class = pool.ConnectionPool
156
    response_class = HTTPResponse
157
     
158
    def __init__(self, follow_redirect=True, force_follow_redirect=False, 
159
            response_class=None, pool_class=None, **conn_opts):
102
160
        self.authorizations = []
103
        self.use_proxy = use_proxy
161
        self.use_proxy = conn_opts.get("use_proxy", False)
104
162
        self.follow_redirect = follow_redirect
105
163
        self.force_follow_redirect = force_follow_redirect
106
        self.min_size = min_size
107
        self.max_size = max_size
108
        self.connections = {}
109
        if pool_class is None:
110
            self.pool_class = ConnectionPool
111
        else:
164
        self.conn_opts = conn_opts
165
        self._http_pool = {}
166
        if response_class is not None:
167
            self.response_class = response_class
168
        if pool_class is not None:
112
169
            self.pool_class = pool_class
113
        
170
114
171
    def add_authorization(self, obj_auth):
115
172
        self.authorizations.append(obj_auth)
116
173
        
117
    def _get_connection(self, uri, headers=None):
118
        connection = None
174
    def _get_pool(self, uri):
119
175
        conn_key = (uri.scheme, uri.netloc, self.use_proxy)
176
        if conn_key in self._http_pool:
177
            pool = self._http_pool[conn_key]
178
        else:
179
            pool = self.pool_class(uri, **self.conn_opts)
180
            self._http_pool[conn_key] = pool
181
        return pool
120
182
121
        if conn_key in self.connections:
122
            pool = self.connections[conn_key]
123
        else:
124
            pool = self.connections[conn_key] = self.pool_class(uri, 
125
                use_proxy=self.use_proxy, min_size=self.min_size,
126
                max_size=self.max_size)
127
        connection = pool.get()
128
        return connection
183
    def _get_connection(self, uri):
184
        pool = self._get_pool(uri)
185
        return  pool.get()
129
186
        
130
187
    def _release_connection(self, uri, connection):
131
        conn_key = (uri.scheme, uri.netloc, self.use_proxy)
132
133
        if conn_key in self.connections:
134
            pool = self.connections[conn_key]
135
        else:
136
            pool = self.connections[conn_key] =self.pool_class(uri,  
137
                use_proxy=self.use_proxy, min_size=self.min_size,
138
                max_size=self.max_size)
188
        pool = self._get_pool(uri)
139
189
        pool.put(connection)
140
141
            
190
        
191
    def _clean_pool(self, uri):
192
        pool = self._get_pool(uri)
193
        pool.clear()
194
        
142
195
    def _make_request(self, uri, method, body, headers): 
143
196
        for i in range(2):
144
            connection = self._get_connection(uri, headers)
197
            connection = self._get_connection(uri)
145
198
            connection.debuglevel = restkit.debuglevel
146
199
            try:
147
200
                if connection.host != uri.hostname:
@@ -173,9 +226,7 @@ class HttpClient(object):
173
226
                    if i > 0 and hasattr(body, 'seek'):
174
227
                        body.seek(0)
175
228
                        
176
                    if isinstance(body, types.StringTypes) and len(body) == 0:
177
                        connection.send("")
178
                    elif isinstance(body, types.StringTypes) or hasattr(body, 'read'):
229
                    if isinstance(body, types.StringTypes) or hasattr(body, 'read'):
179
230
                        _send_body_part(body, connection)
180
231
                    elif hasattr(body, "__iter__"):
181
232
                        for body_part in body:
@@ -185,22 +236,21 @@ class HttpClient(object):
185
236
                            _send_body_part(body_part, connection)
186
237
                    else:
187
238
                        _send_body_part(body, connection)
188
                    
189
            except socket.gaierror:
190
                connection.close()
191
                self._release_connection(uri, connection)
239
                response = connection.getresponse()
240
            except socket.gaierror, e:
241
                self._clean_pool(uri)
192
242
                raise errors.ResourceNotFound("Unable to find the server at %s" % connection.host, 404)
193
            except (socket.error, httplib.HTTPException):
194
                connection.close()
195
                self._release_connection(uri, connection)
243
            except (socket.error, httplib.BadStatusLine), e:
244
                # we should do better error parsing here
245
                self._clean_pool(uri)
196
246
                if i == 0:
197
247
                    continue
198
248
                else:
199
                    raise
249
                    raise errors.RequestFailed("socket error %s" % str(e), 500)
200
250
            break
201
251
                    
202
252
        # Return the HTTP Response from the server.
203
        return connection
253
        return response, connection
204
254
        
205
255
    def _request(self, uri, method, body, headers, nb_redirections=0):
206
256
        auths = [(auth.depth(uri), auth) for auth in self.authorizations if auth.inscope(uri.hostname, uri)]
@@ -209,18 +259,15 @@ class HttpClient(object):
209
259
            auth.request(uri, method, body, headers)
210
260
            
211
261
        headers = _normalize_headers(headers)
212
        old_response = None
213
262
        
214
        connection = self._make_request(uri, method, body, headers)
215
        response = connection.getresponse()
263
        response, connection = self._make_request(uri, method, body, headers)
216
264
        
217
        if auth and auth.response(response, body):
265
        if auth and auth.response(response):
218
266
            auth.request(uri, method, headers, body)
219
            connection = self._make_request(uri, method, body, headers)
220
            response = connection.getresponse()
267
            response, connection = self._make_request(uri, method, body, headers)
221
268
            
222
269
        if self.follow_redirect:
223
            if nb_redirections < self.MAX_REDIRECTIONS: 
270
            if nb_redirections < self.max_redirections: 
224
271
                if response.status in [301, 302, 307]:
225
272
                    if method in ["GET", "HEAD"] or self.force_follow_redirect:
226
273
                        if method not in ["GET", "HEAD"] and hasattr(body, 'seek'):
@@ -250,148 +297,143 @@ class HttpClient(object):
250
297
                raise errors.RedirectLimit("Redirection limit is reached")
251
298
        return response, connection
252
299
        
253
    def request(self, url, method='GET', body=None, headers=None, stream=False, 
254
            stream_size=16384):  
300
    def request(self, url, method='GET', body=None, headers=None):  
255
301
        headers = headers or {}
256
302
        uri = url_parser(url)
257
303
        self.final_url = url
258
304
        
259
305
        headers.setdefault('User-Agent', restkit.USER_AGENT)
260
306
        if method in ["POST", "PUT"] and body is None:
261
            body = ""
262
            headers.setdefault("Content-Length", str(len(body)))
307
            headers.setdefault("Content-Length", "0")
263
308
            
264
309
        if self.use_proxy and uri.scheme != "https":
265
            proxy_auth = get_proxy_auth()
310
            proxy_auth = pool.get_proxy_auth()
266
311
            if proxy_auth:
267
312
                headers['Proxy-Authorization'] = proxy_auth.strip()
268
313
            
269
314
        response, connection = self._request(uri, method, body, headers)
270
        resp = HTTPResponse(response)
315
        release_callback =  lambda: self._release_connection(uri, connection)
316
        resp = self.response_class(response, release_callback)
271
317
        resp.final_url = self.final_url
272
318
        
273
319
        if method == "HEAD":
274
            response.close()
275
            self._release_connection(uri, connection)
276
            return resp, ""
277
        else:
278
            return resp, _decompress_content(resp, response, 
279
                lambda: self._release_connection(uri, connection), 
280
                stream, stream_size)
281
                
282
def _decompress_content(resp, response, release_callback, stream=False, stream_size=16384):
320
            resp.close()
321
        return resp
322
     
323
def _complain_ifclosed(closed):
324
    if closed:
325
        raise ValueError, "I/O operation on closed response"        
326
            
327
class ResponseStream(object):
328
    
329
    def __init__(self, response):
330
        self.response = response
331
        self.resp = response.resp
332
        self._rbuf = ''
333
        self.stream_size = MAX_CHUNK_SIZE
334
        
335
    def close(self):
336
        if not self.response.closed:
337
            self._buffer = ""
338
            self.response.close()
339
        
340
    def read(self, amt=None):
341
        if self._rbuf and not amt is None:
342
            L = len(self._rbuf)
343
            if amt > L:
344
                amt -= L
345
            else:
346
                s = self._rbuf[:amt]
347
                self._rbuf = self._rbuf[amt:]
348
                return s
349
        data = self.resp.read(amt)
350
        if not data:
351
            self.close()
352
        s = self._rbuf + data
353
        self._rbuf = ''
354
        return s
355
356
    def readline(self, amt=-1):
357
        i = self._rbuf.find('\n')
358
        while i < 0 and not (0 < amt <= len(self._rbuf)):
359
            new = self.resp.read(self.stream_size)
360
            if not new: 
361
                self.close()
362
                break
363
            i = new.find('\n')
364
            if i >= 0: 
365
                i = i + len(self._rbuf)
366
            self._rbuf = self._rbuf + new
367
        if i < 0: 
368
            i = len(self._rbuf)
369
        else: 
370
            i = i+1
371
        if 0 <= amt < len(self._rbuf): 
372
            i = amt
373
        data, self._rbuf = self._rbuf[:i], self._rbuf[i:]
374
        return data
375
376
    def readlines(self, sizehint=0):
377
        total = 0
378
        lines = []
379
        line = self.readline()
380
        while line:
381
            lines.append(line)
382
            total += len(line)
383
            if 0 < sizehint <= total:
384
                break
385
            line = self.readline()
386
        return lines
387
    
388
    def next(self):
389
        r = self.readline()
390
        if not r:
391
            raise StopIteration
392
        return r
393
        
394
    def __iter__(self):
395
        return self
396
397
def _get_content(response):
398
    data = response.resp.read()
399
    response.close()
400
    return data
401
402
def _decompress_content(response, stream=False):
403
    resp = response.resp
283
404
    try:
284
        encoding = resp.get('content-encoding', None)
285
        if encoding in ['gzip', 'deflate']:
286
            
405
        encoding = response.headers.get('content-encoding', None)
406
        if encoding in ('gzip', 'deflate'):
287
407
            if encoding == 'gzip':
288
                compressedstream = StringIO.StringIO(response.read())
289
                release_callback()
290
                data = gzip.GzipFile(fileobj=compressedstream)
291
408
                if stream:
292
                    return ResponseStream(data, stream_size)
409
                    return gzip.GzipFile(fileobj=ResponseStream(response))
293
410
                else:
294
                    return data.read()
411
                    data =  gzip.GzipFile(fileobj=ResponseStream(response)).read()
412
                    response.close()
413
                    return data
295
414
            else:
296
                data =  zlib.decompress(response.read())
297
                release_callback()
298
415
                if stream:
299
                    return ResponseStream(StringIO.StringIO(data), stream_size)
416
                    return ResponseStream(ResponseStream(response))
300
417
                else:
301
                    return data
418
                    return zlib.decompress(_get_content(response))
302
419
        else:
303
420
            if stream:
304
                return ResponseStream(response, stream_size, release_callback)
421
                return ResponseStream(response)
305
422
            else:
306
                data = response.read()
307
                release_callback()
308
                return data
423
                return _get_content(response)
309
424
    except Exception, e:
310
        release_callback()
425
        response.close()
311
426
        raise errors.ResponseError("Decompression failed %s" % str(e))
312
427
        
313
428
        
314
429
def _send_body_part(data, connection):
315
430
    if isinstance(data, types.StringTypes):
316
        data = StringIO.StringIO(to_bytestring(data))
431
        data = StringIO(to_bytestring(data))
317
432
    elif not hasattr(data, 'read'):
318
        data = StringIO.StringIO(str(data))
433
        data = StringIO(str(data))
319
434
    
320
435
    # we always stream
321
436
    while 1:
322
        binarydata = data.read(100000)
437
        binarydata = data.read(16384)
323
438
        if binarydata == '': break
324
439
        connection.send(binarydata)
325
        
326
        
327
class ResponseStream(object):
328
    
329
    def __init__(self, response, amnt=16384, release_callback=None):
330
        self.response = response
331
        self.amnt = amnt
332
        self.callback = release_callback
333
        
334
    def next(self):
335
        return self.response.read(self.amnt)
336
            
337
    def __iter__(self):
338
        while 1:
339
            data = self.next()
340
            if data:
341
                yield data
342
            else:
343
                break
344
        if self.callback is not None:
345
            self.callback()
346
        
347
class HTTPResponse(dict):
348
    """An object more like email.Message than httplib.HTTPResponse.
349
    
350
        >>> from restclient import Resource
351
        >>> res = Resource('http://e-engura.org')
352
        >>> from restclient import Resource
353
        >>> res = Resource('http://e-engura.org')
354
        >>> page = res.get()
355
        >>> res.status
356
        200
357
        >>> res.response['content-type']
358
        'text/html'
359
        >>> logo = res.get('/images/logo.gif')
360
        >>> res.response['content-type']
361
        'image/gif'
362
    """
363
364
    final_url = None
365
    
366
    "Status code returned by server. "
367
    status = 200
368
369
    """Reason phrase returned by server."""
370
    reason = "Ok"
371
372
    def __init__(self, info):
373
        if hasattr(info, "getheaders"):
374
            for key, value in info.getheaders():
375
                self[key.lower()] = value
376
            self.status = info.status
377
            self['status'] = str(self.status)
378
            self.reason = info.reason
379
            self.version = info.version
380
        else:
381
            print info
382
            for key, value in info.iteritems(): 
383
                self[key.lower()] = value 
384
            self.status = int(self.get('status', self.status))
385
            
386
        self.final_url = self.get('final_url', self.final_url)
387
388
    def __getattr__(self, name):
389
        if name == 'dict':
390
            return self 
391
        else:  
392
            raise AttributeError, name
393
394
    def __repr__(self):
395
        return "<%s status %s for %s>" % (self.__class__.__name__,
396
                                          self.status,
397
                                          self.final_url)

Up to file-list restkit/pool.py:

16
16
17
17
18
18
"""
19
Threadsafe Pool class based on eventlet.pools.Pool but using Queue.Queue
19
Threadsafe Pool class 
20
20
21
21
TODO:
22
- add our own way to share socket across connections. We shouldn't need to rely
23
  on eventlet for that
24
22
- log errors
25
23
"""
26
24
27
28
25
import os
26
import time
29
27
import collections
30
28
import httplib
31
import Queue
32
import threading
29
import socket
30
import urlparse
33
31
32
import restkit
34
33
from restkit import errors
35
34
35
has_timeout = hasattr(socket, '_GLOBAL_DEFAULT_TIMEOUT')
36
url_parser = urlparse.urlparse
37
36
38
def get_proxy_auth():
37
39
  import base64
38
40
  proxy_username = os.environ.get('proxy-username')
@@ -48,176 +50,138 @@ def get_proxy_auth():
48
50
  else:
49
51
    return ''
50
52
51
def make_proxy_connection(uri):
52
    headers = headers or {}
53
    proxy = None
54
    if uri.scheme == 'https':
55
        proxy = os.environ.get('https_proxy')
56
    elif uri.scheme == 'http':
57
        proxy = os.environ.get('http_proxy')
53
class PoolInterface(object):
54
    """ abstract class from which all connection 
55
    pool should inherit.
56
    """
58
57
59
    if not proxy:
60
        return make_connection(uri, use_proxy=False)
61
  
62
    if uri.scheme == 'https':
63
        proxy_auth = get_proxy_auth()
64
        if proxy_auth:
65
            proxy_auth = 'Proxy-authorization: %s' % proxy_auth
66
        port = uri.port
67
        if not port:
68
            port = 443
69
        proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % (uri.hostname, port)
70
        user_agent = 'User-Agent: %s\r\n' % restkit.USER_AGENT
71
        proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth, user_agent)
72
        proxy_uri = url_parser(proxy)
73
        if not proxy_uri.port:
74
            proxy_uri.port = '80'
75
        # Connect to the proxy server, very simple recv and error checking
76
        p_sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
77
        p_sock.connect((proxy_uri.host, int(proxy_uri.port)))
78
        p_sock.sendall(proxy_pieces)
79
        response = ''
80
        # Wait for the full response.
81
        while response.find("\r\n\r\n") == -1:
82
            response += p_sock.recv(8192)
83
        p_status = response.split()[1]
84
        if p_status != str(200):
85
            raise ProxyError('Error status=%s' % str(p_status))
86
        # Trivial setup for ssl socket.
87
        ssl = socket.ssl(p_sock, None, None)
88
        fake_sock = httplib.FakeSocket(p_sock, ssl)
89
        # Initalize httplib and replace with the proxy socket.
90
        connection = httplib.HTTPConnection(proxy_uri.host)
91
        connection.sock=fake_sock
92
        return connection
93
    else:
94
        proxy_uri = url_parser(proxy)
95
        if not proxy_uri.port:
96
            proxy_uri.port = '80'
97
        return httplib.HTTPConnection(proxy_uri.hostname, proxy_uri.port)
98
    return None
58
    def get(self):
59
        """ method used to return a connection from the pool"""
60
        raise NotImplementedError
61
        
62
    def put(self):
63
        """ Put an item back into the pool, when done """
64
        raise NotImplementedError
65
        
66
    def clear(self):
67
        """ method used to release all connections """
68
        raise NotImplementedError
99
69
    
100
def make_connection(uri, use_proxy=True):
101
    if use_proxy:
102
        return make_proxy_connection(uri)
103
    
104
    if uri.scheme == 'https':
105
        if not uri.port:
106
            connection = httplib.HTTPSConnection(uri.hostname)
107
        else:
108
            connection = httplib.HTTPSConnection(uri.hostname, uri.port)
109
    else:
110
        if not uri.port:
111
            connection = httplib.HTTPConnection(uri.hostname)
112
        else:
113
            connection = httplib.HTTPConnection(uri.hostname, uri.port)
114
    return connection
115
70
116
class Pool(object):
117
    def __init__(self, min_size=0, max_size=4, order_as_stack=False):
71
72
class ConnectionPool(PoolInterface):
73
    def __init__(self, uri, use_proxy=False, key_file=None, cert_file=None, 
74
            timeout=300, min_size=0, max_size=4):
75
        
76
        self.uri = uri
77
        self.use_proxy = use_proxy
78
        self.key_file = key_file
79
        self.cert_file = cert_file
80
        self.timeout = timeout
118
81
        self.min_size = min_size
119
82
        self.max_size = max_size
120
        self.order_as_stack = order_as_stack
121
        self.current_size = 0
122
        self.channel = Queue.Queue(0)
123
        self.free_items = collections.deque()
83
                    
84
        self.connections = collections.deque()
124
85
        for x in xrange(min_size):
125
86
            self.current_size += 1
126
            self.free_items.append(self.create())
87
            self.connections.append(self.make_connection())
127
88
            
128
        self.lock = threading.Lock()
89
    def _make_proxy_connection(self, proxy):
90
        if self.uri.scheme == 'https':
91
            proxy_auth = get_proxy_auth()
92
            if proxy_auth:
93
                proxy_auth = 'Proxy-authorization: %s' % proxy_auth
94
            port = self.uri.port
95
            if not port:
96
                port = 443
97
            proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % (self.uri.hostname, port)
98
            user_agent = 'User-Agent: %s\r\n' % restkit.USER_AGENT
99
            proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth, user_agent)
100
            proxy_uri = url_parser(proxy)
101
            if not proxy_uri.port:
102
                proxy_uri.port = '80'
103
            # Connect to the proxy server, very simple recv and error checking
104
            p_sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
105
            p_sock.connect((proxy_uri.host, int(proxy_uri.port)))
106
            p_sock.sendall(proxy_pieces)
107
            response = ''
108
            # Wait for the full response.
109
            while response.find("\r\n\r\n") == -1:
110
                response += p_sock.recv(8192)
111
            p_status = response.split()[1]
112
            if p_status != str(200):
113
                raise errors.ProxyError('Error status=%s' % str(p_status))
114
            # Trivial setup for ssl socket.
115
            ssl = socket.ssl(p_sock, None, None)
116
            fake_sock = httplib.FakeSocket(p_sock, ssl)
117
            # Initalize httplib and replace with the proxy socket.
118
            connection = httplib.HTTPConnection(proxy_uri.host)
119
            connection.sock=fake_sock
120
            return connection
121
        else:
122
            proxy_uri = url_parser(proxy)
123
            if not proxy_uri.port:
124
                proxy_uri.port = '80'
125
            return httplib.HTTPConnection(proxy_uri.hostname, proxy_uri.port)
126
        return None
127
128
    def make_connection(self):
129
        if self.use_proxy:
130
            proxy = ''
131
            if self.uri.scheme == 'https':
132
                proxy = os.environ.get('https_proxy')
133
            elif self.uri.scheme == 'http':
134
                proxy = os.environ.get('http_proxy')
135
                
136
            if proxy:
137
                return self._make_proxy_connection(proxy)
138
            
139
        kwargs = {}
140
        if hasattr(httplib.HTTPConnection, 'timeout'):
141
            kwargs['timeout'] = self.timeout
142
        
143
        if self.uri.port:
144
            kwargs['port'] = self.uri.port
145
146
        if self.uri.scheme == "https":
147
            kwargs.update(dict(key_file=self.key_file, cert_file=self.cert_file))
148
            connection = httplib.HTTPSConnection(self.uri.hostname, **kwargs)
149
        else:
150
            connection = httplib.HTTPConnection(self.uri.hostname, **kwargs)
151
            
152
        setattr(connection, "started", time.time())
153
        return connection
129
154
            
130
155
    def do_get(self):
131
156
        """
132
157
        Return an item from the pool, when one is available
133
158
        """ 
134
        self.lock.acquire()
135
        try:
136
            if self.free_items:
137
                return self.free_items.popleft()
138
                
139
            try:
140
                return self.channel.get(False)
141
            except Queue.Empty:
142
                created = self.create()
143
                self.current_size += 1
144
                return created
145
  
146
        finally:
147
            self.lock.release()
159
        if self.connections:
160
            connection = self.connections.popleft()
161
            return connection
162
        else:
163
            return self.make_connection()
148
164
149
165
    def get(self):
150
        connection =  self.do_get()
151
        return connection
166
        while True:
167
            connection = self.do_get()
168
            since = time.time() - connection.started
169
            if since < self.timeout:
170
                if connection._HTTPConnection__response:
171
                    connection._HTTPConnection__response.read()
172
                return connection
173
            else:
174
                connection.close()
152
175
        
153
    def put(self, item):
154
        """Put an item back into the pool, when done
155
        """
156
        self.lock.acquire()
157
        try:
158
            if self.current_size > self.max_size:
159
                self.current_size -= 1
160
                return
176
    def put(self, connection):
177
        if len(self.connections) >= self.max_size:
178
            connection.close()
179
            return
180
        if connection.sock is None:
181
            connection = self.make_connection()
182
        self.connections.append(connection)
161
183
            
162
            if self.waiting():
163
                self.channel.put(item, False)
164
            else:
165
                if self.order_as_stack:
166
                    self.free_items.appendleft(item)
167
                else:
168
                    self.free_items.append(item)
169
        finally:
170
            self.lock.release()
171
            
172
    def resize(self, new_size):
173
        """Resize the pool
174
        """
175
        self.max_size = new_size
176
    
177
    def free(self):
178
        """Return the number of free items in the pool.
179
        """
180
        return len(self.free_items) + self.max_size - self.current_size
181
    
182
    def waiting(self):
183
        """Return the number of routines waiting for a pool item.
184
        """
185
        return (self.channel.qsize() < self.max_size)
186
    
187
    def create(self):
188
        """Generate a new pool item
189
        """
190
        raise NotImplementedError("Implement in subclass")
191
                
192
class ConnectionPool(Pool):
193
    def __init__(self, uri, use_proxy=False, min_size=0, max_size=4):
194
        self.uri = uri
195
        self.use_proxy = use_proxy
196
        Pool.__init__(self, min_size, max_size)
197
    
198
    def create(self):
199
        return make_connection(self.uri, self.use_proxy)
200
201
    def put(self, connection):
202
        if self.current_size > self.max_size:
203
            self.lock.acquire()
204
            self.current_size -= 1
205
            # close the connection if needed
206
            if connection.sock is not None:
207
                connection.close()
208
            self.lock.release()
209
            return
210
        
211
        try:
212
            response = connection.getresponse()
213
            response.read()
214
        except httplib.ResponseNotReady:
215
            pass
216
        except:
184
    def clear(self):
185
        while self.connections:
186
            connection = self.connections.pop()
217
187
            connection.close()
218
            
219
            
220
        if connection.sock is None:
221
            connection = self.create()
222
            
223
        Pool.put(self, connection)

Up to file-list restkit/rest.py:

-
Diff size exceeds threshold (19.6 KB)view raw?

Up to file-list restkit/utils.py:

@@ -45,6 +45,49 @@ Converts an IRI to a URI.
45
45
import re
46
46
import urlparse
47
47
48
from restkit.errors import InvalidUrl
49
import urllib
50
51
52
# code borrowed to Wekzeug with minor changes
53
54
def url_quote(s, charset='utf-8', safe='/:'):
55
    """URL encode a single string with a given encoding."""
56
    if isinstance(s, unicode):
57
        s = s.encode(charset)
58
    elif not isinstance(s, str):
59
        s = str(s)
60
    return urllib.quote(s, safe=safe)
61
62
def url_encode(obj, charset="utf8", encode_keys=False):
63
    if isinstance(obj, dict):
64
        items = []
65
        for k, v in obj.iteritems():
66
            if not isinstance(v, (tuple, list)):
67
                v = [v]
68
            items.append((k, v))
69
    else:
70
        items = obj or ()
71
72
    tmp = []
73
    for key, values in items:
74
        if encode_keys and isinstance(key, unicode):
75
            key = key.encode(charset)
76
        else:
77
            key = str(key)
78
79
        for value in values:
80
            if value is None:
81
                continue
82
            elif isinstance(value, unicode):
83
                value = value.encode(charset)
84
            else:
85
                value = str(value)
86
        tmp.append('%s=%s' % (urllib.quote(key),
87
            urllib.quote_plus(value)))
88
89
    return '&'.join(tmp)
90
48
91
49
92
def to_bytestring(s):
50
93
    if not isinstance(s, basestring):