david / django-roa (http://welldev.org/)
Turn your models into remote resources that you can access through Django's ORM. ROA stands for Resource Oriented Architecture.
| commit 134: | abc332ad5029 |
| parent 133: | 580369e844a4 |
| branch: | default |
8 months ago
Changed (Δ31.7 KB):
django_roa/db/models.py (2 lines added, 1 lines removed)
django_roa/db/query.py (3 lines added, 3 lines removed)
examples/django_roa_client/tests.py (1 lines added, 1 lines removed)
restkit/__init__.py (2 lines added, 2 lines removed)
restkit/bin/rest_cli.py (10 lines added, 14 lines removed)
restkit/errors.py (63 lines added, 4 lines removed)
restkit/ext/eventlet_pool.py (94 lines added, 90 lines removed)
restkit/ext/webob_helper.py (61 lines added, 0 lines removed)
restkit/forms.py (150 lines added, 0 lines removed)
restkit/httpc.py (219 lines added, 177 lines removed)
restkit/pool.py (126 lines added, 162 lines removed)
restkit/rest.py (212 lines added, 678 lines removed)
restkit/utils.py (43 lines added, 0 lines removed)
Up to file-list django_roa/db/models.py:
| … | … | @@ -353,10 +353,11 @@ class ROAModel(models.Model): |
353 |
353 |
except RequestFailed, e: |
354 |
354 |
raise ROAException(e) |
355 |
355 |
|
356 |
response = force_unicode(response.body).encode(settings.DEFAULT_CHARSET) |
|
357 |
||
356 |
358 |
for local_name, remote_name in ROA_MODEL_NAME_MAPPING: |
357 |
359 |
response = response.replace(remote_name, local_name) |
358 |
360 |
|
359 |
response = force_unicode(response).encode(settings.DEFAULT_CHARSET) |
|
360 |
361 |
deserializer = serializers.get_deserializer(ROA_FORMAT) |
361 |
362 |
if hasattr(deserializer, 'deserialize_object'): |
362 |
363 |
result = deserializer(response).deserialize_object(response) |
Up to file-list django_roa/db/query.py:
| … | … | @@ -176,7 +176,7 @@ class RemoteQuerySet(query.QuerySet): |
176 |
176 |
except Exception, e: |
177 |
177 |
raise ROAException(e) |
178 |
178 |
|
179 |
response = force_unicode(response |
|
179 |
response = force_unicode(response.body).encode(settings.DEFAULT_CHARSET) |
|
180 |
180 |
for local_name, remote_name in ROA_MODEL_NAME_MAPPING: |
181 |
181 |
response = response.replace(remote_name, local_name) |
182 |
182 |
|
| … | … | @@ -211,7 +211,7 @@ class RemoteQuerySet(query.QuerySet): |
211 |
211 |
except Exception, e: |
212 |
212 |
raise ROAException(e) |
213 |
213 |
|
214 |
return int(response |
|
214 |
return int(response.body) |
|
215 |
215 |
|
216 |
216 |
def _get_from_id_or_pk(self, id=None, pk=None): |
217 |
217 |
""" |
| … | … | @@ -242,7 +242,7 @@ class RemoteQuerySet(query.QuerySet): |
242 |
242 |
except Exception, e: |
243 |
243 |
raise ROAException(e) |
244 |
244 |
|
245 |
response = force_unicode(response |
|
245 |
response = force_unicode(response.body).encode(settings.DEFAULT_CHARSET) |
|
246 |
246 |
for local_name, remote_name in ROA_MODEL_NAME_MAPPING: |
247 |
247 |
response = response.replace(remote_name, local_name) |
248 |
248 |
Up to file-list examples/django_roa_client/tests.py:
| … | … | @@ -638,7 +638,7 @@ class ROASettingsTests(ROATestCase): |
638 |
638 |
self.assertEqual(repr(page), '<RemotePage: A custom serialized page (1)>') |
639 |
639 |
rc = RestClient() |
640 |
640 |
response = rc.get('http://127.0.0.1:8081/django_roa_server/remotepage/?format=custom') |
641 |
self.assertEqual(repr(response |
|
641 |
self.assertEqual(repr(response.body), '\'<?xml version="1.0" encoding="utf-8"?>\\n<django-test version="1.0">\\n <object pk="1" model="django_roa_server.remotepage">\\n <field type="CharField" name="title">A custom serialized page</field>\\n </object>\\n</django-test>\'') |
|
642 |
642 |
self.assertEqual(len(RemotePage.objects.all()), 1) |
643 |
643 |
page = RemotePage.objects.get(id=page.id) |
644 |
644 |
self.assertEqual(repr(page), '<RemotePage: A custom serialized page (1)>') |
Up to file-list restkit/__init__.py:
| … | … | @@ -24,8 +24,8 @@ USER_AGENT = "restkit/%s" % __version__ |
24 |
24 |
debuglevel = 0 |
25 |
25 |
|
26 |
26 |
from restkit.errors import * |
27 |
from restkit.httpc import HttpClient |
|
28 |
from restkit.rest import * |
|
27 |
from restkit.httpc import HttpClient, HTTPResponse, ResponseStream |
|
28 |
from restkit.rest import Resource, RestClient |
|
29 |
29 |
|
30 |
30 |
|
31 |
31 |
Up to file-list restkit/bin/rest_cli.py:
16 |
16 |
# |
17 |
17 |
import os |
18 |
18 |
import sys |
19 |
from optparse import OptionParser |
|
19 |
from optparse import OptionParser |
|
20 |
20 |
import urlparse |
21 |
21 |
import urllib |
22 |
22 |
|
23 |
# python 2.6 and above compatibility |
|
24 |
try: |
|
25 |
from urlparse import parse_qs as _parse_qs |
|
26 |
except ImportError: |
|
27 |
|
|
23 |
from urlparse import parse_qs as parse_qs |
|
28 |
24 |
|
29 |
25 |
import restkit |
30 |
26 |
from restkit import httpc |
| … | … | @@ -62,7 +58,7 @@ class Url(object): |
62 |
58 |
self.path = '' |
63 |
59 |
|
64 |
60 |
if parts[3]: |
65 |
self.query = |
|
61 |
self.query = parse_qs(parts[3]) |
|
66 |
62 |
else: |
67 |
63 |
self.query = {} |
68 |
64 |
|
| … | … | @@ -79,9 +75,8 @@ def make_query(string, method='GET', fna |
79 |
75 |
return |
80 |
76 |
|
81 |
77 |
if uri.username: |
82 |
transport = ProxiedHttpClient() |
|
83 |
transport.add_authorization(httpc.BasicAuth((uri.username, uri.password))) |
|
84 |
res = restkit.Resource(uri.uri |
|
78 |
res = restkit.Resource(uri.uri) |
|
79 |
res.add_authorization(httpc.BasicAuth((uri.username, uri.password))) |
|
85 |
80 |
else: |
86 |
81 |
res = restkit.Resource(uri.uri) |
87 |
82 |
|
| … | … | @@ -103,15 +98,16 @@ def make_query(string, method='GET', fna |
103 |
98 |
headers['Content-Length'] = os.path.getsize(fname) |
104 |
99 |
payload = open(fname, 'r') |
105 |
100 |
|
106 |
|
|
101 |
result = res.request(method, path=uri.path, payload=payload, headers=headers, **uri.query) |
|
107 |
102 |
|
108 |
103 |
output = output or '' |
109 |
104 |
if not output or output == '-': |
110 |
return |
|
105 |
return result.body |
|
111 |
106 |
else: |
112 |
107 |
try: |
113 |
108 |
f = open(output, 'wb') |
114 |
f |
|
109 |
for block in result.body_file: |
|
110 |
f.write(block) |
|
115 |
111 |
f.close() |
116 |
112 |
except: |
117 |
113 |
print >>sys.stderr, "Can't save result in %s" % output |
| … | … | @@ -126,7 +122,7 @@ def main(): |
126 |
122 |
parser.add_option('-i', '--input', action='store', dest='input', metavar='FILE', |
127 |
123 |
help='the name of the file to read from') |
128 |
124 |
parser.add_option('-o', '--output', action='store', dest='output', |
129 |
help='the name of the file to |
|
125 |
help='the name of the file to write to') |
|
130 |
126 |
|
131 |
127 |
parser.add_option('--proxy', action='store', dest='proxy', |
132 |
128 |
help='Full uri of proxy, ex:\n'+ |
Up to file-list restkit/errors.py:
18 |
18 |
""" |
19 |
19 |
exception classes. |
20 |
20 |
""" |
21 |
import restkit |
|
22 |
import warnings |
|
21 |
23 |
|
24 |
class deprecated_property(object): |
|
25 |
""" |
|
26 |
Wraps a decorator, with a deprecation warning or error |
|
27 |
""" |
|
28 |
def __init__(self, decorator, attr, message, warning=True): |
|
29 |
self.decorator = decorator |
|
30 |
self.attr = attr |
|
31 |
self.message = message |
|
32 |
self.warning = warning |
|
33 |
||
34 |
def __get__(self, obj, type=None): |
|
35 |
if obj is None: |
|
36 |
return self |
|
37 |
self.warn() |
|
38 |
return self.decorator.__get__(obj, type) |
|
39 |
||
40 |
def __set__(self, obj, value): |
|
41 |
self.warn() |
|
42 |
self.decorator.__set__(obj, value) |
|
43 |
||
44 |
def __delete__(self, obj): |
|
45 |
self.warn() |
|
46 |
self.decorator.__delete__(obj) |
|
47 |
||
48 |
def __repr__(self): |
|
49 |
return '<Deprecated attribute %s: %r>' % ( |
|
50 |
self.attr, |
|
51 |
self.decorator) |
|
52 |
||
53 |
def warn(self): |
|
54 |
if not self.warning: |
|
55 |
raise DeprecationWarning( |
|
56 |
'The attribute %s is deprecated: %s' % (self.attr, self.message)) |
|
57 |
else: |
|
58 |
warnings.warn( |
|
59 |
'The attribute %s is deprecated: %s' % (self.attr, self.message), |
|
60 |
DeprecationWarning, |
|
61 |
stacklevel=3) |
|
62 |
||
63 |
||
22 |
64 |
class ResourceError(Exception): |
23 |
||
65 |
""" default error class """ |
|
24 |
66 |
def __init__(self, msg=None, http_code=None, response=None): |
25 |
67 |
self.msg = msg or '' |
26 |
self.status |
|
68 |
self.status = http_code |
|
27 |
69 |
self.response = response |
28 |
70 |
Exception.__init__(self) |
29 |
71 |
|
72 |
def _status_int__get(self): |
|
73 |
""" |
|
74 |
The status as an integer |
|
75 |
""" |
|
76 |
return self.status |
|
77 |
def _status_int__set(self, http_code): |
|
78 |
self.status = http_code |
|
79 |
status_int = property(_status_int__get, _status_int__set, doc=_status_int__get.__doc__) |
|
80 |
||
81 |
status_code = deprecated_property( |
|
82 |
status_int, 'status_code', 'use .status_int instead', |
|
83 |
warning=False) |
|
84 |
||
30 |
85 |
def _get_message(self): |
31 |
86 |
return self.msg |
32 |
87 |
def _set_message(self, msg): |
| … | … | @@ -37,11 +92,12 @@ class ResourceError(Exception): |
37 |
92 |
if self.msg: |
38 |
93 |
return self.msg |
39 |
94 |
try: |
40 |
return s |
|
95 |
return str(self.__dict__) |
|
41 |
96 |
except (NameError, ValueError, KeyError), e: |
42 |
97 |
return 'Unprintable exception %s: %s' \ |
43 |
98 |
% (self.__class__.__name__, str(e)) |
44 |
||
99 |
||
100 |
||
45 |
101 |
class ResourceNotFound(ResourceError): |
46 |
102 |
"""Exception raised when no resource was found at the given url. |
47 |
103 |
""" |
| … | … | @@ -83,3 +139,6 @@ class TransportError(Exception): |
83 |
139 |
class ResponseError(Exception): |
84 |
140 |
""" Error raised while getting response or decompressing response stream""" |
85 |
141 |
|
142 |
||
143 |
class ProxyError(Exception): |
|
144 |
""" raised when proxy error happend""" |
Up to file-list restkit/ext/eventlet_pool.py:
15 |
15 |
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
16 |
16 |
|
17 |
17 |
import os |
18 |
import time |
|
19 |
import urlparse |
|
18 |
20 |
|
19 |
21 |
from eventlet.green import socket |
20 |
from eventlet.green import httplib |
|
22 |
from eventlet.green import httplib as ehttplib |
|
21 |
23 |
from eventlet.pools import Pool |
22 |
24 |
from eventlet.util import wrap_socket_with_coroutine_socket |
23 |
25 |
|
26 |
import restkit |
|
27 |
from restkit import errors |
|
28 |
from restkit.pool import get_proxy_auth, PoolInterface |
|
29 |
||
30 |
url_parser = urlparse.urlparse |
|
31 |
||
32 |
||
24 |
33 |
wrap_socket_with_coroutine_socket() |
25 |
34 |
|
26 |
def make_proxy_connection(uri): |
|
27 |
headers = headers or {} |
|
28 |
proxy = None |
|
29 |
if uri.scheme == 'https': |
|
30 |
proxy = os.environ.get('https_proxy') |
|
31 |
elif uri.scheme == 'http': |
|
32 |
proxy = os.environ.get('http_proxy') |
|
35 |
eventlet_httplib = False |
|
36 |
def wrap_eventlet_ehttplib(): |
|
37 |
if eventlet_httplib: return |
|
38 |
import httplib |
|
39 |
ehttplib.BadStatusLine = httplib.BadStatusLine |
|
40 |
||
41 |
wrap_eventlet_ehttplib() |
|
33 |
42 |
|
34 |
if not proxy: |
|
35 |
return make_connection(uri, use_proxy=False) |
|
36 |
||
37 |
if uri.scheme == 'https': |
|
38 |
proxy_auth = _get_proxy_auth() |
|
39 |
if proxy_auth: |
|
40 |
proxy_auth = 'Proxy-authorization: %s' % proxy_auth |
|
41 |
port = uri.port |
|
42 |
if not port: |
|
43 |
port = 443 |
|
44 |
proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % (uri.hostname, port) |
|
45 |
user_agent = 'User-Agent: %s\r\n' % restkit.USER_AGENT |
|
46 |
proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth, user_agent) |
|
47 |
proxy_uri = url_parser(proxy) |
|
48 |
if not proxy_uri.port: |
|
49 |
proxy_uri.port = '80' |
|
50 |
# Connect to the proxy server, very simple recv and error checking |
|
51 |
p_sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM) |
|
52 |
p_sock.connect((proxy_uri.host, int(proxy_uri.port))) |
|
53 |
p_sock.sendall(proxy_pieces) |
|
54 |
response = '' |
|
55 |
# Wait for the full response. |
|
56 |
while response.find("\r\n\r\n") == -1: |
|
57 |
response += p_sock.recv(8192) |
|
58 |
p_status = response.split()[1] |
|
59 |
if p_status != str(200): |
|
60 |
raise ProxyError('Error status=%s' % str(p_status)) |
|
61 |
# Trivial setup for ssl socket. |
|
62 |
ssl = socket.ssl(p_sock, None, None) |
|
63 |
fake_sock = httplib.FakeSocket(p_sock, ssl) |
|
64 |
# Initalize httplib and replace with the proxy socket. |
|
65 |
connection = httplib.HTTPConnection(proxy_uri.host) |
|
66 |
connection.sock=fake_sock |
|
67 |
return connection |
|
68 |
else: |
|
69 |
proxy_uri = url_parser(proxy) |
|
70 |
if not proxy_uri.port: |
|
71 |
proxy_uri.port = '80' |
|
72 |
return httplib.HTTPConnection(proxy_uri.hostname, proxy_uri.port) |
|
73 |
return None |
|
74 |
||
75 |
def make_connection(uri, use_proxy=True): |
|
76 |
if use_proxy: |
|
77 |
return make_proxy_connection(uri) |
|
78 |
||
79 |
if uri.scheme == 'https': |
|
80 |
if not uri.port: |
|
81 |
connection = httplib.HTTPSConnection(uri.hostname) |
|
82 |
else: |
|
83 |
connection = httplib.HTTPSConnection(uri.hostname, uri.port) |
|
84 |
else: |
|
85 |
if not uri.port: |
|
86 |
connection = httplib.HTTPConnection(uri.hostname) |
|
87 |
else: |
|
88 |
connection = httplib.HTTPConnection(uri.hostname, uri.port) |
|
89 |
return connection |
|
90 |
||
91 |
||
92 |
class ConnectionPool(Pool): |
|
93 |
def __init__(self, uri, use_proxy=False, min_size=0, max_size=4): |
|
43 |
class ConnectionPool(Pool, PoolInterface): |
|
44 |
def __init__(self, uri, use_proxy=False, key_file=None, |
|
45 |
cert_file=None, min_size=0, max_size=4, **kwargs): |
|
46 |
Pool.__init__(self, min_size, max_size) |
|
94 |
47 |
self.uri = uri |
95 |
48 |
self.use_proxy = use_proxy |
96 |
|
|
49 |
self.key_file = key_file |
|
50 |
self.cert_file = cert_file |
|
51 |
||
52 |
||
53 |
def _make_proxy_connection(self, proxy): |
|
54 |
if self.uri.scheme == 'https': |
|
55 |
proxy_auth = get_proxy_auth() |
|
56 |
if proxy_auth: |
|
57 |
proxy_auth = 'Proxy-authorization: %s' % proxy_auth |
|
58 |
port = self.uri.port |
|
59 |
if not port: |
|
60 |
port = 443 |
|
61 |
proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % (self.uri.hostname, port) |
|
62 |
user_agent = 'User-Agent: %s\r\n' % restkit.USER_AGENT |
|
63 |
proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth, user_agent) |
|
64 |
proxy_uri = url_parser(proxy) |
|
65 |
if not proxy_uri.port: |
|
66 |
proxy_uri.port = '80' |
|
67 |
# Connect to the proxy server, very simple recv and error checking |
|
68 |
p_sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM) |
|
69 |
p_sock.connect((proxy_uri.host, int(proxy_uri.port))) |
|
70 |
p_sock.sendall(proxy_pieces) |
|
71 |
response = '' |
|
72 |
# Wait for the full response. |
|
73 |
while response.find("\r\n\r\n") == -1: |
|
74 |
response += p_sock.recv(8192) |
|
75 |
p_status = response.split()[1] |
|
76 |
if p_status != str(200): |
|
77 |
raise errors.ProxyError('Error status=%s' % str(p_status)) |
|
78 |
# Trivial setup for ssl socket. |
|
79 |
ssl = socket.ssl(p_sock, None, None) |
|
80 |
fake_sock = ehttplib.FakeSocket(p_sock, ssl) |
|
81 |
# Initalize ehttplib and replace with the proxy socket. |
|
82 |
connection = ehttplib.HTTPConnection(proxy_uri.host) |
|
83 |
connection.sock=fake_sock |
|
84 |
return connection |
|
85 |
else: |
|
86 |
proxy_uri = url_parser(proxy) |
|
87 |
if not proxy_uri.port: |
|
88 |
proxy_uri.port = '80' |
|
89 |
return ehttplib.HTTPConnection(proxy_uri.hostname, proxy_uri.port) |
|
90 |
return None |
|
91 |
||
92 |
def make_connection(self): |
|
93 |
if self.use_proxy: |
|
94 |
proxy = '' |
|
95 |
if self.uri.scheme == 'https': |
|
96 |
proxy = os.environ.get('https_proxy') |
|
97 |
elif self.uri.scheme == 'http': |
|
98 |
proxy = os.environ.get('http_proxy') |
|
99 |
||
100 |
if proxy: |
|
101 |
return self._make_proxy_connection(proxy) |
|
102 |
||
103 |
kwargs = {} |
|
104 |
if hasattr(ehttplib.HTTPConnection, 'timeout'): |
|
105 |
kwargs['timeout'] = self.timeout |
|
106 |
||
107 |
if self.uri.port: |
|
108 |
kwargs['port'] = self.uri.port |
|
109 |
||
110 |
if self.uri.scheme == "https": |
|
111 |
kwargs.update(dict(key_file=self.key_file, cert_file=self.cert_file)) |
|
112 |
connection = ehttplib.HTTPSConnection(self.uri.hostname, **kwargs) |
|
113 |
else: |
|
114 |
connection = ehttplib.HTTPConnection(self.uri.hostname, **kwargs) |
|
115 |
||
116 |
setattr(connection, "started", time.time()) |
|
117 |
return connection |
|
97 |
118 |
|
98 |
119 |
def create(self): |
99 |
return make_connection(self.uri, self.use_proxy) |
|
100 |
||
101 |
def put(self, connection): |
|
102 |
if self.current_size > self.max_size: |
|
103 |
self.current_size -= 1 |
|
104 |
# close the connection if needed |
|
105 |
if connection.sock is not None: |
|
106 |
connection.close() |
|
107 |
return |
|
108 |
||
109 |
try: |
|
110 |
response = connection.getresponse() |
|
111 |
response.read() |
|
112 |
except httplib.ResponseNotReady: |
|
113 |
pass |
|
114 |
except: |
|
115 |
connection.close() |
|
116 |
||
117 |
if connection.sock is None: |
|
118 |
connection = self.create() |
|
119 |
|
|
120 |
return self.make_connection() |
|
121 |
||
122 |
def clear(self): |
|
123 |
self.free() |
Up to file-list restkit/ext/webob_helper.py:
1 |
# -*- coding: utf-8 - |
|
2 |
# |
|
3 |
# Copyright (c) 2008, 2009 Benoit Chesneau <benoitc@e-engura.com> |
|
4 |
# |
|
5 |
# Permission to use, copy, modify, and distribute this software for any |
|
6 |
# purpose with or without fee is hereby granted, provided that the above |
|
7 |
# copyright notice and this permission notice appear in all copies. |
|
8 |
# |
|
9 |
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
|
10 |
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
|
11 |
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR |
|
12 |
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
|
13 |
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN |
|
14 |
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF |
|
15 |
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
|
16 |
||
17 |
import restkit.errors |
|
18 |
import webob.exc |
|
19 |
||
20 |
class WebobResourceError(webob.exc.WSGIHTTPException): |
|
21 |
||
22 |
def __init__(self, msg=None, http_code=None, response=None): |
|
23 |
webob.exc.WSGIHTTPException.__init__(self) |
|
24 |
||
25 |
http_code = http_code or 500 |
|
26 |
klass = webob.exc.status_map[http_code] |
|
27 |
self.code = http_code |
|
28 |
self.title = klass.title |
|
29 |
self.status = '%s %s' % (self.code, self.title) |
|
30 |
self.explanation = msg |
|
31 |
self.response = response |
|
32 |
# default params |
|
33 |
self.msg = msg |
|
34 |
||
35 |
def _status_int__get(self): |
|
36 |
""" |
|
37 |
The status as an integer |
|
38 |
""" |
|
39 |
return int(self.status.split()[0]) |
|
40 |
def _status_int__set(self, value): |
|
41 |
self.status = value |
|
42 |
status_int = property(_status_int__get, _status_int__set, doc=_status_int__get.__doc__) |
|
43 |
||
44 |
status_code = restkit.errors.deprecated_property( |
|
45 |
status_int, 'status_code', 'use .status_int instead', |
|
46 |
warning=False) |
|
47 |
||
48 |
def _get_message(self): |
|
49 |
return self.explanation |
|
50 |
def _set_message(self, msg): |
|
51 |
self.explanation = msg or '' |
|
52 |
message = property(_get_message, _set_message) |
|
53 |
||
54 |
webob_exceptions = False |
|
55 |
def wrap_exceptions(): |
|
56 |
""" wrap restkit exception to return WebBob exceptions""" |
|
57 |
global webob_exceptions |
|
58 |
if webob_exceptions: return |
|
59 |
restkit.errors.ResourceError = WebobResourceError |
|
60 |
webob_exceptions = True |
|
61 |
Up to file-list restkit/forms.py:
1 |
# -*- coding: utf-8 - |
|
2 |
# |
|
3 |
# Copyright (c) 2008, 2009 Benoit Chesneau <benoitc@e-engura.com> |
|
4 |
# |
|
5 |
# Permission to use, copy, modify, and distribute this software for any |
|
6 |
# purpose with or without fee is hereby granted, provided that the above |
|
7 |
# copyright notice and this permission notice appear in all copies. |
|
8 |
# |
|
9 |
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
|
10 |
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
|
11 |
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR |
|
12 |
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
|
13 |
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN |
|
14 |
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF |
|
15 |
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
|
16 |
# |
|
17 |
||
18 |
import mimetypes |
|
19 |
import os |
|
20 |
import re |
|
21 |
import urllib |
|
22 |
||
23 |
||
24 |
from restkit.utils import to_bytestring, url_quote |
|
25 |
||
26 |
MIME_BOUNDARY = 'END_OF_PART' |
|
27 |
||
28 |
def form_encode(obj, charser="utf8"): |
|
29 |
tmp = [] |
|
30 |
for key, value in obj.items(): |
|
31 |
tmp.append("%s=%s" % (url_quote(key), |
|
32 |
url_quote(value))) |
|
33 |
return to_bytestring("&".join(tmp)) |
|
34 |
||
35 |
||
36 |
class BoundaryItem(object): |
|
37 |
def __init__(self, name, value, fname=None, filetype=None, filesize=None): |
|
38 |
self.name = url_quote(name) |
|
39 |
if value is not None and not hasattr(value, 'read'): |
|
40 |
value = url_quote(value) |
|
41 |
self.size = len(value) |
|
42 |
self.value = value |
|
43 |
if fname is not None: |
|
44 |
if isinstance(fname, unicode): |
|
45 |
fname = fname.encode("utf-8").encode("string_escape").replace('"', '\\"') |
|
46 |
else: |
|
47 |
fname = fname.encode("string_escape").replace('"', '\\"') |
|
48 |
self.fname = fname |
|
49 |
if filetype is not None: |
|
50 |
filetype = to_bytestring(filetype) |
|
51 |
self.filetype = filetype |
|
52 |
||
53 |
if isinstance(value, file) and filesize is None: |
|
54 |
try: |
|
55 |
value.flush() |
|
56 |
except IOError: |
|
57 |
pass |
|
58 |
self.size = int(os.fstat(value.fileno())[6]) |
|
59 |
||
60 |
def encode_hdr(self, boundary): |
|
61 |
"""Returns the header of the encoding of this parameter""" |
|
62 |
boundary = url_quote(boundary) |
|
63 |
headers = ["--%s" % boundary] |
|
64 |
if self.fname: |
|
65 |
disposition = 'form-data; name="%s"; filename="%s"' % (self.name, |
|
66 |
self.fname) |
|
67 |
else: |
|
68 |
disposition = 'form-data; name="%s"' % self.name |
|
69 |
headers.append("Content-Disposition: %s" % disposition) |
|
70 |
if self.filetype: |
|
71 |
filetype = self.filetype |
|
72 |
else: |
|
73 |
filetype = "text/plain; charset=utf-8" |
|
74 |
headers.append("Content-Type: %s" % filetype) |
|
75 |
headers.append("Content-Length: %i" % self.size) |
|
76 |
headers.append("") |
|
77 |
headers.append("") |
|
78 |
return "\r\n".join(headers) |
|
79 |
||
80 |
def encode(self, boundary): |
|
81 |
"""Returns the string encoding of this parameter""" |
|
82 |
value = self.value |
|
83 |
if re.search("^--%s$" % re.escape(boundary), value, re.M): |
|
84 |
raise ValueError("boundary found in encoded string") |
|
85 |
||
86 |
return "%s%s\r\n" % (self.encode_hdr(boundary), value) |
|
87 |
||
88 |
def iter_encode(self, boundary, blocksize=16384): |
|
89 |
if not hasattr(self.value, "read"): |
|
90 |
yield self.encode(boundary) |
|
91 |
else: |
|
92 |
yield self.encode_hdr(boundary) |
|
93 |
while True: |
|
94 |
block = self.value.read(blocksize) |
|
95 |
if not block: |
|
96 |
yield "\r\n" |
|
97 |
break |
|
98 |
yield block |
|
99 |
||
100 |
||
101 |
class MultipartForm(object): |
|
102 |
||
103 |
def __init__(self, params, boundary, headers): |
|
104 |
self.boundary = boundary |
|
105 |
self.boundaries = [] |
|
106 |
self.size = 0 |
|
107 |
||
108 |
self.content_length = headers.get('Content-Length') |
|
109 |
||
110 |
if hasattr(params, 'items'): |
|
111 |
params = params.items() |
|
112 |
||
113 |
for param in params: |
|
114 |
name, value = param |
|
115 |
if hasattr(value, "read"): |
|
116 |
fname = getattr(value, 'name') |
|
117 |
if fname is not None: |
|
118 |
filetype = ';'.join(filter(None, mimetypes.guess_type(fname))) |
|
119 |
else: |
|
120 |
filetype = None |
|
121 |
if not isinstance(value, file) and self.content_length is None: |
|
122 |
value = value.read() |
|
123 |
||
124 |
boundary = BoundaryItem(name, value, fname, filetype) |
|
125 |
else: |
|
126 |
boundary = BoundaryItem(name, value) |
|
127 |
self.boundaries.append(boundary) |
|
128 |
||
129 |
def get_size(self): |
|
130 |
if self.content_length is not None: |
|
131 |
return int(self.content_length) |
|
132 |
size = 0 |
|
133 |
for boundary in self.boundaries: |
|
134 |
size = size + boundary.size |
|
135 |
return size |
|
136 |
||
137 |
def __iter__(self): |
|
138 |
for boundary in self.boundaries: |
|
139 |
for block in boundary.iter_encode(self.boundary): |
|
140 |
yield block |
|
141 |
yield "--%s--\r\n" % self.boundary |
|
142 |
||
143 |
||
144 |
def multipart_form_encode(params, headers, boundary): |
|
145 |
headers = headers or {} |
|
146 |
boundary = urllib.quote_plus(boundary) |
|
147 |
body = MultipartForm(params, boundary, headers) |
|
148 |
headers['Content-Type'] = "multipart/form-data; boundary=%s" % boundary |
|
149 |
headers['Content-Length'] = str(body.get_size()) |
|
150 |
return body, headers |
Up to file-list restkit/httpc.py:
29 |
29 |
# limitations under the License. |
30 |
30 |
|
31 |
31 |
import base64 |
32 |
import |
|
32 |
import gzip |
|
33 |
33 |
import httplib |
34 |
import os |
|
35 |
34 |
import re |
36 |
35 |
import socket |
37 |
|
|
36 |
from StringIO import StringIO |
|
38 |
37 |
import types |
39 |
import urllib |
|
40 |
38 |
import urlparse |
39 |
import zlib |
|
41 |
40 |
|
42 |
41 |
import restkit |
43 |
42 |
from restkit import errors |
44 |
from restkit.pool import ConnectionPool, get_proxy_auth |
|
43 |
||
44 |
from restkit import pool |
|
45 |
45 |
from restkit.utils import to_bytestring |
46 |
46 |
|
47 |
47 |
|
48 |
MAX_CHUNK_SIZE = 16384 |
|
48 |
49 |
url_parser = urlparse.urlparse |
49 |
50 |
|
50 |
51 |
NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') |
| … | … | @@ -63,9 +64,7 @@ def _relative_uri(uri): |
63 |
64 |
|
64 |
65 |
class Auth(object): |
65 |
66 |
""" Interface for Auth classes """ |
66 |
def __init__(self, credentials, **kwargs): |
|
67 |
self.credentials = credentials |
|
68 |
||
67 |
||
69 |
68 |
def depth(self, uri): |
70 |
69 |
return uri.path.count("/") |
71 |
70 |
|
| … | … | @@ -75,73 +74,127 @@ class Auth(object): |
75 |
74 |
return True |
76 |
75 |
|
77 |
76 |
def request(self, uri, method, body, headers): |
77 |
""" path auth info to the request """ |
|
78 |
78 |
pass |
79 |
79 |
|
80 |
def response(self, response, content): |
|
81 |
""" allow us to store new auth info from the response.""" |
|
80 |
def response(self, response): |
|
81 |
""" allow us to store new auth info from the response. |
|
82 |
if something is wrong, should return True to redo |
|
83 |
the request. Else return False. |
|
84 |
""" |
|
82 |
85 |
return False |
83 |
||
84 |
def add_credentials(self, *args, **kwargs): |
|
85 |
raise NotImplementedError |
|
86 |
86 |
|
87 |
87 |
class BasicAuth(Auth): |
88 |
88 |
""" basic authentification """ |
89 |
||
90 |
def __init__(self, username, password): |
|
91 |
self.credentials = (username, password) |
|
92 |
||
89 |
93 |
def request(self, uri, method, body, headers): |
90 |
headers['authorization'] = 'Basic ' + base64. |
|
94 |
headers['authorization'] = 'Basic ' + base64.encodestring("%s:%s" % self.credentials)[:-1] |
|
95 |
||
96 |
class HTTPResponse(object): |
|
97 |
""" Object containing response.""" |
|
98 |
||
99 |
charset = "utf8" |
|
100 |
unicode_errors = 'strict' |
|
101 |
||
102 |
def __init__(self, response, release_callback): |
|
103 |
self.resp = response |
|
104 |
self.release_callback = release_callback |
|
105 |
self.headerslist = response.getheaders() |
|
106 |
self.status = "%s %s" % (response.status, response.reason) |
|
107 |
self.status_int = response.status |
|
108 |
self.version = response.version |
|
91 |
109 |
|
92 |
def add_credentials(self, username, password=None): |
|
93 |
password = password or "" |
|
94 |
|
|
110 |
headers = {} |
|
111 |
for key, value in self.headerslist: |
|
112 |
headers[key.lower()] = value |
|
113 |
self.headers = headers |
|
114 |
self.closed = False |
|
115 |
self._body = "" |
|
116 |
||
117 |
def get_body(self, stream=False): |
|
118 |
if self._body: |
|
119 |
return self._body |
|
120 |
body = _decompress_content(self, stream=stream) |
|
121 |
if not stream: |
|
122 |
self._body = body |
|
123 |
return body |
|
124 |
||
125 |
@property |
|
126 |
def unicode_body(self): |
|
127 |
if not self.charset: |
|
128 |
raise AttributeError( |
|
129 |
"You cannot access HTTPResponse.unicode_body unless charset is set") |
|
130 |
body = self.get_body() |
|
131 |
return body.decode(self.charset, self.unicode_errors) |
|
132 |
||
133 |
@property |
|
134 |
def body(self): |
|
135 |
""" get body in one bytestring """ |
|
136 |
return self.get_body() |
|
137 |
||
138 |
@property |
|
139 |
def body_file(self): |
|
140 |
""" get body as a file object """ |
|
141 |
return self.get_body(stream=True) |
|
142 |
||
143 |
def close(self): |
|
144 |
""" close the response""" |
|
145 |
if not self.closed: |
|
146 |
self.closed = True |
|
147 |
self.release_callback() |
|
148 |
if not self.resp.isclosed(): |
|
149 |
self.resp.close() |
|
150 |
||
95 |
151 |
|
96 |
152 |
#TODO : manage authentification detection |
97 |
153 |
class HttpClient(object): |
98 |
MAX_REDIRECTIONS = 5 |
|
99 |
||
100 |
def __init__(self, follow_redirect=True, force_follow_redirect=False, |
|
101 |
use_proxy=False, min_size=0, max_size=4, pool_class=None): |
|
154 |
max_redirections = 5 |
|
155 |
pool_class = pool.ConnectionPool |
|
156 |
response_class = HTTPResponse |
|
157 |
||
158 |
def __init__(self, follow_redirect=True, force_follow_redirect=False, |
|
159 |
response_class=None, pool_class=None, **conn_opts): |
|
102 |
160 |
self.authorizations = [] |
103 |
self.use_proxy = |
|
161 |
self.use_proxy = conn_opts.get("use_proxy", False) |
|
104 |
162 |
self.follow_redirect = follow_redirect |
105 |
163 |
self.force_follow_redirect = force_follow_redirect |
106 |
self.min_size = min_size |
|
107 |
self.max_size = max_size |
|
108 |
self.connections = {} |
|
109 |
if pool_class is None: |
|
110 |
self.pool_class = ConnectionPool |
|
111 |
else: |
|
164 |
self.conn_opts = conn_opts |
|
165 |
self._http_pool = {} |
|
166 |
if response_class is not None: |
|
167 |
self.response_class = response_class |
|
168 |
if pool_class is not None: |
|
112 |
169 |
self.pool_class = pool_class |
113 |
||
170 |
||
114 |
171 |
def add_authorization(self, obj_auth): |
115 |
172 |
self.authorizations.append(obj_auth) |
116 |
173 |
|
117 |
def _get_connection(self, uri, headers=None): |
|
118 |
connection = None |
|
174 |
def _get_pool(self, uri): |
|
119 |
175 |
conn_key = (uri.scheme, uri.netloc, self.use_proxy) |
176 |
if conn_key in self._http_pool: |
|
177 |
pool = self._http_pool[conn_key] |
|
178 |
else: |
|
179 |
pool = self.pool_class(uri, **self.conn_opts) |
|
180 |
self._http_pool[conn_key] = pool |
|
181 |
return pool |
|
120 |
182 |
|
121 |
if conn_key in self.connections: |
|
122 |
pool = self.connections[conn_key] |
|
123 |
else: |
|
124 |
pool = self.connections[conn_key] = self.pool_class(uri, |
|
125 |
use_proxy=self.use_proxy, min_size=self.min_size, |
|
126 |
max_size=self.max_size) |
|
127 |
connection = pool.get() |
|
128 |
return connection |
|
183 |
def _get_connection(self, uri): |
|
184 |
pool = self._get_pool(uri) |
|
185 |
return pool.get() |
|
129 |
186 |
|
130 |
187 |
def _release_connection(self, uri, connection): |
131 |
conn_key = (uri.scheme, uri.netloc, self.use_proxy) |
|
132 |
||
133 |
if conn_key in self.connections: |
|
134 |
pool = self.connections[conn_key] |
|
135 |
else: |
|
136 |
pool = self.connections[conn_key] =self.pool_class(uri, |
|
137 |
use_proxy=self.use_proxy, min_size=self.min_size, |
|
138 |
max_size=self.max_size) |
|
188 |
pool = self._get_pool(uri) |
|
139 |
189 |
pool.put(connection) |
140 |
||
141 |
||
190 |
||
191 |
def _clean_pool(self, uri): |
|
192 |
pool = self._get_pool(uri) |
|
193 |
pool.clear() |
|
194 |
||
142 |
195 |
def _make_request(self, uri, method, body, headers): |
143 |
196 |
for i in range(2): |
144 |
connection = self._get_connection(uri |
|
197 |
connection = self._get_connection(uri) |
|
145 |
198 |
connection.debuglevel = restkit.debuglevel |
146 |
199 |
try: |
147 |
200 |
if connection.host != uri.hostname: |
| … | … | @@ -173,9 +226,7 @@ class HttpClient(object): |
173 |
226 |
if i > 0 and hasattr(body, 'seek'): |
174 |
227 |
body.seek(0) |
175 |
228 |
|
176 |
if isinstance(body, types.StringTypes) and len(body) == 0: |
|
177 |
connection.send("") |
|
178 |
|
|
229 |
if isinstance(body, types.StringTypes) or hasattr(body, 'read'): |
|
179 |
230 |
_send_body_part(body, connection) |
180 |
231 |
elif hasattr(body, "__iter__"): |
181 |
232 |
for body_part in body: |
| … | … | @@ -185,22 +236,21 @@ class HttpClient(object): |
185 |
236 |
_send_body_part(body_part, connection) |
186 |
237 |
else: |
187 |
238 |
_send_body_part(body, connection) |
188 |
||
189 |
except socket.gaierror: |
|
190 |
connection.close() |
|
191 |
self._release_connection(uri, connection) |
|
239 |
response = connection.getresponse() |
|
240 |
except socket.gaierror, e: |
|
241 |
self._clean_pool(uri) |
|
192 |
242 |
raise errors.ResourceNotFound("Unable to find the server at %s" % connection.host, 404) |
193 |
except (socket.error, httplib.HTTPException): |
|
194 |
connection.close() |
|
195 |
|
|
243 |
except (socket.error, httplib.BadStatusLine), e: |
|
244 |
# we should do better error parsing here |
|
245 |
self._clean_pool(uri) |
|
196 |
246 |
if i == 0: |
197 |
247 |
continue |
198 |
248 |
else: |
199 |
raise |
|
249 |
raise errors.RequestFailed("socket error %s" % str(e), 500) |
|
200 |
250 |
break |
201 |
251 |
|
202 |
252 |
# Return the HTTP Response from the server. |
203 |
return |
|
253 |
return response, connection |
|
204 |
254 |
|
205 |
255 |
def _request(self, uri, method, body, headers, nb_redirections=0): |
206 |
256 |
auths = [(auth.depth(uri), auth) for auth in self.authorizations if auth.inscope(uri.hostname, uri)] |
| … | … | @@ -209,18 +259,15 @@ class HttpClient(object): |
209 |
259 |
auth.request(uri, method, body, headers) |
210 |
260 |
|
211 |
261 |
headers = _normalize_headers(headers) |
212 |
old_response = None |
|
213 |
262 |
|
214 |
connection = self._make_request(uri, method, body, headers) |
|
215 |
response = connection.getresponse() |
|
263 |
response, connection = self._make_request(uri, method, body, headers) |
|
216 |
264 |
|
217 |
if auth and auth.response(response |
|
265 |
if auth and auth.response(response): |
|
218 |
266 |
auth.request(uri, method, headers, body) |
219 |
connection = self._make_request(uri, method, body, headers) |
|
220 |
response = connection.getresponse() |
|
267 |
response, connection = self._make_request(uri, method, body, headers) |
|
221 |
268 |
|
222 |
269 |
if self.follow_redirect: |
223 |
if nb_redirections < self. |
|
270 |
if nb_redirections < self.max_redirections: |
|
224 |
271 |
if response.status in [301, 302, 307]: |
225 |
272 |
if method in ["GET", "HEAD"] or self.force_follow_redirect: |
226 |
273 |
if method not in ["GET", "HEAD"] and hasattr(body, 'seek'): |
| … | … | @@ -250,148 +297,143 @@ class HttpClient(object): |
250 |
297 |
raise errors.RedirectLimit("Redirection limit is reached") |
251 |
298 |
return response, connection |
252 |
299 |
|
253 |
def request(self, url, method='GET', body=None, headers=None, stream=False, |
|
254 |
stream_size=16384): |
|
300 |
def request(self, url, method='GET', body=None, headers=None): |
|
255 |
301 |
headers = headers or {} |
256 |
302 |
uri = url_parser(url) |
257 |
303 |
self.final_url = url |
258 |
304 |
|
259 |
305 |
headers.setdefault('User-Agent', restkit.USER_AGENT) |
260 |
306 |
if method in ["POST", "PUT"] and body is None: |
261 |
body = "" |
|
262 |
headers.setdefault("Content-Length", str(len(body))) |
|
307 |
headers.setdefault("Content-Length", "0") |
|
263 |
308 |
|
264 |
309 |
if self.use_proxy and uri.scheme != "https": |
265 |
proxy_auth = |
|
310 |
proxy_auth = pool.get_proxy_auth() |
|
266 |
311 |
if proxy_auth: |
267 |
312 |
headers['Proxy-Authorization'] = proxy_auth.strip() |
268 |
313 |
|
269 |
314 |
response, connection = self._request(uri, method, body, headers) |
270 |
re |
|
315 |
release_callback = lambda: self._release_connection(uri, connection) |
|
316 |
resp = self.response_class(response, release_callback) |
|
271 |
317 |
resp.final_url = self.final_url |
272 |
318 |
|
273 |
319 |
if method == "HEAD": |
274 |
response.close() |
|
275 |
self._release_connection(uri, connection) |
|
276 |
return resp, "" |
|
277 |
else: |
|
278 |
return resp, _decompress_content(resp, response, |
|
279 |
lambda: self._release_connection(uri, connection), |
|
280 |
stream, stream_size) |
|
281 |
||
282 |
def _decompress_content(resp, response, release_callback, stream=False, stream_size=16384): |
|
320 |
resp.close() |
|
321 |
return resp |
|
322 |
||
323 |
def _complain_ifclosed(closed): |
|
324 |
if closed: |
|
325 |
raise ValueError, "I/O operation on closed response" |
|
326 |
||
327 |
class ResponseStream(object): |
|
328 |
||
329 |
def __init__(self, response): |
|
330 |
self.response = response |
|
331 |
self.resp = response.resp |
|
332 |
self._rbuf = '' |
|
333 |
self.stream_size = MAX_CHUNK_SIZE |
|
334 |
||
335 |
def close(self): |
|
336 |
if not self.response.closed: |
|
337 |
self._buffer = "" |
|
338 |
self.response.close() |
|
339 |
||
340 |
def read(self, amt=None): |
|
341 |
if self._rbuf and not amt is None: |
|
342 |
L = len(self._rbuf) |
|
343 |
if amt > L: |
|
344 |
amt -= L |
|
345 |
else: |
|
346 |
s = self._rbuf[:amt] |
|
347 |
self._rbuf = self._rbuf[amt:] |
|
348 |
return s |
|
349 |
data = self.resp.read(amt) |
|
350 |
if not data: |
|
351 |
self.close() |
|
352 |
s = self._rbuf + data |
|
353 |
self._rbuf = '' |
|
354 |
return s |
|
355 |
||
356 |
def readline(self, amt=-1): |
|
357 |
i = self._rbuf.find('\n') |
|
358 |
while i < 0 and not (0 < amt <= len(self._rbuf)): |
|
359 |
new = self.resp.read(self.stream_size) |
|
360 |
if not new: |
|
361 |
self.close() |
|
362 |
break |
|
363 |
i = new.find('\n') |
|
364 |
if i >= 0: |
|
365 |
i = i + len(self._rbuf) |
|
366 |
self._rbuf = self._rbuf + new |
|
367 |
if i < 0: |
|
368 |
i = len(self._rbuf) |
|
369 |
else: |
|
370 |
i = i+1 |
|
371 |
if 0 <= amt < len(self._rbuf): |
|
372 |
i = amt |
|
373 |
data, self._rbuf = self._rbuf[:i], self._rbuf[i:] |
|
374 |
return data |
|
375 |
||
376 |
def readlines(self, sizehint=0): |
|
377 |
total = 0 |
|
378 |
lines = [] |
|
379 |
line = self.readline() |
|
380 |
while line: |
|
381 |
lines.append(line) |
|
382 |
total += len(line) |
|
383 |
if 0 < sizehint <= total: |
|
384 |
break |
|
385 |
line = self.readline() |
|
386 |
return lines |
|
387 |
||
388 |
def next(self): |
|
389 |
r = self.readline() |
|
390 |
if not r: |
|
391 |
raise StopIteration |
|
392 |
return r |
|
393 |
||
394 |
def __iter__(self): |
|
395 |
return self |
|
396 |
||
397 |
def _get_content(response): |
|
398 |
data = response.resp.read() |
|
399 |
response.close() |
|
400 |
return data |
|
401 |
||
402 |
def _decompress_content(response, stream=False): |
|
403 |
resp = response.resp |
|
283 |
404 |
try: |
284 |
encoding = resp.get('content-encoding', None) |
|
285 |
if encoding in ['gzip', 'deflate']: |
|
286 |
||
405 |
encoding = response.headers.get('content-encoding', None) |
|
406 |
if encoding in ('gzip', 'deflate'): |
|
287 |
407 |
if encoding == 'gzip': |
288 |
compressedstream = StringIO.StringIO(response.read()) |
|
289 |
release_callback() |
|
290 |
data = gzip.GzipFile(fileobj=compressedstream) |
|
291 |
408 |
if stream: |
292 |
return |
|
409 |
return gzip.GzipFile(fileobj=ResponseStream(response)) |
|
293 |
410 |
else: |
294 |
|
|
411 |
data = gzip.GzipFile(fileobj=ResponseStream(response)).read() |
|
412 |
response.close() |
|
413 |
return data |
|
295 |
414 |
else: |
296 |
data = zlib.decompress(response.read()) |
|
297 |
release_callback() |
|
298 |
415 |
if stream: |
299 |
return ResponseStream( |
|
416 |
return ResponseStream(ResponseStream(response)) |
|
300 |
417 |
else: |
301 |
return |
|
418 |
return zlib.decompress(_get_content(response)) |
|
302 |
419 |
else: |
303 |
420 |
if stream: |
304 |
return ResponseStream(response |
|
421 |
return ResponseStream(response) |
|
305 |
422 |
else: |
306 |
data = response.read() |
|
307 |
release_callback() |
|
308 |
return |
|
423 |
return _get_content(response) |
|
309 |
424 |
except Exception, e: |
310 |
re |
|
425 |
response.close() |
|
311 |
426 |
raise errors.ResponseError("Decompression failed %s" % str(e)) |
312 |
427 |
|
313 |
428 |
|
314 |
429 |
def _send_body_part(data, connection): |
315 |
430 |
if isinstance(data, types.StringTypes): |
316 |
data = StringIO |
|
431 |
data = StringIO(to_bytestring(data)) |
|
317 |
432 |
elif not hasattr(data, 'read'): |
318 |
data = StringIO |
|
433 |
data = StringIO(str(data)) |
|
319 |
434 |
|
320 |
435 |
# we always stream |
321 |
436 |
while 1: |
322 |
binarydata = data.read(1 |
|
437 |
binarydata = data.read(16384) |
|
323 |
438 |
if binarydata == '': break |
324 |
439 |
connection.send(binarydata) |
325 |
||
326 |
||
327 |
class ResponseStream(object): |
|
328 |
||
329 |
def __init__(self, response, amnt=16384, release_callback=None): |
|
330 |
self.response = response |
|
331 |
self.amnt = amnt |
|
332 |
self.callback = release_callback |
|
333 |
||
334 |
def next(self): |
|
335 |
return self.response.read(self.amnt) |
|
336 |
||
337 |
def __iter__(self): |
|
338 |
while 1: |
|
339 |
data = self.next() |
|
340 |
if data: |
|
341 |
yield data |
|
342 |
else: |
|
343 |
break |
|
344 |
if self.callback is not None: |
|
345 |
self.callback() |
|
346 |
||
347 |
class HTTPResponse(dict): |
|
348 |
"""An object more like email.Message than httplib.HTTPResponse. |
|
349 |
||
350 |
>>> from restclient import Resource |
|
351 |
>>> res = Resource('http://e-engura.org') |
|
352 |
>>> from restclient import Resource |
|
353 |
>>> res = Resource('http://e-engura.org') |
|
354 |
>>> page = res.get() |
|
355 |
>>> res.status |
|
356 |
200 |
|
357 |
>>> res.response['content-type'] |
|
358 |
'text/html' |
|
359 |
>>> logo = res.get('/images/logo.gif') |
|
360 |
>>> res.response['content-type'] |
|
361 |
'image/gif' |
|
362 |
""" |
|
363 |
||
364 |
final_url = None |
|
365 |
||
366 |
"Status code returned by server. " |
|
367 |
status = 200 |
|
368 |
||
369 |
"""Reason phrase returned by server.""" |
|
370 |
reason = "Ok" |
|
371 |
||
372 |
def __init__(self, info): |
|
373 |
if hasattr(info, "getheaders"): |
|
374 |
for key, value in info.getheaders(): |
|
375 |
self[key.lower()] = value |
|
376 |
self.status = info.status |
|
377 |
self['status'] = str(self.status) |
|
378 |
self.reason = info.reason |
|
379 |
self.version = info.version |
|
380 |
else: |
|
381 |
print info |
|
382 |
for key, value in info.iteritems(): |
|
383 |
self[key.lower()] = value |
|
384 |
self.status = int(self.get('status', self.status)) |
|
385 |
||
386 |
self.final_url = self.get('final_url', self.final_url) |
|
387 |
||
388 |
def __getattr__(self, name): |
|
389 |
if name == 'dict': |
|
390 |
return self |
|
391 |
else: |
|
392 |
raise AttributeError, name |
|
393 |
||
394 |
def __repr__(self): |
|
395 |
return "<%s status %s for %s>" % (self.__class__.__name__, |
|
396 |
self.status, |
|
397 |
self.final_url) |
Up to file-list restkit/pool.py:
16 |
16 |
|
17 |
17 |
|
18 |
18 |
""" |
19 |
Threadsafe Pool class |
|
19 |
Threadsafe Pool class |
|
20 |
20 |
|
21 |
21 |
TODO: |
22 |
- add our own way to share socket across connections. We shouldn't need to rely |
|
23 |
on eventlet for that |
|
24 |
22 |
- log errors |
25 |
23 |
""" |
26 |
24 |
|
27 |
||
28 |
||
25 |
import os |
|
26 |
import time |
|
29 |
27 |
import collections |
30 |
28 |
import httplib |
31 |
import Queue |
|
32 |
import threading |
|
29 |
import socket |
|
30 |
import urlparse |
|
33 |
31 |
|
32 |
import restkit |
|
34 |
33 |
from restkit import errors |
35 |
34 |
|
35 |
has_timeout = hasattr(socket, '_GLOBAL_DEFAULT_TIMEOUT') |
|
36 |
url_parser = urlparse.urlparse |
|
37 |
||
36 |
38 |
def get_proxy_auth(): |
37 |
39 |
import base64 |
38 |
40 |
proxy_username = os.environ.get('proxy-username') |
| … | … | @@ -48,176 +50,138 @@ def get_proxy_auth(): |
48 |
50 |
else: |
49 |
51 |
return '' |
50 |
52 |
|
51 |
def make_proxy_connection(uri): |
|
52 |
headers = headers or {} |
|
53 |
proxy = None |
|
54 |
if uri.scheme == 'https': |
|
55 |
proxy = os.environ.get('https_proxy') |
|
56 |
elif uri.scheme == 'http': |
|
57 |
proxy = os.environ.get('http_proxy') |
|
53 |
class PoolInterface(object): |
|
54 |
""" abstract class from which all connection |
|
55 |
pool should inherit. |
|
56 |
""" |
|
58 |
57 |
|
59 |
if not proxy: |
|
60 |
return make_connection(uri, use_proxy=False) |
|
61 |
||
62 |
if uri.scheme == 'https': |
|
63 |
proxy_auth = get_proxy_auth() |
|
64 |
if proxy_auth: |
|
65 |
proxy_auth = 'Proxy-authorization: %s' % proxy_auth |
|
66 |
port = uri.port |
|
67 |
if not port: |
|
68 |
port = 443 |
|
69 |
proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % (uri.hostname, port) |
|
70 |
user_agent = 'User-Agent: %s\r\n' % restkit.USER_AGENT |
|
71 |
proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth, user_agent) |
|
72 |
proxy_uri = url_parser(proxy) |
|
73 |
if not proxy_uri.port: |
|
74 |
proxy_uri.port = '80' |
|
75 |
# Connect to the proxy server, very simple recv and error checking |
|
76 |
p_sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM) |
|
77 |
p_sock.connect((proxy_uri.host, int(proxy_uri.port))) |
|
78 |
p_sock.sendall(proxy_pieces) |
|
79 |
response = '' |
|
80 |
# Wait for the full response. |
|
81 |
while response.find("\r\n\r\n") == -1: |
|
82 |
response += p_sock.recv(8192) |
|
83 |
p_status = response.split()[1] |
|
84 |
if p_status != str(200): |
|
85 |
raise ProxyError('Error status=%s' % str(p_status)) |
|
86 |
# Trivial setup for ssl socket. |
|
87 |
ssl = socket.ssl(p_sock, None, None) |
|
88 |
fake_sock = httplib.FakeSocket(p_sock, ssl) |
|
89 |
# Initalize httplib and replace with the proxy socket. |
|
90 |
connection = httplib.HTTPConnection(proxy_uri.host) |
|
91 |
connection.sock=fake_sock |
|
92 |
return connection |
|
93 |
else: |
|
94 |
proxy_uri = url_parser(proxy) |
|
95 |
if not proxy_uri.port: |
|
96 |
proxy_uri.port = '80' |
|
97 |
return httplib.HTTPConnection(proxy_uri.hostname, proxy_uri.port) |
|
98 |
return None |
|
58 |
def get(self): |
|
59 |
""" method used to return a connection from the pool""" |
|
60 |
raise NotImplementedError |
|
61 |
||
62 |
def put(self): |
|
63 |
""" Put an item back into the pool, when done """ |
|
64 |
raise NotImplementedError |
|
65 |
||
66 |
def clear(self): |
|
67 |
""" method used to release all connections """ |
|
68 |
raise NotImplementedError |
|
99 |
69 |
|
100 |
def make_connection(uri, use_proxy=True): |
|
101 |
if use_proxy: |
|
102 |
return make_proxy_connection(uri) |
|
103 |
||
104 |
if uri.scheme == 'https': |
|
105 |
if not uri.port: |
|
106 |
connection = httplib.HTTPSConnection(uri.hostname) |
|
107 |
else: |
|
108 |
connection = httplib.HTTPSConnection(uri.hostname, uri.port) |
|
109 |
else: |
|
110 |
if not uri.port: |
|
111 |
connection = httplib.HTTPConnection(uri.hostname) |
|
112 |
else: |
|
113 |
connection = httplib.HTTPConnection(uri.hostname, uri.port) |
|
114 |
return connection |
|
115 |
70 |
|
116 |
class Pool(object): |
|
117 |
def __init__(self, min_size=0, max_size=4, order_as_stack=False): |
|
71 |
||
72 |
class ConnectionPool(PoolInterface): |
|
73 |
def __init__(self, uri, use_proxy=False, key_file=None, cert_file=None, |
|
74 |
timeout=300, min_size=0, max_size=4): |
|
75 |
||
76 |
self.uri = uri |
|
77 |
self.use_proxy = use_proxy |
|
78 |
self.key_file = key_file |
|
79 |
self.cert_file = cert_file |
|
80 |
self.timeout = timeout |
|
118 |
81 |
self.min_size = min_size |
119 |
82 |
self.max_size = max_size |
120 |
self.order_as_stack = order_as_stack |
|
121 |
self.current_size = 0 |
|
122 |
self.channel = Queue.Queue(0) |
|
123 |
self.free_items = collections.deque() |
|
83 |
||
84 |
self.connections = collections.deque() |
|
124 |
85 |
for x in xrange(min_size): |
125 |
86 |
self.current_size += 1 |
126 |
self. |
|
87 |
self.connections.append(self.make_connection()) |
|
127 |
88 |
|
128 |
|
|
89 |
def _make_proxy_connection(self, proxy): |
|
90 |
if self.uri.scheme == 'https': |
|
91 |
proxy_auth = get_proxy_auth() |
|
92 |
if proxy_auth: |
|
93 |
proxy_auth = 'Proxy-authorization: %s' % proxy_auth |
|
94 |
port = self.uri.port |
|
95 |
if not port: |
|
96 |
port = 443 |
|
97 |
proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % (self.uri.hostname, port) |
|
98 |
user_agent = 'User-Agent: %s\r\n' % restkit.USER_AGENT |
|
99 |
proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth, user_agent) |
|
100 |
proxy_uri = url_parser(proxy) |
|
101 |
if not proxy_uri.port: |
|
102 |
proxy_uri.port = '80' |
|
103 |
# Connect to the proxy server, very simple recv and error checking |
|
104 |
p_sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM) |
|
105 |
p_sock.connect((proxy_uri.host, int(proxy_uri.port))) |
|
106 |
p_sock.sendall(proxy_pieces) |
|
107 |
response = '' |
|
108 |
# Wait for the full response. |
|
109 |
while response.find("\r\n\r\n") == -1: |
|
110 |
response += p_sock.recv(8192) |
|
111 |
p_status = response.split()[1] |
|
112 |
if p_status != str(200): |
|
113 |
raise errors.ProxyError('Error status=%s' % str(p_status)) |
|
114 |
# Trivial setup for ssl socket. |
|
115 |
ssl = socket.ssl(p_sock, None, None) |
|
116 |
fake_sock = httplib.FakeSocket(p_sock, ssl) |
|
117 |
# Initalize httplib and replace with the proxy socket. |
|
118 |
connection = httplib.HTTPConnection(proxy_uri.host) |
|
119 |
connection.sock=fake_sock |
|
120 |
return connection |
|
121 |
else: |
|
122 |
proxy_uri = url_parser(proxy) |
|
123 |
if not proxy_uri.port: |
|
124 |
proxy_uri.port = '80' |
|
125 |
return httplib.HTTPConnection(proxy_uri.hostname, proxy_uri.port) |
|
126 |
return None |
|
127 |
||
128 |
def make_connection(self): |
|
129 |
if self.use_proxy: |
|
130 |
proxy = '' |
|
131 |
if self.uri.scheme == 'https': |
|
132 |
proxy = os.environ.get('https_proxy') |
|
133 |
elif self.uri.scheme == 'http': |
|
134 |
proxy = os.environ.get('http_proxy') |
|
135 |
||
136 |
if proxy: |
|
137 |
return self._make_proxy_connection(proxy) |
|
138 |
||
139 |
kwargs = {} |
|
140 |
if hasattr(httplib.HTTPConnection, 'timeout'): |
|
141 |
kwargs['timeout'] = self.timeout |
|
142 |
||
143 |
if self.uri.port: |
|
144 |
kwargs['port'] = self.uri.port |
|
145 |
||
146 |
if self.uri.scheme == "https": |
|
147 |
kwargs.update(dict(key_file=self.key_file, cert_file=self.cert_file)) |
|
148 |
connection = httplib.HTTPSConnection(self.uri.hostname, **kwargs) |
|
149 |
else: |
|
150 |
connection = httplib.HTTPConnection(self.uri.hostname, **kwargs) |
|
151 |
||
152 |
setattr(connection, "started", time.time()) |
|
153 |
return connection |
|
129 |
154 |
|
130 |
155 |
def do_get(self): |
131 |
156 |
""" |
132 |
157 |
Return an item from the pool, when one is available |
133 |
158 |
""" |
134 |
self.lock.acquire() |
|
135 |
try: |
|
136 |
if self.free_items: |
|
137 |
return self.free_items.popleft() |
|
138 |
||
139 |
try: |
|
140 |
return self.channel.get(False) |
|
141 |
except Queue.Empty: |
|
142 |
created = self.create() |
|
143 |
self.current_size += 1 |
|
144 |
return created |
|
145 |
||
146 |
finally: |
|
147 |
self.lock.release() |
|
159 |
if self.connections: |
|
160 |
connection = self.connections.popleft() |
|
161 |
return connection |
|
162 |
else: |
|
163 |
return self.make_connection() |
|
148 |
164 |
|
149 |
165 |
def get(self): |
150 |
connection = self.do_get() |
|
151 |
return connection |
|
166 |
while True: |
|
167 |
connection = self.do_get() |
|
168 |
since = time.time() - connection.started |
|
169 |
if since < self.timeout: |
|
170 |
if connection._HTTPConnection__response: |
|
171 |
connection._HTTPConnection__response.read() |
|
172 |
return connection |
|
173 |
else: |
|
174 |
connection.close() |
|
152 |
175 |
|
153 |
def put(self, item): |
|
154 |
"""Put an item back into the pool, when done |
|
155 |
""" |
|
156 |
self.lock.acquire() |
|
157 |
try: |
|
158 |
if self.current_size > self.max_size: |
|
159 |
self.current_size -= 1 |
|
160 |
return |
|
176 |
def put(self, connection): |
|
177 |
if len(self.connections) >= self.max_size: |
|
178 |
connection.close() |
|
179 |
return |
|
180 |
if connection.sock is None: |
|
181 |
connection = self.make_connection() |
|
182 |
self.connections.append(connection) |
|
161 |
183 |
|
162 |
if self.waiting(): |
|
163 |
self.channel.put(item, False) |
|
164 |
else: |
|
165 |
if self.order_as_stack: |
|
166 |
self.free_items.appendleft(item) |
|
167 |
else: |
|
168 |
self.free_items.append(item) |
|
169 |
finally: |
|
170 |
self.lock.release() |
|
171 |
||
172 |
def resize(self, new_size): |
|
173 |
"""Resize the pool |
|
174 |
""" |
|
175 |
self.max_size = new_size |
|
176 |
||
177 |
def free(self): |
|
178 |
"""Return the number of free items in the pool. |
|
179 |
""" |
|
180 |
return len(self.free_items) + self.max_size - self.current_size |
|
181 |
||
182 |
def waiting(self): |
|
183 |
"""Return the number of routines waiting for a pool item. |
|
184 |
""" |
|
185 |
return (self.channel.qsize() < self.max_size) |
|
186 |
||
187 |
def create(self): |
|
188 |
"""Generate a new pool item |
|
189 |
""" |
|
190 |
raise NotImplementedError("Implement in subclass") |
|
191 |
||
192 |
class ConnectionPool(Pool): |
|
193 |
def __init__(self, uri, use_proxy=False, min_size=0, max_size=4): |
|
194 |
self.uri = uri |
|
195 |
self.use_proxy = use_proxy |
|
196 |
Pool.__init__(self, min_size, max_size) |
|
197 |
||
198 |
def create(self): |
|
199 |
return make_connection(self.uri, self.use_proxy) |
|
200 |
||
201 |
def put(self, connection): |
|
202 |
if self.current_size > self.max_size: |
|
203 |
self.lock.acquire() |
|
204 |
self.current_size -= 1 |
|
205 |
# close the connection if needed |
|
206 |
if connection.sock is not None: |
|
207 |
connection.close() |
|
208 |
self.lock.release() |
|
209 |
return |
|
210 |
||
211 |
try: |
|
212 |
response = connection.getresponse() |
|
213 |
response.read() |
|
214 |
except httplib.ResponseNotReady: |
|
215 |
pass |
|
216 |
|
|
184 |
def clear(self): |
|
185 |
while self.connections: |
|
186 |
connection = self.connections.pop() |
|
217 |
187 |
connection.close() |
218 |
||
219 |
||
220 |
if connection.sock is None: |
|
221 |
connection = self.create() |
|
222 |
||
223 |
Pool.put(self, connection) |
Up to file-list restkit/rest.py:
- |
Diff size exceeds threshold (19.6 KB) — view raw? |
Up to file-list restkit/utils.py:
| … | … | @@ -45,6 +45,49 @@ Converts an IRI to a URI. |
45 |
45 |
import re |
46 |
46 |
import urlparse |
47 |
47 |
|
48 |
from restkit.errors import InvalidUrl |
|
49 |
import urllib |
|
50 |
||
51 |
||
52 |
# code borrowed to Wekzeug with minor changes |
|
53 |
||
54 |
def url_quote(s, charset='utf-8', safe='/:'): |
|
55 |
"""URL encode a single string with a given encoding.""" |
|
56 |
if isinstance(s, unicode): |
|
57 |
s = s.encode(charset) |
|
58 |
elif not isinstance(s, str): |
|
59 |
s = str(s) |
|
60 |
return urllib.quote(s, safe=safe) |
|
61 |
||
62 |
def url_encode(obj, charset="utf8", encode_keys=False): |
|
63 |
if isinstance(obj, dict): |
|
64 |
items = [] |
|
65 |
for k, v in obj.iteritems(): |
|
66 |
if not isinstance(v, (tuple, list)): |
|
67 |
v = [v] |
|
68 |
items.append((k, v)) |
|
69 |
else: |
|
70 |
items = obj or () |
|
71 |
||
72 |
tmp = [] |
|
73 |
for key, values in items: |
|
74 |
if encode_keys and isinstance(key, unicode): |
|
75 |
key = key.encode(charset) |
|
76 |
else: |
|
77 |
key = str(key) |
|
78 |
||
79 |
for value in values: |
|
80 |
if value is None: |
|
81 |
continue |
|
82 |
elif isinstance(value, unicode): |
|
83 |
value = value.encode(charset) |
|
84 |
else: |
|
85 |
value = str(value) |
|
86 |
tmp.append('%s=%s' % (urllib.quote(key), |
|
87 |
urllib.quote_plus(value))) |
|
88 |
||
89 |
return '&'.join(tmp) |
|
90 |
||
48 |
91 |
|
49 |
92 |
def to_bytestring(s): |
50 |
93 |
if not isinstance(s, basestring): |
