BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-26 21:04:34 +00:00
Code Releases Wiki Activity

Normalized the usage of ints for storing http status codes

Browse Source 
--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40770
This commit is contained in:
elpolilla 2009-01-26 15:03:47 +00:00
parent 74661d54d0
commit f63a661320
3 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py
View File

@ -20,7 +20,7 @@ class RedirectMiddleware(object):
status = exception.status
response = exception.response
if status in ['302', '303']:
if status in [302, 303]:
redirected_url = urljoin(request.url, response.headers['location'][0])
if not getattr(spider, "no_redirect", False):
redirected = request.replace(url=redirected_url, method='GET', body=None)
@ -34,21 +34,21 @@ class RedirectMiddleware(object):
redirected.dont_filter -= 1
else:
redirected.dont_filter = False
log.msg("Redirecting (%s) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name)
log.msg("Redirecting (%d) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name)
return redirected
log.msg("Ignored redirecting (%s) to %s from %s (disabled by spider)" % (status, redirected_url, request), level=log.DEBUG, domain=spider.domain_name)
log.msg("Ignored redirecting (%d) to %s from %s (disabled by spider)" % (status, redirected_url, request), level=log.DEBUG, domain=spider.domain_name)
return response
if status in ['301', '307']:
if status in [301, 307]:
redirected_url = urljoin(request.url, response.headers['location'][0])
if not getattr(spider, "no_redirect", False):
redirected = request.replace(url=redirected_url)
# This is needed to avoid redirection loops with requests that contain dont_filter = True
# Example (9 May 2008): http://www.55max.com/product/001_photography.asp?3233,0,0,0,Michael+Banks
redirected.dont_filter = False
log.msg("Redirecting (%s) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name)
log.msg("Redirecting (%d) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name)
return redirected
log.msg("Ignored redirecting (%s) to %s from %s (disabled by spider)" % (status, redirected_url, request), level=log.DEBUG, domain=spider.domain_name)
log.msg("Ignored redirecting (%d) to %s from %s (disabled by spider)" % (status, redirected_url, request), level=log.DEBUG, domain=spider.domain_name)
return response
def process_response(self, request, response, spider):

2
scrapy/trunk/scrapy/core/exceptions.py
View File

@ -31,7 +31,7 @@ class HttpException(Exception):
from twisted.web import http
message = http.responses.get(int(status))
self.status = status
self.status = int(status)
self.message = message
self.response = response
Exception.__init__(self, status, message, response)

4
scrapy/trunk/scrapy/http/response/__init__.py
View File

@ -17,7 +17,7 @@ class Response(object):
def __init__(self, url, status=200, headers=None, body='', meta=None, flags=None):
self.url = Url(url)
self.headers = Headers(headers or {})
self.status = status
self.status = int(status)
self.set_body(body)
self.cached = False
self.request = None
@ -71,7 +71,7 @@ class Response(object):
received (that's not exposed by Twisted).
"""
s = "HTTP/1.1 %s %s\r\n" % (self.status, RESPONSES[self.status])
s = "HTTP/1.1 %d %s\r\n" % (self.status, RESPONSES[self.status])
if self.headers:
s += self.headers.to_string() + "\r\n"
s += "\r\n"