From 8ff4dc4d02cb8c75bdb053b9023d8655759ab1b8 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Thu, 22 Jan 2009 03:19:53 +0000 Subject: [PATCH] RetryMiddleware: added ConnectionLost to retried exceptions twisted >8.0 has a ConnectionClosed exception parent of ConnectionLost and ConnectionDone, but twisted 2.5 hasn't. I add ConnectionLost until we can move forward to twisted >8.0 this is the docstring of ConnectionLost, hopes it is self explanatory: """Connection to the other side was lost in a non-clean fashion""" --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40757 --- .../scrapy/contrib/downloadermiddleware/retry.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py index 1ea1018a7..dbdb6277b 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py @@ -2,7 +2,7 @@ An extension to retry failed requests that are potentially caused by temporary problems such as a connection timeout or HTTP 500 error. -You can change the behaviour of this moddileware by modifing the scraping settings: +You can change the behaviour of this moddileware by modifing the scraping settings: RETRY_TIMES - how many times to retry a failed page RETRY_HTTP_CODES - which HTTP response codes to retry @@ -21,12 +21,13 @@ About HTTP errors to consider: indicate server overload, which would be something we want to retry - 200 is included by default (and shoudln't be removed) to check for partial downloads errors, which means the TCP connection has broken in the middle of - a HTTP download -""" + a HTTP download +""" from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError -from twisted.internet.defer import TimeoutError as UserTimeoutError + ConnectionRefusedError, ConnectionDone, ConnectError, \ + ConnectionLost +from twisted.internet.defer import TimeoutError as UserTimeoutError from scrapy import log from scrapy.core.exceptions import HttpException @@ -35,8 +36,9 @@ from scrapy.conf import settings class RetryMiddleware(object): - EXCEPTIONS_TO_RETRY = (ServerTimeoutError, UserTimeoutError, DNSLookupError, - ConnectionRefusedError, ConnectionDone, ConnectError) + EXCEPTIONS_TO_RETRY = (ServerTimeoutError, UserTimeoutError, DNSLookupError, + ConnectionRefusedError, ConnectionDone, ConnectError, + ConnectionLost) def __init__(self): self.failed_count = {}