BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-28 09:03:45 +00:00
Code Releases Wiki Activity

added more verbosity for log and for exception when download is cancelled because of a size limit

Browse Source
This commit is contained in:
Leonid Amirov 2015-12-03 15:30:06 +03:00
parent eb4daa34a2
commit 016875fd51
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
scrapy/core/downloader/handlers/http11.py
View File

@ -239,11 +239,16 @@ class ScrapyAgent(object):
expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1
if maxsize and expected_size > maxsize: if maxsize and expected_size > maxsize:
logger.error("Expected response size (%(size)s) larger than " error_message = (
"download max size (%(maxsize)s).", "Cancelling download of {url}: expected response "
{'size': expected_size, 'maxsize': maxsize}) "size ({size}) larger than "
"download max size ({maxsize}).".format(
url=request.url, size=expected_size, maxsize=maxsize
)
)
logger.error(error_message)
txresponse._transport._producer.loseConnection() txresponse._transport._producer.loseConnection()
raise defer.CancelledError() raise defer.CancelledError(error_message)
if warnsize and expected_size > warnsize: if warnsize and expected_size > warnsize:
logger.warning("Expected response size (%(size)s) larger than " logger.warning("Expected response size (%(size)s) larger than "