BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-27 14:03:40 +00:00
Code Releases Wiki Activity

show download warnsize once

Browse Source
This commit is contained in:
Aron Bordin 2015-12-30 13:10:13 -02:00
parent a5db7f862c
commit f57121c77b
1 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scrapy/core/downloader/handlers/http11.py
View File

@ -292,6 +292,7 @@ class _ResponseReader(protocol.Protocol):
self._bodybuf = BytesIO()
self._maxsize = maxsize
self._warnsize = warnsize
self._reached_warnsize = False
self._bytes_received = 0
def dataReceived(self, bodyBytes):
@ -305,11 +306,12 @@ class _ResponseReader(protocol.Protocol):
'maxsize': self._maxsize})
self._finished.cancel()
if self._warnsize and self._bytes_received > self._warnsize:
logger.warning("Received (%(bytes)s) bytes larger than download "
"warn size (%(warnsize)s).",
{'bytes': self._bytes_received,
'warnsize': self._warnsize})
if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize:
self._reached_warnsize = True
logger.warning("Received more bytes than download "
"warn size (%(warnsize)s) in request %(request)s.",
{'warnsize': self._warnsize,
'request': self._request})
def connectionLost(self, reason):
if self._finished.called: