BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-26 17:44:02 +00:00
Code Releases Wiki Activity

move AutoThrottle default options to default_settings.py

Browse Source
This commit is contained in:
Mikhail Korobov 2015-06-26 18:58:29 +05:00
parent e6272e5209
commit 584252e8f2
2 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/extensions/throttle.py
View File

@ -33,10 +33,10 @@ class AutoThrottle(object):
s.getfloat('DOWNLOAD_DELAY')
def _max_delay(self, spider):
return self.crawler.settings.getfloat('AUTOTHROTTLE_MAX_DELAY', 60.0)
return self.crawler.settings.getfloat('AUTOTHROTTLE_MAX_DELAY')
def _start_delay(self, spider):
return max(self.mindelay, self.crawler.settings.getfloat('AUTOTHROTTLE_START_DELAY', 5.0))
return max(self.mindelay, self.crawler.settings.getfloat('AUTOTHROTTLE_START_DELAY'))
def _response_downloaded(self, response, request, spider):
key, slot = self._get_slot(request, spider)

5
scrapy/settings/default_settings.py
View File

@ -20,6 +20,11 @@ from os.path import join, abspath, dirname
AJAXCRAWL_ENABLED = False
AUTOTHROTTLE_ENABLED = False
AUTOTHROTTLE_DEBUG = False
AUTOTHROTTLE_MAX_DELAY = 60.0
AUTOTHROTTLE_START_DELAY = 5.0
BOT_NAME = 'scrapybot'
CLOSESPIDER_TIMEOUT = 0