BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-27 04:24:05 +00:00
Code Releases Wiki Activity

more efficient ExecutionEngine.spider_is_idle

Browse Source
This commit is contained in:
Mikhail Korobov 2016-02-03 05:37:40 +05:00
parent 8a391a557f
commit c6591b5c9f
1 changed files with 17 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
scrapy/core/engine.py
View File

@ -177,12 +177,23 @@ class ExecutionEngine(object):
return d
def spider_is_idle(self, spider):
scraper_idle = self.scraper.slot.is_idle()
pending = self.slot.scheduler.has_pending_requests()
downloading = bool(self.downloader.active)
pending_start_requests = self.slot.start_requests is not None
idle = scraper_idle and not (pending or downloading or pending_start_requests)
return idle
if not self.scraper.slot.is_idle():
# scraper is not idle
return False
if self.downloader.active:
# downloader has pending requests
return False
if self.slot.start_requests is not None:
# not all start requests are handled
return False
if self.slot.scheduler.has_pending_requests():
# scheduler has pending requests
return False
return True
@property
def open_spiders(self):