diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index c8d94dd71..adb7f5d9b 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -57,7 +57,6 @@ class Downloader(object): self.slots = {} self.handlers = DownloadHandlers() self.middleware = DownloaderMiddlewareManager.from_settings(settings) - self.concurrent_spiders = settings.getint('CONCURRENT_SPIDERS') def fetch(self, request, spider): slot = self.slots[spider] diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b2c8baf07..7759ccd89 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -59,6 +59,7 @@ class ExecutionEngine(object): self.scheduler = load_object(settings['SCHEDULER'])() self.downloader = Downloader() self.scraper = Scraper(self, self.settings) + self._concurrent_spiders = settings.getint('CONCURRENT_SPIDERS') self._spider_closed_callback = spider_closed_callback @defer.inlineCallbacks @@ -164,7 +165,7 @@ class ExecutionEngine(object): def has_capacity(self): """Does the engine have capacity to handle more spiders""" - return len(self.downloader.slots) < self.downloader.concurrent_spiders + return len(self.slots) < self._concurrent_spiders def crawl(self, request, spider): assert spider in self.open_spiders, \