BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-21 05:13:16 +00:00
Code Releases Wiki Activity

Raise an exception if ASYNCIO_ENABLED but the reactor is wrong.

Browse Source
This commit is contained in:
Andrey Rakhmatullin 2019-12-21 13:08:29 +05:00
parent e342de5038
commit 8de80f59db
3 changed files with 11 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/crawler.py
View File

@ -14,7 +14,7 @@ from scrapy.extension import ExtensionManager
from scrapy.settings import overridden_settings, Settings
from scrapy.signalmanager import SignalManager
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.asyncio import install_asyncio_reactor
from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_reactor_installed
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
from scrapy.utils.misc import load_object
from scrapy.utils.log import (
@ -259,6 +259,10 @@ class CrawlerProcess(CrawlerRunner):
super(CrawlerProcess, self).__init__(settings)
if self.settings.getbool('ASYNCIO_ENABLED'):
install_asyncio_reactor()
if not is_asyncio_reactor_installed():
raise Exception("ASYNCIO_ENABLED is on but the Twisted asyncio "
"reactor is not installed, this is not supported.")
install_shutdown_handlers(self._signal_shutdown)
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)

8
scrapy/utils/log.py
View File

@ -11,7 +11,6 @@ from twisted.python import log as twisted_log
import scrapy
from scrapy.settings import Settings
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.asyncio import is_asyncio_reactor_installed
from scrapy.utils.versions import scrapy_components_versions
@ -150,12 +149,7 @@ def log_scrapy_info(settings):
for name, version in scrapy_components_versions()
if name != "Scrapy")})
if settings.getbool('ASYNCIO_ENABLED'):
if is_asyncio_reactor_installed():
logger.debug("Asyncio support enabled")
else:
logger.error("ASYNCIO_ENABLED is on but the Twisted asyncio "
"reactor is not installed, this is not supported "
"and asyncio coroutines will not work.")
logger.debug("Asyncio support enabled")
class StreamLogger(object):

9
tests/test_crawler.py
View File

@ -264,13 +264,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
@defer.inlineCallbacks
def test_crawler_process_asyncio_enabled_true(self):
with LogCapture(level=logging.DEBUG) as log:
runner = CrawlerProcess(settings={'ASYNCIO_ENABLED': True})
yield runner.crawl(NoRequestsSpider)
if self.reactor_pytest == 'asyncio':
runner = CrawlerProcess(settings={'ASYNCIO_ENABLED': True})
yield runner.crawl(NoRequestsSpider)
self.assertIn("Asyncio support enabled", str(log))
else:
self.assertNotIn("Asyncio support enabled", str(log))
self.assertIn("ASYNCIO_ENABLED is on but the Twisted asyncio reactor is not installed", str(log))
msg = "ASYNCIO_ENABLED is on but the Twisted asyncio reactor is not installed"
with self.assertRaisesRegex(Exception, msg):
runner = CrawlerProcess(settings={'ASYNCIO_ENABLED': True})
@defer.inlineCallbacks
def test_crawler_process_asyncio_enabled_false(self):