scrapy/tests/test_command_check.py

from tests.test_commands import CommandTest


class CheckCommandTest(CommandTest):

    command = 'check'

    def setUp(self):
        super(CheckCommandTest, self).setUp()
        self.spider_name = 'check_spider'
        self.spider = (self.proj_mod_path / 'spiders' / 'checkspider.py').resolve()

    def _write_contract(self, contracts, parse_def):
        self.spider.write_text(f"""
import scrapy

class CheckSpider(scrapy.Spider):
    name = '{self.spider_name}'
    start_urls = ['http://toscrape.com']

    def parse(self, response, **cb_kwargs):
        \"\"\"
        @url http://toscrape.com
        {contracts}
        \"\"\"
        {parse_def}
        """)

    def _test_contract(self, contracts='', parse_def='pass'):
        self._write_contract(contracts, parse_def)
        p, out, err = self.proc('check')
        self.assertNotIn('F', out)
        self.assertIn('OK', err)
        self.assertEqual(p.returncode, 0)

    def test_check_returns_requests_contract(self):
        contracts = """
        @returns requests 1
        """
        parse_def = """
        yield scrapy.Request(url='http://next-url.com')
        """
        self._test_contract(contracts, parse_def)

    def test_check_returns_items_contract(self):
        contracts = """
        @returns items 1
        """
        parse_def = """
        yield {'key1': 'val1', 'key2': 'val2'}
        """
        self._test_contract(contracts, parse_def)

    def test_check_cb_kwargs_contract(self):
        contracts = """
        @cb_kwargs {"arg1": "val1", "arg2": "val2"}
        """
        parse_def = """
        if len(cb_kwargs.items()) == 0:
            raise Exception("Callback args not set")
        """
        self._test_contract(contracts, parse_def)

    def test_check_scrapes_contract(self):
        contracts = """
        @scrapes key1 key2
        """
        parse_def = """
        yield {'key1': 'val1', 'key2': 'val2'}
        """
        self._test_contract(contracts, parse_def)

    def test_check_all_default_contracts(self):
        contracts = """
        @returns items 1
        @returns requests 1
        @scrapes key1 key2
        @cb_kwargs {"arg1": "val1", "arg2": "val2"}
        """
        parse_def = """
        yield {'key1': 'val1', 'key2': 'val2'}
        yield scrapy.Request(url='http://next-url.com')
        if len(cb_kwargs.items()) == 0:
            raise Exception("Callback args not set")
        """
        self._test_contract(contracts, parse_def)

    def test_SCRAPY_CHECK_set(self):
        parse_def = """
        import os
        if not os.environ.get('SCRAPY_CHECK'):
            raise Exception('SCRAPY_CHECK not set')
        """
        self._test_contract(parse_def=parse_def)