mirror of
https://github.com/scrapy/scrapy.git
synced 2025-02-06 08:33:33 +00:00
ba5df629a2
* Make download handler test base classes abstract. * Small cleanup. * Don't run the full test suite for special HTTP cases. * Don't run tests in imported base classes. * Remove an obsolete service_identity check. * Move FTP imports back to the top level. * Simplify the H2DownloadHandler import. * Forbig pytest 8.2.x. * Revert "Simplify the H2DownloadHandler import." This reverts commit ed187046ac53c395c7423c0f5e6fb2bc7c27838f.
281 lines
6.8 KiB
INI
281 lines
6.8 KiB
INI
# Tox (https://tox.readthedocs.io/) is a tool for running tests
|
|
# in multiple virtualenvs. This configuration file will run the
|
|
# test suite on all supported python versions. To use it, "pip install tox"
|
|
# and then run "tox" from this directory.
|
|
|
|
[tox]
|
|
envlist = pre-commit,pylint,typing,py
|
|
minversion = 1.7.0
|
|
|
|
[test-requirements]
|
|
deps =
|
|
attrs
|
|
coverage >= 7.4.0
|
|
pexpect >= 4.8.0
|
|
pyftpdlib >= 2.0.1
|
|
pygments
|
|
pytest != 8.2.* # https://github.com/pytest-dev/pytest/issues/12275
|
|
pytest-cov >= 4.0.0
|
|
pytest-xdist
|
|
sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
|
|
testfixtures
|
|
pywin32; sys_platform == "win32"
|
|
|
|
[testenv]
|
|
deps =
|
|
{[test-requirements]deps}
|
|
|
|
# mitmproxy does not support PyPy
|
|
mitmproxy; implementation_name != 'pypy'
|
|
setenv =
|
|
COVERAGE_CORE=sysmon
|
|
passenv =
|
|
S3_TEST_FILE_URI
|
|
AWS_ACCESS_KEY_ID
|
|
AWS_SECRET_ACCESS_KEY
|
|
AWS_SESSION_TOKEN
|
|
GCS_TEST_FILE_URI
|
|
GCS_PROJECT_ID
|
|
#allow tox virtualenv to upgrade pip/wheel/setuptools
|
|
download = true
|
|
commands =
|
|
pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} --doctest-modules
|
|
install_command =
|
|
python -I -m pip install -ctests/upper-constraints.txt {opts} {packages}
|
|
|
|
[testenv:typing]
|
|
basepython = python3
|
|
deps =
|
|
mypy==1.14.0
|
|
typing-extensions==4.12.2
|
|
types-lxml==2024.12.13
|
|
types-Pygments==2.18.0.20240506
|
|
botocore-stubs==1.35.90
|
|
boto3-stubs[s3]==1.35.90
|
|
attrs >= 18.2.0
|
|
Pillow >= 10.3.0
|
|
pyOpenSSL >= 24.2.1
|
|
pytest >= 8.2.0
|
|
w3lib >= 2.2.0
|
|
commands =
|
|
mypy {posargs: scrapy tests}
|
|
|
|
[testenv:typing-tests]
|
|
basepython = python3.9
|
|
deps =
|
|
{[test-requirements]deps}
|
|
{[testenv:typing]deps}
|
|
pytest-mypy-testing==0.1.3
|
|
commands =
|
|
pytest {posargs: tests_typing}
|
|
|
|
[testenv:pre-commit]
|
|
basepython = python3
|
|
deps =
|
|
pre-commit
|
|
commands =
|
|
pre-commit run {posargs:--all-files}
|
|
|
|
[testenv:pylint]
|
|
basepython = python3
|
|
deps =
|
|
{[testenv:extra-deps]deps}
|
|
pylint==3.3.3
|
|
commands =
|
|
pylint conftest.py docs extras scrapy tests
|
|
|
|
[testenv:twinecheck]
|
|
basepython = python3
|
|
deps =
|
|
twine==6.0.1
|
|
build==1.2.2.post1
|
|
commands =
|
|
python -m build --sdist
|
|
twine check dist/*
|
|
|
|
[pinned]
|
|
basepython = python3.9
|
|
deps =
|
|
cryptography==37.0.0
|
|
cssselect==0.9.1
|
|
h2==3.0
|
|
itemadapter==0.1.0
|
|
parsel==1.5.0
|
|
Protego==0.1.15
|
|
pyOpenSSL==22.0.0
|
|
queuelib==1.4.2
|
|
service_identity==18.1.0
|
|
Twisted[http2]==21.7.0
|
|
w3lib==1.17.0
|
|
zope.interface==5.1.0
|
|
lxml==4.6.0
|
|
{[test-requirements]deps}
|
|
|
|
# mitmproxy 8.0.0 requires upgrading some of the pinned dependencies
|
|
# above, hence we do not install it in pinned environments at the moment
|
|
setenv =
|
|
_SCRAPY_PINNED=true
|
|
install_command =
|
|
python -I -m pip install {opts} {packages}
|
|
commands =
|
|
; tests for docs fail with parsel < 1.8.0
|
|
pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests}
|
|
|
|
[testenv:pinned]
|
|
basepython = {[pinned]basepython}
|
|
deps =
|
|
{[pinned]deps}
|
|
PyDispatcher==2.0.5
|
|
install_command = {[pinned]install_command}
|
|
setenv =
|
|
{[pinned]setenv}
|
|
commands = {[pinned]commands}
|
|
|
|
[testenv:windows-pinned]
|
|
basepython = {[pinned]basepython}
|
|
deps =
|
|
{[pinned]deps}
|
|
PyDispatcher==2.0.5
|
|
install_command = {[pinned]install_command}
|
|
setenv =
|
|
{[pinned]setenv}
|
|
commands = {[pinned]commands}
|
|
|
|
[testenv:extra-deps]
|
|
basepython = python3
|
|
deps =
|
|
{[testenv]deps}
|
|
boto3
|
|
google-cloud-storage
|
|
robotexclusionrulesparser
|
|
Pillow
|
|
Twisted[http2]
|
|
uvloop; platform_system != "Windows"
|
|
bpython # optional for shell wrapper tests
|
|
brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests
|
|
brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests
|
|
zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests
|
|
ipython
|
|
|
|
[testenv:extra-deps-pinned]
|
|
basepython = {[pinned]basepython}
|
|
deps =
|
|
{[pinned]deps}
|
|
boto3==1.20.0
|
|
google-cloud-storage==1.29.0
|
|
Pillow==7.1.0
|
|
robotexclusionrulesparser==1.6.2
|
|
brotlipy
|
|
uvloop==0.14.0; platform_system != "Windows"
|
|
bpython==0.7.1
|
|
zstandard==0.1; implementation_name != 'pypy'
|
|
ipython==2.0.0
|
|
brotli==0.5.2; implementation_name != 'pypy'
|
|
brotlicffi==0.8.0; implementation_name == 'pypy'
|
|
install_command = {[pinned]install_command}
|
|
setenv =
|
|
{[pinned]setenv}
|
|
commands = {[pinned]commands}
|
|
|
|
[testenv:asyncio]
|
|
commands =
|
|
{[testenv]commands} --reactor=asyncio
|
|
|
|
[testenv:asyncio-pinned]
|
|
basepython = {[pinned]basepython}
|
|
deps = {[testenv:pinned]deps}
|
|
commands = {[pinned]commands} --reactor=asyncio
|
|
install_command = {[pinned]install_command}
|
|
setenv =
|
|
{[pinned]setenv}
|
|
|
|
[testenv:pypy3]
|
|
basepython = pypy3
|
|
commands =
|
|
; not enabling coverage as it significantly increases the run time
|
|
pytest {posargs:--durations=10 docs scrapy tests}
|
|
|
|
[testenv:pypy3-extra-deps]
|
|
basepython = pypy3
|
|
deps =
|
|
{[testenv:extra-deps]deps}
|
|
commands = {[testenv:pypy3]commands}
|
|
|
|
[testenv:pypy3-pinned]
|
|
basepython = pypy3.10
|
|
deps =
|
|
cryptography==41.0.5
|
|
cssselect==0.9.1
|
|
h2==3.1
|
|
itemadapter==0.1.0
|
|
parsel==1.5.0
|
|
Protego==0.1.15
|
|
pyOpenSSL==23.3.0
|
|
queuelib==1.4.2
|
|
service_identity==18.1.0
|
|
Twisted[http2]==21.7.0
|
|
w3lib==1.17.0
|
|
zope.interface==5.1.0
|
|
lxml==4.6.0
|
|
{[test-requirements]deps}
|
|
PyPyDispatcher==2.1.0
|
|
commands =
|
|
; disabling both coverage and docs tests
|
|
pytest {posargs:--durations=10 scrapy tests}
|
|
install_command = {[pinned]install_command}
|
|
setenv =
|
|
{[pinned]setenv}
|
|
|
|
[docs]
|
|
changedir = docs
|
|
deps =
|
|
-rdocs/requirements.txt
|
|
setenv =
|
|
READTHEDOCS_PROJECT=scrapy
|
|
READTHEDOCS_VERSION=master
|
|
|
|
[testenv:docs]
|
|
basepython = python3
|
|
changedir = {[docs]changedir}
|
|
deps = {[docs]deps}
|
|
setenv = {[docs]setenv}
|
|
commands =
|
|
sphinx-build -W -b html . {envtmpdir}/html
|
|
|
|
[testenv:docs-coverage]
|
|
basepython = python3
|
|
changedir = {[docs]changedir}
|
|
deps = {[docs]deps}
|
|
setenv = {[docs]setenv}
|
|
commands =
|
|
sphinx-build -b coverage . {envtmpdir}/coverage
|
|
|
|
[testenv:docs-links]
|
|
basepython = python3
|
|
changedir = {[docs]changedir}
|
|
deps = {[docs]deps}
|
|
setenv = {[docs]setenv}
|
|
commands =
|
|
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
|
|
|
|
|
|
# Run S3 tests with botocore installed but without boto3.
|
|
|
|
[testenv:botocore]
|
|
deps =
|
|
{[testenv]deps}
|
|
botocore>=1.4.87
|
|
commands =
|
|
pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore}
|
|
|
|
[testenv:botocore-pinned]
|
|
basepython = {[pinned]basepython}
|
|
deps =
|
|
{[pinned]deps}
|
|
botocore==1.4.87
|
|
install_command = {[pinned]install_command}
|
|
setenv =
|
|
{[pinned]setenv}
|
|
commands =
|
|
pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore}
|