1
0
mirror of https://github.com/scrapy/scrapy.git synced 2025-03-14 19:52:49 +00:00
scrapy/tox.ini
2023-09-12 20:07:10 +02:00

224 lines
5.3 KiB
INI

# Tox (https://tox.readthedocs.io/) is a tool for running tests
# in multiple virtualenvs. This configuration file will run the
# test suite on all supported python versions. To use it, "pip install tox"
# and then run "tox" from this directory.
[tox]
envlist = pre-commit,pylint,typing,py
minversion = 1.7.0
[testenv]
deps =
-rtests/requirements.txt
# mitmproxy does not support PyPy
# Python 3.9+ requires mitmproxy >= 5.3.0
# mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0
#mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
# The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
passenv =
S3_TEST_FILE_URI
AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY
AWS_SESSION_TOKEN
GCS_TEST_FILE_URI
GCS_PROJECT_ID
#allow tox virtualenv to upgrade pip/wheel/setuptools
download = true
commands =
pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} --doctest-modules
install_command =
python -I -m pip install -ctests/upper-constraints.txt {opts} {packages}
[testenv:typing]
basepython = python3
deps =
mypy==1.5.1
typing-extensions==4.7.1
types-attrs==19.1.0
types-lxml==2023.3.28
types-Pillow==10.0.0.3
types-Pygments==2.16.0.0
types-pyOpenSSL==23.2.0.2
types-setuptools==68.2.0.0
# 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211
w3lib >= 2.1.2
commands =
mypy {posargs: scrapy tests}
[testenv:pre-commit]
basepython = python3
deps =
pre-commit
commands =
pre-commit run {posargs:--all-files}
[testenv:pylint]
basepython = python3
deps =
{[testenv:extra-deps]deps}
pylint==2.17.5
commands =
pylint conftest.py docs extras scrapy setup.py tests
[testenv:twinecheck]
basepython = python3
deps =
twine==4.0.2
build==1.0.3
commands =
python -m build --sdist
twine check dist/*
[pinned]
deps =
cryptography==36.0.0
cssselect==0.9.1
h2==3.0
itemadapter==0.1.0
parsel==1.5.0
Protego==0.1.15
pyOpenSSL==21.0.0
queuelib==1.4.2
service_identity==18.1.0
Twisted[http2]==18.9.0
w3lib==1.17.0
zope.interface==5.1.0
lxml==4.4.1
-rtests/requirements.txt
# mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
# above, hence we do not install it in pinned environments at the moment
setenv =
_SCRAPY_PINNED=true
install_command =
python -I -m pip install {opts} {packages}
commands =
pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests}
[testenv:pinned]
basepython = python3.8
deps =
{[pinned]deps}
PyDispatcher==2.0.5
install_command = {[pinned]install_command}
setenv =
{[pinned]setenv}
commands = {[pinned]commands}
[testenv:windows-pinned]
basepython = python3
deps =
{[pinned]deps}
PyDispatcher==2.0.5
install_command = {[pinned]install_command}
setenv =
{[pinned]setenv}
commands = {[pinned]commands}
[testenv:extra-deps]
basepython = python3
deps =
{[testenv]deps}
boto3
google-cloud-storage
# Twisted[http2] currently forces old mitmproxy because of h2 version
# restrictions in their deps, so we need to pin old markupsafe here too.
markupsafe < 2.1.0
robotexclusionrulesparser
Pillow
Twisted[http2]
[testenv:extra-deps-pinned]
basepython = python3.8
deps =
{[pinned]deps}
boto3==1.20.0
google-cloud-storage==1.29.0
Pillow==7.1.0
robotexclusionrulesparser==1.6.2
install_command = {[pinned]install_command}
setenv =
{[pinned]setenv}
commands = {[pinned]commands}
[testenv:asyncio]
commands =
{[testenv]commands} --reactor=asyncio
[testenv:asyncio-pinned]
deps = {[testenv:pinned]deps}
commands = {[pinned]commands} --reactor=asyncio
install_command = {[pinned]install_command}
setenv =
{[pinned]setenv}
[testenv:pypy3]
basepython = pypy3
commands =
pytest {posargs:--durations=10 docs scrapy tests}
[testenv:pypy3-pinned]
basepython = {[testenv:pypy3]basepython}
deps =
{[pinned]deps}
PyPyDispatcher==2.1.0
commands =
pytest --durations=10 scrapy tests
install_command = {[pinned]install_command}
setenv =
{[pinned]setenv}
[docs]
changedir = docs
deps =
-rdocs/requirements.txt
setenv =
READTHEDOCS_PROJECT=scrapy
READTHEDOCS_VERSION=master
[testenv:docs]
basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
sphinx-build -W -b html . {envtmpdir}/html
[testenv:docs-coverage]
basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
sphinx-build -b coverage . {envtmpdir}/coverage
[testenv:docs-links]
basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
# Run S3 tests with botocore installed but without boto3.
[testenv:botocore]
deps =
{[testenv]deps}
botocore>=1.4.87
commands =
pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
[testenv:botocore-pinned]
basepython = python3.8
deps =
{[pinned]deps}
botocore==1.4.87
install_command = {[pinned]install_command}
setenv =
{[pinned]setenv}
commands =
pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}