mirror of
https://github.com/scrapy/scrapy.git
synced 2025-02-06 09:07:32 +00:00
Integrating configs into pyproject.toml (#6547)
This commit is contained in:
Swayam Gupta
GitHub
parent
efb53aafdc
commit
8c23da943c
@ -1,7 +0,0 @@
|
||||
skips:
|
||||
- B101 # assert_used, needed for mypy
|
||||
- B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180
|
||||
- B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180
|
||||
- B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082
|
||||
- B503 # ssl_with_bad_defaults
|
||||
exclude_dirs: ['tests']
|
||||
@ -1,11 +0,0 @@
|
||||
[bumpversion]
|
||||
current_version = 2.12.0
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
|
||||
[bumpversion:file:scrapy/VERSION]
|
||||
|
||||
[bumpversion:file:SECURITY.md]
|
||||
parse = (?P<major>\d+)\.(?P<minor>\d+)\.x
|
||||
serialize = {major}.{minor}.x
|
||||
12
.coveragerc
12
.coveragerc
@ -1,12 +0,0 @@
|
||||
[run]
|
||||
branch = true
|
||||
include = scrapy/*
|
||||
omit =
|
||||
tests/*
|
||||
disable_warnings = include-ignored
|
||||
|
||||
[report]
|
||||
# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185
|
||||
exclude_lines =
|
||||
pragma: no cover
|
||||
if TYPE_CHECKING:
|
||||
@ -1,2 +0,0 @@
|
||||
[settings]
|
||||
profile = black
|
||||
@ -3,7 +3,8 @@ repos:
|
||||
rev: 1.7.9
|
||||
hooks:
|
||||
- id: bandit
|
||||
args: [-r, -c, .bandit.yml]
|
||||
args: ["-c", "pyproject.toml"]
|
||||
additional_dependencies: ["bandit[toml]"]
|
||||
- repo: https://github.com/PyCQA/flake8
|
||||
rev: 7.1.0
|
||||
hooks:
|
||||
|
||||
@ -10,7 +10,6 @@ include scrapy/py.typed
|
||||
|
||||
include codecov.yml
|
||||
include conftest.py
|
||||
include pytest.ini
|
||||
include tox.ini
|
||||
|
||||
recursive-include scrapy/templates *
|
||||
|
||||
73
pylintrc
73
pylintrc
@ -1,73 +0,0 @@
|
||||
[MASTER]
|
||||
persistent=no
|
||||
jobs=1 # >1 hides results
|
||||
|
||||
[MESSAGES CONTROL]
|
||||
disable=abstract-method,
|
||||
arguments-differ,
|
||||
arguments-renamed,
|
||||
attribute-defined-outside-init,
|
||||
bad-classmethod-argument,
|
||||
bare-except,
|
||||
broad-except,
|
||||
broad-exception-raised,
|
||||
c-extension-no-member,
|
||||
consider-using-with,
|
||||
cyclic-import,
|
||||
dangerous-default-value,
|
||||
disallowed-name,
|
||||
duplicate-code, # https://github.com/PyCQA/pylint/issues/214
|
||||
eval-used,
|
||||
fixme,
|
||||
function-redefined,
|
||||
global-statement,
|
||||
implicit-str-concat,
|
||||
import-error,
|
||||
import-outside-toplevel,
|
||||
inherit-non-class,
|
||||
invalid-name,
|
||||
invalid-overridden-method,
|
||||
isinstance-second-argument-not-valid-type,
|
||||
keyword-arg-before-vararg,
|
||||
line-too-long,
|
||||
logging-format-interpolation,
|
||||
logging-fstring-interpolation,
|
||||
logging-not-lazy,
|
||||
lost-exception,
|
||||
missing-docstring,
|
||||
no-member,
|
||||
no-method-argument,
|
||||
no-name-in-module,
|
||||
no-self-argument,
|
||||
no-value-for-parameter, # https://github.com/pylint-dev/pylint/issues/3268
|
||||
not-callable,
|
||||
pointless-statement,
|
||||
pointless-string-statement,
|
||||
protected-access,
|
||||
raise-missing-from,
|
||||
redefined-builtin,
|
||||
redefined-outer-name,
|
||||
reimported,
|
||||
signature-differs,
|
||||
too-few-public-methods,
|
||||
too-many-ancestors,
|
||||
too-many-arguments,
|
||||
too-many-branches,
|
||||
too-many-format-args,
|
||||
too-many-function-args,
|
||||
too-many-instance-attributes,
|
||||
too-many-lines,
|
||||
too-many-locals,
|
||||
too-many-public-methods,
|
||||
too-many-return-statements,
|
||||
unbalanced-tuple-unpacking,
|
||||
unnecessary-dunder-call,
|
||||
unnecessary-pass,
|
||||
unreachable,
|
||||
unused-argument,
|
||||
unused-import,
|
||||
unused-variable,
|
||||
used-before-assignment,
|
||||
useless-return,
|
||||
wildcard-import,
|
||||
wrong-import-position
|
||||
235
pyproject.toml
Normal file
235
pyproject.toml
Normal file
@ -0,0 +1,235 @@
|
||||
[build-system]
|
||||
requires = ["setuptools >= 61.0"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "Scrapy"
|
||||
dynamic = ["version"]
|
||||
description = "A high-level Web Crawling and Web Scraping framework"
|
||||
dependencies = [
|
||||
"Twisted>=21.7.0",
|
||||
"cryptography>=37.0.0",
|
||||
"cssselect>=0.9.1",
|
||||
"itemloaders>=1.0.1",
|
||||
"parsel>=1.5.0",
|
||||
"pyOpenSSL>=22.0.0",
|
||||
"queuelib>=1.4.2",
|
||||
"service_identity>=18.1.0",
|
||||
"w3lib>=1.17.0",
|
||||
"zope.interface>=5.1.0",
|
||||
"protego>=0.1.15",
|
||||
"itemadapter>=0.1.0",
|
||||
"packaging",
|
||||
"tldextract",
|
||||
"lxml>=4.6.0",
|
||||
"defusedxml>=0.7.1",
|
||||
# Platform-specific dependencies
|
||||
'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"',
|
||||
'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"',
|
||||
]
|
||||
classifiers = [
|
||||
"Framework :: Scrapy",
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
"Environment :: Console",
|
||||
"Intended Audience :: Developers",
|
||||
"License :: OSI Approved :: BSD License",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: Implementation :: CPython",
|
||||
"Programming Language :: Python :: Implementation :: PyPy",
|
||||
"Topic :: Internet :: WWW/HTTP",
|
||||
"Topic :: Software Development :: Libraries :: Application Frameworks",
|
||||
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||
]
|
||||
readme = "README.rst"
|
||||
requires-python = ">=3.9"
|
||||
authors = [{ name = "Scrapy developers", email = "pablo@pablohoffman.com" }]
|
||||
maintainers = [{ name = "Pablo Hoffman", email = "pablo@pablohoffman.com" }]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://scrapy.org/"
|
||||
Documentation = "https://docs.scrapy.org/"
|
||||
Source = "https://github.com/scrapy/scrapy"
|
||||
Tracker = "https://github.com/scrapy/scrapy/issues"
|
||||
Changelog = "https://github.com/scrapy/scrapy/commits/master/"
|
||||
releasenotes = "https://docs.scrapy.org/en/latest/news.html"
|
||||
|
||||
[project.scripts]
|
||||
scrapy = "scrapy.cmdline:execute"
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
include = ["scrapy", "scrapy.*",]
|
||||
|
||||
[tool.setuptools.dynamic]
|
||||
version = {file = "./scrapy/VERSION"}
|
||||
|
||||
[tool.mypy]
|
||||
ignore_missing_imports = true
|
||||
|
||||
# Interface classes are hard to support
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "twisted.internet.interfaces"
|
||||
follow_imports = "skip"
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "scrapy.interfaces"
|
||||
ignore_errors = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "twisted.internet.reactor"
|
||||
follow_imports = "skip"
|
||||
|
||||
# FIXME: remove the following section once the issues are solved
|
||||
[[tool.mypy.overrides]]
|
||||
module = "scrapy.settings.default_settings"
|
||||
ignore_errors = true
|
||||
|
||||
[tool.bandit]
|
||||
skips = [
|
||||
"B101", # assert_used, needed for mypy
|
||||
"B321", # ftplib, https://github.com/scrapy/scrapy/issues/4180
|
||||
"B402", # import_ftplib, https://github.com/scrapy/scrapy/issues/4180
|
||||
"B411", # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082
|
||||
"B503", # ssl_with_bad_defaults
|
||||
]
|
||||
exclude_dirs = ["tests"]
|
||||
|
||||
[tool.bumpversion]
|
||||
current_version = "2.12.0"
|
||||
commit = true
|
||||
tag = true
|
||||
tag_name = "{new_version}"
|
||||
|
||||
[[tool.bumpversion.files]]
|
||||
filename = "scrapy/VERSION"
|
||||
|
||||
[[tool.bumpversion.files]]
|
||||
filename = "SECURITY.md"
|
||||
parse = """(?P<major>0|[1-9]\\d*)\\.(?P<minor>0|[1-9]\\d*)"""
|
||||
serialize = ["{major}.{minor}"]
|
||||
|
||||
[tool.coverage.run]
|
||||
branch = true
|
||||
include = ["scrapy/*"]
|
||||
omit = ["tests/*"]
|
||||
disable_warnings = ["include-ignored"]
|
||||
|
||||
[tool.coverage.report]
|
||||
# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185
|
||||
exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"]
|
||||
|
||||
[tool.isort]
|
||||
profile = "black"
|
||||
|
||||
[tool.pylint.MASTER]
|
||||
persistent = "no"
|
||||
jobs = 1 # >1 hides results
|
||||
|
||||
[tool.pylint."MESSAGES CONTROL"]
|
||||
disable = [
|
||||
"abstract-method",
|
||||
"arguments-differ",
|
||||
"arguments-renamed",
|
||||
"attribute-defined-outside-init",
|
||||
"bad-classmethod-argument",
|
||||
"bare-except",
|
||||
"broad-except",
|
||||
"broad-exception-raised",
|
||||
"c-extension-no-member",
|
||||
"consider-using-with",
|
||||
"cyclic-import",
|
||||
"dangerous-default-value",
|
||||
"disallowed-name",
|
||||
"duplicate-code", # https://github.com/PyCQA/pylint/issues/214
|
||||
"eval-used",
|
||||
"fixme",
|
||||
"function-redefined",
|
||||
"global-statement",
|
||||
"implicit-str-concat",
|
||||
"import-error",
|
||||
"import-outside-toplevel",
|
||||
"inherit-non-class",
|
||||
"invalid-name",
|
||||
"invalid-overridden-method",
|
||||
"isinstance-second-argument-not-valid-type",
|
||||
"keyword-arg-before-vararg",
|
||||
"line-too-long",
|
||||
"logging-format-interpolation",
|
||||
"logging-fstring-interpolation",
|
||||
"logging-not-lazy",
|
||||
"lost-exception",
|
||||
"missing-docstring",
|
||||
"no-member",
|
||||
"no-method-argument",
|
||||
"no-name-in-module",
|
||||
"no-self-argument",
|
||||
"no-value-for-parameter", # https://github.com/pylint-dev/pylint/issues/3268
|
||||
"not-callable",
|
||||
"pointless-statement",
|
||||
"pointless-string-statement",
|
||||
"protected-access",
|
||||
"raise-missing-from",
|
||||
"redefined-builtin",
|
||||
"redefined-outer-name",
|
||||
"reimported",
|
||||
"signature-differs",
|
||||
"too-few-public-methods",
|
||||
"too-many-ancestors",
|
||||
"too-many-arguments",
|
||||
"too-many-branches",
|
||||
"too-many-format-args",
|
||||
"too-many-function-args",
|
||||
"too-many-instance-attributes",
|
||||
"too-many-lines",
|
||||
"too-many-locals",
|
||||
"too-many-public-methods",
|
||||
"too-many-return-statements",
|
||||
"unbalanced-tuple-unpacking",
|
||||
"unnecessary-dunder-call",
|
||||
"unnecessary-pass",
|
||||
"unreachable",
|
||||
"unused-argument",
|
||||
"unused-import",
|
||||
"unused-variable",
|
||||
"used-before-assignment",
|
||||
"useless-return",
|
||||
"wildcard-import",
|
||||
"wrong-import-position",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
xfail_strict = true
|
||||
usefixtures = "chdir"
|
||||
python_files = ["test_*.py", "__init__.py"]
|
||||
python_classes = []
|
||||
addopts = [
|
||||
"--assert=plain",
|
||||
"--ignore=docs/_ext",
|
||||
"--ignore=docs/conf.py",
|
||||
"--ignore=docs/news.rst",
|
||||
"--ignore=docs/topics/dynamic-content.rst",
|
||||
"--ignore=docs/topics/items.rst",
|
||||
"--ignore=docs/topics/leaks.rst",
|
||||
"--ignore=docs/topics/loaders.rst",
|
||||
"--ignore=docs/topics/selectors.rst",
|
||||
"--ignore=docs/topics/shell.rst",
|
||||
"--ignore=docs/topics/stats.rst",
|
||||
"--ignore=docs/topics/telnetconsole.rst",
|
||||
"--ignore=docs/utils",
|
||||
]
|
||||
markers = [
|
||||
"only_asyncio: marks tests as only enabled when --reactor=asyncio is passed",
|
||||
"only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed",
|
||||
"requires_uvloop: marks tests as only enabled when uvloop is known to be working",
|
||||
"requires_botocore: marks tests that need botocore (but not boto3)",
|
||||
"requires_boto3: marks tests that need botocore and boto3",
|
||||
]
|
||||
filterwarnings = []
|
||||
26
pytest.ini
26
pytest.ini
@ -1,26 +0,0 @@
|
||||
[pytest]
|
||||
xfail_strict = true
|
||||
usefixtures = chdir
|
||||
python_files=test_*.py __init__.py
|
||||
python_classes=
|
||||
addopts =
|
||||
--assert=plain
|
||||
--ignore=docs/_ext
|
||||
--ignore=docs/conf.py
|
||||
--ignore=docs/news.rst
|
||||
--ignore=docs/topics/dynamic-content.rst
|
||||
--ignore=docs/topics/items.rst
|
||||
--ignore=docs/topics/leaks.rst
|
||||
--ignore=docs/topics/loaders.rst
|
||||
--ignore=docs/topics/selectors.rst
|
||||
--ignore=docs/topics/shell.rst
|
||||
--ignore=docs/topics/stats.rst
|
||||
--ignore=docs/topics/telnetconsole.rst
|
||||
--ignore=docs/utils
|
||||
markers =
|
||||
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
|
||||
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
|
||||
requires_uvloop: marks tests as only enabled when uvloop is known to be working
|
||||
requires_botocore: marks tests that need botocore (but not boto3)
|
||||
requires_boto3: marks tests that need botocore and boto3
|
||||
filterwarnings =
|
||||
24
setup.cfg
24
setup.cfg
@ -1,24 +0,0 @@
|
||||
[bdist_rpm]
|
||||
doc_files = docs AUTHORS INSTALL LICENSE README.rst
|
||||
|
||||
[bdist_wheel]
|
||||
universal=1
|
||||
|
||||
[mypy]
|
||||
ignore_missing_imports = true
|
||||
|
||||
# Interface classes are hard to support
|
||||
|
||||
[mypy-twisted.internet.interfaces]
|
||||
follow_imports = skip
|
||||
|
||||
[mypy-scrapy.interfaces]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-twisted.internet.reactor]
|
||||
follow_imports = skip
|
||||
|
||||
# FIXME: remove the following sections once the issues are solved
|
||||
|
||||
[mypy-scrapy.settings.default_settings]
|
||||
ignore_errors = True
|
||||
75
setup.py
75
setup.py
@ -1,75 +0,0 @@
|
||||
from pathlib import Path
|
||||
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip()
|
||||
|
||||
|
||||
install_requires = [
|
||||
"Twisted>=21.7.0",
|
||||
"cryptography>=37.0.0",
|
||||
"cssselect>=0.9.1",
|
||||
"itemloaders>=1.0.1",
|
||||
"parsel>=1.5.0",
|
||||
"pyOpenSSL>=22.0.0",
|
||||
"queuelib>=1.4.2",
|
||||
"service_identity>=18.1.0",
|
||||
"w3lib>=1.17.0",
|
||||
"zope.interface>=5.1.0",
|
||||
"protego>=0.1.15",
|
||||
"itemadapter>=0.1.0",
|
||||
"packaging",
|
||||
"tldextract",
|
||||
"lxml>=4.6.0",
|
||||
"defusedxml>=0.7.1",
|
||||
]
|
||||
extras_require = {
|
||||
':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"],
|
||||
':platform_python_implementation == "PyPy"': ["PyPyDispatcher>=2.1.0"],
|
||||
}
|
||||
|
||||
|
||||
setup(
|
||||
name="Scrapy",
|
||||
version=version,
|
||||
url="https://scrapy.org",
|
||||
project_urls={
|
||||
"Documentation": "https://docs.scrapy.org/",
|
||||
"Source": "https://github.com/scrapy/scrapy",
|
||||
"Tracker": "https://github.com/scrapy/scrapy/issues",
|
||||
},
|
||||
description="A high-level Web Crawling and Web Scraping framework",
|
||||
long_description=open("README.rst", encoding="utf-8").read(),
|
||||
author="Scrapy developers",
|
||||
author_email="pablo@pablohoffman.com",
|
||||
maintainer="Pablo Hoffman",
|
||||
maintainer_email="pablo@pablohoffman.com",
|
||||
license="BSD",
|
||||
packages=find_packages(exclude=("tests", "tests.*")),
|
||||
include_package_data=True,
|
||||
zip_safe=False,
|
||||
entry_points={"console_scripts": ["scrapy = scrapy.cmdline:execute"]},
|
||||
classifiers=[
|
||||
"Framework :: Scrapy",
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
"Environment :: Console",
|
||||
"Intended Audience :: Developers",
|
||||
"License :: OSI Approved :: BSD License",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: Implementation :: CPython",
|
||||
"Programming Language :: Python :: Implementation :: PyPy",
|
||||
"Topic :: Internet :: WWW/HTTP",
|
||||
"Topic :: Software Development :: Libraries :: Application Frameworks",
|
||||
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||
],
|
||||
python_requires=">=3.9",
|
||||
install_requires=install_requires,
|
||||
extras_require=extras_require,
|
||||
)
|
||||
@ -899,7 +899,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
||||
p.expect_exact("shutting down gracefully")
|
||||
# sending the second signal too fast often causes problems
|
||||
d = Deferred()
|
||||
reactor.callLater(0.1, d.callback, None)
|
||||
reactor.callLater(0.01, d.callback, None)
|
||||
yield d
|
||||
p.kill(sig)
|
||||
p.expect_exact("forcing unclean shutdown")
|
||||
|
||||
@ -144,9 +144,10 @@ class SpiderLoaderTest(unittest.TestCase):
|
||||
self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings)
|
||||
|
||||
def test_syntax_error_warning(self):
|
||||
with warnings.catch_warnings(record=True) as w, mock.patch.object(
|
||||
SpiderLoader, "_load_spiders"
|
||||
) as m:
|
||||
with (
|
||||
warnings.catch_warnings(record=True) as w,
|
||||
mock.patch.object(SpiderLoader, "_load_spiders") as m,
|
||||
):
|
||||
m.side_effect = SyntaxError
|
||||
module = "tests.test_spiderloader.test_spiders.spider1"
|
||||
settings = Settings(
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user