mirror of
https://github.com/scrapy/scrapy.git
synced 2025-02-24 08:03:59 +00:00
changed default user agent to Scrapy/0.15 (+http://scrapy.org) and removed no longer needed BOT_VERSION setting
This commit is contained in:
parent
731c569b5c
commit
27018fced7
@ -222,16 +222,6 @@ also for logging.
|
||||
It's automatically populated with your project name when you create your
|
||||
project with the :command:`startproject` command.
|
||||
|
||||
.. setting:: BOT_VERSION
|
||||
|
||||
BOT_VERSION
|
||||
-----------
|
||||
|
||||
Default: ``1.0``
|
||||
|
||||
The version of the bot implemented by this Scrapy project. This will be used to
|
||||
construct the User-Agent by default.
|
||||
|
||||
.. setting:: CONCURRENT_ITEMS
|
||||
|
||||
CONCURRENT_ITEMS
|
||||
@ -973,7 +963,7 @@ the default value for this setting see: http://www.boutell.com/newfaq/misc/urlle
|
||||
USER_AGENT
|
||||
----------
|
||||
|
||||
Default: ``"%s/%s" % (BOT_NAME, BOT_VERSION)``
|
||||
Default: ``"Scrapy/0.15 (+http://scrapy.org)"``
|
||||
|
||||
The default User-Agent to use when crawling, unless overridden.
|
||||
|
||||
|
@ -17,7 +17,6 @@ import sys, os
|
||||
from os.path import join, abspath, dirname
|
||||
|
||||
BOT_NAME = 'scrapybot'
|
||||
BOT_VERSION = '1.0'
|
||||
|
||||
CLOSESPIDER_TIMEOUT = 0
|
||||
CLOSESPIDER_PAGECOUNT = 0
|
||||
@ -226,7 +225,7 @@ TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates'))
|
||||
|
||||
URLLENGTH_LIMIT = 2083
|
||||
|
||||
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
|
||||
USER_AGENT = 'Scrapy/0.15 (+http://scrapy.org)'
|
||||
|
||||
TELNETCONSOLE_ENABLED = 1
|
||||
TELNETCONSOLE_PORT = [6023, 6073]
|
||||
|
@ -7,9 +7,9 @@
|
||||
#
|
||||
|
||||
BOT_NAME = '$project_name'
|
||||
BOT_VERSION = '1.0'
|
||||
|
||||
SPIDER_MODULES = ['$project_name.spiders']
|
||||
NEWSPIDER_MODULE = '$project_name.spiders'
|
||||
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
|
||||
|
||||
# Crawl responsibly by identifying yourself (and your website) on the user-agent
|
||||
#USER_AGENT = '$project_name (+http://www.yourdomain.com)'
|
||||
|
Loading…
x
Reference in New Issue
Block a user