mirror of
https://github.com/scrapy/scrapy.git
synced 2025-02-26 18:04:27 +00:00
added "Global Options" group to command line options, improved help display, splitted --profile option in --profile + --lsprof
This commit is contained in:
Pablo Hoffman
parent
1339b18218
commit
6b20f1f7d9
@ -8,6 +8,7 @@ import cProfile
|
||||
import scrapy
|
||||
from scrapy import log
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.xlib import lsprofcalltree
|
||||
from scrapy.conf import settings, SETTINGS_MODULE
|
||||
|
||||
def find_commands(dir):
|
||||
@ -48,10 +49,15 @@ def getcmdname(argv):
|
||||
if not arg.startswith('-'):
|
||||
return arg
|
||||
|
||||
def usage(argv):
|
||||
s = "usage: %s <command> [options] [args]\n" % argv[0]
|
||||
s += " %s <command> -h\n\n" % argv[0]
|
||||
s += "Available commands:\n\n"
|
||||
def usage(prog):
|
||||
s = "Usage\n"
|
||||
s += "=====\n"
|
||||
s += "%s <command> [options] [args]\n" % prog
|
||||
s += " Run a command\n\n"
|
||||
s += "%s <command> -h\n" % prog
|
||||
s += " Print command help and options\n\n"
|
||||
s += "Available commands\n"
|
||||
s += "===================\n"
|
||||
|
||||
cmds = builtin_commands_dict()
|
||||
cmds.update(custom_commands_dict())
|
||||
@ -103,10 +109,11 @@ def execute_with_args(argv):
|
||||
|
||||
if not cmdname:
|
||||
print "Scrapy %s\n" % scrapy.__version__
|
||||
print usage(argv)
|
||||
print usage(argv[0])
|
||||
sys.exit(2)
|
||||
|
||||
parser = optparse.OptionParser(conflict_handler='resolve')
|
||||
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
|
||||
conflict_handler='resolve', add_help_option=False)
|
||||
|
||||
if cmdname in cmds:
|
||||
cmd = cmds[cmdname]
|
||||
@ -122,6 +129,10 @@ def execute_with_args(argv):
|
||||
(opts, args) = parser.parse_args(args=argv[1:])
|
||||
del args[0] # args[0] is cmdname
|
||||
|
||||
if opts.help:
|
||||
parser.print_help()
|
||||
sys.exit()
|
||||
|
||||
# storing command executed info for later reference
|
||||
command_executed['name'] = cmdname
|
||||
command_executed['class'] = cmd
|
||||
@ -131,20 +142,20 @@ def execute_with_args(argv):
|
||||
cmd.process_options(args, opts)
|
||||
spiders.load()
|
||||
log.start()
|
||||
if opts.profile:
|
||||
log.msg("Profiling enabled. Analyze later with: python -m pstats %s" % opts.profile)
|
||||
if opts.profile or opts.lsprof:
|
||||
if opts.profile:
|
||||
log.msg("writing cProfile stats to %r" % opts.profile)
|
||||
if opts.lsprof:
|
||||
log.msg("writing lsprof stats to %r" % opts.lsprof)
|
||||
loc = locals()
|
||||
p = cProfile.Profile()
|
||||
p.runctx('ret = cmd.run(args, opts)', globals(), loc)
|
||||
p.dump_stats(opts.profile)
|
||||
try:
|
||||
from scrapy.xlib import lsprofcalltree
|
||||
fn = opts.profile + ".cachegrind"
|
||||
k = lsprofcalltree.KCacheGrind(p)
|
||||
with open(fn, 'w') as f:
|
||||
if opts.profile:
|
||||
p.dump_stats(opts.profile)
|
||||
k = lsprofcalltree.KCacheGrind(p)
|
||||
if opts.lsprof:
|
||||
with open(opts.lsprof, 'w') as f:
|
||||
k.output(f)
|
||||
except ImportError:
|
||||
pass
|
||||
ret = loc['ret']
|
||||
else:
|
||||
ret = cmd.run(args, opts)
|
||||
|
||||
@ -6,6 +6,8 @@ from __future__ import with_statement
|
||||
|
||||
import os
|
||||
import sys
|
||||
from optparse import OptionGroup
|
||||
|
||||
from scrapy.conf import settings
|
||||
|
||||
class ScrapyCommand(object):
|
||||
@ -22,16 +24,14 @@ class ScrapyCommand(object):
|
||||
return ""
|
||||
|
||||
def long_desc(self):
|
||||
"""
|
||||
A long description of the command. Return short description when not
|
||||
"""A long description of the command. Return short description when not
|
||||
available. It cannot contain newlines, since contents will be formatted
|
||||
by optparser which removes newlines and wraps text.
|
||||
"""
|
||||
return self.short_desc()
|
||||
|
||||
def help(self):
|
||||
"""
|
||||
An extensive help for the command. It will be shown when using the
|
||||
"""An extensive help for the command. It will be shown when using the
|
||||
"help" command. It can contain newlines, since not post-formatting will
|
||||
be applied to its contents.
|
||||
"""
|
||||
@ -41,22 +41,30 @@ class ScrapyCommand(object):
|
||||
"""
|
||||
Populate option parse with options available for this command
|
||||
"""
|
||||
parser.add_option("--logfile", dest="logfile", metavar="FILE", \
|
||||
group = OptionGroup(parser, "Global Options")
|
||||
group.add_option("-h", "--help", action="store_true", dest="help", \
|
||||
help="print command help and options")
|
||||
group.add_option("--logfile", dest="logfile", metavar="FILE", \
|
||||
help="log file. if omitted stderr will be used")
|
||||
parser.add_option("-L", "--loglevel", dest="loglevel", metavar="LEVEL", \
|
||||
group.add_option("-L", "--loglevel", dest="loglevel", metavar="LEVEL", \
|
||||
default=None, \
|
||||
help="log level. use SILENT level to diasble all log messages")
|
||||
parser.add_option("--default-spider", dest="default_spider", default=None, \
|
||||
help="log level (default: %s)" % settings['LOGLEVEL'])
|
||||
group.add_option("--nolog", action="store_true", dest="nolog", \
|
||||
help="disable logging completely")
|
||||
group.add_option("--default-spider", dest="default_spider", default=None, \
|
||||
help="use this spider when arguments are urls and no spider is found")
|
||||
parser.add_option("--spider", dest="spider", default=None, \
|
||||
group.add_option("--spider", dest="spider", default=None, \
|
||||
help="always use this spider when arguments are urls")
|
||||
parser.add_option("--profile", dest="profile", metavar="FILE", default=None, \
|
||||
group.add_option("--profile", dest="profile", metavar="FILE", default=None, \
|
||||
help="write python cProfile stats to FILE")
|
||||
parser.add_option("--pidfile", dest="pidfile", metavar="FILE", \
|
||||
group.add_option("--lsprof", dest="lsprof", metavar="FILE", default=None, \
|
||||
help="write lsprof profiling stats to FILE")
|
||||
group.add_option("--pidfile", dest="pidfile", metavar="FILE", \
|
||||
help="write process ID to FILE")
|
||||
parser.add_option("--set", dest="settings", action="append", \
|
||||
group.add_option("--set", dest="settings", action="append", \
|
||||
metavar="SETTING=VALUE", default=[], \
|
||||
help="set/override setting (may be repeated)")
|
||||
parser.add_option_group(group)
|
||||
|
||||
def process_options(self, args, opts):
|
||||
if opts.logfile:
|
||||
@ -67,6 +75,9 @@ class ScrapyCommand(object):
|
||||
settings.overrides['LOG_ENABLED'] = True
|
||||
settings.overrides['LOGLEVEL'] = opts.loglevel
|
||||
|
||||
if opts.nolog:
|
||||
settings.overrides['LOG_ENABLED'] = False
|
||||
|
||||
if opts.default_spider:
|
||||
from scrapy.spider import spiders
|
||||
spiders.default_domain = opts.default_spider
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user