From 7b84591ea9dd4bf1c66649b0a863c3d5312e1ae5 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 30 Nov 2010 15:52:15 -0200 Subject: [PATCH] added command for starting a scrapyd server for the current project --- scrapy/commands/scrapyd.py | 22 +++++++++++++++++++++ scrapyd/script.py | 40 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 scrapy/commands/scrapyd.py create mode 100644 scrapyd/script.py diff --git a/scrapy/commands/scrapyd.py b/scrapy/commands/scrapyd.py new file mode 100644 index 000000000..89ade92f7 --- /dev/null +++ b/scrapy/commands/scrapyd.py @@ -0,0 +1,22 @@ +from __future__ import absolute_import + +from scrapy.command import ScrapyCommand +from scrapy.exceptions import UsageError + +class Command(ScrapyCommand): + + requires_project = True + + def short_desc(self): + return "Start Scrapyd server for this project" + + def long_desc(self): + return "Start Scrapyd server for this project, which can be referred " \ + "from the JSON API with the name 'default'" + + def run(self, args, opts): + try: + from scrapyd.script import execute + execute() + except ImportError: + raise UsageError("Scrapyd is not available in this system") diff --git a/scrapyd/script.py b/scrapyd/script.py new file mode 100644 index 000000000..ce0b3af92 --- /dev/null +++ b/scrapyd/script.py @@ -0,0 +1,40 @@ +"""This module can be used to execute Scrapyd from a Scrapy command""" + +import sys +import os +from cStringIO import StringIO + +from twisted.python import log +from twisted.internet import reactor +from twisted.application import app + +from scrapy.utils.project import project_data_dir + +from scrapyd import get_application +from scrapyd.config import Config + +def _get_config(): + datadir = os.path.join(project_data_dir(), '.scrapy', 'scrapyd') + conf = { + 'eggs_dir': os.path.join(datadir, 'eggs'), + 'logs_dir': os.path.join(datadir, 'logs'), + 'dbs_dir': os.path.join(datadir, 'dbs'), + } + for k in ['eggs_dir', 'logs_dir', 'dbs_dir']: # create dirs + d = conf[k] + if not os.path.exists(d): + os.makedirs(d) + scrapyd_conf = """ +[scrapyd] +eggs_dir = %(eggs_dir)s +logs_dir = %(logs_dir)s +dbs_dir = %(dbs_dir)s + """ % conf + return Config(extra_sources=[StringIO(scrapyd_conf)]) + +def execute(): + config = _get_config() + log.startLogging(sys.stderr) + application = get_application(config) + app.startApplication(application, False) + reactor.run()