diff --git a/debian/scrapyd.cfg b/debian/000-default similarity index 100% rename from debian/scrapyd.cfg rename to debian/000-default diff --git a/debian/scrapyd.install b/debian/scrapyd.install index 29c82985e..8e0042970 100644 --- a/debian/scrapyd.install +++ b/debian/scrapyd.install @@ -1,3 +1,3 @@ usr/lib/python*/*-packages/scrapyd -debian/scrapyd.cfg etc +debian/000-default etc/scrapyd/conf.d extras/scrapyd.tac usr/share/scrapyd diff --git a/docs/topics/scrapyd.rst b/docs/topics/scrapyd.rst index 6d874e0ad..d66694879 100644 --- a/docs/topics/scrapyd.rst +++ b/docs/topics/scrapyd.rst @@ -85,10 +85,10 @@ This will install Scrapyd in your Ubuntu server creating a ``scrapy`` user which Scrapyd will run as. It will also create some directories and files that are listed below: -/etc/scrapyd.cfg -~~~~~~~~~~~~~~~~ +/etc/scrapyd +~~~~~~~~~~~~ -Scrapyd configuration file. See :ref:`topics-scrapyd-config`. +Scrapyd configuration files. See :ref:`topics-scrapyd-config`. /var/log/scrapyd/scrapyd.log ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -124,7 +124,16 @@ Directory used to store data files (uploaded eggs and spider queues). Scrapyd Configuration file ========================== -The Scrapyd configuration file supports the following options: +Scrapyd searches for configuration files in the following locations, and parses +them in order with the latest ones taking more priority: + +* ``/etc/scrapyd/scrapyd.conf`` (Unix) +* ``c:\scrapyd\scrapyd.conf`` (Windows) +* ``/etc/scrapyd/conf.d/*`` (in alphabetical order, Unix) +* ``scrapyd.conf`` + +The configuration file supports the following options (see default values in +the :ref:`example `). http_port --------- @@ -160,12 +169,29 @@ logs_dir The directory where the Scrapy processes logs (``slotN.log``) will be stored. +egg_runner +---------- + +The module that will be used for launching sub-processes. You can customize the +Scrapy processes launched from Scrapyd by using your own module. + +application +----------- + +A function that returns the (Twisted) Application object to use. This can be +used if you want to extend Scrapyd by adding and removing your own components +and services. + +For more info see `Twisted Application Framework`_ + +.. _topics-scrapyd-config-example: + Example configuration file -------------------------- Here is an example configuration file with all the defaults: -.. literalinclude:: ../../scrapyd/default_scrapyd.cfg +.. literalinclude:: ../../scrapyd/default_scrapyd.conf Eggifying your project ====================== diff --git a/extras/scrapyd.tac b/extras/scrapyd.tac index bdf548d31..6abce3086 100644 --- a/extras/scrapyd.tac +++ b/extras/scrapyd.tac @@ -1,2 +1,2 @@ -from scrapyd.app import get_application +from scrapyd import get_application application = get_application() diff --git a/scrapyd/__init__.py b/scrapyd/__init__.py index e69de29bb..020997f5e 100644 --- a/scrapyd/__init__.py +++ b/scrapyd/__init__.py @@ -0,0 +1,8 @@ +from scrapy.utils.misc import load_object +from .config import Config + +def get_application(): + config = Config() + apppath = config.get('application', 'scrapyd.app.application') + appfunc = load_object(apppath) + return appfunc() diff --git a/scrapyd/app.py b/scrapyd/app.py index abddfadc1..466b4d44f 100644 --- a/scrapyd/app.py +++ b/scrapyd/app.py @@ -1,7 +1,4 @@ -import sys, os - -from twisted.internet import reactor -from twisted.application.service import Application, Service +from twisted.application.service import Application from twisted.application.internet import TimerService, TCPServer from twisted.web import server @@ -14,8 +11,8 @@ from .environ import Environment from .webservice import Root from .config import Config -def get_application(): - app = Application("Scrapy") +def application(): + app = Application("Scrapyd") config = Config() http_port = config.getint('http_port', 6800) diff --git a/scrapyd/config.py b/scrapyd/config.py index de0b62d19..d91efdad1 100644 --- a/scrapyd/config.py +++ b/scrapyd/config.py @@ -1,23 +1,27 @@ +import glob import pkgutil from cStringIO import StringIO from ConfigParser import SafeConfigParser, NoSectionError, NoOptionError -from scrapy.utils.conf import get_sources - class Config(object): """A ConfigParser wrapper to support defaults when calling instance methods, and also tied to a single section""" - SOURCES = ['scrapyd.cfg', '/etc/scrapyd.cfg'] SECTION = 'scrapyd' def __init__(self): - sources = self.SOURCES + get_sources() - default_config = pkgutil.get_data(__package__, 'default_scrapyd.cfg') + sources = self._getsources() + default_config = pkgutil.get_data(__package__, 'default_scrapyd.conf') self.cp = SafeConfigParser() self.cp.readfp(StringIO(default_config)) self.cp.read(sources) + def _getsources(self): + sources = ['/etc/scrapyd/scrapyd.conf', r'c:\scrapyd\scrapyd.conf'] + sources += sorted(glob.glob('/etc/scrapyd/conf.d/*')) + sources += ['scrapyd.conf'] + return sources + def _getany(self, method, option, default): try: return method(self.SECTION, option) diff --git a/scrapyd/default_scrapyd.cfg b/scrapyd/default_scrapyd.cfg deleted file mode 100644 index ba34a7ab7..000000000 --- a/scrapyd/default_scrapyd.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[scrapyd] -eggs_dir = eggs -logs_dir = logs -dbs_dir = dbs -max_proc = 0 -http_port = 6800 -debug = off -egg_runner = scrapyd.eggrunner diff --git a/scrapyd/default_scrapyd.conf b/scrapyd/default_scrapyd.conf new file mode 100644 index 000000000..b0f046a77 --- /dev/null +++ b/scrapyd/default_scrapyd.conf @@ -0,0 +1,9 @@ +[scrapyd] +eggs_dir = eggs +logs_dir = logs +dbs_dir = dbs +max_proc = 0 +http_port = 6800 +debug = off +egg_runner = scrapyd.eggrunner +application = scrapyd.app.application diff --git a/scrapyd/launcher.py b/scrapyd/launcher.py index a2fda7d16..9afcad16f 100644 --- a/scrapyd/launcher.py +++ b/scrapyd/launcher.py @@ -19,8 +19,8 @@ class Launcher(Service): def startService(self): for slot in range(self.max_proc): self._wait_for_project(slot) - log.msg("Launcher started: max_proc=%r, egg_runner=%r" % \ - (self.max_proc, self.egg_runner), system="Launcher") + log.msg("%s started: max_proc=%r, egg_runner=%r" % (self.parent.name, \ + self.max_proc, self.egg_runner), system="Launcher") def _wait_for_project(self, slot): poller = self.app.getComponent(IPoller)