from datetime import datetime
from twisted.web import resource, static
from twisted.application.service import IServiceCollection
from .interfaces import IPoller, IEggStorage, ISpiderScheduler
from . import webservice
class Root(resource.Resource):
def __init__(self, config, app):
resource.Resource.__init__(self)
self.debug = config.getboolean('debug', False)
self.runner = config.get('runner')
logsdir = config.get('logs_dir')
self.app = app
self.putChild('', Home(self))
self.putChild('schedule.json', webservice.Schedule(self))
self.putChild('addversion.json', webservice.AddVersion(self))
self.putChild('listprojects.json', webservice.ListProjects(self))
self.putChild('listversions.json', webservice.ListVersions(self))
self.putChild('listspiders.json', webservice.ListSpiders(self))
self.putChild('delproject.json', webservice.DeleteProject(self))
self.putChild('delversion.json', webservice.DeleteVersion(self))
self.putChild('listjobs.json', webservice.ListJobs(self))
self.putChild('logs', static.File(logsdir, 'text/plain'))
self.putChild('procmon', ProcessMonitor(self))
self.update_projects()
def update_projects(self):
self.poller.update_projects()
self.scheduler.update_projects()
@property
def launcher(self):
app = IServiceCollection(self.app, self.app)
return app.getServiceNamed('launcher')
@property
def scheduler(self):
return self.app.getComponent(ISpiderScheduler)
@property
def eggstorage(self):
return self.app.getComponent(IEggStorage)
@property
def poller(self):
return self.app.getComponent(IPoller)
class Home(resource.Resource):
def __init__(self, root):
resource.Resource.__init__(self)
self.root = root
def render_GET(self, txrequest):
vars = {
'projects': ', '.join(self.root.scheduler.list_projects()),
}
return """
Scrapyd
Scrapyd
Available projects: %(projects)s
How to schedule a spider?
To schedule a spider you need to use the API (this web UI is only for
monitoring)
Example using curl:
curl http://localhost:6800/schedule.json -d project=default -d spider=somespider
For more information about the API, see the Scrapyd documentation
""" % vars
class ProcessMonitor(resource.Resource):
def __init__(self, root):
resource.Resource.__init__(self)
self.root = root
def render(self, txrequest):
s = "Scrapyd"
s += ""
s += "Process monitor
"
s += "Go back
"
s += ""
s += ""
s += "| Project | Spider | Job | PID | Runtime | Log | "
s += "
"
for p in self.root.launcher.processes.values():
s += ""
for a in ['project', 'spider', 'job', 'pid']:
s += "| %s | " % getattr(p, a)
s += "%s | " % (datetime.now() - p.start_time)
s += "Log | " % (p.project, p.spider, p.job)
s += "
"
s += "
"
s += ""
s += ""
return s