mirror of
https://github.com/scrapy/scrapy.git
synced 2025-02-27 02:43:41 +00:00
123 lines
4.1 KiB
Python
123 lines
4.1 KiB
Python
import traceback
|
|
import uuid
|
|
from cStringIO import StringIO
|
|
|
|
from twisted.python import log
|
|
|
|
from scrapy.utils.txweb import JsonResource
|
|
from .utils import get_spider_list
|
|
|
|
class WsResource(JsonResource):
|
|
|
|
def __init__(self, root):
|
|
JsonResource.__init__(self)
|
|
self.root = root
|
|
|
|
def render(self, txrequest):
|
|
try:
|
|
return JsonResource.render(self, txrequest)
|
|
except Exception, e:
|
|
if self.root.debug:
|
|
return traceback.format_exc()
|
|
log.err()
|
|
r = {"status": "error", "message": str(e)}
|
|
return self.render_object(r, txrequest)
|
|
|
|
class Schedule(WsResource):
|
|
|
|
def render_POST(self, txrequest):
|
|
settings = txrequest.args.pop('setting', [])
|
|
settings = dict(x.split('=', 1) for x in settings)
|
|
args = dict((k, v[0]) for k, v in txrequest.args.items())
|
|
project = args.pop('project')
|
|
spider = args.pop('spider')
|
|
args['settings'] = settings
|
|
jobid = uuid.uuid1().hex
|
|
args['_job'] = jobid
|
|
self.root.scheduler.schedule(project, spider, **args)
|
|
return {"status": "ok", "jobid": jobid}
|
|
|
|
class Cancel(WsResource):
|
|
|
|
def render_POST(self, txrequest):
|
|
args = dict((k, v[0]) for k, v in txrequest.args.items())
|
|
project = args['project']
|
|
jobid = args['job']
|
|
signal = args.get('signal', 'TERM')
|
|
prevstate = None
|
|
queue = self.root.poller.queues[project]
|
|
c = queue.remove(lambda x: x["_job"] == jobid)
|
|
if c:
|
|
prevstate = "pending"
|
|
spiders = self.root.launcher.processes.values()
|
|
for s in spiders:
|
|
if s.job == jobid:
|
|
s.transport.signalProcess(signal)
|
|
prevstate = "running"
|
|
return {"status": "ok", "prevstate": prevstate}
|
|
|
|
class AddVersion(WsResource):
|
|
|
|
def render_POST(self, txrequest):
|
|
project = txrequest.args['project'][0]
|
|
version = txrequest.args['version'][0]
|
|
eggf = StringIO(txrequest.args['egg'][0])
|
|
self.root.eggstorage.put(eggf, project, version)
|
|
spiders = get_spider_list(project)
|
|
self.root.update_projects()
|
|
return {"status": "ok", "project": project, "version": version, \
|
|
"spiders": len(spiders)}
|
|
|
|
class ListProjects(WsResource):
|
|
|
|
def render_GET(self, txrequest):
|
|
projects = self.root.scheduler.list_projects()
|
|
return {"status": "ok", "projects": projects}
|
|
|
|
class ListVersions(WsResource):
|
|
|
|
def render_GET(self, txrequest):
|
|
project = txrequest.args['project'][0]
|
|
versions = self.root.eggstorage.list(project)
|
|
return {"status": "ok", "versions": versions}
|
|
|
|
class ListSpiders(WsResource):
|
|
|
|
def render_GET(self, txrequest):
|
|
project = txrequest.args['project'][0]
|
|
spiders = get_spider_list(project, runner=self.root.runner)
|
|
return {"status": "ok", "spiders": spiders}
|
|
|
|
class ListJobs(WsResource):
|
|
|
|
def render_GET(self, txrequest):
|
|
project = txrequest.args['project'][0]
|
|
spiders = self.root.launcher.processes.values()
|
|
running = [{"id": s.job, "spider": s.spider} for s in spiders if s.project == project]
|
|
queue = self.root.poller.queues[project]
|
|
pending = [{"id": x["_job"], "spider": x["name"]} for x in queue.list()]
|
|
finished = [{"id": s.job, "spider": s.spider,
|
|
"start_time": s.start_time.isoformat(' '),
|
|
"end_time": s.end_time.isoformat(' ')} for s in self.root.launcher.finished
|
|
if s.project == project]
|
|
return {"status":"ok", "pending": pending, "running": running, "finished": finished}
|
|
|
|
class DeleteProject(WsResource):
|
|
|
|
def render_POST(self, txrequest):
|
|
project = txrequest.args['project'][0]
|
|
self._delete_version(project)
|
|
return {"status": "ok"}
|
|
|
|
def _delete_version(self, project, version=None):
|
|
self.root.eggstorage.delete(project, version)
|
|
self.root.update_projects()
|
|
|
|
class DeleteVersion(DeleteProject):
|
|
|
|
def render_POST(self, txrequest):
|
|
project = txrequest.args['project'][0]
|
|
version = txrequest.args['version'][0]
|
|
self._delete_version(project, version)
|
|
return {"status": "ok"}
|