mirror of
https://github.com/scrapy/scrapy.git
synced 2025-02-26 17:44:02 +00:00
bbffa59497
* Always start one process per spider * Added max_proc_per_cpu option (defaults to 4) * Return the number of spiders (instead of a list of them) in schedule.json
35 lines
863 B
Python
35 lines
863 B
Python
from zope.interface import implements
|
|
from twisted.internet.defer import DeferredQueue
|
|
|
|
from .utils import get_spider_queues
|
|
from .interfaces import IPoller
|
|
|
|
class QueuePoller(object):
|
|
|
|
implements(IPoller)
|
|
|
|
def __init__(self, config):
|
|
self.config = config
|
|
self.update_projects()
|
|
self.dq = DeferredQueue(size=1)
|
|
|
|
def poll(self):
|
|
if self.dq.pending:
|
|
return
|
|
for p, q in self.queues.iteritems():
|
|
if q.count():
|
|
msg = q.pop()
|
|
return self.dq.put(self._message(msg, p))
|
|
|
|
def next(self):
|
|
return self.dq.get()
|
|
|
|
def update_projects(self):
|
|
self.queues = get_spider_queues(self.config)
|
|
|
|
def _message(self, queue_msg, project):
|
|
d = queue_msg.copy()
|
|
d['project'] = project
|
|
d['spider'] = d.pop('name')
|
|
return d
|