BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-22 17:04:00 +00:00
Code Releases Wiki Activity

scrapyd: added support for deferred spider queues

Browse Source
This commit is contained in:
Pablo Hoffman 2010-12-10 15:55:40 -02:00
parent 02ccca01eb
commit b19ff21acd
1 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scrapyd/poller.py
View File

@ -1,5 +1,5 @@
from zope.interface import implements
from twisted.internet.defer import DeferredQueue
from twisted.internet.defer import DeferredQueue, inlineCallbacks, maybeDeferred, returnValue
from .utils import get_spider_queues
from .interfaces import IPoller
@ -13,13 +13,15 @@ class QueuePoller(object):
self.update_projects()
self.dq = DeferredQueue(size=1)
@inlineCallbacks
def poll(self):
if self.dq.pending:
return
for p, q in self.queues.iteritems():
if q.count():
msg = q.pop()
return self.dq.put(self._message(msg, p))
c = yield maybeDeferred(q.count)
if c:
msg = yield maybeDeferred(q.pop)
returnValue(self.dq.put(self._message(msg, p)))
def next(self):
return self.dq.get()