mirror of
https://github.com/scrapy/scrapy.git
synced 2025-02-24 08:24:21 +00:00
40 lines
1.1 KiB
Python
40 lines
1.1 KiB
Python
"""
|
|
Scheduler queues
|
|
"""
|
|
|
|
import marshal, cPickle as pickle
|
|
|
|
from queuelib import queue
|
|
|
|
def _serializable_queue(queue_class, serialize, deserialize):
|
|
|
|
class SerializableQueue(queue_class):
|
|
|
|
def push(self, obj):
|
|
s = serialize(obj)
|
|
super(SerializableQueue, self).push(s)
|
|
|
|
def pop(self):
|
|
s = super(SerializableQueue, self).pop()
|
|
if s:
|
|
return deserialize(s)
|
|
|
|
return SerializableQueue
|
|
|
|
def _pickle_serialize(obj):
|
|
try:
|
|
return pickle.dumps(obj, protocol=2)
|
|
except pickle.PicklingError as e:
|
|
raise ValueError(str(e))
|
|
|
|
PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \
|
|
_pickle_serialize, pickle.loads)
|
|
PickleLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, \
|
|
_pickle_serialize, pickle.loads)
|
|
MarshalFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \
|
|
marshal.dumps, marshal.loads)
|
|
MarshalLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, \
|
|
marshal.dumps, marshal.loads)
|
|
FifoMemoryQueue = queue.FifoMemoryQueue
|
|
LifoMemoryQueue = queue.LifoMemoryQueue
|