BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-22 00:33:02 +00:00
Code Releases Wiki Activity

cPickle as pickle

Browse Source 
--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%4089
This commit is contained in:
samus_ 2008-07-22 16:44:30 +00:00
parent 30202c54a4
commit e8b5a07a15
2 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py
View File

@ -4,7 +4,7 @@ import os
import hashlib
import datetime
import urlparse
import cPickle
import cPickle as pickle
from pydispatch import dispatcher
from twisted.internet import defer
@ -114,7 +114,7 @@ class Cache(object):
pickled_meta = os.path.join(self.requestpath(domain, key), 'pickled_meta')
if os.path.exists(pickled_meta):
with open(pickled_meta, 'rb') as f:
metadata = cPickle.load(f)
metadata = pickle.load(f)
if datetime.datetime.utcnow() <= metadata['timestamp'] + datetime.timedelta(seconds=settings.getint('CACHE2_EXPIRATION_SECS')):
return True
else:
@ -134,7 +134,7 @@ class Cache(object):
requestpath = self.requestpath(domain, key)
metadata = responsebody = responseheaders = None
with open(os.path.join(requestpath, 'pickled_meta'), 'rb') as f:
metadata = cPickle.load(f)
metadata = pickle.load(f)
with open(os.path.join(requestpath, 'response_body')) as f:
responsebody = f.read()
with open(os.path.join(requestpath, 'response_headers')) as f:
@ -170,7 +170,7 @@ class Cache(object):
f.write(repr(metadata))
# pickled metadata (to recover without using eval)
with open(os.path.join(requestpath, 'pickled_meta'), 'wb') as f:
cPickle.dump(metadata, f, -1)
pickle.dump(metadata, f, -1)
# response
with open(os.path.join(requestpath, 'response_headers'), 'w') as f:
f.write(headers_dict_to_raw(response.headers))

5
scrapy/trunk/scrapy/xpath/selector.py
View File

@ -86,6 +86,11 @@ class XPathSelector(object):
class XPathSelectorList(list):
"""List of XPathSelector objects"""
def __getitem__(self, i):
print '__getitem__'
self.__getitem__(i)
#return XPathSelectorList(list.__getslice__(self, i, j))
def x(self, xpath):
"""Perform the given XPath query on each XPathSelector of the list and
return a new (flattened) XPathSelectorList of the results"""