mirror of
https://github.com/scrapy/scrapy.git
synced 2025-02-26 19:24:09 +00:00
removed unused (Django) classes from scrapy.utils.datatypes: MergeDict, SortedDict, DotExpandedDict, FileDict. And also removed unused class gzStringIO
--HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40714
This commit is contained in:
Pablo Hoffman
parent
ff637d9a0a
commit
deb960526b
@ -11,135 +11,6 @@ import gzip
|
||||
from cStringIO import StringIO
|
||||
from heapq import heappush, heappop
|
||||
|
||||
class MergeDict(object):
|
||||
"""
|
||||
A simple class for creating new "virtual" dictionaries that actualy look
|
||||
up values in more than one dictionary, passed in the constructor.
|
||||
"""
|
||||
def __init__(self, *dicts):
|
||||
self.dicts = dicts
|
||||
|
||||
def __getitem__(self, key):
|
||||
for dict_ in self.dicts:
|
||||
try:
|
||||
return dict_[key]
|
||||
except KeyError:
|
||||
pass
|
||||
raise KeyError
|
||||
|
||||
def __copy__(self):
|
||||
return self.__class__(*self.dicts)
|
||||
|
||||
def get(self, key, default=None):
|
||||
try:
|
||||
return self[key]
|
||||
except KeyError:
|
||||
return default
|
||||
|
||||
def getlist(self, key):
|
||||
for dict in self.dicts:
|
||||
try:
|
||||
return dict.getlist(key)
|
||||
except KeyError:
|
||||
pass
|
||||
raise KeyError
|
||||
|
||||
def items(self):
|
||||
item_list = []
|
||||
for dict in self.dicts:
|
||||
item_list.extend(dict.items())
|
||||
return item_list
|
||||
|
||||
def has_key(self, key):
|
||||
for dict in self.dicts:
|
||||
if key in dict:
|
||||
return True
|
||||
return False
|
||||
|
||||
__contains__ = has_key
|
||||
|
||||
def copy(self):
|
||||
""" returns a copy of this object"""
|
||||
return self.__copy__()
|
||||
|
||||
class SortedDict(dict):
|
||||
"A dictionary that keeps its keys in the order in which they're inserted."
|
||||
def __init__(self, data=None):
|
||||
if data is None:
|
||||
data = {}
|
||||
dict.__init__(self, data)
|
||||
self.keyOrder = data.keys()
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
dict.__setitem__(self, key, value)
|
||||
if key not in self.keyOrder:
|
||||
self.keyOrder.append(key)
|
||||
|
||||
def __delitem__(self, key):
|
||||
dict.__delitem__(self, key)
|
||||
self.keyOrder.remove(key)
|
||||
|
||||
def __iter__(self):
|
||||
for k in self.keyOrder:
|
||||
yield k
|
||||
|
||||
def items(self):
|
||||
return zip(self.keyOrder, self.values())
|
||||
|
||||
def iteritems(self):
|
||||
for key in self.keyOrder:
|
||||
yield key, dict.__getitem__(self, key)
|
||||
|
||||
def keys(self):
|
||||
return self.keyOrder[:]
|
||||
|
||||
def iterkeys(self):
|
||||
return iter(self.keyOrder)
|
||||
|
||||
def values(self):
|
||||
return [dict.__getitem__(self, k) for k in self.keyOrder]
|
||||
|
||||
def itervalues(self):
|
||||
for key in self.keyOrder:
|
||||
yield dict.__getitem__(self, key)
|
||||
|
||||
def update(self, dict):
|
||||
for k, v in dict.items():
|
||||
self.__setitem__(k, v)
|
||||
|
||||
def setdefault(self, key, default):
|
||||
if key not in self.keyOrder:
|
||||
self.keyOrder.append(key)
|
||||
return dict.setdefault(self, key, default)
|
||||
|
||||
def value_for_index(self, index):
|
||||
"Returns the value of the item at the given zero-based index."
|
||||
return self[self.keyOrder[index]]
|
||||
|
||||
def insert(self, index, key, value):
|
||||
"Inserts the key, value pair before the item with the given index."
|
||||
if key in self.keyOrder:
|
||||
n = self.keyOrder.index(key)
|
||||
del self.keyOrder[n]
|
||||
if n < index:
|
||||
index -= 1
|
||||
self.keyOrder.insert(index, key)
|
||||
dict.__setitem__(self, key, value)
|
||||
|
||||
def copy(self):
|
||||
"Returns a copy of this object."
|
||||
# This way of initializing the copy means it works for subclasses, too.
|
||||
obj = self.__class__(self)
|
||||
obj.keyOrder = self.keyOrder
|
||||
return obj
|
||||
|
||||
def __repr__(self):
|
||||
"""
|
||||
Replaces the normal dict.__repr__ with a version that returns the keys
|
||||
in their sorted order.
|
||||
"""
|
||||
return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in self.items()])
|
||||
|
||||
class MultiValueDictKeyError(KeyError):
|
||||
pass
|
||||
|
||||
@ -267,51 +138,6 @@ class MultiValueDict(dict):
|
||||
for key, value in kwargs.iteritems():
|
||||
self.setlistdefault(key, []).append(value)
|
||||
|
||||
class DotExpandedDict(dict):
|
||||
"""
|
||||
A special dictionary constructor that takes a dictionary in which the keys
|
||||
may contain dots to specify inner dictionaries. It's confusing, but this
|
||||
example should make sense.
|
||||
|
||||
>>> d = DotExpandedDict({'person.1.firstname': ['Simon'], \
|
||||
'person.1.lastname': ['Willison'], \
|
||||
'person.2.firstname': ['Adrian'], \
|
||||
'person.2.lastname': ['Holovaty']})
|
||||
>>> d
|
||||
{'person': {'1': {'lastname': ['Willison'], 'firstname': ['Simon']}, '2': {'lastname': ['Holovaty'], 'firstname': ['Adrian']}}}
|
||||
>>> d['person']
|
||||
{'1': {'lastname': ['Willison'], 'firstname': ['Simon']}, '2': {'lastname': ['Holovaty'], 'firstname': ['Adrian']}}
|
||||
>>> d['person']['1']
|
||||
{'lastname': ['Willison'], 'firstname': ['Simon']}
|
||||
|
||||
# Gotcha: Results are unpredictable if the dots are "uneven":
|
||||
>>> DotExpandedDict({'c.1': 2, 'c.2': 3, 'c': 1})
|
||||
{'c': 1}
|
||||
"""
|
||||
def __init__(self, key_to_list_mapping):
|
||||
for k, v in key_to_list_mapping.items():
|
||||
current = self
|
||||
bits = k.split('.')
|
||||
for bit in bits[:-1]:
|
||||
current = current.setdefault(bit, {})
|
||||
# Now assign value to current position
|
||||
try:
|
||||
current[bits[-1]] = v
|
||||
except TypeError: # Special-case if current isn't a dict.
|
||||
current = {bits[-1] : v}
|
||||
|
||||
class FileDict(dict):
|
||||
"""
|
||||
A dictionary used to hold uploaded file contents. The only special feature
|
||||
here is that repr() of this object won't dump the entire contents of the
|
||||
file to the output. A handy safeguard for a large file upload.
|
||||
"""
|
||||
def __repr__(self):
|
||||
if 'content' in self:
|
||||
d = dict(self, content='<omitted>')
|
||||
return dict.__repr__(d)
|
||||
return dict.__repr__(self)
|
||||
|
||||
class Sitemap(object):
|
||||
"""Sitemap class is used to build a map of the traversed pages"""
|
||||
|
||||
@ -436,14 +262,3 @@ class PriorityStack(PriorityQueue):
|
||||
def push(self, item, priority=0):
|
||||
heappush(self.items, (priority, -time.time(), item))
|
||||
|
||||
class gzStringIO:
|
||||
"""a file like object, similar to StringIO, but gzip-compressed."""
|
||||
def __init__(self, data, compress_level = 9, filename = ""):
|
||||
self._s = StringIO()
|
||||
g = gzip.GzipFile(filename, "wb", compress_level, self._s)
|
||||
g.write(data)
|
||||
g.flush()
|
||||
g.close()
|
||||
def read(self):
|
||||
return self._s.getvalue()
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user