mirror of
https://github.com/scrapy/scrapy.git
synced 2025-02-22 06:13:24 +00:00
236 lines
7.7 KiB
Python
236 lines
7.7 KiB
Python
import gc
|
|
import functools
|
|
import operator
|
|
import unittest
|
|
from itertools import count
|
|
import platform
|
|
import six
|
|
|
|
from scrapy.utils.python import (
|
|
memoizemethod_noargs, binary_is_text, equal_attributes,
|
|
WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode,
|
|
without_none_values)
|
|
|
|
__doctests__ = ['scrapy.utils.python']
|
|
|
|
|
|
class ToUnicodeTest(unittest.TestCase):
|
|
def test_converting_an_utf8_encoded_string_to_unicode(self):
|
|
self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e')
|
|
|
|
def test_converting_a_latin_1_encoded_string_to_unicode(self):
|
|
self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), u'lel\xf1e')
|
|
|
|
def test_converting_a_unicode_to_unicode_should_return_the_same_object(self):
|
|
self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e')
|
|
|
|
def test_converting_a_strange_object_should_raise_TypeError(self):
|
|
self.assertRaises(TypeError, to_unicode, 423)
|
|
|
|
def test_errors_argument(self):
|
|
self.assertEqual(
|
|
to_unicode(b'a\xedb', 'utf-8', errors='replace'),
|
|
u'a\ufffdb'
|
|
)
|
|
|
|
|
|
class ToBytesTest(unittest.TestCase):
|
|
def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self):
|
|
self.assertEqual(to_bytes(u'\xa3 49'), b'\xc2\xa3 49')
|
|
|
|
def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self):
|
|
self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49')
|
|
|
|
def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self):
|
|
self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e')
|
|
|
|
def test_converting_a_strange_object_should_raise_TypeError(self):
|
|
self.assertRaises(TypeError, to_bytes, unittest)
|
|
|
|
def test_errors_argument(self):
|
|
self.assertEqual(
|
|
to_bytes(u'a\ufffdb', 'latin-1', errors='replace'),
|
|
b'a?b'
|
|
)
|
|
|
|
|
|
class MemoizedMethodTest(unittest.TestCase):
|
|
def test_memoizemethod_noargs(self):
|
|
class A(object):
|
|
|
|
@memoizemethod_noargs
|
|
def cached(self):
|
|
return object()
|
|
|
|
def noncached(self):
|
|
return object()
|
|
|
|
a = A()
|
|
one = a.cached()
|
|
two = a.cached()
|
|
three = a.noncached()
|
|
assert one is two
|
|
assert one is not three
|
|
|
|
|
|
class BinaryIsTextTest(unittest.TestCase):
|
|
def test_binaryistext(self):
|
|
assert binary_is_text(b"hello")
|
|
|
|
def test_utf_16_strings_contain_null_bytes(self):
|
|
assert binary_is_text(u"hello".encode('utf-16'))
|
|
|
|
def test_one_with_encoding(self):
|
|
assert binary_is_text(b"<div>Price \xa3</div>")
|
|
|
|
def test_real_binary_bytes(self):
|
|
assert not binary_is_text(b"\x02\xa3")
|
|
|
|
|
|
|
|
class UtilsPythonTestCase(unittest.TestCase):
|
|
|
|
def test_equal_attributes(self):
|
|
class Obj:
|
|
pass
|
|
|
|
a = Obj()
|
|
b = Obj()
|
|
# no attributes given return False
|
|
self.failIf(equal_attributes(a, b, []))
|
|
# not existent attributes
|
|
self.failIf(equal_attributes(a, b, ['x', 'y']))
|
|
|
|
a.x = 1
|
|
b.x = 1
|
|
# equal attribute
|
|
self.assertTrue(equal_attributes(a, b, ['x']))
|
|
|
|
b.y = 2
|
|
# obj1 has no attribute y
|
|
self.failIf(equal_attributes(a, b, ['x', 'y']))
|
|
|
|
a.y = 2
|
|
# equal attributes
|
|
self.assertTrue(equal_attributes(a, b, ['x', 'y']))
|
|
|
|
a.y = 1
|
|
# differente attributes
|
|
self.failIf(equal_attributes(a, b, ['x', 'y']))
|
|
|
|
# test callable
|
|
a.meta = {}
|
|
b.meta = {}
|
|
self.assertTrue(equal_attributes(a, b, ['meta']))
|
|
|
|
# compare ['meta']['a']
|
|
a.meta['z'] = 1
|
|
b.meta['z'] = 1
|
|
|
|
get_z = operator.itemgetter('z')
|
|
get_meta = operator.attrgetter('meta')
|
|
compare_z = lambda obj: get_z(get_meta(obj))
|
|
|
|
self.assertTrue(equal_attributes(a, b, [compare_z, 'x']))
|
|
# fail z equality
|
|
a.meta['z'] = 2
|
|
self.failIf(equal_attributes(a, b, [compare_z, 'x']))
|
|
|
|
def test_weakkeycache(self):
|
|
class _Weakme(object): pass
|
|
_values = count()
|
|
wk = WeakKeyCache(lambda k: next(_values))
|
|
k = _Weakme()
|
|
v = wk[k]
|
|
self.assertEqual(v, wk[k])
|
|
self.assertNotEqual(v, wk[_Weakme()])
|
|
self.assertEqual(v, wk[k])
|
|
del k
|
|
for _ in range(100):
|
|
if wk._weakdict:
|
|
gc.collect()
|
|
self.assertFalse(len(wk._weakdict))
|
|
|
|
@unittest.skipUnless(six.PY2, "deprecated function")
|
|
def test_stringify_dict(self):
|
|
d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'}
|
|
d2 = stringify_dict(d, keys_only=False)
|
|
self.assertEqual(d, d2)
|
|
self.failIf(d is d2) # shouldn't modify in place
|
|
self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()))
|
|
self.failIf(any(isinstance(x, six.text_type) for x in d2.values()))
|
|
|
|
@unittest.skipUnless(six.PY2, "deprecated function")
|
|
def test_stringify_dict_tuples(self):
|
|
tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')]
|
|
d = dict(tuples)
|
|
d2 = stringify_dict(tuples, keys_only=False)
|
|
self.assertEqual(d, d2)
|
|
self.failIf(d is d2) # shouldn't modify in place
|
|
self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys())
|
|
self.failIf(any(isinstance(x, six.text_type) for x in d2.values()))
|
|
|
|
@unittest.skipUnless(six.PY2, "deprecated function")
|
|
def test_stringify_dict_keys_only(self):
|
|
d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
|
|
d2 = stringify_dict(d)
|
|
self.assertEqual(d, d2)
|
|
self.failIf(d is d2) # shouldn't modify in place
|
|
self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()))
|
|
|
|
def test_get_func_args(self):
|
|
def f1(a, b, c):
|
|
pass
|
|
|
|
def f2(a, b=None, c=None):
|
|
pass
|
|
|
|
class A(object):
|
|
def __init__(self, a, b, c):
|
|
pass
|
|
|
|
def method(self, a, b, c):
|
|
pass
|
|
|
|
class Callable(object):
|
|
|
|
def __call__(self, a, b, c):
|
|
pass
|
|
|
|
a = A(1, 2, 3)
|
|
cal = Callable()
|
|
partial_f1 = functools.partial(f1, None)
|
|
partial_f2 = functools.partial(f1, b=None)
|
|
partial_f3 = functools.partial(partial_f2, None)
|
|
|
|
self.assertEqual(get_func_args(f1), ['a', 'b', 'c'])
|
|
self.assertEqual(get_func_args(f2), ['a', 'b', 'c'])
|
|
self.assertEqual(get_func_args(A), ['a', 'b', 'c'])
|
|
self.assertEqual(get_func_args(a.method), ['a', 'b', 'c'])
|
|
self.assertEqual(get_func_args(partial_f1), ['b', 'c'])
|
|
self.assertEqual(get_func_args(partial_f2), ['a', 'c'])
|
|
self.assertEqual(get_func_args(partial_f3), ['c'])
|
|
self.assertEqual(get_func_args(cal), ['a', 'b', 'c'])
|
|
self.assertEqual(get_func_args(object), [])
|
|
|
|
if platform.python_implementation() == 'CPython':
|
|
# TODO: how do we fix this to return the actual argument names?
|
|
self.assertEqual(get_func_args(six.text_type.split), [])
|
|
self.assertEqual(get_func_args(" ".join), [])
|
|
self.assertEqual(get_func_args(operator.itemgetter(2)), [])
|
|
else:
|
|
self.assertEqual(get_func_args(six.text_type.split), ['sep', 'maxsplit'])
|
|
self.assertEqual(get_func_args(" ".join), ['list'])
|
|
self.assertEqual(get_func_args(operator.itemgetter(2)), ['obj'])
|
|
|
|
|
|
def test_without_none_values(self):
|
|
self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])
|
|
self.assertEqual(without_none_values((1, None, 3, 4)), (1, 3, 4))
|
|
self.assertEqual(
|
|
without_none_values({'one': 1, 'none': None, 'three': 3, 'four': 4}),
|
|
{'one': 1, 'three': 3, 'four': 4})
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|