newitem: rename treeadapt as adaptor, and bugfix a mutable dictionary bug

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40904

scrapy/trunk/scrapy/contrib_exp/newitem/extractors.py

@@ -70,7 +70,18 @@ class ExtractorField(object):
         return list(values)
 
 
-def treeadapt(*funcs, **adaptor_args):
+def adaptor(*funcs, **adaptor_args):
+    """A pipe adaptor implementing the tree adaption logic
+    
+    It takes multiples unnamed arguments used as functions of the pipe, and a
+    keywords used as adaptor_args to be passed to functions that supports it
+
+    If an adaptor function returns a list of values, each value is used as
+    input for next adaptor function
+
+    Always returns a list of values
+    """
+
     pipe_adaptor_args = adaptor_args
     _funcs = []
     for func in funcs:
@@ -79,8 +90,8 @@ def treeadapt(*funcs, **adaptor_args):
 
     def _adaptor(value, adaptor_args=None):
         values = value if isinstance(value, (list, tuple)) else [value]
-        pipe_adaptor_args.update(adaptor_args or {})
+        aargs = dict(t for d in [pipe_adaptor_args, adaptor_args or {}] for t in d.iteritems())
-        pipe_kwargs = {'adaptor_args': pipe_adaptor_args}
+        pipe_kwargs = { 'adaptor_args': aargs }
 
         for func, takes_args in _funcs:
             next = []

scrapy/trunk/scrapy/tests/test_itemextractor.py

@@ -0,0 +1,43 @@
+
+import unittest
+from scrapy.contrib_exp.newitem.extractors import adaptor, ItemExtractor
+
+
+class TreeadaptTest(unittest.TestCase):
+
+    def test_1_passtrough(self):
+        ad = adaptor()
+        self.assertEqual(ad('string'), ['string'])
+
+    def test_2_composing(self):
+        addone = lambda v: v+1
+        ad = adaptor(addone)
+        self.assertEqual(ad(0), [1])
+
+        addtwo = lambda v: v+2
+        ad2 = adaptor(addone, addtwo)
+        self.assertEqual(ad2(0), [3])
+
+        ad3 = adaptor(ad, ad2)
+        self.assertEqual(ad3(0), [4])
+
+    def test_3_adaptor_args(self):
+        addn = lambda v, adaptor_args: v + adaptor_args.get('add_value', 0)
+
+        ad = adaptor(addn)
+        self.assertEqual(ad(0), [0])
+        self.assertEqual(ad(0, {'add_value': 3}), [3])
+        self.assertEqual(ad(0), [0])
+
+        ad = adaptor(addn, add_value=5)
+        self.assertEqual(ad(0), [5])
+        self.assertEqual(ad(0, {'add_value': 3}), [3])
+        self.assertEqual(ad(0), [5])
+
+
+    def test_4_treelogic(self):
+        split1 = lambda v: v.split('&')
+        split2 = lambda v: v.split('=')
+        ad = adaptor(split1, split2)
+        self.assertEqual(ad('name=joe&job=joker'), ['name', 'joe', 'job', 'joker'])
+

scrapy/trunk/scrapy/utils/python.py

@@ -245,4 +245,6 @@ def get_func_args(func):
             func_args, _, _, _ = inspect.getargspec(func.__call__)
         except Exception:
             func_args = []
+    else:
+        raise TypeError('%s is not callable' % type(func))
     return func_args