utils: Add walk_packages utility function

--HG--
rename : scrapy/tests/test_utils_misc.py => scrapy/tests/test_utils_misc/__init__.py

scrapy/tests/test_utils_misc.py

@@ -1,29 +0,0 @@
-import unittest
-from cStringIO import StringIO
-
-from scrapy.utils.misc import load_object, arg_to_iter
-
-class UtilsMiscTestCase(unittest.TestCase):
-
-    def test_load_object(self):
-        obj = load_object('scrapy.utils.misc.load_object')
-        assert obj is load_object
-        self.assertRaises(ImportError, load_object, 'nomodule999.mod.function')
-        self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999')
-
-    def test_arg_to_iter(self):
-        assert hasattr(arg_to_iter(None), '__iter__')
-        assert hasattr(arg_to_iter(100), '__iter__')
-        assert hasattr(arg_to_iter('lala'), '__iter__')
-        assert hasattr(arg_to_iter([1,2,3]), '__iter__')
-        assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__')
-
-        self.assertEqual(list(arg_to_iter(None)), [])
-        self.assertEqual(list(arg_to_iter('lala')), ['lala'])
-        self.assertEqual(list(arg_to_iter(100)), [100])
-        self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c'])
-        self.assertEqual(list(arg_to_iter([1,2,3])), [1,2,3])
-        self.assertEqual(list(arg_to_iter({'a':1})), [{'a': 1}])
-
-if __name__ == "__main__":
-    unittest.main()

scrapy/tests/test_utils_misc/__init__.py

@@ -0,0 +1,54 @@
+import unittest
+from cStringIO import StringIO
+
+from scrapy.utils.misc import load_object, arg_to_iter, walk_modules
+
+class UtilsMiscTestCase(unittest.TestCase):
+
+    def test_load_object(self):
+        obj = load_object('scrapy.utils.misc.load_object')
+        assert obj is load_object
+        self.assertRaises(ImportError, load_object, 'nomodule999.mod.function')
+        self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999')
+
+    def test_walk_modules(self):
+        mods = walk_modules('scrapy.tests.test_utils_misc.test_walk_modules')
+        expected = [
+            'scrapy.tests.test_utils_misc.test_walk_modules',
+            'scrapy.tests.test_utils_misc.test_walk_modules.mod',
+            'scrapy.tests.test_utils_misc.test_walk_modules.mod.mod0',
+            'scrapy.tests.test_utils_misc.test_walk_modules.mod1',
+        ]
+        self.assertEquals(set([m.__name__ for m in mods]), set(expected))
+
+        mods = walk_modules('scrapy.tests.test_utils_misc.test_walk_modules.mod')
+        expected = [
+            'scrapy.tests.test_utils_misc.test_walk_modules.mod',
+            'scrapy.tests.test_utils_misc.test_walk_modules.mod.mod0',
+        ]
+        self.assertEquals(set([m.__name__ for m in mods]), set(expected))
+
+        mods = walk_modules('scrapy.tests.test_utils_misc.test_walk_modules.mod1')
+        expected = [
+            'scrapy.tests.test_utils_misc.test_walk_modules.mod1',
+        ]
+        self.assertEquals(set([m.__name__ for m in mods]), set(expected))
+
+        self.assertRaises(ImportError, list, walk_modules('nomodule999'))
+
+    def test_arg_to_iter(self):
+        assert hasattr(arg_to_iter(None), '__iter__')
+        assert hasattr(arg_to_iter(100), '__iter__')
+        assert hasattr(arg_to_iter('lala'), '__iter__')
+        assert hasattr(arg_to_iter([1,2,3]), '__iter__')
+        assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__')
+
+        self.assertEqual(list(arg_to_iter(None)), [])
+        self.assertEqual(list(arg_to_iter('lala')), ['lala'])
+        self.assertEqual(list(arg_to_iter(100)), [100])
+        self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c'])
+        self.assertEqual(list(arg_to_iter([1,2,3])), [1,2,3])
+        self.assertEqual(list(arg_to_iter({'a':1})), [{'a': 1}])
+
+if __name__ == "__main__":
+    unittest.main()

scrapy/utils/misc.py

@@ -2,6 +2,7 @@
 
 import re
 import hashlib
+from pkgutil import walk_packages
 
 from scrapy.utils.python import flatten
 from scrapy.utils.markup import remove_entities
@@ -44,6 +45,18 @@ def load_object(path):
 
     return obj
 
+def walk_modules(path):
+    """Loads a module and all its submodules given its absolute path and
+    returns them.
+
+    path ie: 'scrapy.contrib.downloadermiddelware.redirect'
+    """
+    mod = __import__(path, {}, {}, [''])
+    if hasattr(mod, '__path__'):
+        for _, path, _ in walk_packages(mod.__path__, mod.__name__ + '.'):
+            yield __import__(path, {}, {}, [''])
+    yield mod
+
 def extract_regex(regex, text, encoding='utf-8'):
     """Extract a list of unicode strings from the given text/encoding using the following policies: