Merge pull request #234 from darkrho/httpcache-dbm-module-fullpath

allow to use full path in HTTPCACHE_DBM_MODULE setting

scrapy/contrib/httpcache.py

@@ -163,7 +163,7 @@ class DbmCacheStorage(object):
     def __init__(self, settings):
         self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)
         self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
-        self.dbmodule = __import__(settings['HTTPCACHE_DBM_MODULE'])
+        self.dbmodule = __import__(settings['HTTPCACHE_DBM_MODULE'], {}, {}, [''])
         self.db = None
 
     def open_spider(self, spider):

scrapy/tests/mocks/dummydbm.py

@@ -0,0 +1,22 @@
+"""DBM-like dummy module"""
+import collections
+
+
+class DummyDB(dict):
+    """Provide dummy DBM-like interface."""
+    def close(self):
+        pass
+
+
+error = KeyError
+
+
+_DATABASES = collections.defaultdict(DummyDB)
+
+def open(file, flag='r', mode=0666):
+    """Open or create a dummy database compatible.
+
+    Arguments `flag` and `mode` are ignored.
+    """
+    # return same instance for same file argument
+    return _DATABASES[file]

scrapy/tests/test_downloadermiddleware_httpcache.py

@@ -116,6 +116,20 @@ class DbmStorageTest(DefaultStorageTest):
     storage_class = 'scrapy.contrib.httpcache.DbmCacheStorage'
 
 
+class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):
+
+    dbm_module = 'scrapy.tests.mocks.dummydbm'
+
+    def _get_settings(self, **new_settings):
+        new_settings.setdefault('HTTPCACHE_DBM_MODULE', self.dbm_module)
+        return super(DbmStorageWithCustomDbmModuleTest, self)._get_settings(**new_settings)
+
+    def test_custom_dbm_module_loaded(self):
+        # make sure our dbm module has been loaded
+        with self._storage() as storage:
+            self.assertEqual(storage.dbmodule.__name__, self.dbm_module)
+
+
 class FilesystemStorageTest(DefaultStorageTest):
 
     storage_class = 'scrapy.contrib.httpcache.FilesystemCacheStorage'