Remove LevelDB support (#4112)

scrapy/extensions/httpcache.py

@@ -347,77 +347,6 @@ class FilesystemCacheStorage(object):
             return pickle.load(f)
 
 
-class LeveldbCacheStorage(object):
-
-    def __init__(self, settings):
-        warn("The LevelDB storage backend is deprecated.",
-             ScrapyDeprecationWarning, stacklevel=2)
-        import leveldb
-        self._leveldb = leveldb
-        self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)
-        self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
-        self.db = None
-
-    def open_spider(self, spider):
-        dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name)
-        self.db = self._leveldb.LevelDB(dbpath)
-
-        logger.debug("Using LevelDB cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
-
-    def close_spider(self, spider):
-        # Do compactation each time to save space and also recreate files to
-        # avoid them being removed in storages with timestamp-based autoremoval.
-        self.db.CompactRange()
-        del self.db
-        garbage_collect()
-
-    def retrieve_response(self, spider, request):
-        data = self._read_data(spider, request)
-        if data is None:
-            return  # not cached
-        url = data['url']
-        status = data['status']
-        headers = Headers(data['headers'])
-        body = data['body']
-        respcls = responsetypes.from_args(headers=headers, url=url)
-        response = respcls(url=url, headers=headers, status=status, body=body)
-        return response
-
-    def store_response(self, spider, request, response):
-        key = self._request_key(request)
-        data = {
-            'status': response.status,
-            'url': response.url,
-            'headers': dict(response.headers),
-            'body': response.body,
-        }
-        batch = self._leveldb.WriteBatch()
-        batch.Put(key + b'_data', pickle.dumps(data, protocol=2))
-        batch.Put(key + b'_time', to_bytes(str(time())))
-        self.db.Write(batch)
-
-    def _read_data(self, spider, request):
-        key = self._request_key(request)
-        try:
-            ts = self.db.Get(key + b'_time')
-        except KeyError:
-            return  # not found or invalid entry
-
-        if 0 < self.expiration_secs < time() - float(ts):
-            return  # expired
-
-        try:
-            data = self.db.Get(key + b'_data')
-        except KeyError:
-            return  # invalid entry
-        else:
-            return pickle.loads(data)
-
-    def _request_key(self, request):
-        return to_bytes(request_fingerprint(request))
-
-
-
 def parse_cachecontrol(header):
     """Parse Cache-Control header
 

tests/requirements-py3.txt

@@ -1,6 +1,5 @@
 # Tests requirements
 jmespath
-leveldb; sys_platform != "win32"
 pytest
 pytest-cov
 pytest-twisted

tests/test_downloadermiddleware_httpcache.py

@@ -156,15 +156,6 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):
         return super(FilesystemStorageTest, self)._get_settings(**new_settings)
 
 
-class LeveldbStorageTest(DefaultStorageTest):
-
-    try:
-        pytest.importorskip('leveldb')
-    except SystemError:
-        pytestmark = pytest.mark.skip("Test module skipped - 'SystemError: bad call flags' occurs when >= Python 3.8")
-    storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage'
-
-
 class DummyPolicyTest(_BaseTest):
 
     policy_class = 'scrapy.extensions.httpcache.DummyPolicy'