BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-24 20:23:44 +00:00
Code Releases Wiki Activity

Move cache storage logging to the individual storage classes

Browse Source
This commit is contained in:
jorenham 2017-03-02 16:17:51 +01:00
parent 97d84d920b
commit f96490df2c
2 changed files with 10 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
scrapy/downloadermiddlewares/httpcache.py
View File

@ -1,5 +1,3 @@
import logging
from email.utils import formatdate from email.utils import formatdate
from twisted.internet import defer from twisted.internet import defer
from twisted.internet.error import TimeoutError, DNSLookupError, \ from twisted.internet.error import TimeoutError, DNSLookupError, \
@ -11,9 +9,6 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.utils.misc import load_object from scrapy.utils.misc import load_object
logger = logging.getLogger(__name__)
class HttpCacheMiddleware(object): class HttpCacheMiddleware(object):
DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError, DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
@ -29,8 +24,6 @@ class HttpCacheMiddleware(object):
self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING')
self.stats = stats self.stats = stats
logger.debug("Using cache directory %(cachedir)s" % {'cachedir': self.storage.cachedir})
@classmethod @classmethod
def from_crawler(cls, crawler): def from_crawler(cls, crawler):
o = cls(crawler.settings, crawler.stats) o = cls(crawler.settings, crawler.stats)

10
scrapy/extensions/httpcache.py
View File

@ -1,6 +1,7 @@
from __future__ import print_function from __future__ import print_function
import os import os
import gzip import gzip
import logging
from six.moves import cPickle as pickle from six.moves import cPickle as pickle
from importlib import import_module from importlib import import_module
from time import time from time import time
@ -15,6 +16,9 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.python import to_bytes, to_unicode
logger = logging.getLogger(__name__)
class DummyPolicy(object): class DummyPolicy(object):
def __init__(self, settings): def __init__(self, settings):
@ -216,6 +220,8 @@ class DbmCacheStorage(object):
self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE']) self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE'])
self.db = None self.db = None
logger.debug("Using DBM cache storage in %(cachedir)s" % {'cachedir': self.cachedir})
def open_spider(self, spider): def open_spider(self, spider):
dbpath = os.path.join(self.cachedir, '%s.db' % spider.name) dbpath = os.path.join(self.cachedir, '%s.db' % spider.name)
self.db = self.dbmodule.open(dbpath, 'c') self.db = self.dbmodule.open(dbpath, 'c')
@ -271,6 +277,8 @@ class FilesystemCacheStorage(object):
self.use_gzip = settings.getbool('HTTPCACHE_GZIP') self.use_gzip = settings.getbool('HTTPCACHE_GZIP')
self._open = gzip.open if self.use_gzip else open self._open = gzip.open if self.use_gzip else open
logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir})
def open_spider(self, spider): def open_spider(self, spider):
pass pass
@ -344,6 +352,8 @@ class LeveldbCacheStorage(object):
self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
self.db = None self.db = None
logger.debug("Using LevelDB cache storage in %(cachedir)s" % {'cachedir': self.cachedir})
def open_spider(self, spider): def open_spider(self, spider):
dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name) dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name)
self.db = self._leveldb.LevelDB(dbpath) self.db = self._leveldb.LevelDB(dbpath)