Merge pull request #359 from rocioar/master

Added COMPRESSION_ENABLED setting

docs/topics/downloader-middleware.rst

@@ -533,6 +533,19 @@ HttpCompressionMiddleware
    This middleware allows compressed (gzip, deflate) traffic to be
    sent/received from web sites.
 
+HttpCompressionMiddleware Settings
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. setting:: COMPRESSION_ENABLED
+
+COMPRESSION_ENABLED
+^^^^^^^^^^^^^^^^^^^
+
+Default: ``True``
+
+Whether the Compression middleware will be enabled.
+
+
 ChunkedTransferMiddleware
 -------------------------
 

scrapy/contrib/downloadermiddleware/httpcompression.py

@@ -3,12 +3,19 @@ import zlib
 from scrapy.utils.gz import gunzip
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
+from scrapy.exceptions import NotConfigured
 
 
 class HttpCompressionMiddleware(object):
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
     
+    @classmethod
+    def from_crawler(cls, crawler):
+        if not crawler.settings.getbool('COMPRESSION_ENABLED'):
+            raise NotConfigured
+        return cls()
+    
     def process_request(self, request, spider):
         request.headers.setdefault('Accept-Encoding', 'x-gzip,gzip,deflate')
 

scrapy/settings/default_settings.py

@@ -26,6 +26,8 @@ CLOSESPIDER_ERRORCOUNT = 0
 
 COMMANDS_MODULE = ''
 
+COMPRESSION_ENABLED = True
+
 CONCURRENT_ITEMS = 100
 
 CONCURRENT_REQUESTS = 16