BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-24 21:43:55 +00:00
Code Releases Wiki Activity

Change ImagesPipeline class attributes to instance attributes.

Browse Source 
modified:   scrapy/pipelines/images.py
This commit is contained in:
djunzu 2016-03-31 19:20:43 -03:00
parent 8228a0c491
commit 537083524e
1 changed files with 21 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
scrapy/pipelines/images.py
View File

@ -17,6 +17,7 @@ from PIL import Image
from scrapy.utils.misc import md5sum
from scrapy.utils.python import to_bytes
from scrapy.http import Request
from scrapy.settings import Settings
from scrapy.exceptions import DropItem
#TODO: from scrapy.pipelines.media import MediaPipeline
from scrapy.pipelines.files import FileException, FilesPipeline
@ -37,20 +38,27 @@ class ImagesPipeline(FilesPipeline):
MEDIA_NAME = 'image'
def __init__(self, store_uri, download_func=None, settings=None):
super(ImagesPipeline, self).__init__(store_uri, settings=settings, download_func=download_func)
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
self.expires = settings.getint('IMAGES_EXPIRES')
self.images_urls_field = settings.get('IMAGES_URLS_FIELD')
self.images_result_field = settings.get('IMAGES_RESULT_FIELD')
self.min_width = settings.getint('IMAGES_MIN_WIDTH')
self.min_height = settings.getint('IMAGES_MIN_HEIGHT')
self.thumbs = settings.get('IMAGES_THUMBS')
@classmethod
def from_settings(cls, settings):
cls.MIN_WIDTH = settings.getint('IMAGES_MIN_WIDTH')
cls.MIN_HEIGHT = settings.getint('IMAGES_MIN_HEIGHT')
cls.EXPIRES = settings.getint('IMAGES_EXPIRES')
cls.THUMBS = settings.get('IMAGES_THUMBS')
s3store = cls.STORE_SCHEMES['s3']
s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
cls.IMAGES_URLS_FIELD = settings.get('IMAGES_URLS_FIELD')
cls.IMAGES_RESULT_FIELD = settings.get('IMAGES_RESULT_FIELD')
store_uri = settings['IMAGES_STORE']
return cls(store_uri)
return cls(store_uri, settings=settings)
def file_downloaded(self, response, request, info):
return self.image_downloaded(response, request, info)
@ -73,14 +81,14 @@ class ImagesPipeline(FilesPipeline):
orig_image = Image.open(BytesIO(response.body))
width, height = orig_image.size
if width < self.MIN_WIDTH or height < self.MIN_HEIGHT:
if width < self.min_width or height < self.min_height:
raise ImageException("Image too small (%dx%d < %dx%d)" %
(width, height, self.MIN_WIDTH, self.MIN_HEIGHT))
(width, height, self.min_width, self.min_height))
image, buf = self.convert_image(orig_image)
yield path, image, buf
for thumb_id, size in six.iteritems(self.THUMBS):
for thumb_id, size in six.iteritems(self.thumbs):
thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
thumb_image, thumb_buf = self.convert_image(image, size)
yield thumb_path, thumb_image, thumb_buf
@ -102,11 +110,11 @@ class ImagesPipeline(FilesPipeline):
return image, buf
def get_media_requests(self, item, info):
return [Request(x) for x in item.get(self.IMAGES_URLS_FIELD, [])]
return [Request(x) for x in item.get(self.images_urls_field, [])]
def item_completed(self, results, item, info):
if isinstance(item, dict) or self.IMAGES_RESULT_FIELD in item.fields:
item[self.IMAGES_RESULT_FIELD] = [x for ok, x in results if ok]
if isinstance(item, dict) or self.images_result_field in item.fields:
item[self.images_result_field] = [x for ok, x in results if ok]
return item
def file_path(self, request, response=None, info=None):