BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-25 03:04:06 +00:00
Code Releases Wiki Activity

Merge pull request #1358 from nyov/nyov/boto-anon-connection

Browse Source 
[MRG+1] Support anonymous S3DownloadHandler (boto) connections
This commit is contained in:
Daniel Graña 2016-01-19 13:35:23 -03:00
commit 5ec4319885
2 changed files with 28 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
scrapy/core/downloader/handlers/s3.py
View File

@ -35,7 +35,7 @@ def get_s3_connection():
class S3DownloadHandler(object):
def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
httpdownloadhandler=HTTPDownloadHandler):
httpdownloadhandler=HTTPDownloadHandler, **kw):
_S3Connection = get_s3_connection()
if _S3Connection is None:
@ -46,8 +46,15 @@ class S3DownloadHandler(object):
if not aws_secret_access_key:
aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']
# If no credentials could be found anywhere,
# consider this an anonymous connection request by default;
# unless 'anon' was set explicitly (True/False).
anon = kw.get('anon', None)
if anon is None and not aws_access_key_id and not aws_secret_access_key:
kw['anon'] = True
try:
self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key)
self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key, **kw)
except Exception as ex:
raise NotConfigured(str(ex))
self._download_http = httpdownloadhandler(settings).download_request

21
tests/test_downloader_handlers.py
View File

@ -431,6 +431,23 @@ class HttpDownloadHandlerMock(object):
def download_request(self, request, spider):
return request
class S3AnonTestCase(unittest.TestCase):
skip = 'boto' not in optional_features and 'missing boto library'
def setUp(self):
self.s3reqh = S3DownloadHandler(Settings(),
httpdownloadhandler=HttpDownloadHandlerMock,
#anon=True, # is implicit
)
self.download_request = self.s3reqh.download_request
self.spider = Spider('foo')
def test_anon_request(self):
req = Request('s3://aws-publicdatasets/')
httpreq = self.download_request(req, self.spider)
self.assertEqual(hasattr(self.s3reqh.conn, 'anon'), True)
self.assertEqual(self.s3reqh.conn.anon, True)
class S3TestCase(unittest.TestCase):
download_handler_cls = S3DownloadHandler
try:
@ -448,8 +465,8 @@ class S3TestCase(unittest.TestCase):
AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o'
def setUp(self):
s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID, \
self.AWS_SECRET_ACCESS_KEY, \
s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID,
self.AWS_SECRET_ACCESS_KEY,
httpdownloadhandler=HttpDownloadHandlerMock)
self.download_request = s3reqh.download_request
self.spider = Spider('foo')