1
0
mirror of https://github.com/scrapy/scrapy.git synced 2025-03-14 02:39:27 +00:00

Merge pull request #4768 from maranqz/feature/4606-exporter-from-FEEDS

Pass info from FEEDS to ItemExporter
This commit is contained in:
Mikhail Korobov 2020-10-02 00:10:05 +05:00 committed by GitHub
commit 4f27c5f82b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 46 additions and 0 deletions

View File

@ -303,6 +303,9 @@ For instance::
'store_empty': False,
'fields': None,
'indent': 4,
'item_export_kwargs': {
'export_empty_fields': True,
},
},
'/home/user/documents/items.xml': {
'format': 'xml',
@ -332,6 +335,8 @@ as a fallback value if that key is not provided for a specific feed definition:
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.
- ``overwrite``: whether to overwrite the file if it already exists
(``True``) or append to its content (``False``).

View File

@ -349,6 +349,7 @@ class FeedExporter:
fields_to_export=feed_options['fields'],
encoding=feed_options['encoding'],
indent=feed_options['indent'],
**feed_options['item_export_kwargs'],
)
slot = _FeedSlot(
file=file,

View File

@ -121,6 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings):
out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
out.setdefault("item_export_kwargs", dict())
if settings["FEED_EXPORT_INDENT"] is None:
out.setdefault("indent", None)
else:

View File

@ -1191,6 +1191,43 @@ class FeedExportTest(FeedExportTestBase):
for fmt in ['json', 'xml', 'csv']:
self.assertIn(f'Error storing {fmt} feed (2 items)', str(log))
@defer.inlineCallbacks
def test_extend_kwargs(self):
items = [{'foo': 'FOO', 'bar': 'BAR'}]
expected_with_title_csv = 'foo,bar\r\nFOO,BAR\r\n'.encode('utf-8')
expected_without_title_csv = 'FOO,BAR\r\n'.encode('utf-8')
test_cases = [
# with title
{
'options': {
'format': 'csv',
'item_export_kwargs': {'include_headers_line': True},
},
'expected': expected_with_title_csv,
},
# without title
{
'options': {
'format': 'csv',
'item_export_kwargs': {'include_headers_line': False},
},
'expected': expected_without_title_csv,
},
]
for row in test_cases:
feed_options = row['options']
settings = {
'FEEDS': {
self._random_temp_filename(): feed_options,
},
'FEED_EXPORT_INDENT': None,
}
data = yield self.exported_data(items, settings)
self.assertEqual(row['expected'], data[feed_options['format']])
class BatchDeliveriesTest(FeedExportTestBase):
__test__ = True

View File

@ -176,6 +176,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
"store_empty": True,
"uri_params": (1, 2, 3, 4),
"batch_item_count": 2,
"item_export_kwargs": dict(),
})
def test_feed_complete_default_values_from_settings_non_empty(self):
@ -198,6 +199,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
"store_empty": True,
"uri_params": None,
"batch_item_count": 2,
"item_export_kwargs": dict(),
})