mirror of
https://github.com/scrapy/scrapy.git
synced 2025-03-14 02:39:27 +00:00
Merge pull request #4768 from maranqz/feature/4606-exporter-from-FEEDS
Pass info from FEEDS to ItemExporter
This commit is contained in:
commit
4f27c5f82b
@ -303,6 +303,9 @@ For instance::
|
||||
'store_empty': False,
|
||||
'fields': None,
|
||||
'indent': 4,
|
||||
'item_export_kwargs': {
|
||||
'export_empty_fields': True,
|
||||
},
|
||||
},
|
||||
'/home/user/documents/items.xml': {
|
||||
'format': 'xml',
|
||||
@ -332,6 +335,8 @@ as a fallback value if that key is not provided for a specific feed definition:
|
||||
|
||||
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
|
||||
|
||||
- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.
|
||||
|
||||
- ``overwrite``: whether to overwrite the file if it already exists
|
||||
(``True``) or append to its content (``False``).
|
||||
|
||||
|
@ -349,6 +349,7 @@ class FeedExporter:
|
||||
fields_to_export=feed_options['fields'],
|
||||
encoding=feed_options['encoding'],
|
||||
indent=feed_options['indent'],
|
||||
**feed_options['item_export_kwargs'],
|
||||
)
|
||||
slot = _FeedSlot(
|
||||
file=file,
|
||||
|
@ -121,6 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings):
|
||||
out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
|
||||
out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
|
||||
out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
|
||||
out.setdefault("item_export_kwargs", dict())
|
||||
if settings["FEED_EXPORT_INDENT"] is None:
|
||||
out.setdefault("indent", None)
|
||||
else:
|
||||
|
@ -1191,6 +1191,43 @@ class FeedExportTest(FeedExportTestBase):
|
||||
for fmt in ['json', 'xml', 'csv']:
|
||||
self.assertIn(f'Error storing {fmt} feed (2 items)', str(log))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_extend_kwargs(self):
|
||||
items = [{'foo': 'FOO', 'bar': 'BAR'}]
|
||||
|
||||
expected_with_title_csv = 'foo,bar\r\nFOO,BAR\r\n'.encode('utf-8')
|
||||
expected_without_title_csv = 'FOO,BAR\r\n'.encode('utf-8')
|
||||
test_cases = [
|
||||
# with title
|
||||
{
|
||||
'options': {
|
||||
'format': 'csv',
|
||||
'item_export_kwargs': {'include_headers_line': True},
|
||||
},
|
||||
'expected': expected_with_title_csv,
|
||||
},
|
||||
# without title
|
||||
{
|
||||
'options': {
|
||||
'format': 'csv',
|
||||
'item_export_kwargs': {'include_headers_line': False},
|
||||
},
|
||||
'expected': expected_without_title_csv,
|
||||
},
|
||||
]
|
||||
|
||||
for row in test_cases:
|
||||
feed_options = row['options']
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
self._random_temp_filename(): feed_options,
|
||||
},
|
||||
'FEED_EXPORT_INDENT': None,
|
||||
}
|
||||
|
||||
data = yield self.exported_data(items, settings)
|
||||
self.assertEqual(row['expected'], data[feed_options['format']])
|
||||
|
||||
|
||||
class BatchDeliveriesTest(FeedExportTestBase):
|
||||
__test__ = True
|
||||
|
@ -176,6 +176,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
|
||||
"store_empty": True,
|
||||
"uri_params": (1, 2, 3, 4),
|
||||
"batch_item_count": 2,
|
||||
"item_export_kwargs": dict(),
|
||||
})
|
||||
|
||||
def test_feed_complete_default_values_from_settings_non_empty(self):
|
||||
@ -198,6 +199,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
|
||||
"store_empty": True,
|
||||
"uri_params": None,
|
||||
"batch_item_count": 2,
|
||||
"item_export_kwargs": dict(),
|
||||
})
|
||||
|
||||
|
||||
|
Loading…
x
Reference in New Issue
Block a user