BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-25 10:03:54 +00:00
Code Releases Wiki Activity

TST add a test for JSON lines item exporter

Browse Source
This commit is contained in:
Mikhail Korobov 2015-05-12 18:27:57 +05:00
parent 3a4d57b3f5
commit 8d9e3b7e8d
1 changed files with 27 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
tests/test_feedexport.py
View File

@ -1,6 +1,7 @@
from __future__ import absolute_import
import os
import csv
import json
from io import BytesIO
import tempfile
import shutil
@ -126,7 +127,6 @@ class FeedExportTest(unittest.TestCase):
egg = scrapy.Field()
baz = scrapy.Field()
@defer.inlineCallbacks
def run_and_export(self, spider_cls, settings=None):
""" Run spider with specified settings; return exported data. """
@ -180,7 +180,20 @@ class FeedExportTest(unittest.TestCase):
self.assertEqual(rows, got_rows)
@defer.inlineCallbacks
def test_export_csv_items(self):
def assertExportedJsonLines(self, items, rows, settings=None):
settings = settings or {}
settings.update({'FEED_FORMAT': 'jl'})
data = yield self.exported_data(items, settings)
parsed = [json.loads(line) for line in data.splitlines()]
self.assertEqual(rows, parsed)
@defer.inlineCallbacks
def assertExported(self, items, header, rows, settings=None, ordered=True):
yield self.assertExportedCsv(items, header, rows, settings, ordered)
yield self.assertExportedJsonLines(items, rows, settings)
@defer.inlineCallbacks
def test_export_items(self):
# feed exporters use field names from Item
items = [
self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
@ -191,10 +204,10 @@ class FeedExportTest(unittest.TestCase):
{'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
]
header = self.MyItem.fields.keys()
yield self.assertExportedCsv(items, header, rows, ordered=False)
yield self.assertExported(items, header, rows, ordered=False)
@defer.inlineCallbacks
def test_export_csv_multiple_item_classes(self):
def test_export_multiple_item_classes(self):
class MyItem2(scrapy.Item):
foo = scrapy.Field()
@ -215,7 +228,7 @@ class FeedExportTest(unittest.TestCase):
{'egg': 'spam3', 'foo': 'bar3', 'baz': 'quux3'},
{'egg': 'spam4', 'foo': '', 'baz': ''},
]
yield self.assertExportedCsv(items, header, rows, ordered=False)
yield self.assertExported(items, header, rows, ordered=False)
# but it is possible to override fields using FEED_EXPORT_FIELDS
header = ["foo", "baz", "hello"]
@ -226,11 +239,11 @@ class FeedExportTest(unittest.TestCase):
{'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
{'foo': '', 'baz': '', 'hello': 'world4'},
]
yield self.assertExportedCsv(items, header, rows,
settings=settings, ordered=True)
yield self.assertExported(items, header, rows,
settings=settings, ordered=True)
@defer.inlineCallbacks
def test_export_csv_dicts(self):
def test_export_dicts(self):
# When dicts are used, only keys from the first row are used as
# a header.
items = [
@ -241,10 +254,10 @@ class FeedExportTest(unittest.TestCase):
{'egg': 'spam', 'foo': 'bar'},
{'egg': 'spam', 'foo': 'bar'}
]
yield self.assertExportedCsv(items, ['egg', 'foo'], rows, ordered=False)
yield self.assertExported(items, ['egg', 'foo'], rows, ordered=False)
@defer.inlineCallbacks
def test_export_csv_feed_export_fields(self):
def test_export_feed_export_fields(self):
# FEED_EXPORT_FIELDS option allows to order export fields
# and to select a subset of fields to export, both for Items and dicts.
@ -260,8 +273,8 @@ class FeedExportTest(unittest.TestCase):
{'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
{'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
]
yield self.assertExportedCsv(items, ['foo', 'baz', 'egg'], rows,
settings=settings, ordered=True)
yield self.assertExported(items, ['foo', 'baz', 'egg'], rows,
settings=settings, ordered=True)
# export a subset of columns
settings = {'FEED_EXPORT_FIELDS': 'egg,baz'}
@ -269,5 +282,5 @@ class FeedExportTest(unittest.TestCase):
{'egg': 'spam1', 'baz': ''},
{'egg': 'spam2', 'baz': 'quux2'}
]
yield self.assertExportedCsv(items, ['egg', 'baz'], rows,
settings=settings, ordered=True)
yield self.assertExported(items, ['egg', 'baz'], rows,
settings=settings, ordered=True)