BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-24 14:04:01 +00:00
Code Releases Wiki Activity

SgmlLinkExtractor - fix for parsing <area> tag with Unicode present

Browse Source
This commit is contained in:
yakxxx 2014-08-28 18:47:49 +02:00
parent 774ab74ad2
commit e4689556f0
2 changed files with 12 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
scrapy/contrib/linkextractors/sgml.py
View File

@ -67,6 +67,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
SGMLParser.reset(self)
self.links = []
self.base_url = None
self.current_link = None
def unknown_starttag(self, tag, attrs):
if tag == 'base':

11
tests/test_contrib_linkextractors.py
View File

@ -284,6 +284,17 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
[Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',
fragment='', nofollow=False)])
def test_area_tag_with_unicode_present(self):
body = """<html><body>\xbe\xa9<map><area href="http://example.org/foo" /></map></body></html>"""
response = HtmlResponse("http://example.org", body=body, encoding='utf-8')
lx = self.extractor_cls()
lx.extract_links(response)
lx.extract_links(response)
lx.extract_links(response)
self.assertEqual(lx.extract_links(response),
[Link(url='http://example.org/foo', text=u'',
fragment='', nofollow=False)])
def test_encoded_url(self):
body = """<html><body><div><a href="?page=2">BinB</a></body></html>"""
response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')