scrapy/tests/test_http_headers.py

import copy
import unittest

from scrapy.http import Headers


class HeadersTest(unittest.TestCase):
    def assertSortedEqual(self, first, second, msg=None):
        return self.assertEqual(sorted(first), sorted(second), msg)

    def test_basics(self):
        h = Headers({"Content-Type": "text/html", "Content-Length": 1234})
        assert h["Content-Type"]
        assert h["Content-Length"]

        self.assertRaises(KeyError, h.__getitem__, "Accept")
        self.assertEqual(h.get("Accept"), None)
        self.assertEqual(h.getlist("Accept"), [])

        self.assertEqual(h.get("Accept", "*/*"), b"*/*")
        self.assertEqual(h.getlist("Accept", "*/*"), [b"*/*"])
        self.assertEqual(
            h.getlist("Accept", ["text/html", "images/jpeg"]),
            [b"text/html", b"images/jpeg"],
        )

    def test_single_value(self):
        h = Headers()
        h["Content-Type"] = "text/html"
        self.assertEqual(h["Content-Type"], b"text/html")
        self.assertEqual(h.get("Content-Type"), b"text/html")
        self.assertEqual(h.getlist("Content-Type"), [b"text/html"])

    def test_multivalue(self):
        h = Headers()
        h["X-Forwarded-For"] = hlist = ["ip1", "ip2"]
        self.assertEqual(h["X-Forwarded-For"], b"ip2")
        self.assertEqual(h.get("X-Forwarded-For"), b"ip2")
        self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1", b"ip2"])
        assert h.getlist("X-Forwarded-For") is not hlist

    def test_multivalue_for_one_header(self):
        h = Headers((("a", "b"), ("a", "c")))
        self.assertEqual(h["a"], b"c")
        self.assertEqual(h.get("a"), b"c")
        self.assertEqual(h.getlist("a"), [b"b", b"c"])

    def test_encode_utf8(self):
        h = Headers({"key": "\xa3"}, encoding="utf-8")
        key, val = dict(h).popitem()
        assert isinstance(key, bytes), key
        assert isinstance(val[0], bytes), val[0]
        self.assertEqual(val[0], b"\xc2\xa3")

    def test_encode_latin1(self):
        h = Headers({"key": "\xa3"}, encoding="latin1")
        key, val = dict(h).popitem()
        self.assertEqual(val[0], b"\xa3")

    def test_encode_multiple(self):
        h = Headers({"key": ["\xa3"]}, encoding="utf-8")
        key, val = dict(h).popitem()
        self.assertEqual(val[0], b"\xc2\xa3")

    def test_delete_and_contains(self):
        h = Headers()
        h["Content-Type"] = "text/html"
        assert "Content-Type" in h
        del h["Content-Type"]
        assert "Content-Type" not in h

    def test_setdefault(self):
        h = Headers()
        hlist = ["ip1", "ip2"]
        olist = h.setdefault("X-Forwarded-For", hlist)
        assert h.getlist("X-Forwarded-For") is not hlist
        assert h.getlist("X-Forwarded-For") is olist

        h = Headers()
        olist = h.setdefault("X-Forwarded-For", "ip1")
        self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1"])
        assert h.getlist("X-Forwarded-For") is olist

    def test_iterables(self):
        idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]}

        h = Headers(idict)
        self.assertDictEqual(
            dict(h),
            {b"Content-Type": [b"text/html"], b"X-Forwarded-For": [b"ip1", b"ip2"]},
        )
        self.assertSortedEqual(h.keys(), [b"X-Forwarded-For", b"Content-Type"])
        self.assertSortedEqual(
            h.items(),
            [(b"X-Forwarded-For", [b"ip1", b"ip2"]), (b"Content-Type", [b"text/html"])],
        )
        self.assertSortedEqual(h.values(), [b"ip2", b"text/html"])

    def test_update(self):
        h = Headers()
        h.update({"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]})
        self.assertEqual(h.getlist("Content-Type"), [b"text/html"])
        self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1", b"ip2"])

    def test_copy(self):
        h1 = Headers({"header1": ["value1", "value2"]})
        h2 = copy.copy(h1)
        self.assertEqual(h1, h2)
        self.assertEqual(h1.getlist("header1"), h2.getlist("header1"))
        assert h1.getlist("header1") is not h2.getlist("header1")
        assert isinstance(h2, Headers)

    def test_appendlist(self):
        h1 = Headers({"header1": "value1"})
        h1.appendlist("header1", "value3")
        self.assertEqual(h1.getlist("header1"), [b"value1", b"value3"])

        h1 = Headers()
        h1.appendlist("header1", "value1")
        h1.appendlist("header1", "value3")
        self.assertEqual(h1.getlist("header1"), [b"value1", b"value3"])

    def test_setlist(self):
        h1 = Headers({"header1": "value1"})
        self.assertEqual(h1.getlist("header1"), [b"value1"])
        h1.setlist("header1", [b"value2", b"value3"])
        self.assertEqual(h1.getlist("header1"), [b"value2", b"value3"])

    def test_setlistdefault(self):
        h1 = Headers({"header1": "value1"})
        h1.setlistdefault("header1", ["value2", "value3"])
        h1.setlistdefault("header2", ["value2", "value3"])
        self.assertEqual(h1.getlist("header1"), [b"value1"])
        self.assertEqual(h1.getlist("header2"), [b"value2", b"value3"])

    def test_none_value(self):
        h1 = Headers()
        h1["foo"] = "bar"
        h1["foo"] = None
        h1.setdefault("foo", "bar")
        self.assertEqual(h1.get("foo"), None)
        self.assertEqual(h1.getlist("foo"), [])

    def test_int_value(self):
        h1 = Headers({"hey": 5})
        h1["foo"] = 1
        h1.setdefault("bar", 2)
        h1.setlist("buz", [1, "dos", 3])
        self.assertEqual(h1.getlist("foo"), [b"1"])
        self.assertEqual(h1.getlist("bar"), [b"2"])
        self.assertEqual(h1.getlist("buz"), [b"1", b"dos", b"3"])
        self.assertEqual(h1.getlist("hey"), [b"5"])

    def test_invalid_value(self):
        self.assertRaisesRegex(
            TypeError, "Unsupported value type", Headers, {"foo": object()}
        )
        self.assertRaisesRegex(
            TypeError, "Unsupported value type", Headers().__setitem__, "foo", object()
        )
        self.assertRaisesRegex(
            TypeError, "Unsupported value type", Headers().setdefault, "foo", object()
        )
        self.assertRaisesRegex(
            TypeError, "Unsupported value type", Headers().setlist, "foo", [object()]
        )