mirror of
https://github.com/scrapy/scrapy.git
synced 2025-03-13 16:44:50 +00:00
116 lines
4.0 KiB
Python
116 lines
4.0 KiB
Python
import unittest
|
|
|
|
import pytest
|
|
from w3lib.http import basic_auth_header
|
|
|
|
from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
|
|
from scrapy.exceptions import ScrapyDeprecationWarning
|
|
from scrapy.http import Request
|
|
from scrapy.spiders import Spider
|
|
|
|
|
|
class TestSpiderLegacy(Spider):
|
|
http_user = "foo"
|
|
http_pass = "bar"
|
|
|
|
|
|
class TestSpider(Spider):
|
|
http_user = "foo"
|
|
http_pass = "bar"
|
|
http_auth_domain = "example.com"
|
|
|
|
|
|
class TestSpiderAny(Spider):
|
|
http_user = "foo"
|
|
http_pass = "bar"
|
|
http_auth_domain = None
|
|
|
|
|
|
class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.spider = TestSpiderLegacy("foo")
|
|
|
|
def test_auth(self):
|
|
with pytest.warns(
|
|
ScrapyDeprecationWarning,
|
|
match="Using HttpAuthMiddleware without http_auth_domain is deprecated",
|
|
):
|
|
mw = HttpAuthMiddleware()
|
|
mw.spider_opened(self.spider)
|
|
|
|
# initial request, sets the domain and sends the header
|
|
req = Request("http://example.com/")
|
|
assert mw.process_request(req, self.spider) is None
|
|
self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
|
|
|
|
# subsequent request to the same domain, should send the header
|
|
req = Request("http://example.com/")
|
|
assert mw.process_request(req, self.spider) is None
|
|
self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
|
|
|
|
# subsequent request to a different domain, shouldn't send the header
|
|
req = Request("http://example-noauth.com/")
|
|
assert mw.process_request(req, self.spider) is None
|
|
self.assertNotIn("Authorization", req.headers)
|
|
|
|
def test_auth_already_set(self):
|
|
with pytest.warns(
|
|
ScrapyDeprecationWarning,
|
|
match="Using HttpAuthMiddleware without http_auth_domain is deprecated",
|
|
):
|
|
mw = HttpAuthMiddleware()
|
|
mw.spider_opened(self.spider)
|
|
req = Request("http://example.com/", headers=dict(Authorization="Digest 123"))
|
|
assert mw.process_request(req, self.spider) is None
|
|
self.assertEqual(req.headers["Authorization"], b"Digest 123")
|
|
|
|
|
|
class HttpAuthMiddlewareTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.mw = HttpAuthMiddleware()
|
|
self.spider = TestSpider("foo")
|
|
self.mw.spider_opened(self.spider)
|
|
|
|
def tearDown(self):
|
|
del self.mw
|
|
|
|
def test_no_auth(self):
|
|
req = Request("http://example-noauth.com/")
|
|
assert self.mw.process_request(req, self.spider) is None
|
|
self.assertNotIn("Authorization", req.headers)
|
|
|
|
def test_auth_domain(self):
|
|
req = Request("http://example.com/")
|
|
assert self.mw.process_request(req, self.spider) is None
|
|
self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
|
|
|
|
def test_auth_subdomain(self):
|
|
req = Request("http://foo.example.com/")
|
|
assert self.mw.process_request(req, self.spider) is None
|
|
self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
|
|
|
|
def test_auth_already_set(self):
|
|
req = Request("http://example.com/", headers=dict(Authorization="Digest 123"))
|
|
assert self.mw.process_request(req, self.spider) is None
|
|
self.assertEqual(req.headers["Authorization"], b"Digest 123")
|
|
|
|
|
|
class HttpAuthAnyMiddlewareTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.mw = HttpAuthMiddleware()
|
|
self.spider = TestSpiderAny("foo")
|
|
self.mw.spider_opened(self.spider)
|
|
|
|
def tearDown(self):
|
|
del self.mw
|
|
|
|
def test_auth(self):
|
|
req = Request("http://example.com/")
|
|
assert self.mw.process_request(req, self.spider) is None
|
|
self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))
|
|
|
|
def test_auth_already_set(self):
|
|
req = Request("http://example.com/", headers=dict(Authorization="Digest 123"))
|
|
assert self.mw.process_request(req, self.spider) is None
|
|
self.assertEqual(req.headers["Authorization"], b"Digest 123")
|