marcocosta10000
723ca1506b
Adds `_fetch_page_content()`, which downloads a URL with a browser-like User-Agent and uses BeautifulSoup with the lxml parser to strip scripts, navigation, and other non-content markup, returning up to 3000 characters of readable text. The response validator now also compares generated replies against this scraped text, flagging named entities and claims that do not appear in the retrieved page content. Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0 for parser compatibility.
30 lines
1.1 KiB
Python
30 lines
1.1 KiB
Python
import pytest
|
|
from unittest.mock import patch
|
|
from bs4.builder import DetectsXMLParsedAsHTML
|
|
|
|
class TestDetectsXMLParsedAsHTML(object):
|
|
|
|
@pytest.mark.parametrize(
|
|
"markup,looks_like_xml",
|
|
[("No xml declaration", False),
|
|
("<html>obviously HTML</html", False),
|
|
("<?xml ><html>Actually XHTML</html>", False),
|
|
("<?xml> < html>Tricky XHTML</html>", False),
|
|
("<?xml ><no-html-tag>", True),
|
|
]
|
|
)
|
|
def test_warn_if_markup_looks_like_xml(self, markup, looks_like_xml):
|
|
# Test of our ability to guess at whether markup looks XML-ish
|
|
# _and_ not HTML-ish.
|
|
with patch('bs4.builder.DetectsXMLParsedAsHTML._warn') as mock:
|
|
for data in markup, markup.encode('utf8'):
|
|
result = DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml(
|
|
data
|
|
)
|
|
assert result == looks_like_xml
|
|
if looks_like_xml:
|
|
assert mock.called
|
|
else:
|
|
assert not mock.called
|
|
mock.reset_mock()
|