marcocosta10000
723ca1506b
Adds `_fetch_page_content()`, which downloads a URL with a browser-like User-Agent and uses BeautifulSoup with the lxml parser to strip scripts, navigation, and other non-content markup, returning up to 3000 characters of readable text. The response validator now also compares generated replies against this scraped text, flagging named entities and claims that do not appear in the retrieved page content. Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0 for parser compatibility.
22 lines
503 B
Python
22 lines
503 B
Python
# cython: language_level=3str
|
|
|
|
"""Backward-compatibility module for lxml_html_clean"""
|
|
|
|
try:
|
|
from lxml_html_clean import *
|
|
|
|
__all__ = [
|
|
"clean_html",
|
|
"clean",
|
|
"Cleaner",
|
|
"autolink",
|
|
"autolink_html",
|
|
"word_break",
|
|
"word_break_html",
|
|
]
|
|
except ImportError:
|
|
raise ImportError(
|
|
"lxml.html.clean module is now a separate project lxml_html_clean.\n"
|
|
"Install lxml[html_clean] or lxml_html_clean directly."
|
|
) from None
|