feat(agent): scrape linked pages to ground answers in real content

Adds `_fetch_page_content()`, which downloads a URL with a
browser-like User-Agent and uses BeautifulSoup with the lxml
parser to strip scripts, navigation, and other non-content
markup, returning up to 3000 characters of readable text.

The response validator now also compares generated replies
against this scraped text, flagging named entities and claims
that do not appear in the retrieved page content.

Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0
for parser compatibility.

venv/lib/python3.11/site-packages/lxml/sax.py

@@ -1,3 +1,5 @@
+# cython: language_level=2
+
 """
 SAX-based adapter to copy trees from/to the Python standard library.
 
@@ -16,13 +18,6 @@ from lxml import etree
 from lxml.etree import ElementTree, SubElement
 from lxml.etree import Comment, ProcessingInstruction
 
-try:
-    from types import GenericAlias as _GenericAlias
-except ImportError:
-    # Python 3.8 - we only need this as return value from "__class_getitem__"
-    def _GenericAlias(cls, item):
-        return f"{cls.__name__}[{item.__name__}]"
-
 
 class SaxError(etree.LxmlError):
     """General SAX error.
@@ -30,7 +25,7 @@ class SaxError(etree.LxmlError):
 
 
 def _getNsTag(tag):
-    if tag[0] == '{' and '}' in tag:
+    if tag[0] == '{':
         return tuple(tag[1:].split('}', 1))
     else:
         return None, tag
@@ -150,18 +145,13 @@ class ElementTreeContentHandler(ContentHandler):
         try:
             # if there already is a child element, we must append to its tail
             last_element = last_element[-1]
+            last_element.tail = (last_element.tail or '') + data
         except IndexError:
             # otherwise: append to the text
             last_element.text = (last_element.text or '') + data
-        else:
-            last_element.tail = (last_element.tail or '') + data
 
     ignorableWhitespace = characters
 
-    # Allow subscripting sax.ElementTreeContentHandler in type annotations (PEP 560)
-    def __class_getitem__(cls, item):
-        return _GenericAlias(cls, item)
-
 
 class ElementTreeProducer:
     """Produces SAX events for an element and children.