feat(agent): scrape linked pages to ground answers in real content

Adds `_fetch_page_content()`, which downloads a URL with a
browser-like User-Agent and uses BeautifulSoup with the lxml
parser to strip scripts, navigation, and other non-content
markup, returning up to 3000 characters of readable text.

The response validator now also compares generated replies
against this scraped text, flagging named entities and claims
that do not appear in the retrieved page content.

Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0
for parser compatibility.

venv/lib/python3.11/site-packages/lxml/iterparse.pxi

@@ -1,13 +1,13 @@
 # iterparse -- event-driven parsing
 
+DEF __ITERPARSE_CHUNK_SIZE = 32768
+
 cdef class iterparse:
     """iterparse(self, source, events=("end",), tag=None, \
                   attribute_defaults=False, dtd_validation=False, \
                   load_dtd=False, no_network=True, remove_blank_text=False, \
-                  compact=True, resolve_entities='internal', remove_comments=False, \
-                  remove_pis=False, strip_cdata=True, encoding=None, \
-                  html=False, recover=None, huge_tree=False, schema=None, \
-                  chunk_size=32768)
+                  remove_comments=False, remove_pis=False, encoding=None, \
+                  html=False, recover=None, huge_tree=False, schema=None)
 
     Incremental parser.
 
@@ -42,11 +42,9 @@ cdef class iterparse:
      - remove_blank_text: discard blank text nodes
      - remove_comments: discard comments
      - remove_pis: discard processing instructions
-     - strip_cdata: replace CDATA sections by normal text content (default:
-       True for XML, ignored otherwise)
+     - strip_cdata: replace CDATA sections by normal text content (default: True)
      - compact: safe memory for short text content (default: True)
-     - resolve_entities: replace entities by their text value
-       (default: 'internal' only; True before lxml 6.1)
+     - resolve_entities: replace entities by their text value (default: True)
      - huge_tree: disable security restrictions and support very deep trees
                   and very long text content (only affects libxml2 2.7+)
      - html: parse input as HTML (default: XML)
@@ -56,8 +54,6 @@ cdef class iterparse:
     Other keyword arguments:
      - encoding: override the document encoding
      - schema: an XMLSchema to validate against
-     - chunk_size: the number of bytes to read from the 'source' in one chunk
-                   (default: 32768)
     """
     cdef _FeedParser _parser
     cdef object _tag
@@ -66,16 +62,15 @@ cdef class iterparse:
     cdef object _source
     cdef object _filename
     cdef object _error
-    cdef object _chunk_size
     cdef bint _close_source_after_read
 
     def __init__(self, source, events=("end",), *, tag=None,
                  attribute_defaults=False, dtd_validation=False,
                  load_dtd=False, no_network=True, remove_blank_text=False,
-                 compact=True, resolve_entities='internal', remove_comments=False,
+                 compact=True, resolve_entities=True, remove_comments=False,
                  remove_pis=False, strip_cdata=True, encoding=None,
                  html=False, recover=None, huge_tree=False, collect_ids=True,
-                 XMLSchema schema=None, int chunk_size=32768):
+                 XMLSchema schema=None):
         if not hasattr(source, 'read'):
             source = _getFSPathOrObject(source)
             self._filename = source
@@ -102,6 +97,7 @@ cdef class iterparse:
                 remove_blank_text=remove_blank_text,
                 remove_comments=remove_comments,
                 remove_pis=remove_pis,
+                strip_cdata=strip_cdata,
                 no_network=no_network,
                 target=None,  # TODO
                 schema=schema,
@@ -128,7 +124,6 @@ cdef class iterparse:
                 target=None,  # TODO
                 compact=compact)
 
-        self._chunk_size = chunk_size
         self._events = parser.read_events()
         self._parser = parser
 
@@ -220,7 +215,7 @@ cdef class iterparse:
 
     @cython.final
     cdef bint _read_more_events(self, _SaxParserContext context) except -123:
-        data = self._source.read(self._chunk_size)
+        data = self._source.read(__ITERPARSE_CHUNK_SIZE)
         if not isinstance(data, bytes):
             self._close_source()
             raise TypeError("reading file objects must return bytes objects")