feat(agent): scrape linked pages to ground answers in real content

Adds `_fetch_page_content()`, which downloads a URL with a
browser-like User-Agent and uses BeautifulSoup with the lxml
parser to strip scripts, navigation, and other non-content
markup, returning up to 3000 characters of readable text.

The response validator now also compares generated replies
against this scraped text, flagging named entities and claims
that do not appear in the retrieved page content.

Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0
for parser compatibility.

venv/lib/python3.11/site-packages/lxml/lxml.etree.h

@@ -1,4 +1,4 @@
-/* Generated by Cython 3.2.9 */
+/* Generated by Cython 3.0.11 */
 
 #ifndef __PYX_HAVE__lxml__etree
 #define __PYX_HAVE__lxml__etree
@@ -13,22 +13,22 @@ struct LxmlElementBase;
 struct LxmlElementClassLookup;
 struct LxmlFallbackElementClassLookup;
 
-/* "lxml/etree.pyx":456
+/* "lxml/etree.pyx":355
  * 
  * # type of a function that steps from node to node
  * ctypedef public xmlNode* (*_node_to_node_function)(xmlNode*)             # <<<<<<<<<<<<<<
  * 
  * 
-*/
+ */
 typedef xmlNode *(*_node_to_node_function)(xmlNode *);
 
-/* "lxml/etree.pyx":470
- * # Public Python API
- * 
- * @cython.final             # <<<<<<<<<<<<<<
+/* "lxml/etree.pyx":371
+ * @cython.final
  * @cython.freelist(8)
- * cdef public class _Document [ type LxmlDocumentType, object LxmlDocument ]:
-*/
+ * cdef public class _Document [ type LxmlDocumentType, object LxmlDocument ]:             # <<<<<<<<<<<<<<
+ *     """Internal base class to reference a libxml document.
+ * 
+ */
 struct LxmlDocument {
   PyObject_HEAD
   struct __pyx_vtabstruct_4lxml_5etree__Document *__pyx_vtab;
@@ -38,13 +38,13 @@ struct LxmlDocument {
   struct __pyx_obj_4lxml_5etree__BaseParser *_parser;
 };
 
-/* "lxml/etree.pyx":823
+/* "lxml/etree.pyx":720
  * 
- * 
- * @cython.no_gc_clear             # <<<<<<<<<<<<<<
- * cdef public class _Element [ type LxmlElementType, object LxmlElement ]:
+ * @cython.no_gc_clear
+ * cdef public class _Element [ type LxmlElementType, object LxmlElement ]:             # <<<<<<<<<<<<<<
  *     """Element class.
-*/
+ * 
+ */
 struct LxmlElement {
   PyObject_HEAD
   struct LxmlDocument *_doc;
@@ -52,13 +52,13 @@ struct LxmlElement {
   PyObject *_tag;
 };
 
-/* "lxml/etree.pyx":1997
+/* "lxml/etree.pyx":1894
  * 
  * 
  * cdef public class _ElementTree [ type LxmlElementTreeType,             # <<<<<<<<<<<<<<
  *                                  object LxmlElementTree ]:
  *     cdef _Document _doc
-*/
+ */
 struct LxmlElementTree {
   PyObject_HEAD
   struct __pyx_vtabstruct_4lxml_5etree__ElementTree *__pyx_vtab;
@@ -66,13 +66,13 @@ struct LxmlElementTree {
   struct LxmlElement *_context_node;
 };
 
-/* "lxml/etree.pyx":2771
+/* "lxml/etree.pyx":2668
  * 
  * 
  * cdef public class _ElementTagMatcher [ object LxmlElementTagMatcher,             # <<<<<<<<<<<<<<
  *                                        type LxmlElementTagMatcherType ]:
  *     """
-*/
+ */
 struct LxmlElementTagMatcher {
   PyObject_HEAD
   struct __pyx_vtabstruct_4lxml_5etree__ElementTagMatcher *__pyx_vtab;
@@ -82,13 +82,13 @@ struct LxmlElementTagMatcher {
   char *_name;
 };
 
-/* "lxml/etree.pyx":2802
+/* "lxml/etree.pyx":2699
  *                 self._name = NULL
  * 
  * cdef public class _ElementIterator(_ElementTagMatcher) [             # <<<<<<<<<<<<<<
  *     object LxmlElementIterator, type LxmlElementIteratorType ]:
  *     """
-*/
+ */
 struct LxmlElementIterator {
   struct LxmlElementTagMatcher __pyx_base;
   struct LxmlElement *_node;
@@ -101,7 +101,7 @@ struct LxmlElementIterator {
  * cdef public class ElementBase(_Element) [ type LxmlElementBaseType,             # <<<<<<<<<<<<<<
  *                                           object LxmlElementBase ]:
  *     """ElementBase(*children, attrib=None, nsmap=None, **_extra)
-*/
+ */
 struct LxmlElementBase {
   struct LxmlElement __pyx_base;
 };
@@ -112,7 +112,7 @@ struct LxmlElementBase {
  * ctypedef public object (*_element_class_lookup_function)(object, _Document, xmlNode*)             # <<<<<<<<<<<<<<
  * 
  * # class to store element class lookup functions
-*/
+ */
 typedef PyObject *(*_element_class_lookup_function)(PyObject *, struct LxmlDocument *, xmlNode *);
 
 /* "src/lxml/classlookup.pxi":213
@@ -121,7 +121,7 @@ typedef PyObject *(*_element_class_lookup_function)(PyObject *, struct LxmlDocum
  * cdef public class ElementClassLookup [ type LxmlElementClassLookupType,             # <<<<<<<<<<<<<<
  *                                        object LxmlElementClassLookup ]:
  *     """ElementClassLookup(self)
-*/
+ */
 struct LxmlElementClassLookup {
   PyObject_HEAD
   _element_class_lookup_function _lookup_function;
@@ -133,7 +133,7 @@ struct LxmlElementClassLookup {
  * cdef public class FallbackElementClassLookup(ElementClassLookup) \             # <<<<<<<<<<<<<<
  *          [ type LxmlFallbackElementClassLookupType,
  *            object LxmlFallbackElementClassLookup ]:
-*/
+ */
 struct LxmlFallbackElementClassLookup {
   struct LxmlElementClassLookup __pyx_base;
   struct __pyx_vtabstruct_4lxml_5etree_FallbackElementClassLookup *__pyx_vtab;
@@ -224,6 +224,9 @@ __PYX_EXTERN_C xmlNs *findOrBuildNodeNsPrefix(struct LxmlDocument *, xmlNode *,
 /* WARNING: the interface of the module init function changed in CPython 3.5. */
 /* It now returns a PyModuleDef instance instead of a PyModule instance. */
 
+#if PY_MAJOR_VERSION < 3
+PyMODINIT_FUNC initetree(void);
+#else
 /* WARNING: Use PyImport_AppendInittab("etree", PyInit_etree) instead of calling PyInit_etree directly from Python 3.5 */
 PyMODINIT_FUNC PyInit_etree(void);
 
@@ -240,5 +243,6 @@ static PyObject* __PYX_WARN_IF_PyInit_etree_INIT_CALLED(PyObject* res) {
 }
 #define PyInit_etree() __PYX_WARN_IF_PyInit_etree_INIT_CALLED(PyInit_etree())
 #endif
+#endif
 
 #endif /* !__PYX_HAVE__lxml__etree */