feat(agent): scrape linked pages to ground answers in real content

Adds `_fetch_page_content()`, which downloads a URL with a
browser-like User-Agent and uses BeautifulSoup with the lxml
parser to strip scripts, navigation, and other non-content
markup, returning up to 3000 characters of readable text.

The response validator now also compares generated replies
against this scraped text, flagging named entities and claims
that do not appear in the retrieved page content.

Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0
for parser compatibility.

venv/lib/python3.11/site-packages/lxml/includes/libxml/entities.h

@@ -11,12 +11,10 @@
 #ifndef __XML_ENTITIES_H__
 #define __XML_ENTITIES_H__
 
-/** DOC_DISABLE */
 #include <libxml/xmlversion.h>
 #define XML_TREE_INTERNALS
 #include <libxml/tree.h>
 #undef XML_TREE_INTERNALS
-/** DOC_ENABLE */
 
 #ifdef __cplusplus
 extern "C" {
@@ -59,7 +57,7 @@ struct _xmlEntity {
 
     struct _xmlEntity     *nexte;	/* unused */
     const xmlChar           *URI;	/* the full URI as computed */
-    int                    owner;	/* unused */
+    int                    owner;	/* does the entity own the childrens */
     int                    flags;       /* various flags */
     unsigned long   expandedSize;       /* expanded size */
 };
@@ -76,6 +74,12 @@ typedef xmlEntitiesTable *xmlEntitiesTablePtr;
  * External functions:
  */
 
+#ifdef LIBXML_LEGACY_ENABLED
+XML_DEPRECATED
+XMLPUBFUN void
+		xmlInitializePredefinedEntities	(void);
+#endif /* LIBXML_LEGACY_ENABLED */
+
 XMLPUBFUN xmlEntityPtr
 			xmlNewEntity		(xmlDocPtr doc,
 						 const xmlChar *name,
@@ -85,15 +89,6 @@ XMLPUBFUN xmlEntityPtr
 						 const xmlChar *content);
 XMLPUBFUN void
 			xmlFreeEntity		(xmlEntityPtr entity);
-XMLPUBFUN int
-			xmlAddEntity		(xmlDocPtr doc,
-						 int extSubset,
-						 const xmlChar *name,
-						 int type,
-						 const xmlChar *ExternalID,
-						 const xmlChar *SystemID,
-						 const xmlChar *content,
-						 xmlEntityPtr *out);
 XMLPUBFUN xmlEntityPtr
 			xmlAddDocEntity		(xmlDocPtr doc,
 						 const xmlChar *name,
@@ -119,6 +114,12 @@ XMLPUBFUN xmlEntityPtr
 XMLPUBFUN xmlEntityPtr
 			xmlGetParameterEntity	(xmlDocPtr doc,
 						 const xmlChar *name);
+#ifdef LIBXML_LEGACY_ENABLED
+XML_DEPRECATED
+XMLPUBFUN const xmlChar *
+			xmlEncodeEntities	(xmlDocPtr doc,
+						 const xmlChar *input);
+#endif /* LIBXML_LEGACY_ENABLED */
 XMLPUBFUN xmlChar *
 			xmlEncodeEntitiesReentrant(xmlDocPtr doc,
 						 const xmlChar *input);
@@ -127,8 +128,10 @@ XMLPUBFUN xmlChar *
 						 const xmlChar *input);
 XMLPUBFUN xmlEntitiesTablePtr
 			xmlCreateEntitiesTable	(void);
+#ifdef LIBXML_TREE_ENABLED
 XMLPUBFUN xmlEntitiesTablePtr
 			xmlCopyEntitiesTable	(xmlEntitiesTablePtr table);
+#endif /* LIBXML_TREE_ENABLED */
 XMLPUBFUN void
 			xmlFreeEntitiesTable	(xmlEntitiesTablePtr table);
 #ifdef LIBXML_OUTPUT_ENABLED
@@ -139,6 +142,11 @@ XMLPUBFUN void
 			xmlDumpEntityDecl	(xmlBufferPtr buf,
 						 xmlEntityPtr ent);
 #endif /* LIBXML_OUTPUT_ENABLED */
+#ifdef LIBXML_LEGACY_ENABLED
+XMLPUBFUN void
+			xmlCleanupPredefinedEntities(void);
+#endif /* LIBXML_LEGACY_ENABLED */
+
 
 #ifdef __cplusplus
 }