feat(agent): scrape linked pages to ground answers in real content

Adds `_fetch_page_content()`, which downloads a URL with a
browser-like User-Agent and uses BeautifulSoup with the lxml
parser to strip scripts, navigation, and other non-content
markup, returning up to 3000 characters of readable text.

The response validator now also compares generated replies
against this scraped text, flagging named entities and claims
that do not appear in the retrieved page content.

Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0
for parser compatibility.

venv/lib/python3.11/site-packages/lxml/includes/libxml/parserInternals.h

@@ -22,12 +22,14 @@
 extern "C" {
 #endif
 
-/*
- * Backward compatibility
+/**
+ * xmlParserMaxDepth:
+ *
+ * arbitrary depth limit for the XML documents that we allow to
+ * process. This is not a limitation of the parser but a safety
+ * boundary feature, use XML_PARSE_HUGE option to override it.
  */
-#define inputPush xmlCtxtPushInput
-#define inputPop xmlCtxtPopInput
-#define xmlParserMaxDepth 256
+XMLPUBVAR unsigned int xmlParserMaxDepth;
 
 /**
  * XML_MAX_TEXT_LENGTH:
@@ -66,7 +68,7 @@ extern "C" {
  * use XML_PARSE_HUGE option to override it.
  * Introduced in 2.9.0
  */
-#define XML_MAX_DICTIONARY_LIMIT 100000000
+#define XML_MAX_DICTIONARY_LIMIT 10000000
 
 /**
  * XML_MAX_LOOKUP_LIMIT:
@@ -86,6 +88,14 @@ extern "C" {
  */
 #define XML_MAX_NAMELEN 100
 
+/**
+ * INPUT_CHUNK:
+ *
+ * The parser tries to always have that amount of input ready.
+ * One of the point is providing context when reporting errors.
+ */
+#define INPUT_CHUNK	250
+
 /************************************************************************
  *									*
  * UNICODE version of the macros.					*
@@ -245,8 +255,8 @@ extern "C" {
  * Macro to check [a-zA-Z]
  *
  */
-#define IS_ASCII_LETTER(c)	((0x61 <= ((c) | 0x20)) && \
-                                 (((c) | 0x20) <= 0x7a))
+#define IS_ASCII_LETTER(c)	(((0x41 <= (c)) && ((c) <= 0x5a)) || \
+				 ((0x61 <= (c)) && ((c) <= 0x7a)))
 
 /**
  * IS_ASCII_DIGIT:
@@ -281,10 +291,11 @@ extern "C" {
  */
 XMLPUBVAR const xmlChar xmlStringText[];
 XMLPUBVAR const xmlChar xmlStringTextNoenc[];
-XML_DEPRECATED
 XMLPUBVAR const xmlChar xmlStringComment[];
 
-XML_DEPRECATED
+/*
+ * Function to finish the work of the macros where needed.
+ */
 XMLPUBFUN int                   xmlIsLetter     (int c);
 
 /**
@@ -298,19 +309,13 @@ XMLPUBFUN xmlParserCtxtPtr
 XMLPUBFUN xmlParserCtxtPtr
 			xmlCreateMemoryParserCtxt(const char *buffer,
 						 int size);
-XML_DEPRECATED
 XMLPUBFUN xmlParserCtxtPtr
 			xmlCreateEntityParserCtxt(const xmlChar *URL,
 						 const xmlChar *ID,
 						 const xmlChar *base);
-XMLPUBFUN void
-			xmlCtxtErrMemory	(xmlParserCtxtPtr ctxt);
 XMLPUBFUN int
 			xmlSwitchEncoding	(xmlParserCtxtPtr ctxt,
 						 xmlCharEncoding enc);
-XMLPUBFUN int
-			xmlSwitchEncodingName	(xmlParserCtxtPtr ctxt,
-						 const char *encoding);
 XMLPUBFUN int
 			xmlSwitchToEncoding	(xmlParserCtxtPtr ctxt,
 					 xmlCharEncodingHandlerPtr handler);
@@ -330,16 +335,9 @@ XML_DEPRECATED
 XMLPUBFUN xmlParserInputPtr
 			xmlNewEntityInputStream	(xmlParserCtxtPtr ctxt,
 						 xmlEntityPtr entity);
-XMLPUBFUN int
-			xmlCtxtPushInput	(xmlParserCtxtPtr ctxt,
-						 xmlParserInputPtr input);
-XMLPUBFUN xmlParserInputPtr
-			xmlCtxtPopInput		(xmlParserCtxtPtr ctxt);
-XML_DEPRECATED
 XMLPUBFUN int
 			xmlPushInput		(xmlParserCtxtPtr ctxt,
 						 xmlParserInputPtr input);
-XML_DEPRECATED
 XMLPUBFUN xmlChar
 			xmlPopInput		(xmlParserCtxtPtr ctxt);
 XMLPUBFUN void
@@ -505,7 +503,6 @@ XMLPUBFUN void
 XML_DEPRECATED
 XMLPUBFUN void
 			xmlParseMisc		(xmlParserCtxtPtr ctxt);
-XML_DEPRECATED
 XMLPUBFUN void
 			xmlParseExternalSubset	(xmlParserCtxtPtr ctxt,
 						 const xmlChar *ExternalID,
@@ -553,6 +550,23 @@ XMLPUBFUN xmlChar *
 						 xmlChar  end2,
 						 xmlChar end3);
 
+/*
+ * Generated by MACROS on top of parser.c c.f. PUSH_AND_POP.
+ */
+XML_DEPRECATED
+XMLPUBFUN int			nodePush		(xmlParserCtxtPtr ctxt,
+						 xmlNodePtr value);
+XML_DEPRECATED
+XMLPUBFUN xmlNodePtr		nodePop			(xmlParserCtxtPtr ctxt);
+XMLPUBFUN int			inputPush		(xmlParserCtxtPtr ctxt,
+						 xmlParserInputPtr value);
+XMLPUBFUN xmlParserInputPtr	inputPop		(xmlParserCtxtPtr ctxt);
+XML_DEPRECATED
+XMLPUBFUN const xmlChar *	namePop			(xmlParserCtxtPtr ctxt);
+XML_DEPRECATED
+XMLPUBFUN int			namePush		(xmlParserCtxtPtr ctxt,
+						 const xmlChar *value);
+
 /*
  * other commodities shared between parser.c and parserInternals.
  */
@@ -573,10 +587,8 @@ XMLPUBFUN int			xmlCheckLanguageID	(const xmlChar *lang);
 XML_DEPRECATED
 XMLPUBFUN int			xmlCurrentChar		(xmlParserCtxtPtr ctxt,
 						 int *len);
-XML_DEPRECATED
 XMLPUBFUN int		xmlCopyCharMultiByte	(xmlChar *out,
 						 int val);
-XML_DEPRECATED
 XMLPUBFUN int			xmlCopyChar		(int len,
 						 xmlChar *out,
 						 int val);
@@ -585,6 +597,66 @@ XMLPUBFUN void			xmlNextChar		(xmlParserCtxtPtr ctxt);
 XML_DEPRECATED
 XMLPUBFUN void			xmlParserInputShrink	(xmlParserInputPtr in);
 
+/*
+ * Specific function to keep track of entities references
+ * and used by the XSLT debugger.
+ */
+#ifdef LIBXML_LEGACY_ENABLED
+/**
+ * xmlEntityReferenceFunc:
+ * @ent: the entity
+ * @firstNode:  the fist node in the chunk
+ * @lastNode:  the last nod in the chunk
+ *
+ * Callback function used when one needs to be able to track back the
+ * provenance of a chunk of nodes inherited from an entity replacement.
+ */
+typedef	void	(*xmlEntityReferenceFunc)	(xmlEntityPtr ent,
+						 xmlNodePtr firstNode,
+						 xmlNodePtr lastNode);
+
+XML_DEPRECATED
+XMLPUBFUN void		xmlSetEntityReferenceFunc	(xmlEntityReferenceFunc func);
+
+XML_DEPRECATED
+XMLPUBFUN xmlChar *
+			xmlParseQuotedString	(xmlParserCtxtPtr ctxt);
+XML_DEPRECATED
+XMLPUBFUN void
+                        xmlParseNamespace       (xmlParserCtxtPtr ctxt);
+XML_DEPRECATED
+XMLPUBFUN xmlChar *
+			xmlNamespaceParseNSDef	(xmlParserCtxtPtr ctxt);
+XML_DEPRECATED
+XMLPUBFUN xmlChar *
+			xmlScanName		(xmlParserCtxtPtr ctxt);
+XML_DEPRECATED
+XMLPUBFUN xmlChar *
+			xmlNamespaceParseNCName	(xmlParserCtxtPtr ctxt);
+XML_DEPRECATED
+XMLPUBFUN void	xmlParserHandleReference(xmlParserCtxtPtr ctxt);
+XML_DEPRECATED
+XMLPUBFUN xmlChar *
+			xmlNamespaceParseQName	(xmlParserCtxtPtr ctxt,
+						 xmlChar **prefix);
+/**
+ * Entities
+ */
+XML_DEPRECATED
+XMLPUBFUN xmlChar *
+		xmlDecodeEntities		(xmlParserCtxtPtr ctxt,
+						 int len,
+						 int what,
+						 xmlChar end,
+						 xmlChar  end2,
+						 xmlChar end3);
+XML_DEPRECATED
+XMLPUBFUN void
+			xmlHandleEntity		(xmlParserCtxtPtr ctxt,
+						 xmlEntityPtr entity);
+
+#endif /* LIBXML_LEGACY_ENABLED */
+
 #ifdef __cplusplus
 }
 #endif