feat(agent): scrape linked pages to ground answers in real content

Adds `_fetch_page_content()`, which downloads a URL with a
browser-like User-Agent and uses BeautifulSoup with the lxml
parser to strip scripts, navigation, and other non-content
markup, returning up to 3000 characters of readable text.

The response validator now also compares generated replies
against this scraped text, flagging named entities and claims
that do not appear in the retrieved page content.

Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0
for parser compatibility.

venv/lib/python3.11/site-packages/lxml/includes/libxml/nanohttp.h

@@ -18,21 +18,16 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-XML_DEPRECATED
 XMLPUBFUN void
 	xmlNanoHTTPInit		(void);
-XML_DEPRECATED
 XMLPUBFUN void
 	xmlNanoHTTPCleanup	(void);
-XML_DEPRECATED
 XMLPUBFUN void
 	xmlNanoHTTPScanProxy	(const char *URL);
-XML_DEPRECATED
 XMLPUBFUN int
 	xmlNanoHTTPFetch	(const char *URL,
 				 const char *filename,
 				 char **contentType);
-XML_DEPRECATED
 XMLPUBFUN void *
 	xmlNanoHTTPMethod	(const char *URL,
 				 const char *method,
@@ -40,7 +35,6 @@ XMLPUBFUN void *
 				 char **contentType,
 				 const char *headers,
 				 int   ilen);
-XML_DEPRECATED
 XMLPUBFUN void *
 	xmlNanoHTTPMethodRedir	(const char *URL,
 				 const char *method,
@@ -49,45 +43,34 @@ XMLPUBFUN void *
 				 char **redir,
 				 const char *headers,
 				 int   ilen);
-XML_DEPRECATED
 XMLPUBFUN void *
 	xmlNanoHTTPOpen		(const char *URL,
 				 char **contentType);
-XML_DEPRECATED
 XMLPUBFUN void *
 	xmlNanoHTTPOpenRedir	(const char *URL,
 				 char **contentType,
 				 char **redir);
-XML_DEPRECATED
 XMLPUBFUN int
 	xmlNanoHTTPReturnCode	(void *ctx);
-XML_DEPRECATED
 XMLPUBFUN const char *
 	xmlNanoHTTPAuthHeader	(void *ctx);
-XML_DEPRECATED
 XMLPUBFUN const char *
 	xmlNanoHTTPRedir	(void *ctx);
-XML_DEPRECATED
 XMLPUBFUN int
 	xmlNanoHTTPContentLength( void * ctx );
-XML_DEPRECATED
 XMLPUBFUN const char *
 	xmlNanoHTTPEncoding	(void *ctx);
-XML_DEPRECATED
 XMLPUBFUN const char *
 	xmlNanoHTTPMimeType	(void *ctx);
-XML_DEPRECATED
 XMLPUBFUN int
 	xmlNanoHTTPRead		(void *ctx,
 				 void *dest,
 				 int len);
 #ifdef LIBXML_OUTPUT_ENABLED
-XML_DEPRECATED
 XMLPUBFUN int
 	xmlNanoHTTPSave		(void *ctxt,
 				 const char *filename);
 #endif /* LIBXML_OUTPUT_ENABLED */
-XML_DEPRECATED
 XMLPUBFUN void
 	xmlNanoHTTPClose	(void *ctx);
 #ifdef __cplusplus