feat(agent): scrape linked pages to ground answers in real content

Adds `_fetch_page_content()`, which downloads a URL with a
browser-like User-Agent and uses BeautifulSoup with the lxml
parser to strip scripts, navigation, and other non-content
markup, returning up to 3000 characters of readable text.

The response validator now also compares generated replies
against this scraped text, flagging named entities and claims
that do not appear in the retrieved page content.

Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0
for parser compatibility.
This commit is contained in:
2026-10-26 16:48:29 -03:00
parent 9eed02645b
commit 723ca1506b
231 changed files with 20508 additions and 5330 deletions
@@ -18,7 +18,6 @@
#include <stdio.h>
#include <libxml/encoding.h>
#include <libxml/parser.h>
#include <libxml/tree.h>
#include <libxml/xmlerror.h>
@@ -157,26 +156,22 @@ XMLPUBFUN void
xmlStructuredErrorFunc serror,
void *ctx);
XMLPUBFUN int
xmlSchemaGetParserErrors (xmlSchemaParserCtxtPtr ctxt,
xmlSchemaGetParserErrors(xmlSchemaParserCtxtPtr ctxt,
xmlSchemaValidityErrorFunc * err,
xmlSchemaValidityWarningFunc * warn,
void **ctx);
XMLPUBFUN void
xmlSchemaSetResourceLoader (xmlSchemaParserCtxtPtr ctxt,
xmlResourceLoader loader,
void *data);
XMLPUBFUN int
xmlSchemaIsValid (xmlSchemaValidCtxtPtr ctxt);
xmlSchemaIsValid (xmlSchemaValidCtxtPtr ctxt);
XMLPUBFUN xmlSchemaPtr
xmlSchemaParse (xmlSchemaParserCtxtPtr ctxt);
XMLPUBFUN void
xmlSchemaFree (xmlSchemaPtr schema);
#ifdef LIBXML_DEBUG_ENABLED
#ifdef LIBXML_OUTPUT_ENABLED
XMLPUBFUN void
xmlSchemaDump (FILE *output,
xmlSchemaPtr schema);
#endif /* LIBXML_DEBUG_ENABLED */
#endif /* LIBXML_OUTPUT_ENABLED */
/*
* Interfaces for validating
*/
@@ -217,7 +212,7 @@ XMLPUBFUN int
xmlSchemaValidateStream (xmlSchemaValidCtxtPtr ctxt,
xmlParserInputBufferPtr input,
xmlCharEncoding enc,
const xmlSAXHandler *sax,
xmlSAXHandlerPtr sax,
void *user_data);
XMLPUBFUN int
xmlSchemaValidateFile (xmlSchemaValidCtxtPtr ctxt,