feat(agent): scrape linked pages to ground answers in real content
Adds `_fetch_page_content()`, which downloads a URL with a browser-like User-Agent and uses BeautifulSoup with the lxml parser to strip scripts, navigation, and other non-content markup, returning up to 3000 characters of readable text. The response validator now also compares generated replies against this scraped text, flagging named entities and claims that do not appear in the retrieved page content. Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0 for parser compatibility.
This commit is contained in:
@@ -18,7 +18,6 @@
|
||||
|
||||
#include <stdio.h>
|
||||
#include <libxml/encoding.h>
|
||||
#include <libxml/parser.h>
|
||||
#include <libxml/tree.h>
|
||||
#include <libxml/xmlerror.h>
|
||||
|
||||
@@ -157,26 +156,22 @@ XMLPUBFUN void
|
||||
xmlStructuredErrorFunc serror,
|
||||
void *ctx);
|
||||
XMLPUBFUN int
|
||||
xmlSchemaGetParserErrors (xmlSchemaParserCtxtPtr ctxt,
|
||||
xmlSchemaGetParserErrors(xmlSchemaParserCtxtPtr ctxt,
|
||||
xmlSchemaValidityErrorFunc * err,
|
||||
xmlSchemaValidityWarningFunc * warn,
|
||||
void **ctx);
|
||||
XMLPUBFUN void
|
||||
xmlSchemaSetResourceLoader (xmlSchemaParserCtxtPtr ctxt,
|
||||
xmlResourceLoader loader,
|
||||
void *data);
|
||||
XMLPUBFUN int
|
||||
xmlSchemaIsValid (xmlSchemaValidCtxtPtr ctxt);
|
||||
xmlSchemaIsValid (xmlSchemaValidCtxtPtr ctxt);
|
||||
|
||||
XMLPUBFUN xmlSchemaPtr
|
||||
xmlSchemaParse (xmlSchemaParserCtxtPtr ctxt);
|
||||
XMLPUBFUN void
|
||||
xmlSchemaFree (xmlSchemaPtr schema);
|
||||
#ifdef LIBXML_DEBUG_ENABLED
|
||||
#ifdef LIBXML_OUTPUT_ENABLED
|
||||
XMLPUBFUN void
|
||||
xmlSchemaDump (FILE *output,
|
||||
xmlSchemaPtr schema);
|
||||
#endif /* LIBXML_DEBUG_ENABLED */
|
||||
#endif /* LIBXML_OUTPUT_ENABLED */
|
||||
/*
|
||||
* Interfaces for validating
|
||||
*/
|
||||
@@ -217,7 +212,7 @@ XMLPUBFUN int
|
||||
xmlSchemaValidateStream (xmlSchemaValidCtxtPtr ctxt,
|
||||
xmlParserInputBufferPtr input,
|
||||
xmlCharEncoding enc,
|
||||
const xmlSAXHandler *sax,
|
||||
xmlSAXHandlerPtr sax,
|
||||
void *user_data);
|
||||
XMLPUBFUN int
|
||||
xmlSchemaValidateFile (xmlSchemaValidCtxtPtr ctxt,
|
||||
|
||||
Reference in New Issue
Block a user