feat(agent): scrape linked pages to ground answers in real content

Adds `_fetch_page_content()`, which downloads a URL with a
browser-like User-Agent and uses BeautifulSoup with the lxml
parser to strip scripts, navigation, and other non-content
markup, returning up to 3000 characters of readable text.

The response validator now also compares generated replies
against this scraped text, flagging named entities and claims
that do not appear in the retrieved page content.

Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0
for parser compatibility.
This commit is contained in:
2026-10-26 16:48:29 -03:00
parent 9eed02645b
commit 723ca1506b
231 changed files with 20508 additions and 5330 deletions
@@ -9,11 +9,6 @@
#define __XML_EXPORTS_H__
/** DOC_DISABLE */
/*
* Symbol visibility
*/
#if defined(_WIN32) || defined(__CYGWIN__)
#ifdef LIBXML_STATIC
#define XMLPUBLIC
@@ -25,71 +20,30 @@
#else /* not Windows */
#define XMLPUBLIC
#endif /* platform switch */
/** DOC_ENABLE */
/*
* XMLPUBFUN:
*
* Macro which declares an exportable function
*/
#define XMLPUBFUN XMLPUBLIC
/**
* XMLPUBVAR:
*
* Macro which declares an exportable variable
*/
#define XMLPUBVAR XMLPUBLIC extern
/** DOC_DISABLE */
/* Compatibility */
#define XMLCALL
#define XMLCDECL
#ifndef LIBXML_DLL_IMPORT
#define LIBXML_DLL_IMPORT XMLPUBVAR
#endif
/*
* Attributes
*/
#if !defined(__clang__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
#define LIBXML_ATTR_ALLOC_SIZE(x) __attribute__((alloc_size(x)))
#else
#define LIBXML_ATTR_ALLOC_SIZE(x)
#endif
#if __GNUC__ * 100 + __GNUC_MINOR__ >= 303
#define LIBXML_ATTR_FORMAT(fmt,args) \
__attribute__((__format__(__printf__,fmt,args)))
#else
#define LIBXML_ATTR_FORMAT(fmt,args)
#endif
#ifndef XML_DEPRECATED
#if defined(IN_LIBXML)
#define XML_DEPRECATED
#elif __GNUC__ * 100 + __GNUC_MINOR__ >= 301
#define XML_DEPRECATED __attribute__((deprecated))
#elif defined(_MSC_VER) && _MSC_VER >= 1400
/* Available since Visual Studio 2005 */
#define XML_DEPRECATED __declspec(deprecated)
#else
#define XML_DEPRECATED
#endif
#endif
#ifndef XML_DEPRECATED_MEMBER
#if defined(IN_LIBXML)
#define XML_DEPRECATED_MEMBER
#elif __GNUC__ * 100 + __GNUC_MINOR__ >= 301
#define XML_DEPRECATED_MEMBER __attribute__((deprecated))
#else
#define XML_DEPRECATED_MEMBER
#endif
#endif
/*
* Originally declared in xmlversion.h which is generated
*/
#ifdef __cplusplus
extern "C" {
#endif
XMLPUBFUN void xmlCheckVersion(int version);
#ifdef __cplusplus
}
#if !defined(LIBXML_DLL_IMPORT)
#define LIBXML_DLL_IMPORT XMLPUBVAR
#endif
/** DOC_ENABLE */
#endif /* __XML_EXPORTS_H__ */