feat(agent): scrape linked pages to ground answers in real content
Adds `_fetch_page_content()`, which downloads a URL with a browser-like User-Agent and uses BeautifulSoup with the lxml parser to strip scripts, navigation, and other non-content markup, returning up to 3000 characters of readable text. The response validator now also compares generated replies against this scraped text, flagging named entities and claims that do not appear in the retrieved page content. Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0 for parser compatibility.
This commit is contained in:
@@ -10,26 +10,40 @@
|
||||
#ifndef __XML_VERSION_H__
|
||||
#define __XML_VERSION_H__
|
||||
|
||||
#include <libxml/xmlexports.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*
|
||||
* use those to be sure nothing nasty will happen if
|
||||
* your library and includes mismatch
|
||||
*/
|
||||
#ifndef LIBXML2_COMPILING_MSCCDEF
|
||||
XMLPUBFUN void xmlCheckVersion(int version);
|
||||
#endif /* LIBXML2_COMPILING_MSCCDEF */
|
||||
|
||||
/**
|
||||
* LIBXML_DOTTED_VERSION:
|
||||
*
|
||||
* the version string like "1.2.3"
|
||||
*/
|
||||
#define LIBXML_DOTTED_VERSION "2.14.6"
|
||||
#define LIBXML_DOTTED_VERSION "2.12.9"
|
||||
|
||||
/**
|
||||
* LIBXML_VERSION:
|
||||
*
|
||||
* the version number: 1.2.3 value is 10203
|
||||
*/
|
||||
#define LIBXML_VERSION 21406
|
||||
#define LIBXML_VERSION 21209
|
||||
|
||||
/**
|
||||
* LIBXML_VERSION_STRING:
|
||||
*
|
||||
* the version number string, 1.2.3 value is "10203"
|
||||
*/
|
||||
#define LIBXML_VERSION_STRING "21406"
|
||||
#define LIBXML_VERSION_STRING "21209"
|
||||
|
||||
/**
|
||||
* LIBXML_VERSION_EXTRA:
|
||||
@@ -44,7 +58,32 @@
|
||||
* Macro to check that the libxml version in use is compatible with
|
||||
* the version the software has been compiled against
|
||||
*/
|
||||
#define LIBXML_TEST_VERSION xmlCheckVersion(21406);
|
||||
#define LIBXML_TEST_VERSION xmlCheckVersion(21209);
|
||||
|
||||
#ifndef VMS
|
||||
#if 0
|
||||
/**
|
||||
* WITH_TRIO:
|
||||
*
|
||||
* defined if the trio support need to be configured in
|
||||
*/
|
||||
#define WITH_TRIO
|
||||
#else
|
||||
/**
|
||||
* WITHOUT_TRIO:
|
||||
*
|
||||
* defined if the trio support should not be configured in
|
||||
*/
|
||||
#define WITHOUT_TRIO
|
||||
#endif
|
||||
#else /* VMS */
|
||||
/**
|
||||
* WITH_TRIO:
|
||||
*
|
||||
* defined if the trio support need to be configured in
|
||||
*/
|
||||
#define WITH_TRIO 1
|
||||
#endif /* VMS */
|
||||
|
||||
/**
|
||||
* LIBXML_THREAD_ENABLED:
|
||||
@@ -67,9 +106,11 @@
|
||||
/**
|
||||
* LIBXML_TREE_ENABLED:
|
||||
*
|
||||
* Always enabled since 2.14.0
|
||||
* Whether the DOM like tree manipulation API support is configured in
|
||||
*/
|
||||
#if 1
|
||||
#define LIBXML_TREE_ENABLED
|
||||
#endif
|
||||
|
||||
/**
|
||||
* LIBXML_OUTPUT_ENABLED:
|
||||
@@ -125,12 +166,21 @@
|
||||
#define LIBXML_SAX1_ENABLED
|
||||
#endif
|
||||
|
||||
/**
|
||||
* LIBXML_FTP_ENABLED:
|
||||
*
|
||||
* Whether the FTP support is configured in
|
||||
*/
|
||||
#if 0
|
||||
#define LIBXML_FTP_ENABLED
|
||||
#endif
|
||||
|
||||
/**
|
||||
* LIBXML_HTTP_ENABLED:
|
||||
*
|
||||
* Whether the HTTP support is configured in
|
||||
*/
|
||||
#if 0
|
||||
#if 1
|
||||
#define LIBXML_HTTP_ENABLED
|
||||
#endif
|
||||
|
||||
@@ -155,9 +205,11 @@
|
||||
/**
|
||||
* LIBXML_LEGACY_ENABLED:
|
||||
*
|
||||
* Removed in 2.14
|
||||
* Whether the deprecated APIs are compiled in for compatibility
|
||||
*/
|
||||
#undef LIBXML_LEGACY_ENABLED
|
||||
#if 0
|
||||
#define LIBXML_LEGACY_ENABLED
|
||||
#endif
|
||||
|
||||
/**
|
||||
* LIBXML_C14N_ENABLED:
|
||||
@@ -195,6 +247,15 @@
|
||||
#define LIBXML_XPTR_ENABLED
|
||||
#endif
|
||||
|
||||
/**
|
||||
* LIBXML_XPTR_LOCS_ENABLED:
|
||||
*
|
||||
* Whether support for XPointer locations is configured in
|
||||
*/
|
||||
#if 0
|
||||
#define LIBXML_XPTR_LOCS_ENABLED
|
||||
#endif
|
||||
|
||||
/**
|
||||
* LIBXML_XINCLUDE_ENABLED:
|
||||
*
|
||||
@@ -240,12 +301,32 @@
|
||||
#define LIBXML_DEBUG_ENABLED
|
||||
#endif
|
||||
|
||||
/**
|
||||
* DEBUG_MEMORY_LOCATION:
|
||||
*
|
||||
* Whether the memory debugging is configured in
|
||||
*/
|
||||
#if 0
|
||||
#define DEBUG_MEMORY_LOCATION
|
||||
#endif
|
||||
|
||||
/**
|
||||
* LIBXML_DEBUG_RUNTIME:
|
||||
*
|
||||
* Removed
|
||||
*/
|
||||
#if 0
|
||||
#define LIBXML_DEBUG_RUNTIME
|
||||
#endif
|
||||
|
||||
/**
|
||||
* LIBXML_UNICODE_ENABLED:
|
||||
*
|
||||
* Removed in 2.14
|
||||
* Whether the Unicode related interfaces are compiled in
|
||||
*/
|
||||
#undef LIBXML_UNICODE_ENABLED
|
||||
#if 1
|
||||
#define LIBXML_UNICODE_ENABLED
|
||||
#endif
|
||||
|
||||
/**
|
||||
* LIBXML_REGEXP_ENABLED:
|
||||
@@ -266,12 +347,14 @@
|
||||
#endif
|
||||
|
||||
/**
|
||||
* LIBXML_RELAXNG_ENABLED:
|
||||
* LIBXML_EXPR_ENABLED:
|
||||
*
|
||||
* Whether the RelaxNG validation interfaces are compiled in
|
||||
* Whether the formal expressions interfaces are compiled in
|
||||
*
|
||||
* This code is unused and disabled unconditionally for now.
|
||||
*/
|
||||
#if 1
|
||||
#define LIBXML_RELAXNG_ENABLED
|
||||
#if 0
|
||||
#define LIBXML_EXPR_ENABLED
|
||||
#endif
|
||||
|
||||
/**
|
||||
@@ -325,8 +408,104 @@
|
||||
#define LIBXML_LZMA_ENABLED
|
||||
#endif
|
||||
|
||||
#include <libxml/xmlexports.h>
|
||||
#ifdef __GNUC__
|
||||
/** DOC_DISABLE */
|
||||
|
||||
#ifndef ATTRIBUTE_UNUSED
|
||||
# if ((__GNUC__ > 2) || ((__GNUC__ == 2) && (__GNUC_MINOR__ >= 7)))
|
||||
# define ATTRIBUTE_UNUSED __attribute__((unused))
|
||||
# else
|
||||
# define ATTRIBUTE_UNUSED
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#ifndef LIBXML_ATTR_ALLOC_SIZE
|
||||
# if (!defined(__clang__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 3))))
|
||||
# define LIBXML_ATTR_ALLOC_SIZE(x) __attribute__((alloc_size(x)))
|
||||
# else
|
||||
# define LIBXML_ATTR_ALLOC_SIZE(x)
|
||||
# endif
|
||||
#else
|
||||
# define LIBXML_ATTR_ALLOC_SIZE(x)
|
||||
#endif
|
||||
|
||||
#ifndef LIBXML_ATTR_FORMAT
|
||||
# if ((__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 3)))
|
||||
# define LIBXML_ATTR_FORMAT(fmt,args) __attribute__((__format__(__printf__,fmt,args)))
|
||||
# else
|
||||
# define LIBXML_ATTR_FORMAT(fmt,args)
|
||||
# endif
|
||||
#else
|
||||
# define LIBXML_ATTR_FORMAT(fmt,args)
|
||||
#endif
|
||||
|
||||
#ifndef XML_DEPRECATED
|
||||
# if defined (IN_LIBXML) || (__GNUC__ * 100 + __GNUC_MINOR__ < 301)
|
||||
# define XML_DEPRECATED
|
||||
/* Available since at least GCC 3.1 */
|
||||
# else
|
||||
# define XML_DEPRECATED __attribute__((deprecated))
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#if defined(__clang__) || (__GNUC__ * 100 + __GNUC_MINOR__ >= 406)
|
||||
#if defined(__clang__) || (__GNUC__ * 100 + __GNUC_MINOR__ >= 800)
|
||||
#define XML_IGNORE_FPTR_CAST_WARNINGS \
|
||||
_Pragma("GCC diagnostic push") \
|
||||
_Pragma("GCC diagnostic ignored \"-Wpedantic\"") \
|
||||
_Pragma("GCC diagnostic ignored \"-Wcast-function-type\"")
|
||||
#else
|
||||
#define XML_IGNORE_FPTR_CAST_WARNINGS \
|
||||
_Pragma("GCC diagnostic push") \
|
||||
_Pragma("GCC diagnostic ignored \"-Wpedantic\"")
|
||||
#endif
|
||||
#define XML_POP_WARNINGS \
|
||||
_Pragma("GCC diagnostic pop")
|
||||
#else
|
||||
#define XML_IGNORE_FPTR_CAST_WARNINGS
|
||||
#define XML_POP_WARNINGS
|
||||
#endif
|
||||
|
||||
#else /* ! __GNUC__ */
|
||||
#define ATTRIBUTE_UNUSED
|
||||
#define LIBXML_ATTR_ALLOC_SIZE(x)
|
||||
#define LIBXML_ATTR_FORMAT(fmt,args)
|
||||
#ifndef XML_DEPRECATED
|
||||
# if defined (IN_LIBXML) || !defined (_MSC_VER)
|
||||
# define XML_DEPRECATED
|
||||
/* Available since Visual Studio 2005 */
|
||||
# elif defined (_MSC_VER) && (_MSC_VER >= 1400)
|
||||
# define XML_DEPRECATED __declspec(deprecated)
|
||||
# endif
|
||||
#endif
|
||||
#if defined (_MSC_VER) && (_MSC_VER >= 1400)
|
||||
# define XML_IGNORE_FPTR_CAST_WARNINGS __pragma(warning(push))
|
||||
#else
|
||||
# define XML_IGNORE_FPTR_CAST_WARNINGS
|
||||
#endif
|
||||
#ifndef XML_POP_WARNINGS
|
||||
# if defined (_MSC_VER) && (_MSC_VER >= 1400)
|
||||
# define XML_POP_WARNINGS __pragma(warning(pop))
|
||||
# else
|
||||
# define XML_POP_WARNINGS
|
||||
# endif
|
||||
#endif
|
||||
#endif /* __GNUC__ */
|
||||
|
||||
#define XML_NO_ATTR
|
||||
|
||||
#ifdef LIBXML_THREAD_ENABLED
|
||||
#define XML_DECLARE_GLOBAL(name, type, attrs) \
|
||||
attrs XMLPUBFUN type *__##name(void);
|
||||
#define XML_GLOBAL_MACRO(name) (*__##name())
|
||||
#else
|
||||
#define XML_DECLARE_GLOBAL(name, type, attrs) \
|
||||
attrs XMLPUBVAR type name;
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif /* __cplusplus */
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user