feat(agent): scrape linked pages to ground answers in real content
Adds `_fetch_page_content()`, which downloads a URL with a browser-like User-Agent and uses BeautifulSoup with the lxml parser to strip scripts, navigation, and other non-content markup, returning up to 3000 characters of readable text. The response validator now also compares generated replies against this scraped text, flagging named entities and claims that do not appear in the retrieved page content. Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0 for parser compatibility.
This commit is contained in:
@@ -32,11 +32,11 @@ cdef extern from "libxml/parser.h" nogil:
|
||||
|
||||
ctypedef void (*commentSAXFunc)(void* ctx, const_xmlChar* value) noexcept
|
||||
|
||||
ctypedef void (*processingInstructionSAXFunc)(void* ctx,
|
||||
ctypedef void (*processingInstructionSAXFunc)(void* ctx,
|
||||
const_xmlChar* target,
|
||||
const_xmlChar* data) noexcept
|
||||
|
||||
ctypedef void (*internalSubsetSAXFunc)(void* ctx,
|
||||
ctypedef void (*internalSubsetSAXFunc)(void* ctx,
|
||||
const_xmlChar* name,
|
||||
const_xmlChar* externalID,
|
||||
const_xmlChar* systemID) noexcept
|
||||
@@ -80,7 +80,6 @@ cdef extern from "libxml/tree.h" nogil:
|
||||
cdataBlockSAXFunc cdataBlock
|
||||
referenceSAXFunc reference
|
||||
getEntitySAXFunc getEntity
|
||||
getEntitySAXFunc getParameterEntity
|
||||
commentSAXFunc comment
|
||||
processingInstructionSAXFunc processingInstruction
|
||||
startDocumentSAXFunc startDocument
|
||||
@@ -100,48 +99,11 @@ cdef extern from "libxml/xmlIO.h" nogil:
|
||||
|
||||
cdef extern from "libxml/parser.h" nogil:
|
||||
|
||||
ctypedef enum xmlFeature:
|
||||
XML_WITH_THREAD = 1
|
||||
XML_WITH_TREE = 2
|
||||
XML_WITH_OUTPUT = 3
|
||||
XML_WITH_PUSH = 4
|
||||
XML_WITH_READER = 5
|
||||
XML_WITH_PATTERN = 6
|
||||
XML_WITH_WRITER = 7
|
||||
XML_WITH_SAX1 = 8
|
||||
XML_WITH_FTP = 9
|
||||
XML_WITH_HTTP = 10
|
||||
XML_WITH_VALID = 11
|
||||
XML_WITH_HTML = 12
|
||||
XML_WITH_LEGACY = 13
|
||||
XML_WITH_C14N = 14
|
||||
XML_WITH_CATALOG = 15
|
||||
XML_WITH_XPATH = 16
|
||||
XML_WITH_XPTR = 17
|
||||
XML_WITH_XINCLUDE = 18
|
||||
XML_WITH_ICONV = 19
|
||||
XML_WITH_ISO8859X = 20
|
||||
XML_WITH_UNICODE = 21
|
||||
XML_WITH_REGEXP = 22
|
||||
XML_WITH_AUTOMATA = 23
|
||||
XML_WITH_EXPR = 24
|
||||
XML_WITH_SCHEMAS = 25
|
||||
XML_WITH_SCHEMATRON = 26
|
||||
XML_WITH_MODULES = 27
|
||||
XML_WITH_DEBUG = 28
|
||||
XML_WITH_DEBUG_MEM = 29
|
||||
XML_WITH_DEBUG_RUN = 30
|
||||
XML_WITH_ZLIB = 31
|
||||
XML_WITH_ICU = 32
|
||||
XML_WITH_LZMA = 33
|
||||
|
||||
cdef bint xmlHasFeature(xmlFeature feature)
|
||||
|
||||
cdef xmlDict* xmlDictCreate()
|
||||
cdef xmlDict* xmlDictCreateSub(xmlDict* subdict)
|
||||
cdef void xmlDictFree(xmlDict* sub)
|
||||
cdef int xmlDictReference(xmlDict* dict)
|
||||
|
||||
|
||||
cdef int XML_COMPLETE_ATTRS # SAX option for adding DTD default attributes
|
||||
cdef int XML_SKIP_IDS # SAX option for not building an XML ID dict
|
||||
|
||||
@@ -196,42 +158,34 @@ cdef extern from "libxml/parser.h" nogil:
|
||||
xmlParserInput* inputTab[]
|
||||
|
||||
ctypedef enum xmlParserOption:
|
||||
XML_PARSE_RECOVER = 0x1 # recover on errors
|
||||
XML_PARSE_NOENT = 0x2 # substitute entities
|
||||
XML_PARSE_DTDLOAD = 0x4 # load the external subset
|
||||
XML_PARSE_DTDATTR = 0x8 # default DTD attributes
|
||||
XML_PARSE_DTDVALID = 0x10 # validate with the DTD
|
||||
XML_PARSE_NOERROR = 0x20 # suppress error reports
|
||||
XML_PARSE_NOWARNING = 0x40 # suppress warning reports
|
||||
XML_PARSE_PEDANTIC = 0x80 # pedantic error reporting
|
||||
XML_PARSE_NOBLANKS = 0x100 # remove blank nodes
|
||||
XML_PARSE_SAX1 = 0x200 # use the SAX1 interface internally
|
||||
XML_PARSE_XINCLUDE = 0x400 # Implement XInclude substitution
|
||||
XML_PARSE_NONET = 0x800 # Forbid network access
|
||||
XML_PARSE_NODICT = 0x1000 # Do not reuse the context dictionary
|
||||
XML_PARSE_NSCLEAN = 0x2000 # remove redundant namespaces declarations
|
||||
XML_PARSE_NOCDATA = 0x4000 # merge CDATA as text nodes
|
||||
XML_PARSE_NOXINCNODE = 0x8000 # do not generate XINCLUDE START/END nodes
|
||||
XML_PARSE_RECOVER = 1 # recover on errors
|
||||
XML_PARSE_NOENT = 2 # substitute entities
|
||||
XML_PARSE_DTDLOAD = 4 # load the external subset
|
||||
XML_PARSE_DTDATTR = 8 # default DTD attributes
|
||||
XML_PARSE_DTDVALID = 16 # validate with the DTD
|
||||
XML_PARSE_NOERROR = 32 # suppress error reports
|
||||
XML_PARSE_NOWARNING = 64 # suppress warning reports
|
||||
XML_PARSE_PEDANTIC = 128 # pedantic error reporting
|
||||
XML_PARSE_NOBLANKS = 256 # remove blank nodes
|
||||
XML_PARSE_SAX1 = 512 # use the SAX1 interface internally
|
||||
XML_PARSE_XINCLUDE = 1024 # Implement XInclude substitution
|
||||
XML_PARSE_NONET = 2048 # Forbid network access
|
||||
XML_PARSE_NODICT = 4096 # Do not reuse the context dictionary
|
||||
XML_PARSE_NSCLEAN = 8192 # remove redundant namespaces declarations
|
||||
XML_PARSE_NOCDATA = 16384 # merge CDATA as text nodes
|
||||
XML_PARSE_NOXINCNODE = 32768 # do not generate XINCLUDE START/END nodes
|
||||
# libxml2 2.6.21+ only:
|
||||
XML_PARSE_COMPACT = 0x1_0000 # compact small text nodes
|
||||
XML_PARSE_COMPACT = 65536 # compact small text nodes
|
||||
# libxml2 2.7.0+ only:
|
||||
XML_PARSE_OLD10 = 0x2_0000 # parse using XML-1.0 before update 5
|
||||
XML_PARSE_NOBASEFIX = 0x4_0000 # do not fixup XINCLUDE xml:base uris
|
||||
XML_PARSE_HUGE = 0x8_0000 # relax any hardcoded limit from the parser
|
||||
XML_PARSE_OLD10 = 131072 # parse using XML-1.0 before update 5
|
||||
XML_PARSE_NOBASEFIX = 262144 # do not fixup XINCLUDE xml:base uris
|
||||
XML_PARSE_HUGE = 524288 # relax any hardcoded limit from the parser
|
||||
# libxml2 2.7.3+ only:
|
||||
XML_PARSE_OLDSAX = 0x10_0000 # parse using SAX2 interface before 2.7.0
|
||||
XML_PARSE_OLDSAX = 1048576 # parse using SAX2 interface before 2.7.0
|
||||
# libxml2 2.8.0+ only:
|
||||
XML_PARSE_IGNORE_ENC = 0x20_0000 # ignore internal document encoding hint
|
||||
XML_PARSE_IGNORE_ENC = 2097152 # ignore internal document encoding hint
|
||||
# libxml2 2.9.0+ only:
|
||||
XML_PARSE_BIG_LINES = 0x40_0000 # Store big lines numbers in text PSVI field
|
||||
# libxml2 2.13.0+ only:
|
||||
XML_PARSE_NO_XXE = 0x80_0000 # Disable loading of external DTDs or entities
|
||||
# libxml2 2.14.0+ only:
|
||||
XML_PARSE_UNZIP = 0x100_0000 # Enable input decompression (and potential gzip bombs)
|
||||
XML_PARSE_NO_SYS_CATALOG = 0x200_0000 # Disable the global system XML catalog
|
||||
XML_PARSE_CATALOG_PI = 0x400_0000 # Enable XML catalog processing instructions
|
||||
# libxml2 2.15.0+ only:
|
||||
XML_PARSE_SKIP_IDS = 0x800_0000 # Force the parser to ignore IDs
|
||||
XML_PARSE_BIG_LINES = 4194304 # Store big lines numbers in text PSVI field
|
||||
|
||||
cdef void xmlInitParser()
|
||||
cdef void xmlCleanupParser()
|
||||
@@ -253,9 +207,9 @@ cdef extern from "libxml/parser.h" nogil:
|
||||
cdef xmlDoc* xmlCtxtReadFile(xmlParserCtxt* ctxt,
|
||||
char* filename, char* encoding,
|
||||
int options)
|
||||
cdef xmlDoc* xmlCtxtReadIO(xmlParserCtxt* ctxt,
|
||||
xmlInputReadCallback ioread,
|
||||
xmlInputCloseCallback ioclose,
|
||||
cdef xmlDoc* xmlCtxtReadIO(xmlParserCtxt* ctxt,
|
||||
xmlInputReadCallback ioread,
|
||||
xmlInputCloseCallback ioclose,
|
||||
void* ioctx,
|
||||
char* URL, char* encoding,
|
||||
int options)
|
||||
@@ -302,18 +256,10 @@ cdef extern from "libxml/parser.h" nogil:
|
||||
|
||||
|
||||
cdef extern from "libxml/parserInternals.h" nogil:
|
||||
"""
|
||||
#if LIBXML_VERSION < 21400
|
||||
#define xmlNewInputFromMemory(url, mem, size, flags) (NULL)
|
||||
#endif
|
||||
"""
|
||||
cdef xmlParserInput* xmlNewInputStream(xmlParserCtxt* ctxt)
|
||||
cdef xmlParserInput* xmlNewStringInputStream(xmlParserCtxt* ctxt,
|
||||
cdef xmlParserInput* xmlNewStringInputStream(xmlParserCtxt* ctxt,
|
||||
char* buffer)
|
||||
cdef xmlParserInput* xmlNewInputFromFile(xmlParserCtxt* ctxt,
|
||||
cdef xmlParserInput* xmlNewInputFromFile(xmlParserCtxt* ctxt,
|
||||
char* filename)
|
||||
cdef xmlParserInput* xmlNewInputFromMemory(
|
||||
const char *url, const void *mem, size_t size, int flags) # actually "xmlParserInputFlags flags"
|
||||
cdef void xmlFreeInputStream(xmlParserInput* input)
|
||||
cdef int xmlSwitchEncoding(xmlParserCtxt* ctxt, int enc)
|
||||
cdef bint xmlCtxtIsStopped(xmlParserCtxt* ctxt)
|
||||
|
||||
Reference in New Issue
Block a user