feat(agent): scrape linked pages to ground answers in real content
Adds `_fetch_page_content()`, which downloads a URL with a browser-like User-Agent and uses BeautifulSoup with the lxml parser to strip scripts, navigation, and other non-content markup, returning up to 3000 characters of readable text. The response validator now also compares generated replies against this scraped text, flagging named entities and claims that do not appear in the retrieved page content. Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0 for parser compatibility.
This commit is contained in:
@@ -11,7 +11,6 @@
|
||||
#ifndef __XML_VALID_H__
|
||||
#define __XML_VALID_H__
|
||||
|
||||
/** DOC_DISABLE */
|
||||
#include <libxml/xmlversion.h>
|
||||
#include <libxml/xmlerror.h>
|
||||
#define XML_TREE_INTERNALS
|
||||
@@ -20,7 +19,6 @@
|
||||
#include <libxml/list.h>
|
||||
#include <libxml/xmlautomata.h>
|
||||
#include <libxml/xmlregexp.h>
|
||||
/** DOC_ENABLE */
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
@@ -145,18 +143,16 @@ XMLPUBFUN xmlNotationPtr
|
||||
const xmlChar *name,
|
||||
const xmlChar *PublicID,
|
||||
const xmlChar *SystemID);
|
||||
XML_DEPRECATED
|
||||
#ifdef LIBXML_TREE_ENABLED
|
||||
XMLPUBFUN xmlNotationTablePtr
|
||||
xmlCopyNotationTable (xmlNotationTablePtr table);
|
||||
XML_DEPRECATED
|
||||
#endif /* LIBXML_TREE_ENABLED */
|
||||
XMLPUBFUN void
|
||||
xmlFreeNotationTable (xmlNotationTablePtr table);
|
||||
#ifdef LIBXML_OUTPUT_ENABLED
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN void
|
||||
xmlDumpNotationDecl (xmlBufferPtr buf,
|
||||
xmlNotationPtr nota);
|
||||
/* XML_DEPRECATED, still used in lxml */
|
||||
XMLPUBFUN void
|
||||
xmlDumpNotationTable (xmlBufferPtr buf,
|
||||
xmlNotationTablePtr table);
|
||||
@@ -164,43 +160,37 @@ XMLPUBFUN void
|
||||
|
||||
/* Element Content */
|
||||
/* the non Doc version are being deprecated */
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN xmlElementContentPtr
|
||||
xmlNewElementContent (const xmlChar *name,
|
||||
xmlElementContentType type);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN xmlElementContentPtr
|
||||
xmlCopyElementContent (xmlElementContentPtr content);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN void
|
||||
xmlFreeElementContent (xmlElementContentPtr cur);
|
||||
/* the new versions with doc argument */
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN xmlElementContentPtr
|
||||
xmlNewDocElementContent (xmlDocPtr doc,
|
||||
const xmlChar *name,
|
||||
xmlElementContentType type);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN xmlElementContentPtr
|
||||
xmlCopyDocElementContent(xmlDocPtr doc,
|
||||
xmlElementContentPtr content);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN void
|
||||
xmlFreeDocElementContent(xmlDocPtr doc,
|
||||
xmlElementContentPtr cur);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN void
|
||||
xmlSnprintfElementContent(char *buf,
|
||||
int size,
|
||||
xmlElementContentPtr content,
|
||||
int englob);
|
||||
#ifdef LIBXML_OUTPUT_ENABLED
|
||||
XML_DEPRECATED
|
||||
/* DEPRECATED */
|
||||
XMLPUBFUN void
|
||||
xmlSprintfElementContent(char *buf,
|
||||
xmlElementContentPtr content,
|
||||
int englob);
|
||||
#endif /* LIBXML_OUTPUT_ENABLED */
|
||||
/* DEPRECATED */
|
||||
|
||||
/* Element */
|
||||
XMLPUBFUN xmlElementPtr
|
||||
@@ -209,33 +199,30 @@ XMLPUBFUN xmlElementPtr
|
||||
const xmlChar *name,
|
||||
xmlElementTypeVal type,
|
||||
xmlElementContentPtr content);
|
||||
XML_DEPRECATED
|
||||
#ifdef LIBXML_TREE_ENABLED
|
||||
XMLPUBFUN xmlElementTablePtr
|
||||
xmlCopyElementTable (xmlElementTablePtr table);
|
||||
XML_DEPRECATED
|
||||
#endif /* LIBXML_TREE_ENABLED */
|
||||
XMLPUBFUN void
|
||||
xmlFreeElementTable (xmlElementTablePtr table);
|
||||
#ifdef LIBXML_OUTPUT_ENABLED
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN void
|
||||
xmlDumpElementTable (xmlBufferPtr buf,
|
||||
xmlElementTablePtr table);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN void
|
||||
xmlDumpElementDecl (xmlBufferPtr buf,
|
||||
xmlElementPtr elem);
|
||||
#endif /* LIBXML_OUTPUT_ENABLED */
|
||||
|
||||
/* Enumeration */
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN xmlEnumerationPtr
|
||||
xmlCreateEnumeration (const xmlChar *name);
|
||||
/* XML_DEPRECATED, needed for custom attributeDecl SAX handler */
|
||||
XMLPUBFUN void
|
||||
xmlFreeEnumeration (xmlEnumerationPtr cur);
|
||||
XML_DEPRECATED
|
||||
#ifdef LIBXML_TREE_ENABLED
|
||||
XMLPUBFUN xmlEnumerationPtr
|
||||
xmlCopyEnumeration (xmlEnumerationPtr cur);
|
||||
#endif /* LIBXML_TREE_ENABLED */
|
||||
|
||||
/* Attribute */
|
||||
XMLPUBFUN xmlAttributePtr
|
||||
@@ -248,27 +235,22 @@ XMLPUBFUN xmlAttributePtr
|
||||
xmlAttributeDefault def,
|
||||
const xmlChar *defaultValue,
|
||||
xmlEnumerationPtr tree);
|
||||
XML_DEPRECATED
|
||||
#ifdef LIBXML_TREE_ENABLED
|
||||
XMLPUBFUN xmlAttributeTablePtr
|
||||
xmlCopyAttributeTable (xmlAttributeTablePtr table);
|
||||
XML_DEPRECATED
|
||||
#endif /* LIBXML_TREE_ENABLED */
|
||||
XMLPUBFUN void
|
||||
xmlFreeAttributeTable (xmlAttributeTablePtr table);
|
||||
#ifdef LIBXML_OUTPUT_ENABLED
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN void
|
||||
xmlDumpAttributeTable (xmlBufferPtr buf,
|
||||
xmlAttributeTablePtr table);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN void
|
||||
xmlDumpAttributeDecl (xmlBufferPtr buf,
|
||||
xmlAttributePtr attr);
|
||||
#endif /* LIBXML_OUTPUT_ENABLED */
|
||||
|
||||
/* IDs */
|
||||
XMLPUBFUN int
|
||||
xmlAddIDSafe (xmlAttrPtr attr,
|
||||
const xmlChar *value);
|
||||
XMLPUBFUN xmlIDPtr
|
||||
xmlAddID (xmlValidCtxtPtr ctxt,
|
||||
xmlDocPtr doc,
|
||||
@@ -321,38 +303,31 @@ XMLPUBFUN xmlValidCtxtPtr
|
||||
XMLPUBFUN void
|
||||
xmlFreeValidCtxt(xmlValidCtxtPtr);
|
||||
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int
|
||||
xmlValidateRoot (xmlValidCtxtPtr ctxt,
|
||||
xmlDocPtr doc);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int
|
||||
xmlValidateElementDecl (xmlValidCtxtPtr ctxt,
|
||||
xmlDocPtr doc,
|
||||
xmlElementPtr elem);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN xmlChar *
|
||||
xmlValidNormalizeAttributeValue(xmlDocPtr doc,
|
||||
xmlNodePtr elem,
|
||||
const xmlChar *name,
|
||||
const xmlChar *value);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN xmlChar *
|
||||
xmlValidCtxtNormalizeAttributeValue(xmlValidCtxtPtr ctxt,
|
||||
xmlDocPtr doc,
|
||||
xmlNodePtr elem,
|
||||
const xmlChar *name,
|
||||
const xmlChar *value);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int
|
||||
xmlValidateAttributeDecl(xmlValidCtxtPtr ctxt,
|
||||
xmlDocPtr doc,
|
||||
xmlAttributePtr attr);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int
|
||||
xmlValidateAttributeValue(xmlAttributeType type,
|
||||
const xmlChar *value);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int
|
||||
xmlValidateNotationDecl (xmlValidCtxtPtr ctxt,
|
||||
xmlDocPtr doc,
|
||||
@@ -361,7 +336,6 @@ XMLPUBFUN int
|
||||
xmlValidateDtd (xmlValidCtxtPtr ctxt,
|
||||
xmlDocPtr doc,
|
||||
xmlDtdPtr dtd);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int
|
||||
xmlValidateDtdFinal (xmlValidCtxtPtr ctxt,
|
||||
xmlDocPtr doc);
|
||||
@@ -372,19 +346,16 @@ XMLPUBFUN int
|
||||
xmlValidateElement (xmlValidCtxtPtr ctxt,
|
||||
xmlDocPtr doc,
|
||||
xmlNodePtr elem);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int
|
||||
xmlValidateOneElement (xmlValidCtxtPtr ctxt,
|
||||
xmlDocPtr doc,
|
||||
xmlNodePtr elem);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int
|
||||
xmlValidateOneAttribute (xmlValidCtxtPtr ctxt,
|
||||
xmlDocPtr doc,
|
||||
xmlNodePtr elem,
|
||||
xmlAttrPtr attr,
|
||||
const xmlChar *value);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int
|
||||
xmlValidateOneNamespace (xmlValidCtxtPtr ctxt,
|
||||
xmlDocPtr doc,
|
||||
@@ -392,16 +363,17 @@ XMLPUBFUN int
|
||||
const xmlChar *prefix,
|
||||
xmlNsPtr ns,
|
||||
const xmlChar *value);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int
|
||||
xmlValidateDocumentFinal(xmlValidCtxtPtr ctxt,
|
||||
xmlDocPtr doc);
|
||||
XML_DEPRECATED
|
||||
#endif /* LIBXML_VALID_ENABLED */
|
||||
|
||||
#if defined(LIBXML_VALID_ENABLED) || defined(LIBXML_SCHEMAS_ENABLED)
|
||||
XMLPUBFUN int
|
||||
xmlValidateNotationUse (xmlValidCtxtPtr ctxt,
|
||||
xmlDocPtr doc,
|
||||
const xmlChar *notationName);
|
||||
#endif /* LIBXML_VALID_ENABLED */
|
||||
#endif /* LIBXML_VALID_ENABLED or LIBXML_SCHEMAS_ENABLED */
|
||||
|
||||
XMLPUBFUN int
|
||||
xmlIsMixedElement (xmlDocPtr doc,
|
||||
@@ -452,23 +424,19 @@ XMLPUBFUN int
|
||||
/*
|
||||
* Validation based on the regexp support
|
||||
*/
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int
|
||||
xmlValidBuildContentModel(xmlValidCtxtPtr ctxt,
|
||||
xmlElementPtr elem);
|
||||
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int
|
||||
xmlValidatePushElement (xmlValidCtxtPtr ctxt,
|
||||
xmlDocPtr doc,
|
||||
xmlNodePtr elem,
|
||||
const xmlChar *qname);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int
|
||||
xmlValidatePushCData (xmlValidCtxtPtr ctxt,
|
||||
const xmlChar *data,
|
||||
int len);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int
|
||||
xmlValidatePopElement (xmlValidCtxtPtr ctxt,
|
||||
xmlDocPtr doc,
|
||||
|
||||
Reference in New Issue
Block a user