feat(agent): scrape linked pages to ground answers in real content
Adds `_fetch_page_content()`, which downloads a URL with a browser-like User-Agent and uses BeautifulSoup with the lxml parser to strip scripts, navigation, and other non-content markup, returning up to 3000 characters of readable text. The response validator now also compares generated replies against this scraped text, flagging named entities and claims that do not appear in the retrieved page content. Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0 for parser compatibility.
This commit is contained in:
@@ -21,11 +21,6 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Backward compatibility
|
||||
*/
|
||||
#define UTF8ToHtml htmlUTF8ToHtml
|
||||
|
||||
/*
|
||||
* Most of the back-end structures from XML and HTML are shared.
|
||||
*/
|
||||
@@ -47,22 +42,31 @@ typedef struct _htmlElemDesc htmlElemDesc;
|
||||
typedef htmlElemDesc *htmlElemDescPtr;
|
||||
struct _htmlElemDesc {
|
||||
const char *name; /* The tag name */
|
||||
char startTag; /* unused */
|
||||
char startTag; /* Whether the start tag can be implied */
|
||||
char endTag; /* Whether the end tag can be implied */
|
||||
char saveEndTag; /* Whether the end tag should be saved */
|
||||
char empty; /* Is this an empty element ? */
|
||||
char depr; /* unused */
|
||||
char dtd; /* unused */
|
||||
char depr; /* Is this a deprecated element ? */
|
||||
char dtd; /* 1: only in Loose DTD, 2: only Frameset one */
|
||||
char isinline; /* is this a block 0 or inline 1 element */
|
||||
const char *desc; /* the description */
|
||||
|
||||
const char** subelts XML_DEPRECATED_MEMBER;
|
||||
const char* defaultsubelt XML_DEPRECATED_MEMBER;
|
||||
const char** attrs_opt XML_DEPRECATED_MEMBER;
|
||||
const char** attrs_depr XML_DEPRECATED_MEMBER;
|
||||
const char** attrs_req XML_DEPRECATED_MEMBER;
|
||||
|
||||
int dataMode;
|
||||
/* NRK Jan.2003
|
||||
* New fields encapsulating HTML structure
|
||||
*
|
||||
* Bugs:
|
||||
* This is a very limited representation. It fails to tell us when
|
||||
* an element *requires* subelements (we only have whether they're
|
||||
* allowed or not), and it doesn't tell us where CDATA and PCDATA
|
||||
* are allowed. Some element relationships are not fully represented:
|
||||
* these are flagged with the word MODIFIER
|
||||
*/
|
||||
const char** subelts; /* allowed sub-elements of this element */
|
||||
const char* defaultsubelt; /* subelement for suggested auto-repair
|
||||
if necessary or NULL */
|
||||
const char** attrs_opt; /* Optional Attributes */
|
||||
const char** attrs_depr; /* Additional deprecated attributes */
|
||||
const char** attrs_req; /* Required attributes */
|
||||
};
|
||||
|
||||
/*
|
||||
@@ -76,12 +80,22 @@ struct _htmlEntityDesc {
|
||||
const char *desc; /* the description */
|
||||
};
|
||||
|
||||
/** DOC_DISABLE */
|
||||
#ifdef LIBXML_SAX1_ENABLED
|
||||
#define XML_GLOBALS_HTML \
|
||||
XML_OP(htmlDefaultSAXHandler, xmlSAXHandlerV1, XML_DEPRECATED)
|
||||
#else
|
||||
#define XML_GLOBALS_HTML
|
||||
#endif
|
||||
|
||||
XML_DEPRECATED
|
||||
XMLPUBVAR const xmlSAXHandlerV1 htmlDefaultSAXHandler;
|
||||
#define XML_OP XML_DECLARE_GLOBAL
|
||||
XML_GLOBALS_HTML
|
||||
#undef XML_OP
|
||||
|
||||
#endif /* LIBXML_SAX1_ENABLED */
|
||||
#if defined(LIBXML_THREAD_ENABLED) && !defined(XML_GLOBALS_NO_REDEFINITION)
|
||||
#define htmlDefaultSAXHandler XML_GLOBAL_MACRO(htmlDefaultSAXHandler)
|
||||
#endif
|
||||
/** DOC_ENABLE */
|
||||
|
||||
/*
|
||||
* There is only few public functions.
|
||||
@@ -96,11 +110,9 @@ XMLPUBFUN const htmlEntityDesc *
|
||||
XMLPUBFUN const htmlEntityDesc *
|
||||
htmlEntityValueLookup(unsigned int value);
|
||||
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int
|
||||
htmlIsAutoClosed(htmlDocPtr doc,
|
||||
htmlNodePtr elem);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int
|
||||
htmlAutoCloseTag(htmlDocPtr doc,
|
||||
const xmlChar *name,
|
||||
@@ -150,7 +162,7 @@ XMLPUBFUN htmlDocPtr
|
||||
htmlParseFile (const char *filename,
|
||||
const char *encoding);
|
||||
XMLPUBFUN int
|
||||
htmlUTF8ToHtml (unsigned char *out,
|
||||
UTF8ToHtml (unsigned char *out,
|
||||
int *outlen,
|
||||
const unsigned char *in,
|
||||
int *inlen);
|
||||
@@ -161,7 +173,6 @@ XMLPUBFUN int
|
||||
int *inlen, int quoteChar);
|
||||
XMLPUBFUN int
|
||||
htmlIsScriptAttribute(const xmlChar *name);
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int
|
||||
htmlHandleOmittedElem(int val);
|
||||
|
||||
@@ -196,26 +207,20 @@ XMLPUBFUN void
|
||||
* to the xmlReadDoc() and similar calls.
|
||||
*/
|
||||
typedef enum {
|
||||
HTML_PARSE_RECOVER = 1<<0, /* No effect */
|
||||
HTML_PARSE_RECOVER = 1<<0, /* Relaxed parsing */
|
||||
HTML_PARSE_NODEFDTD = 1<<2, /* do not default a doctype if not found */
|
||||
HTML_PARSE_NOERROR = 1<<5, /* suppress error reports */
|
||||
HTML_PARSE_NOWARNING= 1<<6, /* suppress warning reports */
|
||||
HTML_PARSE_PEDANTIC = 1<<7, /* No effect */
|
||||
HTML_PARSE_PEDANTIC = 1<<7, /* pedantic error reporting */
|
||||
HTML_PARSE_NOBLANKS = 1<<8, /* remove blank nodes */
|
||||
HTML_PARSE_NONET = 1<<11,/* No effect */
|
||||
HTML_PARSE_NONET = 1<<11,/* Forbid network access */
|
||||
HTML_PARSE_NOIMPLIED= 1<<13,/* Do not add implied html/body... elements */
|
||||
HTML_PARSE_COMPACT = 1<<16,/* compact small text nodes */
|
||||
HTML_PARSE_HUGE = 1<<19,/* relax any hardcoded limit from the parser */
|
||||
HTML_PARSE_IGNORE_ENC=1<<21,/* ignore internal document encoding hint */
|
||||
HTML_PARSE_BIG_LINES= 1<<22,/* Store big lines numbers in text PSVI field */
|
||||
HTML_PARSE_HTML5 = 1<<26 /* HTML5 support */
|
||||
HTML_PARSE_IGNORE_ENC=1<<21 /* ignore internal document encoding hint */
|
||||
} htmlParserOption;
|
||||
|
||||
XMLPUBFUN void
|
||||
htmlCtxtReset (htmlParserCtxtPtr ctxt);
|
||||
XMLPUBFUN int
|
||||
htmlCtxtSetOptions (htmlParserCtxtPtr ctxt,
|
||||
int options);
|
||||
XMLPUBFUN int
|
||||
htmlCtxtUseOptions (htmlParserCtxtPtr ctxt,
|
||||
int options);
|
||||
@@ -246,9 +251,6 @@ XMLPUBFUN htmlDocPtr
|
||||
const char *URL,
|
||||
const char *encoding,
|
||||
int options);
|
||||
XMLPUBFUN htmlDocPtr
|
||||
htmlCtxtParseDocument (htmlParserCtxtPtr ctxt,
|
||||
xmlParserInputPtr input);
|
||||
XMLPUBFUN htmlDocPtr
|
||||
htmlCtxtReadDoc (xmlParserCtxtPtr ctxt,
|
||||
const xmlChar *cur,
|
||||
@@ -282,7 +284,7 @@ XMLPUBFUN htmlDocPtr
|
||||
const char *encoding,
|
||||
int options);
|
||||
|
||||
/* deprecated content model
|
||||
/* NRK/Jan2003: further knowledge of HTML structure
|
||||
*/
|
||||
typedef enum {
|
||||
HTML_NA = 0 , /* something we don't check at all */
|
||||
@@ -295,14 +297,10 @@ typedef enum {
|
||||
/* Using htmlElemDesc rather than name here, to emphasise the fact
|
||||
that otherwise there's a lookup overhead
|
||||
*/
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN htmlStatus htmlAttrAllowed(const htmlElemDesc*, const xmlChar*, int) ;
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN int htmlElementAllowedHere(const htmlElemDesc*, const xmlChar*) ;
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN htmlStatus htmlElementStatusHere(const htmlElemDesc*, const htmlElemDesc*) ;
|
||||
XML_DEPRECATED
|
||||
XMLPUBFUN htmlStatus htmlNodeStatus(htmlNodePtr, int) ;
|
||||
XMLPUBFUN htmlStatus htmlNodeStatus(const htmlNodePtr, int) ;
|
||||
/**
|
||||
* htmlDefaultSubelement:
|
||||
* @elt: HTML element
|
||||
@@ -335,5 +333,11 @@ XMLPUBFUN htmlStatus htmlNodeStatus(htmlNodePtr, int) ;
|
||||
}
|
||||
#endif
|
||||
|
||||
#else /* LIBXML_HTML_ENABLED */
|
||||
|
||||
/** DOC_DISABLE */
|
||||
#define XML_GLOBALS_HTML
|
||||
/** DOC_ENABLE */
|
||||
|
||||
#endif /* LIBXML_HTML_ENABLED */
|
||||
#endif /* __HTML_PARSER_H__ */
|
||||
|
||||
Reference in New Issue
Block a user