feat(agent): scrape linked pages to ground answers in real content
Adds `_fetch_page_content()`, which downloads a URL with a browser-like User-Agent and uses BeautifulSoup with the lxml parser to strip scripts, navigation, and other non-content markup, returning up to 3000 characters of readable text. The response validator now also compares generated replies against this scraped text, flagging named entities and claims that do not appear in the retrieved page content. Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0 for parser compatibility.
This commit is contained in:
@@ -109,10 +109,6 @@ XMLPUBFUN void
|
||||
/*
|
||||
* Add a new entry to the hash table.
|
||||
*/
|
||||
XMLPUBFUN int
|
||||
xmlHashAdd (xmlHashTablePtr hash,
|
||||
const xmlChar *name,
|
||||
void *userdata);
|
||||
XMLPUBFUN int
|
||||
xmlHashAddEntry (xmlHashTablePtr hash,
|
||||
const xmlChar *name,
|
||||
@@ -122,11 +118,6 @@ XMLPUBFUN int
|
||||
const xmlChar *name,
|
||||
void *userdata,
|
||||
xmlHashDeallocator dealloc);
|
||||
XMLPUBFUN int
|
||||
xmlHashAdd2 (xmlHashTablePtr hash,
|
||||
const xmlChar *name,
|
||||
const xmlChar *name2,
|
||||
void *userdata);
|
||||
XMLPUBFUN int
|
||||
xmlHashAddEntry2 (xmlHashTablePtr hash,
|
||||
const xmlChar *name,
|
||||
@@ -138,12 +129,6 @@ XMLPUBFUN int
|
||||
const xmlChar *name2,
|
||||
void *userdata,
|
||||
xmlHashDeallocator dealloc);
|
||||
XMLPUBFUN int
|
||||
xmlHashAdd3 (xmlHashTablePtr hash,
|
||||
const xmlChar *name,
|
||||
const xmlChar *name2,
|
||||
const xmlChar *name3,
|
||||
void *userdata);
|
||||
XMLPUBFUN int
|
||||
xmlHashAddEntry3 (xmlHashTablePtr hash,
|
||||
const xmlChar *name,
|
||||
@@ -214,10 +199,6 @@ XMLPUBFUN void *
|
||||
/*
|
||||
* Helpers.
|
||||
*/
|
||||
XMLPUBFUN xmlHashTablePtr
|
||||
xmlHashCopySafe (xmlHashTablePtr hash,
|
||||
xmlHashCopier copy,
|
||||
xmlHashDeallocator dealloc);
|
||||
XMLPUBFUN xmlHashTablePtr
|
||||
xmlHashCopy (xmlHashTablePtr hash,
|
||||
xmlHashCopier copy);
|
||||
|
||||
Reference in New Issue
Block a user