feat(agent): scrape linked pages to ground answers in real content

Adds `_fetch_page_content()`, which downloads a URL with a
browser-like User-Agent and uses BeautifulSoup with the lxml
parser to strip scripts, navigation, and other non-content
markup, returning up to 3000 characters of readable text.

The response validator now also compares generated replies
against this scraped text, flagging named entities and claims
that do not appear in the retrieved page content.

Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0
for parser compatibility.
This commit is contained in:
2026-10-26 16:48:29 -03:00
parent 9eed02645b
commit 723ca1506b
231 changed files with 20508 additions and 5330 deletions
@@ -756,12 +756,8 @@ cdef void _receiveGenericError(void* c_log_handler, int c_domain,
element_size = cstring_h.strlen(c_element)
c_message = <char*>stdlib.malloc(
(text_size + element_size + 1) * sizeof(char))
if c_message is not NULL:
stdio.sprintf(c_message, msg, c_element)
c_error.message = c_message
else:
# Out of memory, report at least the original message.
c_error.message = msg
stdio.sprintf(c_message, msg, c_element)
c_error.message = c_message
else:
c_error.message = ''
elif c_element is NULL:
@@ -987,12 +983,6 @@ ERR_VERSION_MISMATCH=109
ERR_NAME_TOO_LONG=110
ERR_USER_STOP=111
ERR_COMMENT_ABRUPTLY_ENDED=112
WAR_ENCODING_MISMATCH=113
ERR_RESOURCE_LIMIT=114
ERR_ARGUMENT=115
ERR_SYSTEM=116
ERR_REDECL_PREDEF_ENTITY=117
ERR_INT_SUBSET_NOT_FINISHED=118
NS_ERR_XML_NAMESPACE=200
NS_ERR_UNDEFINED_NAMESPACE=201
NS_ERR_QNAME=202
@@ -1043,7 +1033,6 @@ DTD_XMLID_TYPE=540
DTD_DUP_TOKEN=541
HTML_STRUCURE_ERROR=800
HTML_UNKNOWN_TAG=801
HTML_INCORRECTLY_OPENED_COMMENT=802
RNGP_ANYNAME_ATTR_ANCESTOR=1000
RNGP_ATTR_CONFLICT=1001
RNGP_ATTRIBUTE_CHILDREN=1002
@@ -1255,7 +1244,6 @@ IO_ENETUNREACH=1553
IO_EADDRINUSE=1554
IO_EALREADY=1555
IO_EAFNOSUPPORT=1556
IO_UNSUPPORTED_PROTOCOL=1557
XINCLUDE_RECURSION=1600
XINCLUDE_PARSE_VALUE=1601
XINCLUDE_ENTITY_DEF_MISMATCH=1602