feat(agent): scrape linked pages to ground answers in real content
Adds `_fetch_page_content()`, which downloads a URL with a browser-like User-Agent and uses BeautifulSoup with the lxml parser to strip scripts, navigation, and other non-content markup, returning up to 3000 characters of readable text. The response validator now also compares generated replies against this scraped text, flagging named entities and claims that do not appear in the retrieved page content. Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0 for parser compatibility.
This commit is contained in:
@@ -476,50 +476,6 @@ cdef _write_attr_string(tree.xmlOutputBuffer* buf, const char *string):
|
||||
tree.xmlOutputBufferWrite(buf, cur - base, base)
|
||||
|
||||
|
||||
cdef void _write_cdata_section(tree.xmlOutputBuffer* buf, const char* c_data, const char* c_end):
|
||||
tree.xmlOutputBufferWrite(buf, 9, "<![CDATA[")
|
||||
while c_end - c_data > limits.INT_MAX:
|
||||
tree.xmlOutputBufferWrite(buf, limits.INT_MAX, c_data)
|
||||
c_data += limits.INT_MAX
|
||||
tree.xmlOutputBufferWrite(buf, c_end - c_data, c_data)
|
||||
tree.xmlOutputBufferWrite(buf, 3, "]]>")
|
||||
|
||||
|
||||
cdef _write_cdata_string(tree.xmlOutputBuffer* buf, bytes bstring):
|
||||
cdef const char* c_data = bstring
|
||||
cdef const char* c_end = c_data + len(bstring)
|
||||
cdef const char* c_pos = c_data
|
||||
cdef bint nothing_written = True
|
||||
|
||||
while True:
|
||||
c_pos = <const char*> cstring_h.memchr(c_pos, b']', c_end - c_pos)
|
||||
if not c_pos:
|
||||
break
|
||||
c_pos += 1
|
||||
next_char = c_pos[0]
|
||||
c_pos += 1
|
||||
if next_char != b']':
|
||||
continue
|
||||
# Found ']]', c_pos points to next character.
|
||||
while c_pos[0] == b']':
|
||||
c_pos += 1
|
||||
if c_pos[0] != b'>':
|
||||
if c_pos == c_end:
|
||||
break
|
||||
# c_pos[0] is neither ']' nor '>', continue with next character.
|
||||
c_pos += 1
|
||||
continue
|
||||
|
||||
# Write section up to ']]' and start next block at trailing '>'.
|
||||
_write_cdata_section(buf, c_data, c_pos)
|
||||
nothing_written = False
|
||||
c_data = c_pos
|
||||
c_pos += 1
|
||||
|
||||
if nothing_written or c_data < c_end:
|
||||
_write_cdata_section(buf, c_data, c_end)
|
||||
|
||||
|
||||
############################################################
|
||||
# output to file-like objects
|
||||
|
||||
@@ -563,7 +519,6 @@ cdef class _FilelikeWriter:
|
||||
cdef object _close_filelike
|
||||
cdef _ExceptionContext _exc_context
|
||||
cdef _ErrorLog error_log
|
||||
|
||||
def __cinit__(self, filelike, exc_context=None, compression=None, close=False):
|
||||
if compression is not None and compression > 0:
|
||||
filelike = GzipFile(
|
||||
@@ -704,12 +659,6 @@ cdef _FilelikeWriter _create_output_buffer(
|
||||
f"unknown encoding: '{c_enc.decode('UTF-8') if c_enc is not NULL else u''}'")
|
||||
try:
|
||||
f = _getFSPathOrObject(f)
|
||||
|
||||
if c_compression and not HAS_ZLIB_COMPRESSION and _isString(f):
|
||||
# Let "_FilelikeWriter" fall back to Python's GzipFile.
|
||||
f = open(f, mode="wb")
|
||||
close = True
|
||||
|
||||
if _isString(f):
|
||||
filename8 = _encodeFilename(f)
|
||||
if b'%' in filename8 and (
|
||||
@@ -746,10 +695,7 @@ cdef xmlChar **_convert_ns_prefixes(tree.xmlDict* c_dict, ns_prefixes) except NU
|
||||
try:
|
||||
for prefix in ns_prefixes:
|
||||
prefix_utf = _utf8(prefix)
|
||||
c_prefix_len = len(prefix_utf)
|
||||
if c_prefix_len > limits.INT_MAX:
|
||||
raise ValueError("Prefix too long")
|
||||
c_prefix = tree.xmlDictExists(c_dict, _xcstr(prefix_utf), <int> c_prefix_len)
|
||||
c_prefix = tree.xmlDictExists(c_dict, _xcstr(prefix_utf), len(prefix_utf))
|
||||
if c_prefix:
|
||||
# unknown prefixes do not need to get serialised
|
||||
c_ns_prefixes[i] = <xmlChar*>c_prefix
|
||||
@@ -779,13 +725,6 @@ cdef _tofilelikeC14N(f, _Element element, bint exclusive, bint with_comments,
|
||||
if inclusive_ns_prefixes else NULL)
|
||||
|
||||
f = _getFSPathOrObject(f)
|
||||
|
||||
close = False
|
||||
if compression and not HAS_ZLIB_COMPRESSION and _isString(f):
|
||||
# Let "_FilelikeWriter" fall back to Python's GzipFile.
|
||||
f = open(f, mode="wb")
|
||||
close = True
|
||||
|
||||
if _isString(f):
|
||||
filename8 = _encodeFilename(f)
|
||||
c_filename = _cstr(filename8)
|
||||
@@ -794,7 +733,7 @@ cdef _tofilelikeC14N(f, _Element element, bint exclusive, bint with_comments,
|
||||
c_doc, NULL, exclusive, c_inclusive_ns_prefixes,
|
||||
with_comments, c_filename, compression)
|
||||
elif hasattr(f, 'write'):
|
||||
writer = _FilelikeWriter(f, compression=compression, close=close)
|
||||
writer = _FilelikeWriter(f, compression=compression)
|
||||
c_buffer = writer._createOutputBuffer(NULL)
|
||||
try:
|
||||
with writer.error_log:
|
||||
@@ -1617,11 +1556,6 @@ cdef class _IncrementalFileWriter:
|
||||
else:
|
||||
tree.xmlOutputBufferWriteEscape(self._c_out, _xcstr(bstring), NULL)
|
||||
|
||||
elif isinstance(content, CDATA):
|
||||
if self._status > WRITER_IN_ELEMENT:
|
||||
raise LxmlSyntaxError("not in an element")
|
||||
_write_cdata_string(self._c_out, (<CDATA>content)._utf8_data)
|
||||
|
||||
elif iselement(content):
|
||||
if self._status > WRITER_IN_ELEMENT:
|
||||
raise LxmlSyntaxError("cannot append trailing element to complete XML document")
|
||||
@@ -1634,10 +1568,8 @@ cdef class _IncrementalFileWriter:
|
||||
|
||||
elif content is not None:
|
||||
raise TypeError(
|
||||
f"got invalid input value of type {type(content)}, expected string, CDATA or Element")
|
||||
|
||||
f"got invalid input value of type {type(content)}, expected string or Element")
|
||||
self._handle_error(self._c_out.error)
|
||||
|
||||
if not self._buffered:
|
||||
tree.xmlOutputBufferFlush(self._c_out)
|
||||
self._handle_error(self._c_out.error)
|
||||
|
||||
Reference in New Issue
Block a user