marcocosta10000/PsAgentAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): scrape linked pages to ground answers in real content

Browse Source 
Adds `_fetch_page_content()`, which downloads a URL with a
browser-like User-Agent and uses BeautifulSoup with the lxml
parser to strip scripts, navigation, and other non-content
markup, returning up to 3000 characters of readable text.

The response validator now also compares generated replies
against this scraped text, flagging named entities and claims
that do not appear in the retrieved page content.

Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0
for parser compatibility.
This commit is contained in:
marcocosta10000
2026-10-26 16:48:29 -03:00
parent 9eed02645b
commit 723ca1506b
231 changed files with 20508 additions and 5330 deletions
Download Patch File Download Diff File Expand all files Collapse all files
venv/lib/python3.11/site-packages/lxml/etree.pyx
+38 -166
View File
@@ -19,7 +19,6 @@ __all__ = [
'FallbackElementClassLookup', 'FunctionNamespace', 'HTML', 'HTMLParser',
'ICONV_COMPILED_VERSION',
'LIBXML_COMPILED_VERSION', 'LIBXML_VERSION',
'LIBXML_FEATURES',
'LIBXSLT_COMPILED_VERSION', 'LIBXSLT_VERSION',
'LXML_VERSION',
'LxmlError', 'LxmlRegistryError', 'LxmlSyntaxError',
@@ -300,101 +299,6 @@ cdef extern from *:
ICONV_COMPILED_VERSION = __unpackIntVersion(LIBICONV_HEX_VERSION, base=0x100)[:2]
cdef extern from "libxml/xmlversion.h":
"""
static const char* const _lxml_lib_features[] = {
#ifdef LIBXML_HTML_ENABLED
"html",
#endif
#ifdef LIBXML_FTP_ENABLED
"ftp",
#endif
#ifdef LIBXML_HTTP_ENABLED
"http",
#endif
#ifdef LIBXML_CATALOG_ENABLED
"catalog",
#endif
#ifdef LIBXML_XPATH_ENABLED
"xpath",
#endif
#ifdef LIBXML_ICONV_ENABLED
"iconv",
#endif
#ifdef LIBXML_ICU_ENABLED
"icu",
#endif
#ifdef LIBXML_REGEXP_ENABLED
"regexp",
#endif
#ifdef LIBXML_SCHEMAS_ENABLED
"xmlschema",
#endif
#ifdef LIBXML_SCHEMATRON_ENABLED
"schematron",
#endif
#ifdef LIBXML_ZLIB_ENABLED
"zlib",
#endif
#ifdef LIBXML_LZMA_ENABLED
"lzma",
#endif
0
};
"""
const char* const* _LXML_LIB_FEATURES "_lxml_lib_features"
cdef set _copy_lib_features():
features = set()
feature = _LXML_LIB_FEATURES
while feature[0]:
features.add(feature[0].decode('ASCII'))
feature += 1
return features
LIBXML_COMPILED_FEATURES = _copy_lib_features()
LIBXML_FEATURES = {
feature_name for feature_id, feature_name in [
#XML_WITH_THREAD = 1
#XML_WITH_TREE = 2
#XML_WITH_OUTPUT = 3
#XML_WITH_PUSH = 4
#XML_WITH_READER = 5
#XML_WITH_PATTERN = 6
#XML_WITH_WRITER = 7
#XML_WITH_SAX1 = 8
(xmlparser.XML_WITH_FTP, "ftp"), # XML_WITH_FTP = 9
(xmlparser.XML_WITH_HTTP, "http"), # XML_WITH_HTTP = 10
#XML_WITH_VALID = 11
(xmlparser.XML_WITH_HTML, "html"), # XML_WITH_HTML = 12
#XML_WITH_LEGACY = 13
#XML_WITH_C14N = 14
(xmlparser.XML_WITH_CATALOG, "catalog"), # XML_WITH_CATALOG = 15
(xmlparser.XML_WITH_XPATH, "xpath"), # XML_WITH_XPATH = 16
#XML_WITH_XPTR = 17
#XML_WITH_XINCLUDE = 18
(xmlparser.XML_WITH_ICONV, "iconv"), # XML_WITH_ICONV = 19
#XML_WITH_ISO8859X = 20
#XML_WITH_UNICODE = 21
(xmlparser.XML_WITH_REGEXP, "regexp"), # XML_WITH_REGEXP = 22
#XML_WITH_AUTOMATA = 23
#XML_WITH_EXPR = 24
(xmlparser.XML_WITH_SCHEMAS, "xmlschema"), # XML_WITH_SCHEMAS = 25
(xmlparser.XML_WITH_SCHEMATRON, "schematron"), # XML_WITH_SCHEMATRON = 26
#XML_WITH_MODULES = 27
#XML_WITH_DEBUG = 28
#XML_WITH_DEBUG_MEM = 29
#XML_WITH_DEBUG_RUN = 30 # unused
(xmlparser.XML_WITH_ZLIB, "zlib"), # XML_WITH_ZLIB = 31
(xmlparser.XML_WITH_ICU, "icu"), # XML_WITH_ICU = 32
(xmlparser.XML_WITH_LZMA, "lzma"), # XML_WITH_LZMA = 33
] if xmlparser.xmlHasFeature(feature_id)
}
cdef bint HAS_ZLIB_COMPRESSION = xmlparser.xmlHasFeature(xmlparser.XML_WITH_ZLIB)
# class for temporary storage of Python references,
# used e.g. for XPath results
@cython.final
@@ -417,12 +321,10 @@ cdef class _TempStore:
@cython.internal
cdef class _ExceptionContext:
cdef object _exc_info
cdef int clear(self) except -1:
self._exc_info = None
return 0
@cython.final
cdef void _store_raised(self) noexcept:
try:
self._exc_info = sys.exc_info()
@@ -431,16 +333,13 @@ cdef class _ExceptionContext:
finally:
return # and swallow any further exceptions
@cython.final
cdef int _store_exception(self, exception) except -1:
self._exc_info = (exception, None, None)
return 0
@cython.final
cdef bint _has_raised(self) except -1:
return self._exc_info is not None
@cython.final
cdef int _raise_if_stored(self) except -1:
if self._exc_info is None:
return 0
@@ -478,7 +377,6 @@ cdef public class _Document [ type LxmlDocumentType, object LxmlDocument ]:
cdef int _ns_counter
cdef bytes _prefix_tail
cdef xmlDoc* _c_doc
# -- End of public part, '_c_doc' is the only publicly exposed struct field.
cdef _BaseParser _parser
def __dealloc__(self):
@@ -621,15 +519,13 @@ cdef public class _Document [ type LxmlDocumentType, object LxmlDocument ]:
c_ns = self._findOrBuildNodeNs(c_node, c_href, NULL, 0)
tree.xmlSetNs(c_node, c_ns)
cdef tuple __initPrefixCache():
cdef int i
return tuple([ python.PyBytes_FromFormat("ns%d", i)
for i in range(26) ])
for i in range(30) ])
cdef tuple _PREFIX_CACHE = __initPrefixCache()
cdef _Document _documentFactory(xmlDoc* c_doc, _BaseParser parser):
cdef _Document result
result = _Document.__new__(_Document)
@@ -663,7 +559,7 @@ cdef class DocInfo:
return root_name
@cython.final
cdef tree.xmlDtd* _get_c_dtd(self) noexcept:
cdef tree.xmlDtd* _get_c_dtd(self):
""""Return the DTD. Create it if it does not yet exist."""
cdef xmlDoc* c_doc = self._doc._c_doc
cdef xmlNode* c_root_node
@@ -872,7 +768,7 @@ cdef public class _Element [ type LxmlElementType, object LxmlElement ]:
left_to_right = 1
else:
left_to_right = 0
step = -step if step != python.PY_SSIZE_T_MIN else python.PY_SSIZE_T_MAX
step = -step
_replaceSlice(self, c_node, slicelength, step, left_to_right, value)
return
else:
@@ -1287,7 +1183,7 @@ cdef public class _Element [ type LxmlElementType, object LxmlElement ]:
if step > 0:
next_element = _nextElement
else:
step = -step if step != python.PY_SSIZE_T_MIN else python.PY_SSIZE_T_MAX
step = -step
next_element = _previousElement
result = []
c = 0
@@ -1318,8 +1214,7 @@ cdef public class _Element [ type LxmlElementType, object LxmlElement ]:
"""__bool__(self)"""
import warnings
warnings.warn(
"Truth-testing of elements was a source of confusion and will always "
"return True in future versions. "
"The behavior of this method will change in future versions. "
"Use specific 'len(elem)' or 'elem is not None' test instead.",
FutureWarning
)
@@ -1741,6 +1636,11 @@ cdef public class _Element [ type LxmlElementType, object LxmlElement ]:
return CSSSelector(expr, translator=translator)(self)
cdef extern from "includes/etree_defs.h":
# macro call to 't->tp_new()' for fast instantiation
cdef object NEW_ELEMENT "PY_NEW" (object t)
@cython.linetrace(False)
cdef _Element _elementFactory(_Document doc, xmlNode* c_node):
cdef _Element result
@@ -1750,15 +1650,12 @@ cdef _Element _elementFactory(_Document doc, xmlNode* c_node):
if c_node is NULL:
return None
element_class = <type> LOOKUP_ELEMENT_CLASS(
element_class = LOOKUP_ELEMENT_CLASS(
ELEMENT_CLASS_LOOKUP_STATE, doc, c_node)
if type(element_class) is not type:
if not isinstance(element_class, type):
raise TypeError(f"Element class is not a type, got {type(element_class)}")
if hasProxy(c_node):
# prevent re-entry race condition - we just called into Python
return getProxy(c_node)
result = element_class.__new__(element_class)
result = NEW_ELEMENT(element_class)
if hasProxy(c_node):
# prevent re-entry race condition - we just called into Python
result._c_node = NULL
@@ -3162,7 +3059,7 @@ cdef class ElementTextIterator:
return result
cdef xmlNode* _createElement(xmlDoc* c_doc, object name_utf) noexcept:
cdef xmlNode* _createElement(xmlDoc* c_doc, object name_utf) except NULL:
cdef xmlNode* c_node
c_node = tree.xmlNewDocNode(c_doc, NULL, _xcstr(name_utf), NULL)
return c_node
@@ -3184,34 +3081,18 @@ cdef xmlNode* _createEntity(xmlDoc* c_doc, const_xmlChar* name) noexcept:
# module-level API for ElementTree
from abc import ABC
class Element(ABC):
def Element(_tag, attrib=None, nsmap=None, **_extra):
"""Element(_tag, attrib=None, nsmap=None, **_extra)
Element factory, as a class.
An instance of this class is an object implementing the
Element factory. This function returns an object implementing the
Element interface.
>>> element = Element("test")
>>> type(element)
<class 'lxml.etree._Element'>
>>> isinstance(element, Element)
True
>>> issubclass(_Element, Element)
True
Also look at the `_Element.makeelement()` and
`_BaseParser.makeelement()` methods, which provide a faster way to
create an Element within a specific document or parser context.
"""
def __new__(cls, _tag, attrib=None, nsmap=None, **_extra):
return _makeElement(_tag, NULL, None, None, None, None,
attrib, nsmap, _extra)
# Register _Element as a virtual subclass of Element
Element.register(_Element)
return _makeElement(_tag, NULL, None, None, None, None,
attrib, nsmap, _extra)
def Comment(text=None):
@@ -3323,41 +3204,32 @@ def SubElement(_Element _parent not None, _tag,
"""
return _makeSubElement(_parent, _tag, None, None, attrib, nsmap, _extra)
from typing import Generic, TypeVar
T = TypeVar("T")
def ElementTree(_Element element=None, *, file=None, _BaseParser parser=None):
"""ElementTree(element=None, file=None, parser=None)
class ElementTree(ABC, Generic[T]):
def __new__(cls, _Element element=None, *, file=None, _BaseParser parser=None):
"""ElementTree(element=None, file=None, parser=None)
ElementTree wrapper class.
"""
cdef xmlNode* c_next
cdef xmlNode* c_node
cdef xmlNode* c_node_copy
cdef xmlDoc* c_doc
cdef _ElementTree etree
cdef _Document doc
ElementTree wrapper class.
"""
cdef xmlNode* c_next
cdef xmlNode* c_node
cdef xmlNode* c_node_copy
cdef xmlDoc* c_doc
cdef _ElementTree etree
cdef _Document doc
if element is not None:
doc = element._doc
elif file is not None:
try:
doc = _parseDocument(file, parser, None)
except _TargetParserResult as result_container:
return result_container.result
else:
c_doc = _newXMLDoc()
doc = _documentFactory(c_doc, parser)
if element is not None:
doc = element._doc
elif file is not None:
try:
doc = _parseDocument(file, parser, None)
except _TargetParserResult as result_container:
return result_container.result
else:
c_doc = _newXMLDoc()
doc = _documentFactory(c_doc, parser)
return _elementTreeFactory(doc, element)
return _elementTreeFactory(doc, element)
# Register _ElementTree as a virtual subclass of ElementTree
ElementTree.register(_ElementTree)
# Remove "ABC" and typing helpers from module dict
del ABC, Generic, TypeVar, T
def HTML(text, _BaseParser parser=None, *, base_url=None):
"""HTML(text, parser=None, base_url=None)