marcocosta10000/PsAgentAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): scrape linked pages to ground answers in real content

Browse Source 
Adds `_fetch_page_content()`, which downloads a URL with a
browser-like User-Agent and uses BeautifulSoup with the lxml
parser to strip scripts, navigation, and other non-content
markup, returning up to 3000 characters of readable text.

The response validator now also compares generated replies
against this scraped text, flagging named entities and claims
that do not appear in the retrieved page content.

Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0
for parser compatibility.
This commit is contained in:
marcocosta10000
2026-10-26 16:48:29 -03:00
parent 9eed02645b
commit 723ca1506b
231 changed files with 20508 additions and 5330 deletions
Download Patch File Download Diff File Expand all files Collapse all files
venv/lib/python3.11/site-packages/lxml/public-api.pxi
+10 -10
View File
@@ -51,7 +51,7 @@ cdef public api object callLookupFallback(FallbackElementClassLookup lookup,
_Document doc, xmlNode* c_node):
return _callLookupFallback(lookup, doc, c_node)
cdef public api int tagMatches(xmlNode* c_node, const_xmlChar* c_href, const_xmlChar* c_name) noexcept:
cdef public api int tagMatches(xmlNode* c_node, const_xmlChar* c_href, const_xmlChar* c_name):
if c_node is NULL:
return -1
return _tagMatches(c_node, c_href, c_name)
@@ -62,10 +62,10 @@ cdef public api _Document documentOrRaise(object input):
cdef public api _Element rootNodeOrRaise(object input):
return _rootNodeOrRaise(input)
cdef public api bint hasText(xmlNode* c_node) noexcept:
cdef public api bint hasText(xmlNode* c_node):
return _hasText(c_node)
cdef public api bint hasTail(xmlNode* c_node) noexcept:
cdef public api bint hasTail(xmlNode* c_node):
return _hasTail(c_node)
cdef public api unicode textOf(xmlNode* c_node):
@@ -115,25 +115,25 @@ cdef public api int delAttribute(_Element element, key) except -1:
return _delAttribute(element, key)
cdef public api int delAttributeFromNsName(tree.xmlNode* c_element,
const_xmlChar* c_href, const_xmlChar* c_name) noexcept:
const_xmlChar* c_href, const_xmlChar* c_name):
return _delAttributeFromNsName(c_element, c_href, c_name)
cdef public api bint hasChild(xmlNode* c_node) noexcept:
cdef public api bint hasChild(xmlNode* c_node):
return _hasChild(c_node)
cdef public api xmlNode* findChild(xmlNode* c_node, Py_ssize_t index) noexcept:
cdef public api xmlNode* findChild(xmlNode* c_node, Py_ssize_t index):
return _findChild(c_node, index)
cdef public api xmlNode* findChildForwards(xmlNode* c_node, Py_ssize_t index) noexcept:
cdef public api xmlNode* findChildForwards(xmlNode* c_node, Py_ssize_t index):
return _findChildForwards(c_node, index)
cdef public api xmlNode* findChildBackwards(xmlNode* c_node, Py_ssize_t index) noexcept:
cdef public api xmlNode* findChildBackwards(xmlNode* c_node, Py_ssize_t index):
return _findChildBackwards(c_node, index)
cdef public api xmlNode* nextElement(xmlNode* c_node) noexcept:
cdef public api xmlNode* nextElement(xmlNode* c_node):
return _nextElement(c_node)
cdef public api xmlNode* previousElement(xmlNode* c_node) noexcept:
cdef public api xmlNode* previousElement(xmlNode* c_node):
return _previousElement(c_node)
cdef public api void appendChild(_Element parent, _Element child):