marcocosta10000/PsAgentAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): scrape linked pages to ground answers in real content

Browse Source 
Adds `_fetch_page_content()`, which downloads a URL with a
browser-like User-Agent and uses BeautifulSoup with the lxml
parser to strip scripts, navigation, and other non-content
markup, returning up to 3000 characters of readable text.

The response validator now also compares generated replies
against this scraped text, flagging named entities and claims
that do not appear in the retrieved page content.

Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0
for parser compatibility.
This commit is contained in:
marcocosta10000
2026-10-26 16:48:29 -03:00
parent 9eed02645b
commit 723ca1506b
231 changed files with 20508 additions and 5330 deletions
Download Patch File Download Diff File Expand all files Collapse all files
venv/lib/python3.11/site-packages/lxml/saxparser.pxi
+15 -15
View File
@@ -217,7 +217,7 @@ cdef class _SaxParserContext(_ParserContext):
finally:
self._parser = None # clear circular reference ASAP
if self._matcher is not None:
self._matcher.cacheTags(self._doc, force_into_dict=True)
self._matcher.cacheTags(self._doc, True) # force entry in libxml2 dict
return 0
cdef int pushEvent(self, event, xmlNode* c_node) except -1:
@@ -297,7 +297,7 @@ cdef void _handleSaxStart(
cdef int i
cdef size_t c_len
c_ctxt = <xmlparser.xmlParserCtxt*>ctxt
if c_ctxt._private is NULL or xmlparser.xmlCtxtIsStopped(c_ctxt):
if c_ctxt._private is NULL or c_ctxt.disableSAX:
return
context = <_SaxParserContext>c_ctxt._private
cdef int event_filter = context._event_filter
@@ -345,7 +345,7 @@ cdef void _handleSaxTargetStart(
cdef int i
cdef size_t c_len
c_ctxt = <xmlparser.xmlParserCtxt*>ctxt
if c_ctxt._private is NULL or xmlparser.xmlCtxtIsStopped(c_ctxt):
if c_ctxt._private is NULL or c_ctxt.disableSAX:
return
context = <_SaxParserContext>c_ctxt._private
@@ -411,7 +411,7 @@ cdef void _handleSaxTargetStart(
cdef void _handleSaxStartNoNs(void* ctxt, const_xmlChar* c_name,
const_xmlChar** c_attributes) noexcept with gil:
c_ctxt = <xmlparser.xmlParserCtxt*>ctxt
if c_ctxt._private is NULL or xmlparser.xmlCtxtIsStopped(c_ctxt):
if c_ctxt._private is NULL or c_ctxt.disableSAX:
return
context = <_SaxParserContext>c_ctxt._private
try:
@@ -436,7 +436,7 @@ cdef void _handleSaxStartNoNs(void* ctxt, const_xmlChar* c_name,
cdef void _handleSaxTargetStartNoNs(void* ctxt, const_xmlChar* c_name,
const_xmlChar** c_attributes) noexcept with gil:
c_ctxt = <xmlparser.xmlParserCtxt*>ctxt
if c_ctxt._private is NULL or xmlparser.xmlCtxtIsStopped(c_ctxt):
if c_ctxt._private is NULL or c_ctxt.disableSAX:
return
context = <_SaxParserContext>c_ctxt._private
try:
@@ -493,7 +493,7 @@ cdef void _handleSaxEnd(void* ctxt, const_xmlChar* c_localname,
const_xmlChar* c_prefix,
const_xmlChar* c_namespace) noexcept with gil:
c_ctxt = <xmlparser.xmlParserCtxt*>ctxt
if c_ctxt._private is NULL or xmlparser.xmlCtxtIsStopped(c_ctxt):
if c_ctxt._private is NULL or c_ctxt.disableSAX:
return
context = <_SaxParserContext>c_ctxt._private
try:
@@ -516,7 +516,7 @@ cdef void _handleSaxEnd(void* ctxt, const_xmlChar* c_localname,
cdef void _handleSaxEndNoNs(void* ctxt, const_xmlChar* c_name) noexcept with gil:
c_ctxt = <xmlparser.xmlParserCtxt*>ctxt
if c_ctxt._private is NULL or xmlparser.xmlCtxtIsStopped(c_ctxt):
if c_ctxt._private is NULL or c_ctxt.disableSAX:
return
context = <_SaxParserContext>c_ctxt._private
try:
@@ -569,7 +569,7 @@ cdef int _pushSaxEndEvent(_SaxParserContext context,
cdef void _handleSaxData(void* ctxt, const_xmlChar* c_data, int data_len) noexcept with gil:
# can only be called if parsing with a target
c_ctxt = <xmlparser.xmlParserCtxt*>ctxt
if c_ctxt._private is NULL or xmlparser.xmlCtxtIsStopped(c_ctxt):
if c_ctxt._private is NULL or c_ctxt.disableSAX:
return
context = <_SaxParserContext>c_ctxt._private
try:
@@ -586,7 +586,7 @@ cdef void _handleSaxTargetDoctype(void* ctxt, const_xmlChar* c_name,
const_xmlChar* c_system) noexcept with gil:
# can only be called if parsing with a target
c_ctxt = <xmlparser.xmlParserCtxt*>ctxt
if c_ctxt._private is NULL or xmlparser.xmlCtxtIsStopped(c_ctxt):
if c_ctxt._private is NULL or c_ctxt.disableSAX:
return
context = <_SaxParserContext>c_ctxt._private
try:
@@ -602,7 +602,7 @@ cdef void _handleSaxTargetDoctype(void* ctxt, const_xmlChar* c_name,
cdef void _handleSaxStartDocument(void* ctxt) noexcept with gil:
c_ctxt = <xmlparser.xmlParserCtxt*>ctxt
if c_ctxt._private is NULL or xmlparser.xmlCtxtIsStopped(c_ctxt):
if c_ctxt._private is NULL or c_ctxt.disableSAX:
return
context = <_SaxParserContext>c_ctxt._private
context._origSaxStartDocument(ctxt)
@@ -619,7 +619,7 @@ cdef void _handleSaxTargetPI(void* ctxt, const_xmlChar* c_target,
const_xmlChar* c_data) noexcept with gil:
# can only be called if parsing with a target
c_ctxt = <xmlparser.xmlParserCtxt*>ctxt
if c_ctxt._private is NULL or xmlparser.xmlCtxtIsStopped(c_ctxt):
if c_ctxt._private is NULL or c_ctxt.disableSAX:
return
context = <_SaxParserContext>c_ctxt._private
try:
@@ -638,7 +638,7 @@ cdef void _handleSaxPIEvent(void* ctxt, const_xmlChar* target,
const_xmlChar* data) noexcept with gil:
# can only be called when collecting pi events
c_ctxt = <xmlparser.xmlParserCtxt*>ctxt
if c_ctxt._private is NULL or xmlparser.xmlCtxtIsStopped(c_ctxt):
if c_ctxt._private is NULL or c_ctxt.disableSAX:
return
context = <_SaxParserContext>c_ctxt._private
context._origSaxPI(ctxt, target, data)
@@ -656,7 +656,7 @@ cdef void _handleSaxPIEvent(void* ctxt, const_xmlChar* target,
cdef void _handleSaxTargetComment(void* ctxt, const_xmlChar* c_data) noexcept with gil:
# can only be called if parsing with a target
c_ctxt = <xmlparser.xmlParserCtxt*>ctxt
if c_ctxt._private is NULL or xmlparser.xmlCtxtIsStopped(c_ctxt):
if c_ctxt._private is NULL or c_ctxt.disableSAX:
return
context = <_SaxParserContext>c_ctxt._private
try:
@@ -672,7 +672,7 @@ cdef void _handleSaxTargetComment(void* ctxt, const_xmlChar* c_data) noexcept wi
cdef void _handleSaxComment(void* ctxt, const_xmlChar* text) noexcept with gil:
# can only be called when collecting comment events
c_ctxt = <xmlparser.xmlParserCtxt*>ctxt
if c_ctxt._private is NULL or xmlparser.xmlCtxtIsStopped(c_ctxt):
if c_ctxt._private is NULL or c_ctxt.disableSAX:
return
context = <_SaxParserContext>c_ctxt._private
context._origSaxComment(ctxt, text)
@@ -687,7 +687,7 @@ cdef void _handleSaxComment(void* ctxt, const_xmlChar* text) noexcept with gil:
return # swallow any further exceptions
cdef inline xmlNode* _findLastEventNode(xmlparser.xmlParserCtxt* c_ctxt) noexcept:
cdef inline xmlNode* _findLastEventNode(xmlparser.xmlParserCtxt* c_ctxt):
# this mimics what libxml2 creates for comments/PIs
if c_ctxt.inSubset == 1:
return c_ctxt.myDoc.intSubset.last