marcocosta10000/PsAgentAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): scrape linked pages to ground answers in real content

Browse Source 
Adds `_fetch_page_content()`, which downloads a URL with a
browser-like User-Agent and uses BeautifulSoup with the lxml
parser to strip scripts, navigation, and other non-content
markup, returning up to 3000 characters of readable text.

The response validator now also compares generated replies
against this scraped text, flagging named entities and claims
that do not appear in the retrieved page content.

Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0
for parser compatibility.
This commit is contained in:
marcocosta10000
2026-10-26 16:48:29 -03:00
parent 9eed02645b
commit 723ca1506b
231 changed files with 20508 additions and 5330 deletions
Download Patch File Download Diff File Expand all files Collapse all files
venv/lib/python3.11/site-packages/lxml/includes/libxml/xmlIO.h
+34 -32
View File
@@ -10,14 +10,12 @@
#ifndef __XML_IO_H__
#define __XML_IO_H__
/** DOC_DISABLE */
#include <stdio.h>
#include <libxml/xmlversion.h>
#include <libxml/encoding.h>
#define XML_TREE_INTERNALS
#include <libxml/tree.h>
#undef XML_TREE_INTERNALS
/** DOC_ENABLE */
#ifdef __cplusplus
extern "C" {
@@ -176,18 +174,21 @@ struct _xmlOutputBuffer {
#endif /* LIBXML_OUTPUT_ENABLED */
/** DOC_DISABLE */
XML_DEPRECATED
XMLPUBFUN xmlParserInputBufferCreateFilenameFunc *
__xmlParserInputBufferCreateFilenameValue(void);
XML_DEPRECATED
XMLPUBFUN xmlOutputBufferCreateFilenameFunc *
__xmlOutputBufferCreateFilenameValue(void);
#define XML_GLOBALS_IO \
XML_OP(xmlParserInputBufferCreateFilenameValue, \
xmlParserInputBufferCreateFilenameFunc, XML_DEPRECATED) \
XML_OP(xmlOutputBufferCreateFilenameValue, \
xmlOutputBufferCreateFilenameFunc, XML_DEPRECATED)
#ifndef XML_GLOBALS_NO_REDEFINITION
#define XML_OP XML_DECLARE_GLOBAL
XML_GLOBALS_IO
#undef XML_OP
#if defined(LIBXML_THREAD_ENABLED) && !defined(XML_GLOBALS_NO_REDEFINITION)
#define xmlParserInputBufferCreateFilenameValue \
(*__xmlParserInputBufferCreateFilenameValue())
XML_GLOBAL_MACRO(xmlParserInputBufferCreateFilenameValue)
#define xmlOutputBufferCreateFilenameValue \
(*__xmlOutputBufferCreateFilenameValue())
XML_GLOBAL_MACRO(xmlOutputBufferCreateFilenameValue)
#endif
/** DOC_ENABLE */
@@ -208,7 +209,6 @@ XMLPUBFUN xmlParserInputBufferPtr
XMLPUBFUN xmlParserInputBufferPtr
xmlParserInputBufferCreateFilename (const char *URI,
xmlCharEncoding enc);
XML_DEPRECATED
XMLPUBFUN xmlParserInputBufferPtr
xmlParserInputBufferCreateFile (FILE *file,
xmlCharEncoding enc);
@@ -226,15 +226,12 @@ XMLPUBFUN xmlParserInputBufferPtr
xmlInputCloseCallback ioclose,
void *ioctx,
xmlCharEncoding enc);
XML_DEPRECATED
XMLPUBFUN int
xmlParserInputBufferRead (xmlParserInputBufferPtr in,
int len);
XML_DEPRECATED
XMLPUBFUN int
xmlParserInputBufferGrow (xmlParserInputBufferPtr in,
int len);
XML_DEPRECATED
XMLPUBFUN int
xmlParserInputBufferPush (xmlParserInputBufferPtr in,
int len,
@@ -250,7 +247,7 @@ XMLPUBFUN int
xmlInputReadCallback readFunc,
xmlInputCloseCallback closeFunc);
XMLPUBFUN xmlParserInputBufferPtr
xmlParserInputBufferPtr
__xmlParserInputBufferCreateFilename(const char *URI,
xmlCharEncoding enc);
@@ -319,21 +316,19 @@ XMLPUBFUN int
xmlOutputWriteCallback writeFunc,
xmlOutputCloseCallback closeFunc);
XMLPUBFUN xmlOutputBufferPtr
xmlOutputBufferPtr
__xmlOutputBufferCreateFilename(const char *URI,
xmlCharEncodingHandlerPtr encoder,
int compression);
#ifdef LIBXML_HTTP_ENABLED
/* This function only exists if HTTP support built into the library */
XML_DEPRECATED
XMLPUBFUN void
xmlRegisterHTTPPostCallbacks (void );
#endif /* LIBXML_HTTP_ENABLED */
#endif /* LIBXML_OUTPUT_ENABLED */
XML_DEPRECATED
XMLPUBFUN xmlParserInputPtr
xmlCheckHTTPInput (xmlParserCtxtPtr ctxt,
xmlParserInputPtr ret);
@@ -346,28 +341,26 @@ XMLPUBFUN xmlParserInputPtr
const char *ID,
xmlParserCtxtPtr ctxt);
XML_DEPRECATED
/*
* xmlNormalizeWindowsPath is obsolete, don't use it.
* Check xmlCanonicPath in uri.h for a better alternative.
*/
XMLPUBFUN xmlChar *
xmlNormalizeWindowsPath (const xmlChar *path);
XML_DEPRECATED
XMLPUBFUN int
xmlCheckFilename (const char *path);
/**
* Default 'file://' protocol callbacks
*/
XML_DEPRECATED
XMLPUBFUN int
xmlFileMatch (const char *filename);
XML_DEPRECATED
XMLPUBFUN void *
xmlFileOpen (const char *filename);
XML_DEPRECATED
XMLPUBFUN int
xmlFileRead (void * context,
char * buffer,
int len);
XML_DEPRECATED
XMLPUBFUN int
xmlFileClose (void * context);
@@ -375,39 +368,48 @@ XMLPUBFUN int
* Default 'http://' protocol callbacks
*/
#ifdef LIBXML_HTTP_ENABLED
XML_DEPRECATED
XMLPUBFUN int
xmlIOHTTPMatch (const char *filename);
XML_DEPRECATED
XMLPUBFUN void *
xmlIOHTTPOpen (const char *filename);
#ifdef LIBXML_OUTPUT_ENABLED
XML_DEPRECATED
XMLPUBFUN void *
xmlIOHTTPOpenW (const char * post_uri,
int compression );
#endif /* LIBXML_OUTPUT_ENABLED */
XML_DEPRECATED
XMLPUBFUN int
xmlIOHTTPRead (void * context,
char * buffer,
int len);
XML_DEPRECATED
XMLPUBFUN int
xmlIOHTTPClose (void * context);
#endif /* LIBXML_HTTP_ENABLED */
/**
* Default 'ftp://' protocol callbacks
*/
#if defined(LIBXML_FTP_ENABLED)
XMLPUBFUN int
xmlIOFTPMatch (const char *filename);
XMLPUBFUN void *
xmlIOFTPOpen (const char *filename);
XMLPUBFUN int
xmlIOFTPRead (void * context,
char * buffer,
int len);
XMLPUBFUN int
xmlIOFTPClose (void * context);
#endif /* defined(LIBXML_FTP_ENABLED) */
XMLPUBFUN xmlParserInputBufferCreateFilenameFunc
xmlParserInputBufferCreateFilenameDefault(
xmlParserInputBufferCreateFilenameFunc func);
XMLPUBFUN xmlOutputBufferCreateFilenameFunc
xmlOutputBufferCreateFilenameDefault(
xmlOutputBufferCreateFilenameFunc func);
XML_DEPRECATED
XMLPUBFUN xmlOutputBufferCreateFilenameFunc
xmlThrDefOutputBufferCreateFilenameDefault(
xmlOutputBufferCreateFilenameFunc func);
XML_DEPRECATED
XMLPUBFUN xmlParserInputBufferCreateFilenameFunc
xmlThrDefParserInputBufferCreateFilenameDefault(
xmlParserInputBufferCreateFilenameFunc func);