marcocosta10000/PsAgentAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): scrape linked pages to ground answers in real content

Browse Source 
Adds `_fetch_page_content()`, which downloads a URL with a
browser-like User-Agent and uses BeautifulSoup with the lxml
parser to strip scripts, navigation, and other non-content
markup, returning up to 3000 characters of readable text.

The response validator now also compares generated replies
against this scraped text, flagging named entities and claims
that do not appear in the retrieved page content.

Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0
for parser compatibility.
This commit is contained in:
marcocosta10000
2026-10-26 16:48:29 -03:00
parent 9eed02645b
commit 723ca1506b
231 changed files with 20508 additions and 5330 deletions
Download Patch File Download Diff File Expand all files Collapse all files
venv/lib/python3.11/site-packages/lxml/includes/libxml/tree.h
+96 -60
View File
@@ -32,12 +32,6 @@
extern "C" {
#endif
/*
* Backward compatibility
*/
#define xmlBufferAllocScheme XML_BUFFER_ALLOC_EXACT
#define xmlDefaultBufferSize 4096
/*
* Some of the basic types pointer to structures:
*/
@@ -65,6 +59,13 @@ typedef xmlSAXHandler *xmlSAXHandlerPtr;
typedef struct _xmlEntity xmlEntity;
typedef xmlEntity *xmlEntityPtr;
/**
* BASE_BUFFER_SIZE:
*
* default buffer size 4000.
*/
#define BASE_BUFFER_SIZE 4096
/**
* LIBXML_NAMESPACE_DICT:
*
@@ -99,16 +100,11 @@ typedef enum {
typedef struct _xmlBuffer xmlBuffer;
typedef xmlBuffer *xmlBufferPtr;
struct _xmlBuffer {
/* The buffer content UTF8 */
xmlChar *content XML_DEPRECATED_MEMBER;
/* The buffer size used */
unsigned int use XML_DEPRECATED_MEMBER;
/* The buffer size */
unsigned int size XML_DEPRECATED_MEMBER;
/* The realloc method */
xmlBufferAllocationScheme alloc XML_DEPRECATED_MEMBER;
/* in IO mode we may have a different base */
xmlChar *contentIO XML_DEPRECATED_MEMBER;
xmlChar *content; /* The buffer content UTF8 */
unsigned int use; /* The buffer size used */
unsigned int size; /* The buffer size */
xmlBufferAllocationScheme alloc; /* The realloc method */
xmlChar *contentIO; /* in IO mode we may have a different base */
};
/**
@@ -177,13 +173,13 @@ typedef enum {
XML_TEXT_NODE= 3,
XML_CDATA_SECTION_NODE= 4,
XML_ENTITY_REF_NODE= 5,
XML_ENTITY_NODE= 6, /* unused */
XML_ENTITY_NODE= 6,
XML_PI_NODE= 7,
XML_COMMENT_NODE= 8,
XML_DOCUMENT_NODE= 9,
XML_DOCUMENT_TYPE_NODE= 10, /* unused */
XML_DOCUMENT_TYPE_NODE= 10,
XML_DOCUMENT_FRAG_NODE= 11,
XML_NOTATION_NODE= 12, /* unused */
XML_NOTATION_NODE= 12,
XML_HTML_DOCUMENT_NODE= 13,
XML_DTD_NODE= 14,
XML_ELEMENT_DECL= 15,
@@ -453,7 +449,6 @@ struct _xmlAttr {
xmlNs *ns; /* pointer to the associated namespace */
xmlAttributeType atype; /* the attribute type if validating */
void *psvi; /* for type/PSVI information */
struct _xmlID *id; /* the ID struct */
};
/**
@@ -678,16 +673,24 @@ typedef void (*xmlDeregisterNodeFunc) (xmlNodePtr node);
*/
/** DOC_DISABLE */
XML_DEPRECATED
XMLPUBFUN xmlRegisterNodeFunc *__xmlRegisterNodeDefaultValue(void);
XML_DEPRECATED
XMLPUBFUN xmlDeregisterNodeFunc *__xmlDeregisterNodeDefaultValue(void);
#define XML_GLOBALS_TREE \
XML_OP(xmlBufferAllocScheme, xmlBufferAllocationScheme, XML_DEPRECATED) \
XML_OP(xmlDefaultBufferSize, int, XML_DEPRECATED) \
XML_OP(xmlRegisterNodeDefaultValue, xmlRegisterNodeFunc, XML_DEPRECATED) \
XML_OP(xmlDeregisterNodeDefaultValue, xmlDeregisterNodeFunc, \
XML_DEPRECATED)
#ifndef XML_GLOBALS_NO_REDEFINITION
#define XML_OP XML_DECLARE_GLOBAL
XML_GLOBALS_TREE
#undef XML_OP
#if defined(LIBXML_THREAD_ENABLED) && !defined(XML_GLOBALS_NO_REDEFINITION)
#define xmlBufferAllocScheme XML_GLOBAL_MACRO(xmlBufferAllocScheme)
#define xmlDefaultBufferSize XML_GLOBAL_MACRO(xmlDefaultBufferSize)
#define xmlRegisterNodeDefaultValue \
(*__xmlRegisterNodeDefaultValue())
XML_GLOBAL_MACRO(xmlRegisterNodeDefaultValue)
#define xmlDeregisterNodeDefaultValue \
(*__xmlDeregisterNodeDefaultValue())
XML_GLOBAL_MACRO(xmlDeregisterNodeDefaultValue)
#endif
/** DOC_ENABLE */
@@ -698,6 +701,7 @@ XMLPUBFUN int
xmlValidateNCName (const xmlChar *value,
int space);
#if defined(LIBXML_TREE_ENABLED) || defined(LIBXML_SCHEMAS_ENABLED)
XMLPUBFUN int
xmlValidateQName (const xmlChar *value,
int space);
@@ -707,6 +711,7 @@ XMLPUBFUN int
XMLPUBFUN int
xmlValidateNMToken (const xmlChar *value,
int space);
#endif
XMLPUBFUN xmlChar *
xmlBuildQName (const xmlChar *ncname,
@@ -724,10 +729,8 @@ XMLPUBFUN const xmlChar *
* Handling Buffers, the old ones see @xmlBuf for the new ones.
*/
XML_DEPRECATED
XMLPUBFUN void
xmlSetBufferAllocationScheme(xmlBufferAllocationScheme scheme);
XML_DEPRECATED
XMLPUBFUN xmlBufferAllocationScheme
xmlGetBufferAllocationScheme(void);
@@ -738,7 +741,6 @@ XMLPUBFUN xmlBufferPtr
XMLPUBFUN xmlBufferPtr
xmlBufferCreateStatic (void *mem,
size_t size);
XML_DEPRECATED
XMLPUBFUN int
xmlBufferResize (xmlBufferPtr buf,
unsigned int size);
@@ -761,11 +763,9 @@ XMLPUBFUN int
XMLPUBFUN int
xmlBufferCCat (xmlBufferPtr buf,
const char *str);
XML_DEPRECATED
XMLPUBFUN int
xmlBufferShrink (xmlBufferPtr buf,
unsigned int len);
XML_DEPRECATED
XMLPUBFUN int
xmlBufferGrow (xmlBufferPtr buf,
unsigned int len);
@@ -798,6 +798,13 @@ XMLPUBFUN xmlDtdPtr
xmlGetIntSubset (const xmlDoc *doc);
XMLPUBFUN void
xmlFreeDtd (xmlDtdPtr cur);
#ifdef LIBXML_LEGACY_ENABLED
XML_DEPRECATED
XMLPUBFUN xmlNsPtr
xmlNewGlobalNs (xmlDocPtr doc,
const xmlChar *href,
const xmlChar *prefix);
#endif /* LIBXML_LEGACY_ENABLED */
XMLPUBFUN xmlNsPtr
xmlNewNs (xmlNodePtr node,
const xmlChar *href,
@@ -814,10 +821,13 @@ XMLPUBFUN xmlAttrPtr
xmlNewDocProp (xmlDocPtr doc,
const xmlChar *name,
const xmlChar *value);
#if defined(LIBXML_TREE_ENABLED) || defined(LIBXML_HTML_ENABLED) || \
defined(LIBXML_SCHEMAS_ENABLED)
XMLPUBFUN xmlAttrPtr
xmlNewProp (xmlNodePtr node,
const xmlChar *name,
const xmlChar *value);
#endif
XMLPUBFUN xmlAttrPtr
xmlNewNsProp (xmlNodePtr node,
xmlNsPtr ns,
@@ -838,11 +848,15 @@ XMLPUBFUN xmlAttrPtr
XMLPUBFUN xmlAttrPtr
xmlCopyPropList (xmlNodePtr target,
xmlAttrPtr cur);
#ifdef LIBXML_TREE_ENABLED
XMLPUBFUN xmlDtdPtr
xmlCopyDtd (xmlDtdPtr dtd);
#endif /* LIBXML_TREE_ENABLED */
#if defined(LIBXML_TREE_ENABLED) || defined(LIBXML_SCHEMAS_ENABLED)
XMLPUBFUN xmlDocPtr
xmlCopyDoc (xmlDocPtr doc,
int recursive);
#endif /* defined(LIBXML_TREE_ENABLED) || defined(LIBXML_SCHEMAS_ENABLED) */
/*
* Creating new nodes.
*/
@@ -862,11 +876,13 @@ XMLPUBFUN xmlNodePtr
XMLPUBFUN xmlNodePtr
xmlNewNodeEatName (xmlNsPtr ns,
xmlChar *name);
#if defined(LIBXML_TREE_ENABLED) || defined(LIBXML_SCHEMAS_ENABLED)
XMLPUBFUN xmlNodePtr
xmlNewChild (xmlNodePtr parent,
xmlNsPtr ns,
const xmlChar *name,
const xmlChar *content);
#endif
XMLPUBFUN xmlNodePtr
xmlNewDocText (const xmlDoc *doc,
const xmlChar *content);
@@ -913,6 +929,7 @@ XMLPUBFUN xmlNodePtr
xmlNodePtr node);
XMLPUBFUN xmlNodePtr
xmlCopyNodeList (xmlNodePtr node);
#ifdef LIBXML_TREE_ENABLED
XMLPUBFUN xmlNodePtr
xmlNewTextChild (xmlNodePtr parent,
xmlNsPtr ns,
@@ -925,14 +942,17 @@ XMLPUBFUN xmlNodePtr
const xmlChar *content);
XMLPUBFUN xmlNodePtr
xmlNewDocFragment (xmlDocPtr doc);
#endif /* LIBXML_TREE_ENABLED */
/*
* Navigating.
*/
XMLPUBFUN long
xmlGetLineNo (const xmlNode *node);
#if defined(LIBXML_TREE_ENABLED) || defined(LIBXML_DEBUG_ENABLED)
XMLPUBFUN xmlChar *
xmlGetNodePath (const xmlNode *node);
#endif /* defined(LIBXML_TREE_ENABLED) || defined(LIBXML_DEBUG_ENABLED) */
XMLPUBFUN xmlNodePtr
xmlDocGetRootElement (const xmlDoc *doc);
XMLPUBFUN xmlNodePtr
@@ -945,24 +965,33 @@ XMLPUBFUN int
/*
* Changing the structure.
*/
#if defined(LIBXML_TREE_ENABLED) || defined(LIBXML_WRITER_ENABLED)
XMLPUBFUN xmlNodePtr
xmlDocSetRootElement (xmlDocPtr doc,
xmlNodePtr root);
#endif /* defined(LIBXML_TREE_ENABLED) || defined(LIBXML_WRITER_ENABLED) */
#ifdef LIBXML_TREE_ENABLED
XMLPUBFUN void
xmlNodeSetName (xmlNodePtr cur,
const xmlChar *name);
#endif /* LIBXML_TREE_ENABLED */
XMLPUBFUN xmlNodePtr
xmlAddChild (xmlNodePtr parent,
xmlNodePtr cur);
XMLPUBFUN xmlNodePtr
xmlAddChildList (xmlNodePtr parent,
xmlNodePtr cur);
#if defined(LIBXML_TREE_ENABLED) || defined(LIBXML_WRITER_ENABLED)
XMLPUBFUN xmlNodePtr
xmlReplaceNode (xmlNodePtr old,
xmlNodePtr cur);
#endif /* defined(LIBXML_TREE_ENABLED) || defined(LIBXML_WRITER_ENABLED) */
#if defined(LIBXML_TREE_ENABLED) || defined(LIBXML_HTML_ENABLED) || \
defined(LIBXML_SCHEMAS_ENABLED) || defined(LIBXML_XINCLUDE_ENABLED)
XMLPUBFUN xmlNodePtr
xmlAddPrevSibling (xmlNodePtr cur,
xmlNodePtr elem);
#endif /* LIBXML_TREE_ENABLED || LIBXML_HTML_ENABLED || LIBXML_SCHEMAS_ENABLED */
XMLPUBFUN xmlNodePtr
xmlAddSibling (xmlNodePtr cur,
xmlNodePtr elem);
@@ -982,10 +1011,10 @@ XMLPUBFUN void
xmlFreeNodeList (xmlNodePtr cur);
XMLPUBFUN void
xmlFreeNode (xmlNodePtr cur);
XMLPUBFUN int
XMLPUBFUN void
xmlSetTreeDoc (xmlNodePtr tree,
xmlDocPtr doc);
XMLPUBFUN int
XMLPUBFUN void
xmlSetListDoc (xmlNodePtr list,
xmlDocPtr doc);
/*
@@ -999,13 +1028,12 @@ XMLPUBFUN xmlNsPtr
xmlSearchNsByHref (xmlDocPtr doc,
xmlNodePtr node,
const xmlChar *href);
XMLPUBFUN int
xmlGetNsListSafe (const xmlDoc *doc,
const xmlNode *node,
xmlNsPtr **out);
#if defined(LIBXML_TREE_ENABLED) || defined(LIBXML_XPATH_ENABLED) || \
defined(LIBXML_SCHEMAS_ENABLED)
XMLPUBFUN xmlNsPtr *
xmlGetNsList (const xmlDoc *doc,
const xmlNode *node);
#endif /* defined(LIBXML_TREE_ENABLED) || defined(LIBXML_XPATH_ENABLED) */
XMLPUBFUN void
xmlSetNs (xmlNodePtr node,
@@ -1018,6 +1046,8 @@ XMLPUBFUN xmlNsPtr
/*
* Changing the content.
*/
#if defined(LIBXML_TREE_ENABLED) || defined(LIBXML_XINCLUDE_ENABLED) || \
defined(LIBXML_SCHEMAS_ENABLED) || defined(LIBXML_HTML_ENABLED)
XMLPUBFUN xmlAttrPtr
xmlSetProp (xmlNodePtr node,
const xmlChar *name,
@@ -1027,11 +1057,8 @@ XMLPUBFUN xmlAttrPtr
xmlNsPtr ns,
const xmlChar *name,
const xmlChar *value);
XMLPUBFUN int
xmlNodeGetAttrValue (const xmlNode *node,
const xmlChar *name,
const xmlChar *nsUri,
xmlChar **out);
#endif /* defined(LIBXML_TREE_ENABLED) || defined(LIBXML_XINCLUDE_ENABLED) || \
defined(LIBXML_SCHEMAS_ENABLED) || defined(LIBXML_HTML_ENABLED) */
XMLPUBFUN xmlChar *
xmlGetNoNsProp (const xmlNode *node,
const xmlChar *name);
@@ -1060,21 +1087,25 @@ XMLPUBFUN xmlChar *
xmlNodeListGetString (xmlDocPtr doc,
const xmlNode *list,
int inLine);
#ifdef LIBXML_TREE_ENABLED
XMLPUBFUN xmlChar *
xmlNodeListGetRawString (const xmlDoc *doc,
const xmlNode *list,
int inLine);
XMLPUBFUN int
#endif /* LIBXML_TREE_ENABLED */
XMLPUBFUN void
xmlNodeSetContent (xmlNodePtr cur,
const xmlChar *content);
XMLPUBFUN int
#ifdef LIBXML_TREE_ENABLED
XMLPUBFUN void
xmlNodeSetContentLen (xmlNodePtr cur,
const xmlChar *content,
int len);
XMLPUBFUN int
#endif /* LIBXML_TREE_ENABLED */
XMLPUBFUN void
xmlNodeAddContent (xmlNodePtr cur,
const xmlChar *content);
XMLPUBFUN int
XMLPUBFUN void
xmlNodeAddContentLen (xmlNodePtr cur,
const xmlChar *content,
int len);
@@ -1092,28 +1123,29 @@ XMLPUBFUN xmlChar *
xmlNodeGetLang (const xmlNode *cur);
XMLPUBFUN int
xmlNodeGetSpacePreserve (const xmlNode *cur);
XMLPUBFUN int
#ifdef LIBXML_TREE_ENABLED
XMLPUBFUN void
xmlNodeSetLang (xmlNodePtr cur,
const xmlChar *lang);
XMLPUBFUN int
XMLPUBFUN void
xmlNodeSetSpacePreserve (xmlNodePtr cur,
int val);
XMLPUBFUN int
xmlNodeGetBaseSafe (const xmlDoc *doc,
const xmlNode *cur,
xmlChar **baseOut);
#endif /* LIBXML_TREE_ENABLED */
XMLPUBFUN xmlChar *
xmlNodeGetBase (const xmlDoc *doc,
const xmlNode *cur);
XMLPUBFUN int
#if defined(LIBXML_TREE_ENABLED) || defined(LIBXML_XINCLUDE_ENABLED)
XMLPUBFUN void
xmlNodeSetBase (xmlNodePtr cur,
const xmlChar *uri);
#endif
/*
* Removing content.
*/
XMLPUBFUN int
xmlRemoveProp (xmlAttrPtr cur);
#if defined(LIBXML_TREE_ENABLED) || defined(LIBXML_SCHEMAS_ENABLED)
XMLPUBFUN int
xmlUnsetNsProp (xmlNodePtr node,
xmlNsPtr ns,
@@ -1121,6 +1153,7 @@ XMLPUBFUN int
XMLPUBFUN int
xmlUnsetProp (xmlNodePtr node,
const xmlChar *name);
#endif /* defined(LIBXML_TREE_ENABLED) || defined(LIBXML_SCHEMAS_ENABLED) */
/*
* Internal, don't use.
@@ -1142,12 +1175,14 @@ XMLPUBFUN void xmlAttrSerializeTxtContent(xmlBufferPtr buf,
const xmlChar *string);
#endif /* LIBXML_OUTPUT_ENABLED */
#ifdef LIBXML_TREE_ENABLED
/*
* Namespace handling.
*/
XMLPUBFUN int
xmlReconciliateNs (xmlDocPtr doc,
xmlNodePtr tree);
#endif
#ifdef LIBXML_OUTPUT_ENABLED
/*
@@ -1248,10 +1283,8 @@ XMLPUBFUN int
XMLPUBFUN void
xmlSetDocCompressMode (xmlDocPtr doc,
int mode);
XML_DEPRECATED
XMLPUBFUN int
xmlGetCompressMode (void);
XML_DEPRECATED
XMLPUBFUN void
xmlSetCompressMode (int mode);
@@ -1288,6 +1321,7 @@ XMLPUBFUN int
int deep,
int options);
#ifdef LIBXML_TREE_ENABLED
/*
* 5 interfaces from DOM ElementTraversal, but different in entities
* traversal.
@@ -1302,20 +1336,22 @@ XMLPUBFUN xmlNodePtr
xmlLastElementChild (xmlNodePtr parent);
XMLPUBFUN xmlNodePtr
xmlPreviousElementSibling (xmlNodePtr node);
#endif
XML_DEPRECATED
XMLPUBFUN xmlRegisterNodeFunc
xmlRegisterNodeDefault (xmlRegisterNodeFunc func);
XML_DEPRECATED
XMLPUBFUN xmlDeregisterNodeFunc
xmlDeregisterNodeDefault (xmlDeregisterNodeFunc func);
XML_DEPRECATED
XMLPUBFUN xmlRegisterNodeFunc
xmlThrDefRegisterNodeDefault(xmlRegisterNodeFunc func);
XML_DEPRECATED
XMLPUBFUN xmlDeregisterNodeFunc
xmlThrDefDeregisterNodeDefault(xmlDeregisterNodeFunc func);
XML_DEPRECATED XMLPUBFUN xmlBufferAllocationScheme
xmlThrDefBufferAllocScheme (xmlBufferAllocationScheme v);
XML_DEPRECATED XMLPUBFUN int
xmlThrDefDefaultBufferSize (int v);
#ifdef __cplusplus
}
#endif