marcocosta10000/PsAgentAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): scrape linked pages to ground answers in real content

Browse Source 
Adds `_fetch_page_content()`, which downloads a URL with a
browser-like User-Agent and uses BeautifulSoup with the lxml
parser to strip scripts, navigation, and other non-content
markup, returning up to 3000 characters of readable text.

The response validator now also compares generated replies
against this scraped text, flagging named entities and claims
that do not appear in the retrieved page content.

Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0
for parser compatibility.
This commit is contained in:
marcocosta10000
2026-10-26 16:48:29 -03:00
parent 9eed02645b
commit 723ca1506b
231 changed files with 20508 additions and 5330 deletions
Download Patch File Download Diff File Expand all files Collapse all files
venv/lib/python3.11/site-packages/lxml/includes/libxml/xmlmemory.h
+64 -27
View File
@@ -69,25 +69,27 @@ typedef char *(*xmlStrdupFunc)(const char *str);
* - xmlMemStrdup
* - xmlFree
*/
#ifdef LIBXML_THREAD_ALLOC_ENABLED
/** DOC_DISABLE */
XMLPUBFUN xmlMallocFunc *__xmlMalloc(void);
XMLPUBFUN xmlMallocFunc *__xmlMallocAtomic(void);
XMLPUBFUN xmlReallocFunc *__xmlRealloc(void);
XMLPUBFUN xmlFreeFunc *__xmlFree(void);
XMLPUBFUN xmlStrdupFunc *__xmlMemStrdup(void);
#ifndef XML_GLOBALS_NO_REDEFINITION
#define xmlMalloc (*__xmlMalloc())
#define xmlMallocAtomic (*__xmlMallocAtomic())
#define xmlRealloc (*__xmlRealloc())
#define xmlFree (*__xmlFree())
#define xmlMemStrdup (*__xmlMemStrdup())
#endif
/** DOC_ENABLE */
#ifdef LIBXML_THREAD_ALLOC_ENABLED
#define XML_GLOBALS_ALLOC \
XML_OP(xmlMalloc, xmlMallocFunc, XML_NO_ATTR) \
XML_OP(xmlMallocAtomic, xmlMallocFunc, XML_NO_ATTR) \
XML_OP(xmlRealloc, xmlReallocFunc, XML_NO_ATTR) \
XML_OP(xmlFree, xmlFreeFunc, XML_NO_ATTR) \
XML_OP(xmlMemStrdup, xmlStrdupFunc, XML_NO_ATTR)
#define XML_OP XML_DECLARE_GLOBAL
XML_GLOBALS_ALLOC
#undef XML_OP
#if defined(LIBXML_THREAD_ENABLED) && !defined(XML_GLOBALS_NO_REDEFINITION)
#define xmlMalloc XML_GLOBAL_MACRO(xmlMalloc)
#define xmlMallocAtomic XML_GLOBAL_MACRO(xmlMallocAtomic)
#define xmlRealloc XML_GLOBAL_MACRO(xmlRealloc)
#define xmlFree XML_GLOBAL_MACRO(xmlFree)
#define xmlMemStrdup XML_GLOBAL_MACRO(xmlMemStrdup)
#endif
#else
#define XML_GLOBALS_ALLOC
/** DOC_ENABLE */
XMLPUBVAR xmlMallocFunc xmlMalloc;
XMLPUBVAR xmlMallocFunc xmlMallocAtomic;
XMLPUBVAR xmlReallocFunc xmlRealloc;
@@ -110,14 +112,12 @@ XMLPUBFUN int
xmlMallocFunc *mallocFunc,
xmlReallocFunc *reallocFunc,
xmlStrdupFunc *strdupFunc);
XML_DEPRECATED
XMLPUBFUN int
xmlGcMemSetup (xmlFreeFunc freeFunc,
xmlMallocFunc mallocFunc,
xmlMallocFunc mallocAtomicFunc,
xmlReallocFunc reallocFunc,
xmlStrdupFunc strdupFunc);
XML_DEPRECATED
XMLPUBFUN int
xmlGcMemGet (xmlFreeFunc *freeFunc,
xmlMallocFunc *mallocFunc,
@@ -147,16 +147,12 @@ XMLPUBFUN int
xmlMemUsed (void);
XMLPUBFUN int
xmlMemBlocks (void);
XML_DEPRECATED
XMLPUBFUN void
xmlMemDisplay (FILE *fp);
XML_DEPRECATED
XMLPUBFUN void
xmlMemDisplayLast(FILE *fp, long nbBytes);
XML_DEPRECATED
XMLPUBFUN void
xmlMemShow (FILE *fp, int nr);
XML_DEPRECATED
XMLPUBFUN void
xmlMemoryDump (void);
XMLPUBFUN void *
@@ -167,19 +163,60 @@ XMLPUBFUN void
xmlMemFree (void *ptr);
XMLPUBFUN char *
xmlMemoryStrdup (const char *str);
XML_DEPRECATED
XMLPUBFUN void *
xmlMallocLoc (size_t size, const char *file, int line) LIBXML_ATTR_ALLOC_SIZE(1);
XML_DEPRECATED
XMLPUBFUN void *
xmlReallocLoc (void *ptr, size_t size, const char *file, int line);
XML_DEPRECATED
XMLPUBFUN void *
xmlMallocAtomicLoc (size_t size, const char *file, int line) LIBXML_ATTR_ALLOC_SIZE(1);
XML_DEPRECATED
XMLPUBFUN char *
xmlMemStrdupLoc (const char *str, const char *file, int line);
/** DOC_DISABLE */
#ifdef DEBUG_MEMORY_LOCATION
/**
* xmlMalloc:
* @size: number of bytes to allocate
*
* Wrapper for the malloc() function used in the XML library.
*
* Returns the pointer to the allocated area or NULL in case of error.
*/
#define xmlMalloc(size) xmlMallocLoc((size), __FILE__, __LINE__)
/**
* xmlMallocAtomic:
* @size: number of bytes to allocate
*
* Wrapper for the malloc() function used in the XML library for allocation
* of block not containing pointers to other areas.
*
* Returns the pointer to the allocated area or NULL in case of error.
*/
#define xmlMallocAtomic(size) xmlMallocAtomicLoc((size), __FILE__, __LINE__)
/**
* xmlRealloc:
* @ptr: pointer to the existing allocated area
* @size: number of bytes to allocate
*
* Wrapper for the realloc() function used in the XML library.
*
* Returns the pointer to the allocated area or NULL in case of error.
*/
#define xmlRealloc(ptr, size) xmlReallocLoc((ptr), (size), __FILE__, __LINE__)
/**
* xmlMemStrdup:
* @str: pointer to the existing string
*
* Wrapper for the strdup() function, xmlStrdup() is usually preferred.
*
* Returns the pointer to the allocated area or NULL in case of error.
*/
#define xmlMemStrdup(str) xmlMemStrdupLoc((str), __FILE__, __LINE__)
#endif /* DEBUG_MEMORY_LOCATION */
/** DOC_ENABLE */
#ifdef __cplusplus
}
#endif /* __cplusplus */