feat(agent): scrape linked pages to ground answers in real content

Adds `_fetch_page_content()`, which downloads a URL with a browser-like User-Agent and uses BeautifulSoup with the lxml parser to strip scripts, navigation, and other non-content markup, returning up to 3000 characters of readable text. The response validator now also compares generated replies against this scraped text, flagging named entities and claims that do not appear in the retrieved page content. Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0 for parser compatibility.
2026-10-26 16:48:29 -03:00
parent 9eed02645b
commit 723ca1506b
231 changed files with 20508 additions and 5330 deletions
@@ -71,6 +71,144 @@ XMLPUBFUN int
 	xmlDebugCheckDocument	(FILE * output,
 				 xmlDocPtr doc);

+/****************************************************************
+ *								*
+ *			XML shell helpers			*
+ *								*
+ ****************************************************************/
+
+XMLPUBFUN void
+	xmlLsOneNode		(FILE *output, xmlNodePtr node);
+XMLPUBFUN int
+	xmlLsCountNode		(xmlNodePtr node);
+
+XMLPUBFUN const char *
+	xmlBoolToText		(int boolval);
+
+/****************************************************************
+ *								*
+ *	 The XML shell related structures and functions		*
+ *								*
+ ****************************************************************/
+
+#ifdef LIBXML_XPATH_ENABLED
+/**
+ * xmlShellReadlineFunc:
+ * @prompt:  a string prompt
+ *
+ * This is a generic signature for the XML shell input function.
+ *
+ * Returns a string which will be freed by the Shell.
+ */
+typedef char * (* xmlShellReadlineFunc)(char *prompt);
+
+/**
+ * xmlShellCtxt:
+ *
+ * A debugging shell context.
+ * TODO: add the defined function tables.
+ */
+typedef struct _xmlShellCtxt xmlShellCtxt;
+typedef xmlShellCtxt *xmlShellCtxtPtr;
+struct _xmlShellCtxt {
+    char *filename;
+    xmlDocPtr doc;
+    xmlNodePtr node;
+    xmlXPathContextPtr pctxt;
+    int loaded;
+    FILE *output;
+    xmlShellReadlineFunc input;
+};
+
+/**
+ * xmlShellCmd:
+ * @ctxt:  a shell context
+ * @arg:  a string argument
+ * @node:  a first node
+ * @node2:  a second node
+ *
+ * This is a generic signature for the XML shell functions.
+ *
+ * Returns an int, negative returns indicating errors.
+ */
+typedef int (* xmlShellCmd) (xmlShellCtxtPtr ctxt,
+                             char *arg,
+			     xmlNodePtr node,
+			     xmlNodePtr node2);
+
+XMLPUBFUN void
+	xmlShellPrintXPathError	(int errorType,
+				 const char *arg);
+XMLPUBFUN void
+	xmlShellPrintXPathResult(xmlXPathObjectPtr list);
+XMLPUBFUN int
+	xmlShellList		(xmlShellCtxtPtr ctxt,
+				 char *arg,
+				 xmlNodePtr node,
+				 xmlNodePtr node2);
+XMLPUBFUN int
+	xmlShellBase		(xmlShellCtxtPtr ctxt,
+				 char *arg,
+				 xmlNodePtr node,
+				 xmlNodePtr node2);
+XMLPUBFUN int
+	xmlShellDir		(xmlShellCtxtPtr ctxt,
+				 char *arg,
+				 xmlNodePtr node,
+				 xmlNodePtr node2);
+XMLPUBFUN int
+	xmlShellLoad		(xmlShellCtxtPtr ctxt,
+				 char *filename,
+				 xmlNodePtr node,
+				 xmlNodePtr node2);
+#ifdef LIBXML_OUTPUT_ENABLED
+XMLPUBFUN void
+	xmlShellPrintNode	(xmlNodePtr node);
+XMLPUBFUN int
+	xmlShellCat		(xmlShellCtxtPtr ctxt,
+				 char *arg,
+				 xmlNodePtr node,
+				 xmlNodePtr node2);
+XMLPUBFUN int
+	xmlShellWrite		(xmlShellCtxtPtr ctxt,
+				 char *filename,
+				 xmlNodePtr node,
+				 xmlNodePtr node2);
+XMLPUBFUN int
+	xmlShellSave		(xmlShellCtxtPtr ctxt,
+				 char *filename,
+				 xmlNodePtr node,
+				 xmlNodePtr node2);
+#endif /* LIBXML_OUTPUT_ENABLED */
+#ifdef LIBXML_VALID_ENABLED
+XMLPUBFUN int
+	xmlShellValidate	(xmlShellCtxtPtr ctxt,
+				 char *dtd,
+				 xmlNodePtr node,
+				 xmlNodePtr node2);
+#endif /* LIBXML_VALID_ENABLED */
+XMLPUBFUN int
+	xmlShellDu		(xmlShellCtxtPtr ctxt,
+				 char *arg,
+				 xmlNodePtr tree,
+				 xmlNodePtr node2);
+XMLPUBFUN int
+	xmlShellPwd		(xmlShellCtxtPtr ctxt,
+				 char *buffer,
+				 xmlNodePtr node,
+				 xmlNodePtr node2);
+
+/*
+ * The Shell interface.
+ */
+XMLPUBFUN void
+	xmlShell		(xmlDocPtr doc,
+				 char *filename,
+				 xmlShellReadlineFunc input,
+				 FILE *output);
+
+#endif /* LIBXML_XPATH_ENABLED */
+
 #ifdef __cplusplus
 }
 #endif