feat(agent): scrape linked pages to ground answers in real content

Adds `_fetch_page_content()`, which downloads a URL with a browser-like User-Agent and uses BeautifulSoup with the lxml parser to strip scripts, navigation, and other non-content markup, returning up to 3000 characters of readable text. The response validator now also compares generated replies against this scraped text, flagging named entities and claims that do not appear in the retrieved page content. Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0 for parser compatibility.
2026-10-26 16:48:29 -03:00
parent 9eed02645b
commit 723ca1506b
231 changed files with 20508 additions and 5330 deletions
@@ -70,7 +70,7 @@ _forms_xpath = etree.XPath("descendant-or-self::form|descendant-or-self::x:form"
 #_class_xpath = etree.XPath(r"descendant-or-self::*[regexp:match(@class, concat('\b', $class_name, '\b'))]", {'regexp': 'http://exslt.org/regular-expressions'})
 _class_xpath = etree.XPath("descendant-or-self::*[@class and contains(concat(' ', normalize-space(@class), ' '), concat(' ', $class_name, ' '))]")
 _id_xpath = etree.XPath("descendant-or-self::*[@id=$id]")
-_collect_string_content = etree.XPath("string()", smart_strings=False)
+_collect_string_content = etree.XPath("string()")
 _iter_css_urls = re.compile(r'url\(('+'["][^"]*["]|'+"['][^']*[']|"+r'[^)]*)\)', re.I).finditer
 _iter_css_imports = re.compile(r'@import "(.*?)"').finditer
 _label_xpath = etree.XPath("//label[@for=$id]|//x:label[@for=$id]",
@@ -263,9 +263,7 @@ class HtmlMixin:
        Return the <body> element.  Can be called from a child element
        to get the document's head.
        """
-        for element in self.getroottree().iter("body", f"{{{XHTML_NAMESPACE}}}body"):
-            return element
-        return None
+        return self.xpath('//body|//x:body', namespaces={'x':XHTML_NAMESPACE})[0]

    @property
    def head(self):
@@ -273,9 +271,7 @@ class HtmlMixin:
        Returns the <head> element.  Can be called from a child
        element to get the document's head.
        """
-        for element in self.getroottree().iter("head", f"{{{XHTML_NAMESPACE}}}head"):
-            return element
-        return None
+        return self.xpath('//head|//x:head', namespaces={'x':XHTML_NAMESPACE})[0]

    @property
    def label(self):
@@ -41,44 +41,31 @@ ACRONYM = E.acronym  #:
 ADDRESS = E.address  #: information on author
 APPLET = E.applet  #: Java applet (DEPRECATED)
 AREA = E.area  #: client-side image map area
-ARTICLE = E.article  #: self-contained article
-ASIDE = E.aside  #: indirectly-related content
-AUDIO = E.audio  #: embedded audio file
 B = E.b  #: bold text style
 BASE = E.base  #: document base URI
 BASEFONT = E.basefont  #: base font size (DEPRECATED)
-BDI = E.bdi  #: isolate bidirectional text
 BDO = E.bdo  #: I18N BiDi over-ride
 BIG = E.big  #: large text style
 BLOCKQUOTE = E.blockquote  #: long quotation
 BODY = E.body  #: document body
 BR = E.br  #: forced line break
 BUTTON = E.button  #: push button
-CANVAS = E.canvas  #: scriptable graphics container
 CAPTION = E.caption  #: table caption
 CENTER = E.center  #: shorthand for DIV align=center (DEPRECATED)
 CITE = E.cite  #: citation
 CODE = E.code  #: computer code fragment
 COL = E.col  #: table column
 COLGROUP = E.colgroup  #: table column group
-DATA = E.data  #: machine-readable translation
-DATALIST = E.datalist  #: list of options for an input
 DD = E.dd  #: definition description
 DEL = getattr(E, 'del')  #: deleted text
-DETAILS = E.details  #: expandable section
 DFN = E.dfn  #: instance definition
-DIALOG = E.dialog  #: dialog box
 DIR = E.dir  #: directory list (DEPRECATED)
 DIV = E.div  #: generic language/style container
 DL = E.dl  #: definition list
 DT = E.dt  #: definition term
 EM = E.em  #: emphasis
-EMBED = E.embed  #: embedded external content
 FIELDSET = E.fieldset  #: form control group
-FIGCAPTION = E.figcaption  #: figure caption
-FIGURE = E.figure  #: self-contained, possibly-captioned content
 FONT = E.font  #: local change to font (DEPRECATED)
-FOOTER = E.footer  #: footer for nearest ancestor
 FORM = E.form  #: interactive form
 FRAME = E.frame  #: subwindow
 FRAMESET = E.frameset  #: window subdivision
@@ -89,8 +76,6 @@ H4 = E.h4  #: heading
 H5 = E.h5  #: heading
 H6 = E.h6  #: heading
 HEAD = E.head  #: document head
-HEADER = E.header  #: heading content
-HGROUP = E.hgroup  #: heading group
 HR = E.hr  #: horizontal rule
 HTML = E.html  #: document root element
 I = E.i  #: italic text style
@@ -104,68 +89,43 @@ LABEL = E.label  #: form field label text
 LEGEND = E.legend  #: fieldset legend
 LI = E.li  #: list item
 LINK = E.link  #: a media-independent link
-MAIN = E.main  #: main content
 MAP = E.map  #: client-side image map
-MARK = E.mark  #: marked/highlighted text
-MARQUEE = E.marquee  #: scrolling text
 MENU = E.menu  #: menu list (DEPRECATED)
 META = E.meta  #: generic metainformation
-METER = E.meter  #: numerical value display
-NAV = E.nav  #: navigation section
-NOBR = E.nobr  #: prevent wrapping
 NOFRAMES = E.noframes  #: alternate content container for non frame-based rendering
 NOSCRIPT = E.noscript  #: alternate content container for non script-based rendering
 OBJECT = E.object  #: generic embedded object
 OL = E.ol  #: ordered list
 OPTGROUP = E.optgroup  #: option group
 OPTION = E.option  #: selectable choice
-OUTPUT = E.output  #: result of a calculation
 P = E.p  #: paragraph
 PARAM = E.param  #: named property value
-PICTURE = E.picture  #: picture with multiple sources
-PORTAL = E.portal  #: embedded preview
 PRE = E.pre  #: preformatted text
-PROGRESS = E.progress  #: progress bar
 Q = E.q  #: short inline quotation
-RB = E.rb  #: ruby base text
-RP = E.rp  #: ruby parentheses
-RT = E.rt  #: ruby text component
-RTC = E.rtc  #: ruby semantic annotation
-RUBY = E.ruby  #: ruby annotations
 S = E.s  #: strike-through text style (DEPRECATED)
 SAMP = E.samp  #: sample program output, scripts, etc.
 SCRIPT = E.script  #: script statements
-SEARCH = E.search  #: set of form controls for a search
-SECTION = E.section  #: generic standalone section
 SELECT = E.select  #: option selector
-SLOT = E.slot  #: placeholder for JS use
 SMALL = E.small  #: small text style
-SOURCE = E.source  #: source for picture/audio/video element
 SPAN = E.span  #: generic language/style container
 STRIKE = E.strike  #: strike-through text (DEPRECATED)
 STRONG = E.strong  #: strong emphasis
 STYLE = E.style  #: style info
 SUB = E.sub  #: subscript
-SUMMARY = E.summary  #: summary for <details>
 SUP = E.sup  #: superscript
 TABLE = E.table  #: 
 TBODY = E.tbody  #: table body
 TD = E.td  #: table data cell
-TEMPLATE = E.template  #: fragment for JS use
 TEXTAREA = E.textarea  #: multi-line text field
 TFOOT = E.tfoot  #: table footer
 TH = E.th  #: table header cell
 THEAD = E.thead  #: table header
-TIME = E.time  #: date/time
 TITLE = E.title  #: document title
 TR = E.tr  #: table row
-TRACK = E.track  #: audio/video track
 TT = E.tt  #: teletype or monospaced text style
 U = E.u  #: underlined text style (DEPRECATED)
 UL = E.ul  #: unordered list
 VAR = E.var  #: instance of a variable or program argument
-VIDEO = E.video  #: embedded video file
-WBR = E.wbr  #: word break

 # attributes (only reserved words are included here)
 ATTR = dict
@@ -17,5 +17,5 @@ try:
 except ImportError:
    raise ImportError(
        "lxml.html.clean module is now a separate project lxml_html_clean.\n"
-        "Install lxml[html-clean] or lxml_html_clean directly."
+        "Install lxml[html_clean] or lxml_html_clean directly."
    ) from None
@@ -4,13 +4,13 @@

 """
 Data taken from https://www.w3.org/TR/html401/index/elements.html
-and https://html.spec.whatwg.org/multipage/syntax.html#elements-2
+and https://www.w3.org/community/webed/wiki/HTML/New_HTML5_Elements
 for html5_tags.
 """

 empty_tags = frozenset([
-    'area', 'base', 'basefont', 'br', 'col', 'embed', 'frame', 'hr',
-    'img', 'input', 'isindex', 'link', 'meta', 'param', 'source', 'track', 'wbr'])
+    'area', 'base', 'basefont', 'br', 'col', 'frame', 'hr',
+    'img', 'input', 'isindex', 'link', 'meta', 'param', 'source', 'track'])

 deprecated_tags = frozenset([
    'applet', 'basefont', 'center', 'dir', 'font', 'isindex',
@@ -24,9 +24,7 @@ link_attrs = frozenset([
    # Not standard:
    'dynsrc', 'lowsrc',
    # HTML5 formaction
-    'formaction',
-    # XLink as used by HTML5 (including embedded SVG/MathML)
-    'xlink:href',
+    'formaction'
    ])

 # Not in the HTML 4 spec:
@@ -49,23 +47,7 @@ safe_attrs = frozenset([
    'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'prompt', 'readonly',
    'rel', 'rev', 'rows', 'rowspan', 'rules', 'scope', 'selected', 'shape',
    'size', 'span', 'src', 'start', 'summary', 'tabindex', 'target', 'title',
-    'type', 'usemap', 'valign', 'value', 'vspace', 'width',
-    # ARIA attributes from https://www.w3.org/TR/wai-aria-1.3/
-    'aria-activedescendant', 'aria-atomic', 'aria-autocomplete',
-    'aria-braillelabel', 'aria-brailleroledescription', 'aria-busy',
-    'aria-checked', 'aria-colcount', 'aria-colindex', 'aria-colindextext',
-    'aria-colspan', 'aria-controls', 'aria-current', 'aria-describedby',
-    'aria-description', 'aria-details', 'aria-disabled', 'aria-dropeffect',
-    'aria-errormessage', 'aria-expanded', 'aria-flowto', 'aria-grabbed',
-    'aria-haspopup', 'aria-hidden', 'aria-invalid', 'aria-keyshortcuts',
-    'aria-label', 'aria-labelledby', 'aria-level', 'aria-live', 'aria-modal',
-    'aria-multiline', 'aria-multiselectable', 'aria-orientation', 'aria-owns',
-    'aria-placeholder', 'aria-posinset', 'aria-pressed', 'aria-readonly',
-    'aria-relevant', 'aria-required', 'aria-roledescription', 'aria-rowcount',
-    'aria-rowindex', 'aria-rowindextext', 'aria-rowspan', 'aria-selected',
-    'aria-setsize', 'aria-sort', 'aria-valuemax', 'aria-valuemin',
-    'aria-valuenow', 'aria-valuetext', 'role',
-    ])
+    'type', 'usemap', 'valign', 'value', 'vspace', 'width'])

 # From http://htmlhelp.com/reference/html40/olist.html
 top_level_tags = frozenset([
@@ -135,7 +117,7 @@ font_style_tags = frozenset([
 frame_tags = frozenset([
    'frameset', 'frame', 'noframes',
    ])
-
+    
 html5_tags = frozenset([
    'article', 'aside', 'audio', 'canvas', 'command', 'datalist',
    'details', 'embed', 'figcaption', 'figure', 'footer', 'header',