feat(agent): scrape linked pages to ground answers in real content

Adds `_fetch_page_content()`, which downloads a URL with a
browser-like User-Agent and uses BeautifulSoup with the lxml
parser to strip scripts, navigation, and other non-content
markup, returning up to 3000 characters of readable text.

The response validator now also compares generated replies
against this scraped text, flagging named entities and claims
that do not appear in the retrieved page content.

Dependencies: adds beautifulsoup4==4.12.3 and pins lxml==5.3.0
for parser compatibility.

venv/lib/python3.11/site-packages/lxml/html/defs.py

@@ -4,13 +4,13 @@
 
 """
 Data taken from https://www.w3.org/TR/html401/index/elements.html
-and https://html.spec.whatwg.org/multipage/syntax.html#elements-2
+and https://www.w3.org/community/webed/wiki/HTML/New_HTML5_Elements
 for html5_tags.
 """
 
 empty_tags = frozenset([
-    'area', 'base', 'basefont', 'br', 'col', 'embed', 'frame', 'hr',
-    'img', 'input', 'isindex', 'link', 'meta', 'param', 'source', 'track', 'wbr'])
+    'area', 'base', 'basefont', 'br', 'col', 'frame', 'hr',
+    'img', 'input', 'isindex', 'link', 'meta', 'param', 'source', 'track'])
 
 deprecated_tags = frozenset([
     'applet', 'basefont', 'center', 'dir', 'font', 'isindex',
@@ -24,9 +24,7 @@ link_attrs = frozenset([
     # Not standard:
     'dynsrc', 'lowsrc',
     # HTML5 formaction
-    'formaction',
-    # XLink as used by HTML5 (including embedded SVG/MathML)
-    'xlink:href',
+    'formaction'
     ])
 
 # Not in the HTML 4 spec:
@@ -49,23 +47,7 @@ safe_attrs = frozenset([
     'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'prompt', 'readonly',
     'rel', 'rev', 'rows', 'rowspan', 'rules', 'scope', 'selected', 'shape',
     'size', 'span', 'src', 'start', 'summary', 'tabindex', 'target', 'title',
-    'type', 'usemap', 'valign', 'value', 'vspace', 'width',
-    # ARIA attributes from https://www.w3.org/TR/wai-aria-1.3/
-    'aria-activedescendant', 'aria-atomic', 'aria-autocomplete',
-    'aria-braillelabel', 'aria-brailleroledescription', 'aria-busy',
-    'aria-checked', 'aria-colcount', 'aria-colindex', 'aria-colindextext',
-    'aria-colspan', 'aria-controls', 'aria-current', 'aria-describedby',
-    'aria-description', 'aria-details', 'aria-disabled', 'aria-dropeffect',
-    'aria-errormessage', 'aria-expanded', 'aria-flowto', 'aria-grabbed',
-    'aria-haspopup', 'aria-hidden', 'aria-invalid', 'aria-keyshortcuts',
-    'aria-label', 'aria-labelledby', 'aria-level', 'aria-live', 'aria-modal',
-    'aria-multiline', 'aria-multiselectable', 'aria-orientation', 'aria-owns',
-    'aria-placeholder', 'aria-posinset', 'aria-pressed', 'aria-readonly',
-    'aria-relevant', 'aria-required', 'aria-roledescription', 'aria-rowcount',
-    'aria-rowindex', 'aria-rowindextext', 'aria-rowspan', 'aria-selected',
-    'aria-setsize', 'aria-sort', 'aria-valuemax', 'aria-valuemin',
-    'aria-valuenow', 'aria-valuetext', 'role',
-    ])
+    'type', 'usemap', 'valign', 'value', 'vspace', 'width'])
 
 # From http://htmlhelp.com/reference/html40/olist.html
 top_level_tags = frozenset([
@@ -135,7 +117,7 @@ font_style_tags = frozenset([
 frame_tags = frozenset([
     'frameset', 'frame', 'noframes',
     ])
-
+    
 html5_tags = frozenset([
     'article', 'aside', 'audio', 'canvas', 'command', 'datalist',
     'details', 'embed', 'figcaption', 'figure', 'footer', 'header',