marcocosta10000/PsAgentAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(fetch): improve SPA rendering and URL normalization in browser scraper

Browse Source 
Increase hydration wait to 5s and scroll the page three times to
trigger lazy-loaded content before extracting text. Prefer body text
when it yields more content than main-content selectors.

Normalize URLs to https and strip the www. prefix for consistent
deduplication, and allow bare domains with no trailing path to match
the extraction pattern.

Also update agent/user profiles with site reputation and price
comparison guidance, and append conversation logs for 2026-10-27.
This commit is contained in:
marcocosta10000
2026-10-27 20:14:50 -03:00
parent aa70c4f6f6
commit 2323a2d1ab
5 changed files with 259 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
psagentai_agent.py
+98 -41
View File
@@ -435,15 +435,25 @@ def _fetch_page_with_browser(url: str, max_chars: int = _BROWSER_FETCH_MAX_CHARS
except Exception:
# Page may have loaded partially; continue anyway
response = None
# Wait for dynamic content to render
page.wait_for_timeout(3000)
# Scroll through page to trigger lazy-loaded content — go full height multiple times
# Wait for SPA frameworks (React/Vue/Angular) to hydrate
page.wait_for_timeout(5000)
# Scroll through page multiple times to trigger lazy-loaded content
try:
page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
page.wait_for_timeout(800)
for _ in range(3):
page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
page.wait_for_timeout(1500)
page.evaluate("window.scrollTo(0, 0)")
page.wait_for_timeout(500)
except Exception:
pass
# Also extract text from common product/content selectors before body
# Extra wait for any final async content to settle
page.wait_for_timeout(2000)
# Extract visible text from body (most reliable for SPAs)
try:
text = page.inner_text("body")
except Exception:
text = ""
# Also try main content selectors for richer extraction
main_texts = []
for selector in [
'main', '[role="main"]', '.search-results', '.product-list',
@@ -458,11 +468,11 @@ def _fetch_page_with_browser(url: str, max_chars: int = _BROWSER_FETCH_MAX_CHARS
main_texts.append(t)
except Exception:
pass
# Use whichever extraction gave more content
if main_texts:
text = "\n\n".join(main_texts)
else:
# Fallback to full body text
text = page.inner_text("body")
combined = "\n\n".join(main_texts)
if len(combined) > len(text):
text = combined
browser.close()
if not text:
return ""
@@ -3168,9 +3178,18 @@ def _looks_like_url_visit_request(user_message: str) -> bool:
def _normalize_url(raw_url: str) -> str:
"""Ensure URL has a protocol prefix so requests can handle it."""
import re as _re
stripped = raw_url.strip().rstrip(".,;:")
if not stripped.startswith(("http://", "https://")):
stripped = "https://" + stripped
# Remove www. prefix for consistent deduplication
domain = stripped.lower()
if domain.startswith("www."):
domain = domain[4:]
stripped = "https://" + domain
else:
# Normalize to https:// and remove www. for consistent dedup
stripped = _re.sub(r'(?:https?://)www\.', 'https://', stripped, count=1)
stripped = _re.sub(r'^https?://', 'https://', stripped, count=1)
return stripped
@@ -3189,19 +3208,22 @@ def _extract_urls(text: str) -> List[str]:
patterns = [
r'https?://[^\s)>\]"\']+',
r'www\.[^\s)>\]"\']+',
r'[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.(?:com|br|net|org|io|dev|co|info|biz|gov|edu|cloud|ai)[^\s)>\]"\']+',
r'[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.(?:com|br|net|org|io|dev|co|info|biz|gov|edu|cloud|ai)(?:[^\s)>\]"\']*)?',
]
urls = []
for pattern in patterns:
urls.extend(re.findall(pattern, text))
# Deduplicate while preserving order (normalize for comparison)
# Clean trailing punctuation and deduplicate
seen = set()
unique = []
for u in urls:
normalized = _normalize_url(u)
cleaned = u.rstrip(".,;:!?)]")
if not cleaned or len(cleaned) < 3:
continue
normalized = _normalize_url(cleaned)
if normalized not in seen:
seen.add(normalized)
unique.append(u)
unique.append(cleaned)
return unique
@@ -6972,7 +6994,51 @@ async def process_text_message(update: Update, context: ContextTypes.DEFAULT_TYP
modified = user_message + file_context
else:
modified = user_message
# Direct URL visit/scrape FIRST — the visited site is the primary source
# the user asked about. Appending before web search ensures search-context
# instructions ("base answer on information above") also cover it.
direct_url_context = ""
_dbg = []
_dbg.append(f"msg={user_message[:100]!r}")
_dbg.append(f"visit_trigger={_looks_like_url_visit_request(user_message)}")
_dbg.append(f"search_trigger={_looks_like_search_request(user_message)}")
if _looks_like_url_visit_request(user_message):
raw_urls = _extract_urls(user_message)
_dbg.append(f"urls={raw_urls}")
for raw in raw_urls:
normalized = _normalize_url(raw)
domain = ""
try:
from urllib.parse import urlparse
domain = urlparse(normalized).netloc
except Exception:
domain = normalized
# Try headless browser first for JS-rendered sites, fallback to HTTP scrape
page_text = _fetch_page_with_browser(normalized, max_chars=4000)
if not page_text:
page_text = _fetch_page_content(normalized, max_chars=4000)
if page_text:
direct_url_context += f"\n\n[WEBSITE CONTENT FROM {domain}]\n{page_text}\n[END OF WEBSITE CONTENT]"
_dbg.append(f"scraped {normalized}: {len(page_text)} chars")
print(f"[URL_VISIT] scraped url={normalized} chars={len(page_text)}")
else:
_dbg.append(f"FAILED {normalized}")
print(f"[URL_VISIT] failed to fetch url={normalized}")
if direct_url_context:
modified = modified + direct_url_context
# Counter context anchoring: previous assistant messages may say
# "cannot access sites". Force the AI to treat this content as
# authoritative regardless of earlier refusals in history.
modified = (
"[IMPORTANT OVERRIDE] You HAVE successfully accessed and scraped the website below. "
"The [WEBSITE CONTENT] section contains the REAL, CURRENT page content. "
"Ignore any earlier messages in this conversation where you said you cannot access websites — "
"you CAN access them now, and the content is provided below. "
"Base your answer PRIMARILY on this website content.\n"
) + modified
_dbg.append(f"direct_ctx_len={len(direct_url_context)}")
if WEB_SEARCH_ENABLED and _looks_like_search_request(user_message):
search_query = user_message
# Strip the leading intent verb ("search for X" -> "X") before querying.
@@ -7031,7 +7097,10 @@ async def process_text_message(update: Update, context: ContextTypes.DEFAULT_TYP
page_content = result.get("content")
if page_content:
web_context += f"Content:\n{page_content}\n"
web_context += "\n[END OF WEB SEARCH RESULTS - IMPORTANT: Base your answer ONLY on the information above. Use the Content sections when available. If the information is not sufficient or you could not read the pages, say so clearly. NEVER invent facts, URLs, or references.]"
if direct_url_context:
web_context += "\n[END OF WEB SEARCH RESULTS - IMPORTANT: The [WEBSITE CONTENT] section above is the PRIMARY source for answering questions about that specific website. Use search results only as supplementary context. NEVER invent facts, URLs, or references.]"
else:
web_context += "\n[END OF WEB SEARCH RESULTS - IMPORTANT: Base your answer ONLY on the information above. Use the Content sections when available. If the information is not sufficient or you could not read the pages, say so clearly. NEVER invent facts, URLs, or references.]"
modified = modified + "\n\n" + web_context
# Store sources for /sources command
context.user_data["last_web_sources"] = [
@@ -7041,30 +7110,18 @@ async def process_text_message(update: Update, context: ContextTypes.DEFAULT_TYP
else:
web_results = []
# Direct URL visit/scrape when user explicitly asks to enter or analyze a site
direct_url_context = ""
if _looks_like_url_visit_request(user_message):
raw_urls = _extract_urls(user_message)
for raw in raw_urls:
normalized = _normalize_url(raw)
domain = ""
try:
from urllib.parse import urlparse
domain = urlparse(normalized).netloc
except Exception:
domain = normalized
# Try headless browser first for JS-rendered sites, fallback to HTTP scrape
page_text = _fetch_page_with_browser(normalized, max_chars=4000)
if not page_text:
page_text = _fetch_page_content(normalized, max_chars=4000)
if page_text:
direct_url_context += f"\n\n[WEBSITE CONTENT FROM {domain}]\n{page_text}\n[END OF WEBSITE CONTENT]"
print(f"[URL_VISIT] scraped url={normalized} chars={len(page_text)}")
else:
print(f"[URL_VISIT] failed to fetch url={normalized}")
if direct_url_context:
modified = modified + direct_url_context
# Debug log: write to file so we can inspect what the AI actually received
try:
_dbg.append(f"modified_len={len(modified)}")
_dbg.append(f"has_website_content={'[WEBSITE CONTENT' in modified}")
_dbg.append(f"has_web_search={'[WEB SEARCH RESULTS' in modified}")
with open("workspace/debug_prompt.log", "a", encoding="utf-8") as _f:
_f.write("=" * 60 + "\n")
_f.write("\n".join(_dbg) + "\n")
_f.write(f"PROMPT TAIL (last 500): ...{modified[-500:]}\n")
except Exception:
pass
response = await get_psagentai_response(modified)
if response:
# Post-validation: if search results were provided, ensure the