gffrt
This commit is contained in:
+149
-68
@@ -396,6 +396,90 @@ def _fetch_page_content(url: str, max_chars: int = 3000) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
_BROWSER_FETCH_TIMEOUT_SECS = 30
|
||||
_BROWSER_FETCH_MAX_CHARS = 5000
|
||||
|
||||
def _fetch_page_with_browser(url: str, max_chars: int = _BROWSER_FETCH_MAX_CHARS) -> str:
|
||||
"""Fetch page content using headless Chromium via Playwright.
|
||||
|
||||
Falls back to this when _fetch_page_content returns empty and the URL
|
||||
is known to be JS-rendered (SPAs, e-commerce, dynamic sites).
|
||||
|
||||
Args:
|
||||
url: Target webpage URL.
|
||||
max_chars: Maximum characters to return.
|
||||
|
||||
Returns:
|
||||
Extracted text content, or empty string on failure.
|
||||
"""
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
import re
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True)
|
||||
context = browser.new_context(
|
||||
user_agent=(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36"
|
||||
),
|
||||
locale="pt-BR",
|
||||
viewport={"width": 1280, "height": 800},
|
||||
)
|
||||
page = context.new_page()
|
||||
# Increase timeout for slow pages
|
||||
page.set_default_timeout(_BROWSER_FETCH_TIMEOUT_SECS * 1000)
|
||||
try:
|
||||
response = page.goto(url, wait_until="networkidle", timeout=_BROWSER_FETCH_TIMEOUT_SECS * 1000)
|
||||
except Exception:
|
||||
# Page may have loaded partially; continue anyway
|
||||
response = None
|
||||
# Wait for dynamic content to render
|
||||
page.wait_for_timeout(3000)
|
||||
# Scroll through page to trigger lazy-loaded content — go full height multiple times
|
||||
try:
|
||||
page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
|
||||
page.wait_for_timeout(800)
|
||||
except Exception:
|
||||
pass
|
||||
# Also extract text from common product/content selectors before body
|
||||
main_texts = []
|
||||
for selector in [
|
||||
'main', '[role="main"]', '.search-results', '.product-list',
|
||||
'[class*="product"]', '[class*="result"]', '#main-content',
|
||||
'article', '.content', '.listing', '.grid',
|
||||
]:
|
||||
try:
|
||||
el = page.query_selector(selector)
|
||||
if el:
|
||||
t = el.inner_text().strip()
|
||||
if t and len(t) > 20:
|
||||
main_texts.append(t)
|
||||
except Exception:
|
||||
pass
|
||||
if main_texts:
|
||||
text = "\n\n".join(main_texts)
|
||||
else:
|
||||
# Fallback to full body text
|
||||
text = page.inner_text("body")
|
||||
browser.close()
|
||||
if not text:
|
||||
return ""
|
||||
# Clean up whitespace
|
||||
lines = [line.strip() for line in text.splitlines() if line.strip()]
|
||||
cleaned = "\n".join(lines)
|
||||
# Remove excessive repeated sections that come from navigation menus
|
||||
if len(cleaned) > max_chars:
|
||||
cleaned = cleaned[:max_chars] + "..."
|
||||
print(f"[BROWSER] ok url={url} chars={len(cleaned)}")
|
||||
return cleaned
|
||||
except ImportError:
|
||||
# Playwright not installed, return empty to let caller use fallback
|
||||
return ""
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def set_schedule(days: List[int], start_time: str, end_time: str) -> bool:
|
||||
"""Validate and persist the proactive message schedule.
|
||||
@@ -847,33 +931,20 @@ def find_matching_upload(query: str) -> Optional[Dict[str, Any]]:
|
||||
# Keyword phrases that indicate the user wants a live web lookup rather than
|
||||
# a purely conversational answer (see _looks_like_search_request).
|
||||
_WEB_SEARCH_TRIGGERS = [
|
||||
"search",
|
||||
"search for",
|
||||
"look up",
|
||||
"look for",
|
||||
"find",
|
||||
"on the internet",
|
||||
"internet",
|
||||
"web",
|
||||
"google",
|
||||
"what is",
|
||||
"who is",
|
||||
"what's the",
|
||||
"what's a",
|
||||
"how does",
|
||||
"how to",
|
||||
"news",
|
||||
"current",
|
||||
"today",
|
||||
"now",
|
||||
"real time",
|
||||
"forecast",
|
||||
"weather",
|
||||
"where is",
|
||||
"where's",
|
||||
"location",
|
||||
"map",
|
||||
"address",
|
||||
# English action words
|
||||
"search", "search for", "look up", "look for", "find", "google",
|
||||
# Portuguese action words + variants
|
||||
"pesquise", "pesquisa", "pesquisar",
|
||||
"busca", "buscar", "procure", "procurar", "verifique", "verificar",
|
||||
"consulta", "consulte", "consultar",
|
||||
# Content triggers
|
||||
"on the internet", "internet", "web",
|
||||
# Questions
|
||||
"what is", "who is", "what's the", "what's a",
|
||||
"how does", "how to", "where is", "where's",
|
||||
# Time-sensitive keywords
|
||||
"news", "current", "today", "now", "real time",
|
||||
"forecast", "weather", "location", "map", "address",
|
||||
]
|
||||
|
||||
|
||||
@@ -3062,38 +3133,17 @@ def ensure_min_lines(response: str, user_message: str, profile: Dict[str, Any])
|
||||
# ===================== URL VISIT DETECTION =====================
|
||||
|
||||
_URL_VISIT_TRIGGERS = [
|
||||
"entra",
|
||||
"entre",
|
||||
"acesse",
|
||||
"acessar",
|
||||
"visite",
|
||||
"visitar",
|
||||
"abre",
|
||||
"abrir",
|
||||
"analisa",
|
||||
"analisar",
|
||||
"leia",
|
||||
"leitura",
|
||||
"le",
|
||||
"busca",
|
||||
"buscar",
|
||||
"pesquise",
|
||||
"pesquisar",
|
||||
"olha",
|
||||
"olhar",
|
||||
"ver",
|
||||
"veja",
|
||||
"mostra",
|
||||
"mostrar",
|
||||
"check",
|
||||
"visit",
|
||||
"browse",
|
||||
"scrape",
|
||||
"site",
|
||||
"pagina",
|
||||
"página",
|
||||
"url",
|
||||
"page",
|
||||
# Portuguese action words
|
||||
"entra", "entre", "acesse", "acessar", "visite", "visitar",
|
||||
"abre", "abrir", "analisa", "analisar", "leia", "leitura",
|
||||
"verifique", "verificar", "confira", "confirmar", "consulte",
|
||||
"consulta", "busca", "buscar", "pesquise", "pesquisar",
|
||||
"olha", "olhar", "veja", "mostra", "mostar", "mostrar",
|
||||
"mostre", "me mostre", "mostre-me",
|
||||
# Common nouns indicating intent
|
||||
"site", "pagina", "página", "url", "page", "website",
|
||||
# English action words
|
||||
"check", "visit", "browse", "scrape", "fetch", "read",
|
||||
]
|
||||
|
||||
def _looks_like_url_visit_request(user_message: str) -> bool:
|
||||
@@ -3110,7 +3160,7 @@ def _looks_like_url_visit_request(user_message: str) -> bool:
|
||||
urls = _extract_urls(q)
|
||||
if not urls:
|
||||
return False
|
||||
# Must contain at least one action trigger (allow short messages with URLs since the URL itself implies intent)
|
||||
# If message has multiple words (at least 2), check for trigger keywords
|
||||
if len(q.split()) >= 2:
|
||||
return any(t in q for t in _URL_VISIT_TRIGGERS)
|
||||
return False
|
||||
@@ -6926,10 +6976,32 @@ async def process_text_message(update: Update, context: ContextTypes.DEFAULT_TYP
|
||||
if WEB_SEARCH_ENABLED and _looks_like_search_request(user_message):
|
||||
search_query = user_message
|
||||
# Strip the leading intent verb ("search for X" -> "X") before querying.
|
||||
for prefix in ["search", "search for", "look up", "look for", "find", "on the internet", "tell me", "inform me", "what", "who", "how", "where", "when", "why"]:
|
||||
for prefix in [
|
||||
"search for", "look up", "look for", "on the internet",
|
||||
"tell me", "inform me",
|
||||
"pesquise ", "busca ", "buscar ", "procure ", "procurar ",
|
||||
"verifique ", "veja ", "mostra ", "consulte ", "consulta ",
|
||||
"no ", "na ", "nos ", "nas ",
|
||||
]:
|
||||
if search_query.lower().startswith(prefix):
|
||||
search_query = search_query[len(prefix):].strip(" :,-")
|
||||
break
|
||||
# Also strip single-word triggers
|
||||
if not search_query or search_query == user_message:
|
||||
for prefix in [
|
||||
"search", "find", "google",
|
||||
"pesquise", "busca", "buscar", "procure", "procurar",
|
||||
"verifique", "veja", "mostra", "consulte", "consulta",
|
||||
"what is", "who is", "where is", "how to", "how does",
|
||||
]:
|
||||
lower_q = search_query.lower()
|
||||
if lower_q.startswith(prefix + " ") or lower_q == prefix:
|
||||
search_query = search_query[len(prefix):].strip(" :,-")
|
||||
break
|
||||
# Handle possessive/apostrophe forms
|
||||
if lower_q.startswith("what's ") and prefix == "what":
|
||||
search_query = search_query[6:].strip(" :,-")
|
||||
break
|
||||
if search_query:
|
||||
web_results = _web_search(search_query, max_results=WEB_SEARCH_MAX_RESULTS)
|
||||
# Fetch page content for each result to provide richer context.
|
||||
@@ -6942,6 +7014,12 @@ async def process_text_message(update: Update, context: ContextTypes.DEFAULT_TYP
|
||||
page_url = result.get("url", "")
|
||||
if page_url:
|
||||
page_text = _fetch_page_content(page_url, max_chars=per_result_max)
|
||||
# If HTTP scraping failed, try headless browser for JS-rendered sites
|
||||
if not page_text:
|
||||
browser_text = _fetch_page_with_browser(page_url, max_chars=per_result_max)
|
||||
if browser_text:
|
||||
page_text = browser_text
|
||||
print(f"[SCRAPE] browser fallback succeeded url={page_url} chars={len(browser_text)}")
|
||||
if page_text:
|
||||
result["content"] = page_text
|
||||
else:
|
||||
@@ -6969,14 +7047,17 @@ async def process_text_message(update: Update, context: ContextTypes.DEFAULT_TYP
|
||||
raw_urls = _extract_urls(user_message)
|
||||
for raw in raw_urls:
|
||||
normalized = _normalize_url(raw)
|
||||
page_text = _fetch_page_content(normalized, max_chars=4000)
|
||||
domain = ""
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
domain = urlparse(normalized).netloc
|
||||
except Exception:
|
||||
domain = normalized
|
||||
# Try headless browser first for JS-rendered sites, fallback to HTTP scrape
|
||||
page_text = _fetch_page_with_browser(normalized, max_chars=4000)
|
||||
if not page_text:
|
||||
page_text = _fetch_page_content(normalized, max_chars=4000)
|
||||
if page_text:
|
||||
domain = ""
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
domain = urlparse(normalized).netloc
|
||||
except Exception:
|
||||
domain = normalized
|
||||
direct_url_context += f"\n\n[WEBSITE CONTENT FROM {domain}]\n{page_text}\n[END OF WEBSITE CONTENT]"
|
||||
print(f"[URL_VISIT] scraped url={normalized} chars={len(page_text)}")
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user