perf(agent): cache scraped pages and retry failed fetches
Adds a disk-backed scrape cache under workspace/web_cache keyed by the SHA-256 of the URL, honoring WEB_SEARCH_SCRAPE_CACHE_TTL so repeated lookups within the window skip the network entirely. Fetches now retry twice with exponential backoff before giving up, and URLs are screened against SCRAPING_BLOCKED_EXTENSIONS and SCRAPING_BLOCKED_DOMAINS to avoid downloading binaries or denylisted hosts. New scraping knobs (enable flag, max chars, cache TTL, blocklists) are exposed via .env, alongside a provider switch to stepfun with the step-3.7-flash default model.
This commit is contained in:
+137
-9
@@ -38,6 +38,7 @@ import asyncio
|
||||
import base64
|
||||
import logging
|
||||
import mimetypes
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, List, Any
|
||||
@@ -114,6 +115,21 @@ UPLOAD_CONTEXT_MAX_CHARS = int(os.getenv("UPLOAD_CONTEXT_MAX_CHARS", "8000"))
|
||||
WEB_SEARCH_ENABLED = os.getenv("WEB_SEARCH_ENABLED", "true").lower() in ("true", "1", "yes")
|
||||
WEB_SEARCH_MAX_RESULTS = int(os.getenv("WEB_SEARCH_MAX_RESULTS", "5"))
|
||||
|
||||
# Scraping configuration
|
||||
WEB_SEARCH_SCRAPE_ENABLED = os.getenv("WEB_SEARCH_SCRAPE_ENABLED", "true").lower() in ("true", "1", "yes")
|
||||
WEB_SEARCH_SCRAPE_MAX_CHARS = int(os.getenv("WEB_SEARCH_SCRAPE_MAX_CHARS", "2500"))
|
||||
WEB_SEARCH_SCRAPE_CACHE_TTL = int(os.getenv("WEB_SEARCH_SCRAPE_CACHE_TTL", "3600"))
|
||||
SCRAPING_BLOCKED_DOMAINS = [
|
||||
d.strip().lower()
|
||||
for d in os.getenv("SCRAPING_BLOCKED_DOMAINS", "").split(",")
|
||||
if d.strip()
|
||||
]
|
||||
SCRAPING_BLOCKED_EXTENSIONS = [
|
||||
ext.strip().lower()
|
||||
for ext in os.getenv("SCRAPING_BLOCKED_EXTENSIONS", ".pdf,.jpg,.jpeg,.png,.gif,.svg,.zip,.rar,.exe,.dmg,.bin").split(",")
|
||||
if ext.strip()
|
||||
]
|
||||
|
||||
# --- Vision / image analysis ---
|
||||
VISION_MODEL = os.getenv("VISION_MODEL", "meta-llama/llama-4-maverick-17b-128e-instruct")
|
||||
VISION_PROVIDER = os.getenv("VISION_PROVIDER", "").strip().lower()
|
||||
@@ -279,6 +295,40 @@ def _fetch_page_content(url: str, max_chars: int = 3000) -> str:
|
||||
Extracted text content, or an empty string on any failure.
|
||||
"""
|
||||
try:
|
||||
# Check cache first
|
||||
from urllib.parse import urlparse
|
||||
import hashlib
|
||||
parsed = urlparse(url)
|
||||
netloc = parsed.netloc.lower()
|
||||
# Extension blacklist
|
||||
path_lower = parsed.path.lower()
|
||||
if any(path_lower.endswith(ext) for ext in SCRAPING_BLOCKED_EXTENSIONS):
|
||||
print(f"[SCRAPE] blocked extension url={url}")
|
||||
return ""
|
||||
# Blocked domains
|
||||
if any(netloc == blocked or netloc.endswith("." + blocked) for blocked in SCRAPING_BLOCKED_DOMAINS):
|
||||
print(f"[SCRAPE] blocked domain url={url}")
|
||||
return ""
|
||||
# Build cache key
|
||||
cache_key = hashlib.sha256(url.encode("utf-8")).hexdigest()
|
||||
cache_dir = Path("workspace") / "web_cache"
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
cache_file = cache_dir / f"{cache_key}.json"
|
||||
# Check cache TTL
|
||||
if cache_file.exists():
|
||||
try:
|
||||
with open(cache_file, "r", encoding="utf-8") as f:
|
||||
cache_entry = json.load(f)
|
||||
cached_at = cache_entry.get("cached_at", 0)
|
||||
if time.time() - cached_at < WEB_SEARCH_SCRAPE_CACHE_TTL:
|
||||
content = cache_entry.get("content", "")
|
||||
if content:
|
||||
print(f"[SCRAPE] cache hit url={url}")
|
||||
return content[:max_chars] if len(content) > max_chars else content
|
||||
else:
|
||||
cache_file.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
cache_file.unlink(missing_ok=True)
|
||||
headers = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
@@ -288,10 +338,41 @@ def _fetch_page_content(url: str, max_chars: int = 3000) -> str:
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9,pt-BR;q=0.8,pt;q=0.7",
|
||||
}
|
||||
response = requests.get(url, headers=headers, timeout=10)
|
||||
response.raise_for_status()
|
||||
# Retry + backoff
|
||||
session = requests.Session()
|
||||
retries = 2
|
||||
backoff = 1
|
||||
response = None
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
response = session.get(url, headers=headers, timeout=10)
|
||||
if response.status_code == 429:
|
||||
time.sleep(backoff)
|
||||
backoff *= 2
|
||||
continue
|
||||
response.raise_for_status()
|
||||
break
|
||||
except requests.exceptions.RequestException:
|
||||
if attempt < retries - 1:
|
||||
time.sleep(backoff)
|
||||
backoff *= 2
|
||||
else:
|
||||
print(f"[SCRAPE] fail url={url} status=error")
|
||||
return ""
|
||||
if response is None or response.status_code != 200:
|
||||
print(f"[SCRAPE] fail url={url} status={response.status_code if response else 'none'}")
|
||||
return ""
|
||||
# CAPTCHA / block detection
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
if "text/html" not in content_type.lower() and "application/xhtml" not in content_type.lower():
|
||||
print(f"[SCRAPE] blocked non-html url={url}")
|
||||
return ""
|
||||
lowered = response.text.lower()
|
||||
if "attention required" in lowered or "captcha" in lowered or "cloudflare" in lowered:
|
||||
print(f"[SCRAPE] blocked captcha url={url}")
|
||||
return ""
|
||||
if response.status_code in (403, 429):
|
||||
print(f"[SCRAPE] blocked status={response.status_code} url={url}")
|
||||
return ""
|
||||
from bs4 import BeautifulSoup
|
||||
soup = BeautifulSoup(response.text, "lxml")
|
||||
@@ -303,6 +384,13 @@ def _fetch_page_content(url: str, max_chars: int = 3000) -> str:
|
||||
cleaned = "\n".join(lines)
|
||||
if len(cleaned) > max_chars:
|
||||
cleaned = cleaned[:max_chars] + "..."
|
||||
# Save to cache
|
||||
try:
|
||||
with open(cache_file, "w", encoding="utf-8") as f:
|
||||
json.dump({"url": url, "content": cleaned, "cached_at": time.time()}, f, ensure_ascii=False)
|
||||
except Exception:
|
||||
pass
|
||||
print(f"[SCRAPE] ok url={url} chars={len(cleaned)}")
|
||||
return cleaned
|
||||
except Exception:
|
||||
return ""
|
||||
@@ -1082,6 +1170,7 @@ def ensure_workspace_dirs():
|
||||
"""
|
||||
Path(UPLOADS_DIR).mkdir(parents=True, exist_ok=True)
|
||||
Path(WORKSPACE_PROJECTS_DIR).mkdir(parents=True, exist_ok=True)
|
||||
Path("workspace", "web_cache").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def save_workspace_file(relative_path: str, content: str) -> str:
|
||||
@@ -6000,6 +6089,7 @@ Examples:
|
||||
- "search about the new iPhone"
|
||||
- "what is the weather forecast for São Paulo"
|
||||
- "who is the president of Brazil"
|
||||
- /sources - Show the sources used in the last web search
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━
|
||||
8️⃣ SCHEDULING AND ROUTINE
|
||||
@@ -6110,6 +6200,7 @@ Utilities:
|
||||
/configai - Configure the AI provider and model interactively with button selection
|
||||
/viewvision - Shows the current vision provider and model
|
||||
/configvision - Configure the vision model interactively with button selection
|
||||
/sources - Show the sources used in the last web search
|
||||
/about - Displays system information, version, author and license
|
||||
|
||||
Tip: just send a normal message to chat with me! """
|
||||
@@ -6767,12 +6858,17 @@ async def process_text_message(update: Update, context: ContextTypes.DEFAULT_TYP
|
||||
if search_query:
|
||||
web_results = _web_search(search_query, max_results=WEB_SEARCH_MAX_RESULTS)
|
||||
# Fetch page content for each result to provide richer context.
|
||||
for result in web_results:
|
||||
page_url = result.get("url", "")
|
||||
if page_url:
|
||||
page_text = _fetch_page_content(page_url, max_chars=2500)
|
||||
if page_text:
|
||||
result["content"] = page_text
|
||||
if WEB_SEARCH_SCRAPE_ENABLED:
|
||||
# Dynamic content limit: reserve ~60% of remaining context for content,
|
||||
# capped by WEB_SEARCH_SCRAPE_MAX_CHARS.
|
||||
remaining_chars = max(1000, 12000 - len(modified))
|
||||
per_result_max = min(WEB_SEARCH_SCRAPE_MAX_CHARS, max(500, int(remaining_chars * 0.6 / max(len(web_results), 1))))
|
||||
for result in web_results:
|
||||
page_url = result.get("url", "")
|
||||
if page_url:
|
||||
page_text = _fetch_page_content(page_url, max_chars=per_result_max)
|
||||
if page_text:
|
||||
result["content"] = page_text
|
||||
else:
|
||||
web_results = []
|
||||
if web_results:
|
||||
@@ -6782,8 +6878,13 @@ async def process_text_message(update: Update, context: ContextTypes.DEFAULT_TYP
|
||||
page_content = result.get("content")
|
||||
if page_content:
|
||||
web_context += f"Content:\n{page_content}\n"
|
||||
web_context += "\n[END OF WEB SEARCH RESULTS - Base your answer ONLY on the information above. If the information is not sufficient, say so. NEVER invent facts, URLs, or references.]"
|
||||
web_context += "\n[END OF WEB SEARCH RESULTS - IMPORTANT: Base your answer ONLY on the information above. Use the Content sections when available. If the information is not sufficient or you could not read the pages, say so clearly. NEVER invent facts, URLs, or references.]"
|
||||
modified = modified + "\n\n" + web_context
|
||||
# Store sources for /sources command
|
||||
context.user_data["last_web_sources"] = [
|
||||
{"title": r.get("title", ""), "url": r.get("url", ""), "content": r.get("content", "")}
|
||||
for r in web_results
|
||||
]
|
||||
else:
|
||||
web_results = []
|
||||
|
||||
@@ -6800,6 +6901,32 @@ async def process_text_message(update: Update, context: ContextTypes.DEFAULT_TYP
|
||||
else:
|
||||
await update.message.reply_text("Sorry, I'm having a little problem right now. Can you try again? 😔")
|
||||
|
||||
async def sources_command(update: Update, context: ContextTypes.DEFAULT_TYPE):
|
||||
"""Handle /sources: show sources from the last web search.
|
||||
|
||||
Args:
|
||||
update: Telegram update.
|
||||
context: PTB callback context.
|
||||
|
||||
Returns:
|
||||
None.
|
||||
"""
|
||||
await auto_detect_chat_id(update)
|
||||
sources = context.user_data.get("last_web_sources", [])
|
||||
if not sources:
|
||||
await update.message.reply_text("No recent web search sources available. Try asking a search question first!")
|
||||
return
|
||||
lines = ["📚 Sources from last search:"]
|
||||
for i, src in enumerate(sources, 1):
|
||||
title = src.get("title", "Untitled")
|
||||
url = src.get("url", "")
|
||||
content = src.get("content", "")
|
||||
lines.append(f"{i}. {title}\n{url}")
|
||||
if content:
|
||||
preview = content[:300] + ("..." if len(content) > 300 else "")
|
||||
lines.append(f"Preview: {preview}")
|
||||
await update.message.reply_text("\n\n".join(lines))
|
||||
|
||||
async def debug_command(update: Update, context: ContextTypes.DEFAULT_TYPE):
|
||||
"""Handle /debug: print connectivity and configuration diagnostics.
|
||||
|
||||
@@ -7342,6 +7469,7 @@ def main():
|
||||
app_instance.add_handler(CommandHandler("debug", debug_command))
|
||||
app_instance.add_handler(CommandHandler("reload", reload_command))
|
||||
app_instance.add_handler(CommandHandler("about", about_command))
|
||||
app_instance.add_handler(CommandHandler("sources", sources_command))
|
||||
app_instance.add_handler(CommandHandler("viewai", viewai_command))
|
||||
app_instance.add_handler(CommandHandler("configai", configai_command))
|
||||
app_instance.add_handler(CommandHandler("updatesystem", updatesystem_command))
|
||||
|
||||
Reference in New Issue
Block a user