From 2a50b26f4aef60d4eff555cdf6b35d3b3c0cd579 Mon Sep 17 00:00:00 2001
From: Marco Costa <marcocosta@gmx.com>
Date: Tue, 27 Oct 2026 20:32:25 -0300
Subject: [PATCH] refactor(fetch): simplify browser text extraction and improve
 error logging

Remove complex main content selector logic and add structured error
logging for navigation and text extraction failures.
---
 psagentai_agent.py | 41 ++++++++++++++++-------------------------
 1 file changed, 16 insertions(+), 25 deletions(-)

diff --git a/psagentai_agent.py b/psagentai_agent.py
index f27ee24..2dabc17 100755
--- a/psagentai_agent.py
+++ b/psagentai_agent.py
@@ -428,12 +428,11 @@ def _fetch_page_with_browser(url: str, max_chars: int = _BROWSER_FETCH_MAX_CHARS
                 viewport={"width": 1280, "height": 800},
             )
             page = context.new_page()
-            # Increase timeout for slow pages
             page.set_default_timeout(_BROWSER_FETCH_TIMEOUT_SECS * 1000)
             try:
                 response = page.goto(url, wait_until="networkidle", timeout=_BROWSER_FETCH_TIMEOUT_SECS * 1000)
-            except Exception:
-                # Page may have loaded partially; continue anyway
+            except Exception as nav_err:
+                print(f"[BROWSER] nav_error url={url} err={nav_err}")
                 response = None
             # Wait for SPA frameworks (React/Vue/Angular) to hydrate
             page.wait_for_timeout(5000)
@@ -449,30 +448,11 @@ def _fetch_page_with_browser(url: str, max_chars: int = _BROWSER_FETCH_MAX_CHARS
             # Extra wait for any final async content to settle
             page.wait_for_timeout(2000)
             # Extract visible text from body (most reliable for SPAs)
+            text = ""
             try:
                 text = page.inner_text("body")
-            except Exception:
-                text = ""
-            # Also try main content selectors for richer extraction
-            main_texts = []
-            for selector in [
-                'main', '[role="main"]', '.search-results', '.product-list',
-                '[class*="product"]', '[class*="result"]', '#main-content',
-                'article', '.content', '.listing', '.grid',
-            ]:
-                try:
-                    el = page.query_selector(selector)
-                    if el:
-                        t = el.inner_text().strip()
-                        if t and len(t) > 20:
-                            main_texts.append(t)
-                except Exception:
-                    pass
-            # Use whichever extraction gave more content
-            if main_texts:
-                combined = "\n\n".join(main_texts)
-                if len(combined) > len(text):
-                    text = combined
+            except Exception as text_err:
+                print(f"[BROWSER] inner_text_error url={url} err={text_err}")
             browser.close()
             if not text:
                 return ""
@@ -484,6 +464,17 @@ def _fetch_page_with_browser(url: str, max_chars: int = _BROWSER_FETCH_MAX_CHARS
                 cleaned = cleaned[:max_chars] + "..."
             print(f"[BROWSER] ok url={url} chars={len(cleaned)}")
             return cleaned
+    except Exception as e:
+        print(f"[BROWSER] FAILED url={url} err={type(e).__name__}: {e}")
+        return ""
+            # Clean up whitespace
+            lines = [line.strip() for line in text.splitlines() if line.strip()]
+            cleaned = "\n".join(lines)
+            # Remove excessive repeated sections that come from navigation menus
+            if len(cleaned) > max_chars:
+                cleaned = cleaned[:max_chars] + "..."
+            print(f"[BROWSER] ok url={url} chars={len(cleaned)}")
+            return cleaned
     except ImportError:
         # Playwright not installed, return empty to let caller use fallback
         return ""