browsing-web

name	browsing-web
description	Web scraping with agents, browser automation, content extraction, and Use when this capability is needed.
metadata	{"author":"gitwalter"}

Web Browsing

Web scraping with agents, browser automation, content extraction, and ethical scraping practices

Build AI agents that browse the web, scrape content, and automate browser interactions using Playwright, Selenium, and other tools.

Process

1. Review the task requirements.
2. Apply the skill's methodology.
3. Validate the output against the defined criteria.

Step 1: Basic Web Scraping with httpx and BeautifulSoup

import httpx
from bs4 import BeautifulSoup
from typing import Dict, List, Optional

async def fetch_page(url: str, headers: Optional[Dict] = None) -> str:
    """Fetch HTML content from URL."""
    default_headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
    }

    if headers:
        default_headers.update(headers)

    async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
        response = await client.get(url, headers=default_headers)
        response.raise_for_status()
        return response.text

def parse_html(html: str) -> BeautifulSoup:
    """Parse HTML with BeautifulSoup."""
    return BeautifulSoup(html, "lxml")

async def scrape_basic(url: str) -> Dict[str, any]:
    """Basic web scraping example."""
    html = await fetch_page(url)
    soup = parse_html(html)

    return {
        "title": soup.title.string if soup.title else None,
        "headings": [h.get_text() for h in soup.find_all(["h1", "h2", "h3"])],
        "links": [a.get("href") for a in soup.find_all("a", href=True)],
        "text": soup.get_text()[:1000]  # First 1000 chars
    }

# Usage
result = await scrape_basic("https://example.com")
print(result)

Step 2: Structured Content Extraction

from pydantic import BaseModel, Field
from typing import List, Optional

class Article(BaseModel):
    """Structured article data."""
    title: str
    content: str
    author: Optional[str] = None
    published_date: Optional[str] = None
    tags: List[str] = Field(default_factory=list)

def extract_article(soup: BeautifulSoup, url: str) -> Article:
    """Extract structured article data."""

    # Try common article selectors
    title_selectors = [
        "h1.article-title",
        "h1.post-title",
        "article h1",
        "h1"
    ]

    title = None
    for selector in title_selectors:
        elem = soup.select_one(selector)
        if elem:
            title = elem.get_text().strip()
            break

    # Extract content
    content_selectors = [
        "article .content",
        ".post-content",
        "article",
        "main"
    ]

    content = None
    for selector in content_selectors:
        elem = soup.select_one(selector)
        if elem:
            content = elem.get_text().strip()
            

    
    author_elem = soup.select_one()
    author = author_elem.get_text().strip()  author_elem  

    
    date_elem = soup.select_one()
    date = date_elem.get()  (date_elem.get_text()  date_elem  )

    
    tag_elems = soup.select()
    tags = [tag.get_text().strip()  tag  tag_elems]

     Article(
        title=title  ,
        content=content  ,
        author=author,
        published_date=date,
        tags=tags
    )


html =  fetch_page()
soup = parse_html(html)
article = extract_article(soup, )
(article.model_dump_json(indent=))

Step 3: Browser Automation with Playwright

from playwright.async_api import async_playwright, Page, Browser
from typing import Optional

class WebBrowser:
    """Browser automation wrapper."""

    def __init__(self):
        self.browser: Optional[Browser] = None
        self.page: Optional[Page] = None

    async def start(self, headless: bool = True):
        """Start browser instance."""
        playwright = await async_playwright().start()
        self.browser = await playwright.chromium.launch(headless=headless)
        self.page = await self.browser.new_page()

    async def close(self):
        """Close browser."""
        if self.browser:
            await self.browser.close()

    async def navigate(self, url: str, wait_until: str = "networkidle"):
        """Navigate to URL."""
        await .page.goto(url, wait_until=wait_until)

      () -> :
        
          .page.content()

      () -> :
        
          .page.inner_text()

      ():
        
         .page.click(selector)

      ():
        
         .page.fill(selector, text)

      ():
        
         .page.screenshot(path=path)

      ():
        
         .page.wait_for_selector(selector, timeout=timeout)


browser = WebBrowser()
 browser.start(headless=)
 browser.navigate()
content =  browser.get_content()
 browser.close()

Step 4: JavaScript-Heavy Site Scraping

async def scrape_spa(browser: WebBrowser, url: str) -> Dict:
    """Scrape Single Page Application (SPA)."""
    await browser.navigate(url, wait_until="networkidle")

    # Wait for content to load
    await browser.wait_for_selector(".content", timeout=10000)

    # Execute JavaScript to extract data
    data = await browser.page.evaluate("""
        () => {
            return {
                title: document.title,
                headings: Array.from(document.querySelectorAll('h1, h2')).map(h => h.textContent),
                articles: Array.from(document.querySelectorAll('.article')).map(article => ({
                    title: article.querySelector('.title')?.textContent,
                    content: article.querySelector('.content')?.textContent
                }))
            }
        }
    """)

    return data

# Usage
browser = WebBrowser()
await browser.start()
result = await scrape_spa(browser, "https://spa-example.com")
await browser.close()

Step 5: Rate Limiting and Ethical Scraping

import asyncio
from datetime import datetime, timedelta
from collections import deque
from typing import Deque

class RateLimiter:
    """Rate limiter for web requests."""

    def __init__(self, max_requests: int, time_window: int):
        """
        Args:
            max_requests: Maximum requests allowed
            time_window: Time window in seconds
        """
        self.max_requests = max_requests
        self.time_window = time_window
        self.requests: Deque[datetime] = deque()

    async def acquire(self):
        """Wait if necessary to respect rate limit."""
        now = datetime.now()

        # Remove old requests outside time window
        while self.requests and (now - self.requests[0]).total_seconds() > self.time_window:
            self.requests.popleft()

        # Wait if at limit
        if len(self.requests) >= self.max_requests:
            sleep_time = self.time_window - (now - self.requests[0]).total_seconds()
            if sleep_time > 0:
                 asyncio.sleep(sleep_time)
                  .acquire()

        .requests.append(datetime.now())

 :
    

     ():
        .rate_limiter = RateLimiter(requests_per_minute, )
        .session = httpx.AsyncClient(
            timeout=,
            follow_redirects=,
            headers={
                : 
            }
        )

      () -> :
        
         .rate_limiter.acquire()

        
        

        response =  .session.get(url)
        response.raise_for_status()
         response.text

      () -> :
        
        html =  .fetch(url)
        soup = parse_html(html)

         {
            : url,
            : soup.title.string  soup.title  ,
            : soup.get_text()[:]  
        }

      ():
        
         .session.aclose()


scraper = EthicalScraper(requests_per_minute=)
result =  scraper.scrape()
 scraper.close()

Step 6: Content Extraction with LLM

from langchain_google_genai import ChatGoogleGenerativeAI
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import PydanticOutputParser

llm = ChatGoogleGenerativeAI(model="gemini-2.5-flash")

class LLMContentExtractor:
    """Extract structured content using LLM."""

    def __init__(self, llm):
        self.llm = llm

    async def extract(self, html: str, extraction_schema: BaseModel) -> BaseModel:
        """Extract structured data from HTML using LLM."""
        soup = parse_html(html)
        text_content = soup.get_text()[:8000]  # Limit for LLM context

        parser = PydanticOutputParser(pydantic_object=extraction_schema)

        prompt = ChatPromptTemplate.from_messages([
            ("system", """Extract structured data from web page content.

{format_instructions}

Extract only information that is clearly present in the content."""),
            ("user", "Extract data from this page:\n\n{content}")
        ])

        chain = prompt.partial(format_instructions=parser.get_format_instructions()) | llm | parser

        result = await chain.ainvoke({"content": text_content})
        return result

# Usage
class ProductInfo(BaseModel):
    name: str
    price: Optional[str] = None
    description: Optional[str] = 
    rating: [] = 

extractor = LLMContentExtractor(llm)
html =  fetch_page()
product =  extractor.extract(html, ProductInfo)
(product)

Step 7: Complete Web Browsing Agent

from langchain_core.tools import tool
from langchain_core.messages import HumanMessage

class WebBrowsingAgent:
    """Complete web browsing agent."""

    def __init__(self, llm, browser: WebBrowser):
        self.llm = llm
        self.browser = browser
        self.scraper = EthicalScraper()

    @tool
    async def search_web(self, query: str) -> str:
        """Search the web and return results."""
        # In production, use a search API
        # For demo, simulate search
        return f"Search results for: {query}"

    @tool
    async def fetch_page(self, url: str) -> str:
        """Fetch and return page content."""
        html = await self.scraper.fetch(url)
        soup = parse_html(html)
        return soup.get_text()[:5000]

    @tool
    async def navigate_and_extract(self, url: str, selector: str) -> str:
        """Navigate to URL and extract content by selector."""
         .browser.navigate(url)
        :
            element =  .browser.page.query_selector(selector)
             element:
                  element.inner_text()
             
         Exception  e:
             

      () -> :
        
        
        search_prompt = ChatPromptTemplate.from_messages([
            (, ),
            (, )
        ])

        search_chain = search_prompt | .llm | StrOutputParser()
        search_query =  search_chain.ainvoke({: question})

        
        search_results =  .search_web(search_query)

        
        

        
        summary_prompt = ChatPromptTemplate.from_messages([
            (, ),
            (, )
        ])

        
        content =  .fetch_page()  

        summary_chain = summary_prompt | .llm | StrOutputParser()
        answer =  summary_chain.ainvoke({
            : question,
            : content
        })

         answer


browser = WebBrowser()
 browser.start()
agent = WebBrowsingAgent(llm, browser)
answer =  agent.research()
(answer)
 browser.close()

### Step 8: Tavily Integration (New)

Use the `tavily` MCP server for advanced search and research capabilities:

```python
# Perform a comprehensive research task
response = await client.chat.completions.create(
    messages=[{"role": "user", "content": "Research the impact of quantum computing on cryptography"}],
    tools=[{
        "type": "mcp",
        "name": "tavily",
        "command": "npx",
        "args": ["-y", "tavily-mcp"]
    }]
)

Scraping Patterns

| Pattern | Use Case | Tool | ||-|| | Static HTML | Simple sites | httpx + BeautifulSoup | | JavaScript-heavy | SPAs, React apps | Playwright/Selenium | | API endpoints | Data APIs | httpx (direct API calls) | | Rate-limited | Many requests | RateLimiter | | Structured data | E-commerce, articles | LLM extraction |

Best Practices

• Respect robots.txt and rate limits
• Use appropriate User-Agent headers
• Implement retry logic with exponential backoff
• Cache responses when possible
• Handle errors gracefully
• Use async/await for concurrent requests
• Limit content size for LLM processing
• Respect website terms of service
• Use browser automation only when necessary
• Clean and validate extracted data

Anti-Patterns

Anti-Pattern	Fix
No rate limiting	Implement RateLimiter
Ignoring robots.txt	Check and respect robots.txt
Synchronous requests	Use async/await
No error handling	Wrap in try/except
Hardcoded selectors	Use flexible extraction
No timeout handling	Set appropriate timeouts
Ignoring HTTP status	Check response.status_code
Scraping too frequently	Implement delays between requests
No content validation	Validate extracted data
Ignoring legal/ethical	Review ToS and use responsibly

Related

• Knowledge: {directories.knowledge}/api-integration-patterns.json
• Skill: tool-usage
• Skill: using-langchain
• Skill: mcp-integration

When to Use

This skill should be used when strict adherence to the defined process is required.

Prerequisites

• Basic understanding of the agent factory context.
• Access to the necessary tools and resources.

---

Converted and distributed by TomeVault — claim your Tome and manage your conversions.