菜单
Web crawling and scraping toolkit for fetching webpage content, extracting text, links, and structured data from HTML. Use when you need to retrieve information from websites, follow links, or extract specific elements from web pages. Use when this capability is needed.
默认使用会先检查来源的 Prompt；你也可以切换为直接命令，或下载本地副本。
决定是否安装前，请先阅读 SKILL.md，以及 SkillsMP 当前展示的配套文件。
用 Codex 或 Claude 帮你安装 复制这段 Prompt，粘贴到 Codex、Claude 或其他助手里，让它检查 Skill 页面并帮你完成安装。
直接命令不会经过审查 Prompt；运行前请先检查来源。
npx skills add https://github.com/tomevault-io/skills-registry --skill crawl命令会保持在同一行。复制前请横向滚动并检查完整内容。
想先保存到本地？可下载 SkillsMP 当前能够提供的文件。
正在显示 SKILL.md
基于 SOC 职业分类
| name | crawl |
| description | Web crawling and scraping toolkit for fetching webpage content, extracting text, links, and structured data from HTML. Use when you need to retrieve information from websites, follow links, or extract specific elements from web pages. Use when this capability is needed. |
| metadata | {"author":"datalayer"} |
This guide covers essential web crawling and scraping operations using Python libraries. For simple fetching, use httpx. For HTML parsing and data extraction, use beautifulsoup4. For JavaScript-rendered pages, consider playwright.
import httpx
from bs4 import BeautifulSoup
# Fetch and parse a webpage
response = httpx.get("https://example.com")
soup = BeautifulSoup(response.text, "html.parser")
# Extract title
title = soup.title.string if soup.title else "No title"
print(f"Title: {title}")
# Extract all text
text = soup.get_text(separator="\n", strip=True)
print(text)
import httpx
response = httpx.get("https://example.com")
print(f"Status: {response.status_code}")
print(f"Content-Type: {response.headers.get('content-type')}")
print(response.text)
headers = {
"User-Agent": "Mozilla/5.0 (compatible; DataBot/1.0)",
"Accept": "text/html,application/xhtml+xml",
}
params = {"q": "search query", "page": 1}
response = httpx.get(
"https://example.com/search",
headers=headers,
params=params,
timeout=30.0,
)
import httpx
try:
response = httpx.get(
"https://example.com",
follow_redirects=True,
timeout=30.0,
)
response.raise_for_status()
print(response.text)
except httpx.HTTPStatusError as e:
print(f"HTTP error: {e.response.status_code}")
except httpx.RequestError as e:
print(f"Request failed: {e}")
import httpx
import asyncio
async def fetch_pages(urls: list[str]) -> list[str]:
async with httpx.AsyncClient() as client:
tasks = [client.get(url) for url in urls]
responses = await asyncio.gather(*tasks, return_exceptions=True)
return [r.text if isinstance(r, httpx.Response) else str(r) for r in responses]
# Usage
urls = ["https://example.com/page1", "https://example.com/page2"]
results = asyncio.run(fetch_pages(urls))
from bs4 import BeautifulSoup
html = """
<html>
<head><title>Example Page</title></head>
<body>
<h1>Welcome</h1>
<p class="intro">This is an introduction.</p>
<a href="/page1">Link 1</a>
<a href="/page2">Link 2</a>
</body>
</html>
"""
soup = BeautifulSoup(html, "html.parser")
# Find elements
title = soup.title.string
h1 = soup.find("h1").text
intro = soup.find("p", class_="intro").text
# Find all links
links = [(a.text, a.get("href")) for a in soup.find_all("a")]
from bs4 import BeautifulSoup
soup = BeautifulSoup(html, "html.parser")
# Get all text, separated by newlines
text = soup.get_text(separator="\n", strip=True)
# Get text from specific sections
main_content = soup.find("main")
if main_content:
content_text = main_content.get_text(separator=" ", strip=True)
from urllib.parse import urljoin
from bs4 import BeautifulSoup
base_url = "https://example.com"
soup = BeautifulSoup(html, "html.parser")
links = []
for a in soup.find_all("a", href=True):
href = a.get("href")
absolute_url = urljoin(base_url, href)
links.append({
"text": a.get_text(strip=True),
"url": absolute_url,
})
from bs4 import BeautifulSoup
soup = BeautifulSoup(html, "html.parser")
tables = []
for table in soup.find_all("table"):
rows = []
for tr in table.find_all("tr"):
cells = [td.get_text(strip=True) for td in tr.find_all(["td", "th"])]
rows.append(cells)
tables.append(rows)
from bs4 import BeautifulSoup
soup = BeautifulSoup(html, "html.parser")
# Use CSS selectors
articles = soup.select("article.post")
headers = soup.select("h1, h2, h3")
nav_links = soup.select("nav a[href]")
first_para = soup.select_one("p")
import httpx
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from dataclasses import dataclass
@dataclass
class PageContent:
url: str
title: str
text: str
links: list[dict]
def crawl_page(url: str, timeout: float = 30.0) -> PageContent:
"""Crawl a single webpage and extract its content."""
headers = {
"User-Agent": "Mozilla/5.0 (compatible; DataBot/1.0)",
}
response = httpx.get(url, headers=headers, timeout=timeout, follow_redirects=True)
response.raise_for_status()
soup = BeautifulSoup(response.text, "html.parser")
# Remove script and style elements
for element in soup(["script", "style", "nav", "footer"]):
element.decompose()
# Extract title
title = soup.title.string if soup.title else ""
# Extract text
text = soup.get_text(separator="\n", strip=True)
# Extract links
links = []
for a in soup.find_all("a", href=True):
href = a.get("href")
href href.startswith((, , )):
links.append({
: a.get_text(strip=),
: urljoin(url, href),
})
PageContent(url=url, title=title, text=text, links=links)
page = crawl_page()
()
()
import httpx
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
from collections import deque
def crawl_site(
start_url: str,
max_pages: int = 10,
max_depth: int = 2,
same_domain_only: bool = True,
) -> dict[str, dict]:
"""Crawl multiple pages starting from a URL."""
visited = {}
queue = deque([(start_url, 0)]) # (url, depth)
start_domain = urlparse(start_url).netloc
headers = {"User-Agent": "Mozilla/5.0 (compatible; DataBot/1.0)"}
with httpx.Client(headers=headers, timeout=30.0, follow_redirects=True) as client:
while queue and len(visited) < max_pages:
url, depth = queue.popleft()
if url in visited:
continue
if depth > max_depth:
continue
try:
response = client.get(url)
response.raise_for_status()
soup = BeautifulSoup(response.text, "html.parser")
# Remove non-content elements
for element in soup(["script", "style"]):
element.decompose()
visited[url] = {
: soup.title.string soup.title ,
: soup.get_text(separator=, strip=)[:],
: depth,
}
depth < max_depth:
a soup.find_all(, href=):
href = a.get()
href href.startswith((, )):
next_url = urljoin(url, href)
next_domain = urlparse(next_url).netloc
same_domain_only next_domain != start_domain:
next_url visited:
queue.append((next_url, depth + ))
Exception e:
visited[url] = {: (e), : depth}
visited
pages = crawl_site(, max_pages=, max_depth=)
url, data pages.items():
()
For pages that require JavaScript execution, use Playwright:
from playwright.sync_api import sync_playwright
def crawl_js_page(url: str) -> str:
"""Crawl a JavaScript-rendered page."""
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page()
page.goto(url, wait_until="networkidle")
content = page.content()
browser.close()
return content
from urllib.robotparser import RobotFileParser
from urllib.parse import urljoin
def can_crawl(url: str, user_agent: str = "*") -> bool:
"""Check if crawling is allowed by robots.txt."""
from urllib.parse import urlparse
parsed = urlparse(url)
robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
rp = RobotFileParser()
rp.set_url(robots_url)
try:
rp.read()
return rp.can_fetch(user_agent, url)
except Exception:
return True # Allow if robots.txt cannot be read
import time
import httpx
from collections import defaultdict
class RateLimitedClient:
def __init__(self, requests_per_second: float = 1.0):
self.client = httpx.Client(timeout=30.0, follow_redirects=True)
self.delay = 1.0 / requests_per_second
self.last_request = defaultdict(float)
def get(self, url: str) -> httpx.Response:
from urllib.parse import urlparse
domain = urlparse(url).netloc
elapsed = time.time() - self.last_request[domain]
if elapsed < self.delay:
time.sleep(self.delay - elapsed)
response = self.client.get(url)
self.last_request[domain] = time.time()
return response
This skill includes ready-to-use scripts in the scripts/ folder:
| Script | Description |
|---|---|
fetch_page.py | Fetch a single webpage and extract text content |
extract_links.py | Extract all links from a webpage with filtering |
extract_tables.py | Extract HTML tables as JSON or CSV |
crawl_site.py | Multi-page crawler with depth control |
check_robots.py | Check if URL is allowed by robots.txt |
fetch_js_page.py | Fetch JavaScript-rendered pages using Playwright |
# Fetch a page and extract text
python scripts/fetch_page.py https://example.com
# Extract all links from a page
python scripts/extract_links.py https://example.com --output links.json
# Extract tables as CSV
python scripts/extract_tables.py https://example.com --format csv
# Crawl a site (max 10 pages, depth 2)
python scripts/crawl_site.py https://example.com --max-pages 10 --max-depth 2
# Check robots.txt permissions
python scripts/check_robots.py https://example.com/page
# Fetch JavaScript-rendered page
python scripts/fetch_js_page.py https://example.com --wait 1000
Install required packages:
pip install httpx beautifulsoup4 lxml
# For JavaScript rendering:
pip install playwright && playwright install chromium
Converted and distributed by TomeVault — claim your Tome and manage your conversions.