Menu
Extract HTML page content and convert to clean Markdown, preserving headings, images, lists, bold/italic text, tables, and blockquotes.
Install with Codex or Claude Copy this prompt, paste it into Codex, Claude, or another assistant, and let it review the skill page and install it for you.
Based on SOC occupation classification
Integration testing for AI Agent CLI systems — PTY-driven conversation simulation, tool chain verification, OpenSpec-driven test design, and real-API validation.
Extract web article content from WeChat MP, blogs, and other platforms, converting text, images, formatting, and tables to clean Markdown.
Configure, extend, or contribute to Hermes Agent.
Generate images, video, and audio with ComfyUI — install, launch, manage nodes/models, run workflows with parameter injection. Uses the official comfy-cli for lifecycle and direct REST/WebSocket API for execution.
Decomposition playbook + anti-temptation rules for an orchestrator profile routing work through Kanban. The "don't do the work yourself" rule and the basic lifecycle are auto-injected into every kanban worker's system prompt; this skill is the deeper playbook when you're specifically playing the orchestrator role.
Pitfalls, examples, and edge cases for Hermes Kanban workers. The lifecycle itself is auto-injected into every worker's system prompt as KANBAN_GUIDANCE (from agent/prompt_builder.py); this skill is what you load when you want deeper detail on specific scenarios.
| name | web-content-extraction |
| category | research |
| trigger | User asks to extract, scrape, or convert web page content to Markdown. Includes WeChat articles, blog posts, documentation pages, or any HTML-to-Markdown conversion where preserving formatting and images matters. |
| description | Extract HTML page content and convert to clean Markdown, preserving headings, images, lists, bold/italic text, tables, and blockquotes. |
Extract HTML page content and convert to clean Markdown, preserving headings, images, lists, bold/italic text, tables, and blockquotes.
beautifulsoup4 — install with pip: pip install beautifulsoup4execute_code for this — the sandbox lacks conda packages. Use terminal with conda activation instead.curl -s -L -A "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" "URL" > /tmp/page.html
Inspect the HTML to find the main content container:
id="js_content"class="post-content", id="main-content", or <article> taggrep -o 'id="[^"]*"' /tmp/page.html | sort -u to discover IDsUse a depth-first walk collecting leaf-level content blocks:
from bs4 import BeautifulSoup, NavigableString
import re
with open('/tmp/page.html', 'r') as f:
html = f.read()
soup = BeautifulSoup(html, 'html.parser')
content = soup.find(id='js_content') # adjust selector per site
blocks = []
def walk(el):
if isinstance(el, NavigableString):
t = str(el).strip()
if t:
blocks.append(('text', t))
return
if not hasattr(el, 'name') or el.name is None:
return
tag = el.name
if tag == 'img':
src = el.get('data-src') or el.get('src') or ''
alt = el.get('alt', '')
if src:
blocks.append(('image', src, alt))
return
if tag == 'section':
# Background images
style = el.get('style', '')
bg = re.search(r'url\(["\']?([^"\'()]+)["\']?\)', style)
if bg and ('mmbiz' in bg.group(1) or 'wx' in bg.group(1)):
blocks.append(('image', bg.group(1), ''))
# Child images
imgs = el.find_all('img')
if imgs:
for img in imgs:
src = img.get('data-src') or img.get('src') or ''
if src:
blocks.append(('image', src, img.get('alt', '')))
return
# Heading detection by style
has_bold = 'font-weight: bold' in style
has_big = any(f'font-size: {s}px' in style for s in ['16','17','18','20','22','24'])
text = el.get_text(strip=True)
if (has_bold or has_big) and len(text) < 150:
level = 2 if any(f'font-size: {s}px' in style for s in ['18','20','22','24']) else 3
blocks.append(('heading', text, level))
return
for child in el.children:
walk(child)
return
if tag in ['h1','h2','h3','h4','h5','h6']:
text = el.get_text(strip=True)
if text:
blocks.append(('heading', text, int(tag[1])))
return
if tag == 'p':
text = get_inline(el).strip()
if text:
blocks.append(('text', text))
return
if tag in ['strong', 'b']:
text = el.get_text(strip=True)
if text:
blocks.append(('text', f'**{text}**'))
return
if tag in ['em', 'i']:
text = el.get_text(strip=True)
if text:
blocks.append(('text', f'*{text}*'))
return
if tag == 'span':
style = el.get('style', '')
text = get_inline(el).strip()
if text:
if 'font-weight: bold' in style:
blocks.append(('text', f'**{text}**'))
else:
blocks.append(('text', text))
return
if tag in ['ul', 'ol']:
items = []
for i, li in enumerate(el.find_all('li', recursive=False)):
prefix = f"{i+1}." if tag == 'ol' else "-"
t = get_inline(li).strip()
if t:
items.append(f"{prefix} {t}")
if items:
blocks.append(('list', '\n'.join(items)))
return
if tag == 'li':
text = get_inline(el).strip()
if text:
blocks.append(('text', text))
return
if tag == 'blockquote':
text = get_inline(el).strip()
if text:
quoted = '\n'.join(f'> {l}' for l in text.split('\n'))
blocks.append(('blockquote', quoted))
return
if tag == 'table':
text = extract_table(el)
if text:
blocks.append(('table', text))
return
for child in el.children:
walk(child)
def get_inline(el):
parts = []
for child in el.children:
if isinstance(child, NavigableString):
t = str(child)
if t.strip():
parts.append(t)
continue
if not hasattr(child, 'name') or child.name is None:
continue
if child.name == 'img':
continue
if child.name == 'br':
parts.append(' ')
continue
if child.name in ['strong', 'b']:
t = child.get_text(strip=True)
if t:
parts.append(f'**{t}**')
elif child.name in ['em', 'i']:
t = child.get_text(strip=True)
if t:
parts.append(f'*{t}*')
elif child.name == 'span':
inner = get_inline(child).strip()
if inner:
parts.append(inner)
else:
inner = get_inline(child)
if inner.strip():
parts.append(inner)
return ''.join(parts)
def extract_table(el):
rows = []
for tr in el.find_all('tr'):
cells = [td.get_text(strip=True) for td in tr.find_all(['td', 'th'])]
if cells:
rows.append(cells)
if not rows:
return ''
max_cols = max(len(r) for r in rows)
for r in rows:
while len(r) < max_cols:
r.append('')
lines = ['| ' + ' | '.join(rows[0]) + ' |']
lines.append('| ' + ' | '.join(['---'] * max_cols) + ' |')
for r in rows[1:]:
lines.append('| ' + ' | '.join(r) + ' |')
return '\n'.join(lines)
walk(content)
# Convert blocks to markdown
lines = []
for block in blocks:
if block[0] == 'text':
lines.append(block[1])
elif block[0] == 'image':
lines.append(f'![{block[2]}]({block[1]})')
elif block[0] == 'heading':
lines.append(f'{"#" * block[2]} {block[1]}')
elif block[0] == 'list':
lines.append(block[1])
elif block[0] == 'blockquote':
lines.append(block[1])
elif block[0] == 'table':
lines.append(block[1])
# Add spacing between block types
md_parts = []
for i, line in enumerate(lines):
md_parts.append(line)
if i < len(lines) - 1:
curr, nxt = blocks[i][0], blocks[i+1][0]
if curr != nxt or curr in ('heading', 'image', 'list', 'blockquote', 'table'):
md_parts.append('')
md = '\n'.join(md_parts)
md = re.sub(r'\n{4,}', '\n\n\n', md).strip()
print(md)
terminal with conda activation.curl with standard User-Agent.<section> elements. Walk depth-first collecting leaf blocks rather than mapping section hierarchy to headings.<h1>-<h6> tags. Check style attributes.background-image: url(...) instead of <img> tags. Check section styles for url() patterns.