Use this Skill to encode historical documents in TEI XML P5: critical apparatus, named entity markup (persName/placeName), XPath analysis with lxml, and XSLT transformation.
Use this Skill to encode historical documents in TEI XML P5: critical apparatus, named entity markup (persName/placeName), XPath analysis with lxml, and XSLT transformation.
TL;DR — Encode, analyze, and transform TEI P5 XML documents: build teiHeader metadata,
mark up named entities with VIAF authority links, model critical apparatus variant readings,
run XPath queries with lxml, extract entity frequency tables, and transform to HTML or plain
text via XSLT/Saxon-HE.
When to Use
Use this Skill when you need to:
Create a TEI P5 digital edition of a historical text with scholarly apparatus
Mark up personal names, place names, and organization names with authority record links
Represent multiple manuscript witnesses using <app>, <lem>, and <rdg> elements
Run XPath queries to extract named entities, count occurrences, and build frequency tables
Transform TEI XML to HTML reading text or plain text for downstream NLP
Do not use this Skill for:
Lightweight bibliographic XML (use Dublin Core or MODS)
Real-time XML database queries at scale (use eXist-db or BaseX)
Automatic NER tagging without human review (TEI requires scholarly validation)
Background
TEI (Text Encoding Initiative) P5 is the international standard XML vocabulary for
encoding humanities texts. Its hierarchical document model covers:
Element
Purpose
<teiHeader>
Bibliographic and editorial metadata
<text><body>
The encoded text content
<div>
Textual divisions (chapter, section, act)
<p>, <lg>, <l>
Paragraph, line group, line
<lb>, <pb>, <fw>
Line break, page break, running header/footer
<persName>, <placeName>, <orgName>
Named entity markup with @ref to authority
<abbr>, <expan>, <choice>
Abbreviation and expansion pairs
<app>, <lem>, <rdg>
Critical apparatus: lemma + variant readings
@wit
Witness sigil on <rdg> elements
The VIAF (Virtual International Authority File) and Wikidata provide stable URIs for
persons and places. Linking @ref="https://viaf.org/viaf/22146956/" disambiguates
historical figures across editions.
XPath 1.0/2.0 queries via lxml's xpath() method allow systematic extraction and
statistical analysis of encoded features without manual text parsing.
from lxml import etree
from typing importOptional
TEI_NS = "http://www.tei-c.org/ns/1.0"
XML_NS = "http://www.w3.org/XML/1998/namespace"
TEI = f"{{{TEI_NS}}}"deftei_element(tag: str, attrib: dict = None, text: str = None) -> etree._Element:
"""
Create a TEI-namespaced element with optional attributes and text.
Args:
tag: Local element name (without namespace prefix).
attrib: Optional dict of attribute name → value.
text: Optional text content for the element.
Returns:
lxml Element node in the TEI namespace.
"""
el = etree.Element(f"{TEI_NS}{tag}", nsmap={"tei": TEI_NS, None: TEI_NS})
if attrib:
for k, v in attrib.items():
el.set(k, v)
if text:
el.text = text
return el
defbuild_tei_header(
title: str,
author: str,
editor: str,
publisher: str,
date: str,
source_description: str,
language: str = "la",
) -> etree._Element:
"""
Construct a minimal TEI P5 teiHeader.
Args:
title: Work title.
author: Original author's name.
editor: Digital edition editor name.
publisher: Publishing institution.
date: Publication year string.
source_description: Description of the base manuscript/print.
language: ISO 639-1 language code.
Returns:
lxml Element for the complete teiHeader.
"""
NS = TEI_NS
defel(tag, **kwargs):
return etree.SubElement # convenience alias not used directly
header = etree.Element(f"{{{NS}}}teiHeader")
# fileDesc
file_desc = etree.SubElement(header, f"{{{NS}}}fileDesc")
title_stmt = etree.SubElement(file_desc, f"{{{NS}}}titleStmt")
etree.SubElement(title_stmt, f"{{{NS}}}title").text = title
author_el = etree.SubElement(title_stmt, f"{{{NS}}}author")
author_el.text = author
resp_stmt = etree.SubElement(title_stmt, f"{{{NS}}}respStmt")
resp = etree.SubElement(resp_stmt, f"{{{NS}}}resp")
resp.text = "Digital edition created by"
etree.SubElement(resp_stmt, f"{{{NS}}}name").text = editor
pub_stmt = etree.SubElement(file_desc, f"{{{NS}}}publicationStmt")
etree.SubElement(pub_stmt, f"{{{NS}}}publisher").text = publisher
etree.SubElement(pub_stmt, f"{{{NS}}}date").text = date
availability = etree.SubElement(pub_stmt, f"{{{NS}}}availability",
attrib={"status": "free"})
etree.SubElement(availability, f"{{{NS}}}licence",
attrib={"target": "https://creativecommons.org/licenses/by/4.0/"}).text = "CC-BY 4.0"
source_desc = etree.SubElement(file_desc, f"{{{NS}}}sourceDesc")
etree.SubElement(source_desc, f"{{{NS}}}p").text = source_description
# encodingDesc
enc_desc = etree.SubElement(header, f"{{{NS}}}encodingDesc")
project_desc = etree.SubElement(enc_desc, f"{{{NS}}}projectDesc")
etree.SubElement(project_desc, f"{{{NS}}}p").text = (
"Encoded following TEI P5 guidelines. Named entities linked to VIAF and Wikidata."
)
# profileDesc
profile_desc = etree.SubElement(header, f"{{{NS}}}profileDesc")
lang_usage = etree.SubElement(profile_desc, f"{{{NS}}}langUsage")
etree.SubElement(lang_usage, f"{{{NS}}}language",
attrib={"ident": language}).text = f"Language: {language}"return header
defbuild_tei_document(
header: etree._Element,
paragraphs: list[str],
) -> etree._Element:
"""
Assemble a complete TEI P5 document from a teiHeader and plain text paragraphs.
Args:
header: teiHeader element from build_tei_header().
paragraphs: List of paragraph text strings.
Returns:
Root <TEI> element with complete document tree.
"""
NS = TEI_NS
root = etree.Element(
f"{{{NS}}}TEI",
attrib={f"{{{XML_NS}}}lang": "la"},
nsmap={None: NS},
)
root.append(header)
text_el = etree.SubElement(root, f"{{{NS}}}text")
body = etree.SubElement(text_el, f"{{{NS}}}body")
div = etree.SubElement(body, f"{{{NS}}}div", attrib={"type": "chapter", "n": "1"})
for i, para_text inenumerate(paragraphs):
p_el = etree.SubElement(div, f"{{{NS}}}p", attrib={"n": str(i + 1)})
p_el.text = para_text
return root
Step 2 — XPath: Extract Named Entities with VIAF Frequencies
import pandas as pd
from collections import Counter
defextract_named_entities(
tei_root: etree._Element,
entity_types: list[str] = None,
) -> pd.DataFrame:
"""
Extract all named entity elements from a TEI document and tabulate frequencies.
Queries XPath for persName, placeName, and orgName elements.
Groups by normalized text + @ref URI.
Args:
tei_root: Root lxml Element of the TEI document.
entity_types: List of entity element local names to query.
Defaults to ["persName", "placeName", "orgName"].
Returns:
DataFrame with columns: entity_type, text, ref_uri, count, sorted by count desc.
"""if entity_types isNone:
entity_types = ["persName", "placeName", "orgName"]
ns = {"tei": TEI_NS}
records = []
for etype in entity_types:
elements = tei_root.xpath(
f"//tei:{etype}", namespaces=ns
)
for el in elements:
text = "".join(el.itertext()).strip()
ref = el.get("ref", "")
if text:
records.append({
"entity_type": etype,
"text": text,
"ref_uri": ref,
})
df = pd.DataFrame(records)
if df.empty:
return df
freq_df = (
df.groupby(["entity_type", "text", "ref_uri"])
.size()
.reset_index(name="count")
.sort_values("count", ascending=False)
.reset_index(drop=True)
)
return freq_df
defextract_critical_apparatus(
tei_root: etree._Element,
) -> pd.DataFrame:
"""
Extract all critical apparatus entries from a TEI document.
Returns a table of lemma + variant readings per witness.
Args:
tei_root: Root lxml Element of the TEI document.
Returns:
DataFrame with columns: location, lemma_text, witness, reading_text.
"""
ns = {"tei": TEI_NS}
apps = tei_root.xpath("//tei:app", namespaces=ns)
records = []
for i, app_el inenumerate(apps):
# Get lemma text
lem_els = app_el.xpath("tei:lem", namespaces=ns)
lemma_text = "".join(lem_els[0].itertext()).strip() if lem_els else""# Get all variant readings
rdg_els = app_el.xpath("tei:rdg", namespaces=ns)
for rdg in rdg_els:
wit = rdg.get("wit", "").replace("#", "").strip()
rdg_text = "".join(rdg.itertext()).strip()
records.append({
"location": i + 1,
"lemma_text": lemma_text,
"witness": wit,
"reading_text": rdg_text,
})
return pd.DataFrame(records)
Step 3 — TEI to Plain Text and XSLT Transform
import subprocess
from pathlib import Path
deftei_to_plain_text(tei_root: etree._Element) -> str:
"""
Extract clean readable text from a TEI document, stripping all markup.
Preserves paragraph breaks. Expands <abbr>/<expan> pairs to expanded form.
Ignores teiHeader, apparatus elements (app/rdg), and metadata.
Args:
tei_root: Root lxml Element of the TEI document.
Returns:
Plain text string with paragraph separations.
"""
ns = {"tei": TEI_NS}
# Prefer expan over abbr in choice elementsfor choice in tei_root.xpath("//tei:choice", namespaces=ns):
abbr_els = choice.xpath("tei:abbr", namespaces=ns)
for a in abbr_els:
a.getparent().remove(a)
# Remove apparatus (rdg = variant readings, keep only lem)for rdg in tei_root.xpath("//tei:rdg", namespaces=ns):
rdg.getparent().remove(rdg)
# Remove headerfor header in tei_root.xpath("//tei:teiHeader", namespaces=ns):
header.getparent().remove(header)
# Extract paragraph texts
paragraphs = tei_root.xpath("//tei:p | //tei:l", namespaces=ns)
lines = []
for p in paragraphs:
text = "".join(p.itertext()).strip()
if text:
lines.append(text)
return"\n\n".join(lines)
deftransform_tei_xslt(
input_xml_path: str,
xsl_path: str,
output_path: str,
saxon_jar: str,
extra_params: dict = None,
) -> None:
"""
Apply an XSLT stylesheet to a TEI document using Saxon-HE (Java).
Requires Java and the Saxon-HE JAR on disk. Download Saxon-HE from
https://www.saxonica.com/download/java.xml
Args:
input_xml_path: Absolute path to the TEI XML file.
xsl_path: Absolute path to the XSLT 2.0/3.0 stylesheet.
output_path: Absolute path for the transformed output file.
saxon_jar: Absolute path to the saxon-he-*.jar file.
extra_params: Optional dict of XSLT parameter name → value pairs.
"""
cmd = [
"java", "-jar", saxon_jar,
f"-s:{input_xml_path}",
f"-xsl:{xsl_path}",
f"-o:{output_path}",
]
if extra_params:
for k, v in extra_params.items():
cmd.append(f"{k}={v}")
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
raise RuntimeError(f"Saxon-HE XSLT failed:\n{result.stderr}")
print(f"XSLT transform complete. Output: {output_path}")
Advanced Usage
Adding Named Entity Markup with VIAF References
defadd_named_entity_markup(
p_element: etree._Element,
entity_spans: list[dict],
) -> None:
"""
Insert <persName> or <placeName> markup into a paragraph element.
This is a simplified in-place markup inserter. In production, use a
standoff annotation approach or CATMA for complex overlapping markup.
Args:
p_element: A <p> element whose text contains the entities.
entity_spans: List of dicts with: text (str), entity_type (str),
viaf_ref (str, optional), wikidata_ref (str, optional).
"""
NS = TEI_NS
original_text = p_element.text or""for span in entity_spans:
entity_text = span["text"]
entity_type = span.get("entity_type", "persName")
ref = span.get("viaf_ref") or span.get("wikidata_ref", "")
if entity_text notin original_text:
continue# Split text at entity occurrence and wrap in element
idx = original_text.find(entity_text)
before = original_text[:idx]
after = original_text[idx + len(entity_text):]
p_element.text = before
attrib = {}
if ref:
attrib["ref"] = ref
entity_el = etree.SubElement(p_element, f"{{{NS}}}{entity_type}", attrib=attrib)
entity_el.text = entity_text
entity_el.tail = after
original_text = after # continue searching in remaining text
Serialize TEI to File
defserialize_tei(tei_root: etree._Element, output_path: str) -> None:
"""
Serialize a TEI lxml tree to a well-formed XML file with XML declaration.
Args:
tei_root: Root <TEI> element.
output_path: Absolute path to write the XML file.
"""
tree = etree.ElementTree(tei_root)
tree.write(
output_path,
encoding="UTF-8",
xml_declaration=True,
pretty_print=True,
)
print(f"TEI document written to {output_path}")
