document-classification-nlp

name	document-classification-nlp
description	Automatically classify and extract information from construction documents using NLP. Categorize RFIs, submittals, change orders, specifications, and contracts.

Document Classification with NLP

Overview

This skill implements NLP-based document classification and information extraction for construction projects. Automate document sorting, key term extraction, and content analysis.

Document Types:

• RFIs (Requests for Information)
• Submittals and shop drawings
• Change orders and variations
• Specifications and standards
• Contracts and agreements
• Safety reports and permits

Quick Start

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
import pandas as pd

# Sample training data
documents = [
    ("Please clarify the steel reinforcement spacing for the foundation slab", "RFI"),
    ("Attached shop drawing for HVAC ductwork layout", "Submittal"),
    ("Additional cost for unforeseen soil conditions", "Change Order"),
    ("Fire-rated wall assembly specification Section 09 21 16", "Specification"),
]

texts, labels = zip(*documents)

# Train classifier
classifier = Pipeline([
    ('tfidf', TfidfVectorizer(max_features=1000, ngram_range=(1, 2))),
    ('clf', MultinomialNB())
])

classifier.fit(texts, labels)

# Classify new document
new_doc = "Request to approve substitution of specified light fixtures"
prediction = classifier.predict([new_doc])[0]
print(f"Classification: {prediction}")  # Output: Submittal

Advanced Classification System

Document Classifier Class

import re
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import Pipeline
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import LabelEncoder
from typing import List, Dict, Tuple, Optional
import spacy
from dataclasses import dataclass

@dataclass
class ClassificationResult:
    document_id: str
    predicted_class: str
    confidence: float
    alternative_classes: List[Tuple[str, float]]
    extracted_entities: Dict[str, List[str]]
    keywords: List[str]

class ConstructionDocumentClassifier:
    """Classify and analyze construction documents"""

    # Document type patterns
    DOCUMENT_PATTERNS = {
        'RFI': [
            ,
            ,
            ,
            ,
            
        ],
        : [
            ,
            ,
            ,
            ,
            ,
            
        ],
        : [
            ,
            ,
            ,
            ,
            ,
            
        ],
        : [
            ,
            ,
            ,
            ,
            
        ],
        : [
            ,
            ,
            ,
            ,
            ,
            
        ],
        : [
            ,
            ,
            ,
            ,
            
        ]
    }

     ():
        .classifier = 
        .vectorizer = 
        .label_encoder = LabelEncoder()

         use_spacy:
            :
                .nlp = spacy.load()
            :
                .nlp = 
        :
            .nlp = 

     () -> :
        
        
        y = .label_encoder.fit_transform(labels)

        
        .classifier = Pipeline([
            (, TfidfVectorizer(
                max_features=,
                ngram_range=(, ),
                stop_words=,
                sublinear_tf=
            )),
            (, LinearSVC(C=, class_weight=))
        ])

        
        .classifier.fit(documents, y)

        
        scores = cross_val_score(.classifier, documents, y, cv=)

         {
            : scores.mean(),
            : scores.std(),
            : (.label_encoder.classes_)
        }

     () -> ClassificationResult:
        
         .classifier  :
            
             ._rule_based_classify(document)

        
        prediction = .classifier.predict([document])[]
        predicted_class = .label_encoder.inverse_transform([prediction])[]

        
        decision_scores = .classifier.decision_function([document])[]
        probs = ._softmax(decision_scores)

        alternatives = [
            (.label_encoder.inverse_transform([i])[], (probs[i]))
             i  np.argsort(probs)[::-][:]
        ]

        
        entities = ._extract_entities(document)
        keywords = ._extract_keywords(document)

         ClassificationResult(
            document_id=,
            predicted_class=predicted_class,
            confidence=(probs[prediction]),
            alternative_classes=alternatives,
            extracted_entities=entities,
            keywords=keywords
        )

     () -> ClassificationResult:
        
        doc_lower = document.lower()
        scores = {}

         doc_type, patterns  .DOCUMENT_PATTERNS.items():
            score = (
                  pattern  patterns
                 re.search(pattern, doc_lower)
            )
            scores[doc_type] = score

         (scores.values()) == :
            predicted = 
            confidence = 
        :
            predicted = (scores, key=scores.get)
            confidence = scores[predicted] / (.DOCUMENT_PATTERNS[predicted])

         ClassificationResult(
            document_id=,
            predicted_class=predicted,
            confidence=confidence,
            alternative_classes=[],
            extracted_entities=._extract_entities(document),
            keywords=._extract_keywords(document)
        )

     () -> [, []]:
        
        entities = {
            : [],
            : [],
            : [],
            : [],
            : []
        }

        
        date_pattern = 
        entities[] = re.findall(date_pattern, document)

        
        money_pattern = 
        entities[] = re.findall(money_pattern, document)

        
        ref_pattern = 
        entities[] = re.findall(ref_pattern, document, re.IGNORECASE)

        
         .nlp:
            doc = .nlp(document)
             ent  doc.ents:
                 ent.label_ == :
                    entities[].append(ent.text)
                 ent.label_ == :
                    entities[].append(ent.text)

         entities

     () -> []:
        
        
        construction_terms = [
            , , , , ,
            , , , , ,
            , , , , ,
            , , , , 
        ]

        doc_lower = document.lower()
        found_terms = [term  term  construction_terms  term  doc_lower]

         found_terms[:top_n]

     () -> np.ndarray:
        
        exp_x = np.exp(x - np.(x))
         exp_x / exp_x.()

     () -> pd.DataFrame:
        
        results = [.classify(doc)  doc  documents]

         pd.DataFrame([{
            : r.predicted_class,
            : r.confidence,
            : .join(r.keywords),
            : .join(r.extracted_entities[]),
            : .join(r.extracted_entities[])
        }  r  results])

Information Extraction

Key Information Extractor

class ConstructionInfoExtractor:
    """Extract key information from construction documents"""

    def __init__(self):
        self.patterns = {
            'rfi_number': r'RFI\s*[-#]?\s*(\d+)',
            'submittal_number': r'(?:Submittal|SI)\s*[-#]?\s*(\d+)',
            'change_order_number': r'(?:Change Order|CO|PCO)\s*[-#]?\s*(\d+)',
            'spec_section': r'Section\s*(\d{2}\s*\d{2}\s*\d{2})',
            'cost_amount': r'\$\s*([\d,]+(?:\.\d{2})?)',
            'duration_days': r'(\d+)\s*(?:calendar\s+)?days?',
            'drawing_reference': r'(?:Drawing|Dwg|DWG)\s*[-#]?\s*([A-Z\d-]+)',
            'date': r'(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})',
            'contractor_name': r'(?:Contractor|Subcontractor):\s*([^\n]+)',
            'project_name': r'Project:\s*([^\n]+)',
            'priority': r'(?:Priority|Urgency):\s*(Critical|High|Medium|Low)'
        }

    def extract_all(self, document: str) -> Dict:
        """Extract all available information"""
        results = {}

        for field, pattern in self.patterns.items():
            matches = re.findall(pattern, document, re.IGNORECASE)
            results[field] = matches if matches else None

        
         results.get():
            results[] = [
                (amt.replace(, ))
                 amt  results[]
            ]

         results

     () -> :
        
         {
            : ._find_first(document, .patterns[]),
            : ._find_first(document, .patterns[]),
            : ._find_first(document, .patterns[]),
            : ._find_first(document, .patterns[]),
            : ._extract_question(document),
            : ._find_first(document, .patterns[])
        }

     () -> :
        
        costs = re.findall(.patterns[], document)
        total_cost = ((c.replace(, ))  c  costs)  costs  

         {
            : ._find_first(document, .patterns[]),
            : ._find_first(document, .patterns[]),
            : total_cost,
            : ._find_first(document, .patterns[]),
            : ._extract_reason(document),
            : ._find_first(document, .patterns[])
        }

     () -> []:
         = re.search(pattern, document, re.IGNORECASE)
         .group()    

     () -> []:
        
        
        patterns = [
            ,
            ,
        ]
         pattern  patterns:
             = re.search(pattern, document, re.IGNORECASE | re.DOTALL)
             :
                 .group().strip()[:]
         

     () -> []:
        
        patterns = [
            ,
            ,
        ]
         pattern  patterns:
             = re.search(pattern, document, re.IGNORECASE | re.DOTALL)
             :
                 .group().strip()[:]
         

Processing Pipeline

def process_document_batch(documents: List[str], output_path: str):
    """Process and classify a batch of documents"""
    classifier = ConstructionDocumentClassifier()
    extractor = ConstructionInfoExtractor()

    results = []

    for i, doc in enumerate(documents):
        # Classify
        classification = classifier.classify(doc)

        # Extract info based on type
        if classification.predicted_class == 'RFI':
            extracted = extractor.extract_rfi_details(doc)
        elif classification.predicted_class == 'Change Order':
            extracted = extractor.extract_change_order_details(doc)
        else:
            extracted = extractor.extract_all(doc)

        results.append({
            'Document_ID': i + 1,
            'Classification': classification.predicted_class,
            'Confidence': classification.confidence,
            'Keywords': ', '.join(classification.keywords),
            **extracted
        })

    df = pd.DataFrame(results)
    df.to_excel(output_path, index=False)

    return df

Quick Reference

Document Type	Key Patterns	Extracted Info
RFI	"request for information", "clarify"	Number, spec section, question
Submittal	"shop drawing", "approval request"	Number, product, spec section
Change Order	"change order", "additional cost"	Number, cost, duration impact
Specification	"Section XX XX XX"	Section number, requirements
Safety Report	"incident", "hazard"	Date, type, severity

Resources

• spaCy: https://spacy.io
• Scikit-learn: https://scikit-learn.org
• DDC Website: https://datadrivenconstruction.io

Next Steps

• See vector-search for semantic document search
• See llm-data-automation for advanced extraction
• See pdf-to-structured for PDF processing