nlp-basics

name	nlp-basics
description	Process and analyze text using modern NLP techniques - preprocessing, embeddings, and transformers
version	1.4.0
sasmp_version	1.4.0
bonded_agent	05-nlp
bond_type	PRIMARY_BOND
parameters	{"required":[{"name":"text","type":"string|list","validation":"Non-empty text or list of texts"}],"optional":[{"name":"model_name","type":"string","default":"bert-base-uncased"},{"name":"max_length","type":"integer","default":512,"validation":"1 <= max_length <= 512"}]}
retry_logic	{"strategy":"exponential_backoff","max_attempts":3,"base_delay_ms":1000}
logging	{"level":"info","metrics":["tokenization_time","embedding_dim","batch_size"]}

NLP Basics Skill

Transform unstructured text into structured insights.

Quick Start

from transformers import AutoTokenizer, AutoModel
import torch

# Load model and tokenizer
tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
model = AutoModel.from_pretrained('bert-base-uncased')

# Tokenize
text = "Machine learning is transforming industries."
inputs = tokenizer(text, return_tensors='pt', padding=True, truncation=True)

# Get embeddings
with torch.no_grad():
    outputs = model(**inputs)
    embeddings = outputs.last_hidden_state.mean(dim=1)  # [CLS] pooling

print(f"Embedding shape: {embeddings.shape}")

Key Topics

1. Text Preprocessing

import re
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer

class TextPreprocessor:
    def __init__(self):
        self.lemmatizer = WordNetLemmatizer()
        self.stop_words = set(stopwords.words('english'))

    def clean(self, text):
        # Lowercase
        text = text.lower()
        # Remove URLs
        text = re.sub(, , text)
        
        text = re.sub(, , text)
        
        tokens = word_tokenize(text)
        tokens = [.lemmatizer.lemmatize(t)  t  tokens
                   t   .stop_words]
         .join(tokens)