huggingface

name	huggingface
description	HuggingFace ecosystem — transformers, datasets, huggingface_hub. Model loading, tokenization, training, and dataset handling.

HuggingFace

Transformers — model & tokenizer

from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification

# Load pretrained
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
model = AutoModelForSequenceClassification.from_pretrained(
    "bert-base-uncased",
    num_labels=2,
)

# Tokenize
inputs = tokenizer(
    ["Hello world", "Fraud transaction"],
    padding=True,
    truncation=True,
    max_length=128,
    return_tensors="pt",
)

# Forward pass
outputs = model(**inputs)
logits = outputs.logits          # shape: [batch, num_labels]

Datasets

from datasets import load_dataset, Dataset, DatasetDict
import pandas as pd

# Load from HuggingFace Hub
ds = load_dataset("imdb")
ds["train"][0]                   # dict with 'text', 'label'

# Load from local files
ds = load_dataset("csv", data_files={"train": "train.csv", "test": "test.csv"})
ds = load_dataset("json", data_files="data.jsonl")

# From pandas
df = pd.read_csv("data.csv")
ds = Dataset.from_pandas(df)

# Map / preprocess
def tokenize(batch):
    return tokenizer(batch["text"], truncation=True, max_length=128)

ds = ds.map(tokenize, batched=True, batch_size=1000, num_proc=4)
ds = ds.remove_columns(["text"])
ds.set_format("torch", columns=["input_ids", "attention_mask", "label"])

# Train/val split
ds = ds.train_test_split(test_size=0.1, seed=42)

# Save / load from disk (avoid re-downloading)
ds.save_to_disk("data/processed")
ds = load_dataset("data/processed")  # or DatasetDict.load_from_disk(...)

Trainer (HF training loop)

from transformers import TrainingArguments, Trainer
import numpy as np
from sklearn.metrics import f1_score

training_args = TrainingArguments(
    output_dir="outputs/",
    num_train_epochs=3,
    per_device_train_batch_size=32,
    per_device_eval_batch_size=64,
    learning_rate=2e-5,
    weight_decay=0.01,
    evaluation_strategy="epoch",
    save_strategy="epoch",
    load_best_model_at_end=True,
    metric_for_best_model="f1",
    fp16=True,
    dataloader_num_workers=4,
    report_to="none",              # disable wandb/tb unless needed
)

def compute_metrics(eval_pred):
    logits, labels = eval_pred
    preds = np.argmax(logits, axis=-1)
    return {"f1": f1_score(labels, preds, average="binary")}

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=ds["train"],
    eval_dataset=ds["test"],
    compute_metrics=compute_metrics,
)
trainer.train()

huggingface_hub — upload/download

from huggingface_hub import hf_hub_download, snapshot_download, login

# Auth (or set HF_TOKEN env var)
login(token="hf_...")

# Download single file
path = hf_hub_download(repo_id="bert-base-uncased", filename="config.json")

# Download entire model to local dir
snapshot_download(repo_id="bert-base-uncased", local_dir="models/bert")

# Upload model
from huggingface_hub import HfApi
api = HfApi()
api.upload_folder(folder_path="outputs/", repo_id="user/my-model", repo_type="model")

Offline / cache

# Set cache dir (default: ~/.cache/huggingface)
export HF_HOME=/data/hf_cache

# Force offline mode — use only cached files
export TRANSFORMERS_OFFLINE=1
export HF_DATASETS_OFFLINE=1

Pitfalls

• AutoModel returns hidden states, not logits — use AutoModelForSequenceClassification for classification.
• tokenizer(...) returns Python lists by default; add return_tensors="pt" for PyTorch tensors.
• dataset.map(..., batched=True) is significantly faster — always use it for tokenization.
• set_format("torch") only affects __getitem__; save_to_disk always saves as Arrow regardless.
• HF_TOKEN must be set for gated models (Llama, Gemma); login() caches it at ~/.cache/huggingface/token.