Automated data quality reports with comprehensive variable analysis, missing value detection, correlations, and HTML report generation - formerly pandas-profiling
Installer avec Codex ou Claude Copiez ce prompt, collez-le dans Codex, Claude ou un autre assistant, puis laissez-le vérifier la page du skill et l'installer pour vous.
Une commande directe contourne le prompt de vérification. Examinez la source avant de l'exécuter.
Automated data quality reports with comprehensive variable analysis, missing value detection, correlations, and HTML report generation - formerly pandas-profiling
author
workspace-hub
category
data-analysis
capabilities
["Automated data quality reports","Variable type inference and analysis","Missing value detection and patterns","Correlation analysis (Pearson, Spearman, Kendall, Phik)","Duplicate row detection","HTML report generation","Large dataset handling with minimal mode","Comparison reports between datasets","Time series analysis"]
Master YData Profiling (formerly pandas-profiling) for automated data quality reports with comprehensive variable analysis, missing value patterns, correlation detection, and publication-ready HTML reports.
When to Use This Skill
USE YData Profiling when:
Data quality assessment - Evaluating dataset health and completeness
Initial data exploration - Understanding a new dataset quickly
Missing value analysis - Detecting patterns in missing data
Variable analysis - Understanding distributions and characteristics
Data documentation - Creating shareable data quality reports
Dataset comparison - Comparing training vs test data, or before/after
Stakeholder reporting - Generating professional HTML reports
Data validation - Checking data before ML model training
DON'T USE YData Profiling when:
Real-time analysis - Need streaming data profiling
Custom visualizations - Specific chart requirements
Interactive dashboards - Use Streamlit or Dash instead
Very large datasets - Over 10M rows (use sampling or minimal mode)
Production pipelines - Need lightweight validation (use Great Expectations)
from ydata_profiling import ProfileReport
import pandas as pd
df = pd.read_csv("data.csv")
# Customized report
profile = ProfileReport(
df,
title="Custom Styled Report",
dataset={
"description": "This is a sample dataset for analysis",
"creator": "Data Team",
"copyright_holder": "Company Inc.",
"copyright_year": "2025",
"url": "https://company.com/data"
},
variables={
"descriptions": {
"revenue": "Total revenue in USD",
"units": "Number of units sold",
"category": "Product category"
}
},
html={
"style": {
"full_width": True
},
"navbar_show": True,
"minify_html": True
},
progress_bar=True
)
profile.to_file("custom_report.html")
Report Sections Control:
from ydata_profiling import ProfileReport
import pandas as pd
df = pd.read_csv("data.csv")
# Control which sections appear
profile = ProfileReport(
df,
title="Selective Report",
samples={
"head": 10, # Show first 10 rows"tail": 10# Show last 10 rows
},
duplicates={
"head": 10# Show first 10 duplicate rows
},
correlations={
"pearson": {"calculate": True},
"spearman": {"calculate": False}, # Skip Spearman"kendall": {"calculate": False}, # Skip Kendall"phi_k": {"calculate": False} # Skip Phi-K
},
missing_diagrams={
"bar": True,
"matrix": False, # Skip matrix"heatmap": False# Skip heatmap
}
)
profile.to_file("selective_report.html")
Export Options:
from ydata_profiling import ProfileReport
import pandas as pd
import json
df = pd.read_csv("data.csv")
profile = ProfileReport(df, title="Export Demo")
# Export to HTML
profile.to_file("report.html")
# Export to JSON
profile.to_file("report.json")
# Get JSON as string
json_output = profile.to_json()
# Get as dictionary
description_dict = profile.get_description()
# Save widgets for notebook
profile.to_widgets()
from ydata_profiling import ProfileReport, compare
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
import os
defml_dataset_profiling(
X: pd.DataFrame,
y: pd.Series,
output_dir: str,
test_size: float = 0.2) -> dict:
"""
Profile ML dataset with train/test comparison.
Args:
X: Feature DataFrame
y: Target Series
output_dir: Output directory
test_size: Test set proportion
Returns:
Profiling results
"""
os.makedirs(output_dir, exist_ok=True)
# Split data
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=test_size, random_state=42
)
# Combine features and target
df_train = X_train.copy()
df_train["target"] = y_train.values
df_test = X_test.copy()
df_test["target"] = y_test.values
print(f"Training set: {df_train.shape}")
print(f"Test set: {df_test.shape}")
# Generate individual profilesprint("\nProfiling training set...")
profile_train = ProfileReport(
df_train,
title="Training Set Profile",
explorative=True
)
profile_train.to_file(os.path.join(output_dir, "train_profile.html"))
print("Profiling test set...")
profile_test = ProfileReport(
df_test,
title="Test Set Profile",
explorative=True
)
profile_test.to_file(os.path.join(output_dir, "test_profile.html"))
# Generate comparison reportprint("Generating comparison report...")
comparison = compare([profile_train, profile_test])
comparison.to_file(os.path.join(output_dir, "train_test_comparison.html"))
# Check for data drift
drift_metrics = {}
numeric_cols = X.select_dtypes(include=[np.number]).columns
for col in numeric_cols:
train_mean = df_train[col].mean()
test_mean = df_test[col].mean()
train_std = df_train[col].std()
# Calculate drift as standardized difference
drift = abs(train_mean - test_mean) / train_std if train_std > 0else0
drift_metrics[col] = {
"train_mean": train_mean,
"test_mean": test_mean,
"drift_score": drift
}
# Identify significant drift
significant_drift = [
col for col, metrics in drift_metrics.items()
if metrics["drift_score"] > 0.5
]
if significant_drift:
print(f"\nWarning: Significant drift detected in: {significant_drift}")
return {
"train_profile": profile_train,
"test_profile": profile_test,
"comparison": comparison,
"drift_metrics": drift_metrics,
"significant_drift": significant_drift,
"output_dir": output_dir
}
# Generate sample ML datasetdefcreate_ml_dataset(n_samples=10000):
"""Create sample ML dataset."""
np.random.seed(42)
X = pd.DataFrame({
"feature_1": np.random.randn(n_samples),
"feature_2": np.random.exponential(10, n_samples),
"feature_3": np.random.uniform(0, 100, n_samples),
"feature_4": np.random.randn(n_samples) * 5,
"category": np.random.choice(["A", "B", "C"], n_samples)
})
# Target based on features
y = (X["feature_1"] + X["feature_2"] / 10 > 1).astype(int)
y = pd.Series(y, name="target")
return X, y
# X, y = create_ml_dataset(10000)# results = ml_dataset_profiling(X, y, "ml_profiling_output")
Example 3: Time Series Data Profiling
from ydata_profiling import ProfileReport
import pandas as pd
import numpy as np
from datetime import datetime, timedelta
import os
defprofile_time_series(
df: pd.DataFrame,
date_column: str,
value_columns: list,
output_dir: str) -> dict:
"""
Profile time series data with temporal analysis.
Args:
df: Time series DataFrame
date_column: Date/time column name
value_columns: List of value columns to analyze
output_dir: Output directory
Returns:
Profiling results
"""
os.makedirs(output_dir, exist_ok=True)
# Ensure datetime type
df = df.copy()
df[date_column] = pd.to_datetime(df[date_column])
# Sort by date
df = df.sort_values(date_column)
# Add temporal features
df["year"] = df[date_column].dt.year
df["month"] = df[date_column].dt.month
df["day_of_week"] = df[date_column].dt.dayofweek
df["hour"] = df[date_column].dt.hour
df["is_weekend"] = df["day_of_week"].isin([5, 6])
# Generate main profile
profile = ProfileReport(
df,
title="Time Series Data Profile",
tsmode=True, # Enable time series mode
sortby=date_column,
explorative=True
)
profile.to_file(os.path.join(output_dir, "time_series_profile.html"))
# Analyze temporal patterns
temporal_analysis = {}
for col in value_columns:
# Monthly statistics
monthly = df.groupby("month")[col].agg(["mean", "std", "min", "max"])
temporal_analysis[f"{col}_monthly"] = monthly.to_dict()
# Day of week statistics
dow = df.groupby("day_of_week")[col].agg(["mean", "std"])
temporal_analysis[f"{col}_day_of_week"] = dow.to_dict()
# Profile by time period# Most recent period
recent_cutoff = df[date_column].max() - timedelta(days=30)
df_recent = df[df[date_column] >= recent_cutoff]
iflen(df_recent) > 100:
profile_recent = ProfileReport(
df_recent,
title="Recent 30 Days Profile",
minimal=True
)
profile_recent.to_file(os.path.join(output_dir, "recent_30d_profile.html"))
print(f"Time series profiling complete!")
print(f"Date range: {df[date_column].min()} to {df[date_column].max()}")
print(f"Total records: {len(df):,}")
return {
"main_profile": profile,
"temporal_analysis": temporal_analysis,
"output_dir": output_dir
}
# Generate sample time seriesdefcreate_time_series_data():
"""Create sample time series data."""
np.random.seed(42)
# Generate hourly data for 1 year
dates = pd.date_range("2024-01-01", "2024-12-31", freq="H")
n = len(dates)
# Trend + seasonality + noise
trend = np.linspace(100, 150, n)
daily_seasonality = 20 * np.sin(2 * np.pi * np.arange(n) / 24)
weekly_seasonality = 10 * np.sin(2 * np.pi * np.arange(n) / (24 * 7))
noise = np.random.randn(n) * 5return pd.DataFrame({
"timestamp": dates,
"value": trend + daily_seasonality + weekly_seasonality + noise,
"volume": np.random.exponential(1000, n),
"category": np.random.choice(["A", "B", "C"], n)
})
# ts_df = create_time_series_data()# results = profile_time_series(# ts_df,# date_column="timestamp",# value_columns=["value", "volume"],# output_dir="time_series_output"# )
Integration Examples
YData Profiling with Streamlit
import streamlit as st
from ydata_profiling import ProfileReport
import pandas as pd
from streamlit_pandas_profiling import st_profile_report
st.set_page_config(page_title="Data Profiler", layout="wide")
st.title("Interactive Data Profiler")
uploaded_file = st.file_uploader("Upload CSV", type=["csv"])
if uploaded_file:
df = pd.read_csv(uploaded_file)
st.subheader("Data Preview")
st.dataframe(df.head(100))
# Profile optionswith st.sidebar:
st.header("Profile Options")
minimal = st.checkbox("Minimal Mode", value=False)
explorative = st.checkbox("Explorative Mode", value=True)
if st.button("Generate Profile"):
with st.spinner("Generating report..."):
profile = ProfileReport(
df,
title="Data Profile",
minimal=minimal,
explorative=explorative
)
st_profile_report(profile)
YData Profiling with Polars
from ydata_profiling import ProfileReport
import polars as pl
import pandas as pd
defprofile_polars_df(
lf: pl.LazyFrame,
title: str = "Polars Data Profile",
**kwargs
) -> ProfileReport:
"""
Profile Polars LazyFrame using YData Profiling.
Args:
lf: Polars LazyFrame
title: Report title
**kwargs: Additional ProfileReport arguments
Returns:
ProfileReport object
"""# Collect and convert to pandas
df_polars = lf.collect()
df_pandas = df_polars.to_pandas()
return ProfileReport(df_pandas, title=title, **kwargs)
# Usage# lf = pl.scan_parquet("data.parquet")# profile = profile_polars_df(lf, title="Polars Data Profile")# profile.to_file("profile.html")
Best Practices
1. Use Minimal Mode for Large Datasets
# GOOD: Minimal mode for large data
profile = ProfileReport(large_df, minimal=True)
# AVOID: Full explorative on large data# profile = ProfileReport(large_df, explorative=True) # Slow!
2. Sample for Initial Exploration
# GOOD: Sample first, then full profile
sample = df.sample(n=10000, random_state=42)
profile = ProfileReport(sample, title="Sample Profile")
# If interesting, profile full data# profile_full = ProfileReport(df, minimal=True)
# GOOD: Lazy profile, compute when needed
profile = ProfileReport(df, lazy=True)
# ... do other work ...
profile.to_file("report.html") # Computes here
Troubleshooting
Common Issues
Issue: Memory error with large dataset
# Solution 1: Use minimal mode
profile = ProfileReport(df, minimal=True)
# Solution 2: Sample data
profile = ProfileReport(df.sample(50000))
