exploratory-data-analysis

name	exploratory-data-analysis
description	Initial data investigation techniques to discover patterns, spot anomalies, test hypotheses, and check assumptions using summary statistics and visualization.
category	data-science
keywords	["exploratory-data-analysis","eda","data exploration","data profiling","hypothesis testing","pattern discovery","data understanding","initial analysis"]
difficulty	beginner
related_skills	["pandas","numpy","statistics","data-visualization"]

Exploratory Data Analysis

What I do

I provide techniques for initial data investigation to understand its structure, identify patterns, detect anomalies, formulate hypotheses, and guide feature engineering. EDA is the foundational step in any data science project that informs subsequent analysis and modeling decisions through systematic exploration and visualization.

When to use me

• At the start of any data science project
• Understanding data structure and quality
• Identifying relationships between variables
• Detecting outliers and anomalies
• Formulating hypotheses for testing
• Guiding feature engineering decisions
• Communicating data characteristics to stakeholders
• Validating data collection assumptions

Core Concepts

Data Profiling

• Structure discovery: Dimensions, data types, memory usage
• Value distributions: Histograms, density plots, frequency counts
• Missing value analysis: Patterns of missingness
• Duplicate detection: Exact and fuzzy duplicates

Univariate Analysis

• Central tendency: Mean, median, mode
• Dispersion: Range, variance, std, IQR
• Shape: Skewness, kurtosis, percentiles
• Outlier detection: Z-scores, IQR method

Bivariate/Multivariate Analysis

• Correlation: Pearson, Spearman, Kendall
• Relationships: Scatter plots, pair plots
• Group comparisons: Box plots, violin plots
• Dimensionality reduction: PCA for visualization

Statistical Testing

• Normality tests: Shapiro-Wilk, D'Agostino
• Outlier tests: Grubbs, Dixon's Q
• Homogeneity tests: Levene, Bartlett
• Independence tests: Chi-square, Fisher's exact

Code Examples (Python)

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats

# Basic data overview
df = pd.read_csv('data.csv')

# Structure information
print(f"Shape: {df.shape}")
print(f"\nColumn types:\n{df.dtypes}")
print(f"\nMemory usage: {df.memory_usage(deep=True).sum() / 1024**2:.2f} MB")

# First and last rows
print(df.head(10))
print(df.tail(5))

# Summary statistics - numerical
numerical_cols = df.select_dtypes(include=[np.number]).columns
numerical_summary = df[numerical_cols].describe(percentiles=[.01, .05, .25, .5, .75, .95, .99])
print(numerical_summary)

# Summary statistics - categorical
categorical_cols = df.select_dtypes(include=['object', 'category']).columns
for col in categorical_cols:
    print(f"\n{col}:")
    print()
    ()
    ()


missing = df.isnull().()
missing_pct = (df.isnull().() / (df) * ).()
missing_df = pd.DataFrame({: missing, : missing_pct})
missing_df = missing_df[missing_df[] > ].sort_values(, ascending=)
()


missing_pattern = df.isnull().(axis=).value_counts().sort_index()
()


()
()


 ():
    Q1 = series.quantile()
    Q3 = series.quantile()
    IQR = Q3 - Q1
    lower_bound = Q1 -  * IQR
    upper_bound = Q3 +  * IQR
     (series < lower_bound) | (series > upper_bound), lower_bound, upper_bound

 col  numerical_cols:
    outliers, lb, ub = detect_outliers_iqr(df[col].dropna())
    n_outliers = outliers.()
    pct_outliers = n_outliers / (df) * 
    ()


 col  numerical_cols:
     df[col].notna().() > :
        stat, p_value = stats.shapiro(df[col].dropna().sample((, (df[col]))))
        is_normal =   p_value >   
        ()


correlation_matrix = df[numerical_cols].corr()
()


corr_pairs = []
 i  ((correlation_matrix.columns)):
     j  (i+, (correlation_matrix.columns)):
        corr_pairs.append({
            : correlation_matrix.columns[i],
            : correlation_matrix.columns[j],
            : correlation_matrix.iloc[i, j]
        })
corr_df = pd.DataFrame(corr_pairs).sort_values(, key=, ascending=)
()


fig, axes = plt.subplots(, , figsize=(, ))
axes = axes.flatten()

 i, col  (numerical_cols[:]):
    ax = axes[i]
    df[col].hist(bins=, ax=ax, alpha=)
    ax.set_title()
    ax.axvline(df[col].mean(), color=, linestyle=, label=)
    ax.axvline(df[col].median(), color=, linestyle=, label=)
    ax.legend(fontsize=)

plt.tight_layout()
plt.savefig(, dpi=)
plt.show()


fig, axes = plt.subplots(, , figsize=(, ))
axes = axes.flatten()

 i, col  (numerical_cols[:]):
     (categorical_cols) > :
        sns.boxplot(x=categorical_cols[], y=col, data=df, ax=axes[i])
    axes[i].set_title()
    axes[i].tick_params(axis=, rotation=)

plt.tight_layout()
plt.savefig(, dpi=)
plt.show()


fig, axes = plt.subplots(, , figsize=(, ))
axes = axes.flatten()

 i, row  corr_df.head().iterrows():
    ax = axes[i]
    x, y = row[], row[]
    df_sample = df[[x, y]].dropna().sample((, (df)))
    ax.scatter(df_sample[x], df_sample[y], alpha=)
    ax.set_xlabel(x)
    ax.set_ylabel(y)
    ax.set_title()

plt.tight_layout()
plt.savefig(, dpi=)
plt.show()


plt.figure(figsize=(, ))
sns.heatmap(correlation_matrix, annot=, cmap=, center=,
            fmt=, square=, linewidths=)
plt.title()
plt.tight_layout()
plt.savefig(, dpi=)
plt.show()


selected_cols = numerical_cols[:].tolist()
 (categorical_cols) > :
    selected_cols.append(categorical_cols[])
sns.pairplot(df[selected_cols].dropna(), hue=categorical_cols[]  (categorical_cols) >   ,
             diag_kind=, plot_kws={: })
plt.savefig(, dpi=)
plt.show()


fig, axes = plt.subplots(, (categorical_cols[:]), figsize=(, ))

 i, col  (categorical_cols[:]):
    value_counts = df[col].value_counts().head()
    axes[i].barh(value_counts.index, value_counts.values)
    axes[i].set_xlabel()
    axes[i].set_title()
    axes[i].invert_yaxis()

plt.tight_layout()
plt.savefig(, dpi=)
plt.show()


 (categorical_cols) >= :
    ct = pd.crosstab(df[categorical_cols[]], df[categorical_cols[]], 
                     margins=, normalize=)
    ()


date_cols = [col  col  df.columns    col.lower()    col.lower()]
 date_cols:
    df[date_cols[]] = pd.to_datetime(df[date_cols[]], errors=)
    df[] = df[date_cols[]].dt.year
    df[] = df[date_cols[]].dt.month
    df[] = df[date_cols[]].dt.dayofweek
    
    temporal_summary = df.groupby([, ])[numerical_cols[]].agg([, , ])
    ()


cardinality = df[categorical_cols].nunique()
()


 col  categorical_cols:
    ()
     df[col].nunique() < :
        (df[col].value_counts())


 (categorical_cols) >= :
    contingency = pd.crosstab(df[categorical_cols[]], df[categorical_cols[]])
    chi2, p, dof, expected = stats.chi2_contingency(contingency)
    ()
    ()

Best Practices

1. Start simple: Begin with basic structure and summary statistics before complex analysis.

2. Visualize distributions: Always plot your data; summary statistics can be misleading.

3. Check for anomalies: Look for impossible values, duplicates, and unexpected patterns.

4. Understand missingness: Is missingness random, or does it carry information?

5. Iterative exploration: Let findings guide further investigation; don't pre-plan everything.

6. Document everything: Keep track of what you find and what questions remain.

7. Sample wisely: For large datasets, sample intelligently for visualization.

8. Consider context: Domain knowledge should guide and validate your exploration.

Common Patterns

Pattern 1: Automated EDA Report Generator

def generate_eda_report(df, output_dir='.'):
    """Generate comprehensive EDA report."""
    report = {}
    
    # Basic info
    report['shape'] = df.shape
    report['dtypes'] = df.dtypes.to_dict()
    report['missing'] = df.isnull().sum().to_dict()
    report['duplicates'] = df.duplicated().sum()
    
    # Numerical summary
    numerical = df.select_dtypes(include=[np.number])
    report['numerical_summary'] = numerical.describe(percentiles=[.01, .05, .25, .5, .75, .95, .99]).to_dict()
    report['skewness'] = numerical.skew().to_dict()
    report['kurtosis'] = numerical.kurtosis().to_dict()
    
    # Categorical summary
    categorical = df.select_dtypes(include=['object', 'category'])
    report['categorical_summary'] = {
        col: {
            'unique': df[col].nunique(),
            'top_values': df[col].value_counts().head(10).to_dict()
        } for col in categorical.columns
    }
    
    # Correlations
    report['correlations'] = numerical.corr().to_dict()
    
    # Save report
    import json
    with open(f'{output_dir}/eda_report.json', 'w') as f:
        json.dump(report, f, indent=2, default=)
    
     report

Pattern 2: Target Variable Analysis

def analyze_target(df, target_col):
    """Analyze relationship between features and target."""
    results = {}
    
    # Separate by target type
    if df[target_col].dtype in [np.number] and df[target_col].nunique() > 10:
        # Continuous target
        for col in df.select_dtypes(include=[np.number]).columns:
            if col != target_col:
                corr = df[[col, target_col]].corr().iloc[0, 1]
                results[col] = {'correlation': corr}
    
    elif df[target_col].nunique() <= 10:
        # Categorical target
        # ANOVA for numerical features
        for col in df.select_dtypes(include=[np.number]).columns:
            if col != target_col:
                groups = [group[col].values for name, group in df.groupby(target_col)]
                f_stat, p_value = stats.f_oneway(*groups)
                results[col] = {'f_statistic': f_stat, 'p_value': p_value}
        
        # Chi-square for categorical features
        for col in df.select_dtypes(include=['object', 'category']).columns:
            if col != target_col:
                contingency = pd.crosstab(df[col], df[target_col])
                chi2, p_value, _, _ = stats.chi2_contingency(contingency)
                results[col] = {'chi2': chi2, 'p_value': p_value}
    
    return pd.DataFrame(results).T.sort_values('p_value')

Pattern 3: Data Quality Assessment Framework

def assess_data_quality(df):
    """Comprehensive data quality assessment."""
    quality_report = {}
    
    # Completeness
    quality_report['completeness'] = {
        'total_cells': df.size,
        'missing_cells': df.isnull().sum().sum(),
        'missing_pct': df.isnull().mean().mean() * 100,
        'columns_with_missing': (df.isnull().sum() > 0).sum(),
        'rows_with_missing': (df.isnull().sum(axis=1) > 0).sum()
    }
    
    # Validity
    quality_report['validity'] = {}
    for col in df.select_dtypes(include=[np.number]).columns:
        # Check for impossible values
        invalid = ((df[col] < 0) & (col not in ['temperature', 'price_change'])).sum()
        if invalid > 0:
            quality_report['validity'][col] = f'{invalid} negative values'
    
    # Consistency
    quality_report['consistency'] = {}
    for col1 in df.select_dtypes(include=[np.number]).columns:
        for col2 in df.select_dtypes(include=[np.number]).columns:
            if col1 < col2:
                corr = df[[col1, col2]].corr().iloc[, ]
                 (corr) > :
                    quality_report[][] = \
                        
    
    
    quality_report[] = {
        : df.duplicated().(),
        : df.duplicated().mean() * 
    }
    
    
    date_cols = df.select_dtypes(include=[]).columns
     (date_cols) > :
        freshness = (pd.Timestamp.now() - df[date_cols[]].()).days
        quality_report[] = {: freshness}
    
     quality_report