data-science-expert

name	data-science-expert
version	1.0.0
description	Expert-level data science, analytics, visualization, and statistical modeling
category	ai
tags	["data-science","analytics","visualization","statistics","pandas","numpy"]
allowed-tools	["Read","Write","Edit","Bash(python:*)"]

Data Science Expert

Expert guidance for data science, analytics, statistical modeling, and data visualization.

Core Concepts

Data Analysis

• Exploratory Data Analysis (EDA)
• Data cleaning and preprocessing
• Feature engineering
• Statistical inference
• Time series analysis
• A/B testing

Machine Learning

• Supervised learning (classification, regression)
• Unsupervised learning (clustering, PCA)
• Model selection and validation
• Feature importance
• Hyperparameter tuning
• Ensemble methods

Data Visualization

• Matplotlib, Seaborn, Plotly
• Statistical plots
• Interactive dashboards
• Storytelling with data
• Best practices for visualization
• Color theory and accessibility

Data Cleaning and EDA

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from typing import Dict, List

class DataCleaner:
    """Clean and preprocess data"""

    def __init__(self, df: pd.DataFrame):
        self.df = df.copy()
        self.cleaning_log = []

    def handle_missing_values(self, strategy: str = 'drop',
                             fill_value=None) -> pd.DataFrame:
        """Handle missing values"""
        missing_before = self.df.isnull().sum().sum()

        if strategy == 'drop':
            self.df = self.df.dropna()
        elif strategy == 'fill':
            if fill_value is not None:
                self.df = self.df.fillna(fill_value)
            else:
                # Fill numeric with median, categorical with mode
                for col in self.df.columns:
                     .df[col].dtype  [, ]:
                        .df[col].fillna(.df[col].median(), inplace=)
                    :
                        .df[col].fillna(.df[col].mode()[], inplace=)

        missing_after = .df.isnull().().()
        .cleaning_log.append()

         .df

     () -> pd.DataFrame:
        
        before = (.df)
        .df = .df.drop_duplicates()
        after = (.df)

        .cleaning_log.append()
         .df

     () -> pd.DataFrame:
        
        before = (.df)

         col  columns:
             method == :
                Q1 = .df[col].quantile()
                Q3 = .df[col].quantile()
                IQR = Q3 - Q1

                lower = Q1 - threshold * IQR
                upper = Q3 + threshold * IQR

                .df = .df[(.df[col] >= lower) & (.df[col] <= upper)]

             method == :
                z_scores = np.(stats.zscore(.df[col]))
                .df = .df[z_scores < threshold]

        after = (.df)
        .cleaning_log.append()

         .df

 :
    

     ():
        .df = df

     () -> pd.DataFrame:
        
         .df.describe(include=).T

     () -> pd.DataFrame:
        
        numeric_cols = .df.select_dtypes(include=[np.number]).columns
         .df[numeric_cols].corr(method=method)

     ():
        
         columns  :
            columns = .df.select_dtypes(include=[np.number]).columns

        n_cols = (columns)
        n_rows = (n_cols + ) // 

        fig, axes = plt.subplots(n_rows, , figsize=(, *n_rows))
        axes = axes.flatten()

         idx, col  (columns):
            sns.histplot(.df[col], kde=, ax=axes[idx])
            axes[idx].set_title()

        plt.tight_layout()
         fig

     ():
        
        corr = .correlation_analysis()

        plt.figure(figsize=(, ))
        sns.heatmap(corr, annot=, fmt=, cmap=,
                   center=, square=, linewidths=)
        plt.title()
         plt.gcf()

Feature Engineering

from sklearn.preprocessing import StandardScaler, LabelEncoder, OneHotEncoder
from sklearn.feature_selection import SelectKBest, f_classif, mutual_info_classif

class FeatureEngineer:
    """Engineer features for machine learning"""

    def __init__(self, df: pd.DataFrame):
        self.df = df.copy()
        self.transformers = {}

    def create_interaction_features(self, col1: str, col2: str) -> pd.Series:
        """Create interaction features"""
        self.df[f'{col1}_x_{col2}'] = self.df[col1] * self.df[col2]
        return self.df[f'{col1}_x_{col2}']

    def create_polynomial_features(self, col: str, degree: int = 2) -> pd.DataFrame:
        """Create polynomial features"""
        for d in range(2, degree + 1):
            self.df[f'{col}_pow_{d}'] = self.df[col] ** d
        return self.df

     () -> pd.Series:
        
        .df[] = pd.qcut(.df[col], q=n_bins,
                                           labels=, duplicates=)
         .df[]

     () -> pd.DataFrame:
        
         method == :
            le = LabelEncoder()
            .df[] = le.fit_transform(.df[col])
            .transformers[col] = le

         method == :
            dummies = pd.get_dummies(.df[col], prefix=col, drop_first=)
            .df = pd.concat([.df, dummies], axis=)

         .df

     () -> pd.DataFrame:
        
         method == :
            scaler = StandardScaler()
         method == :
             sklearn.preprocessing  MinMaxScaler
            scaler = MinMaxScaler()

        .df[columns] = scaler.fit_transform(.df[columns])
        .transformers[] = scaler

         .df

     () -> []:
        
         method == :
            scorer = f_classif
         method == :
            scorer = mutual_info_classif

        selector = SelectKBest(scorer, k=k)
        selector.fit(X, y)

        selected_features = X.columns[selector.get_support()].tolist()
         selected_features

Time Series Analysis

from statsmodels.tsa.seasonal import seasonal_decompose
from statsmodels.tsa.stattools import adfuller
from statsmodels.tsa.arima.model import ARIMA

class TimeSeriesAnalyzer:
    """Analyze time series data"""

    def __init__(self, data: pd.Series, freq: str = 'D'):
        self.data = data
        self.freq = freq

    def decompose(self, model: str = 'additive'):
        """Decompose time series"""
        result = seasonal_decompose(self.data, model=model, period=30)

        return {
            'trend': result.trend,
            'seasonal': result.seasonal,
            'residual': result.resid
        }

    def test_stationarity(self) -> dict:
        """Test for stationarity using Augmented Dickey-Fuller"""
        result = adfuller(self.data.dropna())

        return {
            'adf_statistic': result[0],
            'p_value': result[1],
            'critical_values': result[4],
            'is_stationary': result[1] < 0.05
        }

    def () -> pd.Series:
        
         method == :
             .data.diff().dropna()
         method == :
             np.log(.data)
         method == :
             np.log(.data).diff().dropna()

     ():
        
        model = ARIMA(.data, order=order)
        fitted_model = model.fit()

         {
            : fitted_model,
            : fitted_model.aic,
            : fitted_model.bic,
            : fitted_model.summary()
        }

     () -> pd.Series:
        
         model.forecast(steps=steps)

A/B Testing

from scipy import stats

class ABTest:
    """Conduct A/B tests"""

    def __init__(self, control: np.ndarray, treatment: np.ndarray):
        self.control = control
        self.treatment = treatment

    def ttest(self) -> dict:
        """Two-sample t-test"""
        statistic, p_value = stats.ttest_ind(self.control, self.treatment)

        # Calculate confidence interval for difference
        diff_mean = self.treatment.mean() - self.control.mean()
        se_diff = np.sqrt(self.control.var()/len(self.control) +
                         self.treatment.var()/len(self.treatment))
        ci_lower = diff_mean - 1.96 * se_diff
        ci_upper = diff_mean + 1.96 * se_diff

        return {
            't_statistic': statistic,
            'p_value': p_value,
            'mean_control': self.control.mean(),
            'mean_treatment': self.treatment.mean(),
            'difference': diff_mean,
            'ci_95': (ci_lower, ci_upper),
            'significant': p_value < 0.05
        }

    def proportion_test() -> :
        
        n_control = (.control)
        n_treatment = (.treatment)

        p_control = conversions_control / n_control
        p_treatment = conversions_treatment / n_treatment

        p_pooled = (conversions_control + conversions_treatment) / (n_control + n_treatment)

        se = np.sqrt(p_pooled * ( - p_pooled) * (/n_control + /n_treatment))
        z = (p_treatment - p_control) / se
        p_value =  * ( - stats.norm.cdf((z)))

         {
            : p_control,
            : p_treatment,
            : (p_treatment - p_control) / p_control * ,
            : z,
            : p_value,
            : p_value < 
        }

Best Practices

Data Analysis

• Always explore data before modeling
• Check data quality and missing values
• Understand variable distributions
• Look for correlations and relationships
• Document data cleaning steps
• Validate assumptions

Feature Engineering

• Create domain-specific features
• Test feature importance
• Avoid data leakage
• Use cross-validation for validation
• Document feature transformations
• Keep features interpretable

Visualization

• Choose appropriate plot types
• Use clear labels and titles
• Consider color accessibility
• Avoid chartjunk
• Tell a story with data
• Make visualizations reproducible

Anti-Patterns

❌ Not exploring data before modeling ❌ Ignoring data quality issues ❌ Data leakage in feature engineering ❌ Over-engineering features ❌ Misleading visualizations ❌ Not documenting analysis steps ❌ Ignoring business context

Resources

• Pandas: https://pandas.pydata.org/
• NumPy: https://numpy.org/
• Scikit-learn: https://scikit-learn.org/
• Seaborn: https://seaborn.pydata.org/
• Plotly: https://plotly.com/python/