experimental-design

name	experimental-design
description	Design rigorous experiments: sample size calculation, randomization strategies, pre-registration, and AB test duration estimation.
tags	["experimental-design","sample-size","randomization","pre-registration","ab-testing","power-analysis"]
version	1.0.0
authors	[{"name":"awesome-rosetta-skills contributors","github":"@awesome-rosetta-skills"}]
license	MIT
platforms	["claude-code","codex","gemini-cli","cursor"]
dependencies	{"python":["statsmodels>=0.14.0","numpy>=1.24.0","pandas>=2.0.0","scipy>=1.10.0","osfclient>=0.0.5"]}
last_updated	2026-03-17

Experimental Design

Plan experiments with correct statistical power, principled randomization, and transparent pre-registration before data collection begins.

---

Key Concepts

Term	Definition
α (alpha)	Type I error rate — false positive probability (typically 0.05)
β (beta)	Type II error rate — false negative probability (typically 0.20)
Power (1−β)	Probability of detecting a true effect (typically 0.80 or 0.90)
MDES	Minimum Detectable Effect Size — smallest effect your design can detect
ICC	Intra-cluster correlation — needed for cluster-randomised trials

---

Setup

pip install statsmodels numpy pandas scipy osfclient

---

Core Implementation

"""
experimental_design.py
Sample size, randomization, and pre-registration utilities.
"""

import hashlib
import math
import os
import warnings
from typing import Dict, List, Optional, Tuple, Union

import numpy as np
import pandas as pd
from scipy import stats
from statsmodels.stats.power import (
    TTestIndPower,
    TTestPower,
    FTestAnovaPower,
    NormalIndPower,
)
from statsmodels.stats.proportion import proportion_effectsize


# ─────────────────────────────────────────────
# 1. Sample Size Calculation
# ─────────────────────────────────────────────

def calculate_sample_size() -> [, ]:
    
    alternative =   two_tailed  

     p1     p2   :
        effect_size = (proportion_effectsize(p1, p2))
        test_type = 

     effect_size  :
         ValueError()

     test_type == :
        analysis = TTestIndPower()
        n = analysis.solve_power(effect_size=effect_size, alpha=alpha, power=power,
                                  ratio=ratio, alternative=alternative)
     test_type == :
        analysis = TTestPower()
        n = analysis.solve_power(effect_size=effect_size, alpha=alpha, power=power,
                                  alternative=alternative)
        ratio = 
     test_type == :
        analysis = TTestPower()
        n = analysis.solve_power(effect_size=effect_size, alpha=alpha, power=power,
                                  alternative=alternative)
        ratio = 
     test_type == :
        
        analysis = FTestAnovaPower()
        n = analysis.solve_power(effect_size=effect_size, alpha=alpha, power=power,
                                  k_groups=k_groups)
     test_type  (, ):
        analysis = NormalIndPower()
        n = analysis.solve_power(effect_size=effect_size, alpha=alpha, power=power,
                                  ratio=ratio, alternative=alternative)
    :
         ValueError()

    n_ceil = math.ceil(n)
    total_n = math.ceil(n_ceil * ( + ratio))  test_type   (, )  n_ceil

    result = {
        : n_ceil,
        : total_n,
        : (effect_size, ),
        : alpha,
        : power,
        : test_type,
        : ratio,
    }

     verbose:
        ()
        ()
        ()
        ()
        ()
        ()
         p1   :
            ()
        ()

     result


 () -> :
    
    analysis = TTestIndPower()  test_type ==   TTestPower()
    es = analysis.solve_power(nobs1=n_per_group, alpha=alpha, power=power,
                               ratio=, alternative=)
     (es, )


 () -> :
    
       <= expected_attrition_rate < :
         ValueError()
     math.ceil(n / ( - expected_attrition_rate))






 () -> pd.DataFrame:
    
    rng = np.random.default_rng(seed)
     group_labels  :
        group_labels = [  i  (n_groups)]
    assignments = rng.choice(group_labels, size=n)
    df = pd.DataFrame({: (, n + ), : assignments})
     df


 () -> pd.DataFrame:
    
     block_size % n_groups != :
         ValueError()

    rng = np.random.default_rng(seed)
     group_labels  :
        group_labels = [  i  (n_groups)]

    per_arm = block_size // n_groups
    block_template = group_labels * per_arm

    assignments = []
     (assignments) < n:
        block = block_template.copy()
        rng.shuffle(block)
        assignments.extend(block)

    assignments = assignments[:n]
    df = pd.DataFrame({: (, n + ), : assignments})
     df


 () -> pd.DataFrame:
    
     group_labels  :
        group_labels = [  i  (n_groups)]

    result_frames = []
    
    df = participants.copy().reset_index(drop=)
    df[] = df[stratify_cols].astype().agg(.join, axis=)

     stratum, group_df  df.groupby():
        n_stratum = (group_df)
        assigned = block_randomize(
            n=n_stratum,
            block_size=block_size,
            n_groups=n_groups,
            group_labels=group_labels,
            seed=((stratum)) % (**)  seed    seed + ((stratum)) % ,
        )
        group_df = group_df.reset_index(drop=)
        group_df[] = assigned[].values
        result_frames.append(group_df)

    result = pd.concat(result_frames).drop(columns=[]).sort_index()
     result


 () -> pd.DataFrame:
    
    rng = np.random.default_rng(seed)
     group_labels  :
        group_labels = [  i  (n_groups)]

    shuffled_clusters = (clusters)
    rng.shuffle(shuffled_clusters)

    assignments = []
     i, cluster  (shuffled_clusters):
        assignments.append({: cluster, : group_labels[i % n_groups]})

     pd.DataFrame(assignments)






 () -> [, ]:
    
    p1 = baseline_rate
    p2 = baseline_rate * ( + minimum_detectable_effect)
    p2 = (p2, )

    size_result = calculate_sample_size(
        p1=p1, p2=p2, alpha=alpha / (n_variants - ),  
        power=power, verbose=
    )
    required_n = size_result[] * n_variants

    daily_n = (daily_traffic * allocation_fraction / n_variants) * n_variants
    days_needed = math.ceil(required_n / daily_n)

     {
        : required_n,
        : daily_n,
        : days_needed,
        : p1,
        : (p2, ),
        : (alpha / (n_variants - ), ),
    }






 () -> :
    
    lines = [
        ,
        ,
        ,
        ,
        ,
        ,
    ]
     i, h  (hypotheses, ):
        lines.append()
    lines += [
        ,
        ,
        ,
        primary_outcome,
        ,
        ,
        ,
        sample_size_justification,
        ,
        ,
        ,
        randomization_method,
        ,
        ,
        ,
        blinding,
        ,
        ,
        ,
        analysis_plan,
        ,
        ,
        ,
    ]
    content = .join(lines)
     (output_path, , encoding=)  fh:
        fh.write(content)
    ()
     content