Menu
Quality assurance and testing protocols for statistical software
Install with Codex or Claude Copy this prompt, paste it into Codex, Claude, or another assistant, and let it review the skill page and install it for you.
Based on SOC occupation classification
| name | statistical-software-qa |
| description | Quality assurance and testing protocols for statistical software |
Quality assurance patterns and testing strategies for statistical R packages
Use this skill when working on: R package testing, numerical accuracy validation, reference implementation comparison, edge case identification, statistical correctness verification, or software quality assurance for methodology packages.
Unlike typical software where "correct output" is clear, statistical software must:
▲
/│\
/ │ \
/ │ \
/Manual│\ <- Human review of outputs
/ Tests │ \
/─────────────\
/ Integration \ <- Cross-function workflows
/ Tests \
/───────────────────\
/ Statistical Tests \ <- Correctness verification
/ \
/─────────────────────────\
/ Reference Tests \ <- Match known results
/ \
/───────────────────────────────\
/ Unit Tests \ <- Individual functions
/___________________________________\
Test against analytically derivable results:
test_that("indirect effect equals a*b for simple mediation", {
# Create data where we know true values
set.seed(42)
n <- 10000
x <- rnorm(n)
m <- 0.5 * x + rnorm(n, sd = 0.1) # a = 0.5
y <- 0.3 * m + rnorm(n, sd = 0.1) # b = 0.3
result <- mediate(y ~ x + m, mediator = "m", data = data.frame(x, m, y))
# True indirect = 0.5 * 0.3 = 0.15
expect_equal(result$indirect, 0.15, tolerance = 0.02)
})
test_that("handles minimum sample size", {
# Minimum viable sample
small_data <- data.frame(
x = c(0, 0, 1, 1),
m = c(0, 1, 1, 2),
y = c(1, 2, 2, 3)
)
# Should work without error
expect_no_error(mediate(y ~ x + m, mediator = "m", data = small_data))
# Should warn about low power
expect_warning(
mediate(y ~ x + m, mediator = "m", data = small_data),
"sample size"
)
})
test_that("bootstrap CI contains delta method CI asymptotically", {
set.seed(123)
data <- simulate_mediation(n = 5000, a = 0.3, b = 0.4)
boot_result <- mediate(data, method = "bootstrap", R = 2000)
delta_result <- mediate(data, method = "delta")
# CIs should be similar for large n
expect_equal(boot_result$ci, delta_result$ci, tolerance = 0.05)
})
test_that("matches Imai et al. (2010) JOBS II example",
# Load reference data from mediation package
data("jobs", package = "mediation")
# Our implementation
our_result <- our_mediate(
outcome = job_seek ~ treat + econ_hard + sex + age,
mediator = job_disc ~ treat + econ_hard + sex + age,
data = jobs
)
# Published results (from paper Table 2)
expected_acme <- 0.015
expected_acme_ci <- c(-0.004, 0.035)
expect_equal(our_result$acme, expected_acme, tolerance = 0.005)
expect_equal(our_result$acme_ci, expected_acme_ci, tolerance = 0.01)
})
test_that("matches lavaan for SEM-based mediation", {
data <- simulate_mediation(n = 1000)
# Our implementation
our_result <- our_mediate(data)
# lavaan implementation
library(lavaan)
model <- '
m ~ a*x
y ~ b*m + c*x
indirect := a*b
'
lavaan_fit <- sem(model, data = data)
lavaan_indirect <- parameterEstimates(lavaan_fit)[
parameterEstimates(lavaan_fit)$label == "indirect", "est"
]
expect_equal(our_result$indirect, lavaan_indirect, tolerance = 0.01)
})
Verify confidence intervals achieve nominal coverage:
test_that("95% CI achieves nominal coverage", {
set.seed(42)
n_sims <- 1000
true_indirect <- 0.15
coverage <- 0
for (i in 1:n_sims) {
data <- simulate_mediation(n = 200, a = 0.5, b = 0.3)
result <- mediate(data, conf.level = 0.95)
if (result$ci[1] <= true_indirect && true_indirect <= result$ci[2]) {
coverage <- coverage + 1
}
}
coverage_rate <- coverage / n_sims
# Coverage should be between 93% and 97% (accounting for MC error)
expect_gte(coverage_rate, 0.93)
expect_lte(coverage_rate, 0.97)
})
test_that("estimator is approximately unbiased", {
set.seed(123)
n_sims <- 500
true_indirect <- 0.2
estimates <- numeric(n_sims)
for (i in 1:n_sims) {
data <- simulate_mediation(n = 500, a = 0.5, b = 0.4)
estimates[i] <- mediate(data)$indirect
}
# Mean should be close to true value
bias <- mean(estimates) - true_indirect
expect_lt(abs(bias), 0.02) # Less than 2% bias
})
test_that("maintains nominal Type I error under null", {
set.seed(456)
n_sims <- 1000
rejections <- 0
for (i in 1:n_sims) {
# Null: no indirect effect (a = 0)
data <- simulate_mediation(n = 200, a = 0, b = 0.5)
result <- mediate(data, conf.level = 0.95)
# Reject if CI excludes 0
if (result$ci[1] > 0 || result$ci[2] < 0) {
rejections <- rejections + 1
}
}
type1_rate <- rejections / n_sims
# Should be close to 5%
expect_lt(type1_rate, 0.07) # Allow some MC error
})
test_that("results invariant to variable scaling", {
data <- simulate_mediation(n = 500)
# Original scale
result1 <- mediate(data)
# Scale variables by 1000
data_scaled <- data
data_scaled$y <- data$y * 1000
data_scaled$m <- data$m * 1000
result2 <- mediate(data_scaled)
# Standardized effects should match
expect_equal(
result1$indirect / (sd(data$y)),
result2$indirect / (sd(data_scaled$y)),
tolerance = 1e-10
)
})
test_that("handles extreme correlations", {
set.seed(789)
# Nearly collinear
x <- rnorm(100)
m <- x + rnorm(100, sd = 0.001) # r ≈ 0.9999
y <- m + rnorm(100, sd = 0.1)
data <- data.frame(x, m, y)
# Should warn about collinearity
expect_warning(mediate(data), "collinear|singular")
})
test_that("handles near-zero variance", {
data <- data.frame(
x = c(rep(0, 99), 1), # Almost constant
m = rnorm(100),
y = rnorm(100)
)
# Should handle gracefully
expect_error(
mediate(data),
"variance|constant"
)
})
#' Generate Edge Case Test Suite
#'
#' @return List of edge case datasets
generate_edge_cases <- function() {
list(
# Minimal sample
minimal = data.frame(
x = c(0, 1, 0, 1),
m = c(0, 0.5, 0.5, 1),
y = c(0, 0.3, 0.3, 0.6)
),
# Zero effect
null_effect = {
set.seed(1)
n <- 100
data.frame(
x = rnorm(n),
m = rnorm(n), # No relationship with x
y = rnorm(n) # No relationship with m
)
},
# Perfect mediation
perfect = {
x <- c(0, 0, 1, 1, 2, 2)
m <- x # Perfect a path
y <- m # Perfect b path
data.frame(x, m, y)
},
# High collinearity
collinear = {
set.seed(2)
x <- rnorm(100)
m <- x + rnorm(100, sd = 0.01)
y <- m + rnorm(100)
data.frame(x, m, y)
},
# Outliers
with_outliers = {
set.seed(3)
n <- 100
x <- c(rnorm(n-2), 10, -10)
m <- 0.5 * x + c(rnorm(n-2), 20, -20)
y <- 0.3 * m + rnorm(n)
data.frame(x, m, y)
},
# Missing data
with_missing = {
set.seed(4)
n <- 100
x <- rnorm(n)
m <- 0.5 * x + rnorm(n)
y <- 0.3 * m + rnorm(n)
# Introduce 10% missing
m[sample(n, 10)] <- NA
y[sample(n, 10)] <- NA
data.frame(x, m, y)
}
)
}
# Run all edge cases
test_that("handles all edge cases", {
edge_cases <- generate_edge_cases()
for (name in names(edge_cases)) {
expect_no_error(
tryCatch(
mediate(edge_cases[[name]]),
warning = function(w) NULL # Warnings OK
),
info = paste("Edge case:", name)
)
}
})
test_that("meets performance requirements", {
skip_on_cran() # Skip during CRAN checks
data <- simulate_mediation(n = 1000)
# Point estimate should be fast
time_point <- system.time({
mediate(data, method = "delta")
})["elapsed"]
expect_lt(time_point, 0.1) # < 100ms
# Bootstrap should be reasonable
time_boot <- system.time({
mediate(data, method = "bootstrap", R = 1000)
})["elapsed"]
expect_lt(time_boot, 10) # < 10s for 1000 bootstraps
})
test_that("memory usage is reasonable", {
skip_on_cran()
# Large dataset
data <- simulate_mediation(n = 100000)
mem_before <- pryr::mem_used()
result <- mediate(data)
mem_after <- pryr::mem_used()
mem_increase <- as.numeric(mem_after - mem_before) / 1e6 # MB
expect_lt(mem_increase, 100) # Less than 100MB increase
})
tests/
├── testthat/
│ ├── test-mediate.R # Main function tests
│ ├── test-mediate-bootstrap.R # Bootstrap-specific
│ ├── test-mediate-delta.R # Delta method-specific
│ ├── test-numerical.R # Numerical precision
│ ├── test-edge-cases.R # Edge cases
│ ├── test-reference.R # Reference implementation
│ ├── test-coverage.R # Statistical coverage
│ └── helper-simulate.R # Test helpers
├── testthat.R
└── reference-results/ # Saved reference results
├── jobs-example.rds
└── known-values.rds
test_that("coverage probability (slow)", {
skip_on_cran()
skip_if_not(Sys.getenv("RUN_SLOW_TESTS") == "true")
# ... slow coverage test ...
})
# .github/workflows/R-CMD-check.yaml
name: R-CMD-check
on: [push, pull_request]
jobs:
R-CMD-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: r-lib/actions/setup-r@v2
- uses: r-lib/actions/setup-r-dependencies@v2
- uses: r-lib/actions/check-r-package@v2
coverage:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: r-lib/actions/setup-r@v2
- uses: r-lib/actions/setup-r-dependencies@v2
- name: Test coverage
run: covr::codecov()
shell: Rscript {0}
slow-tests:
runs-on: ubuntu-latest
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
steps:
- uses: actions/checkout@v3
- uses: r-lib/actions/setup-r@v2
- uses: r-lib/actions/setup-r-dependencies@v2
- name: Run slow tests
run: |
Sys.setenv(RUN_SLOW_TESTS = "true")
testthat::test_local()
shell: Rscript {0}
## Statistical Validation Report
**Package**: [name] v[version]
**Date**: [date]
**Validator**: [name]
### Coverage Probability (95% CI)
| Scenario | N | True Effect | Observed Coverage | Pass |
|----------|---|-------------|-------------------|------|
| Small sample | 50 | 0.15 | 94.2% | Yes |
| Medium sample | 200 | 0.15 | 95.1% | Yes |
| Large sample | 1000 | 0.15 | 94.8% | Yes |
| Null effect | 200 | 0 | 95.3% | Yes |
### Bias Assessment
| Scenario | True | Mean Estimate | Bias | Pass |
|----------|------|---------------|------|------|
| a=0.5, b=0.3 | 0.15 | 0.151 | 0.001 | Yes |
| a=0.1, b=0.1 | 0.01 | 0.012 | 0.002 | Yes |
### Reference Comparison
| Reference | Our Result | Difference | Pass |
|-----------|------------|------------|------|
| Imai et al. Table 2 | Match | <0.001 | Yes |
| lavaan output | Match | <0.001 | Yes |
Version: 1.0.0 Created: 2025-12-09 Domain: Quality assurance for statistical R packages Target: MediationVerse ecosystem and similar methodology packages
Route empirical-research requests through the Auto-Empirical Research Skills catalog when this whole repository is installed as one skill in Codex, CodeBuddy, Claude Code, or another IDE. Use to choose and load the right vendored AERS skill for causal inference, econometrics, replication, data acquisition, manuscript writing, peer review and referee responses, citation checking, de-AIGC editing, or full empirical-paper workflows without reading the entire repository at once.
公司金融实证研究的"漏斗式选题查找器"。互动开场先后询问 (1) 研究方向、(2) 候选标题数量 N， 再扫描全球文献（已出版英文学术期刊 + SSRN working paper + 全球高校 department seminar 1 年内日程），基于 Edmans (2024) "1000 Rejections" 红线生成 N 个候选标题，**通过并行 subagent（Agent 工具）批量生成计划书 + 查新；每个 subagent 必须强制调用 Skill 工具加载 econfin-proposal 与 novelty-check 两个预设 skill 完成各自模块**，**只有当 novelty score >= 9 时（即 JF/JFE/RFS 顶刊层次），subagent 才把 proposal + 查新报告合并的 md 写入 F:\Dropbox\CC\选题大全\<研究方向短名>\（以"简短选题名称-分数"命名，子文件夹名由 Step 0 从用户输入的研究方向派生）；< 9 分的选题在 subagent 内部直接丢弃，绝不写盘、绝不输出**。当用户说"找选题"、"帮我找选题"、"想做 X 方向"、 "empirical CF idea search"、"批量生成研究计划书"、"100 ideas"、"econfin-idea-finder" 时触发。
Create and compile beautiful Beamer presentations following the Rhetoric of Decks philosophy. Use when making slides, creating decks, or compiling .tex presentation files.
Scaffold a new research project with standard directory structure, CLAUDE.md template, and documented README. Use this at the start of every new project to ensure consistent organization.
Download, split, and deeply read academic PDFs. Use when asked to read, review, or summarize an academic paper. Splits PDFs into 4-page chunks, reads them in small batches, and produces structured reading notes — avoiding context window crashes and shallow comprehension.
This skill should be used when the user asks to "create a slash command", "add a command", "write a custom command", "define command arguments", "use command frontmatter", "organize commands", "create command with file references", "interactive command", "use AskUserQuestion in command", or needs guidance on slash command structure, YAML frontmatter fields, dynamic arguments, bash execution in commands, user interaction patterns, or command development best practices for Claude Code.