메뉴
公式リーダーボード準拠のA/B/Cテストを実行するPlanモードスキル。Phi-3.5-mini-instruct、Borea-phi3.5-instinct-jp、AEGIS-Phi3.5mini-jpv2.4を標準化ベンチマークで比較評価し、統計的有意性を検証。
Codex 또는 Claude로 설치 이 Prompt를 복사해 Codex, Claude 또는 다른 어시스턴트에 붙여 넣으면 Skill 페이지를 검토하고 설치를 진행할 수 있습니다.
SOC 직업 분류 기준
| name | plan-mode-official-leaderboard-abctest |
| description | 公式リーダーボード準拠のA/B/Cテストを実行するPlanモードスキル。Phi-3.5-mini-instruct、Borea-phi3.5-instinct-jp、AEGIS-Phi3.5mini-jpv2.4を標準化ベンチマークで比較評価し、統計的有意性を検証。 |
| metadata | {"short-description":"公式準拠A/B/CテストPlanモード","version":"1.0.0","author":"SO8T Assistant","capabilities":["official_benchmark_evaluation","statistical_ab_testing","model_comparison_analysis","leaderboard_compliance"]} |
SO8Tプロジェクト専用に設計された公式リーダーボード準拠のA/B/Cテスト実行Planモード。Phi-3.5-mini-instruct、Borea-phi3.5-instinct-jp、AEGIS-Phi3.5mini-jp v2.4の3モデルをGSM8K/MATH/ARC-Challengeで標準化評価し、統計的有意性を検証します。
from skills.plan_mode_official_leaderboard_abctest import OfficialABCTestPlan
# 3モデル公式準拠比較テスト
abc_test = OfficialABCTestPlan()
test_config = {
"models": {
"Phi-3.5-mini-instruct": "microsoft/Phi-3.5-mini-instruct",
"Borea-phi3.5-instinct-jp": "path/to/borea/model",
"AEGIS-Phi3.5mini-jp-v2.4": "your-username/AEGIS-Phi3.5mini-jp"
},
"benchmarks": ["gsm8k", "math", "arc_challenge"],
"sample_sizes": {"gsm8k": 1000, "math": 500, "arc_challenge": 1000},
"runs_per_model": 3, # 統計的安定性のための複数回実行
"statistical_tests": ["t_test", "cohen_d", "confidence_intervals"],
"significance_level": 0.05
}
# 公式準拠A/B/Cテスト実行
results = abc_test.execute_official_abctest(test_config)
print(f"A/B/C Test completed: {results['summary']['winner']}")
# 詳細な統計分析を含む評価
statistical_analysis = abc_test.perform_statistical_analysis(results)
print("=== STATISTICAL SIGNIFICANCE RESULTS ===")
for benchmark in results['benchmarks']:
print(f"{benchmark.upper()}:")
for comparison in results['comparisons'][benchmark]:
print(f" {comparison['model_a']} vs {comparison['model_b']}:")
print(f" t-statistic: {comparison['t_statistic']:.3f}")
print(f" p-value: {comparison['p_value']:.4f}")
print(f" Cohen's d: {comparison['cohen_d']:.3f}")
print(f" Significant: {'Yes' if comparison['significant'] else 'No'}")
# 比較チャートとレポート生成
visualization = abc_test.generate_comparison_visualization(results)
report = abc_test.generate_official_report(results)
# 保存
abc_test.save_results(results, "official_abctest_results.json")
abc_test.save_visualizations(visualization, "charts/")
abc_test.save_report(report, "reports/official_abctest_report.pdf")
# モデル読み込みと基本検証
models_ready = abc_test.verify_models_availability(test_config['models'])
benchmarks_ready = abc_test.verify_benchmarks_availability(test_config['benchmarks'])
if not (models_ready and benchmarks_ready):
raise ValueError("Models or benchmarks not available")
# リソース割り当て最適化
resource_plan = abc_test.optimize_resource_allocation(test_config)
print(f"GPU allocation: {resource_plan['gpu_allocation']}")
print(f"Estimated time: {resource_plan['estimated_hours']} hours")
# 複数GPUでの並行評価
evaluation_jobs = abc_test.create_evaluation_jobs(test_config)
results = abc_test.execute_parallel_evaluations(evaluation_jobs)
# 進捗監視
with tqdm(total=len(evaluation_jobs), desc="A/B/C Test Progress") as pbar:
for completed_job in abc_test.monitor_evaluation_progress():
pbar.update(1)
print(f"Completed: {completed_job['model']} on {completed_job['benchmark']}")
# 統計的有意性検定
statistical_tests = abc_test.perform_statistical_tests(results, test_config['significance_level'])
# 効果サイズ計算
effect_sizes = abc_test.calculate_effect_sizes(results)
# 信頼区間推定
confidence_intervals = abc_test.calculate_confidence_intervals(results, confidence_level=0.95)
# 多重比較補正
corrected_p_values = abc_test.apply_multiple_comparison_correction(statistical_tests)
# 結果集約
final_results = abc_test.aggregate_results(results, statistical_tests, effect_sizes)
# 勝者決定（統計的有意性ベース）
winner_analysis = abc_test.determine_winner(final_results, test_config)
# 詳細レポート生成
comprehensive_report = abc_test.generate_comprehensive_report(final_results, winner_analysis)
class ABCTestDesign:
def __init__(self, models, benchmarks, significance_level=0.05):
self.models = models # 3つのモデル
self.benchmarks = benchmarks
self.alpha = significance_level
self.adjusted_alpha = self.alpha / 3 # Bonferroni補正
def calculate_required_sample_size(self, effect_size, power=0.8):
"""必要なサンプルサイズ計算"""
# Cohen's dに基づくpower analysis
return self.power_analysis_sample_size(effect_size, power)
def create_comparison_matrix(self):
"""3モデル間の全比較ペア生成"""
return [
(model_a, model_b)
for i, model_a in enumerate(self.models[:-1])
for model_b in self.models[i+1:]
]
class StatisticalTester:
def perform_pairwise_t_tests(self, results_a, results_b):
"""対応のないt-test実行"""
from scipy.stats import ttest_ind
t_stat, p_value = ttest_ind(results_a, results_b, equal_var=False)
# Cohen's d効果サイズ
mean_diff = np.mean(results_a) - np.mean(results_b)
pooled_std = np.sqrt((np.std(results_a)**2 + np.std(results_b)**2) / 2)
cohen_d = mean_diff / pooled_std if pooled_std > 0 else 0
return {
't_statistic': t_stat,
'p_value': p_value,
'cohen_d': cohen_d,
'significant': p_value < self.adjusted_alpha,
'effect_size_interpretation': self.interpret_cohen_d(cohen_d)
}
def interpret_cohen_d(self, d):
"""Cohen's dの解釈"""
abs_d = abs(d)
if abs_d < 0.2:
return "negligible"
elif abs_d < 0.5:
return "small"
elif abs_d < 0.8:
return "medium"
else:
return "large"
class ConfidenceIntervalCalculator:
def calculate_bootstrap_ci(self, data, n_bootstrap=1000, ci_level=0.95):
"""ブートストラップ法による信頼区間"""
bootstrap_means = []
n = len(data)
for _ in range(n_bootstrap):
sample = np.random.choice(data, size=n, replace=True)
bootstrap_means.append(np.mean(sample))
lower_percentile = (1 - ci_level) / 2 * 100
upper_percentile = (1 + ci_level) / 2 * 100
return {
'mean': np.mean(data),
'ci_lower': np.percentile(bootstrap_means, lower_percentile),
'ci_upper': np.percentile(bootstrap_means, upper_percentile),
'ci_level': ci_level
}
OfficialABCTestResults = {
'metadata': {
'timestamp': '2026-01-17 03:00:00',
'models_tested': ['Phi-3.5-mini-instruct', 'Borea-phi3.5-instinct-jp', 'AEGIS-Phi3.5mini-jp-v2.4'],
'benchmarks': ['gsm8k', 'math', 'arc_challenge'],
'sample_sizes': {'gsm8k': 1000, 'math': 500, 'arc_challenge': 1000},
'runs_per_model': 3,
'evaluation_protocols': {
'gsm8k': '8-shot CoT',
'math': '0-shot CoT',
'arc_challenge': '10-shot'
}
},
'raw_results': {
'model_name': {
'benchmark_name': {
'run_1': {'accuracy': 0.xxx, 'individual_scores': [...]},
'run_2': {...},
'run_3': {...},
'aggregate': {'mean': 0.xxx, 'std': 0.xxx, '95ci': [0.xxx, 0.xxx]}
}
}
},
'statistical_analysis': {
'pairwise_comparisons': [
{
'model_a': 'AEGIS-Phi3.5mini-jp-v2.4',
'model_b': 'Phi-3.5-mini-instruct',
'benchmark': 'gsm8k',
't_statistic': 2.345,
'p_value': 0.023,
'cohen_d': 0.678,
'significant': True,
'effect_size': 'medium',
'confidence_intervals': {'model_a': [0.85, 0.87], 'model_b': [0.83, 0.85]}
}
],
'overall_rankings': {
'gsm8k': ['AEGIS-Phi3.5mini-jp-v2.4', 'Borea-phi3.5-instinct-jp', 'Phi-3.5-mini-instruct'],
'math': ['AEGIS-Phi3.5mini-jp-v2.4', 'Phi-3.5-mini-instruct', 'Borea-phi3.5-instinct-jp'],
'arc_challenge': ['Phi-3.5-mini-instruct', 'AEGIS-Phi3.5mini-jp-v2.4', 'Borea-phi3.5-instinct-jp']
}
},
'summary': {
'winner_by_benchmark': {
'gsm8k': 'AEGIS-Phi3.5mini-jp-v2.4',
'math': 'AEGIS-Phi3.5mini-jp-v2.4',
'arc_challenge': 'Phi-3.5-mini-instruct'
},
'overall_winner': 'AEGIS-Phi3.5mini-jp-v2.4',
'confidence_level': 'high',
'key_findings': [
'AEGIS shows significant improvement in GSM8K and MATH',
'Phi-3.5-mini-instruct performs best on ARC-Challenge',
'All differences are statistically significant (p < 0.05)'
]
}
}
# 公式準拠A/B/Cテスト実行
python scripts/plan_mode_official_abctest.py \
--models-config scripts/evaluation/models_config.json \
--benchmarks gsm8k math arc_challenge \
--sample-sizes "gsm8k:1000,math:500,arc_challenge:1000" \
--runs-per-model 3 \
--significance-level 0.05 \
--output-dir evaluation_results/official_abctest/
# 統計分析とレポート生成
python scripts/analyze_abctest_results.py \
--results-dir evaluation_results/official_abctest/ \
--generate-plots \
--create-pdf-report \
--significance-analysis
# 結果比較チャート生成
python scripts/visualize_abctest_comparison.py \
--results-file evaluation_results/official_abctest/final_results.json \
--output-dir charts/abctest_comparison/ \
--chart-types "bar,ranking,effect_size,confidence_intervals"
テスト対象モデル: 3モデル (Phi-3.5-mini-instruct, Borea-phi3.5-instinct-jp, AEGIS-Phi3.5mini-jp-v2.4)
評価ベンチマーク: 3種類 (GSM8K, MATH, ARC-Challenge)
統計的厳密性: 公式リーダーボード準拠 + 統計的有意性検証
このPlanモードは、SO8Tプロジェクトにおける公式準拠のモデル比較評価を完全に自動化し、統計的に信頼できるA/B/Cテスト結果を提供します。
Implement comprehensive mathematical theorem proving capabilities with SFT+GRPO training, MCP/A2A agent integration, and imatrix quantization protection to surpass Boreas-phi3.5-instinct-jp in formal proof generation and scientific discovery. Use when building mathematical reasoning systems, formal verification tools, or AI-assisted theorem proving environments.
Execute GGUF quantization with imatrix protection, perform statistical benchmark evaluation with error bars, generate academic-style methodology documentation, and create comprehensive scorecards. Use when evaluating model quantization quality, comparing quantization methods, or generating publication-ready evaluation results with subagent execution and PowerShell progress visualization.
Advanced multimodal 'thinking' model pipeline with SO8T Grand Design, stability-constrained evolutionary training (EvoFreeze), and specialized OSINT/Military/Bio data collection.
Implement comprehensive mathematical theorem proving capabilities with SFT+GRPO training, MCP/A2A agent integration, and imatrix quantization protection to surpass Boreas-phi3.5-instinct-jp in formal proof generation and scientific discovery. Use when building mathematical reasoning systems, formal verification tools, or AI-assisted theorem proving environments.
Execute GGUF quantization with imatrix protection, perform statistical benchmark evaluation with error bars, generate academic-style methodology documentation, and create comprehensive scorecards. Use when evaluating model quantization quality, comparing quantization methods, or generating publication-ready evaluation results with subagent execution and PowerShell progress visualization.
Create, manage, and update multi-step execution plans for complex tasks in Cursor. Use when the user asks for a plan/roadmap/phases/checklist, or when work has multiple dependent steps, higher risk, or requires coordination. Skip for straightforward tasks. Integrates with Cursor's todo_write tool for plan tracking.