Menu
Review calibration metrics, accuracy trends, and model confidence levels
Install with Codex or Claude Copy this prompt, paste it into Codex, Claude, or another assistant, and let it review the skill page and install it for you.
Based on SOC occupation classification
Deep analysis of a specific segment including roles, topics, and past performance
Display high-level summary of audience segments, sizes, topics, and activity
Score 3 content variations and rank them with comparative analysis
Core skill for scoring a draft post with per-segment predictions and reasoning
| name | prediction-accuracy |
| description | Review calibration metrics, accuracy trends, and model confidence levels |
| version | 1.0.0 |
Purpose: Show how accurate predictions have become and identify where the model needs more data.
{
"min_predictions_for_calibration": "integer (optional, default 10)"
}
{
"overall_status": {
"total_scored_posts": 18,
"total_tracked_results": 4,
"overall_mae": 4.2,
"overall_rmse": 5.1,
"global_calibration": {
"slope": 0.98,
"intercept": 0.3
},
"confidence_level": "low",
"next_milestone": "Need 10 predictions per segment for medium confidence"
},
"confidence_by_segment": [
{
"segment": "Technical Leaders",
"predictions_tracked": 2,
"confidence": "low",
"mae": 3.1,
"rmse": 3.8,
"calibration_slope": 1.02,
"calibration_intercept": -0.5,
"interpretation": "Good performance so far, but need 8 more tracked predictions"
}
],
"topic_calibration_details": [
{
"topic": "AI/ML",
"predictions_tracked": 3,
"mae": 2.8,
"rmse": 3.1,
"segments_with_data": ["Technical Leaders", "Growth & Sales"],
"status": "good_confidence_for_tech"
}
],
"calibration_matrix": {
"Technical Leaders": {
"AI/ML": {
"count": 2,
"mae": 3.1,
"status": "need_8_more"
}
}
},
"accuracy_trends": {
"posts_1_to_5": {
"mae": 5.2,
"improvement": "baseline"
},
"posts_6_to_10": {
"mae": 4.1,
"improvement": "+21%"
}
},
"build_plan": [
{
"priority": "high",
"recommendation": "Post 5+ more about Leadership + Executives",
"rationale": "0 tracked predictions. High affinity topic but no calibration."
}
],
"recommendations": [
"You're on track. After 10-15 total posts with tracked results, medium confidence kicks in.",
"Post about: Leadership (0 data), Growth Hacking (0 data), Founder Stories (0 data)"
]
}
def get_prediction_accuracy(min_predictions=10):
"""Review prediction calibration and accuracy."""
conn = get_db_connection()
try:
cursor = conn.cursor()
# Step 1: Overall status
cursor.execute('SELECT COUNT(*) as count FROM predictions')
total_scored = cursor.fetchone()['count']
cursor.execute('SELECT COUNT(*) as count FROM predictions WHERE actual_reactions IS NOT NULL')
total_tracked = cursor.fetchone()['count']
# Calculate overall MAE/RMSE
cursor.execute('''
SELECT
AVG(ABS(predicted_reactions - actual_reactions)) as mae,
SQRT(AVG((predicted_reactions - actual_reactions) * (predicted_reactions - actual_reactions))) as rmse
FROM predictions
WHERE actual_reactions IS NOT NULL
''')
accuracy_row = cursor.fetchone()
overall_mae = accuracy_row['mae'] if accuracy_row['mae'] else 0.0
overall_rmse = accuracy_row['rmse'] if accuracy_row['rmse'] else 0.0
# Global calibration
cursor.execute('''
SELECT
AVG(calibration_slope) as avg_slope,
AVG(calibration_intercept) as avg_intercept
FROM prediction_accuracy
WHERE prediction_count >= ?
''', (min_predictions,))
calib_row = cursor.fetchone()
global_slope = calib_row['avg_slope'] if calib_row and calib_row['avg_slope'] else 1.0
global_intercept = calib_row['avg_intercept'] if calib_row and calib_row['avg_intercept'] else 0.0
# Overall confidence
avg_predictions = total_tracked / max(4, total_scored / 4) if total_scored > 0 else 0
if avg_predictions < min_predictions / 4:
confidence_level = 'low'
elif avg_predictions < min_predictions / 2:
confidence_level = 'medium'
else:
confidence_level = 'high'
overall_status = {
'total_scored_posts': total_scored,
'total_tracked_results': total_tracked,
'overall_mae': round(overall_mae, 1),
'overall_rmse': round(overall_rmse, 1),
'global_calibration': {
'slope': round(global_slope, 2),
'intercept': round(global_intercept, 2)
},
'confidence_level': confidence_level,
'next_milestone': f"Need {min_predictions - (total_tracked // 4)} more tracked predictions per segment"
}
# Step 2: Confidence by segment
cursor.execute('''
SELECT segment_name, prediction_count, mean_absolute_error, rmse,
calibration_slope, calibration_intercept
FROM prediction_accuracy
GROUP BY segment_name
ORDER BY prediction_count DESC
''')
confidence_by_segment = []
for row in cursor.fetchall():
if row['prediction_count'] == 0:
confidence = 'no_data'
elif row['prediction_count'] < min_predictions / 2:
confidence = 'low'
elif row['prediction_count'] < min_predictions:
confidence = 'medium'
else:
confidence = 'high'
interpretation = get_segment_interpretation(row['prediction_count'], min_predictions)
confidence_by_segment.append({
'segment': row['segment_name'],
'predictions_tracked': row['prediction_count'],
'confidence': confidence,
'mae': round(row['mean_absolute_error'] or 0.0, 1),
'rmse': round(row['rmse'] or 0.0, 1),
'calibration_slope': round(row['calibration_slope'] or 1.0, 2),
'calibration_intercept': round(row['calibration_intercept'] or 0.0, 2),
'interpretation': interpretation
})
# Step 3: Topic calibration details
cursor.execute('''
SELECT topic_name, prediction_count, mean_absolute_error, rmse
FROM prediction_accuracy
WHERE topic_name IS NOT NULL
ORDER BY prediction_count DESC
LIMIT 10
''')
topic_calibration = []
for row in cursor.fetchall():
segments_with_data = get_segments_for_topic(cursor, row['topic_name'])
status = 'need_data'
if row['prediction_count'] >= min_predictions:
status = 'high_confidence'
elif row['prediction_count'] >= min_predictions / 2:
status = 'medium_confidence'
elif row['prediction_count'] > 0:
status = 'building_confidence'
topic_calibration.append({
'topic': row['topic_name'],
'predictions_tracked': row['prediction_count'],
'mae': round(row['mean_absolute_error'] or 0.0, 1),
'rmse': round(row['rmse'] or 0.0, 1),
'segments_with_data': segments_with_data,
'status': status
})
# Step 4: Calibration matrix
cursor.execute('''
SELECT segment_name, topic_name, prediction_count, mean_absolute_error
FROM prediction_accuracy
WHERE segment_name IS NOT NULL AND topic_name IS NOT NULL
ORDER BY segment_name, prediction_count DESC
''')
calibration_matrix = {}
for row in cursor.fetchall():
seg = row['segment_name']
topic = row['topic_name']
if seg not in calibration_matrix:
calibration_matrix[seg] = {}
need_more = max(0, min_predictions - row['prediction_count'])
calibration_matrix[seg][topic] = {
'count': row['prediction_count'],
'mae': round(row['mean_absolute_error'] or 0.0, 1),
'status': f"need_{need_more}_more" if need_more > 0 else 'confident'
}
# Step 5: Accuracy trends
cursor.execute('''
SELECT
CASE
WHEN rowid <= 5 THEN 'posts_1_to_5'
WHEN rowid <= 10 THEN 'posts_6_to_10'
WHEN rowid <= 15 THEN 'posts_11_to_15'
ELSE 'posts_16+'
END as period,
AVG(ABS(predicted_reactions - actual_reactions)) as mae
FROM predictions
WHERE actual_reactions IS NOT NULL
GROUP BY period
ORDER BY period
''')
trends = {}
baseline_mae = None
for row in cursor.fetchall():
mae = round(row['mae'] or 0.0, 1)
if baseline_mae is None:
baseline_mae = mae
improvement = 'baseline'
else:
improvement = f"+{int((baseline_mae - mae) / baseline_mae * 100)}%"
trends[row['period']] = {
'mae': mae,
'improvement': improvement
}
# Step 6: Build plan
build_plan = generate_build_plan(cursor, min_predictions)
return {
'overall_status': overall_status,
'confidence_by_segment': confidence_by_segment,
'topic_calibration_details': topic_calibration,
'calibration_matrix': calibration_matrix,
'accuracy_trends': trends,
'build_plan': build_plan,
'recommendations': generate_recommendations(confidence_by_segment, total_tracked, min_predictions)
}
except Exception as e:
return {'error': f'Error: {str(e)}'}
finally:
conn.close()
def get_segment_interpretation(count, min_needed):
"""Interpret prediction count for a segment."""
if count == 0:
return f"No data yet. Need {min_needed} predictions to build calibration."
elif count < min_needed / 2:
return f"Early stage. Have {count}, need {min_needed - count} more."
elif count < min_needed:
return f"Getting there. Have {count}, need {min_needed - count} more for high confidence."
else:
return f"High confidence! Predictions are reliable ({count} predictions tracked)."
def get_segments_for_topic(cursor, topic_name):
"""Get segments that have prediction data for this topic."""
cursor.execute('''
SELECT DISTINCT segment_name
FROM prediction_accuracy
WHERE topic_name = ? AND prediction_count > 0
''', (topic_name,))
return [row['segment_name'] for row in cursor.fetchall()]
def generate_build_plan(cursor, min_predictions):
"""Generate a prioritized list of what to post next."""
cursor.execute('''
SELECT segment_name, topic_name, prediction_count
FROM prediction_accuracy
WHERE prediction_count < ?
ORDER BY prediction_count ASC
LIMIT 5
''', (min_predictions,))
plan = []
for row in cursor.fetchall():
need_more = min_predictions - row['prediction_count']
if need_more >= 8:
priority = 'high'
rationale = f"{row['prediction_count']} tracked. High affinity topic but no calibration."
elif need_more >= 5:
priority = 'medium'
rationale = f"{row['prediction_count']} tracked. Need more data for confidence."
else:
priority = 'low'
rationale = f"{row['prediction_count']} tracked. Almost there."
plan.append({
'priority': priority,
'segment': row['segment_name'],
'topic': row['topic_name'],
'recommendation': f"Post about {row['topic_name']} for {row['segment_name']}",
'need_more': need_more,
'rationale': rationale
})
return plan
def generate_recommendations(confidence_by_segment, total_tracked, min_predictions):
"""Generate actionable recommendations."""
recs = []
if total_tracked < 5:
recs.append("You're in cold-start phase. Post 5+ more times and track results. Calibration will improve prediction accuracy.")
elif total_tracked < min_predictions:
recs.append(f"Good progress! After {min_predictions - total_tracked} more tracked results, medium confidence kicks in.")
else:
recs.append("High confidence across most segments. Predictions are reliable.")
# Identify missing segment/topic combos
zero_segments = [s['segment'] for s in confidence_by_segment if s['predictions_tracked'] == 0]
if zero_segments:
recs.append(f"Build calibration for: {', '.join(zero_segments)}. Post content targeting these segments.")
recs.append("Timeline: 2 posts/week = medium confidence in 4-5 weeks, high confidence in 8+ weeks.")
return recs