菜单
Improves vision-language model distillation by aligning latent visual reasoning trajectories between teacher and student, enabling 3B parameter models to outperform larger open-source and proprietary systems with +16.9% gains on reasoning tasks.
默认使用会先检查来源的 Prompt；你也可以切换为直接命令，或下载本地副本。
决定是否安装前，请先阅读 SKILL.md，以及 SkillsMP 当前展示的配套文件。
用 Codex 或 Claude 帮你安装 复制这段 Prompt，粘贴到 Codex、Claude 或其他助手里，让它检查 Skill 页面并帮你完成安装。
直接命令不会经过审查 Prompt；运行前请先检查来源。
npx skills add https://github.com/ADu2021/skillXiv --skill lavit-visual-reasoning命令会保持在同一行。复制前请横向滚动并检查完整内容。
想先保存到本地？可下载 SkillsMP 当前能够提供的文件。
正在显示 SKILL.md
| name | lavit-visual-reasoning |
| title | LaViT: Aligning Latent Visual Thoughts for Multi-modal Reasoning |
| version | 0.0.2 |
| engine | skillxiv-v0.0.2-claude-opus-4.6 |
| license | MIT |
| url | https://arxiv.org/abs/2601.10129 |
| keywords | ["vision-language","distillation","latent-alignment","multi-modal-reasoning","knowledge-transfer"] |
| description | Improves vision-language model distillation by aligning latent visual reasoning trajectories between teacher and student, enabling 3B parameter models to outperform larger open-source and proprietary systems with +16.9% gains on reasoning tasks. |
Address the perception gap in knowledge distillation where students generate similar text to teachers but rely on different visual reasoning. Align latent visual thought trajectories between models, ensuring students genuinely understand images rather than taking shortcuts.
Identify when student models are taking language shortcuts.
# Detect language shortcut reliance
class PerceptionGapDetector:
def diagnose_gap(self, teacher_output, student_output, image):
"""Detect if student relies on language vs visual understanding"""
# Compare attention patterns
teacher_visual_attention = teacher_output["visual_attention"]
student_visual_attention = student_output["visual_attention"]
# Compute attention divergence
attention_kl = self.compute_kl_divergence(
teacher_visual_attention,
student_visual_attention
)
# If text outputs match but attention differs, perception gap exists
text_match = teacher_output["text"] == student_output["text"]
attention_mismatch = attention_kl > THRESHOLD
gap_exists = text_match and attention_mismatch
return {
"gap_exists": gap_exists,
"attention_divergence": attention_kl,
"severity": .estimate_gap_severity(attention_kl)
}
():
kl_div < :
kl_div < :
kl_div < :
:
():
p = np.array(p) +
q = np.array(q) +
np.(p * (np.log(p) - np.log(q)))
Align intermediate visual reasoning states.
# Latent thought alignment
class LatentThoughtAligner:
def __init__(self, embedding_dim=768):
self.embedding_dim = embedding_dim
self.projection = None
def extract_latent_thoughts(self, model_output, step_index=None):
"""Extract intermediate visual reasoning states"""
if step_index is None:
# Get all intermediate states from generation
thoughts = model_output["intermediate_embeddings"]
else:
# Get specific step
thoughts = model_output["intermediate_embeddings"][step_index]
return thoughts
def align_thought_trajectories(self, teacher_thoughts, student_thoughts):
"""Align latent trajectories between teacher and student"""
# Teacher trajectory is ground truth
# Student trajectory should match
if len(teacher_thoughts) != len(student_thoughts):
# Interpolate or align differently
student_thoughts = self.align_to_length(student_thoughts, len(teacher_thoughts))
# Compute alignment loss for each step
alignment_losses = []
for t_thought, s_thought in zip(teacher_thoughts, student_thoughts):
# Cosine similarity or L2 distance
loss = self.compute_thought_distance(t_thought, s_thought)
alignment_losses.append(loss)
return {
"per_step_losses": alignment_losses,
"total_loss": np.mean(alignment_losses),
"aligned_trajectory": student_thoughts
}
def compute_thought_distance(self, thought1, thought2):
"""Distance between latent thought vectors"""
# MSE or Cosine distance
return np.mean((thought1 - thought2) ** 2)
def align_to_length(self, trajectory, target_length):
"""Interpolate trajectory to target length"""
if len(trajectory) == target_length:
return trajectory
# Linear interpolation in embedding space
aligned = []
for i in range(target_length):
src_idx = i * (len(trajectory) - 1) / (target_length - 1)
lower_idx = int(src_idx)
upper_idx = min(lower_idx + 1, len(trajectory) - 1)
alpha = src_idx - lower_idx
interpolated = (
(1 - alpha) * trajectory[lower_idx] +
alpha * trajectory[upper_idx]
)
aligned.append(interpolated)
return aligned
Prevent students from taking shortcuts during learning.
# Curriculum sensory gating
class CurriculumGating:
def __init__(self, total_epochs=100):
self.total_epochs = total_epochs
self.current_epoch = 0
def compute_gate_strength(self):
"""Determine how strongly to enforce visual grounding"""
# Early epochs: force visual understanding
# Later epochs: allow more text reliance
if self.current_epoch < self.total_epochs * 0.3:
# Strict visual requirement
return 1.0
elif self.current_epoch < self.total_epochs * 0.7:
# Gradual relaxation
progress = (self.current_epoch - self.total_epochs * 0.3) / (self.total_epochs * 0.4)
return 1.0 - 0.5 * progress
else:
# Allow some text shortcutting
return 0.5
def apply_gate(self, student_trajectory, gate_strength):
"""Apply gating to prevent shortcuts"""
# Mask visual attention, force reconstruction
gated_trajectory = []
for thought in student_trajectory:
# Scale visual signal
gated = thought * gate_strength
gated_trajectory.append(gated)
return gated_trajectory
def gate_forward_pass(self, model, image, text_context, gate_strength):
"""Forward pass with gating applied"""
# Process with visual signal strength controlled
output = model(image, text_context)
# Apply curriculum gating
gated_output = self.apply_gate(output["thoughts"], gate_strength)
output["thoughts"] = gated_output
return output
def update_epoch(self):
"""Increment training epoch"""
self.current_epoch += 1
Train students to reconstruct teacher's visual semantics.
# Autoregressive reconstruction
class AutoregressiveReconstruction:
def __init__(self, student_model, teacher_model):
self.student = student_model
self.teacher = teacher_model
def compute_reconstruction_loss(self, image, text_context):
"""Reconstruct teacher's visual semantics"""
# Get teacher's visual understanding
teacher_output = self.teacher(image, text_context)
teacher_thoughts = teacher_output["intermediate_embeddings"]
teacher_attention = teacher_output["visual_attention"]
# Student attempts to reconstruct
student_output = self.student(image, text_context)
student_thoughts = student_output["intermediate_embeddings"]
student_attention = student_output["visual_attention"]
# Reconstruction loss components
thought_reconstruction = self.compute_thought_loss(
teacher_thoughts,
student_thoughts
)
attention_reconstruction = self.compute_attention_loss(
teacher_attention,
student_attention
)
# Combined reconstruction loss
total_loss = 0.7 * thought_reconstruction + 0.3 * attention_reconstruction
return {
"thought_loss": thought_reconstruction,
"attention_loss": attention_reconstruction,
"total_loss": total_loss
}
def compute_thought_loss(self, teacher_thoughts, student_thoughts):
"""MSE loss on intermediate thoughts"""
return np.mean((np.array(teacher_thoughts) - np.array(student_thoughts)) ** 2)
def compute_attention_loss(self, teacher_attention, student_attention):
"""KL divergence on attention patterns"""
p = np.array(teacher_attention) + 1e-10
q = np.array(student_attention) + 1e-10
return np.sum(p * (np.log(p) - np.log(q)))
def training_step(self, image_text_pairs, gating, learning_rate=1e-3):
"""Single training step with reconstruction"""
total_loss = 0.0
for image, text in image_text_pairs:
# Compute with curriculum gating
gated_output = gating.gate_forward_pass(
self.student,
image,
text,
gating.compute_gate_strength()
)
# Compute reconstruction loss
losses = self.compute_reconstruction_loss(image, text)
# Also include task loss (reasoning quality)
task_loss = self.compute_task_loss(gated_output, text)
# Combined loss
combined = 0.5 * losses["total_loss"] + 0.5 * task_loss
total_loss += combined
# Optimization
avg_loss = total_loss / len(image_text_pairs)
self.student.backward(avg_loss, learning_rate)
return avg_loss.item()
def compute_task_loss(self, model_output, ground_truth):
"""Loss on final reasoning task"""
# Standard supervised loss on task
predicted_text = model_output["text"]
return self.compute_text_loss(predicted_text, ground_truth)
基于 SOC 职业分类
