메뉴
Train multimodal models for long-chain reflective reasoning (iterative thinking, backtracking) using Adaptive Hybrid Policy Optimization. Trigger: improve VLM performance on tasks requiring iterative refinement and error correction.
기본적으로 소스를 먼저 확인하는 Prompt가 선택됩니다. 직접 명령으로 전환하거나 로컬 사본을 다운로드할 수도 있습니다.
설치 여부를 결정하기 전에 SKILL.md와 SkillsMP에 표시된 보조 파일을 읽어 보세요.
Codex 또는 Claude로 설치 이 Prompt를 복사해 Codex, Claude 또는 다른 어시스턴트에 붙여 넣으면 Skill 페이지를 검토하고 설치를 진행할 수 있습니다.
직접 명령은 검토 Prompt를 거치지 않습니다. 실행하기 전에 소스를 확인하세요.
npx skills add https://github.com/ADu2021/skillXiv --skill mm-helix-reasoning명령은 한 줄로 유지됩니다. 복사하기 전에 가로로 스크롤해 전체 내용을 확인하세요.
로컬 사본을 원하시나요? SkillsMP에서 현재 제공할 수 있는 파일을 다운로드하세요.
SOC 직업 분류 기준
SKILL.md 표시 중
| name | mm-helix-reasoning |
| title | MM-HELIX: Multimodal Long-Chain Reflective Reasoning via AHPO |
| version | 0.0.2 |
| engine | skillxiv-v0.0.2-claude-opus-4.6 |
| license | MIT |
| url | https://arxiv.org/abs/2510.08540 |
| keywords | ["multimodal-reasoning","reflective-thinking","reinforcement-learning","backtracking","adaptive-training"] |
| description | Train multimodal models for long-chain reflective reasoning (iterative thinking, backtracking) using Adaptive Hybrid Policy Optimization. Trigger: improve VLM performance on tasks requiring iterative refinement and error correction. |
MM-HELIX addresses a critical gap in multimodal LLMs: they struggle with long-chain reflective reasoning that requires iterative thinking and backtracking. The paper introduces three components: a benchmark (MM-HELIX) with 1,260 samples requiring iterative thinking, a large-scale training dataset (MM-HELIX-100K), and Adaptive Hybrid Policy Optimization (AHPO) that dynamically unifies offline supervision with online RL in a single training stage, achieving 18.6% accuracy improvement.
The key insight: Reflective reasoning requires dynamic training that switches between supervised learning on correct traces and online RL refinement—not static offline or online training alone.
Define what constitutes reflective reasoning in multimodal contexts.
class ReflectiveReasoningStructure:
"""
Represent multi-step reasoning with backtracking.
"""
def __init__(self):
pass
@staticmethod
def extract_reasoning_steps(trace):
"""
Parse reasoning into steps with branch points.
"""
steps = []
current_path = []
lines = trace.split('\n')
for line in lines:
if 'THOUGHT:' in line:
current_path.append({
"type": "thought",
: line.split()[].strip()
})
line:
steps.append({
: current_path,
: line.split()[].strip()
})
current_path = []
line:
steps.append({
: current_path,
: line.split()[].strip()
})
steps
():
steps = ReflectiveReasoningStructure.extract_reasoning_steps(trace)
final_answer = extract_final_answer(trace)
correctness = final_answer == ground_truth
num_iterations = (
step steps step
)
iterativeness = (num_iterations / , )
num_tokens = (trace.split())
efficiency = / ( + num_tokens / )
quality_score = (
* correctness +
* iterativeness +
* efficiency
)
{
: quality_score,
: correctness,
: iterativeness,
: efficiency
}
Create high-quality reasoning traces with explicit step-by-step generation.
class StepElicitedResponseGenerator:
"""
Systematically generate reasoning traces with explicit steps.
"""
def __init__(self, model):
self.model = model
def generate_with_steps(self, problem, image=None, max_iterations=5):
"""
Generate reasoning by eliciting explicit steps.
Args:
problem: Problem statement
image: Multimodal input (optional)
max_iterations: Max refinement iterations
Returns:
Multi-step reasoning trace
"""
trace = f"Problem: {problem}\n\n"
if image:
trace += f"[Image analysis]\n"
iteration = 0
current_solution = None
while iteration < max_iterations:
# Step 1: Generate initial thought
thought_prompt = (
trace +
f"\n[Step {iteration + 1}]\n"
f"Thinking: "
)
thought = self.model.generate(
thought_prompt,
max_tokens=150,
temperature=0.7
)
trace += f"THOUGHT: {thought}\n"
# Step 2: Attempt conclusion
conclusion_prompt = (
trace +
f"Conclusion from this step: "
)
conclusion = self.model.generate(
conclusion_prompt,
max_tokens=100,
temperature=0.3
)
trace += f"CONCLUSION: {conclusion}\n"
current_solution = conclusion
# Step 3: Self-evaluate and potentially backtrack
evaluation_prompt = (
trace +
f"Is this solution correct and complete? "
f"[Yes/No]. If No, what needs to be reconsidered? "
)
evaluation = self.model.generate(
evaluation_prompt,
max_tokens=50,
temperature=0.5
)
if "yes" in evaluation.lower():
trace += "FINAL: Solution is correct.\n"
break
else:
# Extract reason for backtracking
backtrack_reason = evaluation.split("No")[-1].strip()
trace += f"BACKTRACK: {backtrack_reason}\n"
trace += "[Reconsidering...]\n"
iteration += 1
return trace
def generate_dataset_with_serg(self, problems, images, num_samples=100000):
"""
Systematically generate 100K reasoning traces.
"""
dataset = []
for idx, (problem, image) in enumerate(zip(problems, images)):
trace = self.generate_with_steps(problem, image)
quality = ReflectiveReasoningStructure.score_reasoning_quality(
trace,
ground_truth=extract_answer(problem)
)
example = {
"problem": problem,
"image": image,
"reasoning": trace,
"quality_score": quality["quality_score"]
}
dataset.append(example)
if (idx + 1) % 10000 == 0:
print(f"Generated {idx + 1} examples")
if len(dataset) >= num_samples:
break
return dataset
Dynamically blend offline supervision with online RL in a single unified stage.
class AdaptiveHybridPolicyOptimization:
"""
Unified training that adapts between supervised and online RL.
"""
def __init__(self, model):
self.model = model
def compute_ahpo_loss(self, batch, step, total_steps):
"""
Compute loss that adapts between offline and online optimization.
Args:
batch: Training batch with problems, images, reasoning traces
step: Current training step
total_steps: Total steps in training
Returns:
Scalar loss value
"""
total_loss = 0
batch_size = len(batch)
# Adaptive blending: start with more offline supervision, shift to online RL
offline_weight = max(0.7 - (step / total_steps) * 0.5, 0.2)
online_weight = 1.0 - offline_weight
for example in batch:
problem = example["problem"]
image = example["image"]
target_trace = example["reasoning"]
# Phase 1: Offline supervision loss
# Standard next-token prediction on high-quality traces
generated = self.model.generate(
f"Problem: {problem}",
image=image,
max_tokens=2000
)
offline_loss = compute_token_loss(
generated,
target_trace
)
# Phase 2: Online RL loss
# Reward based on reasoning quality
quality = ReflectiveReasoningStructure.score_reasoning_quality(
generated,
extract_answer(problem)
)
reward = quality["quality_score"]
log_prob = self.model.compute_log_prob(generated)
online_loss = -reward * log_prob
# Adaptive combination
combined_loss = (
offline_weight * offline_loss +
online_weight * online_loss
)
total_loss += combined_loss
# Prevent catastrophic forgetting: entropy bonus
entropy = compute_entropy_bonus(
self.model.get_logits(generated)
)
total_loss -= 0.01 * entropy
return total_loss / batch_size, offline_weight, online_weight
def train_ahpo(self, model, dataset, config):
"""
Execute AHPO training loop.
"""
optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)
total_steps = config.num_epochs * len(dataset) // config.batch_size
for epoch in range(config.num_epochs):
epoch_loss = 0
for batch_idx in range(0, len(dataset), config.batch_size):
batch = dataset[batch_idx:batch_idx + config.batch_size]
step = epoch * len(dataset) // config.batch_size + batch_idx // config.batch_size
# Compute adaptive loss
loss, offline_w, online_w = self.compute_ahpo_loss(
batch,
step,
total_steps
)
# Backward and optimize
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
epoch_loss += loss.item()
if (batch_idx // config.batch_size) % 100 == 0:
print(f"Epoch {epoch}, Batch {batch_idx}: "
f"loss={loss:.4f}, "
f"offline_w={offline_w:.2f}, online_w={online_w:.2f}")
print(f"Epoch {epoch} avg loss: {epoch_loss / len(dataset) * config.batch_size:.4f}")
return model
Combine SERG data generation with AHPO training.
def train_mm_helix(base_model, problem_dataset, image_dataset, config):
"""
Complete MM-HELIX training pipeline.
"""
# Step 1: Generate MM-HELIX-100K dataset with SERG
print("Step 1: Generating reasoning dataset with SERG")
serg_generator = StepElicitedResponseGenerator(base_model)
training_dataset = serg_generator.generate_dataset_with_serg(
problem_dataset,
image_dataset,
num_samples=100000
)
# Filter to high-quality examples
quality_filtered = [
ex for ex in training_dataset
if ex["quality_score"] > 0.5
]
print(f"Kept {len(quality_filtered)} high-quality examples")
# Step 2: Train with AHPO
print("\nStep 2: Training with Adaptive Hybrid Policy Optimization")
ahpo_trainer = AdaptiveHybridPolicyOptimization(base_model)
trained_model = ahpo_trainer.train_ahpo(
base_model,
quality_filtered,
config
)
return trained_model
Assess model performance on iterative reasoning tasks.
def evaluate_mm_helix(model, benchmark_dataset):
"""
Evaluate model on long-chain reflective reasoning tasks.
"""
results = {
"accuracy": 0,
"avg_iterations": 0,
"avg_reasoning_length": 0
}
correct = 0
total_iterations = []
reasoning_lengths = []
for example in benchmark_dataset:
problem = example["problem"]
image = example["image"]
ground_truth = example["ground_truth"]
# Generate reasoning
trace = model.generate(
f"Problem: {problem}",
image=image,
max_tokens=2000
)
# Extract answer
final_answer = extract_final_answer(trace)
if final_answer == ground_truth:
correct += 1
# Count iterations (backtracks)
iterations = trace.count("BACKTRACK")
total_iterations.append(iterations)
# Measure efficiency
reasoning_lengths.append(len(trace.split()))
results["accuracy"] = correct / len(benchmark_dataset) * 100
results["avg_iterations"] = np.mean(total_iterations)
results["avg_reasoning_length"] = np.mean(reasoning_lengths)
print(f"Accuracy: {results['accuracy']:.1f}%")
print(f"Avg iterations: {results['avg_iterations']:.1f}")
print(f"Avg reasoning length: {results['avg_reasoning_length']:.0f} tokens")
return results
Hyperparameters:
When to Use:
When NOT to Use: