菜单
Build robust reward models for computer-use agents by processing execution videos with spatiotemporal token pruning (STP+TTP) to remove redundancy while preserving task-relevant UI details. Achieve 84.7% accuracy with adversarial instruction translation.
默认使用会先检查来源的 Prompt；你也可以切换为直接命令，或下载本地副本。
决定是否安装前，请先阅读 SKILL.md，以及 SkillsMP 当前展示的配套文件。
用 Codex 或 Claude 帮你安装 复制这段 Prompt，粘贴到 Codex、Claude 或其他助手里，让它检查 Skill 页面并帮你完成安装。
直接命令不会经过审查 Prompt；运行前请先检查来源。
npx skills add https://github.com/ADu2021/skillXiv --skill exevrm-video-reward-modeling命令会保持在同一行。复制前请横向滚动并检查完整内容。
想先保存到本地？可下载 SkillsMP 当前能够提供的文件。
正在显示 SKILL.md
基于 SOC 职业分类
| name | exevrm-video-reward-modeling |
| title | Video-Based Reward Modeling for Computer-Use Agents |
| version | 0.0.2 |
| engine | skillxiv-v0.0.2-claude-opus-4.6 |
| license | MIT |
| url | https://arxiv.org/abs/2603.10178 |
| keywords | ["Reward Modeling","Computer Use","Video Understanding","RL","Vision Language"] |
| description | Build robust reward models for computer-use agents by processing execution videos with spatiotemporal token pruning (STP+TTP) to remove redundancy while preserving task-relevant UI details. Achieve 84.7% accuracy with adversarial instruction translation. |
Computer-use agents generate long, high-resolution execution videos that are expensive to process. Standard vision models struggle with this modality's inherent redundancy: static backgrounds, repeated layouts, and unchanged UI elements dominate frames. Execution Video Reward Models (ExeVRM) use spatiotemporal token pruning to eliminate this redundancy while preserving subtle visual cues determining task correctness.
The approach combines spatial pruning (removing homogeneous regions) with temporal pruning (suppressing unchanged tokens) and augments training with adversarial instruction translation to handle distribution shifts.
ExeVRM operates through three mechanisms:
This enables efficient processing of long, high-resolution videos while maintaining robustness to instruction variations.
Remove large homogeneous regions while preserving UI-relevant patches.
import torch
import torch.nn as nn
import torch.nn.functional as F
class SpatialTokenPruner:
def __init__(self, homogeneity_threshold=0.95, min_region_size=):
.threshold = homogeneity_threshold
.min_region_size = min_region_size
():
kernel_size =
stride =
patches = F.unfold(
frame.permute(, , ).unsqueeze(),
kernel_size=kernel_size,
stride=stride
)
patch_variance = patches.std(dim=)
patch_variance
():
homogeneity = .compute_region_homogeneity(frame)
keep_mask = homogeneity < (.threshold * homogeneity.())
keep_mask
():
keep_mask = .get_pruning_mask(frame)
keep_indices = torch.nonzero(keep_mask).squeeze(-)
pruned_tokens = visual_tokens[:, keep_indices, :]
pruned_tokens, keep_mask
Suppress tokens unchanged across frames, focus on state transitions.
class TemporalTokenPruner:
def __init__(self, change_threshold=0.1):
self.threshold = change_threshold
self.prev_frame_tokens = None
def compute_token_delta(self, current_tokens, prev_tokens):
"""
Measure L2 distance between consecutive frame representations.
current_tokens: (batch, num_patches, dim)
prev_tokens: (batch, num_patches, dim)
"""
delta = torch.norm(current_tokens - prev_tokens, dim=-1) # (batch, num_patches)
return delta
def get_change_mask(self, current_tokens, prev_tokens):
"""
Mask: 1 for tokens with significant change, 0 for static.
"""
delta = self.compute_token_delta(current_tokens, prev_tokens)
# Adaptive threshold (relative to frame's max change)
frame_max_delta = delta.max(dim=-1, keepdim=True)[0]
threshold = self.threshold * frame_max_delta
change_mask = delta > threshold
return change_mask
def prune_tokens(self, current_tokens, prev_tokens):
"""
Filter tokens to keep only those with significant changes.
"""
if prev_tokens is None:
# First frame: keep all
return current_tokens, torch.ones_like(current_tokens[:, :, 0])
change_mask = self.get_change_mask(current_tokens, prev_tokens)
# Select tokens with changes
batch_size = current_tokens.shape[0]
pruned_tokens_list = []
for b in range(batch_size):
selected = current_tokens[b, change_mask[b]]
pruned_tokens_list.append(selected)
# Pad to same length
max_len = max(t.shape[0] for t in pruned_tokens_list)
padded_tokens = torch.stack([
F.pad(t, (0, 0, 0, max_len - t.shape[0]))
for t in pruned_tokens_list
])
self.prev_frame_tokens = current_tokens # Cache for next frame
return padded_tokens, change_mask
Integrate spatial and temporal pruning into coherent video processing.
class VideoRewardModel(nn.Module):
def __init__(self, backbone_model, hidden_dim=768):
super().__init__()
self.backbone = backbone_model
self.spatial_pruner = SpatialTokenPruner()
self.temporal_pruner = TemporalTokenPruner()
# Reward heads
self.execution_head = nn.Linear(hidden_dim, 1)
self.consistency_head = nn.Linear(hidden_dim, 1)
def forward(self, instruction, video_frames):
"""
instruction: str description
video_frames: (num_frames, height, width, 3)
"""
pruned_frame_features = []
prev_pruned_tokens = None
for frame_idx, frame in enumerate(video_frames):
# Extract visual features
with torch.no_grad():
visual_tokens = self.backbone.encode_image(frame)
# Spatial pruning
spatial_pruned, spatial_mask = self.spatial_pruner.prune_tokens(
visual_tokens,
frame
)
# Temporal pruning (accumulates over frames)
if prev_pruned_tokens is not None:
temporal_pruned, temporal_mask = self.temporal_pruner.prune_tokens(
spatial_pruned,
prev_pruned_tokens
)
else:
temporal_pruned = spatial_pruned
prev_pruned_tokens = spatial_pruned
pruned_frame_features.append(temporal_pruned)
# Aggregate across frames
aggregated_features = torch.cat(pruned_frame_features, dim=1) # (1, total_tokens, dim)
# Process with language model
instruction_tokens = self.backbone.encode_text(instruction)
# Combined understanding
combined = torch.cat([
instruction_tokens.mean(dim=0, keepdim=True),
aggregated_features.mean(dim=0, keepdim=True)
], dim=-1)
# Predict rewards
execution_score = self.execution_head(combined)
consistency_score = self.consistency_head(combined)
return {
'execution': execution_score.sigmoid(),
'consistency': consistency_score.sigmoid()
}
Synthesize hard negatives by pairing trajectories with semantically mismatched instructions.
def adversarial_instruction_translation(
successful_trajectory,
task_instructions,
llm_model,
num_hard_negatives=3
):
"""
Generate plausible but incorrect instruction-trajectory pairs.
"""
hard_negatives = []
# Extract trajectory semantics
trajectory_description = llm_model.summarize_trajectory(successful_trajectory)
# For each instruction, generate semantically different variants
for instruction in task_instructions:
# Generate modifications: addition, deletion, paraphrasing
modifications = [
f"{instruction} and additionally...", # Addition variant
instruction.split()[:-2], # Deletion variant
llm_model.paraphrase(instruction) # Paraphrase variant
]
for modified in modifications[:num_hard_negatives]:
# Verify it's different from original
similarity = cosine_similarity(
llm_model.encode(instruction),
llm_model.encode(str(modified))
)
if similarity < 0.8: # Ensure sufficient divergence
hard_negatives.append({
'trajectory': successful_trajectory,
'instruction': str(modified),
'label': 0 # Mismatch label
})
return hard_negatives
def train_step_with_adversarial_augmentation(
model,
instruction,
video,
label,
adversarial_examples,
optimizer
):
"""
Training step with hard negative mining.
"""
# Standard forward pass
output = model(instruction, video)
execution_loss = F.binary_cross_entropy(output['execution'], label)
# Adversarial hard negatives
adversarial_loss = 0
for adv_ex in adversarial_examples:
adv_output = model(adv_ex['instruction'], video)
# Should output low score (mismatch)
adv_loss = F.binary_cross_entropy(
adv_output['execution'],
torch.tensor(adv_ex['label'])
)
adversarial_loss += adv_loss
total_loss = execution_loss + 0.5 * (adversarial_loss / len(adversarial_examples))
optimizer.zero_grad()
total_loss.backward()
optimizer.step()
return total_loss.item()
When to Use:
When NOT to Use:
Hyperparameter Tuning:
Common Pitfalls: