| name | token-order-prediction |
| title | Predicting Token Order Improves Language Model Performance |
| version | 0.0.2 |
| engine | skillxiv-v0.0.2-claude-opus-4.6 |
| license | MIT |
| url | https://arxiv.org/abs/2508.19228 |
| keywords | ["auxiliary-objective","token-ordering","learning-to-rank","language-modeling","multi-token-prediction"] |
| description | Improve LM performance with token order prediction (TOP) auxiliary loss using learning-to-rank instead of exact multi-token prediction, achieving gains across math, code, and NLP tasks |
Predicting the Order of Upcoming Tokens Improves Language Models
Core Concept
Token Order Prediction (TOP) is a lightweight auxiliary training objective that teaches models to rank upcoming tokens by proximity. Instead of predicting exact future tokens (which is difficult), TOP uses learning-to-rank loss to teach the model to understand token sequencing. This simple auxiliary objective improves performance across mathematics, coding, and standard NLP benchmarks while requiring only a single additional unembedding layer.
Architecture Overview
- Learning-to-Rank Loss: Rank tokens by distance rather than exact prediction
- Single Extra Layer: Minimal architectural overhead vs. multi-token prediction
- Auxiliary Objective: Combined with next-token prediction during training
- Task Generalization: Benefits across math, code, and general NLP
- Efficiency: No increase in inference latency or model size
Implementation Steps
Stage 1: Design Token Order Prediction Objective
Formulate the learning-to-rank loss for token ordering.
import torch
from torch import nn
import torch.nn.functional as F
class TokenOrderPredictionHead(nn.Module):
"""Predict ordering of upcoming tokens"""
def __init__(
self,
hidden_dim: int,
vocab_size: int,
num_future_tokens: int = 5
):
super().__init__()
self.hidden_dim = hidden_dim
self.vocab_size = vocab_size
self.num_future = num_future_tokens
self.ranking_head = nn.Linear(hidden_dim, vocab_size)
def forward(
self,
hidden_states: torch.Tensor,
future_tokens: torch.Tensor
) -> torch.Tensor:
"""
Compute ranking scores for upcoming tokens.
Args:
hidden_states: [batch, seq_len, hidden_dim]
future_tokens: [batch, seq_len, num_future]
Returns:
ranking_scores: [batch, seq_len, vocab_size]
"""
scores = self.ranking_head(hidden_states)
return scores
class TopLoss(nn.Module):
"""Learning-to-rank loss for token ordering"""
def __init__(self, loss_type: str = "listwise"):
().__init__()
.loss_type = loss_type
() -> torch.Tensor:
.loss_type == :
._listwise_loss(predicted_scores, target_tokens, target_positions)
.loss_type == :
._pairwise_loss(predicted_scores, target_tokens, target_positions)
:
._pointwise_loss(predicted_scores, target_tokens, target_positions)
() -> torch.Tensor:
batch_size = predicted_scores.shape[]
loss =
i (batch_size):
scores = predicted_scores[i]
tokens = target_tokens[i]
positions = target_positions[i]
token_scores = scores[tokens]
j ((tokens)):
k (j + , (tokens)):
positions[j] < positions[k]:
margin = token_scores[j] - token_scores[k]
:
margin = token_scores[k] - token_scores[j]
loss += F.relu( - margin)
loss / batch_size
() -> torch.Tensor:
batch_size = predicted_scores.shape[]
batch_idx = torch.arange(batch_size).unsqueeze()
token_scores = predicted_scores[batch_idx, target_tokens]
loss =
dist_close (target_positions.shape[]):
dist_far (dist_close + , target_positions.shape[]):
score_close = token_scores[:, dist_close]
score_far = token_scores[:, dist_far]
margin_loss = F.relu( - (score_close - score_far))
loss += margin_loss.mean()
loss
() -> torch.Tensor:
batch_size = predicted_scores.shape[]
batch_idx = torch.arange(batch_size).unsqueeze()
token_scores = predicted_scores[batch_idx, target_tokens]
max_pos = target_positions.().()
position_targets = (max_pos - target_positions.()) / max_pos
loss = F.mse_loss(token_scores, position_targets)
loss
Stage 2: Integrate TOP with Next-Token Prediction
Combine TOP auxiliary objective with standard language modeling.
class LanguageModelWithTOP(nn.Module):
"""LM with top-k token order prediction"""
def __init__(
self,
hidden_dim: int = 4096,
vocab_size: int = 32000,
num_layers: int = 32
):
super().__init__()
self.transformer = TransformerLM(hidden_dim, vocab_size, num_layers)
self.lm_head = nn.Linear(hidden_dim, vocab_size)
self.top_head = TokenOrderPredictionHead(hidden_dim, vocab_size)
self.top_loss_fn = TopLoss(loss_type="pairwise")
def forward(
self,
input_ids: torch.Tensor,
future_token_ids: torch.Tensor = None
) -> Dict:
"""
Forward pass with both NTP and TOP.
Args:
input_ids: [batch, seq_len]
future_token_ids: [batch, seq_len, num_future] (for TOP)
Returns:
ntp_logits, top_scores
"""
hidden_states = self.transformer(input_ids)
ntp_logits = self.lm_head(hidden_states)
top_scores = None
if future_token_ids is not :
top_scores = .top_head(hidden_states, future_token_ids)
{
: ntp_logits,
: top_scores,
: hidden_states
}
:
():
.model = model
.optimizer = torch.optim.AdamW(model.parameters(), lr=lr)
.ntp_loss_fn = nn.CrossEntropyLoss()
.top_loss_fn = TopLoss(loss_type=)
.top_weight = top_weight
() -> [torch.Tensor, torch.Tensor]:
batch_size, seq_len = input_ids.shape
future_tokens = torch.zeros(
batch_size, seq_len, num_future,
dtype=torch.long, device=input_ids.device
)
future_positions = torch.zeros_like(future_tokens)
i (seq_len):
j (num_future):
i + j + < seq_len:
future_tokens[:, i, j] = input_ids[:, i + j + ]
future_positions[:, i, j] = j +
future_tokens, future_positions
() -> :
input_ids = batch[]
attention_mask = batch.get()
future_tokens, future_positions = .extract_future_tokens(input_ids)
outputs = .model(input_ids, future_tokens)
ntp_logits = outputs[]
top_scores = outputs[]
ntp_loss = .ntp_loss_fn(
ntp_logits.view(-, ntp_logits.shape[-]),
input_ids.view(-)
)
top_loss =
top_scores :
top_loss = .top_loss_fn(
top_scores.view(-, top_scores.shape[-]),
future_tokens.view(-, future_tokens.shape[-]),
future_positions.view(-, future_positions.shape[-])
)
total_loss = ntp_loss + .top_weight * top_loss
.optimizer.zero_grad()
total_loss.backward()
.optimizer.step()
{
: total_loss.item(),
: ntp_loss.item(),
: top_loss.item() (top_loss, torch.Tensor) top_loss
}
():
.top_weight = weight
Stage 3: Evaluation on Benchmarks
Test TOP across diverse tasks.
class TOPEvaluator:
"""Evaluate TOP improvements"""
def __init__(self, model_with_top: LanguageModelWithTOP):
self.model = model_with_top
def evaluate_on_benchmarks(self) -> Dict:
"""
Evaluate on standard benchmarks.
Reproduction of paper results.
"""
benchmarks = {
"arc_challenge": self.evaluate_benchmark("arc_challenge"),
"hellaswag": self.evaluate_benchmark("hellaswag"),
"mmlu": self.evaluate_benchmark("mmlu"),
"gsm8k": self.evaluate_benchmark("gsm8k"),
"math": self.evaluate_benchmark("math"),
"humanevals": self.evaluate_benchmark("humanevals"),
"mbpp": self.evaluate_benchmark("mbpp")
}
return benchmarks
def evaluate_benchmark(self, benchmark_name: str) -> Dict:
"""Evaluate on single benchmark"""
test_set = self.load_benchmark_data(benchmark_name)
correct =
total = (test_set)
example test_set:
prompt = example[]
answer = example[]
generated = .generate(prompt, max_length=)
.check_correctness(generated, answer, benchmark_name):
correct +=
accuracy = correct / total
{
: benchmark_name,
: correct,
: total,
: accuracy
}
() -> :
tokens = .model.transformer.tokenizer.encode(prompt)
tokens = torch.tensor(tokens).unsqueeze()
_ (max_length):
outputs = .model(tokens)
logits = outputs[][:, -, :]
next_token = logits.argmax(dim=-)
tokens = torch.cat([tokens, next_token.unsqueeze()], dim=)
.model.transformer.tokenizer.decode(tokens[])
() -> :
benchmark [, ]:
gen_num = .extract_number(generated)
ref_num = .extract_number(reference)
gen_num == ref_num
:
reference generated
() -> :
re
numbers = re.findall(, text)
numbers:
(numbers[-])
():
[]
Practical Guidance
Hyperparameters
- TOP Weight: 0.1-0.3 relative to NTP loss (higher = more TOP emphasis)
- Future Tokens: 5 is optimal (further tokens have diminishing signal)
- Loss Type: Pairwise generally works best
- Training Duration: TOP converges quickly; add after 10-20% of pretraining
When to Use TOP
- Improving language model performance across diverse tasks
- Math and coding tasks (TOP particularly helps here)
- General pretraining where you want better few-shot performance
- Resource-constrained training (minimal overhead)
When NOT to Use
- Models already performing optimally on target tasks
- Very long-context scenarios (future token extraction becomes expensive)
- Real-time systems where any overhead matters
Performance Expectations
- Math benchmarks: +2-4% improvement
- Coding benchmarks: +1-3% improvement
- General NLP: +0.5-2% improvement
- Computational overhead: <5% training time increase
Design Insights
TOP works because predicting exact future tokens is a harder auxiliary task than necessary. By reformulating as a ranking problem ("which tokens appear first?"), the model learns token sequencing patterns without the regression difficulty. This is a goldilocks auxiliary objective—harder than next-token only, but easier than exact multi-token prediction.
Reference
Predicting the Order of Upcoming Tokens Improves Language Models. arXiv:2508.19228
