slow-fast-policy-optimization-rl

name	slow-fast-policy-optimization-rl
title	Slow-Fast Policy Optimization: Reposition-Before-Update for LLM Reasoning
version	0.0.2
engine	skillxiv-v0.0.2-claude-opus-4.6
license	MIT
url	https://arxiv.org/abs/2510.04072
keywords	["policy optimization","RL stability","gradient control","off-policy divergence","reasoning models"]
description	Stabilize RL for LLM reasoning via three-phase decomposition: fast inner trajectory optimization, repositioning to manage off-policy drift, slow correction for stable updates. Achieve up to 2.80-point math reasoning gains over GRPO while reducing rollouts 4.93x and wall-clock time 4.19x via improved stability without changing reward structure.

Slow-Fast Policy Optimization: Reposition-Before-Update

Core Concept

On-policy RL for LLMs suffers gradient instability from noisy early-training rollouts. Slow-Fast Policy Optimization (SFPO) decomposes each training step into three phases: fast optimization on the current batch, repositioning to constrain off-policy divergence, and slow correction for stable updates. This decomposition is plug-compatible with existing pipelines while dramatically improving stability and efficiency.

Architecture Overview

• Three-Phase Framework: (1) Fast trajectory optimization on same batch; (2) Reposition to manage divergence; (3) Slow correction phase
• Off-Policy Divergence Control: Track KL divergence during each phase to prevent catastrophic divergence
• Plug-Compatible Design: Works with existing policy gradient implementations (GRPO, PPO) without modification
• Efficiency Gains: 4.93× rollout reduction, 4.19× wall-clock speedup while improving performance

Implementation Steps

1. Three-Phase Decomposition Framework

Structure each training iteration into coordinated phases.

class SlowFastPolicyOptimizer:
    def __init__(self, policy_model, learning_rate=1e-6, max_kl=0.1):
        self.policy = policy_model
        self.optimizer = torch.optim.AdamW(self.policy.parameters(), lr=learning_rate)
        self.max_kl = max_kl  # KL divergence threshold

    def train_step_sfpo(self, batch, initial_policy_state=None):
        """
        Single SFPO training step with three phases.

        Args:
            batch: (observations, actions, rewards, values) from single rollout
            initial_policy_state: Reference policy for KL computation
        """

        # Save initial policy state for divergence tracking
        if initial_policy_state is None:
            initial_policy_state = {
                name: param.detach().clone()
                 name, param  .policy.named_parameters()
            }

        
        ()
        fast_loss = ._phase1_fast_optimization(batch)

        
        ()
        repositioned_state = ._phase2_repositioning(initial_policy_state)

        
        ()
        final_loss = ._phase3_slow_correction(batch, repositioned_state)

         {
            : fast_loss,
            : final_loss,
            : ._compute_kl_divergence(initial_policy_state)
        }

     ():
        

        obs, actions, rewards, values = batch

        
        advantages = rewards - values
        advantages = (advantages - advantages.mean()) / (advantages.std() + )

        
        inner_losses = []

         _  ():  
            
            action_logits = .policy(obs)

            
            log_probs = torch.log_softmax(action_logits, dim=-)
            selected_log_probs = log_probs[((actions)), actions]

            
            ratio = torch.exp(selected_log_probs - selected_log_probs.detach())
            clipped_ratio = torch.clamp(ratio, , )

            loss = -torch.(ratio, clipped_ratio) * advantages

            
            .optimizer.zero_grad()
            loss.mean().backward()
            torch.nn.utils.clip_grad_norm_(.policy.parameters(), )
            .optimizer.step()

            inner_losses.append(loss.mean().item())

         (inner_losses) / (inner_losses)

     ():
        

        
        current_kl = ._compute_kl_divergence(initial_state)

         current_kl > .max_kl:
            ()
            ()

            
            
            excess_divergence = (, current_kl - .max_kl)
            alpha = excess_divergence / (current_kl + )

            
             (name, param), (init_name, init_param)  (
                .policy.named_parameters(),
                initial_state.items()
            ):
                 name == init_name:
                    param.data = ( - alpha) * param.data + alpha * init_param

         {
            : current_kl,
            : current_kl > .max_kl
        }

     ():
        

        obs, actions, rewards, values = batch

        
        advantages = rewards - values
        advantages = (advantages - advantages.mean()) / (advantages.std() + )

        
        action_logits = .policy(obs)
        log_probs = torch.log_softmax(action_logits, dim=-)
        selected_log_probs = log_probs[((actions)), actions]

        
        ratio = torch.exp(selected_log_probs - selected_log_probs.detach())
        clipped_ratio = torch.clamp(ratio, , )  

        loss = -torch.(ratio, clipped_ratio) * advantages

        
        .optimizer.param_groups[][] =   
        .optimizer.zero_grad()
        loss.mean().backward()
        torch.nn.utils.clip_grad_norm_(.policy.parameters(), )  
        .optimizer.step()
        .optimizer.param_groups[][] =   

         loss.mean().item()

     ():
        

        total_kl = 

         (name, param), (ref_name, ref_param)  (
            .policy.named_parameters(),
            reference_state.items()
        ):
             name == ref_name:
                kl = torch.norm(param.data - ref_param) / (torch.norm(ref_param) + )
                total_kl += kl.item()

         total_kl / (reference_state)