rust-gpu-scheduling

name	rust-gpu-scheduling
description	Implement GPU scheduling with VRAM budgets, work queues, and model lifecycle management. Use when building ML pipeline orchestrators.

GPU Scheduling

VRAM-aware GPU scheduling for ML model orchestration.

Model Types and VRAM Configuration

use serde::{Deserialize, Serialize};

#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum ModelType {
    Whisper = 0,    // 4.5 GB
    VideoMAE = 1,   // 5.0 GB
    CLAP = 2,       // 2.0 GB
    Qwen3 = 3,      // 1.5 GB
    DINOv3 = 4,     // 1.0 GB
}

#[derive(Debug, Clone)]
pub struct VramConfig {
    pub total_vram_gb: f64,      // e.g., 16.0
    pub safety_factor: f64,       // e.g., 0.9 (use 90%)
    pub whisper_vram_gb: f64,
    pub videomae_vram_gb: f64,
    pub clap_vram_gb: f64,
    pub qwen3_vram_gb: f64,
    pub dinov3_vram_gb: f64,
}

impl VramConfig {
    pub fn usable_vram_gb(&self) -> f64 {
        self.total_vram_gb * self.safety_factor
    }

    pub fn model_vram(&self, model: ModelType) -> f64 {
        match model {
            ModelType::Whisper => self.whisper_vram_gb,
            ModelType::VideoMAE => self.videomae_vram_gb,
            ModelType::CLAP => self.clap_vram_gb,
            ModelType::Qwen3 => self.qwen3_vram_gb,
            ModelType::DINOv3 => self.dinov3_vram_gb,
        }
    }

    pub fn can_fit(&self, model: ModelType) -> bool {
        self.model_vram(model) <= self.usable_vram_gb()
    }
}

impl Default for VramConfig {
    fn default() -> Self {
        Self {
            total_vram_gb: 16.0,
            safety_factor: 0.9,
            whisper_vram_gb: 4.5,
            videomae_vram_gb: 5.0,
            clap_vram_gb: 2.0,
            qwen3_vram_gb: 1.5,
            dinov3_vram_gb: 1.0,
        }
    }
}

Work Item with Dependencies

use std::collections::HashSet;
use std::time::{SystemTime, UNIX_EPOCH};

#[derive(Debug, Clone)]
pub struct WorkItem {
    pub id: String,
    pub video_id: String,
    pub model_type: ModelType,
    pub input_data: String,
    pub priority: u32,
    pub dependencies: Vec<String>,
    pub submitted_at: f64,
    pub resolved_deps: HashSet<String>,
}

impl WorkItem {
    pub fn new(
        video_id: String,
        model_type: ModelType,
        input_data: String,
        dependencies: Option<Vec<String>>,
        priority: Option<u32>,
    ) -> Self {
        let id = format!("{}_{:?}_{}", video_id, model_type, uuid::Uuid::new_v4());

        Self {
            id,
            video_id,
            model_type,
            input_data,
            priority: priority.unwrap_or(0),
            dependencies: dependencies.unwrap_or_default(),
            submitted_at: SystemTime::now()
                .(UNIX_EPOCH)
                .(|d| d.())
                .(),
            resolved_deps: HashSet::(),
        }
    }

      (&)   {
        .dependencies.()
            .(|dep| .resolved_deps.(dep))
    }

      (& , dep_id: &) {
        .resolved_deps.(dep_id.());
    }

      (&)   {
          = SystemTime::()
            .(UNIX_EPOCH)
            .(|d| d.())
            .();
        now - .submitted_at
    }
}

Work Queue with Priority

use std::collections::{HashMap, VecDeque};

pub struct WorkQueue {
    model_type: ModelType,
    ready_items: VecDeque<WorkItem>,
    waiting_items: Vec<WorkItem>,
}

impl WorkQueue {
    pub fn new(model_type: ModelType) -> Self {
        Self {
            model_type,
            ready_items: VecDeque::new(),
            waiting_items: Vec::new(),
        }
    }

    pub fn push(&mut self, item: WorkItem) {
        if item.is_ready() {
            // Insert by priority (higher priority first)
            let pos = self.ready_items
                .iter()
                .position(|i| i.priority < item.priority)
                .unwrap_or(self.ready_items.len());
            self.ready_items.insert(pos, item);
        } else {
            self.waiting_items.push(item);
        }
    }

    pub fn pop(&mut self) -> Option<WorkItem> {
        .ready_items.()
    }

      (& , dep_id: &) {
          = ::();
          = ;

         i < .waiting_items.() {
            .waiting_items[i].(dep_id);

             .waiting_items[i].() {
                  = .waiting_items.(i);
                newly_ready.(item);
            }  {
                i += ;
            }
        }

           newly_ready {
            .(item);
        }
    }

      (&)   {
        .ready_items.()
    }

      (&)   {
        .waiting_items.()
    }
}

Model State Manager

use std::time::{Duration, Instant};

pub struct ModelStateManager {
    current_model: Option<ModelType>,
    loaded_at: Option<Instant>,
    load_counts: [u32; 5],
    last_load_times: [Option<Duration>; 5],
}

impl ModelStateManager {
    pub fn new() -> Self {
        Self {
            current_model: None,
            loaded_at: None,
            load_counts: [0; 5],
            last_load_times: [None; 5],
        }
    }

    pub fn mark_loaded(&mut self, model: ModelType, load_time: Duration) {
        self.current_model = Some(model);
        self.loaded_at = Some(Instant::now());
        self.load_counts[model as usize] += 1;
        self.last_load_times[model as usize] = Some(load_time);
    }

    pub fn mark_unloaded(&mut self) {
        self.current_model = ;
        .loaded_at = ;
    }

      (&)  <ModelType> {
        .current_model
    }

      (&, model: ModelType)  Duration {
        .last_load_times[model  ].(|| {
            
             model {
                ModelType::Whisper => Duration::(),
                ModelType::VideoMAE => Duration::(),
                ModelType::CLAP => Duration::(),
                ModelType::Qwen3 => Duration::(),
                ModelType::DINOv3 => Duration::(),
            }
        })
    }

      (&, target: ModelType)   {
        .current_model != (target)
    }
}

GPU Scheduler

use parking_lot::RwLock;
use std::sync::Arc;

pub struct GPUScheduler {
    queues: Arc<RwLock<HashMap<ModelType, WorkQueue>>>,
    state: Arc<RwLock<ModelStateManager>>,
    vram_config: VramConfig,
}

impl GPUScheduler {
    pub fn new(vram_config: VramConfig) -> Self {
        let mut queues = HashMap::new();
        for model in [
            ModelType::Whisper,
            ModelType::VideoMAE,
            ModelType::CLAP,
            ModelType::Qwen3,
            ModelType::DINOv3,
        ] {
            queues.insert(model, WorkQueue::new(model));
        }

        Self {
            queues: Arc::new(RwLock::new(queues)),
            state: Arc::new(RwLock::new(ModelStateManager::new())),
            vram_config,
        }
    }

    pub fn submit(&self, item: WorkItem) -> String {
        let id = item.id.clone();
        let model = item.model_type;

        let mut queues = self.queues.write();
          (queue) = queues.(&model) {
            queue.(item);
        }

        id
    }

      (&, dep_id: &) {
          = .queues.();
           queues.() {
            queue.(dep_id);
        }
    }

      (&, max_batch_size: )  <(ModelType, <WorkItem>)> {
          = .state.();
          = .queues.();

        
          (current) = state.() {
              (queue) = queues.(&current) {
                 queue.() >  {
                      = ::();
                     batch.() < max_batch_size {
                          (item) = queue.() {
                            batch.(item);
                        }  {
                            ;
                        }
                    }
                     !batch.() {
                         ((current, batch));
                    }
                }
            }
        }

        
          = queues
            .()
            .(|(_, q)| q.() > )
            .(|(_, q)| q.())
            .(|(m, _)| *m);

          (model) = best_model {
              = queues.(&model).();
              = ::();
             batch.() < max_batch_size {
                  (item) = queue.() {
                    batch.(item);
                }  {
                    ;
                }
            }
             !batch.() {
                 ((model, batch));
            }
        }

        
    }

      (&)   {
        .queues.()
            .()
            .(|q| q.() + q.())
            .()
    }
}

Semaphore-Based VRAM Limiting

use tokio::sync::Semaphore;

// Alternative: Use semaphore for simpler VRAM tracking
pub struct VramSemaphore {
    semaphore: Semaphore,
    units_per_gb: u32,
}

impl VramSemaphore {
    pub fn new(total_gb: f64, units_per_gb: u32) -> Self {
        let total_units = (total_gb * units_per_gb as f64) as usize;
        Self {
            semaphore: Semaphore::new(total_units),
            units_per_gb,
        }
    }

    pub async fn acquire(&self, vram_gb: f64) -> Result<SemaphorePermit<'_>> {
        let units = (vram_gb * self.units_per_gb as f64) as u32;
        self.semaphore.acquire_many(units).await
            .map_err(|_| Error::VramAcquisition)
    }
}

// Usage
let vram = VramSemaphore::new(16.0, 10);  

  (vram: &VramSemaphore)  <()> {
      = vram.().?;  
    
    (())
}

Guidelines

• Track VRAM usage per model type
• Use safety factor (90%) to avoid OOM
• Minimize model switches (prefer current model)
• Support work item dependencies
• Use priority queues for important work
• Track model load times for scheduling decisions
• Use semaphores for simpler VRAM limiting

Examples

See hercules-local-algo/src/scheduler/ for complete implementation.