vllm-lora

name	vllm-lora
description	LoRA adapter serving, multi-LoRA deployment
triggers	["When user wants to serve LoRA adapters","When user needs multi-LoRA concurrent serving","When user wants to combine LoRA with quantization","When user needs dynamic adapter loading"]

vllm-lora

Overview

LoRA (Low-Rank Adaptation) allows efficient fine-tuning of LLMs. vLLM supports serving multiple LoRA adapters concurrently on a single base model.

Prerequisites

• vLLM 0.5.0+ installed
• Base model and LoRA adapter weights
• Sufficient GPU memory (LoRA adds small overhead)

Main Workflow

Step 1: Basic LoRA Serving

from vllm import LLM, SamplingParams
from vllm.lora.request import LoRARequest

# Load base model with LoRA support
llm = LLM(
    model="meta-llama/Llama-2-7b-hf",
    enable_lora=True,
    max_loras=1,  # Number of concurrent LoRAs
    max_lora_rank=8
)

# Create LoRA request
lora_request = LoRARequest(
    lora_name="sql_adapter",
    lora_int_id=1,
    lora_path="path/to/sql-lora-adapter"
)

# Generate with LoRA
output = llm.generate(
    "Generate a SQL query to list all users",
    SamplingParams(max_tokens=100),
    lora_request=lora_request
)
print(output[0].outputs[0].text)

Step 2: Multi-LoRA Serving

from vllm import LLM, SamplingParams
from vllm.lora.request import LoRARequest

# Enable multiple LoRAs
llm = LLM(
    model="meta-llama/Llama-2-7b-hf",
    enable_lora=True,
    max_loras=4,      # Support 4 concurrent LoRAs
    max_lora_rank=8,
    max_cpu_loras=16  # Can swap from CPU
)

# Define multiple LoRA adapters
loras = {
    "sql": LoRARequest("sql_adapter", 1, "path/to/sql-lora"),
    "code": LoRARequest("code_adapter", 2, "path/to/code-lora"),
    "medical": LoRARequest("medical_adapter", 3, "path/to/medical-lora"),
}

# Use different LoRAs for different requests
prompts_with_lora = [
    ("Write SQL to join tables", loras["sql"]),
    ("Implement quicksort in Python", loras["code"]),
    ("Explain diabetes symptoms", loras["medical"]),
]

for prompt, lora in prompts_with_lora:
    output = llm.generate(prompt, SamplingParams(max_tokens=100), lora_request=lora)
    print(f"[{lora.lora_name}] {output[0].outputs[0].text[:50]}...")

Step 3: API Server with LoRA

# Start server with LoRA support
vllm serve meta-llama/Llama-2-7b-hf \
  --enable-lora \
  --lora-modules sql=path/to/sql-lora,code=path/to/code-lora \
  --max-loras 4

from openai import OpenAI

client = OpenAI(base_url="http://localhost:8000/v1", api_key="dummy")

# Use specific LoRA
response = client.chat.completions.create(
    model="sql",  # Use LoRA module name
    messages=[{"role": "user", "content": "List all tables"}]
)

Step 4: LoRA with Quantization

from vllm import LLM

# Combine LoRA with AWQ quantization
llm = LLM(
    model="TheBloke/Llama-2-7B-AWQ",
    quantization="awq",
    enable_lora=True,
    max_loras=2
)

Common Patterns

Pattern 1: Dynamic LoRA Loading

from vllm import LLM
from vllm.lora.request import LoRARequest

llm = LLM(
    model="meta-llama/Llama-2-7b-hf",
    enable_lora=True,
    max_loras=8,
    max_cpu_loras=32  # Can load from CPU
)

def load_lora(adapter_path: str, name: str):
    """Dynamically load a LoRA adapter."""
    lora_id = hash(name) % 10000
    return LoRARequest(
        lora_name=name,
        lora_int_id=lora_id,
        lora_path=adapter_path
    )

# Load on demand
sql_lora = load_lora("adapters/sql-lora", "sql_specialist")
output = llm.generate("Query", SamplingParams(), lora_request=sql_lora)

Pattern 2: LoRA Adapter Management

class LoRAManager:
    def __init__(self, llm):
        self.llm = llm
        self.loaded_loras = {}
        self.next_id = 1

    def register(self, name: str, path: str):
        """Register a new LoRA adapter."""
        if name in self.loaded_loras:
            return self.loaded_loras[name]

        lora_req = LoRARequest(
            lora_name=name,
            lora_int_id=self.next_id,
            lora_path=path
        )
        self.loaded_loras[name] = lora_req
        self.next_id += 1
        return lora_req

    def get(self, name: str):
        return self.loaded_loras.get(name)

# Usage
llm = LLM(model="model", enable_lora=True, max_loras=4)
manager = LoRAManager(llm)

manager.register("sql", "path/to/sql-lora")
manager.register("code", "path/to/code-lora")

Pattern 3: A/B Testing with LoRA

from vllm import LLM, SamplingParams

llm = LLM(model="model", enable_lora=True, max_loras=2)

# Load both variants
lora_a = LoRARequest("variant_a", 1, "path/to/lora-a")
lora_b = LoRARequest("variant_b", 2, "path/to/lora-b")

# Test same prompt with both
test_prompt = "Generate a product description"

out_a = llm.generate(test_prompt, SamplingParams(), lora_request=lora_a)
out_b = llm.generate(test_prompt, SamplingParams(), lora_request=lora_b)

print("Variant A:", out_a[0].outputs[0].text)
print("Variant B:", out_b[0].outputs[0].text)

Troubleshooting

Problem: LoRA not found

Solution:

# Verify LoRA path
import os
print(os.listdir("path/to/lora-adapter"))

# Should contain: adapter_config.json, adapter_model.bin

Problem: LoRA rank mismatch

Solution:

# Check adapter config
import json
with open("path/to/lora-adapter/adapter_config.json") as f:
    config = json.load(f)
print(f"LoRA rank: {config['r']}")

# Match in LLM init
llm = LLM(
    model="model",
    enable_lora=True,
    max_lora_rank=config["r"]  # Match adapter rank
)

Problem: OOM with multiple LoRAs

Solution:

llm = LLM(
    model="model",
    enable_lora=True,
    max_loras=2,       # Reduce concurrent LoRAs
    max_cpu_loras=8,   # Keep others on CPU
    gpu_memory_utilization=0.85
)

References

• LoRA Serving Guide - Advanced LoRA serving patterns