ONNX Runtime
Overview
ONNX Runtime is a high-performance inference engine for running models in the Open Neural Network Exchange (ONNX) format. It enables .NET applications to run models trained in PyTorch, TensorFlow, scikit-learn, or any framework that exports to ONNX. The runtime supports multiple execution providers (CPU, CUDA, DirectML, TensorRT) for hardware-accelerated inference with minimal code changes.
NuGet Packages
dotnet add package Microsoft.ML.OnnxRuntime
dotnet add package Microsoft.ML.OnnxRuntime.Gpu
dotnet add package Microsoft.ML.OnnxRuntime.DirectML
Basic Inference
using Microsoft.ML.OnnxRuntime;
using Microsoft.ML.OnnxRuntime.Tensors;
using var session = new InferenceSession("model.onnx");
foreach (var input in session.InputMetadata)
{
Console.WriteLine($"Input: {input.Key}, Shape: [{string.Join(",", input.Value.Dimensions)}], Type: {input.Value.ElementType}");
}
foreach (var output in session.OutputMetadata)
{
Console.WriteLine($"Output: {output.Key}, Shape: [{string.Join(",", output.Value.Dimensions)}], Type: {output.Value.ElementType}");
}
var inputData = new float[] { 1.0f, 2.0f, 3.0f, 4.0f };
var inputTensor = new DenseTensor<float>(inputData, new[] { 1, 4 });
var inputs = new List<NamedOnnxValue>
{
NamedOnnxValue.CreateFromTensor("input", inputTensor)
};
using IDisposableReadOnlyCollection<DisposableNamedOnnxValue> results = session.Run(inputs);
var output = results.First().AsTensor<float>();
Console.WriteLine($"Output shape: [{string.Join(",", output.Dimensions.ToArray())}]");
Console.WriteLine($"Values: [{string.Join(", ", output.ToArray())}]");
Image Classification
using Microsoft.ML.OnnxRuntime;
using Microsoft.ML.OnnxRuntime.Tensors;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
public class ImageClassifier : IDisposable
{
private readonly InferenceSession _session;
private readonly string[] _labels;
public ImageClassifier(string modelPath, string labelsPath)
{
_session = new InferenceSession(modelPath);
_labels = File.ReadAllLines(labelsPath);
}
public (string Label, float Confidence) Classify(string imagePath)
{
using var image = Image.Load<Rgb24>(imagePath);
image.Mutate(x => x.Resize(224, 224));
var tensor = new DenseTensor<float>(new[] { 1, 3, 224, 224 });
for (int y = 0; y < 224; y++)
{
for (int x = 0; x < ; x++)
{
pixel = image[x, y];
tensor[, , y, x] = (pixel.R / - ) / ;
tensor[, , y, x] = (pixel.G / - ) / ;
tensor[, , y, x] = (pixel.B / - ) / ;
}
}
inputs = List<NamedOnnxValue>
{
NamedOnnxValue.CreateFromTensor(, tensor)
};
results = _session.Run(inputs);
probabilities = Softmax(results.First().AsTensor<>().ToArray());
maxIndex = Array.IndexOf(probabilities, probabilities.Max());
(_labels[maxIndex], probabilities[maxIndex]);
}
{
max = logits.Max();
exps = logits.Select(x => MathF.Exp(x - max)).ToArray();
sum = exps.Sum();
exps.Select(x => x / sum).ToArray();
}
=> _session.Dispose();
}
classifier = ImageClassifier(, );
(label, confidence) = classifier.Classify();
Console.WriteLine();
Execution Providers
Configure hardware acceleration by selecting execution providers.
var cpuOptions = new SessionOptions();
using var cpuSession = new InferenceSession("model.onnx", cpuOptions);
var cudaOptions = new SessionOptions();
cudaOptions.AppendExecutionProvider_CUDA(deviceId: 0);
using var gpuSession = new InferenceSession("model.onnx", cudaOptions);
var dmlOptions = new SessionOptions();
dmlOptions.AppendExecutionProvider_DML(deviceId: 0);
using var dmlSession = new InferenceSession("model.onnx", dmlOptions);
var trtOptions = new SessionOptions();
trtOptions.AppendExecutionProvider_Tensorrt(deviceId: 0);
trtOptions.AppendExecutionProvider_CUDA(deviceId: 0);
using var trtSession = new InferenceSession("model.onnx", trtOptions);
Execution Provider Comparison
| Provider | Package | Platform | Best For |
|---|
| CPU | OnnxRuntime | All | Default, universally available |
| CUDA | OnnxRuntime.Gpu | Linux/Windows + NVIDIA | High-throughput GPU inference |
| DirectML | OnnxRuntime.DirectML | Windows | Cross-vendor GPU (AMD, Intel, NVIDIA) |
| TensorRT | OnnxRuntime.Gpu | Linux + NVIDIA | Maximum NVIDIA throughput |
| CoreML | OnnxRuntime | macOS/iOS | Apple Silicon optimization |
Session Options and Optimization
var options = new SessionOptions
{
GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL,
IntraOpNumThreads = Environment.ProcessorCount,
InterOpNumThreads = 2,
EnableMemoryPattern = true,
OptimizedModelFilePath = "model_optimized.onnx"
};
using var session = new InferenceSession("model.onnx", options);
Batch Inference
public class BatchPredictor : IDisposable
{
private readonly InferenceSession _session;
public BatchPredictor(string modelPath)
{
var options = new SessionOptions
{
GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL
};
_session = new InferenceSession(modelPath, options);
}
public float[][] PredictBatch(float[][] inputs, int featureCount)
{
int batchSize = inputs.Length;
var flatInput = new float[batchSize * featureCount];
for (int i = 0; i < batchSize; i++)
{
Array.Copy(inputs[i], 0, flatInput, i * featureCount, featureCount);
}
var tensor = new DenseTensor<float>(flatInput, new[] { batchSize, featureCount });
var onnxInputs = new List<NamedOnnxValue>
{
NamedOnnxValue.CreateFromTensor("input", tensor)
};
using var results = _session.Run(onnxInputs);
var outputTensor = results.First().AsTensor<float>();
int outputSize = outputTensor.Dimensions[1];
var outputs = new [batchSize][];
( i = ; i < batchSize; i++)
{
outputs[i] = [outputSize];
( j = ; j < outputSize; j++)
{
outputs[i][j] = outputTensor[i, j];
}
}
outputs;
}
=> _session.Dispose();
}
ASP.NET Core Integration
var builder = WebApplication.CreateBuilder(args);
builder.Services.AddSingleton(sp =>
{
var options = new SessionOptions
{
GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL,
IntraOpNumThreads = 4
};
return new InferenceSession("model.onnx", options);
});
var app = builder.Build();
app.MapPost("/predict", (PredictRequest request, InferenceSession session) =>
{
var tensor = new DenseTensor<float>(
request.Features, new[] { 1, request.Features.Length });
var inputs = new List<NamedOnnxValue>
{
NamedOnnxValue.CreateFromTensor("input", tensor)
};
using var results = session.Run(inputs);
var output = results.First().AsTensor<float>().ToArray();
return Results.Ok(new { predictions = output });
});
app.Run();
record PredictRequest(float[] Features);
Text Embedding with ONNX
public class TextEmbedder : IDisposable
{
private readonly InferenceSession _session;
public TextEmbedder(string modelPath)
{
_session = new InferenceSession(modelPath);
}
public float[] Embed(long[] inputIds, long[] attentionMask)
{
var idsTensor = new DenseTensor<long>(inputIds, new[] { 1, inputIds.Length });
var maskTensor = new DenseTensor<long>(attentionMask, new[] { 1, attentionMask.Length });
var inputs = new List<NamedOnnxValue>
{
NamedOnnxValue.CreateFromTensor("input_ids", idsTensor),
NamedOnnxValue.CreateFromTensor("attention_mask", maskTensor)
};
using var results = _session.Run(inputs);
var embeddings = results.First().AsTensor<float>();
int seqLen = inputIds.Length;
int hiddenSize = embeddings.Dimensions[2];
var pooled = new float[hiddenSize];
int validTokens = attentionMask.Sum(m => (int)m);
( h = ; h < hiddenSize; h++)
{
sum = ;
( s = ; s < seqLen; s++)
{
(attentionMask[s] == )
sum += embeddings[, s, h];
}
pooled[h] = sum / validTokens;
}
pooled;
}
=> _session.Dispose();
}
Best Practices
- Register
InferenceSession as a singleton in DI because it is thread-safe for Run() calls and expensive to initialize; creating a session per request causes severe performance degradation.
- Inspect
session.InputMetadata and session.OutputMetadata at startup to verify input/output names, shapes, and data types match your code rather than relying on documentation that may be outdated.
- Use
GraphOptimizationLevel.ORT_ENABLE_ALL in SessionOptions to enable constant folding, node fusion, and memory planning optimizations that reduce inference latency by 10-30%.
- Set
OptimizedModelFilePath on SessionOptions to save the optimized graph to disk; subsequent loads skip optimization and start faster.
- Batch multiple inputs into a single tensor (e.g., shape
[N, features]) and call session.Run once rather than looping N times, reducing overhead from session lock acquisition and memory allocation.
- Pre-allocate
DenseTensor<T> instances and reuse them across inference calls in hot paths to avoid repeated heap allocations that trigger garbage collection pauses.
- Add CUDA or DirectML execution providers with a CPU fallback (
AppendExecutionProvider_CUDA followed by default CPU) so the application runs on any machine regardless of GPU availability.
- Normalize input data consistently with the model's training preprocessing (e.g., ImageNet mean/std for vision models, tokenizer for NLP models); mismatched normalization is the most common cause of incorrect predictions.
- Use
session.Run(inputs, outputNames) with explicit output names when the model has multiple outputs to avoid deserializing outputs you do not need.
- Monitor inference latency with
Stopwatch or OpenTelemetry and set up alerts for p99 latency regressions that may indicate model size issues or execution provider misconfiguration.
