菜单
Discover that large pretrained models have dense neighborhoods of task-specific experts—random weight perturbations improve performance. Use RandOpt: sample perturbations, select top performers, ensemble via voting for multi-task adaptation.
默认使用会先检查来源的 Prompt；你也可以切换为直接命令，或下载本地副本。
决定是否安装前，请先阅读 SKILL.md，以及 SkillsMP 当前展示的配套文件。
用 Codex 或 Claude 帮你安装 复制这段 Prompt，粘贴到 Codex、Claude 或其他助手里，让它检查 Skill 页面并帮你完成安装。
直接命令不会经过审查 Prompt；运行前请先检查来源。
npx skills add https://github.com/ADu2021/skillXiv --skill neural-thickets-dense-task-experts命令会保持在同一行。复制前请横向滚动并检查完整内容。
想先保存到本地？可下载 SkillsMP 当前能够提供的文件。
基于 SOC 职业分类
正在显示 SKILL.md
| name | neural-thickets-dense-task-experts |
| title | Neural Thickets: Diverse Task Experts Are Dense Around Pretrained Weights |
| version | 0.0.2 |
| engine | skillxiv-v0.0.2-claude-opus-4.6 |
| license | MIT |
| url | https://arxiv.org/abs/2603.12228 |
| keywords | ["Pretraining","Adaptation","Ensembling","Task Specialization","Model Landscape"] |
| description | Discover that large pretrained models have dense neighborhoods of task-specific experts—random weight perturbations improve performance. Use RandOpt: sample perturbations, select top performers, ensemble via voting for multi-task adaptation. |
Traditional fine-tuning assumes isolated optimal solutions in the loss landscape. Neural Thickets reveals a different regime for large pretrained models: the neighborhood around pretrained weights contains abundant diverse task-improving specialists. This "thicket regime" enables a surprisingly simple approach: randomly perturb the pretrained weights, select performers, and ensemble via majority voting.
This emerges from the fundamental difference in loss landscapes between small models ("needle in haystack" sparsity) and large pretrained models (dense solution neighborhoods).
RandOpt operates in two phases:
Training Phase: Generate N random Gaussian perturbations of pretrained weights θ' = θ + σ·ϵ, evaluate on validation set, select top-K performers.
Inference Phase: Generate predictions using only the K selected models, aggregate via majority voting.
The method requires no additional training—just sampling, selection, and ensemble voting. Yet it often outperforms standard fine-tuning by 5-15% on diverse tasks.
Sample Gaussian noise, create perturbed model copies, and evaluate on validation set.
import torch
import numpy as np
class RandOptEnsemble:
def __init__(self, base_model, perturbation_scale=0.1, k_selected=5):
self.base_model = base_model
self.perturbation_scale = perturbation_scale
self.k_selected = k_selected
self.selected_models = []
.base_weights = .get_model_weights(base_model)
():
weights = []
param model.parameters():
weights.append(param.data.clone().flatten())
torch.cat(weights)
():
offset =
param model.parameters():
param_size = param.numel()
param.data = weights[offset:offset + param_size].reshape(param.shape)
offset += param_size
():
perturbed_models = []
_ (num_perturbations):
perturbation = torch.randn_like(.base_weights) * .perturbation_scale
perturbed_weights = .base_weights + perturbation
perturbed_model = .clone_model(.base_model)
.set_model_weights(perturbed_model, perturbed_weights)
perturbed_models.append(perturbed_model)
perturbed_models
():
copy
copy.deepcopy(model)
Benchmark perturbations on validation set, retain only the best performers.
def select_top_k_models(
perturbed_models,
validation_data,
task_metric,
k=5,
batch_size=32
):
"""
Evaluate all perturbations, return top-k by validation metric.
task_metric: function(outputs, targets) -> score
"""
model_scores = []
for idx, model in enumerate(perturbed_models):
model.eval()
total_score = 0
num_batches = 0
with torch.no_grad():
for batch_idx, (inputs, targets) in enumerate(validation_data):
if batch_idx * batch_size >= 1000: # Use subset for speed
break
outputs = model(inputs)
score = task_metric(outputs, targets)
total_score += score.item()
num_batches += 1
avg_score = total_score / num_batches
model_scores.append((idx, avg_score, model))
# Sort by performance
model_scores.sort(key=lambda x: x[1], reverse=True)
# Select top-k
selected = model_scores[:k]
return [model for _, _, model in selected]
For classification, aggregate predictions across ensemble members.
class RandOptEnsembleInference:
def __init__(self, selected_models):
self.selected_models = selected_models
def forward(self, inputs):
"""
inputs: batch of examples
returns: ensemble predictions via majority voting
"""
batch_size = inputs.shape[0]
num_models = len(self.selected_models)
# Collect predictions from all models
all_predictions = []
for model in self.selected_models:
model.eval()
with torch.no_grad():
outputs = model(inputs)
# For classification: argmax to get class
predictions = torch.argmax(outputs, dim=-1) # (batch_size,)
all_predictions.append(predictions)
# Stack: (num_models, batch_size)
all_predictions = torch.stack(all_predictions)
# Majority voting
ensemble_predictions = torch.mode(all_predictions, dim=0)[0]
return ensemble_predictions
def forward_with_confidence(self, inputs):
"""
Also return confidence from voting agreement.
"""
batch_size = inputs.shape[0]
num_models = len(self.selected_models)
all_predictions = []
for model in self.selected_models:
model.eval()
with torch.no_grad():
outputs = model(inputs)
predictions = torch.argmax(outputs, dim=-1)
all_predictions.append(predictions)
all_predictions = torch.stack(all_predictions)
ensemble_predictions = torch.mode(all_predictions, dim=0)[0]
# Confidence: fraction voting for majority
agreement = (all_predictions == ensemble_predictions.unsqueeze(0)).float()
confidence = agreement.mean(dim=0) # (batch_size,)
return ensemble_predictions, confidence
Extend RandOpt to multi-task scenarios with shared perturbations.
def randopt_multitask_adaptation(
base_model,
tasks,
perturbation_scale=0.1,
num_perturbations=100,
k_per_task=5
):
"""
Adapt base model to multiple tasks via perturbations.
tasks: list of (task_name, validation_data, metric_fn)
"""
# Generate shared perturbations
base_weights = torch.cat([p.data.flatten() for p in base_model.parameters()])
perturbed_models_all = []
for _ in range(num_perturbations):
perturbation = torch.randn_like(base_weights) * perturbation_scale
perturbed_weights = base_weights + perturbation
perturbed_models_all.append(perturbed_weights)
# Per-task selection
task_ensembles = {}
for task_name, val_data, metric_fn in tasks:
# Evaluate all perturbations on this task
model_scores = []
for weight_vec in perturbed_models_all:
# Create model with these weights
temp_model = create_model_with_weights(base_model, weight_vec)
# Evaluate
score = evaluate_model(temp_model, val_data, metric_fn)
model_scores.append((score, weight_vec))
# Select top-k for this task
model_scores.sort(key=lambda x: x[0], reverse=True)
top_weights = [w for _, w in model_scores[:k_per_task]]
task_ensembles[task_name] = top_weights
return task_ensembles
End-to-end example showing RandOpt for adaptation.
def adapt_model_with_randopt(
base_model,
validation_data,
test_data,
task_metric,
num_perturbations=100,
perturbation_scale=0.1,
k_selected=5
):
"""
Full RandOpt pipeline: perturb, select, ensemble.
"""
# Step 1: Generate perturbations
ensemble = RandOptEnsemble(base_model, perturbation_scale, k_selected)
perturbed_models = ensemble.generate_perturbed_models(num_perturbations)
# Step 2: Select top-k
selected_models = select_top_k_models(
perturbed_models,
validation_data,
task_metric,
k=k_selected
)
print(f"Selected {len(selected_models)} specialist models from {num_perturbations} perturbations")
# Step 3: Ensemble inference
ensemble_model = RandOptEnsembleInference(selected_models)
# Evaluate on test set
total_correct = 0
total_samples = 0
with torch.no_grad():
for inputs, targets in test_data:
predictions, confidence = ensemble_model.forward_with_confidence(inputs)
correct = (predictions == targets).sum().item()
total_correct += correct
total_samples += targets.shape[0]
accuracy = total_correct / total_samples
print(f"Test Accuracy: {accuracy:.4f}")
return ensemble_model, accuracy
When to Use:
When NOT to Use:
Hyperparameter Tuning:
Common Pitfalls: