メニュー
Evaluate and optimize tool-integrated reasoning in LLMs through empirical benchmarking, performance-cost metrics (PAC, AUC-PCC), and measurement frameworks for diverse reasoning tasks.
デフォルトでは、最初にソースを確認する Prompt が選択されています。直接コマンドに切り替えるか、ローカルコピーをダウンロードすることもできます。
インストールを決める前に、SKILL.md と SkillsMP に表示されている付属ファイルをお読みください。
Codex または Claude でインストール この Prompt をコピーして Codex、Claude、または他のアシスタントに貼り付けると、Skill ページを確認してインストールできます。
直接コマンドでは確認用 Prompt が省略されます。実行前にソースを確認してください。
npx skills add https://github.com/ADu2021/skillXiv --skill dissecting-tool-integrated-reasoningコマンドは1行のまま表示されます。コピー前に横へスクロールして全体を確認してください。
ローカルで確認しますか？SkillsMP が現在取得できるファイルをダウンロードできます。
SKILL.md を表示中
SOC 職業分類に基づく
| name | dissecting-tool-integrated-reasoning |
| title | Tool-Integrated Reasoning: Empirical Benchmarking and Efficiency Metrics |
| version | 0.0.2 |
| engine | skillxiv-v0.0.2-claude-opus-4.6 |
| license | MIT |
| url | https://arxiv.org/abs/2508.15754 |
| keywords | ["tool-integration","reasoning-efficiency","chain-of-thought","external-tools","benchmarking"] |
| description | Evaluate and optimize tool-integrated reasoning in LLMs through empirical benchmarking, performance-cost metrics (PAC, AUC-PCC), and measurement frameworks for diverse reasoning tasks. |
Tool-Integrated Reasoning (TIR) enables language models to offload computational tasks to external tools (calculators, code interpreters, etc.) rather than solving everything through token generation. This approach reduces "overthinking" and produces more streamlined reasoning traces while improving accuracy. The key innovation is measuring TIR effectiveness beyond raw accuracy through efficiency metrics like Performance-At-Cost (PAC) and Area-Under-Curve-PCC (AUC-PCC), enabling systematic evaluation across diverse reasoning domains.
Create comprehensive benchmark covering diverse reasoning types:
from enum import Enum
from dataclasses import dataclass
class ReasoningCategory(Enum):
ARITHMETIC = "arithmetic"
ALGEBRA = "algebra"
GEOMETRY = "geometry"
LOGIC = "logic"
COUNTING = "counting"
KNOWLEDGE = "knowledge"
SYMBOLIC = "symbolic"
COMMONSENSE = "commonsense"
MULTI_STEP = "multi_step"
@dataclass
class BenchmarkTask:
task_id: str
category: ReasoningCategory
question: str
expected_answer: str
tools_required: list[]
requires_reasoning:
expected_tool_calls:
() -> [BenchmarkTask]:
tasks = []
tasks.append(BenchmarkTask(
task_id=,
category=ReasoningCategory.ARITHMETIC,
question=,
expected_answer=,
tools_required=[],
requires_reasoning=,
expected_tool_calls=
))
tasks.append(BenchmarkTask(
task_id=,
category=ReasoningCategory.MULTI_STEP,
question=,
expected_answer=,
tools_required=[],
requires_reasoning=,
expected_tool_calls=
))
tasks
Create abstraction for external tool binding:
from abc import ABC, abstractmethod
from typing import Any, Dict
class ExternalTool(ABC):
@abstractmethod
def execute(self, input_str: str) -> str:
pass
class Calculator(ExternalTool):
def execute(self, expression: str) -> str:
"""Evaluate mathematical expressions safely."""
try:
result = eval(expression, {"__builtins__": {}}, {})
return str(result)
except Exception as e:
return f"Error: {str(e)}"
class CodeInterpreter(ExternalTool):
def execute(self, code: str) -> str:
"""Execute Python code safely in sandbox."""
import subprocess
try:
result = subprocess.run(
["python", "-c", code],
capture_output=True,
text=True,
timeout=5
)
return result.stdout or result.stderr
except Exception as e:
return f"Error: {str(e)}"
class ToolRouter:
def __init__(self):
self.tools = {
"calculator": Calculator(),
"code_interpreter": CodeInterpreter(),
"search": SearchTool(),
}
def route_and_execute(self, tool_name: str, tool_input: str) -> str:
"""Route execution request to appropriate tool."""
if tool_name not in self.tools:
return f"Unknown tool: {tool_name}"
return self.tools[tool_name].execute(tool_input)
Capture detailed reasoning execution for analysis:
@dataclass
class ReasoningTrace:
task_id: str
model_response: str
tool_calls: list[Dict[str, Any]]
reasoning_steps: list[str]
final_answer: str
correct: bool
tokens_generated: int
tool_execution_time: float
total_time: float
class ReasoningTraceCollector:
def __init__(self, tool_router: ToolRouter):
self.tool_router = tool_router
self.traces = []
def collect_trace(
self,
task: BenchmarkTask,
model: "LLM"
) -> ReasoningTrace:
"""
Execute task and collect detailed reasoning trace.
"""
trace = ReasoningTrace(
task_id=task.task_id,
model_response="",
tool_calls=[],
reasoning_steps=[],
final_answer="",
correct=False,
tokens_generated=0,
tool_execution_time=0.0,
total_time=0.0
)
# Generate response with tool integration
start_time = time.time()
response, tool_calls, steps = model.generate_with_tools(
task.question,
available_tools=task.tools_required
)
trace.model_response = response
trace.tokens_generated = len(response.split())
trace.reasoning_steps = steps
# Execute tool calls
tool_start = time.time()
for tool_call in tool_calls:
result = self.tool_router.route_and_execute(
tool_call["tool_name"],
tool_call["input"]
)
tool_call["result"] = result
trace.tool_calls.append(tool_call)
trace.tool_execution_time = time.time() - tool_start
trace.total_time = time.time() - start_time
# Extract final answer and check correctness
trace.final_answer = extract_answer(response)
trace.correct = trace.final_answer == task.expected_answer
return trace
Measure efficiency beyond accuracy:
def compute_pac_metric(
traces: list[ReasoningTrace],
cost_weights: Dict[str, float] = None
) -> float:
"""
Compute Performance-At-Cost metric balancing accuracy and efficiency.
PAC = Accuracy / (1 + normalized_cost)
where cost includes tokens and tool calls
"""
if cost_weights is None:
cost_weights = {
"token": 0.001, # per token cost
"tool_call": 0.1, # per tool invocation cost
"time": 0.01 # per second cost
}
total_accuracy = 0.0
total_cost = 0.0
for trace in traces:
# Accuracy component
total_accuracy += 1.0 if trace.correct else 0.0
# Cost component
token_cost = trace.tokens_generated * cost_weights["token"]
tool_cost = len(trace.tool_calls) * cost_weights["tool_call"]
time_cost = trace.total_time * cost_weights["time"]
total_cost += token_cost + tool_cost + time_cost
avg_accuracy = total_accuracy / len(traces)
normalized_cost = total_cost / len(traces)
pac = avg_accuracy / (1.0 + normalized_cost)
return pac
Measure efficiency frontier:
from sklearn.metrics import auc
def compute_auc_pcc(
traces: list[ReasoningTrace],
cost_budgets: list[float]
) -> float:
"""
Compute AUC of Performance vs Cost Curve.
Shows how accuracy scales with different cost budgets
(token limits, time limits, tool call limits)
"""
accuracies = []
costs = []
# Sort by total cost
sorted_traces = sorted(traces, key=lambda t: t.total_time)
cumulative_cost = 0.0
cumulative_correct = 0
for i, trace in enumerate(sorted_traces):
cumulative_cost += trace.total_time
if trace.correct:
cumulative_correct += 1
accuracy = cumulative_correct / (i + 1)
accuracies.append(accuracy)
costs.append(cumulative_cost / (i + 1))
# Normalize costs to [0, 1]
min_cost = min(costs)
max_cost = max(costs)
normalized_costs = [(c - min_cost) / (max_cost - min_cost) for c in costs]
# Compute AUC
auc_pcc = auc(normalized_costs, accuracies)
return auc_pcc
def evaluate_tool_integration(
model: "LLM",
benchmark: list[BenchmarkTask]
) -> Dict[str, float]:
"""
Comprehensive evaluation of tool-integrated reasoning.
"""
collector = ReasoningTraceCollector(ToolRouter())
traces = [collector.collect_trace(task, model) for task in benchmark]
metrics = {
"accuracy": sum(1.0 for t in traces if t.correct) / len(traces),
"avg_tokens": sum(t.tokens_generated for t in traces) / len(traces),
"avg_tool_calls": sum(len(t.tool_calls) for t in traces) / len(traces),
"avg_time": sum(t.total_time for t in traces) / len(traces),
"pac": compute_pac_metric(traces),
"auc_pcc": compute_auc_pcc(traces, cost_budgets=[0.5, 1.0, 2.0])
}
return metrics
Researchers. (2024). Dissecting Tool-Integrated Reasoning: An Empirical Study and Analysis. arXiv preprint arXiv:2508.15754.