메뉴
Automate construction of GitHub issue resolution benchmarks via multi-agent LLM coordination, binary file recovery, and exit-code-based validation, achieving 337 valid instances at $0.047 per instance.
기본적으로 소스를 먼저 확인하는 Prompt가 선택됩니다. 직접 명령으로 전환하거나 로컬 사본을 다운로드할 수도 있습니다.
설치 여부를 결정하기 전에 SKILL.md와 SkillsMP에 표시된 보조 파일을 읽어 보세요.
Codex 또는 Claude로 설치 이 Prompt를 복사해 Codex, Claude 또는 다른 어시스턴트에 붙여 넣으면 Skill 페이지를 검토하고 설치를 진행할 수 있습니다.
직접 명령은 검토 Prompt를 거치지 않습니다. 실행하기 전에 소스를 확인하세요.
npx skills add https://github.com/ADu2021/skillXiv --skill swe-factory-benchmark-generation명령은 한 줄로 유지됩니다. 복사하기 전에 가로로 스크롤해 전체 내용을 확인하세요.
로컬 사본을 원하시나요? SkillsMP에서 현재 제공할 수 있는 파일을 다운로드하세요.
SOC 직업 분류 기준
SKILL.md 표시 중
| name | swe-factory-benchmark-generation |
| title | SWE-Factory: Your Automated Factory for Issue Resolution Training Data and Evaluation Benchmarks |
| version | 0.0.2 |
| engine | skillxiv-v0.0.2-claude-opus-4.6 |
| license | MIT |
| url | https://arxiv.org/abs/2506.10954 |
| keywords | ["automated dataset generation","GitHub issues","software engineering","benchmarks","fail2pass validation"] |
| description | Automate construction of GitHub issue resolution benchmarks via multi-agent LLM coordination, binary file recovery, and exit-code-based validation, achieving 337 valid instances at $0.047 per instance. |
SWE-Factory fully automates GitHub issue resolution dataset construction through a multi-agent LLM system that coordinates environment setup, binary file recovery, and fail2pass validation. Instead of manual inspection and custom log parsing, the system uses exit-code-based testing (standardized Unix return codes), successfully generating 337 valid benchmarks from 671 issues across Python, Java, JavaScript, and TypeScript at $0.047 per instance and 96% quality.
import os
import subprocess
from pathlib import Path
class BinaryFileRecoverer:
"""
Automatically downloads missing binary test files from repositories
and removes incomplete binary hunks from patches.
"""
def __init__(self, repo_path, repo_url):
self.repo_path = repo_path
self.repo_url = repo_url
def identify_missing_binaries(self, patch_content):
"""
Parse patch file to identify missing binary files.
Binary patches show "Binary files ... differ" marker.
"""
missing_binaries = []
lines = patch_content.split('\n')
for i, line (lines):
line line:
j (i-, (, i-), -):
lines[j].startswith():
filename = lines[j].split()[].replace(, )
missing_binaries.append(filename)
missing_binaries
():
downloaded = []
filename filenames:
raw_url = .repo_url.replace(, )
raw_url =
:
result = subprocess.run(
[, , , filename, raw_url],
cwd=.repo_path,
timeout=,
capture_output=
)
result.returncode == os.path.exists(os.path.join(.repo_path, filename)):
downloaded.append(filename)
()
Exception e:
()
downloaded
():
lines = patch_content.split()
cleaned = []
skip_binary_section =
i, line (lines):
line line:
skip_binary_section =
cleaned.append(line)
line.startswith() line.startswith():
skip_binary_section =
cleaned.append(line)
skip_binary_section line.startswith((, )):
cleaned.append(line)
.join(cleaned)
class SWEBuilderMultiAgent:
"""
Orchestrates four collaborative agents to construct evaluation environment.
Uses shared memory pool for configuration reuse.
"""
def __init__(self, repo_path, model_name='gpt-4-mini'):
self.repo_path = repo_path
self.model = model_name
self.env_memory_pool = {}
def repository_explorer(self):
"""Agent 1: Collects environment dependencies and test commands."""
exploration_prompt = f"""
Analyze repository at {self.repo_path}:
1. Identify all dependency files (requirements.txt, pom.xml, package.json, etc.)
2. Extract all test commands (pytest, maven, npm test, etc.)
3. List any configuration files (setup.cfg, tsconfig.json, etc.)
4. Identify language versions needed (Python 3.9+, Java 11+, etc.)
Return structured JSON with sections: dependencies, test_commands, configs, versions
"""
return self._llm_call(exploration_prompt)
def environment_manager(self, exploration_result):
"""Agent 2: Generates Dockerfile configuration."""
docker_prompt = f"""
Based on repository analysis:
{exploration_result}
Generate complete Dockerfile that:
1. Installs base image matching language version requirements
2. Installs all dependencies in correct order
3. Sets environment variables
4. Copies repository code
5. Runs any setup scripts
Output valid Dockerfile content.
"""
docker_content = self._llm_call(docker_prompt)
# Check memory pool for similar configurations
similar = self._find_similar_config(exploration_result)
if similar:
print(f"Found similar configuration: {similar}")
docker_content = self._merge_configs(docker_content, similar)
return docker_content
def test_manager(self, exploration_result):
"""Agent 3: Creates executable test scripts with exit code reporting."""
test_prompt = f"""
Based on identified test commands:
{exploration_result['test_commands']}
Create a master test script that:
1. Sets up test environment
2. Runs each test command sequentially
3. Captures exit codes
4. Appends standardized marker: OMNIGRIL_EXIT_CODE=$rc
5. Returns appropriate exit code (0=pass, 1=fail)
Language: {self._detect_language()}
"""
test_script = self._llm_call(test_prompt)
return test_script
def test_analyst(self, docker_content, test_script):
"""Agent 4: Validates environment and orchestrates refinement."""
# Try to build Docker image
build_success = self._build_docker(docker_content)
if not build_success:
# Iteratively refine via LLM
refinement_prompt = f"""
Docker build failed. Previous Dockerfile:
{docker_content}
Common issues:
- Missing base image layer
- Dependency installation order
- Working directory not set
Provide corrected Dockerfile.
"""
docker_content = self._llm_call(refinement_prompt)
return self.test_analyst(docker_content, test_script)
# Build passed; store in memory pool
self.env_memory_pool[hash(docker_content)] = {
'dockerfile': docker_content,
'test_script': test_script,
'success': True
}
return {'dockerfile': docker_content, 'test_script': test_script}
def _find_similar_config(self, exploration_result):
"""Find similar configurations in memory pool."""
repo_hash = hash(exploration_result.get('dependencies', ''))
# Simple similarity: same language/version
for stored_hash, config in self.env_memory_pool.items():
if config['success']:
return config
return None
def _merge_configs(self, new_config, similar_config):
"""Merge new config with similar successful one."""
# Simplified: prefer new, fallback to similar
return new_config
def _llm_call(self, prompt):
"""Call LLM with prompt."""
# Simplified placeholder
return f"Generated from: {prompt[:50]}..."
def _detect_language(self):
"""Detect primary language of repository."""
if os.path.exists(os.path.join(self.repo_path, 'requirements.txt')):
return 'python'
elif os.path.exists(os.path.join(self.repo_path, 'pom.xml')):
return 'java'
elif os.path.exists(os.path.join(self.repo_path, 'package.json')):
return 'javascript'
return 'unknown'
def _build_docker(self, dockerfile_content):
"""Test build Docker image."""
# Simplified: return True for now
return True
import re
import subprocess
class ExitCodeValidator:
"""
Standardized fail2pass validation using Unix exit codes.
Eliminates custom log parsing; achieves 100% precision, 98.3% recall.
"""
def create_test_wrapper(self, original_test_command, language):
"""
Wraps test command to append standardized exit code marker.
Format: OMNIGRIL_EXIT_CODE=$rc
"""
if language == 'python':
wrapper = f"""#!/bin/bash
set -e
bash -c '{original_test_command}'
rc=$?
echo "OMNIGRIL_EXIT_CODE=$rc"
exit $rc
"""
elif language == 'java':
wrapper = f"""#!/bin/bash
set -e
{original_test_command}
rc=$?
echo "OMNIGRIL_EXIT_CODE=$rc"
exit $rc
"""
else:
wrapper = f"""#!/bin/bash
set -e
{original_test_command}
rc=$?
echo "OMNIGRIL_EXIT_CODE=$rc"
exit $rc
"""
return wrapper
def extract_exit_code(self, test_output):
"""
Parse test output and extract exit code marker.
Returns (exit_code, is_valid).
"""
pattern = r'OMNIGRIL_EXIT_CODE=(\d+)'
match = re.search(pattern, test_output)
if match:
exit_code = int(match.group(1))
return exit_code, True
return None, False
def validate_fail2pass(self, buggy_version_output, fixed_version_output):
"""
Verify that test failed on buggy version and passed on fixed version.
Returns (is_valid, buggy_code, fixed_code).
"""
buggy_code, buggy_valid = self.extract_exit_code(buggy_version_output)
fixed_code, fixed_valid = self.extract_exit_code(fixed_version_output)
if not (buggy_valid and fixed_valid):
return False, None, None
# Fail2pass: buggy version failed (non-zero), fixed version passed (zero)
is_fail2pass = (buggy_code != 0) and (fixed_code == 0)
return is_fail2pass, buggy_code, fixed_code
Dataset Construction Workflow:
Cost Optimization:
Quality Assurance:
Supported Languages: