Normalize all collected data into a common schema. Create a unified JSON structure that tags each finding with its source, timestamp, and data type:
cat > /tmp/osint/normalize.py << 'EOF'
import json, csv, sys, os
from datetime import datetime
findings = []
sherlock_path = "/tmp/osint/sherlock-results.txt"
if os.path.exists(sherlock_path):
with open(sherlock_path) as f:
for row in csv.DictReader(f):
findings.append({
"source": "sherlock",
"type": "social_profile",
"platform": row.get("name", ""),
"url": row.get("url_user", ""),
"username": row.get("username", ""),
"status": row.get("status", ""),
"collected_at": datetime.utcnow().isoformat()
})
harvester_path = "/tmp/osint/harvester-results.json"
if os.path.exists(harvester_path):
with open(harvester_path) as f:
data = json.load(f)
for email in data.get("emails", []):
findings.append({
"source": "theHarvester",
"type": "email",
"value": email,
"collected_at": datetime.utcnow().isoformat()
})
for host in data.get("hosts", []):
findings.append({
"source": "theHarvester",
"type": "hostname",
"value": host,
"collected_at": datetime.utcnow().isoformat()
})
sf_path = "/tmp/osint/spiderfoot-results.json"
if os.path.exists(sf_path):
with open(sf_path) as f:
for item in json.load(f):
findings.append({
"source": "spiderfoot",
"type": item.get("type", "unknown"),
"value": item.get("data", ""),
"module": item.get("module", ""),
"collected_at": datetime.utcnow().isoformat()
})
with open("/tmp/osint/normalized-findings.json", "w") as f:
json.dump(findings, f, indent=2)
print(f"Normalized {len(findings)} findings from {len(set(f['source'] for f in findings))} sources")
EOF
python3 /tmp/osint/normalize.py
