| name | detecting-shadow-api-endpoints |
| description | Discover and inventory shadow API endpoints that operate outside documented specifications using traffic analysis, code scanning, and API discovery platforms. |
| domain | cybersecurity |
| subdomain | api-security |
| tags | ["api-security","shadow-apis","api-discovery","undocumented-apis","zombie-apis","api-inventory","attack-surface-management","api-governance"] |
| version | 1.0 |
| author | mahipal |
| license | Apache-2.0 |
Detecting Shadow API Endpoints
Overview
Shadow APIs are API endpoints operating within an organization's environment that are not tracked, documented, or secured. They emerge from rapid development cycles, forgotten test environments, deprecated API versions left running, third-party integrations, or developer side projects deployed without governance. Shadow APIs bypass authentication and monitoring controls, creating hidden entry points for attackers. Studies show that up to 30% of API endpoints in large organizations are undocumented, making shadow API detection a critical component of API security posture management.
Prerequisites
- API gateway or reverse proxy with traffic logging (Kong, AWS API Gateway, Envoy)
- Network traffic capture capability (packet broker, port mirroring)
- Access to source code repositories and CI/CD pipeline configurations
- Cloud provider access for configuration scanning (AWS, GCP, Azure)
- API documentation inventory (OpenAPI specs, Swagger docs)
- Python 3.8+ for custom discovery tooling
Detection Methods
1. Traffic Analysis and Comparison
Compare live API traffic against documented OpenAPI specifications to identify undocumented endpoints:
"""Shadow API Endpoint Detector
Compares observed API traffic patterns against documented
OpenAPI specifications to identify undocumented (shadow) endpoints.
"""
import json
import re
import yaml
import sys
from collections import defaultdict
from datetime import datetime
from typing import Dict, List, Set, Tuple, Optional
from dataclasses import dataclass, field
@dataclass
class DiscoveredEndpoint:
method: str
path_pattern: str
first_seen: str
last_seen: str
request_count: int
source_ips: Set[str] = field(default_factory=set)
status_codes: Set[int] = field(default_factory=set)
has_auth_header: bool = False
documented: bool = False
class ShadowAPIDetector:
PARAM_PATTERNS = [
(re.compile(r'/\d+'), '/{id}'),
(re.compile(r'/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'), '/{uuid}'),
(re.(), ),
]
():
.documented_endpoints: [[, ]] = ()
.discovered_endpoints: [[, ], DiscoveredEndpoint] = {}
():
(spec_path, ) f:
spec_path.endswith():
spec = json.load(f)
:
spec = yaml.safe_load(f)
paths = spec.get(, {})
path, methods paths.items():
normalized_path = re.sub(, , path)
method methods:
method.upper() (, , , , , , ):
.documented_endpoints.add((method.upper(), normalized_path))
()
() -> :
path = path.split()[]
pattern, replacement .PARAM_PATTERNS:
path = pattern.sub(replacement, path)
path
():
patterns = {
: re.(
),
:
}
(log_file, ) f:
line f:
log_format == :
:
entry = json.loads(line)
method = entry.get(, entry.get(, ))
path = entry.get(, entry.get(, ))
status = (entry.get(, entry.get(, )))
ip = entry.get(, entry.get(, ))
timestamp = entry.get(, entry.get(, ))
has_auth = (entry.get(, entry.get(, )))
json.JSONDecodeError:
:
= patterns[log_format].(line)
:
method = .group()
path = .group()
status = (.group())
ip = .group()
timestamp = .group()
has_auth = line
path.startswith() path.startswith():
normalized = .normalize_path(path)
key = (method.upper(), normalized)
key .discovered_endpoints:
.discovered_endpoints[key] = DiscoveredEndpoint(
method=method.upper(),
path_pattern=normalized,
first_seen=timestamp,
last_seen=timestamp,
request_count=,
documented=(key .documented_endpoints)
)
endpoint = .discovered_endpoints[key]
endpoint.request_count +=
endpoint.last_seen = timestamp
endpoint.source_ips.add(ip)
endpoint.status_codes.add(status)
has_auth:
endpoint.has_auth_header =
() -> [DiscoveredEndpoint]:
shadows = []
key, endpoint .discovered_endpoints.items():
endpoint.documented:
shadows.append(endpoint)
shadows.sort(key= e: e.request_count, reverse=)
shadows
() -> :
risk_score =
endpoint.has_auth_header:
risk_score +=
endpoint.request_count > :
risk_score +=
endpoint.request_count > :
risk_score +=
(endpoint.source_ips) > :
risk_score +=
endpoint.status_codes endpoint.status_codes:
risk_score +=
endpoint.method (, , , ):
risk_score +=
sensitive_patterns = [, , , , ,
, , , , ]
pattern sensitive_patterns:
pattern endpoint.path_pattern.lower():
risk_score +=
risk_score >= :
risk_score >= :
risk_score >= :
() -> :
shadows = .identify_shadow_apis()
total_documented = (.documented_endpoints)
total_discovered = (.discovered_endpoints)
report = {
: datetime.now().isoformat(),
: {
: total_documented,
: total_discovered,
: (shadows),
: ,
},
: []
}
endpoint shadows:
risk = .classify_risk(endpoint)
report[].append({
: endpoint.method,
: endpoint.path_pattern,
: risk,
: endpoint.request_count,
: (endpoint.source_ips),
: endpoint.has_auth_header,
: (endpoint.status_codes),
: endpoint.first_seen,
: endpoint.last_seen,
})
report
():
detector = ShadowAPIDetector()
spec_files = sys.argv[:] (sys.argv) > []
spec spec_files:
spec.endswith((, , )):
detector.load_openapi_spec(spec)
detector.process_access_log()
report = detector.generate_report()
()
()
()
()
()
()
()
ep report[]:
risk_marker = {: , : , : , : }
()
()
(, ) f:
json.dump(report, f, indent=, default=)
()
__name__ == :
main()
2. Cloud Configuration Scanning
aws apigateway get-rest-apis --query 'items[*].[name,id]' --output table
aws apigatewayv2 get-apis --query 'Items[*].[Name,ApiId,ProtocolType]' --output table
aws lambda list-function-url-configs --function-name "*" 2>/dev/null
aws elbv2 describe-rules --listener-arn $LISTENER_ARN \
--query 'Rules[*].[Priority,Conditions[0].Values[0],Actions[0].TargetGroupArn]'
3. Source Code Repository Mining
grep -rn "app\.\(get\|post\|put\|delete\|patch\)" --include="*.js" --include="*.ts" src/
grep -rn "@app\.route\|@api\.route\|path(" --include="*.py" src/
grep -rn "@\(Get\|Post\|Put\|Delete\|Patch\)Mapping\|@RequestMapping" --include="*.java" src/
diff <(grep -roh "'/api/[^']*'" src/ | sort -u) \
<(yq '.paths | keys[]' openapi.yaml | sort -u)
Prevention and Governance
API Registration Gateway Policy
plugins:
- name: request-validator
config:
allowed_content_types:
- application/json
body_schema: null
- name: pre-function
config:
access:
- |
-- Block requests to unregistered endpoints
local registered = kong.cache:get("registered_endpoints")
local path = kong.request.get_path()
local method = kong.request.get_method()
local key = method .. ":" .. path
if not registered[key] then
kong.log.warn("Shadow API access attempt: ", key)
return kong.response.exit(404, {error = "Endpoint not registered"})
end
References
