Configure Metrics Collection and Dashboards: The agent defines and deploys metric scrapers, exporters, and custom metrics. It builds dashboards that visualize the golden signals (latency, traffic, errors, saturation) and infrastructure metrics (CPU, memory, disk, network). Dashboards are organized by service tier so teams can quickly triage issues.
Provide the agent with your cloud provider, the services to monitor, your preferred monitoring stack, and any existing SLOs or alerting requirements.
Set up monitoring for our Kubernetes microservices on AWS.
- Use Prometheus and Grafana for metrics and dashboards
- Monitor API latency (p99 < 500ms) and error rate (< 1%)
- Send critical alerts to PagerDuty, warnings to Slack
- Aggregate logs with CloudWatch Logs
groups:
- name: slo-alerts
rules:
- alert: HighErrorRate
expr: |
sum(rate(http_requests_total{status=~"5.."}[5m]))
/
sum(rate(http_requests_total[5m])) > 0.01
for: 5m
labels:
severity: critical
annotations:
summary: "Error rate exceeds 1% SLO"
description: "{{ $labels.job }} error rate is {{ $value | humanizePercentage }}"
- alert: HighP99Latency
expr: |
histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))
> 0.5
for: 10m
labels:
severity: warning
annotations:
summary: "P99 latency exceeds 500ms SLO"
- alert: PodCrashLooping
expr: increase(kube_pod_container_status_restarts_total[1h]) > 3
for: 5m
labels:
severity: critical
annotations:
summary: "Pod {{ $labels.pod }} is crash looping"
- alert: HighMemoryUsage
expr: (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes > 0.9
for: 15m
labels:
severity: warning
annotations:
summary: "Node memory usage above 90%"
{
"AWSTemplateFormatVersion": "2010-09-09",
"Resources": {
"ApiDashboard": {
"Type": "AWS::CloudWatch::Dashboard",
"Properties": {
"DashboardName": "api-service-dashboard",
"DashboardBody": "{\"widgets\":[{\"type\":\"metric\",\"properties\":{\"metrics\":[[\"AWS/ApplicationELB\",\"TargetResponseTime\",\"TargetGroup\",\"my-tg\",{\"stat\":\"p99\"}],[\"AWS/ApplicationELB\",\"HTTPCode_Target_5XX_Count\",\"TargetGroup\",\"my-tg\"]],\"period\":300,\"title\":\"API Latency & Errors\"}},{\"type\":\"metric\",\"properties\":{\"metrics\":[[\"Custom/App\",\"ActiveConnections\"],[\"Custom/App\",\"QueueDepth\"]],\"period\":60,\"title\":\"Application Metrics\"}}]}"
}
},
"HighLatencyAlarm": {
"Type": "AWS::CloudWatch::Alarm",
"Properties": {
"AlarmName": "api-high-latency",
"MetricName": "TargetResponseTime",
"Namespace": "AWS/ApplicationELB",
"Statistic": "p99",
"Period": 300,
"EvaluationPeriods": 3,
"Threshold": 0.5,
"ComparisonOperator": "GreaterThanThreshold",
"AlarmActions": ["arn:aws:sns:us-east-1:123456789012:ops-alerts"],
"Dimensions": [
{"Name": "TargetGroup", "Value": "my-tg"}
]
}
},
"HighErrorRateAlarm": {
"Type": "AWS::CloudWatch::Alarm",
"Properties": {
"AlarmName": "api-high-error-rate",
"MetricName": "HTTPCode_Target_5XX_Count",
"Namespace": "AWS/ApplicationELB",
"Statistic": "Sum",
"Period": 300,
"EvaluationPeriods": 2,
"Threshold": 50,
"ComparisonOperator": "GreaterThanThreshold",
"AlarmActions": ["arn:aws:sns:us-east-1:123456789012:ops-alerts"]
}
}
}
}