mirror of
https://github.com/BagelHole/DevOps-Security-Agent-Skills.git
synced 2026-08-22 12:49:53 +02:00
.
This commit is contained in:
Toby
@@ -0,0 +1,498 @@
|
||||
---
|
||||
name: alerting-oncall
|
||||
description: Set up alerting rules, configure on-call rotations, and manage incident response workflows. Integrate with PagerDuty, Opsgenie, or Grafana OnCall for alert routing and escalation. Use when implementing alerting strategies and on-call management for production systems.
|
||||
license: MIT
|
||||
metadata:
|
||||
author: devops-skills
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
# Alerting & On-Call
|
||||
|
||||
Configure effective alerting and on-call management for production systems.
|
||||
|
||||
## When to Use This Skill
|
||||
|
||||
Use this skill when:
|
||||
- Setting up alerting rules and thresholds
|
||||
- Configuring on-call rotations and schedules
|
||||
- Implementing alert routing and escalation
|
||||
- Reducing alert fatigue
|
||||
- Managing incident response workflows
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Monitoring system (Prometheus, Datadog, etc.)
|
||||
- On-call platform (PagerDuty, Opsgenie, Grafana OnCall)
|
||||
- Communication channels (Slack, email)
|
||||
|
||||
## Alerting Best Practices
|
||||
|
||||
### Alert Categories
|
||||
|
||||
```yaml
|
||||
# Severity levels
|
||||
critical:
|
||||
- Service completely down
|
||||
- Data loss imminent
|
||||
- Security breach
|
||||
response: Immediate page, wake people up
|
||||
|
||||
high:
|
||||
- Service degraded significantly
|
||||
- Error rate above SLO
|
||||
- Capacity near limit
|
||||
response: Page during business hours, notify after hours
|
||||
|
||||
medium:
|
||||
- Performance degradation
|
||||
- Non-critical component failure
|
||||
- Warning thresholds exceeded
|
||||
response: Notify via Slack, review next business day
|
||||
|
||||
low:
|
||||
- Informational alerts
|
||||
- Capacity planning triggers
|
||||
- Routine maintenance needed
|
||||
response: Email notification, weekly review
|
||||
```
|
||||
|
||||
### Alert Design Principles
|
||||
|
||||
```yaml
|
||||
# Good alert characteristics
|
||||
alerts:
|
||||
actionable:
|
||||
- Every alert should require human action
|
||||
- Include runbook links
|
||||
- Clear remediation steps
|
||||
|
||||
relevant:
|
||||
- Alert on symptoms, not causes
|
||||
- Focus on user impact
|
||||
- Avoid alerting on expected behavior
|
||||
|
||||
timely:
|
||||
- Appropriate thresholds
|
||||
- Suitable evaluation windows
|
||||
- Account for normal variance
|
||||
|
||||
unique:
|
||||
- No duplicate alerts
|
||||
- Proper alert grouping
|
||||
- Clear ownership
|
||||
```
|
||||
|
||||
## Prometheus Alerting
|
||||
|
||||
### Alert Rules
|
||||
|
||||
```yaml
|
||||
# prometheus/rules/alerts.yml
|
||||
groups:
|
||||
- name: service_alerts
|
||||
rules:
|
||||
# High-level service health
|
||||
- alert: ServiceDown
|
||||
expr: up{job="myapp"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Service {{ $labels.instance }} is down"
|
||||
description: "{{ $labels.job }} on {{ $labels.instance }} has been down for more than 1 minute."
|
||||
runbook_url: "https://wiki.example.com/runbooks/service-down"
|
||||
|
||||
# Error rate alert
|
||||
- alert: HighErrorRate
|
||||
expr: |
|
||||
sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
|
||||
/ sum(rate(http_requests_total[5m])) by (service) > 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "High error rate for {{ $labels.service }}"
|
||||
description: "Error rate is {{ $value | humanizePercentage }} for the last 5 minutes"
|
||||
|
||||
# Latency alert (SLO-based)
|
||||
- alert: HighLatency
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
|
||||
) > 0.5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: high
|
||||
annotations:
|
||||
summary: "P95 latency above 500ms for {{ $labels.service }}"
|
||||
```
|
||||
|
||||
### Alertmanager Configuration
|
||||
|
||||
```yaml
|
||||
# alertmanager.yml
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
slack_api_url: 'https://hooks.slack.com/services/xxx'
|
||||
pagerduty_url: 'https://events.pagerduty.com/v2/enqueue'
|
||||
|
||||
templates:
|
||||
- '/etc/alertmanager/templates/*.tmpl'
|
||||
|
||||
route:
|
||||
receiver: 'default-receiver'
|
||||
group_by: ['alertname', 'service']
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
|
||||
routes:
|
||||
# Critical alerts go to PagerDuty
|
||||
- match:
|
||||
severity: critical
|
||||
receiver: 'pagerduty-critical'
|
||||
group_wait: 0s
|
||||
repeat_interval: 1h
|
||||
|
||||
# High severity during business hours
|
||||
- match:
|
||||
severity: high
|
||||
receiver: 'slack-high'
|
||||
active_time_intervals:
|
||||
- business-hours
|
||||
|
||||
# Route by team
|
||||
- match_re:
|
||||
team: platform.*
|
||||
receiver: 'platform-team'
|
||||
|
||||
receivers:
|
||||
- name: 'default-receiver'
|
||||
slack_configs:
|
||||
- channel: '#alerts'
|
||||
send_resolved: true
|
||||
|
||||
- name: 'pagerduty-critical'
|
||||
pagerduty_configs:
|
||||
- service_key: 'xxx'
|
||||
severity: critical
|
||||
description: '{{ .CommonAnnotations.summary }}'
|
||||
details:
|
||||
firing: '{{ template "pagerduty.firing" . }}'
|
||||
|
||||
- name: 'slack-high'
|
||||
slack_configs:
|
||||
- channel: '#alerts-high'
|
||||
title: '{{ .CommonAnnotations.summary }}'
|
||||
text: '{{ .CommonAnnotations.description }}'
|
||||
actions:
|
||||
- type: button
|
||||
text: 'Runbook'
|
||||
url: '{{ .CommonAnnotations.runbook_url }}'
|
||||
- type: button
|
||||
text: 'Dashboard'
|
||||
url: '{{ .CommonAnnotations.dashboard_url }}'
|
||||
|
||||
- name: 'platform-team'
|
||||
slack_configs:
|
||||
- channel: '#platform-alerts'
|
||||
|
||||
time_intervals:
|
||||
- name: business-hours
|
||||
time_intervals:
|
||||
- weekdays: ['monday:friday']
|
||||
times:
|
||||
- start_time: '09:00'
|
||||
end_time: '17:00'
|
||||
|
||||
inhibit_rules:
|
||||
- source_match:
|
||||
severity: critical
|
||||
target_match:
|
||||
severity: high
|
||||
equal: ['service']
|
||||
```
|
||||
|
||||
## PagerDuty Integration
|
||||
|
||||
### Service Configuration
|
||||
|
||||
```yaml
|
||||
# Terraform example
|
||||
resource "pagerduty_service" "myapp" {
|
||||
name = "MyApp Production"
|
||||
description = "Production application service"
|
||||
escalation_policy = pagerduty_escalation_policy.default.id
|
||||
alert_creation = "create_alerts_and_incidents"
|
||||
auto_resolve_timeout = 14400 # 4 hours
|
||||
acknowledgement_timeout = 600 # 10 minutes
|
||||
|
||||
incident_urgency_rule {
|
||||
type = "use_support_hours"
|
||||
|
||||
during_support_hours {
|
||||
type = "constant"
|
||||
urgency = "high"
|
||||
}
|
||||
|
||||
outside_support_hours {
|
||||
type = "constant"
|
||||
urgency = "low"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "pagerduty_escalation_policy" "default" {
|
||||
name = "Default Escalation"
|
||||
num_loops = 2
|
||||
|
||||
rule {
|
||||
escalation_delay_in_minutes = 10
|
||||
target {
|
||||
type = "schedule_reference"
|
||||
id = pagerduty_schedule.primary.id
|
||||
}
|
||||
}
|
||||
|
||||
rule {
|
||||
escalation_delay_in_minutes = 15
|
||||
target {
|
||||
type = "user_reference"
|
||||
id = pagerduty_user.manager.id
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Schedule Configuration
|
||||
|
||||
```yaml
|
||||
resource "pagerduty_schedule" "primary" {
|
||||
name = "Primary On-Call"
|
||||
time_zone = "America/New_York"
|
||||
|
||||
layer {
|
||||
name = "Weekly Rotation"
|
||||
start = "2024-01-01T00:00:00-05:00"
|
||||
rotation_virtual_start = "2024-01-01T00:00:00-05:00"
|
||||
rotation_turn_length_seconds = 604800 # 1 week
|
||||
users = [for user in pagerduty_user.oncall : user.id]
|
||||
}
|
||||
|
||||
# Override layer for holidays
|
||||
layer {
|
||||
name = "Holiday Coverage"
|
||||
start = "2024-01-01T00:00:00-05:00"
|
||||
rotation_virtual_start = "2024-01-01T00:00:00-05:00"
|
||||
rotation_turn_length_seconds = 86400
|
||||
users = [pagerduty_user.holiday_coverage.id]
|
||||
|
||||
restriction {
|
||||
type = "daily_restriction"
|
||||
start_time_of_day = "00:00:00"
|
||||
duration_seconds = 86400
|
||||
start_day_of_week = 0 # Sunday
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Grafana OnCall
|
||||
|
||||
### Integration Setup
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml addition
|
||||
services:
|
||||
oncall:
|
||||
image: grafana/oncall
|
||||
environment:
|
||||
- SECRET_KEY=your-secret-key
|
||||
- BASE_URL=http://oncall:8080
|
||||
- GRAFANA_API_URL=http://grafana:3000
|
||||
ports:
|
||||
- "8080:8080"
|
||||
```
|
||||
|
||||
### Escalation Chain
|
||||
|
||||
```yaml
|
||||
# Example escalation chain structure
|
||||
escalation_chains:
|
||||
- name: "Production Critical"
|
||||
steps:
|
||||
- step: 1
|
||||
type: notify
|
||||
persons:
|
||||
- "@oncall-primary"
|
||||
wait_delay: 0
|
||||
|
||||
- step: 2
|
||||
type: notify
|
||||
persons:
|
||||
- "@oncall-secondary"
|
||||
wait_delay: 5m
|
||||
|
||||
- step: 3
|
||||
type: notify
|
||||
persons:
|
||||
- "@engineering-manager"
|
||||
wait_delay: 10m
|
||||
|
||||
- step: 4
|
||||
type: trigger_action
|
||||
action: "escalate_to_incident_commander"
|
||||
wait_delay: 15m
|
||||
```
|
||||
|
||||
## Alert Templates
|
||||
|
||||
### Slack Alert Template
|
||||
|
||||
```go
|
||||
{{ define "slack.title" }}
|
||||
[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }}
|
||||
{{ end }}
|
||||
|
||||
{{ define "slack.text" }}
|
||||
{{ range .Alerts }}
|
||||
*Alert:* {{ .Annotations.summary }}
|
||||
*Severity:* {{ .Labels.severity }}
|
||||
*Description:* {{ .Annotations.description }}
|
||||
*Runbook:* {{ .Annotations.runbook_url }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
### PagerDuty Details Template
|
||||
|
||||
```go
|
||||
{{ define "pagerduty.firing" }}
|
||||
{{ range .Alerts.Firing }}
|
||||
Alert: {{ .Labels.alertname }}
|
||||
Service: {{ .Labels.service }}
|
||||
Instance: {{ .Labels.instance }}
|
||||
Value: {{ .Annotations.value }}
|
||||
Started: {{ .StartsAt.Format "2006-01-02 15:04:05" }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
```
|
||||
|
||||
## On-Call Best Practices
|
||||
|
||||
### Rotation Guidelines
|
||||
|
||||
```yaml
|
||||
on_call_guidelines:
|
||||
rotation_length: 1 week
|
||||
handoff_time: "10:00 AM Monday"
|
||||
|
||||
responsibilities:
|
||||
- Monitor alerts during shift
|
||||
- Respond within SLA (critical: 5min, high: 15min)
|
||||
- Document incidents
|
||||
- Handoff unresolved issues
|
||||
|
||||
support:
|
||||
- Secondary on-call for backup
|
||||
- Clear escalation path
|
||||
- Manager availability for major incidents
|
||||
|
||||
wellness:
|
||||
- Maximum 1 week on-call per month
|
||||
- Comp time after high-alert periods
|
||||
- No-interrupt recovery day after shift
|
||||
```
|
||||
|
||||
### Runbook Template
|
||||
|
||||
```markdown
|
||||
# Alert: High Error Rate
|
||||
|
||||
## Summary
|
||||
Error rate has exceeded the threshold of 5% for the service.
|
||||
|
||||
## Impact
|
||||
Users may experience errors when accessing the application.
|
||||
|
||||
## Investigation Steps
|
||||
1. Check service logs: `kubectl logs -l app=myapp -n production`
|
||||
2. Review recent deployments: `kubectl rollout history deployment/myapp`
|
||||
3. Check database connectivity: `kubectl exec -it myapp -- nc -zv postgres 5432`
|
||||
4. Review error traces in APM dashboard
|
||||
|
||||
## Remediation
|
||||
### If caused by recent deployment:
|
||||
```bash
|
||||
kubectl rollout undo deployment/myapp -n production
|
||||
```
|
||||
|
||||
### If database related:
|
||||
```bash
|
||||
kubectl delete pod -l app=postgres -n production
|
||||
```
|
||||
|
||||
## Escalation
|
||||
If not resolved within 15 minutes, escalate to:
|
||||
- Database team: @db-oncall
|
||||
- Platform team: @platform-oncall
|
||||
```
|
||||
|
||||
## Alert Fatigue Reduction
|
||||
|
||||
### Strategies
|
||||
|
||||
```yaml
|
||||
fatigue_reduction:
|
||||
aggregate_alerts:
|
||||
- Group related alerts
|
||||
- Use inhibit rules
|
||||
- Implement alert correlation
|
||||
|
||||
tune_thresholds:
|
||||
- Base on SLOs, not arbitrary values
|
||||
- Account for normal variance
|
||||
- Use appropriate evaluation windows
|
||||
|
||||
automate_responses:
|
||||
- Auto-remediation for known issues
|
||||
- Self-healing infrastructure
|
||||
- Automated scaling
|
||||
|
||||
regular_review:
|
||||
- Weekly alert review
|
||||
- Remove unused alerts
|
||||
- Update thresholds based on data
|
||||
```
|
||||
|
||||
## Common Issues
|
||||
|
||||
### Issue: Alert Storm
|
||||
**Problem**: Too many alerts firing simultaneously
|
||||
**Solution**: Implement proper grouping and inhibition rules
|
||||
|
||||
### Issue: Missed Alerts
|
||||
**Problem**: Critical alerts not reaching on-call
|
||||
**Solution**: Test escalation policies, verify contact methods
|
||||
|
||||
### Issue: False Positives
|
||||
**Problem**: Alerts firing without actual issues
|
||||
**Solution**: Tune thresholds, increase evaluation windows
|
||||
|
||||
## Best Practices
|
||||
|
||||
- Define clear severity levels
|
||||
- Every alert needs a runbook
|
||||
- Test on-call notifications regularly
|
||||
- Review and tune alerts weekly
|
||||
- Implement proper escalation paths
|
||||
- Use alert grouping and inhibition
|
||||
- Track alert metrics (MTTR, frequency)
|
||||
- Practice incident response regularly
|
||||
|
||||
## Related Skills
|
||||
|
||||
- [prometheus-grafana](../prometheus-grafana/) - Monitoring setup
|
||||
- [incident-response](../../../security/operations/incident-response/) - Incident handling
|
||||
- [runbook-automation](../../../compliance/continuity/runbook-automation/) - Runbook creation
|
||||
@@ -0,0 +1,463 @@
|
||||
---
|
||||
name: datadog
|
||||
description: Implement Datadog monitoring and APM for infrastructure and applications. Configure agents, create dashboards, set up alerts, and implement distributed tracing. Use when implementing enterprise monitoring, APM, or unified observability platforms.
|
||||
license: MIT
|
||||
metadata:
|
||||
author: devops-skills
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
# Datadog
|
||||
|
||||
Monitor infrastructure and applications with Datadog's unified observability platform.
|
||||
|
||||
## When to Use This Skill
|
||||
|
||||
Use this skill when:
|
||||
- Implementing enterprise-grade monitoring
|
||||
- Setting up APM and distributed tracing
|
||||
- Creating unified dashboards for infrastructure and apps
|
||||
- Configuring intelligent alerting
|
||||
- Monitoring cloud infrastructure (AWS, Azure, GCP)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Datadog account and API key
|
||||
- Agent installation access
|
||||
- Application code access for APM
|
||||
|
||||
## Agent Installation
|
||||
|
||||
### Linux
|
||||
|
||||
```bash
|
||||
# Install agent
|
||||
DD_API_KEY=<YOUR_API_KEY> DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_agent7.sh)"
|
||||
|
||||
# Or via package manager
|
||||
apt-get update && apt-get install datadog-agent
|
||||
|
||||
# Configure API key
|
||||
echo "api_key: YOUR_API_KEY" >> /etc/datadog-agent/datadog.yaml
|
||||
|
||||
# Start agent
|
||||
systemctl start datadog-agent
|
||||
systemctl enable datadog-agent
|
||||
```
|
||||
|
||||
### Docker
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
datadog-agent:
|
||||
image: gcr.io/datadoghq/agent:7
|
||||
environment:
|
||||
- DD_API_KEY=${DD_API_KEY}
|
||||
- DD_SITE=datadoghq.com
|
||||
- DD_LOGS_ENABLED=true
|
||||
- DD_APM_ENABLED=true
|
||||
- DD_PROCESS_AGENT_ENABLED=true
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- /proc/:/host/proc/:ro
|
||||
- /sys/fs/cgroup/:/host/sys/fs/cgroup:ro
|
||||
ports:
|
||||
- "8126:8126" # APM
|
||||
- "8125:8125/udp" # DogStatsD
|
||||
```
|
||||
|
||||
### Kubernetes
|
||||
|
||||
```bash
|
||||
# Using Helm
|
||||
helm repo add datadog https://helm.datadoghq.com
|
||||
|
||||
helm install datadog datadog/datadog \
|
||||
--set datadog.apiKey=${DD_API_KEY} \
|
||||
--set datadog.site=datadoghq.com \
|
||||
--set datadog.logs.enabled=true \
|
||||
--set datadog.apm.portEnabled=true \
|
||||
--set datadog.processAgent.enabled=true \
|
||||
--namespace datadog \
|
||||
--create-namespace
|
||||
```
|
||||
|
||||
## Agent Configuration
|
||||
|
||||
```yaml
|
||||
# /etc/datadog-agent/datadog.yaml
|
||||
api_key: YOUR_API_KEY
|
||||
site: datadoghq.com
|
||||
|
||||
# Hostname
|
||||
hostname: myserver.example.com
|
||||
|
||||
# Tags applied to all metrics
|
||||
tags:
|
||||
- env:production
|
||||
- service:myapp
|
||||
- team:platform
|
||||
|
||||
# Log collection
|
||||
logs_enabled: true
|
||||
|
||||
# APM
|
||||
apm_config:
|
||||
enabled: true
|
||||
apm_dd_url: https://trace.agent.datadoghq.com
|
||||
|
||||
# Process monitoring
|
||||
process_config:
|
||||
enabled: true
|
||||
|
||||
# Container monitoring
|
||||
container_collect_all: true
|
||||
docker_labels_as_tags:
|
||||
app: service
|
||||
environment: env
|
||||
```
|
||||
|
||||
## Integration Configuration
|
||||
|
||||
### MySQL
|
||||
|
||||
```yaml
|
||||
# /etc/datadog-agent/conf.d/mysql.d/conf.yaml
|
||||
init_config:
|
||||
|
||||
instances:
|
||||
- host: localhost
|
||||
port: 3306
|
||||
username: datadog
|
||||
password: <PASSWORD>
|
||||
tags:
|
||||
- env:production
|
||||
options:
|
||||
replication: true
|
||||
extra_status_metrics: true
|
||||
```
|
||||
|
||||
### PostgreSQL
|
||||
|
||||
```yaml
|
||||
# /etc/datadog-agent/conf.d/postgres.d/conf.yaml
|
||||
init_config:
|
||||
|
||||
instances:
|
||||
- host: localhost
|
||||
port: 5432
|
||||
username: datadog
|
||||
password: <PASSWORD>
|
||||
dbname: mydb
|
||||
collect_activity_metrics: true
|
||||
collect_database_size_metrics: true
|
||||
```
|
||||
|
||||
### NGINX
|
||||
|
||||
```yaml
|
||||
# /etc/datadog-agent/conf.d/nginx.d/conf.yaml
|
||||
init_config:
|
||||
|
||||
instances:
|
||||
- nginx_status_url: http://localhost:80/nginx_status
|
||||
tags:
|
||||
- env:production
|
||||
```
|
||||
|
||||
## Log Collection
|
||||
|
||||
### File-Based Logs
|
||||
|
||||
```yaml
|
||||
# /etc/datadog-agent/conf.d/myapp.d/conf.yaml
|
||||
logs:
|
||||
- type: file
|
||||
path: /var/log/myapp/*.log
|
||||
service: myapp
|
||||
source: python
|
||||
sourcecategory: custom
|
||||
tags:
|
||||
- env:production
|
||||
|
||||
- type: file
|
||||
path: /var/log/nginx/access.log
|
||||
service: nginx
|
||||
source: nginx
|
||||
log_processing_rules:
|
||||
- type: exclude_at_match
|
||||
name: exclude_healthchecks
|
||||
pattern: health_check
|
||||
```
|
||||
|
||||
### Docker Logs
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
services:
|
||||
myapp:
|
||||
labels:
|
||||
com.datadoghq.ad.logs: '[{"source": "python", "service": "myapp"}]'
|
||||
```
|
||||
|
||||
### Kubernetes Logs
|
||||
|
||||
```yaml
|
||||
# Pod annotation
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
annotations:
|
||||
ad.datadoghq.com/myapp.logs: |
|
||||
[{
|
||||
"source": "python",
|
||||
"service": "myapp",
|
||||
"log_processing_rules": [{
|
||||
"type": "multi_line",
|
||||
"name": "python_tracebacks",
|
||||
"pattern": "^Traceback"
|
||||
}]
|
||||
}]
|
||||
```
|
||||
|
||||
## APM Configuration
|
||||
|
||||
### Python
|
||||
|
||||
```python
|
||||
from ddtrace import patch_all, tracer
|
||||
|
||||
# Automatic instrumentation
|
||||
patch_all()
|
||||
|
||||
# Configure tracer
|
||||
tracer.configure(
|
||||
hostname='localhost',
|
||||
port=8126,
|
||||
service='myapp',
|
||||
env='production',
|
||||
version='1.0.0'
|
||||
)
|
||||
|
||||
# Manual instrumentation
|
||||
@tracer.wrap(service='myapp', resource='process_order')
|
||||
def process_order(order_id):
|
||||
with tracer.trace('validate_order') as span:
|
||||
span.set_tag('order_id', order_id)
|
||||
# Validation logic
|
||||
|
||||
with tracer.trace('save_order'):
|
||||
# Save logic
|
||||
pass
|
||||
```
|
||||
|
||||
```bash
|
||||
# Install library
|
||||
pip install ddtrace
|
||||
|
||||
# Run with auto-instrumentation
|
||||
ddtrace-run python app.py
|
||||
```
|
||||
|
||||
### Node.js
|
||||
|
||||
```javascript
|
||||
const tracer = require('dd-trace').init({
|
||||
service: 'myapp',
|
||||
env: 'production',
|
||||
version: '1.0.0',
|
||||
logInjection: true
|
||||
});
|
||||
|
||||
// Manual instrumentation
|
||||
const span = tracer.startSpan('custom_operation');
|
||||
span.setTag('user_id', userId);
|
||||
// ... operation
|
||||
span.finish();
|
||||
```
|
||||
|
||||
```bash
|
||||
# Install library
|
||||
npm install dd-trace
|
||||
|
||||
# Run with auto-instrumentation
|
||||
DD_TRACE_ENABLED=true node --require dd-trace/init app.js
|
||||
```
|
||||
|
||||
### Go
|
||||
|
||||
```go
|
||||
import (
|
||||
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
|
||||
)
|
||||
|
||||
func main() {
|
||||
tracer.Start(
|
||||
tracer.WithService("myapp"),
|
||||
tracer.WithEnv("production"),
|
||||
tracer.WithServiceVersion("1.0.0"),
|
||||
)
|
||||
defer tracer.Stop()
|
||||
|
||||
// Manual span
|
||||
span, ctx := tracer.StartSpanFromContext(ctx, "process_request")
|
||||
defer span.Finish()
|
||||
span.SetTag("user_id", userID)
|
||||
}
|
||||
```
|
||||
|
||||
## Custom Metrics
|
||||
|
||||
### DogStatsD
|
||||
|
||||
```python
|
||||
from datadog import DogStatsd
|
||||
|
||||
statsd = DogStatsd(host='localhost', port=8125)
|
||||
|
||||
# Counter
|
||||
statsd.increment('myapp.orders.count', tags=['env:production'])
|
||||
|
||||
# Gauge
|
||||
statsd.gauge('myapp.queue.size', queue_size, tags=['queue:orders'])
|
||||
|
||||
# Histogram
|
||||
statsd.histogram('myapp.request.duration', response_time)
|
||||
|
||||
# Distribution
|
||||
statsd.distribution('myapp.response_time', duration, tags=['endpoint:/api/orders'])
|
||||
```
|
||||
|
||||
### API Submission
|
||||
|
||||
```python
|
||||
from datadog_api_client import Configuration, ApiClient
|
||||
from datadog_api_client.v2.api.metrics_api import MetricsApi
|
||||
from datadog_api_client.v2.model.metric_payload import MetricPayload
|
||||
from datadog_api_client.v2.model.metric_series import MetricSeries
|
||||
from datadog_api_client.v2.model.metric_point import MetricPoint
|
||||
|
||||
configuration = Configuration()
|
||||
with ApiClient(configuration) as api_client:
|
||||
api = MetricsApi(api_client)
|
||||
|
||||
payload = MetricPayload(
|
||||
series=[
|
||||
MetricSeries(
|
||||
metric="custom.metric.name",
|
||||
type=MetricSeries.GAUGE,
|
||||
points=[MetricPoint(value=42.0, timestamp=int(time.time()))],
|
||||
tags=["env:production"]
|
||||
)
|
||||
]
|
||||
)
|
||||
api.submit_metrics(body=payload)
|
||||
```
|
||||
|
||||
## Dashboards
|
||||
|
||||
### Dashboard JSON
|
||||
|
||||
```json
|
||||
{
|
||||
"title": "Application Overview",
|
||||
"widgets": [
|
||||
{
|
||||
"definition": {
|
||||
"type": "timeseries",
|
||||
"title": "Request Rate",
|
||||
"requests": [
|
||||
{
|
||||
"q": "sum:trace.http.request.hits{service:myapp}.as_rate()",
|
||||
"display_type": "line"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"definition": {
|
||||
"type": "query_value",
|
||||
"title": "Error Rate",
|
||||
"requests": [
|
||||
{
|
||||
"q": "sum:trace.http.request.errors{service:myapp}.as_rate() / sum:trace.http.request.hits{service:myapp}.as_rate() * 100"
|
||||
}
|
||||
],
|
||||
"precision": 2
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Monitors (Alerts)
|
||||
|
||||
### Metric Monitor
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "High Error Rate",
|
||||
"type": "metric alert",
|
||||
"query": "sum(last_5m):sum:trace.http.request.errors{service:myapp}.as_count() / sum:trace.http.request.hits{service:myapp}.as_count() > 0.05",
|
||||
"message": "Error rate is {{value}}% for {{service.name}}. @slack-alerts",
|
||||
"tags": ["service:myapp", "env:production"],
|
||||
"options": {
|
||||
"thresholds": {
|
||||
"critical": 0.05,
|
||||
"warning": 0.02
|
||||
},
|
||||
"notify_no_data": true,
|
||||
"no_data_timeframe": 10
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### APM Monitor
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "High Latency Alert",
|
||||
"type": "trace-analytics alert",
|
||||
"query": "trace-analytics(\"service:myapp @http.status_code:2*\").rollup(\"avg\", \"@duration\").last(\"5m\") > 2000000000",
|
||||
"message": "Average latency is above 2 seconds. @pagerduty",
|
||||
"options": {
|
||||
"thresholds": {
|
||||
"critical": 2000000000
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Common Issues
|
||||
|
||||
### Issue: Agent Not Reporting
|
||||
**Problem**: No data appearing in Datadog
|
||||
**Solution**: Check API key, verify agent status with `datadog-agent status`
|
||||
|
||||
### Issue: Missing Traces
|
||||
**Problem**: APM traces not appearing
|
||||
**Solution**: Verify APM is enabled, check tracer configuration, verify port 8126
|
||||
|
||||
### Issue: High Cardinality Tags
|
||||
**Problem**: Custom metrics getting dropped
|
||||
**Solution**: Reduce unique tag values, use distributions instead of histograms
|
||||
|
||||
## Best Practices
|
||||
|
||||
- Use consistent service and environment tags
|
||||
- Implement proper tag naming conventions
|
||||
- Use unified service tagging (service, env, version)
|
||||
- Set up service-level monitors
|
||||
- Create dashboards per service
|
||||
- Implement log correlation with traces
|
||||
- Use distributions for latency metrics
|
||||
- Configure proper alert escalation
|
||||
|
||||
## Related Skills
|
||||
|
||||
- [prometheus-grafana](../prometheus-grafana/) - Open source alternative
|
||||
- [alerting-oncall](../alerting-oncall/) - Alert management
|
||||
- [aws-vpc](../../../infrastructure/cloud-aws/aws-vpc/) - AWS monitoring
|
||||
@@ -0,0 +1,119 @@
|
||||
# Datadog Integration Reference
|
||||
|
||||
## Agent Configuration
|
||||
|
||||
```yaml
|
||||
# /etc/datadog-agent/datadog.yaml
|
||||
api_key: YOUR_API_KEY
|
||||
site: datadoghq.com
|
||||
hostname: my-host
|
||||
tags:
|
||||
- env:production
|
||||
- team:platform
|
||||
|
||||
logs_enabled: true
|
||||
apm_config:
|
||||
enabled: true
|
||||
process_config:
|
||||
enabled: true
|
||||
```
|
||||
|
||||
## Docker Integration
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
datadog-agent:
|
||||
image: gcr.io/datadoghq/agent:latest
|
||||
environment:
|
||||
- DD_API_KEY=${DD_API_KEY}
|
||||
- DD_SITE=datadoghq.com
|
||||
- DD_LOGS_ENABLED=true
|
||||
- DD_APM_ENABLED=true
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- /proc/:/host/proc/:ro
|
||||
- /sys/fs/cgroup:/host/sys/fs/cgroup:ro
|
||||
```
|
||||
|
||||
## Kubernetes Integration
|
||||
|
||||
```yaml
|
||||
# Datadog Agent Helm values
|
||||
datadog:
|
||||
apiKey: <API_KEY>
|
||||
site: datadoghq.com
|
||||
|
||||
logs:
|
||||
enabled: true
|
||||
containerCollectAll: true
|
||||
|
||||
apm:
|
||||
portEnabled: true
|
||||
|
||||
processAgent:
|
||||
enabled: true
|
||||
processCollection: true
|
||||
|
||||
clusterAgent:
|
||||
enabled: true
|
||||
metricsProvider:
|
||||
enabled: true
|
||||
```
|
||||
|
||||
## Custom Metrics
|
||||
|
||||
```python
|
||||
from datadog import statsd
|
||||
|
||||
# Counter
|
||||
statsd.increment('page.views')
|
||||
|
||||
# Gauge
|
||||
statsd.gauge('users.online', 123)
|
||||
|
||||
# Histogram
|
||||
statsd.histogram('request.duration', 0.5)
|
||||
|
||||
# Distribution
|
||||
statsd.distribution('request.size', 1024)
|
||||
```
|
||||
|
||||
## Log Integration
|
||||
|
||||
```python
|
||||
import logging
|
||||
import json_log_formatter
|
||||
|
||||
formatter = json_log_formatter.JSONFormatter()
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(formatter)
|
||||
|
||||
logger = logging.getLogger()
|
||||
logger.addHandler(handler)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
logger.info('Request processed', extra={
|
||||
'dd.trace_id': trace_id,
|
||||
'dd.span_id': span_id,
|
||||
'user_id': user_id
|
||||
})
|
||||
```
|
||||
|
||||
## Monitors (Terraform)
|
||||
|
||||
```hcl
|
||||
resource "datadog_monitor" "cpu_high" {
|
||||
name = "High CPU Usage"
|
||||
type = "metric alert"
|
||||
message = "CPU usage is high. @slack-alerts"
|
||||
|
||||
query = "avg(last_5m):avg:system.cpu.user{*} by {host} > 80"
|
||||
|
||||
monitor_thresholds {
|
||||
critical = 80
|
||||
warning = 70
|
||||
}
|
||||
|
||||
tags = ["env:production", "team:platform"]
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,462 @@
|
||||
---
|
||||
name: elk-stack
|
||||
description: Deploy and manage the ELK Stack (Elasticsearch, Logstash, Kibana) for log aggregation and analysis. Configure log pipelines, create visualizations, and implement log-based monitoring. Use when centralizing logs, implementing search functionality, or building log analytics platforms.
|
||||
license: MIT
|
||||
metadata:
|
||||
author: devops-skills
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
# ELK Stack
|
||||
|
||||
Centralize and analyze logs with Elasticsearch, Logstash, and Kibana.
|
||||
|
||||
## When to Use This Skill
|
||||
|
||||
Use this skill when:
|
||||
- Centralizing logs from multiple sources
|
||||
- Building log search and analytics platforms
|
||||
- Creating log-based dashboards and alerts
|
||||
- Implementing full-text search for logs
|
||||
- Processing and transforming log data
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Docker or server infrastructure
|
||||
- Sufficient disk space for log storage
|
||||
- Network access from log sources
|
||||
|
||||
## Docker Deployment
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
elasticsearch:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- xpack.security.enabled=false
|
||||
- "ES_JAVA_OPTS=-Xms1g -Xmx1g"
|
||||
ports:
|
||||
- "9200:9200"
|
||||
volumes:
|
||||
- elasticsearch-data:/usr/share/elasticsearch/data
|
||||
|
||||
logstash:
|
||||
image: docker.elastic.co/logstash/logstash:8.11.0
|
||||
volumes:
|
||||
- ./logstash/pipeline:/usr/share/logstash/pipeline
|
||||
- ./logstash/config:/usr/share/logstash/config
|
||||
ports:
|
||||
- "5044:5044"
|
||||
- "5000:5000"
|
||||
depends_on:
|
||||
- elasticsearch
|
||||
|
||||
kibana:
|
||||
image: docker.elastic.co/kibana/kibana:8.11.0
|
||||
ports:
|
||||
- "5601:5601"
|
||||
environment:
|
||||
- ELASTICSEARCH_HOSTS=http://elasticsearch:9200
|
||||
depends_on:
|
||||
- elasticsearch
|
||||
|
||||
filebeat:
|
||||
image: docker.elastic.co/beats/filebeat:8.11.0
|
||||
user: root
|
||||
volumes:
|
||||
- ./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
|
||||
- /var/lib/docker/containers:/var/lib/docker/containers:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
depends_on:
|
||||
- logstash
|
||||
|
||||
volumes:
|
||||
elasticsearch-data:
|
||||
```
|
||||
|
||||
## Elasticsearch Configuration
|
||||
|
||||
### Index Templates
|
||||
|
||||
```json
|
||||
PUT _index_template/logs-template
|
||||
{
|
||||
"index_patterns": ["logs-*"],
|
||||
"template": {
|
||||
"settings": {
|
||||
"number_of_shards": 1,
|
||||
"number_of_replicas": 1,
|
||||
"index.lifecycle.name": "logs-policy"
|
||||
},
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"@timestamp": { "type": "date" },
|
||||
"message": { "type": "text" },
|
||||
"level": { "type": "keyword" },
|
||||
"service": { "type": "keyword" },
|
||||
"host": { "type": "keyword" },
|
||||
"trace_id": { "type": "keyword" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Index Lifecycle Management
|
||||
|
||||
```json
|
||||
PUT _ilm/policy/logs-policy
|
||||
{
|
||||
"policy": {
|
||||
"phases": {
|
||||
"hot": {
|
||||
"min_age": "0ms",
|
||||
"actions": {
|
||||
"rollover": {
|
||||
"max_size": "50GB",
|
||||
"max_age": "1d"
|
||||
}
|
||||
}
|
||||
},
|
||||
"warm": {
|
||||
"min_age": "7d",
|
||||
"actions": {
|
||||
"shrink": { "number_of_shards": 1 },
|
||||
"forcemerge": { "max_num_segments": 1 }
|
||||
}
|
||||
},
|
||||
"cold": {
|
||||
"min_age": "30d",
|
||||
"actions": {
|
||||
"freeze": {}
|
||||
}
|
||||
},
|
||||
"delete": {
|
||||
"min_age": "90d",
|
||||
"actions": {
|
||||
"delete": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Logstash Pipeline
|
||||
|
||||
### Basic Pipeline
|
||||
|
||||
```ruby
|
||||
# logstash/pipeline/main.conf
|
||||
input {
|
||||
beats {
|
||||
port => 5044
|
||||
}
|
||||
|
||||
tcp {
|
||||
port => 5000
|
||||
codec => json_lines
|
||||
}
|
||||
}
|
||||
|
||||
filter {
|
||||
# Parse JSON logs
|
||||
if [message] =~ /^\{/ {
|
||||
json {
|
||||
source => "message"
|
||||
}
|
||||
}
|
||||
|
||||
# Parse timestamp
|
||||
date {
|
||||
match => ["timestamp", "ISO8601", "yyyy-MM-dd HH:mm:ss"]
|
||||
target => "@timestamp"
|
||||
}
|
||||
|
||||
# Add environment tag
|
||||
mutate {
|
||||
add_field => { "environment" => "production" }
|
||||
}
|
||||
|
||||
# Grok pattern for nginx logs
|
||||
if [type] == "nginx" {
|
||||
grok {
|
||||
match => {
|
||||
"message" => '%{IPORHOST:client_ip} - %{USER:user} \[%{HTTPDATE:timestamp}\] "%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:bytes}'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => ["elasticsearch:9200"]
|
||||
index => "logs-%{+YYYY.MM.dd}"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Advanced Filtering
|
||||
|
||||
```ruby
|
||||
filter {
|
||||
# Parse application logs
|
||||
grok {
|
||||
match => {
|
||||
"message" => "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \[%{DATA:service}\] %{GREEDYDATA:log_message}"
|
||||
}
|
||||
}
|
||||
|
||||
# Extract trace ID from message
|
||||
if [log_message] =~ /trace_id=/ {
|
||||
grok {
|
||||
match => { "log_message" => "trace_id=%{UUID:trace_id}" }
|
||||
}
|
||||
}
|
||||
|
||||
# GeoIP lookup
|
||||
if [client_ip] {
|
||||
geoip {
|
||||
source => "client_ip"
|
||||
target => "geoip"
|
||||
}
|
||||
}
|
||||
|
||||
# Drop debug logs in production
|
||||
if [level] == "DEBUG" and [environment] == "production" {
|
||||
drop {}
|
||||
}
|
||||
|
||||
# Enrich with lookup
|
||||
translate {
|
||||
field => "status"
|
||||
destination => "status_description"
|
||||
dictionary => {
|
||||
"200" => "OK"
|
||||
"404" => "Not Found"
|
||||
"500" => "Internal Server Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Filebeat Configuration
|
||||
|
||||
```yaml
|
||||
# filebeat/filebeat.yml
|
||||
filebeat.inputs:
|
||||
- type: container
|
||||
paths:
|
||||
- '/var/lib/docker/containers/*/*.log'
|
||||
processors:
|
||||
- add_docker_metadata:
|
||||
host: "unix:///var/run/docker.sock"
|
||||
|
||||
- type: log
|
||||
enabled: true
|
||||
paths:
|
||||
- /var/log/nginx/*.log
|
||||
tags: ["nginx"]
|
||||
fields:
|
||||
type: nginx
|
||||
|
||||
output.logstash:
|
||||
hosts: ["logstash:5044"]
|
||||
|
||||
logging.level: info
|
||||
logging.to_files: true
|
||||
logging.files:
|
||||
path: /var/log/filebeat
|
||||
name: filebeat
|
||||
keepfiles: 7
|
||||
```
|
||||
|
||||
## Elasticsearch Queries
|
||||
|
||||
### Basic Queries
|
||||
|
||||
```json
|
||||
// Search all logs
|
||||
GET logs-*/_search
|
||||
{
|
||||
"query": {
|
||||
"match_all": {}
|
||||
}
|
||||
}
|
||||
|
||||
// Search by keyword
|
||||
GET logs-*/_search
|
||||
{
|
||||
"query": {
|
||||
"match": {
|
||||
"message": "error"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Filter by field
|
||||
GET logs-*/_search
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{ "match": { "level": "ERROR" } },
|
||||
{ "range": { "@timestamp": { "gte": "now-1h" } } }
|
||||
],
|
||||
"filter": [
|
||||
{ "term": { "service": "api-gateway" } }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Aggregations
|
||||
|
||||
```json
|
||||
// Count by log level
|
||||
GET logs-*/_search
|
||||
{
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
"log_levels": {
|
||||
"terms": { "field": "level" }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Error rate over time
|
||||
GET logs-*/_search
|
||||
{
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
"errors_over_time": {
|
||||
"date_histogram": {
|
||||
"field": "@timestamp",
|
||||
"fixed_interval": "5m"
|
||||
},
|
||||
"aggs": {
|
||||
"error_count": {
|
||||
"filter": { "term": { "level": "ERROR" } }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Kibana Setup
|
||||
|
||||
### Index Patterns
|
||||
|
||||
1. Go to Stack Management → Index Patterns
|
||||
2. Create pattern: `logs-*`
|
||||
3. Set time field: `@timestamp`
|
||||
|
||||
### Saved Searches
|
||||
|
||||
Create saved searches for common queries:
|
||||
- `level:ERROR` - All errors
|
||||
- `service:api-gateway AND level:ERROR` - API gateway errors
|
||||
- `response_time:>1000` - Slow requests
|
||||
|
||||
### Visualizations
|
||||
|
||||
Common visualization types:
|
||||
- **Line Chart**: Error rate over time
|
||||
- **Pie Chart**: Distribution by log level
|
||||
- **Data Table**: Top error messages
|
||||
- **Metric**: Total error count
|
||||
|
||||
### Dashboard Example
|
||||
|
||||
Create dashboard with:
|
||||
1. Total log count (Metric)
|
||||
2. Error rate trend (Line chart)
|
||||
3. Logs by service (Pie chart)
|
||||
4. Recent errors (Data table)
|
||||
5. Log stream (Discover panel)
|
||||
|
||||
## Alerting
|
||||
|
||||
### Watcher (X-Pack)
|
||||
|
||||
```json
|
||||
PUT _watcher/watch/error_alert
|
||||
{
|
||||
"trigger": {
|
||||
"schedule": { "interval": "5m" }
|
||||
},
|
||||
"input": {
|
||||
"search": {
|
||||
"request": {
|
||||
"indices": ["logs-*"],
|
||||
"body": {
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{ "match": { "level": "ERROR" } },
|
||||
{ "range": { "@timestamp": { "gte": "now-5m" } } }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"condition": {
|
||||
"compare": { "ctx.payload.hits.total.value": { "gt": 100 } }
|
||||
},
|
||||
"actions": {
|
||||
"notify_slack": {
|
||||
"webhook": {
|
||||
"scheme": "https",
|
||||
"host": "hooks.slack.com",
|
||||
"port": 443,
|
||||
"method": "post",
|
||||
"path": "/services/xxx",
|
||||
"body": "{\"text\": \"High error rate detected: {{ctx.payload.hits.total.value}} errors in last 5 minutes\"}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Common Issues
|
||||
|
||||
### Issue: High Disk Usage
|
||||
**Problem**: Elasticsearch consuming too much disk
|
||||
**Solution**: Implement ILM policies, reduce retention
|
||||
|
||||
### Issue: Slow Searches
|
||||
**Problem**: Queries taking too long
|
||||
**Solution**: Optimize index settings, add more shards, use filters
|
||||
|
||||
### Issue: Log Parsing Failures
|
||||
**Problem**: Logs not parsed correctly
|
||||
**Solution**: Test grok patterns, check for log format changes
|
||||
|
||||
### Issue: Memory Pressure
|
||||
**Problem**: Elasticsearch OOM errors
|
||||
**Solution**: Increase heap size (max 50% of RAM), limit field data
|
||||
|
||||
## Best Practices
|
||||
|
||||
- Implement index lifecycle management
|
||||
- Use index templates for consistent mappings
|
||||
- Parse logs at ingestion time
|
||||
- Limit stored fields to reduce storage
|
||||
- Use data streams for time-series data
|
||||
- Monitor cluster health
|
||||
- Implement proper security (X-Pack)
|
||||
- Regular index maintenance
|
||||
|
||||
## Related Skills
|
||||
|
||||
- [loki-logging](../loki-logging/) - Alternative logging stack
|
||||
- [prometheus-grafana](../prometheus-grafana/) - Metrics monitoring
|
||||
- [audit-logging](../../../compliance/auditing/audit-logging/) - Compliance logging
|
||||
@@ -0,0 +1,139 @@
|
||||
# Elasticsearch Query Reference
|
||||
|
||||
## Basic Queries
|
||||
|
||||
```json
|
||||
// Match all
|
||||
GET /logs/_search
|
||||
{
|
||||
"query": { "match_all": {} }
|
||||
}
|
||||
|
||||
// Match query
|
||||
GET /logs/_search
|
||||
{
|
||||
"query": {
|
||||
"match": { "message": "error" }
|
||||
}
|
||||
}
|
||||
|
||||
// Term query (exact match)
|
||||
GET /logs/_search
|
||||
{
|
||||
"query": {
|
||||
"term": { "status": "500" }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Boolean Queries
|
||||
|
||||
```json
|
||||
GET /logs/_search
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{ "match": { "service": "api" } }
|
||||
],
|
||||
"filter": [
|
||||
{ "range": { "@timestamp": { "gte": "now-1h" } } }
|
||||
],
|
||||
"should": [
|
||||
{ "match": { "level": "error" } }
|
||||
],
|
||||
"must_not": [
|
||||
{ "term": { "environment": "test" } }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Aggregations
|
||||
|
||||
```json
|
||||
// Terms aggregation
|
||||
GET /logs/_search
|
||||
{
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
"by_status": {
|
||||
"terms": { "field": "status.keyword" }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Date histogram
|
||||
GET /logs/_search
|
||||
{
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
"over_time": {
|
||||
"date_histogram": {
|
||||
"field": "@timestamp",
|
||||
"fixed_interval": "1h"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Nested aggregations
|
||||
GET /logs/_search
|
||||
{
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
"by_service": {
|
||||
"terms": { "field": "service.keyword" },
|
||||
"aggs": {
|
||||
"error_count": {
|
||||
"filter": { "term": { "level": "error" } }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Index Management
|
||||
|
||||
```bash
|
||||
# Create index
|
||||
PUT /logs-2024
|
||||
{
|
||||
"settings": {
|
||||
"number_of_shards": 3,
|
||||
"number_of_replicas": 1
|
||||
}
|
||||
}
|
||||
|
||||
# Index template
|
||||
PUT /_index_template/logs
|
||||
{
|
||||
"index_patterns": ["logs-*"],
|
||||
"template": {
|
||||
"settings": {
|
||||
"number_of_shards": 3
|
||||
},
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"@timestamp": { "type": "date" },
|
||||
"message": { "type": "text" },
|
||||
"level": { "type": "keyword" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# ILM policy
|
||||
PUT /_ilm/policy/logs-policy
|
||||
{
|
||||
"policy": {
|
||||
"phases": {
|
||||
"hot": { "actions": { "rollover": { "max_size": "50GB" } } },
|
||||
"warm": { "min_age": "7d", "actions": { "shrink": { "number_of_shards": 1 } } },
|
||||
"delete": { "min_age": "30d", "actions": { "delete": {} } }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,455 @@
|
||||
---
|
||||
name: loki-logging
|
||||
description: Configure Grafana Loki for log aggregation and analysis. Set up Promtail for log collection, write LogQL queries, and integrate with Grafana for visualization. Use when implementing lightweight log aggregation, especially in Kubernetes environments.
|
||||
license: MIT
|
||||
metadata:
|
||||
author: devops-skills
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
# Grafana Loki
|
||||
|
||||
Aggregate and query logs with Grafana Loki, the Prometheus-inspired logging system.
|
||||
|
||||
## When to Use This Skill
|
||||
|
||||
Use this skill when:
|
||||
- Implementing cost-effective log aggregation
|
||||
- Building logging for Kubernetes environments
|
||||
- Integrating logs with Grafana dashboards
|
||||
- Querying logs with label-based filtering
|
||||
- Preferring lighter-weight alternative to ELK
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Docker or Kubernetes
|
||||
- Grafana for visualization
|
||||
- Promtail or other log shipper
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
```
|
||||
┌─────────────┐ ┌──────────┐ ┌──────────┐
|
||||
│ Application │────▶│ Promtail │────▶│ Loki │
|
||||
└─────────────┘ └──────────┘ └──────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────┐
|
||||
│ Grafana │
|
||||
└──────────┘
|
||||
```
|
||||
|
||||
## Docker Deployment
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
loki:
|
||||
image: grafana/loki:2.9.0
|
||||
ports:
|
||||
- "3100:3100"
|
||||
volumes:
|
||||
- ./loki-config.yaml:/etc/loki/local-config.yaml
|
||||
- loki-data:/loki
|
||||
command: -config.file=/etc/loki/local-config.yaml
|
||||
|
||||
promtail:
|
||||
image: grafana/promtail:2.9.0
|
||||
volumes:
|
||||
- ./promtail-config.yaml:/etc/promtail/config.yaml
|
||||
- /var/log:/var/log:ro
|
||||
- /var/lib/docker/containers:/var/lib/docker/containers:ro
|
||||
command: -config.file=/etc/promtail/config.yaml
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:10.2.0
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- grafana-data:/var/lib/grafana
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning
|
||||
environment:
|
||||
- GF_AUTH_ANONYMOUS_ENABLED=true
|
||||
- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
|
||||
|
||||
volumes:
|
||||
loki-data:
|
||||
grafana-data:
|
||||
```
|
||||
|
||||
## Loki Configuration
|
||||
|
||||
```yaml
|
||||
# loki-config.yaml
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
|
||||
common:
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2020-10-24
|
||||
store: boltdb-shipper
|
||||
object_store: filesystem
|
||||
schema: v11
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
storage_config:
|
||||
boltdb_shipper:
|
||||
active_index_directory: /loki/index
|
||||
cache_location: /loki/cache
|
||||
shared_store: filesystem
|
||||
|
||||
limits_config:
|
||||
reject_old_samples: true
|
||||
reject_old_samples_max_age: 168h
|
||||
max_query_series: 5000
|
||||
max_query_parallelism: 2
|
||||
|
||||
chunk_store_config:
|
||||
max_look_back_period: 168h
|
||||
|
||||
table_manager:
|
||||
retention_deletes_enabled: true
|
||||
retention_period: 168h
|
||||
```
|
||||
|
||||
## Promtail Configuration
|
||||
|
||||
```yaml
|
||||
# promtail-config.yaml
|
||||
server:
|
||||
http_listen_port: 9080
|
||||
grpc_listen_port: 0
|
||||
|
||||
positions:
|
||||
filename: /tmp/positions.yaml
|
||||
|
||||
clients:
|
||||
- url: http://loki:3100/loki/api/v1/push
|
||||
|
||||
scrape_configs:
|
||||
# System logs
|
||||
- job_name: system
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: varlogs
|
||||
__path__: /var/log/*.log
|
||||
|
||||
# Docker container logs
|
||||
- job_name: docker
|
||||
docker_sd_configs:
|
||||
- host: unix:///var/run/docker.sock
|
||||
refresh_interval: 5s
|
||||
relabel_configs:
|
||||
- source_labels: ['__meta_docker_container_name']
|
||||
regex: '/(.*)'
|
||||
target_label: 'container'
|
||||
- source_labels: ['__meta_docker_container_log_stream']
|
||||
target_label: 'stream'
|
||||
|
||||
# Application logs with parsing
|
||||
- job_name: application
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: application
|
||||
__path__: /var/log/app/*.log
|
||||
pipeline_stages:
|
||||
- json:
|
||||
expressions:
|
||||
level: level
|
||||
message: message
|
||||
timestamp: timestamp
|
||||
- labels:
|
||||
level:
|
||||
- timestamp:
|
||||
source: timestamp
|
||||
format: RFC3339
|
||||
```
|
||||
|
||||
## Kubernetes Deployment
|
||||
|
||||
```bash
|
||||
# Using Helm
|
||||
helm repo add grafana https://grafana.github.io/helm-charts
|
||||
helm install loki grafana/loki-stack \
|
||||
--namespace monitoring \
|
||||
--create-namespace \
|
||||
--set grafana.enabled=true \
|
||||
--set promtail.enabled=true
|
||||
```
|
||||
|
||||
### Promtail DaemonSet
|
||||
|
||||
```yaml
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: promtail
|
||||
namespace: monitoring
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: promtail
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: promtail
|
||||
spec:
|
||||
containers:
|
||||
- name: promtail
|
||||
image: grafana/promtail:2.9.0
|
||||
args:
|
||||
- -config.file=/etc/promtail/promtail.yaml
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/promtail
|
||||
- name: varlog
|
||||
mountPath: /var/log
|
||||
- name: varlibdockercontainers
|
||||
mountPath: /var/lib/docker/containers
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: promtail-config
|
||||
- name: varlog
|
||||
hostPath:
|
||||
path: /var/log
|
||||
- name: varlibdockercontainers
|
||||
hostPath:
|
||||
path: /var/lib/docker/containers
|
||||
```
|
||||
|
||||
## LogQL Queries
|
||||
|
||||
### Basic Queries
|
||||
|
||||
```logql
|
||||
# All logs from a job
|
||||
{job="application"}
|
||||
|
||||
# Filter by label
|
||||
{job="application", level="error"}
|
||||
|
||||
# Multiple labels
|
||||
{namespace="production", container="api"}
|
||||
|
||||
# Regex match
|
||||
{job=~"app.*"}
|
||||
```
|
||||
|
||||
### Log Pipeline
|
||||
|
||||
```logql
|
||||
# Filter by content
|
||||
{job="application"} |= "error"
|
||||
|
||||
# Exclude content
|
||||
{job="application"} != "debug"
|
||||
|
||||
# Regex filter
|
||||
{job="application"} |~ "user_id=[0-9]+"
|
||||
|
||||
# JSON parsing
|
||||
{job="application"} | json | level="error"
|
||||
|
||||
# Line format
|
||||
{job="application"} | json | line_format "{{.level}}: {{.message}}"
|
||||
```
|
||||
|
||||
### Metric Queries
|
||||
|
||||
```logql
|
||||
# Count logs per second
|
||||
count_over_time({job="application"}[5m])
|
||||
|
||||
# Rate of errors
|
||||
rate({job="application", level="error"}[5m])
|
||||
|
||||
# Sum by label
|
||||
sum by (level) (count_over_time({job="application"}[5m]))
|
||||
|
||||
# Top services by error count
|
||||
topk(5, sum by (service) (count_over_time({level="error"}[1h])))
|
||||
```
|
||||
|
||||
### Aggregations
|
||||
|
||||
```logql
|
||||
# Average log line length
|
||||
avg_over_time({job="application"} | unwrap line_length [5m])
|
||||
|
||||
# Percentile of numeric field
|
||||
quantile_over_time(0.95, {job="application"} | json | unwrap response_time [5m])
|
||||
|
||||
# Error percentage
|
||||
sum(rate({job="application", level="error"}[5m]))
|
||||
/
|
||||
sum(rate({job="application"}[5m])) * 100
|
||||
```
|
||||
|
||||
## Pipeline Stages
|
||||
|
||||
```yaml
|
||||
# promtail-config.yaml
|
||||
pipeline_stages:
|
||||
# Parse JSON logs
|
||||
- json:
|
||||
expressions:
|
||||
level: level
|
||||
message: msg
|
||||
trace_id: trace_id
|
||||
|
||||
# Extract with regex
|
||||
- regex:
|
||||
expression: 'user_id=(?P<user_id>\d+)'
|
||||
|
||||
# Add labels from parsed fields
|
||||
- labels:
|
||||
level:
|
||||
user_id:
|
||||
|
||||
# Modify timestamp
|
||||
- timestamp:
|
||||
source: timestamp
|
||||
format: '2006-01-02T15:04:05.000Z'
|
||||
|
||||
# Filter logs
|
||||
- match:
|
||||
selector: '{level="debug"}'
|
||||
action: drop
|
||||
|
||||
# Add static labels
|
||||
- static_labels:
|
||||
environment: production
|
||||
|
||||
# Modify log line
|
||||
- template:
|
||||
source: message
|
||||
template: '{{ ToUpper .Value }}'
|
||||
```
|
||||
|
||||
## Grafana Integration
|
||||
|
||||
### Data Source Configuration
|
||||
|
||||
```yaml
|
||||
# grafana/provisioning/datasources/loki.yaml
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
isDefault: false
|
||||
jsonData:
|
||||
maxLines: 1000
|
||||
```
|
||||
|
||||
### Dashboard Panel
|
||||
|
||||
```json
|
||||
{
|
||||
"title": "Application Logs",
|
||||
"type": "logs",
|
||||
"datasource": "Loki",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "{job=\"application\"} | json",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": true,
|
||||
"wrapLogMessage": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Recording Rules
|
||||
|
||||
```yaml
|
||||
# loki-rules.yaml
|
||||
groups:
|
||||
- name: error_rates
|
||||
interval: 1m
|
||||
rules:
|
||||
- record: job:log_errors:rate5m
|
||||
expr: |
|
||||
sum by (job) (rate({level="error"}[5m]))
|
||||
```
|
||||
|
||||
## Alerting
|
||||
|
||||
```yaml
|
||||
# loki-alerts.yaml
|
||||
groups:
|
||||
- name: log_alerts
|
||||
rules:
|
||||
- alert: HighErrorRate
|
||||
expr: |
|
||||
sum(rate({level="error"}[5m])) > 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "High error rate in logs"
|
||||
description: "Error rate is {{ $value }} errors/second"
|
||||
```
|
||||
|
||||
## Common Issues
|
||||
|
||||
### Issue: High Memory Usage
|
||||
**Problem**: Loki consuming too much memory
|
||||
**Solution**: Reduce max_query_series, limit query time range
|
||||
|
||||
### Issue: Logs Not Appearing
|
||||
**Problem**: Promtail not shipping logs
|
||||
**Solution**: Check positions file, verify file paths, check label configuration
|
||||
|
||||
### Issue: Query Timeout
|
||||
**Problem**: LogQL queries timing out
|
||||
**Solution**: Add more specific label filters, reduce time range
|
||||
|
||||
### Issue: Ingestion Rate Limit
|
||||
**Problem**: Logs being dropped
|
||||
**Solution**: Increase per_stream_rate_limit in limits_config
|
||||
|
||||
## Best Practices
|
||||
|
||||
- Use meaningful labels (avoid high cardinality)
|
||||
- Filter by labels before log content
|
||||
- Parse logs at collection time with Promtail
|
||||
- Set appropriate retention periods
|
||||
- Use recording rules for common queries
|
||||
- Implement proper multitenancy for large deployments
|
||||
- Monitor Loki's own metrics
|
||||
- Use chunk caching for better performance
|
||||
|
||||
## Related Skills
|
||||
|
||||
- [prometheus-grafana](../prometheus-grafana/) - Metrics monitoring
|
||||
- [elk-stack](../elk-stack/) - Alternative logging
|
||||
- [alerting-oncall](../alerting-oncall/) - Alert management
|
||||
@@ -0,0 +1,447 @@
|
||||
---
|
||||
name: new-relic
|
||||
description: Configure New Relic observability platform for infrastructure and application monitoring. Set up APM agents, create dashboards, configure alerts, and implement distributed tracing. Use when implementing full-stack observability with New Relic One.
|
||||
license: MIT
|
||||
metadata:
|
||||
author: devops-skills
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
# New Relic
|
||||
|
||||
Monitor applications and infrastructure with New Relic's observability platform.
|
||||
|
||||
## When to Use This Skill
|
||||
|
||||
Use this skill when:
|
||||
- Implementing full-stack observability
|
||||
- Setting up APM for applications
|
||||
- Monitoring infrastructure health
|
||||
- Creating custom dashboards and alerts
|
||||
- Implementing distributed tracing
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- New Relic account and license key
|
||||
- Application access for APM agents
|
||||
- Infrastructure access for host agents
|
||||
|
||||
## Infrastructure Agent
|
||||
|
||||
### Linux Installation
|
||||
|
||||
```bash
|
||||
# Add repository and install
|
||||
curl -Ls https://download.newrelic.com/install/newrelic-cli/scripts/install.sh | bash
|
||||
|
||||
# Configure license key
|
||||
sudo NEW_RELIC_API_KEY=<YOUR_API_KEY> NEW_RELIC_ACCOUNT_ID=<ACCOUNT_ID> /usr/local/bin/newrelic install
|
||||
|
||||
# Or manual configuration
|
||||
echo "license_key: YOUR_LICENSE_KEY" | sudo tee -a /etc/newrelic-infra.yml
|
||||
sudo systemctl start newrelic-infra
|
||||
```
|
||||
|
||||
### Docker
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
newrelic-infra:
|
||||
image: newrelic/infrastructure:latest
|
||||
cap_add:
|
||||
- SYS_PTRACE
|
||||
privileged: true
|
||||
pid: "host"
|
||||
network_mode: "host"
|
||||
environment:
|
||||
- NRIA_LICENSE_KEY=${NEW_RELIC_LICENSE_KEY}
|
||||
- NRIA_DISPLAY_NAME=docker-host
|
||||
volumes:
|
||||
- /:/host:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
```
|
||||
|
||||
### Kubernetes
|
||||
|
||||
```bash
|
||||
# Using Helm
|
||||
helm repo add newrelic https://helm-charts.newrelic.com
|
||||
|
||||
helm install newrelic-bundle newrelic/nri-bundle \
|
||||
--namespace newrelic \
|
||||
--create-namespace \
|
||||
--set global.licenseKey=${NEW_RELIC_LICENSE_KEY} \
|
||||
--set global.cluster=my-cluster \
|
||||
--set newrelic-infrastructure.privileged=true \
|
||||
--set ksm.enabled=true \
|
||||
--set kubeEvents.enabled=true \
|
||||
--set logging.enabled=true
|
||||
```
|
||||
|
||||
## APM Agents
|
||||
|
||||
### Node.js
|
||||
|
||||
```javascript
|
||||
// At the very start of your application
|
||||
require('newrelic');
|
||||
|
||||
// newrelic.js configuration
|
||||
exports.config = {
|
||||
app_name: ['My Application'],
|
||||
license_key: process.env.NEW_RELIC_LICENSE_KEY,
|
||||
distributed_tracing: {
|
||||
enabled: true
|
||||
},
|
||||
logging: {
|
||||
level: 'info'
|
||||
},
|
||||
error_collector: {
|
||||
enabled: true,
|
||||
ignore_status_codes: [404]
|
||||
},
|
||||
transaction_tracer: {
|
||||
enabled: true,
|
||||
transaction_threshold: 'apdex_f',
|
||||
record_sql: 'obfuscated'
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
```bash
|
||||
# Install agent
|
||||
npm install newrelic
|
||||
|
||||
# Run application
|
||||
NEW_RELIC_LICENSE_KEY=xxx node -r newrelic app.js
|
||||
```
|
||||
|
||||
### Python
|
||||
|
||||
```python
|
||||
# newrelic.ini
|
||||
[newrelic]
|
||||
license_key = YOUR_LICENSE_KEY
|
||||
app_name = My Application
|
||||
distributed_tracing.enabled = true
|
||||
transaction_tracer.enabled = true
|
||||
error_collector.enabled = true
|
||||
browser_monitoring.auto_instrument = true
|
||||
```
|
||||
|
||||
```bash
|
||||
# Install agent
|
||||
pip install newrelic
|
||||
|
||||
# Generate config file
|
||||
newrelic-admin generate-config YOUR_LICENSE_KEY newrelic.ini
|
||||
|
||||
# Run application
|
||||
NEW_RELIC_CONFIG_FILE=newrelic.ini newrelic-admin run-program python app.py
|
||||
|
||||
# Or with gunicorn
|
||||
NEW_RELIC_CONFIG_FILE=newrelic.ini newrelic-admin run-program gunicorn app:app
|
||||
```
|
||||
|
||||
### Java
|
||||
|
||||
```bash
|
||||
# Download agent
|
||||
curl -O https://download.newrelic.com/newrelic/java-agent/newrelic-agent/current/newrelic-java.zip
|
||||
unzip newrelic-java.zip
|
||||
|
||||
# Configure newrelic.yml
|
||||
# license_key: YOUR_LICENSE_KEY
|
||||
# app_name: My Application
|
||||
|
||||
# Run with agent
|
||||
java -javaagent:/path/to/newrelic.jar -jar myapp.jar
|
||||
```
|
||||
|
||||
### Go
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/newrelic/go-agent/v3/newrelic"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func main() {
|
||||
app, err := newrelic.NewApplication(
|
||||
newrelic.ConfigAppName("My Application"),
|
||||
newrelic.ConfigLicense("YOUR_LICENSE_KEY"),
|
||||
newrelic.ConfigDistributedTracerEnabled(true),
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
http.HandleFunc(newrelic.WrapHandleFunc(app, "/", indexHandler))
|
||||
http.ListenAndServe(":8080", nil)
|
||||
}
|
||||
|
||||
func indexHandler(w http.ResponseWriter, r *http.Request) {
|
||||
txn := newrelic.FromContext(r.Context())
|
||||
txn.AddAttribute("user_id", "12345")
|
||||
w.Write([]byte("Hello, World!"))
|
||||
}
|
||||
```
|
||||
|
||||
## Custom Instrumentation
|
||||
|
||||
### Custom Events
|
||||
|
||||
```python
|
||||
import newrelic.agent
|
||||
|
||||
# Record custom event
|
||||
newrelic.agent.record_custom_event('OrderPlaced', {
|
||||
'order_id': '12345',
|
||||
'amount': 99.99,
|
||||
'customer_id': 'cust_001'
|
||||
})
|
||||
```
|
||||
|
||||
### Custom Metrics
|
||||
|
||||
```python
|
||||
import newrelic.agent
|
||||
|
||||
# Record custom metric
|
||||
newrelic.agent.record_custom_metric('Custom/OrderValue', 99.99)
|
||||
|
||||
# With attributes
|
||||
newrelic.agent.record_custom_metric('Custom/ProcessingTime',
|
||||
processing_time,
|
||||
{'unit': 'milliseconds'}
|
||||
)
|
||||
```
|
||||
|
||||
### Custom Spans
|
||||
|
||||
```python
|
||||
import newrelic.agent
|
||||
|
||||
@newrelic.agent.function_trace(name='process_payment')
|
||||
def process_payment(order_id, amount):
|
||||
# This creates a custom span in the trace
|
||||
pass
|
||||
|
||||
# Manual span creation
|
||||
with newrelic.agent.FunctionTrace(name='custom_operation'):
|
||||
# Traced code
|
||||
pass
|
||||
```
|
||||
|
||||
## NRQL Queries
|
||||
|
||||
### Basic Queries
|
||||
|
||||
```sql
|
||||
-- Transaction throughput
|
||||
SELECT rate(count(*), 1 minute) FROM Transaction
|
||||
WHERE appName = 'My Application'
|
||||
SINCE 1 hour ago
|
||||
|
||||
-- Average response time
|
||||
SELECT average(duration) FROM Transaction
|
||||
WHERE appName = 'My Application'
|
||||
SINCE 1 hour ago
|
||||
|
||||
-- Error rate
|
||||
SELECT percentage(count(*), WHERE error IS true) FROM Transaction
|
||||
WHERE appName = 'My Application'
|
||||
SINCE 1 hour ago
|
||||
|
||||
-- Apdex score
|
||||
SELECT apdex(duration, t: 0.5) FROM Transaction
|
||||
WHERE appName = 'My Application'
|
||||
SINCE 1 hour ago
|
||||
```
|
||||
|
||||
### Advanced Queries
|
||||
|
||||
```sql
|
||||
-- Slowest transactions
|
||||
SELECT average(duration) FROM Transaction
|
||||
WHERE appName = 'My Application'
|
||||
FACET name
|
||||
SINCE 1 hour ago
|
||||
ORDER BY average(duration) DESC
|
||||
LIMIT 10
|
||||
|
||||
-- Error breakdown
|
||||
SELECT count(*) FROM TransactionError
|
||||
WHERE appName = 'My Application'
|
||||
FACET error.class
|
||||
SINCE 1 hour ago
|
||||
|
||||
-- Percentile response times
|
||||
SELECT percentile(duration, 50, 90, 95, 99) FROM Transaction
|
||||
WHERE appName = 'My Application'
|
||||
SINCE 1 hour ago TIMESERIES
|
||||
|
||||
-- Custom event analysis
|
||||
SELECT average(amount), count(*) FROM OrderPlaced
|
||||
FACET customer_id
|
||||
SINCE 1 day ago
|
||||
```
|
||||
|
||||
## Dashboards
|
||||
|
||||
### Dashboard JSON
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "Application Dashboard",
|
||||
"pages": [
|
||||
{
|
||||
"name": "Overview",
|
||||
"widgets": [
|
||||
{
|
||||
"title": "Throughput",
|
||||
"visualization": {"id": "viz.line"},
|
||||
"configuration": {
|
||||
"nrqlQueries": [
|
||||
{
|
||||
"accountId": 12345,
|
||||
"query": "SELECT rate(count(*), 1 minute) FROM Transaction WHERE appName = 'My Application' SINCE 1 hour ago TIMESERIES"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "Error Rate",
|
||||
"visualization": {"id": "viz.billboard"},
|
||||
"configuration": {
|
||||
"nrqlQueries": [
|
||||
{
|
||||
"accountId": 12345,
|
||||
"query": "SELECT percentage(count(*), WHERE error IS true) FROM Transaction WHERE appName = 'My Application' SINCE 1 hour ago"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Alerts
|
||||
|
||||
### Alert Condition (NRQL)
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "High Error Rate",
|
||||
"type": "static",
|
||||
"nrql": {
|
||||
"query": "SELECT percentage(count(*), WHERE error IS true) FROM Transaction WHERE appName = 'My Application'"
|
||||
},
|
||||
"valueFunction": "single_value",
|
||||
"terms": [
|
||||
{
|
||||
"threshold": 5,
|
||||
"thresholdOccurrences": "all",
|
||||
"thresholdDuration": 300,
|
||||
"operator": "above",
|
||||
"priority": "critical"
|
||||
},
|
||||
{
|
||||
"threshold": 2,
|
||||
"thresholdOccurrences": "all",
|
||||
"thresholdDuration": 300,
|
||||
"operator": "above",
|
||||
"priority": "warning"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Alert Policy
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "Application Alerts",
|
||||
"incident_preference": "PER_CONDITION_AND_TARGET",
|
||||
"conditions": [
|
||||
{
|
||||
"name": "High Response Time",
|
||||
"type": "apm_app_metric",
|
||||
"entities": ["My Application"],
|
||||
"metric": "response_time_web",
|
||||
"condition_scope": "application",
|
||||
"terms": [
|
||||
{
|
||||
"duration": "5",
|
||||
"operator": "above",
|
||||
"threshold": "1",
|
||||
"priority": "critical"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Logs in Context
|
||||
|
||||
### Python Configuration
|
||||
|
||||
```python
|
||||
# newrelic.ini
|
||||
[newrelic]
|
||||
application_logging.enabled = true
|
||||
application_logging.forwarding.enabled = true
|
||||
application_logging.metrics.enabled = true
|
||||
application_logging.local_decorating.enabled = true
|
||||
```
|
||||
|
||||
### Log Forwarding
|
||||
|
||||
```yaml
|
||||
# newrelic-infra.yml
|
||||
log:
|
||||
- name: application-logs
|
||||
file: /var/log/myapp/*.log
|
||||
attributes:
|
||||
service: myapp
|
||||
environment: production
|
||||
```
|
||||
|
||||
## Common Issues
|
||||
|
||||
### Issue: No Data Appearing
|
||||
**Problem**: Agent not reporting to New Relic
|
||||
**Solution**: Verify license key, check network connectivity, review agent logs
|
||||
|
||||
### Issue: Missing Transactions
|
||||
**Problem**: Some transactions not captured
|
||||
**Solution**: Check instrumentation coverage, verify framework support
|
||||
|
||||
### Issue: High Overhead
|
||||
**Problem**: APM agent impacting performance
|
||||
**Solution**: Adjust sampling rate, disable unnecessary features
|
||||
|
||||
## Best Practices
|
||||
|
||||
- Use meaningful application names
|
||||
- Implement distributed tracing across services
|
||||
- Set up service maps for dependency visualization
|
||||
- Configure appropriate alert thresholds
|
||||
- Use custom attributes for business context
|
||||
- Implement logs in context for correlation
|
||||
- Set up workloads for service grouping
|
||||
- Regular review of unused dashboards and alerts
|
||||
|
||||
## Related Skills
|
||||
|
||||
- [datadog](../datadog/) - Alternative monitoring platform
|
||||
- [prometheus-grafana](../prometheus-grafana/) - Open source monitoring
|
||||
- [alerting-oncall](../alerting-oncall/) - Alert management
|
||||
@@ -0,0 +1,445 @@
|
||||
---
|
||||
name: prometheus-grafana
|
||||
description: Set up metrics collection and visualization with Prometheus and Grafana. Configure scrape targets, create PromQL queries, build dashboards, and implement alerting. Use when implementing monitoring, metrics collection, or visualization for applications and infrastructure.
|
||||
license: MIT
|
||||
metadata:
|
||||
author: devops-skills
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
# Prometheus & Grafana
|
||||
|
||||
Collect metrics and visualize system performance with the Prometheus-Grafana stack.
|
||||
|
||||
## When to Use This Skill
|
||||
|
||||
Use this skill when:
|
||||
- Setting up metrics collection infrastructure
|
||||
- Creating monitoring dashboards
|
||||
- Writing PromQL queries for analysis
|
||||
- Configuring alerting rules
|
||||
- Monitoring Kubernetes clusters
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Docker or Kubernetes for deployment
|
||||
- Network access to monitored targets
|
||||
- Basic understanding of metrics concepts
|
||||
|
||||
## Prometheus Setup
|
||||
|
||||
### Docker Deployment
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.48.0
|
||||
ports:
|
||||
- "9090:9090"
|
||||
volumes:
|
||||
- ./prometheus.yml:/etc/prometheus/prometheus.yml
|
||||
- ./rules:/etc/prometheus/rules
|
||||
- prometheus-data:/prometheus
|
||||
command:
|
||||
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||
- '--storage.tsdb.path=/prometheus'
|
||||
- '--storage.tsdb.retention.time=15d'
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:10.2.0
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- grafana-data:/var/lib/grafana
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_PASSWORD=admin
|
||||
|
||||
volumes:
|
||||
prometheus-data:
|
||||
grafana-data:
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
```yaml
|
||||
# prometheus.yml
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
- alertmanager:9093
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: 'prometheus'
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
|
||||
- job_name: 'node'
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'node-exporter:9100'
|
||||
|
||||
- job_name: 'applications'
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'app1:8080'
|
||||
- 'app2:8080'
|
||||
metrics_path: /metrics
|
||||
```
|
||||
|
||||
## Kubernetes Deployment
|
||||
|
||||
### Using Helm
|
||||
|
||||
```bash
|
||||
# Add Prometheus community Helm repo
|
||||
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
|
||||
|
||||
# Install kube-prometheus-stack
|
||||
helm install prometheus prometheus-community/kube-prometheus-stack \
|
||||
--namespace monitoring \
|
||||
--create-namespace \
|
||||
--set grafana.adminPassword=admin
|
||||
```
|
||||
|
||||
### ServiceMonitor
|
||||
|
||||
```yaml
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: myapp
|
||||
namespace: monitoring
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: myapp
|
||||
endpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
path: /metrics
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- default
|
||||
```
|
||||
|
||||
## PromQL Queries
|
||||
|
||||
### Basic Queries
|
||||
|
||||
```promql
|
||||
# Current CPU usage
|
||||
node_cpu_seconds_total{mode="idle"}
|
||||
|
||||
# Rate of HTTP requests per second
|
||||
rate(http_requests_total[5m])
|
||||
|
||||
# Average response time
|
||||
avg(http_request_duration_seconds_sum / http_request_duration_seconds_count)
|
||||
|
||||
# Memory usage percentage
|
||||
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100
|
||||
```
|
||||
|
||||
### Aggregations
|
||||
|
||||
```promql
|
||||
# Sum requests by status code
|
||||
sum by (status_code) (rate(http_requests_total[5m]))
|
||||
|
||||
# Average CPU by instance
|
||||
avg by (instance) (rate(node_cpu_seconds_total{mode!="idle"}[5m]))
|
||||
|
||||
# Top 5 endpoints by request count
|
||||
topk(5, sum by (endpoint) (rate(http_requests_total[5m])))
|
||||
|
||||
# 95th percentile latency
|
||||
histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))
|
||||
```
|
||||
|
||||
### Time-Based Queries
|
||||
|
||||
```promql
|
||||
# Compare to 1 hour ago
|
||||
http_requests_total - http_requests_total offset 1h
|
||||
|
||||
# Predict disk space in 4 hours
|
||||
predict_linear(node_filesystem_avail_bytes[1h], 4 * 3600)
|
||||
|
||||
# Changes in last 5 minutes
|
||||
changes(up[5m])
|
||||
|
||||
# Average over 24 hours
|
||||
avg_over_time(http_requests_total[24h])
|
||||
```
|
||||
|
||||
## Alerting Rules
|
||||
|
||||
```yaml
|
||||
# rules/alerts.yml
|
||||
groups:
|
||||
- name: application
|
||||
rules:
|
||||
- alert: HighErrorRate
|
||||
expr: |
|
||||
sum(rate(http_requests_total{status=~"5.."}[5m]))
|
||||
/ sum(rate(http_requests_total[5m])) > 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "High error rate detected"
|
||||
description: "Error rate is {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: ServiceDown
|
||||
expr: up == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Service {{ $labels.instance }} is down"
|
||||
|
||||
- alert: HighMemoryUsage
|
||||
expr: |
|
||||
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.9
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High memory usage on {{ $labels.instance }}"
|
||||
description: "Memory usage is {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: DiskSpaceLow
|
||||
expr: |
|
||||
(node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) < 0.1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Disk space low on {{ $labels.instance }}"
|
||||
```
|
||||
|
||||
## Alertmanager
|
||||
|
||||
```yaml
|
||||
# alertmanager.yml
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
slack_api_url: 'https://hooks.slack.com/services/xxx'
|
||||
|
||||
route:
|
||||
receiver: 'slack-notifications'
|
||||
group_by: ['alertname', 'severity']
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
routes:
|
||||
- match:
|
||||
severity: critical
|
||||
receiver: 'pagerduty'
|
||||
|
||||
receivers:
|
||||
- name: 'slack-notifications'
|
||||
slack_configs:
|
||||
- channel: '#alerts'
|
||||
send_resolved: true
|
||||
title: '{{ .Status | toUpper }}: {{ .CommonAnnotations.summary }}'
|
||||
text: '{{ .CommonAnnotations.description }}'
|
||||
|
||||
- name: 'pagerduty'
|
||||
pagerduty_configs:
|
||||
- service_key: 'xxx'
|
||||
severity: critical
|
||||
```
|
||||
|
||||
## Grafana Dashboards
|
||||
|
||||
### Dashboard JSON Structure
|
||||
|
||||
```json
|
||||
{
|
||||
"dashboard": {
|
||||
"title": "Application Metrics",
|
||||
"panels": [
|
||||
{
|
||||
"title": "Request Rate",
|
||||
"type": "graph",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(http_requests_total[5m])) by (status_code)",
|
||||
"legendFormat": "{{ status_code }}"
|
||||
}
|
||||
],
|
||||
"gridPos": {"x": 0, "y": 0, "w": 12, "h": 8}
|
||||
},
|
||||
{
|
||||
"title": "Latency P95",
|
||||
"type": "gauge",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))"
|
||||
}
|
||||
],
|
||||
"gridPos": {"x": 12, "y": 0, "w": 6, "h": 8}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Provisioning Dashboards
|
||||
|
||||
```yaml
|
||||
# grafana/provisioning/dashboards/dashboards.yml
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: 'default'
|
||||
orgId: 1
|
||||
folder: ''
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 30
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
```
|
||||
|
||||
### Data Source Provisioning
|
||||
|
||||
```yaml
|
||||
# grafana/provisioning/datasources/prometheus.yml
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
```
|
||||
|
||||
## Recording Rules
|
||||
|
||||
```yaml
|
||||
# rules/recording.yml
|
||||
groups:
|
||||
- name: aggregations
|
||||
interval: 30s
|
||||
rules:
|
||||
- record: job:http_requests:rate5m
|
||||
expr: sum by (job) (rate(http_requests_total[5m]))
|
||||
|
||||
- record: instance:node_cpu:avg_rate5m
|
||||
expr: |
|
||||
avg by (instance) (
|
||||
rate(node_cpu_seconds_total{mode!="idle"}[5m])
|
||||
)
|
||||
|
||||
- record: job:http_latency:p95
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
sum by (job, le) (rate(http_request_duration_seconds_bucket[5m]))
|
||||
)
|
||||
```
|
||||
|
||||
## Application Instrumentation
|
||||
|
||||
### Go Application
|
||||
|
||||
```go
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
var httpRequests = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "http_requests_total",
|
||||
Help: "Total HTTP requests",
|
||||
},
|
||||
[]string{"method", "endpoint", "status"},
|
||||
)
|
||||
|
||||
func init() {
|
||||
prometheus.MustRegister(httpRequests)
|
||||
}
|
||||
|
||||
// Expose metrics endpoint
|
||||
http.Handle("/metrics", promhttp.Handler())
|
||||
```
|
||||
|
||||
### Node.js Application
|
||||
|
||||
```javascript
|
||||
const client = require('prom-client');
|
||||
|
||||
const httpRequests = new client.Counter({
|
||||
name: 'http_requests_total',
|
||||
help: 'Total HTTP requests',
|
||||
labelNames: ['method', 'endpoint', 'status']
|
||||
});
|
||||
|
||||
// Middleware
|
||||
app.use((req, res, next) => {
|
||||
res.on('finish', () => {
|
||||
httpRequests.inc({
|
||||
method: req.method,
|
||||
endpoint: req.path,
|
||||
status: res.statusCode
|
||||
});
|
||||
});
|
||||
next();
|
||||
});
|
||||
|
||||
// Expose metrics
|
||||
app.get('/metrics', async (req, res) => {
|
||||
res.set('Content-Type', client.register.contentType);
|
||||
res.end(await client.register.metrics());
|
||||
});
|
||||
```
|
||||
|
||||
## Common Issues
|
||||
|
||||
### Issue: Targets Not Discovered
|
||||
**Problem**: Prometheus not scraping targets
|
||||
**Solution**: Check network connectivity, verify target labels
|
||||
|
||||
### Issue: High Memory Usage
|
||||
**Problem**: Prometheus using excessive memory
|
||||
**Solution**: Reduce retention, use recording rules, limit cardinality
|
||||
|
||||
### Issue: Slow Queries
|
||||
**Problem**: PromQL queries timing out
|
||||
**Solution**: Use recording rules, limit time ranges, optimize queries
|
||||
|
||||
### Issue: Missing Data Points
|
||||
**Problem**: Gaps in metrics data
|
||||
**Solution**: Check scrape interval, verify target availability
|
||||
|
||||
## Best Practices
|
||||
|
||||
- Use recording rules for frequently-used queries
|
||||
- Limit label cardinality to prevent memory issues
|
||||
- Set appropriate retention based on storage capacity
|
||||
- Use histogram metrics for latency measurement
|
||||
- Implement proper alerting thresholds
|
||||
- Version control dashboards as code
|
||||
- Use federation for large-scale deployments
|
||||
- Regularly review and prune unused metrics
|
||||
|
||||
## Related Skills
|
||||
|
||||
- [alerting-oncall](../alerting-oncall/) - Alert management
|
||||
- [loki-logging](../loki-logging/) - Log aggregation
|
||||
- [kubernetes-ops](../../orchestration/kubernetes-ops/) - K8s monitoring
|
||||
@@ -0,0 +1,198 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"gnetId": null,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 90}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
}
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 0, "y": 0},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"pluginVersion": "8.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "CPU Usage",
|
||||
"type": "gauge"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 90}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
}
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 6, "y": 0},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Memory Usage",
|
||||
"type": "gauge"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"showPoints": "never"
|
||||
},
|
||||
"unit": "reqps"
|
||||
}
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"legend": {"displayMode": "list", "placement": "bottom"},
|
||||
"tooltip": {"mode": "single"}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(http_requests_total[5m])) by (status)",
|
||||
"legendFormat": "{{status}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Request Rate by Status",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "s"
|
||||
}
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
|
||||
"id": 4,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
|
||||
"legendFormat": "P95",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
|
||||
"legendFormat": "P50",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Request Latency",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10
|
||||
},
|
||||
"unit": "percent"
|
||||
}
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
|
||||
"id": 5,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(http_requests_total{status=~\"5..\"}[5m])) / sum(rate(http_requests_total[5m])) * 100",
|
||||
"legendFormat": "Error Rate",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Error Rate",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"schemaVersion": 30,
|
||||
"style": "dark",
|
||||
"tags": ["template", "prometheus"],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "Application Dashboard Template",
|
||||
"version": 1
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
# Prometheus Configuration Template
|
||||
# Customize for your environment
|
||||
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
external_labels:
|
||||
cluster: production
|
||||
environment: prod
|
||||
|
||||
# Alertmanager configuration
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
- alertmanager:9093
|
||||
|
||||
# Rule files
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yaml
|
||||
|
||||
# Scrape configurations
|
||||
scrape_configs:
|
||||
|
||||
# Prometheus self-monitoring
|
||||
- job_name: 'prometheus'
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
|
||||
# Node Exporter
|
||||
- job_name: 'node'
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'node1:9100'
|
||||
- 'node2:9100'
|
||||
- 'node3:9100'
|
||||
|
||||
# Kubernetes API Server
|
||||
- job_name: 'kubernetes-apiservers'
|
||||
kubernetes_sd_configs:
|
||||
- role: endpoints
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
|
||||
action: keep
|
||||
regex: default;kubernetes;https
|
||||
|
||||
# Kubernetes Nodes
|
||||
- job_name: 'kubernetes-nodes'
|
||||
kubernetes_sd_configs:
|
||||
- role: node
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
relabel_configs:
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_node_label_(.+)
|
||||
|
||||
# Kubernetes Pods with prometheus.io annotations
|
||||
- job_name: 'kubernetes-pods'
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
|
||||
action: keep
|
||||
regex: true
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
|
||||
action: replace
|
||||
target_label: __metrics_path__
|
||||
regex: (.+)
|
||||
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
|
||||
action: replace
|
||||
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||
replacement: $1:$2
|
||||
target_label: __address__
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_pod_label_(.+)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
action: replace
|
||||
target_label: kubernetes_namespace
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
action: replace
|
||||
target_label: kubernetes_pod_name
|
||||
|
||||
# Kubernetes Services with prometheus.io annotations
|
||||
- job_name: 'kubernetes-services'
|
||||
kubernetes_sd_configs:
|
||||
- role: service
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx]
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
|
||||
action: keep
|
||||
regex: true
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter:9115
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_service_label_(.+)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: kubernetes_namespace
|
||||
- source_labels: [__meta_kubernetes_service_name]
|
||||
target_label: kubernetes_name
|
||||
|
||||
# Remote write (optional - for long-term storage)
|
||||
# remote_write:
|
||||
# - url: "http://thanos-receive:19291/api/v1/receive"
|
||||
@@ -0,0 +1,170 @@
|
||||
# Prometheus Alerting Rules Guide
|
||||
|
||||
## Rule Structure
|
||||
|
||||
```yaml
|
||||
groups:
|
||||
- name: example
|
||||
rules:
|
||||
- alert: AlertName
|
||||
expr: <PromQL expression>
|
||||
for: <duration>
|
||||
labels:
|
||||
severity: <critical|warning|info>
|
||||
team: <team-name>
|
||||
annotations:
|
||||
summary: "Brief description"
|
||||
description: "Detailed description with {{ $labels.instance }}"
|
||||
runbook_url: "https://wiki.example.com/alerts/AlertName"
|
||||
```
|
||||
|
||||
## Essential Alerts
|
||||
|
||||
### Infrastructure Alerts
|
||||
|
||||
```yaml
|
||||
groups:
|
||||
- name: infrastructure
|
||||
rules:
|
||||
|
||||
# Node down
|
||||
- alert: NodeDown
|
||||
expr: up{job="node"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Node {{ $labels.instance }} is down"
|
||||
|
||||
# High CPU
|
||||
- alert: HighCPU
|
||||
expr: |
|
||||
100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High CPU on {{ $labels.instance }}"
|
||||
description: "CPU usage is {{ $value | printf \"%.1f\" }}%"
|
||||
|
||||
# High Memory
|
||||
- alert: HighMemory
|
||||
expr: |
|
||||
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High memory on {{ $labels.instance }}"
|
||||
|
||||
# Disk Space Low
|
||||
- alert: DiskSpaceLow
|
||||
expr: |
|
||||
(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"}
|
||||
/ node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) * 100 < 15
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Low disk space on {{ $labels.instance }}"
|
||||
description: "{{ $labels.mountpoint }} has {{ $value | printf \"%.1f\" }}% free"
|
||||
|
||||
# Disk Space Critical
|
||||
- alert: DiskSpaceCritical
|
||||
expr: |
|
||||
(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"}
|
||||
/ node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) * 100 < 5
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical disk space on {{ $labels.instance }}"
|
||||
```
|
||||
|
||||
### Application Alerts
|
||||
|
||||
```yaml
|
||||
groups:
|
||||
- name: application
|
||||
rules:
|
||||
|
||||
# High Error Rate
|
||||
- alert: HighErrorRate
|
||||
expr: |
|
||||
sum by (service) (rate(http_requests_total{status=~"5.."}[5m]))
|
||||
/ sum by (service) (rate(http_requests_total[5m])) > 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "High error rate for {{ $labels.service }}"
|
||||
description: "Error rate is {{ $value | printf \"%.2f\" }}%"
|
||||
|
||||
# High Latency
|
||||
- alert: HighLatency
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
sum by (le, service) (rate(http_request_duration_seconds_bucket[5m]))
|
||||
) > 0.5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High latency for {{ $labels.service }}"
|
||||
description: "P95 latency is {{ $value | printf \"%.2f\" }}s"
|
||||
|
||||
# Service Down
|
||||
- alert: ServiceDown
|
||||
expr: up{job="app"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Service {{ $labels.instance }} is down"
|
||||
```
|
||||
|
||||
### Kubernetes Alerts
|
||||
|
||||
```yaml
|
||||
groups:
|
||||
- name: kubernetes
|
||||
rules:
|
||||
|
||||
# Pod CrashLooping
|
||||
- alert: PodCrashLooping
|
||||
expr: |
|
||||
rate(kube_pod_container_status_restarts_total[15m]) * 60 * 15 > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Pod {{ $labels.pod }} is crash looping"
|
||||
|
||||
# Pod Not Ready
|
||||
- alert: PodNotReady
|
||||
expr: |
|
||||
kube_pod_status_ready{condition="true"} == 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Pod {{ $labels.pod }} is not ready"
|
||||
|
||||
# Deployment Replicas Mismatch
|
||||
- alert: DeploymentReplicasMismatch
|
||||
expr: |
|
||||
kube_deployment_spec_replicas != kube_deployment_status_replicas_available
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Deployment {{ $labels.deployment }} has replica mismatch"
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Use `for` duration** - Avoid alert flapping
|
||||
2. **Include runbook URLs** - Link to remediation docs
|
||||
3. **Use severity labels** - Route alerts appropriately
|
||||
4. **Template annotations** - Include relevant context
|
||||
5. **Test alerts** - Use `promtool check rules`
|
||||
@@ -0,0 +1,168 @@
|
||||
# PromQL Cheat Sheet
|
||||
|
||||
## Basic Queries
|
||||
|
||||
### Instant Vectors
|
||||
```promql
|
||||
# Simple metric
|
||||
http_requests_total
|
||||
|
||||
# With label filter
|
||||
http_requests_total{status="200"}
|
||||
|
||||
# Multiple labels
|
||||
http_requests_total{status="200", method="GET"}
|
||||
|
||||
# Regex matching
|
||||
http_requests_total{status=~"2.."}
|
||||
http_requests_total{status!~"5.."}
|
||||
```
|
||||
|
||||
### Range Vectors
|
||||
```promql
|
||||
# Last 5 minutes
|
||||
http_requests_total[5m]
|
||||
|
||||
# Last 1 hour
|
||||
http_requests_total[1h]
|
||||
|
||||
# Time units: s, m, h, d, w, y
|
||||
```
|
||||
|
||||
## Functions
|
||||
|
||||
### Rate and Increase
|
||||
```promql
|
||||
# Per-second rate over 5m
|
||||
rate(http_requests_total[5m])
|
||||
|
||||
# Total increase over 1h
|
||||
increase(http_requests_total[1h])
|
||||
|
||||
# For gauges that can decrease
|
||||
irate(http_requests_total[5m]) # instant rate
|
||||
```
|
||||
|
||||
### Aggregations
|
||||
```promql
|
||||
# Sum across all instances
|
||||
sum(http_requests_total)
|
||||
|
||||
# Sum by label
|
||||
sum by (status) (http_requests_total)
|
||||
|
||||
# Sum excluding label
|
||||
sum without (instance) (http_requests_total)
|
||||
|
||||
# Other aggregations
|
||||
avg, min, max, count, stddev, stdvar
|
||||
topk(5, http_requests_total)
|
||||
bottomk(3, http_requests_total)
|
||||
```
|
||||
|
||||
### Histogram Quantiles
|
||||
```promql
|
||||
# 95th percentile
|
||||
histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))
|
||||
|
||||
# With grouping
|
||||
histogram_quantile(0.95,
|
||||
sum by (le, endpoint) (
|
||||
rate(http_request_duration_seconds_bucket[5m])
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
## Common Patterns
|
||||
|
||||
### Request Rate
|
||||
```promql
|
||||
# Total request rate
|
||||
sum(rate(http_requests_total[5m]))
|
||||
|
||||
# Request rate by endpoint
|
||||
sum by (endpoint) (rate(http_requests_total[5m]))
|
||||
```
|
||||
|
||||
### Error Rate
|
||||
```promql
|
||||
# Error percentage
|
||||
sum(rate(http_requests_total{status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(http_requests_total[5m]))
|
||||
* 100
|
||||
```
|
||||
|
||||
### Latency
|
||||
```promql
|
||||
# Average latency
|
||||
rate(http_request_duration_seconds_sum[5m])
|
||||
/
|
||||
rate(http_request_duration_seconds_count[5m])
|
||||
|
||||
# P99 latency
|
||||
histogram_quantile(0.99,
|
||||
sum by (le) (rate(http_request_duration_seconds_bucket[5m]))
|
||||
)
|
||||
```
|
||||
|
||||
### Resource Usage
|
||||
```promql
|
||||
# CPU usage percentage
|
||||
100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)
|
||||
|
||||
# Memory usage percentage
|
||||
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100
|
||||
|
||||
# Disk usage percentage
|
||||
(1 - (node_filesystem_avail_bytes / node_filesystem_size_bytes)) * 100
|
||||
```
|
||||
|
||||
### Kubernetes
|
||||
```promql
|
||||
# Pod CPU usage
|
||||
sum by (pod) (rate(container_cpu_usage_seconds_total{container!=""}[5m]))
|
||||
|
||||
# Pod memory usage
|
||||
sum by (pod) (container_memory_usage_bytes{container!=""})
|
||||
|
||||
# Pod restart count
|
||||
sum by (pod) (kube_pod_container_status_restarts_total)
|
||||
```
|
||||
|
||||
## Alert Examples
|
||||
|
||||
### High Error Rate
|
||||
```yaml
|
||||
- alert: HighErrorRate
|
||||
expr: |
|
||||
sum(rate(http_requests_total{status=~"5.."}[5m]))
|
||||
/ sum(rate(http_requests_total[5m])) > 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "High error rate detected"
|
||||
```
|
||||
|
||||
### High Latency
|
||||
```yaml
|
||||
- alert: HighLatency
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
sum by (le) (rate(http_request_duration_seconds_bucket[5m]))
|
||||
) > 0.5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
```
|
||||
|
||||
### Low Disk Space
|
||||
```yaml
|
||||
- alert: LowDiskSpace
|
||||
expr: |
|
||||
(node_filesystem_avail_bytes / node_filesystem_size_bytes) < 0.1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
```
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/bin/bash
|
||||
# Grafana Dashboard Backup Script
|
||||
# Usage: ./backup-grafana.sh [grafana-url] [api-key] [output-dir]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
GRAFANA_URL="${1:-http://localhost:3000}"
|
||||
API_KEY="${2:-$GRAFANA_API_KEY}"
|
||||
OUTPUT_DIR="${3:-./grafana-backup-$(date +%Y%m%d)}"
|
||||
|
||||
if [ -z "$API_KEY" ]; then
|
||||
echo "Error: API key required. Set GRAFANA_API_KEY or pass as argument."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$OUTPUT_DIR/dashboards"
|
||||
mkdir -p "$OUTPUT_DIR/datasources"
|
||||
mkdir -p "$OUTPUT_DIR/folders"
|
||||
|
||||
echo "========================================="
|
||||
echo "Grafana Backup"
|
||||
echo "URL: $GRAFANA_URL"
|
||||
echo "Output: $OUTPUT_DIR"
|
||||
echo "========================================="
|
||||
echo ""
|
||||
|
||||
# Backup datasources
|
||||
echo "Backing up datasources..."
|
||||
curl -s -H "Authorization: Bearer $API_KEY" \
|
||||
"$GRAFANA_URL/api/datasources" > "$OUTPUT_DIR/datasources/datasources.json"
|
||||
DS_COUNT=$(jq length "$OUTPUT_DIR/datasources/datasources.json")
|
||||
echo " Backed up $DS_COUNT datasources"
|
||||
|
||||
# Backup folders
|
||||
echo "Backing up folders..."
|
||||
curl -s -H "Authorization: Bearer $API_KEY" \
|
||||
"$GRAFANA_URL/api/folders" > "$OUTPUT_DIR/folders/folders.json"
|
||||
FOLDER_COUNT=$(jq length "$OUTPUT_DIR/folders/folders.json")
|
||||
echo " Backed up $FOLDER_COUNT folders"
|
||||
|
||||
# Get all dashboards
|
||||
echo "Backing up dashboards..."
|
||||
DASHBOARDS=$(curl -s -H "Authorization: Bearer $API_KEY" \
|
||||
"$GRAFANA_URL/api/search?type=dash-db")
|
||||
|
||||
DASH_COUNT=0
|
||||
echo "$DASHBOARDS" | jq -r '.[].uid' | while read uid; do
|
||||
DASH=$(curl -s -H "Authorization: Bearer $API_KEY" \
|
||||
"$GRAFANA_URL/api/dashboards/uid/$uid")
|
||||
|
||||
TITLE=$(echo "$DASH" | jq -r '.dashboard.title' | tr ' /' '_')
|
||||
echo "$DASH" > "$OUTPUT_DIR/dashboards/${TITLE}_${uid}.json"
|
||||
echo " - $TITLE"
|
||||
DASH_COUNT=$((DASH_COUNT + 1))
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Backup complete!"
|
||||
echo "Location: $OUTPUT_DIR"
|
||||
echo ""
|
||||
echo "To restore:"
|
||||
echo " 1. Datasources: POST to /api/datasources"
|
||||
echo " 2. Folders: POST to /api/folders"
|
||||
echo " 3. Dashboards: POST to /api/dashboards/db"
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/bin/bash
|
||||
# Prometheus Health Check Script
|
||||
# Usage: ./prometheus-health-check.sh [prometheus-url]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PROMETHEUS_URL="${1:-http://localhost:9090}"
|
||||
|
||||
echo "========================================="
|
||||
echo "Prometheus Health Check"
|
||||
echo "URL: $PROMETHEUS_URL"
|
||||
echo "========================================="
|
||||
echo ""
|
||||
|
||||
# Check Prometheus health
|
||||
echo -n "Prometheus Health: "
|
||||
HEALTH=$(curl -s "$PROMETHEUS_URL/-/healthy" 2>/dev/null)
|
||||
if [ "$HEALTH" == "Prometheus Server is Healthy." ]; then
|
||||
echo "✓ Healthy"
|
||||
else
|
||||
echo "✗ Unhealthy"
|
||||
fi
|
||||
|
||||
# Check readiness
|
||||
echo -n "Prometheus Ready: "
|
||||
READY=$(curl -s "$PROMETHEUS_URL/-/ready" 2>/dev/null)
|
||||
if [ "$READY" == "Prometheus Server is Ready." ]; then
|
||||
echo "✓ Ready"
|
||||
else
|
||||
echo "✗ Not Ready"
|
||||
fi
|
||||
|
||||
# Get runtime info
|
||||
echo ""
|
||||
echo "Runtime Information:"
|
||||
echo "--------------------"
|
||||
curl -s "$PROMETHEUS_URL/api/v1/status/runtimeinfo" 2>/dev/null | jq -r '
|
||||
.data |
|
||||
"Start Time: \(.startTime)",
|
||||
"Uptime: \(.CWD // "N/A")",
|
||||
"Storage Retention: \(.storageRetention)",
|
||||
"TSDB Info:",
|
||||
" - Head Chunks: \(.TSDB.headChunks // "N/A")",
|
||||
" - Head Series: \(.TSDB.headSeries // "N/A")"
|
||||
' 2>/dev/null || echo "Could not retrieve runtime info"
|
||||
|
||||
# Get active targets summary
|
||||
echo ""
|
||||
echo "Target Status:"
|
||||
echo "--------------"
|
||||
curl -s "$PROMETHEUS_URL/api/v1/targets" 2>/dev/null | jq -r '
|
||||
.data.activeTargets |
|
||||
group_by(.health) |
|
||||
map({health: .[0].health, count: length}) |
|
||||
.[] |
|
||||
"\(.health): \(.count) targets"
|
||||
' 2>/dev/null || echo "Could not retrieve targets"
|
||||
|
||||
# List unhealthy targets
|
||||
echo ""
|
||||
echo "Unhealthy Targets:"
|
||||
echo "------------------"
|
||||
curl -s "$PROMETHEUS_URL/api/v1/targets" 2>/dev/null | jq -r '
|
||||
.data.activeTargets[] |
|
||||
select(.health != "up") |
|
||||
"- \(.labels.job)/\(.labels.instance): \(.lastError)"
|
||||
' 2>/dev/null || echo "Could not check unhealthy targets"
|
||||
|
||||
# Check for firing alerts
|
||||
echo ""
|
||||
echo "Firing Alerts:"
|
||||
echo "--------------"
|
||||
curl -s "$PROMETHEUS_URL/api/v1/alerts" 2>/dev/null | jq -r '
|
||||
.data.alerts[] |
|
||||
select(.state == "firing") |
|
||||
"- [\(.labels.severity // "unknown")] \(.labels.alertname): \(.annotations.summary // .annotations.description // "No description")"
|
||||
' 2>/dev/null || echo "Could not retrieve alerts"
|
||||
|
||||
echo ""
|
||||
echo "========================================="
|
||||
echo "Health check complete"
|
||||
echo "========================================="
|
||||
Reference in New Issue
Block a user