mirror of
https://github.com/BagelHole/DevOps-Security-Agent-Skills.git
synced 2026-08-22 12:49:53 +02:00
.
This commit is contained in:
Toby
@@ -0,0 +1,445 @@
|
||||
---
|
||||
name: prometheus-grafana
|
||||
description: Set up metrics collection and visualization with Prometheus and Grafana. Configure scrape targets, create PromQL queries, build dashboards, and implement alerting. Use when implementing monitoring, metrics collection, or visualization for applications and infrastructure.
|
||||
license: MIT
|
||||
metadata:
|
||||
author: devops-skills
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
# Prometheus & Grafana
|
||||
|
||||
Collect metrics and visualize system performance with the Prometheus-Grafana stack.
|
||||
|
||||
## When to Use This Skill
|
||||
|
||||
Use this skill when:
|
||||
- Setting up metrics collection infrastructure
|
||||
- Creating monitoring dashboards
|
||||
- Writing PromQL queries for analysis
|
||||
- Configuring alerting rules
|
||||
- Monitoring Kubernetes clusters
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Docker or Kubernetes for deployment
|
||||
- Network access to monitored targets
|
||||
- Basic understanding of metrics concepts
|
||||
|
||||
## Prometheus Setup
|
||||
|
||||
### Docker Deployment
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.48.0
|
||||
ports:
|
||||
- "9090:9090"
|
||||
volumes:
|
||||
- ./prometheus.yml:/etc/prometheus/prometheus.yml
|
||||
- ./rules:/etc/prometheus/rules
|
||||
- prometheus-data:/prometheus
|
||||
command:
|
||||
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||
- '--storage.tsdb.path=/prometheus'
|
||||
- '--storage.tsdb.retention.time=15d'
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:10.2.0
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- grafana-data:/var/lib/grafana
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_PASSWORD=admin
|
||||
|
||||
volumes:
|
||||
prometheus-data:
|
||||
grafana-data:
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
```yaml
|
||||
# prometheus.yml
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
- alertmanager:9093
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: 'prometheus'
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
|
||||
- job_name: 'node'
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'node-exporter:9100'
|
||||
|
||||
- job_name: 'applications'
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'app1:8080'
|
||||
- 'app2:8080'
|
||||
metrics_path: /metrics
|
||||
```
|
||||
|
||||
## Kubernetes Deployment
|
||||
|
||||
### Using Helm
|
||||
|
||||
```bash
|
||||
# Add Prometheus community Helm repo
|
||||
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
|
||||
|
||||
# Install kube-prometheus-stack
|
||||
helm install prometheus prometheus-community/kube-prometheus-stack \
|
||||
--namespace monitoring \
|
||||
--create-namespace \
|
||||
--set grafana.adminPassword=admin
|
||||
```
|
||||
|
||||
### ServiceMonitor
|
||||
|
||||
```yaml
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: myapp
|
||||
namespace: monitoring
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: myapp
|
||||
endpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
path: /metrics
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- default
|
||||
```
|
||||
|
||||
## PromQL Queries
|
||||
|
||||
### Basic Queries
|
||||
|
||||
```promql
|
||||
# Current CPU usage
|
||||
node_cpu_seconds_total{mode="idle"}
|
||||
|
||||
# Rate of HTTP requests per second
|
||||
rate(http_requests_total[5m])
|
||||
|
||||
# Average response time
|
||||
avg(http_request_duration_seconds_sum / http_request_duration_seconds_count)
|
||||
|
||||
# Memory usage percentage
|
||||
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100
|
||||
```
|
||||
|
||||
### Aggregations
|
||||
|
||||
```promql
|
||||
# Sum requests by status code
|
||||
sum by (status_code) (rate(http_requests_total[5m]))
|
||||
|
||||
# Average CPU by instance
|
||||
avg by (instance) (rate(node_cpu_seconds_total{mode!="idle"}[5m]))
|
||||
|
||||
# Top 5 endpoints by request count
|
||||
topk(5, sum by (endpoint) (rate(http_requests_total[5m])))
|
||||
|
||||
# 95th percentile latency
|
||||
histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))
|
||||
```
|
||||
|
||||
### Time-Based Queries
|
||||
|
||||
```promql
|
||||
# Compare to 1 hour ago
|
||||
http_requests_total - http_requests_total offset 1h
|
||||
|
||||
# Predict disk space in 4 hours
|
||||
predict_linear(node_filesystem_avail_bytes[1h], 4 * 3600)
|
||||
|
||||
# Changes in last 5 minutes
|
||||
changes(up[5m])
|
||||
|
||||
# Average over 24 hours
|
||||
avg_over_time(http_requests_total[24h])
|
||||
```
|
||||
|
||||
## Alerting Rules
|
||||
|
||||
```yaml
|
||||
# rules/alerts.yml
|
||||
groups:
|
||||
- name: application
|
||||
rules:
|
||||
- alert: HighErrorRate
|
||||
expr: |
|
||||
sum(rate(http_requests_total{status=~"5.."}[5m]))
|
||||
/ sum(rate(http_requests_total[5m])) > 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "High error rate detected"
|
||||
description: "Error rate is {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: ServiceDown
|
||||
expr: up == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Service {{ $labels.instance }} is down"
|
||||
|
||||
- alert: HighMemoryUsage
|
||||
expr: |
|
||||
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.9
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High memory usage on {{ $labels.instance }}"
|
||||
description: "Memory usage is {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: DiskSpaceLow
|
||||
expr: |
|
||||
(node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) < 0.1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Disk space low on {{ $labels.instance }}"
|
||||
```
|
||||
|
||||
## Alertmanager
|
||||
|
||||
```yaml
|
||||
# alertmanager.yml
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
slack_api_url: 'https://hooks.slack.com/services/xxx'
|
||||
|
||||
route:
|
||||
receiver: 'slack-notifications'
|
||||
group_by: ['alertname', 'severity']
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
routes:
|
||||
- match:
|
||||
severity: critical
|
||||
receiver: 'pagerduty'
|
||||
|
||||
receivers:
|
||||
- name: 'slack-notifications'
|
||||
slack_configs:
|
||||
- channel: '#alerts'
|
||||
send_resolved: true
|
||||
title: '{{ .Status | toUpper }}: {{ .CommonAnnotations.summary }}'
|
||||
text: '{{ .CommonAnnotations.description }}'
|
||||
|
||||
- name: 'pagerduty'
|
||||
pagerduty_configs:
|
||||
- service_key: 'xxx'
|
||||
severity: critical
|
||||
```
|
||||
|
||||
## Grafana Dashboards
|
||||
|
||||
### Dashboard JSON Structure
|
||||
|
||||
```json
|
||||
{
|
||||
"dashboard": {
|
||||
"title": "Application Metrics",
|
||||
"panels": [
|
||||
{
|
||||
"title": "Request Rate",
|
||||
"type": "graph",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(http_requests_total[5m])) by (status_code)",
|
||||
"legendFormat": "{{ status_code }}"
|
||||
}
|
||||
],
|
||||
"gridPos": {"x": 0, "y": 0, "w": 12, "h": 8}
|
||||
},
|
||||
{
|
||||
"title": "Latency P95",
|
||||
"type": "gauge",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))"
|
||||
}
|
||||
],
|
||||
"gridPos": {"x": 12, "y": 0, "w": 6, "h": 8}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Provisioning Dashboards
|
||||
|
||||
```yaml
|
||||
# grafana/provisioning/dashboards/dashboards.yml
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: 'default'
|
||||
orgId: 1
|
||||
folder: ''
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 30
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
```
|
||||
|
||||
### Data Source Provisioning
|
||||
|
||||
```yaml
|
||||
# grafana/provisioning/datasources/prometheus.yml
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
```
|
||||
|
||||
## Recording Rules
|
||||
|
||||
```yaml
|
||||
# rules/recording.yml
|
||||
groups:
|
||||
- name: aggregations
|
||||
interval: 30s
|
||||
rules:
|
||||
- record: job:http_requests:rate5m
|
||||
expr: sum by (job) (rate(http_requests_total[5m]))
|
||||
|
||||
- record: instance:node_cpu:avg_rate5m
|
||||
expr: |
|
||||
avg by (instance) (
|
||||
rate(node_cpu_seconds_total{mode!="idle"}[5m])
|
||||
)
|
||||
|
||||
- record: job:http_latency:p95
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
sum by (job, le) (rate(http_request_duration_seconds_bucket[5m]))
|
||||
)
|
||||
```
|
||||
|
||||
## Application Instrumentation
|
||||
|
||||
### Go Application
|
||||
|
||||
```go
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
var httpRequests = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "http_requests_total",
|
||||
Help: "Total HTTP requests",
|
||||
},
|
||||
[]string{"method", "endpoint", "status"},
|
||||
)
|
||||
|
||||
func init() {
|
||||
prometheus.MustRegister(httpRequests)
|
||||
}
|
||||
|
||||
// Expose metrics endpoint
|
||||
http.Handle("/metrics", promhttp.Handler())
|
||||
```
|
||||
|
||||
### Node.js Application
|
||||
|
||||
```javascript
|
||||
const client = require('prom-client');
|
||||
|
||||
const httpRequests = new client.Counter({
|
||||
name: 'http_requests_total',
|
||||
help: 'Total HTTP requests',
|
||||
labelNames: ['method', 'endpoint', 'status']
|
||||
});
|
||||
|
||||
// Middleware
|
||||
app.use((req, res, next) => {
|
||||
res.on('finish', () => {
|
||||
httpRequests.inc({
|
||||
method: req.method,
|
||||
endpoint: req.path,
|
||||
status: res.statusCode
|
||||
});
|
||||
});
|
||||
next();
|
||||
});
|
||||
|
||||
// Expose metrics
|
||||
app.get('/metrics', async (req, res) => {
|
||||
res.set('Content-Type', client.register.contentType);
|
||||
res.end(await client.register.metrics());
|
||||
});
|
||||
```
|
||||
|
||||
## Common Issues
|
||||
|
||||
### Issue: Targets Not Discovered
|
||||
**Problem**: Prometheus not scraping targets
|
||||
**Solution**: Check network connectivity, verify target labels
|
||||
|
||||
### Issue: High Memory Usage
|
||||
**Problem**: Prometheus using excessive memory
|
||||
**Solution**: Reduce retention, use recording rules, limit cardinality
|
||||
|
||||
### Issue: Slow Queries
|
||||
**Problem**: PromQL queries timing out
|
||||
**Solution**: Use recording rules, limit time ranges, optimize queries
|
||||
|
||||
### Issue: Missing Data Points
|
||||
**Problem**: Gaps in metrics data
|
||||
**Solution**: Check scrape interval, verify target availability
|
||||
|
||||
## Best Practices
|
||||
|
||||
- Use recording rules for frequently-used queries
|
||||
- Limit label cardinality to prevent memory issues
|
||||
- Set appropriate retention based on storage capacity
|
||||
- Use histogram metrics for latency measurement
|
||||
- Implement proper alerting thresholds
|
||||
- Version control dashboards as code
|
||||
- Use federation for large-scale deployments
|
||||
- Regularly review and prune unused metrics
|
||||
|
||||
## Related Skills
|
||||
|
||||
- [alerting-oncall](../alerting-oncall/) - Alert management
|
||||
- [loki-logging](../loki-logging/) - Log aggregation
|
||||
- [kubernetes-ops](../../orchestration/kubernetes-ops/) - K8s monitoring
|
||||
@@ -0,0 +1,198 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"gnetId": null,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 90}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
}
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 0, "y": 0},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"pluginVersion": "8.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "CPU Usage",
|
||||
"type": "gauge"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 90}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
}
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 6, "y": 0},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Memory Usage",
|
||||
"type": "gauge"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"showPoints": "never"
|
||||
},
|
||||
"unit": "reqps"
|
||||
}
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"legend": {"displayMode": "list", "placement": "bottom"},
|
||||
"tooltip": {"mode": "single"}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(http_requests_total[5m])) by (status)",
|
||||
"legendFormat": "{{status}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Request Rate by Status",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "s"
|
||||
}
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
|
||||
"id": 4,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
|
||||
"legendFormat": "P95",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
|
||||
"legendFormat": "P50",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Request Latency",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "palette-classic"},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10
|
||||
},
|
||||
"unit": "percent"
|
||||
}
|
||||
},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
|
||||
"id": 5,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(http_requests_total{status=~\"5..\"}[5m])) / sum(rate(http_requests_total[5m])) * 100",
|
||||
"legendFormat": "Error Rate",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Error Rate",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"schemaVersion": 30,
|
||||
"style": "dark",
|
||||
"tags": ["template", "prometheus"],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "Application Dashboard Template",
|
||||
"version": 1
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
# Prometheus Configuration Template
|
||||
# Customize for your environment
|
||||
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
external_labels:
|
||||
cluster: production
|
||||
environment: prod
|
||||
|
||||
# Alertmanager configuration
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
- alertmanager:9093
|
||||
|
||||
# Rule files
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yaml
|
||||
|
||||
# Scrape configurations
|
||||
scrape_configs:
|
||||
|
||||
# Prometheus self-monitoring
|
||||
- job_name: 'prometheus'
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
|
||||
# Node Exporter
|
||||
- job_name: 'node'
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'node1:9100'
|
||||
- 'node2:9100'
|
||||
- 'node3:9100'
|
||||
|
||||
# Kubernetes API Server
|
||||
- job_name: 'kubernetes-apiservers'
|
||||
kubernetes_sd_configs:
|
||||
- role: endpoints
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
|
||||
action: keep
|
||||
regex: default;kubernetes;https
|
||||
|
||||
# Kubernetes Nodes
|
||||
- job_name: 'kubernetes-nodes'
|
||||
kubernetes_sd_configs:
|
||||
- role: node
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
relabel_configs:
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_node_label_(.+)
|
||||
|
||||
# Kubernetes Pods with prometheus.io annotations
|
||||
- job_name: 'kubernetes-pods'
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
|
||||
action: keep
|
||||
regex: true
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
|
||||
action: replace
|
||||
target_label: __metrics_path__
|
||||
regex: (.+)
|
||||
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
|
||||
action: replace
|
||||
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||
replacement: $1:$2
|
||||
target_label: __address__
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_pod_label_(.+)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
action: replace
|
||||
target_label: kubernetes_namespace
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
action: replace
|
||||
target_label: kubernetes_pod_name
|
||||
|
||||
# Kubernetes Services with prometheus.io annotations
|
||||
- job_name: 'kubernetes-services'
|
||||
kubernetes_sd_configs:
|
||||
- role: service
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx]
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
|
||||
action: keep
|
||||
regex: true
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter:9115
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_service_label_(.+)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: kubernetes_namespace
|
||||
- source_labels: [__meta_kubernetes_service_name]
|
||||
target_label: kubernetes_name
|
||||
|
||||
# Remote write (optional - for long-term storage)
|
||||
# remote_write:
|
||||
# - url: "http://thanos-receive:19291/api/v1/receive"
|
||||
@@ -0,0 +1,170 @@
|
||||
# Prometheus Alerting Rules Guide
|
||||
|
||||
## Rule Structure
|
||||
|
||||
```yaml
|
||||
groups:
|
||||
- name: example
|
||||
rules:
|
||||
- alert: AlertName
|
||||
expr: <PromQL expression>
|
||||
for: <duration>
|
||||
labels:
|
||||
severity: <critical|warning|info>
|
||||
team: <team-name>
|
||||
annotations:
|
||||
summary: "Brief description"
|
||||
description: "Detailed description with {{ $labels.instance }}"
|
||||
runbook_url: "https://wiki.example.com/alerts/AlertName"
|
||||
```
|
||||
|
||||
## Essential Alerts
|
||||
|
||||
### Infrastructure Alerts
|
||||
|
||||
```yaml
|
||||
groups:
|
||||
- name: infrastructure
|
||||
rules:
|
||||
|
||||
# Node down
|
||||
- alert: NodeDown
|
||||
expr: up{job="node"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Node {{ $labels.instance }} is down"
|
||||
|
||||
# High CPU
|
||||
- alert: HighCPU
|
||||
expr: |
|
||||
100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High CPU on {{ $labels.instance }}"
|
||||
description: "CPU usage is {{ $value | printf \"%.1f\" }}%"
|
||||
|
||||
# High Memory
|
||||
- alert: HighMemory
|
||||
expr: |
|
||||
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High memory on {{ $labels.instance }}"
|
||||
|
||||
# Disk Space Low
|
||||
- alert: DiskSpaceLow
|
||||
expr: |
|
||||
(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"}
|
||||
/ node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) * 100 < 15
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Low disk space on {{ $labels.instance }}"
|
||||
description: "{{ $labels.mountpoint }} has {{ $value | printf \"%.1f\" }}% free"
|
||||
|
||||
# Disk Space Critical
|
||||
- alert: DiskSpaceCritical
|
||||
expr: |
|
||||
(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"}
|
||||
/ node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) * 100 < 5
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Critical disk space on {{ $labels.instance }}"
|
||||
```
|
||||
|
||||
### Application Alerts
|
||||
|
||||
```yaml
|
||||
groups:
|
||||
- name: application
|
||||
rules:
|
||||
|
||||
# High Error Rate
|
||||
- alert: HighErrorRate
|
||||
expr: |
|
||||
sum by (service) (rate(http_requests_total{status=~"5.."}[5m]))
|
||||
/ sum by (service) (rate(http_requests_total[5m])) > 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "High error rate for {{ $labels.service }}"
|
||||
description: "Error rate is {{ $value | printf \"%.2f\" }}%"
|
||||
|
||||
# High Latency
|
||||
- alert: HighLatency
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
sum by (le, service) (rate(http_request_duration_seconds_bucket[5m]))
|
||||
) > 0.5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High latency for {{ $labels.service }}"
|
||||
description: "P95 latency is {{ $value | printf \"%.2f\" }}s"
|
||||
|
||||
# Service Down
|
||||
- alert: ServiceDown
|
||||
expr: up{job="app"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Service {{ $labels.instance }} is down"
|
||||
```
|
||||
|
||||
### Kubernetes Alerts
|
||||
|
||||
```yaml
|
||||
groups:
|
||||
- name: kubernetes
|
||||
rules:
|
||||
|
||||
# Pod CrashLooping
|
||||
- alert: PodCrashLooping
|
||||
expr: |
|
||||
rate(kube_pod_container_status_restarts_total[15m]) * 60 * 15 > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Pod {{ $labels.pod }} is crash looping"
|
||||
|
||||
# Pod Not Ready
|
||||
- alert: PodNotReady
|
||||
expr: |
|
||||
kube_pod_status_ready{condition="true"} == 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Pod {{ $labels.pod }} is not ready"
|
||||
|
||||
# Deployment Replicas Mismatch
|
||||
- alert: DeploymentReplicasMismatch
|
||||
expr: |
|
||||
kube_deployment_spec_replicas != kube_deployment_status_replicas_available
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Deployment {{ $labels.deployment }} has replica mismatch"
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Use `for` duration** - Avoid alert flapping
|
||||
2. **Include runbook URLs** - Link to remediation docs
|
||||
3. **Use severity labels** - Route alerts appropriately
|
||||
4. **Template annotations** - Include relevant context
|
||||
5. **Test alerts** - Use `promtool check rules`
|
||||
@@ -0,0 +1,168 @@
|
||||
# PromQL Cheat Sheet
|
||||
|
||||
## Basic Queries
|
||||
|
||||
### Instant Vectors
|
||||
```promql
|
||||
# Simple metric
|
||||
http_requests_total
|
||||
|
||||
# With label filter
|
||||
http_requests_total{status="200"}
|
||||
|
||||
# Multiple labels
|
||||
http_requests_total{status="200", method="GET"}
|
||||
|
||||
# Regex matching
|
||||
http_requests_total{status=~"2.."}
|
||||
http_requests_total{status!~"5.."}
|
||||
```
|
||||
|
||||
### Range Vectors
|
||||
```promql
|
||||
# Last 5 minutes
|
||||
http_requests_total[5m]
|
||||
|
||||
# Last 1 hour
|
||||
http_requests_total[1h]
|
||||
|
||||
# Time units: s, m, h, d, w, y
|
||||
```
|
||||
|
||||
## Functions
|
||||
|
||||
### Rate and Increase
|
||||
```promql
|
||||
# Per-second rate over 5m
|
||||
rate(http_requests_total[5m])
|
||||
|
||||
# Total increase over 1h
|
||||
increase(http_requests_total[1h])
|
||||
|
||||
# For gauges that can decrease
|
||||
irate(http_requests_total[5m]) # instant rate
|
||||
```
|
||||
|
||||
### Aggregations
|
||||
```promql
|
||||
# Sum across all instances
|
||||
sum(http_requests_total)
|
||||
|
||||
# Sum by label
|
||||
sum by (status) (http_requests_total)
|
||||
|
||||
# Sum excluding label
|
||||
sum without (instance) (http_requests_total)
|
||||
|
||||
# Other aggregations
|
||||
avg, min, max, count, stddev, stdvar
|
||||
topk(5, http_requests_total)
|
||||
bottomk(3, http_requests_total)
|
||||
```
|
||||
|
||||
### Histogram Quantiles
|
||||
```promql
|
||||
# 95th percentile
|
||||
histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))
|
||||
|
||||
# With grouping
|
||||
histogram_quantile(0.95,
|
||||
sum by (le, endpoint) (
|
||||
rate(http_request_duration_seconds_bucket[5m])
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
## Common Patterns
|
||||
|
||||
### Request Rate
|
||||
```promql
|
||||
# Total request rate
|
||||
sum(rate(http_requests_total[5m]))
|
||||
|
||||
# Request rate by endpoint
|
||||
sum by (endpoint) (rate(http_requests_total[5m]))
|
||||
```
|
||||
|
||||
### Error Rate
|
||||
```promql
|
||||
# Error percentage
|
||||
sum(rate(http_requests_total{status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(http_requests_total[5m]))
|
||||
* 100
|
||||
```
|
||||
|
||||
### Latency
|
||||
```promql
|
||||
# Average latency
|
||||
rate(http_request_duration_seconds_sum[5m])
|
||||
/
|
||||
rate(http_request_duration_seconds_count[5m])
|
||||
|
||||
# P99 latency
|
||||
histogram_quantile(0.99,
|
||||
sum by (le) (rate(http_request_duration_seconds_bucket[5m]))
|
||||
)
|
||||
```
|
||||
|
||||
### Resource Usage
|
||||
```promql
|
||||
# CPU usage percentage
|
||||
100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)
|
||||
|
||||
# Memory usage percentage
|
||||
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100
|
||||
|
||||
# Disk usage percentage
|
||||
(1 - (node_filesystem_avail_bytes / node_filesystem_size_bytes)) * 100
|
||||
```
|
||||
|
||||
### Kubernetes
|
||||
```promql
|
||||
# Pod CPU usage
|
||||
sum by (pod) (rate(container_cpu_usage_seconds_total{container!=""}[5m]))
|
||||
|
||||
# Pod memory usage
|
||||
sum by (pod) (container_memory_usage_bytes{container!=""})
|
||||
|
||||
# Pod restart count
|
||||
sum by (pod) (kube_pod_container_status_restarts_total)
|
||||
```
|
||||
|
||||
## Alert Examples
|
||||
|
||||
### High Error Rate
|
||||
```yaml
|
||||
- alert: HighErrorRate
|
||||
expr: |
|
||||
sum(rate(http_requests_total{status=~"5.."}[5m]))
|
||||
/ sum(rate(http_requests_total[5m])) > 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "High error rate detected"
|
||||
```
|
||||
|
||||
### High Latency
|
||||
```yaml
|
||||
- alert: HighLatency
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
sum by (le) (rate(http_request_duration_seconds_bucket[5m]))
|
||||
) > 0.5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
```
|
||||
|
||||
### Low Disk Space
|
||||
```yaml
|
||||
- alert: LowDiskSpace
|
||||
expr: |
|
||||
(node_filesystem_avail_bytes / node_filesystem_size_bytes) < 0.1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
```
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/bin/bash
|
||||
# Grafana Dashboard Backup Script
|
||||
# Usage: ./backup-grafana.sh [grafana-url] [api-key] [output-dir]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
GRAFANA_URL="${1:-http://localhost:3000}"
|
||||
API_KEY="${2:-$GRAFANA_API_KEY}"
|
||||
OUTPUT_DIR="${3:-./grafana-backup-$(date +%Y%m%d)}"
|
||||
|
||||
if [ -z "$API_KEY" ]; then
|
||||
echo "Error: API key required. Set GRAFANA_API_KEY or pass as argument."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$OUTPUT_DIR/dashboards"
|
||||
mkdir -p "$OUTPUT_DIR/datasources"
|
||||
mkdir -p "$OUTPUT_DIR/folders"
|
||||
|
||||
echo "========================================="
|
||||
echo "Grafana Backup"
|
||||
echo "URL: $GRAFANA_URL"
|
||||
echo "Output: $OUTPUT_DIR"
|
||||
echo "========================================="
|
||||
echo ""
|
||||
|
||||
# Backup datasources
|
||||
echo "Backing up datasources..."
|
||||
curl -s -H "Authorization: Bearer $API_KEY" \
|
||||
"$GRAFANA_URL/api/datasources" > "$OUTPUT_DIR/datasources/datasources.json"
|
||||
DS_COUNT=$(jq length "$OUTPUT_DIR/datasources/datasources.json")
|
||||
echo " Backed up $DS_COUNT datasources"
|
||||
|
||||
# Backup folders
|
||||
echo "Backing up folders..."
|
||||
curl -s -H "Authorization: Bearer $API_KEY" \
|
||||
"$GRAFANA_URL/api/folders" > "$OUTPUT_DIR/folders/folders.json"
|
||||
FOLDER_COUNT=$(jq length "$OUTPUT_DIR/folders/folders.json")
|
||||
echo " Backed up $FOLDER_COUNT folders"
|
||||
|
||||
# Get all dashboards
|
||||
echo "Backing up dashboards..."
|
||||
DASHBOARDS=$(curl -s -H "Authorization: Bearer $API_KEY" \
|
||||
"$GRAFANA_URL/api/search?type=dash-db")
|
||||
|
||||
DASH_COUNT=0
|
||||
echo "$DASHBOARDS" | jq -r '.[].uid' | while read uid; do
|
||||
DASH=$(curl -s -H "Authorization: Bearer $API_KEY" \
|
||||
"$GRAFANA_URL/api/dashboards/uid/$uid")
|
||||
|
||||
TITLE=$(echo "$DASH" | jq -r '.dashboard.title' | tr ' /' '_')
|
||||
echo "$DASH" > "$OUTPUT_DIR/dashboards/${TITLE}_${uid}.json"
|
||||
echo " - $TITLE"
|
||||
DASH_COUNT=$((DASH_COUNT + 1))
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Backup complete!"
|
||||
echo "Location: $OUTPUT_DIR"
|
||||
echo ""
|
||||
echo "To restore:"
|
||||
echo " 1. Datasources: POST to /api/datasources"
|
||||
echo " 2. Folders: POST to /api/folders"
|
||||
echo " 3. Dashboards: POST to /api/dashboards/db"
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/bin/bash
|
||||
# Prometheus Health Check Script
|
||||
# Usage: ./prometheus-health-check.sh [prometheus-url]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PROMETHEUS_URL="${1:-http://localhost:9090}"
|
||||
|
||||
echo "========================================="
|
||||
echo "Prometheus Health Check"
|
||||
echo "URL: $PROMETHEUS_URL"
|
||||
echo "========================================="
|
||||
echo ""
|
||||
|
||||
# Check Prometheus health
|
||||
echo -n "Prometheus Health: "
|
||||
HEALTH=$(curl -s "$PROMETHEUS_URL/-/healthy" 2>/dev/null)
|
||||
if [ "$HEALTH" == "Prometheus Server is Healthy." ]; then
|
||||
echo "✓ Healthy"
|
||||
else
|
||||
echo "✗ Unhealthy"
|
||||
fi
|
||||
|
||||
# Check readiness
|
||||
echo -n "Prometheus Ready: "
|
||||
READY=$(curl -s "$PROMETHEUS_URL/-/ready" 2>/dev/null)
|
||||
if [ "$READY" == "Prometheus Server is Ready." ]; then
|
||||
echo "✓ Ready"
|
||||
else
|
||||
echo "✗ Not Ready"
|
||||
fi
|
||||
|
||||
# Get runtime info
|
||||
echo ""
|
||||
echo "Runtime Information:"
|
||||
echo "--------------------"
|
||||
curl -s "$PROMETHEUS_URL/api/v1/status/runtimeinfo" 2>/dev/null | jq -r '
|
||||
.data |
|
||||
"Start Time: \(.startTime)",
|
||||
"Uptime: \(.CWD // "N/A")",
|
||||
"Storage Retention: \(.storageRetention)",
|
||||
"TSDB Info:",
|
||||
" - Head Chunks: \(.TSDB.headChunks // "N/A")",
|
||||
" - Head Series: \(.TSDB.headSeries // "N/A")"
|
||||
' 2>/dev/null || echo "Could not retrieve runtime info"
|
||||
|
||||
# Get active targets summary
|
||||
echo ""
|
||||
echo "Target Status:"
|
||||
echo "--------------"
|
||||
curl -s "$PROMETHEUS_URL/api/v1/targets" 2>/dev/null | jq -r '
|
||||
.data.activeTargets |
|
||||
group_by(.health) |
|
||||
map({health: .[0].health, count: length}) |
|
||||
.[] |
|
||||
"\(.health): \(.count) targets"
|
||||
' 2>/dev/null || echo "Could not retrieve targets"
|
||||
|
||||
# List unhealthy targets
|
||||
echo ""
|
||||
echo "Unhealthy Targets:"
|
||||
echo "------------------"
|
||||
curl -s "$PROMETHEUS_URL/api/v1/targets" 2>/dev/null | jq -r '
|
||||
.data.activeTargets[] |
|
||||
select(.health != "up") |
|
||||
"- \(.labels.job)/\(.labels.instance): \(.lastError)"
|
||||
' 2>/dev/null || echo "Could not check unhealthy targets"
|
||||
|
||||
# Check for firing alerts
|
||||
echo ""
|
||||
echo "Firing Alerts:"
|
||||
echo "--------------"
|
||||
curl -s "$PROMETHEUS_URL/api/v1/alerts" 2>/dev/null | jq -r '
|
||||
.data.alerts[] |
|
||||
select(.state == "firing") |
|
||||
"- [\(.labels.severity // "unknown")] \(.labels.alertname): \(.annotations.summary // .annotations.description // "No description")"
|
||||
' 2>/dev/null || echo "Could not retrieve alerts"
|
||||
|
||||
echo ""
|
||||
echo "========================================="
|
||||
echo "Health check complete"
|
||||
echo "========================================="
|
||||
Reference in New Issue
Block a user