.

devops/observability/prometheus-grafana/scripts/backup-grafana.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+# Grafana Dashboard Backup Script
+# Usage: ./backup-grafana.sh [grafana-url] [api-key] [output-dir]
+
+set -euo pipefail
+
+GRAFANA_URL="${1:-http://localhost:3000}"
+API_KEY="${2:-$GRAFANA_API_KEY}"
+OUTPUT_DIR="${3:-./grafana-backup-$(date +%Y%m%d)}"
+
+if [ -z "$API_KEY" ]; then
+    echo "Error: API key required. Set GRAFANA_API_KEY or pass as argument."
+    exit 1
+fi
+
+mkdir -p "$OUTPUT_DIR/dashboards"
+mkdir -p "$OUTPUT_DIR/datasources"
+mkdir -p "$OUTPUT_DIR/folders"
+
+echo "========================================="
+echo "Grafana Backup"
+echo "URL: $GRAFANA_URL"
+echo "Output: $OUTPUT_DIR"
+echo "========================================="
+echo ""
+
+# Backup datasources
+echo "Backing up datasources..."
+curl -s -H "Authorization: Bearer $API_KEY" \
+    "$GRAFANA_URL/api/datasources" > "$OUTPUT_DIR/datasources/datasources.json"
+DS_COUNT=$(jq length "$OUTPUT_DIR/datasources/datasources.json")
+echo "  Backed up $DS_COUNT datasources"
+
+# Backup folders
+echo "Backing up folders..."
+curl -s -H "Authorization: Bearer $API_KEY" \
+    "$GRAFANA_URL/api/folders" > "$OUTPUT_DIR/folders/folders.json"
+FOLDER_COUNT=$(jq length "$OUTPUT_DIR/folders/folders.json")
+echo "  Backed up $FOLDER_COUNT folders"
+
+# Get all dashboards
+echo "Backing up dashboards..."
+DASHBOARDS=$(curl -s -H "Authorization: Bearer $API_KEY" \
+    "$GRAFANA_URL/api/search?type=dash-db")
+
+DASH_COUNT=0
+echo "$DASHBOARDS" | jq -r '.[].uid' | while read uid; do
+    DASH=$(curl -s -H "Authorization: Bearer $API_KEY" \
+        "$GRAFANA_URL/api/dashboards/uid/$uid")
+    
+    TITLE=$(echo "$DASH" | jq -r '.dashboard.title' | tr ' /' '_')
+    echo "$DASH" > "$OUTPUT_DIR/dashboards/${TITLE}_${uid}.json"
+    echo "  - $TITLE"
+    DASH_COUNT=$((DASH_COUNT + 1))
+done
+
+echo ""
+echo "Backup complete!"
+echo "Location: $OUTPUT_DIR"
+echo ""
+echo "To restore:"
+echo "  1. Datasources: POST to /api/datasources"
+echo "  2. Folders: POST to /api/folders"
+echo "  3. Dashboards: POST to /api/dashboards/db"

devops/observability/prometheus-grafana/scripts/prometheus-health-check.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+# Prometheus Health Check Script
+# Usage: ./prometheus-health-check.sh [prometheus-url]
+
+set -euo pipefail
+
+PROMETHEUS_URL="${1:-http://localhost:9090}"
+
+echo "========================================="
+echo "Prometheus Health Check"
+echo "URL: $PROMETHEUS_URL"
+echo "========================================="
+echo ""
+
+# Check Prometheus health
+echo -n "Prometheus Health: "
+HEALTH=$(curl -s "$PROMETHEUS_URL/-/healthy" 2>/dev/null)
+if [ "$HEALTH" == "Prometheus Server is Healthy." ]; then
+    echo "✓ Healthy"
+else
+    echo "✗ Unhealthy"
+fi
+
+# Check readiness
+echo -n "Prometheus Ready: "
+READY=$(curl -s "$PROMETHEUS_URL/-/ready" 2>/dev/null)
+if [ "$READY" == "Prometheus Server is Ready." ]; then
+    echo "✓ Ready"
+else
+    echo "✗ Not Ready"
+fi
+
+# Get runtime info
+echo ""
+echo "Runtime Information:"
+echo "--------------------"
+curl -s "$PROMETHEUS_URL/api/v1/status/runtimeinfo" 2>/dev/null | jq -r '
+  .data | 
+  "Start Time: \(.startTime)",
+  "Uptime: \(.CWD // "N/A")",
+  "Storage Retention: \(.storageRetention)",
+  "TSDB Info:",
+  "  - Head Chunks: \(.TSDB.headChunks // "N/A")",
+  "  - Head Series: \(.TSDB.headSeries // "N/A")"
+' 2>/dev/null || echo "Could not retrieve runtime info"
+
+# Get active targets summary
+echo ""
+echo "Target Status:"
+echo "--------------"
+curl -s "$PROMETHEUS_URL/api/v1/targets" 2>/dev/null | jq -r '
+  .data.activeTargets | 
+  group_by(.health) | 
+  map({health: .[0].health, count: length}) | 
+  .[] | 
+  "\(.health): \(.count) targets"
+' 2>/dev/null || echo "Could not retrieve targets"
+
+# List unhealthy targets
+echo ""
+echo "Unhealthy Targets:"
+echo "------------------"
+curl -s "$PROMETHEUS_URL/api/v1/targets" 2>/dev/null | jq -r '
+  .data.activeTargets[] | 
+  select(.health != "up") | 
+  "- \(.labels.job)/\(.labels.instance): \(.lastError)"
+' 2>/dev/null || echo "Could not check unhealthy targets"
+
+# Check for firing alerts
+echo ""
+echo "Firing Alerts:"
+echo "--------------"
+curl -s "$PROMETHEUS_URL/api/v1/alerts" 2>/dev/null | jq -r '
+  .data.alerts[] | 
+  select(.state == "firing") | 
+  "- [\(.labels.severity // "unknown")] \(.labels.alertname): \(.annotations.summary // .annotations.description // "No description")"
+' 2>/dev/null || echo "Could not retrieve alerts"
+
+echo ""
+echo "========================================="
+echo "Health check complete"
+echo "========================================="